Marown/heissepreise
1
0
Fork 0
mirror of https://github.com/badlogic/heissepreise.git synced 2024-06-24 09:25:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Initial DM Germany support.

Browse Source
This commit is contained in:
Mario Zechner 2023-05-30 19:29:33 +02:00
parent 80dbc296db
commit 1270002b99
4 changed files with 103 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
analysis.js
View File

@ -113,6 +113,7 @@ function compress(items) {
data.push(item.url?.replace("https://shop.billa.at", ""));
break;
case "dm":
case "dmDe":
data.push("");
break;
case "hofer":

8
site/utils.js
View File

@ -29,6 +29,11 @@ const stores = {
budgetBrands: ["balea"],
color: "rgb(255 240 230)",
},
dmDe: {
name: "DM DE",
budgetBrands: ["balea"],
color: "rgb(236 254 253)",
},
unimarkt: {
name: "Unimarkt",
budgetBrands: ["jeden tag", "unipur"],
@ -118,6 +123,9 @@ function decompress(compressedItems) {
case "dm":
url = `https://www.dm.at/product-p${id}.html`;
break;
case "dmDe":
url = `https://www.dm.de/product-p${id}.html`;
break;
case "hofer":
url = "https://www.roksh.at/hofer/produkte/" + url;
break;

93
stores/dm-de.js Normal file
View File

@ -0,0 +1,93 @@
const axios = require("axios");
const utils = require("./utils");
const conversions = {
'g': { unit: 'g', factor: 1 },
'kg': { unit: 'g', factor: 1000 },
'l': { unit: 'ml', factor: 1000 },
'ml': { unit: 'ml', factor: 1 },
'St': { unit: 'stk', factor: 1 },
'Wl': { unit: 'wg', factor: 1 },
'm': { unit: 'cm', factor: 100 },
'mm': { unit: 'cm', factor: .1 },
'Bl': { unit: 'stk', factor: 1 },
'Btl': { unit: 'stk', factor: 1 },
'Paar': { unit: 'stk', factor: 1 },
'Portion': { unit: 'stk', factor: 1 },
'Satz': { unit: 'stk', factor: 1 },
'Tablette': { unit: 'stk', factor: 1 },
};
exports.getCanonical = function(item, today) {
let quantity = item.netQuantityContent || item.basePriceQuantity;
let unit = item.contentUnit || item.basePriceUnit;
return utils.convertUnit({
id: item.gtin,
name: `${item.brandName} ${item.title}`,
price: item.price.value,
priceHistory: [{ date: today, price: item.price.value }],
unit,
quantity,
...(item.brandName === "dmBio" || (item.name ? (item.name.startsWith("Bio ") | item.name.startsWith("Bio-")) : false)) && {bio: true},
url: `https://www.dm.de/product-p${item.gtin}.html`,
}, conversions, 'dmDe');
}
exports.fetchData = async function() {
const DM_BASE_URL = `https://product-search.services.dmtech.com/de/search/crawl?pageSize=1000&`
const QUERIES = [
'allCategories.id=010000&price.value.to=2', //~500 items
'allCategories.id=010000&price.value.from=2&price.value.to=3', //~600 items
'allCategories.id=010000&price.value.from=3&price.value.to=4', //~500 items
'allCategories.id=010000&price.value.from=4&price.value.to=7', //~800 items
'allCategories.id=010000&price.value.from=7&price.value.to=10', //~900 items
'allCategories.id=010000&price.value.from=10&price.value.to=15', //~900 items
'allCategories.id=010000&price.value.from=15', //~300 items
'allCategories.id=020000&price.value.to=2', //~600 items
'allCategories.id=020000&price.value.from=2&price.value.to=3', //~550 items
'allCategories.id=020000&price.value.from=3&price.value.to=4', //~600 items
'allCategories.id=020000&price.value.from=4&price.value.to=6', //~800 items
'allCategories.id=020000&price.value.from=6&price.value.to=10', //~850 items
'allCategories.id=020000&price.value.from=10&price.value.to=18', //~900 items
'allCategories.id=020000&price.value.from=18', //~960 items (!)
'allCategories.id=030000&price.value.to=8', //~900 items
'allCategories.id=030000&price.value.from=8', //~500 items
'allCategories.id=040000&price.value.to=2', //~600 items
'allCategories.id=040000&price.value.from=2&price.value.to=4', //~900 items
'allCategories.id=040000&price.value.from=4', //~400 items
'allCategories.id=050000&price.value.to=4', //~600 items
'allCategories.id=050000&price.value.from=4', //~800 items
'allCategories.id=060000&price.value.to=4', //~900 items
'allCategories.id=060000&price.value.from=4', //~500 items
'allCategories.id=070000', //~300 items
]
let dmItems = [];
for (let query of QUERIES) {
var res = (await axios.get(DM_BASE_URL + query, {
validateStatus: function (status) {
return (status >= 200 && status < 300) || status == 429;
}
}));
// exponential backoff
backoff = 2000;
while (res.status == 429) {
console.info(`DM-DE API returned 429, retrying in ${backoff/1000}s.`);
await new Promise(resolve => setTimeout(resolve, backoff));
backoff *= 2;
res = (await axios.get(DM_BASE_URL + query, {
validateStatus: function (status) {
return (status >= 200 && status < 300) || status == 429;
}
}));
}
let items = res.data;
if (items.count > 1000) {
console.warn(`DM-DE Query returned more than 1000 items! Items may be missing. Adjust queries. Query: ${query}`);
}
dmItems = dmItems.concat(items.products);
await new Promise(resolve => setTimeout(resolve, 1000));
}
return dmItems;
}

1
stores/index.js
View File

@ -1,5 +1,6 @@
exports.billa = require("./billa");
exports.dm = require("./dm");
exports.dmDe = require("./dm-de");
exports.hofer = require("./hofer");
exports.lidl = require("./lidl");
exports.mpreis = require("./mpreis");