diff --git a/analysis.js b/analysis.js index b43630b..8141044 100644 --- a/analysis.js +++ b/analysis.js @@ -113,6 +113,7 @@ function compress(items) { data.push(item.url?.replace("https://shop.billa.at", "")); break; case "dm": + case "dmDe": data.push(""); break; case "hofer": diff --git a/site/utils.js b/site/utils.js index 8bd6f46..f0ddec3 100644 --- a/site/utils.js +++ b/site/utils.js @@ -29,6 +29,11 @@ const stores = { budgetBrands: ["balea"], color: "rgb(255 240 230)", }, + dmDe: { + name: "DM DE", + budgetBrands: ["balea"], + color: "rgb(236 254 253)", + }, unimarkt: { name: "Unimarkt", budgetBrands: ["jeden tag", "unipur"], @@ -118,6 +123,9 @@ function decompress(compressedItems) { case "dm": url = `https://www.dm.at/product-p${id}.html`; break; + case "dmDe": + url = `https://www.dm.de/product-p${id}.html`; + break; case "hofer": url = "https://www.roksh.at/hofer/produkte/" + url; break; diff --git a/stores/dm-de.js b/stores/dm-de.js new file mode 100644 index 0000000..a5716f4 --- /dev/null +++ b/stores/dm-de.js @@ -0,0 +1,93 @@ +const axios = require("axios"); +const utils = require("./utils"); + +const conversions = { + 'g': { unit: 'g', factor: 1 }, + 'kg': { unit: 'g', factor: 1000 }, + 'l': { unit: 'ml', factor: 1000 }, + 'ml': { unit: 'ml', factor: 1 }, + 'St': { unit: 'stk', factor: 1 }, + 'Wl': { unit: 'wg', factor: 1 }, + 'm': { unit: 'cm', factor: 100 }, + 'mm': { unit: 'cm', factor: .1 }, + 'Bl': { unit: 'stk', factor: 1 }, + 'Btl': { unit: 'stk', factor: 1 }, + 'Paar': { unit: 'stk', factor: 1 }, + 'Portion': { unit: 'stk', factor: 1 }, + 'Satz': { unit: 'stk', factor: 1 }, + 'Tablette': { unit: 'stk', factor: 1 }, +}; + +exports.getCanonical = function(item, today) { + let quantity = item.netQuantityContent || item.basePriceQuantity; + let unit = item.contentUnit || item.basePriceUnit; + return utils.convertUnit({ + id: item.gtin, + name: `${item.brandName} ${item.title}`, + price: item.price.value, + priceHistory: [{ date: today, price: item.price.value }], + unit, + quantity, + ...(item.brandName === "dmBio" || (item.name ? (item.name.startsWith("Bio ") | item.name.startsWith("Bio-")) : false)) && {bio: true}, + url: `https://www.dm.de/product-p${item.gtin}.html`, + }, conversions, 'dmDe'); +} + +exports.fetchData = async function() { + const DM_BASE_URL = `https://product-search.services.dmtech.com/de/search/crawl?pageSize=1000&` + const QUERIES = [ + 'allCategories.id=010000&price.value.to=2', //~500 items + 'allCategories.id=010000&price.value.from=2&price.value.to=3', //~600 items + 'allCategories.id=010000&price.value.from=3&price.value.to=4', //~500 items + 'allCategories.id=010000&price.value.from=4&price.value.to=7', //~800 items + 'allCategories.id=010000&price.value.from=7&price.value.to=10', //~900 items + 'allCategories.id=010000&price.value.from=10&price.value.to=15', //~900 items + 'allCategories.id=010000&price.value.from=15', //~300 items + 'allCategories.id=020000&price.value.to=2', //~600 items + 'allCategories.id=020000&price.value.from=2&price.value.to=3', //~550 items + 'allCategories.id=020000&price.value.from=3&price.value.to=4', //~600 items + 'allCategories.id=020000&price.value.from=4&price.value.to=6', //~800 items + 'allCategories.id=020000&price.value.from=6&price.value.to=10', //~850 items + 'allCategories.id=020000&price.value.from=10&price.value.to=18', //~900 items + 'allCategories.id=020000&price.value.from=18', //~960 items (!) + 'allCategories.id=030000&price.value.to=8', //~900 items + 'allCategories.id=030000&price.value.from=8', //~500 items + 'allCategories.id=040000&price.value.to=2', //~600 items + 'allCategories.id=040000&price.value.from=2&price.value.to=4', //~900 items + 'allCategories.id=040000&price.value.from=4', //~400 items + 'allCategories.id=050000&price.value.to=4', //~600 items + 'allCategories.id=050000&price.value.from=4', //~800 items + 'allCategories.id=060000&price.value.to=4', //~900 items + 'allCategories.id=060000&price.value.from=4', //~500 items + 'allCategories.id=070000', //~300 items + ] + + let dmItems = []; + for (let query of QUERIES) { + var res = (await axios.get(DM_BASE_URL + query, { + validateStatus: function (status) { + return (status >= 200 && status < 300) || status == 429; + } + })); + + // exponential backoff + backoff = 2000; + while (res.status == 429) { + console.info(`DM-DE API returned 429, retrying in ${backoff/1000}s.`); + await new Promise(resolve => setTimeout(resolve, backoff)); + backoff *= 2; + res = (await axios.get(DM_BASE_URL + query, { + validateStatus: function (status) { + return (status >= 200 && status < 300) || status == 429; + } + })); + } + let items = res.data; + if (items.count > 1000) { + console.warn(`DM-DE Query returned more than 1000 items! Items may be missing. Adjust queries. Query: ${query}`); + } + dmItems = dmItems.concat(items.products); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + return dmItems; +} diff --git a/stores/index.js b/stores/index.js index 8a82ad1..784a330 100644 --- a/stores/index.js +++ b/stores/index.js @@ -1,5 +1,6 @@ exports.billa = require("./billa"); exports.dm = require("./dm"); +exports.dmDe = require("./dm-de"); exports.hofer = require("./hofer"); exports.lidl = require("./lidl"); exports.mpreis = require("./mpreis");