From b05702aff5900ae15c25e2b404f7498012c68935 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 21 Jun 2023 15:20:28 +0200 Subject: [PATCH] Hofer and MPREIS categories. --- analysis.js | 38 +- restore.js | 2 +- stores/hofer-categories.json | 282 +++++------ stores/hofer.js | 24 +- stores/mpreis-categories.json | 852 ++++++++++++++++++++++++++++++++++ stores/mpreis.js | 56 ++- stores/spar.js | 3 - 7 files changed, 1085 insertions(+), 172 deletions(-) create mode 100644 stores/mpreis-categories.json diff --git a/analysis.js b/analysis.js index 9255095..6aa77d0 100644 --- a/analysis.js +++ b/analysis.js @@ -186,7 +186,7 @@ exports.compress = compress; /// Given a directory of raw data of the form `$store-$date.json`, constructs /// a canonical list of all products and their historical price data. -exports.replay = function (rawDataDir) { +exports.replay = async (rawDataDir) => { const today = currentDate(); const files = fs @@ -209,11 +209,18 @@ exports.replay = function (rawDataDir) { const canonicalFiles = {}; for (const store of STORE_KEYS) { - stores[store].generateCategoryMapping(); + await stores[store].initializeCategoryMapping(); storeFiles[store] = getFilteredFilesFor(store); canonicalFiles[store] = storeFiles[store].map((file) => { console.log(`Creating canonical items for ${file}`); - getCanonicalFor(store, readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]); + const rawItems = readJSON(file); + const items = getCanonicalFor(store, rawItems, file.match(/\d{4}-\d{2}-\d{2}/)[0]); + for (let i = 0; i < items.length; i++) { + const rawItem = rawItems[i]; + const item = items[i]; + item.category = stores[store].mapCategory(rawItem); + } + return items; }); canonicalFiles[store].reverse(); } @@ -248,34 +255,35 @@ exports.updateData = async function (dataDir, done) { console.log("Fetching data for date: " + today); const storeFetchPromises = []; for (const store of STORE_KEYS) { - await stores[store].initializeCategoryMapping(); - storeFetchPromises.push( new Promise(async (resolve) => { const start = performance.now(); try { const rawDataFile = `${dataDir}/${store}-${today}.json`; - let storeItems; + let rawItems; if ("SKIP_FETCHING_STORE_DATA" in process.env && fs.existsSync(rawDataFile + "." + FILE_COMPRESSOR)) - storeItems = await readJSONAsync(rawDataFile + "." + FILE_COMPRESSOR); + rawItems = await readJSONAsync(rawDataFile + "." + FILE_COMPRESSOR); else { - storeItems = await stores[store].fetchData(); - writeJSON(rawDataFile, storeItems, FILE_COMPRESSOR); + rawItems = await stores[store].fetchData(); + writeJSON(rawDataFile, rawItems, FILE_COMPRESSOR); } - const storeItemsCanonical = getCanonicalFor(store, storeItems, today); + const items = getCanonicalFor(store, rawItems, today); + + await stores[store].initializeCategoryMapping(rawItems); let numUncategorized = 0; - for (let i = 0; i < storeItemsCanonical.length; i++) { - const rawItem = storeItems[i]; - const item = storeItemsCanonical[i]; + for (let i = 0; i < items.length; i++) { + const rawItem = rawItems[i]; + const item = items[i]; item.category = stores[store].mapCategory(rawItem); if (item.category == null) numUncategorized++; } + console.log( `Fetched ${store.toUpperCase()} data, took ${(performance.now() - start) / 1000} seconds, ${numUncategorized}/${ - storeItemsCanonical.length + items.length } items without category.` ); - resolve(storeItemsCanonical); + resolve(items); } catch (e) { console.error(`Error while fetching data from ${store}, continuing after ${(performance.now() - start) / 1000} seconds...`, e); resolve([]); diff --git a/restore.js b/restore.js index 5eed18f..a8fdfe0 100644 --- a/restore.js +++ b/restore.js @@ -7,7 +7,7 @@ console.log("Restoring data from raw data."); (async function () { analysis.migrateCompression(dataDir, ".json", ".json.br", false); analysis.migrateCompression(dataDir, ".json.gz", ".json.br"); - const items = analysis.replay(dataDir); + const items = await analysis.replay(dataDir); analysis.writeJSON(`${dataDir}/latest-canonical.json`, items, analysis.FILE_COMPRESSOR); console.log( `Wrote ${ diff --git a/stores/hofer-categories.json b/stores/hofer-categories.json index c7dfb84..6334c24 100644 --- a/stores/hofer-categories.json +++ b/stores/hofer-categories.json @@ -3,709 +3,709 @@ "id": "obst-gemuse", "description": "Obst & Gemüse", "url": "https://www.roksh.at//hofer/angebot/obst-gemuse", - "code": null + "code": "00" }, { "id": "gemuse", "description": "Gemüse", "url": "https://www.roksh.at//hofer/angebot/obst-gemuse/gemuse", - "code": null + "code": "01" }, { "id": "obst", "description": "Obst", "url": "https://www.roksh.at//hofer/angebot/obst-gemuse/obst", - "code": null + "code": "00" }, { "id": "brot-geback", "description": "Brot & Gebäck", "url": "https://www.roksh.at//hofer/angebot/brot-geback", - "code": null + "code": "10" }, { "id": "backbox", "description": "BACKBOX", "url": "https://www.roksh.at//hofer/angebot/brot-geback/backbox", - "code": null + "code": "11" }, { "id": "brot-geback-2", "description": "Brot & Gebäck", "url": "https://www.roksh.at//hofer/angebot/brot-geback/brot-geback-2", - "code": null + "code": "12" }, { "id": "suße-backwaren", "description": "Süße Backwaren", "url": "https://www.roksh.at//hofer/angebot/brot-geback/suße-backwaren", - "code": null + "code": "13" }, { "id": "kuhlwaren", "description": "Kühlwaren", "url": "https://www.roksh.at//hofer/angebot/kuhlwaren", - "code": null + "code": "30" }, { "id": "milch-joghurt-co", "description": "Milch, Joghurt & Co", "url": "https://www.roksh.at//hofer/angebot/kuhlwaren/milch-joghurt-co", - "code": null + "code": "34" }, { "id": "kase-aufstriche", "description": "Käse & Aufstriche", "url": "https://www.roksh.at//hofer/angebot/kuhlwaren/kase-aufstriche", - "code": null + "code": "33" }, { "id": "schnell-kuche", "description": "Schnelle Küche", "url": "https://www.roksh.at//hofer/angebot/kuhlwaren/schnell-kuche", - "code": null + "code": "30" }, { "id": "desserts-und-sußspeise", "description": "Desserts und Süßspeisen", "url": "https://www.roksh.at//hofer/angebot/kuhlwaren/desserts-und-sußspeise", - "code": null + "code": "34" }, { "id": "eier", "description": "Eier", "url": "https://www.roksh.at//hofer/angebot/kuhlwaren/eier", - "code": null + "code": "31" }, { "id": "teig-backwaren", "description": "Teig- & Backwaren", "url": "https://www.roksh.at//hofer/angebot/kuhlwaren/teig-backwaren", - "code": null + "code": "36" }, { "id": "fleisch-fisch-und-wurst", "description": "Fleisch, Wurst & Fisch", "url": "https://www.roksh.at//hofer/angebot/fleisch-fisch-und-wurst", - "code": null + "code": "37" }, { "id": "wurst-aufschnitt", "description": "Wurst & Aufschnitt", "url": "https://www.roksh.at//hofer/angebot/fleisch-fisch-und-wurst/wurst-aufschnitt", - "code": null + "code": "37" }, { "id": "fleisch", "description": "Fleisch", "url": "https://www.roksh.at//hofer/angebot/fleisch-fisch-und-wurst/fleisch", - "code": null + "code": "37" }, { "id": "fisch-meeresfruchte", "description": "Fisch & Meeresfrüchte", "url": "https://www.roksh.at//hofer/angebot/fleisch-fisch-und-wurst/fisch-meeresfruchte", - "code": null + "code": "39" }, { "id": "vorratsschrank", "description": "Vorratsschrank", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank", - "code": null + "code": "50" }, { "id": "mehl-backwaren", "description": "Mehl & Backwaren", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/mehl-backwaren", - "code": null + "code": "52" }, { "id": "suße-aufstriche", "description": "Süße Aufstriche", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/suße-aufstriche", - "code": null + "code": "56" }, { "id": "musli-cerealien", "description": "Müsli & Cerealien", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/musli-cerealien", - "code": null + "code": "5A" }, { "id": "konserven", "description": "Konserven", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/konserven", - "code": null + "code": "57" }, { "id": "fertiggerichte-suppen", "description": "Fertiggerichte & Suppen", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/fertiggerichte-suppen", - "code": null + "code": "54" }, { "id": "essig-ol-saucen", "description": "Essig, Öl & Saucen", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/essig-ol-saucen", - "code": null + "code": "53" }, { "id": "reis-teigwaren-getreide", "description": "Reis, Teigwaren & Getreide", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/reis-teigwaren-getreide", - "code": null + "code": "5B" }, { "id": "gewurze", "description": "Gewürze", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/gewurze", - "code": null + "code": "55" }, { "id": "nusse-trockenfruchte", "description": "Nüsse & Trockenfrüchte", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/nusse-trockenfruchte", - "code": null + "code": "03" }, { "id": "zucker-sußungsmittel", "description": "Zucker & Süßungsmittel", "url": "https://www.roksh.at//hofer/angebot/vorratsschrank/zucker-sußungsmittel", - "code": null + "code": "5E" }, { "id": "sußes-salziges", "description": "Süßes & Salziges", "url": "https://www.roksh.at//hofer/angebot/sußes-salziges", - "code": null + "code": "60" }, { "id": "sußes", "description": "Süßes", "url": "https://www.roksh.at//hofer/angebot/sußes-salziges/sußes", - "code": null + "code": "64" }, { "id": "salziges", "description": "Salziges", "url": "https://www.roksh.at//hofer/angebot/sußes-salziges/salziges", - "code": null + "code": "63" }, { "id": "tiefkuhlwaren", "description": "Tiefkühlwaren", "url": "https://www.roksh.at//hofer/angebot/tiefkuhlwaren", - "code": null + "code": "40" }, { "id": "fleisch-2", "description": "Fleisch", "url": "https://www.roksh.at//hofer/angebot/tiefkuhlwaren/fleisch-2", - "code": null + "code": "42" }, { "id": "gemuse-obst", "description": "Gemüse & Obst", "url": "https://www.roksh.at//hofer/angebot/tiefkuhlwaren/gemuse-obst", - "code": null + "code": "44" }, { "id": "desserts-backwaren-eis", "description": "Desserts, Backwaren & Eis", "url": "https://www.roksh.at//hofer/angebot/tiefkuhlwaren/desserts-backwaren-eis", - "code": null + "code": "40" }, { "id": "pizza-fertiggerichte", "description": "Pizza & Fertiggerichte", "url": "https://www.roksh.at//hofer/angebot/tiefkuhlwaren/pizza-fertiggerichte", - "code": null + "code": "46" }, { "id": "fisch-meeresfruchte-2", "description": "Fisch & Meeresfrüchte", "url": "https://www.roksh.at//hofer/angebot/tiefkuhlwaren/fisch-meeresfruchte-2", - "code": null + "code": "43" }, { "id": "vegetarisch-vegan", "description": "Vegetarisch & Vegan ", "url": "https://www.roksh.at//hofer/angebot/vegetarisch-vegan", - "code": null + "code": "3B" }, { "id": "pflanzliche", "description": "Pflanzliche Produkte", "url": "https://www.roksh.at//hofer/angebot/vegetarisch-vegan/pflanzliche", - "code": null + "code": "3B" }, { "id": "margarine-pflanzliche", "description": "Margarine & pflanzliche Fette", "url": "https://www.roksh.at//hofer/angebot/vegetarisch-vegan/margarine-pflanzliche", - "code": null + "code": "34" }, { "id": "fleischersatzprodukte", "description": "Fleischersatz- Produkte", "url": "https://www.roksh.at//hofer/angebot/vegetarisch-vegan/fleischersatzprodukte", - "code": null + "code": "3B" }, { "id": "getranke", "description": "Getränke", "url": "https://www.roksh.at//hofer/angebot/getranke", - "code": null + "code": "20" }, { "id": "spirituosen", "description": "Spirituosen", "url": "https://www.roksh.at//hofer/angebot/getranke/spirituosen", - "code": null + "code": "24" }, { "id": "wein-sekt", "description": "Wein & Sekt", "url": "https://www.roksh.at//hofer/angebot/getranke/wein-sekt", - "code": null + "code": "25" }, { "id": "bier-radler", "description": "Bier & Radler", "url": "https://www.roksh.at//hofer/angebot/getranke/bier-radler", - "code": null + "code": "21" }, { "id": "kaffe-tee-co", "description": "Kaffee, Tee & Co", "url": "https://www.roksh.at//hofer/angebot/getranke/kaffe-tee-co", - "code": null + "code": "22" }, { "id": "alkoholfreie-getranke", "description": "Alkoholfreie Getränke", "url": "https://www.roksh.at//hofer/angebot/getranke/alkoholfreie-getranke", - "code": null + "code": "20" }, { "id": "drogerie", "description": "Drogerie", "url": "https://www.roksh.at//hofer/angebot/drogerie", - "code": null + "code": "70" }, { "id": "hygieneartikel-2", "description": "Pflege- & Hygieneartikel", "url": "https://www.roksh.at//hofer/angebot/drogerie/hygieneartikel-2", - "code": null + "code": "72" }, { "id": "babyartikel", "description": "Babyartikel", "url": "https://www.roksh.at//hofer/angebot/drogerie/babyartikel", - "code": null + "code": "70" }, { "id": "gesundheit-medizinprodukte", "description": "Hausapotheke & Sportnahrung", "url": "https://www.roksh.at//hofer/angebot/drogerie/gesundheit-medizinprodukte", - "code": null + "code": "5D" }, { "id": "damensocken-feinstrumpfe", "description": "Damensocken & Feinstrümpfe", "url": "https://www.roksh.at//hofer/angebot/drogerie/damensocken-feinstrumpfe", - "code": null + "code": "7C" }, { "id": "haushalt", "description": "Haushalt", "url": "https://www.roksh.at//hofer/angebot/haushalt", - "code": null + "code": "80" }, { "id": "waschen-reinigen", "description": "Waschen & Reinigen", "url": "https://www.roksh.at//hofer/angebot/haushalt/waschen-reinigen", - "code": null + "code": "8A" }, { "id": "hygieneartikel", "description": "Hygieneartikel", "url": "https://www.roksh.at//hofer/angebot/haushalt/hygieneartikel", - "code": null + "code": "84" }, { "id": "servitten", "description": " Servietten, Kerzen & Co", "url": "https://www.roksh.at//hofer/angebot/haushalt/servitten", - "code": null + "code": "89" }, { "id": "haushaltfolien", "description": "Haushaltsfolien", "url": "https://www.roksh.at//hofer/angebot/haushalt/haushaltfolien", - "code": null + "code": "86" }, { "id": "tragetaschen", "description": "Tragetaschen", "url": "https://www.roksh.at//hofer/angebot/haushalt/tragetaschen", - "code": null + "code": "8C" }, { "id": "batterien-akkus", "description": "Batterien & Akkus", "url": "https://www.roksh.at//hofer/angebot/haushalt/batterien-akkus", - "code": null + "code": "85" }, { "id": "tierberdarf", "description": "Tierbedarf", "url": "https://www.roksh.at//hofer/angebot/tierberdarf", - "code": null + "code": "90" }, { "id": "hundenahrung", "description": "Hundenahrung", "url": "https://www.roksh.at//hofer/angebot/tierberdarf/hundenahrung", - "code": null + "code": "90" }, { "id": "katzenzubehor", "description": "Katzenzubehör", "url": "https://www.roksh.at//hofer/angebot/tierberdarf/katzenzubehor", - "code": null + "code": "91" }, { "id": "katzennahrung", "description": "Katzennahrung", "url": "https://www.roksh.at//hofer/angebot/tierberdarf/katzennahrung", - "code": null + "code": "91" }, { "id": "joghurt-topfen", "url": "https://www.roksh.at/hofer/angebot/joghurt-topfen", - "code": "" + "code": "34" }, { "id": "milchgetrank", "url": "https://www.roksh.at/hofer/angebot/milchgetrank", - "code": "" + "code": "34" }, { "id": "fruchtjoghurt", "url": "https://www.roksh.at/hofer/angebot/fruchtjoghurt", - "code": "" + "code": "34" }, { "id": "haltbare-milch-sahne-eiskaffee", "url": "https://www.roksh.at/hofer/angebot/haltbare-milch-sahne-eiskaffee", - "code": "" + "code": "34" }, { "id": "milch-sahne-butter", "url": "https://www.roksh.at/hofer/angebot/milch-sahne-butter", - "code": "" + "code": "34" }, { "id": "kasespezialitaten", "url": "https://www.roksh.at/hofer/angebot/kasespezialitaten", - "code": "" + "code": "33" }, { "id": "frischkase-co", "url": "https://www.roksh.at/hofer/angebot/frischkase-co", - "code": "" + "code": "33" }, { "id": "aufstriche-co", "url": "https://www.roksh.at/hofer/angebot/aufstriche-co", - "code": "" + "code": "33" }, { "id": "hart-schnittkase", "url": "https://www.roksh.at/hofer/angebot/hart-schnittkase", - "code": "" + "code": "33" }, { "id": "weichkase", "url": "https://www.roksh.at/hofer/angebot/weichkase", - "code": "" + "code": "33" }, { "id": "kase-geschnitten-gerieben", "url": "https://www.roksh.at/hofer/angebot/kase-geschnitten-gerieben", - "code": "" + "code": "33" }, { "id": "streichwurst-pasteten", "url": "https://www.roksh.at/hofer/angebot/streichwurst-pasteten", - "code": "" + "code": "37" }, { "id": "wurst-geschnitten", "url": "https://www.roksh.at/hofer/angebot/wurst-geschnitten", - "code": "" + "code": "37" }, { "id": "kochschinken", "url": "https://www.roksh.at/hofer/angebot/kochschinken", - "code": "" + "code": "37" }, { "id": "stangenwurst", "url": "https://www.roksh.at/hofer/angebot/stangenwurst", - "code": "" + "code": "37" }, { "id": "wurstel", "url": "https://www.roksh.at/hofer/angebot/wurstel", - "code": "" + "code": "37" }, { "id": "speck-rohschinken", "url": "https://www.roksh.at/hofer/angebot/speck-rohschinken", - "code": "" + "code": "37" }, { "id": "leberkase-co", "url": "https://www.roksh.at/hofer/angebot/leberkase-co", - "code": "" + "code": "37" }, { "id": "faschiertes", "url": "https://www.roksh.at/hofer/angebot/faschiertes", - "code": "" + "code": "32" }, { "id": "schweinefleisch-spezialitaten", "url": "https://www.roksh.at/hofer/angebot/schweinefleisch-spezialitaten", - "code": "" + "code": "32" }, { "id": "hühnchen-geflügel", "url": "https://www.roksh.at/hofer/angebot/hühnchen-geflügel", - "code": "" + "code": "32" }, { "id": "rind", "url": "https://www.roksh.at/hofer/angebot/rind", - "code": "" - }, - { - "id": "fruchtaufstriche", - "url": "https://www.roksh.at/hofer/angebot/fruchtaufstriche", - "code": "" + "code": "32" }, { "id": "honig-nussaufstriche", "url": "https://www.roksh.at/hofer/angebot/honig-nussaufstriche", - "code": "" + "code": "56" + }, + { + "id": "fruchtaufstriche", + "url": "https://www.roksh.at/hofer/angebot/fruchtaufstriche", + "code": "56" }, { "id": "gemusekonserven", "url": "https://www.roksh.at/hofer/angebot/gemusekonserven", - "code": "" + "code": "57" }, { "id": "obstkonserven", "url": "https://www.roksh.at/hofer/angebot/obstkonserven", - "code": "" + "code": "57" }, { "id": "sauerkonserven", "url": "https://www.roksh.at/hofer/angebot/sauerkonserven", - "code": "" + "code": "57" }, { "id": "fleisch-fischkonserven", "url": "https://www.roksh.at/hofer/angebot/fleisch-fischkonserven", - "code": "" + "code": "57" }, { "id": "nusse", "url": "https://www.roksh.at/hofer/angebot/nusse", - "code": "" + "code": "03" }, { "id": "trockenfruchte", "url": "https://www.roksh.at/hofer/angebot/trockenfruchte", - "code": "" + "code": "03" }, { "id": "tafelschokolade", "url": "https://www.roksh.at/hofer/angebot/tafelschokolade", - "code": "" + "code": "64" }, { "id": "schokoriegel", "url": "https://www.roksh.at/hofer/angebot/schokoriegel", - "code": "" + "code": "64" }, { "id": "bonbons-kaugummi", "url": "https://www.roksh.at/hofer/angebot/bonbons-kaugummi", - "code": "" + "code": "64" }, { "id": "kekse", "url": "https://www.roksh.at/hofer/angebot/kekse", - "code": "" + "code": "64" }, { "id": "waffeln", "url": "https://www.roksh.at/hofer/angebot/waffeln", - "code": "" + "code": "60" }, { "id": "fruchtgummi", "url": "https://www.roksh.at/hofer/angebot/fruchtgummi", - "code": "" + "code": "64" }, { "id": "pralinen", "url": "https://www.roksh.at/hofer/angebot/pralinen", - "code": "" + "code": "64" }, { "id": "salzgeback-kracker", "url": "https://www.roksh.at/hofer/angebot/salzgeback-kracker", - "code": "" + "code": "63" }, { "id": "chips", "url": "https://www.roksh.at/hofer/angebot/chips", - "code": "" + "code": "63" }, { "id": "sekt", "url": "https://www.roksh.at/hofer/angebot/sekt", - "code": "" + "code": "23" }, { "id": "roseweine-spezialweine", "url": "https://www.roksh.at/hofer/angebot/roseweine-spezialweine", - "code": "" + "code": "25" }, { "id": "weißweine", "url": "https://www.roksh.at/hofer/angebot/weißweine", - "code": "" + "code": "25" }, { "id": "rotweine", "url": "https://www.roksh.at/hofer/angebot/rotweine", - "code": "" + "code": "25" }, { "id": "tee", "url": "https://www.roksh.at/hofer/angebot/tee", - "code": "" + "code": "22" }, { "id": "kaffee-kakao-co", "url": "https://www.roksh.at/hofer/angebot/kaffee-kakao-co", - "code": "" + "code": "22" }, { "id": "mineralwasser", "url": "https://www.roksh.at/hofer/angebot/mineralwasser", - "code": "" + "code": "26" }, { "id": "softdrinks", "url": "https://www.roksh.at/hofer/angebot/softdrinks", - "code": "" + "code": "20" }, { "id": "sport-energy-drinks", "url": "https://www.roksh.at/hofer/angebot/sport-energy-drinks", - "code": "" + "code": "20" }, { "id": "fruchtsafte", "url": "https://www.roksh.at/hofer/angebot/fruchtsafte", - "code": "" + "code": "20" }, { "id": "smoothies", "url": "https://www.roksh.at/hofer/angebot/smoothies", - "code": "" + "code": "20" }, { "id": "sirupe", "url": "https://www.roksh.at/hofer/angebot/sirupe", - "code": "" + "code": "20" }, { "id": "deodorants", "url": "https://www.roksh.at/hofer/angebot/deodorants", - "code": "" - }, - { - "id": "zahn-mundhygiene", - "url": "https://www.roksh.at/hofer/angebot/zahn-mundhygiene", - "code": "" + "code": "72" }, { "id": "duschpflege-seife", "url": "https://www.roksh.at/hofer/angebot/duschpflege-seife", - "code": "" + "code": "78" + }, + { + "id": "zahn-mundhygiene", + "url": "https://www.roksh.at/hofer/angebot/zahn-mundhygiene", + "code": "76" }, { "id": "herrenkosmetik-rasur", "url": "https://www.roksh.at/hofer/angebot/herrenkosmetik-rasur", - "code": "" + "code": "77" }, { "id": "intimhygiene", "url": "https://www.roksh.at/hofer/angebot/intimhygiene", - "code": "" + "code": "71" }, { "id": "sonnenpflege", "url": "https://www.roksh.at/hofer/angebot/sonnenpflege", - "code": "" + "code": "79" }, { "id": "haarpflege", "url": "https://www.roksh.at/hofer/angebot/haarpflege", - "code": "" + "code": "73" }, { "id": "beikost-snacks", "url": "https://www.roksh.at/hofer/angebot/beikost-snacks", - "code": "" + "code": "51" }, { "id": "windeln-pflegeprodukte", "url": "https://www.roksh.at/hofer/angebot/windeln-pflegeprodukte", - "code": "" + "code": "70" }, { "id": "milchnahrung", "url": "https://www.roksh.at/hofer/angebot/milchnahrung", - "code": "" - }, - { - "id": "spulmittel-putzmittel", - "url": "https://www.roksh.at/hofer/angebot/spulmittel-putzmittel", - "code": "" + "code": "51" }, { "id": "reinigungstucher-kuchentucher", "url": "https://www.roksh.at/hofer/angebot/reinigungstucher-kuchentucher", - "code": "" + "code": "88" + }, + { + "id": "spulmittel-putzmittel", + "url": "https://www.roksh.at/hofer/angebot/spulmittel-putzmittel", + "code": "88" }, { "id": "wc-lufterfrischer", "url": "https://www.roksh.at/hofer/angebot/wc-lufterfrischer", - "code": "" + "code": "87" }, { "id": "waschmittel", "url": "https://www.roksh.at/hofer/angebot/waschmittel", - "code": "" + "code": "8A" } ] diff --git a/stores/hofer.js b/stores/hofer.js index 6cee6e0..6a1847c 100644 --- a/stores/hofer.js +++ b/stores/hofer.js @@ -87,9 +87,8 @@ exports.getCategories = async () => { exports.urlBase = "https://www.roksh.at/hofer/produkte/"; -exports.initializeCategoryMapping = async () => { - // This is unfortunate, but the API doesn't return all categories - const rawItems = analysis.readJSON("data/hofer-2023-06-21.json.br"); // await exports.fetchData(); +exports.initializeCategoryMapping = async (rawItems) => { + rawItems = rawItems ?? (await exports.fetchData()); const rawCategories = (await exports.getCategories()).categories; const lookup = {}; const processCategory = (category) => { @@ -108,11 +107,9 @@ exports.initializeCategoryMapping = async () => { processCategory(category); } - let total = 0; for (const item of rawItems) { if (!lookup[item.CategorySEOName]) { - console.log(`Couldn't find category '${item.CategorySEOName}' for Hofer product ${item.ProductName}`); - total++; + // console.log(`Couldn't find category '${item.CategorySEOName}' for Hofer product ${item.ProductName}`); lookup[item.CategorySEOName] = { id: item.CategorySEOName, url: "https://www.roksh.at/hofer/angebot/" + item.CategorySEOName, @@ -122,9 +119,16 @@ exports.initializeCategoryMapping = async () => { const category = lookup[item.CategorySEOName]; } } - exports.categories = []; - Object.keys(lookup).forEach((key) => exports.categories.push(lookup[key])); - utils.mergeAndSaveCategories("hofer", exports.categories); + let categories = []; + Object.keys(lookup).forEach((key) => categories.push(lookup[key])); + categories = utils.mergeAndSaveCategories("hofer", categories); + exports.categoryLookup = {}; + for (const category of categories) { + exports.categoryLookup[category.id] = category; + } }; -exports.mapCategory = (rawItem) => {}; +exports.mapCategory = (rawItem) => { + const category = exports.categoryLookup[rawItem.CategorySEOName]; + return category?.code; +}; diff --git a/stores/mpreis-categories.json b/stores/mpreis-categories.json new file mode 100644 index 0000000..25e425e --- /dev/null +++ b/stores/mpreis-categories.json @@ -0,0 +1,852 @@ +[ + { + "id": "Lebensmittel -> To-Go-6920583", + "code": "30", + "url": "https://www.mpreis.at/shop/c/6920583" + }, + { + "id": "Lebensmittel -> To-Go -> Sandwiches & Wraps-42115933", + "code": "30", + "url": "https://www.mpreis.at/shop/c/42115933" + }, + { + "id": "Lebensmittel -> To-Go -> Salate-37528473", + "code": "30", + "url": "https://www.mpreis.at/shop/c/37528473" + }, + { + "id": "Lebensmittel -> To-Go -> Knödel-19773952", + "code": "30", + "url": "https://www.mpreis.at/shop/c/19773952" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Pommes frites & Co.-30944925", + "code": "45", + "url": "https://www.mpreis.at/shop/c/30944925" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Pizza & Baguette-26319064", + "code": "46", + "url": "https://www.mpreis.at/shop/c/26319064" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Mehlspeisen & Teige-44744883", + "code": "47", + "url": "https://www.mpreis.at/shop/c/44744883" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Gemüse & Kräuter-44769631", + "code": "44", + "url": "https://www.mpreis.at/shop/c/44769631" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Früchte-45404641", + "code": "47", + "url": "https://www.mpreis.at/shop/c/45404641" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Fleisch-30248384", + "code": "42", + "url": "https://www.mpreis.at/shop/c/30248384" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Fisch-32280944", + "code": "43", + "url": "https://www.mpreis.at/shop/c/32280944" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Fertiggerichte-24125257", + "code": "42", + "url": "https://www.mpreis.at/shop/c/24125257" + }, + { + "id": "Lebensmittel -> Tiefkühl -> Eis-50054915", + "code": "40", + "url": "https://www.mpreis.at/shop/c/50054915" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Waffeln & Schnitten-30840011", + "code": "60", + "url": "https://www.mpreis.at/shop/c/30840011" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Süßigkeiten-44046508", + "code": "64", + "url": "https://www.mpreis.at/shop/c/44046508" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Schokolade-27652705", + "code": "64", + "url": "https://www.mpreis.at/shop/c/27652705" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Riegel-1040059", + "code": "64", + "url": "https://www.mpreis.at/shop/c/1040059" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Reiswaffeln-38946028", + "code": "63", + "url": "https://www.mpreis.at/shop/c/38946028" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Knabbergebäck-14627798", + "code": "63", + "url": "https://www.mpreis.at/shop/c/14627798" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Kekse & Biskotten-7888241", + "code": "64", + "url": "https://www.mpreis.at/shop/c/7888241" + }, + { + "id": "Lebensmittel -> Süßes & Salziges -> Kaugummi-36233496", + "code": "64", + "url": "https://www.mpreis.at/shop/c/36233496" + }, + { + "id": "Lebensmittel -> Obst -> Zitrusfrüchte-24043806", + "code": "00", + "url": "https://www.mpreis.at/shop/c/24043806" + }, + { + "id": "Lebensmittel -> Obst -> Trauben-30698765", + "code": "00", + "url": "https://www.mpreis.at/shop/c/30698765" + }, + { + "id": "Lebensmittel -> Obst -> Steinobst-44669355", + "code": "00", + "url": "https://www.mpreis.at/shop/c/44669355" + }, + { + "id": "Lebensmittel -> Obst -> Früchte To-Go-36503369", + "code": "00", + "url": "https://www.mpreis.at/shop/c/36503369" + }, + { + "id": "Lebensmittel -> Obst -> Exotische Früchte-39608604", + "code": "00", + "url": "https://www.mpreis.at/shop/c/39608604" + }, + { + "id": "Lebensmittel -> Obst -> Birnen-51716798", + "code": "00", + "url": "https://www.mpreis.at/shop/c/51716798" + }, + { + "id": "Lebensmittel -> Obst -> Beeren-33730528", + "code": "00", + "url": "https://www.mpreis.at/shop/c/33730528" + }, + { + "id": "Lebensmittel -> Obst -> Bananen-45088645", + "code": "00", + "url": "https://www.mpreis.at/shop/c/45088645" + }, + { + "id": "Lebensmittel -> Obst -> Äpfel-40792619", + "code": "00", + "url": "https://www.mpreis.at/shop/c/40792619" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Topfen-19710867", + "code": "34", + "url": "https://www.mpreis.at/shop/c/19710867" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Schlagobers-52987503", + "code": "34", + "url": "https://www.mpreis.at/shop/c/52987503" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Sauerrahm & Crème fraiche-32070142", + "code": "34", + "url": "https://www.mpreis.at/shop/c/32070142" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Pudding-39013909", + "code": "34", + "url": "https://www.mpreis.at/shop/c/39013909" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Pflanzlicher Milchersatz-51710462", + "code": "3B", + "url": "https://www.mpreis.at/shop/c/51710462" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Milch-30552924", + "code": "34", + "url": "https://www.mpreis.at/shop/c/30552924" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Margarine-40393431", + "code": "34", + "url": "https://www.mpreis.at/shop/c/40393431" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Käse-48865859", + "code": "33", + "url": "https://www.mpreis.at/shop/c/48865859" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Joghurt-2307952", + "code": "34", + "url": "https://www.mpreis.at/shop/c/2307952" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Eier-15570058", + "code": "31", + "url": "https://www.mpreis.at/shop/c/15570058" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Desserts-28194646", + "code": "34", + "url": "https://www.mpreis.at/shop/c/28194646" + }, + { + "id": "Lebensmittel -> Milch & Eier -> Butter-48723246", + "code": "34", + "url": "https://www.mpreis.at/shop/c/48723246" + }, + { + "id": "Lebensmittel -> Konserven, Suppen & Fertiggerichte -> Suppen-45429089", + "code": "57", + "url": "https://www.mpreis.at/shop/c/45429089" + }, + { + "id": "Lebensmittel -> Konserven, Suppen & Fertiggerichte -> Saucen-18573555", + "code": "57", + "url": "https://www.mpreis.at/shop/c/18573555" + }, + { + "id": "Lebensmittel -> Konserven, Suppen & Fertiggerichte -> Konserven-30932656", + "code": "57", + "url": "https://www.mpreis.at/shop/c/30932656" + }, + { + "id": "Lebensmittel -> Konserven, Suppen & Fertiggerichte -> Fertiggerichte-5812424", + "code": "54", + "url": "https://www.mpreis.at/shop/c/5812424" + }, + { + "id": "Lebensmittel -> Konserven, Fertiggerichte & Suppen -> Suppen-45429089", + "code": "57", + "url": "https://www.mpreis.at/shop/c/45429089" + }, + { + "id": "Lebensmittel -> Konserven, Fertiggerichte & Suppen -> Saucen-18573555", + "code": "5C", + "url": "https://www.mpreis.at/shop/c/18573555" + }, + { + "id": "Lebensmittel -> Konserven, Fertiggerichte & Suppen -> Konserven-30932656", + "code": "57", + "url": "https://www.mpreis.at/shop/c/30932656" + }, + { + "id": "Lebensmittel -> Konserven, Fertiggerichte & Suppen -> Fertiggerichte-5812424", + "code": "54", + "url": "https://www.mpreis.at/shop/c/5812424" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Zucker-49879760", + "code": "5E", + "url": "https://www.mpreis.at/shop/c/49879760" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Teigwaren-21643018", + "code": "5B", + "url": "https://www.mpreis.at/shop/c/21643018" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Reis-17493179", + "code": "5B", + "url": "https://www.mpreis.at/shop/c/17493179" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Nüsse & Trockenfrüchte-25346130", + "code": "03", + "url": "https://www.mpreis.at/shop/c/25346130" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Mehl-17512389", + "code": "59", + "url": "https://www.mpreis.at/shop/c/17512389" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Ketchup & Senf-29783780", + "code": "5C", + "url": "https://www.mpreis.at/shop/c/29783780" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Gewürze-25428081", + "code": "55", + "url": "https://www.mpreis.at/shop/c/25428081" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Getreide & Samen-42292631", + "code": "57", + "url": "https://www.mpreis.at/shop/c/42292631" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Essig & Öl-18373716", + "code": "53", + "url": "https://www.mpreis.at/shop/c/18373716" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Cerealien & Müsli-3421484", + "code": "5A", + "url": "https://www.mpreis.at/shop/c/3421484" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Brotaufstriche-11264169", + "code": "56", + "url": "https://www.mpreis.at/shop/c/11264169" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Backen-50181948", + "code": "52", + "url": "https://www.mpreis.at/shop/c/50181948" + }, + { + "id": "Lebensmittel -> Grundnahrung -> Babynahrung-45298963", + "code": "51", + "url": "https://www.mpreis.at/shop/c/45298963" + }, + { + "id": "Lebensmittel -> Gemüse -> Zwiebel & Knoblauch-2890053", + "code": "01", + "url": "https://www.mpreis.at/shop/c/2890053" + }, + { + "id": "Lebensmittel -> Gemüse -> Wurzelgemüse-49783396", + "code": "01", + "url": "https://www.mpreis.at/shop/c/49783396" + }, + { + "id": "Lebensmittel -> Gemüse -> Tomaten-24083977", + "code": "01", + "url": "https://www.mpreis.at/shop/c/24083977" + }, + { + "id": "Lebensmittel -> Gemüse -> Salat-15838701", + "code": "01", + "url": "https://www.mpreis.at/shop/c/15838701" + }, + { + "id": "Lebensmittel -> Gemüse -> Pilze-2291441", + "code": "01", + "url": "https://www.mpreis.at/shop/c/2291441" + }, + { + "id": "Lebensmittel -> Gemüse -> Mais, Paprika & Chili-16396458", + "code": "01", + "url": "https://www.mpreis.at/shop/c/16396458" + }, + { + "id": "Lebensmittel -> Gemüse -> Kürbis & Saisongemüse-6676984", + "code": "01", + "url": "https://www.mpreis.at/shop/c/6676984" + }, + { + "id": "Lebensmittel -> Gemüse -> Kräuter-34083247", + "code": "01", + "url": "https://www.mpreis.at/shop/c/34083247" + }, + { + "id": "Lebensmittel -> Gemüse -> Kohlgemüse-21422090", + "code": "01", + "url": "https://www.mpreis.at/shop/c/21422090" + }, + { + "id": "Lebensmittel -> Gemüse -> Kartoffeln-24793838", + "code": "01", + "url": "https://www.mpreis.at/shop/c/24793838" + }, + { + "id": "Lebensmittel -> Gemüse -> Gurken, Zucchini & Auberginen-35313102", + "code": "01", + "url": "https://www.mpreis.at/shop/c/35313102" + }, + { + "id": "Lebensmittel -> Gemüse -> Bohnen & Erbsen-29209051", + "code": "01", + "url": "https://www.mpreis.at/shop/c/29209051" + }, + { + "id": "Lebensmittel -> Fleisch & Wurst -> Würstel-6968763", + "code": "37", + "url": "https://www.mpreis.at/shop/c/6968763" + }, + { + "id": "Lebensmittel -> Fleisch & Wurst -> Wurst-3190180", + "code": "37", + "url": "https://www.mpreis.at/shop/c/3190180" + }, + { + "id": "Lebensmittel -> Fleisch & Wurst -> Speck-27725759", + "code": "37", + "url": "https://www.mpreis.at/shop/c/27725759" + }, + { + "id": "Lebensmittel -> Fleisch & Wurst -> Schinken-24374591", + "code": "37", + "url": "https://www.mpreis.at/shop/c/24374591" + }, + { + "id": "Lebensmittel -> Fleisch & Wurst -> Fleisch-38137534", + "code": "37", + "url": "https://www.mpreis.at/shop/c/38137534" + }, + { + "id": "Lebensmittel -> Fisch-49778040", + "code": "39", + "url": "https://www.mpreis.at/shop/c/49778040" + }, + { + "id": "Lebensmittel -> Fisch -> Fisch-49778040", + "code": "39", + "url": "https://www.mpreis.at/shop/c/49778040" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Zopf, Brioche & Strudel-15366775", + "code": "13", + "url": "https://www.mpreis.at/shop/c/15366775" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Wraps-27642111", + "code": "50", + "url": "https://www.mpreis.at/shop/c/27642111" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Toast-19302036", + "code": "10", + "url": "https://www.mpreis.at/shop/c/19302036" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Kuchen & Konditorei-51377887", + "code": "13", + "url": "https://www.mpreis.at/shop/c/51377887" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Knäckebrot & Zwieback-34477560", + "code": "12", + "url": "https://www.mpreis.at/shop/c/34477560" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Gekühlte Teige-37271369", + "code": "36", + "url": "https://www.mpreis.at/shop/c/37271369" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Gebäck-12919764", + "code": "13", + "url": "https://www.mpreis.at/shop/c/12919764" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Brot-13680199", + "code": "11", + "url": "https://www.mpreis.at/shop/c/13680199" + }, + { + "id": "Lebensmittel -> Brot & Gebäck -> Aufbackbrot-29129924", + "code": "10", + "url": "https://www.mpreis.at/shop/c/29129924" + }, + { + "id": "Getränke -> Wein & Schaumwein -> Wein-46885119", + "code": "25", + "url": "https://www.mpreis.at/shop/c/46885119" + }, + { + "id": "Getränke -> Wein & Schaumwein -> Sekt & Champagner-48612478", + "code": "23", + "url": "https://www.mpreis.at/shop/c/48612478" + }, + { + "id": "Getränke -> Wasser, Limonaden & Energy drinks -> Mineralwasser-40955296", + "code": "26", + "url": "https://www.mpreis.at/shop/c/40955296" + }, + { + "id": "Getränke -> Wasser, Limonaden & Energy drinks -> Energy- & Sportgetränke-17714924", + "code": "20", + "url": "https://www.mpreis.at/shop/c/17714924" + }, + { + "id": "Getränke -> Wasser, Limonaden & Energy drinks -> Eistee-14897036", + "code": "20", + "url": "https://www.mpreis.at/shop/c/14897036" + }, + { + "id": "Getränke -> Wasser, Limonaden & Energy drinks -> Cola & Limonaden-29236807", + "code": "20", + "url": "https://www.mpreis.at/shop/c/29236807" + }, + { + "id": "Getränke -> Spirituosen -> Wodka-48061702", + "code": "24", + "url": "https://www.mpreis.at/shop/c/48061702" + }, + { + "id": "Getränke -> Spirituosen -> Whisky-23353519", + "code": "24", + "url": "https://www.mpreis.at/shop/c/23353519" + }, + { + "id": "Getränke -> Spirituosen -> Wermut-19370553", + "code": "24", + "url": "https://www.mpreis.at/shop/c/19370553" + }, + { + "id": "Getränke -> Spirituosen -> Tequila-23434214", + "code": "24", + "url": "https://www.mpreis.at/shop/c/23434214" + }, + { + "id": "Getränke -> Spirituosen -> Sonstige Bargetränke-45099686", + "code": "24", + "url": "https://www.mpreis.at/shop/c/45099686" + }, + { + "id": "Getränke -> Spirituosen -> Schnaps & Weinbrand-15656322", + "code": "24", + "url": "https://www.mpreis.at/shop/c/15656322" + }, + { + "id": "Getränke -> Spirituosen -> Rum-50367749", + "code": "24", + "url": "https://www.mpreis.at/shop/c/50367749" + }, + { + "id": "Getränke -> Spirituosen -> Portwein & Sherry-40037148", + "code": "24", + "url": "https://www.mpreis.at/shop/c/40037148" + }, + { + "id": "Getränke -> Spirituosen -> Ouzo-48677678", + "code": "24", + "url": "https://www.mpreis.at/shop/c/48677678" + }, + { + "id": "Getränke -> Spirituosen -> Likör-44386568", + "code": "24", + "url": "https://www.mpreis.at/shop/c/44386568" + }, + { + "id": "Getränke -> Spirituosen -> Grappa-24219775", + "code": "24", + "url": "https://www.mpreis.at/shop/c/24219775" + }, + { + "id": "Getränke -> Spirituosen -> Gin-33993661", + "code": "24", + "url": "https://www.mpreis.at/shop/c/33993661" + }, + { + "id": "Getränke -> Spirituosen -> Bitterspirituosen-40175079", + "code": "24", + "url": "https://www.mpreis.at/shop/c/40175079" + }, + { + "id": "Getränke -> Sirup-5799806", + "code": "20", + "url": "https://www.mpreis.at/shop/c/5799806" + }, + { + "id": "Getränke -> Säfte & Sirup -> Sirup-5799806", + "code": "20", + "url": "https://www.mpreis.at/shop/c/5799806" + }, + { + "id": "Getränke -> Säfte & Sirup -> Gemüsesäfte-1366781", + "code": "20", + "url": "https://www.mpreis.at/shop/c/1366781" + }, + { + "id": "Getränke -> Säfte & Sirup -> Fruchtsäfte & Smoothies-31049130", + "code": "20", + "url": "https://www.mpreis.at/shop/c/31049130" + }, + { + "id": "Getränke -> Mineralwasser-6363171", + "code": "26", + "url": "https://www.mpreis.at/shop/c/6363171" + }, + { + "id": "Getränke -> Kaffee, Tee & Kakao -> Tee-15799006", + "code": "22", + "url": "https://www.mpreis.at/shop/c/15799006" + }, + { + "id": "Getränke -> Kaffee, Tee & Kakao -> Kakao-19162087", + "code": "22", + "url": "https://www.mpreis.at/shop/c/19162087" + }, + { + "id": "Getränke -> Kaffee, Tee & Kakao -> Kaffee-20845561", + "code": "22", + "url": "https://www.mpreis.at/shop/c/20845561" + }, + { + "id": "Getränke -> Energy- & Sportgetränke-17714924", + "code": "20", + "url": "https://www.mpreis.at/shop/c/17714924" + }, + { + "id": "Getränke -> Eistee-14897036", + "code": "20", + "url": "https://www.mpreis.at/shop/c/14897036" + }, + { + "id": "Getränke -> Cola & Limonaden-29236807", + "code": "20", + "url": "https://www.mpreis.at/shop/c/29236807" + }, + { + "id": "Getränke -> Bier -> Radler-47039550", + "code": "21", + "url": "https://www.mpreis.at/shop/c/47039550" + }, + { + "id": "Getränke -> Bier -> Bier-41415073", + "code": "21", + "url": "https://www.mpreis.at/shop/c/41415073" + }, + { + "id": "Getränke -> Bier -> alkoholfreies Bier-31647497", + "code": "21", + "url": "https://www.mpreis.at/shop/c/31647497" + }, + { + "id": "Drogerie -> Zeitschriften & Zeitungen-11659073", + "code": null, + "url": "https://www.mpreis.at/shop/c/11659073" + }, + { + "id": "Drogerie -> Wasch- & Putzmittel -> Waschmittel-1856275", + "code": "8A", + "url": "https://www.mpreis.at/shop/c/1856275" + }, + { + "id": "Drogerie -> Wasch- & Putzmittel -> Putzmittel-29619354", + "code": "88", + "url": "https://www.mpreis.at/shop/c/29619354" + }, + { + "id": "Drogerie -> Tchibo – Eduscho-51492995", + "code": "22", + "url": "https://www.mpreis.at/shop/c/51492995" + }, + { + "id": "Drogerie -> Tchibo - Eduscho-51492995", + "code": "22", + "url": "https://www.mpreis.at/shop/c/51492995" + }, + { + "id": "Drogerie -> Specials-28125086", + "code": null, + "url": "https://www.mpreis.at/shop/c/28125086" + }, + { + "id": "Drogerie -> Pflege -> Zahn- & Mundpflege-34880725", + "code": "76", + "url": "https://www.mpreis.at/shop/c/34880725" + }, + { + "id": "Drogerie -> Pflege -> Verhütung & Erotik-9950061", + "code": "7A", + "url": "https://www.mpreis.at/shop/c/9950061" + }, + { + "id": "Drogerie -> Pflege -> Rasur-46343210", + "code": "77", + "url": "https://www.mpreis.at/shop/c/46343210" + }, + { + "id": "Drogerie -> Pflege -> Kosmetik-14566551", + "code": "75", + "url": "https://www.mpreis.at/shop/c/14566551" + }, + { + "id": "Drogerie -> Pflege -> Körperpflege-3011261", + "code": "75", + "url": "https://www.mpreis.at/shop/c/3011261" + }, + { + "id": "Drogerie -> Pflege -> Hygiene-10248480", + "code": "71", + "url": "https://www.mpreis.at/shop/c/10248480" + }, + { + "id": "Drogerie -> Pflege -> Haarpflege-2766928", + "code": "73", + "url": "https://www.mpreis.at/shop/c/2766928" + }, + { + "id": "Drogerie -> Pflege -> Gesundheit-23495476", + "code": "8F", + "url": "https://www.mpreis.at/shop/c/23495476" + }, + { + "id": "Drogerie -> Pflege -> Gesichtspflege-6001187", + "code": "75", + "url": "https://www.mpreis.at/shop/c/6001187" + }, + { + "id": "Drogerie -> Karten & Souvenirs-19001168", + "code": null, + "url": "https://www.mpreis.at/shop/c/19001168" + }, + { + "id": "Drogerie -> Haustiere -> Vogelfutter-45585837", + "code": "93", + "url": "https://www.mpreis.at/shop/c/45585837" + }, + { + "id": "Drogerie -> Haustiere -> Tierstreu-51514384", + "code": "92", + "url": "https://www.mpreis.at/shop/c/51514384" + }, + { + "id": "Drogerie -> Haustiere -> Nagetierfutter-11705135", + "code": "92", + "url": "https://www.mpreis.at/shop/c/11705135" + }, + { + "id": "Drogerie -> Haustiere -> Katzenfutter-14374942", + "code": "91", + "url": "https://www.mpreis.at/shop/c/14374942" + }, + { + "id": "Drogerie -> Haustiere -> Hundefutter-29193940", + "code": "90", + "url": "https://www.mpreis.at/shop/c/29193940" + }, + { + "id": "Drogerie -> Haushalt -> Tragetaschen-35228019", + "code": "86", + "url": "https://www.mpreis.at/shop/c/35228019" + }, + { + "id": "Drogerie -> Haushalt -> Strümpfe & Socken-28080631", + "code": "7C", + "url": "https://www.mpreis.at/shop/c/28080631" + }, + { + "id": "Drogerie -> Haushalt -> Sonstiges-23924996", + "code": null, + "url": "https://www.mpreis.at/shop/c/23924996" + }, + { + "id": "Drogerie -> Haushalt -> Schreibwaren-49971568", + "code": "80", + "url": "https://www.mpreis.at/shop/c/49971568" + }, + { + "id": "Drogerie -> Haushalt -> Partyzubehör-37646513", + "code": "83", + "url": "https://www.mpreis.at/shop/c/37646513" + }, + { + "id": "Drogerie -> Haushalt -> Papierartikel-31159559", + "code": "80", + "url": "https://www.mpreis.at/shop/c/31159559" + }, + { + "id": "Drogerie -> Haushalt -> Müllbeutel-16612791", + "code": "86", + "url": "https://www.mpreis.at/shop/c/16612791" + }, + { + "id": "Drogerie -> Haushalt -> Kerzen-15809798", + "code": "87", + "url": "https://www.mpreis.at/shop/c/15809798" + }, + { + "id": "Drogerie -> Haushalt -> Kaffeefilter-35223819", + "code": "83", + "url": "https://www.mpreis.at/shop/c/35223819" + }, + { + "id": "Drogerie -> Haushalt -> Insektenschutz-25956379", + "code": "8D", + "url": "https://www.mpreis.at/shop/c/25956379" + }, + { + "id": "Drogerie -> Haushalt -> Geschirr-27449830", + "code": "83", + "url": "https://www.mpreis.at/shop/c/27449830" + }, + { + "id": "Drogerie -> Haushalt -> Folien & Beutel-49600533", + "code": "86", + "url": "https://www.mpreis.at/shop/c/49600533" + }, + { + "id": "Drogerie -> Haushalt -> Elektro-20764201", + "code": "83", + "url": "https://www.mpreis.at/shop/c/20764201" + }, + { + "id": "Drogerie -> Haushalt -> Anzündhilfen-21051096", + "code": "83", + "url": "https://www.mpreis.at/shop/c/21051096" + }, + { + "id": "Drogerie -> Gutscheine-19300522", + "code": null, + "url": "https://www.mpreis.at/shop/c/19300522" + }, + { + "id": "Drogerie -> Blumen -> Zimmerpflanzen-12666122", + "code": "81", + "url": "https://www.mpreis.at/shop/c/12666122" + }, + { + "id": "Drogerie -> Blumen -> Schnittblumen-28734182", + "code": "81", + "url": "https://www.mpreis.at/shop/c/28734182" + }, + { + "id": "Drogerie -> Blumen -> Pflanzenzubehör-12464916", + "code": "81", + "url": "https://www.mpreis.at/shop/c/12464916" + }, + { + "id": "Drogerie -> Blumen -> Kränze & Gestecke-16922532", + "code": "81", + "url": "https://www.mpreis.at/shop/c/16922532" + }, + { + "id": "Drogerie -> Blumen -> Freilandpflanzen-29178071", + "code": "81", + "url": "https://www.mpreis.at/shop/c/29178071" + }, + { + "id": "Drogerie -> Blumen -> Blumensamen-14238966", + "code": "81", + "url": "https://www.mpreis.at/shop/c/14238966" + }, + { + "id": "Drogerie -> Baby -> Windeln-31447257", + "code": "79", + "url": "https://www.mpreis.at/shop/c/31447257" + }, + { + "id": "Drogerie -> Baby -> Spiele & Spielzeug-27850986", + "code": "79", + "url": "https://www.mpreis.at/shop/c/27850986" + }, + { + "id": "Drogerie -> Baby -> Flaschen & Schnuller-44286362", + "code": "79", + "url": "https://www.mpreis.at/shop/c/44286362" + }, + { + "id": "Drogerie -> Baby -> Babypflege-6782753", + "code": "79", + "url": "https://www.mpreis.at/shop/c/6782753" + } +] diff --git a/stores/mpreis.js b/stores/mpreis.js index 23bddb8..fa48544 100644 --- a/stores/mpreis.js +++ b/stores/mpreis.js @@ -1,5 +1,6 @@ const axios = require("axios"); const utils = require("./utils"); +const { readJSON } = require("../analysis"); const units = { grm: { unit: "g", factor: 1 }, @@ -58,8 +59,59 @@ exports.fetchData = async function () { return mpreisItems; }; -exports.initializeCategoryMapping = async () => {}; +function categoriesToPath(rawItem) { + if (!rawItem.categories) return null; + const traversePath = (category, result) => { + if (category.name == "ProductRoot") return; + if (category.parent) traversePath(category.parent, result); + result.push({ name: category.name, id: category.id }); + }; + const pathElements = []; + traversePath(rawItem.category, pathElements); + const lastIndex = Math.min(3, pathElements.length) - 1; + const result = + pathElements + .slice(0, lastIndex + 1) + .map((el) => el.name) + .join(" -> ") + + "-" + + pathElements[lastIndex].id; + return result; +} -exports.mapCategory = (rawItem) => {}; +exports.initializeCategoryMapping = async (rawItems) => { + rawItems = rawItems ?? (await exports.fetchData()); + + const categoryLookup = {}; + for (const rawItem of rawItems) { + if (rawItem.categories) { + const path = categoriesToPath(rawItem); + categoryLookup[path] = { + id: path, + code: null, + url: "https://www.mpreis.at/shop/c/" + path.match(/(\d+)$/)[1], + }; + } + } + let categories = []; + Object.keys(categoryLookup).forEach((key) => categories.push(categoryLookup[key])); + categories.sort((a, b) => b.id.localeCompare(a.id)); + categories = utils.mergeAndSaveCategories("mpreis", categories); + exports.categoryLookup = {}; + for (const category of categories) { + exports.categoryLookup[category.id] = category; + } +}; + +exports.mapCategory = (rawItem) => { + const path = categoriesToPath(rawItem); + return exports.categoryLookup[path]; +}; exports.urlBase = "https://www.mpreis.at/shop/p/"; + +if (require.main == module) { + (async () => { + await exports.initializeCategoryMapping(readJSON("data/mpreis-2023-06-21.json.br")); + })(); +} diff --git a/stores/spar.js b/stores/spar.js index 2e257fc..e4a1ae0 100644 --- a/stores/spar.js +++ b/stores/spar.js @@ -1,9 +1,6 @@ const axios = require("axios"); -const fs = require("fs"); -const path = require("path"); const utils = require("./utils"); const HTMLParser = require("node-html-parser"); -const { readJSON } = require("../analysis"); const HITS = Math.floor(30000 + Math.random() * 2000); const units = {