From 8711edb50342a06b4830bde163478a0ad5a30a77 Mon Sep 17 00:00:00 2001 From: simmac Date: Fri, 26 May 2023 00:34:26 +0200 Subject: [PATCH] Added MPREIS support --- analysis.js | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- site/cart.js | 13 +++++++++++++ site/utils.js | 15 +++++++++++---- 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/analysis.js b/analysis.js index e94b6fb..1617b3c 100644 --- a/analysis.js +++ b/analysis.js @@ -98,6 +98,23 @@ function dmToCanonical(rawItems, today) { return canonicalItems; } +function mpreisToCanonical(rawItems, today) { + const canonicalItems = []; + for (let i = 0; i < rawItems.length; i++) { + const item = rawItems[i]; + canonicalItems.push({ + store: "mpreis", + id: item.code, + name: item.name[0], + price: item.prices[0].presentationPrice.effectiveAmount, + priceHistory: [{ date: today, price: item.prices[0].presentationPrice.effectiveAmount }], + unit: `${item.prices[0].presentationPrice.measurementUnit.quantity} ${item.prices[0].presentationPrice.measurementUnit.unitCode}`, + bio: item.mixins.mpreisAttributes.properties?.includes('BIO') + }); + } + return canonicalItems; +} + async function fetchHofer() { const BASE_URL = `https://shopservice.roksh.at` const CATEGORIES = BASE_URL + `/category/GetFullCategoryList/` @@ -188,6 +205,20 @@ async function fetchDm() { return dmItems; } +async function fetchMpreis() { + const url = `https://ax2ixv4hll-dsn.algolia.net/1/indexes/prod_mpreis_8450/browse?X-Algolia-API-Key=NmJlMTI0NjY1NGU4MDUwYTRlMmYzYWFjOWFlY2U4MGFkNGZjMDY2NmNjNjQzNWY3OWJlNDY4OTY0ZjEwOTEwYWZpbHRlcnM9cHVibGlzaGVk&X-Algolia-Application-Id=AX2IXV4HLL&X-Algolia-Agent=Vue.js` + let mpreisItems = []; + let res = (await axios.get(url)).data; + mpreisItems = mpreisItems.concat(res.hits); + cursor = res.cursor; + while (cursor) { + res = (await axios.get(url + `&cursor=${cursor}`)).data; + mpreisItems = mpreisItems.concat(res.hits); + cursor = res.cursor; + } + return mpreisItems; +} + function lidlToCanonical(rawItems, today) { const canonicalItems = []; for (let i = 0; i < rawItems.length; i++) { @@ -258,14 +289,17 @@ exports.replay = function(rawDataDir) { const dmFilesCanonical = dmFiles.map(file => dmToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); const lidlFiles = getFilteredFilesFor("lidl"); const lidlFilesCanonical = lidlFiles.map(file => lidlToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); + const mpreisFiles = getFilteredFilesFor("mpreis"); + const mpreisFilesCanonical = mpreisFiles.map(file => mpreisToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); const allFilesCanonical = []; - const len = Math.max(sparFilesCanonical.length, billaFilesCanonical.length, hoferFilesCanonical.length, lidlFilesCanonical.length, dmFilesCanonical.length); + const len = Math.max(sparFilesCanonical.length, billaFilesCanonical.length, hoferFilesCanonical.length, lidlFilesCanonical.length, dmFilesCanonical.length, mpreisFilesCanonical.length); sparFilesCanonical.reverse(); billaFilesCanonical.reverse(); hoferFilesCanonical.reverse(); dmFilesCanonical.reverse(); lidlFilesCanonical.reverse(); + mpreisFilesCanonical.reverse(); for (let i = 0; i < len; i++) { const canonical = []; let billa = billaFilesCanonical.pop(); @@ -280,6 +314,9 @@ exports.replay = function(rawDataDir) { let lidl = lidlFilesCanonical.pop(); if (lidl) canonical.push(...lidl); allFilesCanonical.push(canonical); + let mpreis = mpreisFilesCanonical.pop(); + if (mpreis) canonical.push(...mpreis); + allFilesCanonical.push(canonical); } if (allFilesCanonical.length == 0) return null; @@ -350,6 +387,16 @@ exports.updateData = async function (dataDir, done) { console.log("Fetched LIDL data, took " + (performance.now() - start) / 1000 + " seconds"); resolve(lidlItemsCanonical) })); + + storeFetchPromises.push(new Promise(async (resolve) => { + const start = performance.now(); + const mpreisItems = await fetchMpreis(); + fs.writeFileSync(`${dataDir}/mpreis-${today}.json`, JSON.stringify(mpreisItems, null, 2)); + const mpreisItemsCanonical = mpreisToCanonical(mpreisItems, today); + console.log("Fetched MPREIS data, took " + (performance.now() - start) / 1000 + " seconds"); + resolve(mpreisItemsCanonical) + })); + const items = [].concat(...await Promise.all(storeFetchPromises)); diff --git a/site/cart.js b/site/cart.js index d172c29..15bf552 100644 --- a/site/cart.js +++ b/site/cart.js @@ -40,6 +40,9 @@ async function load() { document.querySelector("#sumlidl").addEventListener("change", () => { showCharts(canvasDom, cart, lookup); }) + document.querySelector("#summpreis").addEventListener("change", () => { + showCharts(canvasDom, cart, lookup); + }) } function showSearch(cart, items, lookup) { @@ -138,6 +141,16 @@ function showCharts(canvasDom, cart, lookup) { } } + if (document.querySelector("#summpreis").checked) { + const itemsMpreis = items.filter(item => item.store == "mpreis"); + if (itemsMpreis.length > 0) { + itemsToShow.push({ + name: "Summe MPREIS", + priceHistory: calculateOverallPriceChanges(itemsMpreis) + }); + } + } + cart.items.forEach((cartItem) => { const item = lookup[cartItem.id]; if (!item) return; diff --git a/site/utils.js b/site/utils.js index 30401fb..e835cfa 100644 --- a/site/utils.js +++ b/site/utils.js @@ -99,6 +99,8 @@ function itemToStoreLink(item) { return `${item.name}`; if (item.store == "lidl") return `${item.name}`; + if (item.store == "mpreis") + return `${item.name}`; return item.name; } @@ -150,10 +152,11 @@ function itemToDOM(item) { case "dm": row.style["background"] = "rgb(255 240 230)"; break; - case "lidl": row.style["background"] = "rgb(255 225 225)"; break; + case "mpreis": + row.style["background"] = "rgb(255 230 230)"; } row.appendChild(storeDom); row.appendChild(nameDom); @@ -164,7 +167,7 @@ function itemToDOM(item) { let componentId = 0; -function searchItems(items, query, billa, spar, hofer, dm, lidl, eigenmarken, minPrice, maxPrice, exact, bio) { +function searchItems(items, query, billa, spar, hofer, dm, lidl, mpreis, eigenmarken, minPrice, maxPrice, exact, bio) { query = query.trim(); if (query.length < 3) return []; @@ -212,6 +215,7 @@ function searchItems(items, query, billa, spar, hofer, dm, lidl, eigenmarken, mi if (item.store == "hofer" && !hofer) continue; if (item.store == "dm" && !dm) continue; if (item.store == "lidl" && !lidl) continue; + if (item.store == "mpreis" && !mpreis) continue; if (item.price < minPrice) continue; if (item.price > maxPrice) continue; if (eigenmarken && !(name.indexOf("clever") == 0 || name.indexOf("s-budget") == 0 || name.indexOf("milfina") == 0)) continue; @@ -233,6 +237,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi + @@ -255,6 +260,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi const hofer = parentElement.querySelector(`#hofer-${id}`); const dm = parentElement.querySelector(`#dm-${id}`); const lidl = parentElement.querySelector(`#lidl-${id}`); + const mpreis = parentElement.querySelector(`#mpreis-${id}`); const minPrice = parentElement.querySelector(`#minprice-${id}`); const maxPrice = parentElement.querySelector(`#maxprice-${id}`); const numResults = parentElement.querySelector(`#numresults-${id}`); @@ -263,8 +269,8 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi let hits = []; try { hits = searchItems(items, query, - billa.checked, spar.checked, hofer.checked, dm.checked, lidl.checked, eigenmarken.checked, - toNumber(minPrice.value, 0), toNumber(maxPrice.value, 100), exact.checked, bio.checked + billa.checked, spar.checked, hofer.checked, dm.checked, lidl.checked, mpreis.checked, + eigenmarken.checked, toNumber(minPrice.value, 0), toNumber(maxPrice.value, 100), exact.checked, bio.checked ); } catch (e) { console.log("Query: " + query + "\n" + e.message); @@ -312,6 +318,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi hofer.addEventListener("change", () => search(searchInput.value)); dm.addEventListener("change", () => search(searchInput.value)); lidl.addEventListener("change", () => search(searchInput.value)); + mpreis.addEventListener("change", () => search(searchInput.value)); exact.addEventListener("change", () => search(searchInput.value)); minPrice.addEventListener("change", () => search(searchInput.value)); maxPrice.addEventListener("change", () => search(searchInput.value));