From 661ca82f6c359ceb46e45c0d95947b634fa55c56 Mon Sep 17 00:00:00 2001 From: Christian Tschugg Date: Wed, 24 May 2023 20:02:45 +0200 Subject: [PATCH] Add limited support for LIDL --- README.md | 2 ++ analysis.js | 42 +++++++++++++++++++++++++++++++++++++----- site/cart.html | 1 + site/cart.js | 13 +++++++++++++ site/utils.js | 13 +++++++++++-- 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4f425c1..66f3cc5 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ Fetching data for date: 2023-05-23 Fetched SPAR data, took 17.865891209602356 seconds Fetched BILLA data, took 52.95784649944306 seconds Fetched HOFER data, took 64.83968291568756 seconds +Fetched DM data, took 438.77065160000324 seconds +Fetched LIDL data, took 0.77065160000324 seconds Merged price history Example app listening on port 3000 ``` diff --git a/analysis.js b/analysis.js index 6df1599..e700062 100644 --- a/analysis.js +++ b/analysis.js @@ -188,6 +188,23 @@ async function fetchDm() { return dmItems; } +function lidlToCanonical(rawItems, today) { + const canonicalItems = []; + for (let i = 0; i < rawItems.length; i++) { + const item = rawItems[i]; + canonicalItems.push({ + store: "lidl", + id: item.productId, + name: `${item.keyfacts?.supplementalDescription?.concat(" ") ?? ""}${item.fullTitle}`, + price: item.price.price, + priceHistory: [{ date: today, price: item.price.price }], + unit: item.price.basePrice?.text ?? "n/a", + url: item.canonicalUrl + }); + } + return canonicalItems; +} + function mergePriceHistory(oldItems, items) { if (oldItems == null) return items; @@ -228,21 +245,27 @@ exports.replay = function(rawDataDir) { const dateB = new Date(b.match(/\d{4}-\d{2}-\d{2}/)[0]); return dateA - dateB; }; - const sparFiles = files.filter(file => file.indexOf("spar-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file); + + const getFilteredFilesFor = (identifier) => files.filter(file => file.indexOf(`${identifier}-` == 0).sort(dateSort).map(file => rawDataDir + "/" + file)); + + const sparFiles = getFilteredFilesFor("spar"); const sparFilesCanonical = sparFiles.map(file => sparToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); - const billaFiles = files.filter(file => file.indexOf("billa-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file); + const billaFiles = getFilteredFilesFor("billa"); const billaFilesCanonical = billaFiles.map(file => billaToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); - const hoferFiles = files.filter(file => file.indexOf("hofer-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file); + const hoferFiles = getFilteredFilesFor("hofer"); const hoferFilesCanonical = hoferFiles.map(file => hoferToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); const dmFiles = files.filter(file => file.indexOf("dm-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file); const dmFilesCanonical = dmFiles.map(file => dmToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); + const lidlFiles = getFilteredFilesFor("lidl"); + const lidlFilesCanonical = lidlFiles.map(file => lidlToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); const allFilesCanonical = []; - const len = Math.max(Math.max(sparFilesCanonical.length, Math.max(billaFilesCanonical.length, hoferFilesCanonical.length)), dmFilesCanonical.length); + const len = Math.max(sparFilesCanonical.length, billaFilesCanonical.length, hoferFilesCanonical.length, lidlFilesCanonical.length, dmFilesCanonical.length); sparFilesCanonical.reverse(); billaFilesCanonical.reverse(); hoferFilesCanonical.reverse(); dmFilesCanonical.reverse(); + lidlFilesCanonical.reverse(); for (let i = 0; i < len; i++) { const canonical = []; let billa = billaFilesCanonical.pop(); @@ -254,6 +277,8 @@ exports.replay = function(rawDataDir) { allFilesCanonical.push(canonical); let dm = dmFilesCanonical.pop(); if (dm) canonical.push(...dmFilesCanonical.pop()); + let lidl = lidlFilesCanonical.pop(); + if (lidl) canonical.push(...lidl); allFilesCanonical.push(canonical); } @@ -273,6 +298,7 @@ exports.replay = function(rawDataDir) { const HITS = Math.floor(30000 + Math.random() * 2000); const SPAR_SEARCH = `https://search-spar.spar-ics.com/fact-finder/rest/v4/search/products_lmos_at?query=*&q=*&page=1&hitsPerPage=${HITS}`; const BILLA_SEARCH = `https://shop.billa.at/api/search/full?searchTerm=*&storeId=00-10&pageSize=${HITS}`; +const LIDL_SEARCH = `https://www.lidl.at/p/api/gridboxes/AT/de/?max=${HITS}`; exports.updateData = async function (dataDir, done) { const today = currentDate(); @@ -301,8 +327,14 @@ exports.updateData = async function (dataDir, done) { fs.writeFileSync(`${dataDir}/dm-${today}.json`, JSON.stringify(dmItems, null, 2)); const dmItemsCanonical = dmToCanonical(dmItems, today); console.log("Fetched DM data, took " + (performance.now() - start) / 1000 + " seconds"); + + start = performance.now(); + const lidlItems = (await axios.get(LIDL_SEARCH)).data.filter(item => !!item.price.price); + fs.writeFileSync(`${dataDir}/lidl-${today}.json`, JSON.stringify(lidlItems, null, 2)); + const lidlItemsCanonical = lidlToCanonical(lidlItems, today); + console.log("Fetched LIDL data, took " + (performance.now() - start) / 1000 + " seconds"); - const items = [...billaItemsCanonical, ...sparItemsCanonical, ...hoferItemsCanonical, ...dmItemsCanonical]; + const items = [...billaItemsCanonical, ...sparItemsCanonical, ...hoferItemsCanonical, ...dmItemsCanonical, ...lidlItemsCanonical]; if (fs.existsSync(`${dataDir}/latest-canonical.json`)) { const oldItems = JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`)); mergePriceHistory(oldItems, items); diff --git a/site/cart.html b/site/cart.html index b9bc764..4e386e0 100644 --- a/site/cart.html +++ b/site/cart.html @@ -26,6 +26,7 @@ +
diff --git a/site/cart.js b/site/cart.js index 7a38031..d172c29 100644 --- a/site/cart.js +++ b/site/cart.js @@ -37,6 +37,9 @@ async function load() { document.querySelector("#sumdm").addEventListener("change", () => { showCharts(canvasDom, cart, lookup); }) + document.querySelector("#sumlidl").addEventListener("change", () => { + showCharts(canvasDom, cart, lookup); + }) } function showSearch(cart, items, lookup) { @@ -125,6 +128,16 @@ function showCharts(canvasDom, cart, lookup) { } } + if (document.querySelector("#sumlidl").checked) { + const itemsLidl = items.filter(item => item.store == "lidl"); + if (itemsLidl.length > 0) { + itemsToShow.push({ + name: "Summe Lidl", + priceHistory: calculateOverallPriceChanges(itemsLidl) + }); + } + } + cart.items.forEach((cartItem) => { const item = lookup[cartItem.id]; if (!item) return; diff --git a/site/utils.js b/site/utils.js index bfa17ab..a2c4712 100644 --- a/site/utils.js +++ b/site/utils.js @@ -97,6 +97,8 @@ function itemToStoreLink(item) { return `${item.name}`; if (item.store == "dm") return `${item.name}`; + if (item.store == "lidl") + return `${item.name}`; return item.name; } @@ -149,6 +151,9 @@ function itemToDOM(item) { row.style["background"] = "rgb(255 240 230)"; break; + case "lidl": + row.style["background"] = "rgb(255 225 225)"; + break; } row.appendChild(storeDom); row.appendChild(nameDom); @@ -159,7 +164,7 @@ function itemToDOM(item) { let componentId = 0; -function searchItems(items, query, billa, spar, hofer, dm, eigenmarken, minPrice, maxPrice, exact, bio) { +function searchItems(items, query, billa, spar, hofer, dm, lidl, eigenmarken, minPrice, maxPrice, exact, bio) { query = query.trim(); if (query.length < 3) return []; @@ -206,6 +211,7 @@ function searchItems(items, query, billa, spar, hofer, dm, eigenmarken, minPrice if (item.store == "spar" && !spar) continue; if (item.store == "hofer" && !hofer) continue; if (item.store == "dm" && !dm) continue; + if (item.store == "lidl" && !lidl) continue; if (item.price < minPrice) continue; if (item.price > maxPrice) continue; if (eigenmarken && !(name.indexOf("clever") == 0 || name.indexOf("s-budget") == 0 || name.indexOf("milfina") == 0)) continue; @@ -226,6 +232,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi + @@ -247,6 +254,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi const spar = parentElement.querySelector(`#spar-${id}`); const hofer = parentElement.querySelector(`#hofer-${id}`); const dm = parentElement.querySelector(`#dm-${id}`); + const lidl = parentElement.querySelector(`#lidl-${id}`); const minPrice = parentElement.querySelector(`#minprice-${id}`); const maxPrice = parentElement.querySelector(`#maxprice-${id}`); const numResults = parentElement.querySelector(`#numresults-${id}`); @@ -255,7 +263,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi let hits = []; try { hits = searchItems(items, query, - billa.checked, spar.checked, hofer.checked, dm.checked, eigenmarken.checked, + billa.checked, spar.checked, hofer.checked, dm.checked, lidl.checked, eigenmarken.checked, toNumber(minPrice.value, 0), toNumber(maxPrice.value, 100), exact.checked, bio.checked ); } catch (e) { @@ -303,6 +311,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi spar.addEventListener("change", () => search(searchInput.value)); hofer.addEventListener("change", () => search(searchInput.value)); dm.addEventListener("change", () => search(searchInput.value)); + lidl.addEventListener("change", () => search(searchInput.value)); exact.addEventListener("change", () => search(searchInput.value)); minPrice.addEventListener("change", () => search(searchInput.value)); maxPrice.addEventListener("change", () => search(searchInput.value));