Marown/heissepreise
1
0
Fork 0
mirror of https://github.com/badlogic/heissepreise.git synced 2024-06-19 15:05:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add limited support for LIDL

Browse Source
This commit is contained in:
Christian Tschugg 2023-05-24 20:02:45 +02:00
parent 72ec079de2
commit 661ca82f6c
5 changed files with 64 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -29,6 +29,8 @@ Fetching data for date: 2023-05-23
Fetched SPAR data, took 17.865891209602356 seconds
Fetched BILLA data, took 52.95784649944306 seconds
Fetched HOFER data, took 64.83968291568756 seconds
Fetched DM data, took 438.77065160000324 seconds
Fetched LIDL data, took 0.77065160000324 seconds
Merged price history
Example app listening on port 3000
```

42
analysis.js
View File

@ -188,6 +188,23 @@ async function fetchDm() {
return dmItems;
}
function lidlToCanonical(rawItems, today) {
const canonicalItems = [];
for (let i = 0; i < rawItems.length; i++) {
const item = rawItems[i];
canonicalItems.push({
store: "lidl",
id: item.productId,
name: `${item.keyfacts?.supplementalDescription?.concat(" ") ?? ""}${item.fullTitle}`,
price: item.price.price,
priceHistory: [{ date: today, price: item.price.price }],
unit: item.price.basePrice?.text ?? "n/a",
url: item.canonicalUrl
});
}
return canonicalItems;
}
function mergePriceHistory(oldItems, items) {
if (oldItems == null) return items;
@ -228,21 +245,27 @@ exports.replay = function(rawDataDir) {
const dateB = new Date(b.match(/\d{4}-\d{2}-\d{2}/)[0]);
return dateA - dateB;
};
const sparFiles = files.filter(file => file.indexOf("spar-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file);
const getFilteredFilesFor = (identifier) => files.filter(file => file.indexOf(`${identifier}-` == 0).sort(dateSort).map(file => rawDataDir + "/" + file));
const sparFiles = getFilteredFilesFor("spar");
const sparFilesCanonical = sparFiles.map(file => sparToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
const billaFiles = files.filter(file => file.indexOf("billa-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file);
const billaFiles = getFilteredFilesFor("billa");
const billaFilesCanonical = billaFiles.map(file => billaToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
const hoferFiles = files.filter(file => file.indexOf("hofer-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file);
const hoferFiles = getFilteredFilesFor("hofer");
const hoferFilesCanonical = hoferFiles.map(file => hoferToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
const dmFiles = files.filter(file => file.indexOf("dm-") == 0).sort(dateSort).map(file => rawDataDir + "/" + file);
const dmFilesCanonical = dmFiles.map(file => dmToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
const lidlFiles = getFilteredFilesFor("lidl");
const lidlFilesCanonical = lidlFiles.map(file => lidlToCanonical(readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
const allFilesCanonical = [];
const len = Math.max(Math.max(sparFilesCanonical.length, Math.max(billaFilesCanonical.length, hoferFilesCanonical.length)), dmFilesCanonical.length);
const len = Math.max(sparFilesCanonical.length, billaFilesCanonical.length, hoferFilesCanonical.length, lidlFilesCanonical.length, dmFilesCanonical.length);
sparFilesCanonical.reverse();
billaFilesCanonical.reverse();
hoferFilesCanonical.reverse();
dmFilesCanonical.reverse();
lidlFilesCanonical.reverse();
for (let i = 0; i < len; i++) {
const canonical = [];
let billa = billaFilesCanonical.pop();
@ -254,6 +277,8 @@ exports.replay = function(rawDataDir) {
allFilesCanonical.push(canonical);
let dm = dmFilesCanonical.pop();
if (dm) canonical.push(...dmFilesCanonical.pop());
let lidl = lidlFilesCanonical.pop();
if (lidl) canonical.push(...lidl);
allFilesCanonical.push(canonical);
}
@ -273,6 +298,7 @@ exports.replay = function(rawDataDir) {
const HITS = Math.floor(30000 + Math.random() * 2000);
const SPAR_SEARCH = `https://search-spar.spar-ics.com/fact-finder/rest/v4/search/products_lmos_at?query=*&q=*&page=1&hitsPerPage=${HITS}`;
const BILLA_SEARCH = `https://shop.billa.at/api/search/full?searchTerm=*&storeId=00-10&pageSize=${HITS}`;
const LIDL_SEARCH = `https://www.lidl.at/p/api/gridboxes/AT/de/?max=${HITS}`;
exports.updateData = async function (dataDir, done) {
const today = currentDate();
@ -301,8 +327,14 @@ exports.updateData = async function (dataDir, done) {
fs.writeFileSync(`${dataDir}/dm-${today}.json`, JSON.stringify(dmItems, null, 2));
const dmItemsCanonical = dmToCanonical(dmItems, today);
console.log("Fetched DM data, took " + (performance.now() - start) / 1000 + " seconds");
start = performance.now();
const lidlItems = (await axios.get(LIDL_SEARCH)).data.filter(item => !!item.price.price);
fs.writeFileSync(`${dataDir}/lidl-${today}.json`, JSON.stringify(lidlItems, null, 2));
const lidlItemsCanonical = lidlToCanonical(lidlItems, today);
console.log("Fetched LIDL data, took " + (performance.now() - start) / 1000 + " seconds");
const items = [...billaItemsCanonical, ...sparItemsCanonical, ...hoferItemsCanonical, ...dmItemsCanonical];
const items = [...billaItemsCanonical, ...sparItemsCanonical, ...hoferItemsCanonical, ...dmItemsCanonical, ...lidlItemsCanonical];
if (fs.existsSync(`${dataDir}/latest-canonical.json`)) {
const oldItems = JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`));
mergePriceHistory(oldItems, items);

1
site/cart.html
View File

@ -26,6 +26,7 @@
<label>Summe Spar<input type="checkbox" id="sumspar"></label>
<label>Summe Hofer<input type="checkbox" id="sumhofer"></label>
<label>Summe dm<input type="checkbox" id="sumdm"></label>
<label>Summe Lidl<input type="checkbox" id="sumlidl"></label>
</div>
<table id="cartitems"></table>
</div>

13
site/cart.js
View File

@ -37,6 +37,9 @@ async function load() {
document.querySelector("#sumdm").addEventListener("change", () => {
showCharts(canvasDom, cart, lookup);
})
document.querySelector("#sumlidl").addEventListener("change", () => {
showCharts(canvasDom, cart, lookup);
})
}
function showSearch(cart, items, lookup) {
@ -125,6 +128,16 @@ function showCharts(canvasDom, cart, lookup) {
}
}
if (document.querySelector("#sumlidl").checked) {
const itemsLidl = items.filter(item => item.store == "lidl");
if (itemsLidl.length > 0) {
itemsToShow.push({
name: "Summe Lidl",
priceHistory: calculateOverallPriceChanges(itemsLidl)
});
}
}
cart.items.forEach((cartItem) => {
const item = lookup[cartItem.id];
if (!item) return;

13
site/utils.js
View File

@ -97,6 +97,8 @@ function itemToStoreLink(item) {
return `<a target="_blank" href="https://www.roksh.at/hofer/angebot/suche/${encodeURIComponent(item.name)}">${item.name}</a>`;
if (item.store == "dm")
return `<a target="_blank" href="https://www.dm.at/product-p${item.id}.html">${item.name}</a>`;
if (item.store == "lidl")
return `<a target="_blank" href="https://www.lidl.at${item.url}">${item.name}</a>`;
return item.name;
}
@ -149,6 +151,9 @@ function itemToDOM(item) {
row.style["background"] = "rgb(255 240 230)";
break;
case "lidl":
row.style["background"] = "rgb(255 225 225)";
break;
}
row.appendChild(storeDom);
row.appendChild(nameDom);
@ -159,7 +164,7 @@ function itemToDOM(item) {
let componentId = 0;
function searchItems(items, query, billa, spar, hofer, dm, eigenmarken, minPrice, maxPrice, exact, bio) {
function searchItems(items, query, billa, spar, hofer, dm, lidl, eigenmarken, minPrice, maxPrice, exact, bio) {
query = query.trim();
if (query.length < 3) return [];
@ -206,6 +211,7 @@ function searchItems(items, query, billa, spar, hofer, dm, eigenmarken, minPrice
if (item.store == "spar" && !spar) continue;
if (item.store == "hofer" && !hofer) continue;
if (item.store == "dm" && !dm) continue;
if (item.store == "lidl" && !lidl) continue;
if (item.price < minPrice) continue;
if (item.price > maxPrice) continue;
if (eigenmarken && !(name.indexOf("clever") == 0 || name.indexOf("s-budget") == 0 || name.indexOf("milfina") == 0)) continue;
@ -226,6 +232,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi
<label><input id="spar-${id}" type="checkbox" checked="true"> Spar</label>
<label><input id="hofer-${id}" type="checkbox" checked="true"> Hofer</label>
<label><input id="dm-${id}" type="checkbox" checked="true"> DM</label>
<label><input id="lidl-${id}" type="checkbox" checked="true"> Lidl</label>
<label><input id="eigenmarken-${id}" type="checkbox"> Nur CLEVER / S-BUDGET / MILFINA</label>
<label><input id="bio-${id}" type="checkbox"> Nur Bio</label>
</div>
@ -247,6 +254,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi
const spar = parentElement.querySelector(`#spar-${id}`);
const hofer = parentElement.querySelector(`#hofer-${id}`);
const dm = parentElement.querySelector(`#dm-${id}`);
const lidl = parentElement.querySelector(`#lidl-${id}`);
const minPrice = parentElement.querySelector(`#minprice-${id}`);
const maxPrice = parentElement.querySelector(`#maxprice-${id}`);
const numResults = parentElement.querySelector(`#numresults-${id}`);
@ -255,7 +263,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi
let hits = [];
try {
hits = searchItems(items, query,
billa.checked, spar.checked, hofer.checked, dm.checked, eigenmarken.checked,
billa.checked, spar.checked, hofer.checked, dm.checked, lidl.checked, eigenmarken.checked,
toNumber(minPrice.value, 0), toNumber(maxPrice.value, 100), exact.checked, bio.checked
);
} catch (e) {
@ -303,6 +311,7 @@ function newSearchComponent(parentElement, items, searched, filter, headerModifi
spar.addEventListener("change", () => search(searchInput.value));
hofer.addEventListener("change", () => search(searchInput.value));
dm.addEventListener("change", () => search(searchInput.value));
lidl.addEventListener("change", () => search(searchInput.value));
exact.addEventListener("change", () => search(searchInput.value));
minPrice.addEventListener("change", () => search(searchInput.value));
maxPrice.addEventListener("change", () => search(searchInput.value));