Marown
/
heissepreise
mirror of https://github.com/badlogic/heissepreise.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Increased maxWidth to 150 in prettier config, formatted all the things. See #52.

This commit is contained in:
Mario Zechner 2023-06-02 16:45:54 +02:00
parent 1845760384
commit c6bbd0e03b
38 changed files with 11474 additions and 1883 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.prettierignore
View File

@ -1,6 +1,8 @@
data/
.github/
.vscode/
alasql.js
charts.js
latest-canonical*.json
momentum-cart.json
package-lock.json

2
.prettierrc
View File

@ -1,5 +1,5 @@
{
"singleQuote": false,
"bracketSpacing": true,
"printWidth": 100
"printWidth": 150
}

36
README.md
View File

@ -4,13 +4,13 @@ A terrible grocery price search "app". Fetches data from big Austrian grocery ch
You can also get the [raw data](https://heisse-preise.io/api/index). The raw data is returned as a JSON array of items. An item has the following fields:
* `store`: (`billa`, `spar`, `hofer`, `dm`, `lidl`, `mpreis`)
* `name`: the product name.
* `price`: the current price in €.
* `priceHistory`: an array of `{ date: "yyyy-mm-dd", price: number }` objects, sorted in descending order of date.
* `unit`: unit the product is sold at. May be undefined.
* `quantity`: quantity the product is sold at for the given price
* `bio`: whether this product is classified as organic/"Bio"
- `store`: (`billa`, `spar`, `hofer`, `dm`, `lidl`, `mpreis`)
- `name`: the product name.
- `price`: the current price in €.
- `priceHistory`: an array of `{ date: "yyyy-mm-dd", price: number }` objects, sorted in descending order of date.
- `unit`: unit the product is sold at. May be undefined.
- `quantity`: quantity the product is sold at for the given price
- `bio`: whether this product is classified as organic/"Bio"
The project consists of a trivial NodeJS Express server responsible for fetching the product data, massaging it, and serving it to the front end (see `index.js`). The front end is a least-effort vanilla HTML/JS search form (see sources in `site/`).
@ -51,14 +51,14 @@ Create a GitHub account and pick a username. Below, we assume your user name is
1. Log in to your GitHub account.
2. [Fork](https://github.com/badlogic/heissepreise/fork) this repository and name the repository `hotprices123.github.io`.
3. **In your forked repository**:
1. go to `Settings > Pages`, then under `Branch` select the `main` branch, and the `docs/` directory as shown in this screenshot.
![docs/github-pages.png](docs/github-pages.png)
2. go to `Settings > Actions > General`, then under `Workflow permissions`, select `Read and write permissions` as shown in this screenshot.
![docs/github-permissions.png](docs/github-permissions.png)
3. go to the `Actions` tab, then select the `Pages Update` workflow in the list to the left, then click `Enable workflow`. Confirm that you know what you are doing.
![docs/github-workflow.png](docs/github-workflow.png)
1. go to `Settings > Pages`, then under `Branch` select the `main` branch, and the `docs/` directory as shown in this screenshot.
![docs/github-pages.png](docs/github-pages.png)
2. go to `Settings > Actions > General`, then under `Workflow permissions`, select `Read and write permissions` as shown in this screenshot.
![docs/github-permissions.png](docs/github-permissions.png)
3. go to the `Actions` tab, then select the `Pages Update` workflow in the list to the left, then click `Enable workflow`. Confirm that you know what you are doing.
![docs/github-workflow.png](docs/github-workflow.png)
4. Trigger the workflow once manually to build the initial site and data.
![docs/github-workflow2.png](docs/github-workflow2.png)
![docs/github-workflow2.png](docs/github-workflow2.png)
5. Once the workflow has finished, go to `https:/hotprices123.github.io` and enjoy your price comparisons.
The data will be automatically fetched once a day at 8am (no idea what timezone), and the site will be updated.
@ -67,17 +67,17 @@ To get the latest code changes from this repository into your fork:
1. Go to `https://github.com/hotprices123/hotprices123.github.io/compare/main...badlogic:heissepreise:main`
2. Click on `Create pull request`
![docs/github-pullrequest.png](docs/github-pullrequest.png)
![docs/github-pullrequest.png](docs/github-pullrequest.png)
3. Enter a Title like "Updated from upstream", then click `Create pull request``
![docs/github-pullrequest2.png](docs/github-pullrequest2.png)
![docs/github-pullrequest2.png](docs/github-pullrequest2.png)
4. Click `Merge pull request`
![docs/github-pullrequest3.png](docs/github-pullrequest3.png)
![docs/github-pullrequest3.png](docs/github-pullrequest3.png)
Your site will now use the latest source code changes from this repository. It will be automatically updated and is usually live under `https://hotprices123.github.io` within 10-15 minutes.
## Generating a self-contained executable
Run the `package.sh`script in a Bash shell. It will generate a folder `dist/` with executable for Windows, Linux, and MacOS. Run the executable for your OS.
Run the `package.sh`script in a Bash shell. It will generate a folder `dist/` with executable for Windows, Linux, and MacOS. Run the executable for your OS.
## Docker

71
analysis.js
View File

@ -8,8 +8,8 @@ exports.STORE_KEYS = STORE_KEYS;
function currentDate() {
const currentDate = new Date();
const year = currentDate.getFullYear();
const month = String(currentDate.getMonth() + 1).padStart(2, '0');
const day = String(currentDate.getDate()).padStart(2, '0');
const month = String(currentDate.getMonth() + 1).padStart(2, "0");
const day = String(currentDate.getDate()).padStart(2, "0");
return `${year}-${month}-${day}`;
}
@ -28,7 +28,7 @@ function getCanonicalFor(store, rawItems, today) {
if (item)
canonicalItems.push({
store,
...item
...item,
});
}
return canonicalItems;
@ -37,7 +37,7 @@ function getCanonicalFor(store, rawItems, today) {
function mergePriceHistory(oldItems, items) {
if (oldItems == null) return items;
const lookup = {}
const lookup = {};
for (oldItem of oldItems) {
lookup[oldItem.store + oldItem.id] = oldItem;
}
@ -58,7 +58,7 @@ function mergePriceHistory(oldItems, items) {
}
}
console.log(`${Object.keys(lookup).length} not in latest list.`)
console.log(`${Object.keys(lookup).length} not in latest list.`);
for (key of Object.keys(lookup)) {
items.push(lookup[key]);
}
@ -92,8 +92,8 @@ function compress(items) {
const compressed = {
stores: STORE_KEYS,
n: items.length,
data: []
}
data: [],
};
const data = compressed.data;
for (item of items) {
data.push(STORE_KEYS.indexOf(item.store));
@ -148,10 +148,9 @@ exports.compress = compress;
exports.replay = function (rawDataDir) {
const today = currentDate();
const files = fs.readdirSync(rawDataDir).filter(
file => file.indexOf("canonical") == -1 &&
STORE_KEYS.some(store => file.indexOf(`${store}-`) == 0)
);
const files = fs
.readdirSync(rawDataDir)
.filter((file) => file.indexOf("canonical") == -1 && STORE_KEYS.some((store) => file.indexOf(`${store}-`) == 0));
const dateSort = (a, b) => {
const dateA = new Date(a.match(/\d{4}-\d{2}-\d{2}/)[0]);
@ -159,25 +158,29 @@ exports.replay = function (rawDataDir) {
return dateA - dateB;
};
const getFilteredFilesFor = (store) => files.filter(file => file.indexOf(`${store}-`) == 0).sort(dateSort).map(file => rawDataDir + "/" + file);
const getFilteredFilesFor = (store) =>
files
.filter((file) => file.indexOf(`${store}-`) == 0)
.sort(dateSort)
.map((file) => rawDataDir + "/" + file);
const storeFiles = {};
const canonicalFiles = {};
for (const store of STORE_KEYS) {
storeFiles[store] = getFilteredFilesFor(store);
canonicalFiles[store] = storeFiles[store].map(file => getCanonicalFor(store, readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
canonicalFiles[store] = storeFiles[store].map((file) => getCanonicalFor(store, readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
canonicalFiles[store].reverse();
}
const allFilesCanonical = [];
const len = Math.max(...Object.values(canonicalFiles).map(filesByStore => filesByStore.length));
const len = Math.max(...Object.values(canonicalFiles).map((filesByStore) => filesByStore.length));
for (let i = 0; i < len; i++) {
const canonical = [];
Object.values(canonicalFiles).forEach(filesByStore => {
Object.values(canonicalFiles).forEach((filesByStore) => {
const file = filesByStore.pop();
if (file) canonical.push(...file);
})
});
allFilesCanonical.push(canonical);
}
@ -192,29 +195,31 @@ exports.replay = function (rawDataDir) {
prev = curr;
}
return curr;
}
};
exports.updateData = async function (dataDir, done) {
const today = currentDate();
console.log("Fetching data for date: " + today);
const storeFetchPromises = []
const storeFetchPromises = [];
for (const store of STORE_KEYS) {
storeFetchPromises.push(new Promise(async (resolve) => {
const start = performance.now();
try {
const storeItems = await stores[store].fetchData();
fs.writeFileSync(`${dataDir}/${store}-${today}.json`, JSON.stringify(storeItems, null, 2));
const storeItemsCanonical = getCanonicalFor(store, storeItems, today);
console.log(`Fetched ${store.toUpperCase()} data, took ${(performance.now() - start) / 1000} seconds`);
resolve(storeItemsCanonical)
} catch (e) {
console.error(`Error while fetching data from ${store}, continuing after ${(performance.now() - start) / 1000} seconds...`, e);
resolve([])
}
}));
storeFetchPromises.push(
new Promise(async (resolve) => {
const start = performance.now();
try {
const storeItems = await stores[store].fetchData();
fs.writeFileSync(`${dataDir}/${store}-${today}.json`, JSON.stringify(storeItems, null, 2));
const storeItemsCanonical = getCanonicalFor(store, storeItems, today);
console.log(`Fetched ${store.toUpperCase()} data, took ${(performance.now() - start) / 1000} seconds`);
resolve(storeItemsCanonical);
} catch (e) {
console.error(`Error while fetching data from ${store}, continuing after ${(performance.now() - start) / 1000} seconds...`, e);
resolve([]);
}
})
);
}
const items = [].concat(...await Promise.all(storeFetchPromises));
const items = [].concat(...(await Promise.all(storeFetchPromises)));
if (fs.existsSync(`${dataDir}/latest-canonical.json`)) {
const oldItems = JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`));
@ -227,4 +232,4 @@ exports.updateData = async function (dataDir, done) {
if (done) done(items);
return items;
}
};

17
docker/docker-compose.dev.yml
View File

@ -1,11 +1,10 @@
version: "3"
services:
web:
ports:
- 3001:80
site:
ports:
- 9230:9230
environment:
- DEV=true
web:
ports:
- 3001:80
site:
ports:
- 9230:9230
environment:
- DEV=true

27
index.js
View File

@ -5,7 +5,7 @@ function copyItemsToSite(dataDir) {
fs.copyFileSync(`${dataDir}/latest-canonical.json`, `site/latest-canonical.json`);
const items = JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`));
for (const store of analysis.STORE_KEYS) {
const storeItems = items.filter(item => item.store === store);
const storeItems = items.filter((item) => item.store === store);
fs.writeFileSync(`site/latest-canonical.${store}.compressed.json`, JSON.stringify(analysis.compress(storeItems)));
}
}
@ -31,12 +31,11 @@ function scheduleFunction(hour, minute, second, func) {
}, delay);
}
(async () => {
const dataDir = 'data';
const dataDir = "data";
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir)
fs.mkdirSync(dataDir);
}
if (fs.existsSync(`${dataDir}/latest-canonical.json`)) {
@ -45,23 +44,23 @@ function scheduleFunction(hour, minute, second, func) {
copyItemsToSite(dataDir);
});
} else {
await analysis.updateData(dataDir)
await analysis.updateData(dataDir);
copyItemsToSite(dataDir);
}
scheduleFunction(7, 0, 0, async () => {
items = await analysis.updateData(dataDir)
items = await analysis.updateData(dataDir);
copyItemsToSite(dataDir);
});
const express = require('express')
const compression = require('compression');
const app = express()
const port = process?.argv?.[2] ?? 3000
const express = require("express");
const compression = require("compression");
const app = express();
const port = process?.argv?.[2] ?? 3000;
app.use(compression());
app.use(express.static('site'));
app.use(express.static("site"));
app.listen(port, () => {
console.log(`Example app listening on port ${port}`)
})
})();
console.log(`Example app listening on port ${port}`);
});
})();

6
pages.js
View File

@ -3,9 +3,9 @@ const analysis = require("./analysis.js");
if (process.argv.length < 3) {
console.log("Usage: node pages.js <data-dir>");
console.log()
console.log();
console.log("e.g. node pages.js data/");
console.log()
console.log();
process.exit(1);
}
const dataDir = process.argv[2];
@ -19,7 +19,7 @@ if (!fs.existsSync(dataDir)) {
await analysis.updateData(dataDir);
const items = JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`));
for (const store of analysis.STORE_KEYS) {
const storeItems = items.filter(item => item.store === store);
const storeItems = items.filter((item) => item.store === store);
fs.writeFileSync(`${dataDir}/latest-canonical.${store}.compressed.json`, JSON.stringify(analysis.compress(storeItems)));
}
console.log(`Wrote ${items.length} items to ${dataDir}/latest-canonical(-compressed).json`);

2
restore.js
View File

@ -1,6 +1,6 @@
const fs = require("fs");
const analysis = require("./analysis.js");
const dataDir = process?.argv?.[2] ?? "docker/data"
const dataDir = process?.argv?.[2] ?? "docker/data";
console.log("Restoring data from raw data.");
(async function () {
/*console.log("Items: " + JSON.parse(fs.readFileSync("docker/data/latest-canonical.json")).length);

57
site/aktionen.html
View File

@ -1,34 +1,31 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body>
<div class="column">
<h2 id="date"></h2>
<div class="filters" id="filters-store">
<body>
<div class="column">
<h2 id="date"></h2>
<div class="filters" id="filters-store"></div>
<div class="filters" id="filters-changes">
<label><input id="increases" type="checkbox" checked="true" /> Teurer</label>
<label><input id="decreases" type="checkbox" checked="true" /> Billiger</label>
</div>
<input id="filter" type="text" style="max-width: 800px; width: 100%" placeholder="Filtern..." />
<div id="numresults" style="margin-top: 1em"></div>
<table id="result"></table>
</div>
<div class="filters" id="filters-changes">
<label><input id="increases" type="checkbox" checked="true"> Teurer</label>
<label><input id="decreases" type="checkbox" checked="true"> Billiger</label>
</div>
<input id="filter" type="text" style="max-width: 800px; width: 100%;" placeholder="Filtern...">
<div id="numresults" style="margin-top: 1em"></div>
<table id="result"></table>
</div>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="aktionen.js"></script>
</body>
</html>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="aktionen.js"></script>
</body>
</html>

41
site/aktionen.js
View File

@ -23,13 +23,18 @@ async function load() {
document.querySelector("#date").innerText = "Preisänderungen am " + currentDate();
const filtersStore = document.querySelector("#filters-store");
filtersStore.innerHTML = STORE_KEYS.map(store => `<label><input id="${store}" type="checkbox" checked="true">${stores[store].name}</label>`).join(" ");
filtersStore.querySelectorAll("input").forEach(input => {
input.addEventListener("change", () => showResults(items, currentDate()));
});
document.querySelector("#filters-changes").querySelectorAll("input").forEach(input => {
filtersStore.innerHTML = STORE_KEYS.map(
(store) => `<label><input id="${store}" type="checkbox" checked="true">${stores[store].name}</label>`
).join(" ");
filtersStore.querySelectorAll("input").forEach((input) => {
input.addEventListener("change", () => showResults(items, currentDate()));
});
document
.querySelector("#filters-changes")
.querySelectorAll("input")
.forEach((input) => {
input.addEventListener("change", () => showResults(items, currentDate()));
});
document.querySelector("#filter").addEventListener("input", () => showResults(items, currentDate()));
showResults(items, currentDate());
}
@ -37,8 +42,8 @@ async function load() {
function showResults(items, today) {
const increases = document.querySelector("#increases").checked;
const decreases = document.querySelector("#decreases").checked;
const storeCheckboxes = STORE_KEYS.map(store => document.querySelector(`#${store}`));
const checkedStores = STORE_KEYS.filter((store, i) => storeCheckboxes[i].checked)
const storeCheckboxes = STORE_KEYS.map((store) => document.querySelector(`#${store}`));
const checkedStores = STORE_KEYS.filter((store, i) => storeCheckboxes[i].checked);
let changedItems = [];
for (item of items) {
if (item.priceHistory.length < 2) continue;
@ -47,28 +52,32 @@ function showResults(items, today) {
if (!checkedStores.includes(item.store)) continue;
if (item.priceHistory[i].date == today && i + 1 < item.priceHistory.length) {
if (increases && (item.priceHistory[i].price > item.priceHistory[i + 1].price)) changedItems.push(item);
if (decreases && (item.priceHistory[i].price < item.priceHistory[i + 1].price)) changedItems.push(item);
if (increases && item.priceHistory[i].price > item.priceHistory[i + 1].price) changedItems.push(item);
if (decreases && item.priceHistory[i].price < item.priceHistory[i + 1].price) changedItems.push(item);
}
}
}
const query = document.querySelector("#filter").value.trim();
const total = changedItems.length;
if (query.length >= 3) changedItems = searchItems(changedItems, document.querySelector("#filter").value, checkedStores, false, 0, 10000, false, false);
if (query.length >= 3)
changedItems = searchItems(changedItems, document.querySelector("#filter").value, checkedStores, false, 0, 10000, false, false);
document.querySelector("#numresults").innerText = "Resultate: " + changedItems.length + (total > changedItems.length ? " / " + total : "");
const table = document.querySelector("#result");
table.innerHTML = "";
const header = dom("thead", `
const header = dom(
"thead",
`
<tr><th>Kette</th><th>Name</th><th>Menge</th><th>Preis 📈</th></tr>
`)
const showHideAll = header.querySelectorAll('th:nth-child(4)')[0];
`
);
const showHideAll = header.querySelectorAll("th:nth-child(4)")[0];
showHideAll.style["cursor"] = "pointer";
showHideAll.showAll = true;
showHideAll.addEventListener("click", () => {
table.querySelectorAll(".priceinfo").forEach(el => showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide"));
table.querySelectorAll(".priceinfo").forEach((el) => (showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide")));
showHideAll.showAll = !showHideAll.showAll;
})
});
table.appendChild(header);
@ -79,4 +88,4 @@ function showResults(items, today) {
}
}
load();
load();

49
site/billiger.html
View File

@ -1,30 +1,27 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body>
<div class="column">
<h2>Billiger seit letzter Preisänderung</h2>
<div class="filters" id="filters-store">
<body>
<div class="column">
<h2>Billiger seit letzter Preisänderung</h2>
<div class="filters" id="filters-store"></div>
<input id="filter" type="text" style="max-width: 800px; width: 100%" placeholder="Filtern..." />
<div id="numresults" style="margin-top: 1em"></div>
<table id="result"></table>
</div>
<input id="filter" type="text" style="max-width: 800px; width: 100%;" placeholder="Filtern...">
<div id="numresults" style="margin-top: 1em"></div>
<table id="result"></table>
</div>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="billiger.js"></script>
</body>
</html>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="billiger.js"></script>
</body>
</html>

31
site/billiger.js
View File

@ -20,8 +20,10 @@ async function load() {
});
const filtersStore = document.querySelector("#filters-store");
filtersStore.innerHTML = STORE_KEYS.map(store => `<label><input id="${store}" type="checkbox" checked="true">${stores[store].name}</label>`).join(" ");
filtersStore.querySelectorAll("input").forEach(input => {
filtersStore.innerHTML = STORE_KEYS.map(
(store) => `<label><input id="${store}" type="checkbox" checked="true">${stores[store].name}</label>`
).join(" ");
filtersStore.querySelectorAll("input").forEach((input) => {
input.addEventListener("change", () => showResults(items, currentDate()));
});
document.querySelector("#filter").addEventListener("input", () => showResults(items, currentDate()));
@ -29,34 +31,37 @@ async function load() {
}
function showResults(items, _today) {
const storeCheckboxes = STORE_KEYS.map(store => document.querySelector(`#${store}`));
const checkedStores = STORE_KEYS.filter((_store, i) => storeCheckboxes[i].checked)
const storeCheckboxes = STORE_KEYS.map((store) => document.querySelector(`#${store}`));
const checkedStores = STORE_KEYS.filter((_store, i) => storeCheckboxes[i].checked);
let changedItems = [];
for (item of items) {
if (item.priceHistory.length < 2) continue;
if (!checkedStores.includes(item.store)) continue;
if (item.priceHistory[0].price < item.priceHistory[1].price && item.priceHistory[1].date.indexOf("2020") != 0)
changedItems.push(item);
if (item.priceHistory[0].price < item.priceHistory[1].price && item.priceHistory[1].date.indexOf("2020") != 0) changedItems.push(item);
}
const query = document.querySelector("#filter").value.trim();
const total = changedItems.length;
if (query.length >= 3) changedItems = searchItems(changedItems, document.querySelector("#filter").value, checkedStores, false, 0, 10000, false, false);
if (query.length >= 3)
changedItems = searchItems(changedItems, document.querySelector("#filter").value, checkedStores, false, 0, 10000, false, false);
document.querySelector("#numresults").innerText = "Resultate: " + changedItems.length + (total > changedItems.length ? " / " + total : "");
const table = document.querySelector("#result");
table.innerHTML = "";
const header = dom("thead", `
const header = dom(
"thead",
`
<tr><th>Kette</th><th>Name</th><th>Menge</th><th>Preis 📈</th></tr>
`)
const showHideAll = header.querySelectorAll('th:nth-child(4)')[0];
`
);
const showHideAll = header.querySelectorAll("th:nth-child(4)")[0];
showHideAll.style["cursor"] = "pointer";
showHideAll.showAll = true;
showHideAll.addEventListener("click", () => {
table.querySelectorAll(".priceinfo").forEach(el => showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide"));
table.querySelectorAll(".priceinfo").forEach((el) => (showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide")));
showHideAll.showAll = !showHideAll.showAll;
})
});
table.appendChild(header);
@ -67,4 +72,4 @@ function showResults(items, _today) {
}
}
load();
load();

85
site/cart.html
View File

@ -1,48 +1,45 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Heisse Preise</title>
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Heisse Preise</title>
<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body>
<div class="column">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html">Produktsuche</a>
<a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html">Warenkörbe</a>
<body>
<div class="column">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html">Produktsuche</a>
<a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html">Warenkörbe</a>
</div>
<div id="cart" class="cart column">
<h3 id="cartname"></h3>
<input type="button" id="save" value="Speichern" class="hide" />
<canvas id="chart"></canvas>
<div class="filters" style="margin-top: 1em">
<label><input type="checkbox" id="sum" checked /> Preissumme Warenkorb</label>
<label><input type="checkbox" id="sumstores" checked /> Preissumme pro Kette</label>
<label><input type="checkbox" id="todayonly" /> Nur heutige Preise</label>
</div>
<div>
<label>Von <input id="start" type="date" /></label>
<label>Bis <input id="end" type="date" /></label>
</div>
<hr />
<input id="filter" type="text" style="max-width: 800px; width: 100%; margin-bottom: 1em" placeholder="Filtern..." />
<div class="filters" id="filters-store"></div>
<div id="numitems"></div>
<table id="cartitems"></table>
</div>
<div id="search" class="column"></div>
</div>
<div id="cart" class="cart column">
<h3 id="cartname"></h3>
<input type="button" id="save" value="Speichern" class="hide">
<canvas id="chart"></canvas>
<div class="filters" style="margin-top: 1em;">
<label><input type="checkbox" id="sum" checked> Preissumme Warenkorb</label>
<label><input type="checkbox" id="sumstores" checked> Preissumme pro Kette</label>
<label><input type="checkbox" id="todayonly"> Nur heutige Preise</label>
</div>
<div>
<label>Von <input id="start" type="date"></label>
<label>Bis <input id="end"type="date"></label>
</div>
<hr>
<input id="filter" type="text" style="max-width: 800px; width: 100%; margin-bottom: 1em;" placeholder="Filtern...">
<div class="filters" id="filters-store">
</div>
<div id="numitems"></div>
<table id="cartitems"></table>
</div>
<div id="search" class="column"></div>
</div>
<script src="chart.js"></script>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="cart.js"></script>
</body>
</html>
<script src="chart.js"></script>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="cart.js"></script>
</body>
</html>

70
site/cart.js
View File

@ -35,7 +35,7 @@ async function load() {
cart = {
name: tokens[0],
items: [],
linked: true
linked: true,
};
for (let i = 1; i < tokens.length; i++) {
const item = lookup[tokens[i]];
@ -73,16 +73,18 @@ async function load() {
document.querySelector("#end").value = currentDate();
const filtersStore = document.querySelector("#filters-store");
filtersStore.innerHTML = STORE_KEYS.map(store => `<label><input id="${store}" type="checkbox" checked="true">${stores[store].name}</label>`).join(" ");
filtersStore.querySelectorAll("input").forEach(input => input.addEventListener("change", () => showCart(cart)));
filtersStore.innerHTML = STORE_KEYS.map(
(store) => `<label><input id="${store}" type="checkbox" checked="true">${stores[store].name}</label>`
).join(" ");
filtersStore.querySelectorAll("input").forEach((input) => input.addEventListener("change", () => showCart(cart)));
document.querySelector("#filter").addEventListener("input", () => showCart(cart));
showCart(cart);
}
function filter(cartItems) {
const query = document.querySelector("#filter").value.trim();
const storeCheckboxes = STORE_KEYS.map(store => document.querySelector(`#${store}`));
const checkedStores = STORE_KEYS.filter((store, i) => storeCheckboxes[i].checked)
const storeCheckboxes = STORE_KEYS.map((store) => document.querySelector(`#${store}`));
const checkedStores = STORE_KEYS.filter((store, i) => storeCheckboxes[i].checked);
let items = [];
if (query.charAt(0) != "!") {
for (item of cartItems) {
@ -99,21 +101,28 @@ function filter(cartItems) {
function showSearch(cart, items) {
const searchDom = document.querySelector("#search");
searchDom.innerHTML = "";
newSearchComponent(searchDom, items, null, null, (header) => {
header.append(dom("th", ""));
return header;
}, (item, itemDom) => {
const cell = dom("td", `<input type="button" value="+">`);
cell.children[0].addEventListener("click", () => {
cart.items.push(item);
shoppingCarts.save();
document.querySelector("#start").value = getOldestDate(cart.items);
document.querySelector("#end").value = currentDate();
showCart(cart);
});
itemDom.appendChild(cell);
return itemDom;
});
newSearchComponent(
searchDom,
items,
null,
null,
(header) => {
header.append(dom("th", ""));
return header;
},
(item, itemDom) => {
const cell = dom("td", `<input type="button" value="+">`);
cell.children[0].addEventListener("click", () => {
cart.items.push(item);
shoppingCarts.save();
document.querySelector("#start").value = getOldestDate(cart.items);
document.querySelector("#end").value = currentDate();
showCart(cart);
});
itemDom.appendChild(cell);
return itemDom;
}
);
}
function updateCharts(canvasDom, items) {
@ -136,7 +145,7 @@ function updateCharts(canvasDom, items) {
}
function showCart(cart) {
let link = encodeURIComponent(cart.name) + ";"
let link = encodeURIComponent(cart.name) + ";";
for (cartItem of cart.items) {
link += cartItem.store + cartItem.id + ";";
}
@ -157,14 +166,17 @@ function showCart(cart) {
itemTable.append(header);
items.forEach((cartItem, idx) => {
const itemDom = itemToDOM(cartItem)
const itemDom = itemToDOM(cartItem);
const cell = dom("td", `
const cell = dom(
"td",
`
<input type="checkbox">
<input type="button" value="-">
<input type="button" value="⬆️">
<input type="button" value="⬇️">
`);
`
);
if (cartItem.chart) cell.children[0].setAttribute("checked", true);
cell.children[0].addEventListener("change", () => {
@ -179,7 +191,7 @@ function showCart(cart) {
shoppingCarts.save();
document.querySelector("#start").value = getOldestDate(cart.items);
document.querySelector("#end").value = currentDate();
showCart(cart)
showCart(cart);
});
cell.children[2].addEventListener("click", () => {
@ -188,7 +200,7 @@ function showCart(cart) {
cart.items[idx - 1] = cartItem;
cart.items[idx] = otherItem;
shoppingCarts.save();
showCart(cart)
showCart(cart);
});
cell.children[3].addEventListener("click", () => {
@ -197,10 +209,10 @@ function showCart(cart) {
cart.items[idx + 1] = cartItem;
cart.items[idx] = otherItem;
shoppingCarts.save();
showCart(cart)
showCart(cart);
});
} else {
cell.querySelectorAll("input[type='button']").forEach(button => button.classList.add("hide"));
cell.querySelectorAll("input[type='button']").forEach((button) => button.classList.add("hide"));
}
itemDom.append(cell);
@ -208,4 +220,4 @@ function showCart(cart) {
});
}
load();
load();

64
site/carts.html
View File

@ -1,37 +1,35 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body>
<div class="column">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html">Produktsuche</a>
<a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html"><strong>Warenkörbe</strong></a>
<body>
<div class="column">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html">Produktsuche</a>
<a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html"><strong>Warenkörbe</strong></a>
</div>
<div class="filters">
<button id="newcart">Neuer Warenkorb</button>
<button id="export">Exportieren</button>
<button id="import">Importieren</button>
</div>
<table id="carts" class="carts"></table>
</div>
<div class="filters">
<button id="newcart">Neuer Warenkorb</button>
<button id="export">Exportieren</button>
<button id="import">Importieren</button>
</div>
<table id="carts" class="carts"></table>
</div>
<input type="file" id="fileInput" style="display: none;">
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="carts.js"></script>
</body>
</html>
<input type="file" id="fileInput" style="display: none" />
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="carts.js"></script>
</body>
</html>

38
site/carts.js
View File

@ -20,7 +20,7 @@ async function load() {
}
shoppingCarts.save();
if (shoppingCarts.carts.findIndex(cart => cart.name === "Momentum Eigenmarken Vergleich") == -1) {
if (shoppingCarts.carts.findIndex((cart) => cart.name === "Momentum Eigenmarken Vergleich") == -1) {
response = await fetch("momentum-cart.json");
momentumCart = await response.json();
shoppingCarts.carts.unshift(momentumCart);
@ -48,11 +48,11 @@ async function load() {
const importButton = document.querySelector("#import");
importButton.addEventListener("click", () => {
document.getElementById('fileInput').value = null
document.getElementById('fileInput').click();
document.getElementById("fileInput").value = null;
document.getElementById("fileInput").click();
});
document.querySelector("#fileInput").addEventListener('change', function (event) {
document.querySelector("#fileInput").addEventListener("change", function (event) {
const file = event.target.files[0];
const reader = new FileReader();
reader.onload = function (event) {
@ -67,12 +67,12 @@ async function load() {
}
importedCart.items = items;
const index = shoppingCarts.carts.findIndex(cart => cart.name === importedCart.name);
const index = shoppingCarts.carts.findIndex((cart) => cart.name === importedCart.name);
if (index != -1) {
if (confirm("Existierenden Warenkorb '" + importedCart.name + " überschreiben?")) {
console.log(shoppingCarts.carts[index]);
shoppingCarts.carts[index] = importedCart;
console.log(shoppingCarts.carts[index])
console.log(shoppingCarts.carts[index]);
}
} else {
shoppingCarts.carts.push(importedCart);
@ -90,19 +90,24 @@ async function load() {
function showCarts(lookup) {
const cartsTable = document.querySelector("#carts");
cartsTable.innerHTML = "";
cartsTable.appendChild(dom("thead", `
cartsTable.appendChild(
dom(
"thead",
`
<tr>
<th>Name</th>
<th>Produkte</th>
<th>Preis</th>
<th></th>
</tr>
`));
`
)
);
shoppingCarts.carts.forEach(cart => {
shoppingCarts.carts.forEach((cart) => {
let oldPrice = 0;
let currPrice = 0;
let link = encodeURIComponent(cart.name) + ";"
let link = encodeURIComponent(cart.name) + ";";
for (cartItem of cart.items) {
const item = lookup[cartItem.store + cartItem.id];
if (!item) continue;
@ -110,7 +115,7 @@ function showCarts(lookup) {
currPrice += item.priceHistory[0].price;
link += item.store + item.id + ";";
}
const increase = oldPrice != 0 ? Math.round((currPrice - oldPrice) / oldPrice * 100) : 0;
const increase = oldPrice != 0 ? Math.round(((currPrice - oldPrice) / oldPrice) * 100) : 0;
const row = dom("tr", ``);
@ -122,12 +127,15 @@ function showCarts(lookup) {
itemsDom.setAttribute("data-label", "Produkte");
row.appendChild(itemsDom);
const priceDom = dom("td", `<span style="color: ${currPrice > oldPrice ? "red" : "green"}">${currPrice.toFixed(2)} ${(increase > 0 ? "+" : "") + increase + "%"}`);
const priceDom = dom(
"td",
`<span style="color: ${currPrice > oldPrice ? "red" : "green"}">${currPrice.toFixed(2)} ${(increase > 0 ? "+" : "") + increase + "%"}`
);
priceDom.setAttribute("data-label", "Preis");
row.appendChild(priceDom);
const actionsDom = dom("div", "");
actionsDom.classList.add("cartactions")
actionsDom.classList.add("cartactions");
const linkDom = dom("a", "Teilen");
linkDom.setAttribute("href", "cart.html?cart=" + link);
actionsDom.appendChild(linkDom);
@ -137,7 +145,7 @@ function showCarts(lookup) {
jsonDom.addEventListener("click", (event) => {
event.preventDefault();
downloadFile(cart.name + ".json", JSON.stringify(cart, null, 2));
})
});
actionsDom.appendChild(jsonDom);
if (cart.name != "Momentum Eigenmarken Vergleich") {
@ -158,4 +166,4 @@ function showCarts(lookup) {
});
}
load();
load();

1484
site/carts.json
View File

File diff suppressed because it is too large Load Diff

73
site/changes.html
View File

@ -1,40 +1,41 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body>
<div class="column">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html">Produktsuche</a>
<a href="changes.html"><strong>Tagespreisänderungen</strong></a>
<a href="carts.html">Warenkörbe</a>
<body>
<div class="column">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html">Produktsuche</a>
<a href="changes.html"><strong>Tagespreisänderungen</strong></a>
<a href="carts.html">Warenkörbe</a>
</div>
<div class="filters">
<label
>Preisänderungen am
<select id="dates"></select
></label>
</div>
<div class="filters">
<label><input id="increases" type="checkbox" checked="true" />Teurer</label>
<label><input id="decreases" type="checkbox" checked="true" />Billiger</label>
<label><input id="fullhistory" type="checkbox" checked="true" />Gesamte Preishistorie</label>
</div>
<div id="results"></div>
<table id="result"></table>
</div>
<div class="filters">
<label>Preisänderungen am <select id="dates"></select></label>
</div>
<div class="filters">
<label><input id="increases" type="checkbox" checked="true">Teurer</label>
<label><input id="decreases" type="checkbox" checked="true">Billiger</label>
<label><input id="fullhistory" type="checkbox" checked="true">Gesamte Preishistorie</label>
</div>
<div id="results"></div>
<table id="result"></table>
</div>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="changes.js"></script>
</body>
</html>
<script src="alasql.js"></script>
<script src="utils.js"></script>
<script src="changes.js"></script>
</body>
</html>

27
site/changes.js
View File

@ -38,13 +38,13 @@ async function load() {
dateSelection.addEventListener("change", () => {
showResults(items, dateSelection.value);
})
});
document.querySelector("#increases").addEventListener("change", () => {
showResults(items, dateSelection.value);
})
});
document.querySelector("#decreases").addEventListener("change", () => {
showResults(items, dateSelection.value);
})
});
}
function showResults(items, today) {
@ -57,24 +57,27 @@ function showResults(items, today) {
for (let i = 0; i < item.priceHistory.length; i++) {
if (item.priceHistory[i].date == today && i + 1 < item.priceHistory.length) {
if (increases && (item.priceHistory[i].price > item.priceHistory[i + 1].price)) changedItems.push(item);
if (decreases && (item.priceHistory[i].price < item.priceHistory[i + 1].price)) changedItems.push(item);
if (increases && item.priceHistory[i].price > item.priceHistory[i + 1].price) changedItems.push(item);
if (decreases && item.priceHistory[i].price < item.priceHistory[i + 1].price) changedItems.push(item);
}
}
}
const table = document.querySelector("#result");
table.innerHTML = "";
const header = dom("thead", `
const header = dom(
"thead",
`
<tr><th>Kette</th><th>Name</th><th>Menge</th><th>Preis 📈</th></tr>
`)
const showHideAll = header.querySelectorAll('th:nth-child(4)')[0];
`
);
const showHideAll = header.querySelectorAll("th:nth-child(4)")[0];
showHideAll.style["cursor"] = "pointer";
showHideAll.showAll = true;
showHideAll.addEventListener("click", () => {
table.querySelectorAll(".priceinfo").forEach(el => showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide"));
table.querySelectorAll(".priceinfo").forEach((el) => (showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide")));
showHideAll.showAll = !showHideAll.showAll;
})
});
table.appendChild(header);
@ -82,7 +85,7 @@ function showResults(items, today) {
item = JSON.parse(JSON.stringify(item));
if (!fullHistory) {
let priceHistory = [];
for(let i = 0;i < item.priceHistory.length; i++) {
for (let i = 0; i < item.priceHistory.length; i++) {
priceHistory.push(item.priceHistory[i]);
if (item.priceHistory[i].date == today) break;
}
@ -92,4 +95,4 @@ function showResults(items, today) {
document.querySelector("#results").innerText = "Resultate: " + changedItems.length;
}
load();
load();

9673
site/chart.js
View File

File diff suppressed because one or more lines are too long

34
site/index-new.html
View File

@ -1,19 +1,19 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body style="background: black;">
<div class="main">
<img src="joeh.png">
</div>
</body>
</html>
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<body style="background: black">
<div class="main">
<img src="joeh.png" />
</div>
</body>
</html>

75
site/index.html
View File

@ -1,45 +1,42 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Heisse Preise</title>
<link
rel="icon"
href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%2210 0 100 100%22><text y=%22.90em%22 font-size=%2290%22>🔥</text></svg>"
/>
<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body>
<div class="column" style="max-width: 1000px">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html"><strong>Produktsuche</strong></a>
<a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html">Warenkörbe</a>
</div>
<div id="chart" class="column hide" style="width: 80%">
<canvas></canvas>
<div class="filters" style="margin-top: 1em;">
<label><input type="checkbox" id="sum"> Preissumme Gesamt</label>
<label><input type="checkbox" id="sumstores"> Preissumme pro Kette</label>
<label><input type="checkbox" id="todayonly"> Nur heutige Preise</label>
<body>
<div class="column" style="max-width: 1000px">
<h2>Heisse Preise</h2>
<div class="filters">
<a href="index.html"><strong>Produktsuche</strong></a>
<a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html">Warenkörbe</a>
</div>
<div>
<label>Von <input id="start" type="date"></label>
<label>Bis <input id="end"type="date"></label>
<div id="chart" class="column hide" style="width: 80%">
<canvas></canvas>
<div class="filters" style="margin-top: 1em">
<label><input type="checkbox" id="sum" /> Preissumme Gesamt</label>
<label><input type="checkbox" id="sumstores" /> Preissumme pro Kette</label>
<label><input type="checkbox" id="todayonly" /> Nur heutige Preise</label>
</div>
<div>
<label>Von <input id="start" type="date" /></label>
<label>Bis <input id="end" type="date" /></label>
</div>
</div>
<div id="search" class="column"></div>
</div>
<div id="search" class="column">
</div>
</div>
<script src="alasql.js"></script>
<script src="chart.js"></script>
<script src="utils.js"></script>
<script src="main.js"></script>
</body>
</html>
<script src="alasql.js"></script>
<script src="chart.js"></script>
<script src="utils.js"></script>
<script src="main.js"></script>
</body>
</html>

37
site/main.js
View File

@ -1,5 +1,5 @@
function updateCharts(canvasDom, items) {
const now =performance.now();
const now = performance.now();
const sum = document.querySelector("#sum").checked;
const sumStores = document.querySelector("#sumstores").checked;
const todayOnly = document.querySelector("#todayonly").checked;
@ -26,9 +26,11 @@ async function load() {
document.querySelector("#start").value = getOldestDate(items);
document.querySelector("#end").value = currentDate();
newSearchComponent(document.querySelector("#search"), items,
newSearchComponent(
document.querySelector("#search"),
items,
(hits) => {
items.forEach(item => item.chart = false);
items.forEach((item) => (item.chart = false));
if (hits.length > 0) {
chartDom.classList.remove("hide");
} else {
@ -40,36 +42,39 @@ async function load() {
},
null,
(header) => {
header = dom("tr", `<th>Kette</th><th>Name</th><th>Menge</th><th>Preis 📈</th><th></th>`)
const showHideAll = header.querySelectorAll('th:nth-child(4)')[0];
header = dom("tr", `<th>Kette</th><th>Name</th><th>Menge</th><th>Preis 📈</th><th></th>`);
const showHideAll = header.querySelectorAll("th:nth-child(4)")[0];
showHideAll.style["cursor"] = "pointer";
showHideAll.showAll = true;
showHideAll.addEventListener("click", () => {
document.querySelectorAll(".priceinfo").forEach(el => showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide"));
document
.querySelectorAll(".priceinfo")
.forEach((el) => (showHideAll.showAll ? el.classList.remove("hide") : el.classList.add("hide")));
showHideAll.showAll = !showHideAll.showAll;
})
});
return header;
}, (item, itemDom, items, setQuery) => {
const checked = item.chart = (getQueryParameter("c") ?? []).includes(`${item.store}:${item.id}`);
},
(item, itemDom, items, setQuery) => {
const checked = (item.chart = (getQueryParameter("c") ?? []).includes(`${item.store}:${item.id}`));
const dataId = item.store + ":" + item.id;
const cell = dom("td", `<input type="checkbox" ${checked ? "checked" : ""} data-id="${dataId}">`);
itemDom.appendChild(cell);
const handleClick = (eventShouldSetQuery = false) =>{
const handleClick = (eventShouldSetQuery = false) => {
item.chart = cell.children[0].checked;
updateCharts(canvasDom, lastHits)
updateCharts(canvasDom, lastHits);
!!eventShouldSetQuery && setQuery();
}
};
cell.children[0].addEventListener("click", handleClick);
checked && handleClick();
return itemDom;
});
}
);
const query = getQueryParameter("q");
if (query) {
document.querySelector(".search").value = query;
const inputEvent = new Event('input', {
const inputEvent = new Event("input", {
bubbles: true,
cancelable: false
cancelable: false,
});
document.querySelector(".search").dispatchEvent(inputEvent);
}

22
site/style.css
View File

@ -43,7 +43,7 @@ label {
flex-wrap: wrap;
align-items: center;
justify-content: center;
gap: .5em;
gap: 0.5em;
}
.toggle--hidden {
@ -57,7 +57,7 @@ label {
justify-content: center;
align-items: center;
margin: 0 auto;
gap: .5em;
gap: 0.5em;
z-index: 1;
}
.wrapper--search {
@ -88,7 +88,7 @@ label {
}
.wrapper--pinned ~ .wrapper--sticky::before {
content: '';
content: "";
position: absolute;
top: 0;
height: 100%;
@ -96,7 +96,7 @@ label {
max-width: 650px;
left: 50%;
background-color: white;
opacity: .95;
opacity: 0.95;
transform: translateX(-50%);
z-index: -1;
}
@ -106,13 +106,13 @@ label {
opacity: 0;
pointer-events: none;
}
.toggle:checked ~ .wrapper--sticky,
.toggle:checked ~ .wrapper--sticky,
.toggle:checked ~ .wrapper--sticky * {
opacity: 1;
pointer-events: initial;
}
.wrapper--pinned::before {
content: '';
content: "";
position: fixed;
display: block;
top: 0;
@ -214,7 +214,7 @@ td:nth-child(4) {
}
.searchresults td:nth-child(4):before {
content: "€ "
content: "€ ";
}
.querylink {
@ -230,7 +230,7 @@ td:nth-child(4) {
}
.carts td:nth-child(3):before {
content: "€ "
content: "€ ";
}
.carts td:nth-child(3) {
@ -288,7 +288,7 @@ td:nth-child(4) {
table tr {
display: block;
margin-bottom: .625em;
margin-bottom: 0.625em;
}
table td {
@ -309,7 +309,7 @@ td:nth-child(4) {
color: white;
border: 0px;
position: absolute;
padding: .1em .4em;
padding: 0.1em 0.4em;
width: 65px;
height: calc(100%);
left: -1px;
@ -335,4 +335,4 @@ td:nth-child(4) {
table td:last-child {
border-bottom: 0;
}
}
}

278
site/utils.js
View File

@ -46,26 +46,13 @@ const stores = {
},
penny: {
name: "Penny",
budgetBrands: [
"bravo",
"echt bio!",
"san fabio",
"federike",
"blik",
"berida",
"today",
"ich bin österreich",
],
budgetBrands: ["bravo", "echt bio!", "san fabio", "federike", "blik", "berida", "today", "ich bin österreich"],
color: "rgb(255, 180, 180)",
},
};
const STORE_KEYS = Object.keys(stores);
const BUDGET_BRANDS = [
...new Set(
[].concat(...Object.values(stores).map((store) => store.budgetBrands))
),
];
const BUDGET_BRANDS = [...new Set([].concat(...Object.values(stores).map((store) => store.budgetBrands)))];
/**
* @description Returns the current date in ISO format
@ -128,12 +115,7 @@ function decompress(compressedItems) {
const date = data[i++];
const price = data[i++];
prices.push({
date:
date.substring(0, 4) +
"-" +
date.substring(4, 6) +
"-" +
date.substring(6, 8),
date: date.substring(0, 4) + "-" + date.substring(4, 6) + "-" + date.substring(6, 8),
price,
});
}
@ -168,11 +150,7 @@ function decompress(compressedItems) {
url = "https://shop.unimarkt.at" + url;
break;
case "reweDe":
url =
"https://shop.rewe.de/p/" +
name.toLowerCase().replace(/ /g, "-") +
"/" +
id;
url = "https://shop.rewe.de/p/" + name.toLowerCase().replace(/ /g, "-") + "/" + id;
break;
}
@ -200,21 +178,13 @@ async function loadItems() {
new Promise(async (resolve) => {
const now = performance.now();
try {
const response = await fetch(
`latest-canonical.${store}.compressed.json`
);
const response = await fetch(`latest-canonical.${store}.compressed.json`);
const json = await response.json();
console.log(
`Loading compressed items for ${store} took ${
(performance.now() - now) / 1000
} secs`
);
console.log(`Loading compressed items for ${store} took ${(performance.now() - now) / 1000} secs`);
resolve(decompress(json));
} catch {
console.log(
`Error while loading compressed items for ${store}. It took ${
(performance.now() - now) / 1000
} secs, continueing...`
`Error while loading compressed items for ${store}. It took ${(performance.now() - now) / 1000} secs, continueing...`
);
resolve([]);
}
@ -222,27 +192,19 @@ async function loadItems() {
);
}
const items = [].concat(...(await Promise.all(compressedItemsPerStore)));
console.log(
"Loading compressed items in parallel took " +
(performance.now() - now) / 1000 +
" secs"
);
console.log("Loading compressed items in parallel took " + (performance.now() - now) / 1000 + " secs");
now = performance.now();
alasql.fn.hasPriceChange = (priceHistory, date, endDate) => {
if (!endDate) return priceHistory.some((price) => price.date == date);
else
return priceHistory.some(
(price) => price.date >= date && price.date <= endDate
);
else return priceHistory.some((price) => price.date >= date && price.date <= endDate);
};
for (const item of items) {
item.search = item.name + " " + item.quantity + " " + item.unit;
item.search = item.search.toLowerCase().replace(",", ".");
item.numPrices = item.priceHistory.length;
item.priceOldest =
item.priceHistory[item.priceHistory.length - 1].price;
item.priceOldest = item.priceHistory[item.priceHistory.length - 1].price;
item.dateOldest = item.priceHistory[item.priceHistory.length - 1].date;
item.date = item.priceHistory[0].date;
let highestPriceBefore = -1;
@ -261,9 +223,7 @@ async function loadItems() {
item.highestBefore = highestPriceBefore;
item.lowestBefore = lowestPriceBefore;
}
console.log(
"Processing items took " + (performance.now() - now) / 1000 + " secs"
);
console.log("Processing items took " + (performance.now() - now) / 1000 + " secs");
return items;
}
@ -342,51 +302,31 @@ function itemToDOM(item) {
quantity = parseFloat((0.001 * quantity).toFixed(2));
unit = unit == "ml" ? "l" : "kg";
}
let unitDom = dom(
"td",
(item.isWeighted ? "⚖ " : "") + `${quantity} ${unit}`
);
let unitDom = dom("td", (item.isWeighted ? "⚖ " : "") + `${quantity} ${unit}`);
unitDom.setAttribute("data-label", "Menge");
let increase = "";
if (item.priceHistory.length > 1) {
let percentageChange = Math.round(
((item.priceHistory[0].price - item.priceHistory[1].price) /
item.priceHistory[1].price) *
100
);
increase = `<span class="${
percentageChange > 0 ? "increase" : "decrease"
}">${
let percentageChange = Math.round(((item.priceHistory[0].price - item.priceHistory[1].price) / item.priceHistory[1].price) * 100);
increase = `<span class="${percentageChange > 0 ? "increase" : "decrease"}">${
percentageChange > 0 ? "+" + percentageChange : percentageChange
}%</span>`;
}
let priceDomText = `${Number(item.price).toFixed(2)} ${increase} ${
item.priceHistory.length > 1
? "(" + (item.priceHistory.length - 1) + ")"
: ""
item.priceHistory.length > 1 ? "(" + (item.priceHistory.length - 1) + ")" : ""
}`;
let pricesText = "";
for (let i = 0; i < item.priceHistory.length; i++) {
const date = item.priceHistory[i].date;
const currPrice = item.priceHistory[i].price;
const lastPrice = item.priceHistory[i + 1]
? item.priceHistory[i + 1].price
: currPrice;
const increase = Math.round(
((currPrice - lastPrice) / lastPrice) * 100
);
const lastPrice = item.priceHistory[i + 1] ? item.priceHistory[i + 1].price : currPrice;
const increase = Math.round(((currPrice - lastPrice) / lastPrice) * 100);
let priceColor = "black";
if (increase > 0) priceColor = "red";
if (increase < 0) priceColor = "green";
pricesText += `<span style="color: ${priceColor}">${date} ${currPrice} ${
increase > 0 ? "+" + increase : increase
}%</span>`;
pricesText += `<span style="color: ${priceColor}">${date} ${currPrice} ${increase > 0 ? "+" + increase : increase}%</span>`;
if (i != item.priceHistory.length - 1) pricesText += "<br>";
}
let priceDom = dom(
"td",
`${priceDomText}<div class="priceinfo hide">${pricesText}</div>`
);
let priceDom = dom("td", `${priceDomText}<div class="priceinfo hide">${pricesText}</div>`);
priceDom.setAttribute("data-label", "Preis");
if (item.priceHistory.length > 1) {
priceDom.style["cursor"] = "pointer";
@ -410,16 +350,7 @@ function itemToDOM(item) {
let componentId = 0;
function searchItems(
items,
query,
checkedStores,
budgetBrands,
minPrice,
maxPrice,
exact,
bio
) {
function searchItems(items, query, checkedStores, budgetBrands, minPrice, maxPrice, exact, bio) {
query = query.trim();
if (query.length < 3) return [];
@ -428,9 +359,7 @@ function searchItems(
return alasql("select * from ? where " + query, [items]);
}
const tokens = query
.split(/\s+/)
.map((token) => token.toLowerCase().replace(",", "."));
const tokens = query.split(/\s+/).map((token) => token.toLowerCase().replace(",", "."));
let hits = [];
for (const item of items) {
@ -443,18 +372,11 @@ function searchItems(
break;
}
if (exact) {
if (
index > 0 &&
item.search.charAt(index - 1) != " " &&
item.search.charAt(index - 1) != "-"
) {
if (index > 0 && item.search.charAt(index - 1) != " " && item.search.charAt(index - 1) != "-") {
allFound = false;
break;
}
if (
index + token.length < item.search.length &&
item.search.charAt(index + token.length) != " "
) {
if (index + token.length < item.search.length && item.search.charAt(index + token.length) != " ") {
allFound = false;
break;
}
@ -462,17 +384,10 @@ function searchItems(
}
if (allFound) {
const name = item.name.toLowerCase();
if (checkedStores.length && !checkedStores.includes(item.store))
continue;
if (checkedStores.length && !checkedStores.includes(item.store)) continue;
if (item.price < minPrice) continue;
if (item.price > maxPrice) continue;
if (
budgetBrands &&
!BUDGET_BRANDS.some(
(budgetBrand) => name.indexOf(budgetBrand) >= 0
)
)
continue;
if (budgetBrands && !BUDGET_BRANDS.some((budgetBrand) => name.indexOf(budgetBrand) >= 0)) continue;
if (bio && !item.bio) continue;
hits.push(item);
}
@ -480,14 +395,7 @@ function searchItems(
return hits;
}
function newSearchComponent(
parentElement,
items,
searched,
filter,
headerModifier,
itemDomModifier
) {
function newSearchComponent(parentElement, items, searched, filter, headerModifier, itemDomModifier) {
let id = componentId++;
parentElement.innerHTML = "";
parentElement.innerHTML = `
@ -501,17 +409,14 @@ function newSearchComponent(
<a id="json-${id}" href="" class="hide">JSON</a>
<div class="filters filters--store">
<label><input id="all-${id}" type="checkbox" checked="true"><strong>Alle</strong></label>
${STORE_KEYS.map(
(store) =>
`<label><input id="${store}-${id}" type="checkbox" checked="true">${stores[store].name}</label>`
).join(" ")}
${STORE_KEYS.map((store) => `<label><input id="${store}-${id}" type="checkbox" checked="true">${stores[store].name}</label>`).join(
" "
)}
</div>
<div class="filters">
<label>
<input id="budgetBrands-${id}" type="checkbox"> Nur
<abbr title="${BUDGET_BRANDS.map((budgetBrand) =>
budgetBrand.toUpperCase()
).join(", ")}">
<abbr title="${BUDGET_BRANDS.map((budgetBrand) => budgetBrand.toUpperCase()).join(", ")}">
Diskont-Eigenmarken
</abbr>
</label>
@ -536,7 +441,8 @@ function newSearchComponent(
(entries) => {
for (const entry of entries) {
const clientRect = entry.target.getBoundingClientRect();
if (entry.intersectionRatio < 0.999 && (clientRect.top + clientRect.height) < window.innerHeight) { // Fix Edge issue
if (entry.intersectionRatio < 0.999 && clientRect.top + clientRect.height < window.innerHeight) {
// Fix Edge issue
entry.target.classList.add("wrapper--pinned");
} else {
entry.target.classList.remove("wrapper--pinned");
@ -558,9 +464,7 @@ function newSearchComponent(
const budgetBrands = parentElement.querySelector(`#budgetBrands-${id}`);
const bio = parentElement.querySelector(`#bio-${id}`);
const allCheckbox = parentElement.querySelector(`#all-${id}`);
const storeCheckboxes = STORE_KEYS.map((store) =>
parentElement.querySelector(`#${store}-${id}`)
);
const storeCheckboxes = STORE_KEYS.map((store) => parentElement.querySelector(`#${store}-${id}`));
const minPrice = parentElement.querySelector(`#minprice-${id}`);
const maxPrice = parentElement.querySelector(`#maxprice-${id}`);
const numResults = parentElement.querySelector(`#numresults-${id}`);
@ -582,16 +486,9 @@ function newSearchComponent(
queryLink.classList.remove("hide");
jsonLink.classList.remove("hide");
const inputs = [...table.querySelectorAll("input:checked")];
let checked = inputs.length
? inputs.map((item) => item.dataset.id)
: getQueryParameter("c");
let checked = inputs.length ? inputs.map((item) => item.dataset.id) : getQueryParameter("c");
if (typeof checked === "string") checked = [checked];
queryLink.setAttribute(
"href",
`/?q=${encodeURIComponent(query)}${
checked?.length ? `&c=${checked.join("&c=")}` : ""
}`
);
queryLink.setAttribute("href", `/?q=${encodeURIComponent(query)}${checked?.length ? `&c=${checked.join("&c=")}` : ""}`);
};
let search = (query) => {
@ -612,9 +509,7 @@ function newSearchComponent(
} catch (e) {
console.log("Query: " + query + "\n" + e.message);
}
console.log(
"Search took " + (performance.now() - now) / 1000.0 + " secs"
);
console.log("Search took " + (performance.now() - now) / 1000.0 + " secs");
if (searched) hits = searched(hits);
if (filter) hits = hits.filter(filter);
table.innerHTML = "";
@ -622,10 +517,7 @@ function newSearchComponent(
numResults.innerHTML = "Resultate: 0";
return;
}
if (
query.trim().charAt(0) != "!" ||
query.trim().toLowerCase().indexOf("order by") == -1
) {
if (query.trim().charAt(0) != "!" || query.trim().toLowerCase().indexOf("order by") == -1) {
if (sort.value == "priceasc") {
hits.sort((a, b) => a.price - b.price);
} else if (sort.value == "pricedesc") {
@ -636,10 +528,7 @@ function newSearchComponent(
}
}
let header = dom(
"tr",
`<th>Kette</th><th>Name</th><th>Menge</th><th>Preis</th>`
);
let header = dom("tr", `<th>Kette</th><th>Name</th><th>Menge</th><th>Preis</th>`);
if (headerModifier) header = headerModifier(header);
const thead = dom("thead", ``);
thead.appendChild(header);
@ -648,21 +537,15 @@ function newSearchComponent(
now = performance.now();
let num = 0;
let limit = isMobile() ? 500 : 2000;
hits.every(hit => {
hits.every((hit) => {
let itemDom = itemToDOM(hit);
if (itemDomModifier)
itemDom = itemDomModifier(hit, itemDom, hits, setQuery);
if (itemDomModifier) itemDom = itemDomModifier(hit, itemDom, hits, setQuery);
table.appendChild(itemDom);
num++;
return num < limit;
});
console.log(
"Building DOM took: " + (performance.now() - now) / 1000.0 + " secs"
);
numResults.innerHTML =
"Resultate: " +
hits.length +
(num < hits.length ? ", " + num + " angezeigt" : "");
console.log("Building DOM took: " + (performance.now() - now) / 1000.0 + " secs");
numResults.innerHTML = "Resultate: " + hits.length + (num < hits.length ? ", " + num + " angezeigt" : "");
lastHits = hits;
};
@ -676,13 +559,9 @@ function newSearchComponent(
maxPrice.value = 100;
}
if (query?.charAt(0) == "!") {
parentElement
.querySelectorAll(".filters")
.forEach((f) => (f.style.display = "none"));
parentElement.querySelectorAll(".filters").forEach((f) => (f.style.display = "none"));
} else {
parentElement
.querySelectorAll(".filters")
.forEach((f) => (f.style = undefined));
parentElement.querySelectorAll(".filters").forEach((f) => (f.style = undefined));
}
setQuery();
search(searchInput.value);
@ -690,10 +569,8 @@ function newSearchComponent(
});
budgetBrands.addEventListener("change", () => search(searchInput.value));
bio.addEventListener("change", () => search(searchInput.value));
allCheckbox.addEventListener("change", () => storeCheckboxes.forEach(store => store.checked = allCheckbox.checked));
storeCheckboxes.map((store) =>
store.addEventListener("change", () => search(searchInput.value))
);
allCheckbox.addEventListener("change", () => storeCheckboxes.forEach((store) => (store.checked = allCheckbox.checked)));
storeCheckboxes.map((store) => store.addEventListener("change", () => search(searchInput.value)));
sort.addEventListener("change", () => search(searchInput.value));
minPrice.addEventListener("change", () => search(searchInput.value));
maxPrice.addEventListener("change", () => search(searchInput.value));
@ -710,18 +587,14 @@ function showChart(canvasDom, items, chartType) {
canvasDom.style.display = "block";
}
const allDates = items.flatMap((product) =>
product.priceHistory.map((item) => item.date)
);
const allDates = items.flatMap((product) => product.priceHistory.map((item) => item.date));
const uniqueDates = [...new Set(allDates)];
uniqueDates.sort();
const datasets = items.map((product) => {
let price = null;
const prices = uniqueDates.map((date) => {
const priceObj = product.priceHistory.find(
(item) => item.date === date
);
const priceObj = product.priceHistory.find((item) => item.date === date);
if (!price && priceObj) price = priceObj.price;
return priceObj ? priceObj.price : null;
});
@ -776,26 +649,13 @@ function getOldestDate(items) {
return oldestDate;
}
function showCharts(
canvasDom,
items,
sum,
sumStores,
todayOnly,
startDate,
endDate
) {
function showCharts(canvasDom, items, sum, sumStores, todayOnly, startDate, endDate) {
let itemsToShow = [];
if (sum && items.length > 0) {
itemsToShow.push({
name: "Preissumme Warenkorb",
priceHistory: calculateOverallPriceChanges(
items,
todayOnly,
startDate,
endDate
),
priceHistory: calculateOverallPriceChanges(items, todayOnly, startDate, endDate),
});
}
@ -805,12 +665,7 @@ function showCharts(
if (storeItems.length > 0) {
itemsToShow.push({
name: "Preissumme " + store,
priceHistory: calculateOverallPriceChanges(
storeItems,
todayOnly,
startDate,
endDate
),
priceHistory: calculateOverallPriceChanges(storeItems, todayOnly, startDate, endDate),
});
}
});
@ -822,10 +677,7 @@ function showCharts(
name: item.store + " " + item.name,
priceHistory: todayOnly
? [{ date: currentDate(), price: item.price }]
: item.priceHistory.filter(
(price) =>
price.date >= startDate && price.date <= endDate
),
: item.priceHistory.filter((price) => price.date >= startDate && price.date <= endDate),
});
}
});
@ -842,21 +694,15 @@ function calculateOverallPriceChanges(items, todayOnly, startDate, endDate) {
return [{ date: currentDate(), price: sum }];
}
const allDates = items.flatMap((product) =>
product.priceHistory.map((item) => item.date)
);
const allDates = items.flatMap((product) => product.priceHistory.map((item) => item.date));
let uniqueDates = [...new Set(allDates)];
uniqueDates.sort();
uniqueDates = uniqueDates.filter(
(date) => date >= startDate && date <= endDate
);
uniqueDates = uniqueDates.filter((date) => date >= startDate && date <= endDate);
const allPrices = items.map((product) => {
let price = null;
const prices = uniqueDates.map((date) => {
const priceObj = product.priceHistory.find(
(item) => item.date === date
);
const priceObj = product.priceHistory.find((item) => item.date === date);
if (!price && priceObj) price = priceObj.price;
return priceObj ? priceObj.price : null;
});
@ -1247,9 +1093,7 @@ function cluster(items, maxTime) {
if (storeItems.length > 0) itemsPerStore.push(storeItems);
}
itemsPerStore.sort((a, b) => b.length - a.length);
itemsPerStore.forEach((items) =>
console.log(items[0].store + ", " + items.length)
);
itemsPerStore.forEach((items) => console.log(items[0].store + ", " + items.length));
// Take the store with the most items, then try to find the best match
// from each of the other stores
@ -1294,11 +1138,7 @@ function cluster(items, maxTime) {
const time = (performance.now() - now) / 1000;
console.log(maxIterations + ", time " + time);
if (
JSON.stringify(clusters) == JSON.stringify(newClusters) ||
maxIterations == 1 ||
time > maxTime
) {
if (JSON.stringify(clusters) == JSON.stringify(newClusters) || maxIterations == 1 || time > maxTime) {
break;
}
@ -1325,9 +1165,7 @@ function flattenClusters(clusters) {
}
function isMobile() {
return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(
navigator.userAgent
);
return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent);
}
try {

85
stores/billa.js
View File

@ -3,49 +3,54 @@ const utils = require("./utils");
const HITS = Math.floor(30000 + Math.random() * 2000);
const conversions = {
"Beutel": { unit: 'stk', factor: 1 },
"Blatt": { unit: 'stk', factor: 1 },
"Bund": { unit: 'stk', factor: 1 },
"g": { unit: 'g', factor: 1},
"Gramm": { unit: 'g', factor: 1},
"kg": { unit: 'g', factor: 1000},
"Kilogramm": { unit: 'g', factor: 1},
"l": { unit: 'ml', factor: 1000},
"Liter": { unit: 'ml', factor: 1000},
"Meter": { unit: 'cm', factor: 100},
"Milliliter": { unit: 'ml', factor: 1},
"ml": { unit: 'ml', factor: 1},
"Paar": { unit: 'stk', factor: 1 },
"Packung": { unit: 'stk', factor: 1 },
"Portion": { unit: 'stk', factor: 1 },
"Rollen": { unit: 'stk', factor: 1 },
"Stk": { unit: 'stk', factor: 1 },
"Stück": { unit: 'stk', factor: 1 },
"Teebeutel": { unit: 'stk', factor: 1 },
"Waschgang": { unit: 'wg', factor: 1 },
"Zentimeter": { unit: 'cm', factor: 1 },
Beutel: { unit: "stk", factor: 1 },
Blatt: { unit: "stk", factor: 1 },
Bund: { unit: "stk", factor: 1 },
g: { unit: "g", factor: 1 },
Gramm: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
Kilogramm: { unit: "g", factor: 1 },
l: { unit: "ml", factor: 1000 },
Liter: { unit: "ml", factor: 1000 },
Meter: { unit: "cm", factor: 100 },
Milliliter: { unit: "ml", factor: 1 },
ml: { unit: "ml", factor: 1 },
Paar: { unit: "stk", factor: 1 },
Packung: { unit: "stk", factor: 1 },
Portion: { unit: "stk", factor: 1 },
Rollen: { unit: "stk", factor: 1 },
Stk: { unit: "stk", factor: 1 },
Stück: { unit: "stk", factor: 1 },
Teebeutel: { unit: "stk", factor: 1 },
Waschgang: { unit: "wg", factor: 1 },
Zentimeter: { unit: "cm", factor: 1 },
};
exports.getCanonical = function(item, today) {
let quantity = 1, unit = "kg";
if(item.data.grammagePriceFactor == 1) {
const grammage = item.data.grammage !== "" && item.data.grammage.trim().split(' ').length>1 ? item.data.grammage : item.data.price.unit;
if (grammage) [quantity, unit] = grammage.trim().split(' ').splice(0,2);
exports.getCanonical = function (item, today) {
let quantity = 1,
unit = "kg";
if (item.data.grammagePriceFactor == 1) {
const grammage = item.data.grammage !== "" && item.data.grammage.trim().split(" ").length > 1 ? item.data.grammage : item.data.price.unit;
if (grammage) [quantity, unit] = grammage.trim().split(" ").splice(0, 2);
}
return utils.convertUnit({
id: item.data.articleId,
name: item.data.name,
price: item.data.price.final,
priceHistory: [{ date: today, price: item.data.price.final }],
isWeighted : item.data.isWeightArticle,
unit,
quantity,
bio: item.data.attributes && item.data.attributes.includes("s_bio"),
url: `https://shop.billa.at${item.data.canonicalPath}`,
}, conversions, 'billa');
}
return utils.convertUnit(
{
id: item.data.articleId,
name: item.data.name,
price: item.data.price.final,
priceHistory: [{ date: today, price: item.data.price.final }],
isWeighted: item.data.isWeightArticle,
unit,
quantity,
bio: item.data.attributes && item.data.attributes.includes("s_bio"),
url: `https://shop.billa.at${item.data.canonicalPath}`,
},
conversions,
"billa"
);
};
exports.fetchData = async function() {
exports.fetchData = async function () {
const BILLA_SEARCH = `https://shop.billa.at/api/search/full?searchTerm=*&storeId=00-10&pageSize=${HITS}`;
return (await axios.get(BILLA_SEARCH)).data.tiles;
}
};

130
stores/dm-de.js
View File

@ -2,92 +2,96 @@ const axios = require("axios");
const utils = require("./utils");
const conversions = {
'g': { unit: 'g', factor: 1 },
'kg': { unit: 'g', factor: 1000 },
'l': { unit: 'ml', factor: 1000 },
'ml': { unit: 'ml', factor: 1 },
'St': { unit: 'stk', factor: 1 },
'Wl': { unit: 'wg', factor: 1 },
'm': { unit: 'cm', factor: 100 },
'mm': { unit: 'cm', factor: .1 },
'Bl': { unit: 'stk', factor: 1 },
'Btl': { unit: 'stk', factor: 1 },
'Paar': { unit: 'stk', factor: 1 },
'Portion': { unit: 'stk', factor: 1 },
'Satz': { unit: 'stk', factor: 1 },
'Tablette': { unit: 'stk', factor: 1 },
g: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
l: { unit: "ml", factor: 1000 },
ml: { unit: "ml", factor: 1 },
St: { unit: "stk", factor: 1 },
Wl: { unit: "wg", factor: 1 },
m: { unit: "cm", factor: 100 },
mm: { unit: "cm", factor: 0.1 },
Bl: { unit: "stk", factor: 1 },
Btl: { unit: "stk", factor: 1 },
Paar: { unit: "stk", factor: 1 },
Portion: { unit: "stk", factor: 1 },
Satz: { unit: "stk", factor: 1 },
Tablette: { unit: "stk", factor: 1 },
};
exports.getCanonical = function(item, today) {
exports.getCanonical = function (item, today) {
let quantity = item.netQuantityContent || item.basePriceQuantity;
let unit = item.contentUnit || item.basePriceUnit;
return utils.convertUnit({
id: item.gtin,
name: `${item.brandName} ${item.title}`,
price: item.price.value,
priceHistory: [{ date: today, price: item.price.value }],
unit,
quantity,
...(item.brandName === "dmBio" || (item.name ? (item.name.startsWith("Bio ") | item.name.startsWith("Bio-")) : false)) && {bio: true},
url: `https://www.dm.de/product-p${item.gtin}.html`,
}, conversions, 'dmDe');
}
return utils.convertUnit(
{
id: item.gtin,
name: `${item.brandName} ${item.title}`,
price: item.price.value,
priceHistory: [{ date: today, price: item.price.value }],
unit,
quantity,
...((item.brandName === "dmBio" || (item.name ? item.name.startsWith("Bio ") | item.name.startsWith("Bio-") : false)) && { bio: true }),
url: `https://www.dm.de/product-p${item.gtin}.html`,
},
conversions,
"dmDe"
);
};
exports.fetchData = async function() {
const DM_BASE_URL = `https://product-search.services.dmtech.com/de/search/crawl?pageSize=1000&`
exports.fetchData = async function () {
const DM_BASE_URL = `https://product-search.services.dmtech.com/de/search/crawl?pageSize=1000&`;
const QUERIES = [
'allCategories.id=010000&price.value.to=2', //~500 items
'allCategories.id=010000&price.value.from=2&price.value.to=3', //~600 items
'allCategories.id=010000&price.value.from=3&price.value.to=4', //~500 items
'allCategories.id=010000&price.value.from=4&price.value.to=7', //~800 items
'allCategories.id=010000&price.value.from=7&price.value.to=10', //~900 items
'allCategories.id=010000&price.value.from=10&price.value.to=15', //~900 items
'allCategories.id=010000&price.value.from=15', //~300 items
'allCategories.id=020000&price.value.to=2', //~600 items
'allCategories.id=020000&price.value.from=2&price.value.to=3', //~550 items
'allCategories.id=020000&price.value.from=3&price.value.to=4', //~600 items
'allCategories.id=020000&price.value.from=4&price.value.to=6', //~800 items
'allCategories.id=020000&price.value.from=6&price.value.to=10', //~850 items
'allCategories.id=020000&price.value.from=10&price.value.to=18', //~900 items
'allCategories.id=020000&price.value.from=18', //~960 items (!)
'allCategories.id=030000&price.value.to=8', //~900 items
'allCategories.id=030000&price.value.from=8', //~500 items
'allCategories.id=040000&price.value.to=2', //~600 items
'allCategories.id=040000&price.value.from=2&price.value.to=4', //~900 items
'allCategories.id=040000&price.value.from=4', //~400 items
'allCategories.id=050000&price.value.to=4', //~600 items
'allCategories.id=050000&price.value.from=4', //~800 items
'allCategories.id=060000&price.value.to=4', //~900 items
'allCategories.id=060000&price.value.from=4', //~500 items
'allCategories.id=070000', //~300 items
]
"allCategories.id=010000&price.value.to=2", //~500 items
"allCategories.id=010000&price.value.from=2&price.value.to=3", //~600 items
"allCategories.id=010000&price.value.from=3&price.value.to=4", //~500 items
"allCategories.id=010000&price.value.from=4&price.value.to=7", //~800 items
"allCategories.id=010000&price.value.from=7&price.value.to=10", //~900 items
"allCategories.id=010000&price.value.from=10&price.value.to=15", //~900 items
"allCategories.id=010000&price.value.from=15", //~300 items
"allCategories.id=020000&price.value.to=2", //~600 items
"allCategories.id=020000&price.value.from=2&price.value.to=3", //~550 items
"allCategories.id=020000&price.value.from=3&price.value.to=4", //~600 items
"allCategories.id=020000&price.value.from=4&price.value.to=6", //~800 items
"allCategories.id=020000&price.value.from=6&price.value.to=10", //~850 items
"allCategories.id=020000&price.value.from=10&price.value.to=18", //~900 items
"allCategories.id=020000&price.value.from=18", //~960 items (!)
"allCategories.id=030000&price.value.to=8", //~900 items
"allCategories.id=030000&price.value.from=8", //~500 items
"allCategories.id=040000&price.value.to=2", //~600 items
"allCategories.id=040000&price.value.from=2&price.value.to=4", //~900 items
"allCategories.id=040000&price.value.from=4", //~400 items
"allCategories.id=050000&price.value.to=4", //~600 items
"allCategories.id=050000&price.value.from=4", //~800 items
"allCategories.id=060000&price.value.to=4", //~900 items
"allCategories.id=060000&price.value.from=4", //~500 items
"allCategories.id=070000", //~300 items
];
let dmItems = [];
for (let query of QUERIES) {
var res = (await axios.get(DM_BASE_URL + query, {
var res = await axios.get(DM_BASE_URL + query, {
validateStatus: function (status) {
return (status >= 200 && status < 300) || status == 429;
}
}));
},
});
// exponential backoff
backoff = 2000;
while (res.status == 429) {
console.info(`DM-DE API returned 429, retrying in ${backoff/1000}s.`);
await new Promise(resolve => setTimeout(resolve, backoff));
console.info(`DM-DE API returned 429, retrying in ${backoff / 1000}s.`);
await new Promise((resolve) => setTimeout(resolve, backoff));
backoff *= 2;
res = (await axios.get(DM_BASE_URL + query, {
res = await axios.get(DM_BASE_URL + query, {
validateStatus: function (status) {
return (status >= 200 && status < 300) || status == 429;
}
}));
},
});
}
let items = res.data;
if (items.count > 1000) {
console.warn(`DM-DE Query returned more than 1000 items! Items may be missing. Adjust queries. Query: ${query}`);
}
dmItems = dmItems.concat(items.products);
await new Promise(resolve => setTimeout(resolve, 1000));
await new Promise((resolve) => setTimeout(resolve, 1000));
}
return dmItems;
}
};

130
stores/dm.js
View File

@ -2,92 +2,96 @@ const axios = require("axios");
const utils = require("./utils");
const conversions = {
'g': { unit: 'g', factor: 1 },
'kg': { unit: 'g', factor: 1000 },
'l': { unit: 'ml', factor: 1000 },
'ml': { unit: 'ml', factor: 1 },
'St': { unit: 'stk', factor: 1 },
'Wl': { unit: 'wg', factor: 1 },
'm': { unit: 'cm', factor: 100 },
'mm': { unit: 'cm', factor: .1 },
'Bl': { unit: 'stk', factor: 1 },
'Btl': { unit: 'stk', factor: 1 },
'Paar': { unit: 'stk', factor: 1 },
'Portion': { unit: 'stk', factor: 1 },
'Satz': { unit: 'stk', factor: 1 },
'Tablette': { unit: 'stk', factor: 1 },
g: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
l: { unit: "ml", factor: 1000 },
ml: { unit: "ml", factor: 1 },
St: { unit: "stk", factor: 1 },
Wl: { unit: "wg", factor: 1 },
m: { unit: "cm", factor: 100 },
mm: { unit: "cm", factor: 0.1 },
Bl: { unit: "stk", factor: 1 },
Btl: { unit: "stk", factor: 1 },
Paar: { unit: "stk", factor: 1 },
Portion: { unit: "stk", factor: 1 },
Satz: { unit: "stk", factor: 1 },
Tablette: { unit: "stk", factor: 1 },
};
exports.getCanonical = function(item, today) {
exports.getCanonical = function (item, today) {
let quantity = item.netQuantityContent || item.basePriceQuantity;
let unit = item.contentUnit || item.basePriceUnit;
return utils.convertUnit({
id: item.gtin,
name: `${item.brandName} ${item.title}`,
price: item.price.value,
priceHistory: [{ date: today, price: item.price.value }],
unit,
quantity,
...(item.brandName === "dmBio" || (item.name ? (item.name.startsWith("Bio ") | item.name.startsWith("Bio-")) : false)) && {bio: true},
url: `https://www.dm.at/product-p${item.gtin}.html`,
}, conversions, 'dm');
}
return utils.convertUnit(
{
id: item.gtin,
name: `${item.brandName} ${item.title}`,
price: item.price.value,
priceHistory: [{ date: today, price: item.price.value }],
unit,
quantity,
...((item.brandName === "dmBio" || (item.name ? item.name.startsWith("Bio ") | item.name.startsWith("Bio-") : false)) && { bio: true }),
url: `https://www.dm.at/product-p${item.gtin}.html`,
},
conversions,
"dm"
);
};
exports.fetchData = async function() {
const DM_BASE_URL = `https://product-search.services.dmtech.com/at/search/crawl?pageSize=1000&`
exports.fetchData = async function () {
const DM_BASE_URL = `https://product-search.services.dmtech.com/at/search/crawl?pageSize=1000&`;
const QUERIES = [
'allCategories.id=010000&price.value.to=2', //~500 items
'allCategories.id=010000&price.value.from=2&price.value.to=3', //~600 items
'allCategories.id=010000&price.value.from=3&price.value.to=4', //~500 items
'allCategories.id=010000&price.value.from=4&price.value.to=7', //~800 items
'allCategories.id=010000&price.value.from=7&price.value.to=10', //~900 items
'allCategories.id=010000&price.value.from=10&price.value.to=15', //~900 items
'allCategories.id=010000&price.value.from=15', //~300 items
'allCategories.id=020000&price.value.to=2', //~600 items
'allCategories.id=020000&price.value.from=2&price.value.to=3', //~550 items
'allCategories.id=020000&price.value.from=3&price.value.to=4', //~600 items
'allCategories.id=020000&price.value.from=4&price.value.to=6', //~800 items
'allCategories.id=020000&price.value.from=6&price.value.to=10', //~850 items
'allCategories.id=020000&price.value.from=10&price.value.to=18', //~900 items
'allCategories.id=020000&price.value.from=18', //~960 items (!)
'allCategories.id=030000&price.value.to=8', //~900 items
'allCategories.id=030000&price.value.from=8', //~500 items
'allCategories.id=040000&price.value.to=2', //~600 items
'allCategories.id=040000&price.value.from=2&price.value.to=4', //~900 items
'allCategories.id=040000&price.value.from=4', //~400 items
'allCategories.id=050000&price.value.to=4', //~600 items
'allCategories.id=050000&price.value.from=4', //~800 items
'allCategories.id=060000&price.value.to=4', //~900 items
'allCategories.id=060000&price.value.from=4', //~500 items
'allCategories.id=070000', //~300 items
]
"allCategories.id=010000&price.value.to=2", //~500 items
"allCategories.id=010000&price.value.from=2&price.value.to=3", //~600 items
"allCategories.id=010000&price.value.from=3&price.value.to=4", //~500 items
"allCategories.id=010000&price.value.from=4&price.value.to=7", //~800 items
"allCategories.id=010000&price.value.from=7&price.value.to=10", //~900 items
"allCategories.id=010000&price.value.from=10&price.value.to=15", //~900 items
"allCategories.id=010000&price.value.from=15", //~300 items
"allCategories.id=020000&price.value.to=2", //~600 items
"allCategories.id=020000&price.value.from=2&price.value.to=3", //~550 items
"allCategories.id=020000&price.value.from=3&price.value.to=4", //~600 items
"allCategories.id=020000&price.value.from=4&price.value.to=6", //~800 items
"allCategories.id=020000&price.value.from=6&price.value.to=10", //~850 items
"allCategories.id=020000&price.value.from=10&price.value.to=18", //~900 items
"allCategories.id=020000&price.value.from=18", //~960 items (!)
"allCategories.id=030000&price.value.to=8", //~900 items
"allCategories.id=030000&price.value.from=8", //~500 items
"allCategories.id=040000&price.value.to=2", //~600 items
"allCategories.id=040000&price.value.from=2&price.value.to=4", //~900 items
"allCategories.id=040000&price.value.from=4", //~400 items
"allCategories.id=050000&price.value.to=4", //~600 items
"allCategories.id=050000&price.value.from=4", //~800 items
"allCategories.id=060000&price.value.to=4", //~900 items
"allCategories.id=060000&price.value.from=4", //~500 items
"allCategories.id=070000", //~300 items
];
let dmItems = [];
for (let query of QUERIES) {
var res = (await axios.get(DM_BASE_URL + query, {
var res = await axios.get(DM_BASE_URL + query, {
validateStatus: function (status) {
return (status >= 200 && status < 300) || status == 429;
}
}));
},
});
// exponential backoff
backoff = 2000;
while (res.status == 429) {
console.info(`DM API returned 429, retrying in ${backoff/1000}s.`);
await new Promise(resolve => setTimeout(resolve, backoff));
console.info(`DM API returned 429, retrying in ${backoff / 1000}s.`);
await new Promise((resolve) => setTimeout(resolve, backoff));
backoff *= 2;
res = (await axios.get(DM_BASE_URL + query, {
res = await axios.get(DM_BASE_URL + query, {
validateStatus: function (status) {
return (status >= 200 && status < 300) || status == 429;
}
}));
},
});
}
let items = res.data;
if (items.count > 1000) {
console.warn(`Query returned more than 1000 items! Items may be missing. Adjust queries. Query: ${query}`);
}
dmItems = dmItems.concat(items.products);
await new Promise(resolve => setTimeout(resolve, 1000));
await new Promise((resolve) => setTimeout(resolve, 1000));
}
return dmItems;
}
};

98
stores/hofer.js
View File

@ -2,53 +2,68 @@ const axios = require("axios");
const utils = require("./utils");
const conversions = {
"": {unit: "stk", factor: 1},
"blatt": {unit: "stk", factor: 1},
"g": {unit: "g", factor: 1},
"gg": {unit: "g", factor: 1},
"gramm": {unit: "g", factor: 1},
"kg": {unit: "g", factor: 1000},
"cl": {unit: "ml", factor: 100},
"l": {unit: "ml", factor: 1000},
"ml": {unit: "ml", factor: 1},
"paar": {unit: "stk", factor: 1},
"stk.": {unit: "stk", factor: 1},
"stück": {unit: "stk", factor: 1},
"er": {unit: "stk", factor: 1},
"teebeutel": {unit: "stk", factor: 1},
"": { unit: "stk", factor: 1 },
blatt: { unit: "stk", factor: 1 },
g: { unit: "g", factor: 1 },
gg: { unit: "g", factor: 1 },
gramm: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
cl: { unit: "ml", factor: 100 },
l: { unit: "ml", factor: 1000 },
ml: { unit: "ml", factor: 1 },
paar: { unit: "stk", factor: 1 },
"stk.": { unit: "stk", factor: 1 },
stück: { unit: "stk", factor: 1 },
er: { unit: "stk", factor: 1 },
teebeutel: { unit: "stk", factor: 1 },
};
exports.getCanonical = function(item, today) {
exports.getCanonical = function (item, today) {
// try to read quantity and unit from product name
const name = item.ProductName;
let [quantity, unit] = utils.parseUnitAndQuantityAtEnd(name);
if(conversions[unit] === undefined) {
if (conversions[unit] === undefined) {
// fallback: use given quantity and unit (including packaging)
quantity = item.Unit
unit= item.UnitType
quantity = item.Unit;
unit = item.UnitType;
}
return utils.convertUnit({
id: item.ProductID,
name,
price: item.Price,
priceHistory: [{ date: today, price: item.Price }],
isWeighted: item.IsBulk,
unit,
quantity,
bio: item.IsBio,
url: `https://www.roksh.at/hofer/produkte/${item.CategorySEOName}/${item.SEOName}`
}, conversions, 'hofer');
}
return utils.convertUnit(
{
id: item.ProductID,
name,
price: item.Price,
priceHistory: [{ date: today, price: item.Price }],
isWeighted: item.IsBulk,
unit,
quantity,
bio: item.IsBio,
url: `https://www.roksh.at/hofer/produkte/${item.CategorySEOName}/${item.SEOName}`,
},
conversions,
"hofer"
);
};
exports.fetchData = async function() {
const HOFER_BASE_URL = `https://shopservice.roksh.at`
const CATEGORIES = HOFER_BASE_URL + `/category/GetFullCategoryList/`
const CONFIG = { headers: { authorization: null } }
const ITEMS = HOFER_BASE_URL + `/productlist/CategoryProductList`
exports.fetchData = async function () {
const HOFER_BASE_URL = `https://shopservice.roksh.at`;
const CATEGORIES = HOFER_BASE_URL + `/category/GetFullCategoryList/`;
const CONFIG = { headers: { authorization: null } };
const ITEMS = HOFER_BASE_URL + `/productlist/CategoryProductList`;
// fetch access token
const token_data = { "OwnWebshopProviderCode": "", "SetUserSelectedShopsOnFirstSiteLoad": true, "RedirectToDashboardNeeded": false, "ShopsSelectedForRoot": "hofer", "BrandProviderSelectedForRoot": null, "UserSelectedShops": [] }
const token = (await axios.post("https://shopservice.roksh.at/session/configure", token_data, { headers: { 'Accept': 'application/json', 'Content-Type': 'application/json' } })).headers['jwt-auth'];
const token_data = {
OwnWebshopProviderCode: "",
SetUserSelectedShopsOnFirstSiteLoad: true,
RedirectToDashboardNeeded: false,
ShopsSelectedForRoot: "hofer",
BrandProviderSelectedForRoot: null,
UserSelectedShops: [],
};
const token = (
await axios.post("https://shopservice.roksh.at/session/configure", token_data, {
headers: { Accept: "application/json", "Content-Type": "application/json" },
})
).headers["jwt-auth"];
CONFIG.headers.authorization = "Bearer " + token;
// concat all subcategories (categories.[i].ChildList)
@ -57,14 +72,17 @@ exports.fetchData = async function() {
let hoferItems = [];
for (let subCategory of subCategories) {
let categoryData = (await axios.get(`${ITEMS}?progId=${subCategory.ProgID}&firstLoadProductListResultNum=4&listResultProductNum=24`, CONFIG)).data;
let categoryData = (await axios.get(`${ITEMS}?progId=${subCategory.ProgID}&firstLoadProductListResultNum=4&listResultProductNum=24`, CONFIG))
.data;
const numPages = categoryData.ProductListResults[0].ListContext.TotalPages;
for (let iPage = 1; iPage <= numPages; iPage++) {
let items = (await axios.post(`${HOFER_BASE_URL}/productlist/GetProductList`, { CategoryProgId: subCategory.ProgID, Page: iPage }, CONFIG)).data;
let items = (
await axios.post(`${HOFER_BASE_URL}/productlist/GetProductList`, { CategoryProgId: subCategory.ProgID, Page: iPage }, CONFIG)
).data;
hoferItems = hoferItems.concat(items.ProductList);
}
}
return hoferItems;
}
};

2
stores/index.js
View File

@ -7,4 +7,4 @@ exports.mpreis = require("./mpreis");
exports.spar = require("./spar");
exports.unimarkt = require("./unimarkt");
exports.reweDe = require("./rewe-de");
exports.penny = require("./penny");
exports.penny = require("./penny");

83
stores/lidl.js
View File

@ -4,58 +4,59 @@ const utils = require("./utils");
const HITS = Math.floor(30000 + Math.random() * 2000);
const conversions = {
"": {unit: "stk", factor: 1},
"dosen": {unit: "stk", factor: 1},
"blatt": {unit: "stk", factor: 1},
"flaschen": {unit: "stk", factor: 1},
"l": {unit: "ml", factor: 1000},
"liter": {unit: "ml", factor: 1000},
"ml": {unit: "ml", factor: 1},
"g": {unit: "g", factor: 1},
"kg": {unit: "g", factor: 1000},
"stk.": {unit: "stk", factor: 1},
"": { unit: "stk", factor: 1 },
dosen: { unit: "stk", factor: 1 },
blatt: { unit: "stk", factor: 1 },
flaschen: { unit: "stk", factor: 1 },
l: { unit: "ml", factor: 1000 },
liter: { unit: "ml", factor: 1000 },
ml: { unit: "ml", factor: 1 },
g: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
"stk.": { unit: "stk", factor: 1 },
};
exports.getCanonical = function(item, today) {
exports.getCanonical = function (item, today) {
let quantity = 1;
let unit = '';
let text = (item.price.basePrice?.text ?? "").trim().split('(')[0].replaceAll(',', '.').toLowerCase();
let unit = "";
let text = (item.price.basePrice?.text ?? "").trim().split("(")[0].replaceAll(",", ".").toLowerCase();
let isWeighted = false;
if(text === 'per kg') {
isWeighted = true;
unit = 'kg';
}
else {
if(text.startsWith('bei') && text.search('je ') != -1)
text = text.substr(text.search('je '))
if (text === "per kg") {
isWeighted = true;
unit = "kg";
} else {
if (text.startsWith("bei") && text.search("je ") != -1) text = text.substr(text.search("je "));
for (let s of ['ab ', 'je ', 'ca. ', 'z.b.: ', 'z.b. '])
text = text.replace(s, '').trim()
for (let s of ["ab ", "je ", "ca. ", "z.b.: ", "z.b. "]) text = text.replace(s, "").trim();
const regex = /^([0-9.x ]+)(.*)$/;
const matches = text.match(regex);
if(matches) {
matches[1].split('x').forEach( (q)=> {
quantity = quantity * parseFloat(q.split('/')[0])
})
unit = matches[2].split('/')[0].trim().split(' ')[0];
if (matches) {
matches[1].split("x").forEach((q) => {
quantity = quantity * parseFloat(q.split("/")[0]);
});
unit = matches[2].split("/")[0].trim().split(" ")[0];
}
unit = unit.split('-')[0];
unit = unit.split("-")[0];
}
return utils.convertUnit({
id: item.productId,
name: `${item.keyfacts?.supplementalDescription?.concat(" ") ?? ""}${item.fullTitle}`,
price: item.price.price,
priceHistory: [{ date: today, price: item.price.price }],
unit,
quantity,
url: `https://www.lidl.at${item.canonicalUrl}`,
}, conversions, 'lidl');
}
return utils.convertUnit(
{
id: item.productId,
name: `${item.keyfacts?.supplementalDescription?.concat(" ") ?? ""}${item.fullTitle}`,
price: item.price.price,
priceHistory: [{ date: today, price: item.price.price }],
unit,
quantity,
url: `https://www.lidl.at${item.canonicalUrl}`,
},
conversions,
"lidl"
);
};
exports.fetchData = async function() {
exports.fetchData = async function () {
const LIDL_SEARCH = `https://www.lidl.at/p/api/gridboxes/AT/de/?max=${HITS}`;
return (await axios.get(LIDL_SEARCH)).data.filter(item => !!item.price.price);
}
return (await axios.get(LIDL_SEARCH)).data.filter((item) => !!item.price.price);
};

90
stores/mpreis.js
View File

@ -2,57 +2,61 @@ const axios = require("axios");
const utils = require("./utils");
const conversions = {
'cm': { unit: 'cm', factor: 1 },
'dag': { unit: 'g', factor: 10 },
'dl': { unit: 'ml', factor: 10 },
'grm': { unit: 'g', factor: 1 },
'kgm': { unit: 'g', factor: 1000 },
'ltr': { unit: 'ml', factor: 1000 },
'mlt': { unit: 'ml', factor: 1 },
'mtr': { unit: 'm', factor: 1 },
'stk': { unit: 'stk', factor: 1 },
'stk.': { unit: 'stk', factor: 1 },
'g': { unit: 'g', factor: 1 },
'anw': { unit: 'stk', factor: 1 },
'l': { unit: 'ml', factor: 1000 },
'm': { unit: 'cm', factor: 100 },
'ml': { unit: 'ml', factor: 1 },
'kg': { unit: 'g', factor: 1000 },
'paar': { unit: 'stk', factor: 1 },
'stück': { unit: 'stk', factor: 1 },
'bl.': { unit: 'stk', factor: 1 },
'pkg': { unit: 'stk', factor: 1 },
'gr': { unit: 'g', factor: 1 },
'er': { unit: 'stk', factor: 1 },
cm: { unit: "cm", factor: 1 },
dag: { unit: "g", factor: 10 },
dl: { unit: "ml", factor: 10 },
grm: { unit: "g", factor: 1 },
kgm: { unit: "g", factor: 1000 },
ltr: { unit: "ml", factor: 1000 },
mlt: { unit: "ml", factor: 1 },
mtr: { unit: "m", factor: 1 },
stk: { unit: "stk", factor: 1 },
"stk.": { unit: "stk", factor: 1 },
g: { unit: "g", factor: 1 },
anw: { unit: "stk", factor: 1 },
l: { unit: "ml", factor: 1000 },
m: { unit: "cm", factor: 100 },
ml: { unit: "ml", factor: 1 },
kg: { unit: "g", factor: 1000 },
paar: { unit: "stk", factor: 1 },
stück: { unit: "stk", factor: 1 },
"bl.": { unit: "stk", factor: 1 },
pkg: { unit: "stk", factor: 1 },
gr: { unit: "g", factor: 1 },
er: { unit: "stk", factor: 1 },
};
exports.getCanonical = function(item, today) {
let quantity = item.prices[0].presentationPrice.measurementUnit.quantity
let unit = item.prices[0].presentationPrice.measurementUnit.unitCode.toLowerCase()
if(['xro', 'h87', 'hlt'].indexOf(unit)!=-1) {
const q = utils.parseUnitAndQuantityAtEnd(item.mixins.productCustomAttributes.packagingUnit)
exports.getCanonical = function (item, today) {
let quantity = item.prices[0].presentationPrice.measurementUnit.quantity;
let unit = item.prices[0].presentationPrice.measurementUnit.unitCode.toLowerCase();
if (["xro", "h87", "hlt"].indexOf(unit) != -1) {
const q = utils.parseUnitAndQuantityAtEnd(item.mixins.productCustomAttributes.packagingUnit);
quantity = q[0] ?? quantity;
unit = q[1];
}
if (!(unit in conversions)) {
unit = 'stk';
unit = "stk";
}
const isWeighted = (item.mixins.productCustomAttributes?.packagingDescription ?? "").startsWith("Gewichtsware");
return utils.convertUnit({
id: item.code,
name: item.name[0],
isWeighted,
price: isWeighted ? item.prices[0].effectiveAmount : item.prices[0].presentationPrice.effectiveAmount,
priceHistory: [{ date: today, price: item.prices[0].presentationPrice.effectiveAmount }],
unit,
quantity,
bio: item.mixins.mpreisAttributes.properties?.includes('BIO'),
url: `https://www.mpreis.at/shop/p/${item.code}`,
}, conversions, 'mpreis');
}
return utils.convertUnit(
{
id: item.code,
name: item.name[0],
isWeighted,
price: isWeighted ? item.prices[0].effectiveAmount : item.prices[0].presentationPrice.effectiveAmount,
priceHistory: [{ date: today, price: item.prices[0].presentationPrice.effectiveAmount }],
unit,
quantity,
bio: item.mixins.mpreisAttributes.properties?.includes("BIO"),
url: `https://www.mpreis.at/shop/p/${item.code}`,
},
conversions,
"mpreis"
);
};
exports.fetchData = async function() {
const MPREIS_URL = `https://ax2ixv4hll-dsn.algolia.net/1/indexes/prod_mpreis_8450/browse?X-Algolia-API-Key=NmJlMTI0NjY1NGU4MDUwYTRlMmYzYWFjOWFlY2U4MGFkNGZjMDY2NmNjNjQzNWY3OWJlNDY4OTY0ZjEwOTEwYWZpbHRlcnM9cHVibGlzaGVk&X-Algolia-Application-Id=AX2IXV4HLL&X-Algolia-Agent=Vue.js`
exports.fetchData = async function () {
const MPREIS_URL = `https://ax2ixv4hll-dsn.algolia.net/1/indexes/prod_mpreis_8450/browse?X-Algolia-API-Key=NmJlMTI0NjY1NGU4MDUwYTRlMmYzYWFjOWFlY2U4MGFkNGZjMDY2NmNjNjQzNWY3OWJlNDY4OTY0ZjEwOTEwYWZpbHRlcnM9cHVibGlzaGVk&X-Algolia-Application-Id=AX2IXV4HLL&X-Algolia-Agent=Vue.js`;
let mpreisItems = [];
let res = (await axios.get(MPREIS_URL)).data;
mpreisItems = mpreisItems.concat(res.hits);
@ -63,4 +67,4 @@ exports.fetchData = async function() {
cursor = res.cursor;
}
return mpreisItems;
}
};

56
stores/penny.js
View File

@ -3,35 +3,39 @@ const utils = require("./utils");
const MAXITEMS = 10000;
const conversions = {
"bd": { unit: 'stk', factor: 1 },
"g": { unit: 'g', factor: 1 },
"gr": { unit: 'g', factor: 1 },
"kg": { unit: 'g', factor: 1000 },
"lt": { unit: 'ml', factor: 1000 },
"ml": { unit: 'ml', factor: 1 },
"pk": { unit: 'stk', factor: 1 },
"pa": { unit: 'stk', factor: 1 },
"rl": { unit: 'stk', factor: 1 },
"st": { unit: 'stk', factor: 1 },
"tb": { unit: 'stk', factor: 1 },
"wg": { unit: 'wg', factor: 1 },
bd: { unit: "stk", factor: 1 },
g: { unit: "g", factor: 1 },
gr: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
lt: { unit: "ml", factor: 1000 },
ml: { unit: "ml", factor: 1 },
pk: { unit: "stk", factor: 1 },
pa: { unit: "stk", factor: 1 },
rl: { unit: "stk", factor: 1 },
st: { unit: "stk", factor: 1 },
tb: { unit: "stk", factor: 1 },
wg: { unit: "wg", factor: 1 },
};
exports.getCanonical = function (item, today) {
let quantity = item.amount;
let unit = item.volumeLabelKey;
return utils.convertUnit({
id: item.productId,
name: item.name,
price: item.price.regular.value / 100,
priceHistory: [{ date: today, price: item.price.regular.value / 100 }],
isWeighted: item.isWeightArticle,
unit,
quantity,
bio: item.name.toLowerCase().includes("bio") && !item.name.toLowerCase().includes("fabio"),
url: `https://www.penny.at/produkte/${item.slug}`,
}, conversions, 'penny');
}
return utils.convertUnit(
{
id: item.productId,
name: item.name,
price: item.price.regular.value / 100,
priceHistory: [{ date: today, price: item.price.regular.value / 100 }],
isWeighted: item.isWeightArticle,
unit,
quantity,
bio: item.name.toLowerCase().includes("bio") && !item.name.toLowerCase().includes("fabio"),
url: `https://www.penny.at/produkte/${item.slug}`,
},
conversions,
"penny"
);
};
exports.fetchData = async function () {
hits = 100;
@ -41,9 +45,9 @@ exports.fetchData = async function () {
while (!done) {
const PENNY_SEARCH = `https://www.penny.at/api/products?page=${page}&pageSize=${hits}`;
data = (await axios.get(PENNY_SEARCH)).data;
done = (data.count < hits || page * hits > MAXITEMS);
done = data.count < hits || page * hits > MAXITEMS;
page++;
result = result.concat(data.results);
}
return result;
}
};

118
stores/rewe-de.js
View File

@ -1,42 +1,47 @@
const axios = require("axios");
const util = require('util');
const exec = util.promisify(require('child_process').exec);
const utils = require('./utils');
const util = require("util");
const exec = util.promisify(require("child_process").exec);
const utils = require("./utils");
const conversions = {
"Beutel": { unit: 'stk', factor: 1 },
"Blatt": { unit: 'stk', factor: 1 },
"Bund": { unit: 'stk', factor: 1 },
"g": { unit: 'g', factor: 1},
"Gramm": { unit: 'g', factor: 1},
"kg": { unit: 'g', factor: 1000},
"Kilogramm": { unit: 'g', factor: 1},
"l": { unit: 'ml', factor: 1000},
"Liter": { unit: 'ml', factor: 1000},
"cm": { unit: 'cm', factor: 1},
"m": { unit: 'cm', factor: 100},
"Meter": { unit: 'cm', factor: 100},
"Milliliter": { unit: 'ml', factor: 1},
"ml": { unit: 'ml', factor: 1},
"Paar": { unit: 'stk', factor: 1 },
"Packung": { unit: 'stk', factor: 1 },
"Portion": { unit: 'stk', factor: 1 },
"Rollen": { unit: 'stk', factor: 1 },
"Stk": { unit: 'stk', factor: 1 },
"Stück": { unit: 'stk', factor: 1 },
"stück": { unit: 'stk', factor: 1 },
"Teebeutel": { unit: 'stk', factor: 1 },
"Waschgang": { unit: 'wg', factor: 1 },
"Zentimeter": { unit: 'cm', factor: 1 },
Beutel: { unit: "stk", factor: 1 },
Blatt: { unit: "stk", factor: 1 },
Bund: { unit: "stk", factor: 1 },
g: { unit: "g", factor: 1 },
Gramm: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
Kilogramm: { unit: "g", factor: 1 },
l: { unit: "ml", factor: 1000 },
Liter: { unit: "ml", factor: 1000 },
cm: { unit: "cm", factor: 1 },
m: { unit: "cm", factor: 100 },
Meter: { unit: "cm", factor: 100 },
Milliliter: { unit: "ml", factor: 1 },
ml: { unit: "ml", factor: 1 },
Paar: { unit: "stk", factor: 1 },
Packung: { unit: "stk", factor: 1 },
Portion: { unit: "stk", factor: 1 },
Rollen: { unit: "stk", factor: 1 },
Stk: { unit: "stk", factor: 1 },
Stück: { unit: "stk", factor: 1 },
stück: { unit: "stk", factor: 1 },
Teebeutel: { unit: "stk", factor: 1 },
Waschgang: { unit: "wg", factor: 1 },
Zentimeter: { unit: "cm", factor: 1 },
};
exports.getCanonical = function (item, today) {
let quantity = 1, unit = "kg";
let quantity = 1,
unit = "kg";
if (item.grammage && item.grammage.length > 0) {
let grammage = item.grammage.trim().replace(/\([^)]*\)/g, '').replace(",", ".").trim();
let grammage = item.grammage
.trim()
.replace(/\([^)]*\)/g, "")
.replace(",", ".")
.trim();
let multiplier = 1;
if (grammage.indexOf("x") != -1) {
let tokens = grammage.split("x")
let tokens = grammage.split("x");
multiplier = Number.parseFloat(tokens[0]);
grammage = tokens[1];
}
@ -51,22 +56,27 @@ exports.getCanonical = function (item, today) {
}
quantity *= multiplier;
} else {
quantity = 1; unit = "Stk";
quantity = 1;
unit = "Stk";
}
let price = Number.parseFloat(item.currentPrice.split(" ")[0].replace(",", "."));
return utils.convertUnit({
id: item.id,
name: item.name,
price,
priceHistory: [{ date: today, price }],
isWeighted: false,
unit,
quantity,
bio: false,
url: "",
}, conversions, "reweDe");
}
return utils.convertUnit(
{
id: item.id,
name: item.name,
price,
priceHistory: [{ date: today, price }],
isWeighted: false,
unit,
quantity,
bio: false,
url: "",
},
conversions,
"reweDe"
);
};
exports.fetchData = async function () {
// For some unholy reason, Axios returns 403 when accessing the endpoint
@ -83,19 +93,31 @@ exports.fetchData = async function () {
return (await axiosNoDefaults.get('https://mobile-api.rewe.de/api/v3/product-search?searchTerm=*&page=1&sorting=RELEVANCE_DESC&objectsPerPage=250&marketCode=440405&serviceTypes=PICKUP', { headers, httpsAgent: agent })).data;*/
try {
await exec("curl --version")
} catch(e) {
await exec("curl --version");
} catch (e) {
console.log("ERROR: Can't fetch REWE-DE data, no curl installed.");
return [];
}
let pageId = 1;
let result = (await exec(`curl -s "https://mobile-api.rewe.de/api/v3/product-search\?searchTerm\=\*\&page\=${pageId++}\&sorting\=RELEVANCE_DESC\&objectsPerPage\=250\&marketCode\=440405\&serviceTypes\=PICKUP" -H "Rd-Service-Types: PICKUP" -H "Rd-Market-Id: 440405"`)).stdout;
let result = (
await exec(
`curl -s "https://mobile-api.rewe.de/api/v3/product-search\?searchTerm\=\*\&page\=${pageId++}\&sorting\=RELEVANCE_DESC\&objectsPerPage\=250\&marketCode\=440405\&serviceTypes\=PICKUP" -H "Rd-Service-Types: PICKUP" -H "Rd-Market-Id: 440405"`
)
).stdout;
const firstPage = JSON.parse(result);
const totalPages = firstPage.totalPages;
const items = [...firstPage.products];
for (let i = 2; i <= totalPages; i++) {
items.push(...JSON.parse((await exec(`curl -s "https://mobile-api.rewe.de/api/v3/product-search\?searchTerm\=\*\&page\=${pageId++}\&sorting\=RELEVANCE_DESC\&objectsPerPage\=250\&marketCode\=440405\&serviceTypes\=PICKUP" -H "Rd-Service-Types: PICKUP" -H "Rd-Market-Id: 440405"`)).stdout).products);
items.push(
...JSON.parse(
(
await exec(
`curl -s "https://mobile-api.rewe.de/api/v3/product-search\?searchTerm\=\*\&page\=${pageId++}\&sorting\=RELEVANCE_DESC\&objectsPerPage\=250\&marketCode\=440405\&serviceTypes\=PICKUP" -H "Rd-Service-Types: PICKUP" -H "Rd-Market-Id: 440405"`
)
).stdout
).products
);
}
return items;
}
};

93
stores/spar.js
View File

@ -3,64 +3,67 @@ const utils = require("./utils");
const HITS = Math.floor(30000 + Math.random() * 2000);
const conversions = {
'g': { unit: 'g', factor: 1 },
'kg': { unit: 'g', factor: 1000 },
'l': { unit: 'ml', factor: 1000 },
'ml': { unit: 'ml', factor: 1 },
'stk': { unit: 'stk', factor: 1 },
'stück': { unit: 'stk', factor: 1 },
'100ml': { unit: 'ml', factor: 100 },
'wg': { unit: 'wg', factor: 1 },
'100g': { unit: 'g', factor: 100 },
'm': { unit: 'cm', factor: 100 },
'cm': { unit: 'cm', factor: 100 },
'ml': { unit: 'ml', factor: 1 },
'meter': { unit: 'cm', factor: 100 },
'mm': { unit: 'cm', factor: .1 },
'stk.': { unit: 'cm', factor: .1 },
'cl': { unit: 'ml', factor: 10 },
'blatt': { unit: 'stk', factor: 1 },
g: { unit: "g", factor: 1 },
kg: { unit: "g", factor: 1000 },
l: { unit: "ml", factor: 1000 },
ml: { unit: "ml", factor: 1 },
stk: { unit: "stk", factor: 1 },
stück: { unit: "stk", factor: 1 },
"100ml": { unit: "ml", factor: 100 },
wg: { unit: "wg", factor: 1 },
"100g": { unit: "g", factor: 100 },
m: { unit: "cm", factor: 100 },
cm: { unit: "cm", factor: 100 },
ml: { unit: "ml", factor: 1 },
meter: { unit: "cm", factor: 100 },
mm: { unit: "cm", factor: 0.1 },
"stk.": { unit: "cm", factor: 0.1 },
cl: { unit: "ml", factor: 10 },
blatt: { unit: "stk", factor: 1 },
};
exports.getCanonical = function(item, today) {
exports.getCanonical = function (item, today) {
let price, unit, quantity;
const description = item.masterValues["short-description-3"] ?? item.masterValues["short-description-2"];
if (item.masterValues["quantity-selector"]) {
const [str_price, str_unit] = item.masterValues["price-per-unit"].split('/');
const [str_price, str_unit] = item.masterValues["price-per-unit"].split("/");
price = parseFloat(str_price.replace("€", ""));
}
else {
} else {
price = item.masterValues.price;
}
if(description) {
if (description) {
const s = description.replace(" EINWEG", "").replace(" MEHRWEG", "").trim();
const q = utils.parseUnitAndQuantityAtEnd(s);
quantity = q[0]
unit = q[1]
quantity = q[0];
unit = q[1];
}
if(conversions[unit]===undefined) {
// use price per unit to calculate quantity (less accurate)
let [unitPrice, unit_] = item.masterValues['price-per-unit'].split('/');
unitPrice = parseFloat(unitPrice.replace("€", ""));
quantity = parseFloat((price / unitPrice).toFixed(3));
unit = unit_.toLowerCase();
if (conversions[unit] === undefined) {
// use price per unit to calculate quantity (less accurate)
let [unitPrice, unit_] = item.masterValues["price-per-unit"].split("/");
unitPrice = parseFloat(unitPrice.replace("€", ""));
quantity = parseFloat((price / unitPrice).toFixed(3));
unit = unit_.toLowerCase();
}
return utils.convertUnit({
id: item.masterValues["code-internal"],
sparId: item.masterValues["product-number"],
name: item.masterValues.title + " " + item.masterValues["short-description"],
price,
priceHistory: [{ date: today, price }],
unit,
quantity,
isWeighted: item.masterValues['item-type'] === 'WeightProduct',
bio: item.masterValues.biolevel === "Bio",
url: `https://www.interspar.at/shop/lebensmittel${item.masterValues.url}`,
}, conversions, 'spar');
}
return utils.convertUnit(
{
id: item.masterValues["code-internal"],
sparId: item.masterValues["product-number"],
name: item.masterValues.title + " " + item.masterValues["short-description"],
price,
priceHistory: [{ date: today, price }],
unit,
quantity,
isWeighted: item.masterValues["item-type"] === "WeightProduct",
bio: item.masterValues.biolevel === "Bio",
url: `https://www.interspar.at/shop/lebensmittel${item.masterValues.url}`,
},
conversions,
"spar"
);
};
exports.fetchData = async function() {
exports.fetchData = async function () {
const SPAR_SEARCH = `https://search-spar.spar-ics.com/fact-finder/rest/v4/search/products_lmos_at?query=*&q=*&page=1&hitsPerPage=${HITS}`;
const rawItems = (await axios.get(SPAR_SEARCH)).data.hits;
return rawItems?.hits || rawItems;
}
};

82
stores/unimarkt.js
View File

@ -2,52 +2,50 @@ const axios = require("axios");
const HTMLParser = require("node-html-parser");
exports.getCanonical = function (item, today) {
return {
id: item.id,
name: item.name,
price: item.price,
priceHistory: [{ date: today, price: item.price }],
unit: item.unit,
bio: item.name.toLowerCase().includes('bio'),
url: `https://shop.unimarkt.at${item.canonicalUrl}`,
};
return {
id: item.id,
name: item.name,
price: item.price,
priceHistory: [{ date: today, price: item.price }],
unit: item.unit,
bio: item.name.toLowerCase().includes("bio"),
url: `https://shop.unimarkt.at${item.canonicalUrl}`,
};
};
exports.fetchData = async function () {
const UNIMARKT_BASE_URL = `https://shop.unimarkt.at/`;
const UNIMARKT_MAIN_CATEGORIES = [
"obst-gemuese",
"kuehlprodukte",
"fleisch-wurst",
"brot-gebaeck",
"getraenke",
"lebensmittel",
"suesses-snacks",
];
const UNIMARKT_BASE_URL = `https://shop.unimarkt.at/`;
const UNIMARKT_MAIN_CATEGORIES = [
"obst-gemuese",
"kuehlprodukte",
"fleisch-wurst",
"brot-gebaeck",
"getraenke",
"lebensmittel",
"suesses-snacks",
];
let unimarktItems = [];
for (let category of UNIMARKT_MAIN_CATEGORIES) {
var res = await axios.get(UNIMARKT_BASE_URL + category, {
validateStatus: function (status) {
return (status >= 200 && status < 300);
},
});
if (res && res.data) {
var root = HTMLParser.parse(res.data);
root
.querySelectorAll(".articleListItem .produktContainer")
.forEach((product) => {
unimarktItems.push({
id: product._attrs["data-articleid"],
name: product.querySelector(".name").text,
price: parseFloat(product._attrs["data-price"]),
unit: product.querySelector(".grammatur").text,
canonicalUrl: product.querySelector(".image > a")._attrs["href"],
});
let unimarktItems = [];
for (let category of UNIMARKT_MAIN_CATEGORIES) {
var res = await axios.get(UNIMARKT_BASE_URL + category, {
validateStatus: function (status) {
return status >= 200 && status < 300;
},
});
if (res && res.data) {
var root = HTMLParser.parse(res.data);
root.querySelectorAll(".articleListItem .produktContainer").forEach((product) => {
unimarktItems.push({
id: product._attrs["data-articleid"],
name: product.querySelector(".name").text,
price: parseFloat(product._attrs["data-price"]),
unit: product.querySelector(".grammatur").text,
canonicalUrl: product.querySelector(".image > a")._attrs["href"],
});
});
}
}
}
return unimarktItems;
return unimarktItems;
};

35
stores/utils.js
View File

@ -1,39 +1,38 @@
exports.convertUnit = function (item, units, store) {
if(!(item.unit in units)) {
if (!(item.unit in units)) {
console.error(`Unknown unit in ${store}: '${item.unit}' in item ${item.name}`);
return item;
}
if(typeof(item.quantity) == 'string')
item.quantity = parseFloat(item.quantity.replace(',', '.'));
if (typeof item.quantity == "string") item.quantity = parseFloat(item.quantity.replace(",", "."));
const conv = units[item.unit];
item.quantity = conv.factor * item.quantity;
item.unit = conv.unit;
if(item.isWeighted && (item.unit =='g' || item.unit == 'ml')) {
item.price = 100*item.price/item.quantity;
if (item.isWeighted && (item.unit == "g" || item.unit == "ml")) {
item.price = (100 * item.price) / item.quantity;
item.quantity = 100;
}
return item;
}
};
exports.parseUnitAndQuantityAtEnd = function (name) {
let unit, quantity = 1;
const nameTokens = name.trim().replaceAll('(','').replaceAll(')','').replaceAll(',', '.').split(' ');
const lastToken = nameTokens[nameTokens.length-1];
const secondLastToken = nameTokens.length >= 2 ? nameTokens[nameTokens.length-2] : null;
let unit,
quantity = 1;
const nameTokens = name.trim().replaceAll("(", "").replaceAll(")", "").replaceAll(",", ".").split(" ");
const lastToken = nameTokens[nameTokens.length - 1];
const secondLastToken = nameTokens.length >= 2 ? nameTokens[nameTokens.length - 2] : null;
const token = parseFloat(lastToken) ? lastToken : secondLastToken + lastToken;
const regex = /^([0-9.x]+)(.*)$/;
const matches = token.match(regex);
if(matches) {
matches[1].split('x').forEach( (q)=> {
quantity = quantity * parseFloat(q)
})
unit = matches[2];
return [quantity, unit.toLowerCase()];
if (matches) {
matches[1].split("x").forEach((q) => {
quantity = quantity * parseFloat(q);
});
unit = matches[2];
return [quantity, unit.toLowerCase()];
}
return [undefined, undefined];
}
};

54
stuff.js
View File

@ -4,20 +4,19 @@ const STORE_KEYS = Object.keys(stores);
function grammageAnalysis() {
const items = JSON.parse(fs.readFileSync("docker/data/latest-canonical.json"));
items.sort(item => item.priceHistory.length);
items.sort((item) => item.priceHistory.length);
for (item of items) {
if (item.priceHistory.length > 2)
console.log(JSON.stringify(item, null, 2));
if (item.priceHistory.length > 2) console.log(JSON.stringify(item, null, 2));
}
const units = {};
const unitsSmall = {}
const unitsSmall = {};
for (item of items) {
const tokens = item.unit ? item.unit.split(/\s+/) : [];
if (tokens.length == 0) continue;
if (tokens[0].charAt(0) >= '0' && tokens[0].charAt(0) <= '9') {
if (tokens[0].charAt(0) >= "0" && tokens[0].charAt(0) <= "9") {
tokens.splice(0, 1);
}
units[tokens.join(" ")] = item;
@ -29,7 +28,7 @@ function grammageAnalysis() {
console.log(Object.keys(unitsSmall).length);
const hofer = JSON.parse(fs.readFileSync("docker/data/hofer-2023-05-19.json"));
const unitTypes = {}
const unitTypes = {};
for (item of hofer) {
unitTypes[item.UnitType] = true;
}
@ -75,7 +74,7 @@ function grammageAnalysis() {
for (item of billa) {
let unit;
if (item.masterValues["quantity-selector"]) {
const [str_price, str_unit] = item.masterValues["price-per-unit"].split('/');
const [str_price, str_unit] = item.masterValues["price-per-unit"].split("/");
unit = str_unit.trim();
} else {
unit = item.masterValues["short-description-3"];
@ -83,7 +82,9 @@ function grammageAnalysis() {
if (!unit) {
noGrammage.push(item);
noGrammageUnits[item.masterValues["sales-unit"]] = noGrammageUnits[item.masterValues["sales-unit"]] ? noGrammageUnits[item.masterValues["sales-unit"]] + 1 : 1;
noGrammageUnits[item.masterValues["sales-unit"]] = noGrammageUnits[item.masterValues["sales-unit"]]
? noGrammageUnits[item.masterValues["sales-unit"]] + 1
: 1;
continue;
}
let tokens = unit.split(" ");
@ -116,8 +117,8 @@ function momentumCartConversion() {
const lines = fs.readFileSync("momentum-cart.csv").toString().split(/\r?\n/);
const cart = {
name: "Momentum Eigenmarken Vergleich",
items: []
}
items: [],
};
for (line of lines) {
const [sparId, billaId] = line.split(/\s+/);
const sparItem = lookup[sparId];
@ -139,7 +140,7 @@ function momentumCartConversion() {
}
function fixSparHistoricalData(dataDir) {
const files = fs.readdirSync(dataDir).filter(file => file.indexOf("canonical") == -1 && file.indexOf(`spar-`) == 0);
const files = fs.readdirSync(dataDir).filter((file) => file.indexOf("canonical") == -1 && file.indexOf(`spar-`) == 0);
console.log(files);
for (file of files) {
@ -151,15 +152,14 @@ function fixSparHistoricalData(dataDir) {
}
}
const nReadlines = require('n-readlines');
const nReadlines = require("n-readlines");
function convertDossierData(dataDir, file) {
console.log(`Converting ${file}`);
const lookup = {};
for (item of JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`))) {
lookup[item.store + item.id] = item;
if (item.sparId)
lookup[item.store + "-" + item.sparId] = item;
if (item.sparId) lookup[item.store + "-" + item.sparId] = item;
}
const lines = new nReadlines(file);
@ -167,10 +167,10 @@ function convertDossierData(dataDir, file) {
const itemsPerDate = {};
let line = null;
const store = file.indexOf("spar") == 0 ? "spar" : "billa";
lines.next()
lines.next();
let itemsTotal = 0;
let notFound = 0;
while (line = lines.next()) {
while ((line = lines.next())) {
itemsTotal++;
const tokens = line.toString("utf-8").split(";");
const dateTokens = tokens[0].split(".");
@ -181,8 +181,7 @@ function convertDossierData(dataDir, file) {
const price = Number.parseFloat(tokens[7].replace("€", "").trim().replace(",", "."));
const id = tokens[4].replace("ARTIKELNUMMER: ", "").replace("Art. Nr.: ", "");
let item = lookup[store + id];
if (!item)
item = lookup[store + "-" + id]
if (!item) item = lookup[store + "-" + id];
if (!item) {
// console.log("Couldn't find item " + name);
notFound++;
@ -199,8 +198,8 @@ function convertDossierData(dataDir, file) {
title: producer,
"short-description": name,
"short-description-3": unit,
bioLevel: ""
}
bioLevel: "",
},
});
} else {
items.push({
@ -208,12 +207,12 @@ function convertDossierData(dataDir, file) {
articleId: id,
name: name,
price: {
final: price
final: price,
},
grammagePriceFactor: 1,
grammage: unit,
}
})
},
});
}
}
console.log("total: " + itemsTotal);
@ -231,8 +230,8 @@ function clownCompress(dataDir) {
const compressed = {
stores: STORE_KEYS,
n: items.length,
data: []
}
data: [],
};
const data = compressed.data;
for (item of items) {
data.push(STORE_KEYS.indexOf(item.store));
@ -274,7 +273,6 @@ function clownCompress(dataDir) {
fs.writeFileSync(`${dataDir}/clown.json`, JSON.stringify(compressed));
}
const clustering = require("./site/utils");
/*let items = JSON.parse(fs.readFileSync("palmolive.json"));
@ -302,5 +300,5 @@ for (cluster of clusters) {
(async () => {
// let items = await stores.reweDe.fetchData();
let items = JSON.parse(fs.readFileSync("tmp/reweDe-2023-05-31.json"));
for (item of items) stores.reweDe.getCanonical(item)
})();
for (item of items) stores.reweDe.getCanonical(item);
})();