Marown
/
heissepreise
mirror of https://github.com/badlogic/heissepreise.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP esbuild, bundling, components.

This commit is contained in:
Mario Zechner 2023-06-08 13:48:08 +02:00
parent f96745cc81
commit ec3c8f4ed3
14 changed files with 1209 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
.vscode/launch.json vendored
View File

@ -4,68 +4,12 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "restore",
"program": "${workspaceFolder}/restore",
"request": "launch",
"skipFiles": [
"<node_internals>/**"
],
"type": "node"
},
{
"name": "stuff",
"program": "${workspaceFolder}/stuff.js",
"request": "launch",
"skipFiles": [
"<node_internals>/**"
],
"type": "node"
},
{
"name": "analysis",
"program": "${workspaceFolder}/analysis.js",
"request": "launch",
"skipFiles": [
"<node_internals>/**"
],
"type": "node"
},
{
"type": "pwa-chrome",
"request": "launch",
"name": "client",
"url": "http://localhost:3001",
"webRoot": "${workspaceFolder}/site"
},
{
"type": "pwa-chrome",
"request": "launch",
"name": "client2",
"name": "frontend",
"url": "http://localhost:3000",
"webRoot": "${workspaceFolder}/site"
"webRoot": "${workspaceFolder}/site/output"
},
{
"type": "node",
"request": "attach",
"name": "server",
"port": 9230,
"address": "localhost",
"localRoot": "${workspaceFolder}",
"remoteRoot": "/heisse-preise",
"protocol": "inspector",
"restart": true,
"continueOnAttach": true
},
],
"compounds": [
{
"name": "client-server",
"configurations": [
"client",
"server"
],
"stopAll": true
}
]
}

41
template.js → bundle.js Normal file → Executable file
View File

@ -1,5 +1,7 @@
const fs = require("fs");
const path = require("path");
const chokidar = require("chokidar");
const esbuild = require("esbuild");
function deleteDirectory(directory) {
if (fs.existsSync(directory)) {
@ -36,6 +38,7 @@ function processFile(inputFile, outputFile) {
console.log(`${inputFile} -> ${outputFile}`);
const fileDir = path.dirname(inputFile);
const data = fs.readFileSync(inputFile, "utf8");
if (data.includes(`require("`)) return;
const replacedData = replaceFileContents(data, fileDir);
fs.writeFileSync(outputFile, replacedData);
}
@ -64,6 +67,40 @@ function generateSite(inputDir, outputDir, deleteOutput) {
});
}
exports.generateSite = generateSite;
function generateSiteAndWatch(inputDir, outputDir, deleteDir = true, watch = false) {
generateSite(inputDir, outputDir, deleteDir);
if (!watch) return;
// generateSite("site", "site/output", true);
const watcher = chokidar.watch(inputDir, { ignored: /(^|[\/\\])\../ });
let initialScan = true;
watcher.on("ready", () => (initialScan = false));
watcher.on("all", (event, filePath) => {
if (initialScan) return;
if (path.resolve(filePath).startsWith(path.resolve(outputDir))) return;
console.log(`File ${filePath} has been ${event}`);
generateSite(inputDir, outputDir, false);
});
console.log(`Watching directory for changes: ${inputDir}`);
}
async function bundle(inputDir, outputDir, liveReload) {
let buildContext = await esbuild.context({
entryPoints: {
"carts-new": `${inputDir}/carts-new.js`,
},
bundle: true,
sourcemap: true,
outdir: outputDir,
logLevel: "debug",
});
if (!liveReload) {
await buildContext.rebuild();
} else {
buildContext.watch();
}
generateSiteAndWatch(inputDir, outputDir, false, liveReload);
}
exports.deleteDirectory = deleteDirectory;
exports.generateSite = generateSiteAndWatch;
exports.bundle = bundle;

840
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

13
package.json
View File

@ -7,7 +7,8 @@
"prepare": "husky install",
"dev": "NODE_ENV=development PORT=$PORT node server.js",
"start": "NODE_ENV=production PORT=$PORT node server.js",
"format": "npx prettier --write ."
"format": "npx prettier --write .",
"build": "bundle.js"
},
"repository": {
"type": "git",
@ -18,13 +19,6 @@
"bugs": {
"url": "https://github.com/badlogic/heissepreise/issues"
},
"nodemonConfig": {
"ignore": [
"data/*",
"site/output/*",
"node_modules/*"
]
},
"homepage": "https://github.com/badlogic/heissepreise#readme",
"dependencies": {
"axios": "^1.4.0",
@ -35,8 +29,9 @@
"node-html-parser": "^6.1.5"
},
"devDependencies": {
"concurrently": "^8.1.0",
"esbuild": "^0.17.19",
"husky": "^8.0.3",
"nodemon": "^2.0.22",
"prettier": "^2.8.8",
"pretty-quick": "^3.1.3",
"socket.io": "^4.6.2"

4
pages.js
View File

@ -1,7 +1,7 @@
const fs = require("fs");
const path = require("path");
const analysis = require("./analysis.js");
const template = require("./template.js");
const bundle = require("./bundle.js");
const outputDir = path.resolve("docs");
const dataDir = path.join(outputDir, "data");
@ -26,7 +26,7 @@ function deleteFiles(folderPath) {
try {
if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir);
deleteFiles(outputDir);
template.generateSite("site", outputDir, false);
bundle.bundle("site", outputDir, false);
const data = analysis.readJSON(`${dataDir}/latest-canonical.json`);
analysis.writeJSON(`${dataDir}/latest-canonical.json`, data, analysis.FILE_COMPRESSOR);

25
server.js
View File

@ -1,9 +1,9 @@
const fs = require("fs");
const path = require("path");
const http = require("http");
const chokidar = require("chokidar");
const analysis = require("./analysis");
const template = require("./template");
const bundle = require("./bundle");
const chokidar = require("chokidar");
const express = require("express");
const compression = require("compression");
@ -37,21 +37,6 @@ function scheduleFunction(hour, minute, second, func) {
}, delay);
}
function generateSiteAndWatch(inputDir, outputDir) {
template.generateSite(inputDir, outputDir, true);
const watcher = chokidar.watch(inputDir, { ignored: /(^|[\/\\])\../ });
let initialScan = true;
watcher.on("ready", () => (initialScan = false));
watcher.on("all", (event, filePath) => {
if (initialScan) return;
if (path.resolve(filePath).startsWith(path.resolve(outputDir))) return;
console.log(`File ${filePath} has been ${event}`);
template.generateSite(inputDir, outputDir, false);
});
console.log(`Watching directory for changes: ${inputDir}`);
}
(async () => {
const dataDir = "data";
@ -81,7 +66,11 @@ function generateSiteAndWatch(inputDir, outputDir) {
fs.mkdirSync(dataDir);
}
generateSiteAndWatch("site", "site/output");
const outputDir = "site/output";
bundle.deleteDirectory(outputDir);
fs.mkdirSync(outputDir);
fs.mkdirSync(outputDir + "/data");
bundle.bundle("site", outputDir, liveReload);
analysis.migrateCompression(dataDir, ".json", ".json.br");
analysis.migrateCompression(dataDir, ".json.gz", ".json.br");

17
site/carts-new.html Normal file
View File

@ -0,0 +1,17 @@
%%_templates/_header.html%% %%_templates/_menu.html%%
<div class="w-full relative px-4">
<div class="max-w-3xl mx-auto">
<h1 class="text-2xl font-bold pb-2 pt-8 text-center">Warenkörbe</h1>
<div class="px-4 py-2 my-4 text-sm border rounded-xl md:mt-8 md:rounded-b-none md:mb-0 bg-gray-100 flex gap-4" id="numresults">
<input type="button" id="new" value="Neuer Warenkorb" class="text-primary font-medium hover:underline cursor-pointer" />
<input type="button" id="export" value="Exportieren" class="text-primary font-medium hover:underline cursor-pointer" />
<input type="button" id="import" value="Importieren" class="text-primary font-medium hover:underline cursor-pointer" />
</div>
<table id="carts" class="carts w-full"></table>
<input type="file" id="fileInput" class="hidden" />
</div>
</div>
<script src="carts-new.js"></script>
%%_templates/_footer.html%%

57
site/carts-new.js Normal file
View File

@ -0,0 +1,57 @@
const { dom, downloadJSON } = require("./misc");
const model = require("./model");
function render(carts) {}
(async () => {
await model.load();
const carts = model.carts.carts;
document.querySelector("#new").addEventListener("click", () => {
let name = prompt("Name für Warenkorb eingeben:");
if (!name || name.trim().length == 0) return;
name = name.trim();
if (carts.some((cart) => cart.name === name)) {
alert("Warenkorb mit Namen '" + name + "' existiert bereits");
return;
}
model.carts.add(name);
location.href = `/cart.html?name=${encodeURIComponent(name)}`;
});
document.querySelector("#export").addEventListener("click", () => {
downloadJSON("carts.json", carts);
});
document.querySelector("#import").addEventListener("click", () => {});
document.querySelector("#fileInput").addEventListener("change", function (event) {
const reader = new FileReader();
reader.onload = (event) => {
const importedCarts = JSON.parse(event.target.result);
for (const importedCart of importedCarts) {
const items = [];
for (const cartItem of importedCart.items) {
const item = model.items.lookup[cartItem.store + cartItem.id];
if (!item) continue;
items.push(item);
}
importedCart.items = items;
const index = carts.findIndex((cart) => cart.name === importedCart.name);
if (index != -1) {
if (confirm("Existierenden Warenkorb '" + importedCart.name + " überschreiben?")) {
carts[index] = importedCart;
}
} else {
carts.push(importedCart);
}
}
model.carts.save();
render(carts);
};
reader.readAsText(event.target.files[0]);
});
render(carts);
})();

46
site/misc.js Normal file
View File

@ -0,0 +1,46 @@
if (typeof window !== "undefined") {
function setupLiveEdit() {
if (window.location.host.indexOf("localhost") < 0 && window.location.host.indexOf("127.0.0.1") < 0) return;
var script = document.createElement("script");
script.type = "text/javascript";
script.onload = () => {
let lastChangeTimestamp = null;
let socket = io({ transports: ["websocket"] });
socket.on("connect", () => console.log("Connected"));
socket.on("disconnect", () => console.log("Disconnected"));
socket.on("message", (timestamp) => {
if (lastChangeTimestamp != timestamp) {
setTimeout(() => location.reload(), 100);
lastChangeTimestamp = timestamp;
}
});
};
script.src = "js/socket.io.js";
document.body.appendChild(script);
}
setupLiveEdit();
}
exports.fetchJSON = async (url) => {
const response = await fetch(url);
return await response.json();
};
exports.downloadJSON = (filename, content) => {
const json = JSON.stringify(content, null, 2);
const blob = new Blob([json], { type: "text/plain" });
const element = document.createElement("a");
element.href = URL.createObjectURL(blob);
element.download = filename;
element.style.display = "none";
document.body.appendChild(element);
element.click();
document.body.removeChild(element);
URL.revokeObjectURL(element.href);
};
exports.dom = (html) => {
const div = document.createElement("div");
element.innerHTML = html;
return element.children[0];
};

43
site/model/carts.js Normal file
View File

@ -0,0 +1,43 @@
const misc = require("../misc");
exports.carts = [];
exports.load = async (itemsLookup) => {
const val = localStorage.getItem("carts");
const carts = (exports.carts = val ? JSON.parse(val) : []);
// Add Momentum cart if it is not in the list of carts
if (!carts.some((cart) => cart.name === "Momentum Eigenmarken Vergleich")) {
const momentumCart = await misc.fetchJSON("data/momentum-cart.json");
carts.unshift(momentumCart);
}
// Update items in cart to their latest version.
for (const cart of carts) {
const items = [];
for (const cartItem of cart.items) {
const item = itemsLookup[cartItem.store + cartItem.id];
if (!item) items.push(cartItem);
else items.push(item);
}
cart.items = items;
}
exports.save();
};
exports.save = () => {
localStorage.setItem("carts", JSON.stringify(exports.carts, null, 2));
};
exports.add = (name) => {
exports.carts.push({
name: name,
items: [],
});
exports.save();
};
exports.remove = (name) => {
exports.carts = exports.carts.filter((cart) => cart.name !== name);
exports.save();
};

116
site/model/categories.js Normal file
View File

@ -0,0 +1,116 @@
// These are a match of the Billa categories, which are organized in a 2-level hierarchy.
// Each category in the top level gets a code from 1-Z, each sub category also gets a code.
// Together the two codes from a unique id for the category, which we store in the item.category
// field. E.g. "Obst & Gemüse > Salate" has the code "13", "Kühlwaren > Tofu" has the code "4C"
exports.categories = [
{
name: "Obst & Gemüse",
subcategories: ["Obst", "Gemüse", "Salate", "Trockenfrüchte & Nüsse"],
},
{
name: "Brot & Gebäck",
subcategories: ["Aufbackbrötchen & Toast", "Brot & Gebäck", "Knäckebrot & Zwieback", "Kuchen & Co.", "Semmelwürfel & Brösel"],
},
{
name: "Getränke",
subcategories: ["Alkoholfreie Getränke", "Bier & Radler", "Kaffee, Tee & Co.", "Sekt & Champagner", "Spirituosen", "Wein", "Mineralwasser"],
},
{
name: "Kühlwaren",
subcategories: [
"Schnelle Küche",
"Eier",
"Fleisch",
"Käse, Aufstriche & Salate",
"Milchprodukte",
"Feinkostplatten & Brötchen",
"Blätterteig, Strudelteig",
"Wurst, Schinken & Speck",
"Feinkost",
"Fisch",
"Unbekannt", // Not available in Billa hierarchy, left blank
"Tofu",
],
},
{
name: "Tiefkühl",
subcategories: [
"Eis",
"Unbekannt", // Not available in Billa hierarchy, left blank
"Fertiggerichte",
"Fisch & Garnelen",
"Gemüse & Kräuter",
"Pommes Frites & Co.",
"Pizza & Baguette",
"Desserts & Früchte",
],
},
{
name: "Grundnahrungsmittel",
subcategories: [
"Asia & Mexican Produkte",
"Baby",
"Backen",
"Essig & Öl",
"Fertiggerichte",
"Gewürze & Würzmittel",
"Honig, Marmelade & Co.",
"Konserven & Sauerwaren",
"Kuchen & Co.",
"Mehl & Getreideprodukte",
"Müsli & Cerealien",
"Reis, Teigwaren & Sugo",
"Saucen & Dressings",
"Spezielle Ernährung",
"Zucker & Süßstoffe",
"Fixprodukte",
],
},
{
name: "Süßes & Salziges",
subcategories: ["Biskotten & Eiswaffeln", "Für kluge Naschkatzen", "Müsliriegel", "Chips & Co.", "Süßes"],
},
{
name: "Pflege",
subcategories: [
"Baby",
"Damenhygiene",
"Deodorants",
"Haarpflege & Haarfarben",
"Pflaster & Verbandsmaterial",
"Haut- & Lippenpflege",
"Mund- & Zahnhygiene",
"Rasierbedarf",
"Seife & Duschbäder",
"Sonnen- & Gelsenschutzmittel",
"Verhütungsmittel",
"Fußpflege",
"Strumpfhosen & Socken",
],
},
{
name: "Haushalt",
subcategories: [
"Büro- & Schulartikel",
"Garten",
"Kleben & Befestigen",
"Küchenartikel",
"Küchenrollen & WC-Papier",
"Lampen & Batterien",
"Müllsäcke, Gefrierbeutel & Co.",
"Raumsprays & Kerzen",
"Reinigen & Pflegen",
"Taschentücher & Servietten",
"Waschmittel & Weichspüler",
"Schuhpflege",
"Kunststoffbehälter",
"Insektenschutz",
"Spielwaren",
"Hygiene-Schutzartikel",
],
},
{
name: "Haustier",
subcategories: ["Hunde", "Katzen", "Nager", "Vögel"],
},
];

9
site/model/index.js Normal file
View File

@ -0,0 +1,9 @@
exports.stores = require("./stores");
exports.categories = require("./categories");
exports.items = require("./items");
exports.carts = require("./carts");
exports.load = async () => {
await exports.items.load();
await exports.carts.load(exports.items.lookup);
};

104
site/model/items.js Normal file
View File

@ -0,0 +1,104 @@
const { stores, STORE_KEYS } = require("./stores");
function decompress(compressedItems) {
const items = [];
const storeLookup = compressedItems.stores;
const data = compressedItems.data;
const numItems = compressedItems.n;
let i = 0;
while (items.length < numItems) {
const store = storeLookup[data[i++]];
const id = data[i++];
const name = data[i++];
const numPrices = data[i++];
const prices = [];
for (let j = 0; j < numPrices; j++) {
const date = data[i++];
const price = data[i++];
prices.push({
date: date.substring(0, 4) + "-" + date.substring(4, 6) + "-" + date.substring(6, 8),
price,
});
}
const unit = data[i++];
const quantity = data[i++];
const isWeighted = data[i++] == 1;
const bio = data[i++] == 1;
const url = stores[store].getUrl({ id, name, url: data[i++] });
items.push({
store,
id,
name,
price: prices[0].price,
priceHistory: prices,
isWeighted,
unit,
quantity,
bio,
url,
});
}
return items;
}
exports.items = [];
exports.lookup = {};
exports.load = async () => {
now = performance.now();
const compressedItemsPerStore = [];
for (const store of STORE_KEYS) {
compressedItemsPerStore.push(
new Promise(async (resolve) => {
const now = performance.now();
try {
const response = await fetch(`data/latest-canonical.${store}.compressed.json`);
const json = await response.json();
console.log(`Loading compressed items for ${store} took ${(performance.now() - now) / 1000} secs`);
resolve(decompress(json));
} catch (e) {
console.error(e);
console.log(
`Error while loading compressed items for ${store}. It took ${(performance.now() - now) / 1000} secs, continueing...`
);
resolve([]);
}
})
);
}
const items = [].concat(...(await Promise.all(compressedItemsPerStore)));
console.log("Loading compressed items in parallel took " + (performance.now() - now) / 1000 + " secs");
const lookup = {};
now = performance.now();
for (const item of items) {
lookup[item.store + item.id] = item;
item.search = item.name + " " + item.quantity + " " + item.unit;
item.search = item.search.toLowerCase().replace(",", ".");
item.numPrices = item.priceHistory.length;
item.priceOldest = item.priceHistory[item.priceHistory.length - 1].price;
item.dateOldest = item.priceHistory[item.priceHistory.length - 1].date;
item.date = item.priceHistory[0].date;
let highestPriceBefore = -1;
let lowestPriceBefore = 100000;
for (let i = 1; i < item.priceHistory.length; i++) {
const price = item.priceHistory[i];
if (i < 10) {
item["price" + i] = price.price;
item["date" + i] = price.date;
}
highestPriceBefore = Math.max(highestPriceBefore, price.price);
lowestPriceBefore = Math.min(lowestPriceBefore, price.price);
}
if (highestPriceBefore == -1) highestPriceBefore = item.price;
if (lowestPriceBefore == 100000) lowestPriceBefore = item.price;
item.highestBefore = highestPriceBefore;
item.lowestBefore = lowestPriceBefore;
}
console.log("Processing items took " + (performance.now() - now) / 1000 + " secs");
exports.items = items;
exports.lookup = lookup;
};

65
site/model/stores.js Normal file
View File

@ -0,0 +1,65 @@
exports.stores = {
billa: {
name: "Billa",
budgetBrands: ["clever"],
color: "yellow",
getUrl: (item) => `https://shop.billa.at${item.url}`,
},
spar: {
name: "Spar",
budgetBrands: ["s-budget"],
color: "green",
getUrl: (item) => `https://www.interspar.at/shop/lebensmittel${item.url}`,
},
hofer: {
name: "Hofer",
budgetBrands: ["milfina"],
color: "purple",
getUrl: (item) => `https://www.roksh.at/hofer/produkte/${item.url}`,
},
lidl: {
name: "Lidl",
budgetBrands: ["milbona"],
color: "pink",
getUrl: (item) => `https://www.lidl.at${item.url}`,
},
mpreis: {
name: "MPREIS",
budgetBrands: [],
color: "rose",
getUrl: (item) => `https://www.mpreis.at/shop/p/${item.id}`,
},
dm: {
name: "DM",
budgetBrands: ["balea"],
color: "orange",
getUrl: (item) => `https://www.dm.at/product-p${item.id}.html`,
},
unimarkt: {
name: "Unimarkt",
budgetBrands: ["jeden tag", "unipur"],
color: "blue",
getUrl: (item) => `https://shop.unimarkt.at/${item.url}`,
},
penny: {
name: "Penny",
budgetBrands: ["bravo", "echt bio!", "san fabio", "federike", "blik", "berida", "today", "ich bin österreich"],
color: "purple",
getUrl: (item) => `https://www.penny.at/produkte/${item.url}`,
},
dmDe: {
name: "DM DE",
budgetBrands: ["balea"],
color: "teal",
getUrl: (item) => `https://www.dm.de/product-p${item.id}.html`,
},
reweDe: {
name: "REWE DE",
budgetBrands: ["ja!"],
color: "stone",
getUrl: (item) => `https://shop.rewe.de/p/${item.name.toLowerCase().replace(/ /g, "-")}/${item.id}`,
},
};
exports.STORE_KEYS = Object.keys(exports.stores);
exports.BUDGET_BRANDS = [...new Set([].concat(...Object.values(exports.stores).map((store) => store.budgetBrands)))];