Merge branch 'badlogic:main' into main

2024-06-16 05:34:21 +02:00 · 2023-06-23 08:08:33 +02:00 · 2023-06-23 08:08:33 +02:00 · 2e2af585ef
commit 2e2af585ef
parent eeff2abab6 9420c3ae3e
7 changed files with 385 additions and 10 deletions
--- a/site/js/misc.js
+++ b/site/js/misc.js
@ -120,9 +120,10 @@ exports.queryItemsAlasql = (query, items) => {
 };

 exports.queryItems = (query, items, exactWord) => {
-    query = query.trim();
-    if (query.length < 3) return [];
-    let tokens = query.split(/\s+/).map((token) => token.toLowerCase().replace(",", "."));
+    query = query.trim().replace(",", ".").toLowerCase();
+    if (query.length < 3) return { items: [], queryTokens: [] };
+    const regex = /([\p{L}-][\p{L}\p{N}-]*)|(>=|<=|=|>|<)|(\d+(\.\d+)?)/gu;
+    let tokens = query.match(regex);

    // Find quantity/unit query
    let newTokens = [];
@ -212,7 +213,7 @@ exports.queryItems = (query, items, exactWord) => {
            if (allUnitsMatched) hits.push(item);
        }
    }
-    return hits;
+    return { items: hits, queryTokens: tokens.filter((token) => !token.startsWith("-")) };
 };

 exports.onVisibleOnce = (target, callback) => {
--- a/site/model/stores.js
+++ b/site/model/stores.js
@ -24,7 +24,7 @@ exports.stores = {
    },
    lidl: {
        name: "Lidl",
-        budgetBrands: ["milbona"],
+        budgetBrands: ["milbona", "alpengut", "cien", "livarno", "wiesentaler"],
        color: "pink",
        defaultChecked: true,
        getUrl: (item) => `https://www.lidl.at${item.url}`,
--- a/site/views/items-filter.js
+++ b/site/views/items-filter.js
@ -262,9 +262,12 @@ class ItemsFilter extends View {
                    elements.sqlError.classList.remove("hidden");
                    elements.sqlError.innerText = e.message;
                    filteredItems = [];
+                    this.lastQueryTokens = [];
                }
            } else {
-                filteredItems = queryItems(query, filteredItems, elements.exact.checked);
+                const result = queryItems(query, filteredItems, elements.exact.checked);
+                filteredItems = result.items;
+                this.model.lastQueryTokens = result.queryTokens;
            }
        }

--- a/site/views/items-list.js
+++ b/site/views/items-list.js
@ -184,6 +184,16 @@ class ItemsList extends View {
        return items;
    }

+    highlightMatches(keywords, name) {
+        let highlightedName = name;
+        for (let i = 0; i < keywords.length; i++) {
+            const string = keywords[i];
+            const regex = new RegExp(string, "gi");
+            highlightedName = highlightedName.replace(regex, "<strong>$&</strong>");
+        }
+        return `${highlightedName}`;
+    }
+
    renderItem(item) {
        if (!this._itemTemplate) {
            this._itemTemplate = dom(
@ -294,7 +304,7 @@ class ItemsList extends View {
        const elements = View.elements(itemDom);
        elements.store.innerText = item.store;
        elements.name.href = stores[item.store].getUrl(item);
-        elements.name.innerText = item.name + (item.unavailable ? " 💀" : "");
+        elements.name.innerHTML = this.highlightMatches(this.model.lastQueryTokens ?? [], item.name) + (item.unavailable ? " 💀" : "");
        elements.quantity.innerText = (item.isWeighted ? "⚖ " : "") + `${quantity} ${unit}`;
        elements.price.innerText = `€ ${Number(showUnitPrice ? unitPrice : price).toFixed(2)} ${priceUnit}`;
        elements.priceHistory.innerHTML = priceHistory;
--- a/stores/lidl.js
+++ b/stores/lidl.js
@ -36,16 +36,19 @@ exports.getCanonical = function (item, today) {
        unit = unit.split("-")[0];
    }

+    const name = `${item.keyfacts?.supplementalDescription?.concat(" ") ?? ""}${item.fullTitle}`;
+
    return utils.convertUnit(
        {
            id: item.productId,
-            name: `${item.keyfacts?.supplementalDescription?.concat(" ") ?? ""}${item.fullTitle}`,
+            name,
            description: item.keyfacts?.description ?? "",
            price: item.price.price,
            priceHistory: [{ date: today, price: item.price.price }],
            unit,
            quantity,
            url: item.canonicalUrl,
+            bio: name.toLowerCase().includes("bio"),
        },
        units,
        "lidl"
--- a/stores/penny-categories.json
+++ b/stores/penny-categories.json
@ -0,0 +1,322 @@
+[
+  {
+    "id": "Obst & Gemüse",
+    "url": "https://www.penny.at/kategorie/obst-und-gemuese-13013",
+    "code": "00"
+  },
+  {
+    "id": "Obst & Gemüse -> Gemüse & Salate",
+    "url": "https://www.penny.at/kategorie/gemuese-und-salate-13015",
+    "code": "01"
+  },
+  {
+    "id": "Obst & Gemüse -> Obst",
+    "url": "https://www.penny.at/kategorie/obst-13014",
+    "code": "00"
+  },
+  {
+    "id": "Brot & Gebäck",
+    "url": "https://www.penny.at/kategorie/brot-und-gebaeck-13018",
+    "code": "10"
+  },
+  {
+    "id": "Brot & Gebäck -> Kuchen & Co.",
+    "url": "https://www.penny.at/kategorie/kuchen-und-co-13022",
+    "code": "13"
+  },
+  {
+    "id": "Brot & Gebäck -> Brot & Gebäck",
+    "url": "https://www.penny.at/kategorie/brot-und-gebaeck-13021",
+    "code": "11"
+  },
+  {
+    "id": "Brot & Gebäck -> Aufbackbrötchen & Toast",
+    "url": "https://www.penny.at/kategorie/aufbackbroetchen-und-toast-13019",
+    "code": "10"
+  },
+  {
+    "id": "Getränke",
+    "url": "https://www.penny.at/kategorie/getraenke-13024",
+    "code": ""
+  },
+  {
+    "id": "Getränke -> Bier & Radler",
+    "url": "https://www.penny.at/kategorie/bier-und-radler-13026",
+    "code": "21"
+  },
+  {
+    "id": "Getränke -> Wein",
+    "url": "https://www.penny.at/kategorie/wein-13030",
+    "code": "25"
+  },
+  {
+    "id": "Getränke -> Kaffee, Tee & Co.",
+    "url": "https://www.penny.at/kategorie/kaffee-tee-und-co-13027",
+    "code": "22"
+  },
+  {
+    "id": "Getränke -> Sekt & Champagner",
+    "url": "https://www.penny.at/kategorie/sekt-und-champagner-13028",
+    "code": "23"
+  },
+  {
+    "id": "Getränke -> Alkoholfreie Getränke",
+    "url": "https://www.penny.at/kategorie/alkoholfreie-getraenke-13025",
+    "code": "20"
+  },
+  {
+    "id": "Getränke -> Spirituosen",
+    "url": "https://www.penny.at/kategorie/spirituosen-13029",
+    "code": "24"
+  },
+  {
+    "id": "Getränke -> Mineralwasser",
+    "url": "https://www.penny.at/kategorie/mineralwasser-13031",
+    "code": "26"
+  },
+  {
+    "id": "Kühlwaren",
+    "url": "https://www.penny.at/kategorie/kuehlwaren-13032",
+    "code": ""
+  },
+  {
+    "id": "Kühlwaren -> Milchprodukte",
+    "url": "https://www.penny.at/kategorie/milchprodukte-13039",
+    "code": "34"
+  },
+  {
+    "id": "Kühlwaren -> Wurst, Schinken & Speck",
+    "url": "https://www.penny.at/kategorie/wurst-schinken-und-speck-13044",
+    "code": "37"
+  },
+  {
+    "id": "Kühlwaren -> Fleisch",
+    "url": "https://www.penny.at/kategorie/fleisch-13035",
+    "code": "32"
+  },
+  {
+    "id": "Kühlwaren -> Fisch",
+    "url": "https://www.penny.at/kategorie/fisch-13045",
+    "code": "39"
+  },
+  {
+    "id": "Kühlwaren -> Käse, Aufstriche & Salate",
+    "url": "https://www.penny.at/kategorie/kaese-aufstriche-und-salate-13038",
+    "code": "33"
+  },
+  {
+    "id": "Kühlwaren -> Eier",
+    "url": "https://www.penny.at/kategorie/eier-13034",
+    "code": "31"
+  },
+  {
+    "id": "Kühlwaren -> Tofu & Vegetarische/Vegane Produkte",
+    "url": "https://www.penny.at/kategorie/tofu-und-vegetarischevegane-produkte-13046",
+    "code": "3B"
+  },
+  {
+    "id": "Tiefkühl",
+    "url": "https://www.penny.at/kategorie/tiefkuehl-13047",
+    "code": "40"
+  },
+  {
+    "id": "Tiefkühl -> Desserts & Früchte",
+    "url": "https://www.penny.at/kategorie/desserts-und-fruechte-13054",
+    "code": "47"
+  },
+  {
+    "id": "Tiefkühl -> Eis",
+    "url": "https://www.penny.at/kategorie/eis-13048",
+    "code": "40"
+  },
+  {
+    "id": "Tiefkühl -> Fertiggerichte",
+    "url": "https://www.penny.at/kategorie/fertiggerichte-13049",
+    "code": "42"
+  },
+  {
+    "id": "Tiefkühl -> Fisch & Garnelen",
+    "url": "https://www.penny.at/kategorie/fisch-und-garnelen-13050",
+    "code": "43"
+  },
+  {
+    "id": "Tiefkühl -> Gemüse & Kräuter",
+    "url": "https://www.penny.at/kategorie/gemuese-und-kraeuter-13051",
+    "code": "44"
+  },
+  {
+    "id": "Grundnahrungsmittel",
+    "url": "https://www.penny.at/kategorie/grundnahrungsmittel-13055",
+    "code": "50"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Fertiggerichte",
+    "url": "https://www.penny.at/kategorie/fertiggerichte-13059",
+    "code": "54"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Konserven & Sauerwaren",
+    "url": "https://www.penny.at/kategorie/konserven-und-sauerwaren-13062",
+    "code": "57"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Essig & Öle",
+    "url": "https://www.penny.at/kategorie/essig-und-oele-13058",
+    "code": "53"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Müsli & Cerealien",
+    "url": "https://www.penny.at/kategorie/muesli-und-cerealien-13065",
+    "code": "5A"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Asia Produkte",
+    "url": "https://www.penny.at/kategorie/asia-produkte-13056",
+    "code": "50"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Spezielle Ernährung",
+    "url": "https://www.penny.at/kategorie/spezielle-ernaehrung-13068",
+    "code": "5D"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Reis, Teigwaren & Sugo",
+    "url": "https://www.penny.at/kategorie/reis-teigwaren-und-sugo-13066",
+    "code": "5B"
+  },
+  {
+    "id": "Grundnahrungsmittel -> Backen",
+    "url": "https://www.penny.at/kategorie/backen-13057",
+    "code": "52"
+  },
+  {
+    "id": "Süßes & Salziges",
+    "url": "https://www.penny.at/kategorie/suesses-und-salziges-13071",
+    "code": "60"
+  },
+  {
+    "id": "Süßes & Salziges -> Süßwaren",
+    "url": "https://www.penny.at/kategorie/suesswaren-13075",
+    "code": "64"
+  },
+  {
+    "id": "Süßes & Salziges -> Schokolade",
+    "url": "https://www.penny.at/kategorie/schokolade-13074",
+    "code": "64"
+  },
+  {
+    "id": "Süßes & Salziges -> Chips & Co.",
+    "url": "https://www.penny.at/kategorie/chips-und-co-13073",
+    "code": "63"
+  },
+  {
+    "id": "Süßes & Salziges -> Müsliriegel",
+    "url": "https://www.penny.at/kategorie/muesliriegel-13072",
+    "code": "62"
+  },
+  {
+    "id": "Haustier",
+    "url": "https://www.penny.at/kategorie/haustier-13090",
+    "code": "90"
+  },
+  {
+    "id": "Haustier -> Hunde",
+    "url": "https://www.penny.at/kategorie/hunde-13091",
+    "code": "90"
+  },
+  {
+    "id": "Haustier -> Katzen",
+    "url": "https://www.penny.at/kategorie/katzen-13092",
+    "code": "91"
+  },
+  {
+    "id": "Haushalt",
+    "url": "https://www.penny.at/kategorie/haushalt-13089",
+    "code": "80"
+  },
+  {
+    "id": "Haushalt -> Küchenrollen & WC-Papier",
+    "url": "https://www.penny.at/kategorie/kuechenrollen-und-wcpapier-13094",
+    "code": "84"
+  },
+  {
+    "id": "Haushalt -> Waschmittel & Weichspüler",
+    "url": "https://www.penny.at/kategorie/waschmittel-und-weichspueler-13099",
+    "code": "8A"
+  },
+  {
+    "id": "Haushalt -> Reinigen & Pflegen",
+    "url": "https://www.penny.at/kategorie/reinigen-und-pflegen-13097",
+    "code": "88"
+  },
+  {
+    "id": "Haushalt -> Pflanzen & Blumen",
+    "url": "https://www.penny.at/kategorie/pflanzen-und-blumen-13100",
+    "code": "81"
+  },
+  {
+    "id": "Haushalt -> Taschentücher & Servietten",
+    "url": "https://www.penny.at/kategorie/taschentuecher-und-servietten-13098",
+    "code": "89"
+  },
+  {
+    "id": "Pflege",
+    "url": "https://www.penny.at/kategorie/pflege-13076",
+    "code": "70"
+  },
+  {
+    "id": "Pflege -> Haut- & Lippenpflege",
+    "url": "https://www.penny.at/kategorie/haut-und-lippenpflege-13082",
+    "code": "75"
+  },
+  {
+    "id": "Pflege -> Seifen & Duschbäder",
+    "url": "https://www.penny.at/kategorie/seifen-und-duschbaeder-13085",
+    "code": "78"
+  },
+  {
+    "id": "Pflege -> Deodrants",
+    "url": "https://www.penny.at/kategorie/deodrants-13079",
+    "code": "72"
+  },
+  {
+    "id": "Pflege -> Sonnen- & Insektenschutzmittel",
+    "url": "https://www.penny.at/kategorie/sonnen-und-insektenschutzmittel-13088",
+    "code": "79"
+  },
+  {
+    "id": "Pflege -> Strumpfhosen & Socken",
+    "url": "https://www.penny.at/kategorie/strumpfhosen-und-socken-13086",
+    "code": "7C"
+  },
+  {
+    "id": "Non-Food",
+    "url": "https://www.penny.at/kategorie/nonfood-13106",
+    "code": "80"
+  },
+  {
+    "id": "Non-Food -> Spiele, Bücher & Co.",
+    "url": "https://www.penny.at/kategorie/spiele-buecher-und-co-13110",
+    "code": "8E"
+  },
+  {
+    "id": "Non-Food -> Haushalt",
+    "url": "https://www.penny.at/kategorie/haushalt-13109",
+    "code": "82"
+  },
+  {
+    "id": "Non-Food -> Bekleidung & Textilien",
+    "url": "https://www.penny.at/kategorie/bekleidung-und-textilien-13107",
+    "code": null
+  },
+  {
+    "id": "Non-Food -> Saison",
+    "url": "https://www.penny.at/kategorie/saison-13111",
+    "code": null
+  },
+  {
+    "id": "Non-Food -> Küche",
+    "url": "https://www.penny.at/kategorie/kueche-13108",
+    "code": null
+  }
+]
--- a/stores/penny.js
+++ b/stores/penny.js
@ -1,5 +1,6 @@
 const axios = require("axios");
 const utils = require("./utils");
+const HTMLParser = require("node-html-parser");
 const MAXITEMS = 10000;

 const units = {
@ -48,8 +49,43 @@ exports.fetchData = async function () {
    return result;
 };

-exports.initializeCategoryMapping = async () => {};
+async function parseCategory(url, parent, result) {
+    const data = (await axios.get(url)).data;
+    const dom = HTMLParser.parse(data);
+    const categories = dom.querySelectorAll('[data-test="category-tree-navigation-button"]');
+    for (const category of categories) {
+        const link = "https://www.penny.at" + category.getAttribute("href");
+        if (!category.querySelector(".subtitle-2")) continue;
+        const name = (parent ? parent + " -> " : "") + category.querySelector(".subtitle-2").innerText.trim().replace("&amp;", "&");
+        if (name.startsWith("Alle Angebote")) continue;

-exports.mapCategory = (rawItem) => {};
+        result.push({
+            id: name,
+            url: link,
+            code: null,
+        });
+
+        await parseCategory(link, name, result);
+    }
+}
+
+exports.initializeCategoryMapping = async () => {
+    const categories = [];
+    await parseCategory("https://www.penny.at/kategorie", null, categories);
+    utils.mergeAndSaveCategories("penny", categories);
+
+    exports.categoryLookup = {};
+    for (const category of categories) {
+        exports.categoryLookup[category.id] = category;
+    }
+};
+
+exports.mapCategory = (rawItem) => {
+    const categoryPath = rawItem.parentCategories.filter((path) => path.length > 0 && !path[0].name.includes("ngebot"))[0];
+    const categoryName = categoryPath.map((path) => path.name).join(" -> ");
+    const category = exports.categoryLookup[categoryName];
+    if (category) return category.code;
+    return null;
+};

 exports.urlBase = "https://www.penny.at/produkte/";