Remove old entries for discount only stores. Closes #102

.prettierignore

@@ -7,4 +7,5 @@ latest-canonical*.json
 momentum-cart.json
 package-lock.json
 site/_templates
-docs/
+docs/
+stores/*.json

analysis.js

@@ -2,6 +2,7 @@ const fs = require("fs");
 const fsAsync = require("fs").promises;
 const zlib = require("zlib");
 const stores = require("./stores");
+const model = require("./site/model/stores");
 const { promisify } = require("util");
 
 const STORE_KEYS = Object.keys(stores);
@@ -92,9 +93,17 @@ function mergePriceHistory(oldItems, items) {
     }
 
     console.log(`${Object.keys(lookup).length} not in latest list.`);
+    let removed = {};
     for (key of Object.keys(lookup)) {
-        items.push(lookup[key]);
+        const item = lookup[key];
+        if (model.stores[item.store]?.removeOld) {
+            removed[item.store] = removed[item.store] ? removed[item.store] + 1 : 1;
+        } else {
+            items.push(item);
+        }
     }
+    console.log("Removed items for discount-only stores");
+    console.log(JSON.stringify(removed, null, 2));
 
     sortItems(items);
     console.log(`Items: ${items.length}`);

site/model/stores.js

@@ -22,14 +22,14 @@ exports.stores = {
         defaultChecked: true,
         getUrl: (item) => `https://www.roksh.at/hofer/produkte/${item.url}`,
     },
-    // Disabled as they only give us discounted products.
-    /*lidl: {
+    lidl: {
         name: "Lidl",
         budgetBrands: ["milbona"],
         color: "pink",
         defaultChecked: true,
         getUrl: (item) => `https://www.lidl.at${item.url}`,
-    },*/
+        removeOld: true,
+    },
     mpreis: {
         name: "MPREIS",
         budgetBrands: [],
@@ -57,6 +57,7 @@ exports.stores = {
         color: "purple",
         defaultChecked: true,
         getUrl: (item) => `https://www.penny.at/produkte/${item.id}`,
+        removeOld: true,
     },
     dmDe: {
         name: "DM DE",

stores/billa.js

@@ -76,7 +76,6 @@ exports.fetchData = async function () {
 
 exports.initializeCategoryMapping = async () => {
     // FIXME check if categories have changed.
-    console.log("No mapping for Billa");
 };
 
 exports.mapCategory = (rawItem) => {