const fs = require("fs"); const analysis = require("./analysis"); const knn = require("./site/js/knn"); const { itemsToCSV } = require("./site/js/misc"); function similaritySort(items, filter, filterA, filterB) { const filteredItems = items.filter(filter); knn.vectorizeItems(filteredItems); const itemsA = filteredItems.filter(filterA); const itemsB = filteredItems.filter(filterB); const sortedItems = []; itemsB.forEach((item) => (item.sorted = false)); const total = itemsA.length; while (itemsA.length > 0) { const refItem = itemsA.shift(); const similar = knn.findMostSimilarItem(refItem, itemsB); if (similar.item != null) { sortedItems.push(refItem); sortedItems.push(similar.item); } else { console.log("No similar item found for " + refItem.name); } if (sortedItems.length % 100 == 0) console.log(sortedItems.length / 2 + "/" + total); } return sortedItems; } function filterSimilarItems(items) { const filteredItems = []; for (let i = 0; i < items.length; i += 2) { const a = items[i]; const b = items[i + 1]; if (b.priceHistory.some((price) => price.price == a.price) && a.quantity == b.quantity) { filteredItems.push(a); filteredItems.push(b); } } return filteredItems; } if (!fs.existsSync("patterns")) fs.mkdirSync("patterns"); if (!fs.existsSync("patterns/sorted-billa-spar.json")) { console.log("Sorting 3rd party brands Billa/Spar"); const items = analysis.readJSON("data/latest-canonical.json.br"); const sortedItems = similaritySort( items, (item) => { if (!(item.store == "billa" || item.store == "spar")) return false; return !["Clever", "S-BUDGET", "Ja! Natürlich", "SPAR", "BILLA"].some((str) => item.name.includes(str)); }, (item) => item.store === "billa", (item) => item.store === "spar" ); analysis.writeJSON("patterns/sorted-billa-spar.json", sortedItems); } if (!fs.existsSync("site/data/billa-spar-cart.json")) { console.log("Creating cart 3rd party brands Billa/Spar"); const sortedItems = analysis.readJSON("patterns/sorted-billa-spar.json"); const filteredItems = filterSimilarItems(sortedItems); analysis.writeJSON("site/data/billa-spar-cart.json", { name: "Markenprodukte Billa/Spar", items: filteredItems.map((item) => { return { store: item.store, id: item.id }; }), }); } if (!fs.existsSync("patterns/sorted-budget.json")) { console.log("Sorting budget brands Billa/Spar"); const items = analysis.readJSON("data/latest-canonical.json.br"); const sortedItems = similaritySort( items, (item) => { if (!(item.store == "billa" || item.store == "spar")) return false; return ["Clever", "S-BUDGET"].some((str) => item.name.includes(str)); }, (item) => item.store === "billa", (item) => item.store === "spar" ); analysis.writeJSON("patterns/sorted-budget.json", sortedItems); } if (!fs.existsSync("site/data/budget-cart.json")) { console.log("Creating cart budget brands Billa/Spar"); const sortedItems = analysis.readJSON("patterns/sorted-budget.json"); const filteredItems = filterSimilarItems(sortedItems); analysis.writeJSON("site/data/budget-cart.json", { name: "Diskont-Marken Produkte Billa/Spar", items: filteredItems.map((item) => { return { store: item.store, id: item.id }; }), }); } if (!fs.existsSync("patterns/sorted-bio.json")) { console.log("Sorting bio brands Billa/Spar"); const items = analysis.readJSON("data/latest-canonical.json.br"); const sortedItems = similaritySort( items, (item) => { if (!(item.store == "billa" || item.store == "spar")) return false; return ["Ja! Natürlich", "SPAR Natur*pur"].some((str) => item.name.includes(str)); }, (item) => item.store === "billa", (item) => item.store === "spar" ); analysis.writeJSON("patterns/sorted-bio.json", sortedItems); } if (!fs.existsSync("site/data/bio-cart.json")) { console.log("Sorting bio brands Billa/Spar"); const sortedItems = analysis.readJSON("patterns/sorted-bio.json"); const filteredItems = filterSimilarItems(sortedItems); analysis.writeJSON("site/data/bio-cart.json", { name: "Bio Eigenmarken Produkte Billa/Spar", items: filteredItems.map((item) => { return { store: item.store, id: item.id }; }), }); } if (!fs.existsSync("patterns/sorted-midrange.json")) { console.log("Sorting mid-range brands Billa/Spar"); const items = analysis.readJSON("data/latest-canonical.json.br"); const sortedItems = similaritySort( items, (item) => { if (!(item.store == "billa" || item.store == "spar")) return false; return ( ["BILLA", "SPAR"].some((str) => item.name.includes(str)) && !["Ja! Natürlich", "SPAR Natur*pur"].some((str) => item.name.includes(str)) ); }, (item) => item.store === "billa", (item) => item.store === "spar" ); analysis.writeJSON("patterns/sorted-midrange.json", sortedItems); } if (!fs.existsSync("site/data/midrange-cart.json")) { console.log("Creating cart mid-range brands Billa/Spar"); const sortedItems = analysis.readJSON("patterns/sorted-midrange.json"); const filteredItems = filterSimilarItems(sortedItems); analysis.writeJSON("site/data/midrange-cart.json", { name: "Mittelpreisige Eigenmarken Produkte Billa/Spar", items: filteredItems.map((item) => { return { store: item.store, id: item.id }; }), }); } const canonicalItems = analysis.readJSON("data/latest-canonical.json.br"); const lookup = {}; canonicalItems.forEach((item) => (lookup[item.store + item.id] = item)); const files = fs.readdirSync("site/data/"); for (const file of files) { if (file.endsWith("-cart.json")) { const cart = analysis.readJSON("site/data/" + file); const items = cart.items; if (items.length % 2 != 0) { console.log("Uneven number of items in cart " + file); // throw Error(); } for (let i = 1; i < items.length; i++) { if (items[i].store == items[i - 1].store) { console.log("--- " + items[i - 1].store + " " + items[i - 1].id + " " + items[i].store + " " + items[i].id); } } let samePrice = 0; let samePriceItems = []; let otherItems = []; for (let i = 0; i < items.length; i += 2) { let a = items[i].priceHistory ? items[i] : lookup[items[i].store + items[i].id]; let b = items[i + 1].priceHistory ? items[i + 1] : lookup[items[i + 1].store + items[i + 1].id]; if (!a || !b) { console.log("Couldn't find item for product pair"); throw Error(); } if (a.store == b.store) { console.log("Subsequent items from same store. " + a.store + " " + a.id + " " + b.store + " " + b.id); // throw Error(); } if (a.price == b.price) { samePrice++; samePriceItems.push(a); samePriceItems.push(b); } else { otherItems.push(a); otherItems.push(b); } } cart.items = []; cart.items.push(...samePriceItems); cart.items.push(...otherItems); // analysis.writeJSON("site/data/" + file, cart); console.log(`${file}: ${samePrice}/${items.length / 2} product pairs have the same price.`); const csv = itemsToCSV(cart.items); fs.writeFileSync("site/data/" + file.replace(".json", ".csv"), csv, "utf-8"); } }