From 9646f07db5786e5328c1cdb73c9697ea072f7091 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 14 Jun 2023 17:07:02 +0200 Subject: [PATCH] More charting changes. restore.js can now take a h43z.json file and merge it with the restored history. Ask Mario for the data. --- analysis.js | 6 ++- h43z.js | 103 +++++++++++++++++++++----------------- restore.js | 6 +++ site/js/misc.js | 4 ++ site/views/items-chart.js | 60 +++++++++++++--------- 5 files changed, 106 insertions(+), 73 deletions(-) diff --git a/analysis.js b/analysis.js index a5de717..53e4040 100644 --- a/analysis.js +++ b/analysis.js @@ -210,7 +210,10 @@ exports.replay = function (rawDataDir) { for (const store of STORE_KEYS) { storeFiles[store] = getFilteredFilesFor(store); - canonicalFiles[store] = storeFiles[store].map((file) => getCanonicalFor(store, readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0])); + canonicalFiles[store] = storeFiles[store].map((file) => { + console.log(`Creating canonical items for ${file}`); + return getCanonicalFor(store, readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]); + }); canonicalFiles[store].reverse(); } @@ -232,6 +235,7 @@ exports.replay = function (rawDataDir) { let curr = null; for (let i = 1; i < allFilesCanonical.length; i++) { curr = allFilesCanonical[i]; + console.log(`Merging ${i}/${allFilesCanonical.length} canonical files.`); mergePriceHistory(prev, curr); prev = curr; } diff --git a/h43z.js b/h43z.js index 98c8639..56bc45e 100644 --- a/h43z.js +++ b/h43z.js @@ -1,11 +1,12 @@ const fs = require("fs"); const analysis = require("./analysis"); const Database = require("better-sqlite3"); -const db = new Database("/Users/badlogic/Downloads/shops.db", { verbose: console.log }); -let items = []; -const lookup = {}; -if (!fs.existsSync("h43z.json")) { +exports.importH43zData = (sqliteFile, outputFile) => { + const db = new Database(sqliteFile, { verbose: console.log }); + let items = []; + const lookup = {}; + let stmt = db.prepare("select * from product"); for (const row of stmt.iterate()) { const item = { @@ -48,50 +49,58 @@ if (!fs.existsSync("h43z.json")) { item.priceHistory.reverse(); item.price = item.priceHistory[0]; }); - analysis.writeJSON("h43z.json", items); -} - -items = analysis.readJSON("h43z.json"); -items.forEach((item) => { - // item.priceHistory = item.priceHistory.filter(price => price.date > "2020-01-01") -}); -const currItems = analysis.readJSON("data/latest-canonical.json.br"); -const currLookup = {}; -currItems.forEach((item) => (currLookup[item.store + (item.sparId ? item.sparId : item.id)] = item)); -let missingItems = { - spar: 0, - billa: 0, + analysis.writeJSON(outputFile, items); }; -let foundItems = { - spar: 0, - billa: 0, -}; -for (item of items) { - const i = lookup[item.id]; - const currItem = currLookup[item.store + item.id]; - if (!currItem) { - missingItems[item.store]++; - } else { - foundItems[item.store]++; - const oldHistory = [...currItem.priceHistory]; - currItem.priceHistory.push(...item.priceHistory); - currItem.priceHistory.sort((a, b) => new Date(a.date) - new Date(b.date)); - const mergedHistory = []; - currItem.priceHistory.forEach((price) => { - if (mergedHistory.length == 0) { - mergedHistory.push(price); - return; - } - if (mergedHistory[mergedHistory.length - 1].price != price.price) { - mergedHistory.push(price); - } - }); - mergedHistory.reverse(); - currItem.priceHistory = mergedHistory; +exports.mergeWithLatestCanonical = (h43zFile, latestCanonicalFile) => { + const items = analysis.readJSON(h43zFile); + const lookup = {}; + items.forEach((item) => { + // item.priceHistory = item.priceHistory.filter(price => price.date > "2020-01-01") + lookup[item.id] = item; + }); + const currItems = analysis.readJSON(latestCanonicalFile + "." + analysis.FILE_COMPRESSOR); + const currLookup = {}; + currItems.forEach((item) => (currLookup[item.store + (item.sparId ? item.sparId : item.id)] = item)); + let missingItems = { + spar: 0, + billa: 0, + }; + let foundItems = { + spar: 0, + billa: 0, + }; + for (item of items) { + const i = lookup[item.id]; + const currItem = currLookup[item.store + item.id]; + if (!currItem) { + missingItems[item.store]++; + } else { + foundItems[item.store]++; + const oldHistory = [...currItem.priceHistory]; + currItem.priceHistory.push(...item.priceHistory); + currItem.priceHistory.sort((a, b) => new Date(a.date) - new Date(b.date)); + + const mergedHistory = []; + currItem.priceHistory.forEach((price) => { + if (mergedHistory.length == 0) { + mergedHistory.push(price); + return; + } + if (mergedHistory[mergedHistory.length - 1].price != price.price) { + mergedHistory.push(price); + } + }); + mergedHistory.reverse(); + currItem.priceHistory = mergedHistory; + } } + console.log(JSON.stringify(missingItems, null, 2)); + console.log(JSON.stringify(foundItems, null, 2)); + analysis.writeJSON(latestCanonicalFile, currItems, analysis.FILE_COMPRESSOR); +}; + +if (require.main === module) { + exports.importH43zData("/Users/badlogic/Downloads/shops.db", "h43z.json"); + exports.mergeWithLatestCanonical("h43z.json", "data/latest-canonical.json"); } -console.log(JSON.stringify(missingItems, null, 2)); -console.log(JSON.stringify(foundItems, null, 2)); -analysis.writeJSON("h43z.compressed.json", items, false, 0, true); -analysis.writeJSON("data/latest-canonical.h43z.json", currItems, analysis.FILE_COMPRESSOR); diff --git a/restore.js b/restore.js index 1841351..5eed18f 100644 --- a/restore.js +++ b/restore.js @@ -1,5 +1,8 @@ +const fs = require("fs"); const analysis = require("./analysis.js"); +const h43z = require("./h43z"); const dataDir = process?.argv?.[2] ?? "data"; +const h43zFile = process?.argv?.[3] ?? null; console.log("Restoring data from raw data."); (async function () { analysis.migrateCompression(dataDir, ".json", ".json.br", false); @@ -11,4 +14,7 @@ console.log("Restoring data from raw data."); analysis.readJSON(`${dataDir}/latest-canonical.json.${analysis.FILE_COMPRESSOR}`).length } items to ${dataDir}/latest-canonical.json.${analysis.FILE_COMPRESSOR}` ); + if (h43zFile && fs.existsSync(h43zFile)) { + h43z.mergeWithLatestCanonical(h43zFile, `${dataDir}/latest-canonical.json`); + } })(); diff --git a/site/js/misc.js b/site/js/misc.js index 137c038..297d101 100644 --- a/site/js/misc.js +++ b/site/js/misc.js @@ -111,6 +111,10 @@ exports.queryItems = (query, items, exactWord) => { else return priceHistory.some((price) => price.date >= date && price.date <= endDate); }; + alasql.fn.hasPriceChangeLike = (priceHistory, date) => { + return priceHistory.some((price) => price.date.indexOf(date) >= 0); + }; + if (query.charAt(0) == "!") { query = query.substring(1); try { diff --git a/site/views/items-chart.js b/site/views/items-chart.js index f0c49a2..0831b35 100644 --- a/site/views/items-chart.js +++ b/site/views/items-chart.js @@ -20,7 +20,7 @@ class ItemsChart extends View {
- + -
@@ -42,38 +42,45 @@ class ItemsChart extends View { return [{ date: today(), price: sum }]; } - const allDates = items.flatMap((product) => product.priceHistory.map((item) => item.date)); + const allDates = items.flatMap((product) => + product.priceHistory.filter((price) => price.date >= startDate && price.date <= endDate).map((item) => item.date) + ); let uniqueDates = [...new Set(allDates)]; uniqueDates.sort(); - const allPrices = items.map((product) => { + let priceChanges = new Array(uniqueDates.length); + for (let i = 0; i < uniqueDates.length; i++) { + priceChanges[i] = { date: uniqueDates[i], price: 0 }; + } + const priceScratch = new Array(uniqueDates.length); + items.forEach((product) => { let price = null; - const prices = uniqueDates.map((date) => { - const priceObj = product.priceHistory.find((item) => item.date === date); + priceScratch.fill(null); + if (!product.priceHistoryLookup) { + product.priceHistoryLookup = {}; + product.priceHistory.forEach((price) => (product.priceHistoryLookup[price.date] = price)); + } + for (let i = 0; i < uniqueDates.length; i++) { + const priceObj = product.priceHistoryLookup[uniqueDates[i]]; if (!price && priceObj) price = priceObj.price; - return priceObj ? priceObj.price : null; - }); + priceScratch[i] = priceObj ? priceObj.price : null; + } - for (let i = 0; i < prices.length; i++) { - if (!prices[i]) { - prices[i] = price; + for (let i = 0; i < priceScratch.length; i++) { + if (!priceScratch[i]) { + priceScratch[i] = price; } else { - price = prices[i]; + price = priceScratch[i]; } } - return prices; + + for (let i = 0; i < priceScratch.length; i++) { + const price = priceScratch[i]; + priceChanges[i].price += price; + } }); - const priceChanges = []; - for (let i = 0; i < uniqueDates.length; i++) { - if (uniqueDates[i] < startDate || uniqueDates[i] > endDate) continue; - let price = 0; - for (let j = 0; j < allPrices.length; j++) { - price += allPrices[j][i]; - } - priceChanges.push({ date: uniqueDates[i], price }); - } - + priceChanges = priceChanges.filter((price) => price.date >= startDate && price.date <= endDate); return priceChanges; } @@ -94,6 +101,7 @@ class ItemsChart extends View { const uniqueDates = [...new Set(allDates)]; uniqueDates.sort(); + const now = performance.now(); const datasets = items.map((item) => { let price = null; const prices = uniqueDates.map((date) => { @@ -142,11 +150,9 @@ class ItemsChart extends View { // stepped: "before" }; - const data = dataset.data; - for (let i = 0; i < data.length; i++) {} - return dataset; }); + log("ItemsChart - Calculating datasets took " + ((performance.now() - now) / 1000).toFixed(2) + " secs"); const ctx = canvasDom.getContext("2d"); let scrollTop = -1; @@ -202,13 +208,16 @@ class ItemsChart extends View { const itemsToShow = []; if (elements.sumTotal.checked && items.length > 0) { + const now = performance.now(); itemsToShow.push({ name: "Preissumme Gesamt", priceHistory: this.calculateOverallPriceChanges(items, onlyToday, startDate, endDate), }); + log("ItemsChart - Calculating overall sum total " + ((performance.now() - now) / 1000).toFixed(2) + " secs"); } if (elements.sumStores.checked && items.length > 0) { + const now = performance.now(); STORE_KEYS.forEach((store) => { const storeItems = items.filter((item) => item.store === store); if (storeItems.length > 0) { @@ -218,6 +227,7 @@ class ItemsChart extends View { }); } }); + log("ItemsChart - Calculating overall sum per store took " + ((performance.now() - now) / 1000).toFixed(2) + " secs"); } items.forEach((item) => {