mirror of
https://github.com/badlogic/heissepreise.git
synced 2024-09-22 00:00:59 +02:00
More charting changes. restore.js can now take a h43z.json file and merge it with the restored history. Ask Mario for the data.
This commit is contained in:
parent
182d08f528
commit
9646f07db5
|
@ -210,7 +210,10 @@ exports.replay = function (rawDataDir) {
|
||||||
|
|
||||||
for (const store of STORE_KEYS) {
|
for (const store of STORE_KEYS) {
|
||||||
storeFiles[store] = getFilteredFilesFor(store);
|
storeFiles[store] = getFilteredFilesFor(store);
|
||||||
canonicalFiles[store] = storeFiles[store].map((file) => getCanonicalFor(store, readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]));
|
canonicalFiles[store] = storeFiles[store].map((file) => {
|
||||||
|
console.log(`Creating canonical items for ${file}`);
|
||||||
|
return getCanonicalFor(store, readJSON(file), file.match(/\d{4}-\d{2}-\d{2}/)[0]);
|
||||||
|
});
|
||||||
canonicalFiles[store].reverse();
|
canonicalFiles[store].reverse();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -232,6 +235,7 @@ exports.replay = function (rawDataDir) {
|
||||||
let curr = null;
|
let curr = null;
|
||||||
for (let i = 1; i < allFilesCanonical.length; i++) {
|
for (let i = 1; i < allFilesCanonical.length; i++) {
|
||||||
curr = allFilesCanonical[i];
|
curr = allFilesCanonical[i];
|
||||||
|
console.log(`Merging ${i}/${allFilesCanonical.length} canonical files.`);
|
||||||
mergePriceHistory(prev, curr);
|
mergePriceHistory(prev, curr);
|
||||||
prev = curr;
|
prev = curr;
|
||||||
}
|
}
|
||||||
|
|
103
h43z.js
103
h43z.js
|
@ -1,11 +1,12 @@
|
||||||
const fs = require("fs");
|
const fs = require("fs");
|
||||||
const analysis = require("./analysis");
|
const analysis = require("./analysis");
|
||||||
const Database = require("better-sqlite3");
|
const Database = require("better-sqlite3");
|
||||||
const db = new Database("/Users/badlogic/Downloads/shops.db", { verbose: console.log });
|
|
||||||
let items = [];
|
|
||||||
const lookup = {};
|
|
||||||
|
|
||||||
if (!fs.existsSync("h43z.json")) {
|
exports.importH43zData = (sqliteFile, outputFile) => {
|
||||||
|
const db = new Database(sqliteFile, { verbose: console.log });
|
||||||
|
let items = [];
|
||||||
|
const lookup = {};
|
||||||
|
|
||||||
let stmt = db.prepare("select * from product");
|
let stmt = db.prepare("select * from product");
|
||||||
for (const row of stmt.iterate()) {
|
for (const row of stmt.iterate()) {
|
||||||
const item = {
|
const item = {
|
||||||
|
@ -48,50 +49,58 @@ if (!fs.existsSync("h43z.json")) {
|
||||||
item.priceHistory.reverse();
|
item.priceHistory.reverse();
|
||||||
item.price = item.priceHistory[0];
|
item.price = item.priceHistory[0];
|
||||||
});
|
});
|
||||||
analysis.writeJSON("h43z.json", items);
|
analysis.writeJSON(outputFile, items);
|
||||||
}
|
|
||||||
|
|
||||||
items = analysis.readJSON("h43z.json");
|
|
||||||
items.forEach((item) => {
|
|
||||||
// item.priceHistory = item.priceHistory.filter(price => price.date > "2020-01-01")
|
|
||||||
});
|
|
||||||
const currItems = analysis.readJSON("data/latest-canonical.json.br");
|
|
||||||
const currLookup = {};
|
|
||||||
currItems.forEach((item) => (currLookup[item.store + (item.sparId ? item.sparId : item.id)] = item));
|
|
||||||
let missingItems = {
|
|
||||||
spar: 0,
|
|
||||||
billa: 0,
|
|
||||||
};
|
};
|
||||||
let foundItems = {
|
|
||||||
spar: 0,
|
|
||||||
billa: 0,
|
|
||||||
};
|
|
||||||
for (item of items) {
|
|
||||||
const i = lookup[item.id];
|
|
||||||
const currItem = currLookup[item.store + item.id];
|
|
||||||
if (!currItem) {
|
|
||||||
missingItems[item.store]++;
|
|
||||||
} else {
|
|
||||||
foundItems[item.store]++;
|
|
||||||
const oldHistory = [...currItem.priceHistory];
|
|
||||||
currItem.priceHistory.push(...item.priceHistory);
|
|
||||||
currItem.priceHistory.sort((a, b) => new Date(a.date) - new Date(b.date));
|
|
||||||
|
|
||||||
const mergedHistory = [];
|
exports.mergeWithLatestCanonical = (h43zFile, latestCanonicalFile) => {
|
||||||
currItem.priceHistory.forEach((price) => {
|
const items = analysis.readJSON(h43zFile);
|
||||||
if (mergedHistory.length == 0) {
|
const lookup = {};
|
||||||
mergedHistory.push(price);
|
items.forEach((item) => {
|
||||||
return;
|
// item.priceHistory = item.priceHistory.filter(price => price.date > "2020-01-01")
|
||||||
}
|
lookup[item.id] = item;
|
||||||
if (mergedHistory[mergedHistory.length - 1].price != price.price) {
|
});
|
||||||
mergedHistory.push(price);
|
const currItems = analysis.readJSON(latestCanonicalFile + "." + analysis.FILE_COMPRESSOR);
|
||||||
}
|
const currLookup = {};
|
||||||
});
|
currItems.forEach((item) => (currLookup[item.store + (item.sparId ? item.sparId : item.id)] = item));
|
||||||
mergedHistory.reverse();
|
let missingItems = {
|
||||||
currItem.priceHistory = mergedHistory;
|
spar: 0,
|
||||||
|
billa: 0,
|
||||||
|
};
|
||||||
|
let foundItems = {
|
||||||
|
spar: 0,
|
||||||
|
billa: 0,
|
||||||
|
};
|
||||||
|
for (item of items) {
|
||||||
|
const i = lookup[item.id];
|
||||||
|
const currItem = currLookup[item.store + item.id];
|
||||||
|
if (!currItem) {
|
||||||
|
missingItems[item.store]++;
|
||||||
|
} else {
|
||||||
|
foundItems[item.store]++;
|
||||||
|
const oldHistory = [...currItem.priceHistory];
|
||||||
|
currItem.priceHistory.push(...item.priceHistory);
|
||||||
|
currItem.priceHistory.sort((a, b) => new Date(a.date) - new Date(b.date));
|
||||||
|
|
||||||
|
const mergedHistory = [];
|
||||||
|
currItem.priceHistory.forEach((price) => {
|
||||||
|
if (mergedHistory.length == 0) {
|
||||||
|
mergedHistory.push(price);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (mergedHistory[mergedHistory.length - 1].price != price.price) {
|
||||||
|
mergedHistory.push(price);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
mergedHistory.reverse();
|
||||||
|
currItem.priceHistory = mergedHistory;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
console.log(JSON.stringify(missingItems, null, 2));
|
||||||
|
console.log(JSON.stringify(foundItems, null, 2));
|
||||||
|
analysis.writeJSON(latestCanonicalFile, currItems, analysis.FILE_COMPRESSOR);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
exports.importH43zData("/Users/badlogic/Downloads/shops.db", "h43z.json");
|
||||||
|
exports.mergeWithLatestCanonical("h43z.json", "data/latest-canonical.json");
|
||||||
}
|
}
|
||||||
console.log(JSON.stringify(missingItems, null, 2));
|
|
||||||
console.log(JSON.stringify(foundItems, null, 2));
|
|
||||||
analysis.writeJSON("h43z.compressed.json", items, false, 0, true);
|
|
||||||
analysis.writeJSON("data/latest-canonical.h43z.json", currItems, analysis.FILE_COMPRESSOR);
|
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
|
const fs = require("fs");
|
||||||
const analysis = require("./analysis.js");
|
const analysis = require("./analysis.js");
|
||||||
|
const h43z = require("./h43z");
|
||||||
const dataDir = process?.argv?.[2] ?? "data";
|
const dataDir = process?.argv?.[2] ?? "data";
|
||||||
|
const h43zFile = process?.argv?.[3] ?? null;
|
||||||
console.log("Restoring data from raw data.");
|
console.log("Restoring data from raw data.");
|
||||||
(async function () {
|
(async function () {
|
||||||
analysis.migrateCompression(dataDir, ".json", ".json.br", false);
|
analysis.migrateCompression(dataDir, ".json", ".json.br", false);
|
||||||
|
@ -11,4 +14,7 @@ console.log("Restoring data from raw data.");
|
||||||
analysis.readJSON(`${dataDir}/latest-canonical.json.${analysis.FILE_COMPRESSOR}`).length
|
analysis.readJSON(`${dataDir}/latest-canonical.json.${analysis.FILE_COMPRESSOR}`).length
|
||||||
} items to ${dataDir}/latest-canonical.json.${analysis.FILE_COMPRESSOR}`
|
} items to ${dataDir}/latest-canonical.json.${analysis.FILE_COMPRESSOR}`
|
||||||
);
|
);
|
||||||
|
if (h43zFile && fs.existsSync(h43zFile)) {
|
||||||
|
h43z.mergeWithLatestCanonical(h43zFile, `${dataDir}/latest-canonical.json`);
|
||||||
|
}
|
||||||
})();
|
})();
|
||||||
|
|
|
@ -111,6 +111,10 @@ exports.queryItems = (query, items, exactWord) => {
|
||||||
else return priceHistory.some((price) => price.date >= date && price.date <= endDate);
|
else return priceHistory.some((price) => price.date >= date && price.date <= endDate);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
alasql.fn.hasPriceChangeLike = (priceHistory, date) => {
|
||||||
|
return priceHistory.some((price) => price.date.indexOf(date) >= 0);
|
||||||
|
};
|
||||||
|
|
||||||
if (query.charAt(0) == "!") {
|
if (query.charAt(0) == "!") {
|
||||||
query = query.substring(1);
|
query = query.substring(1);
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -20,7 +20,7 @@ class ItemsChart extends View {
|
||||||
<custom-checkbox x-id="onlyToday" x-change x-state label="Nur heutige Preise"></custom-checkbox>
|
<custom-checkbox x-id="onlyToday" x-change x-state label="Nur heutige Preise"></custom-checkbox>
|
||||||
<div
|
<div
|
||||||
class="cursor-pointer inline-flex items-center gap-x-1 rounded-full bg-white border border-gray-400 px-2 py-1 text-xs font-medium text-gray-600">
|
class="cursor-pointer inline-flex items-center gap-x-1 rounded-full bg-white border border-gray-400 px-2 py-1 text-xs font-medium text-gray-600">
|
||||||
<input x-id="startDate" x-change x-state type="date" value="2020-01-01" />
|
<input x-id="startDate" x-change x-state type="date" value="2017-01-01" />
|
||||||
-
|
-
|
||||||
<input x-id="endDate" x-change x-state type="date" value="${today()}"/>
|
<input x-id="endDate" x-change x-state type="date" value="${today()}"/>
|
||||||
</div>
|
</div>
|
||||||
|
@ -42,38 +42,45 @@ class ItemsChart extends View {
|
||||||
return [{ date: today(), price: sum }];
|
return [{ date: today(), price: sum }];
|
||||||
}
|
}
|
||||||
|
|
||||||
const allDates = items.flatMap((product) => product.priceHistory.map((item) => item.date));
|
const allDates = items.flatMap((product) =>
|
||||||
|
product.priceHistory.filter((price) => price.date >= startDate && price.date <= endDate).map((item) => item.date)
|
||||||
|
);
|
||||||
let uniqueDates = [...new Set(allDates)];
|
let uniqueDates = [...new Set(allDates)];
|
||||||
uniqueDates.sort();
|
uniqueDates.sort();
|
||||||
|
|
||||||
const allPrices = items.map((product) => {
|
let priceChanges = new Array(uniqueDates.length);
|
||||||
|
for (let i = 0; i < uniqueDates.length; i++) {
|
||||||
|
priceChanges[i] = { date: uniqueDates[i], price: 0 };
|
||||||
|
}
|
||||||
|
const priceScratch = new Array(uniqueDates.length);
|
||||||
|
items.forEach((product) => {
|
||||||
let price = null;
|
let price = null;
|
||||||
const prices = uniqueDates.map((date) => {
|
priceScratch.fill(null);
|
||||||
const priceObj = product.priceHistory.find((item) => item.date === date);
|
if (!product.priceHistoryLookup) {
|
||||||
|
product.priceHistoryLookup = {};
|
||||||
|
product.priceHistory.forEach((price) => (product.priceHistoryLookup[price.date] = price));
|
||||||
|
}
|
||||||
|
for (let i = 0; i < uniqueDates.length; i++) {
|
||||||
|
const priceObj = product.priceHistoryLookup[uniqueDates[i]];
|
||||||
if (!price && priceObj) price = priceObj.price;
|
if (!price && priceObj) price = priceObj.price;
|
||||||
return priceObj ? priceObj.price : null;
|
priceScratch[i] = priceObj ? priceObj.price : null;
|
||||||
});
|
}
|
||||||
|
|
||||||
for (let i = 0; i < prices.length; i++) {
|
for (let i = 0; i < priceScratch.length; i++) {
|
||||||
if (!prices[i]) {
|
if (!priceScratch[i]) {
|
||||||
prices[i] = price;
|
priceScratch[i] = price;
|
||||||
} else {
|
} else {
|
||||||
price = prices[i];
|
price = priceScratch[i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return prices;
|
|
||||||
|
for (let i = 0; i < priceScratch.length; i++) {
|
||||||
|
const price = priceScratch[i];
|
||||||
|
priceChanges[i].price += price;
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
const priceChanges = [];
|
priceChanges = priceChanges.filter((price) => price.date >= startDate && price.date <= endDate);
|
||||||
for (let i = 0; i < uniqueDates.length; i++) {
|
|
||||||
if (uniqueDates[i] < startDate || uniqueDates[i] > endDate) continue;
|
|
||||||
let price = 0;
|
|
||||||
for (let j = 0; j < allPrices.length; j++) {
|
|
||||||
price += allPrices[j][i];
|
|
||||||
}
|
|
||||||
priceChanges.push({ date: uniqueDates[i], price });
|
|
||||||
}
|
|
||||||
|
|
||||||
return priceChanges;
|
return priceChanges;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -94,6 +101,7 @@ class ItemsChart extends View {
|
||||||
const uniqueDates = [...new Set(allDates)];
|
const uniqueDates = [...new Set(allDates)];
|
||||||
uniqueDates.sort();
|
uniqueDates.sort();
|
||||||
|
|
||||||
|
const now = performance.now();
|
||||||
const datasets = items.map((item) => {
|
const datasets = items.map((item) => {
|
||||||
let price = null;
|
let price = null;
|
||||||
const prices = uniqueDates.map((date) => {
|
const prices = uniqueDates.map((date) => {
|
||||||
|
@ -142,11 +150,9 @@ class ItemsChart extends View {
|
||||||
// stepped: "before"
|
// stepped: "before"
|
||||||
};
|
};
|
||||||
|
|
||||||
const data = dataset.data;
|
|
||||||
for (let i = 0; i < data.length; i++) {}
|
|
||||||
|
|
||||||
return dataset;
|
return dataset;
|
||||||
});
|
});
|
||||||
|
log("ItemsChart - Calculating datasets took " + ((performance.now() - now) / 1000).toFixed(2) + " secs");
|
||||||
|
|
||||||
const ctx = canvasDom.getContext("2d");
|
const ctx = canvasDom.getContext("2d");
|
||||||
let scrollTop = -1;
|
let scrollTop = -1;
|
||||||
|
@ -202,13 +208,16 @@ class ItemsChart extends View {
|
||||||
const itemsToShow = [];
|
const itemsToShow = [];
|
||||||
|
|
||||||
if (elements.sumTotal.checked && items.length > 0) {
|
if (elements.sumTotal.checked && items.length > 0) {
|
||||||
|
const now = performance.now();
|
||||||
itemsToShow.push({
|
itemsToShow.push({
|
||||||
name: "Preissumme Gesamt",
|
name: "Preissumme Gesamt",
|
||||||
priceHistory: this.calculateOverallPriceChanges(items, onlyToday, startDate, endDate),
|
priceHistory: this.calculateOverallPriceChanges(items, onlyToday, startDate, endDate),
|
||||||
});
|
});
|
||||||
|
log("ItemsChart - Calculating overall sum total " + ((performance.now() - now) / 1000).toFixed(2) + " secs");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (elements.sumStores.checked && items.length > 0) {
|
if (elements.sumStores.checked && items.length > 0) {
|
||||||
|
const now = performance.now();
|
||||||
STORE_KEYS.forEach((store) => {
|
STORE_KEYS.forEach((store) => {
|
||||||
const storeItems = items.filter((item) => item.store === store);
|
const storeItems = items.filter((item) => item.store === store);
|
||||||
if (storeItems.length > 0) {
|
if (storeItems.length > 0) {
|
||||||
|
@ -218,6 +227,7 @@ class ItemsChart extends View {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
log("ItemsChart - Calculating overall sum per store took " + ((performance.now() - now) / 1000).toFixed(2) + " secs");
|
||||||
}
|
}
|
||||||
|
|
||||||
items.forEach((item) => {
|
items.forEach((item) => {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user