106 lines
3.5 KiB
JavaScript
106 lines
3.5 KiB
JavaScript
const fs = require("fs");
|
|
const analysis = require("./analysis");
|
|
const Database = require("better-sqlite3");
|
|
|
|
exports.importH43zData = (sqliteFile, outputFile) => {
|
|
const db = new Database(sqliteFile, { verbose: console.log });
|
|
let items = [];
|
|
const lookup = {};
|
|
|
|
let stmt = db.prepare("select * from product");
|
|
for (const row of stmt.iterate()) {
|
|
const item = {
|
|
store: row.shop == "billa" ? "billa" : "spar",
|
|
id: row.product_id,
|
|
name: row.name,
|
|
price: 0,
|
|
priceHistory: [],
|
|
isWeighted: false,
|
|
unit: "g",
|
|
quantity: 0,
|
|
bio: false,
|
|
url: "",
|
|
};
|
|
items.push(item);
|
|
lookup[row.product_id] = item;
|
|
}
|
|
console.log(items.length + " products");
|
|
|
|
stmt = db.prepare("select * from pricehistory order by date asc");
|
|
let i = 0;
|
|
let unknown = 0;
|
|
for (const row of stmt.iterate()) {
|
|
i++;
|
|
const item = lookup[row.product_id];
|
|
if (!item) {
|
|
unknown++;
|
|
} else {
|
|
if (item.priceHistory.length == 0 || item.priceHistory[item.priceHistory.length - 1].price != row.price)
|
|
item.priceHistory.push({ date: row.date, price: row.price });
|
|
}
|
|
if (i % 1000 == 0) {
|
|
console.log("Processed " + i + " prices, " + unknown + " unknown");
|
|
}
|
|
}
|
|
|
|
items = items.filter((item) => item.priceHistory.length > 0);
|
|
items.forEach((item) => {
|
|
item.priceHistory.reverse();
|
|
item.price = item.priceHistory[0];
|
|
});
|
|
analysis.writeJSON(outputFile, items);
|
|
};
|
|
|
|
exports.mergeWithLatestCanonical = (h43zFile, latestCanonicalFile) => {
|
|
const items = analysis.readJSON(h43zFile);
|
|
const lookup = {};
|
|
items.forEach((item) => {
|
|
// item.priceHistory = item.priceHistory.filter(price => price.date > "2020-01-01")
|
|
lookup[item.id] = item;
|
|
});
|
|
const currItems = analysis.readJSON(latestCanonicalFile + "." + analysis.FILE_COMPRESSOR);
|
|
const currLookup = {};
|
|
currItems.forEach((item) => (currLookup[item.store + item.id] = item));
|
|
let missingItems = {
|
|
spar: 0,
|
|
billa: 0,
|
|
};
|
|
let foundItems = {
|
|
spar: 0,
|
|
billa: 0,
|
|
};
|
|
for (item of items) {
|
|
const i = lookup[item.id];
|
|
const currItem = currLookup[item.store + item.id];
|
|
if (!currItem) {
|
|
missingItems[item.store]++;
|
|
} else {
|
|
foundItems[item.store]++;
|
|
const oldHistory = [...currItem.priceHistory];
|
|
currItem.priceHistory.push(...item.priceHistory);
|
|
currItem.priceHistory.sort((a, b) => new Date(a.date) - new Date(b.date));
|
|
|
|
const mergedHistory = [];
|
|
currItem.priceHistory.forEach((price) => {
|
|
if (mergedHistory.length == 0) {
|
|
mergedHistory.push(price);
|
|
return;
|
|
}
|
|
if (mergedHistory[mergedHistory.length - 1].price != price.price) {
|
|
mergedHistory.push(price);
|
|
}
|
|
});
|
|
mergedHistory.reverse();
|
|
currItem.priceHistory = mergedHistory;
|
|
}
|
|
}
|
|
console.log(JSON.stringify(missingItems, null, 2));
|
|
console.log(JSON.stringify(foundItems, null, 2));
|
|
analysis.writeJSON(latestCanonicalFile, currItems, analysis.FILE_COMPRESSOR);
|
|
};
|
|
|
|
if (require.main === module) {
|
|
exports.importH43zData("/Users/badlogic/Downloads/shops.db", "h43z.json");
|
|
exports.mergeWithLatestCanonical("h43z.json", "data/latest-canonical.json");
|
|
}
|