mirror of
https://github.com/badlogic/heissepreise.git
synced 2024-06-13 20:24:16 +02:00
Compare commits
4 Commits
6ecaafa12b
...
20c96f1e25
Author | SHA1 | Date | |
---|---|---|---|
|
20c96f1e25 | ||
|
f2d9939adf | ||
|
479395d897 | ||
|
80dfea61d5 |
18
analysis.js
18
analysis.js
|
@ -244,9 +244,10 @@ exports.replay = async (rawDataDir) => {
|
||||||
canonicalFiles[store] = storeFiles[store].map((file) => {
|
canonicalFiles[store] = storeFiles[store].map((file) => {
|
||||||
console.log(`Creating canonical items for ${file}`);
|
console.log(`Creating canonical items for ${file}`);
|
||||||
const rawItems = readJSON(file);
|
const rawItems = readJSON(file);
|
||||||
const items = exports.dedupItems(getCanonicalFor(store, rawItems, file.match(/\d{4}-\d{2}-\d{2}/)[0]));
|
let sourceIdx = [];
|
||||||
|
const items = exports.dedupItems(getCanonicalFor(store, rawItems, file.match(/\d{4}-\d{2}-\d{2}/)[0]), sourceIdx);
|
||||||
for (let i = 0; i < items.length; i++) {
|
for (let i = 0; i < items.length; i++) {
|
||||||
const rawItem = rawItems[i];
|
const rawItem = rawItems[sourceIdx[i]];
|
||||||
const item = items[i];
|
const item = items[i];
|
||||||
item.category = stores[store].mapCategory(rawItem);
|
item.category = stores[store].mapCategory(rawItem);
|
||||||
}
|
}
|
||||||
|
@ -297,14 +298,15 @@ exports.updateData = async function (dataDir, done) {
|
||||||
rawItems = await stores[store].fetchData();
|
rawItems = await stores[store].fetchData();
|
||||||
writeJSON(rawDataFile, rawItems, FILE_COMPRESSOR);
|
writeJSON(rawDataFile, rawItems, FILE_COMPRESSOR);
|
||||||
}
|
}
|
||||||
const items = exports.dedupItems(getCanonicalFor(store, rawItems, today));
|
let sourceIdx = [];
|
||||||
|
const items = exports.dedupItems(getCanonicalFor(store, rawItems, today), sourceIdx);
|
||||||
|
|
||||||
await stores[store].initializeCategoryMapping(rawItems);
|
await stores[store].initializeCategoryMapping(rawItems);
|
||||||
let numUncategorized = 0;
|
let numUncategorized = 0;
|
||||||
for (let i = 0; i < items.length; i++) {
|
for (let i = 0; i < items.length; i++) {
|
||||||
const rawItem = rawItems[i];
|
const rawItem = rawItems[sourceIdx[i]];
|
||||||
const item = items[i];
|
const item = items[i];
|
||||||
item.category = stores[store].mapCategory(rawItem);
|
item.category = stores[store].mapCategory(rawItem, item); // both could be unrelated ?!
|
||||||
if (item.category == null) numUncategorized++;
|
if (item.category == null) numUncategorized++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -367,14 +369,16 @@ exports.migrateCompression = (dataDir, fromSuffix, toSuffix, remove = true) => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
exports.dedupItems = (items) => {
|
exports.dedupItems = (items, sourceIdx = []) => {
|
||||||
const lookup = {};
|
const lookup = {};
|
||||||
const dedupItems = [];
|
const dedupItems = [];
|
||||||
let duplicates = {};
|
let duplicates = {};
|
||||||
for (const item of items) {
|
for (let i = 0; i < items.length; i++) {
|
||||||
|
const item = items[i];
|
||||||
const seenItem = lookup[item.store + item.id];
|
const seenItem = lookup[item.store + item.id];
|
||||||
if (!seenItem) {
|
if (!seenItem) {
|
||||||
lookup[item.store + item.id] = item;
|
lookup[item.store + item.id] = item;
|
||||||
|
sourceIdx.push(i);
|
||||||
dedupItems.push(item);
|
dedupItems.push(item);
|
||||||
} else {
|
} else {
|
||||||
if (seenItem.quantity != item.quantity || seenItem.unit != item.unit) {
|
if (seenItem.quantity != item.quantity || seenItem.unit != item.unit) {
|
||||||
|
|
|
@ -45,7 +45,7 @@ exports.getCanonical = function (item, today) {
|
||||||
};
|
};
|
||||||
|
|
||||||
exports.fetchData = async function () {
|
exports.fetchData = async function () {
|
||||||
const MPREIS_URL = `https://ax2ixv4hll-dsn.algolia.net/1/indexes/prod_mpreis_8450/browse?X-Algolia-API-Key=NmJlMTI0NjY1NGU4MDUwYTRlMmYzYWFjOWFlY2U4MGFkNGZjMDY2NmNjNjQzNWY3OWJlNDY4OTY0ZjEwOTEwYWZpbHRlcnM9cHVibGlzaGVk&X-Algolia-Application-Id=AX2IXV4HLL&X-Algolia-Agent=Vue.js`;
|
const MPREIS_URL = `https://uzxors8tl2-dsn.algolia.net/1/indexes/prod_mpreis_8450/browse?X-Algolia-API-Key=6d27574257fd3a92542ff880585333f1&X-Algolia-Application-Id=UZXORS8TL2&X-Algolia-Agent=Vue.js`;
|
||||||
let mpreisItems = [];
|
let mpreisItems = [];
|
||||||
let res = (await axios.get(MPREIS_URL)).data;
|
let res = (await axios.get(MPREIS_URL)).data;
|
||||||
mpreisItems = mpreisItems.concat(res.hits);
|
mpreisItems = mpreisItems.concat(res.hits);
|
||||||
|
|
Loading…
Reference in New Issue
Block a user