Marown
/
heissepreise
mirror of https://github.com/badlogic/heissepreise.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: Marown:6ecaafa12b9654bb2f6c8050619bdea21183eb30
Branches Tags
Marown:main
Marown:patch-1
...
compare: Marown:20c96f1e25d0694933c87a58237ddb380d34c5e1
Branches Tags
Marown:patch-1
Marown:main

4 Commits
6ecaafa12b ... 20c96f1e25

Author SHA1 Message Date
Jan Elzein 20c96f1e25
Merge 80dfea61d5 into f2d9939adf 2024-02-26 23:37:59 +01:00
Mario Zechner f2d9939adf
Merge pull request #175 from dakralex/main 
Update MPREIS API URL
 2024-02-26 23:37:27 +01:00
Daniel Kral 479395d897
Update MPREIS API URL to the one that is used by the current web application 2024-02-25 19:31:18 +01:00
Jan Elzein 80dfea61d5 Match category bugfix. 2024-01-08 09:21:06 +01:00
2 changed files with 12 additions and 8 deletions
Show Stats Expand all files Collapse all files

18
analysis.js
View File

@ -244,9 +244,10 @@ exports.replay = async (rawDataDir) => {
canonicalFiles[store] = storeFiles[store].map((file) => {
console.log(`Creating canonical items for ${file}`);
const rawItems = readJSON(file);
const items = exports.dedupItems(getCanonicalFor(store, rawItems, file.match(/\d{4}-\d{2}-\d{2}/)[0]));
let sourceIdx = [];
const items = exports.dedupItems(getCanonicalFor(store, rawItems, file.match(/\d{4}-\d{2}-\d{2}/)[0]), sourceIdx);
for (let i = 0; i < items.length; i++) {
const rawItem = rawItems[i];
const rawItem = rawItems[sourceIdx[i]];
const item = items[i];
item.category = stores[store].mapCategory(rawItem);
}
@ -297,14 +298,15 @@ exports.updateData = async function (dataDir, done) {
rawItems = await stores[store].fetchData();
writeJSON(rawDataFile, rawItems, FILE_COMPRESSOR);
}
const items = exports.dedupItems(getCanonicalFor(store, rawItems, today));
let sourceIdx = [];
const items = exports.dedupItems(getCanonicalFor(store, rawItems, today), sourceIdx);
await stores[store].initializeCategoryMapping(rawItems);
let numUncategorized = 0;
for (let i = 0; i < items.length; i++) {
const rawItem = rawItems[i];
const rawItem = rawItems[sourceIdx[i]];
const item = items[i];
item.category = stores[store].mapCategory(rawItem);
item.category = stores[store].mapCategory(rawItem, item); // both could be unrelated ?!
if (item.category == null) numUncategorized++;
}
@ -367,14 +369,16 @@ exports.migrateCompression = (dataDir, fromSuffix, toSuffix, remove = true) => {
}
};
exports.dedupItems = (items) => {
exports.dedupItems = (items, sourceIdx = []) => {
const lookup = {};
const dedupItems = [];
let duplicates = {};
for (const item of items) {
for (let i = 0; i < items.length; i++) {
const item = items[i];
const seenItem = lookup[item.store + item.id];
if (!seenItem) {
lookup[item.store + item.id] = item;
sourceIdx.push(i);
dedupItems.push(item);
} else {
if (seenItem.quantity != item.quantity || seenItem.unit != item.unit) {

2
stores/mpreis.js
View File

@ -45,7 +45,7 @@ exports.getCanonical = function (item, today) {
};
exports.fetchData = async function () {
const MPREIS_URL = `https://ax2ixv4hll-dsn.algolia.net/1/indexes/prod_mpreis_8450/browse?X-Algolia-API-Key=NmJlMTI0NjY1NGU4MDUwYTRlMmYzYWFjOWFlY2U4MGFkNGZjMDY2NmNjNjQzNWY3OWJlNDY4OTY0ZjEwOTEwYWZpbHRlcnM9cHVibGlzaGVk&X-Algolia-Application-Id=AX2IXV4HLL&X-Algolia-Agent=Vue.js`;
const MPREIS_URL = `https://uzxors8tl2-dsn.algolia.net/1/indexes/prod_mpreis_8450/browse?X-Algolia-API-Key=6d27574257fd3a92542ff880585333f1&X-Algolia-Application-Id=UZXORS8TL2&X-Algolia-Agent=Vue.js`;
let mpreisItems = [];
let res = (await axios.get(MPREIS_URL)).data;
mpreisItems = mpreisItems.concat(res.hits);