2023-05-28 09:53:19 +02:00
|
|
|
const fs = require("fs");
|
|
|
|
const analysis = require("./analysis.js");
|
|
|
|
|
2023-05-28 11:34:56 +02:00
|
|
|
if (process.argv.length < 3) {
|
2023-05-28 11:26:17 +02:00
|
|
|
console.log("Usage: node pages.js <data-dir>");
|
2023-05-28 09:53:19 +02:00
|
|
|
console.log()
|
2023-05-28 11:26:17 +02:00
|
|
|
console.log("e.g. node pages.js data/");
|
2023-05-28 09:53:19 +02:00
|
|
|
console.log()
|
|
|
|
process.exit(1);
|
|
|
|
}
|
|
|
|
const dataDir = process.argv[2];
|
|
|
|
|
|
|
|
if (!fs.existsSync(dataDir)) {
|
|
|
|
fs.mkdirSync(dataDir);
|
|
|
|
}
|
|
|
|
|
|
|
|
(async function () {
|
|
|
|
try {
|
|
|
|
await analysis.updateData(dataDir);
|
2023-05-30 10:34:25 +02:00
|
|
|
const items = JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`));
|
2023-05-30 15:46:10 +02:00
|
|
|
for (const store of analysis.STORE_KEYS) {
|
|
|
|
const storeItems = items.filter(item => item.store === store);
|
|
|
|
fs.writeFileSync(`${dataDir}/latest-canonical.${store}.compressed.json`, JSON.stringify(analysis.compress(storeItems)));
|
|
|
|
}
|
2023-05-30 10:34:25 +02:00
|
|
|
console.log(`Wrote ${items.length} items to ${dataDir}/latest-canonical(-compressed).json`);
|
2023-05-30 15:46:10 +02:00
|
|
|
} catch (e) {
|
2023-05-28 09:53:19 +02:00
|
|
|
process.exit(1);
|
|
|
|
}
|
|
|
|
})();
|