119 lines
4.1 KiB
JavaScript
119 lines
4.1 KiB
JavaScript
const axios = require("axios");
|
|
const util = require("util");
|
|
const exec = util.promisify(require("child_process").exec);
|
|
const utils = require("./utils");
|
|
const { decompress } = require("../site/model/items");
|
|
|
|
const units = {
|
|
beutel: { unit: "stk", factor: 1 },
|
|
bund: { unit: "stk", factor: 1 },
|
|
packung: { unit: "stk", factor: 1 },
|
|
portion: { unit: "stk", factor: 1 },
|
|
rollen: { unit: "stk", factor: 1 },
|
|
teebeutel: { unit: "stk", factor: 1 },
|
|
waschgang: { unit: "wg", factor: 1 },
|
|
};
|
|
|
|
exports.getCanonical = function (item, today) {
|
|
if (item.isCanonical) return item;
|
|
|
|
let quantity = 1,
|
|
unit = "kg";
|
|
if (item.grammage && item.grammage.length > 0) {
|
|
let grammage = item.grammage
|
|
.trim()
|
|
.replace(/\([^)]*\)/g, "")
|
|
.replace(",", ".")
|
|
.trim();
|
|
let multiplier = 1;
|
|
if (grammage.indexOf("x") != -1) {
|
|
let tokens = grammage.split("x");
|
|
multiplier = Number.parseFloat(tokens[0]);
|
|
grammage = tokens[1];
|
|
}
|
|
|
|
let tokens = grammage.split(" ");
|
|
if (tokens.length > 1) {
|
|
quantity = Number.parseFloat(tokens[0]);
|
|
unit = tokens[1];
|
|
} else {
|
|
[quantity, unit] = grammage.match(/^(\d+(?:\.\d+)?)(\D+)$/).slice(1);
|
|
quantity = Number.parseFloat(quantity);
|
|
}
|
|
quantity *= multiplier;
|
|
} else {
|
|
quantity = 1;
|
|
unit = "Stk";
|
|
}
|
|
|
|
let price = Number.parseFloat(item.currentPrice.split(" ")[0].replace(",", "."));
|
|
return utils.convertUnit(
|
|
{
|
|
id: item.id,
|
|
name: item.name,
|
|
// description: "", not available
|
|
price,
|
|
priceHistory: [{ date: today, price }],
|
|
isWeighted: false,
|
|
unit,
|
|
quantity,
|
|
bio: false,
|
|
},
|
|
units,
|
|
"reweDe"
|
|
);
|
|
};
|
|
|
|
exports.fetchData = async function () {
|
|
// For some unholy reason, Axios returns 403 when accessing the endpoint
|
|
// Hack: use curl...
|
|
/*const agent = new https.Agent({
|
|
rejectUnauthorized: false
|
|
});
|
|
let axiosNoDefaults = axios.create({ headers: {} });
|
|
const headers = {
|
|
'Rd-Service-Types': 'PICKUP',
|
|
'Rd-Market-Id': '440405',
|
|
"User-Agent": "curl/7.84.0"
|
|
}
|
|
return (await axiosNoDefaults.get('https://mobile-api.rewe.de/api/v3/product-search?searchTerm=*&page=1&sorting=RELEVANCE_DESC&objectsPerPage=250&marketCode=440405&serviceTypes=PICKUP', { headers, httpsAgent: agent })).data;*/
|
|
|
|
try {
|
|
let pageId = 1;
|
|
let result = (
|
|
await exec(
|
|
`curl -s "https://mobile-api.rewe.de/api/v3/product-search\?searchTerm\=\*\&page\=${pageId++}\&sorting\=RELEVANCE_DESC\&objectsPerPage\=250\&marketCode\=440405\&serviceTypes\=PICKUP" -H "Rd-Service-Types: PICKUP" -H "Rd-Market-Id: 440405"`
|
|
)
|
|
).stdout;
|
|
const firstPage = JSON.parse(result);
|
|
const totalPages = firstPage.totalPages;
|
|
const items = [...firstPage.products];
|
|
for (let i = 2; i <= totalPages; i++) {
|
|
items.push(
|
|
...JSON.parse(
|
|
(
|
|
await exec(
|
|
`curl -s "https://mobile-api.rewe.de/api/v3/product-search\?searchTerm\=\*\&page\=${pageId++}\&sorting\=RELEVANCE_DESC\&objectsPerPage\=250\&marketCode\=440405\&serviceTypes\=PICKUP" -H "Rd-Service-Types: PICKUP" -H "Rd-Market-Id: 440405"`
|
|
)
|
|
).stdout
|
|
).products
|
|
);
|
|
}
|
|
return items;
|
|
} catch (e) {
|
|
console.log("Failed to fetch REWE-DE data, either CURL is not installed, or CloudFlare protection kicked in.");
|
|
const compressedItems = (await axios.get("https://heissepreise.github.io/data/latest-canonical.reweDe.compressed.json")).data;
|
|
const items = decompress(compressedItems);
|
|
for (const item of items) {
|
|
item.isCanonical = true;
|
|
}
|
|
return items;
|
|
}
|
|
};
|
|
|
|
exports.initializeCategoryMapping = async () => {};
|
|
|
|
exports.mapCategory = (rawItem) => {};
|
|
|
|
exports.urlBase = "";
|