heissepreise/stores/billa.js

const axios = require("axios");
const fs = require("fs");
const utils = require("./utils");

const units = {
    beutel: { unit: "stk", factor: 1 },
    bund: { unit: "stk", factor: 1 },
    packung: { unit: "stk", factor: 1 },
    pa: { unit: "stk", factor: 1 },
    fl: { unit: "stk", factor: 1 },
    portion: { unit: "stk", factor: 1 },
    rollen: { unit: "stk", factor: 1 },
    teebeutel: { unit: "stk", factor: 1 },
    waschgang: { unit: "wg", factor: 1 },
};

const baseCategorySlugs = [
    "obst-und-gemuese-13751",
    "brot-und-gebaeck-13766",
    "getraenke-13784",
    "kuehlwaren-13841",
    "tiefkuehl-13916",
    "nahrungsmittel-13943",
    "suesses-und-salziges-14057",
    "pflege-14083",
    "haushalt-14126",
    "haustier-14181",
];

// Map from old categories (see categories.js) to new categories in new Billa Store
const subCategoryMap = {
    "vegan-13911": "3A",
    "tofu-14529": "3A",
    "fleisch-und-gefluegel-13921": "42",
    "fisch-und-meeresfruechte-13930": "43",
    "pizza-baguette-und-gebaeck-13936": "46",
    "pommes-frites-und-mehr-13935": "45",
    "tiefkuehlgerichte-13922": "42",
    "gemuese-kraeuter-und-obst-13934": "44",
    "eis-13917": "40",
    "suessspeisen-und-torten-13939": "47",
    "vegane-ernaehrung-14048": "5D",
    "filter-und-entkalker-14535": "83",
    "hygiene-schutzartikel-14536": "8F",
};

exports.getCanonical = function (item, today) {
    const price = item.price.regular.value / 100;
    return utils.convertUnit(
        {
            id: item.sku,
            name: item.name,
            description: item.descriptionShort ?? "",
            price,
            priceHistory: [{ date: today, price }],
            isWeighted: item.weightArticle,
            unit: item.volumeLabelShort ?? item.price.baseUnitShort,
            quantity: parseFloat(item.amount),
            bio: item.badges && item.badges.includes("pp-bio") ? true : false,
            url: item.slug,
        },
        units,
        "billa"
    );
};

exports.generateCategories = async function () {
    const categories = [];
    let baseIndex = 0;
    for (const baseSlug of baseCategorySlugs) {
        const data = await axios.get(`https://shop.billa.at/api/categories/${baseSlug}/child-properties?storeId=00-10`);
        data.data.forEach((value, index) => {
            const code = subCategoryMap[value.slug] ?? baseIndex.toString(16).toUpperCase() + index.toString(16).toUpperCase();
            categories.push({
                id: value.slug,
                description: value.name,
                url: "https://shop.billa.at/kategorie/" + value.slug,
                code,
            });
        });
        baseIndex++;
    }
    return categories;
};

exports.fetchData = async function () {
    const categories = await exports.generateCategories();
    const rawItems = [];
    for (const category of categories) {
        let page = 0;
        while (true) {
            const data = await axios.get(`https://shop.billa.at/api/categories/${category.id}/products?pageSize=500&storeId=00-10&page=${page}`);
            page++;
            if (data.data.count == 0) break;
            for (const rawItem of data.data.results) {
                rawItem.mappedCategory = category.code;
            }
            rawItems.push(...data.data.results);
        }
        // console.log("Fetched Billa category " + category.id);
    }
    const lookup = {};
    const dedupRawItems = [];
    let duplicates = 0;
    for (const rawItem of rawItems) {
        if (lookup[rawItem.sku]) {
            duplicates++;
        } else {
            lookup[rawItem.sku] = rawItem;
            dedupRawItems.push(rawItem);
        }
    }
    // console.log("Billa duplicates: " + duplicates);
    // console.log("Billa total items: " + dedupRawItems.length);
    return dedupRawItems;
};

exports.initializeCategoryMapping = async () => {
    // FIXME check if categories have changed.
};

exports.mapCategory = (rawItem) => {
    return rawItem.mappedCategory;
};

exports.urlBase = "https://shop.billa.at";

if (require.main === module) {
    main();
}

/* Test code, ensuring the data from the old Billa store is still compatible with the data from the new Billa store.
 you can get an old latest-canonical.json from here:
 https://marioslab.io/uploads/latest-canonical-2023-10-22.json
*/
function currentDate() {
    const currentDate = new Date();
    const year = currentDate.getFullYear();
    const month = String(currentDate.getMonth() + 1).padStart(2, "0");
    const day = String(currentDate.getDate()).padStart(2, "0");
    return `${year}-${month}-${day}`;
}

async function main() {
    const categories = await exports.generateCategories();
    console.log(categories);
    const oldCanonical = fs.existsSync("old-canonical.json")
        ? JSON.parse(fs.readFileSync("old-canonical.json"))
        : (await axios.get("https://heisse-preise.io/data/latest-canonical.json")).data;
    fs.writeFileSync("old-canonical.json", JSON.stringify(oldCanonical, null, 2));
    const oldItems = oldCanonical.filter((item) => item.store == "billa");
    const oldItemsLookup = {};
    for (const oldItem of oldItems) {
        oldItemsLookup[oldItem.id] = oldItem;
    }
    const newItems = fs.existsSync("billa-new.json") ? JSON.parse(fs.readFileSync("billa-new.json")) : await exports.fetchData();
    fs.writeFileSync("billa-new.json", JSON.stringify(newItems, null, 2));
    const canonicalItems = [];
    for (const newItem of newItems) {
        const canonicalItem = exports.getCanonical(newItem, currentDate());
        canonicalItems.push(canonicalItem);
        const oldItem = oldItemsLookup[newItem.sku];
        if (!oldItem) continue;

        let change = false;

        if (Math.abs(canonicalItem.price - oldItem.price) / oldItem.price > 1) {
            console.log(
                "Too big a price difference " +
                    canonicalItem.id +
                    " " +
                    canonicalItem.name +
                    ", old: " +
                    oldItem.price +
                    ", new: " +
                    canonicalItem.price
            );
            change = true;
        }
        if (canonicalItem.isWeighted != oldItem.isWeighted) {
            console.log(
                "!= isWeighted " + canonicalItem.id + " " + canonicalItem.name + ", old: " + oldItem.isWeighted + ", new: " + canonicalItem.isWeighted
            );
            change = true;
        }
        if (canonicalItem.unit != oldItem.unit) {
            console.log("!= unit " + canonicalItem.id + " " + canonicalItem.name + ", old: " + oldItem.unit + ", new: " + canonicalItem.unit);
            change = true;
        }
        if (canonicalItem.quantity != oldItem.quantity) {
            console.log(
                "!= quantity " + canonicalItem.id + " " + canonicalItem.name + ", old: " + oldItem.quantity + ", new: " + canonicalItem.quantity
            );
            change = true;
        }
        if (canonicalItem.bio != oldItem.bio) {
            console.log("!= bio " + canonicalItem.id + " " + canonicalItem.name + ", old: " + oldItem.bio + ", new: " + canonicalItem.bio);
            change = true;
        }
        if (canonicalItem.category != oldItem.category) {
            console.log(
                "!= category " + canonicalItem.id + " " + canonicalItem.name + ", old: " + oldItem.category + ", new: " + canonicalItem.category
            );
            change = true;
        }
        if (change) console.log("\n");
    }

    console.log("Canonical items");
}