Binary compression (it's worse), unit prices in charts, small improvements.

analysis.js

@@ -148,6 +148,139 @@ function sortItems(items) {
     });
 }
 
+function compressBinary(items) {
+    const buffer = [];
+
+    for (const item of items) {
+        // Serialize 'bio', 'isWeighted', and 'unit' into a single byte
+        let flagsByte = 0;
+        if (item.bio) flagsByte |= 1;
+        if (item.isWeighted) flagsByte |= 2;
+        if (item.unit === "ml") flagsByte |= 4;
+        if (item.unit === "stk") flagsByte |= 8;
+        buffer.push(flagsByte);
+
+        // Serialize 'quantity' as a 4-byte float
+        const quantityBuffer = Buffer.allocUnsafe(4);
+        quantityBuffer.writeFloatLE(item.quantity, 0);
+        buffer.push(...quantityBuffer);
+
+        // Serialize 'price' as a 4-byte float
+        const priceBuffer = Buffer.allocUnsafe(4);
+        priceBuffer.writeFloatLE(item.price, 0);
+        buffer.push(...priceBuffer);
+
+        // Serialize 'store' as a byte
+        const storeByte = STORE_KEYS.findIndex((store) => store == item.store);
+        buffer.push(storeByte);
+
+        // Serialize 'name' as UTF-8 with 2 bytes encoding the string length
+        const nameBuffer = Buffer.from(item.name, "utf8");
+        const nameLengthBuffer = Buffer.allocUnsafe(2);
+        nameLengthBuffer.writeUInt16LE(nameBuffer.length, 0);
+        buffer.push(...nameLengthBuffer, ...nameBuffer);
+
+        // Serialize 'url' as UTF-8 with 2 bytes encoding the string length
+        if (item.url !== undefined) {
+            const urlBuffer = Buffer.from(item.url, "utf8");
+            const urlLengthBuffer = Buffer.allocUnsafe(2);
+            urlLengthBuffer.writeUInt16LE(urlBuffer.length, 0);
+            buffer.push(...urlLengthBuffer, ...urlBuffer);
+        } else {
+            const urlLengthBuffer = Buffer.allocUnsafe(2).fill(0);
+            buffer.push(...urlLengthBuffer);
+        }
+
+        // Serialize 'priceHistory' array
+        const priceHistoryLengthBuffer = Buffer.allocUnsafe(2);
+        priceHistoryLengthBuffer.writeUInt16LE(item.priceHistory.length, 0);
+        buffer.push(...priceHistoryLengthBuffer);
+
+        for (const priceEntry of item.priceHistory) {
+            // Serialize price as a 4-byte float
+            const priceEntryBuffer = Buffer.allocUnsafe(4);
+            priceEntryBuffer.writeFloatLE(priceEntry.price, 0);
+            buffer.push(...priceEntryBuffer);
+
+            // Calculate the days since 2000-01-01
+            const entryDate = new Date(priceEntry.date);
+            const baseDate = new Date("2000-01-01");
+            const daysSince2000 = Math.floor((entryDate - baseDate) / (1000 * 60 * 60 * 24));
+
+            // Serialize days as a 32-bit integer
+            const daysBuffer = Buffer.allocUnsafe(4);
+            daysBuffer.writeInt32LE(daysSince2000, 0);
+            buffer.push(...daysBuffer);
+        }
+    }
+
+    return Buffer.from(buffer);
+}
+exports.compressBinary = compressBinary;
+
+function decompressBinary(buffer) {
+    const objects = [];
+    let offset = 0;
+
+    while (offset < buffer.length) {
+        const obj = {};
+
+        // Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
+        const flagsByte = buffer[offset++];
+        obj.bio = (flagsByte & 1) !== 0;
+        obj.isWeighted = (flagsByte & 2) !== 0;
+        obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
+
+        // Deserialize 'quantity' as a 4-byte float
+        obj.quantity = buffer.readFloatLE(offset);
+        offset += 4;
+
+        // Deserialize 'price' as a 4-byte float
+        obj.price = buffer.readFloatLE(offset);
+        offset += 4;
+
+        // Deserialize 'store' as a byte
+        obj.store = STORE_KEYS[buffer[offset++]];
+
+        // Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
+        const nameLength = buffer.readUInt16LE(offset);
+        offset += 2;
+        obj.name = buffer.toString("utf8", offset, offset + nameLength);
+        offset += nameLength;
+
+        // Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
+        const urlLength = buffer.readUInt16LE(offset);
+        offset += 2;
+        obj.url = urlLength !== 0 ? buffer.toString("utf8", offset, offset + urlLength) : undefined;
+        offset += urlLength;
+
+        // Deserialize 'priceHistory' array
+        const priceHistoryLength = buffer.readUInt16LE(offset);
+        offset += 2;
+        obj.priceHistory = [];
+
+        for (let i = 0; i < priceHistoryLength; i++) {
+            // Deserialize price as a 4-byte float
+            const price = buffer.readFloatLE(offset);
+            offset += 4;
+
+            // Deserialize days as a 32-bit integer
+            const daysSince2000 = buffer.readInt32LE(offset);
+            offset += 4;
+
+            // Calculate the date from days since 2000-01-01
+            const baseDate = new Date("2000-01-01");
+            const entryDate = new Date(baseDate.getTime() + daysSince2000 * 24 * 60 * 60 * 1000);
+
+            obj.priceHistory.push({ date: entryDate.toISOString().substring(0, 10), price });
+        }
+
+        objects.push(obj);
+    }
+
+    return objects;
+}
+
 // Keep this in sync with utils.js:decompress
 function compress(items) {
     const compressed = {

server.js

@@ -13,6 +13,7 @@ function copyItemsToSite(dataDir) {
     for (const store of analysis.STORE_KEYS) {
         const storeItems = items.filter((item) => item.store === store);
         analysis.writeJSON(`site/output/data/latest-canonical.${store}.compressed.json`, storeItems, false, 0, true);
+        fs.writeFileSync(`site/output/data/latest-canonical.${store}.bin.json`, analysis.compressBinary(storeItems));
     }
 }
 

site/cart.js

@@ -48,7 +48,7 @@ class CartHeader extends View {
             } else {
                 carts.push(cart);
             }
-            // model.carts.save();
+            models.carts.save();
             location.href = location.pathname + "?name=" + encodeURIComponent(cart.name);
         });
     }
@@ -65,7 +65,7 @@ class CartHeader extends View {
             for (const cartItem of cart.items) {
                 link += cartItem.store + cartItem.id + ";";
             }
-            elements.share.href = "cart.html?cart=" + link;
+            elements.share.href = "cart.html?cart=" + link + (this.stateToUrl ? stateToUrl() : "");
         }
     }
 }
@@ -150,6 +150,50 @@ function loadCart() {
         cartList.classList.remove("hidden");
     };
 
+    const itemsFilter = cartFilter;
+    const itemsList = cartList;
+    const itemsChart = cartList.querySelector("items-chart");
+    itemsList.elements.sort.value = "store-and-name";
+    let baseUrl = location.href.split("&")[0];
+
+    const stateToUrl = () => {
+        const filterState = itemsFilter.shareableState;
+        const listState = itemsList.shareableState;
+        const chartState = itemsChart.shareableState;
+        const chartedItems = cart.filteredItems
+            .filter((item) => item.chart)
+            .map((item) => item.store + item.id)
+            .join(";");
+        return baseUrl + "&f=" + filterState + "&l=" + listState + "&c=" + chartState + "&d=" + chartedItems;
+    };
+    cartHeader.stateToUrl = stateToUrl;
+    itemsFilter.addEventListener("x-change", () => {
+        const url = stateToUrl();
+        history.pushState({}, null, url);
+        cartHeader.render();
+    });
+    itemsList.addEventListener("x-change", () => {
+        const url = stateToUrl();
+        history.pushState({}, null, url);
+        cartHeader.render();
+    });
+
+    const f = getQueryParameter("f");
+    const l = getQueryParameter("l");
+    const c = getQueryParameter("c");
+    const d = getQueryParameter("d");
+
+    if (f) itemsFilter.shareableState = f;
+    if (l) itemsList.shareableState = l;
+    if (c) itemsChart.shareableState = c;
+    if (d) {
+        cart.items.lookup = {};
+        for (const item of cart.items) cart.items.lookup[item.store + item.id] = item;
+        for (const id of d.split(";")) {
+            cart.items.lookup[id].chart = true;
+        }
+    }
     cartList.model = cartFilter.model = cart;
     productsList.model = productsFilter.model = models.items;
+    if (c || d) itemsChart.render();
 })();

site/model/items.js

@@ -1,6 +1,78 @@
 const { deltaTime, log } = require("../js/misc");
 const { stores, STORE_KEYS } = require("./stores");
 const { Model } = require("./model");
+const { Settings } = require("./settings");
+
+function decompressBinary(buffer) {
+    const objects = [];
+    let offset = 0;
+    const view = new DataView(buffer);
+    const baseDate = new Date("2000-01-01");
+    const textDecoder = new TextDecoder("utf-8");
+
+    while (offset < buffer.byteLength) {
+        const obj = {};
+
+        // Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
+        const flagsByte = view.getUint8(offset++);
+        obj.bio = (flagsByte & 1) !== 0;
+        obj.isWeighted = (flagsByte & 2) !== 0;
+        obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
+
+        // Deserialize 'quantity' as a 4-byte float
+        obj.quantity = view.getFloat32(offset, true);
+        offset += 4;
+
+        // Deserialize 'price' as a 4-byte float
+        obj.price = view.getFloat32(offset, true);
+        offset += 4;
+
+        // Deserialize 'store' as a byte
+        obj.store = STORE_KEYS[view.getUint8(offset++)];
+
+        // Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
+        const nameLength = view.getUint16(offset, true);
+        offset += 2;
+        const nameBuffer = new Uint8Array(buffer, offset, nameLength);
+        obj.name = textDecoder.decode(nameBuffer);
+        offset += nameLength;
+
+        // Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
+        const urlLength = view.getUint16(offset, true);
+        offset += 2;
+        if (urlLength !== 0) {
+            const urlBuffer = new Uint8Array(buffer, offset, urlLength);
+            obj.url = textDecoder.decode(urlBuffer);
+        } else {
+            obj.url = undefined;
+        }
+        offset += urlLength;
+
+        // Deserialize 'priceHistory' array
+        const priceHistoryLength = view.getUint16(offset, true);
+        offset += 2;
+        obj.priceHistory = new Array(priceHistoryLength);
+
+        for (let i = 0; i < priceHistoryLength; i++) {
+            // Deserialize price as a 4-byte float
+            const price = view.getFloat32(offset, true);
+            offset += 4;
+
+            // Deserialize days as a 32-bit integer
+            const daysSince2000 = view.getInt32(offset, true);
+            offset += 4;
+
+            // Calculate the date from days since 2000-01-01
+            const entryDate = new Date(baseDate.getTime() + daysSince2000 * 24 * 60 * 60 * 1000);
+
+            obj.priceHistory[i] = { date: entryDate.toISOString().substring(0, 10), price };
+        }
+
+        objects.push(obj);
+    }
+
+    return objects;
+}
 
 function decompress(compressedItems) {
     const storeLookup = compressedItems.stores;
@@ -126,16 +198,31 @@ class Items extends Model {
 
     async load() {
         let start = performance.now();
+        const settings = new Settings();
         const compressedItemsPerStore = [];
         for (const store of STORE_KEYS) {
             compressedItemsPerStore.push(
                 new Promise(async (resolve) => {
-                    const start = performance.now();
+                    let start = performance.now();
                     try {
-                        const response = await fetch(`data/latest-canonical.${store}.compressed.json`);
-                        const json = await response.json();
-                        log(`Items - loading compressed items for ${store} took ${deltaTime(start)} secs`);
-                        resolve(decompress(json));
+                        const useJSON = settings.useJson;
+                        if (useJSON) {
+                            const response = await fetch(`data/latest-canonical.${store}.compressed.json`);
+                            const json = await response.json();
+                            log(`Items - loading compressed items for ${store} took ${deltaTime(start)} secs`);
+                            start = performance.now();
+                            let items = decompress(json);
+                            log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`);
+                            resolve(items);
+                        } else {
+                            const response = await fetch(`data/latest-canonical.${store}.bin.json`);
+                            const binary = await response.arrayBuffer();
+                            log(`Items - loading compressed binary items for ${store} took ${deltaTime(start)} secs`);
+                            start = performance.now();
+                            let items = decompressBinary(binary);
+                            log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`);
+                            resolve(items);
+                        }
                     } catch (e) {
                         log(`Items - error while loading compressed items for ${store} ${e.message}`);
                         resolve([]);

site/model/settings.js

@@ -1,7 +1,7 @@
 const { STORE_KEYS, stores } = require("./stores");
 const { Model } = require("./model");
 
-export class Settings extends Model {
+class Settings extends Model {
     constructor() {
         super();
         this.startDate = "2017-01-01";
@@ -9,6 +9,7 @@ export class Settings extends Model {
         STORE_KEYS.forEach((store) => {
             this[store] = stores[store].defaultChecked;
         });
+        this.jsonData = true;
 
         let settings = localStorage.getItem("settings");
         if (settings) {
@@ -30,3 +31,5 @@ export class Settings extends Model {
         localStorage.setItem("settings", JSON.stringify(settings));
     }
 }
+
+exports.Settings = Settings;

site/settings.js

@@ -37,6 +37,8 @@ class SettingsView extends View {
                         <option value="lines">Linien</option>
                     </select>
                 </label>
+                <custom-checkbox x-id="useJson" x-change x-state checked label="Daten als JSON downloaden">
+                </custom-checkbox>
             </div>
         `;
         this.setupEventHandlers();

site/views/items-chart.js

@@ -12,6 +12,7 @@ class ItemsChart extends View {
     constructor() {
         super();
 
+        this.unitPrice = false;
         this.innerHTML = /*html*/ `
             <div class="bg-stone-200 p-4 mx-auto">
                 <div class="w-full  h-[calc(100vw*0.66)] md:h-[calc(100vw*0.5)] lg:h-[calc(100vw*0.30)]" style="position: relative;">
@@ -41,9 +42,11 @@ class ItemsChart extends View {
     calculateOverallPriceChanges(items, onlyToday, startDate, endDate) {
         if (items.length == 0) return { dates: [], changes: [] };
 
+        const getPrice = this.unitPrice ? (o) => o.unitPrice : (o) => o.price;
+
         if (onlyToday) {
             let sum = 0;
-            for (const item of items) sum += item.price;
+            for (const item of items) sum += getPrice(item);
             return [{ date: today(), price: sum }];
         }
 
@@ -67,8 +70,8 @@ class ItemsChart extends View {
             }
             for (let i = 0; i < uniqueDates.length; i++) {
                 const priceObj = product.priceHistoryLookup[uniqueDates[i]];
-                if (!price && priceObj) price = priceObj.price;
-                priceScratch[i] = priceObj ? priceObj.price : null;
+                if (!price && priceObj) price = getPrice(priceObj);
+                priceScratch[i] = priceObj ? getPrice(priceObj) : null;
             }
 
             for (let i = 0; i < priceScratch.length; i++) {
@@ -90,6 +93,7 @@ class ItemsChart extends View {
     }
 
     renderChart(items, chartType) {
+        const getPrice = this.unitPrice ? (o) => o.unitPrice : (o) => o.price;
         const canvasDom = this.elements.canvas;
         const noData = this.elements.noData;
         if (items.length === 0) {
@@ -113,7 +117,7 @@ class ItemsChart extends View {
                 data: prices.map((price) => {
                     return {
                         x: moment(price.date),
-                        y: price.price,
+                        y: getPrice(price),
                     };
                 }),
             };
@@ -212,11 +216,12 @@ class ItemsChart extends View {
             log("ItemsChart - Calculating overall sum per store took " + ((performance.now() - now) / 1000).toFixed(2) + " secs");
         }
 
+        const getPrice = this.unitPrice ? (o) => o.unitPrice : (o) => o.price;
         items.forEach((item) => {
             if (item.chart) {
                 const chartItem = {
                     name: item.store + " " + item.name,
-                    priceHistory: onlyToday ? [{ date: today(), price: item.price }] : item.priceHistory,
+                    priceHistory: onlyToday ? [{ date: today(), price: getPrice(item) }] : item.priceHistory,
                 };
                 itemsToShow.push(chartItem);
             }

site/views/items-filter.js

@@ -129,7 +129,10 @@ class ItemsFilter extends View {
         const start = performance.now();
         const elements = this.elements;
         this.model.totalItems = this.model.items.length;
-        let filteredItems = [...this.model.items];
+        let filteredItems = new Array(this.model.items.length);
+        for (let i = 0; i < this.model.items.length; i++) {
+            filteredItems[i] = this.model.items[i];
+        }
         let query = elements.query.value.trim();
         if (query.length == 0 && this._emptyQuery) {
             this.model.removeListener(this._listener);
@@ -189,8 +192,7 @@ class ItemsFilter extends View {
         // Don't apply store and misc filters if query is an alasql query.
         if (query.charAt(0) != "!") {
             if (this._filterByStores) {
-                const checkedStores = this.checkedStores;
-                filteredItems = filteredItems.filter((item) => checkedStores.includes(item.store));
+                filteredItems = filteredItems.filter((item) => elements[item.store].checked);
             }
 
             if (this._filterByMisc) {
@@ -228,37 +230,35 @@ class ItemsFilter extends View {
         const start = performance.now();
         const elements = this.elements;
         const items = this.model.items;
-        const dates = {};
-        for (const item of items) {
-            if (item.priceHistory.length == 1) continue;
-            for (let i = 0; i < item.priceHistory.length; i++) {
-                const price = item.priceHistory[i];
-                if (i + 1 < item.priceHistory.length) {
-                    if (item.priceHistory[i].price != item.priceHistory[i + 1].price) {
-                        if (i == 0 || item.priceHistory[i].date != item.priceHistory[i - 1].date) {
-                            dates[price.date] = dates[price.date] ? dates[price.date] + 1 : 1;
+        if (this._filterByPriceChanges) {
+            const dates = {};
+            for (const item of items) {
+                if (item.priceHistory.length == 1) continue;
+                for (let i = 0; i < item.priceHistory.length; i++) {
+                    const price = item.priceHistory[i];
+                    if (i + 1 < item.priceHistory.length) {
+                        if (item.priceHistory[i].price != item.priceHistory[i + 1].price) {
+                            if (i == 0 || item.priceHistory[i].date != item.priceHistory[i - 1].date) {
+                                dates[price.date] = dates[price.date] ? dates[price.date] + 1 : 1;
+                            }
                         }
                     }
                 }
             }
-        }
 
-        const priceChangesDates = elements.priceChangesDate;
-        priceChangesDates.innerHTML = "";
-        for (const date of Object.keys(dates).sort((a, b) => b.localeCompare(a))) {
-            const dateDom = dom("option");
-            dateDom.value = date;
-            dateDom.innerText = `${date} (${dates[date]})`;
-            priceChangesDates.append(dateDom);
+            const priceChangesDates = elements.priceChangesDate;
+            priceChangesDates.innerHTML = "";
+            for (const date of Object.keys(dates).sort((a, b) => b.localeCompare(a))) {
+                const dateDom = dom("option");
+                dateDom.value = date;
+                dateDom.innerText = `${date} (${dates[date]})`;
+                priceChangesDates.append(dateDom);
+            }
         }
 
         log(`ItemsFilter - rendering items filter took ${deltaTime(start)}`);
     }
 
-    get checkedStores() {
-        return STORE_KEYS.filter((store) => this.elements[store].checked);
-    }
-
     get shareableState() {
         const state = this.state;
         const shareableState = Object.keys(state)

site/views/items-list.js

@@ -5,6 +5,8 @@ const { View } = require("./view");
 const { ItemsChart } = require("./items-chart");
 
 class ItemsList extends View {
+    static priceTypeId = 0;
+
     constructor() {
         super();
 
@@ -27,8 +29,10 @@ class ItemsList extends View {
                         <custom-checkbox x-id="enableChart" x-change x-state label="Diagramm" class="${
                             this._chart ? "" : "hidden"
                         }"></custom-checkbox>
-                        <label><input x-id="salesPrice" x-change x-state type="radio" name="priceType" checked> Verkaufspreis</label>
-                        <label><input x-id="unitPrice" x-change x-state type="radio" name="priceType"> Mengenpreis</label>
+                        <label><input x-id="salesPrice" x-change x-state type="radio" name="priceType${
+                            ItemsList.priceTypeId
+                        }" checked> Verkaufspreis</label>
+                        <label><input x-id="unitPrice" x-change x-state type="radio" name="priceType${ItemsList.priceTypeId++}"> Mengenpreis</label>
                     </div>
                 </div>
                 <label class="${hideSort}">
@@ -83,6 +87,18 @@ class ItemsList extends View {
             elements.tableBody.querySelectorAll(".priceinfo").forEach((el) => (showAll ? el.classList.remove("hidden") : el.classList.add("hidden")));
         });
 
+        elements.chart.unitPrice = elements.unitPrice.checked;
+
+        elements.unitPrice.addEventListener("change", () => {
+            elements.chart.unitPrice = elements.unitPrice.checked;
+            elements.chart.render();
+        });
+
+        elements.salesPrice.addEventListener("change", () => {
+            elements.chart.unitPrice = elements.unitPrice.checked;
+            elements.chart.render();
+        });
+
         this.setupEventHandlers();
 
         this.addEventListener("x-change", (event) => {
@@ -369,6 +385,7 @@ class ItemsList extends View {
         const start = performance.now();
         const elements = this.elements;
         if (!this.model) return;
+        elements.chart.unitPrice = elements.unitPrice.checked;
         if (this.model.filteredItems.length != 0 && this.model.filteredItems.length <= (isMobile() ? 200 : 1500)) {
             elements.nameSimilarity.removeAttribute("disabled");
         } else {