From ea5c13300367654d627964f13de676678be9a28d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Fri, 16 Jun 2023 16:01:13 +0200 Subject: [PATCH] Binary compression (it's worse), unit prices in charts, small improvements. --- analysis.js | 133 +++++++++++++++++++++++++++++++++++++ server.js | 1 + site/cart.js | 48 ++++++++++++- site/model/items.js | 97 +++++++++++++++++++++++++-- site/model/settings.js | 5 +- site/settings.js | 2 + site/views/items-chart.js | 15 +++-- site/views/items-filter.js | 48 ++++++------- site/views/items-list.js | 21 +++++- 9 files changed, 331 insertions(+), 39 deletions(-) diff --git a/analysis.js b/analysis.js index 53e4040..54a7b8e 100644 --- a/analysis.js +++ b/analysis.js @@ -148,6 +148,139 @@ function sortItems(items) { }); } +function compressBinary(items) { + const buffer = []; + + for (const item of items) { + // Serialize 'bio', 'isWeighted', and 'unit' into a single byte + let flagsByte = 0; + if (item.bio) flagsByte |= 1; + if (item.isWeighted) flagsByte |= 2; + if (item.unit === "ml") flagsByte |= 4; + if (item.unit === "stk") flagsByte |= 8; + buffer.push(flagsByte); + + // Serialize 'quantity' as a 4-byte float + const quantityBuffer = Buffer.allocUnsafe(4); + quantityBuffer.writeFloatLE(item.quantity, 0); + buffer.push(...quantityBuffer); + + // Serialize 'price' as a 4-byte float + const priceBuffer = Buffer.allocUnsafe(4); + priceBuffer.writeFloatLE(item.price, 0); + buffer.push(...priceBuffer); + + // Serialize 'store' as a byte + const storeByte = STORE_KEYS.findIndex((store) => store == item.store); + buffer.push(storeByte); + + // Serialize 'name' as UTF-8 with 2 bytes encoding the string length + const nameBuffer = Buffer.from(item.name, "utf8"); + const nameLengthBuffer = Buffer.allocUnsafe(2); + nameLengthBuffer.writeUInt16LE(nameBuffer.length, 0); + buffer.push(...nameLengthBuffer, ...nameBuffer); + + // Serialize 'url' as UTF-8 with 2 bytes encoding the string length + if (item.url !== undefined) { + const urlBuffer = Buffer.from(item.url, "utf8"); + const urlLengthBuffer = Buffer.allocUnsafe(2); + urlLengthBuffer.writeUInt16LE(urlBuffer.length, 0); + buffer.push(...urlLengthBuffer, ...urlBuffer); + } else { + const urlLengthBuffer = Buffer.allocUnsafe(2).fill(0); + buffer.push(...urlLengthBuffer); + } + + // Serialize 'priceHistory' array + const priceHistoryLengthBuffer = Buffer.allocUnsafe(2); + priceHistoryLengthBuffer.writeUInt16LE(item.priceHistory.length, 0); + buffer.push(...priceHistoryLengthBuffer); + + for (const priceEntry of item.priceHistory) { + // Serialize price as a 4-byte float + const priceEntryBuffer = Buffer.allocUnsafe(4); + priceEntryBuffer.writeFloatLE(priceEntry.price, 0); + buffer.push(...priceEntryBuffer); + + // Calculate the days since 2000-01-01 + const entryDate = new Date(priceEntry.date); + const baseDate = new Date("2000-01-01"); + const daysSince2000 = Math.floor((entryDate - baseDate) / (1000 * 60 * 60 * 24)); + + // Serialize days as a 32-bit integer + const daysBuffer = Buffer.allocUnsafe(4); + daysBuffer.writeInt32LE(daysSince2000, 0); + buffer.push(...daysBuffer); + } + } + + return Buffer.from(buffer); +} +exports.compressBinary = compressBinary; + +function decompressBinary(buffer) { + const objects = []; + let offset = 0; + + while (offset < buffer.length) { + const obj = {}; + + // Deserialize 'bio', 'isWeighted', and 'unit' from the single byte + const flagsByte = buffer[offset++]; + obj.bio = (flagsByte & 1) !== 0; + obj.isWeighted = (flagsByte & 2) !== 0; + obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g"; + + // Deserialize 'quantity' as a 4-byte float + obj.quantity = buffer.readFloatLE(offset); + offset += 4; + + // Deserialize 'price' as a 4-byte float + obj.price = buffer.readFloatLE(offset); + offset += 4; + + // Deserialize 'store' as a byte + obj.store = STORE_KEYS[buffer[offset++]]; + + // Deserialize 'name' as UTF-8 with 2 bytes encoding the string length + const nameLength = buffer.readUInt16LE(offset); + offset += 2; + obj.name = buffer.toString("utf8", offset, offset + nameLength); + offset += nameLength; + + // Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0) + const urlLength = buffer.readUInt16LE(offset); + offset += 2; + obj.url = urlLength !== 0 ? buffer.toString("utf8", offset, offset + urlLength) : undefined; + offset += urlLength; + + // Deserialize 'priceHistory' array + const priceHistoryLength = buffer.readUInt16LE(offset); + offset += 2; + obj.priceHistory = []; + + for (let i = 0; i < priceHistoryLength; i++) { + // Deserialize price as a 4-byte float + const price = buffer.readFloatLE(offset); + offset += 4; + + // Deserialize days as a 32-bit integer + const daysSince2000 = buffer.readInt32LE(offset); + offset += 4; + + // Calculate the date from days since 2000-01-01 + const baseDate = new Date("2000-01-01"); + const entryDate = new Date(baseDate.getTime() + daysSince2000 * 24 * 60 * 60 * 1000); + + obj.priceHistory.push({ date: entryDate.toISOString().substring(0, 10), price }); + } + + objects.push(obj); + } + + return objects; +} + // Keep this in sync with utils.js:decompress function compress(items) { const compressed = { diff --git a/server.js b/server.js index cf09f8e..a5ecd1a 100644 --- a/server.js +++ b/server.js @@ -13,6 +13,7 @@ function copyItemsToSite(dataDir) { for (const store of analysis.STORE_KEYS) { const storeItems = items.filter((item) => item.store === store); analysis.writeJSON(`site/output/data/latest-canonical.${store}.compressed.json`, storeItems, false, 0, true); + fs.writeFileSync(`site/output/data/latest-canonical.${store}.bin.json`, analysis.compressBinary(storeItems)); } } diff --git a/site/cart.js b/site/cart.js index ed7a27e..03e3e96 100644 --- a/site/cart.js +++ b/site/cart.js @@ -48,7 +48,7 @@ class CartHeader extends View { } else { carts.push(cart); } - // model.carts.save(); + models.carts.save(); location.href = location.pathname + "?name=" + encodeURIComponent(cart.name); }); } @@ -65,7 +65,7 @@ class CartHeader extends View { for (const cartItem of cart.items) { link += cartItem.store + cartItem.id + ";"; } - elements.share.href = "cart.html?cart=" + link; + elements.share.href = "cart.html?cart=" + link + (this.stateToUrl ? stateToUrl() : ""); } } } @@ -150,6 +150,50 @@ function loadCart() { cartList.classList.remove("hidden"); }; + const itemsFilter = cartFilter; + const itemsList = cartList; + const itemsChart = cartList.querySelector("items-chart"); + itemsList.elements.sort.value = "store-and-name"; + let baseUrl = location.href.split("&")[0]; + + const stateToUrl = () => { + const filterState = itemsFilter.shareableState; + const listState = itemsList.shareableState; + const chartState = itemsChart.shareableState; + const chartedItems = cart.filteredItems + .filter((item) => item.chart) + .map((item) => item.store + item.id) + .join(";"); + return baseUrl + "&f=" + filterState + "&l=" + listState + "&c=" + chartState + "&d=" + chartedItems; + }; + cartHeader.stateToUrl = stateToUrl; + itemsFilter.addEventListener("x-change", () => { + const url = stateToUrl(); + history.pushState({}, null, url); + cartHeader.render(); + }); + itemsList.addEventListener("x-change", () => { + const url = stateToUrl(); + history.pushState({}, null, url); + cartHeader.render(); + }); + + const f = getQueryParameter("f"); + const l = getQueryParameter("l"); + const c = getQueryParameter("c"); + const d = getQueryParameter("d"); + + if (f) itemsFilter.shareableState = f; + if (l) itemsList.shareableState = l; + if (c) itemsChart.shareableState = c; + if (d) { + cart.items.lookup = {}; + for (const item of cart.items) cart.items.lookup[item.store + item.id] = item; + for (const id of d.split(";")) { + cart.items.lookup[id].chart = true; + } + } cartList.model = cartFilter.model = cart; productsList.model = productsFilter.model = models.items; + if (c || d) itemsChart.render(); })(); diff --git a/site/model/items.js b/site/model/items.js index e865ff9..c1cde04 100644 --- a/site/model/items.js +++ b/site/model/items.js @@ -1,6 +1,78 @@ const { deltaTime, log } = require("../js/misc"); const { stores, STORE_KEYS } = require("./stores"); const { Model } = require("./model"); +const { Settings } = require("./settings"); + +function decompressBinary(buffer) { + const objects = []; + let offset = 0; + const view = new DataView(buffer); + const baseDate = new Date("2000-01-01"); + const textDecoder = new TextDecoder("utf-8"); + + while (offset < buffer.byteLength) { + const obj = {}; + + // Deserialize 'bio', 'isWeighted', and 'unit' from the single byte + const flagsByte = view.getUint8(offset++); + obj.bio = (flagsByte & 1) !== 0; + obj.isWeighted = (flagsByte & 2) !== 0; + obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g"; + + // Deserialize 'quantity' as a 4-byte float + obj.quantity = view.getFloat32(offset, true); + offset += 4; + + // Deserialize 'price' as a 4-byte float + obj.price = view.getFloat32(offset, true); + offset += 4; + + // Deserialize 'store' as a byte + obj.store = STORE_KEYS[view.getUint8(offset++)]; + + // Deserialize 'name' as UTF-8 with 2 bytes encoding the string length + const nameLength = view.getUint16(offset, true); + offset += 2; + const nameBuffer = new Uint8Array(buffer, offset, nameLength); + obj.name = textDecoder.decode(nameBuffer); + offset += nameLength; + + // Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0) + const urlLength = view.getUint16(offset, true); + offset += 2; + if (urlLength !== 0) { + const urlBuffer = new Uint8Array(buffer, offset, urlLength); + obj.url = textDecoder.decode(urlBuffer); + } else { + obj.url = undefined; + } + offset += urlLength; + + // Deserialize 'priceHistory' array + const priceHistoryLength = view.getUint16(offset, true); + offset += 2; + obj.priceHistory = new Array(priceHistoryLength); + + for (let i = 0; i < priceHistoryLength; i++) { + // Deserialize price as a 4-byte float + const price = view.getFloat32(offset, true); + offset += 4; + + // Deserialize days as a 32-bit integer + const daysSince2000 = view.getInt32(offset, true); + offset += 4; + + // Calculate the date from days since 2000-01-01 + const entryDate = new Date(baseDate.getTime() + daysSince2000 * 24 * 60 * 60 * 1000); + + obj.priceHistory[i] = { date: entryDate.toISOString().substring(0, 10), price }; + } + + objects.push(obj); + } + + return objects; +} function decompress(compressedItems) { const storeLookup = compressedItems.stores; @@ -126,16 +198,31 @@ class Items extends Model { async load() { let start = performance.now(); + const settings = new Settings(); const compressedItemsPerStore = []; for (const store of STORE_KEYS) { compressedItemsPerStore.push( new Promise(async (resolve) => { - const start = performance.now(); + let start = performance.now(); try { - const response = await fetch(`data/latest-canonical.${store}.compressed.json`); - const json = await response.json(); - log(`Items - loading compressed items for ${store} took ${deltaTime(start)} secs`); - resolve(decompress(json)); + const useJSON = settings.useJson; + if (useJSON) { + const response = await fetch(`data/latest-canonical.${store}.compressed.json`); + const json = await response.json(); + log(`Items - loading compressed items for ${store} took ${deltaTime(start)} secs`); + start = performance.now(); + let items = decompress(json); + log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`); + resolve(items); + } else { + const response = await fetch(`data/latest-canonical.${store}.bin.json`); + const binary = await response.arrayBuffer(); + log(`Items - loading compressed binary items for ${store} took ${deltaTime(start)} secs`); + start = performance.now(); + let items = decompressBinary(binary); + log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`); + resolve(items); + } } catch (e) { log(`Items - error while loading compressed items for ${store} ${e.message}`); resolve([]); diff --git a/site/model/settings.js b/site/model/settings.js index 28716c4..3ac92d6 100644 --- a/site/model/settings.js +++ b/site/model/settings.js @@ -1,7 +1,7 @@ const { STORE_KEYS, stores } = require("./stores"); const { Model } = require("./model"); -export class Settings extends Model { +class Settings extends Model { constructor() { super(); this.startDate = "2017-01-01"; @@ -9,6 +9,7 @@ export class Settings extends Model { STORE_KEYS.forEach((store) => { this[store] = stores[store].defaultChecked; }); + this.jsonData = true; let settings = localStorage.getItem("settings"); if (settings) { @@ -30,3 +31,5 @@ export class Settings extends Model { localStorage.setItem("settings", JSON.stringify(settings)); } } + +exports.Settings = Settings; diff --git a/site/settings.js b/site/settings.js index b57bfe1..a7d9717 100644 --- a/site/settings.js +++ b/site/settings.js @@ -37,6 +37,8 @@ class SettingsView extends View { + + `; this.setupEventHandlers(); diff --git a/site/views/items-chart.js b/site/views/items-chart.js index ae0ef5b..cc53843 100644 --- a/site/views/items-chart.js +++ b/site/views/items-chart.js @@ -12,6 +12,7 @@ class ItemsChart extends View { constructor() { super(); + this.unitPrice = false; this.innerHTML = /*html*/ `
@@ -41,9 +42,11 @@ class ItemsChart extends View { calculateOverallPriceChanges(items, onlyToday, startDate, endDate) { if (items.length == 0) return { dates: [], changes: [] }; + const getPrice = this.unitPrice ? (o) => o.unitPrice : (o) => o.price; + if (onlyToday) { let sum = 0; - for (const item of items) sum += item.price; + for (const item of items) sum += getPrice(item); return [{ date: today(), price: sum }]; } @@ -67,8 +70,8 @@ class ItemsChart extends View { } for (let i = 0; i < uniqueDates.length; i++) { const priceObj = product.priceHistoryLookup[uniqueDates[i]]; - if (!price && priceObj) price = priceObj.price; - priceScratch[i] = priceObj ? priceObj.price : null; + if (!price && priceObj) price = getPrice(priceObj); + priceScratch[i] = priceObj ? getPrice(priceObj) : null; } for (let i = 0; i < priceScratch.length; i++) { @@ -90,6 +93,7 @@ class ItemsChart extends View { } renderChart(items, chartType) { + const getPrice = this.unitPrice ? (o) => o.unitPrice : (o) => o.price; const canvasDom = this.elements.canvas; const noData = this.elements.noData; if (items.length === 0) { @@ -113,7 +117,7 @@ class ItemsChart extends View { data: prices.map((price) => { return { x: moment(price.date), - y: price.price, + y: getPrice(price), }; }), }; @@ -212,11 +216,12 @@ class ItemsChart extends View { log("ItemsChart - Calculating overall sum per store took " + ((performance.now() - now) / 1000).toFixed(2) + " secs"); } + const getPrice = this.unitPrice ? (o) => o.unitPrice : (o) => o.price; items.forEach((item) => { if (item.chart) { const chartItem = { name: item.store + " " + item.name, - priceHistory: onlyToday ? [{ date: today(), price: item.price }] : item.priceHistory, + priceHistory: onlyToday ? [{ date: today(), price: getPrice(item) }] : item.priceHistory, }; itemsToShow.push(chartItem); } diff --git a/site/views/items-filter.js b/site/views/items-filter.js index aa49632..4bb3124 100644 --- a/site/views/items-filter.js +++ b/site/views/items-filter.js @@ -129,7 +129,10 @@ class ItemsFilter extends View { const start = performance.now(); const elements = this.elements; this.model.totalItems = this.model.items.length; - let filteredItems = [...this.model.items]; + let filteredItems = new Array(this.model.items.length); + for (let i = 0; i < this.model.items.length; i++) { + filteredItems[i] = this.model.items[i]; + } let query = elements.query.value.trim(); if (query.length == 0 && this._emptyQuery) { this.model.removeListener(this._listener); @@ -189,8 +192,7 @@ class ItemsFilter extends View { // Don't apply store and misc filters if query is an alasql query. if (query.charAt(0) != "!") { if (this._filterByStores) { - const checkedStores = this.checkedStores; - filteredItems = filteredItems.filter((item) => checkedStores.includes(item.store)); + filteredItems = filteredItems.filter((item) => elements[item.store].checked); } if (this._filterByMisc) { @@ -228,37 +230,35 @@ class ItemsFilter extends View { const start = performance.now(); const elements = this.elements; const items = this.model.items; - const dates = {}; - for (const item of items) { - if (item.priceHistory.length == 1) continue; - for (let i = 0; i < item.priceHistory.length; i++) { - const price = item.priceHistory[i]; - if (i + 1 < item.priceHistory.length) { - if (item.priceHistory[i].price != item.priceHistory[i + 1].price) { - if (i == 0 || item.priceHistory[i].date != item.priceHistory[i - 1].date) { - dates[price.date] = dates[price.date] ? dates[price.date] + 1 : 1; + if (this._filterByPriceChanges) { + const dates = {}; + for (const item of items) { + if (item.priceHistory.length == 1) continue; + for (let i = 0; i < item.priceHistory.length; i++) { + const price = item.priceHistory[i]; + if (i + 1 < item.priceHistory.length) { + if (item.priceHistory[i].price != item.priceHistory[i + 1].price) { + if (i == 0 || item.priceHistory[i].date != item.priceHistory[i - 1].date) { + dates[price.date] = dates[price.date] ? dates[price.date] + 1 : 1; + } } } } } - } - const priceChangesDates = elements.priceChangesDate; - priceChangesDates.innerHTML = ""; - for (const date of Object.keys(dates).sort((a, b) => b.localeCompare(a))) { - const dateDom = dom("option"); - dateDom.value = date; - dateDom.innerText = `${date} (${dates[date]})`; - priceChangesDates.append(dateDom); + const priceChangesDates = elements.priceChangesDate; + priceChangesDates.innerHTML = ""; + for (const date of Object.keys(dates).sort((a, b) => b.localeCompare(a))) { + const dateDom = dom("option"); + dateDom.value = date; + dateDom.innerText = `${date} (${dates[date]})`; + priceChangesDates.append(dateDom); + } } log(`ItemsFilter - rendering items filter took ${deltaTime(start)}`); } - get checkedStores() { - return STORE_KEYS.filter((store) => this.elements[store].checked); - } - get shareableState() { const state = this.state; const shareableState = Object.keys(state) diff --git a/site/views/items-list.js b/site/views/items-list.js index 34620fa..0d5db58 100644 --- a/site/views/items-list.js +++ b/site/views/items-list.js @@ -5,6 +5,8 @@ const { View } = require("./view"); const { ItemsChart } = require("./items-chart"); class ItemsList extends View { + static priceTypeId = 0; + constructor() { super(); @@ -27,8 +29,10 @@ class ItemsList extends View { - - + +