diff --git a/bundle.js b/bundle.js index 5a7d0a9..82bfd85 100755 --- a/bundle.js +++ b/bundle.js @@ -108,6 +108,7 @@ async function bundleJS(inputDir, outputDir, watch) { changes: `${inputDir}/changes.js`, settings: `${inputDir}/settings.js`, index: `${inputDir}/index.js`, + loader: `${inputDir}/model/loader.js`, }, bundle: true, sourcemap: true, diff --git a/site/model/items.js b/site/model/items.js index b1319b1..89f5a03 100644 --- a/site/model/items.js +++ b/site/model/items.js @@ -1,142 +1,6 @@ -const { deltaTime, log, uint16ToDate } = require("../js/misc"); -const { stores, STORE_KEYS } = require("./stores"); const { Model } = require("./model"); const { Settings } = require("./settings"); - -function decompressBinary(buffer) { - const objects = []; - let offset = 0; - const view = new DataView(buffer); - const baseDate = new Date("2000-01-01"); - const textDecoder = new TextDecoder("utf-8"); - - const numStores = view.getUint8(offset++); - const stores = []; - for (let i = 0; i < numStores; i++) { - const nameLength = view.getUint16(offset, true); - offset += 2; - const nameBuffer = new Uint8Array(buffer, offset, nameLength); - stores.push(textDecoder.decode(nameBuffer)); - offset += nameLength; - } - - const numWords = view.getUint32(offset, true); - offset += 4; - const words = new Array(numWords); - for (let i = 0; i < numWords; i++) { - const nameLength = view.getUint8(offset++); - const nameBuffer = new Uint8Array(buffer, offset, nameLength); - words[i] = textDecoder.decode(nameBuffer); - offset += nameLength; - } - - while (offset < buffer.byteLength) { - const obj = {}; - const idLength = view.getUint8(offset++); - const idBuffer = new Uint8Array(buffer, offset, idLength); - obj.id = textDecoder.decode(idBuffer); - offset += idLength; - - const flagsByte = view.getUint8(offset++); - obj.bio = (flagsByte & 1) !== 0; - obj.isWeighted = (flagsByte & 2) !== 0; - if (flagsByte & 4) obj.unit = "ml"; - if (flagsByte & 8) obj.unit = "g"; - if (flagsByte & 16) obj.unit = "stk"; - if (flagsByte & 32) obj.unit = "cm"; - if (flagsByte & 64) obj.unit = "wg"; - - obj.quantity = view.getUint16(offset, true); - offset += 2; - - obj.store = stores[view.getUint8(offset++)]; - - let name = ""; - const numTokens = view.getUint8(offset++); - for (let i = 0; i < numTokens; i++) { - const b1 = view.getUint8(offset++); - const b2 = view.getUint8(offset++); - const b3 = view.getUint8(offset++); - const tokenId = (b3 << 16) | (b2 << 8) | b1; - name += words[tokenId]; - if (i < numTokens - 1) name += " "; - } - obj.name = name; - - const urlLength = view.getUint16(offset, true); - offset += 2; - if (urlLength !== 0) { - const urlBuffer = new Uint8Array(buffer, offset, urlLength); - obj.url = textDecoder.decode(urlBuffer); - } else { - obj.url = undefined; - } - offset += urlLength; - - const priceHistoryLength = view.getUint16(offset, true); - offset += 2; - obj.priceHistory = new Array(priceHistoryLength); - - for (let i = 0; i < priceHistoryLength; i++) { - const price = view.getUint16(offset, true) / 100; - offset += 2; - - const date = uint16ToDate(view.getUint16(offset, true)); - offset += 2; - - obj.priceHistory[i] = { date, price }; - } - - obj.price = obj.priceHistory[0].price; - - objects.push(obj); - } - - return objects; -} - -function decompress(compressedItems) { - const storeLookup = compressedItems.stores; - const data = compressedItems.data; - const dates = compressedItems.dates; - const numItems = compressedItems.n; - const items = new Array(numItems); - let i = 0; - for (let l = 0; l < numItems; l++) { - const store = storeLookup[data[i++]]; - const id = data[i++]; - const name = data[i++]; - const numPrices = data[i++]; - const prices = new Array(numPrices); - for (let j = 0; j < numPrices; j++) { - const date = dates[data[i++]]; - const price = data[i++]; - prices[j] = { - date: date.substring(0, 4) + "-" + date.substring(4, 6) + "-" + date.substring(6, 8), - price, - }; - } - const unit = data[i++]; - const quantity = data[i++]; - const isWeighted = data[i++] == 1; - const bio = data[i++] == 1; - const url = stores[store].getUrl({ id, name, url: data[i++] }); - - items[l] = { - store, - id, - name, - price: prices[0].price, - priceHistory: prices, - isWeighted, - unit, - quantity, - bio, - url, - }; - } - return items; -} +const { loadItems } = require("./loader"); class Items extends Model { constructor() { @@ -163,103 +27,25 @@ class Items extends Model { return this._lookup; } - processItems(items) { - const lookup = new Set(); - const start = performance.now(); - for (const item of items) { - lookup[item.store + item.id] = item; - item.search = item.name + " " + item.quantity + " " + item.unit; - item.search = item.search.toLowerCase().replace(",", "."); - - const unitPriceFactor = item.unit == "g" || item.unit == "ml" ? 1000 : 1; - item.unitPrice = (item.price / item.quantity) * unitPriceFactor; - item.numPrices = item.priceHistory.length; - item.priceOldest = item.priceHistory[item.priceHistory.length - 1].price; - item.dateOldest = item.priceHistory[item.priceHistory.length - 1].date; - item.date = item.priceHistory[0].date; - let highestPriceBefore = -1; - let lowestPriceBefore = 100000; - for (let i = 0; i < item.priceHistory.length; i++) { - const price = item.priceHistory[i]; - price.unitPrice = (price.price / item.quantity) * unitPriceFactor; - if (i == 0) continue; - if (i < 10) { - item["price" + i] = price.price; - item["unitPrice" + i] = price.unitPrice; - item["date" + i] = price.date; - } - highestPriceBefore = Math.max(highestPriceBefore, price.price); - lowestPriceBefore = Math.min(lowestPriceBefore, price.price); - } - if (highestPriceBefore == -1) highestPriceBefore = item.price; - if (lowestPriceBefore == 100000) lowestPriceBefore = item.price; - item.highestBefore = highestPriceBefore; - item.lowestBefore = lowestPriceBefore; - } - - items.sort((a, b) => { - if (a.store < b.store) { - return -1; - } else if (a.store > b.store) { - return 1; - } - - if (a.name < b.name) { - return -1; - } else if (a.name > b.name) { - return 1; - } - - return 0; - }); - - log(`Items - processing ${items.length} items took ${deltaTime(start).toFixed(4)} secs`); - this._items = items; - this._lookup = lookup; - } - async load() { - let start = performance.now(); const settings = new Settings(); - const compressedItemsPerStore = []; - log(`Items - load using JSON: ${settings.useJson}`); - for (const store of STORE_KEYS) { - compressedItemsPerStore.push( - new Promise(async (resolve) => { - let start = performance.now(); - try { - const useJSON = settings.useJson; - if (useJSON) { - const response = await fetch(`data/latest-canonical.${store}.compressed.json`); - const json = await response.json(); - log(`Items - loading compressed items for ${store} took ${deltaTime(start)} secs`); - start = performance.now(); - let items = decompress(json); - log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`); - resolve(items); - } else { - const response = await fetch(`data/latest-canonical.${store}.bin.json`); - const binary = await response.arrayBuffer(); - log(`Items - loading compressed binary items for ${store} took ${deltaTime(start)} secs`); - start = performance.now(); - let items = decompressBinary(binary); - log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`); - resolve(items); - } - } catch (e) { - log(`Items - error while loading compressed items for ${store} ${e.message}`); - resolve([]); - } - }) - ); + if (window.Worker && false) { + const self = this; + return new Promise((resolve, reject) => { + const loader = new Worker("loader.js"); + loader.onmessage = (event) => { + self._items = event.data.items; + self._lookup = event.data.lookup; + resolve(); + }; + loader.postMessage({ settings }); + }); + } else { + const { items, lookup } = await loadItems(settings); + this._items = items; + this._lookup = lookup; } - const items = [].concat(...(await Promise.all(compressedItemsPerStore))); - log(`Items - loaded ${items.length} items took ${deltaTime(start).toFixed(4)} secs`); - - this.processItems(items); - log(`Items - total loading took ${deltaTime(start).toFixed(4)} secs`); } } -exports.decompress = decompress; exports.Items = Items; diff --git a/site/model/loader.js b/site/model/loader.js new file mode 100644 index 0000000..bde19f5 --- /dev/null +++ b/site/model/loader.js @@ -0,0 +1,240 @@ +const { deltaTime, log, uint16ToDate } = require("../js/misc"); +const { stores, STORE_KEYS } = require("./stores"); + +function decompressBinary(buffer) { + const objects = []; + let offset = 0; + const view = new DataView(buffer); + const baseDate = new Date("2000-01-01"); + const textDecoder = new TextDecoder("utf-8"); + + const numStores = view.getUint8(offset++); + const stores = []; + for (let i = 0; i < numStores; i++) { + const nameLength = view.getUint16(offset, true); + offset += 2; + const nameBuffer = new Uint8Array(buffer, offset, nameLength); + stores.push(textDecoder.decode(nameBuffer)); + offset += nameLength; + } + + const numWords = view.getUint32(offset, true); + offset += 4; + const words = new Array(numWords); + for (let i = 0; i < numWords; i++) { + const nameLength = view.getUint8(offset++); + const nameBuffer = new Uint8Array(buffer, offset, nameLength); + words[i] = textDecoder.decode(nameBuffer); + offset += nameLength; + } + + while (offset < buffer.byteLength) { + const obj = {}; + const idLength = view.getUint8(offset++); + const idBuffer = new Uint8Array(buffer, offset, idLength); + obj.id = textDecoder.decode(idBuffer); + offset += idLength; + + const flagsByte = view.getUint8(offset++); + obj.bio = (flagsByte & 1) !== 0; + obj.isWeighted = (flagsByte & 2) !== 0; + if (flagsByte & 4) obj.unit = "ml"; + if (flagsByte & 8) obj.unit = "g"; + if (flagsByte & 16) obj.unit = "stk"; + if (flagsByte & 32) obj.unit = "cm"; + if (flagsByte & 64) obj.unit = "wg"; + + obj.quantity = view.getUint16(offset, true); + offset += 2; + + obj.store = stores[view.getUint8(offset++)]; + + let name = ""; + const numTokens = view.getUint8(offset++); + for (let i = 0; i < numTokens; i++) { + const b1 = view.getUint8(offset++); + const b2 = view.getUint8(offset++); + const b3 = view.getUint8(offset++); + const tokenId = (b3 << 16) | (b2 << 8) | b1; + name += words[tokenId]; + if (i < numTokens - 1) name += " "; + } + obj.name = name; + + const urlLength = view.getUint16(offset, true); + offset += 2; + if (urlLength !== 0) { + const urlBuffer = new Uint8Array(buffer, offset, urlLength); + obj.url = textDecoder.decode(urlBuffer); + } else { + obj.url = undefined; + } + offset += urlLength; + + const priceHistoryLength = view.getUint16(offset, true); + offset += 2; + obj.priceHistory = new Array(priceHistoryLength); + + for (let i = 0; i < priceHistoryLength; i++) { + const price = view.getUint16(offset, true) / 100; + offset += 2; + + const date = uint16ToDate(view.getUint16(offset, true)); + offset += 2; + + obj.priceHistory[i] = { date, price }; + } + + obj.price = obj.priceHistory[0].price; + + objects.push(obj); + } + + return objects; +} + +function decompress(compressedItems) { + const storeLookup = compressedItems.stores; + const data = compressedItems.data; + const dates = compressedItems.dates; + const numItems = compressedItems.n; + const items = new Array(numItems); + let i = 0; + for (let l = 0; l < numItems; l++) { + const store = storeLookup[data[i++]]; + const id = data[i++]; + const name = data[i++]; + const numPrices = data[i++]; + const prices = new Array(numPrices); + for (let j = 0; j < numPrices; j++) { + const date = dates[data[i++]]; + const price = data[i++]; + prices[j] = { + date: date.substring(0, 4) + "-" + date.substring(4, 6) + "-" + date.substring(6, 8), + price, + }; + } + const unit = data[i++]; + const quantity = data[i++]; + const isWeighted = data[i++] == 1; + const bio = data[i++] == 1; + const url = stores[store].getUrl({ id, name, url: data[i++] }); + + items[l] = { + store, + id, + name, + price: prices[0].price, + priceHistory: prices, + isWeighted, + unit, + quantity, + bio, + url, + }; + } + return items; +} + +function processItems(items) { + const lookup = {}; + const start = performance.now(); + for (const item of items) { + lookup[item.store + item.id] = item; + item.search = item.name + " " + item.quantity + " " + item.unit; + item.search = item.search.toLowerCase().replace(",", "."); + + const unitPriceFactor = item.unit == "g" || item.unit == "ml" ? 1000 : 1; + item.unitPrice = (item.price / item.quantity) * unitPriceFactor; + item.numPrices = item.priceHistory.length; + item.priceOldest = item.priceHistory[item.priceHistory.length - 1].price; + item.dateOldest = item.priceHistory[item.priceHistory.length - 1].date; + item.date = item.priceHistory[0].date; + let highestPriceBefore = -1; + let lowestPriceBefore = 100000; + for (let i = 0; i < item.priceHistory.length; i++) { + const price = item.priceHistory[i]; + price.unitPrice = (price.price / item.quantity) * unitPriceFactor; + if (i == 0) continue; + if (i < 10) { + item["price" + i] = price.price; + item["unitPrice" + i] = price.unitPrice; + item["date" + i] = price.date; + } + highestPriceBefore = Math.max(highestPriceBefore, price.price); + lowestPriceBefore = Math.min(lowestPriceBefore, price.price); + } + if (highestPriceBefore == -1) highestPriceBefore = item.price; + if (lowestPriceBefore == 100000) lowestPriceBefore = item.price; + item.highestBefore = highestPriceBefore; + item.lowestBefore = lowestPriceBefore; + } + + items.sort((a, b) => { + if (a.store < b.store) { + return -1; + } else if (a.store > b.store) { + return 1; + } + + if (a.name < b.name) { + return -1; + } else if (a.name > b.name) { + return 1; + } + + return 0; + }); + + log(`Items - processing ${items.length} items took ${deltaTime(start).toFixed(4)} secs`); + return { items, lookup }; +} + +exports.loadItems = async (settings) => { + let start = performance.now(); + const compressedItemsPerStore = []; + log(`Items - load using JSON: ${settings.useJson}`); + for (const store of STORE_KEYS) { + compressedItemsPerStore.push( + new Promise(async (resolve) => { + let start = performance.now(); + try { + const useJSON = settings.useJson; + if (useJSON) { + const response = await fetch(`data/latest-canonical.${store}.compressed.json`); + const json = await response.json(); + log(`Items - loading compressed items for ${store} took ${deltaTime(start)} secs`); + start = performance.now(); + let items = decompress(json); + log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`); + resolve(items); + } else { + const response = await fetch(`data/latest-canonical.${store}.bin.json`); + const binary = await response.arrayBuffer(); + log(`Items - loading compressed binary items for ${store} took ${deltaTime(start)} secs`); + start = performance.now(); + let items = decompressBinary(binary); + log(`Items - Decompressing items for ${store} took ${deltaTime(start)} secs`); + resolve(items); + } + } catch (e) { + log(`Items - error while loading compressed items for ${store} ${e.message}`); + resolve([]); + } + }) + ); + } + const items = [].concat(...(await Promise.all(compressedItemsPerStore))); + log(`Items - loaded ${items.length} items took ${deltaTime(start).toFixed(4)} secs`); + + const result = processItems(items); + log(`Items - total loading took ${deltaTime(start).toFixed(4)} secs`); + + return result; +}; + +onmessage = async (event) => { + const settings = event.data.settings; + const result = await exports.loadItems(settings); + postMessage(result); +};