Improved compression, 4.8mb -> 4.4mb and faster decoding.

analysis.js

@@ -4,6 +4,7 @@ const zlib = require("zlib");
 const stores = require("./stores");
 const { FILE } = require("dns");
 const { promisify } = require("util");
+const { dateToUint16 } = require("./site/js/misc");
 
 const STORE_KEYS = Object.keys(stores);
 exports.STORE_KEYS = STORE_KEYS;
@@ -159,7 +160,6 @@ function compressBinary(items) {
     }
 
     for (const item of items) {
-        // Serialize 'bio', 'isWeighted', and 'unit' into a single byte
         let flagsByte = 0;
         if (item.bio) flagsByte |= 1;
         if (item.isWeighted) flagsByte |= 2;
@@ -167,27 +167,22 @@ function compressBinary(items) {
         if (item.unit === "stk") flagsByte |= 8;
         buffer.push(flagsByte);
 
-        // Serialize 'quantity' as a 4-byte float
+        const quantityBuffer = Buffer.allocUnsafe(2);
-        const quantityBuffer = Buffer.allocUnsafe(4);
+        let quantity = Math.min(64000, item.quantity);
-        quantityBuffer.writeFloatLE(item.quantity, 0);
+        if (quantity > 64000) {
+            console.log(`Item quantity > 64000 ${item.id} - ${item.store} - ${item.name}`);
+        }
+        quantityBuffer.writeUint16LE(quantity, 0);
         buffer.push(...quantityBuffer);
 
-        // Serialize 'price' as a 4-byte float
-        const priceBuffer = Buffer.allocUnsafe(4);
-        priceBuffer.writeFloatLE(item.price, 0);
-        buffer.push(...priceBuffer);
-
-        // Serialize 'store' as a byte
         const storeByte = STORE_KEYS.findIndex((store) => store == item.store);
         buffer.push(storeByte);
 
-        // Serialize 'name' as UTF-8 with 2 bytes encoding the string length
         const nameBuffer = Buffer.from(item.name, "utf8");
         const nameLengthBuffer = Buffer.allocUnsafe(2);
         nameLengthBuffer.writeUInt16LE(nameBuffer.length, 0);
         buffer.push(...nameLengthBuffer, ...nameBuffer);
 
-        // Serialize 'url' as UTF-8 with 2 bytes encoding the string length
         if (item.url !== undefined) {
             const urlBuffer = Buffer.from(item.url, "utf8");
             const urlLengthBuffer = Buffer.allocUnsafe(2);
@@ -198,26 +193,24 @@ function compressBinary(items) {
             buffer.push(...urlLengthBuffer);
         }
 
-        // Serialize 'priceHistory' array
         const priceHistoryLengthBuffer = Buffer.allocUnsafe(2);
         priceHistoryLengthBuffer.writeUInt16LE(item.priceHistory.length, 0);
         buffer.push(...priceHistoryLengthBuffer);
 
         for (const priceEntry of item.priceHistory) {
-            // Serialize price as a 4-byte float
+            const priceEntryBuffer = Buffer.allocUnsafe(2);
-            const priceEntryBuffer = Buffer.allocUnsafe(4);
+            if (priceEntry.price == 999) priceEntry.price = 9.99;
-            priceEntryBuffer.writeFloatLE(priceEntry.price, 0);
+            let price = Math.round(priceEntry.price * 100);
+            if (price > 64000) {
+                console.log(`Item price > 64000 ${item.id} - ${item.store} - ${item.name}`);
+                price = 64000;
+            }
+            priceEntryBuffer.writeUint16LE(price, 0);
             buffer.push(...priceEntryBuffer);
 
-            // Calculate the days since 2000-01-01
+            const dateBuffer = Buffer.allocUnsafe(2);
-            const entryDate = new Date(priceEntry.date);
+            dateBuffer.writeUint16LE(dateToUint16(priceEntry.date), 0);
-            const baseDate = new Date("2000-01-01");
+            buffer.push(...dateBuffer);
-            const daysSince2000 = Math.floor((entryDate - baseDate) / (1000 * 60 * 60 * 24));
-
-            // Serialize days as a 32-bit integer
-            const daysBuffer = Buffer.allocUnsafe(4);
-            daysBuffer.writeInt32LE(daysSince2000, 0);
-            buffer.push(...daysBuffer);
         }
     }
 
@@ -225,69 +218,6 @@ function compressBinary(items) {
 }
 exports.compressBinary = compressBinary;
 
-function decompressBinary(buffer) {
-    const objects = [];
-    let offset = 0;
-
-    while (offset < buffer.length) {
-        const obj = {};
-
-        // Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
-        const flagsByte = buffer[offset++];
-        obj.bio = (flagsByte & 1) !== 0;
-        obj.isWeighted = (flagsByte & 2) !== 0;
-        obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
-
-        // Deserialize 'quantity' as a 4-byte float
-        obj.quantity = buffer.readFloatLE(offset);
-        offset += 4;
-
-        // Deserialize 'price' as a 4-byte float
-        obj.price = buffer.readFloatLE(offset);
-        offset += 4;
-
-        // Deserialize 'store' as a byte
-        obj.store = STORE_KEYS[buffer[offset++]];
-
-        // Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
-        const nameLength = buffer.readUInt16LE(offset);
-        offset += 2;
-        obj.name = buffer.toString("utf8", offset, offset + nameLength);
-        offset += nameLength;
-
-        // Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
-        const urlLength = buffer.readUInt16LE(offset);
-        offset += 2;
-        obj.url = urlLength !== 0 ? buffer.toString("utf8", offset, offset + urlLength) : undefined;
-        offset += urlLength;
-
-        // Deserialize 'priceHistory' array
-        const priceHistoryLength = buffer.readUInt16LE(offset);
-        offset += 2;
-        obj.priceHistory = [];
-
-        for (let i = 0; i < priceHistoryLength; i++) {
-            // Deserialize price as a 4-byte float
-            const price = buffer.readFloatLE(offset);
-            offset += 4;
-
-            // Deserialize days as a 32-bit integer
-            const daysSince2000 = buffer.readInt32LE(offset);
-            offset += 4;
-
-            // Calculate the date from days since 2000-01-01
-            const baseDate = new Date("2000-01-01");
-            const entryDate = new Date(baseDate.getTime() + daysSince2000 * 24 * 60 * 60 * 1000);
-
-            obj.priceHistory.push({ date: entryDate.toISOString().substring(0, 10), price });
-        }
-
-        objects.push(obj);
-    }
-
-    return objects;
-}
-
 // Keep this in sync with utils.js:decompress
 function compress(items) {
     const compressed = {

site/js/misc.js

@@ -63,44 +63,16 @@ exports.today = () => {
     return `${year}-${month}-${day}`;
 };
 
-exports.isoDate = (daysSince2000) => {
+exports.dateToUint16 = (dateString) => {
-    // Number of days in each month (non-leap year)
+    const [year, month, day] = dateString.split("-").map(Number);
-    const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
+    return (((year - 2000) << 9) | (month << 5) | day) & 0xffff;
+};
 
-    // Number of days in each month (leap year)
+exports.uint16ToDate = (encodedDate) => {
-    const daysInMonthLeap = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
+    const year = (encodedDate >> 9) + 2000;
-
+    const month = (encodedDate >> 5) & 0xf;
-    // Start date: January 1, 2000
+    const day = encodedDate & 0x1f;
-    const startYear = 2000;
+    return `${year}-${month.toString().padStart(2, "0")}-${day.toString().padStart(2, "0")}`;
-    const startMonth = 1;
-    const startDay = 1;
-
-    // Calculate the number of leap years
-    const numLeapYears = Math.floor(daysSince2000 / 365) - Math.floor((startYear - 2000) / 4);
-
-    // Calculate the number of non-leap years
-    const numNonLeapYears = Math.floor(daysSince2000 / 365) - numLeapYears;
-
-    // Calculate the number of days remaining after accounting for years
-    const remainingDays = daysSince2000 - (numLeapYears * 366 + numNonLeapYears * 365);
-
-    // Determine the current year
-    const currentYear = startYear + numLeapYears * 4 + numNonLeapYears;
-    const isLeapYear = currentYear % 4 === 0 && (currentYear % 100 !== 0 || currentYear % 400 === 0);
-    const daysInMonthArray = isLeapYear ? daysInMonthLeap : daysInMonth;
-
-    // Determine the current month and day
-    let currentMonth = 1;
-    let currentDay = remainingDays + 1;
-    for (const days of daysInMonthArray) {
-        if (currentDay <= days) {
-            break;
-        }
-        currentMonth++;
-        currentDay -= days;
-    }
-
-    return `${currentYear}-${currentMonth.toString().padStart(2, "0")}-${currentDay.toString().padStart(2, "0")}`;
 };
 
 exports.fetchJSON = async (url) => {

site/model/items.js

@@ -1,4 +1,4 @@
-const { deltaTime, log, isoDate } = require("../js/misc");
+const { deltaTime, log, uint16ToDate } = require("../js/misc");
 const { stores, STORE_KEYS } = require("./stores");
 const { Model } = require("./model");
 const { Settings } = require("./settings");
@@ -23,31 +23,22 @@ function decompressBinary(buffer) {
     while (offset < buffer.byteLength) {
         const obj = {};
 
-        // Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
         const flagsByte = view.getUint8(offset++);
         obj.bio = (flagsByte & 1) !== 0;
         obj.isWeighted = (flagsByte & 2) !== 0;
         obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
 
-        // Deserialize 'quantity' as a 4-byte float
+        obj.quantity = view.getUint16(offset, true);
-        obj.quantity = view.getFloat32(offset, true);
+        offset += 2;
-        offset += 4;
 
-        // Deserialize 'price' as a 4-byte float
-        obj.price = view.getFloat32(offset, true);
-        offset += 4;
-
-        // Deserialize 'store' as a byte
         obj.store = stores[view.getUint8(offset++)];
 
-        // Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
         const nameLength = view.getUint16(offset, true);
         offset += 2;
         const nameBuffer = new Uint8Array(buffer, offset, nameLength);
         obj.name = textDecoder.decode(nameBuffer);
         offset += nameLength;
 
-        // Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
         const urlLength = view.getUint16(offset, true);
         offset += 2;
         if (urlLength !== 0) {
@@ -58,25 +49,22 @@ function decompressBinary(buffer) {
         }
         offset += urlLength;
 
-        // Deserialize 'priceHistory' array
         const priceHistoryLength = view.getUint16(offset, true);
         offset += 2;
         obj.priceHistory = new Array(priceHistoryLength);
 
         for (let i = 0; i < priceHistoryLength; i++) {
-            // Deserialize price as a 4-byte float
+            const price = view.getUint16(offset, true) / 100;
-            const price = view.getFloat32(offset, true);
+            offset += 2;
-            offset += 4;
 
-            // Deserialize days as a 32-bit integer
+            const date = uint16ToDate(view.getUint16(offset, true));
-            const daysSince2000 = view.getInt32(offset, true);
+            offset += 2;
-            offset += 4;
-            // Calculate the date from days since 2000-01-01
-            const dateStr = isoDate(daysSince2000);
 
-            obj.priceHistory[i] = { date: dateStr, price };
+            obj.priceHistory[i] = { date, price };
         }
 
+        obj.price = obj.priceHistory[0].price;
+
         objects.push(obj);
     }