Marown/heissepreise
1
0
Fork 0
mirror of https://github.com/badlogic/heissepreise.git synced 2024-09-22 00:00:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

Improved compression, 4.8mb -> 4.4mb and faster decoding.

Browse Source
This commit is contained in:
Mario Zechner 2023-06-16 23:12:37 +02:00
parent d63d42d623
commit c9740b8660
3 changed files with 37 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
analysis.js
View File

@ -4,6 +4,7 @@ const zlib = require("zlib");
const stores = require("./stores");
const { FILE } = require("dns");
const { promisify } = require("util");
const { dateToUint16 } = require("./site/js/misc");
const STORE_KEYS = Object.keys(stores);
exports.STORE_KEYS = STORE_KEYS;
@ -159,7 +160,6 @@ function compressBinary(items) {
}
for (const item of items) {
// Serialize 'bio', 'isWeighted', and 'unit' into a single byte
let flagsByte = 0;
if (item.bio) flagsByte |= 1;
if (item.isWeighted) flagsByte |= 2;
@ -167,27 +167,22 @@ function compressBinary(items) {
if (item.unit === "stk") flagsByte |= 8;
buffer.push(flagsByte);
// Serialize 'quantity' as a 4-byte float
const quantityBuffer = Buffer.allocUnsafe(4);
quantityBuffer.writeFloatLE(item.quantity, 0);
const quantityBuffer = Buffer.allocUnsafe(2);
let quantity = Math.min(64000, item.quantity);
if (quantity > 64000) {
console.log(`Item quantity > 64000 ${item.id} - ${item.store} - ${item.name}`);
}
quantityBuffer.writeUint16LE(quantity, 0);
buffer.push(...quantityBuffer);
// Serialize 'price' as a 4-byte float
const priceBuffer = Buffer.allocUnsafe(4);
priceBuffer.writeFloatLE(item.price, 0);
buffer.push(...priceBuffer);
// Serialize 'store' as a byte
const storeByte = STORE_KEYS.findIndex((store) => store == item.store);
buffer.push(storeByte);
// Serialize 'name' as UTF-8 with 2 bytes encoding the string length
const nameBuffer = Buffer.from(item.name, "utf8");
const nameLengthBuffer = Buffer.allocUnsafe(2);
nameLengthBuffer.writeUInt16LE(nameBuffer.length, 0);
buffer.push(...nameLengthBuffer, ...nameBuffer);
// Serialize 'url' as UTF-8 with 2 bytes encoding the string length
if (item.url !== undefined) {
const urlBuffer = Buffer.from(item.url, "utf8");
const urlLengthBuffer = Buffer.allocUnsafe(2);
@ -198,26 +193,24 @@ function compressBinary(items) {
buffer.push(...urlLengthBuffer);
}
// Serialize 'priceHistory' array
const priceHistoryLengthBuffer = Buffer.allocUnsafe(2);
priceHistoryLengthBuffer.writeUInt16LE(item.priceHistory.length, 0);
buffer.push(...priceHistoryLengthBuffer);
for (const priceEntry of item.priceHistory) {
// Serialize price as a 4-byte float
const priceEntryBuffer = Buffer.allocUnsafe(4);
priceEntryBuffer.writeFloatLE(priceEntry.price, 0);
const priceEntryBuffer = Buffer.allocUnsafe(2);
if (priceEntry.price == 999) priceEntry.price = 9.99;
let price = Math.round(priceEntry.price * 100);
if (price > 64000) {
console.log(`Item price > 64000 ${item.id} - ${item.store} - ${item.name}`);
price = 64000;
}
priceEntryBuffer.writeUint16LE(price, 0);
buffer.push(...priceEntryBuffer);
// Calculate the days since 2000-01-01
const entryDate = new Date(priceEntry.date);
const baseDate = new Date("2000-01-01");
const daysSince2000 = Math.floor((entryDate - baseDate) / (1000 * 60 * 60 * 24));
// Serialize days as a 32-bit integer
const daysBuffer = Buffer.allocUnsafe(4);
daysBuffer.writeInt32LE(daysSince2000, 0);
buffer.push(...daysBuffer);
const dateBuffer = Buffer.allocUnsafe(2);
dateBuffer.writeUint16LE(dateToUint16(priceEntry.date), 0);
buffer.push(...dateBuffer);
}
}
@ -225,69 +218,6 @@ function compressBinary(items) {
}
exports.compressBinary = compressBinary;
function decompressBinary(buffer) {
const objects = [];
let offset = 0;
while (offset < buffer.length) {
const obj = {};
// Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
const flagsByte = buffer[offset++];
obj.bio = (flagsByte & 1) !== 0;
obj.isWeighted = (flagsByte & 2) !== 0;
obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
// Deserialize 'quantity' as a 4-byte float
obj.quantity = buffer.readFloatLE(offset);
offset += 4;
// Deserialize 'price' as a 4-byte float
obj.price = buffer.readFloatLE(offset);
offset += 4;
// Deserialize 'store' as a byte
obj.store = STORE_KEYS[buffer[offset++]];
// Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
const nameLength = buffer.readUInt16LE(offset);
offset += 2;
obj.name = buffer.toString("utf8", offset, offset + nameLength);
offset += nameLength;
// Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
const urlLength = buffer.readUInt16LE(offset);
offset += 2;
obj.url = urlLength !== 0 ? buffer.toString("utf8", offset, offset + urlLength) : undefined;
offset += urlLength;
// Deserialize 'priceHistory' array
const priceHistoryLength = buffer.readUInt16LE(offset);
offset += 2;
obj.priceHistory = [];
for (let i = 0; i < priceHistoryLength; i++) {
// Deserialize price as a 4-byte float
const price = buffer.readFloatLE(offset);
offset += 4;
// Deserialize days as a 32-bit integer
const daysSince2000 = buffer.readInt32LE(offset);
offset += 4;
// Calculate the date from days since 2000-01-01
const baseDate = new Date("2000-01-01");
const entryDate = new Date(baseDate.getTime() + daysSince2000 * 24 * 60 * 60 * 1000);
obj.priceHistory.push({ date: entryDate.toISOString().substring(0, 10), price });
}
objects.push(obj);
}
return objects;
}
// Keep this in sync with utils.js:decompress
function compress(items) {
const compressed = {

46
site/js/misc.js
View File

@ -63,44 +63,16 @@ exports.today = () => {
return `${year}-${month}-${day}`;
};
exports.isoDate = (daysSince2000) => {
// Number of days in each month (non-leap year)
const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
exports.dateToUint16 = (dateString) => {
const [year, month, day] = dateString.split("-").map(Number);
return (((year - 2000) << 9) | (month << 5) | day) & 0xffff;
};
// Number of days in each month (leap year)
const daysInMonthLeap = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
// Start date: January 1, 2000
const startYear = 2000;
const startMonth = 1;
const startDay = 1;
// Calculate the number of leap years
const numLeapYears = Math.floor(daysSince2000 / 365) - Math.floor((startYear - 2000) / 4);
// Calculate the number of non-leap years
const numNonLeapYears = Math.floor(daysSince2000 / 365) - numLeapYears;
// Calculate the number of days remaining after accounting for years
const remainingDays = daysSince2000 - (numLeapYears * 366 + numNonLeapYears * 365);
// Determine the current year
const currentYear = startYear + numLeapYears * 4 + numNonLeapYears;
const isLeapYear = currentYear % 4 === 0 && (currentYear % 100 !== 0 || currentYear % 400 === 0);
const daysInMonthArray = isLeapYear ? daysInMonthLeap : daysInMonth;
// Determine the current month and day
let currentMonth = 1;
let currentDay = remainingDays + 1;
for (const days of daysInMonthArray) {
if (currentDay <= days) {
break;
}
currentMonth++;
currentDay -= days;
}
return `${currentYear}-${currentMonth.toString().padStart(2, "0")}-${currentDay.toString().padStart(2, "0")}`;
exports.uint16ToDate = (encodedDate) => {
const year = (encodedDate >> 9) + 2000;
const month = (encodedDate >> 5) & 0xf;
const day = encodedDate & 0x1f;
return `${year}-${month.toString().padStart(2, "0")}-${day.toString().padStart(2, "0")}`;
};
exports.fetchJSON = async (url) => {

32
site/model/items.js
View File

@ -1,4 +1,4 @@
const { deltaTime, log, isoDate } = require("../js/misc");
const { deltaTime, log, uint16ToDate } = require("../js/misc");
const { stores, STORE_KEYS } = require("./stores");
const { Model } = require("./model");
const { Settings } = require("./settings");
@ -23,31 +23,22 @@ function decompressBinary(buffer) {
while (offset < buffer.byteLength) {
const obj = {};
// Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
const flagsByte = view.getUint8(offset++);
obj.bio = (flagsByte & 1) !== 0;
obj.isWeighted = (flagsByte & 2) !== 0;
obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
// Deserialize 'quantity' as a 4-byte float
obj.quantity = view.getFloat32(offset, true);
offset += 4;
obj.quantity = view.getUint16(offset, true);
offset += 2;
// Deserialize 'price' as a 4-byte float
obj.price = view.getFloat32(offset, true);
offset += 4;
// Deserialize 'store' as a byte
obj.store = stores[view.getUint8(offset++)];
// Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
const nameLength = view.getUint16(offset, true);
offset += 2;
const nameBuffer = new Uint8Array(buffer, offset, nameLength);
obj.name = textDecoder.decode(nameBuffer);
offset += nameLength;
// Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
const urlLength = view.getUint16(offset, true);
offset += 2;
if (urlLength !== 0) {
@ -58,25 +49,22 @@ function decompressBinary(buffer) {
}
offset += urlLength;
// Deserialize 'priceHistory' array
const priceHistoryLength = view.getUint16(offset, true);
offset += 2;
obj.priceHistory = new Array(priceHistoryLength);
for (let i = 0; i < priceHistoryLength; i++) {
// Deserialize price as a 4-byte float
const price = view.getFloat32(offset, true);
offset += 4;
const price = view.getUint16(offset, true) / 100;
offset += 2;
// Deserialize days as a 32-bit integer
const daysSince2000 = view.getInt32(offset, true);
offset += 4;
// Calculate the date from days since 2000-01-01
const dateStr = isoDate(daysSince2000);
const date = uint16ToDate(view.getUint16(offset, true));
offset += 2;
obj.priceHistory[i] = { date: dateStr, price };
obj.priceHistory[i] = { date, price };
}
obj.price = obj.priceHistory[0].price;
objects.push(obj);
}