mirror of
https://github.com/badlogic/heissepreise.git
synced 2024-09-22 00:00:59 +02:00
Improved compression, 4.8mb -> 4.4mb and faster decoding.
This commit is contained in:
parent
d63d42d623
commit
c9740b8660
106
analysis.js
106
analysis.js
|
@ -4,6 +4,7 @@ const zlib = require("zlib");
|
||||||
const stores = require("./stores");
|
const stores = require("./stores");
|
||||||
const { FILE } = require("dns");
|
const { FILE } = require("dns");
|
||||||
const { promisify } = require("util");
|
const { promisify } = require("util");
|
||||||
|
const { dateToUint16 } = require("./site/js/misc");
|
||||||
|
|
||||||
const STORE_KEYS = Object.keys(stores);
|
const STORE_KEYS = Object.keys(stores);
|
||||||
exports.STORE_KEYS = STORE_KEYS;
|
exports.STORE_KEYS = STORE_KEYS;
|
||||||
|
@ -159,7 +160,6 @@ function compressBinary(items) {
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
// Serialize 'bio', 'isWeighted', and 'unit' into a single byte
|
|
||||||
let flagsByte = 0;
|
let flagsByte = 0;
|
||||||
if (item.bio) flagsByte |= 1;
|
if (item.bio) flagsByte |= 1;
|
||||||
if (item.isWeighted) flagsByte |= 2;
|
if (item.isWeighted) flagsByte |= 2;
|
||||||
|
@ -167,27 +167,22 @@ function compressBinary(items) {
|
||||||
if (item.unit === "stk") flagsByte |= 8;
|
if (item.unit === "stk") flagsByte |= 8;
|
||||||
buffer.push(flagsByte);
|
buffer.push(flagsByte);
|
||||||
|
|
||||||
// Serialize 'quantity' as a 4-byte float
|
const quantityBuffer = Buffer.allocUnsafe(2);
|
||||||
const quantityBuffer = Buffer.allocUnsafe(4);
|
let quantity = Math.min(64000, item.quantity);
|
||||||
quantityBuffer.writeFloatLE(item.quantity, 0);
|
if (quantity > 64000) {
|
||||||
|
console.log(`Item quantity > 64000 ${item.id} - ${item.store} - ${item.name}`);
|
||||||
|
}
|
||||||
|
quantityBuffer.writeUint16LE(quantity, 0);
|
||||||
buffer.push(...quantityBuffer);
|
buffer.push(...quantityBuffer);
|
||||||
|
|
||||||
// Serialize 'price' as a 4-byte float
|
|
||||||
const priceBuffer = Buffer.allocUnsafe(4);
|
|
||||||
priceBuffer.writeFloatLE(item.price, 0);
|
|
||||||
buffer.push(...priceBuffer);
|
|
||||||
|
|
||||||
// Serialize 'store' as a byte
|
|
||||||
const storeByte = STORE_KEYS.findIndex((store) => store == item.store);
|
const storeByte = STORE_KEYS.findIndex((store) => store == item.store);
|
||||||
buffer.push(storeByte);
|
buffer.push(storeByte);
|
||||||
|
|
||||||
// Serialize 'name' as UTF-8 with 2 bytes encoding the string length
|
|
||||||
const nameBuffer = Buffer.from(item.name, "utf8");
|
const nameBuffer = Buffer.from(item.name, "utf8");
|
||||||
const nameLengthBuffer = Buffer.allocUnsafe(2);
|
const nameLengthBuffer = Buffer.allocUnsafe(2);
|
||||||
nameLengthBuffer.writeUInt16LE(nameBuffer.length, 0);
|
nameLengthBuffer.writeUInt16LE(nameBuffer.length, 0);
|
||||||
buffer.push(...nameLengthBuffer, ...nameBuffer);
|
buffer.push(...nameLengthBuffer, ...nameBuffer);
|
||||||
|
|
||||||
// Serialize 'url' as UTF-8 with 2 bytes encoding the string length
|
|
||||||
if (item.url !== undefined) {
|
if (item.url !== undefined) {
|
||||||
const urlBuffer = Buffer.from(item.url, "utf8");
|
const urlBuffer = Buffer.from(item.url, "utf8");
|
||||||
const urlLengthBuffer = Buffer.allocUnsafe(2);
|
const urlLengthBuffer = Buffer.allocUnsafe(2);
|
||||||
|
@ -198,26 +193,24 @@ function compressBinary(items) {
|
||||||
buffer.push(...urlLengthBuffer);
|
buffer.push(...urlLengthBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Serialize 'priceHistory' array
|
|
||||||
const priceHistoryLengthBuffer = Buffer.allocUnsafe(2);
|
const priceHistoryLengthBuffer = Buffer.allocUnsafe(2);
|
||||||
priceHistoryLengthBuffer.writeUInt16LE(item.priceHistory.length, 0);
|
priceHistoryLengthBuffer.writeUInt16LE(item.priceHistory.length, 0);
|
||||||
buffer.push(...priceHistoryLengthBuffer);
|
buffer.push(...priceHistoryLengthBuffer);
|
||||||
|
|
||||||
for (const priceEntry of item.priceHistory) {
|
for (const priceEntry of item.priceHistory) {
|
||||||
// Serialize price as a 4-byte float
|
const priceEntryBuffer = Buffer.allocUnsafe(2);
|
||||||
const priceEntryBuffer = Buffer.allocUnsafe(4);
|
if (priceEntry.price == 999) priceEntry.price = 9.99;
|
||||||
priceEntryBuffer.writeFloatLE(priceEntry.price, 0);
|
let price = Math.round(priceEntry.price * 100);
|
||||||
|
if (price > 64000) {
|
||||||
|
console.log(`Item price > 64000 ${item.id} - ${item.store} - ${item.name}`);
|
||||||
|
price = 64000;
|
||||||
|
}
|
||||||
|
priceEntryBuffer.writeUint16LE(price, 0);
|
||||||
buffer.push(...priceEntryBuffer);
|
buffer.push(...priceEntryBuffer);
|
||||||
|
|
||||||
// Calculate the days since 2000-01-01
|
const dateBuffer = Buffer.allocUnsafe(2);
|
||||||
const entryDate = new Date(priceEntry.date);
|
dateBuffer.writeUint16LE(dateToUint16(priceEntry.date), 0);
|
||||||
const baseDate = new Date("2000-01-01");
|
buffer.push(...dateBuffer);
|
||||||
const daysSince2000 = Math.floor((entryDate - baseDate) / (1000 * 60 * 60 * 24));
|
|
||||||
|
|
||||||
// Serialize days as a 32-bit integer
|
|
||||||
const daysBuffer = Buffer.allocUnsafe(4);
|
|
||||||
daysBuffer.writeInt32LE(daysSince2000, 0);
|
|
||||||
buffer.push(...daysBuffer);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -225,69 +218,6 @@ function compressBinary(items) {
|
||||||
}
|
}
|
||||||
exports.compressBinary = compressBinary;
|
exports.compressBinary = compressBinary;
|
||||||
|
|
||||||
function decompressBinary(buffer) {
|
|
||||||
const objects = [];
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
while (offset < buffer.length) {
|
|
||||||
const obj = {};
|
|
||||||
|
|
||||||
// Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
|
|
||||||
const flagsByte = buffer[offset++];
|
|
||||||
obj.bio = (flagsByte & 1) !== 0;
|
|
||||||
obj.isWeighted = (flagsByte & 2) !== 0;
|
|
||||||
obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
|
|
||||||
|
|
||||||
// Deserialize 'quantity' as a 4-byte float
|
|
||||||
obj.quantity = buffer.readFloatLE(offset);
|
|
||||||
offset += 4;
|
|
||||||
|
|
||||||
// Deserialize 'price' as a 4-byte float
|
|
||||||
obj.price = buffer.readFloatLE(offset);
|
|
||||||
offset += 4;
|
|
||||||
|
|
||||||
// Deserialize 'store' as a byte
|
|
||||||
obj.store = STORE_KEYS[buffer[offset++]];
|
|
||||||
|
|
||||||
// Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
|
|
||||||
const nameLength = buffer.readUInt16LE(offset);
|
|
||||||
offset += 2;
|
|
||||||
obj.name = buffer.toString("utf8", offset, offset + nameLength);
|
|
||||||
offset += nameLength;
|
|
||||||
|
|
||||||
// Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
|
|
||||||
const urlLength = buffer.readUInt16LE(offset);
|
|
||||||
offset += 2;
|
|
||||||
obj.url = urlLength !== 0 ? buffer.toString("utf8", offset, offset + urlLength) : undefined;
|
|
||||||
offset += urlLength;
|
|
||||||
|
|
||||||
// Deserialize 'priceHistory' array
|
|
||||||
const priceHistoryLength = buffer.readUInt16LE(offset);
|
|
||||||
offset += 2;
|
|
||||||
obj.priceHistory = [];
|
|
||||||
|
|
||||||
for (let i = 0; i < priceHistoryLength; i++) {
|
|
||||||
// Deserialize price as a 4-byte float
|
|
||||||
const price = buffer.readFloatLE(offset);
|
|
||||||
offset += 4;
|
|
||||||
|
|
||||||
// Deserialize days as a 32-bit integer
|
|
||||||
const daysSince2000 = buffer.readInt32LE(offset);
|
|
||||||
offset += 4;
|
|
||||||
|
|
||||||
// Calculate the date from days since 2000-01-01
|
|
||||||
const baseDate = new Date("2000-01-01");
|
|
||||||
const entryDate = new Date(baseDate.getTime() + daysSince2000 * 24 * 60 * 60 * 1000);
|
|
||||||
|
|
||||||
obj.priceHistory.push({ date: entryDate.toISOString().substring(0, 10), price });
|
|
||||||
}
|
|
||||||
|
|
||||||
objects.push(obj);
|
|
||||||
}
|
|
||||||
|
|
||||||
return objects;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keep this in sync with utils.js:decompress
|
// Keep this in sync with utils.js:decompress
|
||||||
function compress(items) {
|
function compress(items) {
|
||||||
const compressed = {
|
const compressed = {
|
||||||
|
|
|
@ -63,44 +63,16 @@ exports.today = () => {
|
||||||
return `${year}-${month}-${day}`;
|
return `${year}-${month}-${day}`;
|
||||||
};
|
};
|
||||||
|
|
||||||
exports.isoDate = (daysSince2000) => {
|
exports.dateToUint16 = (dateString) => {
|
||||||
// Number of days in each month (non-leap year)
|
const [year, month, day] = dateString.split("-").map(Number);
|
||||||
const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
return (((year - 2000) << 9) | (month << 5) | day) & 0xffff;
|
||||||
|
};
|
||||||
|
|
||||||
// Number of days in each month (leap year)
|
exports.uint16ToDate = (encodedDate) => {
|
||||||
const daysInMonthLeap = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
const year = (encodedDate >> 9) + 2000;
|
||||||
|
const month = (encodedDate >> 5) & 0xf;
|
||||||
// Start date: January 1, 2000
|
const day = encodedDate & 0x1f;
|
||||||
const startYear = 2000;
|
return `${year}-${month.toString().padStart(2, "0")}-${day.toString().padStart(2, "0")}`;
|
||||||
const startMonth = 1;
|
|
||||||
const startDay = 1;
|
|
||||||
|
|
||||||
// Calculate the number of leap years
|
|
||||||
const numLeapYears = Math.floor(daysSince2000 / 365) - Math.floor((startYear - 2000) / 4);
|
|
||||||
|
|
||||||
// Calculate the number of non-leap years
|
|
||||||
const numNonLeapYears = Math.floor(daysSince2000 / 365) - numLeapYears;
|
|
||||||
|
|
||||||
// Calculate the number of days remaining after accounting for years
|
|
||||||
const remainingDays = daysSince2000 - (numLeapYears * 366 + numNonLeapYears * 365);
|
|
||||||
|
|
||||||
// Determine the current year
|
|
||||||
const currentYear = startYear + numLeapYears * 4 + numNonLeapYears;
|
|
||||||
const isLeapYear = currentYear % 4 === 0 && (currentYear % 100 !== 0 || currentYear % 400 === 0);
|
|
||||||
const daysInMonthArray = isLeapYear ? daysInMonthLeap : daysInMonth;
|
|
||||||
|
|
||||||
// Determine the current month and day
|
|
||||||
let currentMonth = 1;
|
|
||||||
let currentDay = remainingDays + 1;
|
|
||||||
for (const days of daysInMonthArray) {
|
|
||||||
if (currentDay <= days) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
currentMonth++;
|
|
||||||
currentDay -= days;
|
|
||||||
}
|
|
||||||
|
|
||||||
return `${currentYear}-${currentMonth.toString().padStart(2, "0")}-${currentDay.toString().padStart(2, "0")}`;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
exports.fetchJSON = async (url) => {
|
exports.fetchJSON = async (url) => {
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
const { deltaTime, log, isoDate } = require("../js/misc");
|
const { deltaTime, log, uint16ToDate } = require("../js/misc");
|
||||||
const { stores, STORE_KEYS } = require("./stores");
|
const { stores, STORE_KEYS } = require("./stores");
|
||||||
const { Model } = require("./model");
|
const { Model } = require("./model");
|
||||||
const { Settings } = require("./settings");
|
const { Settings } = require("./settings");
|
||||||
|
@ -23,31 +23,22 @@ function decompressBinary(buffer) {
|
||||||
while (offset < buffer.byteLength) {
|
while (offset < buffer.byteLength) {
|
||||||
const obj = {};
|
const obj = {};
|
||||||
|
|
||||||
// Deserialize 'bio', 'isWeighted', and 'unit' from the single byte
|
|
||||||
const flagsByte = view.getUint8(offset++);
|
const flagsByte = view.getUint8(offset++);
|
||||||
obj.bio = (flagsByte & 1) !== 0;
|
obj.bio = (flagsByte & 1) !== 0;
|
||||||
obj.isWeighted = (flagsByte & 2) !== 0;
|
obj.isWeighted = (flagsByte & 2) !== 0;
|
||||||
obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
|
obj.unit = (flagsByte & 4) !== 0 ? "ml" : (flagsByte & 8) !== 0 ? "stk" : "g";
|
||||||
|
|
||||||
// Deserialize 'quantity' as a 4-byte float
|
obj.quantity = view.getUint16(offset, true);
|
||||||
obj.quantity = view.getFloat32(offset, true);
|
offset += 2;
|
||||||
offset += 4;
|
|
||||||
|
|
||||||
// Deserialize 'price' as a 4-byte float
|
|
||||||
obj.price = view.getFloat32(offset, true);
|
|
||||||
offset += 4;
|
|
||||||
|
|
||||||
// Deserialize 'store' as a byte
|
|
||||||
obj.store = stores[view.getUint8(offset++)];
|
obj.store = stores[view.getUint8(offset++)];
|
||||||
|
|
||||||
// Deserialize 'name' as UTF-8 with 2 bytes encoding the string length
|
|
||||||
const nameLength = view.getUint16(offset, true);
|
const nameLength = view.getUint16(offset, true);
|
||||||
offset += 2;
|
offset += 2;
|
||||||
const nameBuffer = new Uint8Array(buffer, offset, nameLength);
|
const nameBuffer = new Uint8Array(buffer, offset, nameLength);
|
||||||
obj.name = textDecoder.decode(nameBuffer);
|
obj.name = textDecoder.decode(nameBuffer);
|
||||||
offset += nameLength;
|
offset += nameLength;
|
||||||
|
|
||||||
// Deserialize 'url' as UTF-8 with 2 bytes encoding the string length (or undefined if length is 0)
|
|
||||||
const urlLength = view.getUint16(offset, true);
|
const urlLength = view.getUint16(offset, true);
|
||||||
offset += 2;
|
offset += 2;
|
||||||
if (urlLength !== 0) {
|
if (urlLength !== 0) {
|
||||||
|
@ -58,25 +49,22 @@ function decompressBinary(buffer) {
|
||||||
}
|
}
|
||||||
offset += urlLength;
|
offset += urlLength;
|
||||||
|
|
||||||
// Deserialize 'priceHistory' array
|
|
||||||
const priceHistoryLength = view.getUint16(offset, true);
|
const priceHistoryLength = view.getUint16(offset, true);
|
||||||
offset += 2;
|
offset += 2;
|
||||||
obj.priceHistory = new Array(priceHistoryLength);
|
obj.priceHistory = new Array(priceHistoryLength);
|
||||||
|
|
||||||
for (let i = 0; i < priceHistoryLength; i++) {
|
for (let i = 0; i < priceHistoryLength; i++) {
|
||||||
// Deserialize price as a 4-byte float
|
const price = view.getUint16(offset, true) / 100;
|
||||||
const price = view.getFloat32(offset, true);
|
offset += 2;
|
||||||
offset += 4;
|
|
||||||
|
|
||||||
// Deserialize days as a 32-bit integer
|
const date = uint16ToDate(view.getUint16(offset, true));
|
||||||
const daysSince2000 = view.getInt32(offset, true);
|
offset += 2;
|
||||||
offset += 4;
|
|
||||||
// Calculate the date from days since 2000-01-01
|
|
||||||
const dateStr = isoDate(daysSince2000);
|
|
||||||
|
|
||||||
obj.priceHistory[i] = { date: dateStr, price };
|
obj.priceHistory[i] = { date, price };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
obj.price = obj.priceHistory[0].price;
|
||||||
|
|
||||||
objects.push(obj);
|
objects.push(obj);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user