mirror of
https://github.com/badlogic/heissepreise.git
synced 2024-06-22 08:25:52 +02:00
Server-side generation with watcher and live-reload. pages.js still broken.
This commit is contained in:
Mario Zechner
parent
d357caf7f1
commit
2ef5be9e9b
210
package-lock.json
generated
210
package-lock.json
generated
|
|
@ -10,11 +10,13 @@
|
|||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"axios": "^1.4.0",
|
||||
"chokidar": "^3.5.3",
|
||||
"compression": "^1.7.4",
|
||||
"express": "^4.18.2",
|
||||
"n-readlines": "^1.0.1",
|
||||
"node-html-parser": "^6.1.5",
|
||||
"nodemon": "^2.0.22"
|
||||
"nodemon": "^2.0.22",
|
||||
"socket.io": "^4.6.2"
|
||||
},
|
||||
"bin": {
|
||||
"heisse-preise": "index.js"
|
||||
|
|
@ -173,12 +175,35 @@
|
|||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@socket.io/component-emitter": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz",
|
||||
"integrity": "sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg=="
|
||||
},
|
||||
"node_modules/@types/cookie": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz",
|
||||
"integrity": "sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q=="
|
||||
},
|
||||
"node_modules/@types/cors": {
|
||||
"version": "2.8.13",
|
||||
"resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.13.tgz",
|
||||
"integrity": "sha512-RG8AStHlUiV5ysZQKq97copd2UmVYw3/pRMLefISZ3S1hK104Cwm7iLQ3fTKx+lsUH2CE8FlLaYeEA2LSeqYUA==",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/minimatch": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz",
|
||||
"integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "20.2.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.2.5.tgz",
|
||||
"integrity": "sha512-JJulVEQXmiY9Px5axXHeYGLSjhkZEnD+MDPDGbCbIAbMslkKwmygtZFy1X6s/075Yo94sf8GuSlFfPzysQrWZQ=="
|
||||
},
|
||||
"node_modules/abbrev": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
|
||||
|
|
@ -348,6 +373,14 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"node_modules/base64id": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz",
|
||||
"integrity": "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==",
|
||||
"engines": {
|
||||
"node": "^4.5.0 || >= 5.9"
|
||||
}
|
||||
},
|
||||
"node_modules/binary-extensions": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
|
||||
|
|
@ -680,6 +713,18 @@
|
|||
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/cors": {
|
||||
"version": "2.8.5",
|
||||
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
|
||||
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
|
||||
"dependencies": {
|
||||
"object-assign": "^4",
|
||||
"vary": "^1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/cross-spawn": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
|
||||
|
|
@ -877,6 +922,63 @@
|
|||
"once": "^1.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/engine.io": {
|
||||
"version": "6.4.2",
|
||||
"resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.4.2.tgz",
|
||||
"integrity": "sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg==",
|
||||
"dependencies": {
|
||||
"@types/cookie": "^0.4.1",
|
||||
"@types/cors": "^2.8.12",
|
||||
"@types/node": ">=10.0.0",
|
||||
"accepts": "~1.3.4",
|
||||
"base64id": "2.0.0",
|
||||
"cookie": "~0.4.1",
|
||||
"cors": "~2.8.5",
|
||||
"debug": "~4.3.1",
|
||||
"engine.io-parser": "~5.0.3",
|
||||
"ws": "~8.11.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/engine.io-parser": {
|
||||
"version": "5.0.7",
|
||||
"resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.0.7.tgz",
|
||||
"integrity": "sha512-P+jDFbvK6lE3n1OL+q9KuzdOFWkkZ/cMV9gol/SbVfpyqfvrfrFTOFJ6fQm2VC3PZHlU3QPhVwmbsCnauHF2MQ==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/engine.io/node_modules/cookie": {
|
||||
"version": "0.4.2",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz",
|
||||
"integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/engine.io/node_modules/debug": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
||||
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
|
||||
"dependencies": {
|
||||
"ms": "2.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"supports-color": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/engine.io/node_modules/ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
},
|
||||
"node_modules/entities": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
|
||||
|
|
@ -1938,6 +2040,14 @@
|
|||
"url": "https://github.com/fb55/nth-check?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/object-inspect": {
|
||||
"version": "1.12.3",
|
||||
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
|
||||
|
|
@ -2657,6 +2767,84 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/socket.io": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.6.2.tgz",
|
||||
"integrity": "sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ==",
|
||||
"dependencies": {
|
||||
"accepts": "~1.3.4",
|
||||
"base64id": "~2.0.0",
|
||||
"debug": "~4.3.2",
|
||||
"engine.io": "~6.4.2",
|
||||
"socket.io-adapter": "~2.5.2",
|
||||
"socket.io-parser": "~4.2.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/socket.io-adapter": {
|
||||
"version": "2.5.2",
|
||||
"resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz",
|
||||
"integrity": "sha512-87C3LO/NOMc+eMcpcxUBebGjkpMDkNBS9tf7KJqcDsmL936EChtVva71Dw2q4tQcuVC+hAUy4an2NO/sYXmwRA==",
|
||||
"dependencies": {
|
||||
"ws": "~8.11.0"
|
||||
}
|
||||
},
|
||||
"node_modules/socket.io-parser": {
|
||||
"version": "4.2.4",
|
||||
"resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz",
|
||||
"integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==",
|
||||
"dependencies": {
|
||||
"@socket.io/component-emitter": "~3.1.0",
|
||||
"debug": "~4.3.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/socket.io-parser/node_modules/debug": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
||||
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
|
||||
"dependencies": {
|
||||
"ms": "2.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"supports-color": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/socket.io-parser/node_modules/ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
},
|
||||
"node_modules/socket.io/node_modules/debug": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
||||
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
|
||||
"dependencies": {
|
||||
"ms": "2.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"supports-color": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/socket.io/node_modules/ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
},
|
||||
"node_modules/statuses": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
|
||||
|
|
@ -2959,6 +3147,26 @@
|
|||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.11.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
|
||||
"integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": "^5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
|
|
|
|||
|
|
@ -20,11 +20,13 @@
|
|||
"homepage": "https://github.com/badlogic/heissepreise#readme",
|
||||
"dependencies": {
|
||||
"axios": "^1.4.0",
|
||||
"chokidar": "^3.5.3",
|
||||
"compression": "^1.7.4",
|
||||
"express": "^4.18.2",
|
||||
"n-readlines": "^1.0.1",
|
||||
"node-html-parser": "^6.1.5",
|
||||
"nodemon": "^2.0.22"
|
||||
"nodemon": "^2.0.22",
|
||||
"socket.io": "^4.6.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"husky": "^8.0.3",
|
||||
|
|
|
|||
43
server.js
43
server.js
|
|
@ -1,6 +1,12 @@
|
|||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const http = require("http");
|
||||
const chokidar = require("chokidar");
|
||||
const analysis = require("./analysis");
|
||||
const template = require("./template");
|
||||
const socketIO = require("socket.io");
|
||||
const express = require("express");
|
||||
const compression = require("compression");
|
||||
|
||||
function copyItemsToSite(dataDir) {
|
||||
const items = analysis.readJSON(`${dataDir}/latest-canonical.json.${analysis.FILE_COMPRESSOR}`);
|
||||
|
|
@ -31,14 +37,31 @@ function scheduleFunction(hour, minute, second, func) {
|
|||
}, delay);
|
||||
}
|
||||
|
||||
function generateSiteAndWatch(inputDir, outputDir) {
|
||||
template.generateSite(inputDir, outputDir, true);
|
||||
const watcher = chokidar.watch(inputDir, { ignored: /(^|[\/\\])\../ });
|
||||
|
||||
let initialScan = true;
|
||||
watcher.on("ready", () => (initialScan = false));
|
||||
watcher.on("all", (event, filePath) => {
|
||||
if (initialScan) return;
|
||||
if (path.resolve(filePath).startsWith(path.resolve(outputDir))) return;
|
||||
console.log(`File ${filePath} has been ${event}`);
|
||||
template.generateSite(inputDir, outputDir, false);
|
||||
});
|
||||
console.log(`Watching directory for changes: ${inputDir}`);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const dataDir = "data";
|
||||
const port = process?.argv?.[2] ?? 3000;
|
||||
const liveReload = process?.argv?.[3] ?? false;
|
||||
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir);
|
||||
}
|
||||
|
||||
template.generateSite("site", "site/output");
|
||||
generateSiteAndWatch("site", "site/output");
|
||||
|
||||
analysis.migrateCompression(dataDir, ".json", ".json.br");
|
||||
analysis.migrateCompression(dataDir, ".json.gz", ".json.br");
|
||||
|
|
@ -57,15 +80,21 @@ function scheduleFunction(hour, minute, second, func) {
|
|||
copyItemsToSite(dataDir);
|
||||
});
|
||||
|
||||
const express = require("express");
|
||||
const compression = require("compression");
|
||||
const app = express();
|
||||
const port = process?.argv?.[2] ?? 3000;
|
||||
|
||||
app.use(compression());
|
||||
app.use(express.static("site/output"));
|
||||
|
||||
app.listen(port, () => {
|
||||
const server = http.createServer(app).listen(port, () => {
|
||||
console.log(`Example app listening on port ${port}`);
|
||||
});
|
||||
if (liveReload === "true") {
|
||||
const sockets = [];
|
||||
const io = socketIO(server);
|
||||
io.on("connection", (socket) => sockets.push(socket));
|
||||
chokidar.watch("site/output").on("all", () => {
|
||||
lastChangeTimestamp = Date.now();
|
||||
for (let i = 0; i < sockets.length; i++) {
|
||||
sockets[i].send(`${lastChangeTimestamp}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
%%templates/_header.html%%
|
||||
%%_templates/_header.html%%
|
||||
|
||||
<h2 id="date"></h2>
|
||||
<div class="filters" id="filters-store"></div>
|
||||
|
|
@ -10,8 +10,8 @@
|
|||
<div id="numresults" style="margin-top: 1em"></div>
|
||||
<table id="result"></table>
|
||||
|
||||
<script src="alasql.js"></script>
|
||||
<script src="js/alasql.js"></script>
|
||||
<script src="utils.js"></script>
|
||||
<script src="aktionen.js"></script>
|
||||
|
||||
%%templates/_footer.html%%
|
||||
%%_templates/_footer.html%%
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
%%templates/_header.html%%
|
||||
%%_templates/_header.html%%
|
||||
|
||||
<h2>Billiger seit letzter Preisänderung</h2>
|
||||
<div class="filters" id="filters-store"></div>
|
||||
|
|
@ -6,8 +6,8 @@
|
|||
<div id="numresults" style="margin-top: 1em"></div>
|
||||
<table id="result"></table>
|
||||
|
||||
<script src="alasql.js"></script>
|
||||
<script src="js/alasql.js"></script>
|
||||
<script src="utils.js"></script>
|
||||
<script src="billiger.js"></script>
|
||||
|
||||
%%templates/_footer.html%%
|
||||
%%_templates/_footer.html%%
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
%%templates/_header.html%%
|
||||
%%_templates/_header.html%%
|
||||
|
||||
<div id="cart" class="cart column">
|
||||
<h3 id="cartname"></h3>
|
||||
|
|
@ -21,9 +21,9 @@
|
|||
</div>
|
||||
<div id="search" class="column"></div>
|
||||
|
||||
<script src="chart.js"></script>
|
||||
<script src="alasql.js"></script>
|
||||
<script src="js/alasql.js"></script>
|
||||
<script src="js/chart.js"></script>
|
||||
<script src="utils.js"></script>
|
||||
<script src="cart.js"></script>
|
||||
|
||||
%%templates/_footer.html%%
|
||||
%%_templates/_footer.html%%
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
%%templates/_header.html%%
|
||||
%%_templates/_header.html%%
|
||||
|
||||
<div class="filters">
|
||||
<button id="newcart">Neuer Warenkorb</button>
|
||||
|
|
@ -8,8 +8,8 @@
|
|||
<table id="carts" class="carts"></table>
|
||||
<input type="file" id="fileInput" style="display: none" />
|
||||
|
||||
<script src="alasql.js"></script>
|
||||
<script src="js/alasql.js"></script>
|
||||
<script src="utils.js"></script>
|
||||
<script src="carts.js"></script>
|
||||
|
||||
%%templates/_header.html%%
|
||||
%%_templates/_footer.html%%
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
%%templates/_header.html%%
|
||||
%%_templates/_header.html%%
|
||||
|
||||
<div class="filters">
|
||||
<label
|
||||
|
|
@ -14,8 +14,8 @@
|
|||
<div id="results"></div>
|
||||
<table id="result"></table>
|
||||
|
||||
<script src="alasql.js"></script>
|
||||
<script src="js/alasql.js"></script>
|
||||
<script src="utils.js"></script>
|
||||
<script src="changes.js"></script>
|
||||
|
||||
%%templates/_footer.html%%
|
||||
%%_templates/_footer.html%%
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
%%templates/_header.html%%
|
||||
%%_templates/_header.html%%
|
||||
|
||||
<div id="chart" class="column hide" style="width: 80%">
|
||||
<canvas></canvas>
|
||||
|
|
@ -14,9 +14,9 @@
|
|||
</div>
|
||||
<div id="search" class="column"></div>
|
||||
|
||||
<script src="alasql.js"></script>
|
||||
<script src="chart.js"></script>
|
||||
<script src="js/alasql.js"></script>
|
||||
<script src="js/chart.js"></script>
|
||||
<script src="utils.js"></script>
|
||||
<script src="index.js"></script>
|
||||
|
||||
%%templates/_footer.html%%
|
||||
%%_templates/_footer.html%%
|
||||
|
|
|
|||
4881
site/js/socket.io.js
Normal file
4881
site/js/socket.io.js
Normal file
File diff suppressed because it is too large
Load Diff
112
site/utils.js
112
site/utils.js
|
|
@ -971,9 +971,6 @@ function stem(word) {
|
|||
|
||||
function vector(tokens) {
|
||||
const vector = {};
|
||||
/*for (const token of tokens) {
|
||||
vector[token] = (vector[token] || 0) + 1;
|
||||
}*/
|
||||
for (token of tokens) {
|
||||
if (token.length > 3) {
|
||||
for (let i = 0; i < token.length - 3; i++) {
|
||||
|
|
@ -1025,10 +1022,6 @@ function magnitude(vector) {
|
|||
return Math.sqrt(sumOfSquares);
|
||||
}
|
||||
|
||||
function deepCopy(obj) {
|
||||
return JSON.parse(JSON.stringify(obj));
|
||||
}
|
||||
|
||||
function similaritySortItems(items) {
|
||||
if (items.length == 0) return items;
|
||||
sortedItems = [items.shift()];
|
||||
|
|
@ -1065,90 +1058,6 @@ function vectorizeItems(items) {
|
|||
});
|
||||
}
|
||||
|
||||
function cluster(items, maxTime) {
|
||||
if (!maxTime) maxTime = 0.25;
|
||||
|
||||
// Tokenize, stem, and vectorize item names
|
||||
vectorizeItems(items);
|
||||
|
||||
// Split by store and sort by number of items in descending order
|
||||
const itemsPerStore = [];
|
||||
for (const store of STORE_KEYS) {
|
||||
const storeItems = items.filter((item) => item.store === store);
|
||||
if (storeItems.length > 0) itemsPerStore.push(storeItems);
|
||||
}
|
||||
itemsPerStore.sort((a, b) => b.length - a.length);
|
||||
itemsPerStore.forEach((items) => console.log(items[0].store + ", " + items.length));
|
||||
|
||||
// Take the store with the most items, then try to find the best match
|
||||
// from each of the other stores
|
||||
const baseStore = itemsPerStore.shift();
|
||||
itemsPerStore.push(baseStore);
|
||||
const otherItems = itemsPerStore.flat();
|
||||
|
||||
let clusters = [];
|
||||
for (item of baseStore) {
|
||||
clusters.push({ centroid: deepCopy(item.vector), item, items: [] });
|
||||
}
|
||||
|
||||
const now = performance.now();
|
||||
let maxIterations = 100;
|
||||
while (maxIterations-- > 0) {
|
||||
for (item of otherItems) {
|
||||
let maxSimilarity = -1;
|
||||
let nearestCluster = null;
|
||||
for (cluster of clusters) {
|
||||
const similarity = dotProduct(cluster.centroid, item.vector);
|
||||
if (similarity > maxSimilarity) {
|
||||
maxSimilarity = similarity;
|
||||
nearestCluster = cluster;
|
||||
item.similarity = similarity;
|
||||
}
|
||||
}
|
||||
nearestCluster.items.push(item);
|
||||
}
|
||||
|
||||
const newClusters = [];
|
||||
for (cluster of clusters) {
|
||||
const newCluster = { centroid: {}, item: cluster.item, items: [] };
|
||||
for (item of cluster.items) {
|
||||
addVector(newCluster.centroid, item.vector);
|
||||
}
|
||||
if (cluster.items.length > 0) {
|
||||
scaleVector(newCluster.centroid, 1 / cluster.items.length);
|
||||
normalizeVector(newCluster.centroid);
|
||||
}
|
||||
newClusters.push(newCluster);
|
||||
}
|
||||
|
||||
const time = (performance.now() - now) / 1000;
|
||||
console.log(maxIterations + ", time " + time);
|
||||
if (JSON.stringify(clusters) == JSON.stringify(newClusters) || maxIterations == 1 || time > maxTime) {
|
||||
break;
|
||||
}
|
||||
|
||||
clusters = newClusters;
|
||||
}
|
||||
|
||||
const finalClusters = [];
|
||||
for (cluster of clusters) {
|
||||
cluster.items = similaritySortItems(cluster.items);
|
||||
finalClusters.push(cluster);
|
||||
}
|
||||
|
||||
return finalClusters;
|
||||
}
|
||||
|
||||
function flattenClusters(clusters) {
|
||||
const items = [];
|
||||
for (cluster of clusters) {
|
||||
for (item of cluster.items) {
|
||||
items.push(item);
|
||||
}
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
function isMobile() {
|
||||
return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent);
|
||||
}
|
||||
|
|
@ -1167,3 +1076,24 @@ try {
|
|||
} catch (e) {
|
||||
// hax
|
||||
}
|
||||
|
||||
function setupLiveEdit() {
|
||||
if (window.location.host.indexOf("localhost") < 0 && window.location.host.indexOf("127.0.0.1") < 0) return;
|
||||
var script = document.createElement("script");
|
||||
script.type = "text/javascript";
|
||||
script.onload = () => {
|
||||
let lastChangeTimestamp = null;
|
||||
this.socket = io({ transports: ["websocket"] });
|
||||
this.socket.on("connect", () => console.log("Connected"));
|
||||
this.socket.on("disconnect", () => console.log("Disconnected"));
|
||||
this.socket.on("message", (timestamp) => {
|
||||
if (lastChangeTimestamp != timestamp) {
|
||||
setTimeout(() => location.reload(), 100);
|
||||
lastChangeTimestamp = timestamp;
|
||||
}
|
||||
});
|
||||
};
|
||||
script.src = "js/socket.io.js";
|
||||
document.body.appendChild(script);
|
||||
}
|
||||
setupLiveEdit();
|
||||
|
|
|
|||
19
template.js
19
template.js
|
|
@ -1,19 +1,6 @@
|
|||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
function findUniqueFilenames(str) {
|
||||
const regex = /%%([^%]+)%%/g;
|
||||
const filenames = new Set();
|
||||
let match;
|
||||
|
||||
while ((match = regex.exec(str))) {
|
||||
const filename = match[1];
|
||||
filenames.add(filename);
|
||||
}
|
||||
|
||||
return Array.from(filenames);
|
||||
}
|
||||
|
||||
function deleteDirectory(directory) {
|
||||
if (fs.existsSync(directory)) {
|
||||
fs.readdirSync(directory).forEach((file) => {
|
||||
|
|
@ -28,12 +15,6 @@ function deleteDirectory(directory) {
|
|||
}
|
||||
}
|
||||
|
||||
function isSameDirectory(directory, potentialSubdirectory) {
|
||||
return path.resolve(directory);
|
||||
const absolutePotentialSubdirectory = path.resolve(potentialSubdirectory);
|
||||
return absolutePotentialSubdirectory.startsWith(absoluteDirectory);
|
||||
}
|
||||
|
||||
function processFile(inputFile, outputFile) {
|
||||
console.log(`${inputFile} -> ${outputFile}`);
|
||||
const fileDir = path.dirname(inputFile);
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user