Marown/heissepreise
1
0
Fork 0
mirror of https://github.com/badlogic/heissepreise.git synced 2024-06-27 19:05:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Style changes, Ingest Dossier Data from xls converted to csv see https://www.dossier.at/dossiers/supermaerkte/quellen/anatomie-eines-supermarkts-die-methodik/

Browse Source
This commit is contained in:
Mario Zechner 2023-05-27 12:33:18 +02:00
parent 42b89fa325
commit ce8e0fb102
11 changed files with 234 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -6,3 +6,7 @@ dist
heissepreise.zip heissepreise.zip
site/latest-canonical.json site/latest-canonical.json
latest-canonical.json latest-canonical.json
access.log
billa-2020.csv
report.html
spar-2020.csv

9
package-lock.json generated
View File

@ -12,6 +12,7 @@
"axios": "^1.4.0", "axios": "^1.4.0",
"compression": "^1.7.4", "compression": "^1.7.4",
"express": "^4.18.2", "express": "^4.18.2",
"n-readlines": "^1.0.1",
"nodemon": "^2.0.22" "nodemon": "^2.0.22"
}, },
"bin": { "bin": {
@ -1441,6 +1442,14 @@
"node": ">= 6" "node": ">= 6"
} }
}, },
"node_modules/n-readlines": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/n-readlines/-/n-readlines-1.0.1.tgz",
"integrity": "sha512-z4SyAIVgMy7CkgsoNw7YVz40v0g4+WWvvqy8+ZdHrCtgevcEO758WQyrYcw3XPxcLxF+//RszTz/rO48nzD0wQ==",
"engines": {
"node": ">=6.x.x"
}
},
"node_modules/napi-build-utils": { "node_modules/napi-build-utils": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",

1
package.json
View File

@ -20,6 +20,7 @@
"axios": "^1.4.0", "axios": "^1.4.0",
"compression": "^1.7.4", "compression": "^1.7.4",
"express": "^4.18.2", "express": "^4.18.2",
"n-readlines": "^1.0.1",
"nodemon": "^2.0.22" "nodemon": "^2.0.22"
}, },
"devDependencies": { "devDependencies": {

2
site/cart.html
View File

@ -11,12 +11,12 @@
<body> <body>
<div class="column"> <div class="column">
<h2>Heisse Preise</h2>
<div class="filters"> <div class="filters">
<a href="index.html">Produktsuche</a> <a href="index.html">Produktsuche</a>
<a href="changes.html">Tagespreisänderungen</a> <a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html">Warenkörbe</a> <a href="carts.html">Warenkörbe</a>
</div> </div>
<h2>Heisse Preise</h2>
<div id="cart" class="cart"> <div id="cart" class="cart">
<h3 id="cartname"></h3> <h3 id="cartname"></h3>
<canvas id="chart"></canvas> <canvas id="chart"></canvas>

6
site/carts.html
View File

@ -11,15 +11,13 @@
<body> <body>
<div class="column"> <div class="column">
<h2>Heisse Preise</h2>
<div class="filters"> <div class="filters">
<a href="index.html">Produktsuche</a> <a href="index.html">Produktsuche</a>
<a href="changes.html">Tagespreisänderungen</a> <a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html"><strong>Warenkörbe</strong></a> <a href="carts.html"><strong>Warenkörbe</strong></a>
</div> </div>
<h2>Heisse Preise</h2> <button id="newcart">Neuer Warenkorb</button>
<div class="filters">
<button id="newcart">Neuer Warenkorb</button>
</div>
<table id="carts"></table> <table id="carts"></table>
<table id="result"></table> <table id="result"></table>
</div> </div>

2
site/changes.html
View File

@ -11,12 +11,12 @@
<body> <body>
<div class="column"> <div class="column">
<h2>Heisse Preise</h2>
<div class="filters"> <div class="filters">
<a href="index.html">Produktsuche</a> <a href="index.html">Produktsuche</a>
<a href="changes.html"><strong>Tagespreisänderungen</strong></a> <a href="changes.html"><strong>Tagespreisänderungen</strong></a>
<a href="carts.html">Warenkörbe</a> <a href="carts.html">Warenkörbe</a>
</div> </div>
<h2>Heisse Preise</h2>
<div class="filters"> <div class="filters">
<label>Preisänderungen am <select id="dates"></select></label> <label>Preisänderungen am <select id="dates"></select></label>
</div> </div>

2
site/index.html
View File

@ -11,12 +11,12 @@
<body> <body>
<div class="column" style="max-width: 1000px"> <div class="column" style="max-width: 1000px">
<h2>Heisse Preise</h2>
<div class="filters"> <div class="filters">
<a href="index.html"><strong>Produktsuche</strong></a> <a href="index.html"><strong>Produktsuche</strong></a>
<a href="changes.html">Tagespreisänderungen</a> <a href="changes.html">Tagespreisänderungen</a>
<a href="carts.html">Warenkörbe</a> <a href="carts.html">Warenkörbe</a>
</div> </div>
<h2>Heisse Preise</h2>
<canvas id="chart" style="display: none;"></canvas> <canvas id="chart" style="display: none;"></canvas>
<div id="search" class="column"> <div id="search" class="column">
</div> </div>

167
site/style.css
View File

@ -1,11 +1,17 @@
* { * {
box-sizing: border-box; box-sizing: border-box;
} }
body { body {
margin: 0; margin: 0;
padding: 1em; padding: 1em;
font-family: sans-serif; font-family: Helvetica;
background: #f5f5f5;
color: #393939;
}
a {
color: #c9543a;
} }
.column { .column {
@ -15,6 +21,7 @@ body {
align-items: center; align-items: center;
margin: 0 auto; margin: 0 auto;
width: 100%; width: 100%;
max-width: 1000px;
} }
.search { .search {
@ -28,33 +35,97 @@ body {
margin-bottom: 1em; margin-bottom: 1em;
} }
input {
border: 2px solid;
padding: 0.4em;
border-color: #ee907b;
accent-color: #c9543a;
}
input[type="number"] { input[type="number"] {
max-width: 3em; max-width: 6em;
border: 2px solid;
padding: 0.4em;
border-color: #ee907b;
}
h2 {
background-color: #c9543a;
color: white;
width: 100%;
text-transform: uppercase;
text-align: center;
padding: 0.5em;
margin-top: 0;
} }
table { table {
border-collapse: collapse; border-collapse: collapse;
border: 1px solid; border: 0px;
margin-top: 1em; margin-top: 1em;
} }
th { th {
border: 1px solid;
padding: 0.2em; padding: 0.2em;
background: #aaa; text-align: left;
background: #c9543a;
color: white;
border: 0px;
}
th:nth-child(1) {
text-align: center;
}
th:nth-child(3) {
text-align: right;
padding: 0.7em;
}
th:nth-child(4) {
text-align: center;
padding: 0.6em;
} }
tr { tr {
border-collapse: collapse; border-collapse: collapse;
border: 1px solid #ddd; border: 1px solid #ddd;
border-left: 0px;
border-right: 0px;
} }
td { td {
padding: 0.2em; padding: 0.2em;
} }
th { td a {
text-align: left; color: #393939;
text-decoration: none;
}
td:nth-child(1) {
text-align: center;
padding: 0.5em;
}
td:nth-child(2) {
background-color: white;
padding: 0.4em;
}
td:nth-child(3) {
background-color: white;
font-size: small;
text-align: right;
padding-right: 0.7em;
}
td:nth-child(4) {
padding-left: 0.5em;
}
td:nth-child(4):before {
content: "€ "
} }
.itemname { .itemname {
@ -88,51 +159,67 @@ th {
@media screen and (max-width: 600px) { @media screen and (max-width: 600px) {
table { table {
border: 0; border: 0;
} }
table thead { table thead {
border: none; border: none;
clip: rect(0 0 0 0); clip: rect(0 0 0 0);
height: 1px; height: 1px;
margin: -1px; margin: -1px;
overflow: hidden; overflow: hidden;
padding: 0; padding: 0;
position: absolute; position: absolute;
width: 1px; width: 1px;
} }
table tr { table tr {
display: block; display: block;
margin-bottom: .625em; margin-bottom: .625em;
} }
table td { table td {
border-bottom: 1px solid #ddd; border-bottom: 1px solid #ddd;
padding-left: calc(65px + 1.2em); padding: 0 0 0 0 !important;
position: relative; padding-left: calc(65px + 1.2em) !important;
display: block; position: relative;
display: block;
} }
table td::before { table td::before {
/* /*
* aria-label has no advantage, it won't be read inside a table * aria-label has no advantage, it won't be read inside a table
content: attr(aria-label); content: attr(aria-label);
*/ */
content: attr(data-label); content: attr(data-label);
background-color: #aaa; background-color: #c9543a;
border: 1px solid; color: white;
position: absolute; border: 0px;
padding: .2em .4em; position: absolute;
width: 65px; padding: .1em .4em;
height: calc(100% - .4em); width: 65px;
left: -1px; height: calc(100%);
top: -1px; left: -1px;
text-align: right; top: -1px;
font-weight: bold; text-align: right;
font-weight: bold;
font-size: medium;
}
td:nth-child(1) {
text-align: center;
font-size: medium;
padding: 0.5em;
}
td:nth-child(3) {
background-color: white;
font-size: medium;
text-align: left;
padding-right: 0.7em;
} }
table td:last-child { table td:last-child {
border-bottom: 0; border-bottom: 0;
} }
} }

10
site/utils.js
View File

@ -375,7 +375,15 @@ function showChart(canvasDom, items) {
}, },
options: { options: {
responsive: true, responsive: true,
aspectRation: 16 / 9 aspectRation: 16 / 9,
scales: {
y: {
title: {
display: true,
text: "EURO"
}
}
}
} }
}); });
} }

1
stores/spar.js
View File

@ -14,6 +14,7 @@ exports.getCanonical = function(item, today) {
} }
return { return {
id: item.masterValues["code-internal"], id: item.masterValues["code-internal"],
sparId: item.masterValues["product-number"],
name: item.masterValues.title + " " + item.masterValues["short-description"], name: item.masterValues.title + " " + item.masterValues["short-description"],
price, price,
priceHistory: [{ date: today, price }], priceHistory: [{ date: today, price }],

79
stuff.js
View File

@ -150,4 +150,81 @@ function fixSparHistoricalData(dataDir) {
} }
} }
// momentumCartConversion(); const nReadlines = require('n-readlines');
function convertDossierData(dataDir, file) {
console.log(`Converting ${file}`);
const lookup = {};
for (item of JSON.parse(fs.readFileSync(`${dataDir}/latest-canonical.json`))) {
lookup[item.store + item.id] = item;
if (item.sparId)
lookup[item.store + "-" + item.sparId] = item;
}
const lines = new nReadlines(file);
const itemsPerDate = {};
let line = null;
const store = file.indexOf("spar") == 0 ? "spar" : "billa";
lines.next()
let itemsTotal = 0;
let notFound = 0;
while(line = lines.next()) {
itemsTotal++;
const tokens = line.toString("utf-8").split(";");
const dateTokens = tokens[0].split(".");
const date = "20" + dateTokens[2] + "-" + dateTokens[1] + "-" + dateTokens[0];
const producer = tokens[5];
const name = tokens[3];
const unit = tokens[6];
const price = Number.parseFloat(tokens[7].replace("€", "").trim().replace(",", "."));
const id = tokens[4].replace("ARTIKELNUMMER: ", "").replace("Art. Nr.: ", "");
let item = lookup[store + id];
if (!item)
item = lookup[store + "-" + id]
if (!item) {
// console.log("Couldn't find item " + name);
notFound++;
continue;
}
let items = itemsPerDate[date];
if (!items) itemsPerDate[date] = items = [];
if (store == "spar") {
items.push({
masterValues: {
"code-internal": item.id,
"product-number": id,
price,
title: producer,
"short-description": name,
"short-description-3": unit,
bioLevel: ""
}
});
} else {
items.push({
data: {
articleId: id,
name: name,
price: {
final: price
},
grammagePriceFactor: 1,
grammage: unit,
}
})
}
}
console.log("total: " + itemsTotal);
console.log("not found: " + notFound);
const dates = Object.keys(itemsPerDate).sort((a, b) => b.localeCompare(a));
for (date of dates) {
fs.writeFileSync(`${dataDir}/${store}-${date}.json`, JSON.stringify(itemsPerDate[date], null, 2));
}
console.log(`Wrote files for ${file}`);
}
// momentumCartConversion();
convertDossierData("data", "spar-2020.csv");
convertDossierData("data", "billa-2020.csv");