Marown/heissepreise
1
0
Fork 0
mirror of https://github.com/badlogic/heissepreise.git synced 2024-06-24 09:25:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add stopword filtering, filter by unit mismatch when finding similar items.

Browse Source
This commit is contained in:
Mario Zechner 2023-06-06 16:55:56 +02:00
parent 353e3cc77e
commit b9ed07eabb
3 changed files with 11804 additions and 996 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
categorize.js
View File

@ -42,8 +42,6 @@ const HITS = Math.floor(30000 + Math.random() * 2000);
const canonicalItem = stores.billa.getCanonical(item);
canonicalItem.categoryCode = `${categoryCode}${subCategoryCode}`;
canonicalItem.categoryName = `${categoryName.name} > ${subCategoryName}`;
if (canonicalItem.name.toLowerCase().indexOf("erdapfel")) canonicalItem.name + " kartoffel";
if (canonicalItem.name.toLowerCase().indexOf("erdäpfel")) canonicalItem.name + " kartoffeln";
subCategory.items.push(canonicalItem);
} catch (e) {
// Ignore super tiles
@ -64,24 +62,26 @@ const HITS = Math.floor(30000 + Math.random() * 2000);
for (const category of categories) {
for (const subCategory of category.subCategories) {
for (const item of subCategory.items) {
if (item.name.toLowerCase().indexOf("erdapfel") >= 0) item.name += " kartoffel";
if (item.name.toLowerCase().indexOf("erdäpfel") >= 0) item.name += " kartoffeln";
items.push(item);
lookup[item.id] = item;
}
}
}
const vectorize = true;
const useUnits = true;
console.log("Vectorizing items");
siteUtils.vectorizeItems(items, vectorize);
siteUtils.vectorizeItems(items, useUnits);
console.log("Categorizing items");
const file = process?.argv?.[2] ?? "site/data/momentum-cart.json";
let momentumItems = analysis.readJSON(file);
if (momentumItems.items) momentumItems = momentumItems.items;
siteUtils.vectorizeItems(momentumItems, vectorize);
siteUtils.vectorizeItems(momentumItems, useUnits);
const start = performance.now();
for (const item of momentumItems) {
const similar = siteUtils.findMostSimilarItems(item, items, 9);
const similar = siteUtils.findMostSimilarItems(item, items, 9, (ref, other) => ref.unit == other.unit);
console.log(`${item.name}`);
similar.sort((a, b) => b.similarity - a.similarity);
similar.forEach((s) => console.log(`${s.item.categoryName}, ${s.item.name}, ${s.similarity}`));

12044
site/data/momentum-cart.json
View File

File diff suppressed because it is too large Load Diff

744
site/utils.js
View File

@ -1160,11 +1160,12 @@ function findMostSimilarItem(refItem, items) {
};
}
function findMostSimilarItems(refItem, items, k = 5) {
function findMostSimilarItems(refItem, items, k = 5, accept = (ref, item) => true) {
let topSimilarItems = [];
let topSimilarities = [];
items.forEach((item, idx) => {
if (!accept(refItem, item)) return;
let similarity = dotProduct(refItem.vector, item.vector);
if (topSimilarItems.length < k) {
@ -1205,19 +1206,24 @@ function similaritySortItems(items) {
return sortedItems;
}
function vectorizeItems(items, useUnit = true) {
items.forEach((item) => {
let name = item.name
.toLowerCase()
.replace(/[^\w\s]|_/g, "")
.replace("-", " ");
item.tokens = name.split(/\s+/).map((token) => stem(token));
if (useUnit) {
if (item.quantity) item.tokens.push("" + item.quantity);
if (item.unit) item.tokens.push(item.unit);
}
item.vector = vector(item.tokens);
});
function vectorizeItem(item, useUnit = true) {
let name = item.name
.toLowerCase()
.replace(/[^\w\s]|_/g, "")
.replace("-", " ");
item.tokens = name
.split(/\s+/)
.filter((token) => !globalStopwords.includes(token))
.map((token) => stem(token));
if (useUnit) {
if (item.quantity) item.tokens.push("" + item.quantity);
if (item.unit) item.tokens.push(item.unit);
}
item.vector = vector(item.tokens);
}
function vectorizeItems(items, useUnit = true, accept = () => {}) {
items.forEach((item) => vectorizeItem(item, useUnit));
}
function isMobile() {
@ -1232,6 +1238,7 @@ try {
exports.scaleVector = scaleVector;
exports.normalizeVector = normalizeVector;
exports.stem = stem;
exports.vectorizeItem = vectorizeItem;
exports.vectorizeItems = vectorizeItems;
exports.findMostSimilarItem = findMostSimilarItem;
exports.findMostSimilarItems = findMostSimilarItems;
@ -1262,3 +1269,712 @@ if (typeof window !== "undefined") {
}
setupLiveEdit();
}
const globalStopwords = [
"ab",
"aber",
"alle",
"allein",
"allem",
"allen",
"aller",
"allerdings",
"allerlei",
"alles",
"allmählich",
"allzu",
"als",
"alsbald",
"also",
"am",
"an",
"and",
"ander",
"andere",
"anderem",
"anderen",
"anderer",
"andererseits",
"anderes",
"anderm",
"andern",
"andernfalls",
"anders",
"anstatt",
"auch",
"auf",
"aus",
"ausgenommen",
"ausser",
"ausserdem",
"außer",
"außerdem",
"außerhalb",
"bald",
"bei",
"beide",
"beiden",
"beiderlei",
"beides",
"beim",
"beinahe",
"bereits",
"besonders",
"besser",
"beträchtlich",
"bevor",
"bezüglich",
"bin",
"bis",
"bisher",
"bislang",
"bist",
"bloß",
"bsp.",
"bzw",
"ca",
"ca.",
"content",
"da",
"dabei",
"dadurch",
"dafür",
"dagegen",
"daher",
"dahin",
"damals",
"damit",
"danach",
"daneben",
"dann",
"daran",
"darauf",
"daraus",
"darin",
"darum",
"darunter",
"darüber",
"darüberhinaus",
"das",
"dass",
"dasselbe",
"davon",
"davor",
"dazu",
"daß",
"dein",
"deine",
"deinem",
"deinen",
"deiner",
"deines",
"dem",
"demnach",
"demselben",
"den",
"denen",
"denn",
"dennoch",
"denselben",
"der",
"derart",
"derartig",
"derem",
"deren",
"derer",
"derjenige",
"derjenigen",
"derselbe",
"derselben",
"derzeit",
"des",
"deshalb",
"desselben",
"dessen",
"desto",
"deswegen",
"dich",
"die",
"diejenige",
"dies",
"diese",
"dieselbe",
"dieselben",
"diesem",
"diesen",
"dieser",
"dieses",
"diesseits",
"dir",
"direkt",
"direkte",
"direkten",
"direkter",
"doch",
"dort",
"dorther",
"dorthin",
"drauf",
"drin",
"drunter",
"drüber",
"du",
"dunklen",
"durch",
"durchaus",
"eben",
"ebenfalls",
"ebenso",
"eher",
"eigenen",
"eigenes",
"eigentlich",
"ein",
"eine",
"einem",
"einen",
"einer",
"einerseits",
"eines",
"einfach",
"einführen",
"einführte",
"einführten",
"eingesetzt",
"einig",
"einige",
"einigem",
"einigen",
"einiger",
"einigermaßen",
"einiges",
"einmal",
"eins",
"einseitig",
"einseitige",
"einseitigen",
"einseitiger",
"einst",
"einstmals",
"einzig",
"entsprechend",
"entweder",
"er",
"erst",
"es",
"etc",
"etliche",
"etwa",
"etwas",
"euch",
"euer",
"eure",
"eurem",
"euren",
"eurer",
"eures",
"falls",
"fast",
"ferner",
"folgende",
"folgenden",
"folgender",
"folgendes",
"folglich",
"fuer",
"für",
"gab",
"ganze",
"ganzem",
"ganzen",
"ganzer",
"ganzes",
"gar",
"gegen",
"gemäss",
"ggf",
"gleich",
"gleichwohl",
"gleichzeitig",
"glücklicherweise",
"gänzlich",
"hab",
"habe",
"haben",
"haette",
"hast",
"hat",
"hatte",
"hatten",
"hattest",
"hattet",
"heraus",
"herein",
"hier",
"hier",
"hinter",
"hiermit",
"hiesige",
"hin",
"hinein",
"hinten",
"hinter",
"hinterher",
"http",
"hätt",
"hätte",
"hätten",
"höchstens",
"ich",
"igitt",
"ihm",
"ihn",
"ihnen",
"ihr",
"ihre",
"ihrem",
"ihren",
"ihrer",
"ihres",
"im",
"immer",
"immerhin",
"in",
"indem",
"indessen",
"infolge",
"innen",
"innerhalb",
"ins",
"insofern",
"inzwischen",
"irgend",
"irgendeine",
"irgendwas",
"irgendwen",
"irgendwer",
"irgendwie",
"irgendwo",
"ist",
"ja",
"je",
"jed",
"jede",
"jedem",
"jeden",
"jedenfalls",
"jeder",
"jederlei",
"jedes",
"jedoch",
"jemand",
"jene",
"jenem",
"jenen",
"jener",
"jenes",
"jenseits",
"jetzt",
"jährig",
"jährige",
"jährigen",
"jähriges",
"kam",
"kann",
"kannst",
"kaum",
"kein",
"keine",
"keinem",
"keinen",
"keiner",
"keinerlei",
"keines",
"keineswegs",
"klar",
"klare",
"klaren",
"klares",
"klein",
"kleinen",
"kleiner",
"kleines",
"koennen",
"koennt",
"koennte",
"koennten",
"komme",
"kommen",
"kommt",
"konkret",
"konkrete",
"konkreten",
"konkreter",
"konkretes",
"können",
"könnt",
"künftig",
"leider",
"machen",
"man",
"manche",
"manchem",
"manchen",
"mancher",
"mancherorts",
"manches",
"manchmal",
"mehr",
"mehrere",
"mein",
"meine",
"meinem",
"meinen",
"meiner",
"meines",
"mich",
"mir",
"mit",
"mithin",
"muessen",
"muesst",
"muesste",
"muss",
"musst",
"musste",
"mussten",
"muß",
"mußt",
"müssen",
"müsste",
"müssten",
"müßt",
"müßte",
"nach",
"nachdem",
"nachher",
"nachhinein",
"nahm",
"natürlich",
"neben",
"nebenan",
"nehmen",
"nein",
"nicht",
"nichts",
"nie",
"niemals",
"niemand",
"nirgends",
"nirgendwo",
"noch",
"nun",
"nur",
"nächste",
"nämlich",
"nötigenfalls",
"ob",
"oben",
"oberhalb",
"obgleich",
"obschon",
"obwohl",
"oder",
"oft",
"per",
"plötzlich",
"schließlich",
"schon",
"sehr",
"sehrwohl",
"seid",
"sein",
"seine",
"seinem",
"seinen",
"seiner",
"seines",
"seit",
"seitdem",
"seither",
"selber",
"selbst",
"sich",
"sicher",
"sicherlich",
"sie",
"sind",
"so",
"sobald",
"sodass",
"sofort",
"sofern",
"sog",
"sogar",
"solange",
"solch",
"solche",
"solchem",
"solchen",
"solcher",
"solches",
"soll",
"sollen",
"sollst",
"sollt",
"sollte",
"sollten",
"somit",
"sondern",
"sonst",
"sonstige",
"sonstigen",
"sonstiger",
"sonstiges",
"sooft",
"soviel",
"soweit",
"sowie",
"sowieso",
"sowohl",
"später",
"statt",
"stattfinden",
"stattfand",
"stattgefunden",
"steht",
"stets",
"such",
"suche",
"suchen",
"tatsächlich",
"tatsächlichen",
"tatsächlicher",
"tatsächliches",
"tatsächlich",
"tatsächlichen",
"tatsächlicher",
"tatsächliches",
"tief",
"tiefer",
"trotz",
"trotzdem",
"tun",
"über",
"überall",
"überallhin",
"überdies",
"überhaupt",
"übrig",
"übrigens",
"um",
"umso",
"umsoweniger",
"unbedingt",
"und",
"unmöglich",
"unnötig",
"unser",
"unsere",
"unserem",
"unseren",
"unserer",
"unseres",
"unserseits",
"unter",
"unterhalb",
"unterhalb",
"untereinander",
"untergebracht",
"unterhalb",
"unterhalb",
"unterhalb",
"unterhalb",
"unterhalb",
"unterhalb",
"unterschiedlich",
"unterschiedliche",
"unterschiedlichen",
"unterschiedlicher",
"unterschiedliches",
"unterschiedlich",
"unterschiedliche",
"unterschiedlichen",
"unterschiedlicher",
"unterschiedliches",
"unzwar",
"usw",
"usw.",
"vermag",
"vermögen",
"vermutlich",
"verrate",
"verraten",
"verrätst",
"verschieden",
"verschiedene",
"verschiedenen",
"verschiedener",
"verschiedenes",
"versorgen",
"versorgt",
"versorgte",
"versorgten",
"viel",
"viele",
"vielem",
"vielen",
"vieler",
"vieles",
"vielleicht",
"vielmals",
"vier",
"vierte",
"viertel",
"vierten",
"vierter",
"viertes",
"vom",
"von",
"vor",
"vorbei",
"vorgestern",
"vorher",
"vorüber",
"wach",
"wachen",
"wahrend",
"wann",
"war",
"warauf",
"ward",
"waren",
"warst",
"wart",
"warum",
"was",
"weder",
"weil",
"weiter",
"weitere",
"weiterem",
"weiteren",
"weiterer",
"weiteres",
"weiterhin",
"weitgehend",
"welche",
"welchem",
"welchen",
"welcher",
"welches",
"wem",
"wen",
"wenig",
"wenige",
"wenigem",
"wenigen",
"weniger",
"wenigstens",
"wenn",
"wenngleich",
"wer",
"werde",
"werden",
"werdet",
"weshalb",
"wessen",
"wichtig",
"wie",
"wieder",
"wiederum",
"wieso",
"will",
"willst",
"wir",
"wird",
"wirklich",
"wirst",
"wissen",
"wo",
"woanders",
"wohl",
"woher",
"wohin",
"wohingegen",
"wohl",
"wohlweislich",
"wollen",
"wollt",
"wollte",
"wollten",
"womit",
"woraufhin",
"woraus",
"woraussichtlich",
"worauf",
"woraus",
"worin",
"worüber",
"wovon",
"wovor",
"wozu",
"während",
"währenddessen",
"wär",
"wäre",
"wären",
"wärst",
"wäre",
"wären",
"wärst",
"würde",
"würden",
"würdest",
"würdet",
"zB",
"z.b.",
"zehn",
"zeigen",
"zeitweise",
"zu",
"zufolge",
"zugleich",
"zuletzt",
"zum",
"zumal",
"zumeist",
"zunächst",
"zur",
"zurück",
"zurückgehend",
"zurückgehen",
"zurückgegangen",
"zurückgekommen",
"zurückgekommen",
"zurückgekommen",
"zurückgekommen",
"zurückgezogen",
"zusammen",
"zusätzlich",
"zusammen",
"zuvor",
"zuviel",
"zuweilen",
"zwanzig",
"zwar",
"zwei",
"zweite",
"zweiten",
"zweiter",
"zweites",
"zwischen",
"zwischendurch",
"zwölf",
"überall",
"überallhin",
"überdies",
"überhaupt",
"übrig",
"übrigens",
];