// Identificator.java // ----------------------- // (C) by Marc Nause; marc.nause@audioattack.de // first published on http://www.yacy.net // Braunschweig, Germany, 2008 // // $LastChangedDate$ // $LastChangedRevision$ // $LastChangedBy$ // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA package net.yacy.document.language; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; /** * This class can try to identify the language a text is written in. */ public final class Identificator { private static final LanguageStatisticsHolder languages = LanguageStatisticsHolder.getInstance(); private final Map letter; private int letters; private String language; public Identificator() { this.letter = new HashMap(); this.letters = 0; this.language = null; } /** * This method tries to return the language a text is written in. The method will only * use the first 100000 characters of the text which should be enough. Using more * characters probably only slows down the process without gaining much accuracy. * @param text the text that is to be analyzed * @return the language or "unknown" if the method was not able to find out the language */ public static String getLanguage(final String text) { // only test the first 100000 characters of a text return getLanguage(text, 100000); } /** * This method tries to return the language a text is written in. The method will * use the number characters defined in the parameter limit. * @param text the text that is to be analyzed * @param limit the number of characters that are supposed to be considered * @return the language or null if the method was not able to find out the language */ public static String getLanguage(final String text, final int limit) { int upperLimit = text.length(); if (upperLimit > limit) { upperLimit = limit; } final Identificator id = new Identificator(); // count number of characters in text for (int i = 0; i < upperLimit; i++) id.inc(text.charAt(i)); return id.getLanguage(); } public void inc(final char c) { if (!Character.isLetter(c)) return; final Character cc = Character.toLowerCase(c); final AtomicInteger i = this.letter.get(cc); if (i == null) { this.letter.put(cc, new AtomicInteger(1)); } else { i.incrementAndGet(); } this.letters++; } public void add(final String word) { if (word == null) return; for (int i = 0; i < word.length(); i++) inc(word.charAt(i)); } //modified by copperdust; Ukraine, 2012 public String getLanguage() { if (this.language != null) return this.language; // don't compute that twice if (this.letters == 0) return null; // not enough information available final LanguageStatistics testStat = new LanguageStatistics("text"); // calculate percentage Character character; Character maxChar = null; int count = 0; int max = 0; for (final Map.Entry e: this.letter.entrySet()) { character = e.getKey(); count = e.getValue().intValue(); if (count > max) { maxChar = character; max = count; } testStat.put(character, ((float) 100) * ((float) count) / (this.letters)); } // create list with relevant languages final List relevantLanguages = new Vector (); for (int i = 0; i < languages.size(); i++) { // only languages that contain the most common character in the text will be tested if (languages.get(i).contains(maxChar)) { relevantLanguages.add(i); } } if (relevantLanguages.isEmpty()) return null; // compare characters in text with characters in statistics final float[] offsetList = new float[relevantLanguages.size()]; final float[] sumList = new float[relevantLanguages.size()]; final Iterator iter = testStat.keySet().iterator(); float offset = 0; float valueCharacter; float value; while (iter.hasNext()) { character = iter.next(); valueCharacter = testStat.get(character); for (int i = 0; i < relevantLanguages.size(); i++) { value = languages.get(relevantLanguages.get(i)).get(character); if (value > 0) { offset = Math.abs(value - valueCharacter); offsetList[i] = offsetList[i] + offset; sumList[i] = sumList[i] + value;// accumulation processed characters // normally must be 100 after loop for language written in } } } //50/50 //abs(50-40) + abs(50-10) = 10 + 40 = 50 -- 50 = 0 [60 must be] //abs(50-25) + abs(50-25) = 25 + 25 = 50 -- 50 = 0 [0 must be] //75/25 //abs(50-60) + abs(50-15) = 10 + 35 = 45 -- 25 = 20 [60 must be] //abs(50-12,5) + abs(50-12,5) = 37,5 + 37,5 = 75 -- 75 = 0 [0 must be] //25/75 //abs(50-20) + abs(50-5) = 30 + 45 = 75 -- 75 = 0 [60 must be] //abs(50-37,5) + abs(50-37,5) = 12,5 + 12,5 = 25 -- 25 = 0 [0 must be] // Now we can count how closer each language to current pattern. float minOffset = 100.1f; int bestLanguage = -1; for (int i = 0; i < sumList.length; i++) { offset = offsetList[i] + 100 - sumList[i];// actual difference if (offset < minOffset) { minOffset = offset; bestLanguage = i; } } // Return name of language only if offset is smaller than 30%. // Prevents wrong language detection due to actual language not in langstats. if (minOffset < 30) { this.language = languages.get(relevantLanguages.get(bestLanguage)).getName(); return this.language; } return null; } }