2008-05-18 23:24:05 +02:00
|
|
|
// Identificator.java
|
|
|
|
// -----------------------
|
|
|
|
// (C) by Marc Nause; marc.nause@audioattack.de
|
|
|
|
// first published on http://www.yacy.net
|
|
|
|
// Braunschweig, Germany, 2008
|
|
|
|
//
|
2010-12-27 18:07:21 +01:00
|
|
|
// $LastChangedDate$
|
|
|
|
// $LastChangedRevision$
|
|
|
|
// $LastChangedBy$
|
2008-05-18 23:24:05 +02:00
|
|
|
//
|
|
|
|
// This program is free software; you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU General Public License as published by
|
|
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU General Public License
|
|
|
|
// along with this program; if not, write to the Free Software
|
|
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
|
2009-10-18 02:53:43 +02:00
|
|
|
package net.yacy.document.language;
|
2008-05-18 23:24:05 +02:00
|
|
|
|
2015-07-02 18:41:13 +02:00
|
|
|
import java.io.File;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import com.cybozu.labs.langdetect.Detector;
|
|
|
|
import com.cybozu.labs.langdetect.DetectorFactory;
|
|
|
|
import com.cybozu.labs.langdetect.LangDetectException;
|
|
|
|
import com.cybozu.labs.langdetect.Language;
|
|
|
|
|
|
|
|
import net.yacy.cora.util.ConcurrentLog;
|
2008-05-18 23:24:05 +02:00
|
|
|
|
|
|
|
/**
|
|
|
|
* This class can try to identify the language a text is written in.
|
|
|
|
*/
|
|
|
|
public final class Identificator {
|
|
|
|
|
2015-07-02 18:41:13 +02:00
|
|
|
private Detector detector;
|
|
|
|
private Language language;
|
2011-06-19 15:31:10 +02:00
|
|
|
|
2008-05-23 20:12:44 +02:00
|
|
|
public Identificator() {
|
2015-07-02 18:41:13 +02:00
|
|
|
try {
|
|
|
|
if(DetectorFactory.getLangList().isEmpty()) DetectorFactory.loadProfile(new File("langdetect").toString());
|
|
|
|
this.detector = DetectorFactory.create();
|
|
|
|
} catch (LangDetectException e) {
|
|
|
|
ConcurrentLog.logException(e);
|
2008-05-18 23:24:05 +02:00
|
|
|
}
|
2008-09-18 15:12:33 +02:00
|
|
|
}
|
2011-06-19 15:31:10 +02:00
|
|
|
|
|
|
|
public void add(final String word) {
|
2008-09-18 15:12:33 +02:00
|
|
|
if (word == null) return;
|
2016-02-07 01:52:32 +01:00
|
|
|
this.detector.append(" " + word); // detector internally caches text up to maxtextlen = default = 10000 chars
|
2008-09-18 15:12:33 +02:00
|
|
|
}
|
2011-06-19 15:31:10 +02:00
|
|
|
|
2008-09-18 15:12:33 +02:00
|
|
|
public String getLanguage() {
|
2015-07-02 18:41:13 +02:00
|
|
|
try {
|
|
|
|
ArrayList<Language> probabilities = this.detector.getProbabilities();
|
|
|
|
if(probabilities.isEmpty()) return null;
|
|
|
|
this.language = this.detector.getProbabilities().get(0);
|
|
|
|
} catch (LangDetectException e) {
|
2015-12-03 00:39:15 +01:00
|
|
|
// this contains mostly the message "no features in text"
|
|
|
|
//ConcurrentLog.logException(e);
|
2015-07-02 18:41:13 +02:00
|
|
|
return null;
|
2008-05-18 23:24:05 +02:00
|
|
|
}
|
2015-07-02 18:41:13 +02:00
|
|
|
// Return language only if probability is higher than 30% to account for missing language profiles
|
|
|
|
if (this.language.prob > 0.3) {
|
|
|
|
return this.language.lang;
|
2008-09-18 15:12:33 +02:00
|
|
|
}
|
2011-06-19 15:31:10 +02:00
|
|
|
|
2008-09-18 15:12:33 +02:00
|
|
|
return null;
|
2011-06-19 15:31:10 +02:00
|
|
|
|
2008-05-18 23:24:05 +02:00
|
|
|
}
|
2011-06-19 15:31:10 +02:00
|
|
|
|
2008-05-18 23:24:05 +02:00
|
|
|
}
|