mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
16e8ed3f01
If language is set to "browser" the client/user browser language is used to choose from available translation. simply: one users browser speaks English -> YaCy responds in English, other users browser speaks French -> YaCy responds in French. ! To make a translation/language available you have to activate the language once ! (or manually use the utility class TranslateAll) In ConfigBasic.html availabel translations are marked green on setting language=Browser The client language is determined by http header Accept-Language (checked in DefaultServlet)
349 lines
16 KiB
Java
349 lines
16 KiB
Java
// translator.java
|
|
// -------------------------------------
|
|
// part of YACY
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004
|
|
//
|
|
// This file ist contributed by Alexander Schier
|
|
// last major change: 25.05.2005
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
package net.yacy.data;
|
|
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.io.FileOutputStream;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.io.OutputStreamWriter;
|
|
import java.io.PrintWriter;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map.Entry;
|
|
import java.util.Set;
|
|
|
|
import net.yacy.cora.util.CommonPattern;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.document.SentenceReader;
|
|
import net.yacy.kelondro.util.FileUtils;
|
|
import net.yacy.kelondro.util.Formatter;
|
|
import net.yacy.peers.Seed;
|
|
import net.yacy.search.Switchboard;
|
|
import net.yacy.search.SwitchboardConstants;
|
|
import net.yacy.server.serverSwitch;
|
|
import net.yacy.utils.translation.ExtensionsFileFilter;
|
|
|
|
/**
|
|
* Wordlist based translator
|
|
*
|
|
* Uses a Property like file with phrases or single words to translate a string or a file
|
|
* */
|
|
public class Translator {
|
|
|
|
public final static String LANG_FILENAME_FILTER = "^.*\\.lng$";
|
|
|
|
/**
|
|
* Translate source using entries in translationTable
|
|
* @param source text to translate. Must be non null.
|
|
* @param translationTable translation entries : text to translate -> translation
|
|
* @return source translated
|
|
*/
|
|
public String translate(final StringBuilder source,
|
|
final Map<String, String> translationTable) {
|
|
final Set<Map.Entry<String, String>> entries = translationTable.entrySet();
|
|
StringBuilder builder = new StringBuilder(source);
|
|
for (final Entry<String, String> entry : entries) {
|
|
String key = entry.getKey();
|
|
/* We have to check key is not empty or indexOf would always return a positive value */
|
|
if (key != null && !key.isEmpty()) {
|
|
String translation = entry.getValue();
|
|
int index = builder.indexOf(key);
|
|
if (index < 0 || translation == null ) {
|
|
// Filename not available, but it will be printed in Log
|
|
// after all untranslated Strings as "Translated file: "
|
|
if (ConcurrentLog.isFine("TRANSLATOR"))
|
|
ConcurrentLog.fine("TRANSLATOR", "Unused String: " + key);
|
|
} else {
|
|
while (index >= 0) {
|
|
|
|
// check for word boundary before and after translation key
|
|
// to avoid translation just on char sequence e.g. as in key="bug" source="mybugfix"
|
|
boolean boundary = index + key.length() >= builder.length(); // eof text = end-bondary
|
|
|
|
if (!boundary) {
|
|
char c = builder.charAt(index + key.length() - 1);
|
|
char lc = builder.charAt(index + key.length());
|
|
boundary |= (SentenceReader.punctuation(c) || SentenceReader.invisible(c)); // special case, basically last char of key
|
|
boundary |= (SentenceReader.punctuation(lc) || SentenceReader.invisible(lc)); // char after key = end-boundary
|
|
}
|
|
|
|
// if end-boundary ok check begin-boundary
|
|
if (boundary && index > 0) {
|
|
char c = builder.charAt(index - 1); // char before key = begin-boundary
|
|
boundary = (SentenceReader.punctuation(c) || SentenceReader.invisible(c));
|
|
char fc = builder.charAt(index); // special case for key >name< , currently to allow <label>name</label (basically fist char of key)
|
|
boundary |= (SentenceReader.punctuation(fc) || SentenceReader.invisible(fc));
|
|
}
|
|
|
|
if (boundary) { // boundary check ok -> translate
|
|
builder.replace(index, index + key.length(), translation);
|
|
index = builder.indexOf(key, index + translation.length());
|
|
} else { // otherwise just skip to next occurence
|
|
index = builder.indexOf(key, index + key.length());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return builder.toString();
|
|
}
|
|
|
|
/**
|
|
* Load multiple translationLists from one File. Each List starts with #File: relative/path/to/file
|
|
* (within each file section translation is done in order of the language file entries, on conflicts
|
|
* put the shorter key to the end of the list)
|
|
* @param translationFile the File, which contains the Lists
|
|
* @return a HashMap, which contains for each File a HashMap with translations.
|
|
*/
|
|
public Map<String, Map<String, String>> loadTranslationsLists(final File translationFile) {
|
|
final Map<String, Map<String, String>> lists = new HashMap<String, Map<String, String>>(); //list of translationLists for different files.
|
|
Map<String, String> translationList = new LinkedHashMap<String, String>(); //current Translation Table (maintaining input order)
|
|
|
|
final List<String> list = FileUtils.getListArray(translationFile);
|
|
String forFile = "";
|
|
|
|
for (final String line : list) {
|
|
if (!line.isEmpty()) {
|
|
if (line.charAt(0) != '#') {
|
|
final String[] split = line.split("==", 2);
|
|
if (split.length == 2) {
|
|
translationList.put(split[0], split[1]);
|
|
//}else{ //Invalid line
|
|
}
|
|
} else if (line.startsWith("#File:")) {
|
|
if (!forFile.isEmpty()) {
|
|
lists.put(forFile, translationList);
|
|
}
|
|
forFile = line.substring(6).trim(); //skip "#File:"
|
|
if (lists.containsKey(forFile)) {
|
|
translationList = lists.get(forFile);
|
|
} else {
|
|
translationList = new LinkedHashMap<String, String>();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
lists.put(forFile, translationList);
|
|
return lists;
|
|
}
|
|
|
|
/**
|
|
* Translate sourceFile to destFile using translationList.
|
|
* @param sourceFile file to translate
|
|
* @param destFile file to write
|
|
* @param translationList map of translations
|
|
* @return true when destFile was sucessfully written, false otherwise
|
|
*/
|
|
public boolean translateFile(final File sourceFile, final File destFile, final Map<String, String> translationList){
|
|
|
|
StringBuilder content = new StringBuilder();
|
|
BufferedReader br = null;
|
|
try{
|
|
br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile), StandardCharsets.UTF_8));
|
|
String line = null;
|
|
while( (line = br.readLine()) != null){
|
|
content.append(line).append(net.yacy.server.serverCore.CRLF_STRING);
|
|
}
|
|
br.close();
|
|
} catch(final IOException e) {
|
|
return false;
|
|
} finally {
|
|
if (br !=null) {
|
|
try {
|
|
br.close();
|
|
} catch (final Exception e) {}
|
|
}
|
|
}
|
|
|
|
String processedContent = translate(content, translationList);
|
|
BufferedWriter bw = null;
|
|
try{
|
|
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(destFile), StandardCharsets.UTF_8));
|
|
bw.write(processedContent);
|
|
bw.close();
|
|
}catch(final IOException e){
|
|
return false;
|
|
} finally {
|
|
if (bw != null) {
|
|
try {
|
|
bw.close();
|
|
} catch (final Exception e) {}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Translate files in sourceDir (relative path of baseDir) write result to destDir
|
|
*
|
|
* @param sourceDir relative path
|
|
* @param destDir destination
|
|
* @param baseDir base dir of source
|
|
* @param translationLists translation to use
|
|
* @param extensions file extension to include in translation
|
|
* @return
|
|
*/
|
|
public boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final Map<String, Map<String, String>> translationLists, final String extensions){
|
|
destDir.mkdirs();
|
|
final List<String> exts = ListManager.string2vector(extensions);
|
|
final File[] sourceFiles = sourceDir.listFiles(new ExtensionsFileFilter(exts));
|
|
String relativePath;
|
|
for (final File sourceFile : sourceFiles) {
|
|
try {
|
|
relativePath=sourceFile.getAbsolutePath().substring(baseDir.getAbsolutePath().length()+1); //+1 to get the "/"
|
|
relativePath = relativePath.replace(File.separatorChar, '/');
|
|
} catch (final IndexOutOfBoundsException e) {
|
|
ConcurrentLog.severe("TRANSLATOR", "Error creating relative Path for "+sourceFile.getAbsolutePath());
|
|
relativePath = "wrong path"; //not in translationLists
|
|
}
|
|
if (translationLists.containsKey(relativePath)) {
|
|
ConcurrentLog.info("TRANSLATOR", "Translating file: "+ relativePath);
|
|
if(!translateFile(
|
|
sourceFile,
|
|
new File(destDir, sourceFile.getName().replace('/', File.separatorChar)),
|
|
translationLists.get(relativePath)))
|
|
{
|
|
ConcurrentLog.severe("TRANSLATOR", "File error while translating file "+relativePath);
|
|
}
|
|
//}else{
|
|
//serverLog.logInfo("TRANSLATOR", "No translation for file: "+relativePath);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Translate files starting with sourceDir and all subdirectories.
|
|
*
|
|
* @param sourceDir
|
|
* @param destDir
|
|
* @param translationFile translation language file
|
|
* @param extensions extension of files to include in translation
|
|
* @param notdir directory to exclude
|
|
* @return true if all files translated (or none)
|
|
*/
|
|
public boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir) {
|
|
final List<File> dirList = FileUtils.getDirsRecursive(sourceDir, notdir);
|
|
dirList.add(sourceDir);
|
|
final Map<String, Map<String, String>> translationLists = loadTranslationsLists(translationFile);
|
|
boolean erg = true;
|
|
for (final File file : dirList) {
|
|
if (file.isDirectory() && !file.getName().equals(notdir)) {
|
|
//cuts the sourcePath and prepends the destPath
|
|
File file2 = new File(destDir, file.getPath().substring(sourceDir.getPath().length()));
|
|
erg &= translateFiles(file, file2, sourceDir, translationLists, extensions);
|
|
}
|
|
}
|
|
return erg;
|
|
}
|
|
|
|
public static Map<String, String> langMap(@SuppressWarnings("unused") final serverSwitch env) {
|
|
final String[] ms = CommonPattern.COMMA.split(
|
|
"browser/Browser Language," +
|
|
"default/English,de/Deutsch,fr/Français,nl/Nederlands,it/Italiano,es/Español,pt/Portugês,fi/Suomi,se/Svenska,dk/Dansk," +
|
|
"gr/Eλληvικα,sk/Slovensky,cn/汉语/漢語," +
|
|
"ru/Русский,uk/Українська," +
|
|
"hi/हिन्दी,ja/日本語"
|
|
);
|
|
final Map<String, String> map = new HashMap<String, String>();
|
|
for (final String element : ms) {
|
|
int p = element.indexOf('/');
|
|
if (p > 0)
|
|
map.put(element.substring(0, p), element.substring(p + 1));
|
|
}
|
|
return map;
|
|
}
|
|
|
|
public boolean changeLang(final serverSwitch env, final File langPath, final String lang) {
|
|
boolean ret = false;
|
|
|
|
if ("default".equals(lang) || "default.lng".equals(lang)) {
|
|
env.setConfig("locale.language", "default");
|
|
ret = true;
|
|
} else if ("browser".equals(lang) || "browser.lng".equals(lang)) {
|
|
env.setConfig("locale.language", "browser");
|
|
ret = true;
|
|
} else {
|
|
final String htRootPath = env.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT);
|
|
final File sourceDir = new File(env.getAppPath(), htRootPath);
|
|
final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut .lng
|
|
final File translationFile = new File(langPath, lang);
|
|
|
|
if (translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
|
|
env.setConfig("locale.language", lang.substring(0, lang.length() - 4));
|
|
Formatter.setLocale(env.getConfig("locale.language", "en"));
|
|
try {
|
|
final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
|
|
bw.write(env.getConfig(Seed.VERSION, "Error getting Version"));
|
|
bw.close();
|
|
} catch (final IOException e) {
|
|
// Error
|
|
}
|
|
ret = true;
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
public static List<String> langFiles(File langPath) {
|
|
return FileUtils.getDirListing(langPath, Translator.LANG_FILENAME_FILTER);
|
|
}
|
|
|
|
/**
|
|
* Helper to collect a list of available translations
|
|
* This looks in the locale directory for the subdirectories created on translation
|
|
* @return list of language-codes of available/active translations
|
|
*/
|
|
public static List<String> activeTranslations() {
|
|
Switchboard sb = Switchboard.getSwitchboard();
|
|
File localePath;
|
|
if (sb != null)
|
|
localePath = sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot");
|
|
else
|
|
localePath = new File ("DATA/LOCALE/htroot");
|
|
List<String> dirlist = new ArrayList<String>(); // get list of language subdirectories
|
|
File[] list = localePath.listFiles();
|
|
for (File f : list) {
|
|
if (f.isDirectory()) dirlist.add(f.getName()); // filter directories to add to result
|
|
}
|
|
return dirlist;
|
|
}
|
|
}
|