Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

introduce a translation edit servlet Translator_p.html YaCy's UI text translation

Browse Source 
This is the 1st rudimentary approach to support the translatio utilities.
It allows currently to edit untranslated text and save it in a local translation file
in the DATA/LOCALE directory.
+ refactor Translator (less static's) to leverage on class overrides and support garbage collection for this 1 time routine
+ adjust TranslatorXliff to check for local translations in DATA/LOCALE,
  this includes storing manually downloaded translation files in DATA as well 
  (to keep default untouched)
+ on 1st call of Translator_p a master tanslation file is generated, checking
the supported languages for missing translation text (later this masterfile is planned to part of the distribution, to harmonize translation key text between the languages)
Outlook: the local modifications (possibly as translation fragments instead of complete file) to be shared with maintainer using xlif features.
This commit is contained in:
reger 2016-06-03 01:46:30 +02:00
parent ab7f82f803
commit a6ba1faa80
14 changed files with 230 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
htroot/ConfigBasic.java
View File

@ -36,7 +36,6 @@ import java.util.regex.Pattern;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.Translator;
import net.yacy.data.WorkTables;
import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.workflow.InstantBusyThread;
@ -46,6 +45,7 @@ import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.utils.translation.TranslatorXliff;
import net.yacy.utils.upnp.UPnPMappingType;
import net.yacy.utils.upnp.UPnP;
@ -85,7 +85,7 @@ public class ConfigBasic {
// language settings
if (post != null && post.containsKey("language") && !lang.equals(post.get("language", "default")) &&
(Translator.changeLang(env, langPath, post.get("language", "default") + ".lng"))) {
(new TranslatorXliff().changeLang(env, langPath, post.get("language", "default") + ".lng"))) {
prop.put("changedLanguage", "1");
}

1
htroot/ConfigLanguage_p.html
View File

@ -57,6 +57,7 @@
<p>Make sure that you only download data from trustworthy sources. The new language file
might overwrite existing data if a file of the same name exists already.</p>
</fieldset>
<p><a href="Translator_p.html">Simple Editor</a> to add untranslated text</p>
</form>
#(status)#
::<p><strong>Unable to get URL: #[url]#</strong></p>

10
htroot/ConfigLanguage_p.java
View File

@ -82,7 +82,7 @@ public class ConfigLanguage_p {
* directory traversal attacks!
*/
if (langFiles.contains(selectedLanguage) || selectedLanguage.startsWith("default")) {
Translator.changeLang(env, langPath, selectedLanguage);
new TranslatorXliff().changeLang(env, langPath, selectedLanguage);
}
//delete language file
@ -105,7 +105,8 @@ public class ConfigLanguage_p {
final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, null, null));
try {
File langFile = new File(langPath, u.getFileName());
TranslatorXliff tx = new TranslatorXliff();
File langFile = tx.getScratchFile(new File(langPath, u.getFileName()));
final OutputStreamWriter bw = new OutputStreamWriter(new FileOutputStream(langFile), StandardCharsets.UTF_8.name());
while (it.hasNext()) {
@ -116,14 +117,13 @@ public class ConfigLanguage_p {
// convert downloaded xliff to internal lng file
final String ext = Files.getFileExtension(langFile.getName());
if (ext.equalsIgnoreCase("xlf") || ext.equalsIgnoreCase("xliff")) {
TranslatorXliff tx = new TranslatorXliff();
Map<String,Map<String,String>> lng = TranslatorXliff.loadTranslationsListsFromXliff(langFile);
Map<String,Map<String,String>> lng = tx.loadTranslationsListsFromXliff(langFile);
langFile = new File(langPath, Files.getNameWithoutExtension(langFile.getName())+".lng");
tx.saveAsLngFile(null, langFile, lng);
}
if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) {
Translator.changeLang(env, langPath, langFile.getName());
tx.changeLang(env, langPath, langFile.getName());
}
} catch (final IOException e) {
prop.put("status", "2");//error saving the language file

53
htroot/Translator_p.html Normal file
View File

@ -0,0 +1,53 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" >
<head>
<title>YaCy '#[clientname]#': Translation Editor</title>
#%env/templates/metas.template%#
</head>
<body id="Translator">
#%env/templates/header.template%#
#%env/templates/submenuDesign.template%#
<h2>Translation Editor</h2>
<p>Translate untranslated text of the user interface (current language). The modified translation file is stored in DATA/LOCALE directory</p>
<fieldset>
<legend>
<label>UI Translation</label>
</legend>
<p>Target Language: <b>#[targetlang]#</b></p><p class="error">#[errmsg]#</p>
<form id="Translation" method="post" action="Translator_p.html" enctype="multipart/form-data" accept-charset="UTF-8">
<label for="sourcefile">Source File</label>
<select name="sourcefile" onchange="submit();">
#{filelist}#
<option value="#[filename]#" #(selected)#::selected="selected"#(/selected)#>#[filename]#</option>
#{/filelist}#
</select> <a href="#[sourcefile]#" target="sourcefilewindow" >view it</a>
<label for="filteruntranslated">filter untranslated</label>
<input type="checkbox" name="filteruntranslated" id="filteruntranslated" value="true" onclick="submit();" #(filter.checked)#::checked="checked"#(/filter.checked)# />
<table>
<tr><th style="border-bottom: solid gray; border-bottom-width: 1px;">Source Text</th>
<th style="border-bottom: solid gray; border-bottom-width: 1px;">Translated Text ( #[targetlang]# )</th></tr>
#{textlist}#
<tr>
<td style="border-bottom: solid gray; border-bottom-width: 1px;" valign="top">
<!--<input type="text" name="sourcetxt" id="sourcetxt#[tokenid]#" size="80" disabled="true" value="#[sourcetxt]#"/>-->
<label for="targettxt#[tokenid]#" >#[sourcetxt]#</label>
</td>
<td style="border-bottom: solid gray; border-bottom-width: 1px;" valign="top" nowrap>
<input type="text" name="targettxt#[tokenid]#" id="targettxt#[tokenid]#" size="80" value="#[targettxt]#"/>#(filteruntranslated)#::<button name="approve" type="submit" value="#[tokenid]#" class="btn btn-sm"><span class="glyphicon glyphicon-ok-sign" style="color: green"/></button>#(/filteruntranslated)#
</td>
</tr>
#{/textlist}#
</table>
<input type="submit" name="savetranslationlist" value="Save translation" class="btn btn-primary"/>
</form>
</fieldset>
#%env/templates/footer.template%#
</body>
</html>

127
htroot/Translator_p.java Normal file
View File

@ -0,0 +1,127 @@
/**
* Translator_p
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 14.09.2011 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.servletProperties;
import net.yacy.utils.translation.CreateTranslationMasters;
public class Translator_p {
public static servletProperties respond(@SuppressWarnings("unused") final RequestHeader requestHeader, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
try {
final servletProperties prop = new servletProperties();
final Switchboard sb = (Switchboard) env;
String langcfg = env.getConfig("locale.language", "default");
prop.put("targetlang", langcfg);
if ("default".equals(langcfg)) {
prop.put("errmsg", "activate a different language");
return prop;
} else {
prop.put("errmsg", "");
}
File lngfile = new File("locales", langcfg + ".lng");
CreateTranslationMasters ctm = new CreateTranslationMasters(/*new File ("locales","master.lng.xlf")*/);
File masterxlf = new File("locales", "master.lng.xlf");
if (!masterxlf.exists()) ctm.createMasterTranslationLists(masterxlf);
Map<String, Map<String, String>> origTrans = ctm.joinMasterTranslationLists(masterxlf, lngfile);
int i = 0;
if (origTrans.size() > 0) {
String filename = origTrans.keySet().iterator().next();
if (post != null && post.containsKey("sourcefile")) {
filename = post.get("sourcefile", filename);
}
Iterator<String> filenameit = origTrans.keySet().iterator();
while (filenameit.hasNext()) {
String tmp = filenameit.next();
prop.put("filelist_" + i + "_filename", tmp);
prop.put("filelist_" + i + "_selected", tmp.equals(filename));
i++;
}
prop.put("filelist", i);
prop.add("sourcefile", filename);
Map<String, String> origTextList = origTrans.get(filename);
i = 0;
boolean filteruntranslated = false;
int textlistid = -1;
if (post != null) {
filteruntranslated = post.getBoolean("filteruntranslated");
if (filteruntranslated) {
prop.put("filter.checked", 1);
} else {
prop.put("filter.checked", 0);
}
textlistid = post.getInt("approve", -1);
}
boolean changed = false;
for (String sourcetext : origTextList.keySet()) {
String targettxt = origTextList.get(sourcetext);
if (targettxt == null || targettxt.isEmpty()) {
prop.put("textlist_" + i + "_filteruntranslated", true);
} else if (filteruntranslated) {
continue;
}
if (i == textlistid && post != null) {
String t = post.get("targettxt" + Integer.toString(textlistid));
// correct common partial html markup (part of text identification for words also used as html parameter)
if (!t.isEmpty()) {
if (sourcetext.startsWith(">") && !t.startsWith(">")) t=">"+t;
if (sourcetext.endsWith("<") && !t.endsWith("<")) t=t+"<";
}
targettxt = t;
origTextList.replace(sourcetext, targettxt);
changed = true;
}
prop.putHTML("textlist_" + i + "_sourcetxt", sourcetext);
prop.putHTML("textlist_" + i + "_targettxt", targettxt);
prop.put("textlist_" + i + "_tokenid", Integer.toString(i));
prop.put("textlist_" + i + "_filteruntranslated_tokenid", Integer.toString(i));
//prop.put("textlist_" + i +"_filteruntranslated", filteruntranslated);
i++;
}
if (post != null && post.containsKey("savetranslationlist")) {
changed = true;
}
if (changed) {
ctm.saveAsLngFile(langcfg, ctm.getScratchFile(lngfile), origTrans);
}
}
prop.put("textlist", i);
return prop;
} catch (IOException ex) {
ConcurrentLog.logException(ex);
}
return null;
}
}

4
source/net/yacy/crawler/CrawlSwitchboard.java
View File

@ -521,8 +521,8 @@ public final class CrawlSwitchboard {
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE),
-1,
true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
false,
false,
false, // indexText
false, // indexMedia
true,
false,
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,

16
source/net/yacy/data/Translator.java
View File

@ -107,7 +107,7 @@ public class Translator {
* @param translationFile the File, which contains the Lists
* @return a HashMap, which contains for each File a HashMap with translations.
*/
public static Map<String, Map<String, String>> loadTranslationsLists(final File translationFile) {
public Map<String, Map<String, String>> loadTranslationsLists(final File translationFile) {
final Map<String, Map<String, String>> lists = new HashMap<String, Map<String, String>>(); //list of translationLists for different files.
Map<String, String> translationList = new LinkedHashMap<String, String>(); //current Translation Table (maintaining input order)
@ -186,7 +186,7 @@ public class Translator {
return true;
}
public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){
public boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){
return translateFiles(sourceDir, destDir, baseDir, loadTranslationsLists(translationFile), extensions);
}
@ -219,7 +219,7 @@ public class Translator {
return true;
}
public static boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){
public boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){
final List<File> dirList=FileUtils.getDirsRecursive(sourceDir, notdir);
dirList.add(sourceDir);
for (final File file : dirList) {
@ -248,7 +248,7 @@ public class Translator {
return map;
}
public static boolean changeLang(final serverSwitch env, final File langPath, final String lang) {
public boolean changeLang(final serverSwitch env, final File langPath, final String lang) {
boolean ret = false;
if ("default".equals(lang) || "default.lng".equals(lang)) {
@ -257,14 +257,10 @@ public class Translator {
} else {
final String htRootPath = env.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT);
final File sourceDir = new File(env.getAppPath(), htRootPath);
final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut
// .lng
//File destDir = new File(env.getRootPath(), htRootPath + "/locale/" + lang.substring(0, lang.length() - 4));// cut
// .lng
final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut .lng
final File translationFile = new File(langPath, lang);
//if (translator.translateFiles(sourceDir, destDir, translationFile, "html")) {
if (Translator.translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
if (translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
env.setConfig("locale.language", lang.substring(0, lang.length() - 4));
Formatter.setLocale(env.getConfig("locale.language", "en"));
try {

8
source/net/yacy/utils/translation/CreateTranslationMasters.java
View File

@ -89,7 +89,7 @@ public class CreateTranslationMasters extends TranslatorXliff {
public void createMasterTranslationLists(File masterOutputFile) throws IOException {
Map<String, Map<String, String>> xliffTrans;
if (masterOutputFile.exists()) // if file exists, conserve existing master content (may be updated by external tool)
xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(masterOutputFile);
xliffTrans = loadTranslationsListsFromXliff(masterOutputFile);
else
xliffTrans = new TreeMap<String, Map<String, String>>();
@ -97,7 +97,7 @@ public class CreateTranslationMasters extends TranslatorXliff {
for (String filename : lngFiles) {
// load translation list
ConcurrentLog.info("TRANSLATOR", "include translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(new File("locales", filename));
Map<String, Map<String, String>> origTrans = loadTranslationsLists(new File("locales", filename));
for (String transfilename : origTrans.keySet()) { // get translation filename
File checkfile = new File("htroot", transfilename);
@ -154,10 +154,10 @@ public class CreateTranslationMasters extends TranslatorXliff {
public Map<String, Map<String, String>> joinMasterTranslationLists(File xlifmaster, File lngfile) throws IOException {
final String filename = lngfile.getName();
Map<String, Map<String, String>> xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(xlifmaster);
Map<String, Map<String, String>> xliffTrans = loadTranslationsListsFromXliff(xlifmaster);
// load translation list
System.out.println("join into master translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(lngfile);
Map<String, Map<String, String>> origTrans = loadTranslationsLists(lngfile);
for (String transfilename : origTrans.keySet()) { // get translation filename
// compare translation list

2
source/net/yacy/utils/translation/ListNonTranslatedFiles.java
View File

@ -87,7 +87,7 @@ public class ListNonTranslatedFiles extends TranslatorUtil {
+ translationFile);
try {
Set<String> translatedRelativePaths = Translator.loadTranslationsLists(translationFile).keySet();
Set<String> translatedRelativePaths = new Translator().loadTranslationsLists(translationFile).keySet();
List<File> srcFiles = FileUtils.getFilesRecursive(sourceDir, excludedDir, fileFilter);

2
source/net/yacy/utils/translation/TranslateAll.java
View File

@ -86,7 +86,7 @@ public class TranslateAll extends TranslatorUtil {
File localeDestDir = new File(destDir, localeCode);
localeDestDir.mkdirs();
Translator.translateFilesRecursive(sourceDir, localeDestDir,
new Translator().translateFilesRecursive(sourceDir, localeDestDir,
translationFile, extensions, "locale");
}
}

2
source/net/yacy/utils/translation/TranslateAllToOneLanguage.java
View File

@ -66,7 +66,7 @@ public class TranslateAllToOneLanguage extends TranslatorUtil {
+ translationFile);
try {
Translator.translateFilesRecursive(sourceDir, destDir,
new Translator().translateFilesRecursive(sourceDir, destDir,
translationFile, extensions, "locale");
} finally {
ConcurrentLog.shutdown();

28
source/net/yacy/utils/translation/TranslatorXliff.java
View File

@ -44,6 +44,7 @@ import javax.xml.bind.Unmarshaller;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.Translator;
import net.yacy.search.Switchboard;
import org.oasis.xliff.core_12.Body;
import org.oasis.xliff.core_12.Target;
@ -66,7 +67,7 @@ public class TranslatorXliff extends Translator {
* @return a HashMap, which contains for each File a HashMap with
* translations.
*/
public static Map<String, Map<String, String>> loadTranslationsListsFromXliff(final File xliffFile) {
public Map<String, Map<String, String>> loadTranslationsListsFromXliff(final File xliffFile) {
final Map<String, Map<String, String>> lngLists = new TreeMap<String, Map<String, String>>(); //list of translationLists for different files.
/**
@ -144,11 +145,13 @@ public class TranslatorXliff extends Translator {
* @param xliffFile
* @return translatio map
*/
public static Map<String, Map<String, String>> loadTranslationsLists(final File xliffFile) {
@Override
public Map<String, Map<String, String>> loadTranslationsLists(final File xliffFile) {
File locallng = getScratchFile(xliffFile);
if (xliffFile.getName().toLowerCase().endsWith(".xlf") || xliffFile.getName().toLowerCase().endsWith(".xliff")) {
return loadTranslationsListsFromXliff(xliffFile);
return locallng.exists() ? loadTranslationsListsFromXliff(locallng) : loadTranslationsListsFromXliff(xliffFile);
} else {
return Translator.loadTranslationsLists(xliffFile);
return locallng.exists() ? super.loadTranslationsLists(locallng) : super.loadTranslationsLists(xliffFile);
}
}
@ -317,4 +320,21 @@ public class TranslatorXliff extends Translator {
}
return s;
}
/**
* Get the path to a work/scratch file in the DATA/LOCALE directory with the
* same name as given in the langPath
*
* @param langFile the path with filename to the language file
* @return a path to DATA/LOCALE/langFile.filename()
*/
public File getScratchFile(final File langFile) {
if (Switchboard.getSwitchboard() != null) { // for debug and testing were switchboard is null
File f = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE");
f = new File(f.getParentFile(), langFile.getName());
return f;
} else {
return langFile;
}
}
}

4
source/net/yacy/yacy.java
View File

@ -46,7 +46,6 @@ import net.yacy.cora.protocol.TimeoutRequest;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.sorting.Array;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.Translator;
import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser;
import net.yacy.http.Jetty9HttpServerImpl;
@ -67,6 +66,7 @@ import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.crawler.retrieval.Response;
import net.yacy.peers.Seed;
import net.yacy.server.serverSwitch;
import net.yacy.utils.translation.TranslatorXliff;
/**
@ -362,7 +362,7 @@ public final class yacy {
if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) try { //is this another version?!
final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang);
if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
if (new TranslatorXliff().translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
//write the new Versionnumber
final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
bw.write(sb.getConfig(Seed.VERSION, "Error getting Version"));

2
test/java/net/yacy/utils/translation/TranslatorXliffTest.java
View File

@ -31,7 +31,7 @@ public class TranslatorXliffTest {
for (String filename : lngFiles) {
// load translation list
System.out.println("Test translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(new File("locales", filename));
Map<String, Map<String, String>> origTrans = new Translator().loadTranslationsLists(new File("locales", filename));
TranslatorXliff txlif = new TranslatorXliff();
// save as xliff file