mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
added some UTF-8 handling.
hope this will help somehow.. for shure not THE solution to our UTF-8 problem git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1308 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter
parent
7586fcb1a4
commit
9544c47684
|
|
@ -44,6 +44,7 @@
|
|||
// javac -classpath .:../Classes MessageSend_p.java
|
||||
// if the shell's current path is HTROOT
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -128,7 +129,13 @@ public class MessageSend_p {
|
|||
if (messagesize < 1000) messagesize = 1000; // debug
|
||||
if (subject.length() > 100) subject = subject.substring(0, 100);
|
||||
if (message.length() > messagesize) message = message.substring(0, messagesize);
|
||||
HashMap result = yacyClient.postMessage(hash, subject, message.getBytes());
|
||||
byte[] mb;
|
||||
try {
|
||||
mb = message.getBytes("UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
mb = message.getBytes();
|
||||
}
|
||||
HashMap result = yacyClient.postMessage(hash, subject, mb);
|
||||
body += "<p>Your message has been sent. The target peer responded:</p>";
|
||||
body += "<p><i>" + result.get("response") + "</i></p>";
|
||||
} catch (NumberFormatException e) {
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ public class Wiki {
|
|||
}
|
||||
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws IOException {
|
||||
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
|
||||
serverObjects prop = new serverObjects();
|
||||
if (post == null) {
|
||||
|
|
@ -94,11 +94,15 @@ public class Wiki {
|
|||
}
|
||||
}
|
||||
|
||||
if (post.containsKey("submit")) {
|
||||
// store a new page
|
||||
switchboard.wikiDB.write(switchboard.wikiDB.newEntry(pagename, author, ip,
|
||||
post.get("reason", "edit"),
|
||||
post.get("content", "").getBytes()));
|
||||
if (post.containsKey("submit")) {
|
||||
// store a new page
|
||||
byte[] content;
|
||||
try {
|
||||
content = post.get("content", "").getBytes("UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
content = post.get("content", "").getBytes();
|
||||
}
|
||||
switchboard.wikiDB.write(switchboard.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content));
|
||||
// create a news message
|
||||
HashMap map = new HashMap();
|
||||
map.put("page", pagename);
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ public class dir {
|
|||
try {
|
||||
serverFileUtils.write(binary, newfile);
|
||||
String md5s = serverCodings.encodeMD5Hex(newfile);
|
||||
serverFileUtils.write((md5s + "\n" + description).getBytes(), newfilemd5); // generate md5
|
||||
serverFileUtils.write((md5s + "\n" + description).getBytes("UTF-8"), newfilemd5); // generate md5
|
||||
|
||||
// index file info
|
||||
if (post.get("indexing", "").equals("on")) {
|
||||
|
|
@ -262,7 +262,7 @@ public class dir {
|
|||
// generate md5 on-the-fly
|
||||
md5s = serverCodings.encodeMD5Hex(f);
|
||||
description = "";
|
||||
serverFileUtils.write((md5s + "\n" + description).getBytes(), fmd5);
|
||||
serverFileUtils.write((md5s + "\n" + description).getBytes("UTF-8"), fmd5);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
md5s = "";
|
||||
|
|
@ -478,7 +478,7 @@ public class dir {
|
|||
public static void deletePhrase(plasmaSwitchboard switchboard, String urlstring, String phrase, String descr) {
|
||||
try {
|
||||
final String urlhash = plasmaURL.urlHash(new URL(urlstring));
|
||||
final Set words = plasmaCondenser.getWords(("yacyshare " + phrase + " " + descr).getBytes());
|
||||
final Set words = plasmaCondenser.getWords(("yacyshare " + phrase + " " + descr).getBytes("UTF-8"));
|
||||
switchboard.removeReferences(urlhash, words);
|
||||
switchboard.urlPool.loadedURL.remove(urlhash);
|
||||
} catch (Exception e) {
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import de.anomic.data.messageBoard;
|
||||
|
|
@ -125,11 +126,17 @@ public final class message {
|
|||
|
||||
// save message
|
||||
messageBoard.entry msgEntry = null;
|
||||
byte[] mb;
|
||||
try {
|
||||
mb = message.getBytes("UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
mb = message.getBytes();
|
||||
}
|
||||
sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
|
||||
"remote",
|
||||
otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,
|
||||
yacyCore.seedDB.mySeed.getName(), yacyCore.seedDB.mySeed.hash,
|
||||
subject, message.getBytes()));
|
||||
subject, mb));
|
||||
|
||||
messageForwardingViaEmail(ss, msgEntry);
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ public class dbtest {
|
|||
|
||||
public final static int keylength = 12;
|
||||
public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
|
||||
//public final static long buffer = 0;
|
||||
public final static long buffer = 8192 * 1024; // 8 MB buffer
|
||||
public static byte[] dummyvalue1 = new byte[valuelength];
|
||||
public static byte[] dummyvalue2 = new byte[valuelength];
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ public class wikiBoard {
|
|||
return author;
|
||||
}
|
||||
|
||||
public entry newEntry(String subject, String author, String ip, String reason, byte[] page) {
|
||||
public entry newEntry(String subject, String author, String ip, String reason, byte[] page) throws IOException {
|
||||
return new entry(normalize(subject), author, ip, reason, page);
|
||||
}
|
||||
|
||||
|
|
@ -158,17 +158,17 @@ public class wikiBoard {
|
|||
String key;
|
||||
Map record;
|
||||
|
||||
public entry(String subject, String author, String ip, String reason, byte[] page) {
|
||||
public entry(String subject, String author, String ip, String reason, byte[] page) throws IOException {
|
||||
record = new HashMap();
|
||||
key = subject;
|
||||
if (key.length() > keyLength) key = key.substring(0, keyLength);
|
||||
record.put("date", dateString());
|
||||
if ((author == null) || (author.length() == 0)) author = "anonymous";
|
||||
record.put("author", kelondroBase64Order.enhancedCoder.encode(author.getBytes()));
|
||||
record.put("author", kelondroBase64Order.enhancedCoder.encode(author.getBytes("UTF-8")));
|
||||
if ((ip == null) || (ip.length() == 0)) ip = "";
|
||||
record.put("ip", ip);
|
||||
if ((reason == null) || (reason.length() == 0)) reason = "";
|
||||
record.put("reason", kelondroBase64Order.enhancedCoder.encode(reason.getBytes()));
|
||||
record.put("reason", kelondroBase64Order.enhancedCoder.encode(reason.getBytes("UTF-8")));
|
||||
if (page == null)
|
||||
record.put("page", "");
|
||||
else
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ import java.io.BufferedReader;
|
|||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
|
||||
import de.anomic.plasma.plasmaSwitchboard;
|
||||
import de.anomic.server.serverCore;
|
||||
|
|
@ -73,7 +74,11 @@ public class wikiCode {
|
|||
}
|
||||
|
||||
public String transform(String content){
|
||||
return transform(content.getBytes(), sb);
|
||||
try {
|
||||
return transform(content.getBytes("UTF-8"), sb);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return transform(content.getBytes(), sb);
|
||||
}
|
||||
}
|
||||
public String transform(byte[] content){
|
||||
return transform(content, sb);
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -230,7 +230,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
|
|||
}
|
||||
|
||||
public byte[] getText() {
|
||||
return content.getBytes();
|
||||
return content.getBytes();
|
||||
}
|
||||
|
||||
public Map getAnchors() {
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ package de.anomic.htmlFilter;
|
|||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.text.Collator;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Locale;
|
||||
|
|
@ -117,7 +118,12 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
|
|||
|
||||
private boolean hit(byte[] text) {
|
||||
if (text == null || bluelist == null) return false;
|
||||
String lc = new String(text).toLowerCase();
|
||||
String lc;
|
||||
try {
|
||||
lc = new String(text, "UTF-8").toLowerCase();
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
lc = new String(text).toLowerCase();
|
||||
}
|
||||
for (int i = 0; i < bluelist.size(); i++) {
|
||||
if (lc.indexOf((String) bluelist.get(i)) >= 0) return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -710,7 +710,7 @@ public final class httpd implements serverHandler {
|
|||
bout.close(); bout = null;
|
||||
}
|
||||
|
||||
int argc = parseArgs(args, new String(buffer));
|
||||
int argc = parseArgs(args, new String(buffer, "UTF-8"));
|
||||
buffer = null;
|
||||
return argc;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ abstract class kelondroAbstractRA implements kelondroRA {
|
|||
return new String(bb, 0, bbsize);
|
||||
}
|
||||
if (c == cr) continue;
|
||||
if (c == lf) return new String(bb, 0, bbsize);
|
||||
if (c == lf) return new String(bb, 0, bbsize, "UTF-8");
|
||||
|
||||
// append to bb
|
||||
if (bbsize == bb.length) {
|
||||
|
|
|
|||
|
|
@ -98,7 +98,6 @@ public class kelondroArray extends kelondroRecords {
|
|||
return getNode(new Handle(index)).getValues();
|
||||
}
|
||||
|
||||
|
||||
public synchronized int seti(int index, int value) throws IOException {
|
||||
int before = getHandle(index).hashCode();
|
||||
setHandle(index, new Handle(value));
|
||||
|
|
@ -109,13 +108,28 @@ public class kelondroArray extends kelondroRecords {
|
|||
return getHandle(index).hashCode();
|
||||
}
|
||||
|
||||
public synchronized int add(byte[][] row) throws IOException {
|
||||
if (row.length != columns())
|
||||
throw new IllegalArgumentException("add: wrong row length " + row.length + "; must be " + columns());
|
||||
|
||||
Node n = newNode();
|
||||
n.commit(CP_LOW);
|
||||
int index = n.handle().hashCode();
|
||||
set(index, row);
|
||||
return index;
|
||||
}
|
||||
|
||||
public synchronized void remove(int index) throws IOException {
|
||||
deleteNode(new Handle(index));
|
||||
}
|
||||
|
||||
public void print() throws IOException {
|
||||
System.out.println("PRINTOUT of table, length=" + size());
|
||||
byte[][] row;
|
||||
for (int i = 0; i < size(); i++) {
|
||||
System.out.print("row " + i + ": ");
|
||||
row = get(i);
|
||||
for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j])) + ", ");
|
||||
for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j], "UTF-8")) + ", ");
|
||||
System.out.println();
|
||||
}
|
||||
System.out.println("EndOfTable");
|
||||
|
|
@ -160,6 +174,20 @@ public class kelondroArray extends kelondroRecords {
|
|||
fm.set(Integer.parseInt(args[2]), row);
|
||||
fm.close();
|
||||
} else
|
||||
if ((args.length == 3) && (args[0].equals("-a"))) {
|
||||
// add <filename> <value>
|
||||
kelondroArray fm = new kelondroArray(new File(args[1]));
|
||||
byte[][] row = new byte[][] { args[2].getBytes() };
|
||||
int index = fm.add(row);
|
||||
System.out.println("Added to row " + index);
|
||||
fm.close();
|
||||
} else
|
||||
if ((args.length == 3) && (args[0].equals("-d"))) {
|
||||
// delete <filename> <index>
|
||||
kelondroArray fm = new kelondroArray(new File(args[1]));
|
||||
fm.remove(Integer.parseInt(args[2]));
|
||||
fm.close();
|
||||
} else
|
||||
if ((args.length == 1) && (args[0].equals("-test"))) {
|
||||
File testfile = new File("test.array");
|
||||
if (testfile.exists()) testfile.delete();
|
||||
|
|
|
|||
|
|
@ -52,6 +52,19 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
|
|||
|
||||
private static final char[] alpha_standard = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray();
|
||||
private static final char[] alpha_enhanced = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray();
|
||||
private static final byte[] ahpla_standard = new byte[256];
|
||||
private static final byte[] ahpla_enhanced = new byte[256];
|
||||
|
||||
static {
|
||||
for (int i = 0; i < 256; i++) {
|
||||
ahpla_standard[i] = -1;
|
||||
ahpla_enhanced[i] = -1;
|
||||
}
|
||||
for (int i = 0; i < alpha_standard.length; i++) {
|
||||
ahpla_standard[alpha_standard[i]] = (byte) i;
|
||||
ahpla_enhanced[alpha_enhanced[i]] = (byte) i;
|
||||
}
|
||||
}
|
||||
|
||||
public static final kelondroBase64Order standardCoder = new kelondroBase64Order(true);
|
||||
public static final kelondroBase64Order enhancedCoder = new kelondroBase64Order(false);
|
||||
|
|
@ -59,15 +72,14 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
|
|||
final boolean rfc1113compliant;
|
||||
|
||||
private final char[] alpha;
|
||||
private final byte[] ahpla = new byte[256];
|
||||
private final byte[] ahpla;
|
||||
|
||||
public kelondroBase64Order(boolean rfc1113compliant) {
|
||||
// if we choose not to be rfc1113compliant,
|
||||
// then we get shorter base64 results which are also filename-compatible
|
||||
this.rfc1113compliant = rfc1113compliant;
|
||||
alpha = (rfc1113compliant) ? alpha_standard : alpha_enhanced;
|
||||
for (int i = 0; i < 256; i++) ahpla[i] = -1;
|
||||
for (int i = 0; i < alpha.length; i++) ahpla[alpha[i]] = (byte) i;
|
||||
ahpla = (rfc1113compliant) ? ahpla_standard : ahpla_enhanced;
|
||||
}
|
||||
|
||||
public char encodeByte(byte b) {
|
||||
|
|
@ -143,7 +155,8 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
|
|||
|
||||
public String decodeString(String in) {
|
||||
try {
|
||||
return new String(decode(in), "ISO-8859-1");
|
||||
//return new String(decode(in), "ISO-8859-1");
|
||||
return new String(decode(in), "UTF-8");
|
||||
} catch (java.io.UnsupportedEncodingException e) {
|
||||
System.out.println("internal error in base64: " + e.getMessage());
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -230,7 +230,7 @@ public class kelondroHashtable {
|
|||
rowNumber = hash.node();
|
||||
if (rowNumber >= hashArray.size()) return new Object[]{new Integer(rowNumber), null};
|
||||
row = hashArray.get(rowNumber);
|
||||
rowKey = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[0]));
|
||||
rowKey = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[0], "UTF-8"));
|
||||
if (rowKey == 0) return new Object[]{new Integer(rowNumber), null};
|
||||
hash.rehash();
|
||||
} while (rowKey != hash.key());
|
||||
|
|
|
|||
|
|
@ -68,6 +68,10 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
|
|||
// two arrays are also equal if one array is a subset of the other's array
|
||||
// with filled-up char(0)-values
|
||||
public int compare(byte[] a, byte[] b) {
|
||||
return compares(a, b);
|
||||
}
|
||||
|
||||
public static final int compares(byte[] a, byte[] b) {
|
||||
int i = 0;
|
||||
final int al = a.length;
|
||||
final int bl = b.length;
|
||||
|
|
|
|||
|
|
@ -790,7 +790,7 @@ public class kelondroRecords {
|
|||
if (h == null) s = s + ":hNULL"; else s = s + ":h" + h.toString();
|
||||
}
|
||||
byte[][] content = getValues();
|
||||
for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i])).trim());
|
||||
for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i], "UTF-8")).trim());
|
||||
} catch (IOException e) {
|
||||
s = s + ":***LOAD ERROR***:" + e.getMessage();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -130,11 +130,11 @@ public class odtParser extends AbstractParser implements Parser {
|
|||
if (docShortTitle != null) {
|
||||
docLongTitle = docShortTitle;
|
||||
} else if (docContent.length <= 80) {
|
||||
docLongTitle = new String(docContent);
|
||||
docLongTitle = new String(docContent, "UTF-8");
|
||||
} else {
|
||||
byte[] title = new byte[80];
|
||||
System.arraycopy(docContent, 0, title, 0, 80);
|
||||
docLongTitle = new String(title);
|
||||
docLongTitle = new String(title, "UTF-8");
|
||||
}
|
||||
docLongTitle.
|
||||
replaceAll("\r\n"," ").
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ public class pdfParser extends AbstractParser implements Parser {
|
|||
out = null;
|
||||
|
||||
if ((docTitle == null) || (docTitle.length() == 0)) {
|
||||
docTitle = ((contents.length > 80)? new String(contents, 0, 80):new String(contents)).
|
||||
docTitle = ((contents.length > 80)? new String(contents, 0, 80, "UTF-8"):new String(contents, "UTF-8")).
|
||||
replaceAll("\r\n"," ").
|
||||
replaceAll("\n"," ").
|
||||
replaceAll("\r"," ").
|
||||
|
|
|
|||
|
|
@ -171,15 +171,15 @@ public class plasmaCrawlEURL extends plasmaURL {
|
|||
this.hash = hash;
|
||||
byte[][] entry = urlHashCache.get(hash.getBytes());
|
||||
if (entry != null) {
|
||||
this.referrer = new String(entry[1]);
|
||||
this.initiator = new String(entry[2]);
|
||||
this.executor = new String(entry[3]);
|
||||
this.url = new URL(new String(entry[4]).trim());
|
||||
this.name = new String(entry[5]).trim();
|
||||
this.initdate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6])));
|
||||
this.trydate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[7])));
|
||||
this.trycount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8]));
|
||||
this.failreason = new String(entry[9]);
|
||||
this.referrer = new String(entry[1], "UTF-8");
|
||||
this.initiator = new String(entry[2], "UTF-8");
|
||||
this.executor = new String(entry[3], "UTF-8");
|
||||
this.url = new URL(new String(entry[4], "UTF-8").trim());
|
||||
this.name = new String(entry[5], "UTF-8").trim();
|
||||
this.initdate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6], "UTF-8")));
|
||||
this.trydate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[7], "UTF-8")));
|
||||
this.trycount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8], "UTF-8"));
|
||||
this.failreason = new String(entry[9], "UTF-8");
|
||||
this.flags = new bitfield(entry[10]);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -475,18 +475,18 @@ public final class plasmaCrawlLURL extends plasmaURL {
|
|||
if (entry == null) throw new IOException("url hash " + urlHash + " not found in LURL");
|
||||
try {
|
||||
if (entry != null) {
|
||||
this.url = new URL(new String(entry[1]).trim());
|
||||
this.descr = (entry[2] == null) ? this.url.toString() : new String(entry[2]).trim();
|
||||
this.moddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[3])));
|
||||
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[4])));
|
||||
this.referrerHash = (entry[5] == null) ? dummyHash : new String(entry[5]);
|
||||
this.copyCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6]));
|
||||
this.flags = new String(entry[7]);
|
||||
this.quality = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8]));
|
||||
this.language = new String(entry[9]);
|
||||
this.url = new URL(new String(entry[1], "UTF-8").trim());
|
||||
this.descr = (entry[2] == null) ? this.url.toString() : new String(entry[2], "UTF-8").trim();
|
||||
this.moddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[3], "UTF-8")));
|
||||
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[4], "UTF-8")));
|
||||
this.referrerHash = (entry[5] == null) ? dummyHash : new String(entry[5], "UTF-8");
|
||||
this.copyCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6], "UTF-8"));
|
||||
this.flags = new String(entry[7], "UTF-8");
|
||||
this.quality = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8], "UTF-8"));
|
||||
this.language = new String(entry[9], "UTF-8");
|
||||
this.doctype = (char) entry[10][0];
|
||||
this.size = kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[11]));
|
||||
this.wordCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[12]));
|
||||
this.size = kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[11], "UTF-8"));
|
||||
this.wordCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[12], "UTF-8"));
|
||||
this.snippet = null;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -189,43 +189,43 @@ public class plasmaCrawlNURL extends plasmaURL {
|
|||
Iterator i;
|
||||
try {
|
||||
i = coreStack.iterator();
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
coreStack.reset();
|
||||
}
|
||||
try {
|
||||
i = limitStack.iterator();
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
limitStack.reset();
|
||||
}
|
||||
try {
|
||||
i = overhangStack.iterator();
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
overhangStack.reset();
|
||||
}
|
||||
try {
|
||||
i = remoteStack.iterator();
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
|
||||
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
remoteStack.reset();
|
||||
}
|
||||
try {
|
||||
i = imageStack.iterator();
|
||||
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
|
||||
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
imageStack = kelondroStack.reset(imageStack);
|
||||
}
|
||||
try {
|
||||
i = movieStack.iterator();
|
||||
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
|
||||
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
movieStack = kelondroStack.reset(movieStack);
|
||||
}
|
||||
try {
|
||||
i = musicStack.iterator();
|
||||
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
|
||||
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
musicStack = kelondroStack.reset(musicStack);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -401,17 +401,17 @@ public final class plasmaCrawlStacker {
|
|||
|
||||
try {
|
||||
this.urlHash = urlHash;
|
||||
this.initiator = new String(entryBytes[1]);
|
||||
this.url = new String(entryBytes[2]).trim();
|
||||
this.referrerHash = (entryBytes[3]==null) ? plasmaURL.dummyHash : new String(entryBytes[3]);
|
||||
this.name = (entryBytes[4] == null) ? "" : new String(entryBytes[4]).trim();
|
||||
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5])));
|
||||
this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6]).trim();
|
||||
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7]));
|
||||
this.anchors = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8]));
|
||||
this.forkfactor = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9]));
|
||||
this.initiator = new String(entryBytes[1], "UTF-8");
|
||||
this.url = new String(entryBytes[2], "UTF-8").trim();
|
||||
this.referrerHash = (entryBytes[3]==null) ? plasmaURL.dummyHash : new String(entryBytes[3], "UTF-8");
|
||||
this.name = (entryBytes[4] == null) ? "" : new String(entryBytes[4], "UTF-8").trim();
|
||||
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5], "UTF-8")));
|
||||
this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6], "UTF-8").trim();
|
||||
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7], "UTF-8"));
|
||||
this.anchors = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8], "UTF-8"));
|
||||
this.forkfactor = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9], "UTF-8"));
|
||||
this.flags = new bitfield(entryBytes[10]);
|
||||
this.handle = Integer.parseInt(new String(entryBytes[11]));
|
||||
this.handle = Integer.parseInt(new String(entryBytes[11], "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw new IllegalStateException();
|
||||
|
|
|
|||
|
|
@ -201,21 +201,21 @@ public class plasmaSwitchboardQueue {
|
|||
this.referrerURL = null;
|
||||
}
|
||||
|
||||
public Entry(byte[][] row) {
|
||||
long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2]));
|
||||
public Entry(byte[][] row) throws IOException {
|
||||
long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2], "UTF-8"));
|
||||
byte flags = (row[3] == null) ? 0 : row[3][0];
|
||||
try {
|
||||
this.url = new URL(new String(row[0]));
|
||||
this.url = new URL(new String(row[0], "UTF-8"));
|
||||
} catch (MalformedURLException e) {
|
||||
this.url = null;
|
||||
}
|
||||
this.referrerHash = (row[1] == null) ? null : new String(row[1]);
|
||||
this.referrerHash = (row[1] == null) ? null : new String(row[1], "UTF-8");
|
||||
this.ifModifiedSince = (ims == 0) ? null : new Date(ims);
|
||||
this.flags = ((flags & 1) == 1) ? (byte) 1 : (byte) 0;
|
||||
this.initiator = (row[4] == null) ? null : new String(row[4]);
|
||||
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[5]));
|
||||
this.profileHandle = new String(row[6]);
|
||||
this.anchorName = (row[7] == null) ? null : (new String(row[7])).trim();
|
||||
this.initiator = (row[4] == null) ? null : new String(row[4], "UTF-8");
|
||||
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[5], "UTF-8"));
|
||||
this.profileHandle = new String(row[6], "UTF-8");
|
||||
this.anchorName = (row[7] == null) ? null : (new String(row[7], "UTF-8")).trim();
|
||||
|
||||
this.profileEntry = null;
|
||||
this.responseHeader = null;
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ public class plasmaWordConnotation {
|
|||
//reference = reference.toLowerCase();
|
||||
byte[][] record = refDB.get(word, reference.getBytes());
|
||||
long c;
|
||||
if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1]));
|
||||
if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1], "UTF-8"));
|
||||
record[1] = kelondroBase64Order.enhancedCoder.encodeLong(c++, countlength).getBytes();
|
||||
refDB.put(word, record);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -200,8 +200,7 @@ public final class plasmaWordIndexAssortment {
|
|||
for (int i = 0; i < assortmentLength; i++) {
|
||||
container.add(
|
||||
new plasmaWordIndexEntry[] { new plasmaWordIndexEntry(
|
||||
new String(row[3 + 2 * i]), new String(
|
||||
row[4 + 2 * i])) }, updateTime);
|
||||
new String(row[3 + 2 * i]), new String(row[4 + 2 * i])) }, updateTime);
|
||||
}
|
||||
return container;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -193,9 +193,9 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
|
|||
// get out one entry
|
||||
row = dumpArray.get(i);
|
||||
if ((row[0] == null) || (row[1] == null) || (row[2] == null) || (row[3] == null) || (row[4] == null)) continue;
|
||||
wordHash = new String(row[0]);
|
||||
wordHash = new String(row[0], "UTF-8");
|
||||
creationTime = kelondroRecords.bytes2long(row[2]);
|
||||
wordEntry = new plasmaWordIndexEntry(new String(row[3]), new String(row[4]));
|
||||
wordEntry = new plasmaWordIndexEntry(new String(row[3], "UTF-8"), new String(row[4], "UTF-8"));
|
||||
// store to cache
|
||||
addEntry(wordHash, wordEntry, creationTime);
|
||||
urlCount++;
|
||||
|
|
|
|||
|
|
@ -241,7 +241,7 @@ public final class serverFileUtils {
|
|||
public static Set loadSet(File file, String sep, boolean tree) throws IOException {
|
||||
Set set = (tree) ? (Set) new TreeSet() : (Set) new HashSet();
|
||||
byte[] b = read(file);
|
||||
StringTokenizer st = new StringTokenizer(new String(b), sep);
|
||||
StringTokenizer st = new StringTokenizer(new String(b, "UTF-8"), sep);
|
||||
while (st.hasMoreTokens()) {
|
||||
set.add(st.nextToken());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -468,7 +468,7 @@ public final class httpdSoapHandler extends httpdAbstractHandler implements http
|
|||
if (templates[i].endsWith(".template")) try {
|
||||
//System.out.println("TEMPLATE " + templates[i].substring(0, templates[i].length() - 9) + ": " + new String(buf, 0, c));
|
||||
result.put(templates[i].substring(0, templates[i].length() - 9),
|
||||
new String(serverFileUtils.read(new File(path, templates[i]))));
|
||||
new String(serverFileUtils.read(new File(path, templates[i])), "UTF-8"));
|
||||
} catch (Exception e) {}
|
||||
}
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -232,7 +232,7 @@ public class httpdSoapService
|
|||
|
||||
// convert it into a byte array and send it back as result
|
||||
byte[] result = o.toByteArray();
|
||||
return new String(result);
|
||||
return new String(result, "UTF-8");
|
||||
} catch (Exception e) {
|
||||
throw new AxisFault(e.getMessage());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ public class cryptbig {
|
|||
if (b64dec == null) return null; // error in input string (inconsistency)
|
||||
byte[] dec = decryptArray(b64dec);
|
||||
if (dec == null) return null;
|
||||
return new String(dec, "UTF8");
|
||||
return new String(dec, "UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
}
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ public class gzip {
|
|||
copy(fout, fin, 128);
|
||||
fin.close();
|
||||
fout.close();
|
||||
return new String(fout.toByteArray(), "UTF8");
|
||||
return new String(fout.toByteArray(), "UTF-8");
|
||||
} catch (IOException e) {
|
||||
System.err.println("ERROR: IO trouble");
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@
|
|||
package de.anomic.yacy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
|
|
@ -550,8 +551,11 @@ public final class yacyClient {
|
|||
post.put("youare", targetHash);
|
||||
post.put("subject", subject);
|
||||
post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
|
||||
post.put("message", new String(message));
|
||||
|
||||
try {
|
||||
post.put("message", new String(message, "UTF-8"));
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
post.put("message", new String(message));
|
||||
}
|
||||
// get target address
|
||||
String address = targetAddress(targetHash);
|
||||
|
||||
|
|
|
|||
|
|
@ -1165,7 +1165,7 @@ public final class yacy {
|
|||
entry = (plasmaCrawlLURL.Entry) eiter.next();
|
||||
if ((entry != null) && (entry.url() != null)) {
|
||||
if (html) {
|
||||
bos.write(("<a href=\"" + entry.url() + "\">" + entry.descr() + "</a><br>").getBytes());
|
||||
bos.write(("<a href=\"" + entry.url() + "\">" + entry.descr() + "</a><br>").getBytes("UTF-8"));
|
||||
bos.write(serverCore.crlf);
|
||||
} else {
|
||||
bos.write(entry.url().toString().getBytes());
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user