mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
replaced some more .getBytes() with UTF8/ASCII.getBytes()
This commit is contained in:
parent
bd769de604
commit
a06930662c
|
@ -193,7 +193,7 @@ public final class transferRWI {
|
|||
|
||||
// learn entry
|
||||
try {
|
||||
sb.index.storeRWI(wordHash.getBytes(), iEntry);
|
||||
sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry);
|
||||
} catch (final Exception e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
|
|
|
@ -32,7 +32,9 @@ import java.io.IOException;
|
|||
import java.io.PrintStream;
|
||||
import java.util.Date;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.protocol.ResponseHeader;
|
||||
|
@ -126,14 +128,14 @@ public class FTPLoader {
|
|||
final ResponseHeader responseHeader = new ResponseHeader(200);
|
||||
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date()));
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
responseHeader,
|
||||
profile,
|
||||
false,
|
||||
dirList.toString().getBytes());
|
||||
UTF8.getBytes(dirList.toString()));
|
||||
}
|
||||
} else {
|
||||
// file -> download
|
||||
|
@ -152,7 +154,7 @@ public class FTPLoader {
|
|||
if (berr.size() > 0 || response == null) {
|
||||
// some error logging
|
||||
final String detail = (berr.size() > 0) ? "Errorlog: " + berr.toString() : "";
|
||||
this.sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, " ftp server download, " + detail, -1);
|
||||
this.sb.crawlQueues.errorURL.push(request, ASCII.getBytes(this.sb.peers.mySeed().hash), new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, " ftp server download, " + detail, -1);
|
||||
throw new IOException("FTPLoader: Unable to download URL '" + request.url().toString() + "': " + detail);
|
||||
}
|
||||
|
||||
|
@ -245,7 +247,7 @@ public class FTPLoader {
|
|||
|
||||
// create response with metadata only
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain");
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
final Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
|
@ -260,7 +262,7 @@ public class FTPLoader {
|
|||
final byte[] b = ftpClient.get(path);
|
||||
|
||||
// create a response
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
final Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
|
|
|
@ -30,7 +30,9 @@ import java.util.ArrayList;
|
|||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.Classification;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.protocol.ResponseHeader;
|
||||
|
@ -68,14 +70,6 @@ public class FileLoader {
|
|||
String[] l = null;
|
||||
try {l = url.list();} catch (IOException e) {}
|
||||
if (l != null) {
|
||||
/*
|
||||
if (l == null) {
|
||||
// this can only happen if there is no connection or the directory does not exist
|
||||
//log.logInfo("directory listing not available. URL = " + request.url().toString());
|
||||
sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, "directory listing not available. URL = " + request.url().toString());
|
||||
throw new IOException("directory listing not available. URL = " + request.url().toString());
|
||||
}
|
||||
*/
|
||||
String u = url.toNormalform(true, true);
|
||||
List<String> list = new ArrayList<String>();
|
||||
for (String s: l) {
|
||||
|
@ -87,14 +81,14 @@ public class FileLoader {
|
|||
ResponseHeader responseHeader = new ResponseHeader(200);
|
||||
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date()));
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
responseHeader,
|
||||
profile,
|
||||
false,
|
||||
content.toString().getBytes());
|
||||
UTF8.getBytes(content.toString()));
|
||||
|
||||
return response;
|
||||
}
|
||||
|
@ -127,14 +121,14 @@ public class FileLoader {
|
|||
|
||||
// create response with metadata only
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain");
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
responseHeader,
|
||||
profile,
|
||||
false,
|
||||
url.toTokens().getBytes());
|
||||
UTF8.getBytes(url.toTokens()));
|
||||
return response;
|
||||
}
|
||||
|
||||
|
@ -144,7 +138,7 @@ public class FileLoader {
|
|||
is.close();
|
||||
|
||||
// create response with loaded content
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
|
|
|
@ -27,6 +27,7 @@ package net.yacy.crawler.retrieval;
|
|||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
|
@ -78,7 +79,7 @@ public final class HTTPLoader {
|
|||
|
||||
private Response load(final Request request, final int retryCount, final int maxFileSize, final BlacklistType blacklistType) throws IOException {
|
||||
|
||||
byte[] myHash = this.sb.peers.mySeed().hash.getBytes();
|
||||
byte[] myHash = ASCII.getBytes(this.sb.peers.mySeed().hash);
|
||||
|
||||
if (retryCount < 0) {
|
||||
this.sb.crawlQueues.errorURL.push(request, myHash, new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, "retry counter exceeded", -1);
|
||||
|
@ -198,7 +199,7 @@ public final class HTTPLoader {
|
|||
}
|
||||
|
||||
// create a new cache entry
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
|
|
|
@ -29,6 +29,7 @@ package net.yacy.crawler.retrieval;
|
|||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.order.Base64Order;
|
||||
import net.yacy.cora.order.NaturalOrder;
|
||||
|
@ -212,11 +213,11 @@ public class Request extends WorkflowJob
|
|||
new byte[][] {
|
||||
this.url.hash(),
|
||||
this.initiator,
|
||||
this.url.toString().getBytes(),
|
||||
UTF8.getBytes(this.url.toString()),
|
||||
this.refhash,
|
||||
namebytes,
|
||||
appdatestr,
|
||||
(this.profileHandle == null) ? null : this.profileHandle.getBytes(),
|
||||
(this.profileHandle == null) ? null : ASCII.getBytes(this.profileHandle),
|
||||
NaturalOrder.encodeLong(this.depth, rowdef.width(7)),
|
||||
NaturalOrder.encodeLong(this.anchors, rowdef.width(8)),
|
||||
NaturalOrder.encodeLong(this.forkfactor, rowdef.width(9)),
|
||||
|
|
|
@ -173,7 +173,7 @@ public class Response {
|
|||
if (!request.isEmpty()) this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(request.size()));
|
||||
this.profile = profile;
|
||||
this.status = QUEUE_STATE_FRESH;
|
||||
this.content = request.name().length() > 0 ? request.name().getBytes() : request.url().toTokens().getBytes();
|
||||
this.content = request.name().length() > 0 ? UTF8.getBytes(request.name()) : UTF8.getBytes(request.url().toTokens());
|
||||
this.fromCache = true;
|
||||
}
|
||||
|
||||
|
|
|
@ -38,8 +38,10 @@ import java.util.List;
|
|||
import jcifs.smb.SmbException;
|
||||
import jcifs.smb.SmbFile;
|
||||
import jcifs.smb.SmbFileInputStream;
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.Classification;
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.protocol.ResponseHeader;
|
||||
|
@ -80,14 +82,6 @@ public class SMBLoader {
|
|||
String[] l = null;
|
||||
try {l = url.list();} catch (IOException e) {}
|
||||
if (l != null) {
|
||||
/*
|
||||
if (l == null) {
|
||||
// this can only happen if there is no connection or the directory does not exist
|
||||
//log.logInfo("directory listing not available. URL = " + request.url().toString());
|
||||
sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, "directory listing not available. URL = " + request.url().toString());
|
||||
throw new IOException("directory listing not available. URL = " + request.url().toString());
|
||||
}
|
||||
*/
|
||||
String u = url.toNormalform(true, true);
|
||||
List<String> list = new ArrayList<String>();
|
||||
for (String s: l) {
|
||||
|
@ -106,14 +100,14 @@ public class SMBLoader {
|
|||
ResponseHeader responseHeader = new ResponseHeader(200);
|
||||
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date()));
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
|
||||
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
||||
Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
responseHeader,
|
||||
profile,
|
||||
false,
|
||||
content.toString().getBytes());
|
||||
UTF8.getBytes(content.toString()));
|
||||
|
||||
return response;
|
||||
}
|
||||
|
|
|
@ -35,10 +35,6 @@ import net.yacy.cora.document.UTF8;
|
|||
|
||||
public final class ByteBuffer extends OutputStream {
|
||||
|
||||
private static final byte singlequote = (byte) 39;
|
||||
private static final byte doublequote = (byte) 34;
|
||||
private static final byte equal = (byte) '=';
|
||||
|
||||
private byte[] buffer;
|
||||
private int offset;
|
||||
private int length;
|
||||
|
|
|
@ -259,7 +259,7 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
|
|||
|
||||
// check if url is in blacklist
|
||||
if (Switchboard.urlBlacklist.isListed(blacklistType, url.getHost().toLowerCase(), url.getFile())) {
|
||||
Switchboard.getSwitchboard().crawlQueues.errorURL.push(new Request(url, null), Switchboard.getSwitchboard().peers.mySeed().hash.getBytes(), new Date(), 1, FailCategory.FINAL_LOAD_CONTEXT, "url in blacklist", -1);
|
||||
Switchboard.getSwitchboard().crawlQueues.errorURL.push(new Request(url, null), ASCII.getBytes(Switchboard.getSwitchboard().peers.mySeed().hash), new Date(), 1, FailCategory.FINAL_LOAD_CONTEXT, "url in blacklist", -1);
|
||||
Log.logFine("snippet fetch", "MEDIA-SNIPPET Rejecting URL '" + url.toString() + "'. URL is in blacklist.");
|
||||
isBlacklisted = true;
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user