From a06930662c38b22b7e7c4f4f40ea055af0229639 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 9 Oct 2012 12:14:28 +0200 Subject: [PATCH] replaced some more .getBytes() with UTF8/ASCII.getBytes() --- htroot/yacy/transferRWI.java | 2 +- .../net/yacy/crawler/retrieval/FTPLoader.java | 12 ++++++----- .../yacy/crawler/retrieval/FileLoader.java | 20 +++++++------------ .../yacy/crawler/retrieval/HTTPLoader.java | 5 +++-- .../net/yacy/crawler/retrieval/Request.java | 5 +++-- .../net/yacy/crawler/retrieval/Response.java | 2 +- .../net/yacy/crawler/retrieval/SMBLoader.java | 14 ++++--------- source/net/yacy/kelondro/util/ByteBuffer.java | 4 ---- .../net/yacy/search/snippet/MediaSnippet.java | 2 +- 9 files changed, 27 insertions(+), 39 deletions(-) diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index af1223b90..e54f15ee4 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -193,7 +193,7 @@ public final class transferRWI { // learn entry try { - sb.index.storeRWI(wordHash.getBytes(), iEntry); + sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry); } catch (final Exception e) { Log.logException(e); } diff --git a/source/net/yacy/crawler/retrieval/FTPLoader.java b/source/net/yacy/crawler/retrieval/FTPLoader.java index d6a5f1523..448c4715d 100644 --- a/source/net/yacy/crawler/retrieval/FTPLoader.java +++ b/source/net/yacy/crawler/retrieval/FTPLoader.java @@ -32,7 +32,9 @@ import java.io.IOException; import java.io.PrintStream; import java.util.Date; +import net.yacy.cora.document.ASCII; import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; @@ -126,14 +128,14 @@ public class FTPLoader { final ResponseHeader responseHeader = new ResponseHeader(200); responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date())); responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html"); - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); response = new Response( request, requestHeader, responseHeader, profile, false, - dirList.toString().getBytes()); + UTF8.getBytes(dirList.toString())); } } else { // file -> download @@ -152,7 +154,7 @@ public class FTPLoader { if (berr.size() > 0 || response == null) { // some error logging final String detail = (berr.size() > 0) ? "Errorlog: " + berr.toString() : ""; - this.sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, " ftp server download, " + detail, -1); + this.sb.crawlQueues.errorURL.push(request, ASCII.getBytes(this.sb.peers.mySeed().hash), new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, " ftp server download, " + detail, -1); throw new IOException("FTPLoader: Unable to download URL '" + request.url().toString() + "': " + detail); } @@ -245,7 +247,7 @@ public class FTPLoader { // create response with metadata only responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain"); - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); final Response response = new Response( request, requestHeader, @@ -260,7 +262,7 @@ public class FTPLoader { final byte[] b = ftpClient.get(path); // create a response - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); final Response response = new Response( request, requestHeader, diff --git a/source/net/yacy/crawler/retrieval/FileLoader.java b/source/net/yacy/crawler/retrieval/FileLoader.java index 0454b3739..17bebdcda 100644 --- a/source/net/yacy/crawler/retrieval/FileLoader.java +++ b/source/net/yacy/crawler/retrieval/FileLoader.java @@ -30,7 +30,9 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import net.yacy.cora.document.ASCII; import net.yacy.cora.document.Classification; +import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; @@ -68,14 +70,6 @@ public class FileLoader { String[] l = null; try {l = url.list();} catch (IOException e) {} if (l != null) { - /* - if (l == null) { - // this can only happen if there is no connection or the directory does not exist - //log.logInfo("directory listing not available. URL = " + request.url().toString()); - sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, "directory listing not available. URL = " + request.url().toString()); - throw new IOException("directory listing not available. URL = " + request.url().toString()); - } - */ String u = url.toNormalform(true, true); List list = new ArrayList(); for (String s: l) { @@ -87,14 +81,14 @@ public class FileLoader { ResponseHeader responseHeader = new ResponseHeader(200); responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date())); responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html"); - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); Response response = new Response( request, requestHeader, responseHeader, profile, false, - content.toString().getBytes()); + UTF8.getBytes(content.toString())); return response; } @@ -127,14 +121,14 @@ public class FileLoader { // create response with metadata only responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain"); - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); Response response = new Response( request, requestHeader, responseHeader, profile, false, - url.toTokens().getBytes()); + UTF8.getBytes(url.toTokens())); return response; } @@ -144,7 +138,7 @@ public class FileLoader { is.close(); // create response with loaded content - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); Response response = new Response( request, requestHeader, diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.java index ffcdfbd22..17f07bb70 100644 --- a/source/net/yacy/crawler/retrieval/HTTPLoader.java +++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java @@ -27,6 +27,7 @@ package net.yacy.crawler.retrieval; import java.io.IOException; import java.util.Date; +import net.yacy.cora.document.ASCII; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; @@ -78,7 +79,7 @@ public final class HTTPLoader { private Response load(final Request request, final int retryCount, final int maxFileSize, final BlacklistType blacklistType) throws IOException { - byte[] myHash = this.sb.peers.mySeed().hash.getBytes(); + byte[] myHash = ASCII.getBytes(this.sb.peers.mySeed().hash); if (retryCount < 0) { this.sb.crawlQueues.errorURL.push(request, myHash, new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, "retry counter exceeded", -1); @@ -198,7 +199,7 @@ public final class HTTPLoader { } // create a new cache entry - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); response = new Response( request, requestHeader, diff --git a/source/net/yacy/crawler/retrieval/Request.java b/source/net/yacy/crawler/retrieval/Request.java index 414883eba..3559f2699 100644 --- a/source/net/yacy/crawler/retrieval/Request.java +++ b/source/net/yacy/crawler/retrieval/Request.java @@ -29,6 +29,7 @@ package net.yacy.crawler.retrieval; import java.io.IOException; import java.util.Date; +import net.yacy.cora.document.ASCII; import net.yacy.cora.document.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; @@ -212,11 +213,11 @@ public class Request extends WorkflowJob new byte[][] { this.url.hash(), this.initiator, - this.url.toString().getBytes(), + UTF8.getBytes(this.url.toString()), this.refhash, namebytes, appdatestr, - (this.profileHandle == null) ? null : this.profileHandle.getBytes(), + (this.profileHandle == null) ? null : ASCII.getBytes(this.profileHandle), NaturalOrder.encodeLong(this.depth, rowdef.width(7)), NaturalOrder.encodeLong(this.anchors, rowdef.width(8)), NaturalOrder.encodeLong(this.forkfactor, rowdef.width(9)), diff --git a/source/net/yacy/crawler/retrieval/Response.java b/source/net/yacy/crawler/retrieval/Response.java index 4267d6a3f..7d61d16a3 100644 --- a/source/net/yacy/crawler/retrieval/Response.java +++ b/source/net/yacy/crawler/retrieval/Response.java @@ -173,7 +173,7 @@ public class Response { if (!request.isEmpty()) this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(request.size())); this.profile = profile; this.status = QUEUE_STATE_FRESH; - this.content = request.name().length() > 0 ? request.name().getBytes() : request.url().toTokens().getBytes(); + this.content = request.name().length() > 0 ? UTF8.getBytes(request.name()) : UTF8.getBytes(request.url().toTokens()); this.fromCache = true; } diff --git a/source/net/yacy/crawler/retrieval/SMBLoader.java b/source/net/yacy/crawler/retrieval/SMBLoader.java index 2a63cfe52..aba7f0d35 100644 --- a/source/net/yacy/crawler/retrieval/SMBLoader.java +++ b/source/net/yacy/crawler/retrieval/SMBLoader.java @@ -38,8 +38,10 @@ import java.util.List; import jcifs.smb.SmbException; import jcifs.smb.SmbFile; import jcifs.smb.SmbFileInputStream; +import net.yacy.cora.document.ASCII; import net.yacy.cora.document.Classification; import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; @@ -80,14 +82,6 @@ public class SMBLoader { String[] l = null; try {l = url.list();} catch (IOException e) {} if (l != null) { - /* - if (l == null) { - // this can only happen if there is no connection or the directory does not exist - //log.logInfo("directory listing not available. URL = " + request.url().toString()); - sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, "directory listing not available. URL = " + request.url().toString()); - throw new IOException("directory listing not available. URL = " + request.url().toString()); - } - */ String u = url.toNormalform(true, true); List list = new ArrayList(); for (String s: l) { @@ -106,14 +100,14 @@ public class SMBLoader { ResponseHeader responseHeader = new ResponseHeader(200); responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date())); responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html"); - final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes()); + final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle())); Response response = new Response( request, requestHeader, responseHeader, profile, false, - content.toString().getBytes()); + UTF8.getBytes(content.toString())); return response; } diff --git a/source/net/yacy/kelondro/util/ByteBuffer.java b/source/net/yacy/kelondro/util/ByteBuffer.java index ed58b119c..37377e221 100644 --- a/source/net/yacy/kelondro/util/ByteBuffer.java +++ b/source/net/yacy/kelondro/util/ByteBuffer.java @@ -35,10 +35,6 @@ import net.yacy.cora.document.UTF8; public final class ByteBuffer extends OutputStream { - private static final byte singlequote = (byte) 39; - private static final byte doublequote = (byte) 34; - private static final byte equal = (byte) '='; - private byte[] buffer; private int offset; private int length; diff --git a/source/net/yacy/search/snippet/MediaSnippet.java b/source/net/yacy/search/snippet/MediaSnippet.java index 85a4933c8..d82651d6c 100644 --- a/source/net/yacy/search/snippet/MediaSnippet.java +++ b/source/net/yacy/search/snippet/MediaSnippet.java @@ -259,7 +259,7 @@ public class MediaSnippet implements Comparable, Comparator