Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

more UTF8 getBytes() performance hacks

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7649 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2011-04-12 05:02:36 +00:00
parent 78d6d6ca06
commit 4c013d9088
43 changed files with 288 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.xml
View File

@ -455,8 +455,8 @@
<!-- packing all files into a gzipped tar --> <!-- packing all files into a gzipped tar -->
<tarfileset dir="${release_main}" prefix="${releaseFileParentDir}/" dirmode="${accessRightsDir}" mode="${accessRightsExecutable}"> <tarfileset dir="${release_main}" prefix="${releaseFileParentDir}/" dirmode="${accessRightsDir}" mode="${accessRightsExecutable}">
<include name="**/*.sh"/> <include name="**/*.sh"/>
<include name="**/*.command"/> <include name="**/*.command"/>
</tarfileset> </tarfileset>
<tarfileset dir="${release_main}" prefix="${releaseFileParentDir}/" dirmode="${accessRightsDir}" mode="${accessRightsFile}" > <tarfileset dir="${release_main}" prefix="${releaseFileParentDir}/" dirmode="${accessRightsDir}" mode="${accessRightsFile}" >
<include name="**/*"/> <include name="**/*"/>
@ -529,7 +529,7 @@
</fileset> </fileset>
</copy> </copy>
<!-- copy language statistics files --> <!-- copy shell scripts from bin/ -->
<!-- i'm not sure, if this is consistent with the debian policy --> <!-- i'm not sure, if this is consistent with the debian policy -->
<!-- but for /usr/bin or /usr/lib we need an extra environment variable --> <!-- but for /usr/bin or /usr/lib we need an extra environment variable -->
<copy todir="${DESTDIR}/usr/share/yacy/bin"> <copy todir="${DESTDIR}/usr/share/yacy/bin">

2
htroot/yacysearch_location.java
View File

@ -107,6 +107,8 @@ public class yacysearch_location {
if (search_publisher) words.append(message.getCopyright().trim()).append(space); if (search_publisher) words.append(message.getCopyright().trim()).append(space);
if (search_creator) words.append(message.getAuthor().trim()).append(space); if (search_creator) words.append(message.getAuthor().trim()).append(space);
String subject = ""; String subject = "";
assert message != null;
assert message.getSubject() != null;
for (String s: message.getSubject()) subject += s.trim() + space; for (String s: message.getSubject()) subject += s.trim() + space;
if (search_subject) words.append(subject).append(space); if (search_subject) words.append(subject).append(space);
String[] wordlist = words.toString().trim().split(space); String[] wordlist = words.toString().trim().split(space);

2
source/de/anomic/crawler/Balancer.java
View File

@ -406,7 +406,7 @@ public class Balancer {
// at this point we must check if the crawlEntry has relevance because the crawl profile still exists // at this point we must check if the crawlEntry has relevance because the crawl profile still exists
// if not: return null. A calling method must handle the null value and try again // if not: return null. A calling method must handle the null value and try again
final CrawlProfile profileEntry = cs.getActive(crawlEntry.profileHandle().getBytes()); final CrawlProfile profileEntry = cs.getActive(UTF8.getBytes(crawlEntry.profileHandle()));
if (profileEntry == null) { if (profileEntry == null) {
Log.logWarning("Balancer", "no profile entry for handle " + crawlEntry.profileHandle()); Log.logWarning("Balancer", "no profile entry for handle " + crawlEntry.profileHandle());
return null; return null;

14
source/de/anomic/crawler/CrawlQueues.java
View File

@ -242,7 +242,7 @@ public class CrawlQueues {
log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
return true; return true;
} }
CrawlProfile profile = sb.crawler.getActive(profileHandle.getBytes()); CrawlProfile profile = sb.crawler.getActive(UTF8.getBytes(profileHandle));
if (profile == null) { if (profile == null) {
log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
return true; return true;
@ -284,7 +284,7 @@ public class CrawlQueues {
* @return * @return
*/ */
private void load(Request urlEntry, final String stats, final String profileHandle) { private void load(Request urlEntry, final String stats, final String profileHandle) {
final CrawlProfile profile = sb.crawler.getActive(profileHandle.getBytes()); final CrawlProfile profile = sb.crawler.getActive(UTF8.getBytes(profileHandle));
if (profile != null) { if (profile != null) {
// check if the protocol is supported // check if the protocol is supported
@ -467,7 +467,7 @@ public class CrawlQueues {
// stack url // stack url
if (sb.getLog().isFinest()) sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'"); if (sb.getLog().isFinest()) sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'");
sb.crawlStacker.enqueueEntry(new Request( sb.crawlStacker.enqueueEntry(new Request(
hash.getBytes(), UTF8.getBytes(hash),
url, url,
(referrer == null) ? null : referrer.hash(), (referrer == null) ? null : referrer.hash(),
item.getDescription(), item.getDescription(),
@ -578,7 +578,7 @@ public class CrawlQueues {
//if (log.isFine()) log.logFine("Crawling of URL '" + request.url().toString() + "' disallowed by robots.txt."); //if (log.isFine()) log.logFine("Crawling of URL '" + request.url().toString() + "' disallowed by robots.txt.");
errorURL.push( errorURL.push(
this.request, this.request,
sb.peers.mySeed().hash.getBytes(), UTF8.getBytes(sb.peers.mySeed().hash),
new Date(), new Date(),
1, 1,
"denied by robots.txt"); "denied by robots.txt");
@ -593,7 +593,7 @@ public class CrawlQueues {
try { try {
request.setStatus("loading", WorkflowJob.STATUS_RUNNING); request.setStatus("loading", WorkflowJob.STATUS_RUNNING);
final long maxFileSize = sb.getConfigLong("crawler.http.maxFileSize", HTTPLoader.DEFAULT_MAXFILESIZE); final long maxFileSize = sb.getConfigLong("crawler.http.maxFileSize", HTTPLoader.DEFAULT_MAXFILESIZE);
final CrawlProfile e = sb.crawler.getActive(request.profileHandle().getBytes()); final CrawlProfile e = sb.crawler.getActive(UTF8.getBytes(request.profileHandle()));
Response response = sb.loader.load(request, e == null ? CrawlProfile.CacheStrategy.IFEXIST : e.cacheStrategy(), maxFileSize, true); Response response = sb.loader.load(request, e == null ? CrawlProfile.CacheStrategy.IFEXIST : e.cacheStrategy(), maxFileSize, true);
if (response == null) { if (response == null) {
request.setStatus("error", WorkflowJob.STATUS_FINISHED); request.setStatus("error", WorkflowJob.STATUS_FINISHED);
@ -614,7 +614,7 @@ public class CrawlQueues {
if (result != null) { if (result != null) {
errorURL.push( errorURL.push(
this.request, this.request,
sb.peers.mySeed().hash.getBytes(), UTF8.getBytes(sb.peers.mySeed().hash),
new Date(), new Date(),
1, 1,
"cannot load: " + result); "cannot load: " + result);
@ -626,7 +626,7 @@ public class CrawlQueues {
} catch (final Exception e) { } catch (final Exception e) {
errorURL.push( errorURL.push(
this.request, this.request,
sb.peers.mySeed().hash.getBytes(), UTF8.getBytes(sb.peers.mySeed().hash),
new Date(), new Date(),
1, 1,
e.getMessage() + " - in worker"); e.getMessage() + " - in worker");

6
source/de/anomic/crawler/CrawlStacker.java
View File

@ -201,7 +201,7 @@ public final class CrawlStacker {
// if the url was rejected we store it into the error URL db // if the url was rejected we store it into the error URL db
if (rejectReason != null) { if (rejectReason != null) {
nextQueue.errorURL.push(entry, peers.mySeed().hash.getBytes(), new Date(), 1, rejectReason); nextQueue.errorURL.push(entry, UTF8.getBytes(peers.mySeed().hash), new Date(), 1, rejectReason);
} }
} catch (final Exception e) { } catch (final Exception e) {
CrawlStacker.this.log.logWarning("Error while processing stackCrawl entry.\n" + "Entry: " + entry.toString() + "Error: " + e.toString(), e); CrawlStacker.this.log.logWarning("Error while processing stackCrawl entry.\n" + "Entry: " + entry.toString() + "Error: " + e.toString(), e);
@ -327,7 +327,7 @@ public final class CrawlStacker {
// returns null if successful, a reason string if not successful // returns null if successful, a reason string if not successful
//this.log.logFinest("stackCrawl: nexturlString='" + nexturlString + "'"); //this.log.logFinest("stackCrawl: nexturlString='" + nexturlString + "'");
final CrawlProfile profile = crawler.getActive(entry.profileHandle().getBytes()); final CrawlProfile profile = crawler.getActive(UTF8.getBytes(entry.profileHandle()));
String error; String error;
if (profile == null) { if (profile == null) {
error = "LOST STACKER PROFILE HANDLE '" + entry.profileHandle() + "' for URL " + entry.url(); error = "LOST STACKER PROFILE HANDLE '" + entry.profileHandle() + "' for URL " + entry.url();
@ -339,7 +339,7 @@ public final class CrawlStacker {
if (error != null) return error; if (error != null) return error;
// store information // store information
final boolean local = Base64Order.enhancedCoder.equal(entry.initiator(), peers.mySeed().hash.getBytes()); final boolean local = Base64Order.enhancedCoder.equal(entry.initiator(), UTF8.getBytes(peers.mySeed().hash));
final boolean proxy = (entry.initiator() == null || entry.initiator().length == 0 || UTF8.String(entry.initiator()).equals("------------")) && profile.handle().equals(crawler.defaultProxyProfile.handle()); final boolean proxy = (entry.initiator() == null || entry.initiator().length == 0 || UTF8.String(entry.initiator()).equals("------------")) && profile.handle().equals(crawler.defaultProxyProfile.handle());
final boolean remote = profile.handle().equals(crawler.defaultRemoteProfile.handle()); final boolean remote = profile.handle().equals(crawler.defaultRemoteProfile.handle());
final boolean global = final boolean global =

17
source/de/anomic/crawler/CrawlSwitchboard.java
View File

@ -31,6 +31,7 @@ import java.io.IOException;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.blob.MapHeap; import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -214,44 +215,44 @@ public final class CrawlSwitchboard {
true, true,
false /*getConfigBool(PROXY_INDEXING_REMOTE, false)*/, true, true, true, false /*getConfigBool(PROXY_INDEXING_REMOTE, false)*/, true, true, true,
CrawlProfile.CacheStrategy.IFFRESH); CrawlProfile.CacheStrategy.IFFRESH);
this.profilesActiveCrawls.put(this.defaultProxyProfile.handle().getBytes(), this.defaultProxyProfile); this.profilesActiveCrawls.put(UTF8.getBytes(this.defaultProxyProfile.handle()), this.defaultProxyProfile);
} }
if (this.defaultRemoteProfile == null) { if (this.defaultRemoteProfile == null) {
// generate new default entry for remote crawling // generate new default entry for remote crawling
this.defaultRemoteProfile = new CrawlProfile(CRAWL_PROFILE_REMOTE, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0, this.defaultRemoteProfile = new CrawlProfile(CRAWL_PROFILE_REMOTE, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0,
-1, -1, true, true, true, false, false, true, true, false, CrawlProfile.CacheStrategy.IFFRESH); -1, -1, true, true, true, false, false, true, true, false, CrawlProfile.CacheStrategy.IFFRESH);
this.profilesActiveCrawls.put(this.defaultRemoteProfile.handle().getBytes(), this.defaultRemoteProfile); this.profilesActiveCrawls.put(UTF8.getBytes(this.defaultRemoteProfile.handle()), this.defaultRemoteProfile);
} }
if (this.defaultTextSnippetLocalProfile == null) { if (this.defaultTextSnippetLocalProfile == null) {
// generate new default entry for snippet fetch and optional crawling // generate new default entry for snippet fetch and optional crawling
this.defaultTextSnippetLocalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_LOCAL_TEXT, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0, this.defaultTextSnippetLocalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_LOCAL_TEXT, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0,
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE), -1, true, false, false, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST); CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE), -1, true, false, false, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST);
this.profilesActiveCrawls.put(this.defaultTextSnippetLocalProfile.handle().getBytes(), this.defaultTextSnippetLocalProfile); this.profilesActiveCrawls.put(UTF8.getBytes(this.defaultTextSnippetLocalProfile.handle()), this.defaultTextSnippetLocalProfile);
} }
if (this.defaultTextSnippetGlobalProfile == null) { if (this.defaultTextSnippetGlobalProfile == null) {
// generate new default entry for snippet fetch and optional crawling // generate new default entry for snippet fetch and optional crawling
this.defaultTextSnippetGlobalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0, this.defaultTextSnippetGlobalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0,
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT_RECRAWL_CYCLE), -1, true, true, true, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST); CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT_RECRAWL_CYCLE), -1, true, true, true, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST);
this.profilesActiveCrawls.put(this.defaultTextSnippetGlobalProfile.handle().getBytes(), this.defaultTextSnippetGlobalProfile); this.profilesActiveCrawls.put(UTF8.getBytes(this.defaultTextSnippetGlobalProfile.handle()), this.defaultTextSnippetGlobalProfile);
} }
this.defaultTextSnippetGlobalProfile.setCacheStrategy(CrawlProfile.CacheStrategy.IFEXIST); this.defaultTextSnippetGlobalProfile.setCacheStrategy(CrawlProfile.CacheStrategy.IFEXIST);
if (this.defaultMediaSnippetLocalProfile == null) { if (this.defaultMediaSnippetLocalProfile == null) {
// generate new default entry for snippet fetch and optional crawling // generate new default entry for snippet fetch and optional crawling
this.defaultMediaSnippetLocalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0, this.defaultMediaSnippetLocalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0,
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), -1, true, false, false, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST); CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), -1, true, false, false, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST);
this.profilesActiveCrawls.put(this.defaultMediaSnippetLocalProfile.handle().getBytes(), this.defaultMediaSnippetLocalProfile); this.profilesActiveCrawls.put(UTF8.getBytes(this.defaultMediaSnippetLocalProfile.handle()), this.defaultMediaSnippetLocalProfile);
} }
if (this.defaultMediaSnippetGlobalProfile == null) { if (this.defaultMediaSnippetGlobalProfile == null) {
// generate new default entry for snippet fetch and optional crawling // generate new default entry for snippet fetch and optional crawling
this.defaultMediaSnippetGlobalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0, this.defaultMediaSnippetGlobalProfile = new CrawlProfile(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0,
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE), -1, true, false, true, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST); CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE), -1, true, false, true, true, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST);
this.profilesActiveCrawls.put(this.defaultMediaSnippetGlobalProfile.handle().getBytes(), this.defaultMediaSnippetGlobalProfile); this.profilesActiveCrawls.put(UTF8.getBytes(this.defaultMediaSnippetGlobalProfile.handle()), this.defaultMediaSnippetGlobalProfile);
} }
if (this.defaultSurrogateProfile == null) { if (this.defaultSurrogateProfile == null) {
// generate new default entry for surrogate parsing // generate new default entry for surrogate parsing
this.defaultSurrogateProfile = new CrawlProfile(CRAWL_PROFILE_SURROGATE, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0, this.defaultSurrogateProfile = new CrawlProfile(CRAWL_PROFILE_SURROGATE, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_NEVER, 0,
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE), -1, true, true, false, false, false, true, true, false, CrawlProfile.CacheStrategy.NOCACHE); CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE), -1, true, true, false, false, false, true, true, false, CrawlProfile.CacheStrategy.NOCACHE);
this.profilesActiveCrawls.put(this.defaultSurrogateProfile.handle().getBytes(), this.defaultSurrogateProfile); this.profilesActiveCrawls.put(UTF8.getBytes(this.defaultSurrogateProfile.handle()), this.defaultSurrogateProfile);
} }
} }
@ -285,7 +286,7 @@ public final class CrawlSwitchboard {
(entry.name().equals(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA)) || (entry.name().equals(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA)) ||
(entry.name().equals(CRAWL_PROFILE_SURROGATE)))) { (entry.name().equals(CRAWL_PROFILE_SURROGATE)))) {
CrawlProfile p = new CrawlProfile(entry); CrawlProfile p = new CrawlProfile(entry);
profilesPassiveCrawls.put(p.handle().getBytes(), p); profilesPassiveCrawls.put(UTF8.getBytes(p.handle()), p);
profilesActiveCrawls.remove(handle); profilesActiveCrawls.remove(handle);
hasDoneSomething = true; hasDoneSomething = true;
} }

5
source/de/anomic/crawler/RSSLoader.java
View File

@ -31,6 +31,7 @@ import java.util.Date;
import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.RSSReader; import net.yacy.cora.document.RSSReader;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ARC;
import net.yacy.cora.storage.ComparableARC; import net.yacy.cora.storage.ComparableARC;
import net.yacy.document.Parser.Failure; import net.yacy.document.Parser.Failure;
@ -108,7 +109,7 @@ public class RSSLoader extends Thread {
int lastAvg = rssRow.get("avg_upd_per_day", 0); int lastAvg = rssRow.get("avg_upd_per_day", 0);
long thisAvg = 1000 * 60 * 60 * 24 / deltaTime * loadCount; long thisAvg = 1000 * 60 * 60 * 24 / deltaTime * loadCount;
long nextAvg = lastAvg == 0 ? thisAvg : (thisAvg + lastAvg * 2) / 3; long nextAvg = lastAvg == 0 ? thisAvg : (thisAvg + lastAvg * 2) / 3;
rssRow.put("url", url.toNormalform(true, false).getBytes()); rssRow.put("url", UTF8.getBytes(url.toNormalform(true, false)));
rssRow.put("title", feed.getChannel().getTitle()); rssRow.put("title", feed.getChannel().getTitle());
rssRow.put("last_load_date", new Date()); rssRow.put("last_load_date", new Date());
rssRow.put("last_load_count", loadCount); rssRow.put("last_load_count", loadCount);
@ -140,7 +141,7 @@ public class RSSLoader extends Thread {
// store pk of api table into rss table to show that the entry has been recorded // store pk of api table into rss table to show that the entry has been recorded
assert pk != null; assert pk != null;
Tables.Data rssRow = new Tables.Data(); Tables.Data rssRow = new Tables.Data();
rssRow.put("url", url.toNormalform(true, false).getBytes()); rssRow.put("url", UTF8.getBytes(url.toNormalform(true, false)));
rssRow.put("title", feed.getChannel().getTitle()); rssRow.put("title", feed.getChannel().getTitle());
rssRow.put("api_pk", pk); rssRow.put("api_pk", pk);
try { try {

37
source/de/anomic/crawler/ResourceObserver.java
View File

@ -38,19 +38,20 @@ public class ResourceObserver {
public static final Log log = new Log("RESOURCE OBSERVER"); public static final Log log = new Log("RESOURCE OBSERVER");
// return values for available disk/memory // return values for available disk/memory
private static final int LOW = 0; public enum Space implements Comparable<Space> {
private static final int MEDIUM = 1; LOW, MEDIUM, HIGH; // according to the order of the definition, LOW is smaller than MEDIUM and MEDIUM is smaller than HIGH
private static final int HIGH = 2; }
private final Switchboard sb; private final Switchboard sb;
private final File path; // path to check private final File path; // path to check
private int normalizedDiskFree = HIGH; private Space normalizedDiskFree = Space.HIGH;
private int normalizedMemoryFree = HIGH; private Space normalizedMemoryFree = Space.HIGH;
public ResourceObserver(final Switchboard sb) { public ResourceObserver(final Switchboard sb) {
this.sb = sb; this.sb = sb;
this.path = sb.getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, ""); this.path = sb.getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, "");
log.logInfo("path for disc space measurement: " + this.path);
} }
public static void initThread() { public static void initThread() {
@ -68,9 +69,9 @@ public class ResourceObserver {
normalizedDiskFree = getNormalizedDiskFree(); normalizedDiskFree = getNormalizedDiskFree();
normalizedMemoryFree = getNormalizedMemoryFree(); normalizedMemoryFree = getNormalizedMemoryFree();
if (normalizedDiskFree < HIGH || normalizedMemoryFree < HIGH) { if (normalizedDiskFree.compareTo(Space.HIGH) < 0 || normalizedMemoryFree.compareTo(Space.HIGH) < 0 ) {
if (normalizedDiskFree < HIGH) { // pause crawls if (normalizedDiskFree.compareTo(Space.HIGH) < 0 ) { // pause crawls
if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) { if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
log.logInfo("pausing local crawls"); log.logInfo("pausing local crawls");
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
@ -81,7 +82,7 @@ public class ResourceObserver {
} }
} }
if ((normalizedDiskFree == LOW || normalizedMemoryFree < HIGH) && sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) { if ((normalizedDiskFree == Space.LOW || normalizedMemoryFree.compareTo(Space.HIGH) < 0) && sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
log.logInfo("disabling index receive"); log.logInfo("disabling index receive");
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false); sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
sb.peers.mySeed().setFlagAcceptRemoteIndex(false); sb.peers.mySeed().setFlagAcceptRemoteIndex(false);
@ -108,38 +109,38 @@ public class ResourceObserver {
* <li><code>LOW</code> if lower than hardlimit disk space is available</li> * <li><code>LOW</code> if lower than hardlimit disk space is available</li>
* </ul> * </ul>
*/ */
private int getNormalizedDiskFree() { private Space getNormalizedDiskFree() {
final long currentSpace = getUsableSpace(this.path); final long currentSpace = getUsableSpace(this.path);
if(currentSpace < 1L) return HIGH; if (currentSpace < 1L) return Space.HIGH;
int ret = HIGH; Space ret = Space.HIGH;
if (currentSpace < getMinFreeDiskSpace()) { if (currentSpace < getMinFreeDiskSpace()) {
log.logWarning("Volume " + this.path.toString() + ": free space (" + (currentSpace / 1024 / 1024) + " MB) is too low (< " + (getMinFreeDiskSpace() / 1024 / 1024) + " MB)"); log.logWarning("Volume " + this.path.toString() + ": free space (" + (currentSpace / 1024 / 1024) + " MB) is too low (< " + (getMinFreeDiskSpace() / 1024 / 1024) + " MB)");
ret = MEDIUM; ret = Space.MEDIUM;
} }
if (currentSpace < getMinFreeDiskSpace_hardlimit()) { if (currentSpace < getMinFreeDiskSpace_hardlimit()) {
ret = LOW; ret = Space.LOW;
} }
return ret; return ret;
} }
private int getNormalizedMemoryFree() { private Space getNormalizedMemoryFree() {
if(!MemoryControl.getDHTallowed()) return LOW; if(!MemoryControl.getDHTallowed()) return Space.LOW;
return HIGH; return Space.HIGH;
} }
/** /**
* @return <code>true</code> if disk space is available * @return <code>true</code> if disk space is available
*/ */
public boolean getDiskAvailable() { public boolean getDiskAvailable() {
return normalizedDiskFree == HIGH; return normalizedDiskFree == Space.HIGH;
} }
/** /**
* @return <code>true</code> if memory is available * @return <code>true</code> if memory is available
*/ */
public boolean getMemoryAvailable() { public boolean getMemoryAvailable() {
return normalizedMemoryFree == HIGH; return normalizedMemoryFree == Space.HIGH;
} }
/** /**

2
source/de/anomic/crawler/ResultURLs.java
View File

@ -221,7 +221,7 @@ public final class ResultURLs {
public static void main(final String[] args) { public static void main(final String[] args) {
try { try {
final DigestURI url = new DigestURI("http", "www.yacy.net", 80, "/"); final DigestURI url = new DigestURI("http", "www.yacy.net", 80, "/");
final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0f, 0.0f, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), "de".getBytes(), 0, 0, 0, 0, 0, 0); final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0f, 0.0f, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
EventOrigin stackNo = EventOrigin.LOCAL_CRAWLING; EventOrigin stackNo = EventOrigin.LOCAL_CRAWLING;
System.out.println("valid test:\n======="); System.out.println("valid test:\n=======");
// add // add

8
source/de/anomic/crawler/RobotsEntry.java
View File

@ -125,7 +125,7 @@ public class RobotsEntry {
pathListStr.append(element) pathListStr.append(element)
.append(RobotsTxt.ROBOTS_DB_PATH_SEPARATOR); .append(RobotsTxt.ROBOTS_DB_PATH_SEPARATOR);
} }
this.mem.put(ALLOW_PATH_LIST, pathListStr.substring(0,pathListStr.length()-1).getBytes()); this.mem.put(ALLOW_PATH_LIST, UTF8.getBytes(pathListStr.substring(0,pathListStr.length()-1)));
} }
if (disallowPathList != null && !disallowPathList.isEmpty()) { if (disallowPathList != null && !disallowPathList.isEmpty()) {
@ -136,7 +136,7 @@ public class RobotsEntry {
pathListStr.append(element) pathListStr.append(element)
.append(RobotsTxt.ROBOTS_DB_PATH_SEPARATOR); .append(RobotsTxt.ROBOTS_DB_PATH_SEPARATOR);
} }
this.mem.put(DISALLOW_PATH_LIST,pathListStr.substring(0, pathListStr.length()-1).getBytes()); this.mem.put(DISALLOW_PATH_LIST, UTF8.getBytes(pathListStr.substring(0, pathListStr.length()-1)));
} }
} }
@ -149,7 +149,7 @@ public class RobotsEntry {
} }
public Map<String, byte[]> getMem() { public Map<String, byte[]> getMem() {
if (!this.mem.containsKey(HOST_NAME)) this.mem.put(HOST_NAME, this.hostName.getBytes()); if (!this.mem.containsKey(HOST_NAME)) this.mem.put(HOST_NAME, UTF8.getBytes(this.hostName));
return this.mem; return this.mem;
} }
@ -184,7 +184,7 @@ public class RobotsEntry {
public void setLoadedDate(final Date newLoadedDate) { public void setLoadedDate(final Date newLoadedDate) {
if (newLoadedDate != null) { if (newLoadedDate != null) {
this.mem.put(LOADED_DATE, Long.toString(newLoadedDate.getTime()).getBytes()); this.mem.put(LOADED_DATE, UTF8.getBytes(Long.toString(newLoadedDate.getTime())));
} }
} }

3
source/de/anomic/crawler/SitemapImporter.java
View File

@ -28,6 +28,7 @@ package de.anomic.crawler;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.util.Date; import java.util.Date;
import net.yacy.cora.document.UTF8;
import net.yacy.document.parser.sitemapParser; import net.yacy.document.parser.sitemapParser;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.meta.URIMetadataRow;
@ -89,7 +90,7 @@ public class SitemapImporter extends Thread {
// URL needs to crawled // URL needs to crawled
this.sb.crawlStacker.enqueueEntry(new Request( this.sb.crawlStacker.enqueueEntry(new Request(
this.sb.peers.mySeed().hash.getBytes(), UTF8.getBytes(this.sb.peers.mySeed().hash),
url, url,
null, // this.siteMapURL.toString(), null, // this.siteMapURL.toString(),
entry.url(), entry.url(),

5
source/de/anomic/crawler/ZURL.java
View File

@ -33,6 +33,7 @@ import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.Index; import net.yacy.kelondro.index.Index;
@ -198,7 +199,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
newrow.setCol(1, entry.executor); newrow.setCol(1, entry.executor);
newrow.setCol(2, entry.workdate.getTime()); newrow.setCol(2, entry.workdate.getTime());
newrow.setCol(3, entry.workcount); newrow.setCol(3, entry.workcount);
newrow.setCol(4, entry.anycause.getBytes()); newrow.setCol(4, UTF8.getBytes(entry.anycause));
newrow.setCol(5, entry.bentry.toRow().bytes()); newrow.setCol(5, entry.bentry.toRow().bytes());
try { try {
if (urlIndex != null) urlIndex.put(newrow); if (urlIndex != null) urlIndex.put(newrow);
@ -295,7 +296,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
private boolean error = false; private boolean error = false;
private kiter(final boolean up, final String firstHash) throws IOException { private kiter(final boolean up, final String firstHash) throws IOException {
i = urlIndex.rows(up, (firstHash == null) ? null : firstHash.getBytes()); i = urlIndex.rows(up, (firstHash == null) ? null : UTF8.getBytes(firstHash));
error = false; error = false;
} }

10
source/de/anomic/data/BlogBoard.java
View File

@ -85,7 +85,7 @@ public class BlogBoard {
* @return true if the database contains the element, else false * @return true if the database contains the element, else false
*/ */
public boolean contains(final String key) { public boolean contains(final String key) {
return database.containsKey(key.getBytes()); return database.containsKey(UTF8.getBytes(key));
} }
public void close() { public void close() {
@ -126,7 +126,7 @@ public class BlogBoard {
public String writeBlogEntry(final BlogEntry page) { public String writeBlogEntry(final BlogEntry page) {
String ret = null; String ret = null;
try { try {
database.insert(page.key.getBytes(), page.record); database.insert(UTF8.getBytes(page.key), page.record);
ret = page.key; ret = page.key;
} catch (IOException ex) { } catch (IOException ex) {
Log.logException(ex); Log.logException(ex);
@ -144,7 +144,7 @@ public class BlogBoard {
final String normalized = normalize(key); final String normalized = normalize(key);
Map<String, String> record; Map<String, String> record;
try { try {
record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH)).getBytes()); record = base.get(UTF8.getBytes(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH))));
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
record = null; record = null;
@ -153,7 +153,7 @@ public class BlogBoard {
record = null; record = null;
} }
return (record == null) ? return (record == null) ?
newEntry(key, new byte[0], "anonymous".getBytes(), "127.0.0.1", new Date(), new byte[0], null, null) : newEntry(key, new byte[0], UTF8.getBytes("anonymous"), "127.0.0.1", new Date(), new byte[0], null, null) :
new BlogEntry(key, record); new BlogEntry(key, record);
} }
@ -229,7 +229,7 @@ public class BlogBoard {
public void deleteBlogEntry(final String key) { public void deleteBlogEntry(final String key) {
try { try {
database.delete(normalize(key).getBytes()); database.delete(UTF8.getBytes(normalize(key)));
} catch (final IOException e) { } } catch (final IOException e) { }
} }

10
source/de/anomic/data/BlogBoardComments.java
View File

@ -116,7 +116,7 @@ public class BlogBoardComments {
public String write(final CommentEntry page) { public String write(final CommentEntry page) {
// writes a new page and returns key // writes a new page and returns key
try { try {
database.insert(page.key.getBytes(), page.record); database.insert(UTF8.getBytes(page.key), page.record);
return page.key; return page.key;
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
@ -133,7 +133,7 @@ public class BlogBoardComments {
copyOfKey = copyOfKey.substring(0, Math.min(copyOfKey.length(), KEY_LENGTH)); copyOfKey = copyOfKey.substring(0, Math.min(copyOfKey.length(), KEY_LENGTH));
Map<String, String> record; Map<String, String> record;
try { try {
record = base.get(copyOfKey.getBytes()); record = base.get(UTF8.getBytes(copyOfKey));
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
record = null; record = null;
@ -142,7 +142,7 @@ public class BlogBoardComments {
record = null; record = null;
} }
return (record == null) ? return (record == null) ?
newEntry(copyOfKey, new byte[0], "anonymous".getBytes(), "127.0.0.1", new Date(), new byte[0]) : newEntry(copyOfKey, new byte[0], UTF8.getBytes("anonymous"), "127.0.0.1", new Date(), new byte[0]) :
new CommentEntry(copyOfKey, record); new CommentEntry(copyOfKey, record);
} }
@ -151,7 +151,7 @@ public class BlogBoardComments {
final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
try { try {
final DocumentBuilder builder = factory.newDocumentBuilder(); final DocumentBuilder builder = factory.newDocumentBuilder();
final Document doc = builder.parse(new ByteArrayInputStream(input.getBytes())); final Document doc = builder.parse(new ByteArrayInputStream(UTF8.getBytes(input)));
ret = parseXMLimport(doc); ret = parseXMLimport(doc);
} }
catch (final ParserConfigurationException e) {} catch (final ParserConfigurationException e) {}
@ -219,7 +219,7 @@ public class BlogBoardComments {
public void delete(final String key) { public void delete(final String key) {
try { try {
database.delete(normalize(key).getBytes()); database.delete(UTF8.getBytes(normalize(key)));
} }
catch (final IOException e) { } catch (final IOException e) { }
} }

7
source/de/anomic/data/BookmarkDate.java
View File

@ -39,6 +39,7 @@ import java.util.Map;
import de.anomic.data.BookmarksDB.Bookmark; import de.anomic.data.BookmarksDB.Bookmark;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.blob.MapHeap; import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -60,7 +61,7 @@ public class BookmarkDate {
public Entry getDate(final String date) { public Entry getDate(final String date) {
Map<String, String> map; Map<String, String> map;
try { try {
map = datesTable.get(date.getBytes()); map = datesTable.get(UTF8.getBytes(date));
} catch (final IOException e) { } catch (final IOException e) {
map = null; map = null;
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
@ -146,13 +147,13 @@ public class BookmarkDate {
public void setDatesTable() { public void setDatesTable() {
if (this.size() >0) { if (this.size() >0) {
try { try {
datesTable.insert(getDateString().getBytes(), mem); datesTable.insert(UTF8.getBytes(getDateString()), mem);
} catch (Exception e) { } catch (Exception e) {
Log.logException(e); Log.logException(e);
} }
} else { } else {
try { try {
datesTable.delete(getDateString().getBytes()); datesTable.delete(UTF8.getBytes(getDateString()));
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);
} }

6
source/de/anomic/data/BookmarksDB.java
View File

@ -132,7 +132,7 @@ public class BookmarksDB {
// adding a bookmark to the bookmarksDB // adding a bookmark to the bookmarksDB
public void saveBookmark(final Bookmark bookmark){ public void saveBookmark(final Bookmark bookmark){
try { try {
bookmarks.insert(bookmark.getUrlHash().getBytes(), bookmark.entry); bookmarks.insert(UTF8.getBytes(bookmark.getUrlHash()), bookmark.entry);
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
} }
@ -145,7 +145,7 @@ public class BookmarksDB {
public Bookmark getBookmark(final String urlHash){ public Bookmark getBookmark(final String urlHash){
try { try {
final Map<String, String> map = bookmarks.get(urlHash.getBytes()); final Map<String, String> map = bookmarks.get(UTF8.getBytes(urlHash));
return (map == null) ? null : new Bookmark(map); return (map == null) ? null : new Bookmark(map);
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
@ -172,7 +172,7 @@ public class BookmarksDB {
Bookmark b; Bookmark b;
try { try {
b = getBookmark(urlHash); b = getBookmark(urlHash);
bookmarks.delete(urlHash.getBytes()); bookmarks.delete(UTF8.getBytes(urlHash));
} catch (final IOException e) { } catch (final IOException e) {
b = null; b = null;
} }

6
source/de/anomic/data/MessageBoard.java
View File

@ -184,7 +184,7 @@ public class MessageBoard {
public String write(final entry message) { public String write(final entry message) {
// writes a message and returns key // writes a message and returns key
try { try {
database.insert(message.key.getBytes(), message.record); database.insert(UTF8.getBytes(message.key), message.record);
return message.key; return message.key;
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
@ -195,7 +195,7 @@ public class MessageBoard {
public entry read(final String key) { public entry read(final String key) {
Map<String, String> record; Map<String, String> record;
try { try {
record = database.get(key.getBytes()); record = database.get(UTF8.getBytes(key));
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
return null; return null;
@ -208,7 +208,7 @@ public class MessageBoard {
public void remove(final String key) { public void remove(final String key) {
try { try {
database.delete(key.getBytes()); database.delete(UTF8.getBytes(key));
} catch (final IOException e) { } catch (final IOException e) {
} }
} }

7
source/de/anomic/data/URLAnalysis.java
View File

@ -50,6 +50,7 @@ import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.WordReferenceRow; import net.yacy.kelondro.data.word.WordReferenceRow;
@ -240,9 +241,9 @@ public class URLAnalysis {
if (gz) os = new GZIPOutputStream(os); if (gz) os = new GZIPOutputStream(os);
count = 0; count = 0;
for (final Map.Entry<String, Integer> e: results.entrySet()) { for (final Map.Entry<String, Integer> e: results.entrySet()) {
os.write(e.getKey().getBytes()); os.write(UTF8.getBytes(e.getKey()));
os.write(new byte[]{'\t'}); os.write(new byte[]{'\t'});
os.write(Integer.toString(e.getValue()).getBytes()); os.write(UTF8.getBytes(Integer.toString(e.getValue())));
os.write(new byte[]{'\n'}); os.write(new byte[]{'\n'});
count++; count++;
if (System.currentTimeMillis() - time > 10000) { if (System.currentTimeMillis() - time > 10000) {
@ -330,7 +331,7 @@ public class URLAnalysis {
if (gz) os = new GZIPOutputStream(os); if (gz) os = new GZIPOutputStream(os);
int count = 0; int count = 0;
for (final String h: set) { for (final String h: set) {
os.write(h.getBytes()); os.write(UTF8.getBytes(h));
os.write(new byte[]{'\n'}); os.write(new byte[]{'\n'});
count++; count++;
if (System.currentTimeMillis() - time > 10000) { if (System.currentTimeMillis() - time > 10000) {

66
source/de/anomic/data/UserDB.java
View File

@ -88,7 +88,7 @@ public final class UserDB {
public void removeEntry(final String hostName) { public void removeEntry(final String hostName) {
try { try {
userTable.delete(hostName.toLowerCase().getBytes()); userTable.delete(UTF8.getBytes(hostName.toLowerCase()));
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
@ -100,7 +100,7 @@ public final class UserDB {
} }
Map<String, String> record = null; Map<String, String> record = null;
try { try {
record = userTable.get(userName.getBytes()); record = userTable.get(UTF8.getBytes(userName));
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
@ -117,7 +117,7 @@ public final class UserDB {
public String addEntry(final Entry entry) { public String addEntry(final Entry entry) {
try { try {
userTable.insert(entry.userName.getBytes(), entry.mem); userTable.insert(UTF8.getBytes(entry.userName), entry.mem);
return entry.userName; return entry.userName;
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
@ -522,7 +522,7 @@ public final class UserDB {
} }
try { try {
UserDB.this.userTable.insert(getUserName().getBytes(), this.mem); UserDB.this.userTable.insert(UTF8.getBytes(getUserName()), this.mem);
} catch(final Exception e){ } catch(final Exception e){
Log.logException(e); Log.logException(e);
} }
@ -539,7 +539,7 @@ public final class UserDB {
public void setProperty(final String propName, final String newValue) throws IOException, RowSpaceExceededException { public void setProperty(final String propName, final String newValue) throws IOException, RowSpaceExceededException {
this.mem.put(propName, newValue); this.mem.put(propName, newValue);
UserDB.this.userTable.insert(getUserName().getBytes(), this.mem); UserDB.this.userTable.insert(UTF8.getBytes(getUserName()), this.mem);
} }
public String getProperty(final String propName, final String defaultValue) { public String getProperty(final String propName, final String defaultValue) {
@ -550,62 +550,6 @@ public final class UserDB {
return (this.mem.containsKey(accessRight.toString())) ? this.mem.get(accessRight.toString()).equals("true") : false; return (this.mem.containsKey(accessRight.toString())) ? this.mem.get(accessRight.toString()).equals("true") : false;
} }
/**
* @deprecated use hasRight(UPLOAD_RIGHT) instead
*/
@Deprecated
public boolean hasUploadRight() {
return this.hasRight(AccessRight.UPLOAD_RIGHT);
}
/**
* @deprecated use hasRight(DOWNLOAD_RIGHT) instead
*/
@Deprecated
public boolean hasDownloadRight() {
return this.hasRight(AccessRight.DOWNLOAD_RIGHT);
}
/**
* @deprecated use hasRight(PROXY_RIGHT) instead
*/
@Deprecated
public boolean hasProxyRight() {
return this.hasRight(AccessRight.PROXY_RIGHT);
}
/**
* @deprecated use hasRight(ADMIN_RIGHT) instead
*/
@Deprecated
public boolean hasAdminRight() {
return this.hasRight(AccessRight.ADMIN_RIGHT);
}
/**
* @deprecated use hasRight(BLOG_RIGHT) instead
*/
@Deprecated
public boolean hasBlogRight() {
return this.hasRight(AccessRight.BLOG_RIGHT);
}
/**
* @deprecated use hasRight(WIKIADMIN_RIGHT) instead
*/
@Deprecated
public boolean hasWikiAdminRight() {
return this.hasRight(AccessRight.WIKIADMIN_RIGHT);
}
/**
* @deprecated use hasRight(BOOKMARK_RIGHT) instead
*/
@Deprecated
public boolean hasBookmarkRight() {
return this.hasRight(AccessRight.BOOKMARK_RIGHT);
}
public boolean isLoggedOut(){ public boolean isLoggedOut(){
return (this.mem.containsKey(LOGGED_OUT) ? this.mem.get(LOGGED_OUT).equals("true") : false); return (this.mem.containsKey(LOGGED_OUT) ? this.mem.get(LOGGED_OUT).equals("true") : false);
} }

32
source/de/anomic/data/WorkTables.java
View File

@ -99,7 +99,7 @@ public class WorkTables extends Tables {
public byte[] recordAPICall(final serverObjects post, final String servletName, final String type, final String comment) { public byte[] recordAPICall(final serverObjects post, final String servletName, final String type, final String comment) {
// remove the apicall attributes from the post object // remove the apicall attributes from the post object
String pks = post.remove(TABLE_API_COL_APICALL_PK); String pks = post.remove(TABLE_API_COL_APICALL_PK);
byte[] pk = pks == null ? null : pks.getBytes(); byte[] pk = pks == null ? null : UTF8.getBytes(pks);
// generate the apicall url - without the apicall attributes // generate the apicall url - without the apicall attributes
final String apiurl = /*"http://localhost:" + getConfig("port", "8090") +*/ "/" + servletName + "?" + post.toString(); final String apiurl = /*"http://localhost:" + getConfig("port", "8090") +*/ "/" + servletName + "?" + post.toString();
@ -119,12 +119,12 @@ public class WorkTables extends Tables {
if (row == null) { if (row == null) {
// create and insert new entry // create and insert new entry
Data data = new Data(); Data data = new Data();
data.put(TABLE_API_COL_TYPE, type.getBytes()); data.put(TABLE_API_COL_TYPE, UTF8.getBytes(type));
data.put(TABLE_API_COL_COMMENT, comment.getBytes()); data.put(TABLE_API_COL_COMMENT, UTF8.getBytes(comment));
byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes(); byte[] date = UTF8.getBytes(GenericFormatter.SHORT_MILSEC_FORMATTER.format());
data.put(TABLE_API_COL_DATE_RECORDING, date); data.put(TABLE_API_COL_DATE_RECORDING, date);
data.put(TABLE_API_COL_DATE_LAST_EXEC, date); data.put(TABLE_API_COL_DATE_LAST_EXEC, date);
data.put(TABLE_API_COL_URL, apiurl.getBytes()); data.put(TABLE_API_COL_URL, UTF8.getBytes(apiurl));
// insert APICALL attributes // insert APICALL attributes
data.put(TABLE_API_COL_APICALL_COUNT, "1"); data.put(TABLE_API_COL_APICALL_COUNT, "1");
@ -133,7 +133,7 @@ public class WorkTables extends Tables {
// modify and update existing entry // modify and update existing entry
// modify date attributes and patch old values // modify date attributes and patch old values
row.put(TABLE_API_COL_DATE_LAST_EXEC, GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes()); row.put(TABLE_API_COL_DATE_LAST_EXEC, UTF8.getBytes(GenericFormatter.SHORT_MILSEC_FORMATTER.format()));
if (!row.containsKey(TABLE_API_COL_DATE_RECORDING)) row.put(TABLE_API_COL_DATE_RECORDING, row.get(TABLE_API_COL_DATE)); if (!row.containsKey(TABLE_API_COL_DATE_RECORDING)) row.put(TABLE_API_COL_DATE_RECORDING, row.get(TABLE_API_COL_DATE));
row.remove(TABLE_API_COL_DATE); row.remove(TABLE_API_COL_DATE);
@ -180,17 +180,17 @@ public class WorkTables extends Tables {
try { try {
// create and insert new entry // create and insert new entry
Data data = new Data(); Data data = new Data();
data.put(TABLE_API_COL_TYPE, type.getBytes()); data.put(TABLE_API_COL_TYPE, UTF8.getBytes(type));
data.put(TABLE_API_COL_COMMENT, comment.getBytes()); data.put(TABLE_API_COL_COMMENT, UTF8.getBytes(comment));
byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes(); byte[] date = UTF8.getBytes(GenericFormatter.SHORT_MILSEC_FORMATTER.format());
data.put(TABLE_API_COL_DATE_RECORDING, date); data.put(TABLE_API_COL_DATE_RECORDING, date);
data.put(TABLE_API_COL_DATE_LAST_EXEC, date); data.put(TABLE_API_COL_DATE_LAST_EXEC, date);
data.put(TABLE_API_COL_URL, apiurl.getBytes()); data.put(TABLE_API_COL_URL, UTF8.getBytes(apiurl));
// insert APICALL attributes // insert APICALL attributes
data.put(TABLE_API_COL_APICALL_COUNT, "1".getBytes()); data.put(TABLE_API_COL_APICALL_COUNT, UTF8.getBytes("1"));
data.put(TABLE_API_COL_APICALL_SCHEDULE_TIME, Integer.toString(time).getBytes()); data.put(TABLE_API_COL_APICALL_SCHEDULE_TIME, UTF8.getBytes(Integer.toString(time)));
data.put(TABLE_API_COL_APICALL_SCHEDULE_UNIT, unit.getBytes()); data.put(TABLE_API_COL_APICALL_SCHEDULE_UNIT, UTF8.getBytes(unit));
calculateAPIScheduler(data, false); // set next execution time calculateAPIScheduler(data, false); // set next execution time
pk = super.insert(TABLE_API_NAME, data); pk = super.insert(TABLE_API_NAME, data);
} catch (IOException e) { } catch (IOException e) {
@ -220,7 +220,7 @@ public class WorkTables extends Tables {
for (String pk: pks) { for (String pk: pks) {
Tables.Row row = null; Tables.Row row = null;
try { try {
row = select(WorkTables.TABLE_API_NAME, pk.getBytes()); row = select(WorkTables.TABLE_API_NAME, UTF8.getBytes(pk));
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
@ -307,11 +307,11 @@ public class WorkTables extends Tables {
try { try {
// create and insert new entry // create and insert new entry
Data data = new Data(); Data data = new Data();
byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes(); byte[] date = UTF8.getBytes(GenericFormatter.SHORT_MILSEC_FORMATTER.format());
data.put(TABLE_SEARCH_FAILURE_COL_URL, url.toNormalform(true, false)); data.put(TABLE_SEARCH_FAILURE_COL_URL, url.toNormalform(true, false));
data.put(TABLE_SEARCH_FAILURE_COL_DATE, date); data.put(TABLE_SEARCH_FAILURE_COL_DATE, date);
data.put(TABLE_SEARCH_FAILURE_COL_WORDS, queryHashes.export()); data.put(TABLE_SEARCH_FAILURE_COL_WORDS, queryHashes.export());
data.put(TABLE_SEARCH_FAILURE_COL_COMMENT, reason.getBytes()); data.put(TABLE_SEARCH_FAILURE_COL_COMMENT, UTF8.getBytes(reason));
super.insert(TABLE_SEARCH_FAILURE_NAME, url.hash(), data); super.insert(TABLE_SEARCH_FAILURE_NAME, url.hash(), data);
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);

10
source/de/anomic/data/wiki/WikiBoard.java
View File

@ -321,7 +321,7 @@ public class WikiBoard {
* @param subject subject of child of current Entry. * @param subject subject of child of current Entry.
*/ */
void setChild(final String subject) { void setChild(final String subject) {
record.put("child", Base64Order.enhancedCoder.encode(subject.getBytes())); record.put("child", Base64Order.enhancedCoder.encode(UTF8.getBytes(subject)));
} }
/** /**
@ -369,9 +369,9 @@ public class WikiBoard {
entry.setAncestorDate(oldDate); entry.setAncestorDate(oldDate);
oldEntry.setChild(entry.subject()); oldEntry.setChild(entry.subject());
// write the backup // write the backup
bkpbase.insert((entry.key + dateString(oldDate)).getBytes(), oldEntry.record); bkpbase.insert(UTF8.getBytes(entry.key + dateString(oldDate)), oldEntry.record);
// write the new page // write the new page
datbase.insert(entry.key.getBytes(), entry.record); datbase.insert(UTF8.getBytes(entry.key), entry.record);
key = entry.key; key = entry.key;
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
@ -401,8 +401,8 @@ public class WikiBoard {
if (copyOfKey.length() > keyLength) { if (copyOfKey.length() > keyLength) {
copyOfKey = copyOfKey.substring(0, keyLength); copyOfKey = copyOfKey.substring(0, keyLength);
} }
final Map<String, String> record = base.get(copyOfKey.getBytes()); final Map<String, String> record = base.get(UTF8.getBytes(copyOfKey));
ret = (record == null) ? newEntry(copyOfKey, ANONYMOUS, "127.0.0.1", "New Page", "".getBytes()) : new Entry(copyOfKey, record); ret = (record == null) ? newEntry(copyOfKey, ANONYMOUS, "127.0.0.1", "New Page", UTF8.getBytes("")) : new Entry(copyOfKey, record);
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {

6
source/de/anomic/http/server/HTTPDFileHandler.java
View File

@ -1325,17 +1325,19 @@ public final class HTTPDFileHandler {
sbuffer = sbuffer.replaceAll("(href|src)='([^:\"]+)'", "$1='/proxy.html?url=http://"+proxyurl.getHost()+directory+"/$2'"); sbuffer = sbuffer.replaceAll("(href|src)='([^:\"]+)'", "$1='/proxy.html?url=http://"+proxyurl.getHost()+directory+"/$2'");
sbuffer = sbuffer.replaceAll("url\\(", "url(/proxy.html?url=http://"+proxyurl.getHost()+proxyurl.getPath()); sbuffer = sbuffer.replaceAll("url\\(", "url(/proxy.html?url=http://"+proxyurl.getHost()+proxyurl.getPath());
byte[] sbb = UTF8.getBytes(sbuffer);
if (outgoingHeader.containsKey(HeaderFramework.TRANSFER_ENCODING)) { if (outgoingHeader.containsKey(HeaderFramework.TRANSFER_ENCODING)) {
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader); HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader);
out = new ChunkedOutputStream(out); out = new ChunkedOutputStream(out);
} else { } else {
outgoingHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(sbuffer.getBytes().length)); outgoingHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(sbb.length));
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader); HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader);
} }
out.write(UTF8.getBytes(sbuffer)); out.write(sbb);
} else { } else {
if (!outgoingHeader.containsKey(HeaderFramework.CONTENT_LENGTH)) if (!outgoingHeader.containsKey(HeaderFramework.CONTENT_LENGTH))
outgoingHeader.put(HeaderFramework.CONTENT_LENGTH, prop.getProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_SIZE)); outgoingHeader.put(HeaderFramework.CONTENT_LENGTH, prop.getProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_SIZE));

24
source/de/anomic/http/server/HTTPDemon.java
View File

@ -410,7 +410,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
HTTPDFileHandler.doGet(prop, header, session.out); HTTPDFileHandler.doGet(prop, header, session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted, 1)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this server, because you are using a non-granted IP (" + session.userAddress.getHostAddress() + "). allowed are only connections that match with the following filter: " + switchboard.getConfig("serverClient", "*") + serverCore.CRLF_STRING).getBytes()); session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted, 1)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this server, because you are using a non-granted IP (" + session.userAddress.getHostAddress() + "). allowed are only connections that match with the following filter: " + switchboard.getConfig("serverClient", "*") + serverCore.CRLF_STRING));
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} else { } else {
@ -420,7 +420,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
HTTPDProxyHandler.doGet(prop, header, session.out); HTTPDProxyHandler.doGet(prop, header, session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted, 2)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using a non-granted IP (" + session.userAddress.getHostAddress() + "). allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING).getBytes()); session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted, 2)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using a non-granted IP (" + session.userAddress.getHostAddress() + "). allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING));
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} }
@ -477,8 +477,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
HTTPDFileHandler.doHead(prop, header, session.out); HTTPDFileHandler.doHead(prop, header, session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING));
serverCore.CRLF_STRING).getBytes());
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} else { } else {
@ -488,8 +487,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
HTTPDProxyHandler.doHead(prop, header, session.out); HTTPDProxyHandler.doHead(prop, header, session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING));
serverCore.CRLF_STRING).getBytes());
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} }
@ -523,7 +521,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// "A server which receives an entity-body with a transfer-coding it does // "A server which receives an entity-body with a transfer-coding it does
// not understand SHOULD return 501 (Unimplemented), and close the // not understand SHOULD return 501 (Unimplemented), and close the
// connection." [RFC 2616, section 3.6] // connection." [RFC 2616, section 3.6]
session.out.write((httpVersion + " 501 transfer-encoding not implemented" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you send a transfer-encoding to this server, which is not supported: " + transferEncoding + serverCore.CRLF_STRING).getBytes()); session.out.write(UTF8.getBytes(httpVersion + " 501 transfer-encoding not implemented" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you send a transfer-encoding to this server, which is not supported: " + transferEncoding + serverCore.CRLF_STRING));
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} else { } else {
@ -543,7 +541,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
HTTPDFileHandler.doPost(prop, header, session.out, sessionIn); HTTPDFileHandler.doPost(prop, header, session.out, sessionIn);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this server, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("serverClient", "*") + serverCore.CRLF_STRING).getBytes()); session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this server, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("serverClient", "*") + serverCore.CRLF_STRING));
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} else { } else {
@ -553,7 +551,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
HTTPDProxyHandler.doPost(prop, header, session.out, sessionIn); HTTPDProxyHandler.doPost(prop, header, session.out, sessionIn);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING).getBytes()); session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING));
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} }
@ -619,15 +617,15 @@ public final class HTTPDemon implements serverHandler, Cloneable {
if (!(allowProxy(session))) { if (!(allowProxy(session))) {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING).getBytes()); session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING));
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
if (port != 443 && switchboard.getConfig("secureHttps", "true").equals("true")) { if (port != 443 && switchboard.getConfig("secureHttps", "true").equals("true")) {
// security: connection only to ssl port // security: connection only to ssl port
// we send a 403 (forbidden) error back // we send a 403 (forbidden) error back
session.out.write((httpVersion + " 403 Connection to non-443 forbidden" + session.out.write(UTF8.getBytes(httpVersion + " 403 Connection to non-443 forbidden" +
serverCore.CRLF_STRING + serverCore.CRLF_STRING).getBytes()); serverCore.CRLF_STRING + serverCore.CRLF_STRING));
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
@ -637,7 +635,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
HTTPDProxyHandler.doConnect(prop, header, session.in, session.out); HTTPDProxyHandler.doConnect(prop, header, session.in, session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING).getBytes()); session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING));
} }
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;

6
source/de/anomic/http/server/TemplateEngine.java
View File

@ -401,9 +401,9 @@ public final class TemplateEngine {
if (br != null) try { br.close(); br=null; } catch (final Exception e) {} if (br != null) try { br.close(); br=null; } catch (final Exception e) {}
} }
final PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes())); final PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes()));
structure.append("<fileinclude file=\"".getBytes()).append(filename).append(">\n".getBytes()); structure.append(UTF8.getBytes("<fileinclude file=\"")).append(filename).append(UTF8.getBytes(">\n"));
structure.append(writeTemplate(pis2, out, pattern, dflt, prefix)); structure.append(writeTemplate(pis2, out, pattern, dflt, prefix));
structure.append("</fileinclude>\n".getBytes()); structure.append(UTF8.getBytes("</fileinclude>\n"));
} }
} }
@ -450,7 +450,7 @@ public final class TemplateEngine {
private final static byte[] newPrefix(final byte[] oldPrefix, final byte[] multi_key, final int i) { private final static byte[] newPrefix(final byte[] oldPrefix, final byte[] multi_key, final int i) {
final ByteBuffer newPrefix = new ByteBuffer(oldPrefix.length + multi_key.length + 8); final ByteBuffer newPrefix = new ByteBuffer(oldPrefix.length + multi_key.length + 8);
newPrefix.append(oldPrefix).append(multi_key).append(ul).append(Integer.toString(i).getBytes()).append(ul); newPrefix.append(oldPrefix).append(multi_key).append(ul).append(UTF8.getBytes(Integer.toString(i))).append(ul);
try { try {
newPrefix.close(); newPrefix.close();
} catch (IOException e) { } catch (IOException e) {

12
source/de/anomic/search/MetadataRepository.java
View File

@ -213,7 +213,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
public kiter(final boolean up, final String firstHash) throws IOException { public kiter(final boolean up, final String firstHash) throws IOException {
this.up = up; this.up = up;
this.iter = urlIndexFile.rows(up, (firstHash == null) ? null : firstHash.getBytes()); this.iter = urlIndexFile.rows(up, (firstHash == null) ? null : UTF8.getBytes(firstHash));
this.error = false; this.error = false;
} }
@ -274,7 +274,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
final Iterator<String> eiter2 = damagedURLS.iterator(); final Iterator<String> eiter2 = damagedURLS.iterator();
byte[] urlHashBytes; byte[] urlHashBytes;
while (eiter2.hasNext()) { while (eiter2.hasNext()) {
urlHashBytes = eiter2.next().getBytes(); urlHashBytes = UTF8.getBytes(eiter2.next());
// trying to fix the invalid URL // trying to fix the invalid URL
String oldUrlStr = null; String oldUrlStr = null;
@ -293,7 +293,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
if (client.HEADResponse(newUrl.toString()) != null if (client.HEADResponse(newUrl.toString()) != null
&& client.getHttpResponse().getStatusLine().getStatusCode() == 200) { && client.getHttpResponse().getStatusLine().getStatusCode() == 200) {
entry.setCol(1, newUrl.toString().getBytes()); entry.setCol(1, UTF8.getBytes(newUrl.toString()));
urlIndexFile.put(entry); urlIndexFile.put(entry);
if (log.isInfo()) log.logInfo("UrlDB-Entry with urlHash '" + UTF8.String(urlHashBytes) + "' corrected\n\tURL: " + oldUrlStr + " -> " + newUrlStr); if (log.isInfo()) log.logInfo("UrlDB-Entry with urlHash '" + UTF8.String(urlHashBytes) + "' corrected\n\tURL: " + oldUrlStr + " -> " + newUrlStr);
} else { } else {
@ -585,7 +585,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
TreeSet<String> set = new TreeSet<String>(); TreeSet<String> set = new TreeSet<String>();
for (hashStat hs: map.values()) { for (hashStat hs: map.values()) {
if (hs == null) continue; if (hs == null) continue;
urlref = this.load(hs.urlhash.getBytes(), null, 0); urlref = this.load(UTF8.getBytes(hs.urlhash), null, 0);
if (urlref == null || urlref.metadata() == null || urlref.metadata().url() == null || urlref.metadata().url().getHost() == null) continue; if (urlref == null || urlref.metadata() == null || urlref.metadata().url() == null || urlref.metadata().url().getHost() == null) continue;
set.add(urlref.metadata().url().getHost()); set.add(urlref.metadata().url().getHost());
count--; count--;
@ -619,7 +619,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
while (j.hasNext()) { while (j.hasNext()) {
urlhash = j.next(); urlhash = j.next();
if (urlhash == null) continue; if (urlhash == null) continue;
urlref = this.load(urlhash.getBytes(), null, 0); urlref = this.load(UTF8.getBytes(urlhash), null, 0);
if (urlref == null || urlref.metadata() == null || urlref.metadata().url() == null || urlref.metadata().url().getHost() == null) continue; if (urlref == null || urlref.metadata() == null || urlref.metadata().url() == null || urlref.metadata().url().getHost() == null) continue;
if (statsDump == null) return new ArrayList<hostStat>().iterator(); // some other operation has destroyed the object if (statsDump == null) return new ArrayList<hostStat>().iterator(); // some other operation has destroyed the object
comps = urlref.metadata(); comps = urlref.metadata();
@ -675,7 +675,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
// then delete the urls using this list // then delete the urls using this list
int cnt = 0; int cnt = 0;
for (String h: l) { for (String h: l) {
if (urlIndexFile.delete(h.getBytes())) cnt++; if (urlIndexFile.delete(UTF8.getBytes(h))) cnt++;
} }
// finally remove the line with statistics // finally remove the line with statistics

11
source/de/anomic/search/QueryParams.java
View File

@ -107,19 +107,20 @@ public final class QueryParams {
public final String userAgent; public final String userAgent;
public boolean filterfailurls; public boolean filterfailurls;
public QueryParams(final String queryString, public QueryParams(
final String queryString,
final int itemsPerPage, final int itemsPerPage,
final Bitfield constraint, final Bitfield constraint,
final Segment indexSegment, final Segment indexSegment,
final RankingProfile ranking, final RankingProfile ranking,
final String userAgent) { final String userAgent) {
if ((queryString.length() == 12) && (Base64Order.enhancedCoder.wellformed(queryString.getBytes()))) { if ((queryString.length() == 12) && (Base64Order.enhancedCoder.wellformed(UTF8.getBytes(queryString)))) {
this.queryString = null; this.queryString = null;
this.queryHashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); this.queryHashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
this.excludeHashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); this.excludeHashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
try { try {
this.queryHashes.put(queryString.getBytes()); this.queryHashes.put(UTF8.getBytes(queryString));
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} }
@ -255,7 +256,7 @@ public final class QueryParams {
final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
if (query != null) { if (query != null) {
for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try { for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try {
keyhashes.put(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes()); keyhashes.put(UTF8.getBytes(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)));
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} }
@ -267,7 +268,7 @@ public final class QueryParams {
final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
if (query != null) { if (query != null) {
for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try { for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try {
keyhashes.put(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes()); keyhashes.put(UTF8.getBytes(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)));
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} }

2
source/de/anomic/search/RankingProcess.java
View File

@ -587,7 +587,7 @@ public final class RankingProcess extends Thread {
domhash = domhashs.next(); domhash = domhashs.next();
if (domhash == null) continue; if (domhash == null) continue;
urlhash = this.hostResolver.get(domhash); urlhash = this.hostResolver.get(domhash);
row = urlhash == null ? null : this.query.getSegment().urlMetadata().load(urlhash.getBytes(), null, 0); row = urlhash == null ? null : this.query.getSegment().urlMetadata().load(UTF8.getBytes(urlhash), null, 0);
hostname = row == null ? null : row.metadata().url().getHost(); hostname = row == null ? null : row.metadata().url().getHost();
if (hostname != null) { if (hostname != null) {
result.set(hostname, this.hostNavigator.get(domhash)); result.set(hostname, this.hostNavigator.get(domhash));

20
source/de/anomic/search/Switchboard.java
View File

@ -1045,7 +1045,7 @@ public final class Switchboard extends serverSwitch {
return network.indexOf(peer) >= 0; return network.indexOf(peer) >= 0;
} else if (clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER)) { } else if (clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER)) {
// check if we got the request from a peer in the public cluster // check if we got the request from a peer in the public cluster
return this.clusterhashes.containsKey(peer.getBytes()); return this.clusterhashes.containsKey(UTF8.getBytes(peer));
} else { } else {
return false; return false;
} }
@ -1063,7 +1063,7 @@ public final class Switchboard extends serverSwitch {
return network.indexOf(seed.getPublicAddress()) >= 0; return network.indexOf(seed.getPublicAddress()) >= 0;
} else if (clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER)) { } else if (clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER)) {
// check if we got the request from a peer in the public cluster // check if we got the request from a peer in the public cluster
return this.clusterhashes.containsKey(seed.hash.getBytes()); return this.clusterhashes.containsKey(UTF8.getBytes(seed.hash));
} else { } else {
return false; return false;
} }
@ -1363,7 +1363,7 @@ public final class Switchboard extends serverSwitch {
// create a queue entry // create a queue entry
Document document = surrogate.document(); Document document = surrogate.document();
Request request = new Request( Request request = new Request(
peers.mySeed().hash.getBytes(), UTF8.getBytes(peers.mySeed().hash),
surrogate.getIdentifier(true), surrogate.getIdentifier(true),
null, null,
"", "",
@ -1525,7 +1525,7 @@ public final class Switchboard extends serverSwitch {
Long.toString(CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE))); Long.toString(CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE)));
insert = true; insert = true;
} }
if (insert) crawler.putActive(selentry.handle().getBytes(), selentry); if (insert) crawler.putActive(UTF8.getBytes(selentry.handle()), selentry);
} }
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
@ -1550,7 +1550,7 @@ public final class Switchboard extends serverSwitch {
Log.logException(e); Log.logException(e);
} }
for (final String pk: pks) try { for (final String pk: pks) try {
row = this.tables.select(WorkTables.TABLE_API_NAME, pk.getBytes()); row = this.tables.select(WorkTables.TABLE_API_NAME, UTF8.getBytes(pk));
WorkTables.calculateAPIScheduler(row, true); // calculate next update time WorkTables.calculateAPIScheduler(row, true); // calculate next update time
this.tables.update(WorkTables.TABLE_API_NAME, row); this.tables.update(WorkTables.TABLE_API_NAME, row);
} catch (IOException e) { } catch (IOException e) {
@ -1976,7 +1976,7 @@ public final class Switchboard extends serverSwitch {
condenser, condenser,
searchEvent, searchEvent,
sourceName); sourceName);
yacyChannel.channels(Base64Order.enhancedCoder.equal(queueEntry.initiator(), peers.mySeed().hash.getBytes()) ? yacyChannel.LOCALINDEXING : yacyChannel.REMOTEINDEXING).addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform(true, false))); yacyChannel.channels(Base64Order.enhancedCoder.equal(queueEntry.initiator(), UTF8.getBytes(peers.mySeed().hash)) ? yacyChannel.LOCALINDEXING : yacyChannel.REMOTEINDEXING).addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform(true, false)));
} catch (final IOException e) { } catch (final IOException e) {
//if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase); //if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase);
addURLtoErrorDB(queueEntry.url(), (referrerURL == null) ? null : referrerURL.hash(), queueEntry.initiator(), dc_title, "error storing url: " + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase + ", error = " + e.getMessage()); addURLtoErrorDB(queueEntry.url(), (referrerURL == null) ? null : referrerURL.hash(), queueEntry.initiator(), dc_title, "error storing url: " + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase + ", error = " + e.getMessage());
@ -1987,8 +1987,8 @@ public final class Switchboard extends serverSwitch {
for (final Map.Entry<MultiProtocolURI, String> rssEntry : document.getRSS().entrySet()) { for (final Map.Entry<MultiProtocolURI, String> rssEntry : document.getRSS().entrySet()) {
final Tables.Data rssRow = new Tables.Data(); final Tables.Data rssRow = new Tables.Data();
rssRow.put("referrer", queueEntry.url().hash()); rssRow.put("referrer", queueEntry.url().hash());
rssRow.put("url", rssEntry.getKey().toNormalform(true, false).getBytes()); rssRow.put("url", UTF8.getBytes(rssEntry.getKey().toNormalform(true, false)));
rssRow.put("title", rssEntry.getValue().getBytes()); rssRow.put("title", UTF8.getBytes(rssEntry.getValue()));
rssRow.put("recording_date", new Date()); rssRow.put("recording_date", new Date());
try { try {
this.tables.update("rss", new DigestURI(rssEntry.getKey()).hash(), rssRow); this.tables.update("rss", new DigestURI(rssEntry.getKey()).hash(), rssRow);
@ -2001,7 +2001,7 @@ public final class Switchboard extends serverSwitch {
ResultURLs.stack( ResultURLs.stack(
newEntry, // loaded url db entry newEntry, // loaded url db entry
queueEntry.initiator(), // initiator peer hash queueEntry.initiator(), // initiator peer hash
this.peers.mySeed().hash.getBytes(), // executor peer hash UTF8.getBytes(this.peers.mySeed().hash), // executor peer hash
processCase // process case processCase // process case
); );
@ -2079,7 +2079,7 @@ public final class Switchboard extends serverSwitch {
} }
if (indexSegments.segment(process).urlMetadata.exists(url.hash())) return; // don't do double-work if (indexSegments.segment(process).urlMetadata.exists(url.hash())) return; // don't do double-work
final Request request = loader.request(url, true, true); final Request request = loader.request(url, true, true);
final CrawlProfile profile = sb.crawler.getActive(request.profileHandle().getBytes()); final CrawlProfile profile = sb.crawler.getActive(UTF8.getBytes(request.profileHandle()));
String acceptedError = this.crawlStacker.checkAcceptance(url, profile, 0); String acceptedError = this.crawlStacker.checkAcceptance(url, profile, 0);
if (acceptedError != null) { if (acceptedError != null) {
log.logWarning("addToIndex: cannot load " + url.toNormalform(false, false) + ": " + acceptedError); log.logWarning("addToIndex: cannot load " + url.toNormalform(false, false) + ": " + acceptedError);

1
source/de/anomic/search/SwitchboardConstants.java
View File

@ -351,7 +351,6 @@ public final class SwitchboardConstants {
* whole database of known RWIs and URLs as well as dumps of the DHT-In and DHT-Out caches are stored</p> * whole database of known RWIs and URLs as well as dumps of the DHT-In and DHT-Out caches are stored</p>
*/ */
public static final String INDEX_PRIMARY_PATH = "indexPrimaryPath"; // this is a relative path to the data root public static final String INDEX_PRIMARY_PATH = "indexPrimaryPath"; // this is a relative path to the data root
public static final String INDEX_SECONDARY_PATH = "indexSecondaryPath"; // this is a absolute path to any location
public static final String INDEX_PATH_DEFAULT = "DATA/INDEX"; public static final String INDEX_PATH_DEFAULT = "DATA/INDEX";
/** /**
* <p><code>public static final String <strong>LISTS_PATH</strong> = "listsPath"</code></p> * <p><code>public static final String <strong>LISTS_PATH</strong> = "listsPath"</code></p>

8
source/de/anomic/tools/cryptbig.java
View File

@ -261,10 +261,10 @@ public class cryptbig {
final String A = UTF8.String(ecipher.doFinal(X.getBytes("UTF8"))); final String A = UTF8.String(ecipher.doFinal(X.getBytes("UTF8")));
final String B = UTF8.String(ecipher.doFinal(Base64Order.standardCoder.encodeLongSB(A.length(), 2).toString().getBytes("UTF8"))); // most probable not longer than 4 final String B = UTF8.String(ecipher.doFinal(Base64Order.standardCoder.encodeLongSB(A.length(), 2).toString().getBytes("UTF8"))); // most probable not longer than 4
final String C = Base64Order.standardCoder.encodeLongSB(B.length(), 1).toString(); // fixed length 1 (6 bits, that should be enough) final String C = Base64Order.standardCoder.encodeLongSB(B.length(), 1).toString(); // fixed length 1 (6 bits, that should be enough)
fout.write(magicString.getBytes()); // the magic string, used to identify a 'crypt'-file fout.write(UTF8.getBytes(magicString)); // the magic string, used to identify a 'crypt'-file
fout.write(C.getBytes()); fout.write(UTF8.getBytes(C));
fout.write(B.getBytes()); fout.write(UTF8.getBytes(B));
fout.write(A.getBytes()); fout.write(UTF8.getBytes(A));
// write content of file // write content of file
copy(fout, fin, 512); copy(fout, fin, 512);

2
source/de/anomic/yacy/dht/PeerSelection.java
View File

@ -191,7 +191,7 @@ public class PeerSelection {
} }
public static byte[] selectTransferStart() { public static byte[] selectTransferStart() {
return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + Word.commonHashLength).getBytes(); return UTF8.getBytes(Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + Word.commonHashLength));
} }
public static byte[] limitOver(final yacySeedDB seedDB, final byte[] startHash) { public static byte[] limitOver(final yacySeedDB seedDB, final byte[] startHash) {

5
source/de/anomic/yacy/yacyAccessible.java
View File

@ -30,6 +30,7 @@ package de.anomic.yacy;
import java.io.File; import java.io.File;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
@ -47,7 +48,7 @@ public class yacyAccessible {
final Switchboard sb = Switchboard.getSwitchboard(); final Switchboard sb = Switchboard.getSwitchboard();
final File shortcut = new File(sb.getAppPath() + "/addon/YaCy-Search.html".replace("/", File.separator)); final File shortcut = new File(sb.getAppPath() + "/addon/YaCy-Search.html".replace("/", File.separator));
final String content = "<meta http-equiv=\"refresh\" content=\"0;url=http://localhost:" + newPort + "/\">"; final String content = "<meta http-equiv=\"refresh\" content=\"0;url=http://localhost:" + newPort + "/\">";
FileUtils.copy(content.getBytes(), shortcut); FileUtils.copy(UTF8.getBytes(content), shortcut);
} catch (final Exception e) { } catch (final Exception e) {
return; return;
} }
@ -62,7 +63,7 @@ public class yacyAccessible {
final Switchboard sb = Switchboard.getSwitchboard(); final Switchboard sb = Switchboard.getSwitchboard();
final File shortcut = new File(sb.getAppPath() + "/addon/YaCy-Search.bat".replace("/", File.separator)); final File shortcut = new File(sb.getAppPath() + "/addon/YaCy-Search.bat".replace("/", File.separator));
final String content = "rundll32 url.dll,FileProtocolHandler \"http://localhost:" + newPort + "\""; final String content = "rundll32 url.dll,FileProtocolHandler \"http://localhost:" + newPort + "\"";
FileUtils.copy(content.getBytes(), shortcut); FileUtils.copy(UTF8.getBytes(content), shortcut);
} catch (final Exception e) { } catch (final Exception e) {
return; return;
} }

12
source/de/anomic/yacy/yacyClient.java
View File

@ -437,7 +437,7 @@ public final class yacyClient {
final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words]; final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words];
for (int i = 0; i < words; i++) { for (int i = 0; i < words; i++) {
try { try {
container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count); container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, UTF8.getBytes(wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)), count);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
return -1; return -1;
@ -479,7 +479,7 @@ public final class yacyClient {
// passed all checks, store url // passed all checks, store url
try { try {
indexSegment.urlMetadata().store(urlEntry); indexSegment.urlMetadata().store(urlEntry);
ResultURLs.stack(urlEntry, mySeed.hash.getBytes(), target.hash.getBytes(), EventOrigin.QUERIES); ResultURLs.stack(urlEntry, mySeed.hash.getBytes(), UTF8.getBytes(target.hash), EventOrigin.QUERIES);
} catch (final IOException e) { } catch (final IOException e) {
yacyCore.log.logWarning("could not store search result", e); yacyCore.log.logWarning("could not store search result", e);
continue; // db-error continue; // db-error
@ -672,10 +672,10 @@ public final class yacyClient {
indexabstract = new TreeMap<byte[], String>(Base64Order.enhancedCoder); indexabstract = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
for (Map.Entry<String, String> entry: resultMap.entrySet()) { for (Map.Entry<String, String> entry: resultMap.entrySet()) {
if (entry.getKey().startsWith("indexcount.")) { if (entry.getKey().startsWith("indexcount.")) {
indexcount.put(entry.getKey().substring(11).getBytes(), Integer.parseInt(entry.getValue())); indexcount.put(UTF8.getBytes(entry.getKey().substring(11)), Integer.parseInt(entry.getValue()));
} }
if (entry.getKey().startsWith("indexabstract.")) { if (entry.getKey().startsWith("indexabstract.")) {
indexabstract.put(entry.getKey().substring(14).getBytes(), entry.getValue()); indexabstract.put(UTF8.getBytes(entry.getKey().substring(14)), entry.getValue());
} }
} }
references = resultMap.get("references").split(","); references = resultMap.get("references").split(",");
@ -862,7 +862,7 @@ public final class yacyClient {
// extract the urlCache from the result // extract the urlCache from the result
final URIMetadataRow[] urls = new URIMetadataRow[uhs.length]; final URIMetadataRow[] urls = new URIMetadataRow[uhs.length];
for (int i = 0; i < uhs.length; i++) { for (int i = 0; i < uhs.length; i++) {
urls[i] = urlCache.get(uhs[i].getBytes()); urls[i] = urlCache.get(UTF8.getBytes(uhs[i]));
if (urls[i] == null) { if (urls[i] == null) {
if (yacyCore.log.isFine()) yacyCore.log.logFine("DEBUG transferIndex: requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'"); if (yacyCore.log.isFine()) yacyCore.log.logFine("DEBUG transferIndex: requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'");
} }
@ -1027,7 +1027,7 @@ public final class yacyClient {
searchlines.add(args[2]); searchlines.add(args[2]);
} }
for (final String line: searchlines) { for (final String line: searchlines) {
final byte[] wordhashe = QueryParams.hashSet2hashString(Word.words2hashesHandles(QueryParams.cleanQuery(line)[0])).getBytes(); final byte[] wordhashe = UTF8.getBytes(QueryParams.hashSet2hashString(Word.words2hashesHandles(QueryParams.cleanQuery(line)[0])));
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
SearchResult result; SearchResult result;
try { try {

8
source/de/anomic/yacy/yacyNewsDB.java
View File

@ -137,7 +137,7 @@ public class yacyNewsDB {
} }
public void remove(final String id) throws IOException { public void remove(final String id) throws IOException {
news.delete(id.getBytes()); news.delete(UTF8.getBytes(id));
} }
public synchronized Record put(final Record record) throws IOException, RowSpaceExceededException { public synchronized Record put(final Record record) throws IOException, RowSpaceExceededException {
@ -151,7 +151,7 @@ public class yacyNewsDB {
public synchronized Record get(final String id) throws IOException { public synchronized Record get(final String id) throws IOException {
try { try {
return b2r(news.get(id.getBytes())); return b2r(news.get(UTF8.getBytes(id)));
} catch (final kelondroException e) { } catch (final kelondroException e) {
resetDB(); resetDB();
return null; return null;
@ -180,9 +180,9 @@ public class yacyNewsDB {
attributes = new HashMap<String, String>().toString(); attributes = new HashMap<String, String>().toString();
} }
final Row.Entry entry = this.news.row().newEntry(); final Row.Entry entry = this.news.row().newEntry();
entry.setCol(0, r.id().getBytes()); entry.setCol(0, UTF8.getBytes(r.id()));
entry.setCol(1, UTF8.getBytes(r.category())); entry.setCol(1, UTF8.getBytes(r.category()));
entry.setCol(2, (r.received() == null) ? null : my_SHORT_SECOND_FORMATTER.format(r.received()).getBytes()); entry.setCol(2, (r.received() == null) ? null : UTF8.getBytes(my_SHORT_SECOND_FORMATTER.format(r.received())));
entry.setCol(3, Base64Order.enhancedCoder.encodeLongBA(r.distributed(), 2)); entry.setCol(3, Base64Order.enhancedCoder.encodeLongBA(r.distributed(), 2));
entry.setCol(4, UTF8.getBytes(attributes)); entry.setCol(4, UTF8.getBytes(attributes));
return entry; return entry;

7
source/de/anomic/yacy/yacyNewsQueue.java
View File

@ -50,6 +50,7 @@ import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.index.Column; import net.yacy.kelondro.index.Column;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
@ -146,7 +147,7 @@ public class yacyNewsQueue {
record = i.next(); record = i.next();
if ((record != null) && (record.id().equals(id))) { if ((record != null) && (record.id().equals(id))) {
try { try {
this.queueStack.remove(id.getBytes()); this.queueStack.remove(UTF8.getBytes(id));
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);
} }
@ -167,8 +168,8 @@ public class yacyNewsQueue {
if (r == null) return null; if (r == null) return null;
newsDB.put(r); newsDB.put(r);
final Row.Entry b = queueStack.row().newEntry(new byte[][]{ final Row.Entry b = queueStack.row().newEntry(new byte[][]{
r.id().getBytes(), UTF8.getBytes(r.id()),
GenericFormatter.SHORT_SECOND_FORMATTER.format().getBytes()}); UTF8.getBytes(GenericFormatter.SHORT_SECOND_FORMATTER.format())});
return b; return b;
} }

3
source/de/anomic/yacy/yacyPeerActions.java
View File

@ -28,6 +28,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MapTools; import net.yacy.kelondro.util.MapTools;
@ -205,7 +206,7 @@ public class yacyPeerActions {
// we do this if we did not get contact with the other peer // we do this if we did not get contact with the other peer
if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: no contact to a " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause); if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: no contact to a " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause);
synchronized (seedDB) { synchronized (seedDB) {
if (!seedDB.hasDisconnected(peer.hash.getBytes())) { disconnects++; } if (!seedDB.hasDisconnected(UTF8.getBytes(peer.hash))) { disconnects++; }
peer.put("dct", Long.toString(System.currentTimeMillis())); peer.put("dct", Long.toString(System.currentTimeMillis()));
seedDB.addDisconnected(peer); // update info seedDB.addDisconnected(peer); // update info
} }

2
source/de/anomic/yacy/yacySearch.java
View File

@ -220,7 +220,7 @@ public class yacySearch extends Thread {
// prepare seed targets and threads // prepare seed targets and threads
final yacySeed targetPeer = peers.getConnected(targethash); final yacySeed targetPeer = peers.getConnected(targethash);
if (targetPeer == null || targetPeer.hash == null) return null; if (targetPeer == null || targetPeer.hash == null) return null;
if (clusterselection != null) targetPeer.setAlternativeAddress(clusterselection.get(targetPeer.hash.getBytes())); if (clusterselection != null) targetPeer.setAlternativeAddress(clusterselection.get(UTF8.getBytes(targetPeer.hash)));
final yacySearch searchThread = new yacySearch( final yacySearch searchThread = new yacySearch(
wordhashes, "", urlhashes, Pattern.compile(""), Pattern.compile(".*"), "", "", "", 20, time, 9999, true, 0, targetPeer, wordhashes, "", urlhashes, Pattern.compile(""), Pattern.compile(".*"), "", "", "", 20, time, 9999, true, 0, targetPeer,
indexSegment, peers, containerCache, null, blacklist, rankingProfile, constraint); indexSegment, peers, containerCache, null, blacklist, rankingProfile, constraint);

18
source/de/anomic/yacy/yacySeed.java
View File

@ -615,13 +615,13 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
private boolean getFlag(final int flag) { private boolean getFlag(final int flag) {
final String flags = get(yacySeed.FLAGS, yacySeed.FLAGSZERO); final String flags = get(yacySeed.FLAGS, yacySeed.FLAGSZERO);
return (new bitfield(flags.getBytes())).get(flag); return (new bitfield(UTF8.getBytes(flags))).get(flag);
} }
private void setFlag(final int flag, final boolean value) { private void setFlag(final int flag, final boolean value) {
String flags = get(yacySeed.FLAGS, yacySeed.FLAGSZERO); String flags = get(yacySeed.FLAGS, yacySeed.FLAGSZERO);
if (flags.length() != 4) { flags = yacySeed.FLAGSZERO; } if (flags.length() != 4) { flags = yacySeed.FLAGSZERO; }
final bitfield f = new bitfield(flags.getBytes()); final bitfield f = new bitfield(UTF8.getBytes(flags));
f.set(flag, value); f.set(flag, value);
dna.put(yacySeed.FLAGS, UTF8.String(f.getBytes())); dna.put(yacySeed.FLAGS, UTF8.String(f.getBytes()));
} }
@ -692,8 +692,8 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
if (interval == null) return randomHash(); if (interval == null) return randomHash();
// find dht position and size of gap // find dht position and size of gap
long left = FlatWordPartitionScheme.std.dhtPosition(interval.substring(0, 12).getBytes(), null); long left = FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(interval.substring(0, 12)), null);
long right = FlatWordPartitionScheme.std.dhtPosition(interval.substring(12).getBytes(), null); long right = FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(interval.substring(12)), null);
final long gap8 = FlatWordPartitionScheme.dhtDistance(left, right) >> 3; // 1/8 of a gap final long gap8 = FlatWordPartitionScheme.dhtDistance(left, right) >> 3; // 1/8 of a gap
long gapx = gap8 + (Math.abs(random.nextLong()) % (6 * gap8)); long gapx = gap8 + (Math.abs(random.nextLong()) % (6 * gap8));
long gappos = (Long.MAX_VALUE - left >= gapx) ? left + gapx : (left - Long.MAX_VALUE) + gapx; long gappos = (Long.MAX_VALUE - left >= gapx) ? left + gapx : (left - Long.MAX_VALUE) + gapx;
@ -728,16 +728,16 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
continue; continue;
} }
l = FlatWordPartitionScheme.dhtDistance( l = FlatWordPartitionScheme.dhtDistance(
FlatWordPartitionScheme.std.dhtPosition(s0.hash.getBytes(), null), FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(s0.hash), null),
FlatWordPartitionScheme.std.dhtPosition(s1.hash.getBytes(), null)); FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(s1.hash), null));
gaps.put(l, s0.hash + s1.hash); gaps.put(l, s0.hash + s1.hash);
s0 = s1; s0 = s1;
} }
// compute also the last gap // compute also the last gap
if ((first != null) && (s0 != null)) { if ((first != null) && (s0 != null)) {
l = FlatWordPartitionScheme.dhtDistance( l = FlatWordPartitionScheme.dhtDistance(
FlatWordPartitionScheme.std.dhtPosition(s0.hash.getBytes(), null), FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(s0.hash), null),
FlatWordPartitionScheme.std.dhtPosition(first.hash.getBytes(), null)); FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(first.hash), null));
gaps.put(l, s0.hash + first.hash); gaps.put(l, s0.hash + first.hash);
} }
return gaps; return gaps;
@ -768,7 +768,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
final String hash = final String hash =
Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6) + Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6) +
Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6); Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6);
return hash.getBytes(); return UTF8.getBytes(hash);
} }
public static yacySeed genRemoteSeed(final String seedStr, final String key, final boolean ownSeed) throws IOException { public static yacySeed genRemoteSeed(final String seedStr, final String key, final boolean ownSeed) throws IOException {

45
source/de/anomic/yacy/yacySeedDB.java
View File

@ -41,6 +41,7 @@ import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
@ -293,9 +294,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seedActiveDB.isEmpty() && seedPassiveDB.isEmpty() && seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early if (seedActiveDB.isEmpty() && seedPassiveDB.isEmpty() && seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early
if (this.mySeed == null) initMySeed(); if (this.mySeed == null) initMySeed();
try { try {
seedActiveDB.delete(mySeed.hash.getBytes()); seedActiveDB.delete(UTF8.getBytes(mySeed.hash));
seedPassiveDB.delete(mySeed.hash.getBytes()); seedPassiveDB.delete(UTF8.getBytes(mySeed.hash));
seedPotentialDB.delete(mySeed.hash.getBytes()); seedPotentialDB.delete(UTF8.getBytes(mySeed.hash));
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
} }
@ -401,7 +402,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed == null) { if (seed == null) {
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found."); yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
} else { } else {
clustermap.put(hash.getBytes(), ipport); clustermap.put(UTF8.getBytes(hash), ipport);
} }
} else if (yacydom.endsWith(".yacy")) { } else if (yacydom.endsWith(".yacy")) {
// find a peer with its name // find a peer with its name
@ -409,7 +410,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed == null) { if (seed == null) {
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found."); yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
} else { } else {
clustermap.put(seed.hash.getBytes(), ipport); clustermap.put(UTF8.getBytes(seed.hash), ipport);
} }
} else { } else {
yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh"); yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh");
@ -494,10 +495,10 @@ public final class yacySeedDB implements AlternativeDomainNames {
nameLookupCache.put(seed.getName(), seed.hash); nameLookupCache.put(seed.getName(), seed.hash);
final Map<String, String> seedPropMap = seed.getMap(); final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) { synchronized (seedPropMap) {
seedActiveDB.insert(seed.hash.getBytes(), seedPropMap); seedActiveDB.insert(UTF8.getBytes(seed.hash), seedPropMap);
} }
seedPassiveDB.delete(seed.hash.getBytes()); seedPassiveDB.delete(UTF8.getBytes(seed.hash));
seedPotentialDB.delete(seed.hash.getBytes()); seedPotentialDB.delete(UTF8.getBytes(seed.hash));
} catch (final Exception e) { } catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable(); resetActiveTable();
@ -508,14 +509,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed.isProper(false) != null) return; if (seed.isProper(false) != null) return;
try { try {
nameLookupCache.remove(seed.getName()); nameLookupCache.remove(seed.getName());
seedActiveDB.delete(seed.hash.getBytes()); seedActiveDB.delete(UTF8.getBytes(seed.hash));
seedPotentialDB.delete(seed.hash.getBytes()); seedPotentialDB.delete(UTF8.getBytes(seed.hash));
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime()))); //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try { try {
final Map<String, String> seedPropMap = seed.getMap(); final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) { synchronized (seedPropMap) {
seedPassiveDB.insert(seed.hash.getBytes(), seedPropMap); seedPassiveDB.insert(UTF8.getBytes(seed.hash), seedPropMap);
} }
} catch (final Exception e) { } catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -527,14 +528,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed.isProper(false) != null) return; if (seed.isProper(false) != null) return;
try { try {
nameLookupCache.remove(seed.getName()); nameLookupCache.remove(seed.getName());
seedActiveDB.delete(seed.hash.getBytes()); seedActiveDB.delete(UTF8.getBytes(seed.hash));
seedPassiveDB.delete(seed.hash.getBytes()); seedPassiveDB.delete(UTF8.getBytes(seed.hash));
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime()))); //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try { try {
final Map<String, String> seedPropMap = seed.getMap(); final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) { synchronized (seedPropMap) {
seedPotentialDB.insert(seed.hash.getBytes(), seedPropMap); seedPotentialDB.insert(UTF8.getBytes(seed.hash), seedPropMap);
} }
} catch (final Exception e) { } catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -545,14 +546,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
public synchronized void removeDisconnected(final String peerHash) { public synchronized void removeDisconnected(final String peerHash) {
if(peerHash == null) return; if(peerHash == null) return;
try { try {
seedPassiveDB.delete(peerHash.getBytes()); seedPassiveDB.delete(UTF8.getBytes(peerHash));
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
} }
public synchronized void removePotential(final String peerHash) { public synchronized void removePotential(final String peerHash) {
if(peerHash == null) return; if(peerHash == null) return;
try { try {
seedPotentialDB.delete(peerHash.getBytes()); seedPotentialDB.delete(UTF8.getBytes(peerHash));
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
} }
@ -573,7 +574,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed; if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>(); ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
try { try {
Map<String, String> map = database.get(hash.getBytes()); Map<String, String> map = database.get(UTF8.getBytes(hash));
if (map == null) return null; if (map == null) return null;
entry.putAll(map); entry.putAll(map);
} catch (final IOException e) { } catch (final IOException e) {
@ -613,13 +614,13 @@ public final class yacySeedDB implements AlternativeDomainNames {
} }
yacySeed s = get(hash, seedActiveDB); yacySeed s = get(hash, seedActiveDB);
if (s != null) try { seedActiveDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} if (s != null) try { seedActiveDB.insert(UTF8.getBytes(hash), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPassiveDB); s = get(hash, seedPassiveDB);
if (s != null) try { seedPassiveDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} if (s != null) try { seedPassiveDB.insert(UTF8.getBytes(hash), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPotentialDB); s = get(hash, seedPotentialDB);
if (s != null) try { seedPotentialDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} if (s != null) try { seedPotentialDB.insert(UTF8.getBytes(hash), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
} }
public yacySeed lookupByName(String peerName) { public yacySeed lookupByName(String peerName) {
@ -700,7 +701,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (addressStr == null) { if (addressStr == null) {
Log.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null."); Log.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try { try {
badPeerHashes.put(seed.hash.getBytes()); badPeerHashes.put(UTF8.getBytes(seed.hash));
} catch (RowSpaceExceededException e1) { } catch (RowSpaceExceededException e1) {
Log.logException(e1); Log.logException(e1);
break; break;
@ -733,7 +734,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (addressStr == null) { if (addressStr == null) {
Log.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null."); Log.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try { try {
badPeerHashes.put(seed.hash.getBytes()); badPeerHashes.put(UTF8.getBytes(seed.hash));
} catch (RowSpaceExceededException e1) { } catch (RowSpaceExceededException e1) {
Log.logException(e1); Log.logException(e1);
break; break;

77
source/net/yacy/cora/document/UTF8.java
View File

@ -71,6 +71,83 @@ public class UTF8 {
return new String(bytes, offset, length, charset); return new String(bytes, offset, length, charset);
} }
/**
* getBytes() as method for String synchronizes during the look-up for the
* Charset object for the default charset as given with a default charset name.
* This is the normal process:
public byte[] getBytes() {
return StringCoding.encode(value, offset, count);
}
static byte[] encode(char[] ca, int off, int len) {
String csn = Charset.defaultCharset().name();
try {
return encode(csn, ca, off, len);
...
static byte[] encode(String charsetName, char[] ca, int off, int len)
throws UnsupportedEncodingException
{
StringEncoder se = (StringEncoder)deref(encoder);
String csn = (charsetName == null) ? "ISO-8859-1" : charsetName;
if ((se == null) || !(csn.equals(se.requestedCharsetName())
|| csn.equals(se.charsetName()))) {
se = null;
try {
Charset cs = lookupCharset(csn);
....
private static Charset lookupCharset(String csn) {
if (Charset.isSupported(csn)) {
try {
return Charset.forName(csn);
....
public static Charset forName(String charsetName) {
Charset cs = lookup(charsetName);
....
private static Charset lookup(String charsetName) {
if (charsetName == null)
throw new IllegalArgumentException("Null charset name");
Object[] a;
if ((a = cache1) != null && charsetName.equals(a[0]))
return (Charset)a[1];
// We expect most programs to use one Charset repeatedly.
// We convey a hint to this effect to the VM by putting the
// level 1 cache miss code in a separate method.
return lookup2(charsetName);
}
private static Charset lookup2(String charsetName) {
Object[] a;
if ((a = cache2) != null && charsetName.equals(a[0])) {
cache2 = cache1;
cache1 = a;
return (Charset)a[1];
}
Charset cs;
if ((cs = standardProvider.charsetForName(charsetName)) != null ||
(cs = lookupExtendedCharset(charsetName)) != null ||
(cs = lookupViaProviders(charsetName)) != null)
{
cache(charsetName, cs);
....
At this point the getBytes() call synchronizes at one of the methods
standardProvider.charsetForName
lookupExtendedCharset
lookupViaProviders
* with our call using a given charset object, the call is much easier to perform
* and it omits the synchronization for the charset lookup.
*
* @param s
* @return
*/
public final static byte[] getBytes(final String s) { public final static byte[] getBytes(final String s) {
if (s == null) return null; if (s == null) return null;
return s.getBytes(charset); return s.getBytes(charset);

4
source/net/yacy/document/Document.java
View File

@ -235,7 +235,7 @@ dc_rights
public InputStream getText() { public InputStream getText() {
try { try {
if (this.text == null) return new ByteArrayInputStream("".getBytes()); if (this.text == null) return new ByteArrayInputStream(UTF8.getBytes(""));
if (this.text instanceof String) { if (this.text instanceof String) {
return new ByteArrayInputStream(UTF8.getBytes(((String) this.text))); return new ByteArrayInputStream(UTF8.getBytes(((String) this.text)));
} else if (this.text instanceof InputStream) { } else if (this.text instanceof InputStream) {
@ -252,7 +252,7 @@ dc_rights
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
} }
return new ByteArrayInputStream("".getBytes()); return new ByteArrayInputStream(UTF8.getBytes(""));
} }
public byte[] getTextBytes() { public byte[] getTextBytes() {

5
source/net/yacy/document/parser/vcfParser.java
View File

@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser; import net.yacy.document.AbstractParser;
import net.yacy.document.Document; import net.yacy.document.Document;
import net.yacy.document.Parser; import net.yacy.document.Parser;
@ -199,7 +200,7 @@ public class vcfParser extends AbstractParser implements Parser {
} }
final String[] sections = parsedNames.toArray(new String[parsedNames.size()]); final String[] sections = parsedNames.toArray(new String[parsedNames.size()]);
final byte[] text = parsedDataText.toString().getBytes(); final byte[] text = UTF8.getBytes(parsedDataText.toString());
return new Document[]{new Document( return new Document[]{new Document(
url, // url of the source document url, // url of the source document
mimeType, // the documents mime type mimeType, // the documents mime type
@ -227,7 +228,7 @@ public class vcfParser extends AbstractParser implements Parser {
private String decodeQuotedPrintable(final String s) { private String decodeQuotedPrintable(final String s) {
if (s == null) return null; if (s == null) return null;
final byte[] b = s.getBytes(); final byte[] b = UTF8.getBytes(s);
final StringBuilder sb = new StringBuilder(); final StringBuilder sb = new StringBuilder();
for (int i = 0; i < b.length; i++) { for (int i = 0; i < b.length; i++) {
final int c = b[i]; final int c = b[i];

2
source/net/yacy/document/parser/xml/opensearchdescriptionReader.java
View File

@ -146,7 +146,7 @@ public class opensearchdescriptionReader extends DefaultHandler {
Log.logWarning("opensearchdescriptionReader", "response=" + UTF8.String(a)); Log.logWarning("opensearchdescriptionReader", "response=" + UTF8.String(a));
return null; return null;
} }
if (!ByteBuffer.equals(a, "<?xml".getBytes())) { if (!ByteBuffer.equals(a, UTF8.getBytes("<?xml"))) {
Log.logWarning("opensearchdescriptionReader", "response does not contain valid xml"); Log.logWarning("opensearchdescriptionReader", "response does not contain valid xml");
return null; return null;
} }