mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
Merge branch 'master' of https://git.gitorious.org/yacy/rc1.git
This commit is contained in:
commit
e95011fac0
|
@ -1,4 +1,4 @@
|
|||
// ConfigPortal.java
|
||||
// ConfigPortal.java
|
||||
// -----------------------
|
||||
// part of YaCy
|
||||
// (C) by Michael Peter Christen; mc@yacy.net
|
||||
|
@ -10,7 +10,7 @@
|
|||
//$LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
|
@ -38,14 +38,14 @@ public class ConfigPortal {
|
|||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final serverObjects prop = new serverObjects();
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
|
||||
|
||||
if (post != null) {
|
||||
// AUTHENTICATE
|
||||
if (!header.containsKey(RequestHeader.AUTHORIZATION)) {
|
||||
prop.putHTML("AUTHENTICATE","log-in");
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
if (post.containsKey("popup")) {
|
||||
final String popup = post.get("popup", "status");
|
||||
if ("front".equals(popup)) {
|
||||
|
@ -62,7 +62,7 @@ public class ConfigPortal {
|
|||
final String newGreeting = post.get(SwitchboardConstants.GREETING, "");
|
||||
// store this call as api call
|
||||
sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
|
||||
|
||||
|
||||
sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
|
||||
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
|
||||
sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
|
||||
|
@ -80,7 +80,7 @@ public class ConfigPortal {
|
|||
sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser", false));
|
||||
sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures", false));
|
||||
sb.setConfig("search.verify", post.get("search.verify", "ifexist"));
|
||||
sb.setConfig("search.verify.delete", post.getBoolean("search.verify.delete", false));
|
||||
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete", false));
|
||||
// construct navigation String
|
||||
String nav = "";
|
||||
if (post.getBoolean("search.navigation.hosts", false)) nav += "hosts,";
|
||||
|
@ -109,8 +109,8 @@ public class ConfigPortal {
|
|||
sb.setConfig("search.result.show.parser", true);
|
||||
sb.setConfig("search.result.show.pictures", true);
|
||||
sb.setConfig("search.verify", "iffresh");
|
||||
sb.setConfig("search.verify.delete", "true");
|
||||
}
|
||||
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true");
|
||||
}
|
||||
}
|
||||
|
||||
prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, ""));
|
||||
|
@ -121,7 +121,7 @@ public class ConfigPortal {
|
|||
prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);
|
||||
prop.put("publicSearchpage", sb.getConfigBool("publicSearchpage", false) ? 1 : 0);
|
||||
prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);
|
||||
|
||||
|
||||
prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0);
|
||||
prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
|
||||
prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
|
||||
|
@ -138,7 +138,7 @@ public class ConfigPortal {
|
|||
prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0);
|
||||
prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0);
|
||||
prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0);
|
||||
prop.put("search.verify.delete", sb.getConfigBool("search.verify.delete", true) ? 1 : 0);
|
||||
prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0);
|
||||
|
||||
final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
|
||||
prop.put("popupFront", 0);
|
||||
|
@ -156,14 +156,14 @@ public class ConfigPortal {
|
|||
}
|
||||
|
||||
prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10));
|
||||
|
||||
|
||||
final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self");
|
||||
prop.put("selected_blank", "_blank".equals(target) ? 1 : 0);
|
||||
prop.put("selected_self", "_self".equals(target) ? 1 : 0);
|
||||
prop.put("selected_parent", "_parent".equals(target) ? 1 : 0);
|
||||
prop.put("selected_top", "_top".equals(target) ? 1 : 0);
|
||||
prop.put("selected_searchresult", "searchresult".equals(target) ? 1 : 0);
|
||||
|
||||
|
||||
String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress();
|
||||
if (myaddress == null) {
|
||||
myaddress = "localhost:" + sb.getConfig("port", "8090");
|
||||
|
|
|
@ -597,7 +597,7 @@ public class yacysearch {
|
|||
indexSegment,
|
||||
ranking,
|
||||
header.get(RequestHeader.USER_AGENT, ""),
|
||||
sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
|
||||
sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
|
||||
EventTracker.delete(EventTracker.EClass.SEARCH);
|
||||
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);
|
||||
|
||||
|
|
|
@ -133,10 +133,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
if (UTCOffset == null || UTCOffset.length() == 0) { return new Date(); }
|
||||
try {
|
||||
return new Date(this.dateFormat.parse(timeString).getTime() - UTCDiff() + UTCDiff(UTCOffset));
|
||||
} catch (final java.text.ParseException e) {
|
||||
//serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
|
||||
return new Date();
|
||||
} catch (final java.lang.NumberFormatException e) {
|
||||
} catch (final Throwable e) {
|
||||
//serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
|
||||
return new Date();
|
||||
}
|
||||
|
|
|
@ -28,7 +28,6 @@ package net.yacy.document.parser.images;
|
|||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.EOFException;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
|
@ -227,14 +226,8 @@ public class genericImageParser extends AbstractParser implements Parser {
|
|||
try {
|
||||
ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM
|
||||
image = ImageIO.read(sourceStream);
|
||||
} catch (final EOFException e) {
|
||||
Log.logException(e);
|
||||
throw new Parser.Failure(e.getMessage(), location);
|
||||
} catch (final IOException e) {
|
||||
Log.logException(e);
|
||||
throw new Parser.Failure(e.getMessage(), location);
|
||||
} catch (final Throwable e) { // may appear in case of an OutOfMemoryError
|
||||
Log.logException(e);
|
||||
} catch (final Throwable e) {
|
||||
//Log.logException(e);
|
||||
throw new Parser.Failure(e.getMessage(), location);
|
||||
}
|
||||
if (image == null) throw new Parser.Failure("ImageIO returned NULL", location);
|
||||
|
|
|
@ -437,6 +437,7 @@ public final class SwitchboardConstants {
|
|||
|
||||
public static final String SEARCH_ITEMS = "search.items";
|
||||
public static final String SEARCH_TARGET = "search.target";
|
||||
public static final String SEARCH_VERIFY_DELETE = "search.verify.delete";
|
||||
|
||||
/**
|
||||
* system tray
|
||||
|
|
|
@ -35,6 +35,7 @@ import java.util.SortedMap;
|
|||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
|
@ -79,7 +80,7 @@ public final class RWIProcess extends Thread {
|
|||
private int remote_resourceSize, remote_indexCount, remote_peerCount;
|
||||
private int local_indexCount;
|
||||
private final WeakPriorityBlockingQueue<WordReferenceVars> stack;
|
||||
private int feeders;
|
||||
private final AtomicInteger feeders;
|
||||
private final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
|
||||
//private final HandleSet handover; // key = urlhash; used for double-check of urls that had been handed over to search process
|
||||
|
||||
|
@ -123,7 +124,7 @@ public final class RWIProcess extends Thread {
|
|||
this.protocolNavigator = new ConcurrentScoreMap<String>();
|
||||
this.filetypeNavigator = new ConcurrentScoreMap<String>();
|
||||
this.ref = new ConcurrentScoreMap<String>();
|
||||
this.feeders = 1;
|
||||
this.feeders = new AtomicInteger(1);
|
||||
this.startTime = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
|
@ -288,16 +289,16 @@ public final class RWIProcess extends Thread {
|
|||
* method to signal the incoming stack that one feeder has terminated
|
||||
*/
|
||||
public void oneFeederTerminated() {
|
||||
this.feeders--;
|
||||
assert this.feeders >= 0 : "feeders = " + this.feeders;
|
||||
final int c = this.feeders.decrementAndGet();
|
||||
assert c >= 0 : "feeders = " + c;
|
||||
}
|
||||
|
||||
public void moreFeeders(final int countMoreFeeders) {
|
||||
this.feeders += countMoreFeeders;
|
||||
this.feeders.addAndGet(countMoreFeeders);
|
||||
}
|
||||
|
||||
public boolean feedingIsFinished() {
|
||||
return System.currentTimeMillis() - this.startTime > 50 && this.feeders == 0;
|
||||
return this.feeders.get() == 0;
|
||||
}
|
||||
|
||||
private boolean testFlags(final WordReference ientry) {
|
||||
|
|
|
@ -371,14 +371,13 @@ public final class SearchEvent {
|
|||
}
|
||||
|
||||
public ResultEntry oneResult(final int item, final long timeout) {
|
||||
if ((this.query.domType == QueryParams.Searchdom.GLOBAL) ||
|
||||
(this.query.domType == QueryParams.Searchdom.CLUSTER)) {
|
||||
if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || (this.query.domType == QueryParams.Searchdom.CLUSTER)) {
|
||||
// this is a search using remote search threads. Also the local
|
||||
// search thread is started as background process
|
||||
if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {
|
||||
// in case that the local search takes longer than some other
|
||||
// remote search requests, wait that the local process terminates first
|
||||
try {this.localSearchThread.join();} catch (final InterruptedException e) {}
|
||||
try {this.localSearchThread.join(300);} catch (final InterruptedException e) {}
|
||||
}
|
||||
}
|
||||
return this.resultFetcher.oneResult(item, timeout);
|
||||
|
|
|
@ -37,6 +37,7 @@ import net.yacy.kelondro.util.MemoryControl;
|
|||
import net.yacy.peers.SeedDB;
|
||||
import net.yacy.repository.LoaderDispatcher;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.SwitchboardConstants;
|
||||
import net.yacy.search.index.Segment;
|
||||
import net.yacy.search.ranking.RankingProfile;
|
||||
import net.yacy.search.snippet.ContentDomain;
|
||||
|
@ -178,7 +179,7 @@ public class SearchEventCache {
|
|||
Log.logInfo("SearchEventCache", "getEvent: " + SearchEventCache.lastEvents.size() + " in cache; " + countAliveThreads() + " alive");
|
||||
|
||||
// start a new event
|
||||
final boolean delete = Switchboard.getSwitchboard() == null | Switchboard.getSwitchboard().getConfigBool("search.verify.delete", true);
|
||||
final boolean delete = Switchboard.getSwitchboard() == null || Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true);
|
||||
event = new SearchEvent(query, peers, workTables, preselectedPeerHashes, generateAbstracts, loader, remote_maxcount, remote_maxtime, burstRobinsonPercent, burstMultiwordPercent, delete);
|
||||
MemoryControl.request(100 * 1024 * 1024, false); // this may trigger a short memory status which causes a reducing of cache space of other threads
|
||||
}
|
||||
|
|
|
@ -147,7 +147,8 @@ public class SnippetProcess {
|
|||
final long waittimeout = System.currentTimeMillis() + 300;
|
||||
while (
|
||||
item == 0 &&
|
||||
this.result.sizeAvailable() < this.query.neededResults() + this.query.itemsPerPage &&
|
||||
(!this.rankingProcess.feedingIsFinished() || this.rankingProcess.sizeQueue() > 0) &&
|
||||
this.result.sizeAvailable() < this.query.neededResults() /* + this.query.itemsPerPage */ &&
|
||||
System.currentTimeMillis() < waittimeout &&
|
||||
anyWorkerAlive()
|
||||
) {
|
||||
|
@ -324,7 +325,7 @@ public class SnippetProcess {
|
|||
}
|
||||
|
||||
private boolean anyWorkerAlive() {
|
||||
if (this.workerThreads == null) return false;
|
||||
if (this.workerThreads == null || this.workerThreads.length == 0) return false;
|
||||
synchronized(this.workerThreads) {
|
||||
for (final Worker workerThread : this.workerThreads) {
|
||||
if ((workerThread != null) &&
|
||||
|
@ -377,7 +378,7 @@ public class SnippetProcess {
|
|||
|
||||
// check if we have enough
|
||||
if (SnippetProcess.this.result.sizeAvailable() >= this.neededResults) {
|
||||
//Log.logWarning("ResultFetcher", ResultFetcher.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
|
||||
//Log.logWarning("ResultFetcher", SnippetProcess.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -391,7 +392,7 @@ public class SnippetProcess {
|
|||
page = SnippetProcess.this.rankingProcess.takeURL(true, Math.min(100, this.timeout - System.currentTimeMillis()));
|
||||
//if (page == null) page = rankedCache.takeURL(false, this.timeout - System.currentTimeMillis());
|
||||
if (page == null) {
|
||||
//System.out.println("page == null");
|
||||
//Log.logWarning("ResultFetcher", "page == null");
|
||||
break; // no more available
|
||||
}
|
||||
if (SnippetProcess.this.query.filterfailurls && SnippetProcess.this.workTables.failURLsContains(page.hash())) continue;
|
||||
|
@ -405,7 +406,6 @@ public class SnippetProcess {
|
|||
if (sd != null) solrContent = this.solr.getScheme().solrGetText(sd);
|
||||
}
|
||||
|
||||
|
||||
loops++;
|
||||
resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
|
||||
if (resultEntry == null) continue; // the entry had some problems, cannot be used
|
||||
|
|
Loading…
Reference in New Issue
Block a user