Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Removed some unnecessary uses of java.lang.reflect api.

Browse Source 
This improves code browsing and readability, making search by references
or call hierarchy IDE features more accurate.
This commit is contained in:
luccioman 2017-08-24 18:47:18 +02:00
parent da3dbf9ea1
commit 4eba88f2ff
10 changed files with 402 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
htroot/ConfigBasic.java
View File

@ -35,12 +35,11 @@ import java.util.List;
import java.util.regex.Pattern;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.Translator;
import net.yacy.data.WorkTables;
import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.workflow.InstantBusyThread;
import net.yacy.peers.OnePeerPingBusyThread;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@ -48,8 +47,8 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.utils.translation.TranslatorXliff;
import net.yacy.utils.upnp.UPnPMappingType;
import net.yacy.utils.upnp.UPnP;
import net.yacy.utils.upnp.UPnPMappingType;
public class ConfigBasic {
@ -58,7 +57,7 @@ public class ConfigBasic {
private static final int NEXTSTEP_PEERNAME = 2;
private static final int NEXTSTEP_PEERPORT = 3;
private static final int NEXTSTEP_RECONNECT = 4;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException {
// return variable that accumulates replacements
@ -81,7 +80,7 @@ public class ConfigBasic {
//boolean doPeerPing = false;
if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {
InstantBusyThread.oneTimeJob(sb.yc, "peerPing", 0);
new OnePeerPingBusyThread(sb.yc).start();
//doPeerPing = true;
}

18
htroot/api/ymarks/import_ymark.java
View File

@ -33,7 +33,7 @@ import net.yacy.data.ymark.YMarkUtil;
import net.yacy.data.ymark.YMarkXBELImporter;
import net.yacy.document.Parser.Failure;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.workflow.InstantBusyThread;
import net.yacy.kelondro.workflow.OneTimeBusyThread;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@ -71,7 +71,7 @@ public class import_ymark {
empty = true;
}
YMarkAutoTagger autoTagger = new YMarkAutoTagger(autoTaggingQueue, sb.loader, sb.tables.bookmarks, bmk_user, merge);
InstantBusyThread.oneTimeJob(autoTagger, 0);
OneTimeBusyThread.startFromRunnable(autoTagger, 0);
}
if(isAdmin && post.containsKey("table") && post.get("table").length() > 0) {
@ -109,8 +109,8 @@ public class import_ymark {
MonitoredReader reader = new MonitoredReader(new InputStreamReader(stream, StandardCharsets.UTF_8), 1024*16, bytes.length);
if(post.get("importer").equals("html") && reader != null) {
final YMarkHTMLImporter htmlImporter = new YMarkHTMLImporter(reader, queueSize, root);
InstantBusyThread.oneTimeJob(htmlImporter, 0);
InstantBusyThread.oneTimeJob(htmlImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);
OneTimeBusyThread.startFromRunnable(htmlImporter, 0);
OneTimeBusyThread.startFromRunnable(htmlImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);
prop.put("status", "1");
} else if(post.get("importer").equals("xbel") && reader != null) {
final YMarkXBELImporter xbelImporter;
@ -123,13 +123,13 @@ public class import_ymark {
prop.put("status", "0");
return prop;
}
InstantBusyThread.oneTimeJob(xbelImporter, 0);
InstantBusyThread.oneTimeJob(xbelImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);
OneTimeBusyThread.startFromRunnable(xbelImporter, 0);
OneTimeBusyThread.startFromRunnable(xbelImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);
prop.put("status", "1");
} else if(post.get("importer").equals("json") && reader != null) {
YMarkJSONImporter jsonImporter;
jsonImporter = new YMarkJSONImporter(reader, queueSize, root);
InstantBusyThread.oneTimeJob(jsonImporter, 0);
OneTimeBusyThread.startFromRunnable(jsonImporter, 0);
while ((bmk = jsonImporter.take()) != YMarkEntry.POISON) {
putBookmark(sb, bmk_user, bmk, autoTaggingQueue, autotag, empty, indexing, medialink);
}
@ -207,8 +207,8 @@ public class import_ymark {
mreader.addChangeListener(sb.tables.bookmarks.getProgressListener("DMOZImporter"));
DMOZImporter.setDepth(6);
InstantBusyThread.oneTimeJob(DMOZImporter, 0);
InstantBusyThread.oneTimeJob(DMOZImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);
OneTimeBusyThread.startFromRunnable(DMOZImporter, 0);
OneTimeBusyThread.startFromRunnable(DMOZImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);
prop.put("status", "1");
} catch (final Exception e) {

3
source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java
View File

@ -8,7 +8,7 @@ import net.yacy.kelondro.blob.Tables.Row;
import net.yacy.repository.FilterEngine;
import net.yacy.search.Switchboard;
public class ContentControlFilterUpdateThread {
public class ContentControlFilterUpdateThread implements Runnable {
private final Switchboard sb;
@ -22,6 +22,7 @@ public class ContentControlFilterUpdateThread {
}
@Override
public final void run() {
if (!this.locked) {

3
source/net/yacy/contentcontrol/SMWListSyncThread.java
View File

@ -13,7 +13,7 @@ import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.Switchboard;
public class SMWListSyncThread {
public class SMWListSyncThread implements Runnable {
private final Switchboard sb;
private Boolean locked = false;
@ -55,6 +55,7 @@ public class SMWListSyncThread {
return ret;
}
@Override
public final void run() {
if (!this.locked) {

175
source/net/yacy/kelondro/workflow/InstantBusyThread.java
View File

@ -24,77 +24,36 @@
package net.yacy.kelondro.workflow;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.TreeMap;
import net.yacy.cora.util.ConcurrentLog;
public final class InstantBusyThread extends AbstractBusyThread implements BusyThread {
public abstract class InstantBusyThread extends AbstractBusyThread implements BusyThread {
private Method jobExecMethod, jobCountMethod, freememExecMethod;
private final Object environment;
private final Long handle;
private static final TreeMap<Long, String> jobs = new TreeMap<Long, String>();
public InstantBusyThread(
final Object env,
final String jobExec,
final String jobCount,
final String freemem,
final long idleSleep,
final long busySleep) {
/**
* @param idleSleep defines min idle sleep time that can be set via setIdleSleep()
* @param busySleep defines min busy sleep time that can be set via setBusySleep()
*/
public InstantBusyThread(final long idleSleep, final long busySleep) {
this("InstantBusyThread.job", idleSleep, busySleep);
}
/**
* @param jobName the job name used to monitor the thread
* @param idleSleep defines min idle sleep time that can be set via setIdleSleep()
* @param busySleep defines min busy sleep time that can be set via setBusySleep()
*/
public InstantBusyThread(final String jobName, final long idleSleep, final long busySleep) {
super(idleSleep, busySleep);
// jobExec is the name of a method of the object 'env' that executes the one-step-run
// jobCount is the name of a method that returns the size of the job
// freemem is the name of a method that tries to free memory and returns void
final Class<?> theClass = (env instanceof Class<?>) ? (Class<?>) env : env.getClass();
try {
this.jobExecMethod = theClass.getMethod(jobExec);
} catch (final NoSuchMethodException e) {
throw new RuntimeException("serverInstantThread, wrong declaration of jobExec: " + e.getMessage());
}
try {
if (jobCount == null)
this.jobCountMethod = null;
else
this.jobCountMethod = theClass.getMethod(jobCount);
} catch (final NoSuchMethodException e) {
throw new RuntimeException("serverInstantThread, wrong declaration of jobCount: " + e.getMessage());
}
try {
if (freemem == null)
this.freememExecMethod = null;
else
this.freememExecMethod = theClass.getMethod(freemem);
} catch (final NoSuchMethodException e) {
throw new RuntimeException("serverInstantThread, wrong declaration of freemem: " + e.getMessage());
}
this.environment = (env instanceof Class<?>) ? null : env;
setName("BusyThread " + theClass.getName() + "." + jobExec);
setName("BusyThread " + jobName);
this.handle = Long.valueOf(System.currentTimeMillis() + getName().hashCode());
}
@Override
public int getJobCount() {
if (this.jobCountMethod == null) return Integer.MAX_VALUE;
try {
final Object result = this.jobCountMethod.invoke(this.environment);
return (result instanceof Integer) ? ((Integer) result).intValue() : -1;
} catch (final IllegalAccessException e) {
return -1;
} catch (final IllegalArgumentException e) {
return -1;
} catch (final InvocationTargetException e) {
ConcurrentLog.severe("BUSYTHREAD", "invocation serverInstantThread of thread '" + getName() + "': " + e.getMessage(), e);
return -1;
}
}
@Override
public boolean job() throws Exception {
@ -102,25 +61,13 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
synchronized(jobs) {jobs.put(this.handle, getName());}
boolean jobHasDoneSomething = false;
try {
final Object result = this.jobExecMethod.invoke(this.environment);
if (result == null) jobHasDoneSomething = true;
else if (result instanceof Boolean) jobHasDoneSomething = ((Boolean) result).booleanValue();
} catch (final IllegalAccessException e) {
ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.job: " + e.getMessage());
ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");
terminate(false);
jobHasDoneSomething = jobImpl();
} catch (final IllegalArgumentException e) {
ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.job: " + e.getMessage());
ConcurrentLog.severe("BUSYTHREAD", "Internal Error in InstantBusyThread.job: " + e.getMessage());
ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");
terminate(false);
} catch (final InvocationTargetException e) {
final String targetException = e.getTargetException().getMessage();
ConcurrentLog.logException(e);
ConcurrentLog.logException(e.getCause());
ConcurrentLog.logException(e.getTargetException());
ConcurrentLog.severe("BUSYTHREAD", "Runtime Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException());
} catch (final OutOfMemoryError e) {
ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage());
ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in InstantBusyThread.job, thread '" + getName() + "': " + e.getMessage());
ConcurrentLog.logException(e);
freemem();
} catch (final Exception e) {
@ -130,46 +77,36 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
synchronized(jobs) {jobs.remove(this.handle);}
return jobHasDoneSomething;
}
/**
* The job's main logic implementation
* @return true if it has done something, false if it is idle and does not expect to work on more for a longer time
* @throws Exception when an unexpected error occurred
*/
public abstract boolean jobImpl() throws Exception;
@Override
public void freemem() {
if (this.freememExecMethod == null) return;
try {
this.freememExecMethod.invoke(this.environment);
} catch (final IllegalAccessException e) {
ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.freemem: " + e.getMessage());
ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");
terminate(false);
} catch (final IllegalArgumentException e) {
ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.freemem: " + e.getMessage());
ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");
terminate(false);
} catch (final InvocationTargetException e) {
final String targetException = e.getTargetException().getMessage();
if (targetException.indexOf("heap space",0) > 0) ConcurrentLog.logException(e.getTargetException());
ConcurrentLog.severe("BUSYTHREAD", "Runtime Error in serverInstantThread.freemem, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException());
ConcurrentLog.logException(e.getTargetException());
freememImpl();
} catch (final OutOfMemoryError e) {
ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in serverInstantThread.freemem, thread '" + getName() + "': " + e.getMessage());
ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in InstantBusyThread.freemem, thread '" + getName() + "': " + e.getMessage());
ConcurrentLog.logException(e);
}
}
@Override
public int getJobCount() {
return Integer.MAX_VALUE;
}
/**
* Called when an outOfMemoryCycle is performed.
*/
public void freememImpl() {
// Do nothing in this implementation, please override
}
public static BusyThread oneTimeJob(final Object env, final String jobExec, final long startupDelay) {
// start the job and execute it once as background process
final BusyThread thread = new InstantBusyThread(env, jobExec, null, null, Long.MIN_VALUE, Long.MIN_VALUE);
thread.setStartupSleep(startupDelay);
thread.setIdleSleep(-1);
thread.setBusySleep(-1);
thread.setMemPreReqisite(0);
thread.setLoadPreReqisite(Double.MAX_VALUE); // this is called during initialization phase and some code parts depend on it; therefore we cannot set a prerequisite that prevents the start of that thread
thread.start();
return thread;
}
public static WorkflowThread oneTimeJob(final Runnable thread, final long startupDelay) {
return oneTimeJob(thread, "run", startupDelay);
}
@Override
public void open() {
@ -180,5 +117,35 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
public synchronized void close() {
// Not implemented in this thread
}
/**
* Construct an InstantBusyThread instance from a runnable task.
*
* @param task
* the task to run as a job
* @param idleSleep
* defines min idle sleep time that can be set via setIdleSleep()
* @param busySleep
* defines min busy sleep time that can be set via setBusySleep()
* @return a InstantBusyThread instance
* @throws IllegalArgumentException
* when the task is null
*/
public static InstantBusyThread createFromRunnable(final Runnable task, final long idleSleep,
final long busySleep) {
if (task == null) {
throw new IllegalArgumentException("Runnable task must not be null");
}
InstantBusyThread busyThread = new InstantBusyThread(task.getClass().getName() + ".run", idleSleep, busySleep) {
@Override
public boolean jobImpl() throws Exception {
task.run();
return true;
}
};
return busyThread;
}
}

80
source/net/yacy/kelondro/workflow/OneTimeBusyThread.java Normal file
View File

@ -0,0 +1,80 @@
// OneTimeBusyThread.java
// ---------------------------
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.workflow;
/**
* A busy thread to run a job only once
*/
public abstract class OneTimeBusyThread extends InstantBusyThread {
/**
* Construct an instance able to run a job once, after a given delay
* @param jobName the job name used to monitor the thread
* @param startupDelay the delay in milliseconds to wait before starting the job
*/
public OneTimeBusyThread(final String jobName, final long startupDelay) {
super(jobName, Long.MIN_VALUE, Long.MIN_VALUE);
this.setStartupSleep(startupDelay);
this.setIdleSleep(-1);
this.setBusySleep(-1);
this.setMemPreReqisite(0);
this.setLoadPreReqisite(
Double.MAX_VALUE); /*
* this is called during initialization phase and some code parts depend on it;
* therefore we cannot set a prerequisite that prevents the start of that thread
*/
}
/**
* Construct an instance able to run a job once and immediately
* @param jobName the job name used to monitor the thread
*/
public OneTimeBusyThread(final String jobName) {
this(jobName, 0);
}
/**
* Construct and start a OneTimeBusyThread instance from a runnable task.
* @param task the task to run once
* @param startupDelay the delay in milliseconds to wait before starting the job
* @return a OneTimeBusyThread instance
* @throws IllegalArgumentException when the task is null
*/
public static OneTimeBusyThread startFromRunnable(final Runnable task, final long startupDelay) {
if(task == null) {
throw new IllegalArgumentException("Runnable task must not be null");
}
OneTimeBusyThread busyThread = new OneTimeBusyThread(task.getClass().getName() + ".run", startupDelay) {
@Override
public boolean jobImpl() throws Exception {
task.run();
return true;
}
};
busyThread.start();
return busyThread;
}
}

53
source/net/yacy/peers/OnePeerPingBusyThread.java Normal file
View File

@ -0,0 +1,53 @@
// OnePeerPingBusyThread.java
// ---------------------------
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.peers;
import net.yacy.kelondro.workflow.OneTimeBusyThread;
/**
* A busy thread running peer ping only once.
*/
public class OnePeerPingBusyThread extends OneTimeBusyThread {
/** Access to the peer to peer network */
final net.yacy.peers.Network peersNetwork;
/**
* @param peersNetwork a net.yacy.peers.Network instance
* @throws IllegalArgumentException when peersNetwork is null
*/
public OnePeerPingBusyThread(final net.yacy.peers.Network peersNetwork) {
super("Network.peerPing");
if(peersNetwork == null) {
throw new IllegalArgumentException("peersNetwork parameter must not be null");
}
this.peersNetwork = peersNetwork;
}
@Override
public boolean jobImpl() throws Exception {
this.peersNetwork.peerPing();
return true;
}
}

265
source/net/yacy/search/Switchboard.java
View File

@ -193,6 +193,7 @@ import net.yacy.kelondro.util.OS;
import net.yacy.kelondro.util.SetTools;
import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.InstantBusyThread;
import net.yacy.kelondro.workflow.OneTimeBusyThread;
import net.yacy.kelondro.workflow.WorkflowProcessor;
import net.yacy.kelondro.workflow.WorkflowThread;
import net.yacy.peers.DHTSelection;
@ -337,7 +338,14 @@ public final class Switchboard extends serverSwitch {
// UPnP port mapping
if ( getConfigBool(SwitchboardConstants.UPNP_ENABLED, false) ) {
InstantBusyThread.oneTimeJob(UPnP.class, "addPortMappings", 0);
new OneTimeBusyThread("UPnP.addPortMappings") {
@Override
public boolean jobImpl() throws Exception {
UPnP.addPortMappings();
return true;
}
}.start();
}
// init TrayIcon if possible
@ -611,7 +619,14 @@ public final class Switchboard extends serverSwitch {
// start yacy core
this.log.config("Starting YaCy Protocol Core");
this.yc = new Network(this);
InstantBusyThread.oneTimeJob(this, "loadSeedLists", 0);
new OneTimeBusyThread("Switchboard.loadSeedLists") {
@Override
public boolean jobImpl() throws Exception {
loadSeedLists();
return true;
}
}.start();
//final long startedSeedListAquisition = System.currentTimeMillis();
// init a DHT transmission dispatcher
@ -941,7 +956,17 @@ public final class Switchboard extends serverSwitch {
// initializing the resourceObserver
this.observer = new ResourceObserver(this);
InstantBusyThread.oneTimeJob(this.observer, "resourceObserverJob", 0);
final ResourceObserver resourceObserver = this.observer;
new OneTimeBusyThread("ResourceObserver.resourceObserverJob") {
@Override
public boolean jobImpl() throws Exception {
resourceObserver.resourceObserverJob();
return true;
}
}.start();
// initializing the stackCrawlThread
this.crawlStacker =
@ -1032,101 +1057,142 @@ public final class Switchboard extends serverSwitch {
// deploy busy threads
this.log.config("Starting Threads");
MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq
deployThread(
SwitchboardConstants.CLEANUP,
"Cleanup",
"cleaning process",
null,
new InstantBusyThread(
this,
SwitchboardConstants.CLEANUP_METHOD_START,
SwitchboardConstants.CLEANUP_METHOD_JOBCOUNT,
SwitchboardConstants.CLEANUP_METHOD_FREEMEM,
30000,
10000),
new InstantBusyThread("Switchboard.cleanupJob", 30000, 10000) {
@Override
public boolean jobImpl() throws Exception {
return cleanupJob();
}
@Override
public int getJobCount() {
return cleanupJobSize();
}
@Override
public void freememImpl() {
}
},
60000); // all 10 minutes, wait 1 minute until first run
deployThread(
SwitchboardConstants.SCHEDULER,
"Scheduler",
"starts scheduled processes from the API Processing table",
null,
new InstantBusyThread(
this,
SwitchboardConstants.SCHEDULER_METHOD_START,
SwitchboardConstants.SCHEDULER_METHOD_JOBCOUNT,
SwitchboardConstants.SCHEDULER_METHOD_FREEMEM,
30000,
10000),
new InstantBusyThread("Switchboard.schedulerJob", 30000, 10000) {
@Override
public boolean jobImpl() throws Exception {
return schedulerJob();
}
@Override
public int getJobCount() {
return schedulerJobSize();
}
@Override
public void freememImpl() {
}
},
60000); // all 10 minutes, wait 1 minute until first run
deployThread(
SwitchboardConstants.SURROGATES,
"Surrogates",
"A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.",
null,
new InstantBusyThread(
this,
SwitchboardConstants.SURROGATES_METHOD_START,
SwitchboardConstants.SURROGATES_METHOD_JOBCOUNT,
SwitchboardConstants.SURROGATES_METHOD_FREEMEM,
20000,
0),
new InstantBusyThread("Switchboard.surrogateProcess", 20000, 0) {
@Override
public boolean jobImpl() throws Exception {
return surrogateProcess();
}
@Override
public int getJobCount() {
return surrogateQueueSize();
}
@Override
public void freememImpl() {
surrogateFreeMem();
}
},
10000);
this.initRemoteCrawler(this.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));
this.initAutocrawl(this.getConfigBool(SwitchboardConstants.AUTOCRAWL, false));
final CrawlQueues crawlQueue = this.crawlQueues;
deployThread(
SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL,
"Local Crawl",
"thread that performes a single crawl step from the local crawl queue",
"/IndexCreateQueues_p.html?stack=LOCAL",
new InstantBusyThread(
this.crawlQueues,
SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START,
SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT,
SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM,
0,
0),
new InstantBusyThread("CrawlQueues.coreCrawlJob", 0, 0) {
@Override
public boolean jobImpl() throws Exception {
return crawlQueue.coreCrawlJob();
}
@Override
public int getJobCount() {
return crawlQueue.coreCrawlJobSize();
}
@Override
public void freememImpl() {
crawlQueue.freemem();
}
},
10000);
final Network net = this.yc;
deployThread(
SwitchboardConstants.SEED_UPLOAD,
"Seed-List Upload",
"task that a principal peer performes to generate and upload a seed-list to a ftp account",
null,
new InstantBusyThread(
this.yc,
SwitchboardConstants.SEED_UPLOAD_METHOD_START,
SwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT,
SwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM,
600000,
300000),
new InstantBusyThread("Network.publishSeedList", 600000, 300000) {
@Override
public boolean jobImpl() throws Exception {
net.publishSeedList();
return true;
}
},
180000);
deployThread(
SwitchboardConstants.PEER_PING,
"YaCy Core",
"this is the p2p-control and peer-ping task",
null,
new InstantBusyThread(
this.yc,
SwitchboardConstants.PEER_PING_METHOD_START,
SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT,
SwitchboardConstants.PEER_PING_METHOD_FREEMEM,
30000,
30000),
new InstantBusyThread("Network.peerPing", 30000, 30000) {
@Override
public boolean jobImpl() throws Exception {
net.peerPing();
return true;
}
},
10000);
deployThread(
SwitchboardConstants.INDEX_DIST,
"DHT Distribution",
"selection, transfer and deletion of index entries that are not searched on your peer, but on others",
null,
new InstantBusyThread(
this,
SwitchboardConstants.INDEX_DIST_METHOD_START,
SwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT,
SwitchboardConstants.INDEX_DIST_METHOD_FREEMEM,
10000,
1000),
new InstantBusyThread("Switchboard.dhtTransferJob", 10000, 1000) {
@Override
public boolean jobImpl() throws Exception {
return dhtTransferJob();
}
},
60000,
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")),
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")),
@ -1138,28 +1204,20 @@ public final class Switchboard extends serverSwitch {
"720_ccimport",
"Content Control Import",
"this is the content control import thread",
null,
new InstantBusyThread(
new SMWListSyncThread(this, sb.getConfig("contentcontrol.bookmarklist", "contentcontrol"), "Category:Content Source", "/?Url/?Filter/?Category/?Modification date", sb.getConfigBool(
"contentcontrol.smwimport.purgelistoninit", false)),
"run",
SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT,
SwitchboardConstants.PEER_PING_METHOD_FREEMEM,
3000,
3000),
null,
InstantBusyThread.createFromRunnable(
new SMWListSyncThread(this, sb.getConfig("contentcontrol.bookmarklist", "contentcontrol"),
"Category:Content Source", "/?Url/?Filter/?Category/?Modification date",
sb.getConfigBool("contentcontrol.smwimport.purgelistoninit", false)),
3000, 3000),
2000);
deployThread(
"730_ccfilter",
"Content Control Filter",
"this is the content control filter update thread",
null,
new InstantBusyThread(
new ContentControlFilterUpdateThread(this),
"run",
SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT,
SwitchboardConstants.PEER_PING_METHOD_FREEMEM,
3000,
3000),
InstantBusyThread.createFromRunnable(new ContentControlFilterUpdateThread(this), 3000, 3000),
2000);
// set network-specific performance attributes
@ -1513,6 +1571,8 @@ public final class Switchboard extends serverSwitch {
this.peers.mySeed().setFlagAcceptRemoteCrawl(activate);
if (activate) {
this.crawlQueues.initRemoteCrawlQueues();
final CrawlQueues queues = this.crawlQueues;
BusyThread rct = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
if (rct == null) {
@ -1521,13 +1581,19 @@ public final class Switchboard extends serverSwitch {
"Remote Crawl Job",
"thread that performes a single crawl/indexing step triggered by a remote peer",
"/IndexCreateQueues_p.html?stack=REMOTE",
new InstantBusyThread(
this.crawlQueues,
SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START,
SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT,
SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM,
0,
0),
new InstantBusyThread("CrawlQueues.remoteTriggeredCrawlJob", 0, 0) {
@Override
public boolean jobImpl() throws Exception {
return queues.remoteTriggeredCrawlJob();
}
@Override
public int getJobCount() {
return queues.remoteTriggeredCrawlJobSize();
}
},
10000);
rct = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
}
@ -1541,13 +1607,12 @@ public final class Switchboard extends serverSwitch {
"Remote Crawl URL Loader",
"thread that loads remote crawl lists from other peers",
null,
new InstantBusyThread(
this.crawlQueues,
SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START,
SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT,
SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM,
10000,
10000),
new InstantBusyThread("CrawlQueues.remoteCrawlLoaderJob", 10000, 10000) {
@Override
public boolean jobImpl() throws Exception {
return queues.remoteCrawlLoaderJob();
}
},
10000);
rcl = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER);
@ -1569,18 +1634,19 @@ public final class Switchboard extends serverSwitch {
if (activate) {
BusyThread acr = getThread(SwitchboardConstants.CRAWLJOB_AUTOCRAWL);
if (acr == null) {
final CrawlQueues queues = this.crawlQueues;
deployThread(
SwitchboardConstants.CRAWLJOB_AUTOCRAWL,
"Autocrawl",
"Thread that selects and automatically adds crawling jobs to the local queue",
null,
new InstantBusyThread(
this.crawlQueues,
SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_START,
SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_JOBCOUNT,
SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_FREEMEM,
10000,
10000),
new InstantBusyThread("CrawlQueues.autocrawlJob", 10000, 10000) {
@Override
public boolean jobImpl() throws Exception {
return queues.autocrawlJob();
}
},
10000);
acr = getThread(SwitchboardConstants.CRAWLJOB_AUTOCRAWL);
@ -2832,6 +2898,11 @@ public final class Switchboard extends serverSwitch {
}
}
/**
* Parse a response to produce a new document to add to the index.
* <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.
* @param in an indexing workflow entry containing a response to parse
*/
public IndexingQueueEntry parseDocument(final IndexingQueueEntry in) {
in.queueEntry.updateStatus(Response.QUEUE_STATE_PARSING);
Document[] documents = null;
@ -3009,6 +3080,10 @@ public final class Switchboard extends serverSwitch {
return documents;
}
/**
* <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.
* @param in an indexing workflow entry containing a response and the related parsed document(s)
*/
public IndexingQueueEntry condenseDocument(final IndexingQueueEntry in) {
in.queueEntry.updateStatus(Response.QUEUE_STATE_CONDENSING);
CrawlProfile profile = in.queueEntry.profile();
@ -3083,6 +3158,11 @@ public final class Switchboard extends serverSwitch {
return new IndexingQueueEntry(in.queueEntry, in.documents, condenser);
}
/**
* Perform web structure analysis on parsed documents and update the web structure graph.
* <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.
* @param in an indexing workflow entry containing parsed document(s)
*/
public IndexingQueueEntry webStructureAnalysis(final IndexingQueueEntry in) {
in.queueEntry.updateStatus(Response.QUEUE_STATE_STRUCTUREANALYSIS);
for (Document document : in.documents) {
@ -3096,6 +3176,11 @@ public final class Switchboard extends serverSwitch {
return in;
}
/**
* Store a new entry to the local index.
* <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.
* @param in an indexing workflow entry containing parsed document(s) and a condenser instance
*/
public void storeDocumentIndex(final IndexingQueueEntry in) {
in.queueEntry.updateStatus(Response.QUEUE_STATE_INDEXSTORAGE);
// the condenser may be null in case that an indexing is not wanted (there may be a no-indexing flag in the file)

21
source/net/yacy/search/SwitchboardConstants.java
View File

@ -87,9 +87,6 @@ public final class SwitchboardConstants {
* connected to the YaCy-network</p>
*/
public static final String PEER_PING = "30_peerping";
public static final String PEER_PING_METHOD_START = "peerPing";
public static final String PEER_PING_METHOD_JOBCOUNT = null;
public static final String PEER_PING_METHOD_FREEMEM = null;
public static final String PEER_PING_IDLESLEEP = "30_peerping_idlesleep";
public static final String PEER_PING_BUSYSLEEP = "30_peerping_busysleep";
// 40_peerseedcycle
@ -98,9 +95,6 @@ public final class SwitchboardConstants {
* <p>Name of the seed upload thread, providing the so-called seed-lists needed during bootstrapping</p>
*/
public static final String SEED_UPLOAD = "40_peerseedcycle";
public static final String SEED_UPLOAD_METHOD_START = "publishSeedList";
public static final String SEED_UPLOAD_METHOD_JOBCOUNT = null;
public static final String SEED_UPLOAD_METHOD_FREEMEM = null;
public static final String SEED_UPLOAD_IDLESLEEP = "40_peerseedcycle_idlesleep";
public static final String SEED_UPLOAD_BUSYSLEEP = "40_peerseedcycle_busysleep";
// 50_localcrawl
@ -111,9 +105,6 @@ public final class SwitchboardConstants {
*
*/
public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl";
public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_START = "coreCrawlJob";
public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT = "coreCrawlJobSize";
public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM = "freemem";
public static final String CRAWLJOB_LOCAL_CRAWL_IDLESLEEP = "50_localcrawl_idlesleep";
public static final String CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP = "50_localcrawl_busysleep";
public static final String CRAWLJOB_LOCAL_CRAWL_LOADPREREQ = "50_localcrawl_loadprereq";
@ -148,9 +139,6 @@ public final class SwitchboardConstants {
* <p>Name of the remote triggered crawl thread, responsible for processing a remote crawl received from another peer</p>
*/
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL = "62_remotetriggeredcrawl";
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START = "remoteTriggeredCrawlJob";
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT = "remoteTriggeredCrawlJobSize";
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM = null;
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP = "62_remotetriggeredcrawl_idlesleep";
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP = "62_remotetriggeredcrawl_busysleep";
// 70_surrogates
@ -163,18 +151,12 @@ public final class SwitchboardConstants {
public static final String SURROGATES_LOADPREREQ = "70_surrogates_loadprereq";
public static final String SURROGATES_IDLESLEEP = "70_surrogates_idlesleep";
public static final String SURROGATES_BUSYSLEEP = "70_surrogates_busysleep";
public static final String SURROGATES_METHOD_START = "surrogateProcess";
public static final String SURROGATES_METHOD_JOBCOUNT = "surrogateQueueSize";
public static final String SURROGATES_METHOD_FREEMEM = "surrogateFreeMem";
// 85_scheduler
/**
* <p><code>public static final String <strong>SCHEDULER</strong> = "85_scheduler"</code></p>
* <p>The cleanup thread which is responsible for the start of scheduled processes from the API table</p>
*/
public static final String SCHEDULER = "85_scheduler";
public static final String SCHEDULER_METHOD_START = "schedulerJob";
public static final String SCHEDULER_METHOD_JOBCOUNT = "schedulerJobSize";
public static final String SCHEDULER_METHOD_FREEMEM = null;
public static final String SCHEDULER_IDLESLEEP = "85_scheduler_idlesleep";
public static final String SCHEDULER_BUSYSLEEP = "85_scheduler_busysleep";
// 90_cleanup
@ -183,9 +165,6 @@ public final class SwitchboardConstants {
* <p>The cleanup thread which is responsible for pendant cleanup-jobs, news/ranking distribution, etc.</p>
*/
public static final String CLEANUP = "90_cleanup";
public static final String CLEANUP_METHOD_START = "cleanupJob";
public static final String CLEANUP_METHOD_JOBCOUNT = "cleanupJobSize";
public static final String CLEANUP_METHOD_FREEMEM = null;
public static final String CLEANUP_IDLESLEEP = "90_cleanup_idlesleep";
public static final String CLEANUP_BUSYSLEEP = "90_cleanup_busysleep";
/**

6
source/net/yacy/search/query/SearchEvent.java
View File

@ -521,6 +521,9 @@ public final class SearchEvent {
SearchEventCache.put(this.query.id(false), this);
}
/**
* A concurrent task to perform the current search query on the local RWI.
*/
private class RWIProcess extends Thread {
final Thread waitForThread;
@ -530,6 +533,9 @@ public final class SearchEvent {
this.waitForThread = waitForThread;
}
/**
* Query the local RWI and feed the search event with the obtained results.
*/
@Override
public void run() {