better assortment monitoring and enhanced profile menue

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@416 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2005-07-20 13:03:41 +00:00
parent 5485c0e802
commit b73557ed2d
12 changed files with 94 additions and 113 deletions

View File

@ -22,8 +22,8 @@
<td class="small">Idle<br>Cycles</td>
<td class="small">Busy<br>Cycles</td>
<td class="small">Short Mem<br>Cycles</td>
<td class="small">Sleep Time<br>per Cycle<br>(milliseconds)</td>
<td class="small">Exec Time<br>per Busy-Cycle<br>(milliseconds)</td>
<td class="small">Sleep Time<br>per Cycle<br>(millis)</td>
<td class="small">Exec Time<br>per Busy-Cycle<br>(millis)</td>
<td class="small">Delay between<br>idle loops</td>
<td class="small">Delay between<br>busy loops</td>
<td class="small">Minimum of<br>Required Memory</td>
@ -57,17 +57,13 @@
Changes take effect immediately</td>
</tr>
</table>
</p></form>
</form>
</p>
<p>
<div class=small><b>Indexing Cache Settings:</b></div>
<table border="0" cellpadding="5" cellspacing="0" width="100%">
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<tr valign="top" class="TableCellDark">
<td width="120"></td>
<td></td>
<td></td>
</tr>
<table border="0" cellpadding="5" cellspacing="1" width="100%">
<tr valign="top" class="TableCellDark">
<td class=small>Words in RAM Cache:</td>
<td class=small>#[wordCacheRAMSize]#</td>
@ -85,13 +81,6 @@ Changes take effect immediately</td>
If this is a big number, it shows that the caching works efficiently.
</td>
</tr>
#{assortmentCluster}#
<tr valign="top" class="TableCellDark">
<td class=small>Assortment #[assortmentSlot]#:</td>
<td class=small>#[assortmentSize]#</td>
<td class=small>Buffer for words appearing exactly on #[assortmentSlot]# different pages.</td>
</tr>
#{/assortmentCluster}#
<tr valign="top" class="TableCellDark">
<td class=small>Maximum number of Word Caches:</td>
<td class=small><input name="wordCacheMax" type="text" size="20" maxlength="100" value="#[wordCacheMax]#"></td>
@ -101,27 +90,16 @@ Changes take effect immediately</td>
flushed to disc; this may last some minutes.
</td>
</tr>
<!--
<tr valign="top" class="TableCellDark">
<td class=small>Maximum waitingtime to flush word cache<br>during shut-down (seconds):</td>
<td class=small><input name="maxWaitingWordFlush" type="text" size="6" maxlength="6" value="#[maxWaitingWordFlush]#"></td>
<td class=small>
This gives the number of seconds that the shutdown
may last for the final word flush. Words that cannot be flushed within this
time are lost.
</td>
</tr>
-->
<tr valign="top" class="TableCellLight">
<td class=small colspan="3"><input type="submit" name="cacheSizeSubmit" value="Enter New Cache Size">
<td class=small colspan="3">
<input type="submit" name="cacheSizeSubmit" value="Enter New Cache Size">
Changes take effect immediately</td>
</td>
</tr>
</form>
</table>
</form>
</p>
<p>
<div class=small><b>Thread pool settings:</b></div>
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="2" cellspacing="1" width="100%">
@ -149,6 +127,26 @@ Changes take effect immediately</td>
</table>
</p></form>
<p>
<div class=small><b>Index Assortments:</b></div>
<table border="0" cellpadding="5" cellspacing="1" width="100%">
#{assortmentCluster}#
<tr valign="top" class="TableCellDark">
<td class=small>Assortments #[assortmentSlots]#:</td>
<td class=small align="right">#[assortmentSizeA]#</td>
<td class=small align="right">#[assortmentSizeB]#</td>
<td class=small align="right">#[assortmentSizeC]#</td>
<td class=small align="right">#[assortmentSizeD]#</td>
<td class=small align="right">#[assortmentSizeE]#</td>
<td class=small align="right">#[assortmentSizeF]#</td>
<td class=small align="right">#[assortmentSizeG]#</td>
<td class=small align="right">#[assortmentSizeH]#</td>
</tr>
#{/assortmentCluster}#
</table>
</p>
#[footer]#
</body>

View File

@ -91,8 +91,8 @@ public class Performance_p {
thread = switchboard.getThread(threadName);
// set values to templates
prop.put("table_" + c + "_threadname", threadName);
prop.put("table_" + c + "_shortdescr", thread.getShortDescription());
//prop.put("table_" + c + "_threadname", threadName);
prop.put("table_" + c + "_shortdescr", (thread.getMonitorURL() == null) ? thread.getShortDescription() : "<a href=\"" + thread.getMonitorURL() + "\" class=?\"small\">" + thread.getShortDescription() + "</a>");
prop.put("table_" + c + "_longdescr", thread.getLongDescription());
queuesize = thread.getJobCount();
prop.put("table_" + c + "_queuesize", (queuesize == Integer.MAX_VALUE) ? "unlimited" : ("" + queuesize));
@ -186,11 +186,18 @@ public class Performance_p {
prop.put("wordCacheMax", switchboard.getConfig("wordCacheMax", "10000"));
int[] asizes = switchboard.wordIndex.assortmentSizes();
for (int i = 0; i < asizes.length; i++) {
prop.put("assortmentCluster_" + i + "_assortmentSlot", i + 1);
prop.put("assortmentCluster_" + i + "_assortmentSize", asizes[i]);
for (int i = 0; i < asizes.length; i += 8) {
prop.put("assortmentCluster_" + (i/8) + "_assortmentSlots", (i + 1) + "-" + (i + 8));
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeA", asizes[i]);
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeB", asizes[i + 1]);
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeC", asizes[i + 2]);
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeD", asizes[i + 3]);
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeE", asizes[i + 4]);
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeF", asizes[i + 5]);
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeG", asizes[i + 6]);
prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeH", asizes[i + 7]);
}
prop.put("assortmentCluster", asizes.length);
prop.put("assortmentCluster", asizes.length / 8);
// table thread pool settings
GenericObjectPool.Config crawlerPoolConfig = switchboard.cacheLoader.getPoolConfig();

View File

@ -202,6 +202,7 @@ public class plasmaSnippetCache {
}
private String computeSnippet(String[] sentences, Set queryhashes, int minLength, int maxLength) {
try {
if ((sentences == null) || (sentences.length == 0)) return null;
if ((queryhashes == null) || (queryhashes.size() == 0)) return null;
kelondroMScoreCluster hitTable = new kelondroMScoreCluster();
@ -297,6 +298,10 @@ public class plasmaSnippetCache {
if (maxLength < 20) maxLength = 20;
String nextSnippet = computeSnippet(sentences, remaininghashes, minLength, maxLength);
return result + ((nextSnippet == null) ? "" : (" / " + nextSnippet));
} catch (IndexOutOfBoundsException e) {
e.printStackTrace();
return "";
}
}
private HashMap hashSentence(String sentence) {

View File

@ -351,36 +351,36 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
System.gc(); // help for profiler
int indexing_cluster = Integer.parseInt(getConfig("80_indexing_cluster", "1"));
if (indexing_cluster < 1) indexing_cluster = 1;
deployThread("90_cleanup", "Cleanup", "simple cleaning process for monitoring information" ,
deployThread("90_cleanup", "Cleanup", "simple cleaning process for monitoring information", null,
new serverInstantThread(this, "cleanupJob", "cleanupJobSize"), 10000); // all 5 Minutes
deployThread("80_indexing", "Parsing/Indexing", "thread that performes document parsing and indexing" ,
deployThread("80_indexing", "Parsing/Indexing", "thread that performes document parsing and indexing", "/IndexCreateIndexingQueue_p.html",
new serverInstantThread(this, "deQueue", "queueSize"), 10000);
for (int i = 1; i < indexing_cluster; i++) {
setConfig((i + 80) + "_indexing_idlesleep", getConfig("80_indexing_idlesleep", ""));
setConfig((i + 80) + "_indexing_busysleep", getConfig("80_indexing_busysleep", ""));
deployThread((i + 80) + "_indexing", "Parsing/Indexing (cluster job)", "thread that performes document parsing and indexing" ,
deployThread((i + 80) + "_indexing", "Parsing/Indexing (cluster job)", "thread that performes document parsing and indexing", null,
new serverInstantThread(this, "deQueue", "queueSize"), 10000 + (i * 1000));
}
deployThread("70_cachemanager", "Proxy Cache Enqueue", "job takes new proxy files from RAM stack, stores them, and hands over to the Indexing Stack",
deployThread("70_cachemanager", "Proxy Cache Enqueue", "job takes new proxy files from RAM stack, stores them, and hands over to the Indexing Stack", null,
new serverInstantThread(this, "htEntryStoreJob", "htEntrySize"), 10000);
deployThread("62_remotetriggeredcrawl", "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer",
deployThread("62_remotetriggeredcrawl", "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null,
new serverInstantThread(this, "remoteTriggeredCrawlJob", "remoteTriggeredCrawlJobSize"), 30000);
deployThread("61_globalcrawltrigger", "Global Crawl Trigger", "thread that triggeres remote peers for crawling",
deployThread("61_globalcrawltrigger", "Global Crawl Trigger", "thread that triggeres remote peers for crawling", "/IndexCreateWWWGlobalQueue_p.html",
new serverInstantThread(this, "limitCrawlTriggerJob", "limitCrawlTriggerJobSize"), 30000); // error here?
deployThread("50_localcrawl", "Local Crawl", "thread that performes a single crawl step from the local crawl queue",
deployThread("50_localcrawl", "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html",
new serverInstantThread(this, "coreCrawlJob", "coreCrawlJobSize"), 10000);
deployThread("40_peerseedcycle", "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account",
deployThread("40_peerseedcycle", "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null,
new serverInstantThread(yc, "publishSeedList", null), 180000);
serverInstantThread peerPing = null;
deployThread("30_peerping", "YaCy Core", "this is the p2p-control and peer-ping task",
deployThread("30_peerping", "YaCy Core", "this is the p2p-control and peer-ping task", null,
peerPing = new serverInstantThread(yc, "peerPing", null), 2000);
peerPing.setSyncObject(new Object());
indexDistribution = new plasmaWordIndexDistribution(urlPool, wordIndex, log,
getConfig("allowDistributeIndex", "false").equals("true"));
indexDistribution.setCounts(100, 1, 8000);
deployThread("20_dhtdistribution", "DHT Distribution (currently by juniors only)", "selection, transfer and deletion of index entries that are not searched on your peer, but on others",
deployThread("20_dhtdistribution", "DHT Distribution (currently by juniors only)", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null,
new serverInstantThread(indexDistribution, "job", null), 120000);
// init migratiion from 0.37 -> 0.38
@ -389,7 +389,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if (classicCache.size() > 0) {
setConfig("99_indexcachemigration_idlesleep" , 10000);
setConfig("99_indexcachemigration_busysleep" , 40);
deployThread("99_indexcachemigration", "index cache migration", "migration of index cache data structures 0.37 -> 0.38",
deployThread("99_indexcachemigration", "index cache migration", "migration of index cache data structures 0.37 -> 0.38", null,
new serverInstantThread(classicCache, "oneStepMigration", "size"), 30000);
}

View File

@ -118,13 +118,29 @@ public final class plasmaWordIndexAssortmentCluster {
private void storeStretched(String wordHash, plasmaWordIndexEntryContainer newContainer) {
// this stores the record and stretches the storage over
// all the assortments that are necessary to fit in the record
// IMPORTANT: it must be ensured that the wordHash does not exist in the cluster before
// i.e. by calling removeFromAll
if (newContainer.size() <= clusterCount) {
storeForced(wordHash, newContainer);
return;
}
// calculate appropriate cluster insert point
int clusterStart = clusterCount;
if ((((byte) wordHash.charAt(0)) & 1) == 1) {
// for every second hash, place the entries in the middle of the assortments
// this balances the entries within the assortments-cluster
int cap = clusterCapacity - newContainer.size() - 2 * clusterCount;
while (cap > 0) {
cap -= clusterStart;
clusterStart--;
}
}
// do the insert
plasmaWordIndexEntryContainer c;
Iterator i = newContainer.entries();
for (int j = clusterCount; j >= 1; j--) {
for (int j = clusterStart; j >= 1; j--) {
c = new plasmaWordIndexEntryContainer(wordHash);
for (int k = 0; k < j; k++) {
if (i.hasNext()) {

View File

@ -67,7 +67,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
private static final String oldSingletonFileName = "indexSingletons0.db";
private static final String newSingletonFileName = "indexAssortment001.db";
private static final String indexAssortmentClusterPath = "ACLUSTER";
private static final int assortmentCount = 50;
private static final int assortmentCount = 64;
private static final int ramCacheLimit = 200;
@ -243,63 +243,6 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
return urlCount;
}
/*
private void dump(int waitingSeconds) throws IOException {
log.logSystem("creating dump for index cache, " + cache.size() + " words (and much more urls)");
File indexDumpFile = new File(databaseRoot, indexStackFileName);
if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroStack dumpStack = new kelondroStack(indexDumpFile, 1024, plasmaWordIndexAssortment.bufferStructureBasis);
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
synchronized (cache) {
Iterator i = cache.entrySet().iterator();
Map.Entry entry;
String wordHash;
plasmaWordIndexEntryContainer container;
long updateTime;
plasmaWordIndexEntry wordEntry;
byte[][] row = new byte[5][];
System.gc(); // this can speed up the assortment, because they may better use the cache
while (i.hasNext()) {
// get entries
entry = (Map.Entry) i.next();
wordHash = (String) entry.getKey();
updateTime = getUpdateTime(wordHash);
container = (plasmaWordIndexEntryContainer) entry.getValue();
// put entries on stack
if (container != null) {
Iterator ci = container.entries();
while (ci.hasNext()) {
wordEntry = (plasmaWordIndexEntry) ci.next();
row[0] = wordHash.getBytes();
row[1] = kelondroRecords.long2bytes(container.size(), 4);
row[2] = kelondroRecords.long2bytes(updateTime, 8);
row[3] = wordEntry.getUrlHash().getBytes();
row[4] = wordEntry.toEncodedForm(true).getBytes();
dumpStack.push(row);
urlcount++;
}
}
wordcount++;
i.remove(); // free some mem
// write a log
if (System.currentTimeMillis() > messageTime) {
System.gc(); // for better statistic
wordsPerSecond = wordcount * 1000 / (1 + System.currentTimeMillis() - startTime);
log.logInfo("dumping status: " + wordcount + " words done, " + (cache.size() / (wordsPerSecond + 1)) + " seconds remaining, free mem = " + (Runtime.getRuntime().freeMemory() / 1024 / 1024) + "MB");
messageTime = System.currentTimeMillis() + 5000;
}
}
}
dumpStack.close();
log.logSystem("dumped " + urlcount + " word/url relations in " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds");
}
*/
private long restoreStack() throws IOException {
File indexDumpFile = new File(databaseRoot, indexStackFileName);
if (!(indexDumpFile.exists())) return 0;

View File

@ -257,9 +257,10 @@ public abstract class serverAbstractSwitch implements serverSwitch {
String threadName,
String threadShortDescription,
String threadLongDescription,
String threadMonitorURL,
serverThread newThread,
long startupDelay) {
deployThread(threadName, threadShortDescription, threadLongDescription,
deployThread(threadName, threadShortDescription, threadLongDescription, threadMonitorURL,
newThread, startupDelay,
Long.parseLong(getConfig(threadName + "_idlesleep" , "100")),
Long.parseLong(getConfig(threadName + "_busysleep" , "1000")),
@ -270,6 +271,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
String threadName,
String threadShortDescription,
String threadLongDescription,
String threadMonitorURL,
serverThread newThread,
long startupDelay,
long initialIdleSleep,
@ -300,7 +302,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
}
newThread.setLog(log);
newThread.setDescription(threadShortDescription, threadLongDescription);
newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL);
workerThreads.put(threadName, newThread);
// start the thread
if (workerThreads.containsKey(threadName)) newThread.start();

View File

@ -59,6 +59,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
private serverLog log = null;
private long idletime = 0, busytime = 0, memprereq = 0;
private String shortDescr = "", longDescr = "";
private String monitorURL = null;
private long threadBlockTimestamp = System.currentTimeMillis();
private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
private Object syncObject = null;
@ -87,10 +88,11 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
this.idletime += millis;
}
public final void setDescription(String shortText, String longText) {
public final void setDescription(String shortText, String longText, String monitorURL) {
// sets a visible description string
this.shortDescr = shortText;
this.longDescr = longText;
this.monitorURL = monitorURL;
}
public final void setStartupSleep(long milliseconds) {
@ -121,6 +123,10 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
return this.longDescr;
}
public String getMonitorURL() {
return this.monitorURL;
}
public final long getIdleCycles() {
// returns the total number of cycles of job execution with idle-result
return this.idleCycles;

View File

@ -138,7 +138,7 @@ public class serverPortForwardingSch implements serverPortForwarding{
// using a timer task to control if the session remains open
if (sessionWatcher == null) {
this.log.logDebug("Deploying port forwarding session watcher thread.");
this.switchboard.deployThread("portForwardingWatcher", "Remote Port Forwarding Watcher", "this thread is used to detect broken connections and to re-establish it if necessary.",
this.switchboard.deployThread("portForwardingWatcher", "Remote Port Forwarding Watcher", "this thread is used to detect broken connections and to re-establish it if necessary.", null,
sessionWatcher = new serverInstantThread(this, "reconnect", null), 30000,30000,30000,1000);
sessionWatcher.setSyncObject(new Object());
}

View File

@ -71,6 +71,7 @@ public interface serverSwitch {
public void deployThread(String threadName,
String threadShortDescription,
String threadLongDescription,
String threadMonitorURL,
serverThread newThread,
long startupDelay,
long initialIdleSleep, long initialBusySleep,

View File

@ -53,7 +53,7 @@ public interface serverThread {
// these method are implemented by serverThread and do not need to be altered
// this includes also the run()-Method
public void setDescription(String shortText, String longText);
public void setDescription(String shortText, String longText, String monitorURL);
// sets a visible description string
public void setStartupSleep(long milliseconds);
@ -74,6 +74,9 @@ public interface serverThread {
public String getLongDescription();
// returns long description string for online display
public String getMonitorURL();
// returns an URL that can be used to monitor the thread and it's queue
public long getIdleCycles();
// returns the total number of cycles of job execution with idle-result

View File

@ -300,7 +300,7 @@ public final class yacy {
serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
} else {
// first start the server
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, 0, 0, 0, 0);
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
//server.start();
// open the browser window