Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

added new performance page for memory settings

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@751 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2005-09-20 10:10:34 +00:00
parent 0f1bbff459
commit fb52a82008
22 changed files with 534 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
htroot/PerformanceMemory_p.html Normal file
View File

@ -0,0 +1,189 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
<html>
<head>
<title>YaCy '#[clientname]#': Performance Settings for Memory</title>
#[metas]#
</head>
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
#[header]#
<br><br>
<h2>Performance Settings for Memory</h2>
<p>
<div class=small><b>RAM Cache for Database Files:</b></div>
<table border="0" cellpadding="2" cellspacing="1">
<form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
<tr class="TableHeader" valign="bottom">
<td class="small">Database<br></td>
<td class="small">Chunk&nbsp;Size<br>(bytes)</td>
<td class="small">#Needed&nbsp;Slots<br>(=&nbsp;DB&nbsp;Size)</td>
<td class="small">#Empty&nbsp;Slots<br>&nbsp;</td>
<td class="small">#Used&nbsp;Slots<br>High&nbsp;Priority</td>
<td class="small">#Used&nbsp;Slots<br>Medium&nbsp;Priority</td>
<td class="small">#Used&nbsp;Slots<br>Low&nbsp;Priority</td>
<td class="small">Used<br>Size</td>
<td class="small">Assigned<br>Max-Size</td>
<td class="small">Recommended<br>Max-Size</td>
<td class="small">Optimum<br>Max-Size</td>
<td class="small">Description<br>&nbsp;</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">RWI Assortment Cluster</td>
<td class="small" align="right">#[chunkRWI]#</td>
<td class="small" align="right">#[slreqRWI]#</td>
<td class="small" align="right">#[slempRWI]#</td>
<td class="small" align="right">#[slhigRWI]#</td>
<td class="small" align="right">#[slmedRWI]#</td>
<td class="small" align="right">#[sllowRWI]#</td>
<td class="small" align="right">#[usedRWI]#</td>
<td class="small" align="right"><input name="ramCacheRWI" type="text" align="right" size="8" maxlength="9" value="#[ramCacheRWI]#"> bytes</td>
<td class="small" align="right">#[goodRWI]#</td>
<td class="small" align="right">#[bestRWI]#</td>
<td class="small" align="left">The Assortment Cluster stores most of the page indexes.
Flushing speed of the temporary RWI cache depends on the size of this file cache. Increasing the space of this
cache will speed up crawls with a depth > 3.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">HTTP Response Header</td>
<td class="small" align="right">#[chunkHTTP]#</td>
<td class="small" align="right">#[slreqHTTP]#</td>
<td class="small" align="right">#[slempHTTP]#</td>
<td class="small" align="right">#[slhigHTTP]#</td>
<td class="small" align="right">#[slmedHTTP]#</td>
<td class="small" align="right">#[sllowHTTP]#</td>
<td class="small" align="right">#[usedHTTP]#</td>
<td class="small" align="right"><input name="ramCacheHTTP" type="text" align="right" size="8" maxlength="9" value="#[ramCacheHTTP]#"> bytes</td>
<td class="small" align="right">#[goodHTTP]#</td>
<td class="small" align="right">#[bestHTTP]#</td>
<td class="small" align="left">The Response Header database stores the HTTP heades that other servers send when YaCy retrieves web pages
during proxy mode, when performing crawls or if it fetches pages for snippet generation.
Increasing this cache will be most important for a fast proxy mode.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">'loaded' URLs</td>
<td class="small" align="right">#[chunkLURL]#</td>
<td class="small" align="right">#[slreqLURL]#</td>
<td class="small" align="right">#[slempLURL]#</td>
<td class="small" align="right">#[slhigLURL]#</td>
<td class="small" align="right">#[slmedLURL]#</td>
<td class="small" align="right">#[sllowLURL]#</td>
<td class="small" align="right">#[usedLURL]#</td>
<td class="small" align="right"><input name="ramCacheLURL" type="text" align="right" size="8" maxlength="9" value="#[ramCacheLURL]#"> bytes</td>
<td class="small" align="right">#[goodLURL]#</td>
<td class="small" align="right">#[bestLURL]#</td>
<td class="small" align="left">This is the database that holds the hash/url - relation and properties regarding the url like load date and server date.
This cache is very important for a fast search process. Increasing the cache size will result in more search results and less IO during DHT transfer.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">'noticed' URLs</td>
<td class="small" align="right">#[chunkNURL]#</td>
<td class="small" align="right">#[slreqNURL]#</td>
<td class="small" align="right">#[slempNURL]#</td>
<td class="small" align="right">#[slhigNURL]#</td>
<td class="small" align="right">#[slmedNURL]#</td>
<td class="small" align="right">#[sllowNURL]#</td>
<td class="small" align="right">#[usedNURL]#</td>
<td class="small" align="right"><input name="ramCacheNURL" type="text" align="right" size="8" maxlength="9" value="#[ramCacheNURL]#"> bytes</td>
<td class="small" align="right">#[goodNURL]#</td>
<td class="small" align="right">#[bestNURL]#</td>
<td class="small" align="left">A noticed URL is one that was discovered during crawling but was not loaded yet.
Increasing the cache size will result in faster double-check during URL recognition when doing crawls.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">'error' URLs</td>
<td class="small" align="right">#[chunkEURL]#</td>
<td class="small" align="right">#[slreqEURL]#</td>
<td class="small" align="right">#[slempEURL]#</td>
<td class="small" align="right">#[slhigEURL]#</td>
<td class="small" align="right">#[slmedEURL]#</td>
<td class="small" align="right">#[sllowEURL]#</td>
<td class="small" align="right">#[usedEURL]#</td>
<td class="small" align="right"><input name="ramCacheEURL" type="text" align="right" size="8" maxlength="9" value="#[ramCacheEURL]#"> bytes</td>
<td class="small" align="right">#[goodEURL]#</td>
<td class="small" align="right">#[bestEURL]#</td>
<td class="small" align="left">URLs that cannot be loaded are stored in this database. It is also used for double-checked during crawling.
Increasing the cache size will most probably speed up crawling slightly, but not significantly.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">DHT Control</td>
<td class="small" align="right">#[chunkDHT]#</td>
<td class="small" align="right">#[slreqDHT]#</td>
<td class="small" align="right">#[slempDHT]#</td>
<td class="small" align="right">#[slhigDHT]#</td>
<td class="small" align="right">#[slmedDHT]#</td>
<td class="small" align="right">#[sllowDHT]#</td>
<td class="small" align="right">#[usedDHT]#</td>
<td class="small" align="right"><input name="ramCacheDHT" type="text" align="right" size="8" maxlength="9" value="#[ramCacheDHT]#"> bytes</td>
<td class="small" align="right">#[goodDHT]#</td>
<td class="small" align="right">#[bestDHT]#</td>
<td class="small" align="left">This is simply the cache for the seed-dbs (active, passive, potential).
This cache is divided into three equal parts.
Increasing this cache may speed up many functions, but we need to test this to see the effects.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">Messages</td>
<td class="small" align="right">#[chunkMessage]#</td>
<td class="small" align="right">#[slreqMessage]#</td>
<td class="small" align="right">#[slempMessage]#</td>
<td class="small" align="right">#[slhigMessage]#</td>
<td class="small" align="right">#[slmedMessage]#</td>
<td class="small" align="right">#[sllowMessage]#</td>
<td class="small" align="right">#[usedMessage]#</td>
<td class="small" align="right"><input name="ramCacheMessage" type="text" align="right" size="8" maxlength="9" value="#[ramCacheMessage]#"> bytes</td>
<td class="small" align="right">#[goodMessage]#</td>
<td class="small" align="right">#[bestMessage]#</td>
<td class="small" align="left">The Message cache for peer-to-peer messages. Less important.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">Wiki</td>
<td class="small" align="right">#[chunkWiki]#</td>
<td class="small" align="right">#[slreqWiki]#</td>
<td class="small" align="right">#[slempWiki]#</td>
<td class="small" align="right">#[slhigWiki]#</td>
<td class="small" align="right">#[slmedWiki]#</td>
<td class="small" align="right">#[sllowWiki]#</td>
<td class="small" align="right">#[usedWiki]#</td>
<td class="small" align="right"><input name="ramCacheWiki" type="text" align="right" size="8" maxlength="9" value="#[ramCacheWiki]#"> bytes</td>
<td class="small" align="right">#[goodWiki]#</td>
<td class="small" align="right">#[bestWiki]#</td>
<td class="small" align="left">The YaCy-Wiki uses a database to store its pages.
This cache is divided in two parts, one for the wiki database and one for its backup.
Increasing this cache may speed up access to the wiki pages.</td>
</tr>
<tr class="TableCellDark">
<td class="small" align="left">News</td>
<td class="small" align="right">#[chunkNews]#</td>
<td class="small" align="right">#[slreqNews]#</td>
<td class="small" align="right">#[slempNews]#</td>
<td class="small" align="right">#[slhigNews]#</td>
<td class="small" align="right">#[slmedNews]#</td>
<td class="small" align="right">#[sllowNews]#</td>
<td class="small" align="right">#[usedNews]#</td>
<td class="small" align="right"><input name="ramCacheNews" type="text" align="right" size="8" maxlength="9" value="#[ramCacheNews]#"> bytes</td>
<td class="small" align="right">#[goodNews]#</td>
<td class="small" align="right">#[bestNews]#</td>
<td class="small" align="left">The News-DB stores property-lists for news that are included in seeds.
Increasing this cache may speed up the peer-ping.</td>
</tr>
<tr class="TableCellLight">
<td class="small" align="left" colspan="18">
<input type="submit" name="submitcache" value="Submit New Cache Values">&nbsp;&nbsp;&nbsp;
Changes take effect after re-start of YaCy</td>
</tr>
</form>
</table>
</p>
#[footer]#
</body>
</html>

159
htroot/PerformanceMemory_p.java Normal file
View File

@ -0,0 +1,159 @@
//PerformaceMemory_p.java
//-----------------------
//part of YaCy
//(C) by Michael Peter Christen; mc@anomic.de
//first published on http://www.anomic.de
//Frankfurt, Germany, 2005
//last major change: 19.09.2005
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
//the Free Software Foundation; either version 2 of the License, or
//(at your option) any later version.
//
//This program is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//GNU General Public License for more details.
//
//You should have received a copy of the GNU General Public License
//along with this program; if not, write to the Free Software
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//Using this software in any meaning (reading, learning, copying, compiling,
//running) means that you agree that the Author(s) is (are) not responsible
//for cost, loss of data or any harm that may be caused directly or indirectly
//by usage of this softare or this documentation. The usage of this software
//is on your own risk. The installation and usage (starting/running) of this
//software may allow other people or application to access your computer and
//any attached devices and is highly dependent on the configuration of the
//software which must be done by the user of the software; the author(s) is
//(are) also not responsible for proper configuration and usage of the
//software, even if provoked by documentation provided together with
//the software.
//
//Any changes to this file according to the GPL as documented in the file
//gpl.txt aside this file in the shipment you received can be done to the
//lines that follows this copyright notice here, but changes must not be
//done inside the copyright notive above. A re-distribution must contain
//the intact and unchanged copyright notice.
//Contributions and changes to the program code must be marked as such.
//You must compile this file with
//javac -classpath .:../classes PerformanceMemory_p.java
//if the shell's current path is HTROOT
import java.util.Iterator;
import java.util.Map;
import java.io.File;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverThread;
import de.anomic.server.serverFileUtils;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeedDB;
public class PerformanceMemory_p {
private static int[] slt;
private static int req,chk,usd,bst,god;
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
serverObjects prop = new serverObjects();
File defaultSettingsFile = new File(switchboard.getRootPath(), "yacy.init");
Map defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : serverFileUtils.loadHashMap(defaultSettingsFile);
if ((post != null) && (post.containsKey("submitcache"))) {
sb.setConfig("ramCacheRWI", post.get("ramCacheRWI", "0"));
sb.setConfig("ramCacheHTTP", post.get("ramCacheHTTP", "0"));
sb.setConfig("ramCacheLURL", post.get("ramCacheLURL", "0"));
sb.setConfig("ramCacheNURL", post.get("ramCacheNURL", "0"));
sb.setConfig("ramCacheEURL", post.get("ramCacheEURL", "0"));
sb.setConfig("ramCacheDHT", post.get("ramCacheDHT", "0"));
sb.setConfig("ramCacheMessage", post.get("ramCacheMessage", "0"));
sb.setConfig("ramCacheWiki", post.get("ramCacheWiki", "0"));
sb.setConfig("ramCacheNews", post.get("ramCacheNews", "0"));
}
req = switchboard.wordIndex.size();
chk = switchboard.wordIndex.assortmentsCacheChunkSizeAvg();
slt = switchboard.wordIndex.assortmentsCacheFillStatusCml();
calc(); putprop(prop, "RWI");
prop.put("ramCacheRWI", sb.getConfig("ramCacheRWI", "0"));
req = switchboard.cacheManager.size();
chk = switchboard.cacheManager.dbCacheChunkSize();
slt = switchboard.cacheManager.dbCacheFillStatus();
calc(); putprop(prop, "HTTP");
prop.put("ramCacheHTTP", sb.getConfig("ramCacheHTTP", "0"));
req = switchboard.urlPool.loadedURL.urlHashCache.size();
chk = switchboard.urlPool.loadedURL.urlHashCache.cacheChunkSize();
slt = switchboard.urlPool.loadedURL.urlHashCache.cacheFillStatus();
calc(); putprop(prop, "LURL");
prop.put("ramCacheLURL", sb.getConfig("ramCacheLURL", "0"));
req = switchboard.urlPool.noticeURL.urlHashCache.size();
chk = switchboard.urlPool.noticeURL.urlHashCache.cacheChunkSize();
slt = switchboard.urlPool.noticeURL.urlHashCache.cacheFillStatus();
calc(); putprop(prop, "NURL");
prop.put("ramCacheNURL", sb.getConfig("ramCacheNURL", "0"));
req = switchboard.urlPool.errorURL.urlHashCache.size();
chk = switchboard.urlPool.errorURL.urlHashCache.cacheChunkSize();
slt = switchboard.urlPool.errorURL.urlHashCache.cacheFillStatus();
calc(); putprop(prop, "EURL");
prop.put("ramCacheEURL", sb.getConfig("ramCacheEURL", "0"));
req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential();
chk = yacyCore.seedDB.dbCacheChunkSize();
slt = yacyCore.seedDB.dbCacheFillStatus();
calc(); putprop(prop, "DHT");
prop.put("ramCacheDHT", sb.getConfig("ramCacheDHT", "0"));
req = switchboard.messageDB.size();
chk = switchboard.messageDB.dbCacheChunkSize();
slt = switchboard.messageDB.dbCacheFillStatus();
calc(); putprop(prop, "Message");
prop.put("ramCacheMessage", sb.getConfig("ramCacheMessage", "0"));
req = switchboard.wikiDB.sizeOfTwo();
chk = switchboard.wikiDB.dbCacheChunkSize();
slt = switchboard.wikiDB.dbCacheFillStatus();
calc(); putprop(prop, "Wiki");
prop.put("ramCacheWiki", sb.getConfig("ramCacheWiki", "0"));
req = yacyCore.newsPool.dbSize();
chk = yacyCore.newsPool.dbCacheChunkSize();
slt = yacyCore.newsPool.dbCacheFillStatus();
calc(); putprop(prop, "News");
prop.put("ramCacheNews", sb.getConfig("ramCacheNews", "0"));
// return rewrite values for templates
return prop;
}
private static void calc() {
usd = chk * (slt[1]+slt[2]+slt[3]);
bst = (((chk * req) >> 10) + 1) << 10;
god = (((bst / (1+slt[1]+slt[2]+slt[3]) * slt[1]) >> 10) + 1) << 10;
}
private static void putprop(serverObjects prop, String db) {
prop.put("chunk" + db, chk);
prop.put("slreq" + db, req);
prop.put("slemp" + db, slt[0]);
prop.put("slhig" + db, slt[1]);
prop.put("slmed" + db, slt[2]);
prop.put("sllow" + db, slt[3]);
prop.put("used" + db, usd);
prop.put("good" + db, god);
prop.put("best" + db, bst);
}
}

12
htroot/Performance_p.html → htroot/PerformanceQueues_p.html
View File

@ -1,17 +1,17 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
<html>
<head>
<title>YaCy '#[clientname]#': Performance</title>
<title>YaCy '#[clientname]#': Performance Settings of Queues and Processes</title>
#[metas]#
</head>
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
#[header]#
<br><br>
<h2>Performance</h2>
<h2>Performance Settings of Queues and Processes</h2>
<p>
<div class=small><b>Scheduled tasks overview and waiting time settings:</b></div>
<table border="0" cellpadding="2" cellspacing="1">
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<tr class="TableHeader" valign="bottom">
<td class="small">Thread</td>
<td class="small">Queue Size</td>
@ -63,7 +63,7 @@ Changes take effect immediately</td>
<p>
<div class=small><b>Indexing Cache Settings:</b></div>
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="5" cellspacing="1" width="100%">
<tr valign="top" class="TableCellDark">
<td class=small>Words in RAM Cache:</td>
@ -102,7 +102,7 @@ Changes take effect immediately</td>
<p>
<div class=small><b>Thread pool settings:</b></div>
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="2" cellspacing="1" width="100%">
<tr class="TableHeader" valign="bottom">
<td class="small" width="150">Thread Pool</td>
@ -149,7 +149,7 @@ Changes take effect immediately</td>
<p>
<div class=small><b>Proxy Performance Settings:</b></div>
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="5" cellspacing="1" width="100%">
<tr valign="top" class="TableCellDark">
<td class=small>Online Caution Delay (milliseconds):</td>

4
htroot/Performance_p.java → htroot/PerformanceQueues_p.java
View File

@ -1,4 +1,4 @@
//Performace_p.java
//PerformaceQueues_p.java
//-----------------------
//part of YaCy
//(C) by Michael Peter Christen; mc@anomic.de
@ -57,7 +57,7 @@ import de.anomic.server.serverSwitch;
import de.anomic.server.serverThread;
import de.anomic.server.serverFileUtils;
public class Performance_p {
public class PerformanceQueues_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
// return variable that accumulates replacements

2
htroot/env/templates/header.template vendored
View File

@ -64,7 +64,7 @@
<tr><td class="MenuItem">&nbsp;<a href="/News.html" accesskey="n" class="MenuItemLink">News</a></td></tr>
<tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/ViewLog_p.html" class="MenuItemLink">Log</a></td></tr>
<tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/Settings_p.html" class="MenuItemLink">Settings</a></td></tr>
<tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/Performance_p.html" class="MenuItemLink">Performance</a></td></tr>
<tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/PerformanceQueues_p.html" class="MenuItemLink">Performance</a></td></tr>
<tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/Language_p.html" class="MenuItemLink">Language</a></td></tr>
<tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/Skins_p.html" class="MenuItemLink">Skins</a></td></tr>
<tr><td class="MenuSpacer"></td></tr>

12
source/de/anomic/data/messageBoard.java
View File

@ -78,6 +78,18 @@ public class messageBoard {
sn = 0;
}
public int size() {
return database.size();
}
public int dbCacheChunkSize() {
return database.cacheChunkSize();
}
public int[] dbCacheFillStatus() {
return database.cacheFillStatus();
}
public void close() throws IOException {
database.close();
}

22
source/de/anomic/data/wikiBoard.java
View File

@ -74,20 +74,34 @@ public class wikiBoard {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
if (actpath.exists())
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb * 0x40));
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x40));
else
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb * 0x400, keyLength, recordSize));
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x400, keyLength, recordSize));
}
new File(bkppath.getParent()).mkdir();
if (bkpbase == null) {
if (bkppath.exists())
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb * 0x400));
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400));
else
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb * 0x400, keyLength + dateFormat.length(), recordSize));
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400, keyLength + dateFormat.length(), recordSize));
}
sn = 0;
}
public int sizeOfTwo() {
return datbase.size() + bkpbase.size();
}
public int dbCacheChunkSize() {
return (datbase.cacheChunkSize() + bkpbase.cacheChunkSize()) / 2;
}
public int[] dbCacheFillStatus() {
int[] a = datbase.cacheFillStatus();
int[] b = bkpbase.cacheFillStatus();
return new int[]{a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3]};
}
public void close() throws IOException {
datbase.close();
bkpbase.close();

8
source/de/anomic/kelondro/kelondroMap.java
View File

@ -127,6 +127,14 @@ public class kelondroMap {
return dyn.columnSize(0);
}
public int cacheChunkSize() {
return dyn.cacheChunkSize();
}
public int[] cacheFillStatus() {
return dyn.cacheFillStatus();
}
public synchronized void set(String key, Map newMap) throws IOException {
// update elementCount
if ((sortfields != null) || (accfields != null)) {

78
source/de/anomic/kelondro/kelondroRecords.java
View File

@ -159,19 +159,7 @@ public class kelondroRecords {
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename));
//kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0])));
if (XcacheSize <= 0) {
XcacheSize = 0;
this.XcacheHeaders = null;
this.XcacheScore = null;
} else {
// initialize cache
// in case that we have more cache space than elements
// we dont need cache control by the XcacheScore
this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
}
this.XcacheStartup = System.currentTimeMillis();
initCache(buffersize);
}
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */,
@ -179,16 +167,7 @@ public class kelondroRecords {
int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException {
this.filename = null;
init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0])));
if (XcacheSize <= 0) {
XcacheSize = 0;
this.XcacheHeaders = null;
this.XcacheScore = null;
} else {
this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
}
this.XcacheStartup = System.currentTimeMillis();
initCache(buffersize);
}
private void init(kelondroRA ra, short ohbytec, short ohhandlec,
@ -275,31 +254,13 @@ public class kelondroRecords {
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
init(raf);
this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0])));
if (XcacheSize <= 0) {
XcacheSize = 0;
this.XcacheHeaders = null;
this.XcacheScore = null;
} else {
this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
}
this.XcacheStartup = System.currentTimeMillis();
initCache(buffersize);
}
public kelondroRecords(kelondroRA ra, long buffersize) throws IOException{
this.filename = null;
init(ra);
this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0])));
if (XcacheSize <= 0) {
XcacheSize = 0;
this.XcacheHeaders = null;
this.XcacheScore = null;
} else {
this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
}
this.XcacheStartup = System.currentTimeMillis();
initCache(buffersize);
}
private void init(kelondroRA ra) throws IOException {
@ -350,7 +311,38 @@ public class kelondroRecords {
this.tailchunksize = this.recordsize - this.headchunksize;
}
private void initCache(long buffersize) {
if (buffersize <= 0) {
this.XcacheSize = 0;
this.XcacheHeaders = null;
this.XcacheScore = null;
} else {
if ((buffersize / cacheChunkSize(false)) > size()) {
this.XcacheSize = (int) (buffersize / cacheChunkSize(false));
this.XcacheScore = null;
} else {
this.XcacheSize = (int) (buffersize / cacheChunkSize(true));
this.XcacheScore = new kelondroMScoreCluster();
}
this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
}
this.XcacheStartup = System.currentTimeMillis();
}
private int cacheChunkSize(boolean cacheControl) {
return this.headchunksize + 14 + ((cacheControl) ? 16 : 0);
}
public int cacheChunkSize() {
return cacheChunkSize(this.XcacheScore != null);
}
public int[] cacheFillStatus() {
if (XcacheHeaders == null) return new int[]{0,0,0,0};
return new int[]{XcacheSize - (XcacheHeaders[CP_HIGH].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_LOW].size()), XcacheHeaders[CP_HIGH].size(), XcacheHeaders[CP_MEDIUM].size(), XcacheHeaders[CP_LOW].size()};
}
protected Node newNode() throws IOException {
return new Node();
}

8
source/de/anomic/plasma/plasmaHTCache.java
View File

@ -135,6 +135,14 @@ public final class plasmaHTCache {
}
}
public int dbCacheChunkSize() {
return responseHeaderDB.cacheChunkSize();
}
public int[] dbCacheFillStatus() {
return responseHeaderDB.cacheFillStatus();
}
public void push(Entry entry) {
synchronized (this.cacheStack) {
this.cacheStack.add(entry);

2
source/de/anomic/plasma/plasmaSwitchboard.java
View File

@ -154,7 +154,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public static plasmaURLPattern urlBlacklist;
// storage management
private File cachePath; // do we need that ?
private File cachePath;
private File plasmaPath;
public File listsPath;
public plasmaURLPool urlPool;

2
source/de/anomic/plasma/plasmaURL.java
View File

@ -93,7 +93,7 @@ public class plasmaURL {
}
// the class object
protected kelondroTree urlHashCache;
public kelondroTree urlHashCache;
private HashSet existsIndex;
public plasmaURL() throws IOException {

10
source/de/anomic/plasma/plasmaWordIndex.java
View File

@ -82,7 +82,15 @@ public final class plasmaWordIndex {
public int[] assortmentSizes() {
return ramCache.assortmentsSizes();
}
public int assortmentsCacheChunkSizeAvg() {
return ramCache.assortmentsCacheChunkSizeAvg();
}
public int[] assortmentsCacheFillStatusCml() {
return ramCache.assortmentsCacheFillStatusCml();
}
public void setMaxWords(int maxWords) {
ramCache.setMaxWords(maxWords);
}

8
source/de/anomic/plasma/plasmaWordIndexAssortment.java
View File

@ -213,6 +213,14 @@ public final class plasmaWordIndexAssortment {
return assortments.size();
}
public int cacheChunkSize() {
return assortments.cacheChunkSize();
}
public int[] cacheFillStatus() {
return assortments.cacheFillStatus();
}
public void close() {
try {
assortments.close();

17
source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java
View File

@ -190,7 +190,22 @@ public final class plasmaWordIndexAssortmentCluster {
for (int i = 0; i < clusterCount; i++) sizes[i] = assortments[i].size();
return sizes;
}
public int cacheChunkSizeAvg() {
int total = 0;
for (int i = 0; i < clusterCount; i++) total += assortments[i].cacheChunkSize();
return total / clusterCount;
}
public int[] cacheFillStatusCml() {
int[] a, cml = new int[]{0, 0, 0, 0};
for (int i = 0; i < clusterCount; i++) {
a = assortments[i].cacheFillStatus();
for (int j = 0; j < 4; j++) cml[j] += a[j];
}
return cml;
}
public void close() {
for (int i = 0; i < clusterCount; i++) assortments[i].close();
}

8
source/de/anomic/plasma/plasmaWordIndexCache.java
View File

@ -265,6 +265,14 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
return assortmentCluster.sizes();
}
public int assortmentsCacheChunkSizeAvg() {
return assortmentCluster.cacheChunkSizeAvg();
}
public int[] assortmentsCacheFillStatusCml() {
return assortmentCluster.cacheFillStatusCml();
}
public int size() {
return java.lang.Math.max(assortmentCluster.sizeTotal(), java.lang.Math.max(backend.size(), cache.size()));
}

12
source/de/anomic/yacy/yacyCore.java
View File

@ -153,20 +153,20 @@ public class yacyCore {
yacyDBPath = new File(sb.getRootPath(), sb.getConfig("yacyDB", "DATA/YACYDB"));
if (!(yacyDBPath.exists())) yacyDBPath.mkdir();
// read memory amount
int mem = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024;
log.logConfig("DHT Cache memory = " + mem + " KB");
// create or init seed cache
int memDHT = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024;
log.logConfig("DHT Cache memory = " + memDHT + " KB");
seedDB = new yacySeedDB(
sb,
new File(yacyDBPath, "seed.new.db"),
new File(yacyDBPath, "seed.old.db"),
new File(yacyDBPath, "seed.pot.db"),
mem);
memDHT);
// create or init news database
newsPool = new yacyNewsPool(yacyDBPath, 1024);
int memNews = Integer.parseInt(switchboard.getConfig("ramCacheNews", "1024")) / 1024;
log.logConfig("News Cache memory = " + memNews + " KB");
newsPool = new yacyNewsPool(yacyDBPath, memNews);
loadSeedUploadMethods();

8
source/de/anomic/yacy/yacyNewsDB.java
View File

@ -89,6 +89,14 @@ public class yacyNewsDB {
news = createDB(path, bufferkb);
}
public int dbCacheChunkSize() {
return news.cacheChunkSize();
}
public int[] dbCacheFillStatus() {
return news.cacheFillStatus();
}
public void close() {
if (news != null) try {news.close();} catch (IOException e) {}
news = null;

12
source/de/anomic/yacy/yacyNewsPool.java
View File

@ -99,6 +99,18 @@ public class yacyNewsPool {
maxDistribution = 30;
}
public int dbSize() {
return newsDB.size();
}
public int dbCacheChunkSize() {
return newsDB.dbCacheChunkSize();
}
public int[] dbCacheFillStatus() {
return newsDB.dbCacheFillStatus();
}
public void publishMyNews(yacyNewsRecord record) throws IOException {
// this shall be called if our peer generated a new news record and wants to publish it
if (newsDB.get(record.id()) == null) {

15
source/de/anomic/yacy/yacySeedDB.java
View File

@ -147,10 +147,21 @@ public final class yacySeedDB {
} catch (IOException e) {}
}
public int dbCacheChunkSize() {
return (seedActiveDB.cacheChunkSize() + seedPassiveDB.cacheChunkSize() + seedPotentialDB.cacheChunkSize()) / 3;
}
public int[] dbCacheFillStatus() {
int[] ac = seedActiveDB.cacheFillStatus();
int[] pa = seedPassiveDB.cacheFillStatus();
int[] po = seedPotentialDB.cacheFillStatus();
return new int[]{ac[0] + pa[0] + po[0], ac[1] + pa[1] + po[1], ac[2] + pa[2] + po[2], ac[3] + pa[3] + po[3]};
}
private synchronized kelondroMap openSeedTable(File seedDBFile) throws IOException {
if (seedDBFile.exists()) try {
// open existing seed database
return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB * 0x400), sortFields, accFields);
return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB/3 * 0x400), sortFields, accFields);
} catch (kelondroException e) {
// if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete();
@ -160,7 +171,7 @@ public final class yacySeedDB {
}
// create new seed database
new File(seedDBFile.getParent()).mkdir();
return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB * 0x400, commonHashLength, 480), sortFields, accFields);
return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB/3 * 0x400, commonHashLength, 480), sortFields, accFields);
}
private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {

14
source/yacy.java
View File

@ -166,8 +166,9 @@ public final class yacy {
* the main threads.
*
* @param homePath Root-path where all information is to be found.
* @param startupFree free memory at startup time, to be used later for statistics
*/
private static void startup(String homePath) {
private static void startup(String homePath, long startupFree) {
long startup = yacyCore.universalTime();
try {
// start up
@ -201,6 +202,9 @@ public final class yacy {
plasmaSwitchboard sb = new plasmaSwitchboard(homePath, "yacy.init", "DATA/SETTINGS/httpProxy.conf");
// save information about available memory at startup time
sb.setConfig("memoryFreeAfterStartup", startupFree); // for statistics and memory configuration
// hardcoded, forced, temporary value-migration
sb.setConfig("htTemplatePath", "htroot/env/templates");
sb.setConfig("parseableExt", "html,htm,txt,php,shtml,asp");
@ -413,6 +417,9 @@ public final class yacy {
Runtime run = Runtime.getRuntime();
run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb));
// save information about available memory after all initializations
System.gc(); sb.setConfig("memoryFreeAfterInit", Runtime.getRuntime().freeMemory());
// wait for server shutdown
try {
sb.waitForShutdown();
@ -766,13 +773,14 @@ public final class yacy {
* @param args Given arguments from the command line.
*/
public static void main(String args[]) {
System.gc(); long startupFree = Runtime.getRuntime().freeMemory();
String applicationRoot = System.getProperty("user.dir").replace('\\', '/');
//System.out.println("args.length=" + args.length);
//System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
if ((args.length >= 1) && ((args[0].equals("-startup")) || (args[0].equals("-start")))) {
// normal start-up of yacy
if (args.length == 2) applicationRoot= args[1];
startup(applicationRoot);
startup(applicationRoot, startupFree);
} else if ((args.length >= 1) && ((args[0].equals("-shutdown")) || (args[0].equals("-stop")))) {
// normal shutdown of yacy
if (args.length == 2) applicationRoot= args[1];
@ -800,7 +808,7 @@ public final class yacy {
cleanwordlist(args[1], minlength, maxlength);
} else {
if (args.length == 1) applicationRoot= args[0];
startup(applicationRoot);
startup(applicationRoot, startupFree);
}
}
}

3
yacy.init
View File

@ -461,6 +461,9 @@ ramCacheMessage = 8192
# ram cache for wiki.db
ramCacheWiki = 8192
# ram cache for news1.db
ramCacheNews = 8192
# performance properties for the word index cache
# wordCacheMax is the number of word indexes that shall be held in the
# ram cache during indexing. When YaCy is shut down, this cache must be