mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
- fix for possible NPE in queues_p
- WatchCrawler_p: - display crawler traffic - pause/resume local- and global crawler git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3389 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
9702d3abba
commit
bf7a69197d
|
@ -16,26 +16,39 @@
|
|||
<tr class="TableHeader">
|
||||
<th>Queue</th>
|
||||
<th>Size</th>
|
||||
<th> </th>
|
||||
<th>Max</th>
|
||||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td align="left">Indexing</td>
|
||||
<td align="right"><span id="indexingqueuesize"> </span></td>
|
||||
<td> </td>
|
||||
<td align="right"><span id="indexingqueuemax"> </span></td>
|
||||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td align="left">Loader</td>
|
||||
<td align="right"><span id="loaderqueuesize"> </span></td>
|
||||
<td> </td>
|
||||
<td align="right"><span id="loaderqueuemax"> </span></td>
|
||||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td align="left">Local Crawler</td>
|
||||
<td align="right"><span id="localcrawlerqueuesize"> </span></td>
|
||||
<td>
|
||||
<a href="" id="localcrawlerstateA">
|
||||
<img src="" alt="" style="width:12px; height:12px;" id="localcrawlerstateIMG" />
|
||||
</a>
|
||||
</td>
|
||||
<td align="right">unlimited</td>
|
||||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td align="left">Remote Crawler</td>
|
||||
<td align="right"><span id="remotecrawlerqueuesize"> </span></td>
|
||||
<td>
|
||||
<a href="" title="" id="remotecrawlerstateA">
|
||||
<img src="" alt="" style="width:12px; height:12px;" id="remotecrawlerstateIMG" />
|
||||
</a>
|
||||
</td>
|
||||
<td align="right">unlimited</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
|
@ -69,6 +82,11 @@
|
|||
<td align="left"><span id="ppmNum"> </span></td>
|
||||
<td align="left"><span id="ppmSpan"> </span></td>
|
||||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td align="left">Traffic (Crawler)</td>
|
||||
<td align="left"><span id="trafficCrawler"> </span> MB</td>
|
||||
<td> </td>
|
||||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td align="left">RWI RAM (Word Cache)</td>
|
||||
<td align="left"><span id="wordcacheNum"> </span></td>
|
||||
|
|
|
@ -73,6 +73,26 @@ public class WatchCrawler_p {
|
|||
if (handle != null) switchboard.profiles.removeEntry(handle);
|
||||
}
|
||||
|
||||
if (post.containsKey("continue")) {
|
||||
// continue queue
|
||||
String queue = post.get("continue", "");
|
||||
if (queue.equals("localcrawler")) {
|
||||
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
} else if (queue.equals("remotecrawler")) {
|
||||
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
|
||||
}
|
||||
}
|
||||
|
||||
if (post.containsKey("pause")) {
|
||||
// pause queue
|
||||
String queue = post.get("pause", "");
|
||||
if (queue.equals("localcrawler")) {
|
||||
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
} else if (queue.equals("remotecrawler")) {
|
||||
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
|
||||
}
|
||||
}
|
||||
|
||||
if (post.containsKey("crawlingstart")) {
|
||||
// init crawl
|
||||
if (yacyCore.seedDB == null) {
|
||||
|
@ -301,7 +321,6 @@ public class WatchCrawler_p {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// crawl profiles
|
||||
|
|
|
@ -77,6 +77,13 @@ function handleStatus(){
|
|||
ppmSpan.appendChild(img);
|
||||
}
|
||||
|
||||
// traffic output (no bar up to now)
|
||||
traffic = getFirstChild(statusTag, "traffic");
|
||||
trafficCrawler = getValue(getFirstChild(traffic, "crawler"));
|
||||
trafCrawlerSpan = document.getElementById("trafficCrawler");
|
||||
removeAllChildren(trafCrawlerSpan);
|
||||
trafCrawlerSpan.appendChild(document.createTextNode(Math.round((trafficCrawler) / 1024 / 10.24) / 100));
|
||||
|
||||
var wordCache=getValue(getFirstChild(statusTag, "wordCacheWSize"));
|
||||
var wordCacheMax=getValue(getFirstChild(statusTag, "wordCacheMaxCount"));
|
||||
|
||||
|
@ -133,18 +140,39 @@ function handleQueues(){
|
|||
|
||||
localcrawlerqueue=getFirstChild(xml, "localcrawlerqueue");
|
||||
localcrawlerqueue_size=getValue(getFirstChild(localcrawlerqueue, "size"));
|
||||
localcrawlerqueue_state=getValue(getFirstChild(localcrawlerqueue, "state"));
|
||||
document.getElementById("localcrawlerqueuesize").firstChild.nodeValue=localcrawlerqueue_size;
|
||||
putQueueState("localcrawler", localcrawlerqueue_state);
|
||||
|
||||
updateTable(localcrawlerqueue, "local crawler");
|
||||
|
||||
remotecrawlerqueue=getFirstChild(xml, "remotecrawlerqueue");
|
||||
updateTable(remotecrawlerqueue, "remoteCrawlerTable");
|
||||
|
||||
remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size"));
|
||||
remotecrawlerqueue_state=getValue(getFirstChild(remotecrawlerqueue, "state"));
|
||||
document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size;
|
||||
putQueueState("remotecrawler", remotecrawlerqueue_state);
|
||||
updateTable(remotecrawlerqueue, "remote crawler");
|
||||
}
|
||||
}
|
||||
|
||||
function putQueueState(queue, state) {
|
||||
a = document.getElementById(queue + "stateA");
|
||||
img = document.getElementById(queue + "stateIMG");
|
||||
if (state == "paused") {
|
||||
a.href = "WatchCrawler_p.html?continue=" + queue;
|
||||
a.title = "Continue this queue";
|
||||
img.src = "/env/grafics/start.gif";
|
||||
img.alt = "Continue this queue";
|
||||
} else {
|
||||
a.href = "WatchCrawler_p.html?pause=" + queue;
|
||||
a.title = "Pause this queue";
|
||||
img.src = "/env/grafics/stop.gif";
|
||||
img.alt = "Pause this queue";
|
||||
}
|
||||
}
|
||||
|
||||
function updateTable(indexingqueue, tablename){
|
||||
indexingTable=document.getElementById("queueTable");
|
||||
entries=indexingqueue.getElementsByTagName("entry");
|
||||
|
|
|
@ -66,6 +66,9 @@ import de.anomic.yacy.yacySeed;
|
|||
|
||||
public class queues_p {
|
||||
|
||||
public static final String STATE_RUNNING = "running";
|
||||
public static final String STATE_PAUSED = "paused";
|
||||
|
||||
private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US);
|
||||
private static String daydate(Date date) {
|
||||
if (date == null) return "";
|
||||
|
@ -83,7 +86,7 @@ public class queues_p {
|
|||
yacySeed initiator;
|
||||
|
||||
//indexing queue
|
||||
prop.put("indexingSize", switchboard.getThread("80_indexing").getJobCount()+switchboard.indexingTasksInProcess.size());
|
||||
prop.put("indexingSize", switchboard.getThread(plasmaSwitchboard.INDEXER).getJobCount()+switchboard.indexingTasksInProcess.size());
|
||||
prop.put("indexingMax", plasmaSwitchboard.indexingSlots);
|
||||
prop.put("urlpublictextSize", switchboard.wordIndex.loadedURL.size());
|
||||
prop.put("rwipublictextSize", switchboard.wordIndex.size());
|
||||
|
@ -114,9 +117,9 @@ public class queues_p {
|
|||
for (i = 0; i < size; i++) {
|
||||
boolean inProcess = i < inProcessCount;
|
||||
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
|
||||
long entrySize = pcentry.size();
|
||||
totalSize += entrySize;
|
||||
if ((pcentry != null)&&(pcentry.url() != null)) {
|
||||
long entrySize = pcentry.size();
|
||||
totalSize += entrySize;
|
||||
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
|
||||
prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
|
||||
prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : wikiCode.replaceHTML(initiator.getName())));
|
||||
|
@ -160,13 +163,14 @@ public class queues_p {
|
|||
}
|
||||
|
||||
//local crawl queue
|
||||
prop.put("localCrawlSize", Integer.toString(switchboard.getThread("50_localcrawl").getJobCount()));
|
||||
prop.put("localCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
|
||||
prop.put("localCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
|
||||
int stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
|
||||
addNTable(prop, "list-local", switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
|
||||
|
||||
//global crawl queue
|
||||
prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("61_globalcrawltrigger").getJobCount()));
|
||||
//prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("62_remotetriggeredcrawl").getJobCount()));
|
||||
prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount()));
|
||||
prop.put("remoteCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? STATE_PAUSED : STATE_RUNNING);
|
||||
stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
|
||||
if (stackSize == 0) {
|
||||
prop.put("list-remote", 0);
|
||||
|
|
|
@ -34,6 +34,7 @@
|
|||
</loaderqueue>
|
||||
<localcrawlerqueue>
|
||||
<size>#[localCrawlSize]#</size>
|
||||
<state>#[localCrawlState]#</state>
|
||||
#{list-local}#
|
||||
<entry>
|
||||
<profile>#[profile]#</profile>
|
||||
|
@ -49,6 +50,7 @@
|
|||
</localcrawlerqueue>
|
||||
<remotecrawlerqueue>
|
||||
<size>#[remoteCrawlSize]#</size>
|
||||
<state>#[remoteCrawlState]#</state>
|
||||
#{list-remote}#
|
||||
<entry>
|
||||
<profile>#[profile]#</profile>
|
||||
|
|
Loading…
Reference in New Issue
Block a user