- fix for possible NPE in queues_p

- WatchCrawler_p:
  - display crawler traffic
  - pause/resume local- and global crawler


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3389 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/WatchCrawler_p.html

@@ -16,26 +16,39 @@
           <tr class="TableHeader"> 
             <th>Queue</th>
             <th>Size</th>
+            <th>&nbsp;</th>
             <th>Max</th>
           </tr>
           <tr class="TableCellLight"> 
             <td align="left">Indexing</td>
             <td align="right"><span id="indexingqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+            <td>&nbsp;</td>
             <td align="right"><span id="indexingqueuemax">&nbsp;&nbsp;&nbsp;</span></td>
           </tr>
           <tr class="TableCellLight"> 
             <td align="left">Loader</td>
             <td align="right"><span id="loaderqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+            <td>&nbsp;</td>
             <td align="right"><span id="loaderqueuemax">&nbsp;&nbsp;&nbsp;</span></td>
           </tr>
           <tr class="TableCellLight"> 
             <td align="left">Local Crawler</td>
             <td align="right"><span id="localcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+            <td>
+              <a href="" id="localcrawlerstateA">
+                <img src="" alt="" style="width:12px; height:12px;" id="localcrawlerstateIMG" />
+              </a>
+            </td>
             <td align="right">unlimited</td>
           </tr>
           <tr class="TableCellLight"> 
             <td align="left">Remote Crawler</td>
             <td align="right"><span id="remotecrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+            <td>
+              <a href="" title="" id="remotecrawlerstateA">
+                <img src="" alt="" style="width:12px; height:12px;" id="remotecrawlerstateIMG" />
+              </a>
+            </td>
             <td align="right">unlimited</td>
           </tr>
         </tbody>
@@ -69,6 +82,11 @@
             <td align="left"><span id="ppmNum">&nbsp;&nbsp;&nbsp;</span></td>
             <td align="left"><span id="ppmSpan">&nbsp;&nbsp;&nbsp;</span></td>
           </tr>
+          <tr class="TableCellLight">
+            <td align="left">Traffic (Crawler)</td>
+            <td align="left"><span id="trafficCrawler">&nbsp;&nbsp;&nbsp;</span> MB</td>
+            <td>&nbsp;</td>
+          </tr>
           <tr class="TableCellLight"> 
             <td align="left">RWI RAM (Word Cache)</td>
             <td align="left"><span id="wordcacheNum">&nbsp;</span></td>

htroot/WatchCrawler_p.java

@@ -73,6 +73,26 @@ public class WatchCrawler_p {
                 if (handle != null) switchboard.profiles.removeEntry(handle);
             }
             
+            if (post.containsKey("continue")) {
+                // continue queue
+                String queue = post.get("continue", "");
+                if (queue.equals("localcrawler")) {
+                    switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
+                } else if (queue.equals("remotecrawler")) {
+                    switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
+                }
+            }
+
+            if (post.containsKey("pause")) {
+                // pause queue
+                String queue = post.get("pause", "");
+                if (queue.equals("localcrawler")) {
+                    switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
+                } else if (queue.equals("remotecrawler")) {
+                    switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
+                }
+            }
+            
             if (post.containsKey("crawlingstart")) {
                 // init crawl
                 if (yacyCore.seedDB == null) {
@@ -301,7 +321,6 @@ public class WatchCrawler_p {
                     }
                 }
             }
-
         }
         
         // crawl profiles

htroot/js/WatchCrawler.js

@@ -77,6 +77,13 @@ function handleStatus(){
 		ppmSpan.appendChild(img);
 	}
 	
+	// traffic output (no bar up to now)
+    traffic = getFirstChild(statusTag, "traffic");
+    trafficCrawler = getValue(getFirstChild(traffic, "crawler"));
+    trafCrawlerSpan = document.getElementById("trafficCrawler");
+    removeAllChildren(trafCrawlerSpan);
+	trafCrawlerSpan.appendChild(document.createTextNode(Math.round((trafficCrawler) / 1024 / 10.24) / 100));
+    
 	var wordCache=getValue(getFirstChild(statusTag, "wordCacheWSize"));
 	var wordCacheMax=getValue(getFirstChild(statusTag, "wordCacheMaxCount"));
 
@@ -133,18 +140,39 @@ function handleQueues(){
 		
 		localcrawlerqueue=getFirstChild(xml, "localcrawlerqueue");
 		localcrawlerqueue_size=getValue(getFirstChild(localcrawlerqueue, "size"));
+		localcrawlerqueue_state=getValue(getFirstChild(localcrawlerqueue, "state"));
 		document.getElementById("localcrawlerqueuesize").firstChild.nodeValue=localcrawlerqueue_size;
+		putQueueState("localcrawler", localcrawlerqueue_state);
+		
 		updateTable(localcrawlerqueue, "local crawler");
 		
 		remotecrawlerqueue=getFirstChild(xml, "remotecrawlerqueue");
 		updateTable(remotecrawlerqueue, "remoteCrawlerTable");
 		
 		remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size"));
+		remotecrawlerqueue_state=getValue(getFirstChild(remotecrawlerqueue, "state"));
 		document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size;
+		putQueueState("remotecrawler", remotecrawlerqueue_state);
 		updateTable(remotecrawlerqueue, "remote crawler");
 	}
 }
 
+function putQueueState(queue, state) {
+	a = document.getElementById(queue + "stateA");
+	img = document.getElementById(queue + "stateIMG");
+	if (state == "paused") {
+		a.href = "WatchCrawler_p.html?continue=" + queue;
+		a.title = "Continue this queue";
+		img.src = "/env/grafics/start.gif";
+		img.alt = "Continue this queue";
+	} else {
+		a.href = "WatchCrawler_p.html?pause=" + queue;
+		a.title = "Pause this queue";
+		img.src = "/env/grafics/stop.gif";
+		img.alt = "Pause this queue";
+	}
+}
+
 function updateTable(indexingqueue, tablename){
 	indexingTable=document.getElementById("queueTable");
 	entries=indexingqueue.getElementsByTagName("entry");

htroot/xml/queues_p.java

@@ -66,6 +66,9 @@ import de.anomic.yacy.yacySeed;
 
 public class queues_p {
     
+    public static final String STATE_RUNNING = "running";
+    public static final String STATE_PAUSED = "paused";
+    
     private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US);
     private static String daydate(Date date) {
         if (date == null) return "";
@@ -83,7 +86,7 @@ public class queues_p {
         yacySeed initiator;
         
         //indexing queue
-        prop.put("indexingSize", switchboard.getThread("80_indexing").getJobCount()+switchboard.indexingTasksInProcess.size());
+        prop.put("indexingSize", switchboard.getThread(plasmaSwitchboard.INDEXER).getJobCount()+switchboard.indexingTasksInProcess.size());
         prop.put("indexingMax", plasmaSwitchboard.indexingSlots);
         prop.put("urlpublictextSize", switchboard.wordIndex.loadedURL.size());
         prop.put("rwipublictextSize", switchboard.wordIndex.size());
@@ -114,9 +117,9 @@ public class queues_p {
                 for (i = 0; i < size; i++) {
                     boolean inProcess = i < inProcessCount;
                     pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
-                    long entrySize = pcentry.size();
-                    totalSize += entrySize;
                     if ((pcentry != null)&&(pcentry.url() != null)) {
+                        long entrySize = pcentry.size();
+                        totalSize += entrySize;
                         initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
                         prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
                         prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : wikiCode.replaceHTML(initiator.getName())));
@@ -160,13 +163,14 @@ public class queues_p {
         }
         
         //local crawl queue
-        prop.put("localCrawlSize", Integer.toString(switchboard.getThread("50_localcrawl").getJobCount()));
+        prop.put("localCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
+        prop.put("localCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
         int stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
         addNTable(prop, "list-local", switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
         
         //global crawl queue
-        prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("61_globalcrawltrigger").getJobCount()));
+        prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount()));
-        //prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("62_remotetriggeredcrawl").getJobCount()));
+        prop.put("remoteCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? STATE_PAUSED : STATE_RUNNING);
         stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
         if (stackSize == 0) {
             prop.put("list-remote", 0);

htroot/xml/queues_p.xml

@@ -34,6 +34,7 @@
 </loaderqueue>
 <localcrawlerqueue>
   <size>#[localCrawlSize]#</size>
+  <state>#[localCrawlState]#</state>
 #{list-local}#
   <entry>
     <profile>#[profile]#</profile>
@@ -49,6 +50,7 @@
 </localcrawlerqueue>
 <remotecrawlerqueue>
   <size>#[remoteCrawlSize]#</size>
+  <state>#[remoteCrawlState]#</state>
 #{list-remote}#
   <entry>
     <profile>#[profile]#</profile>