Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

*) Adding option which allows to do a Index-Transfer without deletion of index

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@722 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
theli 2005-09-14 07:14:24 +00:00
parent 4036ee812a
commit beefddf0e8
3 changed files with 38 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
htroot/IndexTransfer_p.html
View File

@ -16,15 +16,22 @@
<tr class="TableHeader" valign="bottom">
<td class="small" >Status</td>
<td class="small" >Transfered Words</td>
<td class="small" >Words Range</td>
<td class="small" >Chunk Size: Words Range</td>
<td class="small" >Delete Index</td>
<td class="small" >Remote Peer</td>
<td class="small" >Start/Stop Transfer</td>
</tr>
#(running)#
<tr valign="top" class="TableCellLight">
<td class="small">Not running</td>
<td class="small">#[status]#</td>
<td class="small" align="center">-</td>
<td class="small" align="center">-</td>
<td class="small">
<select name="deleteIndex">
<option value="1">true</option>
<option value="0">false</option>
</select>
</td>
<td class="small">
<select name="hostHash">
#{hosts}#
@ -42,6 +49,7 @@
<td class="small"><font color="#(stopped)#green::red#(/stopped)#">#[status]#</font></td>
<td class="small" align="right">#[twcount]# (#[twpercent]#%) </td>
<td class="small" align="rigth">#[twchunk]# words: <tt>#[twrange]#</tt></td>
<td class="small">#(deleteIndex)#false::true#(/deleteIndex)#</td>
<td class="small">#[peerName]#</td>
<td class="small">#(stopped)#<input type="submit" name="stopIndexTransfer" value="Stop Index Transfer">::
<input type="submit" name="newIndexTransfer" value="Start New Index Transfer">#(/stopped)#</td>

22
htroot/IndexTransfer_p.java
View File

@ -67,18 +67,25 @@ public class IndexTransfer_p {
if (post != null) {
if (post.containsKey("startIndexTransfer")) {
yacySeed seed = yacyCore.seedDB.getConnected(post.get("hostHash", ""));
if (seed == null) return prop;
switchboard.indexDistribution.startTransferWholeIndex(seed,true);
yacySeed seed = yacyCore.seedDB.getConnected(post.get("hostHash", ""));
if (seed == null) {
prop.put("running_status","Disconnected peer");
} else {
boolean deleteIndex = ((String)post.get("deleteIndex", "0")).equals("1");
switchboard.indexDistribution.startTransferWholeIndex(seed,deleteIndex);
prop.put("LOCATION","");
return prop;
}
} else if (post.containsKey("stopIndexTransfer")) {
switchboard.indexDistribution.stopTransferWholeIndex(true);
prop.put("LOCATION","");
return prop;
} else if (post.containsKey("newIndexTransfer")) {
switchboard.indexDistribution.abortTransferWholeIndex(true);
prop.put("LOCATION","");
return prop;
}
prop.put("LOCATION","");
return prop;
}
@ -95,8 +102,11 @@ public class IndexTransfer_p {
prop.put("running_twpercent",Float.toString(transfThread.getTransferedIndexPercent()));
prop.put("running_twrange", transfThread.getRange());
prop.put("running_twchunk", Integer.toString(transfThread.getChunkSize()));
prop.put("running_deleteIndex", transfThread.deleteIndex()?1:0);
prop.put("running_peerName",transfThread.getSeed().getName());
prop.put("running_stopped",(transfThread.isFinished()) || (!transfThread.isAlive())?1:0);
} else {
if (!prop.containsKey("running_status")) prop.put("running_status","Not running");
}

16
source/de/anomic/plasma/plasmaWordIndexDistribution.java
View File

@ -464,7 +464,7 @@ public class plasmaWordIndexDistribution {
private boolean delete = false;
private boolean finished = false;
private int transferedIndexCount = 0;
private String status = "running";
private String status = "Running";
private String oldStartingPointHash = "------------", startPointHash = "------------";
private int wordsDBSize = 0;
private int chunkSize = 500;
@ -484,11 +484,15 @@ public class plasmaWordIndexDistribution {
this.finished = true;
this.join();
}
public boolean isFinished() {
return this.finished;
}
public boolean deleteIndex() {
return this.delete;
}
public int getChunkSize() {
return this.chunkSize;
}
@ -526,19 +530,21 @@ public class plasmaWordIndexDistribution {
* - detected a server shutdown or user interruption
* - detected a failure
*/
long start, retryCount = 0;
long start, retryCount = 0, iteration = 0;
while (!finished && !Thread.currentThread().isInterrupted()) {
iteration++;
int idxCount = 0;
start = System.currentTimeMillis();
// selecting 500 words to transfer
this.status = "Running: Selection of chunk " + iteration;
Object[] selectResult = selectTransferIndexes(startPointHash, chunkSize);
plasmaWordIndexEntity[] indexEntities = (plasmaWordIndexEntity[]) selectResult[0];
HashMap urlCache = (HashMap) selectResult[1]; // String (url-hash) / plasmaCrawlLURL.Entry
if ((indexEntities == null) || (indexEntities.length == 0)) {
plasmaWordIndexDistribution.this.log.logFine("No index available for index transfer, hash start-point " + startPointHash);
this.status = "finished.";
this.status = "Finished. " + iteration + " chunks transfered.";
return;
}
// count the indexes again, can be smaller as expected
@ -563,6 +569,7 @@ public class plasmaWordIndexDistribution {
if (isAborted()) return;
// transfering seleted words to remote peer
this.status = "Running: Transfer of chunk " + iteration;
String error = yacyClient.transferIndex(seed, indexEntities, urlCache);
if (error == null) {
// words successfully transfered
@ -625,6 +632,7 @@ public class plasmaWordIndexDistribution {
// deleting transfered words from index
if (delete) {
this.status = "Running: Deletion of chunk " + iteration;
try {
if (deleteTransferIndexes(indexEntities)) {
plasmaWordIndexDistribution.this.log.logFine("Deleted all " + indexEntities.length + " transferred whole-word indexes locally");