Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

simplified DHT classes

Browse Source
This commit is contained in:
Michael Peter Christen 2012-09-24 01:04:39 +02:00
parent a053b356ee
commit e57bf2ca39
11 changed files with 224 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
htroot/yacy/transferRWI.java
View File

@ -35,7 +35,7 @@ import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.yacy.dht.HorizontalPartition;
import net.yacy.cora.services.federated.yacy.Distribution;
import net.yacy.cora.storage.HandleSet;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.WordReferenceRow;
@ -228,7 +228,7 @@ public final class transferRWI {
} else {
final String firstHash = wordhashes.get(0);
final String lastHash = wordhashes.get(wordhashes.size() - 1);
final long avdist = (HorizontalPartition.std.dhtDistance(firstHash.getBytes(), null, ASCII.getBytes(sb.peers.mySeed().hash)) + HorizontalPartition.std.dhtDistance(lastHash.getBytes(), null, ASCII.getBytes(sb.peers.mySeed().hash))) / 2;
final long avdist = (Distribution.horizontalDHTDistance(firstHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash)) + Distribution.horizontalDHTDistance(lastHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash))) / 2;
sb.getLog().logInfo("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + receivedURL + " URLs from " + otherPeerName);
EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + receivedURL + " URLs from " + otherPeerName, "", otherPeer.hash));
}

193
source/net/yacy/cora/services/federated/yacy/Distribution.java Normal file
View File

@ -0,0 +1,193 @@
/**
* VerticalPartition
* Copyright 2009 by Michael Peter Christen
* First released 28.01.2009 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.services.federated.yacy;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.order.Base64Order;
/**
* calculate the DHT position for horizontal and vertical performance scaling:
* horizontal: scale with number of words
* vertical: scale with number of references for every word
* The vertical scaling is selected using the corresponding reference hash, the url hash
* This has the effect that every vertical position accumulates references for the same url
* and the urls are not spread over all positions of the DHT. To use this effect, the
* horizontal DHT position must be normed to a 'rest' value of a partition size.
* @param wordHash, the hash of the RWI
* @param urlHash, the hash of a reference
* @return a double in the range 0 .. 1.0 (including 0, excluding 1.0), the DHT position
*/
public class Distribution {
private final int verticalPartitionExponent;
private final int shiftLength;
private final int partitionCount;
private final long partitionSize;
private final long partitionMask;
/**
*
* @param verticalPartitionExponent, the number of partitions should be computed with partitions = 2**n, n = scaling factor
*/
public Distribution(int verticalPartitionExponent) {
assert verticalPartitionExponent > 0;
this.verticalPartitionExponent = verticalPartitionExponent;
this.partitionCount = 1 << verticalPartitionExponent;
this.shiftLength = Long.SIZE - 1 - this.verticalPartitionExponent;
this.partitionSize = 1L << this.shiftLength;
this.partitionMask = (1L << shiftLength) - 1L;
}
public int verticalPartitions() {
return 1 << verticalPartitionExponent;
}
public final static long horizontalDHTPosition(byte[] wordHash) {
assert wordHash != null;
return Base64Order.enhancedCoder.cardinal(wordHash);
}
public final static long horizontalDHTDistance(final byte[] from, final byte[] to) {
// the dht distance is a positive value between 0 and 1
// if the distance is small, the word more probably belongs to the peer
assert to != null;
assert from != null;
final long toPos = horizontalDHTPosition(to);
final long fromPos = horizontalDHTPosition(from);
return horizontalDHTDistance(fromPos, toPos);
}
public final static long horizontalDHTDistance(final long fromPos, final long toPos) {
return (toPos >= fromPos) ? toPos - fromPos : (Long.MAX_VALUE - fromPos) + toPos + 1;
}
public final static byte[] positionToHash(final long l) {
// transform the position of a peer position into a close peer hash
byte[] h = Base64Order.enhancedCoder.uncardinal(l);
assert h.length == 12;
return h;
}
public final long verticalDHTPosition(final byte[] wordHash, final String urlHash) {
// this creates 1^^e different positions for the same word hash (according to url hash)
assert wordHash != null;
assert urlHash != null;
if (urlHash == null || verticalPartitionExponent < 1) return Distribution.horizontalDHTPosition(wordHash);
// the partition size is (Long.MAX + 1) / 2 ** e == 2 ** (63 - e)
// compute the position using a specific fragment of the word hash and the url hash:
// - from the word hash take the 63 - <partitionExponent> lower bits
// - from the url hash take the <partitionExponent> higher bits
// in case that the partitionExpoent is 1, only one bit is taken from the urlHash,
// which means that the partition is in two parts.
// With partitionExponent = 2 it is divided in four parts and so on.
return (Distribution.horizontalDHTPosition(wordHash) & partitionMask) | (Distribution.horizontalDHTPosition(ASCII.getBytes(urlHash)) & ~partitionMask);
}
public final long verticalDHTPosition(final byte[] wordHash, final int verticalPosition) {
assert wordHash != null;
assert wordHash[2] != '@';
long verticalMask = ((long) verticalPosition) << this.shiftLength; // don't remove the cast! it will become an integer result which is wrong.
return (Distribution.horizontalDHTPosition(wordHash) & partitionMask) | verticalMask;
}
public final int verticalDHTPosition(final byte[] urlHash) {
assert urlHash != null;
return (int) (Distribution.horizontalDHTPosition(urlHash) >> this.shiftLength); // take only the top-<partitionExponent> bits
}
/**
* compute all vertical DHT positions for a given word
* This is used when a word is searched and the peers holding the word must be computed
* @param wordHash, the hash of the word
* @param partitions, the number of partitions of the DHT
* @return a vector of long values, the possible DHT positions
*/
public final long[] verticalDHTPositions(final byte[] wordHash) {
assert wordHash != null;
long[] l = new long[this.partitionCount];
l[0] = Distribution.horizontalDHTPosition(wordHash) & (partitionSize - 1L); // this is the lowest possible position
for (int i = 1; i < this.partitionCount; i++) {
l[i] = l[i - 1] + partitionSize; // no overflow, because we started with the lowest
}
return l;
}
/*
public static void main(String[] args) {
long c1 = Base64Order.enhancedCoder.cardinal("AAAAAAAAAAAA".getBytes());
System.out.println(ASCII.String(Base64Order.enhancedCoder.uncardinal(c1)));
long c2 = Base64Order.enhancedCoder.cardinal("____________".getBytes());
System.out.println(ASCII.String(Base64Order.enhancedCoder.uncardinal(c2)));
Random r = new Random(System.currentTimeMillis());
for (int i = 0; i < 10000; i++) {
long l = r.nextLong();
byte[] h = positionToHash(l);
if (l != Base64Order.enhancedCoder.cardinal(h)) System.out.println(l);
}
}
*/
public static void main(String[] args) {
// java -classpath classes de.anomic.yacy.yacySeed hHJBztzcFn76
// java -classpath classes de.anomic.yacy.yacySeed hHJBztzcFG76 M8hgtrHG6g12 3
// test the DHT position calculation
String wordHash = "hHJBztzcFn76";
//double dhtd;
long dhtl;
int partitionExponent = 0;
Distribution partition = new Distribution(0);
if (args.length == 3) {
// the horizontal and vertical position calculation
String urlHash = args[1];
partitionExponent = Integer.parseInt(args[2]);
dhtl = partition.verticalDHTPosition(UTF8.getBytes(wordHash), urlHash);
} else {
// only a horizontal position calculation
dhtl = Distribution.horizontalDHTPosition(UTF8.getBytes(wordHash));
}
//System.out.println("DHT Double = " + dhtd);
System.out.println("DHT Long = " + dhtl);
System.out.println("DHT as Double from Long = " + ((double) dhtl) / ((double) Long.MAX_VALUE));
//System.out.println("DHT as Long from Double = " + (long) (Long.MAX_VALUE * dhtd));
//System.out.println("DHT as b64 from Double = " + positionToHash(dhtd));
System.out.println("DHT as b64 from Long = " + ASCII.String(Distribution.positionToHash(dhtl)));
System.out.print("all " + (1 << partitionExponent) + " DHT positions from doubles: ");
/*
double[] d = dhtPositionsDouble(wordHash, partitionExponent);
for (int i = 0; i < d.length; i++) {
if (i > 0) System.out.print(", ");
System.out.print(positionToHash(d[i]));
}
System.out.println();
*/
System.out.print("all " + (1 << partitionExponent) + " DHT positions from long : ");
long[] l = partition.verticalDHTPositions(UTF8.getBytes(wordHash));
for (int i = 0; i < l.length; i++) {
if (i > 0) System.out.print(", ");
System.out.print(ASCII.String(Distribution.positionToHash(l[i])));
}
System.out.println();
}
}

93
source/net/yacy/cora/services/federated/yacy/dht/HorizontalPartition.java
View File

@ -1,93 +0,0 @@
/**
* HorizontalPartition
* Copyright 2009 by Michael Peter Christen
* First released 28.01.2009 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.services.federated.yacy.dht;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.order.Base64Order;
/**
* A flat word partition scheme is a metric for words on the range of a distributed
* hash table. The dht is reflected by a 0..Long.MAX_VALUE integer range, each word gets
* a number on that range. To compute a number, the hash representation is used to compute
* the hash position from the first 63 bits of the b64 hash string.
*/
public class HorizontalPartition implements Partition {
public static final HorizontalPartition std = new HorizontalPartition();
public HorizontalPartition() {
// nothing to initialize
}
@Override
public int verticalPartitions() {
return 1;
}
@Override
public long dhtPosition(byte[] wordHash, String urlHash) {
// the urlHash has no relevance here
// normalized to Long.MAX_VALUE
return Base64Order.enhancedCoder.cardinal(wordHash);
}
public final long dhtDistance(final byte[] from, final String urlHash, final byte[] to) {
// the dht distance is a positive value between 0 and 1
// if the distance is small, the word more probably belongs to the peer
assert to != null;
assert from != null;
final long toPos = dhtPosition(to, null);
final long fromPos = dhtPosition(from, urlHash);
return dhtDistance(fromPos, toPos);
}
@Override
public long dhtPosition(byte[] wordHash, int verticalPosition) {
return dhtPosition(wordHash, null);
}
@Override
public long[] dhtPositions(byte[] wordHash) {
long[] l = new long[1];
l[1] = dhtPosition(wordHash, null);
return l;
}
@Override
public int verticalPosition(byte[] urlHash) {
return 0; // this is not a method stub, this is actually true for all FlatWordPartitionScheme
}
public final static long dhtDistance(final long fromPos, final long toPos) {
return (toPos >= fromPos) ?
toPos - fromPos :
(Long.MAX_VALUE - fromPos) + toPos + 1;
}
public static byte[] positionToHash(final long l) {
// transform the position of a peer position into a close peer hash
String s = ASCII.String(Base64Order.enhancedCoder.uncardinal(l));
while (s.length() < 12) s += "A";
return ASCII.getBytes(s);
}
}

65
source/net/yacy/cora/services/federated/yacy/dht/Partition.java
View File

@ -1,65 +0,0 @@
/**
* Partition
* Copyright 2009 by Michael Peter Christen
* First released 28.01.2009 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.services.federated.yacy.dht;
/**
* A PartitionScheme is a calculation of index storage positions in the network of peers.
* Depending on the size of the network and the requirements of use cases the network
* may be defined in different ways. A search network can be constructed in two different ways:
* * partition by document
* * partition by word
* Both schemes have different advances and disadvantages:
* A partition by document has the following properties:
* + all documents are distinct from each other, no double occurrences of documents and document infos
* + good performance scaling up to about 100 peers
* + no index distribution necessary
* - not good scaling for a very large number of peers (1000+ servers cannot be requested simulanously)
* A partition by word has the following properties:
* + can scale for a very large number of peers
* + can almost unlimited scale in number of documents because the number of peers may scale to really large numbers
* + in case of a search only a very small number of peers must be asked
* - double occurrences of documents cannot be avoided very easy
* - index distribution is necessary and is IO and bandwidth-intensive
* - a search request does not scale good, because so many references are accumulated at a single peer
* The partition by word may be enhanced: a vertical scaling provides a better scaling for search,
* but increases further the complexity of the distribution process.
*
* In YaCy we implement a word partition with vertical scaling. To organize the complexity of the
* index distribution and peers selection in case of a search request, this interface is needed to
* implement different distribution schemes.
*
* @author Michael Christen
*
*/
public interface Partition {
public int verticalPartitions();
public long dhtPosition(final byte[] wordHash, final String urlHash);
public long dhtPosition(final byte[] wordHash, final int verticalPosition);
public int verticalPosition(final byte[] urlHash);
public long[] dhtPositions(final byte[] wordHash);
}

149
source/net/yacy/cora/services/federated/yacy/dht/VerticalPartition.java
View File

@ -1,149 +0,0 @@
/**
* VerticalPartition
* Copyright 2009 by Michael Peter Christen
* First released 28.01.2009 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.services.federated.yacy.dht;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
public class VerticalPartition implements Partition {
int partitionExponent;
public VerticalPartition(int partitionExponent) {
this.partitionExponent = partitionExponent;
}
public int verticalPartitions() {
return 1 << partitionExponent;
}
/**
* calculate the DHT position for horizontal and vertical performance scaling:
* horizontal: scale with number of words
* vertical: scale with number of references for every word
* The vertical scaling is selected using the corresponding reference hash, the url hash
* This has the effect that every vertical position accumulates references for the same url
* and the urls are not spread over all positions of the DHT. To use this effect, the
* horizontal DHT position must be normed to a 'rest' value of a partition size
* This method is compatible to the classic DHT computation as always one of the vertical
* DHT position corresponds to the classic horizontal position.
* @param wordHash, the hash of the RWI
* @param partitions, the number of partitions should be computed with partitions = 2**n, n = scaling factor
* @param urlHash, the hash of a reference
* @return a double in the range 0 .. 1.0 (including 0, excluding 1.0), the DHT position
*/
public final long dhtPosition(final byte[] wordHash, final String urlHash) {
// this creates 1^^e different positions for the same word hash (according to url hash)
assert wordHash != null;
assert urlHash != null;
if (urlHash == null || partitionExponent < 1) return HorizontalPartition.std.dhtPosition(wordHash, null);
// the partition size is (Long.MAX + 1) / 2 ** e == 2 ** (63 - e)
assert partitionExponent > 0;
long partitionMask = (1L << (Long.SIZE - 1 - partitionExponent)) - 1L;
// compute the position using a specific fragment of the word hash and the url hash:
// - from the word hash take the 63 - <partitionExponent> lower bits
// - from the url hash take the <partitionExponent> higher bits
// in case that the partitionExpoent is 1, only one bit is taken from the urlHash,
// which means that the partition is in two parts.
// With partitionExponent = 2 it is divided in four parts and so on.
return (HorizontalPartition.std.dhtPosition(wordHash, null) & partitionMask) | (HorizontalPartition.std.dhtPosition(ASCII.getBytes(urlHash), null) & ~partitionMask);
}
public final long dhtPosition(final byte[] wordHash, final int verticalPosition) {
assert wordHash != null;
assert wordHash[2] != '@';
if (partitionExponent == 0) return HorizontalPartition.std.dhtPosition(wordHash, null);
long partitionMask = (1L << (Long.SIZE - 1 - partitionExponent)) - 1L;
long verticalMask = ((long) verticalPosition) << (Long.SIZE - 1 - partitionExponent); // don't remove the cast! it will become an integer result which is wrong.
return (HorizontalPartition.std.dhtPosition(wordHash, null) & partitionMask) | verticalMask;
}
public final int verticalPosition(final byte[] urlHash) {
assert urlHash != null;
if (urlHash == null || partitionExponent < 1) return 0;
assert partitionExponent > 0;
return (int) (HorizontalPartition.std.dhtPosition(urlHash, null) >> (Long.SIZE - 1 - partitionExponent)); // take only the top-<partitionExponent> bits
}
/**
* compute all vertical DHT positions for a given word
* This is used when a word is searched and the peers holding the word must be computed
* @param wordHash, the hash of the word
* @param partitions, the number of partitions of the DHT
* @return a vector of long values, the possible DHT positions
*/
public final long[] dhtPositions(final byte[] wordHash) {
assert wordHash != null;
int partitions = 1 << partitionExponent;
long[] l = new long[partitions];
long partitionSize = 1L << (Long.SIZE - 1 - partitionExponent);
l[0] = HorizontalPartition.std.dhtPosition(wordHash, null) & (partitionSize - 1L); // this is the lowest possible position
for (int i = 1; i < partitions; i++) {
l[i] = l[i - 1] + partitionSize; // no overflow, because we started with the lowest
}
return l;
}
public static void main(String[] args) {
// java -classpath classes de.anomic.yacy.yacySeed hHJBztzcFn76
// java -classpath classes de.anomic.yacy.yacySeed hHJBztzcFG76 M8hgtrHG6g12 3
// test the DHT position calculation
String wordHash = args[0];
//double dhtd;
long dhtl;
int partitionExponent = 0;
VerticalPartition partition = new VerticalPartition(0);
if (args.length == 3) {
// the horizontal and vertical position calculation
String urlHash = args[1];
partitionExponent = Integer.parseInt(args[2]);
dhtl = partition.dhtPosition(UTF8.getBytes(wordHash), urlHash);
} else {
// only a horizontal position calculation
dhtl = HorizontalPartition.std.dhtPosition(UTF8.getBytes(wordHash), null);
}
//System.out.println("DHT Double = " + dhtd);
System.out.println("DHT Long = " + dhtl);
System.out.println("DHT as Double from Long = " + ((double) dhtl) / ((double) Long.MAX_VALUE));
//System.out.println("DHT as Long from Double = " + (long) (Long.MAX_VALUE * dhtd));
//System.out.println("DHT as b64 from Double = " + positionToHash(dhtd));
System.out.println("DHT as b64 from Long = " + HorizontalPartition.positionToHash(dhtl));
System.out.print("all " + (1 << partitionExponent) + " DHT positions from doubles: ");
/*
double[] d = dhtPositionsDouble(wordHash, partitionExponent);
for (int i = 0; i < d.length; i++) {
if (i > 0) System.out.print(", ");
System.out.print(positionToHash(d[i]));
}
System.out.println();
*/
System.out.print("all " + (1 << partitionExponent) + " DHT positions from long : ");
long[] l = partition.dhtPositions(UTF8.getBytes(wordHash));
for (int i = 0; i < l.length; i++) {
if (i > 0) System.out.print(", ");
System.out.print(HorizontalPartition.positionToHash(l[i]));
}
System.out.println();
}
}

6
source/net/yacy/peers/DHTSelection.java
View File

@ -37,7 +37,7 @@ import java.util.SortedMap;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.order.Digest;
import net.yacy.cora.services.federated.yacy.dht.HorizontalPartition;
import net.yacy.cora.services.federated.yacy.Distribution;
import net.yacy.cora.storage.HandleSet;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.data.word.Word;
@ -197,10 +197,10 @@ public class DHTSelection {
int redundancy,
Map<String, Seed> regularSeeds) {
// this method is called from the search target computation
final long[] dhtVerticalTargets = seedDB.scheme.dhtPositions(wordhash);
final long[] dhtVerticalTargets = seedDB.scheme.verticalDHTPositions(wordhash);
Seed seed;
for (long dhtVerticalTarget : dhtVerticalTargets) {
wordhash = HorizontalPartition.positionToHash(dhtVerticalTarget);
wordhash = Distribution.positionToHash(dhtVerticalTarget);
Iterator<Seed> dhtEnum = getAcceptRemoteIndexSeeds(seedDB, wordhash, redundancy, false);
int c = Math.min(seedDB.sizeConnected(), redundancy);
int cc = 2; // select a maximum of 3, this is enough redundancy

6
source/net/yacy/peers/Dispatcher.java
View File

@ -34,7 +34,7 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.services.federated.yacy.dht.HorizontalPartition;
import net.yacy.cora.services.federated.yacy.Distribution;
import net.yacy.cora.storage.HandleSet;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.data.meta.URIMetadataRow;
@ -251,7 +251,7 @@ public class Dispatcher {
while (i.hasNext()) {
re = i.next();
if (re == null) continue;
partitionBuffer[this.seeds.scheme.verticalPosition(re.urlhash())].add(re);
partitionBuffer[this.seeds.scheme.verticalDHTPosition(re.urlhash())].add(re);
}
// add the containers to the result vector
@ -281,7 +281,7 @@ public class Dispatcher {
for (int vertical = 0; vertical < containers.length; vertical++) {
// the 'new' primary target is the word hash of the last container in the array
lastContainer = containers[vertical].get(containers[vertical].size() - 1);
primaryTarget = HorizontalPartition.positionToHash(this.seeds.scheme.dhtPosition(lastContainer.getTermHash(), vertical));
primaryTarget = Distribution.positionToHash(this.seeds.scheme.verticalDHTPosition(lastContainer.getTermHash(), vertical));
assert primaryTarget[2] != '@';
pTArray = new ByteArray(primaryTarget);

26
source/net/yacy/peers/Seed.java
View File

@ -65,7 +65,7 @@ import net.yacy.cora.document.UTF8;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.order.Digest;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.services.federated.yacy.dht.HorizontalPartition;
import net.yacy.cora.services.federated.yacy.Distribution;
import net.yacy.cora.sorting.ClusteredScoreMap;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.cora.storage.HandleSet;
@ -859,14 +859,12 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
}
// find dht position and size of gap
final long left =
HorizontalPartition.std.dhtPosition(ASCII.getBytes(interval.substring(0, 12)), null);
final long right =
HorizontalPartition.std.dhtPosition(ASCII.getBytes(interval.substring(12)), null);
final long gap8 = HorizontalPartition.dhtDistance(left, right) >> 3; // 1/8 of a gap
final long left = Distribution.horizontalDHTPosition(ASCII.getBytes(interval.substring(0, 12)));
final long right = Distribution.horizontalDHTPosition(ASCII.getBytes(interval.substring(12)));
final long gap8 = Distribution.horizontalDHTDistance(left, right) >> 3; // 1/8 of a gap
final long gapx = gap8 + (Math.abs(random.nextLong()) % (6 * gap8));
final long gappos = (Long.MAX_VALUE - left >= gapx) ? left + gapx : (left - Long.MAX_VALUE) + gapx;
final byte[] computedHash = HorizontalPartition.positionToHash(gappos);
final byte[] computedHash = Distribution.positionToHash(gappos);
// the computed hash is the perfect position (modulo gap4 population and gap alternatives)
// this is too tight. The hash must be more randomized. We take only (!) the first two bytes
// of the computed hash and add random bytes at the remaining positions. The first two bytes
@ -904,19 +902,17 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
first = s0;
continue;
}
l =
HorizontalPartition.dhtDistance(
HorizontalPartition.std.dhtPosition(ASCII.getBytes(s0.hash), null),
HorizontalPartition.std.dhtPosition(ASCII.getBytes(s1.hash), null));
l = Distribution.horizontalDHTDistance(
Distribution.horizontalDHTPosition(ASCII.getBytes(s0.hash)),
Distribution.horizontalDHTPosition(ASCII.getBytes(s1.hash)));
gaps.put(l, s0.hash + s1.hash);
s0 = s1;
}
// compute also the last gap
if ( (first != null) && (s0 != null) ) {
l =
HorizontalPartition.dhtDistance(
HorizontalPartition.std.dhtPosition(ASCII.getBytes(s0.hash), null),
HorizontalPartition.std.dhtPosition(ASCII.getBytes(first.hash), null));
l = Distribution.horizontalDHTDistance(
Distribution.horizontalDHTPosition(ASCII.getBytes(s0.hash)),
Distribution.horizontalDHTPosition(ASCII.getBytes(first.hash)));
gaps.put(l, s0.hash + first.hash);
}
return gaps;

9
source/net/yacy/peers/SeedDB.java
View File

@ -48,8 +48,7 @@ import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.services.federated.yacy.dht.Partition;
import net.yacy.cora.services.federated.yacy.dht.VerticalPartition;
import net.yacy.cora.services.federated.yacy.Distribution;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.blob.MapDataMining;
import net.yacy.kelondro.data.meta.DigestURI;
@ -93,7 +92,7 @@ public final class SeedDB implements AlternativeDomainNames {
public NewsPool newsPool;
private int netRedundancy;
public Partition scheme;
public Distribution scheme;
private Seed mySeed; // my own seed
private final Set<String> myBotIDs; // list of id's that this bot accepts as robots.txt identification
@ -118,7 +117,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.myBotIDs.add("yacybot");
this.myBotIDs.add("yacyproxy");
this.netRedundancy = redundancy;
this.scheme = new VerticalPartition(partitionExponent);
this.scheme = new Distribution(partitionExponent);
// set up seed database
this.seedActiveDB = openSeedTable(this.seedActiveDBFile);
@ -166,7 +165,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.myOwnSeedFile = new File(newNetworkRoot, SeedDB.DBFILE_OWN_SEED);
this.netRedundancy = redundancy;
this.scheme = new VerticalPartition(partitionExponent);
this.scheme = new Distribution(partitionExponent);
// set up seed database
this.seedActiveDB = openSeedTable(this.seedActiveDBFile);

12
source/net/yacy/peers/graphics/NetworkGraph.java
View File

@ -37,7 +37,7 @@ import java.util.List;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.Hit;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.services.federated.yacy.dht.HorizontalPartition;
import net.yacy.cora.services.federated.yacy.Distribution;
import net.yacy.kelondro.logging.Log;
import net.yacy.peers.EventChannel;
import net.yacy.peers.RemoteSearch;
@ -139,7 +139,7 @@ public class NetworkGraph {
for (final RemoteSearch primarySearche : primarySearches) {
if (primarySearche == null) continue;
eventPicture.setColor((primarySearche.isAlive()) ? RasterPlotter.RED : RasterPlotter.GREEN);
angle = cyc + (360.0d * ((HorizontalPartition.std.dhtPosition(UTF8.getBytes(primarySearche.target().hash), null)) / DOUBLE_LONG_MAX_VALUE));
angle = cyc + (360.0d * ((Distribution.horizontalDHTPosition(UTF8.getBytes(primarySearche.target().hash))) / DOUBLE_LONG_MAX_VALUE));
eventPicture.arcLine(cx, cy, cr - 20, cr, angle, true, null, null, -1, -1, -1, false);
}
@ -161,7 +161,7 @@ public class NetworkGraph {
final Iterator<byte[]> i = query.query_include_hashes.iterator();
eventPicture.setColor(RasterPlotter.GREY);
while (i.hasNext()) {
final long[] positions = seedDB.scheme.dhtPositions(i.next());
final long[] positions = seedDB.scheme.verticalDHTPositions(i.next());
for (final long position : positions) {
angle = cyc + (360.0d * ((position) / DOUBLE_LONG_MAX_VALUE));
eventPicture.arcLine(cx, cy, cr - 20, cr, angle, true, null, null, -1, -1, -1, false);
@ -313,8 +313,8 @@ public class NetworkGraph {
}
private static void drawNetworkPictureDHT(final RasterPlotter img, final int centerX, final int centerY, final int innerradius, final Seed mySeed, final Seed otherSeed, final String colorLine, final int coronaangle, final boolean out, final int cyc) {
final int angleMy = cyc + (int) (360.0d * HorizontalPartition.std.dhtPosition(ASCII.getBytes(mySeed.hash), null) / DOUBLE_LONG_MAX_VALUE);
final int angleOther = cyc + (int) (360.0d * HorizontalPartition.std.dhtPosition(ASCII.getBytes(otherSeed.hash), null) / DOUBLE_LONG_MAX_VALUE);
final int angleMy = cyc + (int) (360.0d * Distribution.horizontalDHTPosition(ASCII.getBytes(mySeed.hash)) / DOUBLE_LONG_MAX_VALUE);
final int angleOther = cyc + (int) (360.0d * Distribution.horizontalDHTPosition(ASCII.getBytes(otherSeed.hash)) / DOUBLE_LONG_MAX_VALUE);
// draw line
img.arcLine(centerX, centerY, innerradius, innerradius - 20, angleMy, !out,
colorLine, null, 12, (coronaangle < 0) ? -1 : coronaangle / 30, 2, true);
@ -355,7 +355,7 @@ public class NetworkGraph {
final String name = this.seed.getName().toUpperCase() /*+ ":" + seed.hash + ":" + (((double) ((int) (100 * (((double) yacySeed.dhtPosition(seed.hash)) / ((double) yacySeed.maxDHTDistance))))) / 100.0)*/;
if (name.length() < shortestName) shortestName = name.length();
if (name.length() > longestName) longestName = name.length();
final double angle = this.cyc + (360.0d * HorizontalPartition.std.dhtPosition(ASCII.getBytes(this.seed.hash), null) / DOUBLE_LONG_MAX_VALUE);
final double angle = this.cyc + (360.0d * Distribution.horizontalDHTPosition(ASCII.getBytes(this.seed.hash)) / DOUBLE_LONG_MAX_VALUE);
//System.out.println("Seed " + seed.hash + " has distance " + seed.dhtDistance() + ", angle = " + angle);
int linelength = 20 + this.outerradius * (20 * (name.length() - shortestName) / (longestName - shortestName) + Math.abs(this.seed.hash.hashCode() % 20)) / 80;
if (linelength > this.outerradius) linelength = this.outerradius;

4
source/net/yacy/search/query/SearchEvent.java
View File

@ -43,7 +43,7 @@ import java.util.concurrent.TimeUnit;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.services.federated.yacy.dht.HorizontalPartition;
import net.yacy.cora.services.federated.yacy.Distribution;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.data.WorkTables;
import net.yacy.document.LargeNumberCache;
@ -232,7 +232,7 @@ public final class SearchEvent {
this.IAmaxcounthash = wordhash;
maxcount = container.size();
}
l = HorizontalPartition.std.dhtDistance(wordhash, null, ASCII.getBytes(peers.mySeed().hash));
l = Distribution.horizontalDHTDistance(wordhash, ASCII.getBytes(peers.mySeed().hash));
if ( l < mindhtdistance ) {
// calculate the word hash that is closest to our dht position
mindhtdistance = l;