2005-05-07 23:11:18 +02:00
// yacyClient.java
2005-04-07 21:19:42 +02:00
// -------------------------------------
2008-07-20 19:14:51 +02:00
// (C) by Michael Peter Christen; mc@yacy.net
2005-04-07 21:19:42 +02:00
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
2005-09-21 23:32:43 +02:00
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
2005-04-07 21:19:42 +02:00
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notice above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.yacy ;
2008-02-01 00:40:47 +01:00
import java.io.File ;
2005-05-05 07:32:19 +02:00
import java.io.IOException ;
2006-01-10 17:48:59 +01:00
import java.io.UnsupportedEncodingException ;
2008-05-04 12:53:04 +02:00
import java.net.MalformedURLException ;
import java.net.URL ;
2005-10-05 12:45:33 +02:00
import java.util.ArrayList ;
2005-05-05 07:32:19 +02:00
import java.util.HashMap ;
2005-10-13 15:57:15 +02:00
import java.util.Iterator ;
2008-04-12 10:12:51 +02:00
import java.util.List ;
2006-09-30 00:27:20 +02:00
import java.util.Map ;
import java.util.TreeMap ;
2010-03-23 11:17:28 +01:00
import java.util.regex.Pattern ;
2006-01-30 01:42:38 +01:00
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.xml.RSSFeed ;
import net.yacy.document.parser.xml.RSSReader ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.URIMetadataRow ;
import net.yacy.kelondro.data.word.Word ;
import net.yacy.kelondro.data.word.WordReference ;
2009-12-10 00:27:26 +01:00
import net.yacy.kelondro.index.RowSpaceExceededException ;
2009-11-05 21:28:37 +01:00
import net.yacy.kelondro.logging.Log ;
2009-10-10 01:22:22 +02:00
import net.yacy.kelondro.order.Base64Order ;
import net.yacy.kelondro.order.Bitfield ;
import net.yacy.kelondro.order.Digest ;
2009-10-10 02:39:15 +02:00
import net.yacy.kelondro.rwi.Reference ;
import net.yacy.kelondro.rwi.ReferenceContainer ;
import net.yacy.kelondro.rwi.ReferenceContainerCache ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.util.ByteBuffer ;
import net.yacy.kelondro.util.FileUtils ;
2009-10-21 22:14:30 +02:00
import net.yacy.repository.Blacklist ;
2009-10-10 01:22:22 +02:00
2008-04-12 10:12:51 +02:00
import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource ;
import org.apache.commons.httpclient.methods.multipart.Part ;
2008-05-06 02:32:41 +02:00
import de.anomic.crawler.ResultURLs ;
2009-10-21 22:14:30 +02:00
import de.anomic.crawler.retrieval.EventOrigin ;
2009-07-15 23:07:46 +02:00
import de.anomic.crawler.retrieval.HTTPLoader ;
2009-07-19 22:37:44 +02:00
import de.anomic.http.client.DefaultCharsetFilePart ;
import de.anomic.http.client.DefaultCharsetStringPart ;
import de.anomic.http.client.Client ;
import de.anomic.http.client.RemoteProxyConfig ;
2009-10-11 02:24:42 +02:00
import de.anomic.http.server.HeaderFramework ;
import de.anomic.http.server.RequestHeader ;
import de.anomic.http.server.ResponseContainer ;
2009-06-16 23:45:40 +02:00
import de.anomic.search.RankingProfile ;
2009-07-09 00:14:57 +02:00
import de.anomic.search.RankingProcess ;
2009-10-11 02:12:19 +02:00
import de.anomic.search.Segment ;
2009-07-19 22:37:44 +02:00
import de.anomic.search.Switchboard ;
import de.anomic.search.SwitchboardConstants ;
2009-08-27 16:34:41 +02:00
import de.anomic.search.TextSnippet ;
2005-05-05 07:32:19 +02:00
import de.anomic.server.serverCore ;
import de.anomic.tools.crypt ;
2005-04-07 21:19:42 +02:00
2005-10-05 12:45:33 +02:00
public final class yacyClient {
2005-09-21 23:32:43 +02:00
2008-08-02 15:57:00 +02:00
/ * *
* this is called to enrich the seed information by
* - own address ( if peer is behind a nat / router )
* - check peer type ( virgin / junior / senior / principal )
*
* to do this , we send a ' Hello ' to another peer
* this carries the following information :
* ' iam ' - own hash
* ' youare ' - remote hash , to verify that we are correct
* ' key ' - a session key that the remote peer may use to answer
* and the own seed string
* we expect the following information to be send back :
* - ' yourip ' the ip of the connection peer ( we )
* - ' yourtype ' the type of this peer that the other peer checked by asking for a specific word
* and the remote seed string
*
* one exceptional failure case is when we know the other ' s peers hash , the other peers responds correctly
* but they appear to be another peer by comparisment of the other peer ' s hash
* this works of course only if we know the other peer ' s hash .
*
* @return the number of new seeds
* /
2008-08-02 14:12:04 +02:00
public static int publishMySeed ( final yacySeed mySeed , final yacyPeerActions peerActions , final String address , final String otherHash ) {
2005-05-07 23:11:18 +02:00
2008-01-17 19:43:01 +01:00
HashMap < String , String > result = null ;
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , null , salt ) ;
2008-03-15 01:49:16 +01:00
for ( int retry = 0 ; retry < 4 ; retry + + ) try {
2007-07-05 01:48:52 +02:00
// generate request
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " count " , " 20 " ) ) ;
post . add ( new DefaultCharsetStringPart ( " seed " , mySeed . genSeedStr ( salt ) ) ) ;
2007-07-05 01:48:52 +02:00
// send request
2008-08-02 14:12:04 +02:00
final long start = System . currentTimeMillis ( ) ;
2009-04-21 00:08:38 +02:00
final byte [ ] content = wput ( " http:// " + address + " /yacy/hello.html " , yacySeed . b64Hash2hexHash ( otherHash ) + " .yacyh " , post , 30000 , false ) ;
2008-06-05 14:52:27 +02:00
yacyCore . log . logInfo ( " yacyClient.publishMySeed thread ' " + Thread . currentThread ( ) . getName ( ) + " ' contacted peer at " + address + " , received " + ( ( content = = null ) ? " null " : content . length ) + " bytes, time = " + ( System . currentTimeMillis ( ) - start ) + " milliseconds " ) ;
2009-10-05 22:11:41 +02:00
result = FileUtils . table ( content ) ;
2007-09-12 02:42:53 +02:00
break ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-07-07 15:58:54 +02:00
if ( Thread . currentThread ( ) . isInterrupted ( ) ) {
2009-06-02 00:45:28 +02:00
yacyCore . log . logInfo ( " yacyClient.publishMySeed thread ' " + Thread . currentThread ( ) . getName ( ) + " ' interrupted. " ) ;
2007-09-12 02:42:53 +02:00
return - 1 ;
2005-07-07 15:58:54 +02:00
}
2009-06-02 00:45:28 +02:00
yacyCore . log . logInfo ( " yacyClient.publishMySeed thread ' " + Thread . currentThread ( ) . getName ( ) + " ', peer " + address + " ; exception: " + e . getMessage ( ) + " ; retry = " + retry ) ;
2008-08-02 15:57:00 +02:00
// try again (go into loop)
2007-09-12 02:42:53 +02:00
result = null ;
2005-05-07 23:11:18 +02:00
}
2007-09-12 02:42:53 +02:00
2005-09-21 23:32:43 +02:00
if ( result = = null | | result . size ( ) < 3 ) {
2008-09-03 02:30:21 +02:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " yacyClient.publishMySeed result error: " +
2005-05-07 23:11:18 +02:00
( ( result = = null ) ? " result null " : ( " result= " + result . toString ( ) ) ) ) ;
return - 1 ;
}
2005-09-21 23:32:43 +02:00
2005-05-07 23:11:18 +02:00
// check consistency with expectation
2005-06-23 13:00:26 +02:00
yacySeed otherPeer = null ;
2007-06-13 15:21:19 +02:00
String seed ;
if ( ( otherHash ! = null ) & &
( otherHash . length ( ) > 0 ) & &
2008-06-06 18:01:27 +02:00
( ( seed = result . get ( " seed0 " ) ) ! = null ) ) {
2008-03-12 01:24:20 +01:00
if ( seed . length ( ) > yacySeed . maxsize ) {
yacyCore . log . logInfo ( " hello/client 0: rejected contacting seed; too large ( " + seed . length ( ) + " > " + yacySeed . maxsize + " ) " ) ;
2007-06-13 00:08:33 +02:00
} else {
2008-06-05 00:24:00 +02:00
otherPeer = yacySeed . genRemoteSeed ( seed , salt , false ) ;
2008-03-12 01:24:20 +01:00
if ( otherPeer = = null | | ! otherPeer . hash . equals ( otherHash ) ) {
2008-09-03 02:30:21 +02:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " yacyClient.publishMySeed: consistency error: other peer ' " + ( ( otherPeer = = null ) ? " unknown " : otherPeer . getName ( ) ) + " ' wrong " ) ;
2008-03-12 01:24:20 +01:00
return - 1 ; // no success
}
2005-05-07 23:11:18 +02:00
}
}
2005-09-21 23:32:43 +02:00
2005-05-07 23:11:18 +02:00
// set my own seed according to new information
2008-05-06 01:13:47 +02:00
// we overwrite our own IP number only
if ( serverCore . useStaticIP ) {
2009-10-11 02:12:19 +02:00
mySeed . setIP ( Switchboard . getSwitchboard ( ) . myPublicIP ( ) ) ;
2005-09-21 23:32:43 +02:00
} else {
2008-08-02 14:12:04 +02:00
final String myIP = result . get ( " yourip " ) ;
final String properIP = yacySeed . isProperIP ( myIP ) ;
2008-06-05 13:01:20 +02:00
if ( properIP = = null ) mySeed . setIP ( myIP ) ;
2005-06-16 09:28:07 +02:00
}
2005-09-21 23:32:43 +02:00
2008-05-06 01:13:47 +02:00
// change our seed-type
2008-06-06 18:01:27 +02:00
String mytype = result . get ( yacySeed . YOURTYPE ) ;
2008-05-06 01:13:47 +02:00
if ( mytype = = null ) { mytype = " " ; }
2008-08-02 14:12:04 +02:00
final yacyAccessible accessible = new yacyAccessible ( ) ;
2008-05-06 01:13:47 +02:00
if ( mytype . equals ( yacySeed . PEERTYPE_SENIOR ) | | mytype . equals ( yacySeed . PEERTYPE_PRINCIPAL ) ) {
accessible . IWasAccessed = true ;
if ( mySeed . isPrincipal ( ) ) {
mytype = yacySeed . PEERTYPE_PRINCIPAL ;
2005-07-04 13:09:48 +02:00
}
2008-05-06 01:13:47 +02:00
} else {
accessible . IWasAccessed = false ;
}
accessible . lastUpdated = System . currentTimeMillis ( ) ;
yacyCore . amIAccessibleDB . put ( otherHash , accessible ) ;
2005-09-21 23:32:43 +02:00
2008-05-06 01:13:47 +02:00
/ *
* If we were reported as junior we have to check if your port forwarding channel is broken
* If this is true we try to reconnect the sch channel to the remote server now .
* /
if ( mytype . equalsIgnoreCase ( yacySeed . PEERTYPE_JUNIOR ) ) {
yacyCore . log . logInfo ( " yacyClient.publishMySeed: Peer ' " + ( ( otherPeer = = null ) ? " unknown " : otherPeer . getName ( ) ) + " ' reported us as junior. " ) ;
} else if ( ( mytype . equalsIgnoreCase ( yacySeed . PEERTYPE_SENIOR ) ) | |
( mytype . equalsIgnoreCase ( yacySeed . PEERTYPE_PRINCIPAL ) ) ) {
2008-09-03 02:30:21 +02:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " yacyClient.publishMySeed: Peer ' " + ( ( otherPeer = = null ) ? " unknown " : otherPeer . getName ( ) ) + " ' reported us as " + mytype + " , accepted other peer. " ) ;
2008-05-06 01:13:47 +02:00
} else {
// wrong type report
2008-09-03 02:30:21 +02:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " yacyClient.publishMySeed: Peer ' " + ( ( otherPeer = = null ) ? " unknown " : otherPeer . getName ( ) ) + " ' reported us as " + mytype + " , rejecting other peer. " ) ;
2008-05-06 01:13:47 +02:00
return - 1 ;
2005-06-16 09:28:07 +02:00
}
2008-05-06 01:13:47 +02:00
if ( mySeed . orVirgin ( ) . equals ( yacySeed . PEERTYPE_VIRGIN ) )
mySeed . put ( yacySeed . PEERTYPE , mytype ) ;
2005-09-21 23:32:43 +02:00
2008-06-05 00:24:00 +02:00
final String error = mySeed . isProper ( true ) ;
2005-09-21 23:32:43 +02:00
if ( error ! = null ) {
2007-10-01 14:30:23 +02:00
yacyCore . log . logSevere ( " yacyClient.publishMySeed mySeed error - not proper: " + error ) ;
2005-05-07 23:11:18 +02:00
return - 1 ;
}
2005-09-21 23:32:43 +02:00
2005-10-17 17:46:12 +02:00
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
2007-10-19 17:31:38 +02:00
2005-05-07 23:11:18 +02:00
// read the seeds that the peer returned and integrate them into own database
2005-04-07 21:19:42 +02:00
int i = 0 ;
2005-05-07 23:11:18 +02:00
int count = 0 ;
2005-09-21 23:32:43 +02:00
String seedStr ;
2008-06-06 18:01:27 +02:00
while ( ( seedStr = result . get ( " seed " + i + + ) ) ! = null ) {
2005-05-07 23:11:18 +02:00
// integrate new seed into own database
// the first seed, "seed0" is the seed of the responding peer
2008-03-12 01:24:20 +01:00
if ( seedStr . length ( ) > yacySeed . maxsize ) {
yacyCore . log . logInfo ( " hello/client: rejected contacting seed; too large ( " + seedStr . length ( ) + " > " + yacySeed . maxsize + " ) " ) ;
2007-06-13 00:08:33 +02:00
} else {
2008-06-05 00:24:00 +02:00
if ( peerActions . peerArrival ( yacySeed . genRemoteSeed ( seedStr , salt , false ) , ( i = = 1 ) ) ) count + + ;
2007-06-13 00:08:33 +02:00
}
2005-05-07 23:11:18 +02:00
}
return count ;
2005-04-07 21:19:42 +02:00
}
2005-09-21 23:32:43 +02:00
2008-05-04 12:53:04 +02:00
/ * *
* send data to the server named by vhost
*
* @param address address of the server
* @param vhost name of the server at address which should respond
* @param post data to send ( name - value - pairs )
* @param gzipBody send with content gzip encoded
* @return response body
* @throws IOException
* /
2008-07-03 15:08:37 +02:00
/ *
2008-05-04 12:53:04 +02:00
private static byte [ ] wput ( final String url , String vhost , final List < Part > post , boolean gzipBody ) throws IOException {
return wput ( url , vhost , post , 10000 , gzipBody ) ;
2008-04-09 13:02:14 +02:00
}
2008-07-03 15:08:37 +02:00
* /
2008-04-09 13:02:14 +02:00
/ * *
* send data to the server named by vhost
*
* @param address address of the server
* @param vhost name of the server at address which should respond
* @param post data to send ( name - value - pairs )
* @param timeout in milliseconds
* @return response body
* @throws IOException
* /
2008-08-02 14:12:04 +02:00
private static byte [ ] wput ( final String url , final String vhost , final List < Part > post , final int timeout ) throws IOException {
2008-05-04 12:53:04 +02:00
return wput ( url , vhost , post , timeout , false ) ;
}
/ * *
* send data to the server named by vhost
*
* @param address address of the server
* @param vhost name of the server at address which should respond
* @param post data to send ( name - value - pairs )
* @param timeout in milliseconds
* @param gzipBody send with content gzip encoded
* @return response body
* @throws IOException
* /
2008-08-02 14:12:04 +02:00
private static byte [ ] wput ( final String url , final String vhost , final List < Part > post , final int timeout , final boolean gzipBody ) throws IOException {
2009-07-19 22:37:44 +02:00
final RequestHeader header = new RequestHeader ( ) ;
header . put ( HeaderFramework . USER_AGENT , HTTPLoader . yacyUserAgent ) ;
header . put ( HeaderFramework . HOST , vhost ) ;
final Client client = new Client ( timeout , header ) ;
2008-07-04 13:03:03 +02:00
client . setProxy ( proxyConfig ( ) ) ;
2008-04-05 15:17:16 +02:00
2009-07-19 22:37:44 +02:00
ResponseContainer res = null ;
2008-04-05 15:17:16 +02:00
byte [ ] content = null ;
try {
// send request/data
2008-05-04 12:53:04 +02:00
res = client . POST ( url , post , gzipBody ) ;
2008-04-05 15:17:16 +02:00
content = res . getData ( ) ;
} finally {
if ( res ! = null ) {
// release connection
res . closeStream ( ) ;
}
}
return content ;
}
/ * *
* @see wput
* @param target
* @param filename
* @param post
* @return
* @throws IOException
* /
2008-08-02 14:12:04 +02:00
private static byte [ ] postToFile ( final yacySeed target , final String filename , final List < Part > post , final int timeout ) throws IOException {
2008-06-05 13:01:20 +02:00
return wput ( " http:// " + target . getClusterAddress ( ) + " /yacy/ " + filename , target . getHexHash ( ) + " .yacyh " , post , timeout , false ) ;
2008-04-05 15:17:16 +02:00
}
2008-08-02 14:12:04 +02:00
private static byte [ ] postToFile ( final yacySeedDB seedDB , final String targetHash , final String filename , final List < Part > post , final int timeout ) throws IOException {
2008-06-05 13:01:20 +02:00
return wput ( " http:// " + targetAddress ( seedDB , targetHash ) + " /yacy/ " + filename , yacySeed . b64Hash2hexHash ( targetHash ) + " .yacyh " , post , timeout , false ) ;
2008-04-05 15:17:16 +02:00
}
2008-08-02 14:12:04 +02:00
public static yacySeed querySeed ( final yacySeed target , final String seedHash ) {
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , target . hash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " object " , " seed " ) ) ;
post . add ( new DefaultCharsetStringPart ( " env " , seedHash ) ) ;
2008-03-12 01:24:20 +01:00
2007-07-05 01:48:52 +02:00
// send request
try {
2008-08-02 14:12:04 +02:00
final byte [ ] content = postToFile ( target , " query.html " , post , 10000 ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2008-03-12 01:24:20 +01:00
2009-12-02 01:37:59 +01:00
if ( result = = null | | result . isEmpty ( ) ) { return null ; }
2005-10-17 17:46:12 +02:00
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
2008-06-06 18:01:27 +02:00
return yacySeed . genRemoteSeed ( result . get ( " response " ) , salt , false ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-08-30 23:32:59 +02:00
yacyCore . log . logSevere ( " yacyClient.querySeed error: " + e . getMessage ( ) ) ;
2005-04-07 21:19:42 +02:00
return null ;
}
}
2005-09-21 23:32:43 +02:00
2008-08-02 14:12:04 +02:00
public static int queryRWICount ( final yacySeed target , final String wordHash ) {
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , target . hash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " object " , " rwicount " ) ) ;
post . add ( new DefaultCharsetStringPart ( " ttl " , " 0 " ) ) ;
post . add ( new DefaultCharsetStringPart ( " env " , wordHash ) ) ;
2005-10-22 15:28:04 +02:00
2007-07-05 01:48:52 +02:00
// send request
try {
2008-08-02 14:12:04 +02:00
final byte [ ] content = postToFile ( target , " query.html " , post , 5000 ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2005-10-22 15:28:04 +02:00
2009-12-02 01:37:59 +01:00
if ( result = = null | | result . isEmpty ( ) ) { return - 1 ; }
2008-06-06 18:01:27 +02:00
return Integer . parseInt ( result . get ( " response " ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-08-30 23:32:59 +02:00
yacyCore . log . logSevere ( " yacyClient.queryRWICount error: " + e . getMessage ( ) ) ;
2005-05-07 23:11:18 +02:00
return - 1 ;
}
2005-04-07 21:19:42 +02:00
}
2005-09-21 23:32:43 +02:00
2008-08-02 14:12:04 +02:00
public static int queryUrlCount ( final yacySeed target ) {
2005-09-21 23:32:43 +02:00
if ( target = = null ) { return - 1 ; }
2008-03-12 01:24:20 +01:00
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , target . hash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " object " , " lurlcount " ) ) ;
post . add ( new DefaultCharsetStringPart ( " ttl " , " 0 " ) ) ;
post . add ( new DefaultCharsetStringPart ( " env " , " " ) ) ;
2008-03-12 01:24:20 +01:00
2007-07-05 01:48:52 +02:00
// send request
2005-05-07 23:11:18 +02:00
try {
2008-08-02 14:12:04 +02:00
final byte [ ] content = postToFile ( target , " query.html " , post , 5000 ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2008-03-12 01:24:20 +01:00
2009-12-02 01:37:59 +01:00
if ( result = = null | | result . isEmpty ( ) ) return - 1 ;
2008-06-06 18:01:27 +02:00
final String resp = result . get ( " response " ) ;
2006-09-07 03:13:03 +02:00
if ( resp = = null ) {
return - 1 ;
2008-08-02 15:57:00 +02:00
}
try {
2006-09-07 03:13:03 +02:00
return Integer . parseInt ( resp ) ;
2008-08-02 14:12:04 +02:00
} catch ( final NumberFormatException e ) {
2006-09-07 03:13:03 +02:00
return - 1 ;
}
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2010-03-07 22:02:08 +01:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " yacyClient.queryUrlCount error asking peer ' " + target . getName ( ) + " ': " + e . toString ( ) ) ;
2005-05-07 23:11:18 +02:00
return - 1 ;
}
2005-04-07 21:19:42 +02:00
}
2005-09-21 23:32:43 +02:00
2008-11-06 11:07:53 +01:00
public static RSSFeed queryRemoteCrawlURLs ( final yacySeedDB seedDB , final yacySeed target , final int maxCount , final long maxTime ) {
2007-11-29 03:07:37 +01:00
// returns a list of
if ( target = = null ) { return null ; }
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , target . hash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " call " , " remotecrawl " ) ) ;
2008-11-06 11:07:53 +01:00
post . add ( new DefaultCharsetStringPart ( " count " , Integer . toString ( maxCount ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " time " , Long . toString ( maxTime ) ) ) ;
2007-11-29 03:07:37 +01:00
// send request
try {
2008-04-09 13:02:14 +02:00
/* a long time-out is needed */
2009-02-04 17:55:09 +01:00
final byte [ ] result = wput ( " http:// " + target . getClusterAddress ( ) + " /yacy/urls.xml " , target . getHexHash ( ) + " .yacyh " , post , ( int ) maxTime ) ;
2008-08-02 14:12:04 +02:00
final RSSReader reader = RSSReader . parse ( result ) ;
2008-05-22 01:07:37 +02:00
if ( reader = = null ) {
yacyCore . log . logWarning ( " yacyClient.queryRemoteCrawlURLs failed asking peer ' " + target . getName ( ) + " ': probably bad response from remote peer (1), reader == null " ) ;
target . put ( yacySeed . RCOUNT , " 0 " ) ;
seedDB . update ( target . hash , target ) ; // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)
2009-11-05 21:28:37 +01:00
//Log.logException(e);
2008-05-22 01:07:37 +02:00
return null ;
}
2008-08-02 14:12:04 +02:00
final RSSFeed feed = reader . getFeed ( ) ;
2008-04-24 23:31:07 +02:00
if ( feed = = null ) {
2007-11-29 03:07:37 +01:00
// case where the rss reader does not understand the content
2008-05-22 01:07:37 +02:00
yacyCore . log . logWarning ( " yacyClient.queryRemoteCrawlURLs failed asking peer ' " + target . getName ( ) + " ': probably bad response from remote peer (2) " ) ;
2009-11-20 13:11:56 +01:00
//System.out.println("***DEBUG*** rss input = " + new String(result));
2007-11-29 03:07:37 +01:00
target . put ( yacySeed . RCOUNT , " 0 " ) ;
2008-05-06 01:13:47 +02:00
seedDB . update ( target . hash , target ) ; // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)
2009-11-05 21:28:37 +01:00
//Log.logException(e);
2007-11-29 03:07:37 +01:00
return null ;
}
2008-04-24 23:31:07 +02:00
return feed ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-11-29 03:07:37 +01:00
yacyCore . log . logSevere ( " yacyClient.queryRemoteCrawlURLs error asking peer ' " + target . getName ( ) + " ': " + e . toString ( ) ) ;
return null ;
}
}
2009-04-15 08:34:27 +02:00
@SuppressWarnings ( " unchecked " )
2006-10-04 00:55:59 +02:00
public static String [ ] search (
2008-08-02 14:12:04 +02:00
final yacySeed mySeed ,
final String wordhashes ,
final String excludehashes ,
final String urlhashes ,
2010-03-23 11:17:28 +01:00
final Pattern prefer ,
final Pattern filter ,
2008-09-21 02:04:42 +02:00
final String language ,
2009-06-09 11:07:52 +02:00
final String sitehash ,
final String authorhash ,
2008-08-02 14:12:04 +02:00
final int count ,
final int maxDistance ,
final boolean global ,
final int partitions ,
final yacySeed target ,
2009-05-28 16:26:05 +02:00
final Segment indexSegment ,
2008-08-02 14:12:04 +02:00
final ResultURLs crawlResults ,
2009-07-09 00:14:57 +02:00
final RankingProcess containerCache ,
2008-08-02 14:12:04 +02:00
final Map < String , TreeMap < String , String > > abstractCache ,
2009-03-02 12:04:13 +01:00
final Blacklist blacklist ,
2009-06-16 23:45:40 +02:00
final RankingProfile rankingProfile ,
2009-01-30 16:33:00 +01:00
final Bitfield constraint
2005-11-08 13:14:51 +01:00
) {
2005-05-07 23:11:18 +02:00
// send a search request to peer with remote Hash
// this mainly converts the words into word hashes
2005-09-21 23:32:43 +02:00
2005-05-07 23:11:18 +02:00
// INPUT:
2007-02-01 14:27:23 +01:00
// iam : complete seed of the requesting peer
// youare : seed hash of the target peer, used for testing network stability
// key : transmission key for response
// search : a list of search words
// hsearch : a string of word hashes
// fwdep : forward depth. if "0" then peer may NOT ask another peer for more results
// fwden : forward deny, a list of seed hashes. They may NOT be target of forward hopping
// count : maximum number of wanted results
// global : if "true", then result may consist of answers from other peers
// partitions : number of remote peers that are asked (for evaluation of QPM)
// duetime : maximum time that a peer should spent to create a result
2005-09-21 23:32:43 +02:00
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , target . hash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " myseed " , mySeed . genSeedStr ( salt ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " count " , Integer . toString ( Math . max ( 10 , count ) ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " resource " , ( ( global ) ? " global " : " local " ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " partitions " , Integer . toString ( partitions ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " query " , wordhashes ) ) ;
post . add ( new DefaultCharsetStringPart ( " exclude " , excludehashes ) ) ;
post . add ( new DefaultCharsetStringPart ( " duetime " , " 1000 " ) ) ;
post . add ( new DefaultCharsetStringPart ( " urls " , urlhashes ) ) ;
2010-03-23 11:17:28 +01:00
post . add ( new DefaultCharsetStringPart ( " prefer " , prefer . toString ( ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " filter " , filter . toString ( ) ) ) ;
2008-09-21 02:04:42 +02:00
post . add ( new DefaultCharsetStringPart ( " language " , language ) ) ;
2009-06-09 11:07:52 +02:00
post . add ( new DefaultCharsetStringPart ( " sitehash " , sitehash ) ) ;
post . add ( new DefaultCharsetStringPart ( " authorhash " , authorhash ) ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " ttl " , " 0 " ) ) ;
post . add ( new DefaultCharsetStringPart ( " maxdist " , Integer . toString ( maxDistance ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " profile " , crypt . simpleEncode ( rankingProfile . toExternalString ( ) ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " constraint " , ( constraint = = null ) ? " " : constraint . exportB64 ( ) ) ) ;
if ( abstractCache ! = null ) post . add ( new DefaultCharsetStringPart ( " abstracts " , " auto " ) ) ;
2007-07-05 01:48:52 +02:00
final long timestamp = System . currentTimeMillis ( ) ;
2007-10-19 17:31:38 +02:00
2007-07-05 01:48:52 +02:00
// send request
2008-01-17 19:43:01 +01:00
HashMap < String , String > result = null ;
2007-07-05 01:48:52 +02:00
try {
2009-10-05 22:11:41 +02:00
result = FileUtils . table ( wput ( " http:// " + target . getClusterAddress ( ) + " /yacy/search.html " , target . getHexHash ( ) + " .yacyh " , post , 60000 ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
yacyCore . log . logInfo ( " SEARCH failed, Peer: " + target . hash + " : " + target . getName ( ) + " ( " + e . getMessage ( ) + " ), score= " + target . selectscore ) ;
2008-03-12 01:56:18 +01:00
//yacyCore.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage());
2007-07-05 01:48:52 +02:00
return null ;
}
2005-10-23 19:50:27 +02:00
2009-12-02 01:37:59 +01:00
if ( result = = null | | result . isEmpty ( ) ) {
2008-09-03 02:30:21 +02:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " SEARCH failed FROM "
2008-03-12 01:24:20 +01:00
+ target . hash
+ " : "
+ target . getName ( )
+ " (zero response), score= "
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
+ target . selectscore ) ;
2008-03-12 01:24:20 +01:00
return null ;
}
2005-09-21 23:32:43 +02:00
2008-03-12 01:24:20 +01:00
// compute all computation times
final long totalrequesttime = System . currentTimeMillis ( ) - timestamp ;
// OUTPUT:
// version : application version of responder
// uptime : uptime in seconds of responder
// total : number of total available LURL's for this search
// count : number of returned LURL's for this search
// resource<n> : LURL of search
// fwhop : hops (depth) of forwards that had been performed to construct this result
// fwsrc : peers that helped to construct this result
// fwrec : peers that would have helped to construct this result (recommendations)
// searchtime : time that the peer actually spent to create the result
// references : references (search hints) that was calculated during search
// now create a plasmaIndex out of this result
// System.out.println("yacyClient: " + ((urlhashes.length() == 0) ? "primary" : "secondary")+ " search result = " + result.toString()); // debug
int results = 0 , joincount = 0 ;
2007-10-03 17:45:12 +02:00
try {
2008-01-30 22:58:30 +01:00
results = Integer . parseInt ( result . get ( " count " ) ) ;
joincount = Integer . parseInt ( result . get ( " joincount " ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final NumberFormatException e ) {
2008-03-12 01:56:18 +01:00
yacyCore . log . logInfo ( " SEARCH failed FROM " + target . hash + " : " + target . getName ( ) + " , wrong output format: " + e . getMessage ( ) ) ;
//yacyCore.peerActions.peerDeparture(target, "search request to peer created number format exception");
2007-10-03 17:45:12 +02:00
return null ;
}
2008-03-12 01:24:20 +01:00
// System.out.println("***result count " + results);
2005-09-21 23:32:43 +02:00
2008-03-12 01:24:20 +01:00
// create containers
2009-10-11 02:12:19 +02:00
final int words = wordhashes . length ( ) / Word . commonHashLength ;
2009-06-02 00:45:28 +02:00
assert words > 0 : " wordhashes = " + wordhashes ;
2009-04-15 08:34:27 +02:00
final ReferenceContainer < WordReference > [ ] container = new ReferenceContainer [ words ] ;
2008-03-12 01:24:20 +01:00
for ( int i = 0 ; i < words ; i + + ) {
2010-01-09 01:08:16 +01:00
try {
container [ i ] = ReferenceContainer . emptyContainer ( Segment . wordReferenceFactory , wordhashes . substring ( i * Word . commonHashLength , ( i + 1 ) * Word . commonHashLength ) . getBytes ( ) , count ) ;
} catch ( RowSpaceExceededException e ) {
Log . logException ( e ) ;
return null ;
}
2008-03-12 01:24:20 +01:00
}
2007-06-22 16:29:14 +02:00
2008-03-12 01:24:20 +01:00
// insert results to containers
2009-10-11 02:12:19 +02:00
URIMetadataRow urlEntry ;
2008-08-02 14:12:04 +02:00
final String [ ] urls = new String [ results ] ;
2008-03-12 01:24:20 +01:00
for ( int n = 0 ; n < results ; n + + ) {
// get one single search result
2009-10-11 02:12:19 +02:00
urlEntry = URIMetadataRow . importEntry ( result . get ( " resource " + n ) ) ;
2008-03-12 01:24:20 +01:00
if ( urlEntry = = null ) continue ;
2010-03-26 19:33:20 +01:00
assert ( urlEntry . hash ( ) . length = = 12 ) : " urlEntry.hash() = " + new String ( urlEntry . hash ( ) ) ;
if ( urlEntry . hash ( ) . length ! = 12 ) continue ; // bad url hash
2009-10-11 02:12:19 +02:00
final URIMetadataRow . Components metadata = urlEntry . metadata ( ) ;
2010-01-29 16:59:24 +01:00
if ( metadata = = null ) continue ;
2009-03-02 12:04:13 +01:00
if ( blacklist . isListed ( Blacklist . BLACKLIST_SEARCH , metadata . url ( ) ) ) {
2010-03-26 19:33:20 +01:00
if ( yacyCore . log . isInfo ( ) ) yacyCore . log . logInfo ( " remote search (client): filtered blacklisted url " + metadata . url ( ) + " from peer " + target . getName ( ) ) ;
2008-03-12 01:24:20 +01:00
continue ; // block with backlist
}
2007-07-24 02:46:17 +02:00
2009-07-19 22:37:44 +02:00
final String urlRejectReason = Switchboard . getSwitchboard ( ) . crawlStacker . urlInAcceptedDomain ( metadata . url ( ) ) ;
2008-04-20 23:36:25 +02:00
if ( urlRejectReason ! = null ) {
2010-03-26 19:33:20 +01:00
if ( yacyCore . log . isInfo ( ) ) yacyCore . log . logInfo ( " remote search (client): rejected url ' " + metadata . url ( ) + " ' ( " + urlRejectReason + " ) from peer " + target . getName ( ) ) ;
2007-07-24 02:46:17 +02:00
continue ; // reject url outside of our domain
}
2006-12-05 03:47:51 +01:00
2008-03-12 01:24:20 +01:00
// save the url entry
2009-03-02 00:58:14 +01:00
Reference entry ;
2008-03-12 01:24:20 +01:00
if ( urlEntry . word ( ) = = null ) {
2010-03-26 19:33:20 +01:00
if ( yacyCore . log . isWarning ( ) ) yacyCore . log . logWarning ( " remote search (client): no word attached from peer " + target . getName ( ) + " , version " + target . getVersion ( ) ) ;
2008-03-12 01:24:20 +01:00
continue ; // no word attached
}
2005-09-21 23:32:43 +02:00
2008-03-12 01:24:20 +01:00
// the search-result-url transports all the attributes of word indexes
entry = urlEntry . word ( ) ;
2010-03-26 19:33:20 +01:00
if ( ! Base64Order . enhancedCoder . equal ( entry . metadataHash ( ) . getBytes ( ) , urlEntry . hash ( ) ) ) {
if ( yacyCore . log . isInfo ( ) ) yacyCore . log . logInfo ( " remote search (client): url-hash " + new String ( urlEntry . hash ( ) ) + " does not belong to word-attached-hash " + entry . metadataHash ( ) + " ; url = " + metadata . url ( ) + " from peer " + target . getName ( ) ) ;
2008-03-12 01:24:20 +01:00
continue ; // spammed
}
2005-05-07 23:11:18 +02:00
2008-03-12 01:24:20 +01:00
// passed all checks, store url
try {
2009-05-29 12:03:35 +02:00
indexSegment . urlMetadata ( ) . store ( urlEntry ) ;
2010-04-08 02:11:32 +02:00
crawlResults . stack ( urlEntry , mySeed . hash . getBytes ( ) , target . hash . getBytes ( ) , EventOrigin . QUERIES ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-03-12 01:24:20 +01:00
yacyCore . log . logSevere ( " could not store search result " , e ) ;
continue ; // db-error
}
2007-07-05 01:48:52 +02:00
2008-03-12 01:24:20 +01:00
if ( urlEntry . snippet ( ) ! = null ) {
// we don't store the snippets along the url entry,
2007-08-15 13:36:59 +02:00
// because they are search-specific.
2008-03-12 01:24:20 +01:00
// instead, they are placed in a snipped-search cache.
// System.out.println("--- RECEIVED SNIPPET '" + link.snippet() + "'");
2010-03-26 19:33:20 +01:00
TextSnippet . storeToCache ( wordhashes , new String ( urlEntry . hash ( ) ) , urlEntry . snippet ( ) ) ;
2008-03-12 01:24:20 +01:00
}
2007-09-04 01:43:55 +02:00
2008-03-12 01:24:20 +01:00
// add the url entry to the word indexes
for ( int m = 0 ; m < words ; m + + ) {
2009-12-10 00:27:26 +01:00
try {
container [ m ] . add ( entry ) ;
} catch ( RowSpaceExceededException e ) {
Log . logException ( e ) ;
break ;
}
2008-03-12 01:24:20 +01:00
}
2007-09-04 01:43:55 +02:00
2008-03-12 01:24:20 +01:00
// store url hash for statistics
2010-03-26 19:33:20 +01:00
urls [ n ] = new String ( urlEntry . hash ( ) ) ;
2008-03-12 01:24:20 +01:00
}
2007-07-05 01:48:52 +02:00
2007-09-04 01:43:55 +02:00
// store remote result to local result container
synchronized ( containerCache ) {
// insert one container into the search result buffer
2009-08-27 17:19:48 +02:00
containerCache . add ( container [ 0 ] , false , joincount ) ; // one is enough
2007-09-04 01:43:55 +02:00
// integrate remote topwords
2008-08-02 14:12:04 +02:00
final String references = result . get ( " references " ) ;
2007-09-08 13:50:19 +02:00
yacyCore . log . logInfo ( " remote search (client): peer " + target . getName ( ) + " sent references " + references ) ;
if ( references ! = null ) {
2009-12-10 15:40:32 +01:00
// add references twice, so they can be counted (must have at least 2 entries)
2009-06-02 17:20:10 +02:00
containerCache . addTopic ( references . split ( " , " ) ) ;
containerCache . addTopic ( references . split ( " , " ) ) ;
2007-09-08 13:50:19 +02:00
}
2007-09-04 01:43:55 +02:00
}
2008-03-12 01:24:20 +01:00
// read index abstract
if ( abstractCache ! = null ) {
2008-08-02 14:12:04 +02:00
final Iterator < Map . Entry < String , String > > i = result . entrySet ( ) . iterator ( ) ;
2008-03-12 01:24:20 +01:00
Map . Entry < String , String > entry ;
TreeMap < String , String > singleAbstract ;
String wordhash ;
2009-01-30 16:33:00 +01:00
ByteBuffer ci ;
2008-03-12 01:24:20 +01:00
while ( i . hasNext ( ) ) {
entry = i . next ( ) ;
if ( entry . getKey ( ) . startsWith ( " indexabstract. " ) ) {
wordhash = entry . getKey ( ) . substring ( 14 ) ;
synchronized ( abstractCache ) {
2008-06-06 18:01:27 +02:00
singleAbstract = abstractCache . get ( wordhash ) ; // a mapping from url-hashes to a string of peer-hashes
2008-03-12 01:24:20 +01:00
if ( singleAbstract = = null ) singleAbstract = new TreeMap < String , String > ( ) ;
2008-11-08 21:24:31 +01:00
try {
2009-01-30 16:33:00 +01:00
ci = new ByteBuffer ( entry . getValue ( ) . getBytes ( " UTF-8 " ) ) ;
2008-11-08 21:24:31 +01:00
} catch ( UnsupportedEncodingException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-11-08 21:24:31 +01:00
return null ;
}
2008-03-12 01:24:20 +01:00
//System.out.println("DEBUG-ABSTRACTFETCH: for word hash " + wordhash + " received " + ci.toString());
2009-03-02 00:58:14 +01:00
ReferenceContainer . decompressIndex ( singleAbstract , ci , target . hash ) ;
2008-03-12 01:24:20 +01:00
abstractCache . put ( wordhash , singleAbstract ) ;
}
}
}
}
2007-10-19 17:31:38 +02:00
2008-03-12 01:24:20 +01:00
// insert the containers to the index
2009-03-16 01:18:37 +01:00
for ( int m = 0 ; m < words ; m + + ) try {
2009-05-29 12:03:35 +02:00
indexSegment . termIndex ( ) . add ( container [ m ] ) ;
2009-12-10 00:27:26 +01:00
} catch ( Exception e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2009-03-16 01:18:37 +01:00
}
2008-02-03 13:40:40 +01:00
// generate statistics
2008-03-12 01:24:20 +01:00
long searchtime ;
try {
2008-06-06 18:01:27 +02:00
searchtime = Integer . parseInt ( result . get ( " searchtime " ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final NumberFormatException e ) {
2008-03-12 01:24:20 +01:00
searchtime = totalrequesttime ;
}
2008-09-03 02:30:21 +02:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " SEARCH "
2008-03-12 01:24:20 +01:00
+ results
+ " URLS FROM "
+ target . hash
+ " : "
+ target . getName ( )
+ " , score= "
+ target . selectscore
+ " , searchtime= " + searchtime + " , netdelay= "
+ ( totalrequesttime - searchtime ) + " , references= "
+ result . get ( " references " ) ) ;
return urls ;
}
2005-09-21 23:32:43 +02:00
2008-08-02 14:12:04 +02:00
public static HashMap < String , String > permissionMessage ( final yacySeedDB seedDB , final String targetHash ) {
2005-05-07 23:11:18 +02:00
// ask for allowed message size and attachement size
// if this replies null, the peer does not answer
2008-05-06 01:13:47 +02:00
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , targetHash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " process " , " permission " ) ) ;
2005-10-22 15:28:04 +02:00
2007-07-05 01:48:52 +02:00
// send request
2005-05-07 23:11:18 +02:00
try {
2008-06-05 13:01:20 +02:00
final byte [ ] content = postToFile ( seedDB , targetHash , " message.html " , post , 5000 ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2007-07-05 01:48:52 +02:00
return result ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-05-07 23:11:18 +02:00
// most probably a network time-out exception
2005-08-30 23:32:59 +02:00
yacyCore . log . logSevere ( " yacyClient.permissionMessage error: " + e . getMessage ( ) ) ;
2005-05-07 23:11:18 +02:00
return null ;
}
2005-04-07 21:19:42 +02:00
}
2005-09-21 23:32:43 +02:00
2008-08-02 14:12:04 +02:00
public static HashMap < String , String > postMessage ( final yacySeedDB seedDB , final String targetHash , final String subject , final byte [ ] message ) {
2005-05-07 23:11:18 +02:00
// this post a message to the remote message board
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , targetHash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " process " , " post " ) ) ;
post . add ( new DefaultCharsetStringPart ( " myseed " , seedDB . mySeed ( ) . genSeedStr ( salt ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " subject " , subject ) ) ;
2007-07-05 01:48:52 +02:00
try {
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " message " , new String ( message , " UTF-8 " ) ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final UnsupportedEncodingException e ) {
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " message " , new String ( message ) ) ) ;
2007-07-05 01:48:52 +02:00
}
2007-08-08 20:23:45 +02:00
2007-07-05 01:48:52 +02:00
// send request
2005-11-11 00:48:20 +01:00
try {
2008-06-05 13:01:20 +02:00
final byte [ ] content = postToFile ( seedDB , targetHash , " message.html " , post , 20000 ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2007-07-05 01:48:52 +02:00
return result ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-11-11 00:48:20 +01:00
yacyCore . log . logSevere ( " yacyClient.postMessage error: " + e . getMessage ( ) ) ;
return null ;
}
}
2008-08-02 14:12:04 +02:00
public static String targetAddress ( final yacySeedDB seedDB , final String targetHash ) {
2005-11-11 00:48:20 +01:00
// find target address
2005-05-07 23:11:18 +02:00
String address ;
2008-05-06 01:13:47 +02:00
if ( targetHash . equals ( seedDB . mySeed ( ) . hash ) ) {
address = seedDB . mySeed ( ) . getClusterAddress ( ) ;
2005-09-21 23:32:43 +02:00
} else {
2008-05-06 01:13:47 +02:00
final yacySeed targetSeed = seedDB . getConnected ( targetHash ) ;
2005-11-11 00:48:20 +01:00
if ( targetSeed = = null ) { return null ; }
2007-04-30 00:05:34 +02:00
address = targetSeed . getClusterAddress ( ) ;
2005-09-21 23:32:43 +02:00
}
2005-11-11 00:48:20 +01:00
if ( address = = null ) address = " localhost:8080 " ;
return address ;
}
2008-08-02 14:12:04 +02:00
public static HashMap < String , String > transferPermission ( final String targetAddress , final long filesize , final String filename ) {
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , null , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " process " , " permission " ) ) ;
post . add ( new DefaultCharsetStringPart ( " purpose " , " crcon " ) ) ;
post . add ( new DefaultCharsetStringPart ( " filename " , filename ) ) ;
post . add ( new DefaultCharsetStringPart ( " filesize " , Long . toString ( filesize ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " can-send-protocol " , " http " ) ) ;
2005-11-11 00:48:20 +01:00
// send request
try {
2008-06-05 14:41:55 +02:00
final byte [ ] content = wput ( " http:// " + targetAddress + " /yacy/transfer.html " , targetAddress , post , 10000 ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2007-07-05 01:48:52 +02:00
return result ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-11-11 00:48:20 +01:00
// most probably a network time-out exception
yacyCore . log . logSevere ( " yacyClient.permissionTransfer error: " + e . getMessage ( ) ) ;
return null ;
}
}
2008-08-02 14:12:04 +02:00
public static HashMap < String , String > transferStore ( final String targetAddress , final String access , final String filename , final byte [ ] file ) {
2005-11-11 00:48:20 +01:00
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , null , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " process " , " store " ) ) ;
post . add ( new DefaultCharsetStringPart ( " purpose " , " crcon " ) ) ;
post . add ( new DefaultCharsetStringPart ( " filesize " , Long . toString ( file . length ) ) ) ;
2009-01-30 16:33:00 +01:00
post . add ( new DefaultCharsetStringPart ( " md5 " , Digest . encodeMD5Hex ( file ) ) ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " access " , access ) ) ;
post . add ( new DefaultCharsetFilePart ( " filename " , new ByteArrayPartSource ( filename , file ) ) ) ;
2005-10-22 15:28:04 +02:00
2007-07-05 01:48:52 +02:00
// send request
2005-04-07 21:19:42 +02:00
try {
2008-06-05 14:41:55 +02:00
final byte [ ] content = wput ( " http:// " + targetAddress + " /yacy/transfer.html " , targetAddress , post , 20000 ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2007-07-05 01:48:52 +02:00
return result ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-08-30 23:32:59 +02:00
yacyCore . log . logSevere ( " yacyClient.postMessage error: " + e . getMessage ( ) ) ;
2005-04-07 21:19:42 +02:00
return null ;
}
}
2005-11-11 00:48:20 +01:00
2008-08-02 14:12:04 +02:00
public static String transfer ( final String targetAddress , final String filename , final byte [ ] file ) {
final HashMap < String , String > phase1 = transferPermission ( targetAddress , file . length , filename ) ;
2005-11-11 00:48:20 +01:00
if ( phase1 = = null ) return " no connection to remote address " + targetAddress + " ; phase 1 " ;
2008-08-02 14:12:04 +02:00
final String access = phase1 . get ( " access " ) ;
final String nextaddress = phase1 . get ( " address " ) ;
final String protocol = phase1 . get ( " protocol " ) ;
2005-12-05 15:24:13 +01:00
//String path = (String) phase1.get("path");
//String maxsize = (String) phase1.get("maxsize");
2008-06-06 18:01:27 +02:00
String response = phase1 . get ( " response " ) ;
2005-11-11 00:48:20 +01:00
if ( ( response = = null ) | | ( protocol = = null ) | | ( access = = null ) ) return " wrong return values from other peer; phase 1 " ;
if ( ! ( response . equals ( " ok " ) ) ) return " remote peer rejected transfer: " + response ;
2009-01-30 16:33:00 +01:00
final String accesscode = Digest . encodeMD5Hex ( Base64Order . standardCoder . encodeString ( access ) ) ;
2005-11-11 00:48:20 +01:00
if ( protocol . equals ( " http " ) ) {
2008-08-02 14:12:04 +02:00
final HashMap < String , String > phase2 = transferStore ( nextaddress , accesscode , filename , file ) ;
2005-11-11 00:48:20 +01:00
if ( phase2 = = null ) return " no connection to remote address " + targetAddress + " ; phase 2 " ;
2008-06-06 18:01:27 +02:00
response = phase2 . get ( " response " ) ;
2005-11-11 00:48:20 +01:00
if ( response = = null ) return " wrong return values from other peer; phase 2 " ;
if ( ! ( response . equals ( " ok " ) ) ) {
return " remote peer failed with transfer: " + response ;
}
return null ;
}
return " wrong protocol: " + protocol ;
}
2005-09-21 23:32:43 +02:00
2009-10-11 02:12:19 +02:00
public static HashMap < String , String > crawlReceipt ( final yacySeed mySeed , final yacySeed target , final String process , final String result , final String reason , final URIMetadataRow entry , final String wordhashes ) {
2007-07-05 01:48:52 +02:00
assert ( target ! = null ) ;
2008-05-06 01:13:47 +02:00
assert ( mySeed ! = null ) ;
assert ( mySeed ! = target ) ;
2005-09-21 23:32:43 +02:00
2005-04-07 21:19:42 +02:00
/ *
the result can have one of the following values :
negative cases , no retry
unavailable - the resource is not avaiable ( a broken link ) ; not found or interrupted
robot - a robot - file has denied to crawl that resource
2005-05-07 23:11:18 +02:00
2005-04-07 21:19:42 +02:00
negative cases , retry possible
rejected - the peer has rejected to load the resource
dequeue - peer too busy - rejected to crawl
positive cases with crawling
fill - the resource was loaded and processed
update - the resource was already in database but re - loaded and processed
2005-05-07 23:11:18 +02:00
positive cases without crawling
2005-04-07 21:19:42 +02:00
known - the resource is already in database , believed to be fresh and not reloaded
stale - the resource was reloaded but not processed because source had no changes
2005-05-07 23:11:18 +02:00
* /
2005-10-22 15:28:04 +02:00
2007-07-05 01:48:52 +02:00
// prepare request
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , target . hash , salt ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " process " , process ) ) ;
2010-03-26 19:33:20 +01:00
post . add ( new DefaultCharsetStringPart ( " urlhash " , ( ( entry = = null ) ? " " : new String ( entry . hash ( ) ) ) ) ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " result " , result ) ) ;
post . add ( new DefaultCharsetStringPart ( " reason " , reason ) ) ;
post . add ( new DefaultCharsetStringPart ( " wordh " , wordhashes ) ) ;
post . add ( new DefaultCharsetStringPart ( " lurlEntry " , ( ( entry = = null ) ? " " : crypt . simpleEncode ( entry . toString ( ) , salt ) ) ) ) ;
2007-07-05 01:48:52 +02:00
2005-10-22 15:28:04 +02:00
// determining target address
2007-07-05 01:48:52 +02:00
final String address = target . getClusterAddress ( ) ;
2005-09-21 23:32:43 +02:00
if ( address = = null ) { return null ; }
2007-07-05 01:48:52 +02:00
// send request
2005-04-07 21:19:42 +02:00
try {
2008-08-02 14:12:04 +02:00
final byte [ ] content = wput ( " http:// " + address + " /yacy/crawlReceipt.html " , target . getHexHash ( ) + " .yacyh " , post , 10000 ) ;
2009-10-05 22:11:41 +02:00
return FileUtils . table ( content ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-05-07 23:11:18 +02:00
// most probably a network time-out exception
2005-08-30 23:32:59 +02:00
yacyCore . log . logSevere ( " yacyClient.crawlReceipt error: " + e . getMessage ( ) ) ;
2005-05-07 23:11:18 +02:00
return null ;
}
2005-04-07 21:19:42 +02:00
}
2005-09-21 23:32:43 +02:00
2009-02-16 22:28:48 +01:00
/ * *
* transfer the index . If the transmission fails , return a string describing the cause .
* If everything is ok , return null .
* @param targetSeed
* @param indexes
* @param urlCache
* @param gzipBody
* @param timeout
* @return
* /
public static String transferIndex (
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
final yacySeed targetSeed ,
2009-04-15 08:34:27 +02:00
final ReferenceContainerCache < WordReference > indexes ,
2009-10-11 02:12:19 +02:00
final HashMap < String , URIMetadataRow > urlCache ,
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
final boolean gzipBody ,
final int timeout ) {
2005-10-22 15:28:04 +02:00
2008-08-02 14:12:04 +02:00
final HashMap < String , Object > resultObj = new HashMap < String , Object > ( ) ;
2006-06-14 11:40:42 +02:00
int payloadSize = 0 ;
try {
// check if we got all necessary urls in the urlCache (only for debugging)
2009-04-15 08:34:27 +02:00
Iterator < WordReference > eenum ;
2009-03-02 00:58:14 +01:00
Reference entry ;
2009-04-15 08:34:27 +02:00
for ( ReferenceContainer < WordReference > ic : indexes ) {
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
eenum = ic . entries ( ) ;
2006-06-14 11:40:42 +02:00
while ( eenum . hasNext ( ) ) {
2008-06-06 18:01:27 +02:00
entry = eenum . next ( ) ;
2009-04-07 11:34:41 +02:00
if ( urlCache . get ( entry . metadataHash ( ) ) = = null ) {
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " DEBUG transferIndex: to-send url hash ' " + entry . metadataHash ( ) + " ' is not contained in urlCache " ) ;
2006-06-14 11:40:42 +02:00
}
2006-01-30 13:42:06 +01:00
}
2006-06-14 11:40:42 +02:00
}
// transfer the RWI without the URLs
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
HashMap < String , String > in = transferRWI ( targetSeed , indexes , gzipBody , timeout ) ;
2006-06-14 11:40:42 +02:00
resultObj . put ( " resultTransferRWI " , in ) ;
2008-03-12 01:24:20 +01:00
2006-06-14 11:40:42 +02:00
if ( in = = null ) {
2009-02-16 22:28:48 +01:00
return " no connection from transferRWI " ;
}
2008-01-17 19:43:01 +01:00
if ( in . containsKey ( " indexPayloadSize " ) ) payloadSize + = Integer . parseInt ( in . get ( " indexPayloadSize " ) ) ;
2008-03-12 01:24:20 +01:00
2008-06-06 18:01:27 +02:00
String result = in . get ( " result " ) ;
2009-02-16 22:28:48 +01:00
if ( result = = null ) {
return " no result from transferRWI " ;
2006-01-30 13:42:06 +01:00
}
2009-02-16 22:28:48 +01:00
2006-06-14 11:40:42 +02:00
if ( ! ( result . equals ( " ok " ) ) ) {
2009-02-16 22:28:48 +01:00
return result ;
2006-01-30 13:42:06 +01:00
}
2008-03-12 01:24:20 +01:00
2006-06-14 11:40:42 +02:00
// in now contains a list of unknown hashes
2009-02-16 22:28:48 +01:00
String uhss = in . get ( " unknownURL " ) ;
2006-06-14 11:40:42 +02:00
if ( uhss = = null ) {
2009-02-16 22:28:48 +01:00
return " no unknownURL tag in response " ;
2006-06-14 11:40:42 +02:00
}
2009-02-16 22:28:48 +01:00
uhss = uhss . trim ( ) ;
if ( uhss . length ( ) = = 0 | | uhss . equals ( " , " ) ) { return null ; } // all url's known, we are ready here
2008-03-12 01:24:20 +01:00
2006-06-14 11:40:42 +02:00
final String [ ] uhs = uhss . split ( " , " ) ;
2009-02-16 22:28:48 +01:00
if ( uhs . length = = 0 ) { return null ; } // all url's known
2008-03-12 01:24:20 +01:00
2006-06-14 11:40:42 +02:00
// extract the urlCache from the result
2009-10-11 02:12:19 +02:00
final URIMetadataRow [ ] urls = new URIMetadataRow [ uhs . length ] ;
2006-06-14 11:40:42 +02:00
for ( int i = 0 ; i < uhs . length ; i + + ) {
2008-06-06 18:01:27 +02:00
urls [ i ] = urlCache . get ( uhs [ i ] ) ;
2006-06-14 11:40:42 +02:00
if ( urls [ i ] = = null ) {
2008-09-03 02:30:21 +02:00
if ( yacyCore . log . isFine ( ) ) yacyCore . log . logFine ( " DEBUG transferIndex: requested url hash ' " + uhs [ i ] + " ', unknownURL=' " + uhss + " ' " ) ;
2006-06-14 11:40:42 +02:00
}
}
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
in = transferURL ( targetSeed , urls , gzipBody , timeout ) ;
2006-06-14 11:40:42 +02:00
resultObj . put ( " resultTransferURL " , in ) ;
if ( in = = null ) {
2009-02-16 22:28:48 +01:00
return " no connection from transferURL " ;
2006-06-14 11:40:42 +02:00
}
2009-02-16 22:28:48 +01:00
2008-01-17 19:43:01 +01:00
if ( in . containsKey ( " urlPayloadSize " ) ) payloadSize + = Integer . parseInt ( in . get ( " urlPayloadSize " ) ) ;
2006-06-14 11:40:42 +02:00
2008-06-06 18:01:27 +02:00
result = in . get ( " result " ) ;
2006-06-14 11:40:42 +02:00
if ( result = = null ) {
2009-02-16 22:28:48 +01:00
return " no result from transferURL " ;
2006-06-14 11:40:42 +02:00
}
2009-02-16 22:28:48 +01:00
2006-06-14 11:40:42 +02:00
if ( ! ( result . equals ( " ok " ) ) ) {
2009-02-16 22:28:48 +01:00
return result ;
2006-06-14 11:40:42 +02:00
}
2009-02-16 22:28:48 +01:00
return null ;
2006-06-14 11:40:42 +02:00
} finally {
2008-08-06 21:43:12 +02:00
resultObj . put ( " payloadSize " , Integer . valueOf ( payloadSize ) ) ;
2005-04-07 21:19:42 +02:00
}
}
2005-09-21 23:32:43 +02:00
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
private static HashMap < String , String > transferRWI (
final yacySeed targetSeed ,
2009-04-15 08:34:27 +02:00
final ReferenceContainerCache < WordReference > indexes ,
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
boolean gzipBody ,
final int timeout ) {
2007-04-30 00:05:34 +02:00
final String address = targetSeed . getPublicAddress ( ) ;
2008-08-02 15:57:00 +02:00
if ( address = = null ) { yacyCore . log . logWarning ( " no address for transferRWI " ) ; return null ; }
2007-07-05 01:48:52 +02:00
2005-04-07 21:19:42 +02:00
// prepare post values
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , targetSeed . hash , salt ) ;
2005-09-22 12:30:55 +02:00
// enabling gzip compression for post request body
2008-05-04 12:53:04 +02:00
if ( gzipBody & & ( targetSeed . getVersion ( ) < yacyVersion . YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED ) ) {
gzipBody = false ;
}
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
post . add ( new DefaultCharsetStringPart ( " wordc " , Integer . toString ( indexes . size ( ) ) ) ) ;
2005-10-22 15:28:04 +02:00
2005-04-07 21:19:42 +02:00
int indexcount = 0 ;
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
final StringBuilder entrypost = new StringBuilder ( indexes . size ( ) * 73 ) ;
2009-04-15 08:34:27 +02:00
Iterator < WordReference > eenum ;
2009-03-02 00:58:14 +01:00
Reference entry ;
2009-04-15 08:34:27 +02:00
for ( ReferenceContainer < WordReference > ic : indexes ) {
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
eenum = ic . entries ( ) ;
2005-10-13 15:57:15 +02:00
while ( eenum . hasNext ( ) ) {
2008-06-06 18:01:27 +02:00
entry = eenum . next ( ) ;
2009-04-20 21:51:20 +02:00
entrypost . append ( ic . getTermHashAsString ( ) )
2006-11-17 15:17:20 +01:00
. append ( entry . toPropertyForm ( ) )
2007-12-14 20:17:54 +01:00
. append ( serverCore . CRLF_STRING ) ;
2005-08-13 00:14:24 +02:00
indexcount + + ;
2005-04-07 21:19:42 +02:00
}
}
2005-09-21 23:32:43 +02:00
2005-08-13 00:14:24 +02:00
if ( indexcount = = 0 ) {
// nothing to do but everything ok
2008-01-22 12:51:43 +01:00
final HashMap < String , String > result = new HashMap < String , String > ( 2 ) ;
2005-08-13 00:14:24 +02:00
result . put ( " result " , " ok " ) ;
result . put ( " unknownURL " , " " ) ;
return result ;
2005-04-07 21:19:42 +02:00
}
2005-09-21 23:32:43 +02:00
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " entryc " , Integer . toString ( indexcount ) ) ) ;
post . add ( new DefaultCharsetStringPart ( " indexes " , entrypost . toString ( ) ) ) ;
2005-05-07 23:11:18 +02:00
try {
2008-06-08 01:48:46 +02:00
final byte [ ] content = wput ( " http:// " + address + " /yacy/transferRWI.html " , targetSeed . getHexHash ( ) + " .yacyh " , post , timeout , gzipBody ) ;
2009-10-05 22:11:41 +02:00
final Iterator < String > v = FileUtils . strings ( content ) ;
2008-05-04 12:53:04 +02:00
// this should return a list of urlhashes that are unknown
2005-04-07 21:19:42 +02:00
2009-01-31 02:06:56 +01:00
final HashMap < String , String > result = FileUtils . table ( v ) ;
2006-06-14 11:40:42 +02:00
// return the transfered index data in bytes (for debugging only)
2008-01-17 19:43:01 +01:00
result . put ( " indexPayloadSize " , Integer . toString ( entrypost . length ( ) ) ) ;
2005-04-07 21:19:42 +02:00
return result ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2009-06-02 00:45:28 +02:00
yacyCore . log . logInfo ( " yacyClient.transferRWI error: " + e . getMessage ( ) ) ;
2005-04-07 21:19:42 +02:00
return null ;
}
}
2005-09-21 23:32:43 +02:00
2009-10-11 02:12:19 +02:00
private static HashMap < String , String > transferURL ( final yacySeed targetSeed , final URIMetadataRow [ ] urls , boolean gzipBody , final int timeout ) {
2005-05-07 23:11:18 +02:00
// this post a message to the remote message board
2007-04-30 00:05:34 +02:00
final String address = targetSeed . getPublicAddress ( ) ;
2005-09-21 23:32:43 +02:00
if ( address = = null ) { return null ; }
2007-07-05 01:48:52 +02:00
2005-04-07 21:19:42 +02:00
// prepare post values
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , targetSeed . hash , salt ) ;
2005-09-22 12:30:55 +02:00
// enabling gzip compression for post request body
2008-05-04 12:53:04 +02:00
if ( gzipBody & & ( targetSeed . getVersion ( ) < yacyVersion . YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED ) ) {
gzipBody = false ;
}
2005-09-22 12:30:55 +02:00
2010-04-08 02:11:32 +02:00
String resource ;
2005-04-07 21:19:42 +02:00
int urlc = 0 ;
2006-06-14 11:40:42 +02:00
int urlPayloadSize = 0 ;
2005-04-07 21:19:42 +02:00
for ( int i = 0 ; i < urls . length ; i + + ) {
if ( urls [ i ] ! = null ) {
2010-04-08 02:11:32 +02:00
resource = urls [ i ] . toString ( ) ;
//System.out.println("*** DEBUG resource = " + resource);
2010-04-10 02:21:07 +02:00
if ( resource ! = null & & resource . indexOf ( 0 ) = = - 1 ) {
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " url " + urlc , resource ) ) ;
2006-06-14 11:40:42 +02:00
urlPayloadSize + = resource . length ( ) ;
2005-04-07 21:19:42 +02:00
urlc + + ;
}
}
}
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " urlc " , Integer . toString ( urlc ) ) ) ;
2005-05-07 23:11:18 +02:00
try {
2008-06-08 01:48:46 +02:00
final byte [ ] content = wput ( " http:// " + address + " /yacy/transferURL.html " , targetSeed . getHexHash ( ) + " .yacyh " , post , timeout , gzipBody ) ;
2009-10-05 22:11:41 +02:00
final Iterator < String > v = FileUtils . strings ( content ) ;
2005-10-22 15:28:04 +02:00
2009-01-31 02:06:56 +01:00
final HashMap < String , String > result = FileUtils . table ( v ) ;
2006-06-14 11:40:42 +02:00
// return the transfered url data in bytes (for debugging only)
2008-01-17 19:43:01 +01:00
result . put ( " urlPayloadSize " , Integer . toString ( urlPayloadSize ) ) ;
2006-06-14 11:40:42 +02:00
return result ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2006-06-21 16:25:27 +02:00
yacyCore . log . logSevere ( " yacyClient.transferURL error: " + e . getMessage ( ) ) ;
2005-04-07 21:19:42 +02:00
return null ;
}
}
2005-09-21 23:32:43 +02:00
2008-08-02 14:12:04 +02:00
public static HashMap < String , String > getProfile ( final yacySeed targetSeed ) {
2007-07-05 01:48:52 +02:00
2005-05-07 23:11:18 +02:00
// this post a message to the remote message board
2008-04-12 10:12:51 +02:00
final String salt = crypt . randomSalt ( ) ;
2009-07-19 22:37:44 +02:00
final List < Part > post = yacyNetwork . basicRequestPost ( Switchboard . getSwitchboard ( ) , targetSeed . hash , salt ) ;
2007-07-05 01:48:52 +02:00
2007-04-30 00:05:34 +02:00
String address = targetSeed . getClusterAddress ( ) ;
2005-09-21 23:32:43 +02:00
if ( address = = null ) { address = " localhost:8080 " ; }
2005-05-07 23:11:18 +02:00
try {
2008-06-05 13:01:20 +02:00
final byte [ ] content = wput ( " http:// " + address + " /yacy/profile.html " , targetSeed . getHexHash ( ) + " .yacyh " , post , 5000 ) ;
2009-10-05 22:11:41 +02:00
return FileUtils . table ( content ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2005-08-30 23:32:59 +02:00
yacyCore . log . logSevere ( " yacyClient.getProfile error: " + e . getMessage ( ) ) ;
2005-04-07 21:19:42 +02:00
return null ;
}
}
2007-07-05 01:48:52 +02:00
2008-04-05 15:17:16 +02:00
/ * *
* proxy for " to YaCy connections "
* @return
* /
2009-07-19 22:37:44 +02:00
private static final RemoteProxyConfig proxyConfig ( ) {
final RemoteProxyConfig p = RemoteProxyConfig . getRemoteProxyConfig ( ) ;
2007-07-05 01:48:52 +02:00
return ( ( p ! = null ) & & ( p . useProxy ( ) ) & & ( p . useProxy4Yacy ( ) ) ) ? p : null ;
}
2005-09-21 23:32:43 +02:00
2008-08-02 14:12:04 +02:00
public static void main ( final String [ ] args ) {
2008-05-04 12:53:04 +02:00
if ( args . length > 1 ) {
2005-04-07 21:19:42 +02:00
System . out . println ( " yacyClient Test " ) ;
try {
2010-01-21 23:06:03 +01:00
final Switchboard sb = new Switchboard ( new File ( args [ 0 ] ) , " httpProxy.init " , " DATA/SETTINGS/yacy.conf " ) ;
2005-12-05 15:24:13 +01:00
/*final yacyCore core =*/ new yacyCore ( sb ) ;
2008-06-04 23:34:57 +02:00
sb . loadSeedLists ( ) ;
2009-05-28 16:26:05 +02:00
final yacySeed target = sb . peers . getConnected ( args [ 1 ] ) ;
2009-04-16 17:29:00 +02:00
final byte [ ] wordhashe = Word . word2hash ( " test " ) ;
2005-05-07 23:11:18 +02:00
//System.out.println("permission=" + permissionMessage(args[1]));
2009-07-19 22:37:44 +02:00
final RequestHeader reqHeader = new RequestHeader ( ) ;
reqHeader . put ( HeaderFramework . USER_AGENT , HTTPLoader . crawlerUserAgent ) ;
final byte [ ] content = Client . wget (
2008-04-05 15:17:16 +02:00
" http:// " + target . getPublicAddress ( ) + " /yacy/search.html " +
2009-05-28 16:26:05 +02:00
" ?myseed= " + sb . peers . mySeed ( ) . genSeedStr ( null ) +
2008-04-05 15:17:16 +02:00
" &youare= " + target . hash + " &key= " +
2009-05-28 16:26:05 +02:00
" &myseed= " + sb . peers . mySeed ( ) . genSeedStr ( null ) +
2008-04-05 15:17:16 +02:00
" &count=10 " +
" &resource=global " +
2009-04-16 17:29:00 +02:00
" &query= " + new String ( wordhashe ) +
2009-07-19 22:37:44 +02:00
" &network.unit.name= " + Switchboard . getSwitchboard ( ) . getConfig ( SwitchboardConstants . NETWORK_NAME , yacySeed . DFLT_NETWORK_UNIT ) ,
2008-07-04 13:03:03 +02:00
reqHeader , 10000 , target . getHexHash ( ) + " .yacyh " ) ;
2009-10-05 22:11:41 +02:00
final HashMap < String , String > result = FileUtils . table ( content ) ;
2005-04-07 21:19:42 +02:00
System . out . println ( " Result= " + result . toString ( ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2005-05-07 23:11:18 +02:00
}
2005-04-07 21:19:42 +02:00
System . exit ( 0 ) ;
2008-05-04 12:53:04 +02:00
} else if ( args . length = = 1 ) {
System . out . println ( " wput Test " ) ;
// connection params
URL url = null ;
try {
url = new URL ( args [ 0 ] ) ;
2008-08-02 14:12:04 +02:00
} catch ( final MalformedURLException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-05-04 12:53:04 +02:00
}
if ( url = = null ) {
System . exit ( 1 ) ;
return ;
}
final String vhost = url . getHost ( ) ;
final int timeout = 10000 ;
2008-08-02 15:57:00 +02:00
final boolean gzipBody = false ;
2008-05-04 12:53:04 +02:00
// data
final List < Part > post = new ArrayList < Part > ( ) ;
2008-08-02 15:57:00 +02:00
post . add ( new DefaultCharsetStringPart ( " process " , " permission " ) ) ;
post . add ( new DefaultCharsetStringPart ( " purpose " , " crcon " ) ) ;
2008-05-04 12:53:04 +02:00
//post.add(new FilePart("filename", new ByteArrayPartSource(filename, file)));
// do it!
try {
final byte [ ] response = wput ( url . toString ( ) , vhost , post , timeout , gzipBody ) ;
System . out . println ( new String ( response ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-05-04 12:53:04 +02:00
}
}
2005-04-07 21:19:42 +02:00
}
2005-09-21 23:32:43 +02:00
2005-04-07 21:19:42 +02:00
}