2005-05-07 23:11:18 +02:00
|
|
|
// yacyClient.java
|
2005-04-07 21:19:42 +02:00
|
|
|
// -------------------------------------
|
|
|
|
// (C) by Michael Peter Christen; mc@anomic.de
|
|
|
|
// first published on http://www.anomic.de
|
|
|
|
// Frankfurt, Germany, 2004
|
2005-09-21 23:32:43 +02:00
|
|
|
//
|
|
|
|
// $LastChangedDate$
|
|
|
|
// $LastChangedRevision$
|
|
|
|
// $LastChangedBy$
|
2005-04-07 21:19:42 +02:00
|
|
|
//
|
|
|
|
// This program is free software; you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU General Public License as published by
|
|
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU General Public License
|
|
|
|
// along with this program; if not, write to the Free Software
|
|
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
//
|
|
|
|
// Using this software in any meaning (reading, learning, copying, compiling,
|
|
|
|
// running) means that you agree that the Author(s) is (are) not responsible
|
|
|
|
// for cost, loss of data or any harm that may be caused directly or indirectly
|
|
|
|
// by usage of this softare or this documentation. The usage of this software
|
|
|
|
// is on your own risk. The installation and usage (starting/running) of this
|
|
|
|
// software may allow other people or application to access your computer and
|
|
|
|
// any attached devices and is highly dependent on the configuration of the
|
|
|
|
// software which must be done by the user of the software; the author(s) is
|
|
|
|
// (are) also not responsible for proper configuration and usage of the
|
|
|
|
// software, even if provoked by documentation provided together with
|
|
|
|
// the software.
|
|
|
|
//
|
|
|
|
// Any changes to this file according to the GPL as documented in the file
|
|
|
|
// gpl.txt aside this file in the shipment you received can be done to the
|
|
|
|
// lines that follows this copyright notice here, but changes must not be
|
|
|
|
// done inside the copyright notice above. A re-distribution must contain
|
|
|
|
// the intact and unchanged copyright notice.
|
|
|
|
// Contributions and changes to the program code must be marked as such.
|
|
|
|
|
|
|
|
package de.anomic.yacy;
|
|
|
|
|
2005-05-05 07:32:19 +02:00
|
|
|
import java.io.IOException;
|
2006-01-10 17:48:59 +01:00
|
|
|
import java.io.UnsupportedEncodingException;
|
2005-05-05 07:32:19 +02:00
|
|
|
import java.net.URL;
|
2005-10-05 12:45:33 +02:00
|
|
|
import java.util.ArrayList;
|
2005-05-05 07:32:19 +02:00
|
|
|
import java.util.Date;
|
|
|
|
import java.util.HashMap;
|
2005-10-13 15:57:15 +02:00
|
|
|
import java.util.Iterator;
|
2006-01-30 01:42:38 +01:00
|
|
|
|
|
|
|
import de.anomic.htmlFilter.htmlFilterContentScraper;
|
2005-05-05 07:32:19 +02:00
|
|
|
import de.anomic.http.httpc;
|
2006-01-04 01:39:00 +01:00
|
|
|
import de.anomic.kelondro.kelondroBase64Order;
|
2005-05-05 07:32:19 +02:00
|
|
|
import de.anomic.plasma.plasmaCrawlLURL;
|
2006-02-26 12:30:37 +01:00
|
|
|
import de.anomic.plasma.plasmaSearchRankingProfile;
|
2005-07-06 16:48:41 +02:00
|
|
|
import de.anomic.plasma.plasmaSnippetCache;
|
2005-05-05 07:32:19 +02:00
|
|
|
import de.anomic.plasma.plasmaSwitchboard;
|
|
|
|
import de.anomic.plasma.plasmaWordIndexEntry;
|
2005-05-07 23:11:18 +02:00
|
|
|
import de.anomic.plasma.plasmaWordIndexEntryContainer;
|
2005-07-12 02:07:09 +02:00
|
|
|
import de.anomic.plasma.plasmaURLPattern;
|
2006-02-05 00:11:31 +01:00
|
|
|
import de.anomic.plasma.plasmaSearchTimingProfile;
|
2005-05-05 07:32:19 +02:00
|
|
|
import de.anomic.server.serverCore;
|
|
|
|
import de.anomic.server.serverObjects;
|
2005-11-11 00:48:20 +01:00
|
|
|
import de.anomic.server.serverCodings;
|
2005-05-05 07:32:19 +02:00
|
|
|
import de.anomic.tools.crypt;
|
|
|
|
import de.anomic.tools.nxTools;
|
2005-09-21 23:32:43 +02:00
|
|
|
import de.anomic.yacy.yacySeed;
|
2005-09-22 12:30:55 +02:00
|
|
|
import de.anomic.yacy.yacyVersion;
|
2005-04-07 21:19:42 +02:00
|
|
|
|
2005-10-05 12:45:33 +02:00
|
|
|
public final class yacyClient {
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
public static int publishMySeed(String address, String otherHash) {
|
2005-05-07 23:11:18 +02:00
|
|
|
// this is called to enrich the seed information by
|
|
|
|
// - own address (if peer is behind a nat/router)
|
|
|
|
// - check peer type (virgin/junior/senior/principal)
|
|
|
|
// to do this, we send a 'Hello' to another peer
|
|
|
|
// this carries the following information:
|
|
|
|
// 'iam' - own hash
|
|
|
|
// 'youare' - remote hash, to verify that we are correct
|
|
|
|
// 'key' - a session key that the remote peer may use to answer
|
|
|
|
// and the own seed string
|
|
|
|
// we expect the following information to be send back:
|
|
|
|
// - 'yourip' the ip of the connection peer (we)
|
|
|
|
// - 'yourtype' the type of this peer that the other peer checked by asking for a specific word
|
|
|
|
// and the remote seed string
|
|
|
|
// the number of new seeds are returned
|
|
|
|
// one exceptional failure case is when we know the other's peers hash, the other peers responds correctly
|
|
|
|
// but they appear to be another peer by comparisment of the other peer's hash
|
|
|
|
// this works of course only if we know the other peer's hash.
|
|
|
|
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-05-07 23:11:18 +02:00
|
|
|
HashMap result = null;
|
|
|
|
try {
|
|
|
|
/*
|
|
|
|
URL url = new URL("http://" + address + "/yacy/hello.html?iam=" +
|
|
|
|
yacyCore.seedCache.mySeed.hash +
|
|
|
|
"&pattern=&count=20" +
|
|
|
|
"&key=" + key + "&seed=" + yacyCore.seedCache.mySeed.genSeedStr(key));
|
|
|
|
yacyCore.log.logDebug("HELLO to URL " + url.toString());
|
|
|
|
result = nxTools.table(httpc.wget(url,
|
|
|
|
10000, null, null, yacyCore.seedCache.sb.remoteProxyHost, yacyCore.seedCache.sb.remoteProxyPort));
|
|
|
|
*/
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-10-22 15:28:04 +02:00
|
|
|
// building URL
|
2005-09-21 23:32:43 +02:00
|
|
|
final URL url = new URL("http://" + address + "/yacy/hello.html");
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// adding all needed parameters
|
2005-10-05 12:45:33 +02:00
|
|
|
final serverObjects obj = new serverObjects(6);
|
2005-10-22 15:28:04 +02:00
|
|
|
obj.put("iam", yacyCore.seedDB.mySeed.hash);
|
|
|
|
obj.put("pattern", "");
|
|
|
|
obj.put("count", "20");
|
|
|
|
obj.put("key", key);
|
|
|
|
obj.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
|
|
|
|
obj.put("myUTC", System.currentTimeMillis());
|
|
|
|
obj.put(yacySeed.SEED, yacyCore.seedDB.mySeed.genSeedStr(key));
|
|
|
|
|
|
|
|
// sending request
|
|
|
|
result = nxTools.table(
|
2006-05-09 15:11:00 +02:00
|
|
|
httpc.wput(url,
|
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
105000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
obj,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
)
|
|
|
|
);
|
2005-05-07 23:11:18 +02:00
|
|
|
} catch (Exception e) {
|
2005-07-07 15:58:54 +02:00
|
|
|
if (Thread.currentThread().isInterrupted()) {
|
2005-08-30 23:10:39 +02:00
|
|
|
yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' interrupted.");
|
2005-07-07 15:58:54 +02:00
|
|
|
} else {
|
2005-08-30 23:10:39 +02:00
|
|
|
yacyCore.log.logFine("yacyClient.publishMySeed exception:" + e.getMessage());
|
2005-07-07 15:58:54 +02:00
|
|
|
}
|
2005-05-07 23:11:18 +02:00
|
|
|
return -1;
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
if (result == null || result.size() < 3) {
|
2005-08-30 23:10:39 +02:00
|
|
|
yacyCore.log.logFine("yacyClient.publishMySeed result error: " +
|
2005-05-07 23:11:18 +02:00
|
|
|
((result == null) ? "result null" : ("result=" + result.toString())));
|
|
|
|
return -1;
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// check consistency with expectation
|
2005-06-23 13:00:26 +02:00
|
|
|
yacySeed otherPeer = null;
|
2005-06-16 09:28:07 +02:00
|
|
|
float otherPeerVersion = 0;
|
2005-09-21 23:32:43 +02:00
|
|
|
if (otherHash != null && otherHash.length() > 0) {
|
2006-04-06 18:28:28 +02:00
|
|
|
otherPeer = yacySeed.genRemoteSeed((String) result.get("seed0"), key, true);
|
2005-09-21 23:32:43 +02:00
|
|
|
if (otherPeer == null || !otherPeer.hash.equals(otherHash)) {
|
2005-08-30 23:10:39 +02:00
|
|
|
yacyCore.log.logFine("yacyClient.publishMySeed: consistency error: other peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' wrong");
|
2005-05-07 23:11:18 +02:00
|
|
|
return -1; // no success
|
|
|
|
}
|
2005-06-16 09:28:07 +02:00
|
|
|
otherPeerVersion = otherPeer.getVersion();
|
2005-05-07 23:11:18 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// set my own seed according to new information
|
2005-09-21 23:32:43 +02:00
|
|
|
final yacySeed mySeedBkp = (yacySeed) yacyCore.seedDB.mySeed.clone();
|
2005-06-16 13:40:36 +02:00
|
|
|
|
|
|
|
// we overwrite our own IP number only, if we do not portForwarding
|
2006-02-28 13:31:57 +01:00
|
|
|
if (serverCore.portForwardingEnabled || serverCore.useStaticIP) {
|
2005-10-17 17:46:12 +02:00
|
|
|
yacyCore.seedDB.mySeed.put(yacySeed.IP, serverCore.publicIP());
|
2005-09-21 23:32:43 +02:00
|
|
|
} else {
|
2005-10-17 17:46:12 +02:00
|
|
|
yacyCore.seedDB.mySeed.put(yacySeed.IP, (String) result.get(yacySeed.YOURIP));
|
2005-06-16 09:28:07 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-06-16 09:28:07 +02:00
|
|
|
/* If we have port forwarding enabled but the other peer uses a too old yacy version
|
|
|
|
* we can ignore the seed-type that was reported by the peer.
|
|
|
|
*
|
|
|
|
* Otherwise we have to change our seed-type
|
|
|
|
*
|
|
|
|
* @see serverCore#portForwardingEnabled
|
|
|
|
*/
|
2005-09-22 12:30:55 +02:00
|
|
|
if (!serverCore.portForwardingEnabled || otherPeerVersion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) {
|
2005-10-17 17:46:12 +02:00
|
|
|
String mytype = (String) result.get(yacySeed.YOURTYPE);
|
2005-09-21 23:32:43 +02:00
|
|
|
if (mytype == null) { mytype = yacySeed.PEERTYPE_JUNIOR; }
|
2006-01-23 00:14:37 +01:00
|
|
|
yacyAccessible accessible = new yacyAccessible();
|
2006-01-25 02:50:24 +01:00
|
|
|
if (mytype.equals(yacySeed.PEERTYPE_SENIOR)||mytype.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
|
2006-01-23 00:14:37 +01:00
|
|
|
accessible.IWasAccessed = true;
|
|
|
|
if (yacyCore.seedDB.mySeed.isPrincipal()) {
|
|
|
|
mytype = yacySeed.PEERTYPE_PRINCIPAL;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
accessible.IWasAccessed = false;
|
2005-07-04 13:09:48 +02:00
|
|
|
}
|
2006-01-23 00:14:37 +01:00
|
|
|
accessible.lastUpdated = System.currentTimeMillis();
|
|
|
|
yacyCore.amIAccessibleDB.put(otherHash, accessible);
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-07-04 13:09:48 +02:00
|
|
|
/*
|
|
|
|
* If we were reported as junior we have to check if your port forwarding channel is broken
|
|
|
|
* If this is true we try to reconnect the sch channel to the remote server now.
|
|
|
|
*/
|
|
|
|
if (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_JUNIOR)) {
|
2005-06-23 13:00:26 +02:00
|
|
|
yacyCore.log.logInfo("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as junior.");
|
2005-07-04 13:09:48 +02:00
|
|
|
if (serverCore.portForwardingEnabled) {
|
2005-09-21 23:32:43 +02:00
|
|
|
if (!Thread.currentThread().isInterrupted() &&
|
|
|
|
serverCore.portForwarding != null &&
|
|
|
|
!serverCore.portForwarding.isConnected()
|
2005-07-04 13:09:48 +02:00
|
|
|
) {
|
|
|
|
yacyCore.log.logWarning("yacyClient.publishMySeed: Broken portForwarding channel detected. Trying to reconnect ...");
|
|
|
|
try {
|
|
|
|
serverCore.portForwarding.reconnect();
|
|
|
|
} catch (IOException e) {
|
|
|
|
yacyCore.log.logWarning("yacyClient.publishMySeed: Unable to reconnect to port forwarding host.");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2005-08-29 13:39:10 +02:00
|
|
|
} else {
|
2005-08-30 23:10:39 +02:00
|
|
|
yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ".");
|
2005-06-23 13:00:26 +02:00
|
|
|
}
|
2006-01-23 00:14:37 +01:00
|
|
|
if (yacyCore.seedDB.mySeed.orVirgin().equals(yacySeed.PEERTYPE_VIRGIN))
|
|
|
|
yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, mytype);
|
2005-06-16 09:28:07 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
|
|
|
final String error = yacyCore.seedDB.mySeed.isProper();
|
|
|
|
if (error != null) {
|
2005-05-07 23:11:18 +02:00
|
|
|
yacyCore.seedDB.mySeed = mySeedBkp;
|
2005-08-30 23:10:39 +02:00
|
|
|
yacyCore.log.logFine("yacyClient.publishMySeed mySeed error - not proper: " + error);
|
2005-05-07 23:11:18 +02:00
|
|
|
return -1;
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-10-17 17:46:12 +02:00
|
|
|
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
|
2005-09-27 18:28:55 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// read the seeds that the peer returned and integrate them into own database
|
2005-04-07 21:19:42 +02:00
|
|
|
int i = 0;
|
2005-05-07 23:11:18 +02:00
|
|
|
int count = 0;
|
2005-09-21 23:32:43 +02:00
|
|
|
String seedStr;
|
2005-10-17 17:46:12 +02:00
|
|
|
while ((seedStr = (String) result.get(yacySeed.SEED + i++)) != null) {
|
2005-05-07 23:11:18 +02:00
|
|
|
// integrate new seed into own database
|
|
|
|
// the first seed, "seed0" is the seed of the responding peer
|
2006-04-06 18:28:28 +02:00
|
|
|
if (yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(seedStr, key, true), (i == 1))) count++;
|
2005-05-07 23:11:18 +02:00
|
|
|
}
|
|
|
|
return count;
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
public static yacySeed querySeed(yacySeed target, String seedHash) {
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-04-07 21:19:42 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// sending request
|
|
|
|
final HashMap result = nxTools.table(
|
|
|
|
httpc.wget(
|
|
|
|
new URL("http://" + target.getAddress() +
|
|
|
|
"/yacy/query.html" +
|
|
|
|
"?iam=" + yacyCore.seedDB.mySeed.hash +
|
|
|
|
"&youare=" + target.hash +
|
|
|
|
"&key=" + key +
|
|
|
|
"&object=seed" +
|
|
|
|
"&env=" + seedHash
|
|
|
|
),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
10000,
|
|
|
|
null,
|
|
|
|
null,
|
2006-05-09 15:11:00 +02:00
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
)
|
|
|
|
);
|
|
|
|
|
2005-09-21 23:32:43 +02:00
|
|
|
if (result == null || result.size() == 0) { return null; }
|
2005-10-17 17:46:12 +02:00
|
|
|
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
|
2006-04-06 18:28:28 +02:00
|
|
|
return yacySeed.genRemoteSeed((String) result.get("response"), key, true);
|
2005-04-07 21:19:42 +02:00
|
|
|
} catch (Exception e) {
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.querySeed error:" + e.getMessage());
|
2005-04-07 21:19:42 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
public static int queryRWICount(yacySeed target, String wordHash) {
|
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// sending request
|
|
|
|
final HashMap result = nxTools.table(
|
|
|
|
httpc.wget(
|
|
|
|
new URL("http://" + target.getAddress() +
|
|
|
|
"/yacy/query.html" +
|
|
|
|
"?iam=" + yacyCore.seedDB.mySeed.hash +
|
|
|
|
"&youare=" + target.hash +
|
|
|
|
"&key=" +
|
|
|
|
"&object=rwicount" +
|
|
|
|
"&env=" + wordHash +
|
|
|
|
"&ttl=0"
|
|
|
|
),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
10000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null
|
|
|
|
)
|
|
|
|
);
|
|
|
|
|
2005-09-21 23:32:43 +02:00
|
|
|
if (result == null || result.size() == 0) { return -1; }
|
2005-05-07 23:11:18 +02:00
|
|
|
return Integer.parseInt((String) result.get("response"));
|
|
|
|
} catch (Exception e) {
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.queryRWICount error:" + e.getMessage());
|
2005-05-07 23:11:18 +02:00
|
|
|
return -1;
|
|
|
|
}
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-10-22 15:28:04 +02:00
|
|
|
public static int queryUrlCount(yacySeed target) {
|
2005-09-21 23:32:43 +02:00
|
|
|
if (target == null) { return -1; }
|
2005-05-07 23:11:18 +02:00
|
|
|
if (yacyCore.seedDB.mySeed == null) return -1;
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// building url
|
2005-09-21 23:32:43 +02:00
|
|
|
final String querystr =
|
2005-10-22 15:28:04 +02:00
|
|
|
"http://" + target.getAddress() +
|
|
|
|
"/yacy/query.html" +
|
|
|
|
"?iam=" + yacyCore.seedDB.mySeed.hash +
|
|
|
|
"&youare=" + target.hash +
|
|
|
|
"&key=" +
|
|
|
|
"&object=lurlcount" +
|
|
|
|
"&env=" +
|
|
|
|
"&ttl=0";
|
|
|
|
|
|
|
|
// seinding request
|
2005-05-07 23:11:18 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
final HashMap result = nxTools.table(
|
|
|
|
httpc.wget(
|
2006-05-09 15:11:00 +02:00
|
|
|
new URL(querystr),
|
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
6000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null
|
|
|
|
)
|
|
|
|
);
|
|
|
|
|
2005-09-21 23:32:43 +02:00
|
|
|
// yacyCore.log("DEBUG QUERY: query=" + querystr + "; result = " + result.toString());
|
2005-05-07 23:11:18 +02:00
|
|
|
if ((result == null) || (result.size() == 0)) return -1;
|
2005-09-21 23:32:43 +02:00
|
|
|
final String resp = (String) result.get("response");
|
|
|
|
if (resp == null) { return -1; } else { return Integer.parseInt(resp); }
|
2005-05-07 23:11:18 +02:00
|
|
|
} catch (Exception e) {
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
|
2005-05-07 23:11:18 +02:00
|
|
|
return -1;
|
|
|
|
}
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-11-08 13:14:51 +01:00
|
|
|
public static int search(
|
2006-01-30 01:42:38 +01:00
|
|
|
String wordhashes,
|
2006-04-20 12:15:00 +02:00
|
|
|
String prefer,
|
|
|
|
String filter,
|
2006-01-30 01:42:38 +01:00
|
|
|
int maxDistance,
|
2005-11-08 13:14:51 +01:00
|
|
|
boolean global,
|
|
|
|
yacySeed targetPeer,
|
2006-01-30 01:42:38 +01:00
|
|
|
plasmaCrawlLURL urlManager,
|
|
|
|
plasmaWordIndexEntryContainer containerCache,
|
|
|
|
plasmaURLPattern blacklist,
|
|
|
|
plasmaSnippetCache snippets,
|
2006-02-26 12:30:37 +01:00
|
|
|
plasmaSearchTimingProfile timingProfile,
|
|
|
|
plasmaSearchRankingProfile rankingProfile
|
2005-11-08 13:14:51 +01:00
|
|
|
) {
|
2005-05-07 23:11:18 +02:00
|
|
|
// send a search request to peer with remote Hash
|
|
|
|
// this mainly converts the words into word hashes
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// INPUT:
|
|
|
|
// iam : complete seed of the requesting peer
|
|
|
|
// youare : seed hash of the target peer, used for testing network stability
|
|
|
|
// key : transmission key for response
|
|
|
|
// search : a list of search words
|
|
|
|
// hsearch : a string of word hashes
|
|
|
|
// fwdep : forward depth. if "0" then peer may NOT ask another peer for more results
|
|
|
|
// fwden : forward deny, a list of seed hashes. They may NOT be target of forward hopping
|
|
|
|
// count : maximum number of wanted results
|
|
|
|
// global : if "true", then result may consist of answers from other peers
|
2005-04-07 21:19:42 +02:00
|
|
|
// duetime : maximum time that a peer should spent to create a result
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// request result
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-10-22 15:28:04 +02:00
|
|
|
try {
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// building url
|
2005-09-21 23:32:43 +02:00
|
|
|
final String url = "http://" + targetPeer.getAddress() + "/yacy/search.html";
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// adding all needed parameters
|
2005-05-07 23:11:18 +02:00
|
|
|
/*
|
|
|
|
String url = "http://" + targetPeer.getAddress() +
|
|
|
|
"/yacy/search.html?myseed=" + yacyCore.seedCache.mySeed.genSeedStr(key) +
|
|
|
|
"&youare=" + targetPeer.hash + "&key=" + key +
|
|
|
|
"&myseed=" + yacyCore.seedCache.mySeed.genSeedStr(key) +
|
|
|
|
"&count=" + count + "&resource=" + ((global) ? "global" : "local") +
|
|
|
|
"&query=" + wordhashes;
|
|
|
|
*/
|
2005-10-05 12:45:33 +02:00
|
|
|
final serverObjects obj = new serverObjects(9);
|
2006-02-26 12:30:37 +01:00
|
|
|
long duetime = timingProfile.duetime();
|
2005-10-23 19:50:27 +02:00
|
|
|
obj.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key));
|
|
|
|
obj.put("youare", targetPeer.hash);
|
|
|
|
obj.put("key", key);
|
2006-02-26 12:30:37 +01:00
|
|
|
obj.put("count", timingProfile.getTargetCount(plasmaSearchTimingProfile.PROCESS_POSTSORT));
|
2005-10-23 19:50:27 +02:00
|
|
|
obj.put("resource", ((global) ? "global" : "local"));
|
|
|
|
obj.put("query", wordhashes);
|
2006-04-20 12:15:00 +02:00
|
|
|
obj.put("prefer", prefer);
|
|
|
|
obj.put("filter", filter);
|
2005-10-23 19:50:27 +02:00
|
|
|
obj.put("ttl", "0");
|
|
|
|
obj.put("duetime", Long.toString(duetime));
|
2006-02-26 12:30:37 +01:00
|
|
|
obj.put("profile", timingProfile.targetToString()); // new duetimes splitted by specific search tasks
|
2006-01-30 01:42:38 +01:00
|
|
|
obj.put("maxdist", maxDistance);
|
2006-02-26 12:30:37 +01:00
|
|
|
obj.put("rankingProfile", rankingProfile.toExternalString());
|
2005-10-23 19:50:27 +02:00
|
|
|
obj.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
|
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
//yacyCore.log.logDebug("yacyClient.search url=" + url);
|
2005-09-21 23:32:43 +02:00
|
|
|
final long timestamp = System.currentTimeMillis();
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// sending request
|
|
|
|
final HashMap result = nxTools.table(
|
|
|
|
httpc.wput(
|
|
|
|
new URL(url),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
300000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
obj,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
)
|
|
|
|
);
|
2005-10-23 19:50:27 +02:00
|
|
|
|
|
|
|
// compute all computation times
|
2005-09-21 23:32:43 +02:00
|
|
|
final long totalrequesttime = System.currentTimeMillis() - timestamp;
|
2005-10-23 19:50:27 +02:00
|
|
|
String returnProfile = (String) result.get("profile");
|
2006-02-26 12:30:37 +01:00
|
|
|
if (returnProfile != null) timingProfile.putYield(returnProfile);
|
2005-04-07 21:19:42 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
/*
|
|
|
|
HashMap result = nxTools.table(httpc.wget(new URL(url),
|
2005-04-07 21:19:42 +02:00
|
|
|
300000, null, null, yacyCore.seedCache.remoteProxyHost, yacyCore.seedCache.remoteProxyPort));
|
2005-05-07 23:11:18 +02:00
|
|
|
*/
|
|
|
|
// OUTPUT:
|
|
|
|
// version : application version of responder
|
|
|
|
// uptime : uptime in seconds of responder
|
|
|
|
// total : number of total available LURL's for this search
|
|
|
|
// count : number of returned LURL's for this search
|
|
|
|
// resource<n> : LURL of search
|
|
|
|
// fwhop : hops (depth) of forwards that had been performed to construct this result
|
|
|
|
// fwsrc : peers that helped to construct this result
|
|
|
|
// fwrec : peers that would have helped to construct this result (recommendations)
|
|
|
|
// searchtime : time that the peer actually spent to create the result
|
|
|
|
// references : references (search hints) that was calculated during search
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// now create a plasmaIndex out of this result
|
|
|
|
//System.out.println("yacyClient: search result = " + result.toString()); // debug
|
2005-09-21 23:32:43 +02:00
|
|
|
final int results = Integer.parseInt((String) result.get("count"));
|
2005-04-07 21:19:42 +02:00
|
|
|
//System.out.println("***result count " + results);
|
2006-01-20 16:14:21 +01:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// create containers
|
2005-09-21 23:32:43 +02:00
|
|
|
final int words = wordhashes.length() / plasmaWordIndexEntry.wordHashLength;
|
2005-09-07 17:20:12 +02:00
|
|
|
plasmaWordIndexEntryContainer[] container = new plasmaWordIndexEntryContainer[words];
|
2005-05-07 23:11:18 +02:00
|
|
|
for (int i = 0; i < words; i++) {
|
|
|
|
container[i] = new plasmaWordIndexEntryContainer(wordhashes.substring(i * plasmaWordIndexEntry.wordHashLength, (i + 1) * plasmaWordIndexEntry.wordHashLength));
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// insert results to containers
|
2006-01-20 16:14:21 +01:00
|
|
|
plasmaCrawlLURL.Entry urlEntry;
|
2005-05-07 23:11:18 +02:00
|
|
|
for (int n = 0; n < results; n++) {
|
2005-06-08 02:52:24 +02:00
|
|
|
// get one single search result
|
2006-01-20 16:14:21 +01:00
|
|
|
urlEntry = urlManager.newEntry((String) result.get("resource" + n), true);
|
2006-03-23 17:05:16 +01:00
|
|
|
if ((urlEntry == null) || (blacklist.isListed(urlEntry.url()))) { continue; } // block with backlist
|
2006-03-17 11:16:07 +01:00
|
|
|
urlEntry.store();
|
2006-01-30 01:42:38 +01:00
|
|
|
int urlLength = urlEntry.url().toString().length();
|
|
|
|
int urlComps = htmlFilterContentScraper.urlComps(urlEntry.url().toString()).length;
|
|
|
|
|
2006-03-17 11:16:07 +01:00
|
|
|
urlManager.stackEntry(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2);
|
2005-06-08 02:52:24 +02:00
|
|
|
// save the url entry
|
2006-01-20 16:14:21 +01:00
|
|
|
final plasmaWordIndexEntry entry;
|
2006-01-22 01:07:00 +01:00
|
|
|
if (urlEntry.word() == null) {
|
|
|
|
// the old way to define words
|
2006-01-20 16:14:21 +01:00
|
|
|
entry = new plasmaWordIndexEntry(
|
|
|
|
urlEntry.hash(),
|
2006-01-30 01:42:38 +01:00
|
|
|
urlLength, urlComps,
|
2006-02-26 12:30:37 +01:00
|
|
|
urlEntry.descr().length(),
|
2006-01-20 16:14:21 +01:00
|
|
|
urlEntry.wordCount(),
|
|
|
|
0, 0, 0, 0, 0, 0,
|
2006-01-27 03:48:27 +01:00
|
|
|
urlEntry.size(),
|
2006-01-20 16:14:21 +01:00
|
|
|
urlEntry.moddate().getTime(),
|
2006-01-27 03:48:27 +01:00
|
|
|
System.currentTimeMillis(),
|
2006-01-20 16:14:21 +01:00
|
|
|
urlEntry.quality(),
|
|
|
|
urlEntry.language(),
|
|
|
|
urlEntry.doctype(),
|
2006-02-26 12:30:37 +01:00
|
|
|
0,0,
|
2006-01-20 16:14:21 +01:00
|
|
|
false
|
|
|
|
);
|
2006-01-22 01:07:00 +01:00
|
|
|
} else {
|
|
|
|
// the new way: the search-result-url transports all the attributes of word indexes
|
|
|
|
entry = urlEntry.word();
|
|
|
|
}
|
2006-01-20 16:14:21 +01:00
|
|
|
if (urlEntry.snippet() != null) {
|
2005-06-08 02:52:24 +02:00
|
|
|
// we don't store the snippets along the url entry, because they are search-specific.
|
|
|
|
// instead, they are placed in a snipped-search cache.
|
|
|
|
//System.out.println("--- RECEIVED SNIPPET '" + link.snippet() + "'");
|
2006-01-20 16:14:21 +01:00
|
|
|
snippets.storeToCache(wordhashes, urlEntry.hash(), urlEntry.snippet());
|
2005-06-08 02:52:24 +02:00
|
|
|
}
|
|
|
|
// add the url entry to the word indexes
|
2005-05-22 15:27:54 +02:00
|
|
|
for (int m = 0; m < words; m++) {
|
|
|
|
container[m].add(new plasmaWordIndexEntry[]{entry}, System.currentTimeMillis());
|
|
|
|
}
|
2005-05-07 23:11:18 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// finally insert the containers to the index
|
2006-04-02 22:40:07 +02:00
|
|
|
for (int m = 0; m < words; m++) { containerCache.add(container[m], -1); }
|
2005-05-07 23:11:18 +02:00
|
|
|
|
|
|
|
// generate statistics
|
2005-04-07 21:19:42 +02:00
|
|
|
long searchtime;
|
|
|
|
try {
|
2005-08-03 04:02:39 +02:00
|
|
|
searchtime = Integer.parseInt((String) result.get("searchtime"));
|
2005-04-07 21:19:42 +02:00
|
|
|
} catch (NumberFormatException e) {
|
|
|
|
searchtime = totalrequesttime;
|
|
|
|
}
|
2005-10-23 19:50:27 +02:00
|
|
|
yacyCore.log.logFine("SEARCH " + results + " URLS FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes) + ", duetime=" + duetime + ", searchtime=" + searchtime + ", netdelay=" + (totalrequesttime - searchtime) + ", references=" + result.get("references"));
|
2005-05-07 23:11:18 +02:00
|
|
|
return results;
|
|
|
|
} catch (Exception e) {
|
2005-10-17 17:46:12 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.search error: '" + targetPeer.get(yacySeed.NAME, "anonymous") + "' failed - " + e);
|
2005-05-07 23:11:18 +02:00
|
|
|
//e.printStackTrace();
|
|
|
|
return 0;
|
|
|
|
}
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
public static HashMap permissionMessage(String targetHash) {
|
2005-05-07 23:11:18 +02:00
|
|
|
// ask for allowed message size and attachement size
|
|
|
|
// if this replies null, the peer does not answer
|
2005-09-21 23:32:43 +02:00
|
|
|
if (yacyCore.seedDB == null || yacyCore.seedDB.mySeed == null) { return null; }
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// adding all needed parameters
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-10-22 15:28:04 +02:00
|
|
|
final serverObjects post = new serverObjects(5);
|
|
|
|
post.put("key", key);
|
|
|
|
post.put("process", "permission");
|
|
|
|
post.put("iam", yacyCore.seedDB.mySeed.hash);
|
|
|
|
post.put("youare", targetHash);
|
|
|
|
post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
|
|
|
|
|
2005-11-11 00:48:20 +01:00
|
|
|
// get target address
|
|
|
|
String address = targetAddress(targetHash);
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// sending request
|
2005-05-07 23:11:18 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
return nxTools.table(
|
|
|
|
httpc.wput(
|
|
|
|
new URL("http://" + address + "/yacy/message.html"),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
8000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
post,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
)
|
|
|
|
);
|
2005-05-07 23:11:18 +02:00
|
|
|
} catch (Exception e) {
|
|
|
|
// most probably a network time-out exception
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.permissionMessage error:" + e.getMessage());
|
2005-05-07 23:11:18 +02:00
|
|
|
return null;
|
|
|
|
}
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
public static HashMap postMessage(String targetHash, String subject, byte[] message) {
|
2005-05-07 23:11:18 +02:00
|
|
|
// this post a message to the remote message board
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// adding all needed parameters
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-10-22 15:28:04 +02:00
|
|
|
final serverObjects post = new serverObjects(7);
|
|
|
|
post.put("key", key);
|
|
|
|
post.put("process", "post");
|
|
|
|
post.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key));
|
|
|
|
post.put("youare", targetHash);
|
|
|
|
post.put("subject", subject);
|
|
|
|
post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
|
2006-01-10 17:48:59 +01:00
|
|
|
try {
|
|
|
|
post.put("message", new String(message, "UTF-8"));
|
|
|
|
} catch (UnsupportedEncodingException e) {
|
|
|
|
post.put("message", new String(message));
|
|
|
|
}
|
2005-11-11 00:48:20 +01:00
|
|
|
// get target address
|
|
|
|
String address = targetAddress(targetHash);
|
|
|
|
|
|
|
|
// sending request
|
|
|
|
try {
|
|
|
|
final ArrayList v = httpc.wput(
|
2006-05-09 15:11:00 +02:00
|
|
|
new URL("http://" + address + "/yacy/message.html"),
|
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-11-11 00:48:20 +01:00
|
|
|
20000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
|
|
|
post,
|
|
|
|
null
|
|
|
|
);
|
|
|
|
|
|
|
|
return nxTools.table(v);
|
|
|
|
} catch (Exception e) {
|
|
|
|
yacyCore.log.logSevere("yacyClient.postMessage error:" + e.getMessage());
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public static String targetAddress(String targetHash) {
|
|
|
|
// find target address
|
2005-05-07 23:11:18 +02:00
|
|
|
String address;
|
2005-09-21 23:32:43 +02:00
|
|
|
if (targetHash.equals(yacyCore.seedDB.mySeed.hash)) {
|
2005-05-07 23:11:18 +02:00
|
|
|
address = yacyCore.seedDB.mySeed.getAddress();
|
2005-09-21 23:32:43 +02:00
|
|
|
} else {
|
2005-11-11 00:48:20 +01:00
|
|
|
final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash);
|
|
|
|
if (targetSeed == null) { return null; }
|
|
|
|
address = targetSeed.getAddress();
|
2005-09-21 23:32:43 +02:00
|
|
|
}
|
2005-11-11 00:48:20 +01:00
|
|
|
if (address == null) address = "localhost:8080";
|
|
|
|
return address;
|
|
|
|
}
|
|
|
|
|
|
|
|
public static HashMap transferPermission(String targetAddress, long filesize, String filename) {
|
|
|
|
// should we use the proxy?
|
|
|
|
boolean useProxy = (yacyCore.seedDB != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
|
|
|
|
|
|
|
// add all needed parameters
|
|
|
|
final String key = crypt.randomSalt();
|
|
|
|
final serverObjects post = new serverObjects();
|
|
|
|
post.put("key", key);
|
|
|
|
post.put("process", "permission");
|
|
|
|
post.put("iam", yacyCore.seedDB.mySeed.hash);
|
|
|
|
post.put("purpose", "crcon");
|
|
|
|
post.put("filename", filename);
|
|
|
|
post.put("filesize", Long.toString(filesize));
|
|
|
|
post.put("can-send-protocol", "http");
|
|
|
|
|
|
|
|
// send request
|
|
|
|
try {
|
|
|
|
return nxTools.table(
|
|
|
|
httpc.wput(
|
|
|
|
new URL("http://" + targetAddress + "/yacy/transfer.html"),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-11-11 00:48:20 +01:00
|
|
|
8000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
|
|
|
post,
|
|
|
|
null
|
|
|
|
)
|
|
|
|
);
|
|
|
|
} catch (Exception e) {
|
|
|
|
// most probably a network time-out exception
|
|
|
|
yacyCore.log.logSevere("yacyClient.permissionTransfer error:" + e.getMessage());
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
public static HashMap transferStore(String targetAddress, String access, String filename, byte[] file) {
|
|
|
|
// should we use the proxy?
|
|
|
|
boolean useProxy = (yacyCore.seedDB != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
|
|
|
|
|
|
|
// adding all needed parameters
|
|
|
|
final String key = crypt.randomSalt();
|
|
|
|
final serverObjects post = new serverObjects(7);
|
|
|
|
post.put("key", key);
|
|
|
|
post.put("process", "store");
|
|
|
|
post.put("iam", yacyCore.seedDB.mySeed.hash);
|
|
|
|
post.put("purpose", "crcon");
|
|
|
|
post.put("filename", filename);
|
|
|
|
post.put("filesize", Long.toString(file.length));
|
|
|
|
post.put("md5", serverCodings.encodeMD5Hex(file));
|
|
|
|
post.put("access", access);
|
|
|
|
HashMap files = new HashMap();
|
|
|
|
files.put("filename", file);
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// sending request
|
2005-04-07 21:19:42 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
final ArrayList v = httpc.wput(
|
2006-05-09 15:11:00 +02:00
|
|
|
new URL("http://" + targetAddress + "/yacy/transfer.html"),
|
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
20000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
post,
|
|
|
|
files
|
2005-10-22 15:28:04 +02:00
|
|
|
);
|
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
return nxTools.table(v);
|
|
|
|
} catch (Exception e) {
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.postMessage error:" + e.getMessage());
|
2005-04-07 21:19:42 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2005-11-11 00:48:20 +01:00
|
|
|
|
|
|
|
public static String transfer(String targetAddress, String filename, byte[] file) {
|
|
|
|
HashMap phase1 = transferPermission(targetAddress, file.length, filename);
|
|
|
|
if (phase1 == null) return "no connection to remote address " + targetAddress + "; phase 1";
|
|
|
|
String access = (String) phase1.get("access");
|
|
|
|
String nextaddress = (String) phase1.get("address");
|
|
|
|
String protocol = (String) phase1.get("protocol");
|
2005-12-05 15:24:13 +01:00
|
|
|
//String path = (String) phase1.get("path");
|
|
|
|
//String maxsize = (String) phase1.get("maxsize");
|
2005-11-11 00:48:20 +01:00
|
|
|
String response = (String) phase1.get("response");
|
|
|
|
if ((response == null) || (protocol == null) || (access == null)) return "wrong return values from other peer; phase 1";
|
|
|
|
if (!(response.equals("ok"))) return "remote peer rejected transfer: " + response;
|
2006-01-04 01:39:00 +01:00
|
|
|
String accesscode = serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access));
|
2005-11-11 00:48:20 +01:00
|
|
|
if (protocol.equals("http")) {
|
|
|
|
HashMap phase2 = transferStore(nextaddress, accesscode, filename, file);
|
|
|
|
if (phase2 == null) return "no connection to remote address " + targetAddress + "; phase 2";
|
|
|
|
response = (String) phase2.get("response");
|
|
|
|
if (response == null) return "wrong return values from other peer; phase 2";
|
|
|
|
if (!(response.equals("ok"))) {
|
|
|
|
return "remote peer failed with transfer: " + response;
|
|
|
|
}
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
return "wrong protocol: " + protocol;
|
|
|
|
}
|
|
|
|
|
2006-04-20 13:53:15 +02:00
|
|
|
public static HashMap crawlOrder(yacySeed targetSeed, URL url, URL referrer, int timeout) {
|
|
|
|
return crawlOrder(targetSeed, new URL[]{url}, new URL[]{referrer}, timeout);
|
2006-02-17 12:04:35 +01:00
|
|
|
}
|
|
|
|
|
2006-04-20 13:53:15 +02:00
|
|
|
public static HashMap crawlOrder(yacySeed targetSeed, URL[] url, URL[] referrer, int timeout) {
|
2005-07-19 02:26:31 +02:00
|
|
|
// this post a message to the remote message board
|
2005-09-21 23:32:43 +02:00
|
|
|
if (targetSeed == null) { return null; }
|
|
|
|
if (yacyCore.seedDB.mySeed == null) { return null; }
|
|
|
|
if (yacyCore.seedDB.mySeed == targetSeed) { return null; }
|
|
|
|
|
2005-10-22 15:28:04 +02:00
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-07-19 02:26:31 +02:00
|
|
|
// construct request
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-10-22 15:28:04 +02:00
|
|
|
final serverObjects post = new serverObjects(9);
|
2005-07-19 02:26:31 +02:00
|
|
|
post.put("key", key);
|
|
|
|
post.put("process", "crawl");
|
|
|
|
post.put("iam", yacyCore.seedDB.mySeed.hash);
|
|
|
|
post.put("youare", targetSeed.hash);
|
2005-10-17 17:46:12 +02:00
|
|
|
post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
|
2006-02-17 12:04:35 +01:00
|
|
|
if (url.length == 1) {
|
2006-02-22 08:18:27 +01:00
|
|
|
post.put("url", crypt.simpleEncode(url[0].toString()));
|
|
|
|
post.put("referrer", crypt.simpleEncode((referrer[0] == null) ? "" : referrer[0].toString()));
|
2006-02-17 12:04:35 +01:00
|
|
|
} else {
|
|
|
|
for (int i=0; i< url.length; i++) {
|
|
|
|
post.put("url" + i, crypt.simpleEncode(url[i].toString()));
|
|
|
|
post.put("ref" + i, crypt.simpleEncode((referrer[i] == null) ? "" : referrer[i].toString()));
|
|
|
|
}
|
|
|
|
}
|
2005-07-19 02:26:31 +02:00
|
|
|
post.put("depth", "0");
|
|
|
|
post.put("ttl", "0");
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-10-22 15:28:04 +02:00
|
|
|
// determining target address
|
2005-09-21 23:32:43 +02:00
|
|
|
final String address = targetSeed.getAddress();
|
|
|
|
if (address == null) { return null; }
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// sending request
|
2005-07-19 02:26:31 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
return nxTools.table(
|
|
|
|
httpc.wput(
|
|
|
|
new URL("http://" + address + "/yacy/crawlOrder.html"),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2006-04-20 13:53:15 +02:00
|
|
|
timeout,
|
2005-10-22 15:28:04 +02:00
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
post,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
)
|
|
|
|
);
|
2005-07-19 02:26:31 +02:00
|
|
|
} catch (Exception e) {
|
|
|
|
// most probably a network time-out exception
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.crawlOrder error: peer=" + targetSeed.getName() + ", error=" + e.getMessage());
|
2005-07-19 02:26:31 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
/*
|
|
|
|
Test:
|
|
|
|
http://217.234.95.114:5777/yacy/crawlOrder.html?key=abc&iam=S-cjM67KhtcJ&youare=EK31N7RgRqTn&process=crawl&referrer=&depth=0&url=p|http://www.heise.de/newsticker/meldung/53245
|
|
|
|
version=0.297 uptime=225 accepted=true reason=ok delay=30 depth=0
|
|
|
|
-er crawlt, Ergebnis erscheint aber unter falschem initiator
|
2005-05-07 23:11:18 +02:00
|
|
|
*/
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-07-12 02:07:09 +02:00
|
|
|
public static HashMap crawlReceipt(yacySeed targetSeed, String process, String result, String reason, plasmaCrawlLURL.Entry entry, String wordhashes) {
|
2005-09-21 23:32:43 +02:00
|
|
|
if (targetSeed == null) { return null; }
|
|
|
|
if (yacyCore.seedDB.mySeed == null) { return null; }
|
|
|
|
if (yacyCore.seedDB.mySeed == targetSeed) { return null; }
|
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
/*
|
|
|
|
the result can have one of the following values:
|
|
|
|
negative cases, no retry
|
|
|
|
unavailable - the resource is not avaiable (a broken link); not found or interrupted
|
|
|
|
robot - a robot-file has denied to crawl that resource
|
2005-05-07 23:11:18 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
negative cases, retry possible
|
|
|
|
rejected - the peer has rejected to load the resource
|
|
|
|
dequeue - peer too busy - rejected to crawl
|
|
|
|
|
|
|
|
positive cases with crawling
|
|
|
|
fill - the resource was loaded and processed
|
|
|
|
update - the resource was already in database but re-loaded and processed
|
2005-05-07 23:11:18 +02:00
|
|
|
|
|
|
|
positive cases without crawling
|
2005-04-07 21:19:42 +02:00
|
|
|
known - the resource is already in database, believed to be fresh and not reloaded
|
|
|
|
stale - the resource was reloaded but not processed because source had no changes
|
2005-05-07 23:11:18 +02:00
|
|
|
|
|
|
|
*/
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
// construct request
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
|
|
|
|
2005-10-22 15:28:04 +02:00
|
|
|
// determining target address
|
2005-04-07 21:19:42 +02:00
|
|
|
String address = targetSeed.getAddress();
|
2005-09-21 23:32:43 +02:00
|
|
|
if (address == null) { return null; }
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// sending request
|
2005-04-07 21:19:42 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
return nxTools.table(
|
|
|
|
httpc.wget(
|
|
|
|
new URL("http://" + address + "/yacy/crawlReceipt.html" +
|
|
|
|
"?iam=" + yacyCore.seedDB.mySeed.hash +
|
|
|
|
"&youare=" + targetSeed.hash +
|
|
|
|
"&process=" + process +
|
|
|
|
"&key=" + key +
|
|
|
|
"&urlhash=" + ((entry == null) ? "" : entry.hash()) +
|
|
|
|
"&result=" + result +
|
|
|
|
"&reason=" + reason +
|
|
|
|
"&wordh=" + wordhashes +
|
|
|
|
"&lurlEntry=" + ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), key))
|
|
|
|
),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
60000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null
|
|
|
|
)
|
|
|
|
);
|
2005-05-07 23:11:18 +02:00
|
|
|
} catch (Exception e) {
|
|
|
|
// most probably a network time-out exception
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.crawlReceipt error:" + e.getMessage());
|
2005-05-07 23:11:18 +02:00
|
|
|
return null;
|
|
|
|
}
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2006-01-30 01:42:38 +01:00
|
|
|
public static String transferIndex(yacySeed targetSeed, plasmaWordIndexEntryContainer[] indexes, HashMap urlCache, boolean gzipBody, int timeout) {
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2006-01-30 13:42:06 +01:00
|
|
|
// check if we got all necessary urls in the urlCache (only for debugging)
|
|
|
|
Iterator eenum;
|
|
|
|
plasmaWordIndexEntry entry;
|
|
|
|
for (int i = 0; i < indexes.length; i++) {
|
|
|
|
eenum = indexes[i].entries();
|
|
|
|
while (eenum.hasNext()) {
|
|
|
|
entry = (plasmaWordIndexEntry) eenum.next();
|
|
|
|
if (urlCache.get(entry.getUrlHash()) == null) {
|
2006-02-05 02:17:08 +01:00
|
|
|
yacyCore.log.logFine("DEBUG transferIndex: to-send url hash '" + entry.getUrlHash() + "' is not contained in urlCache");
|
2006-01-30 13:42:06 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// transfer the RWI without the URLs
|
2005-10-05 12:45:33 +02:00
|
|
|
HashMap in = transferRWI(targetSeed, indexes, gzipBody, timeout);
|
2005-09-21 23:32:43 +02:00
|
|
|
if (in == null) { return "no_connection_1"; }
|
2005-04-07 21:19:42 +02:00
|
|
|
String result = (String) in.get("result");
|
2005-09-21 23:32:43 +02:00
|
|
|
if (result == null) { return "no_result_1"; }
|
2005-04-07 21:19:42 +02:00
|
|
|
if (!(result.equals("ok"))) return result;
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
// in now contains a list of unknown hashes
|
2005-09-21 23:32:43 +02:00
|
|
|
final String uhss = (String) in.get("unknownURL");
|
|
|
|
if (uhss == null) { return "no_unknownURL_tag_in_response"; }
|
|
|
|
if (uhss.length() == 0) { return null; } // all url's known, we are ready here
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-09-21 23:32:43 +02:00
|
|
|
final String[] uhs = uhss.split(",");
|
|
|
|
if (uhs.length == 0) { return null; } // all url's known
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
// extract the urlCache from the result
|
2005-07-12 02:07:09 +02:00
|
|
|
plasmaCrawlLURL.Entry[] urls = new plasmaCrawlLURL.Entry[uhs.length];
|
2005-04-07 21:19:42 +02:00
|
|
|
for (int i = 0; i < uhs.length; i++) {
|
2005-07-12 02:07:09 +02:00
|
|
|
urls[i] = (plasmaCrawlLURL.Entry) urlCache.get(uhs[i]);
|
2006-01-30 13:42:06 +01:00
|
|
|
if (urls[i] == null) {
|
2006-02-09 10:10:49 +01:00
|
|
|
yacyCore.log.logFine("DEBUG transferIndex: requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'");
|
2006-01-30 13:42:06 +01:00
|
|
|
}
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-10-05 12:45:33 +02:00
|
|
|
in = transferURL(targetSeed, urls, gzipBody, timeout);
|
2005-09-21 23:32:43 +02:00
|
|
|
if (in == null) { return "no_connection_2"; }
|
2005-04-07 21:19:42 +02:00
|
|
|
result = (String) in.get("result");
|
2005-09-21 23:32:43 +02:00
|
|
|
if (result == null) { return "no_result_2"; }
|
|
|
|
if (!(result.equals("ok"))) { return result; }
|
|
|
|
// int doubleentries = Integer.parseInt((String) in.get("double"));
|
|
|
|
// System.out.println("DEBUG tansferIndex: transferred " + uhs.length + " URL's, double=" + doubleentries);
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
return null;
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2006-01-30 01:42:38 +01:00
|
|
|
private static HashMap transferRWI(yacySeed targetSeed, plasmaWordIndexEntryContainer[] indexes, boolean gzipBody, int timeout) {
|
2005-09-21 23:32:43 +02:00
|
|
|
final String address = targetSeed.getAddress();
|
|
|
|
if (address == null) { return null; }
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
// prepare post values
|
2005-10-05 12:45:33 +02:00
|
|
|
final serverObjects post = new serverObjects(7);
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-09-22 12:30:55 +02:00
|
|
|
|
|
|
|
// enabling gzip compression for post request body
|
2005-10-05 12:45:33 +02:00
|
|
|
if ((gzipBody) && (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS)) {
|
2005-09-22 12:30:55 +02:00
|
|
|
post.put(httpc.GZIP_POST_BODY,"true");
|
|
|
|
}
|
2005-05-07 23:11:18 +02:00
|
|
|
post.put("key", key);
|
2005-04-07 21:19:42 +02:00
|
|
|
post.put("iam", yacyCore.seedDB.mySeed.hash);
|
2005-05-07 23:11:18 +02:00
|
|
|
post.put("youare", targetSeed.hash);
|
2005-08-03 04:02:39 +02:00
|
|
|
post.put("wordc", Integer.toString(indexes.length));
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
int indexcount = 0;
|
2005-09-21 23:32:43 +02:00
|
|
|
final StringBuffer entrypost = new StringBuffer(indexes.length*73);
|
2005-10-13 15:57:15 +02:00
|
|
|
Iterator eenum;
|
2005-04-07 21:19:42 +02:00
|
|
|
plasmaWordIndexEntry entry;
|
|
|
|
for (int i = 0; i < indexes.length; i++) {
|
2006-01-30 01:42:38 +01:00
|
|
|
eenum = indexes[i].entries();
|
2005-10-13 15:57:15 +02:00
|
|
|
while (eenum.hasNext()) {
|
|
|
|
entry = (plasmaWordIndexEntry) eenum.next();
|
2005-09-12 12:37:16 +02:00
|
|
|
entrypost.append(indexes[i].wordHash())
|
|
|
|
.append(entry.toExternalForm())
|
|
|
|
.append(serverCore.crlfString);
|
2005-08-13 00:14:24 +02:00
|
|
|
indexcount++;
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-08-13 00:14:24 +02:00
|
|
|
if (indexcount == 0) {
|
|
|
|
// nothing to do but everything ok
|
2005-10-05 12:45:33 +02:00
|
|
|
final HashMap result = new HashMap(2);
|
2005-08-13 00:14:24 +02:00
|
|
|
result.put("result", "ok");
|
|
|
|
result.put("unknownURL", "");
|
|
|
|
return result;
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-08-03 04:02:39 +02:00
|
|
|
post.put("entryc", Integer.toString(indexcount));
|
2005-09-12 12:37:16 +02:00
|
|
|
post.put("indexes", entrypost.toString());
|
2005-05-07 23:11:18 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
final ArrayList v = httpc.wput(
|
|
|
|
new URL("http://" + address + "/yacy/transferRWI.html"),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
timeout,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
post,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
);
|
2005-04-07 21:19:42 +02:00
|
|
|
// this should return a list of urlhashes that are unknwon
|
|
|
|
if (v != null) {
|
|
|
|
yacyCore.seedDB.mySeed.incSI(indexcount);
|
|
|
|
}
|
|
|
|
|
2005-09-21 23:32:43 +02:00
|
|
|
final HashMap result = nxTools.table(v);
|
2005-04-07 21:19:42 +02:00
|
|
|
return result;
|
|
|
|
} catch (Exception e) {
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.transferRWI error:" + e.getMessage());
|
2005-04-07 21:19:42 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-10-05 12:45:33 +02:00
|
|
|
private static HashMap transferURL(yacySeed targetSeed, plasmaCrawlLURL.Entry[] urls, boolean gzipBody, int timeout) {
|
2005-05-07 23:11:18 +02:00
|
|
|
// this post a message to the remote message board
|
2005-09-21 23:32:43 +02:00
|
|
|
final String address = targetSeed.getAddress();
|
|
|
|
if (address == null) { return null; }
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
// prepare post values
|
2005-10-05 12:45:33 +02:00
|
|
|
final serverObjects post = new serverObjects(5+urls.length);
|
2005-09-21 23:32:43 +02:00
|
|
|
final String key = crypt.randomSalt();
|
2005-09-22 12:30:55 +02:00
|
|
|
|
|
|
|
// enabling gzip compression for post request body
|
2005-10-05 12:45:33 +02:00
|
|
|
if ((gzipBody) && (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS)) {
|
2005-09-22 12:30:55 +02:00
|
|
|
post.put(httpc.GZIP_POST_BODY,"true");
|
|
|
|
}
|
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
post.put("key", key);
|
2005-04-07 21:19:42 +02:00
|
|
|
post.put("iam", yacyCore.seedDB.mySeed.hash);
|
2005-05-07 23:11:18 +02:00
|
|
|
post.put("youare", targetSeed.hash);
|
|
|
|
String resource = "";
|
2005-04-07 21:19:42 +02:00
|
|
|
int urlc = 0;
|
|
|
|
for (int i = 0; i < urls.length; i++) {
|
|
|
|
if (urls[i] != null) {
|
|
|
|
resource = urls[i].toString();
|
|
|
|
if (resource != null) {
|
2005-09-14 09:20:14 +02:00
|
|
|
post.put("url" + urlc, resource);
|
2005-04-07 21:19:42 +02:00
|
|
|
urlc++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2005-08-03 04:02:39 +02:00
|
|
|
post.put("urlc", Integer.toString(urlc));
|
2005-05-07 23:11:18 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
final ArrayList v = httpc.wput(
|
2006-05-09 15:11:00 +02:00
|
|
|
new URL("http://" + address + "/yacy/transferURL.html"),
|
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
timeout,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
post,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
);
|
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
if (v != null) {
|
|
|
|
yacyCore.seedDB.mySeed.incSU(urlc);
|
|
|
|
}
|
|
|
|
return nxTools.table(v);
|
|
|
|
} catch (Exception e) {
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.transferRWI error:" + e.getMessage());
|
2005-04-07 21:19:42 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
public static HashMap getProfile(yacySeed targetSeed) {
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
2005-05-07 23:11:18 +02:00
|
|
|
// this post a message to the remote message board
|
2005-10-05 12:45:33 +02:00
|
|
|
final serverObjects post = new serverObjects(2);
|
2005-05-07 23:11:18 +02:00
|
|
|
post.put("iam", yacyCore.seedDB.mySeed.hash);
|
|
|
|
post.put("youare", targetSeed.hash);
|
|
|
|
String address = targetSeed.getAddress();
|
2005-09-21 23:32:43 +02:00
|
|
|
if (address == null) { address = "localhost:8080"; }
|
2005-05-07 23:11:18 +02:00
|
|
|
try {
|
2005-10-22 15:28:04 +02:00
|
|
|
final ArrayList v = httpc.wput(
|
|
|
|
new URL("http://" + address + "/yacy/profile.html"),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2006-05-10 11:37:18 +02:00
|
|
|
10000,
|
2005-10-22 15:28:04 +02:00
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
|
2005-11-11 00:48:20 +01:00
|
|
|
post,
|
|
|
|
null
|
2005-10-22 15:28:04 +02:00
|
|
|
);
|
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
return nxTools.table(v);
|
|
|
|
} catch (Exception e) {
|
2005-08-30 23:32:59 +02:00
|
|
|
yacyCore.log.logSevere("yacyClient.getProfile error:" + e.getMessage());
|
2005-04-07 21:19:42 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
public static void main(String[] args) {
|
|
|
|
System.out.println("yacyClient Test");
|
|
|
|
try {
|
2005-09-21 23:32:43 +02:00
|
|
|
final plasmaSwitchboard sb = new plasmaSwitchboard(args[0], "httpProxy.init", "DATA/SETTINGS/httpProxy.conf");
|
2005-12-05 15:24:13 +01:00
|
|
|
/*final yacyCore core =*/ new yacyCore(sb);
|
2005-10-18 09:45:27 +02:00
|
|
|
yacyCore.peerActions.loadSeedLists();
|
|
|
|
final yacySeed target = yacyCore.seedDB.getConnected(args[1]);
|
2005-09-21 23:32:43 +02:00
|
|
|
final String wordhashe = plasmaWordIndexEntry.word2hash("test");
|
2005-05-07 23:11:18 +02:00
|
|
|
//System.out.println("permission=" + permissionMessage(args[1]));
|
|
|
|
|
2005-10-22 15:28:04 +02:00
|
|
|
// should we use the proxy?
|
2005-10-23 10:59:11 +02:00
|
|
|
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
|
|
|
|
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
|
2005-10-22 15:28:04 +02:00
|
|
|
|
|
|
|
final HashMap result = nxTools.table(
|
|
|
|
httpc.wget(
|
|
|
|
new URL("http://" + target.getAddress() + "/yacy/search.html" +
|
|
|
|
"?myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) +
|
|
|
|
"&youare=" + target.hash + "&key=" +
|
|
|
|
"&myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) +
|
|
|
|
"&count=10" +
|
|
|
|
"&resource=global" +
|
|
|
|
"&query=" + wordhashe),
|
2006-05-09 15:11:00 +02:00
|
|
|
yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
|
2005-10-22 15:28:04 +02:00
|
|
|
5000,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null
|
|
|
|
)
|
|
|
|
);
|
2005-04-07 21:19:42 +02:00
|
|
|
System.out.println("Result=" + result.toString());
|
|
|
|
} catch (Exception e) {
|
2005-05-07 23:11:18 +02:00
|
|
|
e.printStackTrace();
|
|
|
|
}
|
2005-04-07 21:19:42 +02:00
|
|
|
System.exit(0);
|
|
|
|
}
|
2005-09-21 23:32:43 +02:00
|
|
|
|
2005-04-07 21:19:42 +02:00
|
|
|
}
|