Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

added new servlet which generates the same file as the principal peers

Browse Source 
upload to a bootstrap position
 you can call it either with
 http://localhost:8090/yacy/seedlist.html
 or to generate json (or jsonp) with
 http://localhost:8090/yacy/seedlist.json
 http://localhost:8090/yacy/seedlist.json?callback=seedlist
This commit is contained in:
orbiter 2013-11-19 15:56:10 +01:00
parent 3e552550d1
commit b7f1e5af51
5 changed files with 133 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
htroot/yacy/seedlist.html Normal file
View File

@ -0,0 +1 @@
#[encoded]#

89
htroot/yacy/seedlist.java Normal file
View File

@ -0,0 +1,89 @@
// seedlist.java
// -----------------------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://yacy.net
// Frankfurt, Germany, 2013
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.ArrayList;
import java.util.Map;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.server.serverCore;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
/**
* this servlet generates the same file as the principal peers upload to a bootstrap position
* you can call it either with
* http://localhost:8090/yacy/seedlist.html
* or to generate json (or jsonp) with
* http://localhost:8090/yacy/seedlist.json
* http://localhost:8090/yacy/seedlist.json?callback=seedlist
*/
public final class seedlist {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
int maxcount = post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE);
final ArrayList<Seed> v = sb.peers.getSeedlist(maxcount, true);
final serverObjects prop = new serverObjects();
// write simple-encoded seed lines or json
String EXT = header.get("EXT");
boolean json = EXT != null && EXT.equals("json");
if (json) {
// check for JSONP
if ( post != null && post.containsKey("callback") ) {
prop.put("jsonp-start", post.get("callback") + "([");
prop.put("jsonp-end", "]);");
} else {
prop.put("jsonp-start", "");
prop.put("jsonp-end", "");
}
// construct json property lists
for (int i = 0; i < v.size(); i++) {
Map<String, String> map = v.get(i).getMap();
int c = 0;
for (Map.Entry<String, String> m: map.entrySet()) {
prop.putJSON("peers_" + i + "_map_" + c + "_k", m.getKey());
prop.putJSON("peers_" + i + "_map_" + c + "_v", m.getValue());
prop.put("peers_" + i + "_map_" + c + "_c", 1);
c++;
}
prop.put("peers_" + i + "_map_" + (c - 1) + "_c", 0);
prop.put("peers_" + i + "_map", c);
prop.put("peers_" + i + "_c", i < v.size() - 1 ? 1 : 0);
}
prop.put("peers", v.size());
} else {
final StringBuilder encoded = new StringBuilder(1024);
for (Seed seed: v) {
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
}
prop.put("encoded", encoded.toString());
}
// return rewrite properties
return prop;
}
}

7
htroot/yacy/seedlist.json Normal file
View File

@ -0,0 +1,7 @@
#[jsonp-start]#{
"peers":[#{peers}#{
#{map}#"#[k]#"="#[v]#"#(c)#::,#(/c)#
#{/map}#}#(c)#::,#(/c)##{/peers}#
]
}#[jsonp-end]#

4
source/net/yacy/peers/Seed.java
View File

@ -1102,8 +1102,8 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
final String r = toString();
final String z = crypt.simpleEncode(r, key, 'z');
final String b = crypt.simpleEncode(r, key, 'b');
// the compressed string may be longer that the uncompressed if there is too much overhead for compression meta-info
// take simply that string that is shorter
// the compressed string may be longer than the uncompressed if there is too much overhead for compression meta-info
// take simply that string which is shorter
return ( b.length() < z.length() ) ? b : z;
}

65
source/net/yacy/peers/SeedDB.java
View File

@ -33,6 +33,7 @@ import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
@ -403,7 +404,7 @@ public final class SeedDB implements AlternativeDomainNames {
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedActiveDB, minVersion);
}
private Iterator<Seed> seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) {
public Iterator<Seed> seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedPassiveDB, minVersion);
}
@ -716,40 +717,13 @@ public final class SeedDB implements AlternativeDomainNames {
try {
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
// store own peer seed
List<Seed> seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed);
String line;
if (this.mySeed == null) initMySeed();
if (addMySeed) {
line = this.mySeed.genSeedStr(null);
for (Seed seed: seedlist) {
line = seed.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
}
// store active peer seeds
Seed ys;
Iterator<Seed> se = seedsConnected(true, false, null, (float) 0.0);
while (se.hasNext()) {
ys = se.next();
if (ys != null) {
line = ys.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
}
}
// store some of the not-so-old passive peer seeds (limit: 1 day)
se = seedsDisconnected(true, false, null, (float) 0.0);
final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
while (se.hasNext()) {
ys = se.next();
if (ys != null) {
if (ys.getLastSeenUTC() < timeout) continue;
line = ys.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
}
}
pw.flush();
} finally {
if (pw != null) try { pw.close(); } catch (final Exception e) {}
@ -757,6 +731,35 @@ public final class SeedDB implements AlternativeDomainNames {
return v;
}
public ArrayList<Seed> getSeedlist(int maxcount, boolean addMySeed) {
final ArrayList<Seed> v = new ArrayList<Seed>(this.seedActiveDB.size() + 1000);
// store own peer seed
if (addMySeed) v.add(this.mySeed);
// store active peer seeds
Seed ys;
Iterator<Seed> se = this.seedsConnected(true, false, null, (float) 0.0);
while (se.hasNext() && v.size() < maxcount) {
ys = se.next();
if (ys != null) v.add(ys);
}
// store some of the not-so-old passive peer seeds (limit: 1 day)
se = this.seedsDisconnected(true, false, null, (float) 0.0);
final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
while (se.hasNext() && v.size() < maxcount) {
ys = se.next();
if (ys != null && ys.getLastSeenUTC() >= timeout) v.add(ys);
}
final StringBuilder encoded = new StringBuilder(1024);
for (Seed seed: v) {
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
}
return v;
}
protected String uploadSeedList(final yacySeedUploader uploader,
final serverSwitch sb,
final SeedDB seedDB,