mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
added new servlet which generates the same file as the principal peers
upload to a bootstrap position you can call it either with http://localhost:8090/yacy/seedlist.html or to generate json (or jsonp) with http://localhost:8090/yacy/seedlist.json http://localhost:8090/yacy/seedlist.json?callback=seedlist
This commit is contained in:
parent
3e552550d1
commit
b7f1e5af51
1
htroot/yacy/seedlist.html
Normal file
1
htroot/yacy/seedlist.html
Normal file
|
@ -0,0 +1 @@
|
||||||
|
#[encoded]#
|
89
htroot/yacy/seedlist.java
Normal file
89
htroot/yacy/seedlist.java
Normal file
|
@ -0,0 +1,89 @@
|
||||||
|
// seedlist.java
|
||||||
|
// -----------------------
|
||||||
|
// part of YaCy
|
||||||
|
// (C) by Michael Peter Christen; mc@yacy.net
|
||||||
|
// first published on http://yacy.net
|
||||||
|
// Frankfurt, Germany, 2013
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.peers.Seed;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.server.serverCore;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* this servlet generates the same file as the principal peers upload to a bootstrap position
|
||||||
|
* you can call it either with
|
||||||
|
* http://localhost:8090/yacy/seedlist.html
|
||||||
|
* or to generate json (or jsonp) with
|
||||||
|
* http://localhost:8090/yacy/seedlist.json
|
||||||
|
* http://localhost:8090/yacy/seedlist.json?callback=seedlist
|
||||||
|
*/
|
||||||
|
public final class seedlist {
|
||||||
|
|
||||||
|
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||||
|
// return variable that accumulates replacements
|
||||||
|
final Switchboard sb = (Switchboard) env;
|
||||||
|
int maxcount = post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE);
|
||||||
|
final ArrayList<Seed> v = sb.peers.getSeedlist(maxcount, true);
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
|
||||||
|
// write simple-encoded seed lines or json
|
||||||
|
String EXT = header.get("EXT");
|
||||||
|
boolean json = EXT != null && EXT.equals("json");
|
||||||
|
|
||||||
|
if (json) {
|
||||||
|
// check for JSONP
|
||||||
|
if ( post != null && post.containsKey("callback") ) {
|
||||||
|
prop.put("jsonp-start", post.get("callback") + "([");
|
||||||
|
prop.put("jsonp-end", "]);");
|
||||||
|
} else {
|
||||||
|
prop.put("jsonp-start", "");
|
||||||
|
prop.put("jsonp-end", "");
|
||||||
|
}
|
||||||
|
// construct json property lists
|
||||||
|
for (int i = 0; i < v.size(); i++) {
|
||||||
|
Map<String, String> map = v.get(i).getMap();
|
||||||
|
int c = 0;
|
||||||
|
for (Map.Entry<String, String> m: map.entrySet()) {
|
||||||
|
prop.putJSON("peers_" + i + "_map_" + c + "_k", m.getKey());
|
||||||
|
prop.putJSON("peers_" + i + "_map_" + c + "_v", m.getValue());
|
||||||
|
prop.put("peers_" + i + "_map_" + c + "_c", 1);
|
||||||
|
c++;
|
||||||
|
}
|
||||||
|
prop.put("peers_" + i + "_map_" + (c - 1) + "_c", 0);
|
||||||
|
prop.put("peers_" + i + "_map", c);
|
||||||
|
prop.put("peers_" + i + "_c", i < v.size() - 1 ? 1 : 0);
|
||||||
|
}
|
||||||
|
prop.put("peers", v.size());
|
||||||
|
} else {
|
||||||
|
final StringBuilder encoded = new StringBuilder(1024);
|
||||||
|
for (Seed seed: v) {
|
||||||
|
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
|
||||||
|
}
|
||||||
|
prop.put("encoded", encoded.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
// return rewrite properties
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
7
htroot/yacy/seedlist.json
Normal file
7
htroot/yacy/seedlist.json
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
#[jsonp-start]#{
|
||||||
|
"peers":[#{peers}#{
|
||||||
|
#{map}#"#[k]#"="#[v]#"#(c)#::,#(/c)#
|
||||||
|
#{/map}#}#(c)#::,#(/c)##{/peers}#
|
||||||
|
]
|
||||||
|
}#[jsonp-end]#
|
||||||
|
|
|
@ -1102,8 +1102,8 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
|
||||||
final String r = toString();
|
final String r = toString();
|
||||||
final String z = crypt.simpleEncode(r, key, 'z');
|
final String z = crypt.simpleEncode(r, key, 'z');
|
||||||
final String b = crypt.simpleEncode(r, key, 'b');
|
final String b = crypt.simpleEncode(r, key, 'b');
|
||||||
// the compressed string may be longer that the uncompressed if there is too much overhead for compression meta-info
|
// the compressed string may be longer than the uncompressed if there is too much overhead for compression meta-info
|
||||||
// take simply that string that is shorter
|
// take simply that string which is shorter
|
||||||
return ( b.length() < z.length() ) ? b : z;
|
return ( b.length() < z.length() ) ? b : z;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -33,6 +33,7 @@ import java.net.InetAddress;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
@ -403,7 +404,7 @@ public final class SeedDB implements AlternativeDomainNames {
|
||||||
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedActiveDB, minVersion);
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedActiveDB, minVersion);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Iterator<Seed> seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) {
|
public Iterator<Seed> seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) {
|
||||||
// enumerates seed-type objects: all seeds sequentially without order
|
// enumerates seed-type objects: all seeds sequentially without order
|
||||||
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedPassiveDB, minVersion);
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedPassiveDB, minVersion);
|
||||||
}
|
}
|
||||||
|
@ -716,40 +717,13 @@ public final class SeedDB implements AlternativeDomainNames {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
|
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
|
||||||
|
List<Seed> seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed);
|
||||||
// store own peer seed
|
|
||||||
String line;
|
String line;
|
||||||
if (this.mySeed == null) initMySeed();
|
for (Seed seed: seedlist) {
|
||||||
if (addMySeed) {
|
line = seed.genSeedStr(null);
|
||||||
line = this.mySeed.genSeedStr(null);
|
|
||||||
v.add(line);
|
v.add(line);
|
||||||
pw.print(line + serverCore.CRLF_STRING);
|
pw.print(line + serverCore.CRLF_STRING);
|
||||||
}
|
}
|
||||||
|
|
||||||
// store active peer seeds
|
|
||||||
Seed ys;
|
|
||||||
Iterator<Seed> se = seedsConnected(true, false, null, (float) 0.0);
|
|
||||||
while (se.hasNext()) {
|
|
||||||
ys = se.next();
|
|
||||||
if (ys != null) {
|
|
||||||
line = ys.genSeedStr(null);
|
|
||||||
v.add(line);
|
|
||||||
pw.print(line + serverCore.CRLF_STRING);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// store some of the not-so-old passive peer seeds (limit: 1 day)
|
|
||||||
se = seedsDisconnected(true, false, null, (float) 0.0);
|
|
||||||
final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
|
|
||||||
while (se.hasNext()) {
|
|
||||||
ys = se.next();
|
|
||||||
if (ys != null) {
|
|
||||||
if (ys.getLastSeenUTC() < timeout) continue;
|
|
||||||
line = ys.genSeedStr(null);
|
|
||||||
v.add(line);
|
|
||||||
pw.print(line + serverCore.CRLF_STRING);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pw.flush();
|
pw.flush();
|
||||||
} finally {
|
} finally {
|
||||||
if (pw != null) try { pw.close(); } catch (final Exception e) {}
|
if (pw != null) try { pw.close(); } catch (final Exception e) {}
|
||||||
|
@ -757,6 +731,35 @@ public final class SeedDB implements AlternativeDomainNames {
|
||||||
return v;
|
return v;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ArrayList<Seed> getSeedlist(int maxcount, boolean addMySeed) {
|
||||||
|
final ArrayList<Seed> v = new ArrayList<Seed>(this.seedActiveDB.size() + 1000);
|
||||||
|
|
||||||
|
// store own peer seed
|
||||||
|
if (addMySeed) v.add(this.mySeed);
|
||||||
|
|
||||||
|
// store active peer seeds
|
||||||
|
Seed ys;
|
||||||
|
Iterator<Seed> se = this.seedsConnected(true, false, null, (float) 0.0);
|
||||||
|
while (se.hasNext() && v.size() < maxcount) {
|
||||||
|
ys = se.next();
|
||||||
|
if (ys != null) v.add(ys);
|
||||||
|
}
|
||||||
|
|
||||||
|
// store some of the not-so-old passive peer seeds (limit: 1 day)
|
||||||
|
se = this.seedsDisconnected(true, false, null, (float) 0.0);
|
||||||
|
final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
|
||||||
|
while (se.hasNext() && v.size() < maxcount) {
|
||||||
|
ys = se.next();
|
||||||
|
if (ys != null && ys.getLastSeenUTC() >= timeout) v.add(ys);
|
||||||
|
}
|
||||||
|
|
||||||
|
final StringBuilder encoded = new StringBuilder(1024);
|
||||||
|
for (Seed seed: v) {
|
||||||
|
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
|
||||||
|
}
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
protected String uploadSeedList(final yacySeedUploader uploader,
|
protected String uploadSeedList(final yacySeedUploader uploader,
|
||||||
final serverSwitch sb,
|
final serverSwitch sb,
|
||||||
final SeedDB seedDB,
|
final SeedDB seedDB,
|
||||||
|
|
Loading…
Reference in New Issue
Block a user