2005-04-07 21:19:42 +02:00
// Network.java
// -----------------------
// part of YaCy
2008-07-20 19:14:51 +02:00
// (C) by Michael Peter Christen; mc@yacy.net
2005-04-07 21:19:42 +02:00
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004, 2005
2005-09-13 13:56:50 +02:00
//
2005-09-30 18:06:42 +02:00
// $LastChangedDate$
// $LastChangedRevision$
2007-02-07 12:20:31 +01:00
// $LastChangedBy$
2005-04-07 21:19:42 +02:00
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2005-05-12 19:50:45 +02:00
// You must compile this file with
2005-04-07 21:19:42 +02:00
// javac -classpath .:../classes Network.java
// if the shell's current path is HTROOT
2005-07-04 13:09:48 +02:00
import java.util.HashMap ;
2005-08-01 03:12:02 +02:00
import java.util.HashSet ;
2007-07-06 00:56:37 +02:00
import java.util.Iterator ;
2012-08-16 16:28:57 +02:00
import java.util.Locale ;
2006-05-16 12:42:22 +02:00
import java.util.Map ;
2014-10-01 03:10:39 +02:00
import java.util.Set ;
2010-02-15 16:57:35 +01:00
import java.util.concurrent.ConcurrentHashMap ;
2011-07-14 09:07:06 +02:00
import java.util.concurrent.ConcurrentMap ;
2007-02-07 12:20:31 +01:00
import java.util.regex.Matcher ;
import java.util.regex.Pattern ;
2007-06-25 21:39:15 +02:00
import java.util.regex.PatternSyntaxException ;
2006-09-30 00:27:20 +02:00
2011-04-26 15:35:29 +02:00
import net.yacy.cora.protocol.ClientIdentification ;
2012-08-16 16:28:57 +02:00
import net.yacy.cora.protocol.Domains ;
2010-08-23 14:32:02 +02:00
import net.yacy.cora.protocol.HeaderFramework ;
import net.yacy.cora.protocol.RequestHeader ;
2012-09-26 10:36:09 +02:00
import net.yacy.cora.protocol.ResponseHeader ;
2009-10-11 23:29:18 +02:00
import net.yacy.kelondro.util.MapTools ;
2011-10-04 11:06:24 +02:00
import net.yacy.peers.NewsDB ;
import net.yacy.peers.NewsPool ;
import net.yacy.peers.PeerActions ;
2011-11-28 21:34:02 +01:00
import net.yacy.peers.Protocol ;
2011-10-04 11:06:24 +02:00
import net.yacy.peers.Seed ;
2011-09-25 18:59:06 +02:00
import net.yacy.peers.operation.yacyVersion ;
import net.yacy.search.Switchboard ;
import net.yacy.search.SwitchboardConstants ;
2012-09-21 15:48:16 +02:00
import net.yacy.server.serverObjects ;
import net.yacy.server.serverSwitch ;
2012-09-26 10:36:09 +02:00
import net.yacy.server.servletProperties ;
2005-04-07 21:19:42 +02:00
public class Network {
2005-09-13 13:56:50 +02:00
private static final String STR_TABLE_LIST = " table_list_ " ;
2009-07-19 22:37:44 +02:00
public static serverObjects respond ( final RequestHeader requestHeader , final serverObjects post , final serverSwitch switchboard ) {
final Switchboard sb = ( Switchboard ) switchboard ;
2006-03-27 17:55:31 +02:00
final long start = System . currentTimeMillis ( ) ;
2011-07-14 09:07:06 +02:00
2012-09-26 10:36:09 +02:00
final servletProperties prop = new servletProperties ( ) ;
2011-01-26 14:59:55 +01:00
prop . put ( " menu " , post = = null ? 2 : ( post . get ( " menu " , " " ) . equals ( " embed " ) ) ? 0 : ( post . get ( " menu " , " " ) . equals ( " simple " ) ) ? 1 : 2 ) ;
2011-02-09 22:32:45 +01:00
if ( sb . peers . mySeed ( ) ! = null ) prop . put ( " menu_newpeer_peerhash " , sb . peers . mySeed ( ) . hash ) ;
2011-07-14 09:07:06 +02:00
2016-11-14 01:37:16 +01:00
prop . setLocalized ( ! requestHeader . getPathInfo ( ) . endsWith ( " .xml " ) ) ;
2007-10-24 23:38:19 +02:00
prop . putHTML ( " page_networkTitle " , sb . getConfig ( " network.unit.description " , " unspecified " ) ) ;
2009-07-19 22:37:44 +02:00
prop . putHTML ( " page_networkName " , sb . getConfig ( SwitchboardConstants . NETWORK_NAME , " unspecified " ) ) ;
2005-12-07 00:51:29 +01:00
final boolean overview = ( post = = null ) | | ( post . get ( " page " , " 0 " ) . equals ( " 0 " ) ) ;
2005-09-13 13:56:50 +02:00
2011-10-04 11:06:24 +02:00
final String mySeedType = sb . peers . mySeed ( ) . get ( Seed . PEERTYPE , Seed . PEERTYPE_VIRGIN ) ;
final boolean iAmActive = ( mySeedType . equals ( Seed . PEERTYPE_SENIOR ) | | mySeedType . equals ( Seed . PEERTYPE_PRINCIPAL ) ) ;
2005-09-13 13:56:50 +02:00
2005-04-07 21:19:42 +02:00
if ( overview ) {
2009-05-28 16:26:05 +02:00
long accActLinks = sb . peers . countActiveURL ( ) ;
long accActWords = sb . peers . countActiveRWI ( ) ;
final long accPassLinks = sb . peers . countPassiveURL ( ) ;
final long accPassWords = sb . peers . countPassiveRWI ( ) ;
long accPotLinks = sb . peers . countPotentialURL ( ) ;
long accPotWords = sb . peers . countPotentialRWI ( ) ;
2005-09-13 13:56:50 +02:00
2009-05-28 16:26:05 +02:00
int conCount = sb . peers . sizeConnected ( ) ;
final int disconCount = sb . peers . sizeDisconnected ( ) ;
int potCount = sb . peers . sizePotential ( ) ;
2005-09-13 13:56:50 +02:00
2012-09-26 10:36:09 +02:00
// final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true"));
2009-05-28 16:26:05 +02:00
final long otherppm = sb . peers . countActivePPM ( ) ;
final double otherqpm = sb . peers . countActiveQPM ( ) ;
2007-02-01 01:17:35 +01:00
long myppm = 0 ;
double myqph = 0d ;
2005-09-13 13:56:50 +02:00
2005-04-07 21:19:42 +02:00
// create own peer info
2011-10-04 11:06:24 +02:00
final Seed seed = sb . peers . mySeed ( ) ;
2009-05-28 16:26:05 +02:00
if ( sb . peers . mySeed ( ) ! = null ) { //our Peer
2006-02-10 14:45:11 +01:00
// update seed info
2008-06-04 23:34:57 +02:00
sb . updateMySeed ( ) ;
2011-07-14 09:07:06 +02:00
2011-03-15 02:03:35 +01:00
final long LCount = seed . getLinkCount ( ) ;
final long ICount = seed . getWordCount ( ) ;
2011-10-04 11:06:24 +02:00
final long RCount = seed . getLong ( Seed . RCOUNT , 0L ) ;
2005-09-13 13:56:50 +02:00
2005-09-30 18:06:42 +02:00
// my-info
2011-10-04 11:06:24 +02:00
prop . putHTML ( " table_my-name " , seed . get ( Seed . NAME , " - " ) ) ;
2007-05-31 16:22:27 +02:00
prop . put ( " table_my-hash " , seed . hash ) ;
2013-05-10 17:32:21 +02:00
prop . put ( " table_my-ssl " , sb . peers . mySeed ( ) . getFlagSSLAvailable ( ) ? 1 : 0 ) ;
2014-10-07 21:57:41 +02:00
if ( seed . isVirgin ( ) ) {
2005-10-01 13:43:48 +02:00
prop . put ( " table_my-info " , 0 ) ;
2014-10-07 21:57:41 +02:00
} else if ( seed . isJunior ( ) ) {
2005-10-01 13:43:48 +02:00
prop . put ( " table_my-info " , 1 ) ;
2007-08-27 00:06:58 +02:00
accPotLinks + = LCount ;
accPotWords + = ICount ;
2014-10-07 21:57:41 +02:00
} else if ( seed . isSenior ( ) ) {
2005-10-01 13:43:48 +02:00
prop . put ( " table_my-info " , 2 ) ;
2007-08-27 00:06:58 +02:00
accActLinks + = LCount ;
accActWords + = ICount ;
2014-10-07 21:57:41 +02:00
} else if ( seed . isPrincipal ( ) ) {
2005-10-01 13:43:48 +02:00
prop . put ( " table_my-info " , 3 ) ;
2007-08-27 00:06:58 +02:00
accActLinks + = LCount ;
accActWords + = ICount ;
2005-04-07 21:19:42 +02:00
}
2014-10-07 21:57:41 +02:00
String port = seed . get ( Seed . PORT , " - " ) ;
Set < String > ips = seed . getIPs ( ) ;
int ipsc = 0 ;
for ( String s : ips ) {
prop . put ( " table_ips_ " + ipsc + " _nodestate " , seed . getFlagRootNode ( ) ? 1 : 0 ) ;
prop . put ( " table_ips_ " + ipsc + " _c " , 0 ) ;
prop . putHTML ( " table_ips_ " + ipsc + " _c_hash " , seed . hash ) ;
prop . putHTML ( " table_ips_ " + ipsc + " _c_ip " , s ) ;
prop . putHTML ( " table_ips_ " + ipsc + " _c_port " , port ) ;
prop . put ( " table_ips_ " + ipsc + + + " _c_ipv6 " , s . indexOf ( ':' ) > = 0 ? 1 : 0 ) ;
}
prop . put ( " table_ips " , ipsc ) ;
2005-10-01 13:43:48 +02:00
prop . put ( " table_my-acceptcrawl " , seed . getFlagAcceptRemoteCrawl ( ) ? 1 : 0 ) ;
prop . put ( " table_my-dhtreceive " , seed . getFlagAcceptRemoteIndex ( ) ? 1 : 0 ) ;
2012-05-29 01:38:54 +02:00
prop . put ( " table_my-nodestate " , seed . getFlagRootNode ( ) ? 1 : 0 ) ;
2005-09-30 18:06:42 +02:00
2012-07-05 12:38:41 +02:00
myppm = Switchboard . currentPPM ( ) ;
2010-02-24 14:53:55 +01:00
myqph = 60d * sb . averageQPM ( ) ;
2011-10-04 11:06:24 +02:00
prop . put ( " table_my-version " , seed . get ( Seed . VERSION , " - " ) ) ;
prop . put ( " table_my-utc " , seed . get ( Seed . UTC , " - " ) ) ;
prop . put ( " table_my-uptime " , PeerActions . formatInterval ( 60000 * seed . getLong ( Seed . UPTIME , 0 ) ) ) ;
2007-10-24 23:38:19 +02:00
prop . putNum ( " table_my-LCount " , LCount ) ;
prop . putNum ( " table_my-ICount " , ICount ) ;
prop . putNum ( " table_my-RCount " , RCount ) ;
2011-10-04 11:06:24 +02:00
prop . putNum ( " table_my-sI " , seed . getLong ( Seed . INDEX_OUT , 0L ) ) ;
prop . putNum ( " table_my-sU " , seed . getLong ( Seed . URL_OUT , 0L ) ) ;
prop . putNum ( " table_my-rI " , seed . getLong ( Seed . INDEX_IN , 0L ) ) ;
prop . putNum ( " table_my-rU " , seed . getLong ( Seed . URL_IN , 0L ) ) ;
2007-10-24 23:38:19 +02:00
prop . putNum ( " table_my-ppm " , myppm ) ;
prop . putNum ( " table_my-qph " , Math . round ( 100d * myqph ) / 100d ) ;
2010-02-24 14:53:55 +01:00
prop . putNum ( " table_my-qph-publocal " , Math . round ( 6000d * sb . averageQPMPublicLocal ( ) ) / 100d ) ;
prop . putNum ( " table_my-qph-pubremote " , Math . round ( 6000d * sb . averageQPMGlobal ( ) ) / 100d ) ;
2011-10-04 11:06:24 +02:00
prop . putNum ( " table_my-seeds " , seed . getLong ( Seed . SCOUNT , 0L ) ) ;
prop . putNum ( " table_my-connects " , seed . getFloat ( Seed . CCOUNT , 0F ) ) ;
prop . put ( " table_my-url " , seed . get ( Seed . SEEDLISTURL , " " ) ) ;
2011-07-14 09:07:06 +02:00
2007-05-31 16:22:27 +02:00
// generating the location string
2011-04-26 15:35:29 +02:00
prop . putHTML ( " table_my-location " , ClientIdentification . generateLocation ( ) ) ;
2005-04-07 21:19:42 +02:00
}
2005-09-13 13:56:50 +02:00
2005-04-07 21:19:42 +02:00
// overall results: Network statistics
2011-10-04 11:06:24 +02:00
if ( iAmActive ) conCount + + ; else if ( mySeedType . equals ( Seed . PEERTYPE_JUNIOR ) ) potCount + + ;
2011-07-14 09:07:06 +02:00
final int activeLastMonth = sb . peers . sizeActiveSince ( 30 * 1440 ) ;
final int activeLastWeek = sb . peers . sizeActiveSince ( 7 * 1440 ) ;
final int activeLastDay = sb . peers . sizeActiveSince ( 1440 ) ;
2013-01-05 00:37:52 +01:00
final int activeLastHour = sb . peers . sizeActiveSince ( 60 ) ;
2011-07-14 09:07:06 +02:00
final int activeSwitch =
2013-01-05 00:37:52 +01:00
( activeLastHour < = conCount ) ? 0 :
( activeLastDay < = activeLastHour ) ? 1 :
( activeLastWeek < = activeLastDay ) ? 2 :
( activeLastMonth < = activeLastWeek ) ? 3 : 4 ;
2010-05-25 00:41:14 +02:00
prop . putNum ( " table_active-switch " , activeSwitch ) ;
prop . putNum ( " table_active-switch_last-month " , activeLastMonth ) ;
prop . putNum ( " table_active-switch_last-week " , activeLastWeek ) ;
prop . putNum ( " table_active-switch_last-day " , activeLastDay ) ;
2013-01-05 00:37:52 +01:00
prop . putNum ( " table_active-switch_last-hour " , activeLastHour ) ;
2007-10-24 23:38:19 +02:00
prop . putNum ( " table_active-count " , conCount ) ;
prop . putNum ( " table_active-links " , accActLinks ) ;
prop . putNum ( " table_active-words " , accActWords ) ;
prop . putNum ( " table_passive-count " , disconCount ) ;
prop . putNum ( " table_passive-links " , accPassLinks ) ;
prop . putNum ( " table_passive-words " , accPassWords ) ;
prop . putNum ( " table_potential-count " , potCount ) ;
prop . putNum ( " table_potential-links " , accPotLinks ) ;
prop . putNum ( " table_potential-words " , accPotWords ) ;
prop . putNum ( " table_all-count " , conCount + disconCount + potCount ) ;
prop . putNum ( " table_all-links " , accActLinks + accPassLinks + accPotLinks ) ;
prop . putNum ( " table_all-words " , accActWords + accPassWords + accPotWords ) ;
2005-09-13 13:56:50 +02:00
2007-10-24 23:38:19 +02:00
prop . putNum ( " table_gppm " , otherppm + ( ( iAmActive ) ? myppm : 0 ) ) ;
prop . putNum ( " table_gqph " , Math . round ( 6000d * otherqpm + 100d * ( ( iAmActive ) ? myqph : 0d ) ) / 100d ) ;
2005-04-07 21:19:42 +02:00
prop . put ( " table " , 2 ) ; // triggers overview
prop . put ( " page " , 0 ) ;
2011-03-15 02:03:35 +01:00
} else if ( post ! = null & & post . getInt ( " page " , 1 ) = = 4 ) {
2014-10-29 13:21:35 +01:00
prop . put ( " table " , 4 ) ; // triggers "Manually contacting Peer"
2011-01-15 19:49:51 +01:00
prop . put ( " page " , 4 ) ;
2011-07-14 09:07:06 +02:00
2011-01-15 19:49:51 +01:00
if ( sb . peers . mySeed ( ) ! = null ) {
prop . put ( " table_my-hash " , sb . peers . mySeed ( ) . hash ) ;
2014-09-30 14:53:52 +02:00
prop . put ( " table_my-ip " , sb . peers . mySeed ( ) . getIPs ( ) . toString ( ) ) ;
2011-01-15 19:49:51 +01:00
prop . put ( " table_my-port " , sb . peers . mySeed ( ) . getPort ( ) ) ;
}
2005-09-13 13:56:50 +02:00
2005-07-04 13:09:48 +02:00
if ( post . containsKey ( " addPeer " ) ) {
2005-09-13 13:56:50 +02:00
2005-07-04 13:09:48 +02:00
// AUTHENTICATE
2014-01-27 15:56:02 +01:00
final int authentication = sb . adminAuthenticated ( requestHeader ) ;
if ( authentication < 2 ) {
prop . authenticationRequired ( ) ; // must authenticate
2005-07-04 13:09:48 +02:00
return prop ;
}
2005-09-13 13:56:50 +02:00
2011-03-15 02:03:35 +01:00
final ConcurrentMap < String , String > map = new ConcurrentHashMap < String , String > ( ) ;
2014-10-01 15:32:10 +02:00
String challengeIP = post . get ( " peerIP " ) ;
String challengePort = post . get ( " peerPort " ) ;
map . put ( Seed . IP , challengeIP ) ;
map . put ( Seed . PORT , challengePort ) ;
2014-10-01 03:47:57 +02:00
Seed peer = post . get ( " peerHash " ) = = null ? null : new Seed ( post . get ( " peerHash " ) , map ) ;
2014-10-01 15:32:10 +02:00
String challengeAddress = peer . getPublicAddress ( challengeIP ) ;
2008-06-04 23:34:57 +02:00
sb . updateMySeed ( ) ;
2014-10-01 15:32:10 +02:00
Seed mySeed = sb . peers . mySeed ( ) ;
final Map < String , String > response = Protocol . hello ( mySeed , sb . peers . peerActions , challengeAddress , peer . hash ) ;
2005-09-13 13:56:50 +02:00
2014-10-01 03:10:39 +02:00
if ( response = = null ) {
2014-10-06 17:44:27 +02:00
Seed peerd = sb . peers . get ( peer . hash ) ;
if ( peerd ! = null ) peer = peerd ;
sb . peers . peerActions . interfaceDeparture ( peer , challengeIP ) ;
2005-07-04 13:09:48 +02:00
prop . put ( " table_comment " , 1 ) ;
2014-10-01 15:32:10 +02:00
prop . put ( " table_comment_status " , " publish: no response from peer ' " + peer . getName ( ) + " / " + post . get ( " peerHash " ) + " ' from <a href= \" http:// " + challengeAddress + " \" target= \" _blank \" > " + challengeAddress + " </a> " ) ;
2005-07-04 13:09:48 +02:00
} else {
2014-10-01 03:10:39 +02:00
String yourtype = response . get ( " yourtype " ) ;
String yourip = response . get ( " yourip " ) ;
2009-05-28 16:26:05 +02:00
peer = sb . peers . getConnected ( peer . hash ) ;
2005-10-20 13:43:43 +02:00
if ( peer = = null ) {
prop . put ( " table_comment " , 1 ) ;
2014-10-01 15:32:10 +02:00
prop . put ( " table_comment_status " , " publish: disconnected peer 'UNKNOWN/ " + post . get ( " peerHash " ) + " ' from <a href= \" http:// " + challengeAddress + " \" target= \" _blank \" > " + challengeAddress + " </a>, yourtype = " + yourtype + " , yourip = " + yourip ) ;
2005-10-20 13:43:43 +02:00
} else {
prop . put ( " table_comment " , 2 ) ;
2014-10-01 15:32:10 +02:00
prop . put ( " table_comment_status " , " publish: handshaked " + peer . get ( Seed . PEERTYPE , Seed . PEERTYPE_SENIOR ) + " peer ' " + peer . getName ( ) + " ' at <a href= \" http:// " + challengeAddress + " \" target= \" _blank \" > " + challengeAddress + " </a>, yourtype = " + yourtype + " , yourip = " + yourip ) ;
2007-10-24 23:38:19 +02:00
prop . putHTML ( " table_comment_details " , peer . toString ( ) ) ;
2005-10-20 13:43:43 +02:00
}
2005-07-04 13:09:48 +02:00
}
2005-09-13 13:56:50 +02:00
2008-09-19 13:45:11 +02:00
prop . putHTML ( " table_peerHash " , post . get ( " peerHash " ) ) ;
prop . putHTML ( " table_peerIP " , post . get ( " peerIP " ) ) ;
2011-07-14 09:07:06 +02:00
prop . putHTML ( " table_peerPort " , post . get ( " peerPort " ) ) ;
2005-07-04 13:09:48 +02:00
} else {
prop . put ( " table_peerHash " , " " ) ;
prop . put ( " table_peerIP " , " " ) ;
2005-09-13 13:56:50 +02:00
prop . put ( " table_peerPort " , " " ) ;
2005-07-04 13:09:48 +02:00
prop . put ( " table_comment " , 0 ) ;
}
2014-10-29 13:21:35 +01:00
} else if ( post ! = null & & post . getInt ( " page " , 1 ) = = 5 ) {
prop . put ( " page " , 5 ) ; // triggers stats page
prop . put ( " table " , 5 ) ;
2005-08-12 16:06:47 +02:00
} else {
2005-04-07 21:19:42 +02:00
// generate table
2011-03-15 02:03:35 +01:00
final int page = ( post = = null ? 1 : post . getInt ( " page " , 1 ) ) ;
2011-11-30 21:50:41 +01:00
final int maxCount = ( post = = null ? 9000 : post . getInt ( " maxCount " , 9000 ) ) ;
2011-07-14 09:07:06 +02:00
int conCount = 0 ;
2009-05-28 16:26:05 +02:00
if ( sb . peers = = null ) {
2005-04-07 21:19:42 +02:00
prop . put ( " table " , 0 ) ; //no remote senior/principal proxies known"
} else {
int size = 0 ;
switch ( page ) {
2009-05-28 16:26:05 +02:00
case 1 : size = sb . peers . sizeConnected ( ) ; break ;
case 2 : size = sb . peers . sizeDisconnected ( ) ; break ;
case 3 : size = sb . peers . sizePotential ( ) ; break ;
2005-09-13 13:56:50 +02:00
default : break ;
2005-04-07 21:19:42 +02:00
}
if ( size = = 0 ) {
prop . put ( " table " , 0 ) ; //no remote senior/principal proxies known"
} else {
// add temporary the own seed to the database
if ( iAmActive ) {
2008-06-04 23:34:57 +02:00
sb . updateMySeed ( ) ;
2009-05-28 16:26:05 +02:00
sb . peers . addConnected ( sb . peers . mySeed ( ) ) ;
2005-04-07 21:19:42 +02:00
}
2005-09-13 13:56:50 +02:00
2005-08-01 03:12:02 +02:00
// find updated Information using YaCyNews
2008-01-29 17:41:09 +01:00
final HashSet < String > updatedProfile = new HashSet < String > ( ) ;
final HashMap < String , Map < String , String > > updatedWiki = new HashMap < String , Map < String , String > > ( ) ;
final HashMap < String , Map < String , String > > updatedBlog = new HashMap < String , Map < String , String > > ( ) ;
final HashMap < String , String > isCrawling = new HashMap < String , String > ( ) ;
2011-10-04 11:06:24 +02:00
NewsDB . Record record ;
2012-07-05 10:23:07 +02:00
final Iterator < NewsDB . Record > recordIterator = sb . peers . newsPool . recordIterator ( NewsPool . INCOMING_DB ) ;
2007-07-06 00:56:37 +02:00
while ( recordIterator . hasNext ( ) ) {
2008-01-29 17:41:09 +01:00
record = recordIterator . next ( ) ;
2007-07-06 00:56:37 +02:00
if ( record = = null ) {
continue ;
2011-10-04 11:06:24 +02:00
} else if ( record . category ( ) . equals ( NewsPool . CATEGORY_PROFILE_UPDATE ) ) {
2007-07-06 00:56:37 +02:00
updatedProfile . add ( record . originator ( ) ) ;
2011-10-04 11:06:24 +02:00
} else if ( record . category ( ) . equals ( NewsPool . CATEGORY_WIKI_UPDATE ) ) {
2007-07-06 00:56:37 +02:00
updatedWiki . put ( record . originator ( ) , record . attributes ( ) ) ;
2011-10-04 11:06:24 +02:00
} else if ( record . category ( ) . equals ( NewsPool . CATEGORY_BLOG_ADD ) ) {
2007-07-06 00:56:37 +02:00
updatedBlog . put ( record . originator ( ) , record . attributes ( ) ) ;
2011-10-04 11:06:24 +02:00
} else if ( record . category ( ) . equals ( NewsPool . CATEGORY_CRAWL_START ) ) {
2007-07-06 00:56:37 +02:00
isCrawling . put ( record . originator ( ) , record . attributes ( ) . get ( " startURL " ) ) ;
2005-08-01 03:12:02 +02:00
}
2007-07-06 00:56:37 +02:00
}
2005-09-13 13:56:50 +02:00
2005-04-07 21:19:42 +02:00
boolean dark = true ;
2011-10-04 11:06:24 +02:00
Seed seed ;
2014-10-01 03:10:39 +02:00
final boolean c = ( post ! = null & & post . containsKey ( " ip " ) ) ;
2013-03-10 02:26:24 +01:00
final boolean onlyIncomingDHT = ( post ! = null & & post . containsKey ( " onlydhtin " ) ) ;
final boolean onlyNode = ( post ! = null & & post . containsKey ( " onlynode " ) ) ;
final long onlyAgeOverDays = post = = null ? 0 : post . getLong ( " onlyageoverdays " , 0 ) ;
final long onlySizeLessDocs = post = = null ? Long . MAX_VALUE : post . getLong ( " onlysizelessdocs " , Long . MAX_VALUE ) ;
2011-10-04 11:06:24 +02:00
Iterator < Seed > e = null ;
2008-06-06 18:01:27 +02:00
final boolean order = ( post ! = null & & post . get ( " order " , " down " ) . equals ( " up " ) ) ;
2010-03-25 08:47:41 +01:00
final String sort = ( post = = null ? null : post . get ( " sort " , null ) ) ;
2005-04-07 21:19:42 +02:00
switch ( page ) {
2011-10-04 11:06:24 +02:00
case 1 : e = sb . peers . seedsSortedConnected ( order , ( sort = = null ? Seed . LCOUNT : sort ) ) ; break ;
case 2 : e = sb . peers . seedsSortedDisconnected ( order , ( sort = = null ? Seed . LASTSEEN : sort ) ) ; break ;
case 3 : e = sb . peers . seedsSortedPotential ( order , ( sort = = null ? Seed . LASTSEEN : sort ) ) ; break ;
2005-09-13 13:56:50 +02:00
default : break ;
2005-04-07 21:19:42 +02:00
}
2005-08-01 14:47:47 +02:00
String startURL ;
2008-01-29 17:41:09 +01:00
Map < String , String > wikiMap ;
Map < String , String > blogMap ;
2012-08-16 16:28:57 +02:00
String userAgent , location ;
2007-02-01 01:17:35 +01:00
int PPM ;
double QPM ;
2007-02-07 12:20:31 +01:00
Pattern peerSearchPattern = null ;
2007-06-25 21:39:15 +02:00
prop . put ( " regexerror " , 0 ) ;
2007-06-25 21:44:54 +02:00
prop . put ( " regexerror_wrongregex " , ( String ) null ) ;
2008-06-06 18:01:27 +02:00
if ( post ! = null & & post . containsKey ( " search " ) ) {
2007-10-03 00:50:47 +02:00
try {
2007-06-25 21:39:15 +02:00
peerSearchPattern = Pattern . compile ( post . get ( " match " , " " ) , Pattern . CASE_INSENSITIVE ) ;
2008-08-02 14:12:04 +02:00
} catch ( final PatternSyntaxException pse ) {
2007-06-25 21:39:15 +02:00
prop . put ( " regexerror " , 1 ) ;
2008-06-06 18:01:27 +02:00
prop . putHTML ( " regexerror_wrongregex " , pse . getPattern ( ) ) ;
2007-06-25 21:39:15 +02:00
}
2007-02-07 12:20:31 +01:00
}
2011-11-30 21:50:41 +01:00
if ( e ! = null ) {
2007-10-01 14:30:23 +02:00
while ( e . hasNext ( ) & & conCount < maxCount ) {
2008-06-06 18:01:27 +02:00
seed = e . next ( ) ;
2010-02-01 23:18:56 +01:00
assert seed ! = null ;
2005-04-07 21:19:42 +02:00
if ( seed ! = null ) {
2013-03-10 02:26:24 +01:00
if ( onlyIncomingDHT & & ! seed . getFlagAcceptRemoteIndex ( ) ) continue ;
if ( onlyNode & & ! seed . getFlagRootNode ( ) ) continue ;
if ( seed . getAge ( ) < onlyAgeOverDays ) continue ;
if ( seed . getLinkCount ( ) > onlySizeLessDocs ) continue ;
2008-06-06 18:01:27 +02:00
if ( ( post ! = null & & post . containsKey ( " search " ) ) & & peerSearchPattern ! = null /*(wrongregex == null)*/ ) {
2007-03-24 17:51:19 +01:00
boolean abort = true ;
2007-02-07 12:20:31 +01:00
Matcher m = peerSearchPattern . matcher ( seed . getName ( ) ) ;
2007-03-24 17:51:19 +01:00
if ( m . find ( ) ) {
abort = false ;
2007-02-07 12:20:31 +01:00
}
2007-03-24 17:51:19 +01:00
m = peerSearchPattern . matcher ( seed . hash ) ;
if ( m . find ( ) ) {
abort = false ;
}
if ( abort ) continue ;
2007-02-07 12:20:31 +01:00
}
2006-01-24 16:32:19 +01:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedProfile " , 0 ) ;
prop . put ( STR_TABLE_LIST + conCount + " _updatedWikiPage " , 0 ) ;
2006-05-15 22:48:39 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedBlog " , 0 ) ;
2006-01-24 16:32:19 +01:00
prop . put ( STR_TABLE_LIST + conCount + " _isCrawling " , 0 ) ;
2014-10-01 03:10:39 +02:00
String ip = seed . getIP ( ) ;
2005-09-13 13:56:50 +02:00
if ( conCount > = maxCount ) { break ; }
2009-05-28 16:26:05 +02:00
if ( sb . peers ! = null & & sb . peers . mySeed ( ) ! = null & & seed . hash ! = null & & seed . hash . equals ( sb . peers . mySeed ( ) . hash ) ) {
2005-10-01 13:43:48 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _dark " , 2 ) ;
2005-04-07 21:19:42 +02:00
} else {
2005-10-01 13:43:48 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _dark " , ( ( dark ) ? 1 : 0 ) ) ; dark = ! dark ;
2005-08-01 03:12:02 +02:00
}
2005-08-02 02:16:19 +02:00
if ( updatedProfile . contains ( seed . hash ) ) {
2006-01-24 16:32:19 +01:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedProfile " , 1 ) ;
prop . put ( STR_TABLE_LIST + conCount + " _updatedProfile_hash " , seed . hash ) ;
2005-08-02 02:16:19 +02:00
}
2008-01-29 17:41:09 +01:00
if ( ( wikiMap = updatedWiki . get ( seed . hash ) ) = = null ) {
2006-05-16 12:42:22 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedWiki " , 0 ) ;
2005-08-01 15:47:18 +02:00
} else {
2006-05-16 12:42:22 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedWiki " , 1 ) ;
2008-06-06 18:01:27 +02:00
prop . putHTML ( STR_TABLE_LIST + conCount + " _updatedWiki_page " , wikiMap . get ( " page " ) ) ;
2014-10-01 03:10:39 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedWiki_address " , seed . getPublicAddress ( ip ) ) ;
2005-08-01 15:47:18 +02:00
}
2008-01-29 17:41:09 +01:00
if ( ( blogMap = updatedBlog . get ( seed . hash ) ) = = null ) {
2006-05-15 22:48:39 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedBlog " , 0 ) ;
} else {
2006-05-15 23:45:26 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedBlog " , 1 ) ;
2008-06-06 18:01:27 +02:00
prop . putHTML ( STR_TABLE_LIST + conCount + " _updatedBlog_page " , blogMap . get ( " page " ) ) ;
prop . putHTML ( STR_TABLE_LIST + conCount + " _updatedBlog_subject " , blogMap . get ( " subject " ) ) ;
2014-10-01 03:10:39 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _updatedBlog_address " , seed . getPublicAddress ( ip ) ) ;
2006-05-15 22:48:39 +02:00
}
2007-02-01 01:17:35 +01:00
PPM = seed . getPPM ( ) ;
QPM = seed . getQPM ( ) ;
2008-06-06 18:01:27 +02:00
if ( ( ( startURL = isCrawling . get ( seed . hash ) ) ! = null ) & & ( PPM > = 4 ) ) {
2006-01-24 16:32:19 +01:00
prop . put ( STR_TABLE_LIST + conCount + " _isCrawling " , 1 ) ;
2006-08-16 21:49:31 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _isCrawling_page " , startURL ) ;
2005-08-01 14:47:47 +02:00
}
2005-10-01 13:43:48 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _hash " , seed . hash ) ;
2011-10-04 11:06:24 +02:00
String shortname = seed . get ( Seed . NAME , " deadlink " ) ;
2005-09-13 13:56:50 +02:00
if ( shortname . length ( ) > 20 ) {
2011-07-14 09:07:06 +02:00
shortname = shortname . substring ( 0 , 20 ) + " ... " ;
2005-09-13 13:56:50 +02:00
}
2007-10-24 23:38:19 +02:00
prop . putHTML ( STR_TABLE_LIST + conCount + " _shortname " , shortname ) ;
2011-10-04 11:06:24 +02:00
prop . putHTML ( STR_TABLE_LIST + conCount + " _fullname " , seed . get ( Seed . NAME , " deadlink " ) ) ;
2012-05-29 01:38:54 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _special " , ( seed . getFlagRootNode ( ) & & ! seed . getFlagAcceptRemoteIndex ( ) ) ? 1 : 0 ) ;
2013-05-10 17:32:21 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _ssl " , ( seed . getFlagSSLAvailable ( ) ) ? 1 : 0 ) ;
2015-04-16 02:36:12 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _ssl_ip " , seed . getIP ( ) ) ;
prop . put ( STR_TABLE_LIST + conCount + " _ssl_portssl " , seed . get ( Seed . PORTSSL , " 8443 " ) ) ;
2006-02-18 11:27:56 +01:00
userAgent = null ;
2009-05-28 16:26:05 +02:00
if ( seed . hash ! = null & & seed . hash . equals ( sb . peers . mySeed ( ) . hash ) ) {
2013-08-22 14:23:47 +02:00
userAgent = ClientIdentification . yacyInternetCrawlerAgent . userAgent ;
2011-04-26 15:35:29 +02:00
location = ClientIdentification . generateLocation ( ) ;
2006-02-18 11:27:56 +01:00
} else {
2014-10-01 03:10:39 +02:00
userAgent = sb . peers . peerActions . getUserAgent ( ip ) ;
2012-08-16 16:28:57 +02:00
location = ClientIdentification . parseLocationInUserAgent ( userAgent ) ;
}
if ( location . length ( ) > 10 ) location = location . substring ( 0 , 10 ) ;
if ( location . length ( ) = = 0 ) {
2014-10-01 03:10:39 +02:00
Locale l = Domains . getLocale ( ip ) ;
2012-08-16 16:28:57 +02:00
if ( l ! = null ) location = l . toString ( ) ;
2006-02-18 11:27:56 +01:00
}
2012-08-15 14:23:21 +02:00
prop . putHTML ( STR_TABLE_LIST + conCount + " _location " , location ) ;
2014-10-01 03:10:39 +02:00
String port = seed . get ( Seed . PORT , " - " ) ;
Set < String > ips = seed . getIPs ( ) ;
int ipsc = 0 ;
for ( String s : ips ) {
prop . put ( STR_TABLE_LIST + conCount + " _ips_ " + ipsc + " _nodestate " , seed . getFlagRootNode ( ) ? 1 : 0 ) ;
prop . put ( STR_TABLE_LIST + conCount + " _ips_ " + ipsc + " _c " , c ? 1 : 0 ) ;
prop . putHTML ( STR_TABLE_LIST + conCount + " _ips_ " + ipsc + " _c_hash " , seed . hash ) ;
prop . putHTML ( STR_TABLE_LIST + conCount + " _ips_ " + ipsc + " _c_ip " , s ) ;
prop . putHTML ( STR_TABLE_LIST + conCount + " _ips_ " + ipsc + " _c_port " , port ) ;
prop . put ( STR_TABLE_LIST + conCount + " _ips_ " + ipsc + + + " _c_ipv6 " , s . indexOf ( ':' ) > = 0 ? 1 : 0 ) ;
}
prop . put ( STR_TABLE_LIST + conCount + " _ips " , ipsc ) ;
prop . put ( STR_TABLE_LIST + conCount + " _port " , port ) ;
prop . put ( STR_TABLE_LIST + conCount + " _hash " , seed . hash ) ;
prop . put ( STR_TABLE_LIST + conCount + " _age " , seed . getAge ( ) ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _seeds " , seed . getLong ( Seed . SCOUNT , 0L ) ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _connects " , seed . getFloat ( Seed . CCOUNT , 0F ) ) ;
if ( c ) {
prop . put ( STR_TABLE_LIST + conCount + " _c " , 1 ) ;
prop . putHTML ( STR_TABLE_LIST + conCount + " _c_userAgent " , userAgent ) ;
2005-09-30 18:06:42 +02:00
} else {
2014-10-01 03:10:39 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _c " , 0 ) ;
2005-04-07 21:19:42 +02:00
}
2005-09-30 18:06:42 +02:00
2005-04-07 21:19:42 +02:00
if ( seed . isJunior ( ) ) {
2005-10-07 00:18:44 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _type " , 0 ) ;
2005-04-07 21:19:42 +02:00
} else if ( seed . isSenior ( ) ) {
2005-10-07 00:18:44 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _type " , 1 ) ;
2005-04-07 21:19:42 +02:00
} else if ( seed . isPrincipal ( ) ) {
2005-10-07 00:18:44 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _type " , 2 ) ;
2005-04-07 21:19:42 +02:00
}
2011-10-04 11:06:24 +02:00
prop . putHTML ( STR_TABLE_LIST + conCount + " _type_url " , seed . get ( Seed . SEEDLISTURL , " http://nowhere/ " ) ) ;
2005-10-04 01:13:41 +02:00
2007-02-03 00:54:27 +01:00
final long lastseen = Math . abs ( ( System . currentTimeMillis ( ) - seed . getLastSeenUTC ( ) ) / 1000 / 60 ) ;
2009-05-12 00:26:21 +02:00
if ( page = = 1 & & lastseen > 720 ) {
continue ;
}
if ( page = = 2 | | ( page = = 1 & & lastseen > 360 ) ) { // Passive Peers should be passive, also Peers without contact greater than 6 hours
2005-10-07 00:18:44 +02:00
// principal/senior/junior: red/red=offline
prop . put ( STR_TABLE_LIST + conCount + " _type_direct " , 2 ) ;
} else {
// principal/senior: green/green=direct or yellow/yellow=passive
// junior: red/green=direct or red/yellow=passive
prop . put ( STR_TABLE_LIST + conCount + " _type_direct " , seed . getFlagDirectConnect ( ) ? 1 : 0 ) ;
}
2011-07-14 09:07:06 +02:00
2005-10-07 00:18:44 +02:00
if ( page = = 1 ) {
2011-07-14 09:07:06 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _acceptcrawl " , seed . getFlagAcceptRemoteCrawl ( ) ? 1 : 0 ) ; // green=on or red=off
2005-10-07 00:18:44 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _dhtreceive " , seed . getFlagAcceptRemoteIndex ( ) ? 1 : 0 ) ; // green=on or red=off
} else { // Passive, Potential Peers
if ( seed . getFlagAcceptRemoteCrawl ( ) ) {
prop . put ( STR_TABLE_LIST + conCount + " _acceptcrawl " , 2 ) ; // red/green: offline, was on
} else {
prop . put ( STR_TABLE_LIST + conCount + " _acceptcrawl " , 0 ) ; // red/red; offline was off
}
if ( seed . getFlagAcceptRemoteIndex ( ) ) {
prop . put ( STR_TABLE_LIST + conCount + " _dhtreceive " , 2 ) ; // red/green: offline, was on
} else {
prop . put ( STR_TABLE_LIST + conCount + " _dhtreceive " , 0 ) ; // red/red; offline was off
}
2005-10-04 01:13:41 +02:00
}
2006-10-10 22:09:26 +02:00
if ( seed . getFlagAcceptRemoteIndex ( ) ) {
prop . put ( STR_TABLE_LIST + conCount + " _dhtreceive_peertags " , " " ) ;
} else {
2009-10-11 23:29:18 +02:00
final String peertags = MapTools . set2string ( seed . getPeerTags ( ) , " , " , false ) ;
2012-07-10 22:59:03 +02:00
prop . putHTML ( STR_TABLE_LIST + conCount + " _dhtreceive_peertags " , ( ( peertags = = null ) | | ( peertags . isEmpty ( ) ) ) ? " no tags given " : ( " tags = " + peertags ) ) ;
2006-10-10 22:09:26 +02:00
}
2012-05-29 01:38:54 +02:00
String [ ] yv = yacyVersion . combined2prettyVersion ( seed . get ( Seed . VERSION , " 0.1 " ) , shortname ) ;
prop . putHTML ( STR_TABLE_LIST + conCount + " _version " , yv [ 0 ] + " / " + yv [ 1 ] ) ;
2007-10-24 23:38:19 +02:00
prop . putNum ( STR_TABLE_LIST + conCount + " _lastSeen " , /*seed.getLastSeenString() + " " +*/ lastseen ) ;
2011-10-04 11:06:24 +02:00
prop . put ( STR_TABLE_LIST + conCount + " _utc " , seed . get ( Seed . UTC , " - " ) ) ;
prop . putHTML ( STR_TABLE_LIST + conCount + " _uptime " , PeerActions . formatInterval ( 60000 * seed . getLong ( Seed . UPTIME , 0 ) ) ) ;
2009-12-21 12:36:48 +01:00
prop . putNum ( STR_TABLE_LIST + conCount + " _LCount " , seed . getLinkCount ( ) ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _ICount " , seed . getWordCount ( ) ) ;
2011-10-04 11:06:24 +02:00
prop . putNum ( STR_TABLE_LIST + conCount + " _RCount " , seed . getLong ( Seed . RCOUNT , 0 ) ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _sI " , seed . getLong ( Seed . INDEX_OUT , 0 ) ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _sU " , seed . getLong ( Seed . URL_OUT , 0 ) ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _rI " , seed . getLong ( Seed . INDEX_IN , 0 ) ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _rU " , seed . getLong ( Seed . URL_IN , 0 ) ) ;
2007-10-24 23:38:19 +02:00
prop . putNum ( STR_TABLE_LIST + conCount + " _ppm " , PPM ) ;
prop . putNum ( STR_TABLE_LIST + conCount + " _qph " , Math . round ( 6000d * QPM ) / 100d ) ;
2005-04-07 21:19:42 +02:00
conCount + + ;
2005-10-01 13:43:48 +02:00
} // seed != null
} // while
2008-06-06 18:01:27 +02:00
}
2009-05-28 16:26:05 +02:00
if ( iAmActive ) { sb . peers . removeMySeed ( ) ; }
2011-11-30 21:50:41 +01:00
prop . put ( " table_list " , conCount ) ;
2005-04-07 21:19:42 +02:00
prop . put ( " table " , 1 ) ;
2007-10-24 23:38:19 +02:00
prop . putNum ( " table_num " , conCount ) ;
prop . putNum ( " table_total " , ( ( page = = 1 ) & & ( iAmActive ) ) ? ( size + 1 ) : size ) ;
2014-10-01 03:10:39 +02:00
prop . put ( " table_c " , ( ( c ) ? 1 : 0 ) ) ;
2005-04-07 21:19:42 +02:00
}
}
prop . put ( " page " , page ) ;
prop . put ( " table_page " , page ) ;
2008-06-06 18:01:27 +02:00
prop . putHTML ( " table_searchpattern " , ( post = = null ? " " : post . get ( " match " , " " ) ) ) ;
2005-04-07 21:19:42 +02:00
switch ( page ) {
2007-10-24 23:38:19 +02:00
case 1 : prop . putHTML ( " table_peertype " , " senior/principal " ) ; break ;
case 2 : prop . putHTML ( " table_peertype " , " senior/principal " ) ; break ;
2011-10-04 11:06:24 +02:00
case 3 : prop . putHTML ( " table_peertype " , Seed . PEERTYPE_JUNIOR ) ; break ;
2005-09-13 13:56:50 +02:00
default : break ;
2005-04-07 21:19:42 +02:00
}
}
2011-07-14 09:07:06 +02:00
2007-10-24 23:38:19 +02:00
prop . putNum ( " table_rt " , System . currentTimeMillis ( ) - start ) ;
2012-09-26 10:36:09 +02:00
// Adding CORS Access header for Network.xml
2016-11-14 01:37:16 +01:00
final String path = requestHeader . getPathInfo ( ) ;
2012-09-26 10:36:09 +02:00
if ( path ! = null & & path . endsWith ( " .xml " ) ) {
final ResponseHeader outgoingHeader = new ResponseHeader ( 200 ) ;
outgoingHeader . put ( HeaderFramework . CORS_ALLOW_ORIGIN , " * " ) ;
prop . setOutgoingHeader ( outgoingHeader ) ;
}
2005-04-07 21:19:42 +02:00
// return rewrite properties
return prop ;
}
2008-04-07 00:17:09 +02:00
2005-09-30 18:06:42 +02:00
}