2012-08-17 15:33:02 +02:00
// Fulltext.java
2008-03-26 15:13:05 +01:00
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2006 as part of 'plasmaCrawlLURL.java' on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
2011-07-15 10:38:10 +02:00
//
2008-03-26 15:13:05 +01:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2011-09-25 18:59:06 +02:00
package net.yacy.search.index ;
2008-03-26 15:13:05 +01:00
import java.io.BufferedOutputStream ;
import java.io.File ;
import java.io.FileOutputStream ;
import java.io.IOException ;
import java.io.PrintWriter ;
2012-11-02 01:22:31 +01:00
import java.net.MalformedURLException ;
2008-09-04 21:41:57 +02:00
import java.util.ArrayList ;
2012-09-26 16:56:33 +02:00
import java.util.Date ;
2008-09-04 21:41:57 +02:00
import java.util.HashMap ;
2008-03-26 15:13:05 +01:00
import java.util.Iterator ;
2012-09-24 17:05:28 +02:00
import java.util.List ;
2008-09-04 21:41:57 +02:00
import java.util.Map ;
2008-09-04 22:28:36 +02:00
import java.util.TreeSet ;
2012-11-02 01:22:31 +01:00
import java.util.concurrent.BlockingQueue ;
2012-11-13 16:54:28 +01:00
import java.util.concurrent.atomic.AtomicInteger ;
2008-03-26 15:13:05 +01:00
2012-09-24 17:05:28 +02:00
import net.yacy.cora.date.GenericFormatter ;
2012-11-13 16:54:28 +01:00
import net.yacy.cora.date.ISO8601Formatter ;
2011-05-27 10:24:54 +02:00
import net.yacy.cora.document.ASCII ;
2010-05-25 14:54:57 +02:00
import net.yacy.cora.document.MultiProtocolURI ;
2012-09-25 21:20:03 +02:00
import net.yacy.cora.federate.solr.YaCySchema ;
2012-11-02 01:22:31 +01:00
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector ;
2012-09-25 21:20:03 +02:00
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector ;
import net.yacy.cora.federate.solr.connector.MirrorSolrConnector ;
import net.yacy.cora.federate.solr.connector.SolrConnector ;
2011-12-16 23:59:29 +01:00
import net.yacy.cora.order.CloneableIterator ;
import net.yacy.cora.sorting.ConcurrentScoreMap ;
import net.yacy.cora.sorting.ScoreMap ;
2012-07-27 12:13:53 +02:00
import net.yacy.cora.storage.HandleSet ;
2012-09-24 17:05:28 +02:00
import net.yacy.cora.storage.ZIPReader ;
import net.yacy.cora.storage.ZIPWriter ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.html.CharacterCoding ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.DigestURI ;
2012-08-10 13:26:51 +02:00
import net.yacy.kelondro.data.meta.URIMetadataNode ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.URIMetadataRow ;
2012-08-10 13:26:51 +02:00
import net.yacy.kelondro.data.word.WordReference ;
2009-10-10 01:32:08 +02:00
import net.yacy.kelondro.index.Cache ;
2010-08-04 15:33:12 +02:00
import net.yacy.kelondro.index.Index ;
2009-10-10 01:32:08 +02:00
import net.yacy.kelondro.index.Row ;
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.table.SplitTable ;
2011-05-26 16:35:32 +02:00
import net.yacy.kelondro.util.MemoryControl ;
2012-09-21 16:46:57 +02:00
import net.yacy.kelondro.util.MergeIterator ;
2012-06-25 11:34:38 +02:00
import net.yacy.search.Switchboard ;
2012-07-19 11:34:05 +02:00
import org.apache.lucene.util.Version ;
2012-08-10 13:26:51 +02:00
import org.apache.solr.common.SolrDocument ;
2012-08-10 15:39:10 +02:00
import org.apache.solr.common.SolrException ;
2012-08-17 15:46:26 +02:00
import org.apache.solr.common.SolrInputDocument ;
2012-07-23 16:28:39 +02:00
2012-08-17 15:33:02 +02:00
public final class Fulltext implements Iterable < byte [ ] > {
2008-03-26 15:13:05 +01:00
2012-11-02 12:29:48 +01:00
private static final String SOLR_PATH = " solr_40 " ; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml
private static final String SOLR_OLD_PATH [ ] = new String [ ] { " solr_36 " } ;
2012-10-26 15:35:42 +02:00
private static final long forcedCommitTimeout = 3000 ; // wait this time until a next forced commit is executed
2008-03-26 15:13:05 +01:00
// class objects
2012-07-22 13:18:45 +02:00
private final File location ;
private Index urlIndexFile ;
2011-09-21 17:08:05 +02:00
private Export exportthread ; // will have a export thread assigned if exporter is running
2012-07-22 13:18:45 +02:00
private String tablename ;
2011-09-21 17:08:05 +02:00
private ArrayList < HostStat > statsDump ;
2012-08-13 13:32:32 +02:00
private final MirrorSolrConnector solr ;
2012-08-05 15:49:27 +02:00
private final SolrConfiguration solrScheme ;
2012-10-26 15:35:42 +02:00
private long forcedCommitTime ;
2009-09-29 10:23:44 +02:00
2012-10-18 15:09:04 +02:00
protected Fulltext ( final File path , final SolrConfiguration solrScheme ) {
2009-12-10 00:27:26 +01:00
this . location = path ;
2012-07-22 13:18:45 +02:00
this . tablename = null ;
this . urlIndexFile = null ;
2009-03-10 14:38:40 +01:00
this . exportthread = null ; // will have a export thread assigned if exporter is running
this . statsDump = null ;
2012-10-28 20:31:29 +01:00
this . solr = new MirrorSolrConnector ( 10000 , 10000 , 100 ) ;
2012-08-05 15:49:27 +02:00
this . solrScheme = solrScheme ;
2012-10-26 15:35:42 +02:00
this . forcedCommitTime = 0 ;
2011-11-08 12:49:04 +01:00
}
2012-07-23 16:28:39 +02:00
2012-10-18 15:09:04 +02:00
protected void connectUrlDb ( final String tablename , final boolean useTailCache , final boolean exceed134217727 ) {
2012-07-22 13:18:45 +02:00
if ( this . urlIndexFile ! = null ) return ;
this . tablename = tablename ;
this . urlIndexFile = new SplitTable ( this . location , tablename , URIMetadataRow . rowdef , useTailCache , exceed134217727 ) ;
2011-11-08 12:49:04 +01:00
}
2012-07-22 13:18:45 +02:00
public void disconnectUrlDb ( ) {
if ( this . urlIndexFile = = null ) return ;
this . urlIndexFile . close ( ) ;
this . urlIndexFile = null ;
2012-07-20 11:40:33 +02:00
}
2012-08-05 15:49:27 +02:00
public SolrConfiguration getSolrScheme ( ) {
return this . solrScheme ;
}
2012-07-23 16:28:39 +02:00
public boolean connectedLocalSolr ( ) {
2012-07-24 17:23:29 +02:00
return this . solr . isConnected0 ( ) ;
2012-07-23 16:28:39 +02:00
}
2012-07-24 17:23:29 +02:00
public void connectLocalSolr ( final int commitWithin ) throws IOException {
2012-11-02 12:29:48 +01:00
File baseLocation = this . location ;
if ( baseLocation . getName ( ) . equals ( " default " ) ) baseLocation = baseLocation . getParentFile ( ) ;
File solrLocation = new File ( baseLocation , SOLR_PATH ) ;
// migrate old solr to new
for ( String oldVersion : SOLR_OLD_PATH ) {
File oldLocation = new File ( baseLocation , oldVersion ) ;
if ( oldLocation . exists ( ) ) oldLocation . renameTo ( solrLocation ) ;
}
2012-07-24 17:23:29 +02:00
EmbeddedSolrConnector esc = new EmbeddedSolrConnector ( solrLocation , new File ( new File ( Switchboard . getSwitchboard ( ) . appPath , " defaults " ) , " solr " ) ) ;
esc . setCommitWithinMs ( commitWithin ) ;
Version luceneVersion = esc . getConfig ( ) . getLuceneVersion ( " luceneMatchVersion " ) ;
2012-07-19 11:34:05 +02:00
String lvn = luceneVersion . name ( ) ;
int p = lvn . indexOf ( '_' ) ;
2012-11-02 12:29:48 +01:00
assert SOLR_PATH . endsWith ( lvn . substring ( p ) ) : " luceneVersion = " + lvn + " , solrPath = " + SOLR_PATH + " , p = " + p + " , check defaults/solr/solrconfig.xml " ;
2012-07-19 11:34:05 +02:00
Log . logInfo ( " MetadataRepository " , " connected solr in " + solrLocation . toString ( ) + " , lucene version " + lvn ) ;
2012-07-24 17:23:29 +02:00
this . solr . connect0 ( esc ) ;
2012-06-25 11:34:38 +02:00
}
2012-07-20 11:40:33 +02:00
public void disconnectLocalSolr ( ) {
2012-07-24 17:23:29 +02:00
this . solr . disconnect0 ( ) ;
2012-07-20 11:40:33 +02:00
}
2012-07-22 13:18:45 +02:00
2012-07-23 16:28:39 +02:00
public boolean connectedRemoteSolr ( ) {
2012-07-24 17:23:29 +02:00
return this . solr . isConnected1 ( ) ;
2012-07-23 16:28:39 +02:00
}
2012-07-24 17:23:29 +02:00
public void connectRemoteSolr ( final SolrConnector rs ) {
this . solr . connect1 ( rs ) ;
2012-07-22 13:18:45 +02:00
}
public void disconnectRemoteSolr ( ) {
2012-07-24 17:23:29 +02:00
this . solr . disconnect1 ( ) ;
2012-07-22 13:18:45 +02:00
}
2012-07-23 16:28:39 +02:00
2012-06-25 11:34:38 +02:00
public SolrConnector getLocalSolr ( ) {
2012-07-24 17:23:29 +02:00
return this . solr . getSolr0 ( ) ;
2008-03-26 15:13:05 +01:00
}
2012-06-26 13:54:48 +02:00
public SolrConnector getRemoteSolr ( ) {
2012-07-24 17:23:29 +02:00
return this . solr . getSolr1 ( ) ;
}
public SolrConnector getSolr ( ) {
return this . solr ;
2012-06-26 13:54:48 +02:00
}
2008-06-16 23:39:58 +02:00
public void clearCache ( ) {
2012-07-22 13:18:45 +02:00
if ( this . urlIndexFile ! = null & & this . urlIndexFile instanceof Cache ) ( ( Cache ) this . urlIndexFile ) . clearCache ( ) ;
2012-06-08 12:48:25 +02:00
if ( this . statsDump ! = null ) this . statsDump . clear ( ) ;
2011-07-15 10:38:10 +02:00
this . statsDump = null ;
2012-09-06 22:07:07 +02:00
this . solr . clearCache ( ) ;
2008-06-16 23:39:58 +02:00
}
2011-07-15 10:38:10 +02:00
2008-05-24 14:30:50 +02:00
public void clear ( ) throws IOException {
2011-07-15 10:38:10 +02:00
if ( this . exportthread ! = null ) this . exportthread . interrupt ( ) ;
2011-09-21 17:08:05 +02:00
if ( this . urlIndexFile = = null ) {
SplitTable . delete ( this . location , this . tablename ) ;
} else {
this . urlIndexFile . clear ( ) ;
}
2012-07-24 17:23:29 +02:00
this . solr . clear ( ) ;
2012-07-22 13:18:45 +02:00
// the remote solr is not cleared here because that shall be done separately
2011-07-15 10:38:10 +02:00
this . statsDump = null ;
2008-05-24 14:30:50 +02:00
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public int size ( ) {
2012-07-22 13:18:45 +02:00
int size = 0 ;
size + = this . urlIndexFile = = null ? 0 : this . urlIndexFile . size ( ) ;
2012-07-24 17:23:29 +02:00
size + = this . solr . getSize ( ) ;
2012-07-22 13:18:45 +02:00
return size ;
2008-03-26 15:13:05 +01:00
}
public void close ( ) {
2011-07-15 10:38:10 +02:00
this . statsDump = null ;
if ( this . urlIndexFile ! = null ) {
this . urlIndexFile . close ( ) ;
this . urlIndexFile = null ;
2008-03-26 15:13:05 +01:00
}
2012-07-24 17:23:29 +02:00
this . solr . close ( ) ;
2008-03-26 15:13:05 +01:00
}
2012-10-26 07:39:07 +02:00
public int getCommitWithinMs ( ) {
return this . solr . getCommitWithinMs ( ) ;
}
public void commit ( ) {
2012-10-26 15:35:42 +02:00
if ( this . forcedCommitTime + forcedCommitTimeout < System . currentTimeMillis ( ) ) return ;
this . forcedCommitTime = Long . MAX_VALUE - forcedCommitTimeout ; // set the time high to prevent that other processes get to this point meanwhile
2012-10-26 07:39:07 +02:00
this . solr . commit ( ) ;
2012-10-26 15:35:42 +02:00
this . forcedCommitTime = System . currentTimeMillis ( ) ; // set the exact time
2012-10-26 07:39:07 +02:00
}
2011-05-13 08:21:40 +02:00
/ * *
* generates an plasmaLURLEntry using the url hash
* if the url cannot be found , this returns null
* @param obrwi
* @return
* /
2012-10-16 18:11:57 +02:00
public URIMetadataNode getMetadata ( WordReference wre , long weight ) {
2012-07-25 14:31:54 +02:00
if ( wre = = null ) return null ; // all time was already wasted in takeRWI to get another element
2012-08-17 01:34:38 +02:00
return getMetadata ( wre . urlhash ( ) , wre , weight ) ;
2011-05-13 08:21:40 +02:00
}
2011-07-15 10:38:10 +02:00
2012-10-16 18:11:57 +02:00
public URIMetadataNode getMetadata ( final byte [ ] urlHash ) {
2011-05-13 08:21:40 +02:00
if ( urlHash = = null ) return null ;
2012-08-17 01:34:38 +02:00
return getMetadata ( urlHash , null , 0 ) ;
2012-08-10 13:26:51 +02:00
}
2012-10-16 18:11:57 +02:00
private URIMetadataNode getMetadata ( final byte [ ] urlHash , WordReference wre , long weight ) {
2012-08-10 13:26:51 +02:00
// get the metadata from Solr
try {
SolrDocument doc = this . solr . get ( ASCII . String ( urlHash ) ) ;
2012-08-18 19:36:21 +02:00
if ( doc ! = null ) {
if ( this . urlIndexFile ! = null ) this . urlIndexFile . remove ( urlHash ) ;
return new URIMetadataNode ( doc , wre , weight ) ;
}
2012-08-10 13:26:51 +02:00
} catch ( IOException e ) {
Log . logException ( e ) ;
}
2012-08-13 10:40:04 +02:00
// get the metadata from the old metadata index
if ( this . urlIndexFile ! = null ) try {
2012-09-26 16:05:11 +02:00
// slow migration to solr
final Row . Entry entry = this . urlIndexFile . remove ( urlHash ) ;
if ( entry = = null ) return null ;
2012-10-18 14:29:11 +02:00
URIMetadataRow row = new URIMetadataRow ( entry , wre ) ;
2012-10-16 18:11:57 +02:00
SolrInputDocument solrInput = this . solrScheme . metadata2solr ( row ) ;
this . putDocument ( solrInput ) ;
2012-10-18 14:29:11 +02:00
return new URIMetadataNode ( solrInput , wre , weight ) ;
2012-08-17 01:34:38 +02:00
} catch ( final IOException e ) {
Log . logException ( e ) ;
}
return null ;
}
2012-08-17 15:46:26 +02:00
public void putDocument ( final SolrInputDocument doc ) throws IOException {
2012-08-18 13:05:27 +02:00
String id = ( String ) doc . getFieldValue ( YaCySchema . id . name ( ) ) ;
byte [ ] idb = ASCII . getBytes ( id ) ;
2012-09-26 16:05:11 +02:00
try {
if ( this . urlIndexFile ! = null ) this . urlIndexFile . remove ( idb ) ;
SolrDocument sd = this . solr . get ( id ) ;
2012-09-26 16:56:33 +02:00
Date now = new Date ( ) ;
2012-10-18 15:09:04 +02:00
Date sdDate = sd = = null ? null : URIMetadataNode . getDate ( sd , YaCySchema . last_modified ) ;
2012-10-18 14:29:11 +02:00
if ( sdDate = = null | | sdDate . after ( now ) ) sdDate = now ;
Date docDate = SolrConfiguration . getDate ( doc , YaCySchema . last_modified ) ;
2012-09-26 16:56:33 +02:00
if ( docDate . after ( now ) ) docDate = now ;
if ( sd = = null | | sdDate . before ( docDate ) ) {
2012-09-26 16:05:11 +02:00
if ( this . solrScheme . contains ( YaCySchema . ip_s ) ) {
// ip_s needs a dns lookup which causes blockings during search here
this . solr . add ( doc ) ;
} else synchronized ( this . solr ) {
this . solr . add ( doc ) ;
}
2012-08-18 13:05:27 +02:00
}
2012-09-26 16:05:11 +02:00
} catch ( SolrException e ) {
throw new IOException ( e . getMessage ( ) , e ) ;
2012-08-18 13:05:27 +02:00
}
this . statsDump = null ;
if ( MemoryControl . shortStatus ( ) ) clearCache ( ) ;
2012-08-17 15:46:26 +02:00
}
2012-10-18 14:29:11 +02:00
public void putMetadata ( final URIMetadataRow entry ) throws IOException {
URIMetadataRow row = entry ;
2012-08-18 13:05:27 +02:00
byte [ ] idb = row . hash ( ) ;
String id = ASCII . String ( idb ) ;
2012-09-26 16:05:11 +02:00
try {
if ( this . urlIndexFile ! = null ) this . urlIndexFile . remove ( idb ) ;
SolrDocument sd = this . solr . get ( id ) ;
if ( sd = = null | | ( new URIMetadataNode ( sd ) ) . isOlder ( row ) ) {
if ( this . solrScheme . contains ( YaCySchema . ip_s ) ) {
// ip_s needs a dns lookup which causes blockings during search here
this . solr . add ( getSolrScheme ( ) . metadata2solr ( row ) ) ;
} else synchronized ( this . solr ) {
this . solr . add ( getSolrScheme ( ) . metadata2solr ( row ) ) ;
2012-09-25 00:19:52 +02:00
}
2012-09-26 16:05:11 +02:00
}
} catch ( SolrException e ) {
throw new IOException ( e . getMessage ( ) , e ) ;
2008-03-26 15:13:05 +01:00
}
2012-08-10 15:39:10 +02:00
this . statsDump = null ;
if ( MemoryControl . shortStatus ( ) ) clearCache ( ) ;
2008-03-26 15:13:05 +01:00
}
2012-08-22 16:30:33 +02:00
2012-11-13 16:54:28 +01:00
/ * *
* using a fragment of the url hash ( 6 bytes : bytes 6 to 11 ) it is possible to address all urls from a specific domain
* here such a fragment can be used to delete all these domains at once
* @param hosthash the hash of the host to be deleted
* @param freshdate either NULL or a date in the past which is the limit for deletion . Only documents older than this date are deleted
* @return number of deleted domains
* @throws IOException
* /
public int deleteDomain ( final String hosthash , Date freshdate , boolean concurrent ) {
// first collect all url hashes that belong to the domain
assert hosthash . length ( ) = = 6 ;
final String q = YaCySchema . host_id_s . getSolrFieldName ( ) + " : \" " + hosthash + " \" " +
( ( freshdate ! = null & & freshdate . before ( new Date ( ) ) ) ? ( " AND " + YaCySchema . load_date_dt . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( freshdate ) + " ] " ) : " " ) ;
final AtomicInteger count = new AtomicInteger ( 0 ) ;
Thread t = new Thread ( ) {
public void run ( ) {
// delete in solr
synchronized ( Fulltext . this . solr ) {
try {
count . addAndGet ( Fulltext . this . solr . deleteByQuery ( q ) ) ;
Fulltext . this . solr . commit ( ) ;
} catch ( IOException e ) { }
}
// delete in old metadata structure
if ( Fulltext . this . urlIndexFile ! = null ) {
final ArrayList < String > l = new ArrayList < String > ( ) ;
synchronized ( this ) {
CloneableIterator < byte [ ] > i ;
try {
i = Fulltext . this . urlIndexFile . keys ( true , null ) ;
String hash ;
while ( i ! = null & & i . hasNext ( ) ) {
hash = ASCII . String ( i . next ( ) ) ;
if ( hosthash . equals ( hash . substring ( 6 ) ) ) l . add ( hash ) ;
}
// then delete the urls using this list
for ( final String h : l ) Fulltext . this . urlIndexFile . delete ( ASCII . getBytes ( h ) ) ;
} catch ( IOException e ) { }
}
}
// finally remove the line with statistics
if ( Fulltext . this . statsDump ! = null ) {
final Iterator < HostStat > hsi = Fulltext . this . statsDump . iterator ( ) ;
HostStat hs ;
while ( hsi . hasNext ( ) ) {
hs = hsi . next ( ) ;
if ( hs . hosthash . equals ( hosthash ) ) {
hsi . remove ( ) ;
break ;
}
}
}
}
} ;
if ( concurrent ) t . start ( ) ; else t . run ( ) ;
return count . get ( ) ;
}
2012-11-02 01:22:31 +01:00
/ * *
* remove a full subpath from the index
* @param subpath the left path of the url ; at least until the end of the host
2012-11-13 16:54:28 +01:00
* @param freshdate either NULL or a date in the past which is the limit for deletion . Only documents older than this date are deleted
2012-11-02 01:22:31 +01:00
* @param concurrently if true , then the method returnes immediately and runs concurrently
* /
2012-11-13 16:54:28 +01:00
public int remove ( String subpath , Date freshdate , final boolean concurrently ) {
2012-11-02 01:22:31 +01:00
int p = subpath . substring ( 0 , subpath . length ( ) - 1 ) . lastIndexOf ( '/' ) ;
final String path = p > 8 ? subpath . substring ( 0 , p + 1 ) : subpath ;
DigestURI uri ;
2012-11-13 16:54:28 +01:00
try { uri = new DigestURI ( path ) ; } catch ( MalformedURLException e ) { return 0 ; }
2012-11-02 01:22:31 +01:00
final String host = uri . getHost ( ) ;
2012-11-13 16:54:28 +01:00
final String q = YaCySchema . host_s . getSolrFieldName ( ) + " : " + host +
( ( freshdate ! = null & & freshdate . before ( new Date ( ) ) ) ? ( " AND " + YaCySchema . load_date_dt . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( freshdate ) + " ] " ) : " " ) ;
final AtomicInteger count = new AtomicInteger ( 0 ) ;
2012-11-02 01:22:31 +01:00
Thread t = new Thread ( ) {
public void run ( ) {
2012-11-13 16:54:28 +01:00
final BlockingQueue < SolrDocument > docs = getSolr ( ) . concurrentQuery ( q , 0 , 1000000 , 600000 , - 1 ) ;
2012-11-02 01:22:31 +01:00
try {
SolrDocument doc ;
while ( ( doc = docs . take ( ) ) ! = AbstractSolrConnector . POISON_DOCUMENT ) {
String u = ( String ) doc . getFieldValue ( YaCySchema . sku . getSolrFieldName ( ) ) ;
if ( u . startsWith ( path ) ) {
2012-11-06 14:32:08 +01:00
remove ( ASCII . getBytes ( ( String ) doc . getFieldValue ( YaCySchema . id . getSolrFieldName ( ) ) ) ) ;
2012-11-13 16:54:28 +01:00
count . incrementAndGet ( ) ;
2012-11-02 01:22:31 +01:00
}
}
2012-11-13 16:54:28 +01:00
if ( count . get ( ) > 0 ) Fulltext . this . solr . commit ( ) ;
2012-11-02 01:22:31 +01:00
} catch ( InterruptedException e ) { }
}
} ;
if ( concurrently ) t . start ( ) ; else t . run ( ) ;
2012-11-13 16:54:28 +01:00
return count . get ( ) ;
2012-11-02 01:22:31 +01:00
}
/ * *
* remove a list of id ' s from the index
* @param deleteIDs a list of urlhashes ; each denoting a document
* @param concurrently if true , then the method returnes immediately and runs concurrently
* /
public void remove ( final List < byte [ ] > deleteIDs , final boolean concurrently ) {
2012-11-02 13:57:43 +01:00
if ( deleteIDs = = null | | deleteIDs . size ( ) = = 0 ) return ;
2012-11-02 01:22:31 +01:00
Thread t = new Thread ( ) {
public void run ( ) {
2012-11-02 13:57:43 +01:00
try {
synchronized ( Fulltext . this . solr ) {
for ( byte [ ] urlHash : deleteIDs ) {
Fulltext . this . solr . delete ( ASCII . String ( urlHash ) ) ;
}
2012-11-06 00:29:37 +01:00
Fulltext . this . solr . commit ( ) ;
2012-11-02 13:57:43 +01:00
}
} catch ( final Throwable e ) {
Log . logException ( e ) ;
}
if ( Fulltext . this . urlIndexFile ! = null ) try {
for ( byte [ ] urlHash : deleteIDs ) {
final Row . Entry r = Fulltext . this . urlIndexFile . remove ( urlHash ) ;
if ( r ! = null ) Fulltext . this . statsDump = null ;
}
} catch ( final IOException e ) { }
} } ;
2012-11-02 01:22:31 +01:00
if ( concurrently ) t . start ( ) ; else t . run ( ) ;
2012-10-31 17:44:45 +01:00
}
2012-06-26 13:54:48 +02:00
public boolean remove ( final byte [ ] urlHash ) {
if ( urlHash = = null ) return false ;
2012-07-24 17:23:29 +02:00
try {
2012-09-25 00:19:52 +02:00
synchronized ( this . solr ) {
this . solr . delete ( ASCII . String ( urlHash ) ) ;
}
2012-07-24 17:23:29 +02:00
} catch ( final Throwable e ) {
Log . logException ( e ) ;
2012-06-26 13:54:48 +02:00
}
2012-07-22 13:18:45 +02:00
if ( this . urlIndexFile ! = null ) try {
2012-06-26 13:54:48 +02:00
final Row . Entry r = this . urlIndexFile . remove ( urlHash ) ;
2011-07-15 10:38:10 +02:00
if ( r ! = null ) this . statsDump = null ;
2008-03-26 15:13:05 +01:00
return r ! = null ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-03-26 15:13:05 +01:00
return false ;
}
2012-07-22 13:18:45 +02:00
return false ;
2008-03-26 15:13:05 +01:00
}
2010-04-08 02:11:32 +02:00
public boolean exists ( final byte [ ] urlHash ) {
2012-05-15 12:25:14 +02:00
if ( urlHash = = null ) return false ;
2012-07-25 14:31:54 +02:00
if ( this . urlIndexFile ! = null & & this . urlIndexFile . has ( urlHash ) ) return true ;
2012-07-24 17:23:29 +02:00
try {
if ( this . solr . exists ( ASCII . String ( urlHash ) ) ) return true ;
} catch ( final Throwable e ) {
Log . logException ( e ) ;
2011-11-08 12:49:04 +01:00
}
2012-07-25 14:31:54 +02:00
return false ;
2008-03-26 15:13:05 +01:00
}
2012-10-02 11:13:06 +02:00
public String failReason ( final String urlHash ) throws IOException {
if ( urlHash = = null ) return null ;
SolrDocument doc = this . solr . get ( urlHash ) ;
if ( doc = = null ) return null ;
2012-10-18 15:09:04 +02:00
String reason = ( String ) doc . getFieldValue ( YaCySchema . failreason_t . getSolrFieldName ( ) ) ;
2012-10-02 11:13:06 +02:00
return reason = = null ? null : reason . length ( ) = = 0 ? null : reason ;
}
2012-07-25 14:31:54 +02:00
@Override
public Iterator < byte [ ] > iterator ( ) {
2012-08-18 15:39:14 +02:00
CloneableIterator < byte [ ] > a = null ;
if ( this . urlIndexFile ! = null ) try { a = this . urlIndexFile . keys ( true , null ) ; } catch ( IOException e ) { }
final Iterator < String > idi = this . solr . iterator ( ) ;
CloneableIterator < byte [ ] > b = new CloneableIterator < byte [ ] > ( ) {
@Override
public boolean hasNext ( ) {
return idi . hasNext ( ) ;
}
@Override
public byte [ ] next ( ) {
String s = idi . next ( ) ;
return s = = null ? null : ASCII . getBytes ( s ) ;
}
@Override
public void remove ( ) {
throw new UnsupportedOperationException ( ) ;
}
@Override
public CloneableIterator < byte [ ] > clone ( Object modifier ) {
return this ;
}
@Override
public void close ( ) {
}
} ;
if ( a = = null ) return b ;
return new MergeIterator < byte [ ] > ( a , b ,
URIMetadataRow . rowdef . objectOrder ,
MergeIterator . simpleMerge ,
true ) ;
}
2012-10-16 18:11:57 +02:00
public CloneableIterator < URIMetadataNode > entries ( ) {
2008-03-26 15:13:05 +01:00
// enumerates entry elements
2012-08-18 15:39:14 +02:00
final Iterator < byte [ ] > ids = iterator ( ) ;
2012-10-16 18:11:57 +02:00
return new CloneableIterator < URIMetadataNode > ( ) {
2012-08-18 15:39:14 +02:00
@Override
2012-10-16 18:11:57 +02:00
public CloneableIterator < URIMetadataNode > clone ( final Object secondHash ) {
2012-08-18 15:39:14 +02:00
return this ;
2008-03-26 15:13:05 +01:00
}
2012-08-18 15:39:14 +02:00
@Override
public final boolean hasNext ( ) {
return ids . hasNext ( ) ;
}
@Override
2012-10-16 18:11:57 +02:00
public final URIMetadataNode next ( ) {
2012-08-18 15:39:14 +02:00
byte [ ] id = ids . next ( ) ;
if ( id = = null ) return null ;
return getMetadata ( id ) ;
}
@Override
public final void remove ( ) {
ids . remove ( ) ;
}
@Override
public void close ( ) {
}
} ;
2008-03-26 15:13:05 +01:00
}
2012-09-24 17:05:28 +02:00
public List < File > dumpFiles ( ) {
EmbeddedSolrConnector esc = ( EmbeddedSolrConnector ) this . solr . getSolr0 ( ) ;
ArrayList < File > zips = new ArrayList < File > ( ) ;
2012-11-07 02:04:08 +01:00
if ( esc = = null ) {
Log . logWarning ( " Fulltext " , " HOT DUMP selected solr0 == NULL, no dump list! " ) ;
return zips ;
}
if ( esc . getStoragePath ( ) = = null ) {
Log . logWarning ( " Fulltext " , " HOT DUMP selected solr0.getStoragePath() == NULL, no dump list! " ) ;
return zips ;
}
File storagePath = esc . getStoragePath ( ) . getParentFile ( ) ;
if ( storagePath = = null ) {
Log . logWarning ( " Fulltext " , " HOT DUMP selected esc.getStoragePath().getParentFile() == NULL, no dump list! " ) ;
return zips ;
}
Log . logInfo ( " Fulltext " , " HOT DUMP dump path = " + storagePath . toString ( ) ) ;
2012-09-24 17:05:28 +02:00
for ( String p : storagePath . list ( ) ) {
if ( p . endsWith ( " zip " ) ) zips . add ( new File ( storagePath , p ) ) ;
}
return zips ;
}
/ * *
* create a dump file from the current solr directory
* @return
* /
public File dumpSolr ( ) {
EmbeddedSolrConnector esc = ( EmbeddedSolrConnector ) this . solr . getSolr0 ( ) ;
int commitWithin = esc . getCommitWithinMs ( ) ;
File storagePath = esc . getStoragePath ( ) ;
File zipOut = new File ( storagePath . toString ( ) + " _ " + GenericFormatter . SHORT_DAY_FORMATTER . format ( ) + " .zip " ) ;
2012-09-25 00:19:52 +02:00
synchronized ( this . solr ) {
this . disconnectLocalSolr ( ) ;
2012-09-24 17:05:28 +02:00
try {
2012-09-25 00:19:52 +02:00
ZIPWriter . zip ( storagePath , zipOut ) ;
2012-09-24 17:05:28 +02:00
} catch ( IOException e ) {
Log . logException ( e ) ;
2012-09-25 00:19:52 +02:00
} finally {
try {
this . connectLocalSolr ( commitWithin ) ;
} catch ( IOException e ) {
Log . logException ( e ) ;
}
2012-09-24 17:05:28 +02:00
}
}
return zipOut ;
}
/ * *
* restore a solr dump to the current solr directory
* @param solrDumpZipFile
* /
public void restoreSolr ( File solrDumpZipFile ) {
EmbeddedSolrConnector esc = ( EmbeddedSolrConnector ) this . solr . getSolr0 ( ) ;
int commitWithin = esc . getCommitWithinMs ( ) ;
File storagePath = esc . getStoragePath ( ) ;
2012-09-25 00:19:52 +02:00
synchronized ( this . solr ) {
this . disconnectLocalSolr ( ) ;
2012-09-24 17:05:28 +02:00
try {
2012-09-25 00:19:52 +02:00
ZIPReader . unzip ( solrDumpZipFile , storagePath ) ;
2012-09-24 17:05:28 +02:00
} catch ( IOException e ) {
Log . logException ( e ) ;
2012-09-25 00:19:52 +02:00
} finally {
try {
this . connectLocalSolr ( commitWithin ) ;
} catch ( IOException e ) {
Log . logException ( e ) ;
}
2012-09-24 17:05:28 +02:00
}
}
}
2008-03-26 15:13:05 +01:00
// export methods
2011-07-15 10:38:10 +02:00
public Export export ( final File f , final String filter , final HandleSet set , final int format , final boolean dom ) {
if ( ( this . exportthread ! = null ) & & ( this . exportthread . isAlive ( ) ) ) {
2009-01-31 00:33:47 +01:00
Log . logWarning ( " LURL-EXPORT " , " cannot start another export thread, already one running " ) ;
2011-07-15 10:38:10 +02:00
return this . exportthread ;
2008-03-26 15:13:05 +01:00
}
2009-03-10 21:52:10 +01:00
this . exportthread = new Export ( f , filter , set , format , dom ) ;
2008-03-26 15:13:05 +01:00
this . exportthread . start ( ) ;
2011-07-15 10:38:10 +02:00
return this . exportthread ;
2008-03-26 15:13:05 +01:00
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public Export export ( ) {
return this . exportthread ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public class Export extends Thread {
2008-08-02 14:12:04 +02:00
private final File f ;
private final String filter ;
2008-03-26 15:13:05 +01:00
private int count ;
private String failure ;
2008-08-02 14:12:04 +02:00
private final int format ;
private final boolean dom ;
2010-01-11 00:09:48 +01:00
private final HandleSet set ;
2011-07-15 10:38:10 +02:00
2012-10-18 15:09:04 +02:00
private Export ( final File f , final String filter , final HandleSet set , final int format , boolean dom ) {
2008-03-26 15:13:05 +01:00
// format: 0=text, 1=html, 2=rss/xml
this . f = f ;
this . filter = filter ;
this . count = 0 ;
this . failure = null ;
this . format = format ;
this . dom = dom ;
2009-03-10 21:52:10 +01:00
this . set = set ;
2008-03-26 15:13:05 +01:00
if ( ( dom ) & & ( format = = 2 ) ) dom = false ;
}
2011-07-15 10:38:10 +02:00
2011-12-17 01:27:08 +01:00
@Override
2008-03-26 15:13:05 +01:00
public void run ( ) {
try {
2011-07-15 10:38:10 +02:00
final File parentf = this . f . getParentFile ( ) ;
2009-03-10 21:52:10 +01:00
if ( parentf ! = null ) parentf . mkdirs ( ) ;
2011-07-15 10:38:10 +02:00
final PrintWriter pw = new PrintWriter ( new BufferedOutputStream ( new FileOutputStream ( this . f ) ) ) ;
if ( this . format = = 1 ) {
2008-03-26 15:13:05 +01:00
pw . println ( " <html><head></head><body> " ) ;
}
2011-07-15 10:38:10 +02:00
if ( this . format = = 2 ) {
2008-03-26 15:13:05 +01:00
pw . println ( " <?xml version= \" 1.0 \" encoding= \" UTF-8 \" ?> " ) ;
pw . println ( " <?xml-stylesheet type='text/xsl' href='/yacysearch.xsl' version='1.0'?> " ) ;
2009-02-23 12:39:20 +01:00
pw . println ( " <rss version= \" 2.0 \" xmlns:yacy= \" http://www.yacy.net/ \" xmlns:opensearch= \" http://a9.com/-/spec/opensearch/1.1/ \" xmlns:atom= \" http://www.w3.org/2005/Atom \" > " ) ;
2008-03-26 15:13:05 +01:00
pw . println ( " <channel> " ) ;
pw . println ( " <title>YaCy Peer-to-Peer - Web-Search LURL Export</title> " ) ;
pw . println ( " <description></description> " ) ;
pw . println ( " <link>http://yacy.net</link> " ) ;
}
2011-07-15 10:38:10 +02:00
if ( this . dom ) {
final TreeSet < String > set = domainNameCollector ( - 1 , domainSampleCollector ( ) ) ;
for ( final String host : set ) {
if ( ! host . matches ( this . filter ) ) continue ;
if ( this . format = = 0 ) pw . println ( host ) ;
if ( this . format = = 1 ) pw . println ( " <a href= \" http:// " + host + " \" > " + host + " </a><br> " ) ;
this . count + + ;
2008-09-04 22:28:36 +02:00
}
} else {
2012-10-16 18:11:57 +02:00
final Iterator < URIMetadataNode > i = entries ( ) ; // iterates indexURLEntry objects
2012-10-18 14:29:11 +02:00
URIMetadataNode entry ;
2008-09-04 22:28:36 +02:00
String url ;
while ( i . hasNext ( ) ) {
entry = i . next ( ) ;
2011-07-15 10:38:10 +02:00
if ( this . set ! = null & & ! this . set . has ( entry . hash ( ) ) ) continue ;
2012-10-10 11:46:22 +02:00
url = entry . url ( ) . toNormalform ( true ) ;
2011-07-15 10:38:10 +02:00
if ( ! url . matches ( this . filter ) ) continue ;
if ( this . format = = 0 ) {
2008-03-26 15:13:05 +01:00
pw . println ( url ) ;
}
2011-07-15 10:38:10 +02:00
if ( this . format = = 1 ) {
2011-12-17 01:27:08 +01:00
pw . println ( " <a href= \" " + url + " \" > " + CharacterCoding . unicode2xml ( entry . dc_title ( ) , true ) + " </a><br> " ) ;
2008-03-26 15:13:05 +01:00
}
2011-07-15 10:38:10 +02:00
if ( this . format = = 2 ) {
2008-03-26 15:13:05 +01:00
pw . println ( " <item> " ) ;
2011-12-17 01:27:08 +01:00
pw . println ( " <title> " + CharacterCoding . unicode2xml ( entry . dc_title ( ) , true ) + " </title> " ) ;
2010-05-25 14:54:57 +02:00
pw . println ( " <link> " + MultiProtocolURI . escape ( url ) + " </link> " ) ;
2012-07-10 22:59:03 +02:00
if ( ! entry . dc_creator ( ) . isEmpty ( ) ) pw . println ( " <author> " + CharacterCoding . unicode2xml ( entry . dc_creator ( ) , true ) + " </author> " ) ;
if ( ! entry . dc_subject ( ) . isEmpty ( ) ) pw . println ( " <description> " + CharacterCoding . unicode2xml ( entry . dc_subject ( ) , true ) + " </description> " ) ;
2008-03-26 15:13:05 +01:00
pw . println ( " <pubDate> " + entry . moddate ( ) . toString ( ) + " </pubDate> " ) ;
2009-02-23 12:39:20 +01:00
pw . println ( " <yacy:size> " + entry . size ( ) + " </yacy:size> " ) ;
2011-05-27 10:24:54 +02:00
pw . println ( " <guid isPermaLink= \" false \" > " + ASCII . String ( entry . hash ( ) ) + " </guid> " ) ;
2008-03-26 15:13:05 +01:00
pw . println ( " </item> " ) ;
}
2011-07-15 10:38:10 +02:00
this . count + + ;
2008-03-26 15:13:05 +01:00
}
}
2011-07-15 10:38:10 +02:00
if ( this . format = = 1 ) {
2008-03-26 15:13:05 +01:00
pw . println ( " </body></html> " ) ;
}
2011-07-15 10:38:10 +02:00
if ( this . format = = 2 ) {
2008-03-26 15:13:05 +01:00
pw . println ( " </channel> " ) ;
pw . println ( " </rss> " ) ;
}
pw . close ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-03-26 15:13:05 +01:00
this . failure = e . getMessage ( ) ;
2009-05-05 08:31:35 +02:00
} catch ( final Exception e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2009-05-05 08:31:35 +02:00
this . failure = e . getMessage ( ) ;
2008-03-26 15:13:05 +01:00
}
// terminate process
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public File file ( ) {
return this . f ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public String failed ( ) {
return this . failure ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public int count ( ) {
return this . count ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
}
2011-07-15 10:38:10 +02:00
2011-05-26 12:57:02 +02:00
/ * *
* collect domain samples : all url hashes from the metadata database is listed and the domain part
* of the url hashes is used to count how many of these domain hashes appear
* @return a map from domain hashes to hash statistics
* @throws IOException
* /
public Map < String , URLHashCounter > domainSampleCollector ( ) throws IOException {
2011-07-15 10:38:10 +02:00
final Map < String , URLHashCounter > map = new HashMap < String , URLHashCounter > ( ) ;
2008-09-04 21:41:57 +02:00
// first collect all domains and calculate statistics about it
2012-01-31 21:13:49 +01:00
synchronized ( this ) {
2012-08-18 15:39:14 +02:00
final Iterator < byte [ ] > i = this . iterator ( ) ;
2012-01-31 21:13:49 +01:00
String hosthash ;
byte [ ] urlhashb ;
URLHashCounter ds ;
if ( i ! = null ) while ( i . hasNext ( ) ) {
urlhashb = i . next ( ) ;
hosthash = ASCII . String ( urlhashb , 6 , 6 ) ;
ds = map . get ( hosthash ) ;
if ( ds = = null ) {
ds = new URLHashCounter ( urlhashb ) ;
map . put ( hosthash , ds ) ;
} else {
ds . count + + ;
}
2008-09-04 21:41:57 +02:00
}
}
2008-09-04 22:28:36 +02:00
return map ;
}
2011-07-15 10:38:10 +02:00
2011-05-26 12:57:02 +02:00
/ * *
* create a list of domain names in this database
* @param count number of entries or - 1 for all
* @param domainSamples a map from domain hashes to hash statistics
* @return a set of domain names , ordered by name of the domains
* /
2012-10-18 15:09:04 +02:00
private TreeSet < String > domainNameCollector ( int count , final Map < String , URLHashCounter > domainSamples ) {
2008-09-04 22:28:36 +02:00
// collect hashes from all domains
2011-07-15 10:38:10 +02:00
2008-09-04 22:28:36 +02:00
// fetch urls from the database to determine the host in clear text
2012-10-16 18:11:57 +02:00
URIMetadataNode urlref ;
2011-05-26 12:57:02 +02:00
if ( count < 0 | | count > domainSamples . size ( ) ) count = domainSamples . size ( ) ;
2011-07-15 10:38:10 +02:00
this . statsDump = new ArrayList < HostStat > ( ) ;
final TreeSet < String > set = new TreeSet < String > ( ) ;
for ( final URLHashCounter hs : domainSamples . values ( ) ) {
2008-09-04 22:28:36 +02:00
if ( hs = = null ) continue ;
2012-08-17 01:34:38 +02:00
urlref = this . getMetadata ( hs . urlhashb ) ;
2011-12-17 01:27:08 +01:00
if ( urlref = = null | | urlref . url ( ) = = null | | urlref . url ( ) . getHost ( ) = = null ) continue ;
set . add ( urlref . url ( ) . getHost ( ) ) ;
2008-09-04 22:28:36 +02:00
count - - ;
if ( count = = 0 ) break ;
}
return set ;
}
2011-05-26 12:57:02 +02:00
/ * *
* calculate a score map for url hash samples : each sample is a single url hash
* that stands for all entries for the corresponding domain . The map counts the number
* of occurrences of the domain
* @param domainSamples a map from domain hashes to hash statistics
* @return a map from url hash samples to counters
* /
2011-07-15 10:38:10 +02:00
public ScoreMap < String > urlSampleScores ( final Map < String , URLHashCounter > domainSamples ) {
final ScoreMap < String > urlSampleScore = new ConcurrentScoreMap < String > ( ) ;
for ( final Map . Entry < String , URLHashCounter > e : domainSamples . entrySet ( ) ) {
2011-05-27 10:24:54 +02:00
urlSampleScore . inc ( ASCII . String ( e . getValue ( ) . urlhashb ) , e . getValue ( ) . count ) ;
2011-05-26 12:57:02 +02:00
}
return urlSampleScore ;
}
2011-07-15 10:38:10 +02:00
2011-05-26 12:57:02 +02:00
/ * *
* calculate all domain names for all domain hashes
* @param domainSamples a map from domain hashes to hash statistics
* @return a map from domain hashes to host stats including domain names
* /
2011-07-15 10:38:10 +02:00
public Map < String , HostStat > domainHashResolver ( final Map < String , URLHashCounter > domainSamples ) {
final HashMap < String , HostStat > hostMap = new HashMap < String , HostStat > ( ) ;
2012-10-16 18:11:57 +02:00
URIMetadataNode urlref ;
2011-07-15 10:38:10 +02:00
final ScoreMap < String > hosthashScore = new ConcurrentScoreMap < String > ( ) ;
for ( final Map . Entry < String , URLHashCounter > e : domainSamples . entrySet ( ) ) {
2011-05-30 09:31:17 +02:00
hosthashScore . inc ( ASCII . String ( e . getValue ( ) . urlhashb , 6 , 6 ) , e . getValue ( ) . count ) ;
2008-09-04 21:41:57 +02:00
}
2011-05-26 12:57:02 +02:00
DigestURI url ;
2011-07-15 10:38:10 +02:00
for ( final Map . Entry < String , URLHashCounter > e : domainSamples . entrySet ( ) ) {
2012-08-17 01:34:38 +02:00
urlref = this . getMetadata ( e . getValue ( ) . urlhashb ) ;
2011-12-17 01:27:08 +01:00
url = urlref . url ( ) ;
2011-05-26 12:57:02 +02:00
hostMap . put ( e . getKey ( ) , new HostStat ( url . getHost ( ) , url . getPort ( ) , e . getKey ( ) , hosthashScore . get ( e . getKey ( ) ) ) ) ;
}
return hostMap ;
}
2011-07-15 10:38:10 +02:00
public Iterator < HostStat > statistics ( int count , final ScoreMap < String > domainScore ) {
2011-05-26 12:57:02 +02:00
// prevent too heavy IO.
2011-07-15 10:38:10 +02:00
if ( this . statsDump ! = null & & count < = this . statsDump . size ( ) ) return this . statsDump . iterator ( ) ;
2008-09-04 21:41:57 +02:00
// fetch urls from the database to determine the host in clear text
2011-07-15 10:38:10 +02:00
final Iterator < String > j = domainScore . keys ( false ) ; // iterate urlhash-examples in reverse order (biggest first)
2012-10-16 18:11:57 +02:00
URIMetadataNode urlref ;
2008-09-04 22:28:36 +02:00
String urlhash ;
2008-09-04 21:41:57 +02:00
count + = 10 ; // make some more to prevent that we have to do this again after deletions too soon.
2011-05-26 12:57:02 +02:00
if ( count < 0 | | domainScore . sizeSmaller ( count ) ) count = domainScore . size ( ) ;
2011-07-15 10:38:10 +02:00
this . statsDump = new ArrayList < HostStat > ( ) ;
2009-10-11 02:12:19 +02:00
DigestURI url ;
2008-09-04 22:28:36 +02:00
while ( j . hasNext ( ) ) {
2008-09-04 21:41:57 +02:00
urlhash = j . next ( ) ;
if ( urlhash = = null ) continue ;
2012-08-17 01:34:38 +02:00
urlref = this . getMetadata ( ASCII . getBytes ( urlhash ) ) ;
2011-12-17 01:27:08 +01:00
if ( urlref = = null | | urlref . url ( ) = = null | | urlref . url ( ) . getHost ( ) = = null ) continue ;
2011-07-15 10:38:10 +02:00
if ( this . statsDump = = null ) return new ArrayList < HostStat > ( ) . iterator ( ) ; // some other operation has destroyed the object
2011-12-17 01:27:08 +01:00
url = urlref . url ( ) ;
2011-07-15 10:38:10 +02:00
this . statsDump . add ( new HostStat ( url . getHost ( ) , url . getPort ( ) , urlhash . substring ( 6 ) , domainScore . get ( urlhash ) ) ) ;
2008-09-04 21:41:57 +02:00
count - - ;
2008-09-04 22:28:36 +02:00
if ( count = = 0 ) break ;
2008-09-04 21:41:57 +02:00
}
// finally return an iterator for the result array
2011-07-15 10:38:10 +02:00
return ( this . statsDump = = null ) ? new ArrayList < HostStat > ( ) . iterator ( ) : this . statsDump . iterator ( ) ;
2008-09-04 21:41:57 +02:00
}
2011-07-15 10:38:10 +02:00
2011-05-26 12:57:02 +02:00
private static class URLHashCounter {
public byte [ ] urlhashb ;
2008-09-04 21:41:57 +02:00
public int count ;
2011-07-15 10:38:10 +02:00
public URLHashCounter ( final byte [ ] urlhashb ) {
2011-05-26 12:57:02 +02:00
this . urlhashb = urlhashb ;
2008-09-04 21:41:57 +02:00
this . count = 1 ;
}
}
2011-07-15 10:38:10 +02:00
2011-05-26 12:57:02 +02:00
public static class HostStat {
2008-09-04 21:41:57 +02:00
public String hostname , hosthash ;
2008-09-08 20:20:05 +02:00
public int port ;
2008-09-04 21:41:57 +02:00
public int count ;
2012-10-18 15:09:04 +02:00
private HostStat ( final String host , final int port , final String urlhashfragment , final int count ) {
2008-09-04 23:28:00 +02:00
assert urlhashfragment . length ( ) = = 6 ;
2008-09-04 21:41:57 +02:00
this . hostname = host ;
2008-09-08 20:20:05 +02:00
this . port = port ;
2008-09-04 21:41:57 +02:00
this . hosthash = urlhashfragment ;
this . count = count ;
}
}
2008-03-26 15:13:05 +01:00
}