2012-08-17 15:33:02 +02:00
// Fulltext.java
2008-03-26 15:13:05 +01:00
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2006 as part of 'plasmaCrawlLURL.java' on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
2011-07-15 10:38:10 +02:00
//
2008-03-26 15:13:05 +01:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2011-09-25 18:59:06 +02:00
package net.yacy.search.index ;
2008-03-26 15:13:05 +01:00
import java.io.BufferedOutputStream ;
import java.io.File ;
import java.io.FileOutputStream ;
import java.io.IOException ;
2015-05-30 19:02:54 +02:00
import java.io.OutputStream ;
2008-03-26 15:13:05 +01:00
import java.io.PrintWriter ;
2014-05-20 21:50:16 +02:00
import java.lang.reflect.Array ;
2012-11-02 01:22:31 +01:00
import java.net.MalformedURLException ;
2008-09-04 21:41:57 +02:00
import java.util.ArrayList ;
2013-02-22 15:45:15 +01:00
import java.util.Collection ;
2012-09-26 16:56:33 +02:00
import java.util.Date ;
2014-02-07 00:22:31 +01:00
import java.util.HashMap ;
2014-03-06 01:43:48 +01:00
import java.util.HashSet ;
2012-09-24 17:05:28 +02:00
import java.util.List ;
2008-09-04 21:41:57 +02:00
import java.util.Map ;
2013-05-17 13:59:37 +02:00
import java.util.Set ;
2012-11-02 01:22:31 +01:00
import java.util.concurrent.BlockingQueue ;
2012-11-13 16:54:28 +01:00
import java.util.concurrent.atomic.AtomicInteger ;
2013-01-24 12:39:19 +01:00
import java.util.regex.Pattern ;
2015-06-01 01:24:33 +02:00
import java.util.zip.Deflater ;
2015-05-30 19:02:54 +02:00
import java.util.zip.GZIPOutputStream ;
2008-03-26 15:13:05 +01:00
2012-09-24 17:05:28 +02:00
import net.yacy.cora.date.GenericFormatter ;
2012-11-13 16:54:28 +01:00
import net.yacy.cora.date.ISO8601Formatter ;
2013-09-15 00:30:23 +02:00
import net.yacy.cora.document.encoding.ASCII ;
import net.yacy.cora.document.id.DigestURL ;
import net.yacy.cora.document.id.MultiProtocolURL ;
2012-11-02 01:22:31 +01:00
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector ;
2012-09-25 21:20:03 +02:00
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector ;
2013-02-21 13:23:55 +01:00
import net.yacy.cora.federate.solr.connector.RemoteSolrConnector ;
import net.yacy.cora.federate.solr.connector.ShardSelection ;
2012-09-25 21:20:03 +02:00
import net.yacy.cora.federate.solr.connector.SolrConnector ;
2013-02-21 13:23:55 +01:00
import net.yacy.cora.federate.solr.instance.EmbeddedInstance ;
import net.yacy.cora.federate.solr.instance.InstanceMirror ;
import net.yacy.cora.federate.solr.instance.RemoteInstance ;
import net.yacy.cora.federate.solr.instance.ShardInstance ;
2015-05-29 15:05:52 +02:00
import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter ;
2013-05-09 00:22:45 +02:00
import net.yacy.cora.protocol.HeaderFramework ;
2013-01-24 12:39:19 +01:00
import net.yacy.cora.sorting.ReversibleScoreMap ;
2013-02-27 22:40:23 +01:00
import net.yacy.cora.sorting.WeakPriorityBlockingQueue ;
2012-09-24 17:05:28 +02:00
import net.yacy.cora.storage.ZIPReader ;
import net.yacy.cora.storage.ZIPWriter ;
2015-05-29 15:05:52 +02:00
import net.yacy.cora.util.CRIgnoreWriter ;
2013-07-09 14:28:25 +02:00
import net.yacy.cora.util.ConcurrentLog ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.html.CharacterCoding ;
2012-08-10 13:26:51 +02:00
import net.yacy.kelondro.data.meta.URIMetadataNode ;
2013-02-27 22:40:23 +01:00
import net.yacy.kelondro.data.word.WordReferenceVars ;
2011-05-26 16:35:32 +02:00
import net.yacy.kelondro.util.MemoryControl ;
2012-06-25 11:34:38 +02:00
import net.yacy.search.Switchboard ;
2013-02-21 13:23:55 +01:00
import net.yacy.search.schema.CollectionConfiguration ;
import net.yacy.search.schema.CollectionSchema ;
import net.yacy.search.schema.WebgraphConfiguration ;
import net.yacy.search.schema.WebgraphSchema ;
2012-07-19 11:34:05 +02:00
2012-08-10 13:26:51 +02:00
import org.apache.solr.common.SolrDocument ;
2012-08-10 15:39:10 +02:00
import org.apache.solr.common.SolrException ;
2012-08-17 15:46:26 +02:00
import org.apache.solr.common.SolrInputDocument ;
2014-02-07 00:02:19 +01:00
import org.apache.solr.core.SolrInfoMBean ;
2013-02-04 10:55:49 +01:00
import org.apache.lucene.util.Version ;
2012-07-23 16:28:39 +02:00
2013-01-24 12:39:19 +01:00
public final class Fulltext {
2008-03-26 15:13:05 +01:00
2015-06-24 01:55:51 +02:00
private static final String SOLR_PATH = " solr_5_2 " ; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml
private static final String SOLR_OLD_PATH [ ] = new String [ ] { " solr_36 " , " solr_40 " , " solr_44 " , " solr_45 " , " solr_46 " , " solr_47 " , " solr_4_9 " , " solr_4_10 " } ;
2012-11-02 12:29:48 +01:00
2008-03-26 15:13:05 +01:00
// class objects
2014-01-07 17:53:49 +01:00
private final File segmentPath ;
private final File archivePath ;
2013-07-23 16:46:44 +02:00
private Export exportthread ; // will have a export thread assigned if exporter is running
private InstanceMirror solrInstances ;
2013-02-21 13:23:55 +01:00
private final CollectionConfiguration collectionConfiguration ;
private final WebgraphConfiguration webgraphConfiguration ;
2013-07-23 16:46:44 +02:00
private boolean writeWebgraph ;
2013-02-21 13:23:55 +01:00
2014-01-07 17:53:49 +01:00
protected Fulltext ( final File segmentPath , final File archivePath ,
final CollectionConfiguration collectionConfiguration , final WebgraphConfiguration webgraphConfiguration ) {
2013-02-21 13:23:55 +01:00
this . segmentPath = segmentPath ;
2014-01-07 17:53:49 +01:00
this . archivePath = archivePath ;
2009-03-10 14:38:40 +01:00
this . exportthread = null ; // will have a export thread assigned if exporter is running
2013-02-21 13:23:55 +01:00
this . solrInstances = new InstanceMirror ( ) ;
this . collectionConfiguration = collectionConfiguration ;
this . webgraphConfiguration = webgraphConfiguration ;
2013-07-23 16:46:44 +02:00
this . writeWebgraph = false ;
}
2013-12-04 01:54:45 +01:00
public void setUseWebgraph ( boolean check ) {
2013-07-23 16:46:44 +02:00
this . writeWebgraph = check ;
}
2013-12-04 01:54:45 +01:00
public boolean useWebgraph ( ) {
2013-07-23 16:46:44 +02:00
return this . writeWebgraph ;
2011-11-08 12:49:04 +01:00
}
2012-07-23 16:28:39 +02:00
2013-02-21 13:23:55 +01:00
public CollectionConfiguration getDefaultConfiguration ( ) {
return this . collectionConfiguration ;
}
public WebgraphConfiguration getWebgraphConfiguration ( ) {
return this . webgraphConfiguration ;
2012-08-05 15:49:27 +02:00
}
2012-07-23 16:28:39 +02:00
public boolean connectedLocalSolr ( ) {
2014-02-06 19:13:29 +01:00
return this . solrInstances . isConnectedEmbedded ( ) ;
2012-07-23 16:28:39 +02:00
}
2013-02-13 02:29:47 +01:00
public void connectLocalSolr ( ) throws IOException {
2013-02-21 13:23:55 +01:00
File solrLocation = new File ( this . segmentPath , SOLR_PATH ) ;
2012-11-02 12:29:48 +01:00
// migrate old solr to new
for ( String oldVersion : SOLR_OLD_PATH ) {
2013-02-21 13:23:55 +01:00
File oldLocation = new File ( this . segmentPath , oldVersion ) ;
2012-11-02 12:29:48 +01:00
if ( oldLocation . exists ( ) ) oldLocation . renameTo ( solrLocation ) ;
}
2013-02-21 13:23:55 +01:00
EmbeddedInstance localCollectionInstance = new EmbeddedInstance ( new File ( new File ( Switchboard . getSwitchboard ( ) . appPath , " defaults " ) , " solr " ) , solrLocation , CollectionSchema . CORE_NAME , new String [ ] { CollectionSchema . CORE_NAME , WebgraphSchema . CORE_NAME } ) ;
2014-02-07 00:02:19 +01:00
Version luceneVersion = localCollectionInstance . getDefaultCore ( ) . getSolrConfig ( ) . getLuceneVersion ( " luceneMatchVersion " ) ;
2014-11-07 22:43:50 +01:00
String lvn = luceneVersion . major + " _ " + luceneVersion . minor ;
2013-07-09 14:28:25 +02:00
ConcurrentLog . info ( " Fulltext " , " using lucene version " + lvn ) ;
2014-11-07 22:43:50 +01:00
assert SOLR_PATH . endsWith ( lvn ) : " luceneVersion = " + lvn + " , solrPath = " + SOLR_PATH + " , check defaults/solr/solrconfig.xml " ;
2014-02-07 00:02:19 +01:00
ConcurrentLog . info ( " Fulltext " , " connected solr in " + solrLocation . toString ( ) + " , lucene version " + lvn ) ;
2014-02-06 19:13:29 +01:00
this . solrInstances . connectEmbedded ( localCollectionInstance ) ;
2012-06-25 11:34:38 +02:00
}
2012-07-20 11:40:33 +02:00
public void disconnectLocalSolr ( ) {
2014-02-06 19:13:29 +01:00
this . solrInstances . disconnectEmbedded ( ) ;
2012-07-20 11:40:33 +02:00
}
2012-07-22 13:18:45 +02:00
2012-07-23 16:28:39 +02:00
public boolean connectedRemoteSolr ( ) {
2014-02-06 19:13:29 +01:00
return this . solrInstances . isConnectedRemote ( ) ;
2012-07-23 16:28:39 +02:00
}
2014-07-16 14:57:51 +02:00
public void connectRemoteSolr ( final ArrayList < RemoteInstance > instances , final ShardSelection . Method shardMethod , final boolean writeEnabled ) {
this . solrInstances . connectRemote ( new ShardInstance ( instances , shardMethod , writeEnabled ) ) ;
2012-07-22 13:18:45 +02:00
}
public void disconnectRemoteSolr ( ) {
2014-02-06 19:13:29 +01:00
this . solrInstances . disconnectRemote ( ) ;
2012-07-22 13:18:45 +02:00
}
2012-07-23 16:28:39 +02:00
2013-02-22 15:45:15 +01:00
public EmbeddedSolrConnector getDefaultEmbeddedConnector ( ) {
return this . solrInstances . getDefaultEmbeddedConnector ( ) ;
}
public EmbeddedSolrConnector getEmbeddedConnector ( String corename ) {
return this . solrInstances . getEmbeddedConnector ( corename ) ;
2008-03-26 15:13:05 +01:00
}
2014-09-22 15:28:54 +02:00
public SolrConnector getConnectorForRead ( String corename ) {
if ( this . solrInstances . isConnectedRemote ( ) ) return this . solrInstances . getRemoteConnector ( corename ) ;
if ( this . solrInstances . isConnectedEmbedded ( ) ) return this . solrInstances . getEmbeddedConnector ( corename ) ;
return null ;
}
2013-02-21 13:23:55 +01:00
public RemoteSolrConnector getDefaultRemoteSolrConnector ( ) {
try {
2014-02-22 01:35:01 +01:00
return this . solrInstances . getDefaultRemoteConnector ( true ) ;
} catch ( IOException e ) {
2013-02-21 13:23:55 +01:00
return null ;
}
2012-07-24 17:23:29 +02:00
}
2013-02-21 13:23:55 +01:00
2014-01-23 19:01:31 +01:00
public EmbeddedInstance getEmbeddedInstance ( ) {
synchronized ( this . solrInstances ) {
2014-02-06 19:13:29 +01:00
if ( this . solrInstances . isConnectedEmbedded ( ) ) return this . solrInstances . getEmbedded ( ) ;
2014-01-23 19:01:31 +01:00
return null ;
}
}
2013-02-21 13:23:55 +01:00
public SolrConnector getDefaultConnector ( ) {
2013-05-29 13:09:34 +02:00
synchronized ( this . solrInstances ) {
return this . solrInstances . getDefaultMirrorConnector ( ) ;
}
2013-02-21 13:23:55 +01:00
}
public SolrConnector getWebgraphConnector ( ) {
2013-07-23 16:46:44 +02:00
if ( ! this . writeWebgraph ) return null ;
2013-05-29 13:09:34 +02:00
synchronized ( this . solrInstances ) {
2014-02-06 19:13:29 +01:00
return this . solrInstances . getGenericMirrorConnector ( WebgraphSchema . CORE_NAME ) ;
2013-05-29 13:09:34 +02:00
}
2012-06-26 13:54:48 +02:00
}
2014-01-16 17:27:14 +01:00
2014-02-07 00:22:31 +01:00
public Map < String , SolrInfoMBean > getSolrInfoBeans ( ) {
2014-02-07 00:02:19 +01:00
EmbeddedSolrConnector esc = this . solrInstances . getDefaultEmbeddedConnector ( ) ;
2014-02-07 00:22:31 +01:00
if ( esc = = null ) return new HashMap < String , SolrInfoMBean > ( ) ;
2014-02-07 00:02:19 +01:00
return esc . getSolrInfoBeans ( ) ;
}
2014-01-16 17:27:14 +01:00
public int bufferSize ( ) {
return this . solrInstances . bufferSize ( ) ;
}
2013-11-07 10:01:44 +01:00
public void clearCaches ( ) {
this . solrInstances . clearCaches ( ) ;
2008-06-16 23:39:58 +02:00
}
2011-07-15 10:38:10 +02:00
2013-01-04 16:39:34 +01:00
public void clearLocalSolr ( ) throws IOException {
2014-02-11 22:01:25 +01:00
if ( this . exportthread ! = null ) this . exportthread . interrupt ( ) ;
2013-05-29 13:09:34 +02:00
synchronized ( this . solrInstances ) {
2014-02-06 19:13:29 +01:00
EmbeddedInstance instance = this . solrInstances . getEmbedded ( ) ;
2013-05-29 13:09:34 +02:00
if ( instance ! = null ) {
2014-02-22 01:35:01 +01:00
for ( String name : instance . getCoreNames ( ) ) {
this . solrInstances . getEmbeddedConnector ( name ) . clear ( ) ;
}
2013-05-29 13:09:34 +02:00
}
this . commit ( false ) ;
2013-11-07 10:01:44 +01:00
this . solrInstances . clearCaches ( ) ;
2013-02-21 13:23:55 +01:00
}
2013-01-04 16:39:34 +01:00
}
public void clearRemoteSolr ( ) throws IOException {
2013-05-29 13:09:34 +02:00
synchronized ( this . solrInstances ) {
2014-02-06 19:13:29 +01:00
ShardInstance instance = this . solrInstances . getRemote ( ) ;
2013-05-29 13:09:34 +02:00
if ( instance ! = null ) {
2014-02-22 01:35:01 +01:00
for ( String name : instance . getCoreNames ( ) ) {
this . solrInstances . getRemoteConnector ( name ) . clear ( ) ;
}
2013-05-29 13:09:34 +02:00
}
2013-11-07 10:01:44 +01:00
this . solrInstances . clearCaches ( ) ;
2013-02-21 13:23:55 +01:00
}
2013-01-04 16:39:34 +01:00
}
2013-02-21 13:23:55 +01:00
/ * *
* get the size of the default index
* @return
* /
2013-05-08 11:50:46 +02:00
private long collectionSizeLastAccess = 0 ;
private long collectionSizeLastValue = 0 ;
2013-02-22 15:45:15 +01:00
public long collectionSize ( ) {
2013-05-08 11:50:46 +02:00
long t = System . currentTimeMillis ( ) ;
if ( t - this . collectionSizeLastAccess < 1000 ) return this . collectionSizeLastValue ;
2014-05-29 13:24:24 +02:00
SolrConnector sc = getDefaultConnector ( ) ;
2014-04-02 02:34:39 +02:00
if ( sc = = null ) return 0 ;
long size = sc . getSize ( ) ;
2013-05-08 11:50:46 +02:00
this . collectionSizeLastAccess = t ;
this . collectionSizeLastValue = size ;
2012-07-22 13:18:45 +02:00
return size ;
2008-03-26 15:13:05 +01:00
}
2013-02-22 15:45:15 +01:00
/ * *
* get the size of the webgraph index
* @return
* /
public long webgraphSize ( ) {
2013-07-23 16:46:44 +02:00
return this . writeWebgraph ? this . getWebgraphConnector ( ) . getSize ( ) : 0 ;
2013-02-22 15:45:15 +01:00
}
2008-03-26 15:13:05 +01:00
public void close ( ) {
2014-02-09 15:04:46 +01:00
try {
this . solrInstances . close ( ) ;
} catch ( Throwable e ) { }
2008-03-26 15:13:05 +01:00
}
2012-10-26 07:39:07 +02:00
2013-05-29 18:27:27 +02:00
private long lastCommit = 0 ;
2013-01-23 14:40:58 +01:00
public void commit ( boolean softCommit ) {
2013-05-29 18:27:27 +02:00
long t = System . currentTimeMillis ( ) ;
if ( lastCommit + 10000 > t ) return ;
lastCommit = t ;
2013-02-21 13:23:55 +01:00
getDefaultConnector ( ) . commit ( softCommit ) ;
2013-07-23 16:46:44 +02:00
if ( this . writeWebgraph ) getWebgraphConnector ( ) . commit ( softCommit ) ;
2012-10-26 07:39:07 +02:00
}
2014-02-24 21:01:56 +01:00
2013-07-15 18:22:35 +02:00
public URIMetadataNode getMetadata ( final WeakPriorityBlockingQueue . Element < WordReferenceVars > element ) {
2013-02-27 22:40:23 +01:00
if ( element = = null ) return null ;
2013-06-01 05:43:08 +02:00
WordReferenceVars wre = element . getElement ( ) ;
2012-07-25 14:31:54 +02:00
if ( wre = = null ) return null ; // all time was already wasted in takeRWI to get another element
2014-05-22 03:01:07 +02:00
float score = element . getWeight ( ) ;
URIMetadataNode node = getMetadata ( wre . urlhash ( ) , wre , score ) ;
2013-02-27 22:40:23 +01:00
return node ;
2011-05-13 08:21:40 +02:00
}
2011-07-15 10:38:10 +02:00
2012-10-16 18:11:57 +02:00
public URIMetadataNode getMetadata ( final byte [ ] urlHash ) {
2011-05-13 08:21:40 +02:00
if ( urlHash = = null ) return null ;
2014-05-22 03:01:07 +02:00
return getMetadata ( urlHash , null , 0 . 0f ) ;
2012-08-10 13:26:51 +02:00
}
2012-11-23 01:35:28 +01:00
2014-05-22 03:01:07 +02:00
private URIMetadataNode getMetadata ( final byte [ ] urlHash , final WordReferenceVars wre , final float score ) {
2013-03-02 10:25:52 +01:00
String u = ASCII . String ( urlHash ) ;
2012-08-10 13:26:51 +02:00
// get the metadata from Solr
try {
2013-04-27 01:32:18 +02:00
SolrDocument doc = this . getDefaultConnector ( ) . getDocumentById ( u ) ;
2012-08-18 19:36:21 +02:00
if ( doc ! = null ) {
2014-05-22 03:01:07 +02:00
return new URIMetadataNode ( doc , wre , score ) ;
2012-08-18 19:36:21 +02:00
}
2013-07-17 18:31:30 +02:00
} catch ( final IOException e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2012-08-10 13:26:51 +02:00
}
2012-08-17 01:34:38 +02:00
return null ;
}
2012-08-17 15:46:26 +02:00
public void putDocument ( final SolrInputDocument doc ) throws IOException {
2013-02-25 00:09:41 +01:00
SolrConnector connector = this . getDefaultConnector ( ) ;
if ( connector = = null ) return ;
2013-02-21 13:23:55 +01:00
String id = ( String ) doc . getFieldValue ( CollectionSchema . id . getSolrFieldName ( ) ) ;
2013-05-30 12:34:53 +02:00
String url = ( String ) doc . getFieldValue ( CollectionSchema . sku . getSolrFieldName ( ) ) ;
2015-05-11 14:42:21 +02:00
assert url ! = null & & url . length ( ) < 30000 ;
2013-07-09 14:28:25 +02:00
ConcurrentLog . info ( " Fulltext " , " indexing: " + id + " " + url ) ;
2012-09-26 16:05:11 +02:00
try {
2013-05-11 10:53:12 +02:00
connector . add ( doc ) ;
2013-07-17 18:31:30 +02:00
} catch ( final SolrException e ) {
2013-02-22 15:45:15 +01:00
throw new IOException ( e . getMessage ( ) , e ) ;
}
2013-11-07 10:01:44 +01:00
if ( MemoryControl . shortStatus ( ) ) clearCaches ( ) ;
2013-02-22 15:45:15 +01:00
}
2013-07-15 18:22:35 +02:00
2013-02-22 15:45:15 +01:00
public void putEdges ( final Collection < SolrInputDocument > edges ) throws IOException {
2013-12-04 01:54:45 +01:00
if ( ! this . useWebgraph ( ) ) return ;
2013-03-05 12:24:01 +01:00
if ( edges = = null | | edges . size ( ) = = 0 ) return ;
2013-02-22 15:45:15 +01:00
try {
this . getWebgraphConnector ( ) . add ( edges ) ;
2013-07-17 18:31:30 +02:00
} catch ( final SolrException e ) {
2012-09-26 16:05:11 +02:00
throw new IOException ( e . getMessage ( ) , e ) ;
2012-08-18 13:05:27 +02:00
}
2013-11-07 10:01:44 +01:00
if ( MemoryControl . shortStatus ( ) ) clearCaches ( ) ;
2012-08-17 15:46:26 +02:00
}
2013-07-15 18:22:35 +02:00
2013-08-20 15:46:04 +02:00
/ * *
* deprecated method to store document metadata , use Solr documents wherever possible
* /
2014-04-03 00:35:15 +02:00
public void putMetadata ( final URIMetadataNode entry ) throws IOException {
2013-02-26 17:16:31 +01:00
byte [ ] idb = entry . hash ( ) ;
2013-03-02 10:25:52 +01:00
String id = ASCII . String ( idb ) ;
2012-09-26 16:05:11 +02:00
try {
2013-03-02 10:25:52 +01:00
// because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten
2014-02-24 23:42:50 +01:00
long date = this . getLoadTime ( id ) ;
2015-03-24 12:32:39 +01:00
if ( date = = - 1 ) {
// document does not exist
2013-03-03 22:38:50 +01:00
putDocument ( getDefaultConfiguration ( ) . metadata2solr ( entry ) ) ;
2015-03-24 12:32:39 +01:00
} else {
// check if document contains rich data
if ( date < entry . loaddate ( ) . getTime ( ) ) {
SolrDocument doc = this . getDefaultConnector ( ) . getDocumentById ( id , CollectionSchema . collection_sxt . getSolrFieldName ( ) ) ;
if ( doc = = null | | ! doc . containsKey ( CollectionSchema . collection_sxt . getSolrFieldName ( ) ) ) {
putDocument ( getDefaultConfiguration ( ) . metadata2solr ( entry ) ) ;
} else {
Collection < Object > collections = doc . getFieldValues ( CollectionSchema . collection_sxt . getSolrFieldName ( ) ) ;
2015-05-10 18:52:33 +02:00
// collection dht is used to identify metadata from full crawled documents (if "dht" exists don't overwrite rich crawldata with metadata
if ( ! collections . contains ( " dht " ) ) return ;
2015-03-24 12:32:39 +01:00
// passed all checks, overwrite document
putDocument ( getDefaultConfiguration ( ) . metadata2solr ( entry ) ) ;
}
}
2013-07-15 18:22:35 +02:00
}
2013-07-17 18:31:30 +02:00
} catch ( final SolrException e ) {
2012-09-26 16:05:11 +02:00
throw new IOException ( e . getMessage ( ) , e ) ;
2008-03-26 15:13:05 +01:00
}
2013-11-07 10:01:44 +01:00
if ( MemoryControl . shortStatus ( ) ) clearCaches ( ) ;
2008-03-26 15:13:05 +01:00
}
2012-08-22 16:30:33 +02:00
2012-11-13 16:54:28 +01:00
/ * *
* using a fragment of the url hash ( 6 bytes : bytes 6 to 11 ) it is possible to address all urls from a specific domain
* here such a fragment can be used to delete all these domains at once
* @param hosthash the hash of the host to be deleted
* @param freshdate either NULL or a date in the past which is the limit for deletion . Only documents older than this date are deleted
* @throws IOException
* /
2013-10-24 16:20:20 +02:00
public void deleteStaleDomainHashes ( final Set < String > hosthashes , Date freshdate ) {
2013-05-11 10:53:12 +02:00
// delete in solr
2013-10-24 16:20:20 +02:00
Date now = new Date ( ) ;
deleteDomainWithConstraint ( this . getDefaultConnector ( ) , CollectionSchema . host_id_s . getSolrFieldName ( ) , hosthashes ,
( freshdate = = null | | freshdate . after ( now ) ) ? null :
( CollectionSchema . load_date_dt . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( freshdate ) + " ] " ) ) ;
if ( this . writeWebgraph ) deleteDomainWithConstraint ( this . getWebgraphConnector ( ) , WebgraphSchema . source_host_id_s . getSolrFieldName ( ) , hosthashes ,
( freshdate = = null | | freshdate . after ( now ) ) ? null :
( WebgraphSchema . load_date_dt . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( freshdate ) + " ] " ) ) ;
2012-11-13 16:54:28 +01:00
}
2013-10-24 16:20:20 +02:00
public void deleteStaleDomainNames ( final Set < String > hostnames , Date freshdate ) {
Date now = new Date ( ) ;
deleteDomainWithConstraint ( this . getDefaultConnector ( ) , CollectionSchema . host_s . getSolrFieldName ( ) , hostnames ,
( freshdate = = null | | freshdate . after ( now ) ) ? null :
( CollectionSchema . load_date_dt . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( freshdate ) + " ] " ) ) ;
if ( this . writeWebgraph ) deleteDomainWithConstraint ( this . getWebgraphConnector ( ) , WebgraphSchema . source_host_s . getSolrFieldName ( ) , hostnames ,
( freshdate = = null | | freshdate . after ( now ) ) ? null :
( WebgraphSchema . load_date_dt . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( freshdate ) + " ] " ) ) ;
}
/ * *
* delete all documents within a domain that are registered as error document
* @param hosthashes
* /
public void deleteDomainErrors ( final Set < String > hosthashes ) {
2014-02-26 14:30:48 +01:00
deleteDomainWithConstraint ( this . getDefaultConnector ( ) , CollectionSchema . host_id_s . getSolrFieldName ( ) , hosthashes , CollectionSchema . failreason_s . getSolrFieldName ( ) + AbstractSolrConnector . CATCHALL_DTERM ) ;
2013-10-24 16:20:20 +02:00
}
private static void deleteDomainWithConstraint ( SolrConnector connector , String fieldname , final Set < String > hosthashes , String constraintQuery ) {
if ( hosthashes = = null | | hosthashes . size ( ) = = 0 ) return ;
int subsetscount = 1 + ( hosthashes . size ( ) / 255 ) ; // if the list is too large, we get a "too many boolean clauses" exception
int c = 0 ;
@SuppressWarnings ( " unchecked " )
2014-05-20 21:50:16 +02:00
List < String > [ ] subsets = ( List < String > [ ] ) Array . newInstance ( ArrayList . class , subsetscount ) ;
2013-10-24 16:20:20 +02:00
for ( int i = 0 ; i < subsetscount ; i + + ) subsets [ i ] = new ArrayList < String > ( ) ;
for ( String hosthash : hosthashes ) subsets [ c + + % subsetscount ] . add ( hosthash ) ;
for ( List < String > subset : subsets ) {
try {
StringBuilder query = new StringBuilder ( ) ;
for ( String hosthash : subset ) {
if ( query . length ( ) > 0 ) query . append ( " OR " ) ;
//query.append(CollectionSchema.host_id_s.getSolrFieldName()).append(":\"").append(hosthash).append(":\"");
2014-11-20 18:45:27 +01:00
query . append ( " ({!cache=false raw f= " ) . append ( fieldname ) . append ( '}' ) . append ( hosthash ) . append ( " ) " ) ;
2013-01-24 12:39:19 +01:00
}
2013-10-24 16:20:20 +02:00
if ( constraintQuery = = null ) connector . deleteByQuery ( query . toString ( ) ) ; else connector . deleteByQuery ( " ( " + query . toString ( ) + " ) AND " + constraintQuery ) ;
} catch ( final IOException e ) {
2013-01-24 12:39:19 +01:00
}
2013-01-31 13:15:28 +01:00
}
2013-01-24 12:39:19 +01:00
}
2014-02-12 01:00:44 +01:00
public void deleteOldDocuments ( final long deltaToNow , final boolean loaddate ) {
Date deleteageDate = new Date ( System . currentTimeMillis ( ) - deltaToNow ) ;
final String collection1Query = ( loaddate ? CollectionSchema . load_date_dt : CollectionSchema . last_modified ) . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( deleteageDate ) + " ] " ;
final String webgraphQuery = ( loaddate ? WebgraphSchema . load_date_dt : WebgraphSchema . last_modified ) . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( deleteageDate ) + " ] " ;
try {
this . getDefaultConnector ( ) . deleteByQuery ( collection1Query ) ;
if ( this . getWebgraphConnector ( ) ! = null ) this . getWebgraphConnector ( ) . deleteByQuery ( webgraphQuery ) ;
} catch ( final IOException e ) {
}
}
2012-11-02 01:22:31 +01:00
/ * *
* remove a full subpath from the index
* @param subpath the left path of the url ; at least until the end of the host
2012-11-13 16:54:28 +01:00
* @param freshdate either NULL or a date in the past which is the limit for deletion . Only documents older than this date are deleted
2012-11-02 01:22:31 +01:00
* @param concurrently if true , then the method returnes immediately and runs concurrently
* /
2013-05-11 10:53:12 +02:00
public int remove ( final String basepath , Date freshdate ) {
2013-09-15 00:30:23 +02:00
DigestURL uri ;
try { uri = new DigestURL ( basepath ) ; } catch ( final MalformedURLException e ) { return 0 ; }
2012-11-02 01:22:31 +01:00
final String host = uri . getHost ( ) ;
2013-02-22 15:45:15 +01:00
final String collectionQuery = CollectionSchema . host_s . getSolrFieldName ( ) + " : \" " + host + " \" " +
2013-02-21 13:23:55 +01:00
( ( freshdate ! = null & & freshdate . before ( new Date ( ) ) ) ? ( " AND " + CollectionSchema . load_date_dt . getSolrFieldName ( ) + " :[* TO " + ISO8601Formatter . FORMATTER . format ( freshdate ) + " ] " ) : " " ) ;
2012-11-13 16:54:28 +01:00
final AtomicInteger count = new AtomicInteger ( 0 ) ;
2014-09-17 13:58:55 +02:00
final BlockingQueue < SolrDocument > docs = Fulltext . this . getDefaultConnector ( ) . concurrentDocumentsByQuery ( collectionQuery , null , 0 , 1000000 , Long . MAX_VALUE , 100 , 1 , false , CollectionSchema . id . getSolrFieldName ( ) , CollectionSchema . sku . getSolrFieldName ( ) ) ;
2013-05-11 10:53:12 +02:00
try {
2014-03-06 01:43:48 +01:00
Set < String > deleteIDs = new HashSet < String > ( ) ;
2013-05-11 10:53:12 +02:00
SolrDocument doc ;
while ( ( doc = docs . take ( ) ) ! = AbstractSolrConnector . POISON_DOCUMENT ) {
String u = ( String ) doc . getFieldValue ( CollectionSchema . sku . getSolrFieldName ( ) ) ;
if ( u . startsWith ( basepath ) ) {
2014-03-06 01:43:48 +01:00
deleteIDs . add ( ( String ) doc . getFieldValue ( CollectionSchema . id . getSolrFieldName ( ) ) ) ;
2013-05-11 10:53:12 +02:00
count . incrementAndGet ( ) ;
}
2012-11-02 01:22:31 +01:00
}
2014-03-06 01:43:48 +01:00
remove ( deleteIDs ) ;
2013-05-11 10:53:12 +02:00
if ( count . get ( ) > 0 ) Fulltext . this . commit ( true ) ;
2013-07-17 18:31:30 +02:00
} catch ( final InterruptedException e ) { }
2012-11-13 16:54:28 +01:00
return count . get ( ) ;
2012-11-02 01:22:31 +01:00
}
/ * *
* remove a list of id ' s from the index
* @param deleteIDs a list of urlhashes ; each denoting a document
* @param concurrently if true , then the method returnes immediately and runs concurrently
* /
2013-05-08 13:26:25 +02:00
public void remove ( final Collection < String > deleteIDs ) {
2012-11-02 13:57:43 +01:00
if ( deleteIDs = = null | | deleteIDs . size ( ) = = 0 ) return ;
2013-05-08 13:26:25 +02:00
try {
2013-05-11 10:53:12 +02:00
this . getDefaultConnector ( ) . deleteByIds ( deleteIDs ) ;
2013-07-23 16:46:44 +02:00
if ( this . writeWebgraph ) this . getWebgraphConnector ( ) . deleteByIds ( deleteIDs ) ;
2013-05-08 13:26:25 +02:00
} catch ( final Throwable e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2013-05-08 13:26:25 +02:00
}
2012-10-31 17:44:45 +01:00
}
2012-06-26 13:54:48 +02:00
public boolean remove ( final byte [ ] urlHash ) {
if ( urlHash = = null ) return false ;
2012-07-24 17:23:29 +02:00
try {
2013-05-08 13:26:25 +02:00
String id = ASCII . String ( urlHash ) ;
2013-05-11 10:53:12 +02:00
this . getDefaultConnector ( ) . deleteById ( id ) ;
2013-07-23 16:46:44 +02:00
if ( this . writeWebgraph ) this . getWebgraphConnector ( ) . deleteById ( id ) ;
2012-07-24 17:23:29 +02:00
} catch ( final Throwable e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2012-06-26 13:54:48 +02:00
}
2012-07-22 13:18:45 +02:00
return false ;
2008-03-26 15:13:05 +01:00
}
2014-08-01 11:00:10 +02:00
public DigestURL getURL ( final String urlHash ) throws IOException {
2014-02-24 22:59:58 +01:00
if ( urlHash = = null | | this . getDefaultConnector ( ) = = null ) return null ;
2014-08-01 11:00:10 +02:00
SolrConnector . LoadTimeURL md = this . getDefaultConnector ( ) . getLoadTimeURL ( urlHash ) ;
if ( md = = null ) return null ;
return new DigestURL ( md . url , ASCII . getBytes ( urlHash ) ) ;
2014-02-24 22:59:58 +01:00
}
2013-05-27 13:45:09 +02:00
/ * *
2014-02-19 04:03:45 +01:00
* get the load time of a resource .
* @param urlHash
* @return the time in milliseconds since epoch for the load time or - 1 if the document does not exist
2013-05-27 13:45:09 +02:00
* /
2014-08-01 11:00:10 +02:00
public long getLoadTime ( final String urlHash ) throws IOException {
2014-02-19 04:03:45 +01:00
if ( urlHash = = null ) return - 1l ;
2014-08-01 11:00:10 +02:00
SolrConnector . LoadTimeURL md = this . getDefaultConnector ( ) . getLoadTimeURL ( urlHash ) ;
if ( md = = null ) return - 1l ;
return md . date ;
2013-05-17 13:59:37 +02:00
}
2012-09-24 17:05:28 +02:00
public List < File > dumpFiles ( ) {
2014-02-06 19:13:29 +01:00
EmbeddedInstance esc = this . solrInstances . getEmbedded ( ) ;
2012-09-24 17:05:28 +02:00
ArrayList < File > zips = new ArrayList < File > ( ) ;
2012-11-07 02:04:08 +01:00
if ( esc = = null ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . warn ( " Fulltext " , " HOT DUMP selected solr0 == NULL, no dump list! " ) ;
2012-11-07 02:04:08 +01:00
return zips ;
}
2013-02-21 13:23:55 +01:00
if ( esc . getContainerPath ( ) = = null ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . warn ( " Fulltext " , " HOT DUMP selected solr0.getStoragePath() == NULL, no dump list! " ) ;
2012-11-07 02:04:08 +01:00
return zips ;
}
2014-01-07 17:53:49 +01:00
if ( this . archivePath = = null ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . warn ( " Fulltext " , " HOT DUMP selected esc.getStoragePath().getParentFile() == NULL, no dump list! " ) ;
2012-11-07 02:04:08 +01:00
return zips ;
}
2014-01-07 17:53:49 +01:00
ConcurrentLog . info ( " Fulltext " , " HOT DUMP dump path = " + this . archivePath . toString ( ) ) ;
for ( String p : this . archivePath . list ( ) ) {
if ( p . endsWith ( " zip " ) ) zips . add ( new File ( this . archivePath , p ) ) ;
2012-09-24 17:05:28 +02:00
}
return zips ;
}
/ * *
* create a dump file from the current solr directory
* @return
* /
public File dumpSolr ( ) {
2014-02-06 19:13:29 +01:00
EmbeddedInstance esc = this . solrInstances . getEmbedded ( ) ;
2013-02-21 13:23:55 +01:00
File storagePath = esc . getContainerPath ( ) ;
2014-01-07 17:53:49 +01:00
File zipOut = new File ( this . archivePath , storagePath . getName ( ) + " _ " + GenericFormatter . SHORT_DAY_FORMATTER . format ( ) + " .zip " ) ;
2013-02-21 13:23:55 +01:00
synchronized ( this . solrInstances ) {
2012-09-25 00:19:52 +02:00
this . disconnectLocalSolr ( ) ;
2012-09-24 17:05:28 +02:00
try {
2012-09-25 00:19:52 +02:00
ZIPWriter . zip ( storagePath , zipOut ) ;
2013-07-17 18:31:30 +02:00
} catch ( final IOException e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2012-09-25 00:19:52 +02:00
} finally {
try {
2013-02-13 02:29:47 +01:00
this . connectLocalSolr ( ) ;
2013-07-17 18:31:30 +02:00
} catch ( final IOException e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2012-09-25 00:19:52 +02:00
}
2012-09-24 17:05:28 +02:00
}
}
return zipOut ;
}
/ * *
* restore a solr dump to the current solr directory
* @param solrDumpZipFile
* /
public void restoreSolr ( File solrDumpZipFile ) {
2014-02-06 19:13:29 +01:00
EmbeddedInstance esc = this . solrInstances . getEmbedded ( ) ;
2013-02-21 13:23:55 +01:00
File storagePath = esc . getContainerPath ( ) ;
synchronized ( this . solrInstances ) {
2012-09-25 00:19:52 +02:00
this . disconnectLocalSolr ( ) ;
2013-02-21 13:23:55 +01:00
this . solrInstances . close ( ) ;
2012-09-24 17:05:28 +02:00
try {
2012-09-25 00:19:52 +02:00
ZIPReader . unzip ( solrDumpZipFile , storagePath ) ;
2013-07-17 18:31:30 +02:00
} catch ( final IOException e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2012-09-25 00:19:52 +02:00
} finally {
2013-02-21 13:23:55 +01:00
this . solrInstances = new InstanceMirror ( ) ;
2012-09-25 00:19:52 +02:00
try {
2013-02-13 02:29:47 +01:00
this . connectLocalSolr ( ) ;
2013-07-17 18:31:30 +02:00
} catch ( final IOException e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2012-09-25 00:19:52 +02:00
}
2012-09-24 17:05:28 +02:00
}
}
}
2013-06-28 14:51:37 +02:00
/ * *
* optimize solr ( experimental to check resource management )
* @param size
* /
public void optimize ( final int size ) {
if ( size < 1 ) return ;
getDefaultConnector ( ) . optimize ( size ) ;
2013-07-23 16:46:44 +02:00
if ( this . writeWebgraph ) getWebgraphConnector ( ) . optimize ( size ) ;
2013-06-28 14:51:37 +02:00
}
2012-09-24 17:05:28 +02:00
2013-05-29 13:09:34 +02:00
/ * *
2013-06-28 14:51:37 +02:00
* reboot solr ( experimental to check resource management )
2013-05-29 13:09:34 +02:00
* /
public void rebootSolr ( ) {
synchronized ( this . solrInstances ) {
this . disconnectLocalSolr ( ) ;
this . solrInstances . close ( ) ;
this . solrInstances = new InstanceMirror ( ) ;
try {
this . connectLocalSolr ( ) ;
2013-07-17 18:31:30 +02:00
} catch ( final IOException e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2013-05-29 13:09:34 +02:00
}
}
}
2015-07-30 03:21:40 +02:00
public static enum ExportFormat {
text , html , rss , solr ;
}
2008-03-26 15:13:05 +01:00
// export methods
2015-07-30 03:21:40 +02:00
public Export export ( final File f , final String filter , final String query , final ExportFormat format , final boolean dom , final boolean text ) {
2011-07-15 10:38:10 +02:00
if ( ( this . exportthread ! = null ) & & ( this . exportthread . isAlive ( ) ) ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . warn ( " LURL-EXPORT " , " cannot start another export thread, already one running " ) ;
2011-07-15 10:38:10 +02:00
return this . exportthread ;
2008-03-26 15:13:05 +01:00
}
2015-07-30 03:21:40 +02:00
this . exportthread = new Export ( f , filter , query , format , dom , text ) ;
2008-03-26 15:13:05 +01:00
this . exportthread . start ( ) ;
2011-07-15 10:38:10 +02:00
return this . exportthread ;
2008-03-26 15:13:05 +01:00
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public Export export ( ) {
return this . exportthread ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public class Export extends Thread {
2008-08-02 14:12:04 +02:00
private final File f ;
2013-01-24 12:39:19 +01:00
private final Pattern pattern ;
2008-03-26 15:13:05 +01:00
private int count ;
2013-11-06 19:22:26 +01:00
private String failure , query ;
2015-07-30 03:21:40 +02:00
private final ExportFormat format ;
private final boolean dom , text ;
2011-07-15 10:38:10 +02:00
2015-07-30 03:21:40 +02:00
private Export ( final File f , final String filter , final String query , final ExportFormat format , final boolean dom , final boolean text ) {
2008-03-26 15:13:05 +01:00
// format: 0=text, 1=html, 2=rss/xml
this . f = f ;
2013-01-24 12:39:19 +01:00
this . pattern = filter = = null ? null : Pattern . compile ( filter ) ;
2013-11-06 19:22:26 +01:00
this . query = query = = null ? " *:* " : query ;
2008-03-26 15:13:05 +01:00
this . count = 0 ;
this . failure = null ;
this . format = format ;
this . dom = dom ;
2015-07-30 03:21:40 +02:00
this . text = text ;
2013-04-22 22:33:13 +02:00
//if ((dom) && (format == 2)) dom = false;
2008-03-26 15:13:05 +01:00
}
2011-07-15 10:38:10 +02:00
2011-12-17 01:27:08 +01:00
@Override
2008-03-26 15:13:05 +01:00
public void run ( ) {
try {
2011-07-15 10:38:10 +02:00
final File parentf = this . f . getParentFile ( ) ;
2009-03-10 21:52:10 +01:00
if ( parentf ! = null ) parentf . mkdirs ( ) ;
2015-07-30 03:21:40 +02:00
OutputStream os = new FileOutputStream ( this . format = = ExportFormat . solr ? new File ( this . f . getAbsolutePath ( ) + " .gz " ) : this . f ) ;
if ( this . format = = ExportFormat . solr ) os = new GZIPOutputStream ( os , 65536 ) { { def . setLevel ( Deflater . BEST_COMPRESSION ) ; } } ;
2015-05-30 19:02:54 +02:00
final PrintWriter pw = new PrintWriter ( new BufferedOutputStream ( os ) ) ;
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . html ) {
2008-03-26 15:13:05 +01:00
pw . println ( " <html><head></head><body> " ) ;
}
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . rss ) {
2008-03-26 15:13:05 +01:00
pw . println ( " <?xml version= \" 1.0 \" encoding= \" UTF-8 \" ?> " ) ;
pw . println ( " <?xml-stylesheet type='text/xsl' href='/yacysearch.xsl' version='1.0'?> " ) ;
2009-02-23 12:39:20 +01:00
pw . println ( " <rss version= \" 2.0 \" xmlns:yacy= \" http://www.yacy.net/ \" xmlns:opensearch= \" http://a9.com/-/spec/opensearch/1.1/ \" xmlns:atom= \" http://www.w3.org/2005/Atom \" > " ) ;
2008-03-26 15:13:05 +01:00
pw . println ( " <channel> " ) ;
2013-01-24 12:39:19 +01:00
pw . println ( " <title>YaCy Peer-to-Peer - Web-Search URL Export</title> " ) ;
2008-03-26 15:13:05 +01:00
pw . println ( " <description></description> " ) ;
pw . println ( " <link>http://yacy.net</link> " ) ;
}
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . solr ) {
2015-05-29 15:05:52 +02:00
pw . println ( " <?xml version= \" 1.0 \" encoding= \" UTF-8 \" ?> " ) ;
pw . println ( " <response> " ) ;
pw . println ( " <result> " ) ;
}
2011-07-15 10:38:10 +02:00
if ( this . dom ) {
2013-11-06 19:22:26 +01:00
Map < String , ReversibleScoreMap < String > > scores = Fulltext . this . getDefaultConnector ( ) . getFacets ( this . query + " AND " + CollectionSchema . httpstatus_i . getSolrFieldName ( ) + " :200 " , 100000000 , CollectionSchema . host_s . getSolrFieldName ( ) ) ;
2013-02-21 13:23:55 +01:00
ReversibleScoreMap < String > stats = scores . get ( CollectionSchema . host_s . getSolrFieldName ( ) ) ;
2013-01-24 12:39:19 +01:00
for ( final String host : stats ) {
if ( this . pattern ! = null & & ! this . pattern . matcher ( host ) . matches ( ) ) continue ;
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . text ) pw . println ( host ) ;
if ( this . format = = ExportFormat . html ) pw . println ( " <a href= \" http:// " + host + " \" > " + host + " </a><br> " ) ;
2011-07-15 10:38:10 +02:00
this . count + + ;
2008-09-04 22:28:36 +02:00
}
} else {
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . solr | | ( this . text & & this . format = = ExportFormat . text ) ) {
BlockingQueue < SolrDocument > docs = Fulltext . this . getDefaultConnector ( ) . concurrentDocumentsByQuery ( this . query + " AND " + CollectionSchema . httpstatus_i . getSolrFieldName ( ) + " :200 " , null , 0 , 100000000 , Long . MAX_VALUE , 100 , 1 , true ) ;
SolrDocument doc ;
while ( ( doc = docs . take ( ) ) ! = AbstractSolrConnector . POISON_DOCUMENT ) {
String url = getStringFrom ( doc . getFieldValue ( CollectionSchema . sku . getSolrFieldName ( ) ) ) ;
if ( this . pattern ! = null & & ! this . pattern . matcher ( url ) . matches ( ) ) continue ;
CRIgnoreWriter sw = new CRIgnoreWriter ( ) ;
if ( this . text ) sw . write ( ( String ) doc . getFieldValue ( CollectionSchema . text_t . getSolrFieldName ( ) ) ) ; else EnhancedXMLResponseWriter . writeDoc ( sw , doc ) ;
sw . close ( ) ;
String d = sw . toString ( ) ;
pw . println ( d ) ;
this . count + + ;
}
} else {
2015-05-29 15:05:52 +02:00
BlockingQueue < SolrDocument > docs = Fulltext . this . getDefaultConnector ( ) . concurrentDocumentsByQuery ( this . query + " AND " + CollectionSchema . httpstatus_i . getSolrFieldName ( ) + " :200 " , null , 0 , 100000000 , Long . MAX_VALUE , 100 , 1 , true ,
CollectionSchema . id . getSolrFieldName ( ) , CollectionSchema . sku . getSolrFieldName ( ) , CollectionSchema . title . getSolrFieldName ( ) ,
CollectionSchema . author . getSolrFieldName ( ) , CollectionSchema . description_txt . getSolrFieldName ( ) , CollectionSchema . size_i . getSolrFieldName ( ) , CollectionSchema . last_modified . getSolrFieldName ( ) ) ;
SolrDocument doc ;
String url , hash , title , author , description ;
Integer size ;
Date date ;
while ( ( doc = docs . take ( ) ) ! = AbstractSolrConnector . POISON_DOCUMENT ) {
hash = getStringFrom ( doc . getFieldValue ( CollectionSchema . id . getSolrFieldName ( ) ) ) ;
url = getStringFrom ( doc . getFieldValue ( CollectionSchema . sku . getSolrFieldName ( ) ) ) ;
title = getStringFrom ( doc . getFieldValue ( CollectionSchema . title . getSolrFieldName ( ) ) ) ;
author = getStringFrom ( doc . getFieldValue ( CollectionSchema . author . getSolrFieldName ( ) ) ) ;
description = getStringFrom ( doc . getFieldValue ( CollectionSchema . description_txt . getSolrFieldName ( ) ) ) ;
size = ( Integer ) doc . getFieldValue ( CollectionSchema . size_i . getSolrFieldName ( ) ) ;
date = ( Date ) doc . getFieldValue ( CollectionSchema . last_modified . getSolrFieldName ( ) ) ;
if ( this . pattern ! = null & & ! this . pattern . matcher ( url ) . matches ( ) ) continue ;
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . text ) {
2015-05-29 15:05:52 +02:00
pw . println ( url ) ;
}
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . html ) {
2015-05-29 15:05:52 +02:00
if ( title ! = null ) pw . println ( " <a href= \" " + MultiProtocolURL . escape ( url ) + " \" > " + CharacterCoding . unicode2xml ( title , true ) + " </a> " ) ;
}
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . rss ) {
2015-05-29 15:05:52 +02:00
pw . println ( " <item> " ) ;
if ( title ! = null ) pw . println ( " <title> " + CharacterCoding . unicode2xml ( title , true ) + " </title> " ) ;
pw . println ( " <link> " + MultiProtocolURL . escape ( url ) + " </link> " ) ;
if ( author ! = null & & ! author . isEmpty ( ) ) pw . println ( " <author> " + CharacterCoding . unicode2xml ( author , true ) + " </author> " ) ;
if ( description ! = null & & ! description . isEmpty ( ) ) pw . println ( " <description> " + CharacterCoding . unicode2xml ( description , true ) + " </description> " ) ;
if ( date ! = null ) pw . println ( " <pubDate> " + HeaderFramework . formatRFC1123 ( date ) + " </pubDate> " ) ;
if ( size ! = null ) pw . println ( " <yacy:size> " + size . intValue ( ) + " </yacy:size> " ) ;
pw . println ( " <guid isPermaLink= \" false \" > " + hash + " </guid> " ) ;
pw . println ( " </item> " ) ;
}
this . count + + ;
2008-03-26 15:13:05 +01:00
}
}
}
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . html ) {
2008-03-26 15:13:05 +01:00
pw . println ( " </body></html> " ) ;
}
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . rss ) {
2008-03-26 15:13:05 +01:00
pw . println ( " </channel> " ) ;
pw . println ( " </rss> " ) ;
}
2015-07-30 03:21:40 +02:00
if ( this . format = = ExportFormat . solr ) {
2015-05-29 15:05:52 +02:00
pw . println ( " </result> " ) ;
pw . println ( " </response> " ) ;
}
2008-03-26 15:13:05 +01:00
pw . close ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2008-03-26 15:13:05 +01:00
this . failure = e . getMessage ( ) ;
2009-05-05 08:31:35 +02:00
} catch ( final Exception e ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2009-05-05 08:31:35 +02:00
this . failure = e . getMessage ( ) ;
2008-03-26 15:13:05 +01:00
}
// terminate process
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public File file ( ) {
return this . f ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public String failed ( ) {
return this . failure ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
public int count ( ) {
return this . count ;
}
2013-11-06 19:22:26 +01:00
@SuppressWarnings ( " unchecked " )
private String getStringFrom ( final Object o ) {
if ( o = = null ) return " " ;
if ( o instanceof ArrayList ) return ( ( ArrayList < String > ) o ) . get ( 0 ) ;
return ( String ) o ;
}
2011-07-15 10:38:10 +02:00
2008-03-26 15:13:05 +01:00
}
2014-02-24 20:51:43 +01:00
2008-03-26 15:13:05 +01:00
}