2007-11-07 23:38:09 +01:00
// plasmaSearchRankingProcess.java
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 07.11.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
2009-09-05 22:41:21 +02:00
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
2007-11-07 23:38:09 +01:00
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2009-07-09 00:14:57 +02:00
package de.anomic.search ;
2007-11-07 23:38:09 +01:00
import java.io.File ;
import java.io.IOException ;
2008-02-07 23:16:36 +01:00
import java.util.ArrayList ;
2009-05-27 00:30:20 +02:00
import java.util.Arrays ;
import java.util.Comparator ;
2007-11-07 23:38:09 +01:00
import java.util.HashMap ;
2009-06-09 17:22:23 +02:00
import java.util.HashSet ;
2007-11-07 23:38:09 +01:00
import java.util.Iterator ;
import java.util.Map ;
import java.util.TreeSet ;
2009-08-28 00:54:32 +02:00
import java.util.concurrent.BlockingQueue ;
2008-02-27 16:16:47 +01:00
import java.util.concurrent.ConcurrentHashMap ;
2009-09-24 21:14:35 +02:00
import java.util.concurrent.TimeUnit ;
2007-11-07 23:38:09 +01:00
2009-10-18 02:53:43 +02:00
import net.yacy.document.Condenser ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.DigestURI ;
import net.yacy.kelondro.data.meta.URIMetadataRow ;
import net.yacy.kelondro.data.word.Word ;
import net.yacy.kelondro.data.word.WordReference ;
import net.yacy.kelondro.data.word.WordReferenceVars ;
2009-10-10 01:32:08 +02:00
import net.yacy.kelondro.index.BinSearch ;
2009-11-05 21:28:37 +01:00
import net.yacy.kelondro.logging.Log ;
2009-10-10 01:22:22 +02:00
import net.yacy.kelondro.order.Digest ;
2009-10-10 02:39:15 +02:00
import net.yacy.kelondro.rwi.Reference ;
import net.yacy.kelondro.rwi.ReferenceContainer ;
import net.yacy.kelondro.rwi.TermSearch ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.util.FileUtils ;
2009-10-20 00:34:44 +02:00
import net.yacy.kelondro.util.MemoryTracker ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.util.SortStack ;
2009-10-10 01:22:22 +02:00
2009-10-20 00:34:44 +02:00
import de.anomic.yacy.graphics.ProfilingGraph ;
2007-11-07 23:38:09 +01:00
2009-08-24 17:24:02 +02:00
public final class RankingProcess extends Thread {
2007-11-07 23:38:09 +01:00
2009-01-30 16:33:00 +01:00
public static BinSearch [ ] ybrTables = null ; // block-rank tables
2008-04-24 15:31:55 +02:00
public static final int maxYBR = 3 ; // the lower this value, the faster the search
2007-11-07 23:38:09 +01:00
private static boolean useYBR = true ;
2008-05-23 11:45:33 +02:00
private static final int maxDoubleDomAll = 20 , maxDoubleDomSpecial = 10000 ;
2007-11-07 23:38:09 +01:00
2009-08-27 17:19:48 +02:00
private final Segment indexSegment ;
2009-07-09 00:14:57 +02:00
private final QueryParams query ;
2008-08-02 14:12:04 +02:00
private final int maxentries ;
2009-03-02 00:58:14 +01:00
private final ReferenceOrder order ;
2008-08-02 14:12:04 +02:00
private final ConcurrentHashMap < String , Integer > urlhashes ; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion)
private final int [ ] flagcount ; // flag counter
private final TreeSet < String > misses ; // contains url-hashes that could not been found in the LURL-DB
private final int [ ] domZones ;
2009-08-27 17:19:48 +02:00
private HashMap < byte [ ] , ReferenceContainer < WordReference > > localSearchInclusion ;
private int remote_peerCount , remote_indexCount , remote_resourceSize , local_resourceSize ;
private final SortStack < WordReferenceVars > stack ;
2009-09-04 01:09:53 +02:00
private int feeders ;
2009-08-27 17:19:48 +02:00
private final HashMap < String , SortStack < WordReferenceVars > > doubleDomCache ; // key = domhash (6 bytes); value = like stack
private final HashSet < String > handover ; // key = urlhash; used for double-check of urls that had been handed over to search process
2009-06-02 17:20:10 +02:00
private final ConcurrentHashMap < String , Integer > ref ; // reference score computation for the commonSense heuristic
2009-06-09 01:30:12 +02:00
private final ConcurrentHashMap < String , HostInfo > hostNavigator ;
private final ConcurrentHashMap < String , AuthorInfo > authorNavigator ;
2007-11-07 23:38:09 +01:00
2009-09-04 01:09:53 +02:00
2009-07-09 00:14:57 +02:00
public RankingProcess (
2009-05-28 16:26:05 +02:00
final Segment indexSegment ,
2009-07-09 00:14:57 +02:00
final QueryParams query ,
2009-04-02 15:26:47 +02:00
final int maxentries ,
final int concurrency ) {
2007-11-07 23:38:09 +01:00
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
2007-11-22 00:14:57 +01:00
// sortorder: 0 = hash, 1 = url, 2 = ranking
2009-06-03 10:49:54 +02:00
this . localSearchInclusion = null ;
2009-04-03 15:23:45 +02:00
this . stack = new SortStack < WordReferenceVars > ( maxentries ) ;
this . doubleDomCache = new HashMap < String , SortStack < WordReferenceVars > > ( ) ;
2009-06-09 17:22:23 +02:00
this . handover = new HashSet < String > ( ) ;
2009-03-02 00:58:14 +01:00
this . order = ( query = = null ) ? null : new ReferenceOrder ( query . ranking , query . targetlang ) ;
2007-11-07 23:38:09 +01:00
this . query = query ;
this . maxentries = maxentries ;
2008-01-30 22:58:30 +01:00
this . remote_peerCount = 0 ;
this . remote_indexCount = 0 ;
this . remote_resourceSize = 0 ;
this . local_resourceSize = 0 ;
2008-02-27 16:16:47 +01:00
this . urlhashes = new ConcurrentHashMap < String , Integer > ( 0 , 0 . 75f , concurrency ) ;
2007-12-27 18:56:59 +01:00
this . misses = new TreeSet < String > ( ) ;
2009-05-28 16:26:05 +02:00
this . indexSegment = indexSegment ;
2007-11-22 00:14:57 +01:00
this . flagcount = new int [ 32 ] ;
for ( int i = 0 ; i < 32 ; i + + ) { this . flagcount [ i ] = 0 ; }
2009-06-09 00:01:26 +02:00
this . hostNavigator = new ConcurrentHashMap < String , HostInfo > ( ) ;
2009-06-09 01:30:12 +02:00
this . authorNavigator = new ConcurrentHashMap < String , AuthorInfo > ( ) ;
2009-06-02 17:20:10 +02:00
this . ref = new ConcurrentHashMap < String , Integer > ( ) ;
this . domZones = new int [ 8 ] ;
2008-03-10 00:58:22 +01:00
for ( int i = 0 ; i < 8 ; i + + ) { this . domZones [ i ] = 0 ; }
2009-09-04 01:09:53 +02:00
this . feeders = concurrency ;
assert this . feeders > = 1 ;
2007-11-22 00:14:57 +01:00
}
2009-08-24 17:24:02 +02:00
public void run ( ) {
2009-08-27 17:19:48 +02:00
// do a search
2009-08-24 17:24:02 +02:00
// sort the local containers and truncate it to a limited count,
// so following sortings together with the global results will be fast
try {
2009-08-27 17:19:48 +02:00
long timer = System . currentTimeMillis ( ) ;
final TermSearch < WordReference > search = this . indexSegment . termIndex ( ) . query (
query . queryHashes ,
query . excludeHashes ,
null ,
Segment . wordReferenceFactory ,
query . maxDistance ) ;
this . localSearchInclusion = search . inclusion ( ) ;
final ReferenceContainer < WordReference > index = search . joined ( ) ;
2009-10-20 00:34:44 +02:00
MemoryTracker . update ( " SEARCH " , new ProfilingGraph . searchEvent ( query . id ( true ) , SearchEvent . JOIN , index . size ( ) , System . currentTimeMillis ( ) - timer ) , false ) ;
2009-08-27 17:19:48 +02:00
if ( index . size ( ) = = 0 ) {
return ;
}
add ( index , true , index . size ( ) ) ;
2009-08-24 17:24:02 +02:00
} catch ( final Exception e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2009-08-24 17:24:02 +02:00
}
2009-09-04 01:09:53 +02:00
oneFeederTerminated ( ) ;
2009-08-24 17:24:02 +02:00
}
2009-04-03 15:23:45 +02:00
public long ranking ( final WordReferenceVars word ) {
2008-02-21 11:06:57 +01:00
return order . cardinal ( word ) ;
}
2008-03-11 12:09:38 +01:00
public int [ ] zones ( ) {
return this . domZones ;
}
2009-08-27 17:19:48 +02:00
public void add ( final ReferenceContainer < WordReference > index , final boolean local , final int fullResource ) {
2007-11-07 23:38:09 +01:00
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
2008-01-29 00:41:39 +01:00
assert ( index ! = null ) ;
if ( index . size ( ) = = 0 ) return ;
2008-01-30 22:58:30 +01:00
if ( local ) {
this . local_resourceSize + = fullResource ;
} else {
this . remote_resourceSize + = fullResource ;
this . remote_peerCount + + ;
}
2007-11-07 23:38:09 +01:00
2007-12-04 21:19:13 +01:00
long timer = System . currentTimeMillis ( ) ;
2008-02-21 11:06:57 +01:00
// normalize entries
2009-08-28 00:54:32 +02:00
final BlockingQueue < WordReferenceVars > decodedEntries = this . order . normalizeWith ( index ) ;
2009-10-20 00:34:44 +02:00
MemoryTracker . update ( " SEARCH " , new ProfilingGraph . searchEvent ( query . id ( true ) , SearchEvent . NORMALIZING , index . size ( ) , System . currentTimeMillis ( ) - timer ) , false ) ;
2007-11-07 23:38:09 +01:00
2008-02-21 11:06:57 +01:00
// iterate over normalized entries and select some that are better than currently stored
2008-02-10 23:50:09 +01:00
timer = System . currentTimeMillis ( ) ;
2009-06-09 00:01:26 +02:00
HostInfo hs ;
2009-05-26 00:27:34 +02:00
String domhash ;
2009-06-08 00:51:15 +02:00
boolean nav_hosts = this . query . navigators . equals ( " all " ) | | this . query . navigators . indexOf ( " hosts " ) > = 0 ;
2009-08-28 00:54:32 +02:00
WordReferenceVars iEntry ;
final ArrayList < WordReferenceVars > filteredEntries = new ArrayList < WordReferenceVars > ( ) ;
2009-08-28 01:20:59 +02:00
// apply all constraints
2009-08-28 00:54:32 +02:00
try {
2009-09-24 21:14:35 +02:00
while ( true ) {
iEntry = decodedEntries . poll ( 1 , TimeUnit . SECONDS ) ;
if ( iEntry = = null | | iEntry = = WordReferenceVars . poison ) break ;
2009-08-28 00:54:32 +02:00
assert ( iEntry . metadataHash ( ) . length ( ) = = index . row ( ) . primaryKeyLength ) ;
//if (iEntry.urlHash().length() != index.row().primaryKeyLength) continue;
2007-11-16 15:48:09 +01:00
2009-08-28 00:54:32 +02:00
// increase flag counts
for ( int j = 0 ; j < 32 ; j + + ) {
if ( iEntry . flags ( ) . get ( j ) ) { flagcount [ j ] + + ; }
}
// check constraints
if ( ! testFlags ( iEntry ) ) continue ;
// check document domain
if ( query . contentdom ! = QueryParams . CONTENTDOM_TEXT ) {
if ( ( query . contentdom = = QueryParams . CONTENTDOM_AUDIO ) & & ( ! ( iEntry . flags ( ) . get ( Condenser . flag_cat_hasaudio ) ) ) ) continue ;
if ( ( query . contentdom = = QueryParams . CONTENTDOM_VIDEO ) & & ( ! ( iEntry . flags ( ) . get ( Condenser . flag_cat_hasvideo ) ) ) ) continue ;
if ( ( query . contentdom = = QueryParams . CONTENTDOM_IMAGE ) & & ( ! ( iEntry . flags ( ) . get ( Condenser . flag_cat_hasimage ) ) ) ) continue ;
if ( ( query . contentdom = = QueryParams . CONTENTDOM_APP ) & & ( ! ( iEntry . flags ( ) . get ( Condenser . flag_cat_hasapp ) ) ) ) continue ;
}
2008-02-21 11:06:57 +01:00
2009-08-28 00:54:32 +02:00
// check tld domain
2009-10-11 02:12:19 +02:00
if ( ! DigestURI . matchesAnyDomDomain ( iEntry . metadataHash ( ) , this . query . zonecode ) ) {
2009-08-28 00:54:32 +02:00
// filter out all tld that do not match with wanted tld domain
continue ;
}
// check site constraints
if ( query . sitehash ! = null & & ! iEntry . metadataHash ( ) . substring ( 6 ) . equals ( query . sitehash ) ) {
// filter out all domains that do not match with the site constraint
continue ;
}
// count domZones
2009-10-11 02:12:19 +02:00
this . domZones [ DigestURI . domDomain ( iEntry . metadataHash ( ) ) ] + + ;
2009-08-28 00:54:32 +02:00
// get statistics for host navigator
if ( nav_hosts ) {
domhash = iEntry . urlHash . substring ( 6 ) ;
hs = this . hostNavigator . get ( domhash ) ;
if ( hs = = null ) {
this . hostNavigator . put ( domhash , new HostInfo ( iEntry . urlHash ) ) ;
} else {
hs . inc ( ) ;
}
}
// accept
filteredEntries . add ( iEntry ) ;
// increase counter for statistics
if ( ! local ) this . remote_indexCount + + ;
}
} catch ( InterruptedException e ) { }
// do the ranking
Long r ;
for ( WordReferenceVars fEntry : filteredEntries ) {
// kick out entries that are too bad according to current findings
r = Long . valueOf ( order . cardinal ( fEntry ) ) ;
if ( ( maxentries > = 0 ) & & ( stack . size ( ) > = maxentries ) & & ( stack . bottom ( r . longValue ( ) ) ) ) continue ;
// insert
if ( ( maxentries < 0 ) | | ( stack . size ( ) < maxentries ) ) {
// in case that we don't have enough yet, accept any new entry
if ( urlhashes . containsKey ( fEntry . metadataHash ( ) ) ) continue ;
stack . push ( fEntry , r ) ;
} else {
// if we already have enough entries, insert only such that are necessary to get a better result
if ( stack . bottom ( r . longValue ( ) ) ) {
continue ;
}
// double-check
if ( urlhashes . containsKey ( fEntry . metadataHash ( ) ) ) continue ;
stack . push ( fEntry , r ) ;
}
}
2009-08-28 01:20:59 +02:00
2007-11-22 00:14:57 +01:00
//if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true);
2009-10-20 00:34:44 +02:00
MemoryTracker . update ( " SEARCH " , new ProfilingGraph . searchEvent ( query . id ( true ) , SearchEvent . PRESORT , index . size ( ) , System . currentTimeMillis ( ) - timer ) , false ) ;
2007-11-16 15:48:09 +01:00
}
2009-05-26 00:27:34 +02:00
2009-09-04 01:09:53 +02:00
/ * *
* method to signal the incoming stack that one feeder has terminated
* /
public void oneFeederTerminated ( ) {
this . feeders - - ;
assert this . feeders > = 0 : " feeders = " + this . feeders ;
}
public void moreFeeders ( final int countMoreFeeders ) {
this . feeders + = countMoreFeeders ;
}
public boolean feedingIsFinished ( ) {
return this . feeders = = 0 ;
}
2009-04-07 11:34:41 +02:00
private boolean testFlags ( final WordReference ientry ) {
2007-11-22 00:14:57 +01:00
if ( query . constraint = = null ) return true ;
// test if ientry matches with filter
// if all = true: let only entries pass that has all matching bits
// if all = false: let all entries pass that has at least one matching bit
if ( query . allofconstraint ) {
for ( int i = 0 ; i < 32 ; i + + ) {
if ( ( query . constraint . get ( i ) ) & & ( ! ientry . flags ( ) . get ( i ) ) ) return false ;
}
return true ;
}
for ( int i = 0 ; i < 32 ; i + + ) {
if ( ( query . constraint . get ( i ) ) & & ( ientry . flags ( ) . get ( i ) ) ) return true ;
}
return false ;
2007-11-07 23:38:09 +01:00
}
2009-06-03 10:49:54 +02:00
public Map < byte [ ] , ReferenceContainer < WordReference > > searchContainerMap ( ) {
2007-11-22 00:14:57 +01:00
// direct access to the result maps is needed for abstract generation
// this is only available if execQuery() was called before
2009-06-03 10:49:54 +02:00
return localSearchInclusion ;
2007-11-22 00:14:57 +01:00
}
// todo:
// - remove redundant urls (sub-path occurred before)
// - move up shorter urls
// - root-domain guessing to prefer the root domain over other urls if search word appears in domain name
2009-08-27 17:19:48 +02:00
private SortStack < WordReferenceVars > . stackElement takeRWI ( final boolean skipDoubleDom ) {
2008-04-24 17:09:06 +02:00
// returns from the current RWI list the best entry and removes this entry from the list
2009-04-03 15:23:45 +02:00
SortStack < WordReferenceVars > m ;
SortStack < WordReferenceVars > . stackElement rwi ;
2008-02-21 11:06:57 +01:00
while ( stack . size ( ) > 0 ) {
rwi = stack . pop ( ) ;
2008-04-24 17:09:06 +02:00
if ( rwi = = null ) continue ; // in case that a synchronization problem occurred just go lazy over it
2008-02-21 11:06:57 +01:00
if ( ! skipDoubleDom ) return rwi ;
2007-11-22 00:14:57 +01:00
// check doubledom
2009-04-07 11:34:41 +02:00
final String domhash = rwi . element . metadataHash ( ) . substring ( 6 ) ;
2008-01-30 01:15:43 +01:00
m = this . doubleDomCache . get ( domhash ) ;
2007-11-22 00:14:57 +01:00
if ( m = = null ) {
// first appearance of dom
2009-04-03 15:23:45 +02:00
m = new SortStack < WordReferenceVars > ( ( query . specialRights ) ? maxDoubleDomSpecial : maxDoubleDomAll ) ;
2007-11-22 00:14:57 +01:00
this . doubleDomCache . put ( domhash , m ) ;
2008-02-21 11:06:57 +01:00
return rwi ;
2007-11-22 00:14:57 +01:00
}
// second appearances of dom
2008-02-21 11:06:57 +01:00
m . push ( rwi ) ;
2007-11-22 00:14:57 +01:00
}
// no more entries in sorted RWI entries. Now take Elements from the doubleDomCache
// find best entry from all caches
2009-04-03 15:23:45 +02:00
final Iterator < SortStack < WordReferenceVars > > i = this . doubleDomCache . values ( ) . iterator ( ) ;
SortStack < WordReferenceVars > . stackElement bestEntry = null ;
SortStack < WordReferenceVars > . stackElement o ;
2007-11-22 00:14:57 +01:00
while ( i . hasNext ( ) ) {
2007-12-28 19:47:45 +01:00
m = i . next ( ) ;
2008-04-24 17:09:06 +02:00
if ( m = = null ) continue ;
2007-11-22 00:14:57 +01:00
if ( m . size ( ) = = 0 ) continue ;
if ( bestEntry = = null ) {
2008-02-21 11:06:57 +01:00
bestEntry = m . top ( ) ;
2007-11-22 00:14:57 +01:00
continue ;
}
2008-02-21 11:06:57 +01:00
o = m . top ( ) ;
2008-02-22 00:40:38 +01:00
if ( o . weight . longValue ( ) < bestEntry . weight . longValue ( ) ) {
2008-02-21 11:06:57 +01:00
bestEntry = o ;
2007-11-22 00:14:57 +01:00
}
}
2008-02-21 11:06:57 +01:00
if ( bestEntry = = null ) return null ;
2007-11-22 00:14:57 +01:00
// finally remove the best entry from the doubledom cache
2009-04-07 11:34:41 +02:00
m = this . doubleDomCache . get ( bestEntry . element . metadataHash ( ) . substring ( 6 ) ) ;
2008-02-21 11:06:57 +01:00
o = m . pop ( ) ;
2009-06-19 01:24:23 +02:00
//assert o == null || o.element.metadataHash().equals(bestEntry.element.metadataHash()) : "bestEntry.element.metadataHash() = " + bestEntry.element.metadataHash() + ", o.element.metadataHash() = " + o.element.metadataHash();
2008-02-21 11:06:57 +01:00
return bestEntry ;
2007-11-22 00:14:57 +01:00
}
2009-10-11 02:12:19 +02:00
public URIMetadataRow takeURL ( final boolean skipDoubleDom , final int timeout ) {
2009-04-30 00:14:12 +02:00
// returns from the current RWI list the best URL entry and removes this entry from the list
2009-09-04 01:09:53 +02:00
long timeLimit = System . currentTimeMillis ( ) + timeout ;
while ( System . currentTimeMillis ( ) < timeLimit ) {
2009-08-27 17:19:48 +02:00
final SortStack < WordReferenceVars > . stackElement obrwi = takeRWI ( skipDoubleDom ) ;
2009-09-04 01:09:53 +02:00
if ( obrwi = = null ) {
if ( this . feedingIsFinished ( ) ) return null ;
try { Thread . sleep ( 50 ) ; } catch ( final InterruptedException e1 ) { }
continue ;
}
2009-10-11 02:12:19 +02:00
final URIMetadataRow page = indexSegment . urlMetadata ( ) . load ( obrwi . element . metadataHash ( ) , obrwi . element , obrwi . weight . longValue ( ) ) ;
2009-08-27 22:20:07 +02:00
if ( page = = null ) {
misses . add ( obrwi . element . metadataHash ( ) ) ;
continue ;
}
// prepare values for constraint check
2009-10-11 02:12:19 +02:00
final URIMetadataRow . Components metadata = page . metadata ( ) ;
2009-08-27 22:20:07 +02:00
// check url constraints
if ( metadata . url ( ) = = null ) {
continue ; // rare case where the url is corrupted
}
final String pageurl = metadata . url ( ) . toNormalform ( true , true ) ;
final String pageauthor = metadata . dc_creator ( ) ;
final String pagetitle = metadata . dc_title ( ) . toLowerCase ( ) ;
// check exclusion
if ( ( QueryParams . matches ( pagetitle , query . excludeHashes ) ) | |
( QueryParams . matches ( pageurl . toLowerCase ( ) , query . excludeHashes ) ) | |
( QueryParams . matches ( pageauthor . toLowerCase ( ) , query . excludeHashes ) ) ) {
continue ;
}
// check url mask
if ( ! ( pageurl . matches ( query . urlMask ) ) ) {
continue ;
}
// check index-of constraint
if ( ( query . constraint ! = null ) & &
( query . constraint . get ( Condenser . flag_cat_indexof ) ) & &
( ! ( pagetitle . startsWith ( " index of " ) ) ) ) {
final Iterator < byte [ ] > wi = query . queryHashes . iterator ( ) ;
while ( wi . hasNext ( ) ) try { indexSegment . termIndex ( ) . remove ( wi . next ( ) , page . hash ( ) ) ; } catch ( IOException e ) { }
continue ;
}
// check content domain
if ( ( query . contentdom = = QueryParams . CONTENTDOM_AUDIO & & page . laudio ( ) = = 0 ) | |
( query . contentdom = = QueryParams . CONTENTDOM_VIDEO & & page . lvideo ( ) = = 0 ) | |
( query . contentdom = = QueryParams . CONTENTDOM_IMAGE & & page . limage ( ) = = 0 ) | |
( query . contentdom = = QueryParams . CONTENTDOM_APP & & page . lapp ( ) = = 0 ) ) {
continue ;
}
// evaluate information of metadata for navigation
// author navigation:
if ( pageauthor ! = null & & pageauthor . length ( ) > 0 ) {
// add author to the author navigator
String authorhash = new String ( Word . word2hash ( pageauthor ) ) ;
//System.out.println("*** DEBUG authorhash = " + authorhash + ", query.authorhash = " + this.query.authorhash + ", author = " + author);
2009-06-09 17:22:23 +02:00
2009-08-27 22:20:07 +02:00
// check if we already are filtering for authors
if ( this . query . authorhash ! = null & & ! this . query . authorhash . equals ( authorhash ) ) {
continue ;
}
// add author to the author navigator
AuthorInfo in = this . authorNavigator . get ( authorhash ) ;
if ( in = = null ) {
this . authorNavigator . put ( authorhash , new AuthorInfo ( pageauthor ) ) ;
} else {
in . inc ( ) ;
this . authorNavigator . put ( authorhash , in ) ;
2008-04-24 15:31:55 +02:00
}
2009-08-27 22:20:07 +02:00
} else if ( this . query . authorhash ! = null ) {
continue ;
2009-06-09 17:22:23 +02:00
}
2009-08-27 22:20:07 +02:00
// accept url
this . handover . add ( page . hash ( ) ) ; // remember that we handed over this url
return page ;
2007-11-22 00:14:57 +01:00
}
return null ;
2009-08-25 23:27:01 +02:00
}
2008-04-24 17:09:06 +02:00
public int size ( ) {
2007-11-22 00:14:57 +01:00
//assert sortedRWIEntries.size() == urlhashes.size() : "sortedRWIEntries.size() = " + sortedRWIEntries.size() + ", urlhashes.size() = " + urlhashes.size();
2008-02-21 11:06:57 +01:00
int c = stack . size ( ) ;
2009-08-27 17:19:48 +02:00
for ( SortStack < WordReferenceVars > s : this . doubleDomCache . values ( ) ) {
c + = s . size ( ) ;
}
2007-11-22 00:14:57 +01:00
return c ;
2007-11-07 23:38:09 +01:00
}
2007-11-16 15:48:09 +01:00
public int [ ] flagCount ( ) {
2007-11-22 00:14:57 +01:00
return flagcount ;
2007-11-16 15:48:09 +01:00
}
2008-01-30 22:58:30 +01:00
// "results from a total number of <remote_resourceSize + local_resourceSize> known (<local_resourceSize> local, <remote_resourceSize> remote), <remote_indexCount> links from <remote_peerCount> other YaCy peers."
2007-11-07 23:38:09 +01:00
public int filteredCount ( ) {
2008-01-30 22:58:30 +01:00
// the number of index entries that are considered as result set
2008-02-21 11:06:57 +01:00
return this . stack . size ( ) ;
2007-11-07 23:38:09 +01:00
}
2008-01-30 22:58:30 +01:00
public int getRemoteIndexCount ( ) {
// the number of result contributions from all the remote peers
return this . remote_indexCount ;
}
public int getRemotePeerCount ( ) {
// the number of remote peers that have contributed
return this . remote_peerCount ;
2007-11-07 23:38:09 +01:00
}
2008-01-30 22:58:30 +01:00
public int getRemoteResourceSize ( ) {
// the number of all hits in all the remote peers
return this . remote_resourceSize ;
}
public int getLocalResourceSize ( ) {
// the number of hits in the local peer (index size, size of the collection in the own index)
return this . local_resourceSize ;
}
2009-03-02 00:58:14 +01:00
public Reference remove ( final String urlHash ) {
2009-04-03 15:23:45 +02:00
final SortStack < WordReferenceVars > . stackElement se = stack . remove ( urlHash . hashCode ( ) ) ;
2008-02-21 11:06:57 +01:00
if ( se = = null ) return null ;
2007-11-07 23:38:09 +01:00
urlhashes . remove ( urlHash ) ;
2008-02-21 11:06:57 +01:00
return se . element ;
2007-11-07 23:38:09 +01:00
}
2007-11-22 00:14:57 +01:00
2007-12-28 19:47:45 +01:00
public Iterator < String > miss ( ) {
2007-11-22 00:14:57 +01:00
return this . misses . iterator ( ) ;
2007-11-07 23:38:09 +01:00
}
2009-06-09 00:01:26 +02:00
public class HostInfo {
2009-06-02 17:20:10 +02:00
public int count ;
public String hashsample ;
2009-06-09 00:01:26 +02:00
public HostInfo ( String urlhash ) {
2009-06-02 17:20:10 +02:00
this . count = 1 ;
this . hashsample = urlhash ;
}
public void inc ( ) {
this . count + + ;
}
}
2009-06-09 00:01:26 +02:00
public class AuthorInfo {
public int count ;
public String author ;
public AuthorInfo ( String author ) {
this . count = 1 ;
this . author = author ;
}
public void inc ( ) {
this . count + + ;
}
}
public static final Comparator < HostInfo > hscomp = new Comparator < HostInfo > ( ) {
public int compare ( HostInfo o1 , HostInfo o2 ) {
if ( o1 . count < o2 . count ) return 1 ;
if ( o2 . count < o1 . count ) return - 1 ;
return 0 ;
}
} ;
public static final Comparator < AuthorInfo > aicomp = new Comparator < AuthorInfo > ( ) {
public int compare ( AuthorInfo o1 , AuthorInfo o2 ) {
2009-06-02 17:20:10 +02:00
if ( o1 . count < o2 . count ) return 1 ;
if ( o2 . count < o1 . count ) return - 1 ;
return 0 ;
}
} ;
public class NavigatorEntry {
public int count ;
public String name ;
public NavigatorEntry ( String name , int count ) {
this . name = name ;
this . count = count ;
}
}
public ArrayList < NavigatorEntry > getHostNavigator ( int count ) {
2009-06-08 00:51:15 +02:00
if ( ! this . query . navigators . equals ( " all " ) & & this . query . navigators . indexOf ( " hosts " ) < 0 ) return new ArrayList < NavigatorEntry > ( 0 ) ;
2009-06-09 00:01:26 +02:00
HostInfo [ ] hsa = this . hostNavigator . values ( ) . toArray ( new HostInfo [ this . hostNavigator . size ( ) ] ) ;
2009-06-02 17:20:10 +02:00
Arrays . sort ( hsa , hscomp ) ;
int rc = Math . min ( count , hsa . length ) ;
ArrayList < NavigatorEntry > result = new ArrayList < NavigatorEntry > ( ) ;
2009-10-11 02:12:19 +02:00
URIMetadataRow mr ;
DigestURI url ;
2009-06-22 14:25:18 +02:00
String hostname ;
2009-06-02 17:20:10 +02:00
for ( int i = 0 ; i < rc ; i + + ) {
mr = indexSegment . urlMetadata ( ) . load ( hsa [ i ] . hashsample , null , 0 ) ;
if ( mr = = null ) continue ;
url = mr . metadata ( ) . url ( ) ;
if ( url = = null ) continue ;
2009-06-22 14:25:18 +02:00
hostname = url . getHost ( ) ;
if ( hostname = = null ) continue ;
2009-06-22 14:30:22 +02:00
if ( query . tenant ! = null & & ! hostname . contains ( query . tenant ) & & ! url . toNormalform ( true , true ) . contains ( query . tenant ) ) continue ;
2009-06-22 14:25:18 +02:00
result . add ( new NavigatorEntry ( hostname , hsa [ i ] . count ) ) ;
2009-06-02 17:20:10 +02:00
}
return result ;
}
public static final Comparator < Map . Entry < String , Integer > > mecomp = new Comparator < Map . Entry < String , Integer > > ( ) {
public int compare ( Map . Entry < String , Integer > o1 , Map . Entry < String , Integer > o2 ) {
if ( o1 . getValue ( ) . intValue ( ) < o2 . getValue ( ) . intValue ( ) ) return 1 ;
if ( o2 . getValue ( ) . intValue ( ) < o1 . getValue ( ) . intValue ( ) ) return - 1 ;
return 0 ;
}
} ;
2009-11-09 20:14:51 +01:00
public Map < String , Integer > getTopics ( ) {
return this . ref ;
}
2009-06-02 17:20:10 +02:00
@SuppressWarnings ( " unchecked " )
public ArrayList < NavigatorEntry > getTopicNavigator ( final int count ) {
2007-11-07 23:38:09 +01:00
// create a list of words that had been computed by statistics over all
// words that appeared in the url or the description of all urls
2009-06-08 00:51:15 +02:00
if ( ! this . query . navigators . equals ( " all " ) & & this . query . navigators . indexOf ( " topics " ) < 0 ) return new ArrayList < NavigatorEntry > ( 0 ) ;
2009-06-02 17:20:10 +02:00
Map . Entry < String , Integer > [ ] a = this . ref . entrySet ( ) . toArray ( new Map . Entry [ this . ref . size ( ) ] ) ;
Arrays . sort ( a , mecomp ) ;
int rc = Math . min ( count , a . length ) ;
ArrayList < NavigatorEntry > result = new ArrayList < NavigatorEntry > ( ) ;
Map . Entry < String , Integer > e ;
int c ;
for ( int i = 0 ; i < rc ; i + + ) {
e = a [ i ] ;
c = e . getValue ( ) . intValue ( ) ;
if ( c = = 0 ) break ;
result . add ( new NavigatorEntry ( e . getKey ( ) , c ) ) ;
2007-11-07 23:38:09 +01:00
}
2009-06-02 17:20:10 +02:00
return result ;
2007-11-07 23:38:09 +01:00
}
2009-06-02 17:20:10 +02:00
public void addTopic ( final String [ ] words ) {
2007-11-07 23:38:09 +01:00
String word ;
for ( int i = 0 ; i < words . length ; i + + ) {
word = words [ i ] . toLowerCase ( ) ;
2009-06-02 17:20:10 +02:00
Integer c ;
2009-06-04 01:49:06 +02:00
if ( word . length ( ) > 2 & &
" http_html_php_ftp_www_com_org_net_gov_edu_index_home_page_for_usage_the_and_ " . indexOf ( word ) < 0 & &
! query . queryHashes . contains ( Word . word2hash ( word ) ) & &
word . matches ( " [a-z]+ " ) & &
2009-07-19 22:37:44 +02:00
! Switchboard . badwords . contains ( word ) & &
! Switchboard . stopwords . contains ( word ) ) {
2009-06-02 17:20:10 +02:00
c = ref . get ( word ) ;
if ( c = = null ) ref . put ( word , 1 ) ; else ref . put ( word , c . intValue ( ) + 1 ) ;
}
2007-11-07 23:38:09 +01:00
}
}
2009-08-24 17:24:02 +02:00
protected void addTopics ( final ResultEntry resultEntry ) {
2007-11-07 23:38:09 +01:00
// take out relevant information for reference computation
if ( ( resultEntry . url ( ) = = null ) | | ( resultEntry . title ( ) = = null ) ) return ;
2009-06-02 17:20:10 +02:00
//final String[] urlcomps = htmlFilterContentScraper.urlComps(resultEntry.url().toNormalform(true, true)); // word components of the url
2009-10-11 02:12:19 +02:00
final String [ ] descrcomps = resultEntry . title ( ) . toLowerCase ( ) . split ( DigestURI . splitrex ) ; // words in the description
2007-11-07 23:38:09 +01:00
// add references
2009-06-02 17:20:10 +02:00
//addTopic(urlcomps);
addTopic ( descrcomps ) ;
2007-11-07 23:38:09 +01:00
}
2009-06-09 00:01:26 +02:00
public ArrayList < NavigatorEntry > getAuthorNavigator ( final int count ) {
// create a list of words that had been computed by statistics over all
// words that appeared in the url or the description of all urls
if ( ! this . query . navigators . equals ( " all " ) & & this . query . navigators . indexOf ( " authors " ) < 0 ) return new ArrayList < NavigatorEntry > ( 0 ) ;
AuthorInfo [ ] a = this . authorNavigator . values ( ) . toArray ( new AuthorInfo [ this . authorNavigator . size ( ) ] ) ;
Arrays . sort ( a , aicomp ) ;
int rc = Math . min ( count , a . length ) ;
ArrayList < NavigatorEntry > result = new ArrayList < NavigatorEntry > ( ) ;
AuthorInfo e ;
for ( int i = 0 ; i < rc ; i + + ) {
e = a [ i ] ;
//System.out.println("*** DEBUG Author = " + e.author + ", count = " + e.count);
result . add ( new NavigatorEntry ( e . author , e . count ) ) ;
}
return result ;
}
2009-03-02 00:58:14 +01:00
public ReferenceOrder getOrder ( ) {
2007-11-07 23:38:09 +01:00
return this . order ;
}
2008-08-02 14:12:04 +02:00
public static void loadYBR ( final File rankingPath , final int count ) {
2007-11-07 23:38:09 +01:00
// load ranking tables
if ( rankingPath . exists ( ) ) {
2009-01-30 16:33:00 +01:00
ybrTables = new BinSearch [ count ] ;
2007-11-07 23:38:09 +01:00
String ybrName ;
File f ;
try {
for ( int i = 0 ; i < count ; i + + ) {
2009-01-30 16:33:00 +01:00
ybrName = " YBR-4- " + Digest . encodeHex ( i , 2 ) + " .idx " ;
2007-11-07 23:38:09 +01:00
f = new File ( rankingPath , ybrName ) ;
if ( f . exists ( ) ) {
2009-01-31 02:06:56 +01:00
ybrTables [ i ] = new BinSearch ( FileUtils . read ( f ) , 6 ) ;
2007-11-07 23:38:09 +01:00
} else {
ybrTables [ i ] = null ;
}
}
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-11-07 23:38:09 +01:00
ybrTables = null ;
}
} else {
ybrTables = null ;
}
}
public static boolean canUseYBR ( ) {
return ybrTables ! = null ;
}
public static boolean isUsingYBR ( ) {
return useYBR ;
}
2008-08-02 14:12:04 +02:00
public static void switchYBR ( final boolean usage ) {
2007-11-07 23:38:09 +01:00
useYBR = usage ;
}
2008-08-02 14:12:04 +02:00
public static int ybr ( final String urlHash ) {
2007-11-07 23:38:09 +01:00
// returns the YBR value in a range of 0..15, where 0 means best ranking and 15 means worst ranking
if ( ybrTables = = null ) return 15 ;
if ( ! ( useYBR ) ) return 15 ;
final String domHash = urlHash . substring ( 6 ) ;
2008-08-02 14:12:04 +02:00
final int m = Math . min ( maxYBR , ybrTables . length ) ;
2008-04-24 15:31:55 +02:00
for ( int i = 0 ; i < m ; i + + ) {
2007-11-07 23:38:09 +01:00
if ( ( ybrTables [ i ] ! = null ) & & ( ybrTables [ i ] . contains ( domHash . getBytes ( ) ) ) ) {
//System.out.println("YBR FOUND: " + urlHash + " (" + i + ")");
return i ;
}
}
//System.out.println("NOT FOUND: " + urlHash);
return 15 ;
}
2007-11-22 00:14:57 +01:00
2007-11-07 23:38:09 +01:00
}