2007-11-15 04:03:18 +01:00
// IndexControlRWIs_p.java
// -----------------------
// (C) 2004-2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2004 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
2009-09-27 00:07:40 +02:00
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
2007-11-15 04:03:18 +01:00
//
// LICENSE
2011-06-13 23:44:03 +02:00
//
2007-11-15 04:03:18 +01:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.File ;
import java.io.FileWriter ;
import java.io.IOException ;
import java.io.PrintWriter ;
2009-05-29 12:03:35 +02:00
import java.util.Date ;
2007-11-15 04:03:18 +01:00
import java.util.Iterator ;
2009-05-29 12:03:35 +02:00
import java.util.List ;
2010-04-15 15:22:59 +02:00
import java.util.TreeMap ;
2007-11-15 04:03:18 +01:00
2011-01-03 21:52:54 +01:00
import net.yacy.cora.date.GenericFormatter ;
2011-06-13 23:44:03 +02:00
import net.yacy.cora.document.ASCII ;
2011-03-07 21:36:40 +01:00
import net.yacy.cora.document.UTF8 ;
2010-08-23 14:32:02 +02:00
import net.yacy.cora.protocol.RequestHeader ;
2011-06-13 23:44:03 +02:00
import net.yacy.cora.services.federated.yacy.CacheStrategy ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.Condenser ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.DigestURI ;
import net.yacy.kelondro.data.meta.URIMetadataRow ;
import net.yacy.kelondro.data.word.Word ;
import net.yacy.kelondro.data.word.WordReference ;
import net.yacy.kelondro.data.word.WordReferenceRow ;
2010-04-15 15:22:59 +02:00
import net.yacy.kelondro.index.HandleSet ;
2009-12-10 00:27:26 +01:00
import net.yacy.kelondro.index.RowSpaceExceededException ;
2009-11-05 21:28:37 +01:00
import net.yacy.kelondro.logging.Log ;
2010-04-15 15:22:59 +02:00
import net.yacy.kelondro.order.Base64Order ;
2009-10-10 01:22:22 +02:00
import net.yacy.kelondro.order.Bitfield ;
2009-10-10 02:39:15 +02:00
import net.yacy.kelondro.rwi.Reference ;
import net.yacy.kelondro.rwi.ReferenceContainer ;
import net.yacy.kelondro.rwi.ReferenceContainerCache ;
2011-05-27 10:24:54 +02:00
import net.yacy.kelondro.util.ByteBuffer ;
2011-06-13 23:44:03 +02:00
import net.yacy.kelondro.util.FileUtils ;
2011-09-25 18:59:06 +02:00
import net.yacy.peers.yacyClient ;
import net.yacy.peers.yacySeed ;
import net.yacy.peers.dht.PeerSelection ;
2009-10-21 22:14:30 +02:00
import net.yacy.repository.Blacklist ;
2011-09-25 18:59:06 +02:00
import net.yacy.search.Switchboard ;
import net.yacy.search.SwitchboardConstants ;
import net.yacy.search.index.Segment ;
import net.yacy.search.index.Segments ;
import net.yacy.search.query.QueryParams ;
2011-09-26 23:42:28 +02:00
import net.yacy.search.query.RWIProcess ;
2011-09-25 18:59:06 +02:00
import net.yacy.search.query.SearchEventCache ;
import net.yacy.search.ranking.BlockRank ;
import net.yacy.search.ranking.ReferenceOrder ;
2011-02-12 01:01:40 +01:00
import de.anomic.crawler.ResultURLs ;
2010-11-21 02:29:32 +01:00
import de.anomic.data.ListManager ;
2011-01-10 22:40:56 +01:00
import de.anomic.data.WorkTables ;
2010-11-03 19:31:36 +01:00
import de.anomic.http.client.Cache ;
2007-11-15 04:03:18 +01:00
import de.anomic.server.serverObjects ;
import de.anomic.server.serverSwitch ;
public class IndexControlRWIs_p {
2011-06-13 23:44:03 +02:00
2011-01-10 22:40:56 +01:00
public static serverObjects respond ( final RequestHeader header , final serverObjects post , final serverSwitch env ) throws IOException {
2007-11-15 04:03:18 +01:00
// return variable that accumulates replacements
2009-07-19 22:37:44 +02:00
final Switchboard sb = ( Switchboard ) env ;
2008-08-02 14:12:04 +02:00
final serverObjects prop = new serverObjects ( ) ;
2007-11-15 04:03:18 +01:00
2009-10-09 16:44:20 +02:00
// set default values
2007-11-15 04:03:18 +01:00
prop . putHTML ( " keystring " , " " ) ;
prop . put ( " keyhash " , " " ) ;
prop . put ( " result " , " " ) ;
2011-09-07 23:47:54 +02:00
prop . put ( " cleanup " , post = = null | | post . containsKey ( " maxReferencesLimit " ) ? 1 : 0 ) ;
2011-09-13 16:39:41 +02:00
prop . put ( " cleanup_solr " , sb . indexSegments . segment ( Segments . Process . LOCALCRAWLING ) . getSolr ( ) = = null | | ! sb . getConfigBool ( " federated.service.solr.indexing.enabled " , false ) ? 0 : 1 ) ;
2011-06-13 23:44:03 +02:00
2009-10-09 16:44:20 +02:00
String segmentName = sb . getConfig ( SwitchboardConstants . SEGMENT_PUBLIC , " default " ) ;
int i = 0 ;
2011-06-13 23:44:03 +02:00
for ( final String s : sb . indexSegments . segmentNames ( ) ) {
2009-10-09 16:44:20 +02:00
prop . put ( " segments_ " + i + " _name " , s ) ;
prop . put ( " segments_ " + i + " _selected " , ( segmentName . equals ( s ) ) ? 1 : 0 ) ;
i + + ;
}
Segment segment = sb . indexSegments . segment ( segmentName ) ;
prop . put ( " segments " , i ) ;
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
// switch off all optional forms/lists
prop . put ( " searchresult " , 0 ) ;
prop . put ( " keyhashsimilar " , 0 ) ;
prop . put ( " genUrlList " , 0 ) ;
2011-06-13 23:44:03 +02:00
2007-11-15 13:49:13 +01:00
// clean up all search events
2009-08-24 17:24:02 +02:00
SearchEventCache . cleanupEvents ( true ) ;
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
if ( post ! = null ) {
// default values
2009-10-09 16:44:20 +02:00
segmentName = post . get ( " segment " , segmentName ) . trim ( ) ;
i = 0 ;
2011-06-13 23:44:03 +02:00
for ( final String s : sb . indexSegments . segmentNames ( ) ) {
2009-10-09 16:44:20 +02:00
prop . put ( " segments_ " + i + " _name " , s ) ;
prop . put ( " segments_ " + i + " _selected " , ( segmentName . equals ( s ) ) ? 1 : 0 ) ;
i + + ;
}
prop . put ( " segments " , i ) ;
segment = sb . indexSegments . segment ( segmentName ) ;
2011-06-13 23:44:03 +02:00
2008-08-02 14:12:04 +02:00
final String keystring = post . get ( " keystring " , " " ) . trim ( ) ;
2009-04-16 17:29:00 +02:00
byte [ ] keyhash = post . get ( " keyhash " , " " ) . trim ( ) . getBytes ( ) ;
2007-11-15 04:03:18 +01:00
prop . putHTML ( " keystring " , keystring ) ;
2011-05-27 10:24:54 +02:00
prop . putHTML ( " keyhash " , ASCII . String ( keyhash ) ) ;
2007-11-15 04:03:18 +01:00
// read values from checkboxes
2011-06-13 23:44:03 +02:00
final String [ ] urls = post . getAll ( " urlhx.* " ) ;
2010-04-15 15:22:59 +02:00
HandleSet urlb = new HandleSet ( URIMetadataRow . rowdef . primaryKeyLength , URIMetadataRow . rowdef . objectOrder , urls . length ) ;
2011-06-13 23:44:03 +02:00
if ( urls ! = null ) for ( final String s : urls ) try { urlb . put ( s . getBytes ( ) ) ; } catch ( final RowSpaceExceededException e ) { Log . logException ( e ) ; }
2008-08-02 14:12:04 +02:00
final boolean delurl = post . containsKey ( " delurl " ) ;
final boolean delurlref = post . containsKey ( " delurlref " ) ;
2007-11-15 04:03:18 +01:00
if ( post . containsKey ( " keystringsearch " ) ) {
2009-03-02 00:58:14 +01:00
keyhash = Word . word2hash ( keystring ) ;
2007-11-15 04:03:18 +01:00
prop . put ( " keyhash " , keyhash ) ;
2011-09-26 23:42:28 +02:00
final RWIProcess ranking = genSearchresult ( prop , sb , segment , keyhash , null ) ;
2007-11-22 00:14:57 +01:00
if ( ranking . filteredCount ( ) = = 0 ) {
2007-11-15 04:03:18 +01:00
prop . put ( " searchresult " , 1 ) ;
2008-09-19 13:45:11 +02:00
prop . putHTML ( " searchresult_word " , keystring ) ;
2007-11-15 04:03:18 +01:00
}
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
if ( post . containsKey ( " keyhashsearch " ) ) {
2011-05-27 10:24:54 +02:00
if ( keystring . length ( ) = = 0 | | ! ByteBuffer . equals ( Word . word2hash ( keystring ) , keyhash ) ) {
2007-11-15 04:03:18 +01:00
prop . put ( " keystring " , " <not possible to compute word from hash> " ) ;
}
2011-09-26 23:42:28 +02:00
final RWIProcess ranking = genSearchresult ( prop , sb , segment , keyhash , null ) ;
2007-11-22 00:14:57 +01:00
if ( ranking . filteredCount ( ) = = 0 ) {
2007-11-15 04:03:18 +01:00
prop . put ( " searchresult " , 2 ) ;
2011-05-27 10:24:54 +02:00
prop . putHTML ( " searchresult_wordhash " , ASCII . String ( keyhash ) ) ;
2007-11-15 04:03:18 +01:00
}
}
2011-06-13 23:44:03 +02:00
2011-09-07 23:47:54 +02:00
// delete everything
2010-11-03 19:31:36 +01:00
if ( post . containsKey ( " deletecomplete " ) ) {
if ( post . get ( " deleteIndex " , " " ) . equals ( " on " ) ) {
segment . clear ( ) ;
}
2011-05-09 00:47:20 +02:00
if ( post . get ( " deleteSolr " , " " ) . equals ( " on " ) & & sb . getConfigBool ( " federated.service.solr.indexing.enabled " , false ) ) try {
2011-09-13 16:39:41 +02:00
sb . indexSegments . segment ( Segments . Process . LOCALCRAWLING ) . getSolr ( ) . clear ( ) ;
2011-06-13 23:44:03 +02:00
} catch ( final Exception e ) {
2011-05-09 00:47:20 +02:00
Log . logException ( e ) ;
2011-04-18 18:11:16 +02:00
}
2010-11-03 19:31:36 +01:00
if ( post . get ( " deleteCrawlQueues " , " " ) . equals ( " on " ) ) {
sb . crawlQueues . clear ( ) ;
sb . crawlStacker . clear ( ) ;
2011-02-12 01:01:40 +01:00
ResultURLs . clearStacks ( ) ;
2010-11-03 19:31:36 +01:00
}
if ( post . get ( " deleteCache " , " " ) . equals ( " on " ) ) {
Cache . clear ( ) ;
}
if ( post . get ( " deleteRobots " , " " ) . equals ( " on " ) ) {
sb . robots . clear ( ) ;
}
2011-01-10 22:40:56 +01:00
if ( post . get ( " deleteSearchFl " , " " ) . equals ( " on " ) ) {
2011-09-07 23:47:54 +02:00
sb . tables . clear ( WorkTables . TABLE_SEARCH_FAILURE_NAME ) ;
2011-01-10 22:40:56 +01:00
}
2008-05-24 14:30:50 +02:00
post . remove ( " deletecomplete " ) ;
}
2011-06-13 23:44:03 +02:00
2011-09-07 23:47:54 +02:00
// set reference limitation
if ( post . containsKey ( " maxReferencesLimit " ) ) {
if ( post . get ( " maxReferencesRadio " , " " ) . equals ( " on " ) ) {
ReferenceContainer . maxReferences = post . getInt ( " maxReferences " , 0 ) ;
} else {
ReferenceContainer . maxReferences = 0 ;
}
sb . setConfig ( " index.maxReferences " , ReferenceContainer . maxReferences ) ;
}
2007-11-15 04:03:18 +01:00
// delete word
2009-03-16 01:18:37 +01:00
if ( post . containsKey ( " keyhashdeleteall " ) ) try {
2007-11-15 04:03:18 +01:00
if ( delurl | | delurlref ) {
2010-02-17 00:06:04 +01:00
// generate urlx: an array of url hashes to be deleted
2009-04-15 08:34:27 +02:00
ReferenceContainer < WordReference > index = null ;
2009-10-09 16:44:20 +02:00
index = segment . termIndex ( ) . get ( keyhash , null ) ;
2009-04-15 08:34:27 +02:00
final Iterator < WordReference > en = index . entries ( ) ;
2010-04-15 15:22:59 +02:00
urlb = new HandleSet ( URIMetadataRow . rowdef . primaryKeyLength , URIMetadataRow . rowdef . objectOrder , index . size ( ) ) ;
2011-06-13 23:44:03 +02:00
while ( en . hasNext ( ) ) try { urlb . put ( en . next ( ) . urlhash ( ) ) ; } catch ( final RowSpaceExceededException e ) { Log . logException ( e ) ; }
2007-11-15 04:03:18 +01:00
index = null ;
}
if ( delurlref ) {
2011-06-13 23:44:03 +02:00
segment . removeAllUrlReferences ( urlb , sb . loader , CacheStrategy . IFEXIST ) ;
2007-11-15 04:03:18 +01:00
}
2010-02-17 00:06:04 +01:00
// delete the word first because that is much faster than the deletion of the urls from the url database
segment . termIndex ( ) . delete ( keyhash ) ;
// now delete all urls if demanded
2007-11-15 04:03:18 +01:00
if ( delurl | | delurlref ) {
2011-06-13 23:44:03 +02:00
for ( final byte [ ] b : urlb ) sb . urlRemove ( segment , b ) ;
2007-11-15 04:03:18 +01:00
}
post . remove ( " keyhashdeleteall " ) ;
post . put ( " urllist " , " generated " ) ;
2011-06-13 23:44:03 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2007-11-15 04:03:18 +01:00
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
// delete selected URLs
2009-03-16 01:18:37 +01:00
if ( post . containsKey ( " keyhashdelete " ) ) try {
2007-11-15 04:03:18 +01:00
if ( delurlref ) {
2011-06-13 23:44:03 +02:00
segment . removeAllUrlReferences ( urlb , sb . loader , CacheStrategy . IFEXIST ) ;
2007-11-15 04:03:18 +01:00
}
if ( delurl | | delurlref ) {
2011-06-13 23:44:03 +02:00
for ( final byte [ ] b : urlb ) sb . urlRemove ( segment , b ) ;
2007-11-15 04:03:18 +01:00
}
2010-04-15 15:22:59 +02:00
final HandleSet urlHashes = new HandleSet ( URIMetadataRow . rowdef . primaryKeyLength , URIMetadataRow . rowdef . objectOrder , 0 ) ;
2011-06-13 23:44:03 +02:00
for ( final byte [ ] b : urlb ) try { urlHashes . put ( b ) ; } catch ( final RowSpaceExceededException e ) { Log . logException ( e ) ; }
2009-10-09 16:44:20 +02:00
segment . termIndex ( ) . remove ( keyhash , urlHashes ) ;
2007-11-15 04:03:18 +01:00
// this shall lead to a presentation of the list; so handle that the remaining program
// thinks that it was called for a list presentation
post . remove ( " keyhashdelete " ) ;
post . put ( " urllist " , " generated " ) ;
2011-06-13 23:44:03 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2007-11-15 04:03:18 +01:00
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
if ( post . containsKey ( " urllist " ) ) {
2011-05-27 10:24:54 +02:00
if ( keystring . length ( ) = = 0 | | ! ByteBuffer . equals ( Word . word2hash ( keystring ) , keyhash ) ) {
2007-11-15 04:03:18 +01:00
prop . put ( " keystring " , " <not possible to compute word from hash> " ) ;
}
2009-05-29 12:03:35 +02:00
final Bitfield flags = compileFlags ( post ) ;
2008-08-02 14:12:04 +02:00
final int count = ( post . get ( " lines " , " all " ) . equals ( " all " ) ) ? - 1 : post . getInt ( " lines " , - 1 ) ;
2011-09-26 23:42:28 +02:00
final RWIProcess ranking = genSearchresult ( prop , sb , segment , keyhash , flags ) ;
2009-05-29 12:03:35 +02:00
genURLList ( prop , keyhash , keystring , ranking , flags , count ) ;
2007-11-15 04:03:18 +01:00
}
// transfer to other peer
2009-03-16 01:18:37 +01:00
if ( post . containsKey ( " keyhashtransfer " ) ) try {
2011-05-27 10:24:54 +02:00
if ( keystring . length ( ) = = 0 | | ! ByteBuffer . equals ( Word . word2hash ( keystring ) , keyhash ) ) {
2007-11-15 04:03:18 +01:00
prop . put ( " keystring " , " <not possible to compute word from hash> " ) ;
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
// find host & peer
String host = post . get ( " host " , " " ) ; // get host from input field
yacySeed seed = null ;
if ( host . length ( ) ! = 0 ) {
if ( host . length ( ) = = 12 ) {
// the host string is a peer hash
2009-05-28 16:26:05 +02:00
seed = sb . peers . getConnected ( host ) ;
2007-11-15 04:03:18 +01:00
} else {
// the host string can be a host name
2009-05-28 16:26:05 +02:00
seed = sb . peers . lookupByName ( host ) ;
2007-11-15 04:03:18 +01:00
}
} else {
host = post . get ( " hostHash " , " " ) ; // if input field is empty, get from select box
2009-05-28 16:26:05 +02:00
seed = sb . peers . getConnected ( host ) ;
2007-11-15 04:03:18 +01:00
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
// prepare index
2009-04-15 08:34:27 +02:00
ReferenceContainer < WordReference > index ;
2008-08-02 14:12:04 +02:00
final long starttime = System . currentTimeMillis ( ) ;
2009-10-09 16:44:20 +02:00
index = segment . termIndex ( ) . get ( keyhash , null ) ;
2007-11-15 04:03:18 +01:00
// built urlCache
2009-04-15 08:34:27 +02:00
final Iterator < WordReference > urlIter = index . entries ( ) ;
2010-04-15 15:22:59 +02:00
final TreeMap < byte [ ] , URIMetadataRow > knownURLs = new TreeMap < byte [ ] , URIMetadataRow > ( Base64Order . enhancedCoder ) ;
final HandleSet unknownURLEntries = new HandleSet ( WordReferenceRow . urlEntryRow . primaryKeyLength , WordReferenceRow . urlEntryRow . objectOrder , index . size ( ) ) ;
2009-03-02 00:58:14 +01:00
Reference iEntry ;
2009-10-11 02:12:19 +02:00
URIMetadataRow lurl ;
2007-11-15 04:03:18 +01:00
while ( urlIter . hasNext ( ) ) {
2008-01-22 20:03:47 +01:00
iEntry = urlIter . next ( ) ;
2011-05-26 12:57:02 +02:00
lurl = segment . urlMetadata ( ) . load ( iEntry . urlhash ( ) ) ;
2007-11-15 04:03:18 +01:00
if ( lurl = = null ) {
2010-04-15 15:22:59 +02:00
try {
2011-05-26 12:57:02 +02:00
unknownURLEntries . put ( iEntry . urlhash ( ) ) ;
2011-06-13 23:44:03 +02:00
} catch ( final RowSpaceExceededException e ) {
2010-04-15 15:22:59 +02:00
Log . logException ( e ) ;
}
2007-11-15 04:03:18 +01:00
urlIter . remove ( ) ;
} else {
2011-05-26 12:57:02 +02:00
knownURLs . put ( iEntry . urlhash ( ) , lurl ) ;
2007-11-15 04:03:18 +01:00
}
}
2011-06-13 23:44:03 +02:00
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
// make an indexContainerCache
2011-06-13 23:44:03 +02:00
final ReferenceContainerCache < WordReference > icc = new ReferenceContainerCache < WordReference > ( Segment . wordReferenceFactory , Segment . wordOrder , Word . commonHashLength ) ;
2009-12-10 00:27:26 +01:00
try {
icc . add ( index ) ;
2011-06-13 23:44:03 +02:00
} catch ( final RowSpaceExceededException e ) {
2009-12-10 00:27:26 +01:00
Log . logException ( e ) ;
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
// transport to other peer
2011-03-15 02:03:35 +01:00
final boolean gzipBody = sb . getConfigBool ( " indexControl.gzipBody " , false ) ;
final int timeout = ( int ) sb . getConfigLong ( " indexControl.timeout " , 60000 ) ;
2009-02-16 22:28:48 +01:00
final String error = yacyClient . transferIndex (
2007-11-15 04:03:18 +01:00
seed ,
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
icc ,
2007-11-15 04:03:18 +01:00
knownURLs ,
2011-03-15 02:03:35 +01:00
gzipBody ,
2007-11-15 04:03:18 +01:00
timeout ) ;
2010-04-15 15:22:59 +02:00
prop . put ( " result " , ( error = = null ) ? ( " Successfully transferred " + knownURLs . size ( ) + " words in " + ( ( System . currentTimeMillis ( ) - starttime ) / 1000 ) + " seconds, " + unknownURLEntries . size ( ) + " URL not found " ) : " error: " + error ) ;
2007-11-15 04:03:18 +01:00
index = null ;
2011-06-13 23:44:03 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2007-11-15 04:03:18 +01:00
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
// generate list
2009-03-16 17:24:53 +01:00
if ( post . containsKey ( " keyhashsimilar " ) ) try {
2011-08-25 12:35:25 +02:00
final Iterator < ReferenceContainer < WordReference > > containerIt = segment . termIndex ( ) . referenceContainer ( keyhash , true , 256 , false ) . iterator ( ) ;
2009-04-15 08:34:27 +02:00
ReferenceContainer < WordReference > container ;
2009-10-09 16:44:20 +02:00
i = 0 ;
2007-11-15 04:03:18 +01:00
int rows = 0 , cols = 0 ;
prop . put ( " keyhashsimilar " , " 1 " ) ;
while ( containerIt . hasNext ( ) & & i < 256 ) {
2008-06-06 18:01:27 +02:00
container = containerIt . next ( ) ;
2009-04-03 15:23:45 +02:00
prop . put ( " keyhashsimilar_rows_ " + rows + " _cols_ " + cols + " _wordHash " , container . getTermHash ( ) ) ;
2007-11-15 04:03:18 +01:00
cols + + ;
if ( cols = = 8 ) {
prop . put ( " keyhashsimilar_rows_ " + rows + " _cols " , cols ) ;
cols = 0 ;
rows + + ;
}
i + + ;
}
prop . put ( " keyhashsimilar_rows_ " + rows + " _cols " , cols ) ;
prop . put ( " keyhashsimilar_rows " , rows + 1 ) ;
prop . put ( " result " , " " ) ;
2011-06-13 23:44:03 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2007-11-15 04:03:18 +01:00
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
if ( post . containsKey ( " blacklist " ) ) {
2008-08-02 14:12:04 +02:00
final String blacklist = post . get ( " blacklist " , " " ) ;
2010-04-15 15:22:59 +02:00
final HandleSet urlHashes = new HandleSet ( URIMetadataRow . rowdef . primaryKeyLength , URIMetadataRow . rowdef . objectOrder , urlb . size ( ) ) ;
2007-11-15 04:03:18 +01:00
if ( post . containsKey ( " blacklisturls " ) ) {
PrintWriter pw ;
try {
2008-08-02 14:12:04 +02:00
final String [ ] supportedBlacklistTypes = env . getConfig ( " BlackLists.types " , " " ) . split ( " , " ) ;
2010-11-21 02:29:32 +01:00
pw = new PrintWriter ( new FileWriter ( new File ( ListManager . listsPath , blacklist ) , true ) ) ;
2009-10-11 02:12:19 +02:00
DigestURI url ;
2011-06-13 23:44:03 +02:00
for ( final byte [ ] b : urlb ) {
2010-04-15 15:22:59 +02:00
try {
urlHashes . put ( b ) ;
2011-06-13 23:44:03 +02:00
} catch ( final RowSpaceExceededException e ) {
2010-04-15 15:22:59 +02:00
Log . logException ( e ) ;
}
2011-05-13 08:21:40 +02:00
final URIMetadataRow e = segment . urlMetadata ( ) . load ( b ) ;
2010-04-15 15:22:59 +02:00
segment . urlMetadata ( ) . remove ( b ) ;
2007-11-15 04:03:18 +01:00
if ( e ! = null ) {
2009-03-02 12:04:13 +01:00
url = e . metadata ( ) . url ( ) ;
2007-11-15 04:03:18 +01:00
pw . println ( url . getHost ( ) + " / " + url . getFile ( ) ) ;
2011-06-13 23:44:03 +02:00
for ( final String supportedBlacklistType : supportedBlacklistTypes ) {
if ( ListManager . listSetContains ( supportedBlacklistType + " .BlackLists " , blacklist ) ) {
2009-07-19 22:37:44 +02:00
Switchboard . urlBlacklist . add (
2011-06-13 23:44:03 +02:00
supportedBlacklistType ,
2007-11-15 04:03:18 +01:00
url . getHost ( ) ,
url . getFile ( ) ) ;
2011-06-13 23:44:03 +02:00
}
2007-11-15 04:03:18 +01:00
}
2009-10-21 22:14:30 +02:00
SearchEventCache . cleanupEvents ( true ) ;
2007-11-15 04:03:18 +01:00
}
}
pw . close ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-11-15 04:03:18 +01:00
}
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
if ( post . containsKey ( " blacklistdomains " ) ) {
PrintWriter pw ;
try {
2009-10-21 22:14:30 +02:00
final String [ ] supportedBlacklistTypes = Blacklist . BLACKLIST_TYPES_STRING . split ( " , " ) ;
2010-11-21 02:29:32 +01:00
pw = new PrintWriter ( new FileWriter ( new File ( ListManager . listsPath , blacklist ) , true ) ) ;
2009-10-11 02:12:19 +02:00
DigestURI url ;
2011-06-13 23:44:03 +02:00
for ( final byte [ ] b : urlb ) {
2010-04-15 15:22:59 +02:00
try {
urlHashes . put ( b ) ;
2011-06-13 23:44:03 +02:00
} catch ( final RowSpaceExceededException e ) {
2010-04-15 15:22:59 +02:00
Log . logException ( e ) ;
}
2011-05-13 08:21:40 +02:00
final URIMetadataRow e = segment . urlMetadata ( ) . load ( b ) ;
2010-04-15 15:22:59 +02:00
segment . urlMetadata ( ) . remove ( b ) ;
2007-11-15 04:03:18 +01:00
if ( e ! = null ) {
2009-03-02 12:04:13 +01:00
url = e . metadata ( ) . url ( ) ;
2007-11-15 04:03:18 +01:00
pw . println ( url . getHost ( ) + " /.* " ) ;
2011-06-13 23:44:03 +02:00
for ( final String supportedBlacklistType : supportedBlacklistTypes ) {
if ( ListManager . listSetContains ( supportedBlacklistType + " .BlackLists " , blacklist ) ) {
2009-07-19 22:37:44 +02:00
Switchboard . urlBlacklist . add (
2011-06-13 23:44:03 +02:00
supportedBlacklistType ,
2007-11-15 04:03:18 +01:00
url . getHost ( ) , " .* " ) ;
2011-06-13 23:44:03 +02:00
}
2007-11-15 04:03:18 +01:00
}
}
}
pw . close ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-11-15 04:03:18 +01:00
}
}
2009-03-16 01:18:37 +01:00
try {
2009-10-09 16:44:20 +02:00
segment . termIndex ( ) . remove ( keyhash , urlHashes ) ;
2011-06-13 23:44:03 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2009-03-16 01:18:37 +01:00
}
2007-11-15 04:03:18 +01:00
}
2011-06-13 23:44:03 +02:00
2009-05-29 12:03:35 +02:00
if ( prop . getInt ( " searchresult " , 0 ) = = 3 ) listHosts ( prop , keyhash , sb ) ;
2007-11-15 04:03:18 +01:00
}
2011-06-13 23:44:03 +02:00
2007-11-15 04:03:18 +01:00
// insert constants
2009-10-09 16:44:20 +02:00
prop . putNum ( " wcount " , segment . termIndex ( ) . sizesMax ( ) ) ;
2011-09-07 23:47:54 +02:00
prop . put ( " cleanup_maxReferencesRadioChecked " , ReferenceContainer . maxReferences > 0 ? 1 : 0 ) ;
prop . put ( " cleanup_maxReferences " , ReferenceContainer . maxReferences > 0 ? ReferenceContainer . maxReferences : 100000 ) ;
2007-11-15 04:03:18 +01:00
// return rewrite properties
return prop ;
}
2011-06-13 23:44:03 +02:00
2011-09-26 23:42:28 +02:00
public static void genURLList ( final serverObjects prop , final byte [ ] keyhash , final String keystring , final RWIProcess ranked , final Bitfield flags , final int maxlines ) {
2009-05-29 12:03:35 +02:00
// search for a word hash and generate a list of url links
2011-06-13 23:44:03 +02:00
final String keyhashs = ASCII . String ( keyhash ) ;
2011-05-30 09:31:17 +02:00
prop . put ( " genUrlList_keyHash " , keyhashs ) ;
2011-06-13 23:44:03 +02:00
2009-05-29 12:03:35 +02:00
if ( ranked . filteredCount ( ) = = 0 ) {
prop . put ( " genUrlList " , 1 ) ;
prop . put ( " genUrlList_count " , 0 ) ;
prop . put ( " searchresult " , 2 ) ;
} else {
prop . put ( " genUrlList " , 2 ) ;
prop . put ( " searchresult " , 3 ) ;
prop . put ( " genUrlList_flags " , ( flags = = null ) ? " " : flags . exportB64 ( ) ) ;
prop . put ( " genUrlList_lines " , maxlines ) ;
int i = 0 ;
2009-10-11 02:12:19 +02:00
DigestURI url ;
URIMetadataRow entry ;
2009-05-29 12:03:35 +02:00
String us ;
long rn = - 1 ;
2010-09-20 15:17:25 +02:00
while ( ! ranked . isEmpty ( ) & & ( entry = ranked . takeURL ( false , 1000 ) ) ! = null ) {
2009-05-29 12:03:35 +02:00
if ( ( entry = = null ) | | ( entry . metadata ( ) = = null ) ) continue ;
url = entry . metadata ( ) . url ( ) ;
if ( url = = null ) continue ;
us = url . toNormalform ( false , false ) ;
if ( rn = = - 1 ) rn = entry . ranking ( ) ;
prop . put ( " genUrlList_urlList_ " + i + " _urlExists " , " 1 " ) ;
prop . put ( " genUrlList_urlList_ " + i + " _urlExists_urlhxCount " , i ) ;
2011-05-26 12:57:02 +02:00
prop . putHTML ( " genUrlList_urlList_ " + i + " _urlExists_urlhxValue " , entry . word ( ) . urlhash ( ) ) ;
2009-05-29 12:03:35 +02:00
prop . putHTML ( " genUrlList_urlList_ " + i + " _urlExists_keyString " , keystring ) ;
2011-05-30 09:31:17 +02:00
prop . put ( " genUrlList_urlList_ " + i + " _urlExists_keyHash " , keyhashs ) ;
2009-05-29 12:03:35 +02:00
prop . putHTML ( " genUrlList_urlList_ " + i + " _urlExists_urlString " , us ) ;
prop . put ( " genUrlList_urlList_ " + i + " _urlExists_urlStringShort " , ( us . length ( ) > 40 ) ? ( us . substring ( 0 , 20 ) + " <br> " + us . substring ( 20 , 40 ) + " ... " ) : ( ( us . length ( ) > 30 ) ? ( us . substring ( 0 , 20 ) + " <br> " + us . substring ( 20 ) ) : us ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_ranking " , ( entry . ranking ( ) - rn ) ) ;
2009-10-11 02:12:19 +02:00
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_domlength " , DigestURI . domLengthEstimation ( entry . hash ( ) ) ) ;
2011-05-18 16:26:28 +02:00
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_ybr " , BlockRank . ranking ( entry . hash ( ) ) ) ;
2009-05-29 12:03:35 +02:00
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_tf " , 1000 . 0 * entry . word ( ) . termFrequency ( ) ) ;
2011-05-30 06:19:20 +02:00
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_authority " , ( ranked . getOrder ( ) = = null ) ? - 1 : ranked . getOrder ( ) . authority ( ASCII . String ( entry . hash ( ) , 6 , 6 ) ) ) ;
2011-01-03 21:52:54 +01:00
prop . put ( " genUrlList_urlList_ " + i + " _urlExists_date " , GenericFormatter . SHORT_DAY_FORMATTER . format ( new Date ( entry . word ( ) . lastModified ( ) ) ) ) ;
2009-05-29 12:03:35 +02:00
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_wordsintitle " , entry . word ( ) . wordsintitle ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_wordsintext " , entry . word ( ) . wordsintext ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_phrasesintext " , entry . word ( ) . phrasesintext ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_llocal " , entry . word ( ) . llocal ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_lother " , entry . word ( ) . lother ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_hitcount " , entry . word ( ) . hitcount ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_worddistance " , 0 ) ;
2011-05-18 16:26:28 +02:00
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_ybr " , BlockRank . ranking ( entry . hash ( ) ) ) ;
2009-05-29 12:03:35 +02:00
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_pos " , entry . word ( ) . minposition ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_phrase " , entry . word ( ) . posofphrase ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_posinphrase " , entry . word ( ) . posinphrase ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_urlcomps " , entry . word ( ) . urlcomps ( ) ) ;
prop . putNum ( " genUrlList_urlList_ " + i + " _urlExists_urllength " , entry . word ( ) . urllength ( ) ) ;
prop . put ( " genUrlList_urlList_ " + i + " _urlExists_props " ,
( ( entry . word ( ) . flags ( ) . get ( Condenser . flag_cat_indexof ) ) ? " appears on index page, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( Condenser . flag_cat_hasimage ) ) ? " contains images, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( Condenser . flag_cat_hasaudio ) ) ? " contains audio, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( Condenser . flag_cat_hasvideo ) ) ? " contains video, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( Condenser . flag_cat_hasapp ) ) ? " contains applications, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( WordReferenceRow . flag_app_dc_identifier ) ) ? " appears in url, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( WordReferenceRow . flag_app_dc_title ) ) ? " appears in title, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( WordReferenceRow . flag_app_dc_creator ) ) ? " appears in author, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( WordReferenceRow . flag_app_dc_subject ) ) ? " appears in subject, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( WordReferenceRow . flag_app_dc_description ) ) ? " appears in description, " : " " ) +
( ( entry . word ( ) . flags ( ) . get ( WordReferenceRow . flag_app_emphasized ) ) ? " appears emphasized, " : " " ) +
2011-05-26 12:57:02 +02:00
( ( DigestURI . probablyRootURL ( entry . word ( ) . urlhash ( ) ) ) ? " probably root url " : " " )
2009-05-29 12:03:35 +02:00
) ;
2009-07-19 22:37:44 +02:00
if ( Switchboard . urlBlacklist . isListed ( Blacklist . BLACKLIST_DHT , url ) ) {
2009-05-29 12:03:35 +02:00
prop . put ( " genUrlList_urlList_ " + i + " _urlExists_urlhxChecked " , " 1 " ) ;
}
i + + ;
if ( ( maxlines > = 0 ) & & ( i > = maxlines ) ) break ;
}
2010-04-15 15:22:59 +02:00
final Iterator < byte [ ] > iter = ranked . miss ( ) ; // iterates url hash strings
byte [ ] b ;
2009-05-29 12:03:35 +02:00
while ( iter . hasNext ( ) ) {
2010-04-15 15:22:59 +02:00
b = iter . next ( ) ;
prop . put ( " genUrlList_urlList_ " + i + " _urlExists " , " 0 " ) ;
prop . put ( " genUrlList_urlList_ " + i + " _urlExists_urlhxCount " , i ) ;
prop . putHTML ( " genUrlList_urlList_ " + i + " _urlExists_urlhxValue " , b ) ;
2009-05-29 12:03:35 +02:00
i + + ;
}
prop . put ( " genUrlList_urlList " , i ) ;
prop . putHTML ( " genUrlList_keyString " , keystring ) ;
prop . put ( " genUrlList_count " , i ) ;
2010-11-21 02:29:32 +01:00
putBlacklists ( prop , FileUtils . getDirListing ( ListManager . listsPath ) ) ;
2009-05-29 12:03:35 +02:00
}
}
2011-06-13 23:44:03 +02:00
2009-05-29 12:03:35 +02:00
public static void putBlacklists ( final serverObjects prop , final List < String > lists ) {
prop . put ( " genUrlList_blacklists " , lists . size ( ) ) ;
int i = 0 ;
2011-06-13 23:44:03 +02:00
for ( final String list : lists )
2009-05-29 12:03:35 +02:00
prop . put ( " genUrlList_blacklists_ " + i + + + " _name " , list ) ;
}
2011-06-13 23:44:03 +02:00
2009-05-29 12:03:35 +02:00
public static Bitfield compileFlags ( final serverObjects post ) {
final Bitfield b = new Bitfield ( 4 ) ;
if ( post . get ( " allurl " , " " ) . equals ( " on " ) ) return null ;
if ( post . get ( " flags " ) ! = null ) {
if ( post . get ( " flags " , " " ) . length ( ) = = 0 ) return null ;
return new Bitfield ( 4 , post . get ( " flags " ) ) ;
}
if ( post . get ( " description " , " " ) . equals ( " on " ) ) b . set ( WordReferenceRow . flag_app_dc_description , true ) ;
if ( post . get ( " title " , " " ) . equals ( " on " ) ) b . set ( WordReferenceRow . flag_app_dc_title , true ) ;
if ( post . get ( " creator " , " " ) . equals ( " on " ) ) b . set ( WordReferenceRow . flag_app_dc_creator , true ) ;
if ( post . get ( " subject " , " " ) . equals ( " on " ) ) b . set ( WordReferenceRow . flag_app_dc_subject , true ) ;
if ( post . get ( " url " , " " ) . equals ( " on " ) ) b . set ( WordReferenceRow . flag_app_dc_identifier , true ) ;
if ( post . get ( " emphasized " , " " ) . equals ( " on " ) ) b . set ( WordReferenceRow . flag_app_emphasized , true ) ;
if ( post . get ( " image " , " " ) . equals ( " on " ) ) b . set ( Condenser . flag_cat_hasimage , true ) ;
if ( post . get ( " audio " , " " ) . equals ( " on " ) ) b . set ( Condenser . flag_cat_hasaudio , true ) ;
if ( post . get ( " video " , " " ) . equals ( " on " ) ) b . set ( Condenser . flag_cat_hasvideo , true ) ;
if ( post . get ( " app " , " " ) . equals ( " on " ) ) b . set ( Condenser . flag_cat_hasapp , true ) ;
if ( post . get ( " indexof " , " " ) . equals ( " on " ) ) b . set ( Condenser . flag_cat_indexof , true ) ;
return b ;
}
2009-07-19 22:37:44 +02:00
public static void listHosts ( final serverObjects prop , final byte [ ] startHash , final Switchboard sb ) {
2009-05-29 12:03:35 +02:00
// list known hosts
yacySeed seed ;
int hc = 0 ;
prop . put ( " searchresult_keyhash " , startHash ) ;
final Iterator < yacySeed > e = PeerSelection . getAcceptRemoteIndexSeeds ( sb . peers , startHash , sb . peers . sizeConnected ( ) , true ) ;
while ( e . hasNext ( ) ) {
seed = e . next ( ) ;
if ( seed ! = null ) {
prop . put ( " searchresult_hosts_ " + hc + " _hosthash " , seed . hash ) ;
prop . putHTML ( " searchresult_hosts_ " + hc + " _hostname " , seed . hash + " " + seed . get ( yacySeed . NAME , " nameless " ) ) ;
hc + + ;
}
}
prop . put ( " searchresult_hosts " , hc ) ;
}
2011-09-26 23:42:28 +02:00
public static RWIProcess genSearchresult ( final serverObjects prop , final Switchboard sb , final Segment segment , final byte [ ] keyhash , final Bitfield filter ) {
2011-05-27 10:24:54 +02:00
final QueryParams query = new QueryParams ( ASCII . String ( keyhash ) , - 1 , filter , segment , sb . getRanking ( ) , " IndexControlRWIs_p " ) ;
2011-03-10 13:35:32 +01:00
final ReferenceOrder order = new ReferenceOrder ( query . ranking , UTF8 . getBytes ( query . targetlang ) ) ;
2011-09-26 23:42:28 +02:00
final RWIProcess ranked = new RWIProcess ( query , order , Integer . MAX_VALUE ) ;
2009-08-27 17:19:48 +02:00
ranked . run ( ) ;
2011-06-13 23:44:03 +02:00
2009-05-29 12:03:35 +02:00
if ( ranked . filteredCount ( ) = = 0 ) {
prop . put ( " searchresult " , 2 ) ;
prop . put ( " searchresult_wordhash " , keyhash ) ;
} else {
prop . put ( " searchresult " , 3 ) ;
prop . put ( " searchresult_allurl " , ranked . filteredCount ( ) ) ;
prop . put ( " searchresult_description " , ranked . flagCount ( ) [ WordReferenceRow . flag_app_dc_description ] ) ;
prop . put ( " searchresult_title " , ranked . flagCount ( ) [ WordReferenceRow . flag_app_dc_title ] ) ;
prop . put ( " searchresult_creator " , ranked . flagCount ( ) [ WordReferenceRow . flag_app_dc_creator ] ) ;
prop . put ( " searchresult_subject " , ranked . flagCount ( ) [ WordReferenceRow . flag_app_dc_subject ] ) ;
prop . put ( " searchresult_url " , ranked . flagCount ( ) [ WordReferenceRow . flag_app_dc_identifier ] ) ;
prop . put ( " searchresult_emphasized " , ranked . flagCount ( ) [ WordReferenceRow . flag_app_emphasized ] ) ;
prop . put ( " searchresult_image " , ranked . flagCount ( ) [ Condenser . flag_cat_hasimage ] ) ;
prop . put ( " searchresult_audio " , ranked . flagCount ( ) [ Condenser . flag_cat_hasaudio ] ) ;
prop . put ( " searchresult_video " , ranked . flagCount ( ) [ Condenser . flag_cat_hasvideo ] ) ;
prop . put ( " searchresult_app " , ranked . flagCount ( ) [ Condenser . flag_cat_hasapp ] ) ;
prop . put ( " searchresult_indexof " , ranked . flagCount ( ) [ Condenser . flag_cat_indexof ] ) ;
}
return ranked ;
}
2007-11-15 04:03:18 +01:00
}