2011-04-14 22:12:14 +02:00
/ * *
* SolrScheme
* Copyright 2011 by Michael Peter Christen
* First released 14 . 04 . 2011 at http : //yacy.net
*
* $LastChangedDate : 2011 - 04 - 14 22 : 05 : 04 + 0200 ( Do , 14 Apr 2011 ) $
* $LastChangedRevision : 7654 $
* $LastChangedBy : orbiter $
*
* This library is free software ; you can redistribute it and / or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation ; either
* version 2 . 1 of the License , or ( at your option ) any later version .
2011-06-30 17:49:21 +02:00
*
2011-04-14 22:12:14 +02:00
* This library is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the GNU
* Lesser General Public License for more details .
2011-06-30 17:49:21 +02:00
*
2011-04-14 22:12:14 +02:00
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21 . txt
* If not , see < http : //www.gnu.org/licenses/>.
* /
2012-05-09 16:46:45 +02:00
package net.yacy.search.index ;
2011-04-14 22:05:04 +02:00
2011-06-30 17:49:21 +02:00
import java.io.File ;
2012-05-14 14:56:21 +02:00
import java.io.IOException ;
2012-05-15 23:10:47 +02:00
import java.io.Serializable ;
2011-04-21 15:58:49 +02:00
import java.net.InetAddress ;
2011-09-13 16:39:41 +02:00
import java.net.MalformedURLException ;
2012-05-15 23:10:47 +02:00
import java.util.ArrayList ;
import java.util.Collection ;
import java.util.Date ;
2012-09-07 22:06:51 +02:00
import java.util.HashMap ;
2012-05-15 23:10:47 +02:00
import java.util.Iterator ;
import java.util.List ;
import java.util.Map ;
import java.util.Properties ;
import java.util.Set ;
2011-04-21 15:58:49 +02:00
2012-05-14 14:56:21 +02:00
import net.yacy.cora.document.ASCII ;
2011-06-30 17:49:21 +02:00
import net.yacy.cora.document.MultiProtocolURI ;
2012-08-05 15:49:27 +02:00
import net.yacy.cora.document.UTF8 ;
2012-11-23 14:00:30 +01:00
import net.yacy.cora.federate.solr.FailType ;
2012-09-25 21:20:03 +02:00
import net.yacy.cora.federate.solr.YaCySchema ;
import net.yacy.cora.federate.yacy.ConfigurationSet ;
2012-08-28 16:58:06 +02:00
import net.yacy.cora.protocol.Domains ;
2011-05-27 14:35:08 +02:00
import net.yacy.cora.protocol.HeaderFramework ;
2011-04-21 15:58:49 +02:00
import net.yacy.cora.protocol.ResponseHeader ;
2012-09-21 15:48:16 +02:00
import net.yacy.crawler.data.CrawlProfile ;
import net.yacy.crawler.retrieval.Response ;
2012-08-05 15:49:27 +02:00
import net.yacy.document.Condenser ;
2011-04-14 22:05:04 +02:00
import net.yacy.document.Document ;
2011-04-21 15:58:49 +02:00
import net.yacy.document.parser.html.ContentScraper ;
import net.yacy.document.parser.html.ImageEntry ;
2011-04-14 22:05:04 +02:00
import net.yacy.kelondro.data.meta.DigestURI ;
2012-09-11 20:15:54 +02:00
import net.yacy.kelondro.data.meta.URIMetadataRow ;
2012-01-13 11:25:15 +01:00
import net.yacy.kelondro.logging.Log ;
2012-09-21 16:46:57 +02:00
import net.yacy.kelondro.util.Bitfield ;
2011-06-30 17:49:21 +02:00
2012-08-18 13:05:27 +02:00
import org.apache.solr.common.SolrInputDocument ;
2011-04-14 22:05:04 +02:00
2012-08-05 15:49:27 +02:00
2012-05-15 23:10:47 +02:00
public class SolrConfiguration extends ConfigurationSet implements Serializable {
private static final long serialVersionUID = - 499100932212840385L ;
2011-06-30 17:49:21 +02:00
2012-06-27 12:17:58 +02:00
private boolean lazy ;
2011-06-30 17:49:21 +02:00
/ * *
* initialize with an empty ConfigurationSet which will cause that all the index
* attributes are used
* /
2012-05-09 17:00:56 +02:00
public SolrConfiguration ( ) {
2011-06-30 17:49:21 +02:00
super ( ) ;
2012-06-27 12:17:58 +02:00
this . lazy = false ;
2011-06-30 17:49:21 +02:00
}
/ * *
* initialize the scheme with a given configuration file
* the configuration file simply contains a list of lines with keywords
2012-05-15 22:34:02 +02:00
* or keyword = value lines ( while value is a custom Solr field name
2011-06-30 17:49:21 +02:00
* @param configurationFile
* /
2012-06-27 12:17:58 +02:00
public SolrConfiguration ( final File configurationFile , boolean lazy ) {
2011-06-30 17:49:21 +02:00
super ( configurationFile ) ;
2012-05-09 16:46:45 +02:00
// check consistency: compare with YaCyField enum
2012-05-15 22:34:02 +02:00
if ( this . isEmpty ( ) ) return ;
Iterator < Entry > it = this . entryIterator ( ) ;
for ( ConfigurationSet . Entry etr = it . next ( ) ; it . hasNext ( ) ; etr = it . next ( ) ) {
2012-01-13 11:25:15 +01:00
try {
2012-08-10 06:47:13 +02:00
YaCySchema f = YaCySchema . valueOf ( etr . key ( ) ) ;
2012-05-15 22:34:02 +02:00
f . setSolrFieldName ( etr . getValue ( ) ) ;
2012-01-13 11:25:15 +01:00
} catch ( IllegalArgumentException e ) {
2012-08-28 16:58:06 +02:00
Log . logFine ( " SolrScheme " , " solr scheme file " + configurationFile . getAbsolutePath ( ) + " defines unknown attribute ' " + etr . toString ( ) + " ' " ) ;
2012-05-15 22:34:02 +02:00
it . remove ( ) ;
2012-01-13 11:25:15 +01:00
}
}
2012-08-05 15:49:27 +02:00
// check consistency the other way: look if all enum constants in SolrField appear in the configuration file
2012-08-10 06:47:13 +02:00
for ( YaCySchema field : YaCySchema . values ( ) ) {
2012-08-05 15:49:27 +02:00
if ( this . get ( field . name ( ) ) = = null ) {
Log . logWarning ( " SolrScheme " , " solr scheme file " + configurationFile . getAbsolutePath ( ) + " is missing declaration for ' " + field . name ( ) + " ' " ) ;
}
}
2012-06-27 12:17:58 +02:00
this . lazy = lazy ;
2012-06-25 18:17:31 +02:00
}
2011-06-30 17:49:21 +02:00
2012-09-03 15:26:08 +02:00
public boolean contains ( YaCySchema field ) {
2012-08-05 15:49:27 +02:00
return this . contains ( field . name ( ) ) ;
}
2012-08-10 13:26:51 +02:00
2012-10-18 15:09:04 +02:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final String value ) {
2012-08-31 10:30:43 +02:00
assert ! key . isMultiValued ( ) ;
2012-08-23 09:51:45 +02:00
if ( ( isEmpty ( ) | | contains ( key ) ) & & ( ! this . lazy | | ( value ! = null & & ! value . isEmpty ( ) ) ) ) key . add ( doc , value ) ;
2011-06-30 17:49:21 +02:00
}
2012-10-18 15:09:04 +02:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final Date value ) {
2012-08-31 10:30:43 +02:00
assert ! key . isMultiValued ( ) ;
2012-08-23 09:51:45 +02:00
if ( ( isEmpty ( ) | | contains ( key ) ) & & ( ! this . lazy | | ( value ! = null & & value . getTime ( ) > 0 ) ) ) key . add ( doc , value ) ;
2011-06-30 17:49:21 +02:00
}
2011-04-14 22:05:04 +02:00
2012-10-18 15:09:04 +02:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final String [ ] value ) {
2012-08-31 10:30:43 +02:00
assert key . isMultiValued ( ) ;
2012-08-23 09:51:45 +02:00
if ( ( isEmpty ( ) | | contains ( key ) ) & & ( ! this . lazy | | ( value ! = null & & value . length > 0 ) ) ) key . add ( doc , value ) ;
2012-06-27 12:17:58 +02:00
}
2012-07-25 01:53:47 +02:00
2012-10-18 15:09:04 +02:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final Integer [ ] value ) {
2012-08-31 10:30:43 +02:00
assert key . isMultiValued ( ) ;
if ( ( isEmpty ( ) | | contains ( key ) ) & & ( ! this . lazy | | ( value ! = null & & value . length > 0 ) ) ) key . add ( doc , value ) ;
}
2012-10-18 15:09:04 +02:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final List < ? > values ) {
2012-08-31 10:30:43 +02:00
assert key . isMultiValued ( ) ;
if ( ( isEmpty ( ) | | contains ( key ) ) & & ( ! this . lazy | | ( values ! = null & & ! values . isEmpty ( ) ) ) ) key . add ( doc , values ) ;
2012-07-04 21:15:38 +02:00
}
2012-06-27 12:17:58 +02:00
2012-10-18 15:09:04 +02:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final int value ) {
2012-08-31 10:30:43 +02:00
assert ! key . isMultiValued ( ) ;
2012-08-23 09:51:45 +02:00
if ( ( isEmpty ( ) | | contains ( key ) ) & & ( ! this . lazy | | value ! = 0 ) ) key . add ( doc , value ) ;
2012-08-05 15:49:27 +02:00
}
2012-11-21 18:46:49 +01:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final long value ) {
assert ! key . isMultiValued ( ) ;
if ( ( isEmpty ( ) | | contains ( key ) ) & & ( ! this . lazy | | value ! = 0 ) ) key . add ( doc , value ) ;
}
2012-10-18 15:09:04 +02:00
private void add ( final SolrInputDocument doc , final YaCySchema key , final boolean value ) {
2012-08-31 10:30:43 +02:00
assert ! key . isMultiValued ( ) ;
2012-08-23 09:51:45 +02:00
if ( isEmpty ( ) | | contains ( key ) ) key . add ( doc , value ) ;
2011-06-30 17:49:21 +02:00
}
2012-08-10 13:26:51 +02:00
2012-10-18 15:09:04 +02:00
protected static Date getDate ( SolrInputDocument doc , final YaCySchema key ) {
2012-11-21 18:46:49 +01:00
Date x = ( Date ) doc . getFieldValue ( key . getSolrFieldName ( ) ) ;
2012-09-26 18:36:32 +02:00
Date now = new Date ( ) ;
return ( x = = null ) ? new Date ( 0 ) : x . after ( now ) ? now : x ;
2012-08-18 13:05:27 +02:00
}
2012-08-27 14:41:33 +02:00
2012-05-15 22:34:02 +02:00
/ * *
* save configuration to file and update enum SolrFields
* @throws IOException
* /
@Override
public void commit ( ) throws IOException {
try {
super . commit ( ) ;
// make sure the enum SolrField.SolrFieldName is current
Iterator < Entry > it = this . entryIterator ( ) ;
for ( ConfigurationSet . Entry etr = it . next ( ) ; it . hasNext ( ) ; etr = it . next ( ) ) {
try {
2012-08-10 06:47:13 +02:00
YaCySchema f = YaCySchema . valueOf ( etr . key ( ) ) ;
2012-05-15 22:34:02 +02:00
f . setSolrFieldName ( etr . getValue ( ) ) ;
} catch ( IllegalArgumentException e ) {
continue ;
}
}
} catch ( final IOException e ) { }
}
2012-08-10 13:26:51 +02:00
2012-10-18 15:09:04 +02:00
protected SolrInputDocument metadata2solr ( final URIMetadataRow md ) {
2012-08-27 14:41:33 +02:00
2012-08-23 09:51:45 +02:00
final SolrInputDocument doc = new SolrInputDocument ( ) ;
2012-10-15 13:17:13 +02:00
final DigestURI digestURI = DigestURI . toDigestURI ( md . url ( ) ) ;
2012-08-05 15:49:27 +02:00
boolean allAttr = this . isEmpty ( ) ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . failreason_t ) ) add ( doc , YaCySchema . failreason_t , " " ) ;
add ( doc , YaCySchema . id , ASCII . String ( md . hash ( ) ) ) ;
2012-10-10 11:46:22 +02:00
String us = digestURI . toNormalform ( true ) ;
2012-08-29 16:11:23 +02:00
add ( doc , YaCySchema . sku , us ) ;
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . ip_s ) ) {
2012-08-05 15:49:27 +02:00
final InetAddress address = digestURI . getInetAddress ( ) ;
2012-08-23 09:51:45 +02:00
if ( address ! = null ) add ( doc , YaCySchema . ip_s , address . getHostAddress ( ) ) ;
2012-08-05 15:49:27 +02:00
}
2012-08-29 16:11:23 +02:00
if ( allAttr | | contains ( YaCySchema . url_protocol_s ) ) add ( doc , YaCySchema . url_protocol_s , digestURI . getProtocol ( ) ) ;
Map < String , String > searchpart = digestURI . getSearchpartMap ( ) ;
if ( searchpart = = null ) {
if ( allAttr | | contains ( YaCySchema . url_parameter_i ) ) add ( doc , YaCySchema . url_parameter_i , 0 ) ;
} else {
if ( allAttr | | contains ( YaCySchema . url_parameter_i ) ) add ( doc , YaCySchema . url_parameter_i , searchpart . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . url_parameter_key_sxt ) ) add ( doc , YaCySchema . url_parameter_key_sxt , searchpart . keySet ( ) . toArray ( new String [ searchpart . size ( ) ] ) ) ;
if ( allAttr | | contains ( YaCySchema . url_parameter_value_sxt ) ) add ( doc , YaCySchema . url_parameter_value_sxt , searchpart . values ( ) . toArray ( new String [ searchpart . size ( ) ] ) ) ;
}
if ( allAttr | | contains ( YaCySchema . url_chars_i ) ) add ( doc , YaCySchema . url_chars_i , us . length ( ) ) ;
2012-08-28 16:58:06 +02:00
String host = null ;
if ( ( host = digestURI . getHost ( ) ) ! = null ) {
String dnc = Domains . getDNC ( host ) ;
String subdomOrga = host . substring ( 0 , host . length ( ) - dnc . length ( ) - 1 ) ;
int p = subdomOrga . lastIndexOf ( '.' ) ;
String subdom = ( p < 0 ) ? " " : subdomOrga . substring ( 0 , p ) ;
String orga = ( p < 0 ) ? subdomOrga : subdomOrga . substring ( p + 1 ) ;
if ( allAttr | | contains ( YaCySchema . host_s ) ) add ( doc , YaCySchema . host_s , host ) ;
if ( allAttr | | contains ( YaCySchema . host_dnc_s ) ) add ( doc , YaCySchema . host_dnc_s , dnc ) ;
if ( allAttr | | contains ( YaCySchema . host_organization_s ) ) add ( doc , YaCySchema . host_organization_s , orga ) ;
if ( allAttr | | contains ( YaCySchema . host_organizationdnc_s ) ) add ( doc , YaCySchema . host_organizationdnc_s , orga + '.' + dnc ) ;
if ( allAttr | | contains ( YaCySchema . host_subdomain_s ) ) add ( doc , YaCySchema . host_subdomain_s , subdom ) ;
}
2012-08-31 10:30:43 +02:00
String title = md . dc_title ( ) ;
if ( allAttr | | contains ( YaCySchema . title ) ) add ( doc , YaCySchema . title , new String [ ] { title } ) ;
if ( allAttr | | contains ( YaCySchema . title_count_i ) ) add ( doc , YaCySchema . title_count_i , 1 ) ;
if ( allAttr | | contains ( YaCySchema . title_chars_val ) ) {
Integer [ ] cv = new Integer [ ] { new Integer ( title . length ( ) ) } ;
add ( doc , YaCySchema . title_chars_val , cv ) ;
}
if ( allAttr | | contains ( YaCySchema . title_words_val ) ) {
Integer [ ] cv = new Integer [ ] { new Integer ( title . split ( " " ) . length ) } ;
add ( doc , YaCySchema . title_words_val , cv ) ;
}
String description = md . snippet ( ) ; if ( description = = null ) description = " " ;
if ( allAttr | | contains ( YaCySchema . description ) ) add ( doc , YaCySchema . description , description ) ;
if ( allAttr | | contains ( YaCySchema . description_count_i ) ) add ( doc , YaCySchema . description_count_i , 1 ) ;
if ( allAttr | | contains ( YaCySchema . description_chars_val ) ) {
Integer [ ] cv = new Integer [ ] { new Integer ( description . length ( ) ) } ;
add ( doc , YaCySchema . description_chars_val , cv ) ;
}
if ( allAttr | | contains ( YaCySchema . description_words_val ) ) {
Integer [ ] cv = new Integer [ ] { new Integer ( description . split ( " " ) . length ) } ;
add ( doc , YaCySchema . description_words_val , cv ) ;
}
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . author ) ) add ( doc , YaCySchema . author , md . dc_creator ( ) ) ;
if ( allAttr | | contains ( YaCySchema . content_type ) ) add ( doc , YaCySchema . content_type , Response . doctype2mime ( digestURI . getFileExtension ( ) , md . doctype ( ) ) ) ;
if ( allAttr | | contains ( YaCySchema . last_modified ) ) add ( doc , YaCySchema . last_modified , md . moddate ( ) ) ;
if ( allAttr | | contains ( YaCySchema . wordcount_i ) ) add ( doc , YaCySchema . wordcount_i , md . wordCount ( ) ) ;
2012-08-27 14:41:33 +02:00
2012-08-18 19:36:21 +02:00
String keywords = md . dc_subject ( ) ;
Bitfield flags = md . flags ( ) ;
if ( flags . get ( Condenser . flag_cat_indexof ) ) {
if ( keywords = = null | | keywords . isEmpty ( ) ) keywords = " indexof " ; else {
if ( keywords . indexOf ( ',' ) > 0 ) keywords + = " , indexof " ; else keywords + = " indexof " ;
}
}
2012-08-27 14:41:33 +02:00
if ( allAttr | | contains ( YaCySchema . keywords ) ) {
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . keywords , keywords ) ;
2012-08-05 15:49:27 +02:00
}
2012-08-10 13:26:51 +02:00
2012-08-05 15:49:27 +02:00
// path elements of link
2012-09-11 22:46:39 +02:00
if ( allAttr | | contains ( YaCySchema . url_paths_sxt ) ) add ( doc , YaCySchema . url_paths_sxt , digestURI . getPaths ( ) ) ;
if ( allAttr | | contains ( YaCySchema . url_file_ext_s ) ) add ( doc , YaCySchema . url_file_ext_s , digestURI . getFileExtension ( ) ) ;
2012-08-05 15:49:27 +02:00
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . imagescount_i ) ) add ( doc , YaCySchema . imagescount_i , md . limage ( ) ) ;
if ( allAttr | | contains ( YaCySchema . inboundlinkscount_i ) ) add ( doc , YaCySchema . inboundlinkscount_i , md . llocal ( ) ) ;
if ( allAttr | | contains ( YaCySchema . outboundlinkscount_i ) ) add ( doc , YaCySchema . outboundlinkscount_i , md . lother ( ) ) ;
if ( allAttr | | contains ( YaCySchema . charset_s ) ) add ( doc , YaCySchema . charset_s , " UTF8 " ) ;
2012-08-05 15:49:27 +02:00
// coordinates
if ( md . lat ( ) ! = 0 . 0f & & md . lon ( ) ! = 0 . 0f ) {
2012-08-27 14:41:33 +02:00
if ( allAttr | | contains ( YaCySchema . coordinate_p ) ) add ( doc , YaCySchema . coordinate_p , Double . toString ( md . lat ( ) ) + " , " + Double . toString ( md . lon ( ) ) ) ;
2012-08-05 15:49:27 +02:00
}
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . httpstatus_i ) ) add ( doc , YaCySchema . httpstatus_i , 200 ) ;
2012-08-05 15:49:27 +02:00
// fields that are in URIMetadataRow additional to yacy2solr basic requirement
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . load_date_dt ) ) add ( doc , YaCySchema . load_date_dt , md . loaddate ( ) ) ;
if ( allAttr | | contains ( YaCySchema . fresh_date_dt ) ) add ( doc , YaCySchema . fresh_date_dt , md . freshdate ( ) ) ;
if ( allAttr | | contains ( YaCySchema . host_id_s ) ) add ( doc , YaCySchema . host_id_s , md . hosthash ( ) ) ;
if ( ( allAttr | | contains ( YaCySchema . referrer_id_txt ) ) & & md . referrerHash ( ) ! = null ) add ( doc , YaCySchema . referrer_id_txt , new String [ ] { ASCII . String ( md . referrerHash ( ) ) } ) ;
if ( allAttr | | contains ( YaCySchema . md5_s ) ) add ( doc , YaCySchema . md5_s , md . md5 ( ) ) ;
if ( allAttr | | contains ( YaCySchema . publisher_t ) ) add ( doc , YaCySchema . publisher_t , md . dc_publisher ( ) ) ;
2012-08-28 16:58:06 +02:00
if ( ( allAttr | | contains ( YaCySchema . language_s ) ) & & md . language ( ) ! = null ) add ( doc , YaCySchema . language_s , UTF8 . String ( md . language ( ) ) ) ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . size_i ) ) add ( doc , YaCySchema . size_i , md . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . audiolinkscount_i ) ) add ( doc , YaCySchema . audiolinkscount_i , md . laudio ( ) ) ;
if ( allAttr | | contains ( YaCySchema . videolinkscount_i ) ) add ( doc , YaCySchema . videolinkscount_i , md . lvideo ( ) ) ;
if ( allAttr | | contains ( YaCySchema . applinkscount_i ) ) add ( doc , YaCySchema . applinkscount_i , md . lapp ( ) ) ;
2012-08-18 19:36:21 +02:00
if ( allAttr | | contains ( YaCySchema . text_t ) ) {
// construct the text from other metadata parts.
// This is necessary here since that is used to search the link when no other data (parsed text body) is available
StringBuilder sb = new StringBuilder ( 120 ) ;
accText ( sb , md . dc_title ( ) ) ;
accText ( sb , md . dc_creator ( ) ) ;
accText ( sb , md . dc_publisher ( ) ) ;
accText ( sb , md . snippet ( ) ) ;
accText ( sb , digestURI . toTokens ( ) ) ;
accText ( sb , keywords ) ;
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . text_t , sb . toString ( ) ) ;
2012-08-18 19:36:21 +02:00
}
2012-08-27 14:41:33 +02:00
2012-08-23 09:51:45 +02:00
return doc ;
2012-08-05 15:49:27 +02:00
}
2012-08-27 14:41:33 +02:00
2012-08-18 19:36:21 +02:00
private static void accText ( final StringBuilder sb , String text ) {
if ( text = = null | | text . length ( ) = = 0 ) return ;
if ( sb . length ( ) ! = 0 ) sb . append ( ' ' ) ;
text = text . trim ( ) ;
2012-08-21 21:03:26 +02:00
if ( ! text . isEmpty ( ) & & text . charAt ( text . length ( ) - 1 ) = = '.' ) sb . append ( text ) ; else sb . append ( text ) . append ( '.' ) ;
2012-08-18 19:36:21 +02:00
}
2012-05-15 22:34:02 +02:00
2012-10-18 15:09:04 +02:00
protected SolrInputDocument yacy2solr ( final String id , final CrawlProfile profile , final ResponseHeader responseHeader , final Document document , Condenser condenser , DigestURI referrerURL , String language ) {
2012-05-15 22:34:02 +02:00
// we use the SolrCell design as index scheme
2012-08-23 09:51:45 +02:00
final SolrInputDocument doc = new SolrInputDocument ( ) ;
2012-10-18 14:29:11 +02:00
final DigestURI digestURI = DigestURI . toDigestURI ( document . dc_source ( ) ) ;
2012-08-05 15:49:27 +02:00
boolean allAttr = this . isEmpty ( ) ;
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . id , id ) ;
if ( allAttr | | contains ( YaCySchema . failreason_t ) ) add ( doc , YaCySchema . failreason_t , " " ) ; // overwrite a possible fail reason (in case that there was a fail reason before)
2012-10-10 11:46:22 +02:00
String us = digestURI . toNormalform ( true ) ;
2012-08-29 16:11:23 +02:00
add ( doc , YaCySchema . sku , us ) ;
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . ip_s ) ) {
2012-08-29 16:11:23 +02:00
final InetAddress address = digestURI . getInetAddress ( ) ;
if ( address ! = null ) add ( doc , YaCySchema . ip_s , address . getHostAddress ( ) ) ;
}
2012-09-03 15:26:08 +02:00
if ( allAttr | | contains ( YaCySchema . collection_sxt ) & & profile ! = null ) add ( doc , YaCySchema . collection_sxt , profile . collections ( ) ) ;
2012-08-29 16:11:23 +02:00
if ( allAttr | | contains ( YaCySchema . url_protocol_s ) ) add ( doc , YaCySchema . url_protocol_s , digestURI . getProtocol ( ) ) ;
Map < String , String > searchpart = digestURI . getSearchpartMap ( ) ;
if ( searchpart = = null ) {
if ( allAttr | | contains ( YaCySchema . url_parameter_i ) ) add ( doc , YaCySchema . url_parameter_i , 0 ) ;
} else {
if ( allAttr | | contains ( YaCySchema . url_parameter_i ) ) add ( doc , YaCySchema . url_parameter_i , searchpart . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . url_parameter_key_sxt ) ) add ( doc , YaCySchema . url_parameter_key_sxt , searchpart . keySet ( ) . toArray ( new String [ searchpart . size ( ) ] ) ) ;
if ( allAttr | | contains ( YaCySchema . url_parameter_value_sxt ) ) add ( doc , YaCySchema . url_parameter_value_sxt , searchpart . values ( ) . toArray ( new String [ searchpart . size ( ) ] ) ) ;
2012-08-05 15:49:27 +02:00
}
2012-08-29 16:11:23 +02:00
if ( allAttr | | contains ( YaCySchema . url_chars_i ) ) add ( doc , YaCySchema . url_chars_i , us . length ( ) ) ;
2012-08-28 16:58:06 +02:00
String host = null ;
if ( ( host = digestURI . getHost ( ) ) ! = null ) {
String dnc = Domains . getDNC ( host ) ;
String subdomOrga = host . substring ( 0 , host . length ( ) - dnc . length ( ) - 1 ) ;
int p = subdomOrga . lastIndexOf ( '.' ) ;
String subdom = ( p < 0 ) ? " " : subdomOrga . substring ( 0 , p ) ;
String orga = ( p < 0 ) ? subdomOrga : subdomOrga . substring ( p + 1 ) ;
if ( allAttr | | contains ( YaCySchema . host_s ) ) add ( doc , YaCySchema . host_s , host ) ;
if ( allAttr | | contains ( YaCySchema . host_dnc_s ) ) add ( doc , YaCySchema . host_dnc_s , dnc ) ;
if ( allAttr | | contains ( YaCySchema . host_organization_s ) ) add ( doc , YaCySchema . host_organization_s , orga ) ;
if ( allAttr | | contains ( YaCySchema . host_organizationdnc_s ) ) add ( doc , YaCySchema . host_organizationdnc_s , orga + '.' + dnc ) ;
if ( allAttr | | contains ( YaCySchema . host_subdomain_s ) ) add ( doc , YaCySchema . host_subdomain_s , subdom ) ;
}
2012-08-31 10:30:43 +02:00
2012-10-18 14:29:11 +02:00
List < String > titles = document . titles ( ) ;
2012-08-31 10:30:43 +02:00
if ( allAttr | | contains ( YaCySchema . title ) ) add ( doc , YaCySchema . title , titles ) ;
if ( allAttr | | contains ( YaCySchema . title_count_i ) ) add ( doc , YaCySchema . title_count_i , titles . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . title_chars_val ) ) {
ArrayList < Integer > cv = new ArrayList < Integer > ( titles . size ( ) ) ;
for ( String s : titles ) cv . add ( new Integer ( s . length ( ) ) ) ;
add ( doc , YaCySchema . title_chars_val , cv ) ;
}
if ( allAttr | | contains ( YaCySchema . title_words_val ) ) {
ArrayList < Integer > cv = new ArrayList < Integer > ( titles . size ( ) ) ;
for ( String s : titles ) cv . add ( new Integer ( s . split ( " " ) . length ) ) ;
add ( doc , YaCySchema . title_words_val , cv ) ;
}
2012-10-18 14:29:11 +02:00
String description = document . dc_description ( ) ;
2012-08-31 10:30:43 +02:00
List < String > descriptions = new ArrayList < String > ( ) ;
for ( String s : description . split ( " \ n " ) ) descriptions . add ( s ) ;
if ( allAttr | | contains ( YaCySchema . description ) ) add ( doc , YaCySchema . description , description ) ;
if ( allAttr | | contains ( YaCySchema . description_count_i ) ) add ( doc , YaCySchema . description_count_i , descriptions . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . description_chars_val ) ) {
ArrayList < Integer > cv = new ArrayList < Integer > ( descriptions . size ( ) ) ;
for ( String s : descriptions ) cv . add ( new Integer ( s . length ( ) ) ) ;
add ( doc , YaCySchema . description_chars_val , cv ) ;
}
if ( allAttr | | contains ( YaCySchema . description_words_val ) ) {
ArrayList < Integer > cv = new ArrayList < Integer > ( descriptions . size ( ) ) ;
for ( String s : descriptions ) cv . add ( new Integer ( s . split ( " " ) . length ) ) ;
add ( doc , YaCySchema . description_words_val , cv ) ;
}
2012-10-18 14:29:11 +02:00
if ( allAttr | | contains ( YaCySchema . author ) ) add ( doc , YaCySchema . author , document . dc_creator ( ) ) ;
if ( allAttr | | contains ( YaCySchema . content_type ) ) add ( doc , YaCySchema . content_type , new String [ ] { document . dc_format ( ) } ) ;
if ( allAttr | | contains ( YaCySchema . last_modified ) ) add ( doc , YaCySchema . last_modified , responseHeader = = null ? new Date ( ) : responseHeader . lastModified ( ) ) ;
if ( allAttr | | contains ( YaCySchema . keywords ) ) add ( doc , YaCySchema . keywords , document . dc_subject ( ' ' ) ) ;
2012-11-07 17:27:13 +01:00
String content = document . getTextString ( ) ;
if ( content = = null | | content . length ( ) = = 0 ) {
content = digestURI . toTokens ( ) ;
}
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . text_t ) ) add ( doc , YaCySchema . text_t , content ) ;
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . wordcount_i ) ) {
2012-11-24 22:30:05 +01:00
if ( content . length ( ) = = 0 ) {
add ( doc , YaCySchema . wordcount_i , 0 ) ;
} else {
int contentwc = 1 ;
for ( int i = content . length ( ) - 1 ; i > = 0 ; i - - ) if ( content . charAt ( i ) = = ' ' ) contentwc + + ;
add ( doc , YaCySchema . wordcount_i , contentwc ) ;
}
2011-06-30 17:49:21 +02:00
}
2012-10-02 11:13:06 +02:00
if ( allAttr | | contains ( YaCySchema . synonyms_sxt ) ) {
List < String > synonyms = condenser . synonyms ( ) ;
add ( doc , YaCySchema . synonyms_sxt , synonyms ) ;
2012-10-02 00:02:50 +02:00
}
2012-11-21 18:46:49 +01:00
add ( doc , YaCySchema . exact_signature_l , condenser . exactSignature ( ) ) ;
add ( doc , YaCySchema . exact_signature_unique_b , true ) ; // this must be corrected afterwards!
add ( doc , YaCySchema . fuzzy_signature_l , condenser . fuzzySignature ( ) ) ;
add ( doc , YaCySchema . fuzzy_signature_text_t , condenser . fuzzySignatureText ( ) ) ;
add ( doc , YaCySchema . fuzzy_signature_unique_b , true ) ; // this must be corrected afterwards!
2011-04-21 15:58:49 +02:00
// path elements of link
2012-09-11 22:46:39 +02:00
if ( allAttr | | contains ( YaCySchema . url_paths_sxt ) ) add ( doc , YaCySchema . url_paths_sxt , digestURI . getPaths ( ) ) ;
if ( allAttr | | contains ( YaCySchema . url_file_ext_s ) ) add ( doc , YaCySchema . url_file_ext_s , digestURI . getFileExtension ( ) ) ;
2011-06-30 17:49:21 +02:00
2012-04-27 16:48:51 +02:00
// get list of all links; they will be shrinked by urls that appear in other fields of the solr scheme
2012-10-18 14:29:11 +02:00
Set < MultiProtocolURI > inboundLinks = document . inboundLinks ( ) ;
Set < MultiProtocolURI > outboundLinks = document . outboundLinks ( ) ;
2011-08-25 17:52:25 +02:00
2012-04-27 16:48:51 +02:00
int c = 0 ;
2012-10-18 14:29:11 +02:00
final Object parser = document . getParserObject ( ) ;
2012-09-07 22:06:51 +02:00
Map < MultiProtocolURI , ImageEntry > images = new HashMap < MultiProtocolURI , ImageEntry > ( ) ;
2011-04-21 15:58:49 +02:00
if ( parser instanceof ContentScraper ) {
2011-06-30 17:49:21 +02:00
final ContentScraper html = ( ContentScraper ) parser ;
2012-09-07 22:06:51 +02:00
images = html . getImages ( ) ;
2011-06-30 17:49:21 +02:00
2011-04-21 15:58:49 +02:00
// header tags
int h = 0 ;
int f = 1 ;
2012-01-13 11:25:15 +01:00
String [ ] hs ;
2012-09-04 14:11:11 +02:00
hs = html . getHeadlines ( 1 ) ; h = h | ( hs . length > 0 ? f : 0 ) ; f = f * 2 ; add ( doc , YaCySchema . h1_txt , hs ) ; add ( doc , YaCySchema . h1_i , hs . length ) ;
hs = html . getHeadlines ( 2 ) ; h = h | ( hs . length > 0 ? f : 0 ) ; f = f * 2 ; add ( doc , YaCySchema . h2_txt , hs ) ; add ( doc , YaCySchema . h2_i , hs . length ) ;
hs = html . getHeadlines ( 3 ) ; h = h | ( hs . length > 0 ? f : 0 ) ; f = f * 2 ; add ( doc , YaCySchema . h3_txt , hs ) ; add ( doc , YaCySchema . h3_i , hs . length ) ;
hs = html . getHeadlines ( 4 ) ; h = h | ( hs . length > 0 ? f : 0 ) ; f = f * 2 ; add ( doc , YaCySchema . h4_txt , hs ) ; add ( doc , YaCySchema . h4_i , hs . length ) ;
hs = html . getHeadlines ( 5 ) ; h = h | ( hs . length > 0 ? f : 0 ) ; f = f * 2 ; add ( doc , YaCySchema . h5_txt , hs ) ; add ( doc , YaCySchema . h5_i , hs . length ) ;
hs = html . getHeadlines ( 6 ) ; h = h | ( hs . length > 0 ? f : 0 ) ; f = f * 2 ; add ( doc , YaCySchema . h6_txt , hs ) ; add ( doc , YaCySchema . h6_i , hs . length ) ;
2012-10-09 13:02:43 +02:00
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . htags_i , h ) ;
2012-10-09 13:02:43 +02:00
add ( doc , YaCySchema . schema_org_breadcrumb_i , html . breadcrumbCount ( ) ) ;
2011-04-21 15:58:49 +02:00
2012-10-09 17:28:48 +02:00
// meta tags: Open Graph properties
String og ;
og = html . getMetas ( ) . get ( " og:title " ) ; if ( og ! = null ) add ( doc , YaCySchema . opengraph_title_t , og ) ;
og = html . getMetas ( ) . get ( " og:type " ) ; if ( og ! = null ) add ( doc , YaCySchema . opengraph_type_s , og ) ;
og = html . getMetas ( ) . get ( " og:url " ) ; if ( og ! = null ) add ( doc , YaCySchema . opengraph_url_s , og ) ;
og = html . getMetas ( ) . get ( " og:image " ) ; if ( og ! = null ) add ( doc , YaCySchema . opengraph_image_s , og ) ;
2011-09-30 15:39:01 +02:00
// noindex and nofollow attributes
// from HTML (meta-tag in HTML header: robots)
// and HTTP header (x-robots property)
// coded as binary value:
// bit 0: "all" contained in html header meta
// bit 1: "index" contained in html header meta
// bit 2: "noindex" contained in html header meta
// bit 3: "nofollow" contained in html header meta
// bit 8: "noarchive" contained in http header properties
// bit 9: "nosnippet" contained in http header properties
// bit 10: "noindex" contained in http header properties
// bit 11: "nofollow" contained in http header properties
// bit 12: "unavailable_after" contained in http header properties
int b = 0 ;
final String robots_meta = html . getMetas ( ) . get ( " robots " ) ;
// this tag may have values: all, index, noindex, nofollow
if ( robots_meta ! = null ) {
2011-11-25 12:23:52 +01:00
if ( robots_meta . indexOf ( " all " , 0 ) > = 0 ) b + = 1 ; // set bit 0
if ( robots_meta . indexOf ( " index " , 0 ) = = 0 | | robots_meta . indexOf ( " index " , 0 ) > = 0 | | robots_meta . indexOf ( " ,index " , 0 ) > = 0 ) b + = 2 ; // set bit 1
if ( robots_meta . indexOf ( " noindex " , 0 ) > = 0 ) b + = 4 ; // set bit 2
if ( robots_meta . indexOf ( " nofollow " , 0 ) > = 0 ) b + = 8 ; // set bit 3
2011-09-30 15:39:01 +02:00
}
2012-07-25 01:53:47 +02:00
String x_robots_tag = " " ;
2012-10-18 14:29:11 +02:00
if ( responseHeader ! = null ) {
x_robots_tag = responseHeader . get ( HeaderFramework . X_ROBOTS_TAG , " " ) ;
2012-07-25 01:53:47 +02:00
if ( x_robots_tag . isEmpty ( ) ) {
2012-10-18 14:29:11 +02:00
x_robots_tag = responseHeader . get ( HeaderFramework . X_ROBOTS , " " ) ;
2012-07-25 01:53:47 +02:00
}
}
if ( ! x_robots_tag . isEmpty ( ) ) {
2012-07-10 22:59:03 +02:00
// this tag may have values: noarchive, nosnippet, noindex, unavailable_after
2011-11-25 12:23:52 +01:00
if ( x_robots_tag . indexOf ( " noarchive " , 0 ) > = 0 ) b + = 256 ; // set bit 8
if ( x_robots_tag . indexOf ( " nosnippet " , 0 ) > = 0 ) b + = 512 ; // set bit 9
if ( x_robots_tag . indexOf ( " noindex " , 0 ) > = 0 ) b + = 1024 ; // set bit 10
if ( x_robots_tag . indexOf ( " nofollow " , 0 ) > = 0 ) b + = 2048 ; // set bit 11
if ( x_robots_tag . indexOf ( " unavailable_after " , 0 ) > = 0 ) b + = 4096 ; // set bit 12
2011-09-30 15:39:01 +02:00
}
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . robots_i , b ) ;
2011-09-30 15:39:01 +02:00
// meta tags: generator
final String generator = html . getMetas ( ) . get ( " generator " ) ;
2012-08-23 09:51:45 +02:00
if ( generator ! = null ) add ( doc , YaCySchema . metagenerator_t , generator ) ;
2011-06-30 17:49:21 +02:00
2011-04-21 15:58:49 +02:00
// bold, italic
2011-06-30 17:49:21 +02:00
final String [ ] bold = html . getBold ( ) ;
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . boldcount_i , bold . length ) ;
2011-04-28 15:09:01 +02:00
if ( bold . length > 0 ) {
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . bold_txt , bold ) ;
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . bold_val ) ) {
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . bold_val , html . getBoldCount ( bold ) ) ;
2011-06-30 17:49:21 +02:00
}
2011-04-28 15:09:01 +02:00
}
2011-06-30 17:49:21 +02:00
final String [ ] italic = html . getItalic ( ) ;
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . italiccount_i , italic . length ) ;
2011-04-28 15:09:01 +02:00
if ( italic . length > 0 ) {
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . italic_txt , italic ) ;
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . italic_val ) ) {
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . italic_val , html . getItalicCount ( italic ) ) ;
2011-06-30 17:49:21 +02:00
}
2011-04-28 15:09:01 +02:00
}
2012-10-01 14:16:49 +02:00
final String [ ] underline = html . getUnderline ( ) ;
add ( doc , YaCySchema . underlinecount_i , underline . length ) ;
if ( underline . length > 0 ) {
add ( doc , YaCySchema . underline_txt , underline ) ;
if ( allAttr | | contains ( YaCySchema . underline_val ) ) {
add ( doc , YaCySchema . underline_val , html . getUnderlineCount ( underline ) ) ;
}
}
2011-06-30 17:49:21 +02:00
final String [ ] li = html . getLi ( ) ;
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . licount_i , li . length ) ;
if ( li . length > 0 ) add ( doc , YaCySchema . li_txt , li ) ;
2011-06-30 17:49:21 +02:00
2011-04-21 15:58:49 +02:00
// images
2012-09-07 22:06:51 +02:00
final Collection < ImageEntry > imagesc = images . values ( ) ;
2012-07-04 21:15:38 +02:00
final List < String > imgtags = new ArrayList < String > ( imagesc . size ( ) ) ;
final List < String > imgprots = new ArrayList < String > ( imagesc . size ( ) ) ;
final List < String > imgstubs = new ArrayList < String > ( imagesc . size ( ) ) ;
final List < String > imgalts = new ArrayList < String > ( imagesc . size ( ) ) ;
2012-09-07 21:33:45 +02:00
int withalt = 0 ;
2011-08-31 18:02:06 +02:00
for ( final ImageEntry ie : imagesc ) {
final MultiProtocolURI uri = ie . url ( ) ;
2012-04-27 16:48:51 +02:00
inboundLinks . remove ( uri ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( uri ) ;
2012-07-04 21:15:38 +02:00
imgtags . add ( ie . toString ( ) ) ;
String protocol = uri . getProtocol ( ) ;
imgprots . add ( protocol ) ;
imgstubs . add ( uri . toString ( ) . substring ( protocol . length ( ) + 3 ) ) ;
imgalts . add ( ie . alt ( ) ) ;
2012-09-07 21:33:45 +02:00
if ( ie . alt ( ) ! = null & & ie . alt ( ) . length ( ) > 0 ) withalt + + ;
2011-06-30 17:49:21 +02:00
}
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . imagescount_i ) ) add ( doc , YaCySchema . imagescount_i , imgtags . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . images_tag_txt ) ) add ( doc , YaCySchema . images_tag_txt , imgtags ) ;
2012-08-28 16:58:06 +02:00
if ( allAttr | | contains ( YaCySchema . images_protocol_sxt ) ) add ( doc , YaCySchema . images_protocol_sxt , protocolList2indexedList ( imgprots ) ) ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . images_urlstub_txt ) ) add ( doc , YaCySchema . images_urlstub_txt , imgstubs ) ;
if ( allAttr | | contains ( YaCySchema . images_alt_txt ) ) add ( doc , YaCySchema . images_alt_txt , imgalts ) ;
2012-09-10 12:30:03 +02:00
if ( allAttr | | contains ( YaCySchema . images_withalt_i ) ) add ( doc , YaCySchema . images_withalt_i , withalt ) ;
2011-04-21 15:58:49 +02:00
// style sheets
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . css_tag_txt ) ) {
2011-06-30 17:49:21 +02:00
final Map < MultiProtocolURI , String > csss = html . getCSS ( ) ;
2011-08-31 18:02:06 +02:00
final String [ ] css_tag = new String [ csss . size ( ) ] ;
final String [ ] css_url = new String [ csss . size ( ) ] ;
2011-06-30 17:49:21 +02:00
c = 0 ;
for ( final Map . Entry < MultiProtocolURI , String > entry : csss . entrySet ( ) ) {
2012-10-10 11:46:22 +02:00
final String url = entry . getKey ( ) . toNormalform ( false ) ;
2012-04-27 16:48:51 +02:00
inboundLinks . remove ( url ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( url ) ;
2011-08-31 18:02:06 +02:00
css_tag [ c ] =
2011-06-30 17:49:21 +02:00
" <link rel= \" stylesheet \" type= \" text/css \" media= \" " + entry . getValue ( ) + " \" " +
2011-08-31 18:02:06 +02:00
" href= \" " + url + " \" /> " ;
css_url [ c ] = url ;
c + + ;
2011-06-30 17:49:21 +02:00
}
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . csscount_i , css_tag . length ) ;
if ( css_tag . length > 0 ) add ( doc , YaCySchema . css_tag_txt , css_tag ) ;
if ( css_url . length > 0 ) add ( doc , YaCySchema . css_url_txt , css_url ) ;
2011-04-21 15:58:49 +02:00
}
2011-06-30 17:49:21 +02:00
2011-04-21 15:58:49 +02:00
// Scripts
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . scripts_txt ) ) {
2011-06-30 17:49:21 +02:00
final Set < MultiProtocolURI > scriptss = html . getScript ( ) ;
final String [ ] scripts = new String [ scriptss . size ( ) ] ;
c = 0 ;
for ( final MultiProtocolURI url : scriptss ) {
2012-04-27 16:48:51 +02:00
inboundLinks . remove ( url ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( url ) ;
2012-10-10 11:46:22 +02:00
scripts [ c + + ] = url . toNormalform ( false ) ;
2011-06-30 17:49:21 +02:00
}
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . scriptscount_i , scripts . length ) ;
if ( scripts . length > 0 ) add ( doc , YaCySchema . scripts_txt , scripts ) ;
2011-04-21 15:58:49 +02:00
}
2011-06-30 17:49:21 +02:00
2011-04-21 15:58:49 +02:00
// Frames
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . frames_txt ) ) {
2011-06-30 17:49:21 +02:00
final Set < MultiProtocolURI > framess = html . getFrames ( ) ;
final String [ ] frames = new String [ framess . size ( ) ] ;
c = 0 ;
2012-04-27 16:48:51 +02:00
for ( final MultiProtocolURI url : framess ) {
inboundLinks . remove ( url ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( url ) ;
2012-10-10 11:46:22 +02:00
frames [ c + + ] = url . toNormalform ( false ) ;
2011-06-30 17:49:21 +02:00
}
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . framesscount_i , frames . length ) ;
if ( frames . length > 0 ) add ( doc , YaCySchema . frames_txt , frames ) ;
2011-04-21 15:58:49 +02:00
}
2011-06-30 17:49:21 +02:00
2011-04-21 15:58:49 +02:00
// IFrames
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . iframes_txt ) ) {
2011-06-30 17:49:21 +02:00
final Set < MultiProtocolURI > iframess = html . getIFrames ( ) ;
final String [ ] iframes = new String [ iframess . size ( ) ] ;
c = 0 ;
2012-04-27 16:48:51 +02:00
for ( final MultiProtocolURI url : iframess ) {
inboundLinks . remove ( url ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( url ) ;
2012-10-10 11:46:22 +02:00
iframes [ c + + ] = url . toNormalform ( false ) ;
2011-06-30 17:49:21 +02:00
}
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . iframesscount_i , iframes . length ) ;
if ( iframes . length > 0 ) add ( doc , YaCySchema . iframes_txt , iframes ) ;
2011-04-21 15:58:49 +02:00
}
2011-06-30 17:49:21 +02:00
2012-06-26 14:51:57 +02:00
// canonical tag
2012-08-28 16:58:06 +02:00
if ( allAttr | | contains ( YaCySchema . canonical_t ) ) {
2012-06-26 14:51:57 +02:00
final MultiProtocolURI canonical = html . getCanonical ( ) ;
if ( canonical ! = null ) {
inboundLinks . remove ( canonical ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( canonical ) ;
2012-10-10 11:46:22 +02:00
add ( doc , YaCySchema . canonical_t , canonical . toNormalform ( false ) ) ;
2012-06-26 14:51:57 +02:00
}
}
2012-06-28 13:27:45 +02:00
// meta refresh tag
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . refresh_s ) ) {
2012-06-28 13:27:45 +02:00
String refresh = html . getRefreshPath ( ) ;
if ( refresh ! = null & & refresh . length ( ) > 0 ) {
MultiProtocolURI refreshURL ;
try {
refreshURL = refresh . startsWith ( " http " ) ? new MultiProtocolURI ( html . getRefreshPath ( ) ) : new MultiProtocolURI ( digestURI , html . getRefreshPath ( ) ) ;
if ( refreshURL ! = null ) {
inboundLinks . remove ( refreshURL ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( refreshURL ) ;
2012-10-10 11:46:22 +02:00
add ( doc , YaCySchema . refresh_s , refreshURL . toNormalform ( false ) ) ;
2012-06-28 13:27:45 +02:00
}
} catch ( MalformedURLException e ) {
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . refresh_s , refresh ) ;
2012-06-28 13:27:45 +02:00
}
}
}
2011-04-21 15:58:49 +02:00
// flash embedded
2012-08-10 06:47:13 +02:00
if ( allAttr | | contains ( YaCySchema . flash_b ) ) {
2012-06-25 18:17:31 +02:00
MultiProtocolURI [ ] flashURLs = html . getFlash ( ) ;
for ( MultiProtocolURI u : flashURLs ) {
// remove all flash links from ibound/outbound links
inboundLinks . remove ( u ) ;
2012-09-07 22:06:51 +02:00
outboundLinks . remove ( u ) ;
2012-06-25 18:17:31 +02:00
}
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . flash_b , flashURLs . length > 0 ) ;
2012-06-25 18:17:31 +02:00
}
2011-06-30 17:49:21 +02:00
2011-04-28 15:09:01 +02:00
// generic evaluation pattern
2011-06-30 17:49:21 +02:00
for ( final String model : html . getEvaluationModelNames ( ) ) {
2012-08-05 15:49:27 +02:00
if ( allAttr | | contains ( " ext_ " + model + " _txt " ) ) {
2011-06-30 17:49:21 +02:00
final String [ ] scorenames = html . getEvaluationModelScoreNames ( model ) ;
if ( scorenames . length > 0 ) {
2012-08-23 09:51:45 +02:00
add ( doc , YaCySchema . valueOf ( " ext_ " + model + " _txt " ) , scorenames ) ;
add ( doc , YaCySchema . valueOf ( " ext_ " + model + " _val " ) , html . getEvaluationModelScoreCounts ( model , scorenames ) ) ;
2011-06-30 17:49:21 +02:00
}
2011-04-28 15:09:01 +02:00
}
}
2011-06-30 17:49:21 +02:00
2011-05-27 14:35:08 +02:00
// response time
2012-10-18 14:29:11 +02:00
add ( doc , YaCySchema . responsetime_i , responseHeader = = null ? 0 : Integer . parseInt ( responseHeader . get ( HeaderFramework . RESPONSE_TIME_MILLIS , " 0 " ) ) ) ;
2011-04-21 15:58:49 +02:00
}
2012-04-27 16:48:51 +02:00
// list all links
2012-10-18 14:29:11 +02:00
final Map < MultiProtocolURI , Properties > alllinks = document . getAnchors ( ) ;
2012-04-27 16:48:51 +02:00
c = 0 ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinkscount_i ) ) add ( doc , YaCySchema . inboundlinkscount_i , inboundLinks . size ( ) ) ;
2012-10-18 14:29:11 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinksnofollowcount_i ) ) add ( doc , YaCySchema . inboundlinksnofollowcount_i , document . inboundLinkNofollowCount ( ) ) ;
2012-07-04 21:15:38 +02:00
final List < String > inboundlinksTag = new ArrayList < String > ( inboundLinks . size ( ) ) ;
final List < String > inboundlinksURLProtocol = new ArrayList < String > ( inboundLinks . size ( ) ) ;
final List < String > inboundlinksURLStub = new ArrayList < String > ( inboundLinks . size ( ) ) ;
final List < String > inboundlinksName = new ArrayList < String > ( inboundLinks . size ( ) ) ;
final List < String > inboundlinksRel = new ArrayList < String > ( inboundLinks . size ( ) ) ;
final List < String > inboundlinksText = new ArrayList < String > ( inboundLinks . size ( ) ) ;
2012-09-07 22:06:51 +02:00
final List < Integer > inboundlinksTextChars = new ArrayList < Integer > ( inboundLinks . size ( ) ) ;
final List < Integer > inboundlinksTextWords = new ArrayList < Integer > ( inboundLinks . size ( ) ) ;
final List < String > inboundlinksAltTag = new ArrayList < String > ( inboundLinks . size ( ) ) ;
2012-04-27 16:48:51 +02:00
for ( final MultiProtocolURI url : inboundLinks ) {
final Properties p = alllinks . get ( url ) ;
2012-07-04 21:15:38 +02:00
if ( p = = null ) continue ;
2012-04-27 16:48:51 +02:00
final String name = p . getProperty ( " name " , " " ) ; // the name attribute
final String rel = p . getProperty ( " rel " , " " ) ; // the rel-attribute
final String text = p . getProperty ( " text " , " " ) ; // the text between the <a></a> tag
2012-10-10 11:46:22 +02:00
final String urls = url . toNormalform ( false ) ;
2012-04-27 16:48:51 +02:00
final int pr = urls . indexOf ( " :// " , 0 ) ;
2012-07-04 21:15:38 +02:00
inboundlinksURLProtocol . add ( urls . substring ( 0 , pr ) ) ;
inboundlinksURLStub . add ( urls . substring ( pr + 3 ) ) ;
inboundlinksName . add ( name . length ( ) > 0 ? name : " " ) ;
inboundlinksRel . add ( rel . length ( ) > 0 ? rel : " " ) ;
inboundlinksText . add ( text . length ( ) > 0 ? text : " " ) ;
2012-09-07 22:06:51 +02:00
inboundlinksTextChars . add ( text . length ( ) > 0 ? text . length ( ) : 0 ) ;
inboundlinksTextWords . add ( text . length ( ) > 0 ? text . split ( " " ) . length : 0 ) ;
2012-07-04 21:15:38 +02:00
inboundlinksTag . add (
2012-10-10 11:46:22 +02:00
" <a href= \" " + url . toNormalform ( false ) + " \" " +
2012-04-27 16:48:51 +02:00
( rel . length ( ) > 0 ? " rel= \" " + rel + " \" " : " " ) +
( name . length ( ) > 0 ? " name= \" " + name + " \" " : " " ) +
" > " +
2012-07-04 21:15:38 +02:00
( ( text . length ( ) > 0 ) ? text : " " ) + " </a> " ) ;
2012-09-07 22:06:51 +02:00
ImageEntry ientry = images . get ( url ) ;
inboundlinksAltTag . add ( ientry = = null ? " " : ientry . alt ( ) ) ;
2012-04-27 16:48:51 +02:00
c + + ;
}
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinks_tag_txt ) ) add ( doc , YaCySchema . inboundlinks_tag_txt , inboundlinksTag ) ;
2012-08-28 16:58:06 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinks_protocol_sxt ) ) add ( doc , YaCySchema . inboundlinks_protocol_sxt , protocolList2indexedList ( inboundlinksURLProtocol ) ) ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinks_urlstub_txt ) ) add ( doc , YaCySchema . inboundlinks_urlstub_txt , inboundlinksURLStub ) ;
if ( allAttr | | contains ( YaCySchema . inboundlinks_name_txt ) ) add ( doc , YaCySchema . inboundlinks_name_txt , inboundlinksName ) ;
2012-08-28 16:58:06 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinks_rel_sxt ) ) add ( doc , YaCySchema . inboundlinks_rel_sxt , inboundlinksRel ) ;
2012-08-31 10:30:43 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinks_relflags_val ) ) add ( doc , YaCySchema . inboundlinks_relflags_val , relEval ( inboundlinksRel ) ) ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinks_text_txt ) ) add ( doc , YaCySchema . inboundlinks_text_txt , inboundlinksText ) ;
2012-09-07 22:06:51 +02:00
if ( allAttr | | contains ( YaCySchema . inboundlinks_text_chars_val ) ) add ( doc , YaCySchema . inboundlinks_text_chars_val , inboundlinksTextChars ) ;
if ( allAttr | | contains ( YaCySchema . inboundlinks_text_words_val ) ) add ( doc , YaCySchema . inboundlinks_text_words_val , inboundlinksTextWords ) ;
if ( allAttr | | contains ( YaCySchema . inboundlinks_alttag_txt ) ) add ( doc , YaCySchema . inboundlinks_alttag_txt , inboundlinksAltTag ) ;
2012-04-27 16:48:51 +02:00
c = 0 ;
2012-09-07 22:06:51 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinkscount_i ) ) add ( doc , YaCySchema . outboundlinkscount_i , outboundLinks . size ( ) ) ;
2012-10-18 14:29:11 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinksnofollowcount_i ) ) add ( doc , YaCySchema . outboundlinksnofollowcount_i , document . outboundLinkNofollowCount ( ) ) ;
2012-09-07 22:06:51 +02:00
final List < String > outboundlinksTag = new ArrayList < String > ( outboundLinks . size ( ) ) ;
final List < String > outboundlinksURLProtocol = new ArrayList < String > ( outboundLinks . size ( ) ) ;
final List < String > outboundlinksURLStub = new ArrayList < String > ( outboundLinks . size ( ) ) ;
final List < String > outboundlinksName = new ArrayList < String > ( outboundLinks . size ( ) ) ;
final List < String > outboundlinksRel = new ArrayList < String > ( outboundLinks . size ( ) ) ;
final List < Integer > outboundlinksTextChars = new ArrayList < Integer > ( outboundLinks . size ( ) ) ;
final List < Integer > outboundlinksTextWords = new ArrayList < Integer > ( outboundLinks . size ( ) ) ;
final List < String > outboundlinksText = new ArrayList < String > ( outboundLinks . size ( ) ) ;
final List < String > outboundlinksAltTag = new ArrayList < String > ( outboundLinks . size ( ) ) ;
for ( final MultiProtocolURI url : outboundLinks ) {
2012-04-27 16:48:51 +02:00
final Properties p = alllinks . get ( url ) ;
2012-07-04 21:15:38 +02:00
if ( p = = null ) continue ;
2012-04-27 16:48:51 +02:00
final String name = p . getProperty ( " name " , " " ) ; // the name attribute
final String rel = p . getProperty ( " rel " , " " ) ; // the rel-attribute
final String text = p . getProperty ( " text " , " " ) ; // the text between the <a></a> tag
2012-10-10 11:46:22 +02:00
final String urls = url . toNormalform ( false ) ;
2012-04-27 16:48:51 +02:00
final int pr = urls . indexOf ( " :// " , 0 ) ;
2012-07-04 21:15:38 +02:00
outboundlinksURLProtocol . add ( urls . substring ( 0 , pr ) ) ;
outboundlinksURLStub . add ( urls . substring ( pr + 3 ) ) ;
outboundlinksName . add ( name . length ( ) > 0 ? name : " " ) ;
outboundlinksRel . add ( rel . length ( ) > 0 ? rel : " " ) ;
outboundlinksText . add ( text . length ( ) > 0 ? text : " " ) ;
2012-09-07 22:06:51 +02:00
outboundlinksTextChars . add ( text . length ( ) > 0 ? text . length ( ) : 0 ) ;
outboundlinksTextWords . add ( text . length ( ) > 0 ? text . split ( " " ) . length : 0 ) ;
2012-07-04 21:15:38 +02:00
outboundlinksTag . add (
2012-10-10 11:46:22 +02:00
" <a href= \" " + url . toNormalform ( false ) + " \" " +
2012-04-27 16:48:51 +02:00
( rel . length ( ) > 0 ? " rel= \" " + rel + " \" " : " " ) +
( name . length ( ) > 0 ? " name= \" " + name + " \" " : " " ) +
" > " +
2012-07-04 21:15:38 +02:00
( ( text . length ( ) > 0 ) ? text : " " ) + " </a> " ) ;
2012-09-07 22:06:51 +02:00
ImageEntry ientry = images . get ( url ) ;
inboundlinksAltTag . add ( ientry = = null ? " " : ientry . alt ( ) ) ;
2012-04-27 16:48:51 +02:00
c + + ;
}
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinks_tag_txt ) ) add ( doc , YaCySchema . outboundlinks_tag_txt , outboundlinksTag ) ;
2012-08-28 16:58:06 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinks_protocol_sxt ) ) add ( doc , YaCySchema . outboundlinks_protocol_sxt , protocolList2indexedList ( outboundlinksURLProtocol ) ) ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinks_urlstub_txt ) ) add ( doc , YaCySchema . outboundlinks_urlstub_txt , outboundlinksURLStub ) ;
if ( allAttr | | contains ( YaCySchema . outboundlinks_name_txt ) ) add ( doc , YaCySchema . outboundlinks_name_txt , outboundlinksName ) ;
2012-08-28 16:58:06 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinks_rel_sxt ) ) add ( doc , YaCySchema . outboundlinks_rel_sxt , outboundlinksRel ) ;
2012-09-07 22:06:51 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinks_relflags_val ) ) add ( doc , YaCySchema . outboundlinks_relflags_val , relEval ( outboundlinksRel ) ) ;
2012-08-23 09:51:45 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinks_text_txt ) ) add ( doc , YaCySchema . outboundlinks_text_txt , outboundlinksText ) ;
2012-09-07 22:06:51 +02:00
if ( allAttr | | contains ( YaCySchema . outboundlinks_text_chars_val ) ) add ( doc , YaCySchema . outboundlinks_text_chars_val , outboundlinksTextChars ) ;
if ( allAttr | | contains ( YaCySchema . outboundlinks_text_words_val ) ) add ( doc , YaCySchema . outboundlinks_text_words_val , outboundlinksTextWords ) ;
if ( allAttr | | contains ( YaCySchema . outboundlinks_alttag_txt ) ) add ( doc , YaCySchema . outboundlinks_alttag_txt , outboundlinksAltTag ) ;
2012-04-27 16:48:51 +02:00
// charset
2012-10-18 14:29:11 +02:00
if ( allAttr | | contains ( YaCySchema . charset_s ) ) add ( doc , YaCySchema . charset_s , document . getCharset ( ) ) ;
2012-04-27 16:48:51 +02:00
// coordinates
2012-10-18 14:29:11 +02:00
if ( document . lat ( ) ! = 0 . 0f & & document . lon ( ) ! = 0 . 0f ) {
if ( allAttr | | contains ( YaCySchema . coordinate_p ) ) add ( doc , YaCySchema . coordinate_p , Double . toString ( document . lat ( ) ) + " , " + Double . toString ( document . lon ( ) ) ) ;
2012-04-27 16:48:51 +02:00
}
2012-10-18 14:29:11 +02:00
if ( allAttr | | contains ( YaCySchema . httpstatus_i ) ) add ( doc , YaCySchema . httpstatus_i , responseHeader = = null ? 200 : responseHeader . getStatusCode ( ) ) ;
// fields that were additionally in URIMetadataRow
Date loadDate = new Date ( ) ;
2012-11-07 02:46:51 +01:00
Date modDate = responseHeader = = null ? new Date ( ) : responseHeader . lastModified ( ) ;
2012-10-18 14:29:11 +02:00
if ( modDate . getTime ( ) > loadDate . getTime ( ) ) modDate = loadDate ;
2012-11-07 02:46:51 +01:00
int size = ( int ) Math . max ( document . dc_source ( ) . length ( ) , responseHeader = = null ? 0 : responseHeader . getContentLength ( ) ) ;
2012-10-18 14:29:11 +02:00
if ( allAttr | | contains ( YaCySchema . load_date_dt ) ) add ( doc , YaCySchema . load_date_dt , loadDate ) ;
if ( allAttr | | contains ( YaCySchema . fresh_date_dt ) ) add ( doc , YaCySchema . fresh_date_dt , new Date ( loadDate . getTime ( ) + Math . max ( 0 , loadDate . getTime ( ) - modDate . getTime ( ) ) / 2 ) ) ; // freshdate, computed with Proxy-TTL formula
if ( allAttr | | contains ( YaCySchema . host_id_s ) ) add ( doc , YaCySchema . host_id_s , document . dc_source ( ) . hosthash ( ) ) ;
if ( ( allAttr | | contains ( YaCySchema . referrer_id_txt ) ) & & referrerURL ! = null ) add ( doc , YaCySchema . referrer_id_txt , new String [ ] { ASCII . String ( referrerURL . hash ( ) ) } ) ;
2012-08-23 09:51:45 +02:00
//if (allAttr || contains(SolrField.md5_s)) add(solrdoc, SolrField.md5_s, new byte[0]);
2012-10-18 14:29:11 +02:00
if ( allAttr | | contains ( YaCySchema . publisher_t ) ) add ( doc , YaCySchema . publisher_t , document . dc_publisher ( ) ) ;
if ( ( allAttr | | contains ( YaCySchema . language_s ) ) & & language ! = null ) add ( doc , YaCySchema . language_s , language ) ;
if ( allAttr | | contains ( YaCySchema . size_i ) ) add ( doc , YaCySchema . size_i , size ) ;
if ( allAttr | | contains ( YaCySchema . audiolinkscount_i ) ) add ( doc , YaCySchema . audiolinkscount_i , document . getAudiolinks ( ) . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . videolinkscount_i ) ) add ( doc , YaCySchema . videolinkscount_i , document . getVideolinks ( ) . size ( ) ) ;
if ( allAttr | | contains ( YaCySchema . applinkscount_i ) ) add ( doc , YaCySchema . applinkscount_i , document . getApplinks ( ) . size ( ) ) ;
2012-08-23 09:51:45 +02:00
return doc ;
2011-04-14 22:05:04 +02:00
}
2011-06-30 17:49:21 +02:00
2012-09-29 02:13:11 +02:00
/ * *
* this method compresses a list of protocol names to an indexed list .
* To do this , all ' http ' entries are removed and considered as default .
* The remaining entries are indexed as follows : a list of < i > - < p > entries is produced , where
* < i > is an index pointing to the original index of the protocol entry and < p > is the protocol entry itself .
* The < i > entry is formatted as a 3 - digit decimal number with leading zero digits .
* @param protocol
* @return a list of indexed protocol entries
* /
2012-07-04 21:15:38 +02:00
private static List < String > protocolList2indexedList ( List < String > protocol ) {
2012-02-01 18:12:59 +01:00
List < String > a = new ArrayList < String > ( ) ;
2012-07-04 21:15:38 +02:00
String p ;
for ( int i = 0 ; i < protocol . size ( ) ; i + + ) {
p = protocol . get ( i ) ;
if ( ! p . equals ( " http " ) ) {
2012-02-01 18:12:59 +01:00
String c = Integer . toString ( i ) ;
while ( c . length ( ) < 3 ) c = " 0 " + c ;
2012-07-04 21:15:38 +02:00
a . add ( c + " - " + p ) ;
2012-02-01 18:12:59 +01:00
}
}
2012-07-04 21:15:38 +02:00
return a ;
2012-02-01 18:12:59 +01:00
}
2012-09-29 02:13:11 +02:00
2012-10-18 14:29:11 +02:00
/ * *
* encode a string containing attributes from anchor rel properties binary :
* bit 0 : " me " contained in rel
* bit 1 : " nofollow " contained in rel
* @param rel
* @return binary encoded information about rel
* /
private static List < Integer > relEval ( final List < String > rel ) {
List < Integer > il = new ArrayList < Integer > ( rel . size ( ) ) ;
for ( final String s : rel ) {
int i = 0 ;
final String s0 = s . toLowerCase ( ) . trim ( ) ;
if ( " me " . equals ( s0 ) ) i + = 1 ;
if ( " nofollow " . equals ( s0 ) ) i + = 2 ;
il . add ( i ) ;
}
return il ;
}
2012-10-02 14:29:45 +02:00
2012-05-14 14:56:21 +02:00
/ * *
* register an entry as error document
* @param digestURI
* @param failReason
* @param httpstatus
* @throws IOException
* /
2012-11-23 14:00:30 +01:00
public SolrInputDocument err ( final DigestURI digestURI , final String failReason , final FailType failType , final int httpstatus ) throws IOException {
2012-08-23 09:51:45 +02:00
final SolrInputDocument solrdoc = new SolrInputDocument ( ) ;
add ( solrdoc , YaCySchema . id , ASCII . String ( digestURI . hash ( ) ) ) ;
2012-10-10 11:46:22 +02:00
add ( solrdoc , YaCySchema . sku , digestURI . toNormalform ( true ) ) ;
2012-05-14 14:56:21 +02:00
final InetAddress address = digestURI . getInetAddress ( ) ;
2012-09-11 22:46:39 +02:00
if ( contains ( YaCySchema . ip_s ) & & address ! = null ) add ( solrdoc , YaCySchema . ip_s , address . getHostAddress ( ) ) ;
if ( contains ( YaCySchema . host_s ) & & digestURI . getHost ( ) ! = null ) add ( solrdoc , YaCySchema . host_s , digestURI . getHost ( ) ) ;
2012-05-14 14:56:21 +02:00
// path elements of link
2012-09-11 22:46:39 +02:00
if ( contains ( YaCySchema . url_paths_sxt ) ) add ( solrdoc , YaCySchema . url_paths_sxt , digestURI . getPaths ( ) ) ;
if ( contains ( YaCySchema . url_file_ext_s ) ) add ( solrdoc , YaCySchema . url_file_ext_s , digestURI . getFileExtension ( ) ) ;
// fail reason and status
if ( contains ( YaCySchema . failreason_t ) ) add ( solrdoc , YaCySchema . failreason_t , failReason ) ;
2012-11-23 14:00:30 +01:00
if ( contains ( YaCySchema . failtype_s ) ) add ( solrdoc , YaCySchema . failtype_s , failType . name ( ) ) ;
2012-09-11 22:46:39 +02:00
if ( contains ( YaCySchema . httpstatus_i ) ) add ( solrdoc , YaCySchema . httpstatus_i , httpstatus ) ;
2012-05-14 14:56:21 +02:00
return solrdoc ;
}
2011-04-14 22:05:04 +02:00
/ *
2012-02-01 18:12:59 +01:00
standard solr schema
2011-04-14 22:05:04 +02:00
< field name = " name " type = " textgen " indexed = " true " stored = " true " / >
< field name = " cat " type = " string " indexed = " true " stored = " true " multiValued = " true " / >
< field name = " features " type = " text " indexed = " true " stored = " true " multiValued = " true " / >
< field name = " includes " type = " text " indexed = " true " stored = " true " termVectors = " true " termPositions = " true " termOffsets = " true " / >
< field name = " weight " type = " float " indexed = " true " stored = " true " / >
< field name = " price " type = " float " indexed = " true " stored = " true " / >
< field name = " popularity " type = " int " indexed = " true " stored = " true " / >
< ! - - Common metadata fields , named specifically to match up with
SolrCell metadata when parsing rich documents such as Word , PDF .
Some fields are multiValued only because Tika currently may return
multiple values for them .
- - >
< field name = " title " type = " text " indexed = " true " stored = " true " multiValued = " true " / >
< field name = " subject " type = " text " indexed = " true " stored = " true " / >
< field name = " description " type = " text " indexed = " true " stored = " true " / >
< field name = " comments " type = " text " indexed = " true " stored = " true " / >
< field name = " author " type = " textgen " indexed = " true " stored = " true " / >
< field name = " keywords " type = " textgen " indexed = " true " stored = " true " / >
< field name = " category " type = " textgen " indexed = " true " stored = " true " / >
< field name = " content_type " type = " string " indexed = " true " stored = " true " multiValued = " true " / >
< field name = " last_modified " type = " date " indexed = " true " stored = " true " / >
< field name = " links " type = " string " indexed = " true " stored = " true " multiValued = " true " / >
* /
}