2012-08-14 12:40:26 +02:00
/ * *
* search
* Copyright 2012 by Michael Peter Christen , mc @yacy.net , Frankfurt am Main , Germany
* First released 14 . 08 . 2012 at http : //yacy.net
*
* This library is free software ; you can redistribute it and / or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation ; either
* version 2 . 1 of the License , or ( at your option ) any later version .
*
* This library is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the GNU
* Lesser General Public License for more details .
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21 . txt
* If not , see < http : //www.gnu.org/licenses/>.
* /
import java.io.IOException ;
import java.io.OutputStream ;
import java.io.OutputStreamWriter ;
import java.io.Writer ;
2013-06-18 16:25:00 +02:00
import java.util.Date ;
2012-08-14 12:40:26 +02:00
2013-06-18 16:25:00 +02:00
import net.yacy.cora.date.ISO8601Formatter ;
2013-09-15 00:30:23 +02:00
import net.yacy.cora.document.encoding.UTF8 ;
2013-03-13 14:47:00 +01:00
import net.yacy.cora.federate.solr.Ranking ;
2012-09-25 21:20:03 +02:00
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector ;
import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter ;
2012-08-16 16:28:57 +02:00
import net.yacy.cora.protocol.HeaderFramework ;
2012-08-14 12:40:26 +02:00
import net.yacy.cora.protocol.RequestHeader ;
2013-07-09 14:28:25 +02:00
import net.yacy.cora.util.ConcurrentLog ;
2012-08-14 12:40:26 +02:00
import net.yacy.search.Switchboard ;
2012-08-22 23:50:40 +02:00
import net.yacy.search.query.AccessTracker ;
2012-11-18 01:22:41 +01:00
import net.yacy.search.query.QueryGoal ;
2013-04-23 20:42:54 +02:00
import net.yacy.search.query.QueryModifier ;
2012-11-01 17:40:06 +01:00
import net.yacy.search.query.SearchEvent ;
2013-02-21 13:23:55 +01:00
import net.yacy.search.schema.CollectionSchema ;
2012-09-21 15:48:16 +02:00
import net.yacy.server.serverObjects ;
import net.yacy.server.serverSwitch ;
2012-08-14 12:40:26 +02:00
2013-09-02 08:05:42 +02:00
import org.apache.solr.common.SolrDocumentList ;
2012-08-14 12:40:26 +02:00
import org.apache.solr.common.SolrException ;
import org.apache.solr.common.params.CommonParams ;
import org.apache.solr.request.SolrQueryRequest ;
2013-06-28 15:19:50 +02:00
import org.apache.solr.request.SolrRequestInfo ;
2012-11-07 14:15:27 +01:00
import org.apache.solr.response.ResultContext ;
2012-08-14 12:40:26 +02:00
import org.apache.solr.response.SolrQueryResponse ;
2012-11-02 12:29:48 +01:00
import org.apache.solr.util.FastWriter ;
2012-08-14 12:40:26 +02:00
// try
2012-08-22 17:37:34 +02:00
// http://localhost:8090/gsa/searchresult?q=chicken+teriyaki&output=xml&client=test&site=test&sort=date:D:S:d1
2012-08-14 12:40:26 +02:00
/ * *
* This is a gsa result formatter for solr search results .
* The result format is implemented according to
* https : //developers.google.com/search-appliance/documentation/68/xml_reference#results_xml
* /
public class searchresult {
private final static GSAResponseWriter responseWriter = new GSAResponseWriter ( ) ;
/ * *
* get the right mime type for this streamed result page
* @param header
* @param post
* @param env
* @return
* /
public static String mime ( final RequestHeader header , final serverObjects post , final serverSwitch env ) {
return " text/xml " ;
}
2012-08-21 01:57:46 +02:00
2012-08-14 12:40:26 +02:00
/ * *
* @param header
* @param post
* @param env
* @param out
* @return
* /
2013-06-28 12:22:33 +02:00
public static serverObjects respond ( final RequestHeader header , serverObjects post , final serverSwitch env , final OutputStream out ) {
2012-08-14 12:40:26 +02:00
// this uses the methods in the jetty servlet environment and can be removed if jetty in implemented
Switchboard sb = ( Switchboard ) env ;
2012-08-16 16:28:57 +02:00
// remember the peer contact for peer statistics
final String clientip = header . get ( HeaderFramework . CONNECTION_PROP_CLIENTIP , " <unknown> " ) ; // read an artificial header addendum
final String userAgent = header . get ( HeaderFramework . USER_AGENT , " <unknown> " ) ;
sb . peers . peerActions . setUserAgent ( clientip , userAgent ) ;
2012-08-14 12:40:26 +02:00
// check if user is allowed to search (can be switched in /ConfigPortal.html)
2012-08-20 17:10:48 +02:00
boolean authenticated = sb . adminAuthenticated ( header ) > = 2 ;
final boolean searchAllowed = authenticated | | sb . getConfigBool ( " publicSearchpage " , true ) ;
2012-08-14 12:40:26 +02:00
if ( ! searchAllowed ) return null ;
// check post
2013-06-28 12:22:33 +02:00
if ( post = = null ) { post = new serverObjects ( ) ; post . put ( " q " , " " ) ; post . put ( " num " , " 0 " ) ; }
2013-07-09 14:28:25 +02:00
ConcurrentLog . info ( " GSA Query " , post . toString ( ) ) ;
2012-08-20 17:10:48 +02:00
sb . intermissionAllThreads ( 3000 ) ; // tell all threads to do nothing for a specific time
2012-12-02 16:54:29 +01:00
2012-08-14 12:40:26 +02:00
// rename post fields according to result style
//post.put(CommonParams.Q, post.remove("q")); // same as solr
//post.put(CommonParams.START, post.remove("start")); // same as solr
//post.put(, post.remove("client"));//required, example: myfrontend
//post.put(, post.remove("output"));//required, example: xml,xml_no_dtd
2012-12-07 14:54:49 +01:00
String originalQuery = post . get ( CommonParams . Q , " " ) ;
post . put ( " originalQuery " , originalQuery ) ;
2012-11-09 16:25:24 +01:00
// get a solr query string
2012-12-15 00:05:46 +01:00
QueryGoal qg = new QueryGoal ( originalQuery , originalQuery ) ;
2013-09-03 11:14:23 +02:00
StringBuilder solrQ = qg . collectionTextQueryString ( sb . index . fulltext ( ) . getDefaultConfiguration ( ) , 0 , false ) ;
2012-12-07 14:54:49 +01:00
post . put ( " defType " , " edismax " ) ;
post . put ( CommonParams . Q , solrQ . toString ( ) ) ;
2012-08-14 12:40:26 +02:00
post . put ( CommonParams . ROWS , post . remove ( " num " ) ) ;
2013-09-10 10:31:57 +02:00
post . put ( CommonParams . ROWS , Math . min ( post . getInt ( CommonParams . ROWS , 10 ) , ( authenticated ) ? 100000000 : 100 ) ) ;
2013-03-13 14:47:00 +01:00
2013-04-12 10:48:41 +02:00
// set ranking
if ( post . containsKey ( " sort " ) ) {
// if a gsa-style sort attribute is given, use this to set the solr sort attribute
GSAResponseWriter . Sort sort = new GSAResponseWriter . Sort ( post . get ( CommonParams . SORT , " " ) ) ;
String sorts = sort . toSolr ( ) ;
if ( sorts = = null ) {
post . remove ( CommonParams . SORT ) ;
} else {
post . put ( CommonParams . SORT , sorts ) ;
}
} else {
// if no such sort attribute is given, use the ranking as configured for YaCy
Ranking ranking = sb . index . fulltext ( ) . getDefaultConfiguration ( ) . getRanking ( 0 ) ;
String bq = ranking . getBoostQuery ( ) ;
String bf = ranking . getBoostFunction ( ) ;
2013-04-16 14:45:14 +02:00
if ( bq . length ( ) > 0 ) post . put ( " bq " , bq ) ;
if ( bf . length ( ) > 0 ) post . put ( " boost " , bf ) ; // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29
2013-04-12 10:48:41 +02:00
}
2013-06-18 16:25:00 +02:00
String daterange [ ] = post . remove ( " daterange " ) ;
if ( daterange ! = null ) {
String origfq = post . get ( CommonParams . FQ ) ;
String datefq = " " ;
for ( String dr : daterange ) {
2013-09-11 12:52:18 +02:00
String from_to [ ] = dr . endsWith ( " .. " ) ? new String [ ] { dr . substring ( 0 , dr . length ( ) - 2 ) , " " } : dr . startsWith ( " .. " ) ? new String [ ] { " " , dr . substring ( 2 ) } : dr . split ( " \\ . \\ . " ) ;
2013-06-18 16:25:00 +02:00
if ( from_to . length ! = 2 ) continue ;
2013-09-11 12:52:18 +02:00
Date from = HeaderFramework . parseGSAFS ( from_to [ 0 ] ) ;
if ( from = = null ) from = new Date ( 0 ) ;
Date to = HeaderFramework . parseGSAFS ( from_to [ 1 ] ) ;
if ( to = = null ) to = new Date ( ) ;
2013-06-21 15:47:12 +02:00
to . setTime ( to . getTime ( ) + 24L * 60L * 60L * 1000L ) ; // we add a day because the day is inclusive
2013-06-18 16:25:00 +02:00
String z = CollectionSchema . last_modified . getSolrFieldName ( ) + " :[ " + ISO8601Formatter . FORMATTER . format ( from ) + " TO " + ISO8601Formatter . FORMATTER . format ( to ) + " ] " ;
datefq = datefq . length ( ) = = 0 ? z : " OR " + z ;
}
if ( datefq . length ( ) > 0 ) post . put ( CommonParams . FQ , origfq = = null | | origfq . length ( ) = = 0 ? datefq : " ( " + origfq + " ) AND ( " + datefq + " ) " ) ;
}
2012-11-22 13:03:33 +01:00
post . put ( CommonParams . FL ,
2013-02-21 13:23:55 +01:00
CollectionSchema . content_type . getSolrFieldName ( ) + ',' +
CollectionSchema . id . getSolrFieldName ( ) + ',' +
CollectionSchema . sku . getSolrFieldName ( ) + ',' +
CollectionSchema . title . getSolrFieldName ( ) + ',' +
2013-07-30 12:48:57 +02:00
CollectionSchema . description_txt . getSolrFieldName ( ) + ',' +
2013-02-21 13:23:55 +01:00
CollectionSchema . load_date_dt . getSolrFieldName ( ) + ',' +
CollectionSchema . last_modified . getSolrFieldName ( ) + ',' +
CollectionSchema . size_i . getSolrFieldName ( ) ) ;
2012-08-22 17:37:34 +02:00
post . put ( " hl " , " true " ) ;
2012-12-07 14:54:49 +01:00
post . put ( " hl.q " , originalQuery ) ;
2013-02-21 13:23:55 +01:00
post . put ( " hl.fl " , CollectionSchema . h1_txt . getSolrFieldName ( ) + " , " + CollectionSchema . h2_txt . getSolrFieldName ( ) + " , " + CollectionSchema . text_t . getSolrFieldName ( ) ) ;
2013-07-30 12:48:57 +02:00
post . put ( " hl.alternateField " , CollectionSchema . description_txt . getSolrFieldName ( ) ) ;
2012-08-22 17:37:34 +02:00
post . put ( " hl.simple.pre " , " <b> " ) ;
post . put ( " hl.simple.post " , " </b> " ) ;
2012-11-01 17:40:06 +01:00
post . put ( " hl.fragsize " , Integer . toString ( SearchEvent . SNIPPET_MAX_LENGTH ) ) ;
2013-04-12 10:48:41 +02:00
//String[] access = post.remove("access");
//String[] entqr = post.remove("entqr");
2012-08-14 12:40:26 +02:00
2012-09-06 22:35:55 +02:00
// add sites operator
2013-04-23 20:42:54 +02:00
String [ ] site = post . remove ( " site " ) ; // example: col1|col2
2013-02-11 22:12:15 +01:00
if ( site ! = null & & site [ 0 ] . length ( ) > 0 ) {
2013-06-06 13:36:58 +02:00
String origfq = post . get ( CommonParams . FQ ) ;
String sitefq = QueryModifier . parseCollectionExpression ( site [ 0 ] ) ;
2013-06-18 16:25:00 +02:00
post . put ( CommonParams . FQ , origfq = = null | | origfq . length ( ) = = 0 ? sitefq : " ( " + origfq + " ) AND ( " + sitefq + " ) " ) ;
2012-09-03 15:26:08 +02:00
}
2012-10-02 14:29:45 +02:00
2012-08-14 12:40:26 +02:00
// get the embedded connector
2013-02-22 15:45:15 +01:00
EmbeddedSolrConnector connector = sb . index . fulltext ( ) . getDefaultEmbeddedConnector ( ) ;
2012-08-14 12:40:26 +02:00
if ( connector = = null ) return null ;
// do the solr request
2012-09-14 12:09:20 +02:00
SolrQueryRequest req = connector . request ( post . toSolrParams ( null ) ) ;
2012-08-14 12:40:26 +02:00
SolrQueryResponse response = null ;
Exception e = null ;
2013-07-17 18:31:30 +02:00
try { response = connector . query ( req ) ; } catch ( final SolrException ee ) { e = ee ; }
2012-08-14 12:40:26 +02:00
if ( response ! = null ) e = response . getException ( ) ;
if ( e ! = null ) {
2013-07-09 14:28:25 +02:00
ConcurrentLog . logException ( e ) ;
2013-06-28 15:19:50 +02:00
if ( req ! = null ) req . close ( ) ;
SolrRequestInfo . clearRequestInfo ( ) ;
2012-08-14 12:40:26 +02:00
return null ;
}
2012-08-21 01:57:46 +02:00
// set some context for the writer
2013-04-12 10:48:41 +02:00
/ *
2012-08-21 01:57:46 +02:00
Map < Object , Object > context = req . getContext ( ) ;
context . put ( " ip " , header . get ( " CLIENTIP " , " " ) ) ;
2012-08-31 14:00:53 +02:00
context . put ( " client " , " vsm_frontent " ) ;
2012-08-21 01:57:46 +02:00
context . put ( " sort " , sort . sort ) ;
context . put ( " site " , site = = null ? " " : site ) ;
2013-02-11 22:12:15 +01:00
context . put ( " access " , access = = null ? " p " : access [ 0 ] ) ;
context . put ( " entqr " , entqr = = null ? " 3 " : entqr [ 0 ] ) ;
2013-04-12 10:48:41 +02:00
* /
2012-08-14 12:40:26 +02:00
// write the result directly to the output stream
Writer ow = new FastWriter ( new OutputStreamWriter ( out , UTF8 . charset ) ) ;
try {
responseWriter . write ( ow , req , response ) ;
ow . flush ( ) ;
2013-07-17 18:31:30 +02:00
} catch ( final IOException e1 ) {
2012-08-14 12:40:26 +02:00
} finally {
req . close ( ) ;
2013-06-28 15:19:50 +02:00
SolrRequestInfo . clearRequestInfo ( ) ;
2013-07-17 18:31:30 +02:00
try { ow . close ( ) ; } catch ( final IOException e1 ) { }
2012-08-14 12:40:26 +02:00
}
2012-08-22 23:50:40 +02:00
// log result
Object rv = response . getValues ( ) . get ( " response " ) ;
2013-09-02 08:05:42 +02:00
int matches = 0 ;
2012-11-07 14:15:27 +01:00
if ( rv ! = null & & rv instanceof ResultContext ) {
2013-09-02 08:05:42 +02:00
matches = ( ( ResultContext ) rv ) . docs . matches ( ) ;
} else if ( rv ! = null & & rv instanceof SolrDocumentList ) {
matches = ( int ) ( ( SolrDocumentList ) rv ) . getNumFound ( ) ;
2012-08-22 23:50:40 +02:00
}
2013-09-02 08:05:42 +02:00
AccessTracker . addToDump ( originalQuery , Integer . toString ( matches ) ) ;
ConcurrentLog . info ( " GSA Query " , " results: " + matches + " , for query: " + post . toString ( ) ) ;
2012-08-14 12:40:26 +02:00
return null ;
}
2013-06-28 15:19:50 +02:00
}