mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
migrated solr to 9.0
This is a major step because solr removed support for embedded solr instances in 9.0 and we want to keep it because we want to ship YaCy with an embedded solr. It was necessary to add parts of solr code into YaCy to make this migration possible. Further on with Solr 9.1 they removed even more parts which are required for embedded operation, therefore we cannot migrate yet further without big changes. If you are running a YaCy instance with Solr 8.x, the migration should be done automatically. If not you require to first migrate to a YaCy version 1.93 with Solr 8.x to migrate to Solr 8 data.
This commit is contained in:
Michael Peter Christen
parent
b8417e5619
commit
8eb0d490aa
|
|
@ -3,7 +3,7 @@ javacSource=11
|
|||
javacTarget=11
|
||||
|
||||
# Release Configuration
|
||||
releaseVersion=1.930
|
||||
releaseVersion=1.940
|
||||
releaseFileParentDir=yacy
|
||||
privateKeyFile=private.key
|
||||
|
||||
|
|
|
|||
|
|
@ -400,8 +400,8 @@
|
|||
<!-- make clean -->
|
||||
<target name="clean" description="make clean">
|
||||
<delete dir="${release_main}" failonerror="false"/>
|
||||
<delete file="${lib}/yacycore.jar" failonerror="false"/>
|
||||
<delete failonerror="false">
|
||||
<fileset dir="${lib}" includes="**/*.jar" exclude="org.restlet.jar" /> <!-- we cannot find the org.restlet.jar in maven -->
|
||||
<fileset dir="${src}" includes="**/*.class" />
|
||||
<fileset dir="${build}" includes="**/*.class" />
|
||||
<fileset dir="test/" includes="**/*.class" />
|
||||
|
|
|
|||
|
|
@ -411,48 +411,32 @@
|
|||
<maxBooleanClauses>1024</maxBooleanClauses>
|
||||
|
||||
|
||||
<!-- Solr Internal Query Caches
|
||||
|
||||
There are two implementations of cache available for Solr,
|
||||
LRUCache, based on a synchronized LinkedHashMap, and
|
||||
FastLRUCache, based on a ConcurrentHashMap.
|
||||
|
||||
FastLRUCache has faster gets and slower puts in single
|
||||
threaded operation and thus is generally faster than LRUCache
|
||||
when the hit ratio of the cache is high (> 75%), and may be
|
||||
faster under other scenarios on multi-cpu systems.
|
||||
-->
|
||||
|
||||
<!-- Filter Cache
|
||||
|
||||
Cache used by SolrIndexSearcher for filters (DocSets),
|
||||
unordered sets of *all* documents that match a query. When a
|
||||
new searcher is opened, its caches may be prepopulated or
|
||||
"autowarmed" using data from caches in the old searcher.
|
||||
autowarmCount is the number of items to prepopulate. For
|
||||
LRUCache, the autowarmed items will be the most recently
|
||||
accessed items.
|
||||
|
||||
autowarmCount is the number of items to prepopulate.
|
||||
Parameters:
|
||||
class - the SolrCache implementation LRUCache or
|
||||
(LRUCache or FastLRUCache)
|
||||
class - the SolrCache implementation (CaffeineCache)
|
||||
size - the maximum number of entries in the cache
|
||||
initialSize - the initial capacity (number of entries) of
|
||||
the cache. (see java.util.HashMap)
|
||||
autowarmCount - the number of entries to prepopulate from
|
||||
and old cache.
|
||||
-->
|
||||
<filterCache class="solr.FastLRUCache"
|
||||
size="100"
|
||||
initialSize="100"
|
||||
autowarmCount="0"/>
|
||||
<filterCache class="solr.CaffeineCache"
|
||||
size="512"
|
||||
initialSize="512"
|
||||
autowarmCount="128"/>
|
||||
|
||||
<!-- Query Result Cache
|
||||
|
||||
Caches results of searches - ordered lists of document ids
|
||||
(DocList) based on a query, a sort, and the range of documents requested.
|
||||
-->
|
||||
<queryResultCache class="solr.LRUCache"
|
||||
<queryResultCache class="solr.CaffeineCache"
|
||||
size="512"
|
||||
initialSize="512"
|
||||
autowarmCount="0"/>
|
||||
|
|
@ -463,18 +447,11 @@
|
|||
document). Since Lucene internal document ids are transient,
|
||||
this cache will not be autowarmed.
|
||||
-->
|
||||
<documentCache class="solr.LRUCache"
|
||||
<documentCache class="solr.CaffeineCache"
|
||||
size="512"
|
||||
initialSize="512"
|
||||
autowarmCount="0"/>
|
||||
|
||||
<!-- custom cache currently used by block join
|
||||
<cache name="perSegFilter"
|
||||
class="solr.search.LRUCache"
|
||||
size="10"
|
||||
initialSize="0"
|
||||
autowarmCount="10"
|
||||
regenerator="solr.NoOpRegenerator" /> -->
|
||||
|
||||
<!-- Field Value Cache
|
||||
|
||||
|
|
@ -482,7 +459,7 @@
|
|||
by document id. The fieldValueCache is created by default
|
||||
even if not configured here.
|
||||
-->
|
||||
<fieldValueCache class="solr.FastLRUCache"
|
||||
<fieldValueCache class="solr.CaffeineCache"
|
||||
size="32"
|
||||
initialSize="0"
|
||||
autowarmCount="0"
|
||||
|
|
@ -497,15 +474,6 @@
|
|||
be specified as an implementation of solr.CacheRegenerator
|
||||
if autowarming is desired.
|
||||
-->
|
||||
<!--
|
||||
<cache name="myUserCache"
|
||||
class="solr.LRUCache"
|
||||
size="4096"
|
||||
initialSize="1024"
|
||||
autowarmCount="1024"
|
||||
regenerator="com.mycompany.MyRegenerator"
|
||||
/>
|
||||
-->
|
||||
|
||||
|
||||
<!-- Lazy Field Loading
|
||||
|
|
@ -1315,7 +1283,7 @@
|
|||
in Solr's conf/xslt directory. Changes to xslt files are checked for
|
||||
every xsltCacheLifetimeSeconds.
|
||||
-->
|
||||
<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
|
||||
<queryResponseWriter name="xslt" class="solr.scripting.xslt.XSLTResponseWriter">
|
||||
<int name="xsltCacheLifetimeSeconds">5</int>
|
||||
</queryResponseWriter>
|
||||
|
||||
|
|
|
|||
55
ivy.xml
55
ivy.xml
|
|
@ -4,6 +4,7 @@
|
|||
<!-- https://wrongnotes.blogspot.com/2014/02/simplest-explanation-of-ivy.html -->
|
||||
<conf name="compile" visibility="public"/>
|
||||
<conf name="test" visibility="public"/>
|
||||
|
||||
</configurations>
|
||||
<dependencies defaultconf="compile->default">
|
||||
<dependency org="commons-fileupload" name="commons-fileupload" rev="1.4" conf="compile->master" />
|
||||
|
|
@ -12,11 +13,11 @@
|
|||
</dependency>
|
||||
<dependency org="com.cybozu.labs" name="langdetect" rev="1.1-20120112" conf="compile->master"/>
|
||||
<dependency org="com.drewnoakes" name="metadata-extractor" rev="2.18.0" />
|
||||
<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.13.5"/>
|
||||
<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.17.0"/>
|
||||
<dependency org="com.github.ben-manes.caffeine" name="caffeine" rev="3.1.8"/>
|
||||
<dependency org="com.google.guava" name="guava" rev="32.1.3-jre" conf="compile->master"/>
|
||||
<dependency org="com.google.guava" name="guava" rev="33.2.0-jre" conf="compile->master"/>
|
||||
<dependency org="com.google.guava" name="failureaccess" rev="1.0.2" />
|
||||
<dependency org="com.ibm.icu" name="icu4j" rev="73.2"/>
|
||||
<dependency org="com.ibm.icu" name="icu4j" rev="75.1"/>
|
||||
<dependency org="com.jcraft" name="jsch" rev="0.1.55" />
|
||||
<dependency org="com.twelvemonkeys.imageio" name="imageio-core" rev="3.9.4"/>
|
||||
<dependency org="com.twelvemonkeys.imageio" name="imageio-bmp" rev="3.9.4" />
|
||||
|
|
@ -28,14 +29,14 @@
|
|||
<dependency org="io.dropwizard.metrics" name="metrics-jmx" rev="4.1.5" conf="compile->master"/>
|
||||
<dependency org="io.opentracing" name="opentracing-noop" rev="0.33.0"/>
|
||||
<dependency org="io.opentracing" name="opentracing-util" rev="0.33.0"/>
|
||||
<dependency org="javax.servlet" name="javax.servlet-api" rev="3.1.0"/>
|
||||
<dependency org="javax.servlet" name="javax.servlet-api" rev="4.0.1"/>
|
||||
<dependency org="javainetlocator" name="inetaddresslocator" rev="2.18" />
|
||||
<dependency org="jcifs" name="jcifs" rev="1.3.17" conf="compile->master" />
|
||||
<dependency org="net.arnx" name="jsonic" rev="1.3.10"/>
|
||||
<dependency org="net.jthink" name="jaudiotagger" rev="2.2.5"/>
|
||||
<dependency org="net.sourceforge.jchardet" name="jchardet" rev="1.0"/>
|
||||
<dependency org="org.apache.commons" name="commons-collections4" rev="4.4" />
|
||||
<dependency org="org.apache.commons" name="commons-compress" rev="1.24.0" />
|
||||
<dependency org="org.apache.commons" name="commons-compress" rev="1.26.0" />
|
||||
<dependency org="org.apache.calcite.avatica" name="avatica-core" rev="1.13.0" conf="compile->master"/>
|
||||
<dependency org="org.apache.calcite" name="calcite-core" rev="1.18.0" conf="compile->master"/>
|
||||
<dependency org="org.apache.calcite" name="calcite-linq4j" rev="1.18.0" conf="compile->master"/>
|
||||
|
|
@ -44,27 +45,30 @@
|
|||
<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.5.12"/>
|
||||
<dependency org="org.apache.httpcomponents" name="httpmime" rev="4.5.12"/>
|
||||
<dependency org="org.apache.james" name="apache-mime4j" rev="0.6"/>
|
||||
<dependency org="org.apache.lucene" name="lucene-analyzers-phonetic" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-backward-codecs" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-classification" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-codecs" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-grouping" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-highlighter" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-join" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-memory" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-misc" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-queries" rev="8.11.2" />
|
||||
<dependency org="org.apache.lucene" name="lucene-queryparser" rev="8.11.2" conf="compile->master"/>
|
||||
<dependency org="org.apache.lucene" name="lucene-spatial-extras" rev="8.11.2" conf="compile->master"/>
|
||||
<dependency org="org.apache.lucene" name="lucene-suggest" rev="8.11.2"/>
|
||||
<dependency org="org.apache.pdfbox" name="pdfbox" rev="3.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-analysis-common" rev="9.0.0"/>
|
||||
<dependency org="org.apache.lucene" name="lucene-backward-codecs" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-classification" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-codecs" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-grouping" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-highlighter" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-join" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-memory" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-misc" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-queries" rev="9.0.0" />
|
||||
<dependency org="org.apache.lucene" name="lucene-queryparser" rev="9.0.0" conf="compile->master"/>
|
||||
<dependency org="org.apache.lucene" name="lucene-spatial-extras" rev="9.0.0" conf="compile->master"/>
|
||||
<dependency org="org.apache.lucene" name="lucene-suggest" rev="9.0.0"/>
|
||||
|
||||
<dependency org="org.apache.pdfbox" name="pdfbox" rev="3.0.2" />
|
||||
<dependency org="org.apache.poi" name="poi" rev="3.17" />
|
||||
<dependency org="org.apache.poi" name="poi-scratchpad" rev="3.17" />
|
||||
<dependency org="org.apache.solr" name="solr-core" rev="8.11.2" conf="compile->master"/>
|
||||
<dependency org="org.apache.solr" name="solr-solrj" rev="8.11.2" conf="compile->master" />
|
||||
|
||||
<dependency org="org.apache.solr" name="solr-core" rev="9.0.0" conf="compile->master"/>
|
||||
<dependency org="org.apache.solr" name="solr-scripting" rev="9.0.0" conf="compile->master"/>
|
||||
<dependency org="org.apache.solr" name="solr-solrj" rev="9.0.0" conf="compile->master" />
|
||||
<dependency org="org.apache.zookeeper" name="zookeeper" rev="3.6.2" conf="compile->master" />
|
||||
<dependency org="org.bitlet" name="weupnp" rev="0.1.4" />
|
||||
<dependency org="org.bouncycastle" name="bcmail-jdk18on" rev="1.76" />
|
||||
<dependency org="org.bouncycastle" name="bcmail-jdk18on" rev="1.78.1" />
|
||||
<dependency org="org.codehaus.woodstox" name="woodstox-core-asl" rev="4.4.1">
|
||||
<exclude module="stax-api" />
|
||||
</dependency>
|
||||
|
|
@ -89,9 +93,9 @@
|
|||
<dependency org="org.locationtech.spatial4j" name="spatial4j" rev="0.8"/>
|
||||
<!--NOT FOUND dependency org="org.restlet.jee" name="org.restlet" rev="2.2.1" / -->
|
||||
<dependency org="org.rrd4j" name="rrd4j" rev="3.2"/>
|
||||
<dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.7.32"/>
|
||||
<dependency org="org.slf4j" name="log4j-over-slf4j" rev="1.7.32"/>
|
||||
<dependency org="org.slf4j" name="slf4j-jdk14" rev="1.7.32"/>
|
||||
<dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.7.36"/>
|
||||
<dependency org="org.slf4j" name="log4j-over-slf4j" rev="1.7.36"/>
|
||||
<dependency org="org.slf4j" name="slf4j-jdk14" rev="1.7.36"/>
|
||||
<!-- This does not match langdetect.jar from pre-ivy -->
|
||||
<dependency org="org.tukaani" name="xz" rev="1.8"/>
|
||||
<dependency org="oro" name="oro" rev="2.0.8"/>
|
||||
|
|
@ -101,5 +105,6 @@
|
|||
<dependency org="org.hamcrest" name="hamcrest-core" rev="2.2" conf="test->default"/>
|
||||
<dependency org="org.hamcrest" name="hamcrest-library" rev="2.2" conf="test->default"/>
|
||||
<dependency org="com.github.luben" name="zstd-jni" rev="1.5.5-6"></dependency>
|
||||
<dependency org="ai.djl" name ="api" rev="0.27.0"></dependency>
|
||||
</dependencies>
|
||||
</ivy-module>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import java.util.List;
|
|||
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
//import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
import org.apache.solr.client.solrj.impl.XMLResponseParser;
|
||||
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||
import org.apache.solr.client.solrj.request.LukeRequest;
|
||||
|
|
@ -43,6 +43,7 @@ import org.apache.solr.common.params.CommonParams;
|
|||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
|
||||
import net.yacy.cora.federate.solr.embedded.EmbeddedSolrServer;
|
||||
import net.yacy.cora.federate.solr.instance.ServerShard;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,395 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// this class was taken from solr 8 package org.apache.solr.client.solrj.embedded;
|
||||
package net.yacy.cora.federate.solr.embedded;
|
||||
|
||||
import static org.apache.solr.common.params.CommonParams.PATH;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
import org.apache.lucene.search.TotalHits.Relation;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.StreamingResponseCallback;
|
||||
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
|
||||
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||
import org.apache.solr.client.solrj.request.RequestWriter;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.ContentStream;
|
||||
import org.apache.solr.common.util.ContentStreamBase;
|
||||
import org.apache.solr.common.util.JavaBinCodec;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.core.NodeConfig;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestHandler;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.response.BinaryResponseWriter;
|
||||
import org.apache.solr.response.ResultContext;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.servlet.SolrRequestParsers;
|
||||
|
||||
/**
|
||||
* SolrClient that connects directly to a CoreContainer.
|
||||
*
|
||||
* @since solr 1.3
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public class EmbeddedSolrServer extends SolrClient {
|
||||
|
||||
private static final long serialVersionUID = -6657217211811383651L;
|
||||
|
||||
protected final CoreContainer coreContainer;
|
||||
protected final String coreName;
|
||||
private final SolrRequestParsers _parser;
|
||||
private final RequestWriterSupplier supplier;
|
||||
private boolean containerIsLocal = false;
|
||||
|
||||
//ClusterPropertiesListener cpl = new ClusterPropertiesListener();
|
||||
|
||||
public enum RequestWriterSupplier {
|
||||
JavaBin(() -> new BinaryRequestWriter()),
|
||||
XML(() -> new RequestWriter());
|
||||
|
||||
private final Supplier<RequestWriter> supplier;
|
||||
|
||||
RequestWriterSupplier(final Supplier<RequestWriter> supplier) {
|
||||
this.supplier = supplier;
|
||||
}
|
||||
|
||||
public RequestWriter newRequestWriter() {
|
||||
return supplier.get();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an EmbeddedSolrServer using a given solr home directory
|
||||
*
|
||||
* @param solrHome the solr home directory
|
||||
* @param defaultCoreName the core to route requests to by default (optional)
|
||||
*/
|
||||
public EmbeddedSolrServer(Path solrHome, String defaultCoreName) {
|
||||
this(load(new CoreContainer(solrHome, new Properties())), defaultCoreName);
|
||||
containerIsLocal = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an EmbeddedSolrServer using a NodeConfig
|
||||
*
|
||||
* @param nodeConfig the configuration
|
||||
* @param defaultCoreName the core to route requests to by default (optional)
|
||||
*/
|
||||
public EmbeddedSolrServer(NodeConfig nodeConfig, String defaultCoreName) {
|
||||
this(load(new CoreContainer(nodeConfig)), defaultCoreName);
|
||||
containerIsLocal = true;
|
||||
}
|
||||
|
||||
private static CoreContainer load(CoreContainer cc) {
|
||||
cc.load();
|
||||
return cc;
|
||||
}
|
||||
|
||||
/** Create an EmbeddedSolrServer wrapping a particular SolrCore */
|
||||
public EmbeddedSolrServer(SolrCore core) {
|
||||
this(core.getCoreContainer(), core.getName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an EmbeddedSolrServer wrapping a CoreContainer.
|
||||
*
|
||||
* @param coreContainer the core container
|
||||
* @param coreName the core to route requests to by default (optional)
|
||||
*/
|
||||
public EmbeddedSolrServer(CoreContainer coreContainer, String coreName) {
|
||||
this(coreContainer, coreName, RequestWriterSupplier.JavaBin);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an EmbeddedSolrServer wrapping a CoreContainer.
|
||||
*
|
||||
* @param coreContainer the core container
|
||||
* @param coreName the core to route requests to by default
|
||||
* @param supplier the supplier used to create a {@link RequestWriter}
|
||||
*/
|
||||
public EmbeddedSolrServer(
|
||||
CoreContainer coreContainer, String coreName, RequestWriterSupplier supplier) {
|
||||
if (coreContainer == null) {
|
||||
throw new NullPointerException("CoreContainer instance required");
|
||||
}
|
||||
this.coreContainer = coreContainer;
|
||||
this.coreName = coreName;
|
||||
_parser = new SolrRequestParsers(null);
|
||||
this.supplier = supplier;
|
||||
}
|
||||
|
||||
// TODO-- this implementation sends the response to XML and then parses it.
|
||||
// It *should* be able to convert the response directly into a named list.
|
||||
|
||||
@Override
|
||||
public NamedList<Object> request(SolrRequest<?> request, String coreName)
|
||||
throws SolrServerException, IOException {
|
||||
|
||||
String path = request.getPath();
|
||||
if (path == null || !path.startsWith("/")) {
|
||||
path = "/select";
|
||||
}
|
||||
|
||||
SolrRequestHandler handler = coreContainer.getRequestHandler(path);
|
||||
if (handler != null) {
|
||||
try {
|
||||
SolrQueryRequest req =
|
||||
_parser.buildRequestFrom(null, request.getParams(), getContentStreams(request));
|
||||
req.getContext().put("httpMethod", request.getMethod().name());
|
||||
req.getContext().put(PATH, path);
|
||||
SolrQueryResponse resp = new SolrQueryResponse();
|
||||
handler.handleRequest(req, resp);
|
||||
checkForExceptions(resp);
|
||||
return BinaryResponseWriter.getParsedResponse(req, resp);
|
||||
} catch (IOException | SolrException iox) {
|
||||
throw iox;
|
||||
} catch (Exception ex) {
|
||||
throw new SolrServerException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
if (coreName == null) {
|
||||
coreName = this.coreName;
|
||||
if (coreName == null) {
|
||||
throw new SolrException(
|
||||
SolrException.ErrorCode.BAD_REQUEST,
|
||||
"No core specified on request and no default core has been set.");
|
||||
}
|
||||
}
|
||||
|
||||
// Check for cores action
|
||||
SolrQueryRequest req = null;
|
||||
try (SolrCore core = coreContainer.getCore(coreName)) {
|
||||
|
||||
if (core == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No such core: " + coreName);
|
||||
}
|
||||
|
||||
SolrParams params = request.getParams();
|
||||
if (params == null) {
|
||||
params = new ModifiableSolrParams();
|
||||
}
|
||||
|
||||
// Extract the handler from the path or params
|
||||
handler = core.getRequestHandler(path);
|
||||
if (handler == null) {
|
||||
if ("/select".equals(path) || "/select/".equalsIgnoreCase(path)) {
|
||||
String qt = params.get(CommonParams.QT);
|
||||
handler = core.getRequestHandler(qt);
|
||||
if (handler == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (handler == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + path);
|
||||
}
|
||||
req = _parser.buildRequestFrom(core, params, getContentStreams(request)/*, request.getUserPrincipal()*/);
|
||||
|
||||
req.getContext().put(PATH, path);
|
||||
req.getContext().put("httpMethod", request.getMethod().name());
|
||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
||||
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
|
||||
|
||||
core.execute(handler, req, rsp);
|
||||
checkForExceptions(rsp);
|
||||
|
||||
// Check if this should stream results
|
||||
if (request.getStreamingResponseCallback() != null) {
|
||||
try {
|
||||
final StreamingResponseCallback callback = request.getStreamingResponseCallback();
|
||||
BinaryResponseWriter.Resolver resolver =
|
||||
new BinaryResponseWriter.Resolver(req, rsp.getReturnFields()) {
|
||||
@Override
|
||||
public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOException {
|
||||
// write an empty list...
|
||||
SolrDocumentList docs = new SolrDocumentList();
|
||||
docs.setNumFound(ctx.getDocList().matches());
|
||||
docs.setNumFoundExact(ctx.getDocList().hitCountRelation() == Relation.EQUAL_TO);
|
||||
docs.setStart(ctx.getDocList().offset());
|
||||
docs.setMaxScore(ctx.getDocList().maxScore());
|
||||
codec.writeSolrDocumentList(docs);
|
||||
|
||||
// This will transform
|
||||
writeResultsBody(ctx, codec);
|
||||
}
|
||||
};
|
||||
|
||||
try (var out =
|
||||
new ByteArrayOutputStream() {
|
||||
ByteArrayInputStream toInputStream() {
|
||||
return new ByteArrayInputStream(buf, 0, count);
|
||||
}
|
||||
}) {
|
||||
createJavaBinCodec(callback, resolver)
|
||||
.setWritableDocFields(resolver)
|
||||
.marshal(rsp.getValues(), out);
|
||||
|
||||
try (ByteArrayInputStream in = out.toInputStream()) {
|
||||
@SuppressWarnings({"unchecked", "resource"})
|
||||
NamedList<Object> resolved = (NamedList<Object>) new JavaBinCodec(resolver).unmarshal(in);
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
// Now write it out
|
||||
NamedList<Object> normalized = BinaryResponseWriter.getParsedResponse(req, rsp);
|
||||
return normalized;
|
||||
} catch (IOException | SolrException iox) {
|
||||
throw iox;
|
||||
} catch (Exception ex) {
|
||||
throw new SolrServerException(ex);
|
||||
} finally {
|
||||
if (req != null) {
|
||||
req.close();
|
||||
SolrRequestInfo.clearRequestInfo();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class BAOS extends ByteArrayOutputStream {
|
||||
public ByteBuffer getByteBuffer() {
|
||||
return ByteBuffer.wrap(super.buf, 0, super.count);
|
||||
}
|
||||
|
||||
/*
|
||||
* A hack to get access to the protected internal buffer and avoid an additional copy
|
||||
*/
|
||||
public byte[] getbuf() {
|
||||
return super.buf;
|
||||
}
|
||||
}
|
||||
|
||||
private Set<ContentStream> getContentStreams(SolrRequest<?> request) throws IOException {
|
||||
if (request.getMethod() == SolrRequest.METHOD.GET) return null;
|
||||
if (request instanceof ContentStreamUpdateRequest) {
|
||||
final ContentStreamUpdateRequest csur = (ContentStreamUpdateRequest) request;
|
||||
final Collection<ContentStream> cs = csur.getContentStreams();
|
||||
if (cs != null) return new HashSet<>(cs);
|
||||
}
|
||||
|
||||
final RequestWriter.ContentWriter contentWriter = request.getContentWriter(null);
|
||||
|
||||
String cType;
|
||||
final BAOS baos = new BAOS();
|
||||
if (contentWriter != null) {
|
||||
contentWriter.write(baos);
|
||||
cType = contentWriter.getContentType();
|
||||
} else {
|
||||
final RequestWriter rw = supplier.newRequestWriter();
|
||||
cType = rw.getUpdateContentType();
|
||||
rw.write(request, baos);
|
||||
}
|
||||
|
||||
final byte[] buf = baos.toByteArray();
|
||||
if (buf.length > 0) {
|
||||
return Collections.singleton(
|
||||
new ContentStreamBase() {
|
||||
|
||||
@Override
|
||||
public InputStream getStream() throws IOException {
|
||||
return new ByteArrayInputStream(buf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType() {
|
||||
return cType;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private JavaBinCodec createJavaBinCodec(
|
||||
final StreamingResponseCallback callback, final BinaryResponseWriter.Resolver resolver) {
|
||||
return new JavaBinCodec(resolver) {
|
||||
|
||||
@Override
|
||||
public void writeSolrDocument(SolrDocument doc) {
|
||||
callback.streamSolrDocument(doc);
|
||||
// super.writeSolrDocument( doc, fields );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeSolrDocumentList(SolrDocumentList docs) throws IOException {
|
||||
if (docs.size() > 0) {
|
||||
SolrDocumentList tmp = new SolrDocumentList();
|
||||
tmp.setMaxScore(docs.getMaxScore());
|
||||
tmp.setNumFound(docs.getNumFound());
|
||||
tmp.setStart(docs.getStart());
|
||||
docs = tmp;
|
||||
}
|
||||
callback.streamDocListInfo(docs.getNumFound(), docs.getStart(), docs.getMaxScore());
|
||||
super.writeSolrDocumentList(docs);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static void checkForExceptions(SolrQueryResponse rsp) throws Exception {
|
||||
if (rsp.getException() != null) {
|
||||
if (rsp.getException() instanceof SolrException) {
|
||||
throw rsp.getException();
|
||||
}
|
||||
throw new SolrServerException(rsp.getException());
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes any resources created by this instance */
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (containerIsLocal) {
|
||||
coreContainer.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Getter method for the CoreContainer
|
||||
*
|
||||
* @return the core container
|
||||
*/
|
||||
public CoreContainer getCoreContainer() {
|
||||
return coreContainer;
|
||||
}
|
||||
}
|
||||
|
|
@ -23,18 +23,23 @@ package net.yacy.cora.federate.solr.instance;
|
|||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
//import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.core.NodeConfig;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.core.SolrXmlConfig;
|
||||
|
||||
import com.google.common.io.Files;
|
||||
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
import net.yacy.cora.federate.solr.embedded.EmbeddedSolrServer;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.kelondro.util.MemoryControl;
|
||||
|
||||
|
|
@ -80,7 +85,19 @@ public class EmbeddedInstance implements SolrInstance {
|
|||
|
||||
// initialize the coreContainer
|
||||
final File configFile = new File(solr_config, "solr.xml"); // the configuration file for all cores
|
||||
this.coreContainer = CoreContainer.createAndLoad(containerPath.toPath(), configFile.toPath()); // this may take indefinitely long if solr files are broken
|
||||
Path solrHome = containerPath.toPath();
|
||||
Path configFilePath = configFile.toPath();
|
||||
//this.coreContainer = CoreContainer.createAndLoad(containerPath.toPath(), configFile.toPath());
|
||||
// this may take indefinitely long if solr files are broken
|
||||
NodeConfig nc = SolrXmlConfig.fromFile(solrHome, configFilePath, new Properties());
|
||||
this.coreContainer = new CoreContainer(nc);
|
||||
try {
|
||||
this.coreContainer.load();
|
||||
} catch (Exception e) {
|
||||
this.coreContainer.shutdown();
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (this.coreContainer == null) throw new IOException("cannot create core container dir = " + containerPath + ", configFile = " + configFile);
|
||||
|
||||
// get the default core from the coreContainer
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ import org.apache.solr.response.QueryResponseWriter;
|
|||
import org.apache.solr.response.RawResponseWriter;
|
||||
import org.apache.solr.response.ResultContext;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.response.XSLTResponseWriter;
|
||||
import org.apache.solr.scripting.xslt.XSLTResponseWriter;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.apache.solr.servlet.SolrRequestParsers;
|
||||
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ public class SolrServlet extends HttpServlet {
|
|||
if (solrRsp.getException() != null) {
|
||||
@SuppressWarnings("rawtypes")
|
||||
NamedList info = new SimpleOrderedMap();
|
||||
@SuppressWarnings("unchecked")
|
||||
int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, null);
|
||||
solrRsp.add("error", info);
|
||||
response.setStatus(code);
|
||||
|
|
|
|||
|
|
@ -874,11 +874,18 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
|
|||
private Date[] getDates(CollectionSchema field) {
|
||||
assert field.isMultiValued();
|
||||
assert field.getType() == SolrType.date;
|
||||
Object content = this.getFieldValue(field.getSolrFieldName());
|
||||
if (content == null) return new Date[0];
|
||||
if (content instanceof Date) {
|
||||
return new Date[] {(Date) content};
|
||||
}
|
||||
if (content instanceof List) {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Date> x = (List<Date>) this.getFieldValue(field.getSolrFieldName());
|
||||
if (x == null) return new Date[0];
|
||||
List<Date> x = (List<Date>) content;
|
||||
return x.toArray(new Date[x.size()]);
|
||||
}
|
||||
return new Date[0];
|
||||
}
|
||||
|
||||
private String getString(CollectionSchema field) {
|
||||
assert !field.isMultiValued();
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ import java.util.regex.Pattern;
|
|||
import java.util.zip.Deflater;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
|
@ -90,10 +89,17 @@ import net.yacy.search.schema.WebgraphSchema;
|
|||
|
||||
public final class Fulltext {
|
||||
|
||||
private static final String SOLR_PATH = "solr_8_11_2"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml
|
||||
private static final String SOLR_PATH = "solr_9_0"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml
|
||||
private static final String SOLR_OLD_PATH[] = new String[]{
|
||||
"solr_36", "solr_40", "solr_44", "solr_45", "solr_46", "solr_47",
|
||||
"solr_4_9", "solr_4_10", "solr_5_2", "solr_5_5", "solr_6_6", "solr_8_8_1"};
|
||||
"solr_4_9", // yacy_v1.80 (solr 4.9.0)
|
||||
"solr_4_10", // yacy_v1.81 (solr 4.10.3), yacy_v1.82 (solr 4.10.3)
|
||||
"solr_5_2", // yacy_v1.90 (solr 5.5.1, sic!)
|
||||
"solr_5_5", // yacy_v1.91 (solr 5.5.2), yacy_v1.92 (solr 5.5.2)
|
||||
"solr_6_6", // yacy_v1.922 (solr 6.6, init fail), yacy_v1.924_20210209_10069 (solr 7.7.3), yacy_v1.926 (solr 8.9.0)
|
||||
"solr_8_8_1", //
|
||||
"solr_8_11_2"
|
||||
};
|
||||
|
||||
// class objects
|
||||
private final File segmentPath;
|
||||
|
|
@ -156,8 +162,7 @@ public final class Fulltext {
|
|||
|
||||
final EmbeddedInstance localCollectionInstance = new EmbeddedInstance(new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr"), solrLocation, CollectionSchema.CORE_NAME, new String[]{CollectionSchema.CORE_NAME, WebgraphSchema.CORE_NAME});
|
||||
final SolrConfig config = localCollectionInstance.getDefaultCore().getSolrConfig();
|
||||
final Version luceneVersion = config.luceneMatchVersion;
|
||||
final String lvn = luceneVersion.major + "_" + luceneVersion.minor + "_" + luceneVersion.bugfix;
|
||||
final String lvn = config.luceneMatchVersion.major + "_" + config.luceneMatchVersion.minor + "_" + config.luceneMatchVersion.bugfix;
|
||||
assert SOLR_PATH.endsWith(lvn) : "luceneVersion = " + lvn + ", solrPath = " + SOLR_PATH + ", check defaults/solr/solrconfig.xml";
|
||||
|
||||
ConcurrentLog.info("Fulltext", "using lucene version " + lvn);
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user