diff --git a/build.properties b/build.properties index 940075c51..3cfbfa0b4 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=11 javacTarget=11 # Release Configuration -releaseVersion=1.930 +releaseVersion=1.940 releaseFileParentDir=yacy privateKeyFile=private.key diff --git a/build.xml b/build.xml index ca24aa5ee..6eb8382af 100644 --- a/build.xml +++ b/build.xml @@ -400,8 +400,8 @@ - + diff --git a/defaults/solr/solrconfig.xml b/defaults/solr/solrconfig.xml index b9d818a49..1dac2a5df 100644 --- a/defaults/solr/solrconfig.xml +++ b/defaults/solr/solrconfig.xml @@ -411,48 +411,32 @@ 1024 - - - + - @@ -463,18 +447,11 @@ document). Since Lucene internal document ids are transient, this cache will not be autowarmed. --> - - - - - + 5 diff --git a/ivy.xml b/ivy.xml index bfa4b96ce..db372e7e5 100644 --- a/ivy.xml +++ b/ivy.xml @@ -4,6 +4,7 @@ + @@ -12,11 +13,11 @@ - + - + - + @@ -28,14 +29,14 @@ - + - + @@ -44,27 +45,30 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - + @@ -89,9 +93,9 @@ - - - + + + @@ -101,5 +105,6 @@ + diff --git a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java index 59f1cb1a0..4a1b7c4fb 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java @@ -28,7 +28,7 @@ import java.util.List; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +//import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; import org.apache.solr.client.solrj.request.LukeRequest; @@ -43,6 +43,7 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; +import net.yacy.cora.federate.solr.embedded.EmbeddedSolrServer; import net.yacy.cora.federate.solr.instance.ServerShard; import net.yacy.cora.util.ConcurrentLog; import net.yacy.search.schema.CollectionSchema; diff --git a/source/net/yacy/cora/federate/solr/embedded/EmbeddedSolrServer.java b/source/net/yacy/cora/federate/solr/embedded/EmbeddedSolrServer.java new file mode 100644 index 000000000..1606775aa --- /dev/null +++ b/source/net/yacy/cora/federate/solr/embedded/EmbeddedSolrServer.java @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// this class was taken from solr 8 package org.apache.solr.client.solrj.embedded; +package net.yacy.cora.federate.solr.embedded; + +import static org.apache.solr.common.params.CommonParams.PATH; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; +import java.util.function.Supplier; +import org.apache.lucene.search.TotalHits.Relation; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.StreamingResponseCallback; +import org.apache.solr.client.solrj.impl.BinaryRequestWriter; +import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; +import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.ContentStreamBase; +import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.BinaryResponseWriter; +import org.apache.solr.response.ResultContext; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.servlet.SolrRequestParsers; + +/** + * SolrClient that connects directly to a CoreContainer. + * + * @since solr 1.3 + */ +@SuppressWarnings("deprecation") +public class EmbeddedSolrServer extends SolrClient { + + private static final long serialVersionUID = -6657217211811383651L; + + protected final CoreContainer coreContainer; + protected final String coreName; + private final SolrRequestParsers _parser; + private final RequestWriterSupplier supplier; + private boolean containerIsLocal = false; + + //ClusterPropertiesListener cpl = new ClusterPropertiesListener(); + + public enum RequestWriterSupplier { + JavaBin(() -> new BinaryRequestWriter()), + XML(() -> new RequestWriter()); + + private final Supplier supplier; + + RequestWriterSupplier(final Supplier supplier) { + this.supplier = supplier; + } + + public RequestWriter newRequestWriter() { + return supplier.get(); + } + } + + /** + * Create an EmbeddedSolrServer using a given solr home directory + * + * @param solrHome the solr home directory + * @param defaultCoreName the core to route requests to by default (optional) + */ + public EmbeddedSolrServer(Path solrHome, String defaultCoreName) { + this(load(new CoreContainer(solrHome, new Properties())), defaultCoreName); + containerIsLocal = true; + } + + /** + * Create an EmbeddedSolrServer using a NodeConfig + * + * @param nodeConfig the configuration + * @param defaultCoreName the core to route requests to by default (optional) + */ + public EmbeddedSolrServer(NodeConfig nodeConfig, String defaultCoreName) { + this(load(new CoreContainer(nodeConfig)), defaultCoreName); + containerIsLocal = true; + } + + private static CoreContainer load(CoreContainer cc) { + cc.load(); + return cc; + } + + /** Create an EmbeddedSolrServer wrapping a particular SolrCore */ + public EmbeddedSolrServer(SolrCore core) { + this(core.getCoreContainer(), core.getName()); + } + + /** + * Create an EmbeddedSolrServer wrapping a CoreContainer. + * + * @param coreContainer the core container + * @param coreName the core to route requests to by default (optional) + */ + public EmbeddedSolrServer(CoreContainer coreContainer, String coreName) { + this(coreContainer, coreName, RequestWriterSupplier.JavaBin); + } + + /** + * Create an EmbeddedSolrServer wrapping a CoreContainer. + * + * @param coreContainer the core container + * @param coreName the core to route requests to by default + * @param supplier the supplier used to create a {@link RequestWriter} + */ + public EmbeddedSolrServer( + CoreContainer coreContainer, String coreName, RequestWriterSupplier supplier) { + if (coreContainer == null) { + throw new NullPointerException("CoreContainer instance required"); + } + this.coreContainer = coreContainer; + this.coreName = coreName; + _parser = new SolrRequestParsers(null); + this.supplier = supplier; + } + + // TODO-- this implementation sends the response to XML and then parses it. + // It *should* be able to convert the response directly into a named list. + + @Override + public NamedList request(SolrRequest request, String coreName) + throws SolrServerException, IOException { + + String path = request.getPath(); + if (path == null || !path.startsWith("/")) { + path = "/select"; + } + + SolrRequestHandler handler = coreContainer.getRequestHandler(path); + if (handler != null) { + try { + SolrQueryRequest req = + _parser.buildRequestFrom(null, request.getParams(), getContentStreams(request)); + req.getContext().put("httpMethod", request.getMethod().name()); + req.getContext().put(PATH, path); + SolrQueryResponse resp = new SolrQueryResponse(); + handler.handleRequest(req, resp); + checkForExceptions(resp); + return BinaryResponseWriter.getParsedResponse(req, resp); + } catch (IOException | SolrException iox) { + throw iox; + } catch (Exception ex) { + throw new SolrServerException(ex); + } + } + + if (coreName == null) { + coreName = this.coreName; + if (coreName == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "No core specified on request and no default core has been set."); + } + } + + // Check for cores action + SolrQueryRequest req = null; + try (SolrCore core = coreContainer.getCore(coreName)) { + + if (core == null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No such core: " + coreName); + } + + SolrParams params = request.getParams(); + if (params == null) { + params = new ModifiableSolrParams(); + } + + // Extract the handler from the path or params + handler = core.getRequestHandler(path); + if (handler == null) { + if ("/select".equals(path) || "/select/".equalsIgnoreCase(path)) { + String qt = params.get(CommonParams.QT); + handler = core.getRequestHandler(qt); + if (handler == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt); + } + } + } + + if (handler == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + path); + } + req = _parser.buildRequestFrom(core, params, getContentStreams(request)/*, request.getUserPrincipal()*/); + + req.getContext().put(PATH, path); + req.getContext().put("httpMethod", request.getMethod().name()); + SolrQueryResponse rsp = new SolrQueryResponse(); + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); + + core.execute(handler, req, rsp); + checkForExceptions(rsp); + + // Check if this should stream results + if (request.getStreamingResponseCallback() != null) { + try { + final StreamingResponseCallback callback = request.getStreamingResponseCallback(); + BinaryResponseWriter.Resolver resolver = + new BinaryResponseWriter.Resolver(req, rsp.getReturnFields()) { + @Override + public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOException { + // write an empty list... + SolrDocumentList docs = new SolrDocumentList(); + docs.setNumFound(ctx.getDocList().matches()); + docs.setNumFoundExact(ctx.getDocList().hitCountRelation() == Relation.EQUAL_TO); + docs.setStart(ctx.getDocList().offset()); + docs.setMaxScore(ctx.getDocList().maxScore()); + codec.writeSolrDocumentList(docs); + + // This will transform + writeResultsBody(ctx, codec); + } + }; + + try (var out = + new ByteArrayOutputStream() { + ByteArrayInputStream toInputStream() { + return new ByteArrayInputStream(buf, 0, count); + } + }) { + createJavaBinCodec(callback, resolver) + .setWritableDocFields(resolver) + .marshal(rsp.getValues(), out); + + try (ByteArrayInputStream in = out.toInputStream()) { + @SuppressWarnings({"unchecked", "resource"}) + NamedList resolved = (NamedList) new JavaBinCodec(resolver).unmarshal(in); + return resolved; + } + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + // Now write it out + NamedList normalized = BinaryResponseWriter.getParsedResponse(req, rsp); + return normalized; + } catch (IOException | SolrException iox) { + throw iox; + } catch (Exception ex) { + throw new SolrServerException(ex); + } finally { + if (req != null) { + req.close(); + SolrRequestInfo.clearRequestInfo(); + } + } + } + + public static class BAOS extends ByteArrayOutputStream { + public ByteBuffer getByteBuffer() { + return ByteBuffer.wrap(super.buf, 0, super.count); + } + + /* + * A hack to get access to the protected internal buffer and avoid an additional copy + */ + public byte[] getbuf() { + return super.buf; + } + } + +private Set getContentStreams(SolrRequest request) throws IOException { + if (request.getMethod() == SolrRequest.METHOD.GET) return null; + if (request instanceof ContentStreamUpdateRequest) { + final ContentStreamUpdateRequest csur = (ContentStreamUpdateRequest) request; + final Collection cs = csur.getContentStreams(); + if (cs != null) return new HashSet<>(cs); + } + + final RequestWriter.ContentWriter contentWriter = request.getContentWriter(null); + + String cType; + final BAOS baos = new BAOS(); + if (contentWriter != null) { + contentWriter.write(baos); + cType = contentWriter.getContentType(); + } else { + final RequestWriter rw = supplier.newRequestWriter(); + cType = rw.getUpdateContentType(); + rw.write(request, baos); + } + + final byte[] buf = baos.toByteArray(); + if (buf.length > 0) { + return Collections.singleton( + new ContentStreamBase() { + + @Override + public InputStream getStream() throws IOException { + return new ByteArrayInputStream(buf); + } + + @Override + public String getContentType() { + return cType; + } + }); + } + + return null; + } + + private JavaBinCodec createJavaBinCodec( + final StreamingResponseCallback callback, final BinaryResponseWriter.Resolver resolver) { + return new JavaBinCodec(resolver) { + + @Override + public void writeSolrDocument(SolrDocument doc) { + callback.streamSolrDocument(doc); + // super.writeSolrDocument( doc, fields ); + } + + @Override + public void writeSolrDocumentList(SolrDocumentList docs) throws IOException { + if (docs.size() > 0) { + SolrDocumentList tmp = new SolrDocumentList(); + tmp.setMaxScore(docs.getMaxScore()); + tmp.setNumFound(docs.getNumFound()); + tmp.setStart(docs.getStart()); + docs = tmp; + } + callback.streamDocListInfo(docs.getNumFound(), docs.getStart(), docs.getMaxScore()); + super.writeSolrDocumentList(docs); + } + }; + } + + private static void checkForExceptions(SolrQueryResponse rsp) throws Exception { + if (rsp.getException() != null) { + if (rsp.getException() instanceof SolrException) { + throw rsp.getException(); + } + throw new SolrServerException(rsp.getException()); + } + } + + /** Closes any resources created by this instance */ + @Override + public void close() throws IOException { + if (containerIsLocal) { + coreContainer.shutdown(); + } + } + + /** + * Getter method for the CoreContainer + * + * @return the core container + */ + public CoreContainer getCoreContainer() { + return coreContainer; + } +} \ No newline at end of file diff --git a/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java b/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java index e9b6cd9c3..4ffa81cef 100644 --- a/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java @@ -23,18 +23,23 @@ package net.yacy.cora.federate.solr.instance; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Path; import java.util.Collection; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +//import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.NodeConfig; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrXmlConfig; import com.google.common.io.Files; import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.federate.solr.embedded.EmbeddedSolrServer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.util.MemoryControl; @@ -80,7 +85,19 @@ public class EmbeddedInstance implements SolrInstance { // initialize the coreContainer final File configFile = new File(solr_config, "solr.xml"); // the configuration file for all cores - this.coreContainer = CoreContainer.createAndLoad(containerPath.toPath(), configFile.toPath()); // this may take indefinitely long if solr files are broken + Path solrHome = containerPath.toPath(); + Path configFilePath = configFile.toPath(); + //this.coreContainer = CoreContainer.createAndLoad(containerPath.toPath(), configFile.toPath()); + // this may take indefinitely long if solr files are broken + NodeConfig nc = SolrXmlConfig.fromFile(solrHome, configFilePath, new Properties()); + this.coreContainer = new CoreContainer(nc); + try { + this.coreContainer.load(); + } catch (Exception e) { + this.coreContainer.shutdown(); + throw e; + } + if (this.coreContainer == null) throw new IOException("cannot create core container dir = " + containerPath + ", configFile = " + configFile); // get the default core from the coreContainer diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index 325757868..ea5590707 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -79,7 +79,7 @@ import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.RawResponseWriter; import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.response.XSLTResponseWriter; +import org.apache.solr.scripting.xslt.XSLTResponseWriter; import org.apache.solr.search.DocList; import org.apache.solr.servlet.SolrRequestParsers; import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; diff --git a/source/net/yacy/http/servlets/SolrServlet.java b/source/net/yacy/http/servlets/SolrServlet.java index 7795a50e9..7b2c621a3 100644 --- a/source/net/yacy/http/servlets/SolrServlet.java +++ b/source/net/yacy/http/servlets/SolrServlet.java @@ -141,6 +141,7 @@ public class SolrServlet extends HttpServlet { if (solrRsp.getException() != null) { @SuppressWarnings("rawtypes") NamedList info = new SimpleOrderedMap(); + @SuppressWarnings("unchecked") int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, null); solrRsp.add("error", info); response.setStatus(code); diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java index 2d93ec8b7..7c2bf2d07 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java @@ -874,10 +874,17 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable x = (List) this.getFieldValue(field.getSolrFieldName()); - if (x == null) return new Date[0]; - return x.toArray(new Date[x.size()]); + Object content = this.getFieldValue(field.getSolrFieldName()); + if (content == null) return new Date[0]; + if (content instanceof Date) { + return new Date[] {(Date) content}; + } + if (content instanceof List) { + @SuppressWarnings("unchecked") + List x = (List) content; + return x.toArray(new Date[x.size()]); + } + return new Date[0]; } private String getString(CollectionSchema field) { diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index cd9680b27..d1d2724d6 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -46,7 +46,6 @@ import java.util.regex.Pattern; import java.util.zip.Deflater; import java.util.zip.GZIPOutputStream; -import org.apache.lucene.util.Version; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; @@ -90,10 +89,17 @@ import net.yacy.search.schema.WebgraphSchema; public final class Fulltext { - private static final String SOLR_PATH = "solr_8_11_2"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml + private static final String SOLR_PATH = "solr_9_0"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml private static final String SOLR_OLD_PATH[] = new String[]{ "solr_36", "solr_40", "solr_44", "solr_45", "solr_46", "solr_47", - "solr_4_9", "solr_4_10", "solr_5_2", "solr_5_5", "solr_6_6", "solr_8_8_1"}; + "solr_4_9", // yacy_v1.80 (solr 4.9.0) + "solr_4_10", // yacy_v1.81 (solr 4.10.3), yacy_v1.82 (solr 4.10.3) + "solr_5_2", // yacy_v1.90 (solr 5.5.1, sic!) + "solr_5_5", // yacy_v1.91 (solr 5.5.2), yacy_v1.92 (solr 5.5.2) + "solr_6_6", // yacy_v1.922 (solr 6.6, init fail), yacy_v1.924_20210209_10069 (solr 7.7.3), yacy_v1.926 (solr 8.9.0) + "solr_8_8_1", // + "solr_8_11_2" + }; // class objects private final File segmentPath; @@ -156,8 +162,7 @@ public final class Fulltext { final EmbeddedInstance localCollectionInstance = new EmbeddedInstance(new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr"), solrLocation, CollectionSchema.CORE_NAME, new String[]{CollectionSchema.CORE_NAME, WebgraphSchema.CORE_NAME}); final SolrConfig config = localCollectionInstance.getDefaultCore().getSolrConfig(); - final Version luceneVersion = config.luceneMatchVersion; - final String lvn = luceneVersion.major + "_" + luceneVersion.minor + "_" + luceneVersion.bugfix; + final String lvn = config.luceneMatchVersion.major + "_" + config.luceneMatchVersion.minor + "_" + config.luceneMatchVersion.bugfix; assert SOLR_PATH.endsWith(lvn) : "luceneVersion = " + lvn + ", solrPath = " + SOLR_PATH + ", check defaults/solr/solrconfig.xml"; ConcurrentLog.info("Fulltext", "using lucene version " + lvn);