mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
Add possibility to set custom Solr field names for the YaCy default Solr attributes.
- Changing the format of YaCy's solr.key.list while maintainig backward compatibility Federated index config screens adjusted accordingly - modified the Solr update request to use a 3 min Solr autocommit intervall
This commit is contained in:
reger
parent
a48f37bf70
commit
b2175ea4ef
|
|
@ -65,17 +65,19 @@
|
|||
</dl>
|
||||
</div>
|
||||
<div>
|
||||
<h3>Index Scheme</h3>
|
||||
<h3>Index Scheme</h3><p>If you use a custom Solr schema you may enter a different field name in the column 'cutom Solr Field Name' of the YacY default attribute name</p>
|
||||
<table class="sortable" border="0" cellpadding="2" cellspacing="1">
|
||||
<tr class="TableHeader" valign="bottom">
|
||||
<td>Active</td>
|
||||
<td>Attribute</td>
|
||||
<td>custom Solr Field Name</td>
|
||||
<td>Comment</td>
|
||||
</tr>
|
||||
#{scheme}#
|
||||
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
|
||||
<td align="center"><input type="checkbox" name="scheme_#[key]#" value="checked" #(checked)#::checked="checked"#(/checked)#/></td>
|
||||
<td align="left">#[key]#</td>
|
||||
<td align="left"><input type="text" name="scheme_solrfieldname_#[key]#" value="#[solrfieldname]#"/></td>
|
||||
<td align="left">#[comment]#</td>
|
||||
</tr>
|
||||
#{/scheme}#
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@
|
|||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -39,10 +38,9 @@ import net.yacy.cora.storage.ConfigurationSet;
|
|||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.index.Segments;
|
||||
import net.yacy.search.index.SolrConfiguration;
|
||||
import net.yacy.search.index.SolrField;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
import net.yacy.search.index.SolrField;
|
||||
|
||||
public class IndexFederated_p {
|
||||
|
||||
|
|
@ -87,8 +85,6 @@ public class IndexFederated_p {
|
|||
sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr(null);
|
||||
}
|
||||
|
||||
final SolrConfiguration scheme = new SolrConfiguration(new File(env.getDataPath(), "DATA/SETTINGS/" + schemename));
|
||||
|
||||
if (solrIsOnAfterwards) {
|
||||
// switch on
|
||||
final boolean usesolr = sb.getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0;
|
||||
|
|
@ -101,23 +97,32 @@ public class IndexFederated_p {
|
|||
}
|
||||
|
||||
// read index scheme table flags
|
||||
final Iterator<ConfigurationSet.Entry> i = scheme.allIterator();
|
||||
final Iterator<ConfigurationSet.Entry> i = sb.solrScheme.entryIterator();
|
||||
ConfigurationSet.Entry entry;
|
||||
boolean modified = false; // flag to remember changes
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
final String v = post.get("scheme_" + entry.key());
|
||||
final String sfn = post.get("scheme_solrfieldname_" + entry.key());
|
||||
if (sfn != null ) {
|
||||
// set custom solr field name
|
||||
if (!sfn.equals(entry.getValue())) {
|
||||
entry.setValue(sfn);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
// set enable flag
|
||||
final boolean c = v != null && v.equals("checked");
|
||||
if (entry.enabled() != c) {
|
||||
entry.setEnable(c);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
if (modified) { // save settings to config file if modified
|
||||
try {
|
||||
if (entry.enabled()) {
|
||||
if (!c) {
|
||||
scheme.disable(entry.key());
|
||||
}
|
||||
} else {
|
||||
if (c) {
|
||||
scheme.enable(entry.key());
|
||||
}
|
||||
}
|
||||
} catch (final IOException e) {}
|
||||
sb.solrScheme.commit();
|
||||
modified = false;
|
||||
} catch (IOException ex) {}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -140,26 +145,29 @@ public class IndexFederated_p {
|
|||
|
||||
// write scheme
|
||||
final String schemename = sb.getConfig("federated.service.solr.indexing.schemefile", "solr.keys.default.list");
|
||||
final Iterator<ConfigurationSet.Entry> i = sb.solrScheme.allIterator();
|
||||
|
||||
int c = 0;
|
||||
boolean dark = false;
|
||||
// use enum SolrField to keep defined order
|
||||
for(SolrField field : SolrField.values()) {
|
||||
prop.put("scheme_" + c + "_dark", dark ? 1 : 0); dark = !dark;
|
||||
prop.put("scheme_" + c + "_checked", sb.solrScheme.contains(field.name()) ? 1 : 0);
|
||||
prop.putHTML("scheme_" + c + "_key", field.name());
|
||||
prop.putHTML("scheme_" + c + "_solrfieldname",field.name().equalsIgnoreCase(field.getSolrFieldName()) ? "" : field.getSolrFieldName());
|
||||
if (field.getComment() != null) prop.putHTML("scheme_" + c + "_comment",field.getComment());
|
||||
c++;
|
||||
}
|
||||
/* final Iterator<ConfigurationSet.Entry> i = sb.solrScheme.entryIterator();
|
||||
ConfigurationSet.Entry entry;
|
||||
SolrField field;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
try {
|
||||
field = SolrField.valueOf(entry.key());
|
||||
} catch (IllegalArgumentException e) {
|
||||
continue;
|
||||
}
|
||||
if (field == null) continue;
|
||||
prop.put("scheme_" + c + "_dark", dark ? 1 : 0); dark = !dark;
|
||||
prop.put("scheme_" + c + "_checked", entry.enabled() ? 1 : 0);
|
||||
prop.putHTML("scheme_" + c + "_key", entry.key());
|
||||
prop.putHTML("scheme_" + c + "_comment",field.getComment() /*scheme.commentHeadline(entry.key())*/);
|
||||
prop.putHTML("scheme_" + c + "_solrfieldname",entry.getValue() == null ? "" : entry.getValue());
|
||||
if (entry.getComment() != null) prop.putHTML("scheme_" + c + "_comment",entry.getComment());
|
||||
c++;
|
||||
}
|
||||
}*/
|
||||
prop.put("scheme", c);
|
||||
|
||||
// fill attribute fields
|
||||
|
|
|
|||
|
|
@ -22,10 +22,7 @@
|
|||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.storage.ConfigurationSet;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.index.SolrField;
|
||||
import de.anomic.server.serverObjects;
|
||||
|
|
@ -39,20 +36,10 @@ public class schema_p {
|
|||
final Switchboard sb = (Switchboard) env;
|
||||
|
||||
// write scheme
|
||||
final Iterator<ConfigurationSet.Entry> i = sb.solrScheme.allIterator();
|
||||
|
||||
int c = 0;
|
||||
ConfigurationSet.Entry entry;
|
||||
SolrField field = null;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
if (!entry.enabled()) continue; //scheme.contains(entry.key())
|
||||
try {
|
||||
field = SolrField.valueOf(entry.key());
|
||||
} catch (IllegalArgumentException e) {
|
||||
continue;
|
||||
}
|
||||
prop.put("fields_" + c + "_name", field.name());
|
||||
for (SolrField field : SolrField.values()) {
|
||||
if (sb.solrScheme.contains(field.name())) {
|
||||
prop.put("fields_" + c + "_solrname", field.getSolrFieldName());
|
||||
prop.put("fields_" + c + "_type", field.getType().printName());
|
||||
prop.put("fields_" + c + "_comment", field.getComment());
|
||||
prop.put("fields_" + c + "_indexedChecked", field.isIndexed() ? 1 : 0);
|
||||
|
|
@ -61,8 +48,11 @@ public class schema_p {
|
|||
prop.put("fields_" + c + "_omitNormsChecked", field.isOmitNorms() ? 1 : 0);
|
||||
c++;
|
||||
}
|
||||
}
|
||||
prop.put("fields", c);
|
||||
|
||||
prop.put("solruniquekey",SolrField.id.getSolrFieldName());
|
||||
prop.put("solrdefaultsearchfield",SolrField.text_t.getSolrFieldName());
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
|
|
|||
6
htroot/api/schema_p.xml
Executable file → Normal file
6
htroot/api/schema_p.xml
Executable file → Normal file
|
|
@ -51,13 +51,13 @@
|
|||
<fields>
|
||||
#{fields}#
|
||||
<!-- #[comment]# -->
|
||||
<field name="#[name]#" type="#[type]#"#(indexedChecked)#:: indexed="true"#(/indexedChecked)##(storedChecked)#:: stored="true"#(/storedChecked)##(multiValuedChecked)#:: multiValued="true"#(/multiValuedChecked)##(omitNormsChecked)#:: omitNorms="true"#(/omitNormsChecked)#/>
|
||||
<field name="#[solrname]#" type="#[type]#"#(indexedChecked)#:: indexed="true"#(/indexedChecked)##(storedChecked)#:: stored="true"#(/storedChecked)##(multiValuedChecked)#:: multiValued="true"#(/multiValuedChecked)##(omitNormsChecked)#:: omitNorms="true"#(/omitNormsChecked)#/>
|
||||
|
||||
#{/fields}#
|
||||
</fields>
|
||||
|
||||
<uniqueKey>id</uniqueKey>
|
||||
<defaultSearchField>sku</defaultSearchField>
|
||||
<uniqueKey>#[solruniquekey]#</uniqueKey>
|
||||
<defaultSearchField>#[solrdefaultsearchfield]#</defaultSearchField>
|
||||
<solrQueryParser defaultOperator="AND"/>
|
||||
</schema>
|
||||
|
||||
|
|
|
|||
|
|
@ -41,35 +41,35 @@ public class SolrDoc extends SolrInputDocument {
|
|||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final String value) {
|
||||
this.setField(key.name(), value);
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final Date value) {
|
||||
this.setField(key.name(), value);
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final int value) {
|
||||
this.setField(key.name(), value);
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final String[] value) {
|
||||
this.setField(key.name(), value);
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final List<String> value) {
|
||||
this.setField(key.name(), value.toArray(new String[value.size()]));
|
||||
this.setField(key.getSolrFieldName(), value.toArray(new String[value.size()]));
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final float value) {
|
||||
this.setField(key.name(), value);
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final boolean value) {
|
||||
this.setField(key.name(), value);
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final String value, final float boost) {
|
||||
this.setField(key.name(), value, boost);
|
||||
this.setField(key.getSolrFieldName(), value, boost);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,8 +31,9 @@ public interface SolrField {
|
|||
* this shall be implemented as enum, thus shall have the name() method
|
||||
* @return the name of the enum constant
|
||||
*/
|
||||
public String name();
|
||||
public String name(); // default field name (according to SolCell default scheme) <= enum.name()
|
||||
|
||||
public String getSolrFieldName(); // return the default or custom solr field name to use for solr requests
|
||||
|
||||
public SolrType getType();
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import java.nio.charset.Charset;
|
|||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import net.yacy.search.index.SolrField;
|
||||
|
||||
public class SolrShardingSelection {
|
||||
|
||||
|
|
@ -58,7 +59,7 @@ public class SolrShardingSelection {
|
|||
|
||||
public int select(final SolrDoc solrdoc) throws IOException {
|
||||
if (this.method == Method.MODULO_HOST_MD5) {
|
||||
final String sku = (String) solrdoc.getField("sku").getValue();
|
||||
final String sku = (String) solrdoc.getField(SolrField.sku.getSolrFieldName()).getValue();
|
||||
return selectURL(sku);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ import org.apache.solr.client.solrj.response.QueryResponse;
|
|||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import net.yacy.search.index.SolrField;
|
||||
|
||||
|
||||
public class SolrSingleConnector implements SolrConnector {
|
||||
|
|
@ -108,7 +109,7 @@ public class SolrSingleConnector implements SolrConnector {
|
|||
}
|
||||
this.server.setAllowCompression(true);
|
||||
this.server.setConnectionTimeout(60000);
|
||||
this.server.setMaxRetries(10);
|
||||
this.server.setMaxRetries(1); // Solr-Doc: No more than 1 recommended (depreciated)
|
||||
this.server.setSoTimeout(60000);
|
||||
}
|
||||
|
||||
|
|
@ -169,7 +170,7 @@ public class SolrSingleConnector implements SolrConnector {
|
|||
@Override
|
||||
public boolean exists(final String id) throws IOException {
|
||||
try {
|
||||
final SolrDocumentList list = get("id:" + id, 0, 1);
|
||||
final SolrDocumentList list = get(SolrField.id.getSolrFieldName() + ":" + id, 0, 1);
|
||||
return list.getNumFound() > 0;
|
||||
} catch (final Throwable e) {
|
||||
Log.logException(e);
|
||||
|
|
@ -195,7 +196,7 @@ public class SolrSingleConnector implements SolrConnector {
|
|||
@Override
|
||||
public void add(final SolrDoc solrdoc) throws IOException, SolrException {
|
||||
try {
|
||||
this.server.add(solrdoc);
|
||||
this.server.add(solrdoc,180000); // commitWithIn 180s
|
||||
//this.server.commit();
|
||||
} catch (SolrServerException e) {
|
||||
Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdoc.toString());
|
||||
|
|
@ -208,7 +209,7 @@ public class SolrSingleConnector implements SolrConnector {
|
|||
ArrayList<SolrInputDocument> l = new ArrayList<SolrInputDocument>();
|
||||
for (SolrDoc d: solrdocs) l.add(d);
|
||||
try {
|
||||
this.server.add(l);
|
||||
this.server.add(l,180000); // commitWithIn 120s
|
||||
//this.server.commit();
|
||||
} catch (SolrServerException e) {
|
||||
Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdocs.toString());
|
||||
|
|
@ -263,12 +264,12 @@ public class SolrSingleConnector implements SolrConnector {
|
|||
public static void main(final String args[]) {
|
||||
SolrSingleConnector solr;
|
||||
try {
|
||||
//SolrScheme scheme = new SolrScheme();
|
||||
solr = new SolrSingleConnector("http://127.0.0.1:8983/solr");
|
||||
solr.clear();
|
||||
final File exampleDir = new File("/Data/workspace2/yacy/test/parsertest/");
|
||||
final File exampleDir = new File("test/parsertest/");
|
||||
long t, t0, a = 0;
|
||||
int c = 0;
|
||||
System.out.println("push files in " + exampleDir.getAbsolutePath() + " to Solr");
|
||||
for (final String s: exampleDir.list()) {
|
||||
if (s.startsWith(".")) continue;
|
||||
t = System.currentTimeMillis();
|
||||
|
|
|
|||
|
|
@ -30,11 +30,12 @@ import java.io.File;
|
|||
import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.AbstractSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.Set;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.cora.storage.ConfigurationSet.Entry;
|
||||
import net.yacy.search.index.SolrField;
|
||||
/**
|
||||
* this class reads configuration attributes as a list of keywords from a list
|
||||
* the list may contain lines with one keyword, comment lines, empty lines and out-commented keyword lines
|
||||
|
|
@ -44,61 +45,85 @@ import java.util.Set;
|
|||
* - all lines beginning with '##' are comments
|
||||
* - all non-empty lines not beginning with '#' are keyword lines
|
||||
* - all lines beginning with '#' and where the second character is not '#' are commented-out keyword lines
|
||||
*
|
||||
* - all text after a '#' not at beginn of line is treated as comment (like 'key = value # comment' )
|
||||
* - a line may contain a key only or a key=value pair
|
||||
* @author Michael Christen
|
||||
*/
|
||||
public class ConfigurationSet extends AbstractSet<String> implements Set<String> {
|
||||
|
||||
public class ConfigurationSet extends TreeMap<String,Entry> {
|
||||
private static final long serialVersionUID = 1L;
|
||||
private final File file;
|
||||
private String[] lines;
|
||||
|
||||
public ConfigurationSet() {
|
||||
this.file = null;
|
||||
this.lines = new String[0];
|
||||
}
|
||||
|
||||
public ConfigurationSet(final File file) {
|
||||
this.file = file;
|
||||
try {
|
||||
final BufferedReader br = new BufferedReader(new FileReader(this.file));
|
||||
final LinkedList<String> sl = new LinkedList<String>();
|
||||
String s;
|
||||
while ((s = br.readLine()) != null) sl.add(s.trim());
|
||||
this.lines = new String[sl.size()];
|
||||
int c = 0;
|
||||
for (final String s0: sl) this.lines[c++] = s0;
|
||||
} catch (final IOException e) {
|
||||
this.lines = new String[0];
|
||||
boolean enabled;
|
||||
String comment, key, value;
|
||||
int i;
|
||||
comment = null;
|
||||
while ((s = br.readLine()) != null) {
|
||||
|
||||
if (s.startsWith("##") || s.isEmpty()){
|
||||
// is comment line - do nothing
|
||||
if (s.startsWith("##")) comment = s.substring(2);
|
||||
continue;
|
||||
} else {
|
||||
if (s.startsWith("#")) {
|
||||
enabled = false ;
|
||||
s = s.substring (1).trim();
|
||||
} else {
|
||||
enabled = true;
|
||||
}
|
||||
if (s.contains("#")) {
|
||||
// second # = text afterwards is a comment
|
||||
i = s.indexOf("#");
|
||||
comment = s.substring(i+1);
|
||||
s = s.substring(0,i).trim();
|
||||
} else {
|
||||
// comment = null;
|
||||
}
|
||||
if (s.contains("=")) {
|
||||
i = s.indexOf("=");
|
||||
key = s.substring(0,i).trim();
|
||||
value = s.substring(i+1).trim();
|
||||
if (value.isEmpty()) value = null;
|
||||
|
||||
} else {
|
||||
key = s.trim();
|
||||
value = null;
|
||||
}
|
||||
if (!key.isEmpty()) {
|
||||
Entry entry = new Entry(key, value, enabled);
|
||||
if (comment != null) {
|
||||
entry.setComment(comment);
|
||||
comment = null;
|
||||
}
|
||||
this.put(key, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (final IOException e) {}
|
||||
}
|
||||
|
||||
/**
|
||||
* override the abstract implementation because that is not stable in concurrent requests
|
||||
*/
|
||||
@Override
|
||||
public boolean contains(Object o) {
|
||||
if (o == null || !(o instanceof String)) return false;
|
||||
String s = (String) o;
|
||||
synchronized (this) {
|
||||
for (String line : this.lines) {
|
||||
if (line != null && line.equals(s)) return true;
|
||||
public boolean contains (String key) {
|
||||
if (key == null) return false;
|
||||
Entry e = this.get(key);
|
||||
return e == null ? false : e.enabled();
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean containsDisabled(final String o) {
|
||||
if (o == null) return false;
|
||||
final Iterator<Entry> i = new EntryIterator();
|
||||
Entry e;
|
||||
while (i.hasNext()) {
|
||||
e = i.next();
|
||||
if (!e.enabled() && o.equals(e.key)) return true;
|
||||
}
|
||||
return false;
|
||||
Entry e = this.get(o);
|
||||
return e == null ? false : !e.enabled();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean add(final String key) {
|
||||
return add(key, null);
|
||||
}
|
||||
|
|
@ -108,244 +133,205 @@ public class ConfigurationSet extends AbstractSet<String> implements Set<String>
|
|||
}
|
||||
|
||||
public boolean add(final String key, final String comment, final boolean enabled) {
|
||||
if (contains(key)) {
|
||||
try {
|
||||
if (!enabled) disable(key);
|
||||
} catch (final IOException e) {
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (containsDisabled(key)) {
|
||||
try {
|
||||
if (enabled) enable(key);
|
||||
} catch (final IOException e) {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
// extend the lines
|
||||
final String[] l = new String[this.lines.length + (comment == null ? 2 : 3)];
|
||||
System.arraycopy(this.lines, 0, l, 0, this.lines.length);
|
||||
l[this.lines.length] = "";
|
||||
if (comment != null) l[this.lines.length + 1] = "## " + comment;
|
||||
l[this.lines.length + (comment == null ? 1 : 2)] = enabled ? key : "#" + key;
|
||||
this.lines = l;
|
||||
boolean modified = false;
|
||||
Entry entry = get(key);
|
||||
if (entry == null) {
|
||||
entry = new Entry (key,enabled);
|
||||
if (comment != null) entry.setComment(comment);
|
||||
this.put (key,entry);
|
||||
modified = true;
|
||||
} else {
|
||||
if (entry.enabled() != enabled) {
|
||||
entry.setEnable(enabled);
|
||||
modified = true;
|
||||
}
|
||||
if ( (comment != null) && ( !comment.equals(entry.getComment()) )) {
|
||||
entry.setComment(comment);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (modified) {
|
||||
commit();
|
||||
try {
|
||||
SolrField f = SolrField.valueOf(key);
|
||||
f.setSolrFieldName(entry.getValue());
|
||||
} catch (IllegalArgumentException e) {}
|
||||
}
|
||||
|
||||
} catch (final IOException e) {
|
||||
}
|
||||
return false;
|
||||
return modified;
|
||||
}
|
||||
|
||||
public void fill(final ConfigurationSet other, final boolean defaultActivated) {
|
||||
final Iterator<Entry> i = other.allIterator();
|
||||
Entry e;
|
||||
final Iterator<Entry> i = other.entryIterator();
|
||||
Entry e, enew = null;
|
||||
while (i.hasNext()) {
|
||||
e = i.next();
|
||||
if (contains(e.key) || containsDisabled(e.key)) continue;
|
||||
this.add(e.key(), other.commentHeadline(e.key()), defaultActivated && e.enabled());
|
||||
// add as new entry
|
||||
enew = new Entry(e.key(),e.getValue(),defaultActivated && e.enabled());
|
||||
enew.setComment(e.getComment());
|
||||
this.put (e.key(),enew);
|
||||
}
|
||||
if (enew != null) {
|
||||
try {
|
||||
commit();
|
||||
} catch (IOException ex) {
|
||||
Logger.getLogger(ConfigurationSet.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
// a shortcut to a fast 'true' in case that we initialized the class without a configuration file
|
||||
return this.lines == null || this.lines.length == 0 || super.isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* save the configuration back to the file
|
||||
* @throws IOException
|
||||
*/
|
||||
private void commit() throws IOException {
|
||||
public void commit() throws IOException {
|
||||
if (this.file == null) return;
|
||||
// create a temporary bak file, use it as template to preserve user comments
|
||||
File bakfile = new File (file.getAbsolutePath() + ".bak");
|
||||
FileUtils.copy (this.file, bakfile);
|
||||
|
||||
TreeMap tclone = (TreeMap) this.clone(); // clone to write appended entries
|
||||
|
||||
final BufferedWriter writer = new BufferedWriter(new FileWriter(this.file));
|
||||
for (final String s: this.lines) {
|
||||
writer.write(s);
|
||||
try {
|
||||
final BufferedReader reader = new BufferedReader(new FileReader(bakfile));
|
||||
String s, sorig;
|
||||
String key;
|
||||
int i;
|
||||
while ((sorig = reader.readLine()) != null) {
|
||||
|
||||
if (sorig.startsWith("##") || sorig.isEmpty()){
|
||||
// is comment line - write as is
|
||||
writer.write(sorig + "\n");
|
||||
continue;
|
||||
} else {
|
||||
if (sorig.startsWith("#")) {
|
||||
s = sorig.substring (1).trim();
|
||||
} else {
|
||||
s = sorig;
|
||||
}
|
||||
if (s.contains("#")) {
|
||||
// second # = is a line comment
|
||||
i = s.indexOf("#");
|
||||
s = s.substring(0,i).trim();
|
||||
}
|
||||
if (s.contains("=")) {
|
||||
i = s.indexOf("=");
|
||||
key = s.substring(0,i).trim();
|
||||
} else {
|
||||
key = s.trim();
|
||||
}
|
||||
if (!key.isEmpty()) {
|
||||
Entry e = this.get(key);
|
||||
if (e != null) {
|
||||
writer.write (e.toString());
|
||||
tclone.remove(key); // remove written entries from clone
|
||||
} else {writer.write(sorig); }
|
||||
writer.write("\n");
|
||||
} else {
|
||||
writer.write(sorig+"\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
reader.close();
|
||||
bakfile.delete();
|
||||
} catch (final IOException e) {}
|
||||
|
||||
// write remainig entries (not already written)
|
||||
Iterator ie = tclone.entrySet().iterator();
|
||||
while (ie.hasNext()) {
|
||||
Object e = ie.next();
|
||||
writer.write (e.toString() + "\n");
|
||||
}
|
||||
writer.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
/*
|
||||
public Iterator<String> iterator() {
|
||||
return new LineIterator(true);
|
||||
return this.keySet().iterator();
|
||||
}
|
||||
|
||||
public Iterator<String> disabledIterator() {
|
||||
return new LineIterator(false);
|
||||
}
|
||||
|
||||
public Iterator<Entry> allIterator() {
|
||||
return new EntryIterator();
|
||||
}
|
||||
|
||||
private boolean isCommentLine(final int line) {
|
||||
return this.lines[line].startsWith("##");
|
||||
}
|
||||
|
||||
private boolean isKeyLine(final int line) {
|
||||
return this.lines[line].length() > 0 && this.lines[line].charAt(0) != '#';
|
||||
}
|
||||
|
||||
private boolean isDisabledLine(final int line) {
|
||||
return this.lines[line].length() > 1 && this.lines[line].charAt(0) == '#' && this.lines[line].charAt(1) != '#';
|
||||
}
|
||||
|
||||
public void enable(final String key) throws IOException {
|
||||
for (int i = 0; i < this.lines.length; i++) {
|
||||
if (isDisabledLine(i) && this.lines[i].substring(1).trim().equals(key)) {
|
||||
this.lines[i] = key;
|
||||
commit();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void disable(final String key) throws IOException {
|
||||
for (int i = 0; i < this.lines.length; i++) {
|
||||
if (isKeyLine(i) && this.lines[i].equals(key)) {
|
||||
this.lines[i] = "#" + key;
|
||||
commit();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String commentHeadline(final String key) {
|
||||
for (int i = 1; i < this.lines.length; i++) {
|
||||
if (this.lines[i].equals(key) ||
|
||||
(isDisabledLine(i) && this.lines[i].substring(1).trim().equals(key))
|
||||
) {
|
||||
return isCommentLine(i - 1) ? this.lines[i - 1].substring(2).trim() : "";
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public class LineIterator implements Iterator<String> {
|
||||
|
||||
EntryIterator i;
|
||||
Entry nextEntry;
|
||||
private final boolean enabled;
|
||||
|
||||
public LineIterator(final boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
this.i = new EntryIterator();
|
||||
findNextValid();
|
||||
}
|
||||
|
||||
public void findNextValid() {
|
||||
while (this.i.hasNext()) {
|
||||
this.nextEntry = this.i.next();
|
||||
if (this.nextEntry.enabled() == this.enabled) return;
|
||||
}
|
||||
this.nextEntry = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.nextEntry != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String next() {
|
||||
if (this.nextEntry == null) return null;
|
||||
final String s = this.nextEntry.key();
|
||||
findNextValid();
|
||||
return s;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class EntryIterator implements Iterator<Entry> {
|
||||
|
||||
private int line;
|
||||
|
||||
public EntryIterator() {
|
||||
this.line = -1;
|
||||
findNextKeywordLine();
|
||||
}
|
||||
|
||||
/**
|
||||
* increase line counter until it points to the next keyword line
|
||||
* @return true if a next line was found, false if EOL
|
||||
*/
|
||||
private boolean findNextKeywordLine() {
|
||||
this.line++;
|
||||
if (this.line >= ConfigurationSet.this.lines.length) return false;
|
||||
while (ConfigurationSet.this.lines[this.line].length() == 0 ||
|
||||
ConfigurationSet.this.lines[this.line].startsWith("##")) {
|
||||
this.line++;
|
||||
if (this.line >= ConfigurationSet.this.lines.length) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.line < ConfigurationSet.this.lines.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry next() {
|
||||
final String s = ConfigurationSet.this.lines[this.line];
|
||||
findNextKeywordLine();
|
||||
if (s.charAt(0) == '#') return new Entry(s.substring(1).trim(), false);
|
||||
return new Entry(s, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
*/
|
||||
public Iterator<Entry> entryIterator() {
|
||||
return this.values().iterator();
|
||||
}
|
||||
|
||||
public class Entry {
|
||||
private final String key;
|
||||
private final boolean enabled;
|
||||
private String value;
|
||||
private boolean enabled;
|
||||
private String comment;
|
||||
|
||||
public Entry(final String key, final boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
// split in key, value if line contains a "=" (equal sign) e.g. myattribute = 123
|
||||
// for backward compatibility here the key parameter is checked to contain a "="
|
||||
if (key.contains("=")) {
|
||||
int i = key.indexOf("=");
|
||||
this.key = key.substring(0,i).trim();
|
||||
this.value = key.substring(i+1).trim();
|
||||
} else {
|
||||
this.key = key;
|
||||
this.value = null;
|
||||
}
|
||||
}
|
||||
public Entry (final String key, String value, final boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
public String key() {
|
||||
return this.key;
|
||||
}
|
||||
public void setValue(String theValue) {
|
||||
//empty string not wanted
|
||||
if ((theValue != null) && theValue.isEmpty()) {
|
||||
this.value = null;
|
||||
} else {
|
||||
this.value = theValue;
|
||||
}
|
||||
}
|
||||
public String getValue() {
|
||||
return this.value;
|
||||
}
|
||||
public void setComment(String comment) {
|
||||
this.comment = comment;
|
||||
}
|
||||
public String getComment() {
|
||||
return this.comment;
|
||||
}
|
||||
public void setEnable(boolean value){
|
||||
this.enabled = value;
|
||||
}
|
||||
public boolean enabled() {
|
||||
return this.enabled;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
int c = 0;
|
||||
for (final String s: this.lines) {
|
||||
if (s.length() > 0 && s.charAt(0) != '#') c++;
|
||||
public String toString(){
|
||||
// output string to write to config file
|
||||
return (this.enabled ? "" : "#") + (this.value != null ? this.key + " = " + this.value : this.key ) + (this.comment != null ? " #" + this.comment : "");
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
if (args.length == 0) return;
|
||||
final File f = new File(args[0]);
|
||||
final File f = new File (args[0]);
|
||||
final ConfigurationSet cs = new ConfigurationSet(f);
|
||||
Iterator<String> i = cs.iterator();
|
||||
String k;
|
||||
Iterator<Entry> i = cs.entryIterator();
|
||||
Entry k;
|
||||
System.out.println("\nall activated attributes:");
|
||||
while (i.hasNext()) {
|
||||
k = i.next();
|
||||
System.out.println(k + " - " + cs.commentHeadline(k));
|
||||
if (k.enabled()) System.out.println(k.toString());
|
||||
}
|
||||
i = cs.disabledIterator();
|
||||
i = cs.entryIterator();
|
||||
System.out.println("\nall deactivated attributes:");
|
||||
while (i.hasNext()) {
|
||||
k = i.next();
|
||||
System.out.println(k + " - " + cs.commentHeadline(k));
|
||||
if (!k.enabled()) System.out.println(k.toString() );
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -645,8 +645,8 @@ public final class Switchboard extends serverSwitch
|
|||
this.solrScheme = new SolrConfiguration(solrWorkProfile);
|
||||
|
||||
// update the working scheme with the backup scheme. This is necessary to include new features.
|
||||
// new features are always activated by default
|
||||
this.solrScheme.fill(backupScheme, false);
|
||||
// new features are always activated by default (if activated in input-backupScheme)
|
||||
this.solrScheme.fill(backupScheme, true);
|
||||
|
||||
// set up the solr interface
|
||||
final String solrurls = getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr");
|
||||
|
|
|
|||
|
|
@ -29,13 +29,7 @@ import java.io.File;
|
|||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
|
|
@ -65,25 +59,23 @@ public class SolrConfiguration extends ConfigurationSet {
|
|||
/**
|
||||
* initialize the scheme with a given configuration file
|
||||
* the configuration file simply contains a list of lines with keywords
|
||||
* or keyword = value lines (while value is a custom Solr field name
|
||||
* @param configurationFile
|
||||
*/
|
||||
public SolrConfiguration(final File configurationFile) {
|
||||
super(configurationFile);
|
||||
// check consistency: compare with YaCyField enum
|
||||
for (String name: this) {
|
||||
if (this.isEmpty()) return;
|
||||
Iterator<Entry> it = this.entryIterator();
|
||||
for (ConfigurationSet.Entry etr = it.next(); it.hasNext(); etr = it.next()) {
|
||||
try {
|
||||
SolrField.valueOf(name);
|
||||
SolrField f = SolrField.valueOf(etr.key());
|
||||
f.setSolrFieldName(etr.getValue());
|
||||
} catch (IllegalArgumentException e) {
|
||||
Log.logWarning("SolrScheme", "solr scheme file " + configurationFile.getAbsolutePath() + " defines unknown attribute '" + name + "'");
|
||||
Log.logWarning("SolrScheme", "solr scheme file " + configurationFile.getAbsolutePath() + " defines unknown attribute '" + etr.toString() + "'");
|
||||
it.remove();
|
||||
}
|
||||
}
|
||||
/*
|
||||
for (YaCyField field: YaCyField.values()) {
|
||||
if (!this.contains(field.name())) {
|
||||
Log.logWarning("SolrScheme", "solr scheme file " + configurationFile.getAbsolutePath() + " omits known attribute '" + field.name() + "'");
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final String value) {
|
||||
|
|
@ -114,8 +106,30 @@ public class SolrConfiguration extends ConfigurationSet {
|
|||
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value, boost);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* save configuration to file and update enum SolrFields
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void commit() throws IOException {
|
||||
try {
|
||||
super.commit();
|
||||
// make sure the enum SolrField.SolrFieldName is current
|
||||
Iterator<Entry> it = this.entryIterator();
|
||||
for (ConfigurationSet.Entry etr = it.next(); it.hasNext(); etr = it.next()) {
|
||||
try {
|
||||
SolrField f = SolrField.valueOf(etr.key());
|
||||
f.setSolrFieldName(etr.getValue());
|
||||
} catch (IllegalArgumentException e) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} catch (final IOException e) {}
|
||||
}
|
||||
|
||||
public SolrDoc yacy2solr(final String id, final ResponseHeader header, final Document yacydoc) {
|
||||
// we user the SolrCell design as index scheme
|
||||
// we use the SolrCell design as index scheme
|
||||
final SolrDoc solrdoc = new SolrDoc();
|
||||
final DigestURI digestURI = new DigestURI(yacydoc.dc_source());
|
||||
addSolr(solrdoc, SolrField.failreason_t, ""); // overwrite a possible fail reason (in case that there was a fail reason before)
|
||||
|
|
@ -453,39 +467,39 @@ public class SolrConfiguration extends ConfigurationSet {
|
|||
}
|
||||
|
||||
public String solrGetID(final SolrDocument solr) {
|
||||
return (String) solr.getFieldValue("id");
|
||||
return (String) solr.getFieldValue(SolrField.id.getSolrFieldName());
|
||||
}
|
||||
|
||||
public DigestURI solrGetURL(final SolrDocument solr) {
|
||||
try {
|
||||
return new DigestURI((String) solr.getFieldValue("sku"));
|
||||
return new DigestURI((String) solr.getFieldValue(SolrField.sku.getSolrFieldName()));
|
||||
} catch (final MalformedURLException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public String solrGetTitle(final SolrDocument solr) {
|
||||
return (String) solr.getFieldValue("title");
|
||||
return (String) solr.getFieldValue(SolrField.title.getSolrFieldName());
|
||||
}
|
||||
|
||||
public String solrGetText(final SolrDocument solr) {
|
||||
return (String) solr.getFieldValue("text_t");
|
||||
return (String) solr.getFieldValue(SolrField.text_t.getSolrFieldName());
|
||||
}
|
||||
|
||||
public String solrGetAuthor(final SolrDocument solr) {
|
||||
return (String) solr.getFieldValue("author");
|
||||
return (String) solr.getFieldValue(SolrField.author.getSolrFieldName());
|
||||
}
|
||||
|
||||
public String solrGetDescription(final SolrDocument solr) {
|
||||
return (String) solr.getFieldValue("description");
|
||||
return (String) solr.getFieldValue(SolrField.description.getSolrFieldName());
|
||||
}
|
||||
|
||||
public Date solrGetDate(final SolrDocument solr) {
|
||||
return (Date) solr.getFieldValue("last_modified");
|
||||
return (Date) solr.getFieldValue(SolrField.last_modified.getSolrFieldName());
|
||||
}
|
||||
|
||||
public Collection<String> solrGetKeywords(final SolrDocument solr) {
|
||||
final Collection<Object> c = solr.getFieldValues("keywords");
|
||||
final Collection<Object> c = solr.getFieldValues(SolrField.keywords.getSolrFieldName());
|
||||
final ArrayList<String> a = new ArrayList<String>();
|
||||
for (final Object s: c) {
|
||||
a.add((String) s);
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import net.yacy.cora.services.federated.solr.SolrType;
|
|||
|
||||
public enum SolrField implements net.yacy.cora.services.federated.solr.SolrField {
|
||||
|
||||
id(SolrType.string, true, true, "primary key of document, the URL hash"),
|
||||
id(SolrType.string, true, true, "primary key of document, the URL hash **mandatory field**"),
|
||||
sku(SolrType.text_en_splitting_tight, true, true, false, true, "url of document"),
|
||||
ip_s(SolrType.string, true, true, "ip of host of url (after DNS lookup)"),
|
||||
host_s(SolrType.string, true, true, "host of the url"),
|
||||
|
|
@ -121,10 +121,11 @@ public enum SolrField implements net.yacy.cora.services.federated.solr.SolrField
|
|||
ext_title_val(SolrType.integer, true, true, true, "number of matching title expressions"),
|
||||
failreason_t(SolrType.text_general, true, true, "fail reason if a page was not loaded. if the page was loaded then this field is empty");
|
||||
|
||||
final SolrType type;
|
||||
final boolean indexed, stored;
|
||||
boolean multiValued, omitNorms;
|
||||
final String comment;
|
||||
private String solrFieldName = null; // solr field name in custom solr schema, defaults to solcell schema field name (= same as this.name() )
|
||||
private final SolrType type;
|
||||
private final boolean indexed, stored;
|
||||
private boolean multiValued, omitNorms;
|
||||
private String comment;
|
||||
|
||||
private SolrField(final SolrType type, final boolean indexed, final boolean stored, final String comment) {
|
||||
this.type = type;
|
||||
|
|
@ -145,6 +146,27 @@ public enum SolrField implements net.yacy.cora.services.federated.solr.SolrField
|
|||
this.omitNorms = omitNorms;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the YaCy default or (if available) custom field name for Solr
|
||||
* @return SolrFieldname String
|
||||
*/
|
||||
public final String getSolrFieldName() {
|
||||
return (this.solrFieldName == null ? this.name() : this.solrFieldName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a custom Solr field name (and converts it to lower case)
|
||||
* @param theValue = the field name
|
||||
*/
|
||||
public final void setSolrFieldName(String theValue) {
|
||||
// make sure no empty string is assigned
|
||||
if ( (theValue != null) && (!theValue.isEmpty()) ) {
|
||||
this.solrFieldName = theValue.toLowerCase();
|
||||
} else {
|
||||
this.solrFieldName = null;
|
||||
}
|
||||
}
|
||||
|
||||
public final SolrType getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ import org.apache.solr.common.SolrDocumentList;
|
|||
|
||||
import de.anomic.crawler.Cache;
|
||||
import de.anomic.data.WorkTables;
|
||||
import net.yacy.search.index.SolrField;
|
||||
|
||||
public class SnippetProcess {
|
||||
|
||||
|
|
@ -497,7 +498,7 @@ public class SnippetProcess {
|
|||
String solrContent = null;
|
||||
if (this.solr != null) {
|
||||
SolrDocument sd = null;
|
||||
final SolrDocumentList sdl = this.solr.get("id:" + ASCII.String(page.hash()), 0, 1);
|
||||
final SolrDocumentList sdl = this.solr.get(SolrField.id.getSolrFieldName()+ ":" + ASCII.String(page.hash()), 0, 1);
|
||||
if (sdl.size() > 0) {
|
||||
sd = sdl.get(0);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user