Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
This commit is contained in:
Michael Peter Christen 2012-05-17 13:58:09 +02:00
commit 3e1bc9477f
20 changed files with 63 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
yacy.logging → defaults/yacy.logging
View File

9
htroot/Load_RSS_p.java
View File

@ -1,5 +1,5 @@
/**
* oad_RSS_p
* Load_RSS_p
* Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 20.08.2010 at http://yacy.net
*
@ -26,6 +26,7 @@ import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import net.yacy.cora.document.Hit;
import net.yacy.cora.document.RSSFeed;
@ -83,7 +84,7 @@ public class Load_RSS_p {
if (messageurl.length() == 0) continue;
final byte[] api_pk = row.get("api_pk");
final Row r = api_pk == null ? null : sb.tables.select("api", api_pk);
if (r == null || !r.get("comment", "").matches(".*\\Q" + messageurl + "\\E.*")) {
if (r == null || !r.get("comment", "").matches(".*" + Pattern.quote(messageurl) + ".*")) {
d.add(row.getPK());
}
}
@ -125,7 +126,7 @@ public class Load_RSS_p {
if (messageurl.length() == 0) continue;
final byte[] api_pk = row.get("api_pk");
final Row r = api_pk == null ? null : sb.tables.select("api", api_pk);
if (r != null && r.get("comment", "").matches(".*\\Q" + messageurl + "\\E.*")) {
if (r != null && r.get("comment", "").matches(".*" + Pattern.quote(messageurl) + ".*")) {
d.add(row.getPK());
}
}
@ -194,7 +195,7 @@ public class Load_RSS_p {
// check if feed is registered in scheduler
final byte[] api_pk = row.get("api_pk");
final Row r = api_pk == null ? null : sb.tables.select("api", api_pk);
if (r != null && r.get("comment", "").matches(".*\\Q" + messageurl + "\\E.*")) {
if (r != null && r.get("comment", "").matches(".*" + Pattern.quote(messageurl) + ".*")) {
// this is a recorded entry
final Date date_next_exec = r.get(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, (Date) null);
prop.put("showscheduledfeeds_list_" + apic + "_pk", UTF8.String(row.getPK()));

1
htroot/index.html
View File

@ -1,4 +1,3 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
#(forward)#::<meta http-equiv="REFRESH" content="0; url=#[target]#" />#(/forward)#

4
htroot/yacyinteractive.java
View File

@ -54,9 +54,9 @@ public class yacyinteractive {
prop.put("startRecord", startRecord);
prop.put("maximumRecords", maximumRecords);
prop.putHTML("querys", query.replaceAll(" ", "+"));
prop.put("serverlist", query.length() == 0 ? 1 : 0);
prop.put("serverlist", query.isEmpty() ? 1 : 0);
prop.put("focus", focus ? 1 : 0);
prop.put("allowrealtime", sb.indexSegments.URLCount() < 100000 ? 1 : 0);
return prop;
}
}
}

2
htroot/yacysearch.html
View File

@ -252,4 +252,4 @@ window.setTimeout('latestinfo();',10000);
</script>
</body>
</html>
</html>

13
htroot/yacysearch.java
View File

@ -28,6 +28,7 @@
// if the shell's current path is HTROOT
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -120,9 +121,8 @@ public class yacysearch {
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
//get focus option
final boolean focus = (post == null) ? true : post.get("focus", "1").equals("1");
prop.put("focus", focus ? 1 : 0);
prop.put("focus", ((post == null) ? true : post.get("focus", "1").equals("1")) ? 1 : 0);
// produce vocabulary navigation sidebars
Collection<Vocabulary> vocabularies = LibraryProvider.autotagging.getVocabularies();
int j = 0;
@ -1104,7 +1104,12 @@ public class yacysearch {
prop.putXML("rss_query", originalquerystring);
prop.putXML("rss_queryenc", originalquerystring.replace(' ', '+'));
sb.localSearchLastAccess = System.currentTimeMillis();
sb.localSearchLastAccess = System.currentTimeMillis();
// hostname and port (assume locahost if nothing helps)
final InetAddress hostIP = Domains.myPublicLocalIP();
prop.put("myhost", hostIP != null ? hostIP.getHostAddress() : "localhost");
prop.put("myport", serverCore.getPortNr(sb.getConfig("port", "8090")));
// return rewrite properties
return prop;

7
source/de/anomic/data/ymark/YMarkCrawlStart.java
View File

@ -119,10 +119,9 @@ public class YMarkCrawlStart extends HashMap<String,String>{
private void load() {
try {
final StringBuilder buffer = new StringBuilder(500);
//buffer.append("^.*crawlingURL=\\Q");
buffer.append("^crawl start for \\Q");
buffer.append(url);
buffer.append("\\E?.*");
buffer.append("^crawl start for ");
buffer.append(Pattern.quote(url));
buffer.append("?.*");
final Pattern pattern = Pattern.compile(buffer.toString());
//final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_URL, pattern);
final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_COMMENT, pattern);

10
source/de/anomic/data/ymark/YMarkTables.java
View File

@ -92,8 +92,6 @@ public class YMarkTables {
public final static String ADMIN_AUTHENTICATE_MSG = "Admin authentication required!";
public final static String p1 = "(?:^|.*,)";
public final static String p2 = "\\Q";
public final static String p3 = "\\E";
public final static String p4 = "(?:,.*|$)";
public final static String p5 = "((?:";
public final static String p6 = ")(?:,.*|$)){";
@ -194,9 +192,7 @@ public class YMarkTables {
patternBuilder.setLength(0);
patternBuilder.append(p1);
patternBuilder.append('(');
patternBuilder.append(p2);
patternBuilder.append(folder);
patternBuilder.append(p3);
patternBuilder.append(Pattern.quote(folder));
patternBuilder.append(')');
patternBuilder.append(p4);
final Pattern p = Pattern.compile(patternBuilder.toString());
@ -210,9 +206,7 @@ public class YMarkTables {
patternBuilder.append(p1);
patternBuilder.append(p5);
for (final String tag : tagArray) {
patternBuilder.append(p2);
patternBuilder.append(tag);
patternBuilder.append(p3);
patternBuilder.append(Pattern.quote(tag));
patternBuilder.append('|');
}
patternBuilder.deleteCharAt(patternBuilder.length()-1);

6
source/de/anomic/tools/tarTools.java
View File

@ -89,8 +89,8 @@ public class tarTools {
}
Log.logInfo("UNTAR", "finished");
}
public static void main(final String args[]){
public static void main(final String args[]) {
// @arg0 source
// @arg1 destination
if(args.length == 2){
@ -103,4 +103,4 @@ public class tarTools {
System.out.println("usage: <source> <destination>");
}
}
}
}

2
source/net/yacy/cora/document/Classification.java
View File

@ -129,7 +129,7 @@ public class Classification {
return appsExtSet.contains(appsExt.trim().toLowerCase());
}
public static ContentDomain getContentDomain(String ext) {
public static ContentDomain getContentDomain(final String ext) {
if (isTextExtension(ext)) return ContentDomain.TEXT;
if (isImageExtension(ext)) return ContentDomain.IMAGE;
if (isAudioExtension(ext)) return ContentDomain.AUDIO;

4
source/net/yacy/kelondro/logging/GuiHandler.java
View File

@ -208,8 +208,8 @@ public class GuiHandler extends Handler {
}
public void close() throws SecurityException {
public synchronized void close() throws SecurityException {
// Nothing implement here
}
}

6
source/net/yacy/kelondro/table/Table.java
View File

@ -690,7 +690,11 @@ public class Table implements Index, Iterable<Row.Entry> {
}
this.file.put(i, p, 0);
byte[] pk = lr.getPrimaryKeyBytes();
if (pk == null) continue;
if (pk == null) {
// Table file might be corrupt
Log.logWarning("TABLE", "Possible corruption found in table " + this.filename() + " detected. i=" + i + ",p=" + p);
continue;
}
this.index.put(pk, i);
break;
}

12
source/net/yacy/kelondro/workflow/InstantBusyThread.java
View File

@ -179,7 +179,17 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
public static WorkflowThread oneTimeJob(final Runnable thread, final long startupDelay, final int maxJobs) {
while (instantThreadCounter >= maxJobs) try {Thread.sleep(100);} catch (final InterruptedException e) {break;}
return oneTimeJob( thread, startupDelay);
return oneTimeJob( thread, startupDelay);
}
@Override
public void open() {
// Not implemented in this thread
}
@Override
public synchronized void close() {
// Not implemented in this thread
}
}

18
source/net/yacy/search/index/DocumentMetadata.java
View File

@ -50,7 +50,7 @@ public class DocumentMetadata implements Metadata {
}
@Override
public void close() {
public synchronized void close() {
// TODO Auto-generated method stub
}
@ -62,37 +62,37 @@ public class DocumentMetadata implements Metadata {
}
@Override
public URIMetadata load(Element<WordReference> obrwi) {
public URIMetadata load(final Element<WordReference> obrwi) {
// TODO Auto-generated method stub
return null;
}
@Override
public URIMetadata load(byte[] urlHash) {
public URIMetadata load(final byte[] urlHash) {
// TODO Auto-generated method stub
return null;
}
@Override
public void store(URIMetadata entry) throws IOException {
public void store(final URIMetadata entry) throws IOException {
// TODO Auto-generated method stub
}
@Override
public boolean remove(byte[] urlHashBytes) {
public boolean remove(final byte[] urlHashBytes) {
// TODO Auto-generated method stub
return false;
}
@Override
public boolean exists(byte[] urlHash) {
public boolean exists(final byte[] urlHash) {
// TODO Auto-generated method stub
return false;
}
@Override
public CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) {
public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) {
// TODO Auto-generated method stub
return null;
}
@ -104,14 +104,14 @@ public class DocumentMetadata implements Metadata {
}
@Override
public CloneableIterator<URIMetadata> entries(boolean up, String firstHash)
public CloneableIterator<URIMetadata> entries(final boolean up, final String firstHash)
throws IOException {
// TODO Auto-generated method stub
return null;
}
@Override
public int deleteDomain(String hosthash) throws IOException {
public int deleteDomain(final String hosthash) throws IOException {
// TODO Auto-generated method stub
return 0;
}

2
source/net/yacy/search/index/DocumentReference.java
View File

@ -42,7 +42,7 @@ public class DocumentReference {
return data.size();
}
public void close() {
public synchronized void close() {
if (data != null) {
data.close();
}

4
source/net/yacy/search/index/Segment.java
View File

@ -72,8 +72,8 @@ import de.anomic.crawler.retrieval.Response;
public class Segment {
// catchall word
final static String catchallString = "yacyall"; // a word that is always in all indexes; can be used for zero-word searches to find ALL documents
final static byte[] catchallHash;
public final static String catchallString = "yacyall"; // a word that is always in all indexes; can be used for zero-word searches to find ALL documents
public final static byte[] catchallHash;
final static Word catchallWord = new Word(0, 0, 0);
static {
catchallHash = Word.word2hash(catchallString); // "KZzU-Vf6h5k-"

2
source/net/yacy/search/index/SolrConfiguration.java
View File

@ -466,7 +466,7 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
* @param rel
* @return binary encoded information about rel
*/
private int relEval(final String[] rel) {
private static int relEval(final String[] rel) {
int i = 0;
for (final String s: rel) {
final String s0 = s.toLowerCase().trim();

5
source/net/yacy/search/query/SnippetProcess.java
View File

@ -62,6 +62,7 @@ import org.apache.solr.common.SolrDocumentList;
import de.anomic.crawler.Cache;
import de.anomic.data.WorkTables;
import net.yacy.search.index.Segment;
import net.yacy.search.index.SolrField;
public class SnippetProcess {
@ -606,6 +607,10 @@ public class SnippetProcess {
return new ResultEntry(page, this.query.getSegment(), this.peers, null, null, dbRetrievalTime, snippetComputationTime); // result without snippet
} else {
// problems with snippet fetch
if (this.snippetFetchWordHashes.has(Segment.catchallHash)) {
// we accept that because the word cannot be on the page
return new ResultEntry(page, this.query.getSegment(), this.peers, null, null, dbRetrievalTime, 0);
}
final String reason = "no text snippet; errorCode = " + snippet.getErrorCode();
if (this.deleteIfSnippetFail) {
this.workTables.failURLsRegisterMissingWord(this.query.getSegment().termIndex(), page.url(), this.query.queryHashes, reason);

2
source/net/yacy/yacy.java
View File

@ -173,7 +173,7 @@ public final class yacy {
f = new File(dataHome, "DATA/LOG/");
mkdirsIfNeseccary(f);
f = new File(dataHome, "DATA/LOG/yacy.logging");
final File f0 = new File(appHome, "yacy.logging");
final File f0 = new File(appHome, "defaults/yacy.logging");
if (!f.exists() || f0.lastModified() > f.lastModified()) try {
FileUtils.copy(f0, f);
} catch (final IOException e){

27
vocabularies/de/Freizeitaktivitäten.vocabulary
View File

@ -1,27 +0,0 @@
Eislaufen
Drachensteigen
Saunabaden=Sauna
Spazieren und Wandern=Spazieren,Wandern,Park,Ufer
Skaten=Inline,Inliner
Skateboarden=Skateboard
Museumsbesuch=Museum
Altstadtbesuch=Altstadt
Parkbesuch=Park,Zoo,Freilichtmuseum,Burg
Ausflug=Schifstouren,Schifffahrt,Aussicht,Aussichtspunkt,Geocaching
Schwimmen=Schwimmbad,Freibad,Hallenbad,Schwimmbäder,Freibäder,Hallenbäder,Therme
Fastnacht
Eislaufen
Saunabaden=Sauna
Theater=Bühnen
Oper=Operette,Opernhaus
Spielplätze=Spielplatz
Museum
Camping
Picknick=Picknicken
Essen und Trinken=Essen,Trinken,Bar,Restaurant,Kneipe
Kochen
Singen=Gesang,Chor
Kegeln
Bowling
Kartbahn=Kart
Kino=Kinoprogramm,Cine