Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

many bigfixes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@475 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2005-08-02 02:18:01 +00:00
parent 9ee8a5ba6c
commit e84a177c49
10 changed files with 103 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
htroot/IndexCreate_p.html
View File

@ -263,7 +263,7 @@ Continue crawling.
<td class="small"><a class="small" href="#[startURL]#">#[startURL]#</a></td>
<td class="small">#[intention]#</td>
<td class="small">#[generalDepth]#</td>
<td class="small">#(crawlingQ)#no::yes#(/withQuery)#</td>
<td class="small">#(crawlingQ)#no::yes#(/crawlingQ)#</td>
</tr>
#{/otherCrawlStart}#
</table>

2
htroot/IndexCreate_p.java
View File

@ -170,7 +170,7 @@ public class IndexCreate_p {
m.remove("storeHTCache");
m.remove("generalFilter");
m.remove("specificFilter");
m.put("intention", ((String) post.get("intention", "")));
m.put("intention", ((String) post.get("intention", "")).replace(',', '/'));
yacyCore.newsPool.publishMyNews(new yacyNewsRecord("crwlstrt", m));
}

2
htroot/Network.html
View File

@ -47,7 +47,7 @@
<td class="small"><b>Type</b><br>&nbsp;</td>
<td class="small"><b>Release/<br>SVN</b><br>&nbsp;</td>
<td class="small"><b>Contact</b><br>&nbsp;</td>
<td class="small"><b>Last Seen</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=down">&gt;</a></td>
<td class="small"><b>Last<br>Seen</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=down">&gt;</a></td>
<td class="small"><b>Uptime</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=Uptime&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=Uptime&order=down">&gt;</a></td>
<td class="small"><b>#Links</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=LCount&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=LCount&order=down">&gt;</a></td>
<td class="small"><b>#RWIs</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=ICount&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=ICount&order=down">&gt;</a></td>

10
htroot/ViewProfile.java
View File

@ -45,6 +45,7 @@
// javac -classpath .:../Classes Blacklist_p.java
// if the shell's current path is HTROOT
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@ -57,6 +58,8 @@ import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.tools.bbCode;
public class ViewProfile {
@ -79,6 +82,13 @@ public class ViewProfile {
}
} else {
prop.put("success","3"); // everything ok
// process news if existent
try {
yacyNewsRecord record = yacyCore.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, "prfleupd", seed.hash);
if (record != null) yacyCore.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id());
} catch (IOException e) {}
// read profile from other peer
HashMap profile = yacyClient.getProfile(seed);
yacyCore.log.logInfo("fetched profile:" + profile);
Iterator i = profile.entrySet().iterator();

3
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
View File

@ -68,6 +68,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
linkTags0 = new TreeSet(insensitiveCollator);
linkTags0.add("img");
linkTags0.add("base");
linkTags0.add("frame");
linkTags1 = new TreeSet(insensitiveCollator);
linkTags1.add("a");
@ -127,6 +128,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
public void scrapeTag0(String tagname, Properties tagopts) {
if (tagname.equalsIgnoreCase("img")) images.put(absolutePath(tagopts.getProperty("src", "")), tagopts.getProperty("alt",""));
if (tagname.equalsIgnoreCase("base")) try {root = new URL(tagopts.getProperty("href", ""));} catch (MalformedURLException e) {}
if (tagname.equalsIgnoreCase("frame")) anchors.put(absolutePath(tagopts.getProperty("src", "")), tagopts.getProperty("name",""));
}
public void scrapeTag1(String tagname, Properties tagopts, byte[] text) {
@ -136,7 +138,6 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
if ((tagname.equalsIgnoreCase("title")) && (text.length < 1024)) title = super.stripAll(new serverByteBuffer(text)).toString();
}
public String getHeadline() {
String hl = "";

36
source/de/anomic/kelondro/kelondroMScoreCluster.java
View File

@ -97,35 +97,35 @@ public class kelondroMScoreCluster {
return (((long) (elementCount & 0xFFFFFFFFL)) << 32) | ((long) (elementNr & 0xFFFFFFFFL));
}
public long totalCount() {
public synchronized long totalCount() {
return gcount;
}
public int size() {
public synchronized int size() {
return refkeyDB.size();
}
public void incScore(Object[] objs) {
public synchronized void incScore(Object[] objs) {
addScore(objs, 1);
}
public void addScore(Object[] objs, int count) {
public synchronized void addScore(Object[] objs, int count) {
if (objs != null)
for (int i = 0; i < objs.length; i++)
addScore(objs[i], count);
}
public void setScore(Object[] objs, int count) {
public synchronized void setScore(Object[] objs, int count) {
if (objs != null)
for (int i = 0; i < objs.length; i++)
setScore(objs[i], count);
}
public void incScore(Object obj) {
public synchronized void incScore(Object obj) {
addScore(obj, 1);
}
public void addScore(Object obj, int count) {
public synchronized void addScore(Object obj, int count) {
if (obj == null) return;
Long cs = (Long) refkeyDB.get(obj);
long c;
@ -154,7 +154,7 @@ public class kelondroMScoreCluster {
gcount += count;
}
public void setScore(Object obj, int count) {
public synchronized void setScore(Object obj, int count) {
if (obj == null) return;
//System.out.println("setScore " + obj.getClass().getName());
Long cs = (Long) refkeyDB.get(obj);
@ -183,7 +183,7 @@ public class kelondroMScoreCluster {
gcount += count;
}
public int deleteScore(Object obj) {
public synchronized int deleteScore(Object obj) {
if (obj == null) return -1;
Long cs = (Long) refkeyDB.get(obj);
if (cs == null) {
@ -199,11 +199,11 @@ public class kelondroMScoreCluster {
}
}
public boolean existsScore(Object obj) {
public synchronized boolean existsScore(Object obj) {
return (refkeyDB.get(obj) != null);
}
public int getScore(Object obj) {
public synchronized int getScore(Object obj) {
if (obj == null) return 0;
Long cs = (Long) refkeyDB.get(obj);
if (cs == null) {
@ -213,7 +213,7 @@ public class kelondroMScoreCluster {
}
}
public int getMaxScore() {
public synchronized int getMaxScore() {
if (refkeyDB.size() == 0) return -1;
return (int) ((((Long) keyrefDB.lastKey()).longValue() & 0xFFFFFFFF00000000L) >> 32);
}
@ -223,23 +223,23 @@ public class kelondroMScoreCluster {
return (int) ((((Long) keyrefDB.firstKey()).longValue() & 0xFFFFFFFF00000000L) >> 32);
}
public Object getMaxObject() {
public synchronized Object getMaxObject() {
if (refkeyDB.size() == 0) return null;
//return getScores(1, false)[0];
return keyrefDB.get((Long) keyrefDB.lastKey());
}
public Object getMinObject() {
public synchronized Object getMinObject() {
if (refkeyDB.size() == 0) return null;
//return getScores(1, true)[0];
return keyrefDB.get((Long) keyrefDB.firstKey());
}
public Object[] getScores(int maxCount, boolean up) {
public synchronized Object[] getScores(int maxCount, boolean up) {
return getScores(maxCount, up, Integer.MIN_VALUE, Integer.MAX_VALUE);
}
public Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) {
public synchronized Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) {
if (maxCount > refkeyDB.size()) maxCount = refkeyDB.size();
Object[] s = new Object[maxCount];
Iterator it = scores(up, minScore, maxScore);
@ -255,12 +255,12 @@ public class kelondroMScoreCluster {
return s;
}
public Iterator scores(boolean up) {
public synchronized Iterator scores(boolean up) {
if (up) return new simpleScoreIterator();
else return scores(false, Integer.MIN_VALUE, Integer.MAX_VALUE);
}
public Iterator scores(boolean up, int minScore, int maxScore) {
public synchronized Iterator scores(boolean up, int minScore, int maxScore) {
return new komplexScoreIterator(up, minScore, maxScore);
}

30
source/de/anomic/kelondro/kelondroMScoreIndex.java
View File

@ -74,35 +74,35 @@ public class kelondroMScoreIndex {
scoreCluster = new kelondroMScoreCluster(); // scores for int-handles
}
public long totalCount() {
public synchronized long totalCount() {
return scoreCluster.totalCount();
}
public int size() {
public synchronized int size() {
return handles.size();
}
public void incScore(Object[] objs) {
public synchronized void incScore(Object[] objs) {
addScore(objs, 1);
}
public void addScore(Object[] objs, int count) {
public synchronized void addScore(Object[] objs, int count) {
if (objs != null)
for (int i = 0; i < objs.length; i++)
addScore(objs[i], count);
}
public void setScore(Object[] objs, int count) {
public synchronized void setScore(Object[] objs, int count) {
if (objs != null)
for (int i = 0; i < objs.length; i++)
setScore(objs[i], count);
}
public void incScore(Object obj) {
public synchronized void incScore(Object obj) {
addScore(obj, 1);
}
public void addScore(Object obj, int count) {
public synchronized void addScore(Object obj, int count) {
// get handle
Integer handle = (Integer) handles.get(obj);
if (handle == null) {
@ -115,7 +115,7 @@ public class kelondroMScoreIndex {
scoreCluster.addScore(handle, count);
}
public void setScore(Object obj, int count) {
public synchronized void setScore(Object obj, int count) {
// get handle
Integer handle = (Integer) handles.get(obj);
if (handle == null) {
@ -128,7 +128,7 @@ public class kelondroMScoreIndex {
scoreCluster.setScore(handle, count);
}
public void deleteScore(Object obj) {
public synchronized void deleteScore(Object obj) {
// get handle
Integer handle = (Integer) handles.get(obj);
if (handle != null) {
@ -138,22 +138,22 @@ public class kelondroMScoreIndex {
}
}
public int getScore(Object obj) {
public synchronized int getScore(Object obj) {
// get handle
Integer handle = (Integer) handles.get(obj);
if (handle == null) return -1;
return scoreCluster.getScore(handle);
}
public Object[] getScores(int count, boolean up, boolean weight, char weightsep) {
public synchronized Object[] getScores(int count, boolean up, boolean weight, char weightsep) {
return new Object[1];
}
public Object[] getScores(int maxCount, boolean up) {
public synchronized Object[] getScores(int maxCount, boolean up) {
return getScores(maxCount, up, Integer.MIN_VALUE, Integer.MAX_VALUE);
}
public Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) {
public synchronized Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) {
if (maxCount > handles.size()) maxCount = handles.size();
Object[] s = new Object[maxCount];
Iterator it = scores(up, minScore, maxScore);
@ -169,11 +169,11 @@ public class kelondroMScoreIndex {
return s;
}
public Iterator scores(boolean up) {
public synchronized Iterator scores(boolean up) {
return scores(up, Integer.MIN_VALUE, Integer.MAX_VALUE);
}
public Iterator scores(boolean up, int minScore, int maxScore) {
public synchronized Iterator scores(boolean up, int minScore, int maxScore) {
return new scoreIterator(up, minScore, maxScore);
}

10
source/de/anomic/plasma/plasmaHTCache.java
View File

@ -217,13 +217,13 @@ public final class plasmaHTCache {
}
private String ageString(long date, File f) {
String s = Integer.toHexString(f.hashCode());
StringBuffer sb = new StringBuffer(32);
for (int i = s.length(); i < 8; i++) sb.append('0');
sb.append(s);
s = Long.toHexString(date);
StringBuffer sb = new StringBuffer(32);
String s = Long.toHexString(date);
for (int i = s.length(); i < 16; i++) sb.append('0');
sb.append(s);
s = Integer.toHexString(f.hashCode());
for (int i = s.length(); i < 8; i++) sb.append('0');
sb.append(s);
return sb.toString();
}

5
source/de/anomic/plasma/plasmaWordIndexAssortment.java
View File

@ -207,6 +207,11 @@ public final class plasmaWordIndexAssortment {
e.printStackTrace();
resetDatabase();
return null;
} catch (kelondroException e) {
log.logFailure("iterateAssortment/kelondro-error: " + e.getMessage() + " - reset assortment-DB");
e.printStackTrace();
resetDatabase();
return null;
}
}

78
source/de/anomic/yacy/yacyNewsPool.java
View File

@ -134,44 +134,56 @@ public class yacyNewsPool {
}
public int size(int dbKey) {
switch (dbKey) {
case INCOMING_DB: return incomingNews.size();
case PROCESSED_DB: return processedNews.size();
case OUTGOING_DB: return outgoingNews.size();
case PUBLISHED_DB: return publishedNews.size();
default: return -1;
}
return switchQueue(dbKey).size();
}
public yacyNewsRecord get(int dbKey, int element) throws IOException {
yacyNewsQueue queue = switchQueue(dbKey);
yacyNewsRecord record;
switch (dbKey) {
case INCOMING_DB:
synchronized (incomingNews) {
record = incomingNews.top(element);
if (record == null) incomingNews.pop(element);
}
return record;
case PROCESSED_DB:
synchronized (processedNews) {
record = processedNews.top(element);
if (record == null) processedNews.pop(element);
}
return record;
case OUTGOING_DB:
synchronized (outgoingNews) {
record = outgoingNews.top(element);
if (record == null) outgoingNews.pop(element);
}
return record;
case PUBLISHED_DB:
synchronized (publishedNews) {
record = publishedNews.top(element);
if (record == null) publishedNews.pop(element);
}
return record;
default: return null;
synchronized (queue) {
record = queue.top(element);
if (record == null) queue.pop(element);
}
return record;
}
public synchronized yacyNewsRecord getSpecific(int dbKey, String category, String key, String value) throws IOException {
yacyNewsQueue queue = switchQueue(dbKey);
yacyNewsRecord record;
String s;
for (int i = queue.size() - 1; i >= 0; i--) {
record = queue.top(i);
if ((record != null) && (record.category().equals(category))) {
s = (String) record.attributes().get(key);
if ((s != null) && (s.equals(value))) return record;
}
}
return null;
}
public synchronized yacyNewsRecord getByOriginator(int dbKey, String category, String originatorHash) throws IOException {
yacyNewsQueue queue = switchQueue(dbKey);
yacyNewsRecord record;
String s;
for (int i = queue.size() - 1; i >= 0; i--) {
record = queue.top(i);
if ((record != null) &&
(record.category().equals(category)) &&
(record.originator().equals(originatorHash))) {
return record;
}
}
return null;
}
private yacyNewsQueue switchQueue(int dbKey) {
switch (dbKey) {
case INCOMING_DB: return incomingNews;
case PROCESSED_DB: return processedNews;
case OUTGOING_DB: return outgoingNews;
case PUBLISHED_DB: return publishedNews;
}
return null;
}
public void moveOff(int dbKey, String id) throws IOException {