mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
added an api recording function: it shall be possible to record
all operations on YaCy in a database that should make it possible 1) to re-create a setting on fresh peers 2) to transmit a setting from one peer to another 3) to re-create crawl starts after a complete deletion of the index This functionality will also support 4) scheduled re-crawls (new implementation) To implement this, a new database structure has been crated that stores maps into blob heaps. to encode maps the b-encoding technique was used (this is the same encoding that torrent files use) - added a b-encoder - enhanced the b-decoder - added a b-encoded map heap data structure - added a table organisation based on b-encoded heaps - added a servlet to maintain such tables (see Tables_p.html) - integrated the servlet into the Advanced Settings menu - added an api recording based on the new tables git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6606 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter
parent
8df1694288
commit
8ce936bcdd
|
|
@ -250,8 +250,12 @@ public class Blacklist_p {
|
|||
* =========================================================== */
|
||||
|
||||
blacklistToUse = post.get("currentBlacklist", "").trim();
|
||||
|
||||
final String temp = addBlacklistEntry(blacklistToUse, post.get("newEntry", "").trim(), header, supportedBlacklistTypes);
|
||||
String blentry = post.get("newEntry", "").trim();
|
||||
|
||||
// store this call as api call
|
||||
listManager.switchboard.recordAPICall(post, "Blacklist_p.html", "blacklist", "add to blacklist: " + blentry);
|
||||
|
||||
final String temp = addBlacklistEntry(blacklistToUse, blentry, header, supportedBlacklistTypes);
|
||||
if (temp != null) {
|
||||
prop.put("LOCATION", temp);
|
||||
return prop;
|
||||
|
|
|
|||
|
|
@ -70,7 +70,10 @@ public class ConfigBasic {
|
|||
return prop;
|
||||
}
|
||||
|
||||
// starting a peer ping
|
||||
// store this call as api call
|
||||
if (post != null && post.containsKey("set")) {
|
||||
sb.recordAPICall(post, "ConfigBasic.html", "configuration", "basic settings");
|
||||
}
|
||||
|
||||
//boolean doPeerPing = false;
|
||||
if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import de.anomic.data.translator;
|
|||
import de.anomic.http.client.Client;
|
||||
import de.anomic.http.server.HeaderFramework;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
import java.util.Collections;
|
||||
|
|
@ -72,6 +73,9 @@ public class ConfigLanguage_p {
|
|||
if (post != null){
|
||||
String selectedLanguage = post.get("language");
|
||||
|
||||
// store this call as api call
|
||||
((Switchboard) env).recordAPICall(post, "ConfigLanguage.html", "configuration", "language settings: " + selectedLanguage);
|
||||
|
||||
//change language
|
||||
if(post.containsKey("use_button") && selectedLanguage != null){
|
||||
/* Only change language if filename is contained in list of filesnames
|
||||
|
|
|
|||
|
|
@ -54,6 +54,9 @@ public class ConfigNetwork_p {
|
|||
|
||||
if (post != null) {
|
||||
|
||||
// store this call as api call
|
||||
sb.recordAPICall(post, "ConfigNetwork.html", "configuration", "network settings");
|
||||
|
||||
if (post.containsKey("changeNetwork")) {
|
||||
final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");
|
||||
if (networkDefinition.equals(sb.getConfig("network.unit.definition", ""))) {
|
||||
|
|
|
|||
|
|
@ -57,7 +57,11 @@ public class ConfigPortal {
|
|||
}
|
||||
}
|
||||
if (post.containsKey("searchpage_set")) {
|
||||
sb.setConfig(SwitchboardConstants.GREETING, post.get(SwitchboardConstants.GREETING, ""));
|
||||
String newGreeting = post.get(SwitchboardConstants.GREETING, "");
|
||||
// store this call as api call
|
||||
sb.recordAPICall(post, "ConfigPortal.html", "appearance", "new portal design. greeting: " + newGreeting);
|
||||
|
||||
sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
|
||||
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
|
||||
sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
|
||||
sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, post.get(SwitchboardConstants.GREETING_SMALL_IMAGE, ""));
|
||||
|
|
|
|||
|
|
@ -130,16 +130,8 @@ public class Crawler_p {
|
|||
if (sb.peers == null) {
|
||||
prop.put("info", "3");
|
||||
} else {
|
||||
// log a GET url for this crawl start for possible use in cronjobs
|
||||
Log.logInfo("CRAWLSTART-URL", "http://localhost:" + sb.getConfig("port", "8080") + "/Crawler_p.html?" + post.toString());
|
||||
|
||||
// set new properties
|
||||
final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start
|
||||
final boolean subPath = post.get("range", "wide").equals("subpath"); // special property in simple crawl start
|
||||
|
||||
String crawlingStart = post.get("crawlingURL","").trim(); // the crawljob start url
|
||||
|
||||
// adding the prefix http:// if necessary
|
||||
// add the prefix http:// if necessary
|
||||
int pos = crawlingStart.indexOf("://");
|
||||
if (pos == -1) crawlingStart = "http://" + crawlingStart;
|
||||
|
||||
|
|
@ -147,6 +139,14 @@ public class Crawler_p {
|
|||
DigestURI crawlingStartURL = null;
|
||||
try {crawlingStartURL = new DigestURI(crawlingStart, null);} catch (final MalformedURLException e1) {}
|
||||
crawlingStart = (crawlingStartURL == null) ? null : crawlingStartURL.toNormalform(true, true);
|
||||
|
||||
// store this call as api call
|
||||
sb.recordAPICall(post, "Crawler_p.html", "crawler", "crawl start for " + crawlingStartURL.getHost());
|
||||
|
||||
// set new properties
|
||||
final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start
|
||||
final boolean subPath = post.get("range", "wide").equals("subpath"); // special property in simple crawl start
|
||||
|
||||
|
||||
// set the crawling filter
|
||||
String newcrawlingMustMatch = post.get("mustmatch", CrawlProfile.MATCH_ALL);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,11 @@
|
|||
<li><a href="PerformanceMemory_p.html">Performance Settings for Memory</a></li>
|
||||
<li><a href="PerformanceSearch_p.html">Performance Settings of Search Sequence</a></li>
|
||||
</ul>
|
||||
<ul class="settingsMenu">
|
||||
<li><a href="Tables_p.html">Viewer and administration for database tables</a></li>
|
||||
<li><a href="News.html">Viewer for Peer-News</a></li>
|
||||
<li><a href="CookieMonitorIncoming_p.html">Viewer for Cookies in Proxy</a></li>
|
||||
</ul>
|
||||
<ul class="settingsMenu">
|
||||
<li><a href="?page=ServerAccess">Server Access Settings</a></li>
|
||||
<li><a href="?page=ProxyAccess">Proxy Access Settings</a></li>
|
||||
|
|
|
|||
86
htroot/Tables_p.html
Normal file
86
htroot/Tables_p.html
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Table Viewer</title>
|
||||
#(showtable)#::
|
||||
<link rel="alternate" type="application/xml" title="Tables" href="Tables.rss?table=#[table]#" />
|
||||
#(/showtable)#
|
||||
#%env/templates/metas.template%#
|
||||
</head>
|
||||
<body id="Tables">
|
||||
#%env/templates/header.template%#
|
||||
#%env/templates/submenuConfig.template%#
|
||||
|
||||
<h2>Table Administration</h2>
|
||||
<form action="Tables_p.html" method="get">
|
||||
<fieldset><legend>Table Selection</legend>
|
||||
<dl>
|
||||
<dt class="TableCellDark">Select Table:
|
||||
<select name="table" size="1">
|
||||
#{tables}#
|
||||
<option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>
|
||||
#{/tables}#
|
||||
</select></dt>
|
||||
<dd>
|
||||
<input type="submit" name="edittable" value="Show Table" />
|
||||
</dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
#(showtable)#::
|
||||
<form action="Tables_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
|
||||
<fieldset>
|
||||
<legend><label for="table">Table Editor: showing table '#[table]#'</label></legend>
|
||||
<table border="0" cellpadding="2" cellspacing="1">
|
||||
<tr class="TableHeader" valign="bottom">
|
||||
<td> </td>
|
||||
<td>PK</td>
|
||||
#{columns}#
|
||||
<td>#[header]#</td>
|
||||
#{/columns}#
|
||||
</tr>
|
||||
#{list}#
|
||||
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
|
||||
<td align="left"><input type="checkbox" name="mark_#[pk]#" /></td>
|
||||
<td>#[pk]#</td>
|
||||
#{columns}#
|
||||
<td align="left">#[cell]#</td>
|
||||
#{/columns}#
|
||||
</tr>
|
||||
#{/list}#
|
||||
</table>
|
||||
<p>
|
||||
<input type="hidden" name="table" value="#[table]#" />
|
||||
<input type="submit" name="editrow" value="Edit Selected Row" />
|
||||
<input type="submit" name="addrow" value="Add a new Row" />
|
||||
<input type="submit" name="deleterows" value="Delete Selected Rows" />
|
||||
<input type="submit" name="deletetable" value="Delete Table" />
|
||||
</p>
|
||||
</fieldset>
|
||||
</form>
|
||||
#(/showtable)#
|
||||
#(showedit)#::
|
||||
<form action="Tables_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
|
||||
<fieldset><legend>Row Editor</legend>
|
||||
<dl>
|
||||
<dt class="TableCellLight">Primary Key</dt>
|
||||
<dd>#[pk]#</dd>
|
||||
#{list}#
|
||||
<dt class="TableCellLight">#[key]#</dt>
|
||||
<dd>
|
||||
<input type="text" name="col_#[key]#" value="#[value]#" size="80"/>
|
||||
</dd>
|
||||
#{/list}#
|
||||
<dt class="TableCellDark"></dt>
|
||||
<dd>
|
||||
<input type="hidden" name="table" value="#[table]#" />
|
||||
<input type="hidden" name="pk" value="#[pk]#" />
|
||||
<input type="submit" name="commitrow" value="Commit" />
|
||||
</dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
#(/showedit)#
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
||||
194
htroot/Tables_p.java
Normal file
194
htroot/Tables_p.java
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
// News.java
|
||||
// -----------------------
|
||||
// part of YaCy
|
||||
// (C) by Michael Peter Christen; mc@yacy.net
|
||||
// first published on http://www.anomic.de
|
||||
// Frankfurt, Germany, 2005
|
||||
// last major change: 29.07.2005
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
// You must compile this file with
|
||||
// javac -classpath .:../classes Network.java
|
||||
// if the shell's current path is HTROOT
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.kelondro.index.RowSpaceExceededException;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
|
||||
public class Tables_p {
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
String table = (post == null) ? null : post.get("table", null);
|
||||
if (table != null && !sb.tables.hasHeap(table)) table = null;
|
||||
|
||||
// show table selection
|
||||
int count = 0;
|
||||
Iterator<String> ti = sb.tables.tables();
|
||||
String tablename;
|
||||
while (ti.hasNext()) {
|
||||
tablename = ti.next();
|
||||
prop.put("tables_" + count + "_name", tablename);
|
||||
prop.put("tables_" + count + "_selected", (table != null && table.equals(tablename)) ? 1 : 0);
|
||||
count++;
|
||||
}
|
||||
prop.put("tables", count);
|
||||
|
||||
List<String> columns = null;
|
||||
if (table != null) try {
|
||||
columns = sb.tables.columns(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
|
||||
// apply deletion requests
|
||||
if (post != null && post.get("deletetable", "").length() > 0) {
|
||||
try {
|
||||
sb.tables.clear(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
if (post != null && post.get("deleterows", "").length() > 0) {
|
||||
try {
|
||||
for (Map.Entry<String, String> entry: post.entrySet()) {
|
||||
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
|
||||
sb.tables.delete(table, entry.getKey().substring(5).getBytes());
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
if (post != null && post.get("commitrow", "").length() > 0) {
|
||||
String pk = post.get("pk");
|
||||
Map<String, byte[]> map = new HashMap<String, byte[]>();
|
||||
for (Map.Entry<String, String> entry: post.entrySet()) {
|
||||
if (entry.getKey().startsWith("col_")) {
|
||||
map.put(entry.getKey().substring(4), entry.getValue().getBytes());
|
||||
}
|
||||
}
|
||||
try {
|
||||
sb.tables.insert(table, pk.getBytes(), map);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
// generate table
|
||||
prop.put("showtable", 0);
|
||||
prop.put("showedit", 0);
|
||||
|
||||
if (table != null && !post.containsKey("editrow") && !post.containsKey("addrow")) try {
|
||||
prop.put("showtable", 1);
|
||||
prop.put("showtable_table", table);
|
||||
|
||||
// insert the columns
|
||||
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
prop.putHTML("showtable_columns_" + i + "_header", columns.get(i));
|
||||
}
|
||||
prop.put("showtable_columns", columns.size());
|
||||
|
||||
// insert all rows
|
||||
final int maxCount = Math.min(1000, sb.tables.size(table));
|
||||
final Iterator<Map.Entry<byte[], Map<String, byte[]>>> mapIterator = sb.tables.iterator(table);
|
||||
Map.Entry<byte[], Map<String, byte[]>> record;
|
||||
Map<String, byte[]> map;
|
||||
byte[] pk;
|
||||
count = 0;
|
||||
boolean dark = true;
|
||||
byte[] cell;
|
||||
while ((mapIterator.hasNext()) && (count < maxCount)) {
|
||||
record = mapIterator.next();
|
||||
if (record == null) continue;
|
||||
pk = record.getKey();
|
||||
map = record.getValue();
|
||||
prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
|
||||
prop.put("showtable_list_" + count + "_pk", new String(pk));
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
cell = map.get(columns.get(i));
|
||||
prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell));
|
||||
}
|
||||
prop.put("showtable_list_" + count + "_columns", columns.size());
|
||||
|
||||
count++;
|
||||
}
|
||||
prop.put("showtable_list", count);
|
||||
} catch (IOException e) {}
|
||||
|
||||
if (post != null && table != null && post.containsKey("editrow")) try {
|
||||
// check if we can find a key
|
||||
String pk = null;
|
||||
for (Map.Entry<String, String> entry: post.entrySet()) {
|
||||
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
|
||||
pk = entry.getKey().substring(5);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (pk != null && sb.tables.has(table, pk.getBytes())) {
|
||||
setEdit(sb, prop, table, pk, columns);
|
||||
}
|
||||
} catch (IOException e) {}
|
||||
|
||||
if (post != null && table != null && post.containsKey("addrow")) try {
|
||||
// get a new key
|
||||
String pk = new String(sb.tables.insert(table, new HashMap<String, byte[]>()));
|
||||
setEdit(sb, prop, table, pk, columns);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
|
||||
// adding the peer address
|
||||
prop.put("address", sb.peers.mySeed().getPublicAddress());
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
||||
private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) throws IOException {
|
||||
prop.put("showedit", 1);
|
||||
prop.put("showedit_table", table);
|
||||
prop.put("showedit_pk", pk);
|
||||
Map<String, byte[]> map = sb.tables.select(table, pk.getBytes());
|
||||
int count = 0;
|
||||
byte[] cell;
|
||||
for (String col: columns) {
|
||||
cell = map.get(col);
|
||||
prop.put("showedit_list_" + count + "_key", col);
|
||||
prop.put("showedit_list_" + count + "_value", cell == null ? "" : new String(cell));
|
||||
count++;
|
||||
}
|
||||
prop.put("showedit_list", count);
|
||||
}
|
||||
}
|
||||
2
htroot/env/templates/header.template
vendored
2
htroot/env/templates/header.template
vendored
|
|
@ -82,8 +82,6 @@
|
|||
<li><a href="/WatchWebStructure_p.html?host=auto&depth=2&time=1000" class="MenuItemLink lock">Web Visualization</a></li>
|
||||
<li><a href="/AccessTracker_p.html" class="MenuItemLink lock">Access Tracker</a></li>
|
||||
<li><a href="/ViewLog_p.html" class="MenuItemLink lock">Server Log</a></li>
|
||||
<li><a href="/News.html" accesskey="n" class="MenuItemLink">News Monitor</a></li>
|
||||
<li><a href="/CookieMonitorIncoming_p.html" class="MenuItemLink lock">Cookies in Proxy</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="menugroup" id="menugroupPeerControl">
|
||||
|
|
|
|||
|
|
@ -110,7 +110,10 @@ public class FTPLoader {
|
|||
if (file.length() == 0) {
|
||||
// directory -> get list of files
|
||||
RequestHeader requestHeader = new RequestHeader();
|
||||
if (request.referrerhash() != null) requestHeader.put(RequestHeader.REFERER, sb.getURL(Segments.Process.LOCALCRAWLING, request.referrerhash()).toNormalform(true, false));
|
||||
if (request.referrerhash() != null) {
|
||||
DigestURI u = sb.getURL(Segments.Process.LOCALCRAWLING, request.referrerhash());
|
||||
if (u != null) requestHeader.put(RequestHeader.REFERER, u.toNormalform(true, false));
|
||||
}
|
||||
|
||||
byte[] dirList = generateDirlist(ftpClient, request, path);
|
||||
|
||||
|
|
|
|||
|
|
@ -550,7 +550,7 @@ public class ftpc {
|
|||
}
|
||||
printElements(l);
|
||||
} catch (final IOException e) {
|
||||
errPrintln("Error: remote list not available");
|
||||
errPrintln("Error: remote list not available (1): " + e.getMessage());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1378,7 +1378,7 @@ public class ftpc {
|
|||
}
|
||||
printElements(l);
|
||||
} catch (final IOException e) {
|
||||
errPrintln("Error: remote list not available");
|
||||
errPrintln("Error: remote list not available (2): " + e.getMessage());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,9 +70,11 @@ import net.yacy.document.content.RSSMessage;
|
|||
import net.yacy.document.content.SurrogateReader;
|
||||
import net.yacy.document.parser.html.ImageEntry;
|
||||
import net.yacy.document.parser.xml.RSSFeed;
|
||||
import net.yacy.kelondro.blob.BEncodedHeapArray;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.data.meta.URIMetadataRow;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
import net.yacy.kelondro.index.RowSpaceExceededException;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.order.Base64Order;
|
||||
import net.yacy.kelondro.order.Digest;
|
||||
|
|
@ -129,6 +131,7 @@ import de.anomic.http.server.ResponseHeader;
|
|||
import de.anomic.http.server.RobotsTxtConfig;
|
||||
import de.anomic.net.UPnP;
|
||||
import de.anomic.search.blockrank.CRDistribution;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
import de.anomic.server.serverCore;
|
||||
import de.anomic.tools.crypt;
|
||||
|
|
@ -214,6 +217,7 @@ public final class Switchboard extends serverSwitch {
|
|||
public Dispatcher dhtDispatcher;
|
||||
public List<String> trail;
|
||||
public yacySeedDB peers;
|
||||
public BEncodedHeapArray tables;
|
||||
|
||||
public WorkflowProcessor<indexingQueueEntry> indexingDocumentProcessor;
|
||||
public WorkflowProcessor<indexingQueueEntry> indexingCondensementProcessor;
|
||||
|
|
@ -234,14 +238,13 @@ public final class Switchboard extends serverSwitch {
|
|||
|
||||
private static Switchboard sb = null;
|
||||
|
||||
public Switchboard(final File rootPath, final String initPath, final String configPath, final boolean applyPro) throws IOException {
|
||||
super(rootPath, initPath, configPath, applyPro);
|
||||
public Switchboard(final File rootPath, final String initPath, final String configPath) throws IOException {
|
||||
super(rootPath, initPath, configPath);
|
||||
MemoryTracker.startSystemProfiling();
|
||||
sb=this;
|
||||
|
||||
// set loglevel and log
|
||||
setLog(new Log("PLASMA"));
|
||||
if (applyPro) this.log.logInfo("This is the pro-version of YaCy");
|
||||
|
||||
// UPnP port mapping
|
||||
if (getConfigBool(SwitchboardConstants.UPNP_ENABLED, false))
|
||||
|
|
@ -274,6 +277,9 @@ public final class Switchboard extends serverSwitch {
|
|||
this.dictionariesPath = getConfigPath(SwitchboardConstants.DICTIONARY_SOURCE_PATH, SwitchboardConstants.DICTIONARY_SOURCE_PATH_DEFAULT);
|
||||
this.log.logConfig("Dictionaries Path:" + this.dictionariesPath.toString());
|
||||
|
||||
// init tables
|
||||
this.tables = new BEncodedHeapArray(this.workPath, 12);
|
||||
|
||||
// init libraries
|
||||
this.log.logConfig("initializing libraries");
|
||||
LibraryProvider.initialize(this.dictionariesPath);
|
||||
|
|
@ -1100,6 +1106,7 @@ public final class Switchboard extends serverSwitch {
|
|||
indexSegments.close();
|
||||
peers.close();
|
||||
Cache.close();
|
||||
tables.close();
|
||||
UPnP.deletePortMapping();
|
||||
Tray.removeTray();
|
||||
log.logConfig("SWITCHBOARD SHUTDOWN TERMINATED");
|
||||
|
|
@ -2091,6 +2098,25 @@ public final class Switchboard extends serverSwitch {
|
|||
yacyCore.log.logInfo("BOOTSTRAP: " + (peers.sizeConnected() - sc) + " new seeds while bootstraping.");
|
||||
}
|
||||
|
||||
|
||||
public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
|
||||
String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
|
||||
try {
|
||||
sb.tables.insert(
|
||||
"api",
|
||||
"type", type.getBytes(),
|
||||
"comment", comment.getBytes(),
|
||||
"date", DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
|
||||
"url", apiurl.getBytes()
|
||||
);
|
||||
} catch (RowSpaceExceededException e2) {
|
||||
Log.logException(e2);
|
||||
} catch (IOException e2) {
|
||||
Log.logException(e2);
|
||||
}
|
||||
Log.logInfo("APICALL", apiurl);
|
||||
}
|
||||
|
||||
public void checkInterruption() throws InterruptedException {
|
||||
final Thread curThread = Thread.currentThread();
|
||||
if ((curThread instanceof WorkflowThread) && ((WorkflowThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ...");
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ public class serverSwitch {
|
|||
private final TreeMap<String, serverSwitchAction> switchActions;
|
||||
private final serverAccessTracker accessTracker;
|
||||
|
||||
public serverSwitch(final File rootPath, final String initPath, final String configPath, final boolean applyPro) {
|
||||
public serverSwitch(final File rootPath, final String initPath, final String configPath) {
|
||||
// we initialize the switchboard with a property file,
|
||||
// but maintain these properties then later in a new 'config' file
|
||||
// to reset all changed configs, the config file must
|
||||
|
|
@ -79,15 +79,7 @@ public class serverSwitch {
|
|||
// if 'pro'-version is selected, overload standard settings with 'pro'-settings
|
||||
Iterator<String> i;
|
||||
String prop;
|
||||
if (applyPro) {
|
||||
i = new HashMap<String, String>(initProps).keySet().iterator(); // clone the map to avoid concurrent modification exceptions
|
||||
while (i.hasNext()) {
|
||||
prop = i.next();
|
||||
if (prop.endsWith("__pro")) {
|
||||
initProps.put(prop.substring(0, prop.length() - 5), initProps.get(prop));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// delete the 'pro' init settings
|
||||
i = initProps.keySet().iterator();
|
||||
while (i.hasNext()) {
|
||||
|
|
|
|||
|
|
@ -1092,7 +1092,7 @@ public final class yacyClient {
|
|||
if(args.length > 1) {
|
||||
System.out.println("yacyClient Test");
|
||||
try {
|
||||
final Switchboard sb = new Switchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf", false);
|
||||
final Switchboard sb = new Switchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf");
|
||||
/*final yacyCore core =*/ new yacyCore(sb);
|
||||
sb.loadSeedLists();
|
||||
final yacySeed target = sb.peers.getConnected(args[1]);
|
||||
|
|
|
|||
|
|
@ -26,8 +26,11 @@ package net.yacy.kelondro.blob;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
|
|
@ -47,6 +50,7 @@ import net.yacy.kelondro.util.BDecoder.BObject;
|
|||
public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte[]>>> {
|
||||
|
||||
private Heap table;
|
||||
private LinkedHashSet<String> columnames;
|
||||
|
||||
/**
|
||||
* produce or open a properties table
|
||||
|
|
@ -62,6 +66,7 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
|
|||
final ByteOrder ordering,
|
||||
int buffermax) throws IOException {
|
||||
this.table = new Heap(location, keylength, ordering, buffermax);
|
||||
this.columnames = new LinkedHashSet<String>();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -73,12 +78,25 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
|
|||
final File location,
|
||||
final int keylength) throws IOException {
|
||||
this.table = new Heap(location, keylength, NaturalOrder.naturalOrder, 100);
|
||||
this.columnames = new LinkedHashSet<String>();
|
||||
}
|
||||
|
||||
public File getFile() {
|
||||
return this.table.heapFile;
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return this.table.size();
|
||||
}
|
||||
|
||||
public void close() {
|
||||
this.table.close();
|
||||
}
|
||||
|
||||
public void clear() throws IOException {
|
||||
this.table.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* insert a map into the table
|
||||
* @param key
|
||||
|
|
@ -86,10 +104,67 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
|
|||
* @throws RowSpaceExceededException
|
||||
* @throws IOException
|
||||
*/
|
||||
public void put(byte[] key, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
|
||||
public void put(byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
|
||||
byte[] b = BEncoder.encode(BEncoder.transcode(map));
|
||||
System.out.println(new String(b));
|
||||
this.table.put(key, b);
|
||||
this.table.put(pk, b);
|
||||
this.columnames.addAll(map.keySet());
|
||||
}
|
||||
|
||||
public void put(
|
||||
byte[] pk,
|
||||
String key, byte[] value
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
byte[] b = BEncoder.encodeMap(key, value);
|
||||
this.table.put(pk, b);
|
||||
this.columnames.add(key);
|
||||
}
|
||||
public void put(
|
||||
byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
byte[] b = BEncoder.encodeMap(
|
||||
key0, value0,
|
||||
key1, value1
|
||||
);
|
||||
this.table.put(pk, b);
|
||||
this.columnames.add(key0);
|
||||
this.columnames.add(key1);
|
||||
}
|
||||
public void put(
|
||||
byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
byte[] b = BEncoder.encodeMap(
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2
|
||||
);
|
||||
this.table.put(pk, b);
|
||||
this.columnames.add(key0);
|
||||
this.columnames.add(key1);
|
||||
this.columnames.add(key2);
|
||||
}
|
||||
public void put(
|
||||
byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2,
|
||||
String key3, byte[] value3
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
byte[] b = BEncoder.encodeMap(
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2,
|
||||
key3, value3
|
||||
);
|
||||
this.table.put(pk, b);
|
||||
this.columnames.add(key0);
|
||||
this.columnames.add(key1);
|
||||
this.columnames.add(key2);
|
||||
this.columnames.add(key3);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -98,15 +173,22 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
|
|||
* @return the map if one found or NULL if no entry exists or the entry is corrupt
|
||||
* @throws IOException
|
||||
*/
|
||||
public Map<String, byte[]> get(byte[] key) throws IOException {
|
||||
byte[] b = this.table.get(key);
|
||||
public Map<String, byte[]> get(byte[] pk) throws IOException {
|
||||
byte[] b = this.table.get(pk);
|
||||
if (b == null) return null;
|
||||
return b2m(b);
|
||||
}
|
||||
|
||||
static Map<String, byte[]> b2m(byte[] b) {
|
||||
|
||||
public byte[] getProp(byte[] pk, String key) throws IOException {
|
||||
byte[] b = this.table.get(pk);
|
||||
if (b == null) return null;
|
||||
System.out.println("b = " + new String(b));
|
||||
Map<String, byte[]> map = b2m(b);
|
||||
return map.get(key);
|
||||
}
|
||||
|
||||
private static Map<String, byte[]> b2m(byte[] b) {
|
||||
if (b == null) return null;
|
||||
//System.out.println("b = " + new String(b));
|
||||
BDecoder decoder = new BDecoder(b);
|
||||
BObject bobj = decoder.parse();
|
||||
if (bobj.getType() != BDecoder.BType.dictionary) return null;
|
||||
|
|
@ -124,8 +206,8 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
|
|||
* @param key
|
||||
* @throws IOException
|
||||
*/
|
||||
public void delete(byte[] key) throws IOException {
|
||||
this.table.remove(key);
|
||||
public void delete(byte[] pk) throws IOException {
|
||||
this.table.remove(pk);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -133,8 +215,8 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
|
|||
* @param key
|
||||
* @return true if the row exists
|
||||
*/
|
||||
public boolean has(byte[] key) {
|
||||
return this.table.has(key);
|
||||
public boolean has(byte[] pk) {
|
||||
return this.table.has(pk);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -226,31 +308,57 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
|
|||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// test the class
|
||||
File f = new File(new File("maptest").getAbsolutePath());
|
||||
//System.out.println(f.getAbsolutePath());
|
||||
//System.out.println(f.getParent());
|
||||
if (f.exists()) FileUtils.deletedelete(f);
|
||||
try {
|
||||
BEncodedHeap map = new BEncodedHeap(f, 4);
|
||||
// put some values into the map
|
||||
Map<String, byte[]> m = new HashMap<String, byte[]>();
|
||||
m.put("k", "000".getBytes()); map.put("123".getBytes(), m);
|
||||
m.put("k", "111".getBytes()); map.put("456".getBytes(), m);
|
||||
m.put("k", "222".getBytes()); map.put("789".getBytes(), m);
|
||||
// iterate over keys
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
|
||||
while (i.hasNext()) {
|
||||
Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
|
||||
System.out.println(new String(entry.getKey(), "UTF-8") + ": " + entry.getValue());
|
||||
public List<String> columns() {
|
||||
if (this.columnames.size() == 0) {
|
||||
for (Map.Entry<byte[], Map<String, byte[]>> row: this) {
|
||||
this.columnames.addAll(row.getValue().keySet());
|
||||
}
|
||||
}
|
||||
List<String> l = new ArrayList<String>();
|
||||
l.addAll(this.columnames);
|
||||
return l;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
if (args.length == 0) {
|
||||
// test the class
|
||||
File f = new File(new File("maptest").getAbsolutePath());
|
||||
//System.out.println(f.getAbsolutePath());
|
||||
//System.out.println(f.getParent());
|
||||
if (f.exists()) FileUtils.deletedelete(f);
|
||||
try {
|
||||
BEncodedHeap map = new BEncodedHeap(f, 4);
|
||||
// put some values into the map
|
||||
Map<String, byte[]> m = new HashMap<String, byte[]>();
|
||||
m.put("k", "000".getBytes()); map.put("123".getBytes(), m);
|
||||
m.put("k", "111".getBytes()); map.put("456".getBytes(), m);
|
||||
m.put("k", "222".getBytes()); map.put("789".getBytes(), m);
|
||||
// iterate over keys
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
|
||||
while (i.hasNext()) {
|
||||
Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
|
||||
System.out.println(new String(entry.getKey(), "UTF-8") + ": " + entry.getValue());
|
||||
}
|
||||
// clean up
|
||||
map.close();
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
} else {
|
||||
File f = new File(args[0]);
|
||||
try {
|
||||
BEncodedHeap map = new BEncodedHeap(f, 12);
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
|
||||
while (i.hasNext()) {
|
||||
Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
|
||||
System.out.println(new String(entry.getKey(), "UTF-8") + ": " + entry.getValue());
|
||||
}
|
||||
map.close();
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
// clean up
|
||||
map.close();
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
333
source/net/yacy/kelondro/blob/BEncodedHeapArray.java
Normal file
333
source/net/yacy/kelondro/blob/BEncodedHeapArray.java
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
// BEncodedHeapArray.java
|
||||
// (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 14.01.2010 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 6539 $
|
||||
// $LastChangedBy: low012 $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
|
||||
package net.yacy.kelondro.blob;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import net.yacy.kelondro.index.RowSpaceExceededException;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
|
||||
|
||||
public class BEncodedHeapArray {
|
||||
|
||||
private static final String suffix = ".bheap";
|
||||
private static final String system_table_pkcounter = "pkcounter";
|
||||
private static final String system_table_pkcounter_counterName = "pk";
|
||||
|
||||
private File location;
|
||||
private ConcurrentHashMap<String, BEncodedHeap> tables;
|
||||
private int keymaxlen;
|
||||
|
||||
public BEncodedHeapArray(final File location, final int keymaxlen) {
|
||||
this.location = location;
|
||||
this.keymaxlen = keymaxlen;
|
||||
this.tables = new ConcurrentHashMap<String, BEncodedHeap>();
|
||||
String[] files = this.location.list();
|
||||
String tablename;
|
||||
File file;
|
||||
for (String f: files) {
|
||||
if (f.endsWith(suffix)) {
|
||||
file = new File(this.location, f);
|
||||
if (file.length() == 0) {
|
||||
file.delete();
|
||||
continue;
|
||||
}
|
||||
tablename = f.substring(0, f.length() - suffix.length());
|
||||
try {
|
||||
getHeap(tablename);
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Iterator<String> tables() {
|
||||
return this.tables.keySet().iterator();
|
||||
}
|
||||
|
||||
public void close(final String tablename) {
|
||||
final BEncodedHeap heap = this.tables.remove(tablename);
|
||||
if (heap == null) return;
|
||||
heap.close();
|
||||
}
|
||||
|
||||
public void close() {
|
||||
for (BEncodedHeap heap: this.tables.values()) heap.close();
|
||||
this.tables.clear();
|
||||
}
|
||||
|
||||
public void clear(final String tablename) throws IOException {
|
||||
BEncodedHeap heap = getHeap(tablename);
|
||||
if (heap == null) return;
|
||||
heap.clear();
|
||||
File f = heap.getFile();
|
||||
heap.close();
|
||||
heap = null;
|
||||
this.tables.remove(tablename);
|
||||
FileUtils.deletedelete(f);
|
||||
}
|
||||
|
||||
public boolean hasHeap(final String tablename) {
|
||||
try {
|
||||
return getHeap(tablename) != null;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private BEncodedHeap getHeap(final String tablename) throws IOException {
|
||||
final String table = tablename + suffix;
|
||||
BEncodedHeap heap = this.tables.get(tablename);
|
||||
if (heap != null) return heap;
|
||||
|
||||
// open a new heap and register it in the tables
|
||||
final File heapf = new File(this.location, table);
|
||||
heap = new BEncodedHeap(heapf, this.keymaxlen);
|
||||
this.tables.put(tablename, heap);
|
||||
return heap;
|
||||
}
|
||||
|
||||
public int size(String table) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.size();
|
||||
}
|
||||
|
||||
private byte[] ukey(String tablename) throws IOException {
|
||||
byte[] pk = select(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName);
|
||||
int pki;
|
||||
if (pk == null) {
|
||||
pki = size(tablename);
|
||||
} else {
|
||||
pki = Integer.parseInt(new String(pk)) + 1;
|
||||
}
|
||||
while (true) {
|
||||
pk = int2key(pki).getBytes();
|
||||
if (!has(tablename, pk)) break;
|
||||
pki++;
|
||||
}
|
||||
return pk;
|
||||
}
|
||||
|
||||
private String int2key(int i) {
|
||||
StringBuilder sb = new StringBuilder(this.keymaxlen);
|
||||
String is = Integer.toString(i);
|
||||
for (int j = 0; j < this.keymaxlen - is.length(); j++) sb.append('0');
|
||||
sb.append(is);
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* insert a map into a table using a new unique key
|
||||
* @param tablename
|
||||
* @param map
|
||||
* @throws RowSpaceExceededException
|
||||
* @throws IOException
|
||||
*/
|
||||
public byte[] insert(final String tablename, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk, map);
|
||||
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
|
||||
return uk;
|
||||
}
|
||||
|
||||
public byte[] insert(final String tablename, String key, byte[] value) throws RowSpaceExceededException, IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk, key, value);
|
||||
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
|
||||
return uk;
|
||||
}
|
||||
|
||||
public byte[] insert(final String tablename,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk,
|
||||
key0, value0,
|
||||
key1, value1
|
||||
);
|
||||
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
|
||||
return uk;
|
||||
}
|
||||
|
||||
public byte[] insert(final String tablename,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2
|
||||
);
|
||||
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
|
||||
return uk;
|
||||
}
|
||||
|
||||
public byte[] insert(final String tablename,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2,
|
||||
String key3, byte[] value3
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2,
|
||||
key3, value3
|
||||
);
|
||||
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
|
||||
return uk;
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk,
|
||||
String key, byte[] value
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
heap.put(pk, key, value);
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
heap.put(pk,
|
||||
key0, value0,
|
||||
key1, value1
|
||||
);
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
heap.put(pk,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2
|
||||
);
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2,
|
||||
String key3, byte[] value3
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
heap.put(pk,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2,
|
||||
key3, value3
|
||||
);
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
heap.put(pk, map);
|
||||
}
|
||||
|
||||
public Map<String, byte[]> select(final String table, byte[] pk) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.get(pk);
|
||||
}
|
||||
|
||||
public byte[] select(final String table, byte[] pk, String key) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.getProp(pk, key);
|
||||
}
|
||||
|
||||
public void delete(final String table, byte[] pk) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
heap.delete(pk);
|
||||
}
|
||||
|
||||
public boolean has(String table, byte[] key) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.has(key);
|
||||
}
|
||||
|
||||
public Iterator<byte[]> keys(String table) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.keys();
|
||||
}
|
||||
|
||||
public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(String table) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.iterator();
|
||||
}
|
||||
|
||||
public List<String> columns(String table) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.columns();
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// test the class
|
||||
File f = new File(new File("maptest").getAbsolutePath());
|
||||
// System.out.println(f.getAbsolutePath());
|
||||
// System.out.println(f.getParent());
|
||||
try {
|
||||
BEncodedHeapArray map = new BEncodedHeapArray(f.getParentFile(), 4);
|
||||
// put some values into the map
|
||||
Map<String, byte[]> m = new HashMap<String, byte[]>();
|
||||
m.put("k", "000".getBytes());
|
||||
map.insert("testdao", "123".getBytes(), m);
|
||||
m.put("k", "111".getBytes());
|
||||
map.insert("testdao", "456".getBytes(), m);
|
||||
m.put("k", "222".getBytes());
|
||||
map.insert("testdao", "789".getBytes(), m);
|
||||
// iterate over keys
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator("testdao");
|
||||
while (i.hasNext()) {
|
||||
Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
|
||||
System.out.println(new String(entry.getKey(), "UTF-8") + ": "
|
||||
+ entry.getValue());
|
||||
}
|
||||
// clean up
|
||||
map.close();
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -119,15 +119,14 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
* @return true if the key exists, false otherwise
|
||||
*/
|
||||
@Override
|
||||
public synchronized boolean has(final byte[] key) {
|
||||
public synchronized boolean has(byte[] key) {
|
||||
assert index != null;
|
||||
assert this.keylength == key.length : this.keylength + "!=" + key.length;
|
||||
|
||||
key = normalizeKey(key);
|
||||
// check the buffer
|
||||
if (this.buffer.containsKey(new String(key))) return true;
|
||||
return super.has(key);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* add a BLOB to the heap: this adds the blob always to the end of the file
|
||||
* @param key
|
||||
|
|
@ -135,19 +134,15 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
* @throws IOException
|
||||
* @throws RowSpaceExceededException
|
||||
*/
|
||||
private void add(final byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
|
||||
private void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
|
||||
assert blob.length > 0;
|
||||
assert key.length == this.keylength;
|
||||
assert this.keylength == key.length : this.keylength + "!=" + key.length;
|
||||
if ((blob == null) || (blob.length == 0)) return;
|
||||
final int pos = (int) file.length();
|
||||
key = normalizeKey(key);
|
||||
index.put(key, pos);
|
||||
file.seek(pos);
|
||||
file.writeInt(this.keylength + blob.length);
|
||||
file.write(key);
|
||||
if (this.keylength > key.length) {
|
||||
for (int i = 0; i < this.keylength - key.length; i++) file.write(HeapWriter.ZERO);
|
||||
}
|
||||
file.write(blob, 0, blob.length);
|
||||
}
|
||||
|
||||
|
|
@ -171,8 +166,7 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
byte[] key, blob;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
key = entry.getKey().getBytes();
|
||||
assert key.length == this.keylength : "key.length = " + key.length + ", this.keylength = " + this.keylength;
|
||||
key = normalizeKey(entry.getKey().getBytes());
|
||||
blob = entry.getValue();
|
||||
posBuffer += 4 + this.keylength + blob.length;
|
||||
}
|
||||
|
|
@ -187,8 +181,7 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
byte[] b;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
key = entry.getKey().getBytes();
|
||||
assert key.length == this.keylength : "key.length = " + key.length + ", this.keylength = " + this.keylength;
|
||||
key = normalizeKey(entry.getKey().getBytes());
|
||||
blob = entry.getValue();
|
||||
index.put(key, posFile);
|
||||
b = AbstractWriter.int2array(this.keylength + blob.length);
|
||||
|
|
@ -218,8 +211,8 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public synchronized byte[] get(final byte[] key) throws IOException {
|
||||
assert this.keylength == key.length : this.keylength + "!=" + key.length;
|
||||
public synchronized byte[] get(byte[] key) throws IOException {
|
||||
key = normalizeKey(key);
|
||||
|
||||
// check the buffer
|
||||
byte[] blob = this.buffer.get(new String(key));
|
||||
|
|
@ -235,9 +228,8 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public synchronized long length(final byte[] key) throws IOException {
|
||||
assert this.keylength == key.length : this.keylength + "!=" + key.length;
|
||||
|
||||
public synchronized long length(byte[] key) throws IOException {
|
||||
key = normalizeKey(key);
|
||||
// check the buffer
|
||||
byte[] blob = this.buffer.get(new String(key));
|
||||
if (blob != null) return blob.length;
|
||||
|
|
@ -297,8 +289,8 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
* @throws RowSpaceExceededException
|
||||
*/
|
||||
@Override
|
||||
public synchronized void put(final byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
|
||||
assert this.keylength == key.length : this.keylength + "!=" + key.length;
|
||||
public synchronized void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
|
||||
key = normalizeKey(key);
|
||||
|
||||
// we do not write records of length 0 into the BLOB
|
||||
if (b.length == 0) return;
|
||||
|
|
@ -329,8 +321,8 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
this.buffersize += b.length;
|
||||
}
|
||||
|
||||
private boolean putToGap(final byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
|
||||
assert this.keylength == key.length : this.keylength + "!=" + key.length;
|
||||
private boolean putToGap(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
|
||||
key = normalizeKey(key);
|
||||
|
||||
// we do not write records of length 0 into the BLOB
|
||||
if (b.length == 0) return true;
|
||||
|
|
@ -423,8 +415,8 @@ public final class Heap extends HeapModifier implements BLOB {
|
|||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public synchronized void remove(final byte[] key) throws IOException {
|
||||
assert this.keylength == key.length : this.keylength + "!=" + key.length;
|
||||
public synchronized void remove(byte[] key) throws IOException {
|
||||
key = normalizeKey(key);
|
||||
|
||||
// check the buffer
|
||||
byte[] blob = this.buffer.remove(new String(key));
|
||||
|
|
|
|||
|
|
@ -94,8 +94,8 @@ public class HeapModifier extends HeapReader implements BLOB {
|
|||
* @param key the primary key
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized void remove(final byte[] key) throws IOException {
|
||||
assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
|
||||
public synchronized void remove(byte[] key) throws IOException {
|
||||
key = normalizeKey(key);
|
||||
|
||||
// check if the index contains the key
|
||||
final long seek = index.get(key);
|
||||
|
|
@ -234,14 +234,14 @@ public class HeapModifier extends HeapReader implements BLOB {
|
|||
}
|
||||
|
||||
public synchronized int replace(byte[] key, Rewriter rewriter) throws IOException {
|
||||
assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
|
||||
|
||||
key = normalizeKey(key);
|
||||
|
||||
// check if the index contains the key
|
||||
final long pos = index.get(key);
|
||||
if (pos < 0) return 0;
|
||||
|
||||
// check consistency of the index
|
||||
assert (checkKey(key, pos)) : "key compare failed; key = " + new String(key) + ", seek = " + pos;
|
||||
assert checkKey(key, pos) : "key compare failed; key = " + new String(key) + ", seek = " + pos;
|
||||
|
||||
// access the file and read the container
|
||||
file.seek(pos);
|
||||
|
|
|
|||
|
|
@ -110,6 +110,21 @@ public class HeapReader {
|
|||
this.file.close();
|
||||
// the file will be opened again automatically when the next access to it comes.
|
||||
}
|
||||
|
||||
protected byte[] normalizeKey(byte[] key) {
|
||||
return normalizeKey(key, this.keylength);
|
||||
}
|
||||
|
||||
protected static byte[] normalizeKey(byte[] key, int keylength) {
|
||||
if (key.length == keylength) return key;
|
||||
byte[] k = new byte[keylength];
|
||||
if (key.length < keylength) {
|
||||
System.arraycopy(key, 0, k, 0, key.length);
|
||||
} else {
|
||||
System.arraycopy(key, 0, k, 0, keylength);
|
||||
}
|
||||
return k;
|
||||
}
|
||||
|
||||
private boolean initIndexReadDump() {
|
||||
// look for an index dump and read it if it exist
|
||||
|
|
@ -278,9 +293,9 @@ public class HeapReader {
|
|||
* @param key
|
||||
* @return true if the key exists, false otherwise
|
||||
*/
|
||||
public synchronized boolean has(final byte[] key) {
|
||||
public synchronized boolean has(byte[] key) {
|
||||
assert index != null;
|
||||
assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
|
||||
key = normalizeKey(key);
|
||||
|
||||
// check if the file index contains the key
|
||||
return index.get(key) >= 0;
|
||||
|
|
@ -344,8 +359,8 @@ public class HeapReader {
|
|||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized byte[] get(final byte[] key) throws IOException {
|
||||
assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
|
||||
public synchronized byte[] get(byte[] key) throws IOException {
|
||||
key = normalizeKey(key);
|
||||
|
||||
// check if the index contains the key
|
||||
final long pos = index.get(key);
|
||||
|
|
@ -377,8 +392,9 @@ public class HeapReader {
|
|||
return blob;
|
||||
}
|
||||
|
||||
protected boolean checkKey(final byte[] key, final long pos) throws IOException {
|
||||
file.seek(pos);
|
||||
protected boolean checkKey(byte[] key, final long pos) throws IOException {
|
||||
key = normalizeKey(key);
|
||||
file.seek(pos);
|
||||
file.readInt(); // skip the size value
|
||||
|
||||
// read the key
|
||||
|
|
@ -394,7 +410,7 @@ public class HeapReader {
|
|||
* @throws IOException
|
||||
*/
|
||||
public synchronized long length(byte[] key) throws IOException {
|
||||
assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
|
||||
key = normalizeKey(key);
|
||||
|
||||
// check if the index contains the key
|
||||
final long pos = index.get(key);
|
||||
|
|
@ -533,13 +549,23 @@ public class HeapReader {
|
|||
|
||||
private Map.Entry<byte[], byte[]> next0() {
|
||||
try {
|
||||
byte b;
|
||||
byte[] payload;
|
||||
final byte[] key = new byte[this.keylen];
|
||||
final int keylen1 = this.keylen - 1;
|
||||
while (true) {
|
||||
int len = is.readInt();
|
||||
byte[] key = new byte[this.keylen];
|
||||
if (is.read(key) < key.length) return null;
|
||||
byte[] payload = new byte[len - this.keylen];
|
||||
if (len == 0) continue;
|
||||
b = is.readByte(); // check for empty record
|
||||
if (b == 0) {
|
||||
// read some more bytes to consume the empty record
|
||||
is.skip(len - 1);
|
||||
continue;
|
||||
}
|
||||
key[0] = b;
|
||||
if (is.read(key, 1, keylen1) < keylen1) return null;
|
||||
payload = new byte[len - this.keylen];
|
||||
if (is.read(payload) < payload.length) return null;
|
||||
if (key[0] == 0) continue; // this is an empty gap
|
||||
return new entry(key, payload);
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
|
|
@ -596,4 +622,19 @@ public class HeapReader {
|
|||
}
|
||||
}
|
||||
|
||||
public static void main(final String args[]) {
|
||||
File f = new File(args[0]);
|
||||
try {
|
||||
entries hr = new HeapReader.entries(f, 12);
|
||||
Map.Entry<byte[], byte[]> entry;
|
||||
while (hr.hasNext()) {
|
||||
entry = hr.next();
|
||||
System.out.println(new String(entry.getKey()) + ":" + new String(entry.getValue()));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,11 +95,11 @@ public final class HeapWriter {
|
|||
* @throws RowSpaceExceededException
|
||||
* @throws RowSpaceExceededException
|
||||
*/
|
||||
public synchronized void add(final byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
|
||||
public synchronized void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
|
||||
//System.out.println("HeapWriter.add: " + new String(key));
|
||||
assert blob.length > 0;
|
||||
assert key.length == this.keylength;
|
||||
assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
|
||||
key = HeapReader.normalizeKey(key, this.keylength);
|
||||
assert index.row().primaryKeyLength == this.keylength : index.row().primaryKeyLength + "!=" + key.length;
|
||||
assert index.get(key) < 0 : "index.get(key) = " + index.get(key) + ", index.size() = " + index.size() + ", file.length() = " + this.heapFileTMP.length() + ", key = " + new String(key); // must not occur before
|
||||
if ((blob == null) || (blob.length == 0)) return;
|
||||
index.putUnique(key, this.seek);
|
||||
|
|
|
|||
|
|
@ -43,10 +43,14 @@ import net.yacy.kelondro.util.ScoreCluster;
|
|||
|
||||
|
||||
public class MapDataMining extends MapHeap {
|
||||
|
||||
private final static Long LONG0 = Long.valueOf(0);
|
||||
private final static Double DOUBLE0 = Double.valueOf(0.0);
|
||||
|
||||
private final String[] sortfields, longaccfields, doubleaccfields;
|
||||
private HashMap<String, ScoreCluster<String>> sortClusterMap; // a String-kelondroMScoreCluster - relation
|
||||
private HashMap<String, Object> accMap; // to store accumulations of specific fields
|
||||
private HashMap<String, Long> accLong; // to store accumulations of Long cells
|
||||
private HashMap<String, Double> accDouble; // to store accumulations of Double cells
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public MapDataMining(final File heapFile,
|
||||
|
|
@ -77,21 +81,22 @@ public class MapDataMining extends MapHeap {
|
|||
|
||||
Long[] longaccumulator = null;
|
||||
Double[] doubleaccumulator = null;
|
||||
if ((longaccfields == null) && (doubleaccfields == null)) {
|
||||
accMap = null;
|
||||
if (longaccfields == null) {
|
||||
accLong = null;
|
||||
} else {
|
||||
accMap = new HashMap<String, Object>();
|
||||
if (longaccfields != null) {
|
||||
longaccumulator = new Long[longaccfields.length];
|
||||
for (int i = 0; i < longaccfields.length; i++) {
|
||||
longaccumulator[i] = Long.valueOf(0);
|
||||
}
|
||||
accLong = new HashMap<String, Long>();
|
||||
longaccumulator = new Long[longaccfields.length];
|
||||
for (int i = 0; i < longaccfields.length; i++) {
|
||||
longaccumulator[i] = LONG0;
|
||||
}
|
||||
if (doubleaccfields != null) {
|
||||
doubleaccumulator = new Double[doubleaccfields.length];
|
||||
for (int i = 0; i < doubleaccfields.length; i++) {
|
||||
doubleaccumulator[i] = Double.valueOf(0);
|
||||
}
|
||||
}
|
||||
if (doubleaccfields == null) {
|
||||
accDouble = null;
|
||||
} else {
|
||||
accDouble = new HashMap<String, Double>();
|
||||
doubleaccumulator = new Double[doubleaccfields.length];
|
||||
for (int i = 0; i < doubleaccfields.length; i++) {
|
||||
doubleaccumulator[i] = DOUBLE0;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -99,7 +104,7 @@ public class MapDataMining extends MapHeap {
|
|||
if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try {
|
||||
final CloneableIterator<byte[]> it = super.keys(true, false);
|
||||
String mapname;
|
||||
Object cell;
|
||||
String cell;
|
||||
long valuel;
|
||||
double valued;
|
||||
Map<String, String> map;
|
||||
|
|
@ -117,8 +122,7 @@ public class MapDataMining extends MapHeap {
|
|||
cell = map.get(longaccfields[i]);
|
||||
valuel = 0;
|
||||
if (cell != null) try {
|
||||
if (cell instanceof Long) valuel = ((Long) cell).longValue();
|
||||
if (cell instanceof String) valuel = Long.parseLong((String) cell);
|
||||
valuel = Long.parseLong(cell);
|
||||
longaccumulator[i] = Long.valueOf(longaccumulator[i].longValue() + valuel);
|
||||
} catch (final NumberFormatException e) {}
|
||||
}
|
||||
|
|
@ -127,8 +131,7 @@ public class MapDataMining extends MapHeap {
|
|||
cell = map.get(doubleaccfields[i]);
|
||||
valued = 0d;
|
||||
if (cell != null) try {
|
||||
if (cell instanceof Double) valued = ((Double) cell).doubleValue();
|
||||
if (cell instanceof String) valued = Double.parseDouble((String) cell);
|
||||
valued = Double.parseDouble(cell);
|
||||
doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued);
|
||||
} catch (final NumberFormatException e) {}
|
||||
}
|
||||
|
|
@ -151,8 +154,8 @@ public class MapDataMining extends MapHeap {
|
|||
if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]);
|
||||
|
||||
// fill acc map
|
||||
if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) accMap.put(longaccfields[i], longaccumulator[i]);
|
||||
if (doubleaccfields != null && doubleaccumulator != null) for (int i = 0; i < doubleaccfields.length; i++) accMap.put(doubleaccfields[i], doubleaccumulator[i]);
|
||||
if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) accLong.put(longaccfields[i], longaccumulator[i]);
|
||||
if (doubleaccfields != null && doubleaccumulator != null) for (int i = 0; i < doubleaccfields.length; i++) accDouble.put(doubleaccfields[i], doubleaccumulator[i]);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -165,19 +168,20 @@ public class MapDataMining extends MapHeap {
|
|||
}
|
||||
}
|
||||
|
||||
if ((longaccfields == null) && (doubleaccfields == null)) {
|
||||
accMap = null;
|
||||
if (longaccfields == null) {
|
||||
accLong = null;
|
||||
} else {
|
||||
accMap = new HashMap<String, Object>();
|
||||
if (longaccfields != null) {
|
||||
for (int i = 0; i < longaccfields.length; i++) {
|
||||
accMap.put(longaccfields[i], Long.valueOf(0));
|
||||
}
|
||||
accLong = new HashMap<String, Long>();
|
||||
for (int i = 0; i < longaccfields.length; i++) {
|
||||
accLong.put(longaccfields[i], LONG0);
|
||||
}
|
||||
if (doubleaccfields != null) {
|
||||
for (int i = 0; i < doubleaccfields.length; i++) {
|
||||
accMap.put(doubleaccfields[i], new Double(0));
|
||||
}
|
||||
}
|
||||
if (doubleaccfields == null) {
|
||||
accDouble = null;
|
||||
} else {
|
||||
accDouble = new HashMap<String, Double>();
|
||||
for (int i = 0; i < doubleaccfields.length; i++) {
|
||||
accDouble.put(doubleaccfields[i], DOUBLE0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -190,6 +194,9 @@ public class MapDataMining extends MapHeap {
|
|||
|
||||
super.put(key, newMap);
|
||||
|
||||
// update sortCluster
|
||||
if (sortClusterMap != null) updateSortCluster(key, newMap);
|
||||
|
||||
// update elementCount
|
||||
if ((longaccfields != null) || (doubleaccfields != null)) {
|
||||
final Map<String, String> oldMap = super.get(key, false);
|
||||
|
|
@ -197,13 +204,10 @@ public class MapDataMining extends MapHeap {
|
|||
// element exists, update acc
|
||||
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(oldMap, false);
|
||||
}
|
||||
}
|
||||
|
||||
// update sortCluster
|
||||
if (sortClusterMap != null) updateSortCluster(key, newMap);
|
||||
|
||||
// update accumulators with new values (add)
|
||||
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(newMap, true);
|
||||
// update accumulators with new values (add)
|
||||
updateAcc(newMap, true);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateAcc(final Map<String, String> map, final boolean add) {
|
||||
|
|
@ -217,11 +221,11 @@ public class MapDataMining extends MapHeap {
|
|||
if (value != null) {
|
||||
try {
|
||||
valuel = Long.parseLong(value);
|
||||
longaccumulator = (Long) accMap.get(longaccfields[i]);
|
||||
longaccumulator = accLong.get(longaccfields[i]);
|
||||
if (add) {
|
||||
accMap.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() + valuel));
|
||||
accLong.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() + valuel));
|
||||
} else {
|
||||
accMap.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() - valuel));
|
||||
accLong.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() - valuel));
|
||||
}
|
||||
} catch (final NumberFormatException e) {}
|
||||
}
|
||||
|
|
@ -231,11 +235,11 @@ public class MapDataMining extends MapHeap {
|
|||
if (value != null) {
|
||||
try {
|
||||
valued = Double.parseDouble(value);
|
||||
doubleaccumulator = (Double) accMap.get(doubleaccfields[i]);
|
||||
doubleaccumulator = accDouble.get(doubleaccfields[i]);
|
||||
if (add) {
|
||||
accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() + valued));
|
||||
accDouble.put(doubleaccfields[i], Double.valueOf(doubleaccumulator.doubleValue() + valued));
|
||||
} else {
|
||||
accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() - valued));
|
||||
accDouble.put(doubleaccfields[i], Double.valueOf(doubleaccumulator.doubleValue() - valued));
|
||||
}
|
||||
} catch (final NumberFormatException e) {}
|
||||
}
|
||||
|
|
@ -330,13 +334,13 @@ public class MapDataMining extends MapHeap {
|
|||
}
|
||||
|
||||
public synchronized long getLongAcc(final String field) {
|
||||
final Long accumulator = (Long) accMap.get(field);
|
||||
final Long accumulator = accLong.get(field);
|
||||
if (accumulator == null) return -1;
|
||||
return accumulator.longValue();
|
||||
}
|
||||
|
||||
public synchronized double getDoubleAcc(final String field) {
|
||||
final Double accumulator = (Double) accMap.get(field);
|
||||
final Double accumulator = accDouble.get(field);
|
||||
if (accumulator == null) return -1;
|
||||
return accumulator.doubleValue();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,6 +109,16 @@ public class BDecoder {
|
|||
os.write(_p);
|
||||
os.write(this.b);
|
||||
}
|
||||
public static void toStream(OutputStream os, byte[] b) throws IOException {
|
||||
os.write(Integer.toString(b.length).getBytes());
|
||||
os.write(_p);
|
||||
os.write(b);
|
||||
}
|
||||
public static void toStream(OutputStream os, String s) throws IOException {
|
||||
os.write(Integer.toString(s.length()).getBytes());
|
||||
os.write(_p);
|
||||
os.write(s.getBytes());
|
||||
}
|
||||
}
|
||||
|
||||
public static class BListObject extends BDfltObject implements BObject {
|
||||
|
|
@ -164,6 +174,60 @@ public class BDecoder {
|
|||
}
|
||||
os.write(_e);
|
||||
}
|
||||
public static void toStream(
|
||||
OutputStream os,
|
||||
String key, byte[] value
|
||||
) throws IOException {
|
||||
os.write(_d);
|
||||
BStringObject.toStream(os, key);
|
||||
BStringObject.toStream(os, value);
|
||||
os.write(_e);
|
||||
}
|
||||
public static void toStream(
|
||||
OutputStream os,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1
|
||||
) throws IOException {
|
||||
os.write(_d);
|
||||
BStringObject.toStream(os, key0);
|
||||
BStringObject.toStream(os, value0);
|
||||
BStringObject.toStream(os, key1);
|
||||
BStringObject.toStream(os, value1);
|
||||
os.write(_e);
|
||||
}
|
||||
public static void toStream(
|
||||
OutputStream os,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2
|
||||
) throws IOException {
|
||||
os.write(_d);
|
||||
BStringObject.toStream(os, key0);
|
||||
BStringObject.toStream(os, value0);
|
||||
BStringObject.toStream(os, key1);
|
||||
BStringObject.toStream(os, value1);
|
||||
BStringObject.toStream(os, key2);
|
||||
BStringObject.toStream(os, value2);
|
||||
os.write(_e);
|
||||
}
|
||||
public static void toStream(
|
||||
OutputStream os,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2,
|
||||
String key3, byte[] value3
|
||||
) throws IOException {
|
||||
os.write(_d);
|
||||
BStringObject.toStream(os, key0);
|
||||
BStringObject.toStream(os, value0);
|
||||
BStringObject.toStream(os, key1);
|
||||
BStringObject.toStream(os, value1);
|
||||
BStringObject.toStream(os, key2);
|
||||
BStringObject.toStream(os, value2);
|
||||
BStringObject.toStream(os, key3);
|
||||
BStringObject.toStream(os, value3);
|
||||
os.write(_e);
|
||||
}
|
||||
}
|
||||
|
||||
public static class BIntegerObject extends BDfltObject implements BObject {
|
||||
|
|
|
|||
|
|
@ -68,8 +68,74 @@ public class BEncoder {
|
|||
dict.toStream(baos);
|
||||
baos.close();
|
||||
return baos.toByteArray();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
} catch (IOException e) {}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static byte[] encodeMap(String key, byte[] value) {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try {
|
||||
BDecoder.BDictionaryObject.toStream(baos, key, value);
|
||||
baos.close();
|
||||
return baos.toByteArray();
|
||||
} catch (IOException e) {}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static byte[] encodeMap(
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1
|
||||
) {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try {
|
||||
BDecoder.BDictionaryObject.toStream(
|
||||
baos,
|
||||
key0, value0,
|
||||
key1, value1
|
||||
);
|
||||
baos.close();
|
||||
return baos.toByteArray();
|
||||
} catch (IOException e) {}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static byte[] encodeMap(
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2
|
||||
) {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try {
|
||||
BDecoder.BDictionaryObject.toStream(
|
||||
baos,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2
|
||||
);
|
||||
baos.close();
|
||||
return baos.toByteArray();
|
||||
} catch (IOException e) {}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static byte[] encodeMap(
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2,
|
||||
String key3, byte[] value3
|
||||
) {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try {
|
||||
BDecoder.BDictionaryObject.toStream(
|
||||
baos,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2,
|
||||
key3, value3
|
||||
);
|
||||
baos.close();
|
||||
return baos.toByteArray();
|
||||
} catch (IOException e) {}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -124,7 +124,6 @@ public final class yacy {
|
|||
// static objects
|
||||
public static final String vString = yacyBuildProperties.getVersion();
|
||||
public static double version = 0.1;
|
||||
public static boolean pro = false;
|
||||
|
||||
public static final String vDATE = yacyBuildProperties.getBuildDate();
|
||||
public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
|
||||
|
|
@ -211,7 +210,6 @@ public final class yacy {
|
|||
Log.logSevere("STARTUP", "WARNING: the file " + f + " can not be created!");
|
||||
f.deleteOnExit();
|
||||
|
||||
pro = new File(homePath, "libx").exists();
|
||||
final String oldconf = "DATA/SETTINGS/httpProxy.conf".replace("/", File.separator);
|
||||
final String newconf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
|
||||
final File oldconffile = new File(homePath, oldconf);
|
||||
|
|
@ -220,7 +218,7 @@ public final class yacy {
|
|||
if(!oldconffile.renameTo(newconfFile))
|
||||
Log.logSevere("STARTUP", "WARNING: the file " + oldconffile + " can not be renamed to "+ newconfFile +"!");
|
||||
}
|
||||
sb = new Switchboard(homePath, "defaults/yacy.init".replace("/", File.separator), newconf, pro);
|
||||
sb = new Switchboard(homePath, "defaults/yacy.init".replace("/", File.separator), newconf);
|
||||
//sbSync.V(); // signal that the sb reference was set
|
||||
|
||||
// save information about available memory at startup time
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user