mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
added a flex-width-array: this is a table where it is
possible to add columns to an existing table git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2163 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
4a907a570f
commit
c75cacda95
|
@ -32,7 +32,7 @@ import java.util.Iterator;
|
|||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.kelondro.kelondroArray;
|
||||
import de.anomic.kelondro.kelondroFixedWidthArray;
|
||||
import de.anomic.kelondro.kelondroException;
|
||||
import de.anomic.kelondro.kelondroMScoreCluster;
|
||||
import de.anomic.kelondro.kelondroNaturalOrder;
|
||||
|
@ -93,8 +93,8 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
|
|||
log.logConfig("creating dump for index cache, " + wCache.size() + " words (and much more urls)");
|
||||
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
|
||||
if (indexDumpFile.exists()) indexDumpFile.delete();
|
||||
kelondroArray dumpArray = null;
|
||||
dumpArray = new kelondroArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
|
||||
kelondroFixedWidthArray dumpArray = null;
|
||||
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
|
||||
long startTime = System.currentTimeMillis();
|
||||
long messageTime = System.currentTimeMillis() + 5000;
|
||||
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
|
||||
|
@ -173,7 +173,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
|
|||
private long restore() throws IOException {
|
||||
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
|
||||
if (!(indexDumpFile.exists())) return 0;
|
||||
kelondroArray dumpArray = new kelondroArray(indexDumpFile);
|
||||
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile);
|
||||
log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations");
|
||||
long startTime = System.currentTimeMillis();
|
||||
long messageTime = System.currentTimeMillis() + 5000;
|
||||
|
|
|
@ -37,7 +37,7 @@ public class kelondroCollectionIndex {
|
|||
private int chunksize;
|
||||
private int partitions;
|
||||
private int maxChunks;
|
||||
private kelondroArray[] array;
|
||||
private kelondroFixedWidthArray[] array;
|
||||
private int[] arrayCapacity;
|
||||
|
||||
private static File arrayFile(File path, String filenameStub, int loadfactor, int chunksize, int partitionNumber) {
|
||||
|
@ -75,7 +75,7 @@ public class kelondroCollectionIndex {
|
|||
index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, columns, 1, 80, true);
|
||||
|
||||
// create array files
|
||||
this.array = new kelondroArray[partitions];
|
||||
this.array = new kelondroFixedWidthArray[partitions];
|
||||
this.arrayCapacity = new int[partitions];
|
||||
|
||||
// open array files
|
||||
|
@ -89,11 +89,11 @@ public class kelondroCollectionIndex {
|
|||
this.maxChunks = load;
|
||||
}
|
||||
|
||||
private kelondroArray openArrayFile(int genericChunkSize, int partitionNumber) throws IOException {
|
||||
private kelondroFixedWidthArray openArrayFile(int genericChunkSize, int partitionNumber) throws IOException {
|
||||
File f = arrayFile(path, filenameStub, loadfactor, genericChunkSize, partitionNumber);
|
||||
|
||||
if (f.exists()) {
|
||||
return new kelondroArray(f);
|
||||
return new kelondroFixedWidthArray(f);
|
||||
} else {
|
||||
int load = 1; for (int i = 0; i < partitionNumber; i++) load = load * loadfactor;
|
||||
int[] columns = new int[4];
|
||||
|
@ -103,7 +103,7 @@ public class kelondroCollectionIndex {
|
|||
columns[3] = 2; // last time wrote
|
||||
columns[4] = 2; // flag string, assigns collection order as currently stored in table
|
||||
columns[5] = load * genericChunkSize;
|
||||
return new kelondroArray(f, columns, 0, true);
|
||||
return new kelondroFixedWidthArray(f, columns, 0, true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -49,13 +49,13 @@ package de.anomic.kelondro;
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
public class kelondroArray extends kelondroRecords {
|
||||
public class kelondroFixedWidthArray extends kelondroRecords implements kelondroArray {
|
||||
|
||||
// define the Over-Head-Array
|
||||
private static short thisOHBytes = 0; // our record definition does not need extra bytes
|
||||
private static short thisOHHandles = 0; // and no handles
|
||||
|
||||
public kelondroArray(File file, int[] columns, int intprops, boolean exitOnFail) {
|
||||
public kelondroFixedWidthArray(File file, int[] columns, int intprops, boolean exitOnFail) {
|
||||
// this creates a new array
|
||||
super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
|
||||
for (int i = 0; i < intprops; i++) try {
|
||||
|
@ -67,7 +67,7 @@ public class kelondroArray extends kelondroRecords {
|
|||
}
|
||||
}
|
||||
|
||||
public kelondroArray(File file) throws IOException{
|
||||
public kelondroFixedWidthArray(File file) throws IOException{
|
||||
// this opens a file with an existing array
|
||||
super(file, 0);
|
||||
}
|
||||
|
@ -144,19 +144,19 @@ public class kelondroArray extends kelondroRecords {
|
|||
// create <filename> <valuelen>
|
||||
File f = new File(args[1]);
|
||||
if (f.exists()) f.delete();
|
||||
kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2, true);
|
||||
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new int[]{Integer.parseInt(args[2])}, 2, true);
|
||||
fm.close();
|
||||
} else
|
||||
if ((args.length == 2) && (args[0].equals("-v"))) {
|
||||
// view <filename>
|
||||
kelondroArray fm = new kelondroArray(new File(args[1]));
|
||||
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
|
||||
fm.print();
|
||||
fm.print(true);
|
||||
fm.close();
|
||||
} else
|
||||
if ((args.length == 3) && (args[0].equals("-g"))) {
|
||||
// get <filename> <index>
|
||||
kelondroArray fm = new kelondroArray(new File(args[1]));
|
||||
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
|
||||
kelondroRow.Entry row = fm.get(Integer.parseInt(args[2]));
|
||||
for (int j = 0; j < fm.columns(); j++) System.out.print(row.getColString(j, null) + " ");
|
||||
System.out.println();
|
||||
|
@ -164,14 +164,14 @@ public class kelondroArray extends kelondroRecords {
|
|||
} else
|
||||
if ((args.length == 4) && (args[0].equals("-s"))) {
|
||||
// set <filename> <index> <value>
|
||||
kelondroArray fm = new kelondroArray(new File(args[1]));
|
||||
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
|
||||
kelondroRow.Entry row = fm.row().newEntry(new byte[][]{args[3].getBytes()});
|
||||
fm.set(Integer.parseInt(args[2]), row);
|
||||
fm.close();
|
||||
} else
|
||||
if ((args.length == 3) && (args[0].equals("-a"))) {
|
||||
// add <filename> <value>
|
||||
kelondroArray fm = new kelondroArray(new File(args[1]));
|
||||
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
|
||||
kelondroRow.Entry row = fm.row().newEntry(new byte[][] {args[2].getBytes()});
|
||||
int index = fm.add(row);
|
||||
System.out.println("Added to row " + index);
|
||||
|
@ -179,14 +179,14 @@ public class kelondroArray extends kelondroRecords {
|
|||
} else
|
||||
if ((args.length == 3) && (args[0].equals("-d"))) {
|
||||
// delete <filename> <index>
|
||||
kelondroArray fm = new kelondroArray(new File(args[1]));
|
||||
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
|
||||
fm.remove(Integer.parseInt(args[2]));
|
||||
fm.close();
|
||||
} else
|
||||
if ((args.length == 1) && (args[0].equals("-test"))) {
|
||||
File testfile = new File("test.array");
|
||||
if (testfile.exists()) testfile.delete();
|
||||
kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9, true);
|
||||
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new int[]{30, 50}, 9, true);
|
||||
for (int i = 0; i < 100; i++) {
|
||||
fm.set(i, fm.row().newEntry(new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()}));
|
||||
}
|
157
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Normal file
157
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Normal file
|
@ -0,0 +1,157 @@
|
|||
// kelondroFlexWidthArray.java
|
||||
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
|
||||
// first published 01.06.2006 on http://www.anomic.de
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
public class kelondroFlexWidthArray implements kelondroArray {
|
||||
|
||||
private kelondroFixedWidthArray[] col;
|
||||
private kelondroRow rowdef;
|
||||
|
||||
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef, boolean exitOnFail) throws IOException {
|
||||
this.rowdef = rowdef;
|
||||
|
||||
// initialize columns
|
||||
col = new kelondroFixedWidthArray[rowdef.columns()];
|
||||
String check = "";
|
||||
for (int i = 0; i < rowdef.columns(); i++) {
|
||||
col = null;
|
||||
check += '_';
|
||||
}
|
||||
|
||||
// open existing files
|
||||
String[] files = path.list();
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
if ((files[i].startsWith("col.") && (files[i].endsWith(".list")))) {
|
||||
int colstart = Integer.parseInt(files[i].substring(4, 7));
|
||||
int colend = (files[i].charAt(7) == '-') ? Integer.parseInt(files[i].substring(8, 11)) : colstart;
|
||||
/*
|
||||
int columns[] = new int[colend - colstart + 1];
|
||||
for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.width(j);
|
||||
col[colstart] = new kelondroFixedWidthArray(new File(path, files[i]), columns, 0, true);
|
||||
*/
|
||||
col[colstart] = new kelondroFixedWidthArray(new File(path, files[i]));
|
||||
for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// check if all columns are there
|
||||
int p, q;
|
||||
while ((p = check.indexOf('_')) >= 0) {
|
||||
q = p;
|
||||
if (p != 0) while ((check.charAt(q) == '_') && (q <= check.length() - 1)) q++;
|
||||
// create new array file
|
||||
int columns[] = new int[q - p + 1];
|
||||
for (int j = p; j <= q; j++) {
|
||||
columns[j - p] = rowdef.width(j);
|
||||
check = check.substring(0, j) + "X" + check.substring(j + 1);
|
||||
}
|
||||
col[p] = new kelondroFixedWidthArray(new File(path, colfilename(p, q)), columns, 0, true);
|
||||
}
|
||||
}
|
||||
|
||||
private static final String colfilename(int start, int end) {
|
||||
String f = Integer.toString(end);
|
||||
while (f.length() < 3) f = "0" + f;
|
||||
if (start == end) return "col." + f + ".list";
|
||||
f = Integer.toString(start) + "-" + f;
|
||||
while (f.length() < 7) f = "0" + f;
|
||||
return "col." + f + ".list";
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return col[0].size();
|
||||
}
|
||||
|
||||
public int columns() {
|
||||
return rowdef.columns();
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
|
||||
int r = 0;
|
||||
kelondroRow.Entry e0, e1, p;
|
||||
p = rowdef.newEntry();
|
||||
while (r < rowdef.columns()) {
|
||||
e0 = col[r].row().newEntry(rowentry.bytes(), rowdef.colstart[r], rowdef.colstart[r] - rowdef.colstart[r + col[r].columns() - 1] + rowdef.width(r));
|
||||
e1 = col[r].set(index, e0);
|
||||
for (int i = 0; i < col[r].columns(); i++)
|
||||
p.setCol(r + i, e1.getColBytes(i));
|
||||
r = r + col[r].columns();
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry get(int index) throws IOException {
|
||||
int r = 0;
|
||||
kelondroRow.Entry e, p;
|
||||
p = rowdef.newEntry();
|
||||
while (r < rowdef.columns()) {
|
||||
e = col[r].get(index);
|
||||
for (int i = 0; i < col[r].columns(); i++)
|
||||
p.setCol(r + i, e.getColBytes(i));
|
||||
r = r + col[r].columns();
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
public synchronized int add(kelondroRow.Entry rowentry) throws IOException {
|
||||
kelondroRow.Entry e;
|
||||
|
||||
e = col[0].row().newEntry(rowentry.bytes(), rowdef.colstart[0], rowdef.colstart[0] - rowdef.colstart[col[0].columns() - 1] + rowdef.width(0));
|
||||
int index = col[0].add(e);
|
||||
int r = col[0].columns();
|
||||
|
||||
while (r < rowdef.columns()) {
|
||||
e = col[r].row().newEntry(rowentry.bytes(), rowdef.colstart[r], rowdef.colstart[r] - rowdef.colstart[r + col[r].columns() - 1] + rowdef.width(r));
|
||||
col[r].set(index, e);
|
||||
r = r + col[r].columns();
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
public synchronized void remove(int index) throws IOException {
|
||||
int r = 0;
|
||||
while (r < rowdef.columns()) {
|
||||
col[r].remove(index);
|
||||
r = r + col[r].columns();
|
||||
}
|
||||
}
|
||||
|
||||
public void print() throws IOException {
|
||||
System.out.println("PRINTOUT of table, length=" + size());
|
||||
kelondroRow.Entry row;
|
||||
for (int i = 0; i < size(); i++) {
|
||||
System.out.print("row " + i + ": ");
|
||||
row = get(i);
|
||||
for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
|
||||
System.out.println();
|
||||
}
|
||||
System.out.println("EndOfTable");
|
||||
}
|
||||
|
||||
}
|
|
@ -134,7 +134,7 @@ import java.io.IOException;
|
|||
|
||||
public class kelondroHashtable {
|
||||
|
||||
private kelondroArray hashArray;
|
||||
private kelondroFixedWidthArray hashArray;
|
||||
private int offset;
|
||||
private int maxk;
|
||||
private int maxrehash;
|
||||
|
@ -152,7 +152,7 @@ public class kelondroHashtable {
|
|||
// this number is needed to omit grow of the table in case of re-hashing
|
||||
// the maxsize is re-computed to a virtual folding height and will result in a tablesize
|
||||
// less than the given maxsize. The actual maxsize can be retrieved by maxsize()
|
||||
this.hashArray = new kelondroArray(file, extCol(columns), 6, exitOnFail);
|
||||
this.hashArray = new kelondroFixedWidthArray(file, extCol(columns), 6, exitOnFail);
|
||||
this.offset = offset;
|
||||
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
|
||||
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
|
||||
|
@ -173,7 +173,7 @@ public class kelondroHashtable {
|
|||
|
||||
public kelondroHashtable(File file) throws IOException{
|
||||
// this opens a file with an existing hashtable
|
||||
this.hashArray = new kelondroArray(file);
|
||||
this.hashArray = new kelondroFixedWidthArray(file);
|
||||
this.offset = hashArray.geti(0);
|
||||
this.maxk = hashArray.geti(1);
|
||||
this.maxrehash = hashArray.geti(2);
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// kelondroTables.java
|
||||
// kelondroapTable.java
|
||||
// -----------------------
|
||||
// (C) by Michael Peter Christen; mc@anomic.de
|
||||
// first published on http://www.anomic.de
|
||||
|
@ -48,12 +48,12 @@ import java.util.HashMap;
|
|||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
public class kelondroTables {
|
||||
public class kelondroMapTable {
|
||||
|
||||
HashMap mTables, tTables, sTables;
|
||||
File tablesPath;
|
||||
|
||||
public kelondroTables(File tablesPath) {
|
||||
public kelondroMapTable(File tablesPath) {
|
||||
this.mTables = new HashMap();
|
||||
this.tTables = new HashMap();
|
||||
this.sTables = new HashMap();
|
|
@ -39,7 +39,7 @@ public class kelondroRow {
|
|||
|
||||
|
||||
private kelondroColumn[] row;
|
||||
private int[] colstart;
|
||||
protected int[] colstart;
|
||||
private HashMap encodedFormConfiguration;
|
||||
private int encodedFormLength;
|
||||
private int objectsize;
|
||||
|
@ -117,6 +117,11 @@ public class kelondroRow {
|
|||
return new Entry(rowinstance);
|
||||
}
|
||||
|
||||
public Entry newEntry(byte[] rowinstance, int start, int length) {
|
||||
if (rowinstance == null) return null;
|
||||
return new Entry(rowinstance);
|
||||
}
|
||||
|
||||
public Entry newEntry(byte[][] cells) {
|
||||
if (cells == null) return null;
|
||||
return new Entry(cells);
|
||||
|
@ -141,6 +146,12 @@ public class kelondroRow {
|
|||
}
|
||||
}
|
||||
|
||||
public Entry(byte[] rowinstance, int start, int length) {
|
||||
this.rowinstance = new byte[length];
|
||||
System.arraycopy(rowinstance, start, this.rowinstance, 0, length);
|
||||
for (int i = rowinstance.length; i < objectsize; i++) this.rowinstance[i] = 0;
|
||||
}
|
||||
|
||||
public Entry(byte[][] cols) {
|
||||
rowinstance = new byte[objectsize];
|
||||
for (int i = 0; i < objectsize; i++) this.rowinstance[i] = 0;
|
||||
|
|
|
@ -138,7 +138,7 @@ import de.anomic.kelondro.kelondroBase64Order;
|
|||
import de.anomic.kelondro.kelondroException;
|
||||
import de.anomic.kelondro.kelondroMSetTools;
|
||||
import de.anomic.kelondro.kelondroNaturalOrder;
|
||||
import de.anomic.kelondro.kelondroTables;
|
||||
import de.anomic.kelondro.kelondroMapTable;
|
||||
import de.anomic.plasma.dbImport.dbImportManager;
|
||||
import de.anomic.server.serverAbstractSwitch;
|
||||
import de.anomic.server.serverCodings;
|
||||
|
@ -201,7 +201,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
public plasmaRankingDistribution rankingOwnDistribution;
|
||||
public plasmaRankingDistribution rankingOtherDistribution;
|
||||
public HashMap outgoingCookies, incomingCookies;
|
||||
public kelondroTables facilityDB;
|
||||
public kelondroMapTable facilityDB;
|
||||
public plasmaParser parser;
|
||||
public long proxyLastAccess;
|
||||
public yacyCore yc;
|
||||
|
|
Loading…
Reference in New Issue
Block a user