Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

added a flex-width-array: this is a table where it is

Browse Source 
possible to add columns to an existing table

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2163 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2006-06-01 16:01:24 +00:00
parent 4a907a570f
commit c75cacda95
8 changed files with 200 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
source/de/anomic/index/indexRAMCacheRI.java
View File

@ -32,7 +32,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import de.anomic.kelondro.kelondroArray;
import de.anomic.kelondro.kelondroFixedWidthArray;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroNaturalOrder;
@ -93,8 +93,8 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
log.logConfig("creating dump for index cache, " + wCache.size() + " words (and much more urls)");
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroArray dumpArray = null;
dumpArray = new kelondroArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
kelondroFixedWidthArray dumpArray = null;
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
@ -173,7 +173,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
private long restore() throws IOException {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (!(indexDumpFile.exists())) return 0;
kelondroArray dumpArray = new kelondroArray(indexDumpFile);
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile);
log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations");
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;

10
source/de/anomic/kelondro/kelondroCollectionIndex.java
View File

@ -37,7 +37,7 @@ public class kelondroCollectionIndex {
private int chunksize;
private int partitions;
private int maxChunks;
private kelondroArray[] array;
private kelondroFixedWidthArray[] array;
private int[] arrayCapacity;
private static File arrayFile(File path, String filenameStub, int loadfactor, int chunksize, int partitionNumber) {
@ -75,7 +75,7 @@ public class kelondroCollectionIndex {
index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, columns, 1, 80, true);
// create array files
this.array = new kelondroArray[partitions];
this.array = new kelondroFixedWidthArray[partitions];
this.arrayCapacity = new int[partitions];
// open array files
@ -89,11 +89,11 @@ public class kelondroCollectionIndex {
this.maxChunks = load;
}
private kelondroArray openArrayFile(int genericChunkSize, int partitionNumber) throws IOException {
private kelondroFixedWidthArray openArrayFile(int genericChunkSize, int partitionNumber) throws IOException {
File f = arrayFile(path, filenameStub, loadfactor, genericChunkSize, partitionNumber);
if (f.exists()) {
return new kelondroArray(f);
return new kelondroFixedWidthArray(f);
} else {
int load = 1; for (int i = 0; i < partitionNumber; i++) load = load * loadfactor;
int[] columns = new int[4];
@ -103,7 +103,7 @@ public class kelondroCollectionIndex {
columns[3] = 2; // last time wrote
columns[4] = 2; // flag string, assigns collection order as currently stored in table
columns[5] = load * genericChunkSize;
return new kelondroArray(f, columns, 0, true);
return new kelondroFixedWidthArray(f, columns, 0, true);
}
}

20
source/de/anomic/kelondro/kelondroArray.java → source/de/anomic/kelondro/kelondroFixedWidthArray.java
View File

@ -49,13 +49,13 @@ package de.anomic.kelondro;
import java.io.File;
import java.io.IOException;
public class kelondroArray extends kelondroRecords {
public class kelondroFixedWidthArray extends kelondroRecords implements kelondroArray {
// define the Over-Head-Array
private static short thisOHBytes = 0; // our record definition does not need extra bytes
private static short thisOHHandles = 0; // and no handles
public kelondroArray(File file, int[] columns, int intprops, boolean exitOnFail) {
public kelondroFixedWidthArray(File file, int[] columns, int intprops, boolean exitOnFail) {
// this creates a new array
super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
for (int i = 0; i < intprops; i++) try {
@ -67,7 +67,7 @@ public class kelondroArray extends kelondroRecords {
}
}
public kelondroArray(File file) throws IOException{
public kelondroFixedWidthArray(File file) throws IOException{
// this opens a file with an existing array
super(file, 0);
}
@ -144,19 +144,19 @@ public class kelondroArray extends kelondroRecords {
// create <filename> <valuelen>
File f = new File(args[1]);
if (f.exists()) f.delete();
kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2, true);
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new int[]{Integer.parseInt(args[2])}, 2, true);
fm.close();
} else
if ((args.length == 2) && (args[0].equals("-v"))) {
// view <filename>
kelondroArray fm = new kelondroArray(new File(args[1]));
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
fm.print();
fm.print(true);
fm.close();
} else
if ((args.length == 3) && (args[0].equals("-g"))) {
// get <filename> <index>
kelondroArray fm = new kelondroArray(new File(args[1]));
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
kelondroRow.Entry row = fm.get(Integer.parseInt(args[2]));
for (int j = 0; j < fm.columns(); j++) System.out.print(row.getColString(j, null) + " ");
System.out.println();
@ -164,14 +164,14 @@ public class kelondroArray extends kelondroRecords {
} else
if ((args.length == 4) && (args[0].equals("-s"))) {
// set <filename> <index> <value>
kelondroArray fm = new kelondroArray(new File(args[1]));
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
kelondroRow.Entry row = fm.row().newEntry(new byte[][]{args[3].getBytes()});
fm.set(Integer.parseInt(args[2]), row);
fm.close();
} else
if ((args.length == 3) && (args[0].equals("-a"))) {
// add <filename> <value>
kelondroArray fm = new kelondroArray(new File(args[1]));
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
kelondroRow.Entry row = fm.row().newEntry(new byte[][] {args[2].getBytes()});
int index = fm.add(row);
System.out.println("Added to row " + index);
@ -179,14 +179,14 @@ public class kelondroArray extends kelondroRecords {
} else
if ((args.length == 3) && (args[0].equals("-d"))) {
// delete <filename> <index>
kelondroArray fm = new kelondroArray(new File(args[1]));
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
fm.remove(Integer.parseInt(args[2]));
fm.close();
} else
if ((args.length == 1) && (args[0].equals("-test"))) {
File testfile = new File("test.array");
if (testfile.exists()) testfile.delete();
kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9, true);
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new int[]{30, 50}, 9, true);
for (int i = 0; i < 100; i++) {
fm.set(i, fm.row().newEntry(new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()}));
}

157
source/de/anomic/kelondro/kelondroFlexWidthArray.java Normal file
View File

@ -0,0 +1,157 @@
// kelondroFlexWidthArray.java
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 01.06.2006 on http://www.anomic.de
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.File;
import java.io.IOException;
public class kelondroFlexWidthArray implements kelondroArray {
private kelondroFixedWidthArray[] col;
private kelondroRow rowdef;
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef, boolean exitOnFail) throws IOException {
this.rowdef = rowdef;
// initialize columns
col = new kelondroFixedWidthArray[rowdef.columns()];
String check = "";
for (int i = 0; i < rowdef.columns(); i++) {
col = null;
check += '_';
}
// open existing files
String[] files = path.list();
for (int i = 0; i < files.length; i++) {
if ((files[i].startsWith("col.") && (files[i].endsWith(".list")))) {
int colstart = Integer.parseInt(files[i].substring(4, 7));
int colend = (files[i].charAt(7) == '-') ? Integer.parseInt(files[i].substring(8, 11)) : colstart;
/*
int columns[] = new int[colend - colstart + 1];
for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.width(j);
col[colstart] = new kelondroFixedWidthArray(new File(path, files[i]), columns, 0, true);
*/
col[colstart] = new kelondroFixedWidthArray(new File(path, files[i]));
for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1);
}
}
// check if all columns are there
int p, q;
while ((p = check.indexOf('_')) >= 0) {
q = p;
if (p != 0) while ((check.charAt(q) == '_') && (q <= check.length() - 1)) q++;
// create new array file
int columns[] = new int[q - p + 1];
for (int j = p; j <= q; j++) {
columns[j - p] = rowdef.width(j);
check = check.substring(0, j) + "X" + check.substring(j + 1);
}
col[p] = new kelondroFixedWidthArray(new File(path, colfilename(p, q)), columns, 0, true);
}
}
private static final String colfilename(int start, int end) {
String f = Integer.toString(end);
while (f.length() < 3) f = "0" + f;
if (start == end) return "col." + f + ".list";
f = Integer.toString(start) + "-" + f;
while (f.length() < 7) f = "0" + f;
return "col." + f + ".list";
}
public int size() {
return col[0].size();
}
public int columns() {
return rowdef.columns();
}
public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
int r = 0;
kelondroRow.Entry e0, e1, p;
p = rowdef.newEntry();
while (r < rowdef.columns()) {
e0 = col[r].row().newEntry(rowentry.bytes(), rowdef.colstart[r], rowdef.colstart[r] - rowdef.colstart[r + col[r].columns() - 1] + rowdef.width(r));
e1 = col[r].set(index, e0);
for (int i = 0; i < col[r].columns(); i++)
p.setCol(r + i, e1.getColBytes(i));
r = r + col[r].columns();
}
return p;
}
public synchronized kelondroRow.Entry get(int index) throws IOException {
int r = 0;
kelondroRow.Entry e, p;
p = rowdef.newEntry();
while (r < rowdef.columns()) {
e = col[r].get(index);
for (int i = 0; i < col[r].columns(); i++)
p.setCol(r + i, e.getColBytes(i));
r = r + col[r].columns();
}
return p;
}
public synchronized int add(kelondroRow.Entry rowentry) throws IOException {
kelondroRow.Entry e;
e = col[0].row().newEntry(rowentry.bytes(), rowdef.colstart[0], rowdef.colstart[0] - rowdef.colstart[col[0].columns() - 1] + rowdef.width(0));
int index = col[0].add(e);
int r = col[0].columns();
while (r < rowdef.columns()) {
e = col[r].row().newEntry(rowentry.bytes(), rowdef.colstart[r], rowdef.colstart[r] - rowdef.colstart[r + col[r].columns() - 1] + rowdef.width(r));
col[r].set(index, e);
r = r + col[r].columns();
}
return index;
}
public synchronized void remove(int index) throws IOException {
int r = 0;
while (r < rowdef.columns()) {
col[r].remove(index);
r = r + col[r].columns();
}
}
public void print() throws IOException {
System.out.println("PRINTOUT of table, length=" + size());
kelondroRow.Entry row;
for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": ");
row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
System.out.println();
}
System.out.println("EndOfTable");
}
}

6
source/de/anomic/kelondro/kelondroHashtable.java
View File

@ -134,7 +134,7 @@ import java.io.IOException;
public class kelondroHashtable {
private kelondroArray hashArray;
private kelondroFixedWidthArray hashArray;
private int offset;
private int maxk;
private int maxrehash;
@ -152,7 +152,7 @@ public class kelondroHashtable {
// this number is needed to omit grow of the table in case of re-hashing
// the maxsize is re-computed to a virtual folding height and will result in a tablesize
// less than the given maxsize. The actual maxsize can be retrieved by maxsize()
this.hashArray = new kelondroArray(file, extCol(columns), 6, exitOnFail);
this.hashArray = new kelondroFixedWidthArray(file, extCol(columns), 6, exitOnFail);
this.offset = offset;
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
@ -173,7 +173,7 @@ public class kelondroHashtable {
public kelondroHashtable(File file) throws IOException{
// this opens a file with an existing hashtable
this.hashArray = new kelondroArray(file);
this.hashArray = new kelondroFixedWidthArray(file);
this.offset = hashArray.geti(0);
this.maxk = hashArray.geti(1);
this.maxrehash = hashArray.geti(2);

6
source/de/anomic/kelondro/kelondroTables.java → source/de/anomic/kelondro/kelondroMapTable.java
View File

@ -1,4 +1,4 @@
// kelondroTables.java
// kelondroapTable.java
// -----------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
@ -48,12 +48,12 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class kelondroTables {
public class kelondroMapTable {
HashMap mTables, tTables, sTables;
File tablesPath;
public kelondroTables(File tablesPath) {
public kelondroMapTable(File tablesPath) {
this.mTables = new HashMap();
this.tTables = new HashMap();
this.sTables = new HashMap();

21
source/de/anomic/kelondro/kelondroRow.java
View File

@ -38,11 +38,11 @@ public class kelondroRow {
public static final int encoder_char = 3;
private kelondroColumn[] row;
private int[] colstart;
private HashMap encodedFormConfiguration;
private int encodedFormLength;
private int objectsize;
private kelondroColumn[] row;
protected int[] colstart;
private HashMap encodedFormConfiguration;
private int encodedFormLength;
private int objectsize;
public kelondroRow(kelondroColumn[] row) {
this.row = row;
@ -117,6 +117,11 @@ public class kelondroRow {
return new Entry(rowinstance);
}
public Entry newEntry(byte[] rowinstance, int start, int length) {
if (rowinstance == null) return null;
return new Entry(rowinstance);
}
public Entry newEntry(byte[][] cells) {
if (cells == null) return null;
return new Entry(cells);
@ -141,6 +146,12 @@ public class kelondroRow {
}
}
public Entry(byte[] rowinstance, int start, int length) {
this.rowinstance = new byte[length];
System.arraycopy(rowinstance, start, this.rowinstance, 0, length);
for (int i = rowinstance.length; i < objectsize; i++) this.rowinstance[i] = 0;
}
public Entry(byte[][] cols) {
rowinstance = new byte[objectsize];
for (int i = 0; i < objectsize; i++) this.rowinstance[i] = 0;

4
source/de/anomic/plasma/plasmaSwitchboard.java
View File

@ -138,7 +138,7 @@ import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroTables;
import de.anomic.kelondro.kelondroMapTable;
import de.anomic.plasma.dbImport.dbImportManager;
import de.anomic.server.serverAbstractSwitch;
import de.anomic.server.serverCodings;
@ -201,7 +201,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public plasmaRankingDistribution rankingOwnDistribution;
public plasmaRankingDistribution rankingOtherDistribution;
public HashMap outgoingCookies, incomingCookies;
public kelondroTables facilityDB;
public kelondroMapTable facilityDB;
public plasmaParser parser;
public long proxyLastAccess;
public yacyCore yc;