2009-07-29 21:43:11 +02:00
// Table.java
2008-01-17 13:12:52 +01:00
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 14.01.2008 on http://yacy.net
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2009-01-30 23:08:08 +01:00
package de.anomic.kelondro.table ;
2008-01-17 13:12:52 +01:00
import java.io.File ;
import java.io.FileNotFoundException ;
import java.io.FileOutputStream ;
import java.io.IOException ;
2008-01-20 02:22:46 +01:00
import java.util.ArrayList ;
2008-10-03 20:57:02 +02:00
import java.util.ConcurrentModificationException ;
2008-01-17 13:12:52 +01:00
import java.util.Date ;
2008-01-17 22:48:08 +01:00
import java.util.HashMap ;
2008-01-17 13:12:52 +01:00
import java.util.Iterator ;
import java.util.List ;
2008-01-17 22:48:08 +01:00
import java.util.Map ;
2008-01-17 13:12:52 +01:00
import java.util.TreeMap ;
2008-01-20 02:22:46 +01:00
import java.util.TreeSet ;
2008-01-17 13:12:52 +01:00
2009-01-30 16:33:00 +01:00
import de.anomic.kelondro.index.Column ;
2009-06-07 23:48:01 +02:00
import de.anomic.kelondro.index.HandleMap ;
2009-01-30 16:33:00 +01:00
import de.anomic.kelondro.index.Row ;
import de.anomic.kelondro.index.RowCollection ;
import de.anomic.kelondro.index.RowSet ;
import de.anomic.kelondro.index.ObjectIndex ;
import de.anomic.kelondro.index.Row.Entry ;
2009-09-09 22:49:16 +02:00
import de.anomic.kelondro.io.records.BufferedRecords ;
import de.anomic.kelondro.io.records.Records ;
2009-01-30 23:08:08 +01:00
import de.anomic.kelondro.order.CloneableIterator ;
import de.anomic.kelondro.order.NaturalOrder ;
2009-03-30 17:31:25 +02:00
import de.anomic.kelondro.util.FileUtils ;
2009-01-30 23:44:20 +01:00
import de.anomic.kelondro.util.MemoryControl ;
import de.anomic.kelondro.util.kelondroException ;
2009-06-15 23:19:54 +02:00
import de.anomic.yacy.logging.Log ;
2008-01-17 13:12:52 +01:00
/ *
2008-04-06 20:47:49 +02:00
* The EcoIndex builds upon the EcoFS and tries to reduce the number of IO requests that the
2008-01-17 13:12:52 +01:00
* EcoFS must do to a minimum . In best cases , no IO has to be done for read operations ( complete database shadow in RAM )
* and a rare number of write IO operations must be done for a large number of table - writings ( using the write buffer of EcoFS )
* To make the EcoIndex scalable in question of available RAM , there are two elements that must be scalable :
* - the access index can be either completely in RAM ( kelondroRAMIndex ) or it is file - based ( kelondroTree )
* - the content cache can be either a complete RAM - based shadow of the File , or empty .
* The content cache can also be deleted during run - time , if the available RAM gets too low .
* /
2009-06-28 23:37:37 +02:00
public class Table implements ObjectIndex {
2008-01-17 13:12:52 +01:00
// static tracker objects
2009-06-28 23:37:37 +02:00
private static TreeMap < String , Table > tableTracker = new TreeMap < String , Table > ( ) ;
2008-01-17 13:12:52 +01:00
2009-06-30 15:25:46 +02:00
public static final long maxarraylength = 134217727L ; // that may be the maxmimum size of array length in some JVMs
private static final long minmemremaining = 20 * 1024 * 1024 ; // if less than this memory is remaininig, the memory copy of a table is abandoned
2009-09-17 22:41:50 +02:00
//private int fail;
2009-06-30 15:25:46 +02:00
private final int buffersize ;
protected HandleMap index ;
2009-09-09 22:49:16 +02:00
protected BufferedRecords file ;
2009-06-30 15:25:46 +02:00
protected Row rowdef ;
protected File tablefile ;
protected RowSet table ;
protected Row taildef ;
2008-01-17 13:12:52 +01:00
2009-09-07 22:30:57 +02:00
public Table (
final File tablefile ,
final Row rowdef ,
final int buffersize ,
final int initialSpace ,
final boolean useTailCache ,
final boolean exceed134217727 ) {
2009-02-24 11:40:20 +01:00
this . tablefile = tablefile ;
2008-01-17 13:12:52 +01:00
this . rowdef = rowdef ;
this . buffersize = buffersize ;
2008-10-03 20:57:02 +02:00
//this.fail = 0;
2008-01-17 13:12:52 +01:00
// define the taildef, a row like the rowdef but without the first column
2009-01-30 16:33:00 +01:00
final Column [ ] cols = new Column [ rowdef . columns ( ) - 1 ] ;
2008-01-17 13:12:52 +01:00
for ( int i = 0 ; i < cols . length ; i + + ) {
cols [ i ] = rowdef . column ( i + 1 ) ;
}
2009-03-13 17:52:31 +01:00
this . taildef = new Row ( cols , NaturalOrder . naturalOrder ) ;
2008-01-17 13:12:52 +01:00
// initialize table file
2009-02-16 17:13:48 +01:00
boolean freshFile = false ;
2008-01-17 13:12:52 +01:00
if ( ! tablefile . exists ( ) ) {
// make new file
2009-02-16 17:13:48 +01:00
freshFile = true ;
2008-01-17 13:12:52 +01:00
FileOutputStream fos = null ;
try {
fos = new FileOutputStream ( tablefile ) ;
2008-08-02 14:12:04 +02:00
} catch ( final FileNotFoundException e ) {
2008-01-17 13:12:52 +01:00
// should not happen
e . printStackTrace ( ) ;
}
2008-08-02 14:12:04 +02:00
if ( fos ! = null ) try { fos . close ( ) ; } catch ( final IOException e ) { }
2008-01-17 13:12:52 +01:00
}
try {
2008-01-17 22:48:08 +01:00
// open an existing table file
2008-08-02 14:12:04 +02:00
final int fileSize = ( int ) tableSize ( tablefile , rowdef . objectsize ) ;
2008-04-06 13:50:15 +02:00
2008-01-17 22:48:08 +01:00
// initialize index and copy table
2008-08-02 14:12:04 +02:00
final int records = Math . max ( fileSize , initialSpace ) ;
final long neededRAM4table = ( records ) * ( ( rowdef . objectsize ) + 4L ) * 3L ;
2009-09-07 22:30:57 +02:00
table = ( ( exceed134217727 | | neededRAM4table < maxarraylength ) & &
( useTailCache & & MemoryControl . available ( ) > neededRAM4table + 200 * 1024 * 1024 ) ) ?
2009-01-30 16:33:00 +01:00
new RowSet ( taildef , records ) : null ;
2009-09-07 21:22:12 +02:00
Log . logInfo ( " TABLE " , " initialization of " + tablefile . getName ( ) + " . table copy: " + ( ( table = = null ) ? " no " : " yes " ) + " , available RAM: " + ( MemoryControl . available ( ) / 1024 / 1024 ) + " MB, needed: " + ( neededRAM4table / 1024 / 1024 + 200 ) + " MB, allocating space for " + records + " entries " ) ;
2008-08-02 14:12:04 +02:00
final long neededRAM4index = 2 * 1024 * 1024 + records * ( rowdef . primaryKeyLength + 4 ) * 3 / 2 ;
2009-01-30 16:33:00 +01:00
if ( ! MemoryControl . request ( neededRAM4index , false ) ) {
2008-01-22 21:44:12 +01:00
// despite calculations seemed to show that there is enough memory for the table AND the index
// there is now not enough memory left for the index. So delete the table again to free the memory
// for the index
2009-09-07 21:22:12 +02:00
Log . logSevere ( " TABLE " , tablefile . getName ( ) + " : not enough RAM ( " + ( MemoryControl . available ( ) / 1024 / 1024 ) + " MB) left for index, deleting allocated table space to enable index space allocation (needed: " + ( neededRAM4index / 1024 / 1024 ) + " MB) " ) ;
2008-01-22 21:44:12 +01:00
table = null ; System . gc ( ) ;
2009-09-07 21:22:12 +02:00
Log . logSevere ( " TABLE " , tablefile . getName ( ) + " : RAM after releasing the table: " + ( MemoryControl . available ( ) / 1024 / 1024 ) + " MB " ) ;
2008-01-22 21:44:12 +01:00
}
2009-06-07 23:48:01 +02:00
index = new HandleMap ( rowdef . primaryKeyLength , rowdef . objectOrder , 4 , records , 100000 ) ;
2009-07-29 21:43:11 +02:00
HandleMap errors = new HandleMap ( rowdef . primaryKeyLength , NaturalOrder . naturalOrder , 4 , records , 10 ) ;
2009-06-28 23:37:37 +02:00
Log . logInfo ( " TABLE " , tablefile + " : TABLE " + tablefile . toString ( ) + " has table copy " + ( ( table = = null ) ? " DISABLED " : " ENABLED " ) ) ;
2008-01-20 22:42:35 +01:00
2008-01-17 22:48:08 +01:00
// read all elements from the file into the copy table
2009-06-28 23:37:37 +02:00
Log . logInfo ( " TABLE " , " initializing RAM index for TABLE " + tablefile . getName ( ) + " , please wait. " ) ;
2008-04-06 13:50:15 +02:00
int i = 0 ;
byte [ ] key ;
if ( table = = null ) {
2009-01-30 23:08:08 +01:00
final Iterator < byte [ ] > ki = new ChunkIterator ( tablefile , rowdef . objectsize , rowdef . primaryKeyLength ) ;
2008-04-06 13:50:15 +02:00
while ( ki . hasNext ( ) ) {
key = ki . next ( ) ;
// write the key into the index table
assert key ! = null ;
if ( key = = null ) { i + + ; continue ; }
2009-07-29 21:43:11 +02:00
if ( rowdef . objectOrder . wellformed ( key ) ) {
index . putUnique ( key , i + + ) ;
} else {
errors . putUnique ( key , i + + ) ;
}
2008-04-06 13:50:15 +02:00
}
} else {
byte [ ] record ;
key = new byte [ rowdef . primaryKeyLength ] ;
2009-01-30 23:08:08 +01:00
final Iterator < byte [ ] > ri = new ChunkIterator ( tablefile , rowdef . objectsize , rowdef . objectsize ) ;
2008-04-06 13:50:15 +02:00
while ( ri . hasNext ( ) ) {
record = ri . next ( ) ;
assert record ! = null ;
if ( record = = null ) { i + + ; continue ; }
System . arraycopy ( record , 0 , key , 0 , rowdef . primaryKeyLength ) ;
// write the key into the index table
2009-07-29 21:43:11 +02:00
if ( rowdef . objectOrder . wellformed ( key ) ) {
index . putUnique ( key , i + + ) ;
// write the tail into the table
table . addUnique ( taildef . newEntry ( record , rowdef . primaryKeyLength , true ) ) ;
if ( abandonTable ( ) ) {
table = null ;
break ;
}
} else {
errors . putUnique ( key , i + + ) ;
2008-04-06 13:50:15 +02:00
}
2008-01-24 23:49:00 +01:00
}
2008-01-17 22:48:08 +01:00
}
2008-04-06 13:50:15 +02:00
2009-02-16 17:13:48 +01:00
// open the file
2009-09-09 22:49:16 +02:00
this . file = new BufferedRecords ( new Records ( tablefile , rowdef . objectsize ) , this . buffersize ) ;
2009-02-16 17:13:48 +01:00
2009-07-29 21:43:11 +02:00
// clean up the file by cleaning badly formed entries
int errorc = errors . size ( ) ;
int errorcc = 0 ;
int idx ;
for ( Entry entry : errors ) {
key = entry . getPrimaryKeyBytes ( ) ;
idx = ( int ) entry . getColLong ( 1 ) ;
Log . logWarning ( " Table " , " removing not well-formed entry " + idx + " with key: " + NaturalOrder . arrayList ( key , 0 , key . length ) + " , " + errorcc + + + " / " + errorc ) ;
removeInFile ( idx ) ;
}
assert index . size ( ) = = this . file . size ( ) : " index.size() = " + index . size ( ) + " , this.file.size() = " + this . file . size ( ) ;
2009-02-16 17:13:48 +01:00
// remove doubles
if ( ! freshFile ) {
2009-06-07 23:48:01 +02:00
final ArrayList < Long [ ] > doubles = index . removeDoubles ( ) ;
2009-02-16 17:13:48 +01:00
//assert index.size() + doubles.size() + fail == i;
//System.out.println(" -removed " + doubles.size() + " doubles- done.");
if ( doubles . size ( ) > 0 ) {
2009-06-28 23:37:37 +02:00
Log . logInfo ( " TABLE " , tablefile + " : WARNING - TABLE " + tablefile + " has " + doubles . size ( ) + " doubles " ) ;
2009-02-16 17:13:48 +01:00
// from all the doubles take one, put it back to the index and remove the others from the file
// first put back one element each
final byte [ ] record = new byte [ rowdef . objectsize ] ;
key = new byte [ rowdef . primaryKeyLength ] ;
2009-06-07 23:48:01 +02:00
for ( final Long [ ] ds : doubles ) {
2009-02-16 17:13:48 +01:00
file . get ( ds [ 0 ] . intValue ( ) , record , 0 ) ;
System . arraycopy ( record , 0 , key , 0 , rowdef . primaryKeyLength ) ;
2009-03-08 22:37:17 +01:00
index . putUnique ( key , ds [ 0 ] . intValue ( ) ) ;
2009-02-16 17:13:48 +01:00
}
// then remove the other doubles by removing them from the table, but do a re-indexing while doing that
// first aggregate all the delete positions because the elements from the top positions must be removed first
2009-06-07 23:48:01 +02:00
final TreeSet < Long > delpos = new TreeSet < Long > ( ) ;
for ( final Long [ ] ds : doubles ) {
2009-02-16 17:13:48 +01:00
for ( int j = 1 ; j < ds . length ; j + + ) delpos . add ( ds [ j ] ) ;
}
// now remove the entries in a sorted way (top-down)
2009-06-07 23:48:01 +02:00
Long top ;
2009-02-16 17:13:48 +01:00
while ( delpos . size ( ) > 0 ) {
top = delpos . last ( ) ;
delpos . remove ( top ) ;
removeInFile ( top . intValue ( ) ) ;
}
2008-01-23 12:13:39 +01:00
}
2008-01-20 22:42:35 +01:00
}
2008-08-02 14:12:04 +02:00
} catch ( final FileNotFoundException e ) {
2008-01-17 22:48:08 +01:00
// should never happen
e . printStackTrace ( ) ;
throw new kelondroException ( e . getMessage ( ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-01-17 22:48:08 +01:00
e . printStackTrace ( ) ;
throw new kelondroException ( e . getMessage ( ) ) ;
2008-01-17 13:12:52 +01:00
}
// track this table
tableTracker . put ( tablefile . toString ( ) , this ) ;
}
2008-11-03 00:53:09 +01:00
private boolean abandonTable ( ) {
// check if not enough memory is there to maintain a memory copy of the table
2009-01-30 16:33:00 +01:00
return MemoryControl . available ( ) < minmemremaining ;
2008-11-03 00:53:09 +01:00
}
2008-08-02 14:12:04 +02:00
public static long tableSize ( final File tablefile , final int recordsize ) {
2008-01-17 13:12:52 +01:00
// returns number of records in table
2009-09-09 22:49:16 +02:00
return Records . tableSize ( tablefile , recordsize ) ;
2008-01-17 13:12:52 +01:00
}
2008-01-17 22:48:08 +01:00
public static final Iterator < String > filenames ( ) {
// iterates string objects; all file names from record tracker
return tableTracker . keySet ( ) . iterator ( ) ;
}
2008-08-02 14:12:04 +02:00
public static final Map < String , String > memoryStats ( final String filename ) {
2008-01-17 22:48:08 +01:00
// returns a map for each file in the tracker;
// the map represents properties for each record objects,
// i.e. for cache memory allocation
2009-06-28 23:37:37 +02:00
final Table theTABLE = tableTracker . get ( filename ) ;
return theTABLE . memoryStats ( ) ;
2008-01-17 22:48:08 +01:00
}
private final Map < String , String > memoryStats ( ) {
// returns statistical data about this object
2009-09-17 22:41:50 +02:00
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-08-02 14:12:04 +02:00
final HashMap < String , String > map = new HashMap < String , String > ( ) ;
2008-01-20 02:51:02 +01:00
map . put ( " tableSize " , Integer . toString ( index . size ( ) ) ) ;
map . put ( " tableKeyChunkSize " , Integer . toString ( index . row ( ) . objectsize ) ) ;
2009-08-28 15:28:11 +02:00
map . put ( " tableKeyMem " , Integer . toString ( ( int ) ( ( ( long ) index . row ( ) . objectsize ) * ( ( long ) index . size ( ) ) * RowCollection . growfactor100 / 100L ) ) ) ;
2008-01-20 02:51:02 +01:00
map . put ( " tableValueChunkSize " , ( table = = null ) ? " 0 " : Integer . toString ( table . row ( ) . objectsize ) ) ;
2009-09-07 22:30:57 +02:00
map . put ( " tableValueMem " , ( table = = null ) ? " 0 " : Integer . toString ( ( int ) ( ( ( long ) table . row ( ) . objectsize ) * ( ( long ) table . size ( ) ) * RowCollection . growfactor100 / 100L ) ) ) ;
2008-01-17 22:48:08 +01:00
return map ;
}
2008-06-18 23:08:56 +02:00
public boolean usesFullCopy ( ) {
return this . table ! = null ;
}
2009-01-30 16:33:00 +01:00
public static int staticRAMIndexNeed ( final File f , final Row rowdef ) {
2009-09-02 15:41:56 +02:00
return ( int ) ( ( ( long ) ( rowdef . primaryKeyLength + 4 ) ) * tableSize ( f , rowdef . objectsize ) * RowCollection . growfactor100 / 100L ) ;
2008-01-17 22:48:08 +01:00
}
2008-01-17 13:12:52 +01:00
2008-11-24 00:55:08 +01:00
public synchronized void addUnique ( final Entry row ) throws IOException {
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-08-02 14:12:04 +02:00
final int i = ( int ) file . size ( ) ;
2009-03-08 22:37:17 +01:00
index . putUnique ( row . getPrimaryKeyBytes ( ) , i ) ;
2008-01-17 13:12:52 +01:00
if ( table ! = null ) {
assert table . size ( ) = = i ;
table . addUnique ( taildef . newEntry ( row . bytes ( ) , rowdef . primaryKeyLength , true ) ) ;
2008-11-03 00:53:09 +01:00
if ( abandonTable ( ) ) table = null ;
2008-01-17 13:12:52 +01:00
}
2008-04-10 15:24:55 +02:00
file . add ( row . bytes ( ) , 0 ) ;
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2009-03-11 21:23:19 +01:00
public synchronized void addUnique ( final List < Entry > rows ) throws IOException {
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-08-02 14:12:04 +02:00
final Iterator < Entry > i = rows . iterator ( ) ;
2008-01-17 13:12:52 +01:00
while ( i . hasNext ( ) ) {
2008-11-24 00:55:08 +01:00
addUnique ( i . next ( ) ) ;
2008-01-17 13:12:52 +01:00
}
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2008-06-18 23:08:56 +02:00
/ * *
* remove double - entries from the table
* this process calls the underlying removeDoubles ( ) method from the table index
* and
* /
2009-01-30 16:33:00 +01:00
public synchronized ArrayList < RowCollection > removeDoubles ( ) throws IOException {
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2009-01-30 16:33:00 +01:00
final ArrayList < RowCollection > report = new ArrayList < RowCollection > ( ) ;
RowSet rows ;
2009-06-07 23:48:01 +02:00
final TreeSet < Long > d = new TreeSet < Long > ( ) ;
2008-08-02 14:12:04 +02:00
final byte [ ] b = new byte [ rowdef . objectsize ] ;
2009-06-07 23:48:01 +02:00
Long L ;
2009-01-30 16:33:00 +01:00
Row . Entry inconsistentEntry ;
2008-06-18 23:08:56 +02:00
// iterate over all entries that have inconsistent index references
2009-02-16 17:13:48 +01:00
long lastlog = System . currentTimeMillis ( ) ;
2009-06-07 23:48:01 +02:00
for ( final Long [ ] is : index . removeDoubles ( ) ) {
2008-06-18 23:08:56 +02:00
// 'is' is the set of all indexes, that have the same reference
// we collect that entries now here
2009-01-30 16:33:00 +01:00
rows = new RowSet ( this . rowdef , is . length ) ;
2008-01-20 02:22:46 +01:00
for ( int j = 0 ; j < is . length ; j + + ) {
2008-06-18 23:08:56 +02:00
L = is [ j ] ;
assert L . intValue ( ) < file . size ( ) : " L.intValue() = " + L . intValue ( ) + " , file.size = " + file . size ( ) ; // prevent ooBounds Exception
d . add ( L ) ;
2008-12-18 23:05:26 +01:00
if ( L . intValue ( ) > = file . size ( ) ) continue ; // prevent IndexOutOfBoundsException
2008-06-18 23:08:56 +02:00
file . get ( L . intValue ( ) , b , 0 ) ; // TODO: fix IndexOutOfBoundsException here
inconsistentEntry = rowdef . newEntry ( b ) ;
rows . addUnique ( inconsistentEntry ) ;
2008-01-20 02:22:46 +01:00
}
report . add ( rows ) ;
}
2008-05-02 00:40:42 +02:00
// finally delete the affected rows, but start with largest id first, otherwise we overwrite wrong entries
2009-06-07 23:48:01 +02:00
Long s ;
2008-01-20 02:22:46 +01:00
while ( d . size ( ) > 0 ) {
s = d . last ( ) ;
d . remove ( s ) ;
this . removeInFile ( s . intValue ( ) ) ;
2009-02-16 17:13:48 +01:00
if ( System . currentTimeMillis ( ) - lastlog > 30000 ) {
2009-06-28 23:37:37 +02:00
Log . logInfo ( " TABLE " , " removing " + d . size ( ) + " entries in " + this . filename ( ) ) ;
2009-02-16 17:21:12 +01:00
lastlog = System . currentTimeMillis ( ) ;
2009-02-16 17:13:48 +01:00
}
2008-01-20 02:22:46 +01:00
}
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-20 02:22:46 +01:00
return report ;
}
2008-01-17 13:12:52 +01:00
public void close ( ) {
2008-12-29 13:22:13 +01:00
this . file . close ( ) ;
this . file = null ;
this . table = null ;
this . index = null ;
2008-01-17 13:12:52 +01:00
}
2008-08-06 21:43:12 +02:00
protected void finalize ( ) {
2008-01-17 13:12:52 +01:00
if ( this . file ! = null ) this . close ( ) ;
}
public String filename ( ) {
return this . file . filename ( ) . toString ( ) ;
}
2008-01-20 02:22:46 +01:00
2008-08-02 14:12:04 +02:00
public synchronized Entry get ( final byte [ ] key ) throws IOException {
2008-05-14 23:36:02 +02:00
if ( ( file = = null ) | | ( index = = null ) ) return null ;
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2009-06-07 23:48:01 +02:00
final int i = ( int ) index . get ( key ) ;
2008-01-17 13:12:52 +01:00
if ( i = = - 1 ) return null ;
2008-08-02 14:12:04 +02:00
final byte [ ] b = new byte [ rowdef . objectsize ] ;
2008-01-17 13:12:52 +01:00
if ( table = = null ) {
// read row from the file
file . get ( i , b , 0 ) ;
} else {
// construct the row using the copy in RAM
2009-01-30 16:33:00 +01:00
final Row . Entry v = table . get ( i , false ) ;
2008-01-17 13:12:52 +01:00
assert v ! = null ;
2008-04-14 07:33:32 +02:00
if ( v = = null ) return null ;
2008-01-17 13:12:52 +01:00
assert key . length = = rowdef . primaryKeyLength ;
System . arraycopy ( key , 0 , b , 0 , key . length ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , rowdef . objectsize - rowdef . primaryKeyLength ) ;
}
return rowdef . newEntry ( b ) ;
}
2008-08-02 14:12:04 +02:00
public synchronized boolean has ( final byte [ ] key ) {
2008-06-18 01:56:39 +02:00
try {
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-06-18 01:56:39 +02:00
// TODO Auto-generated catch block
e . printStackTrace ( ) ;
}
2009-09-17 22:41:50 +02:00
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-06-18 01:56:39 +02:00
return index . has ( key ) ;
2008-01-17 13:12:52 +01:00
}
2009-01-30 23:08:08 +01:00
public synchronized CloneableIterator < byte [ ] > keys ( final boolean up , final byte [ ] firstKey ) throws IOException {
2008-01-17 13:12:52 +01:00
return index . keys ( up , firstKey ) ;
}
2009-03-11 21:23:19 +01:00
public synchronized Entry replace ( final Entry row ) throws IOException {
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-25 12:44:27 +01:00
assert row ! = null ;
assert row . bytes ( ) ! = null ;
if ( ( row = = null ) | | ( row . bytes ( ) = = null ) ) return null ;
2009-06-07 23:48:01 +02:00
final int i = ( int ) index . get ( row . getPrimaryKeyBytes ( ) ) ;
2008-01-17 13:12:52 +01:00
if ( i = = - 1 ) {
addUnique ( row ) ;
return null ;
}
2008-08-02 14:12:04 +02:00
final byte [ ] b = new byte [ rowdef . objectsize ] ;
2008-01-17 13:12:52 +01:00
if ( table = = null ) {
// read old value
file . get ( i , b , 0 ) ;
// write new value
file . put ( i , row . bytes ( ) , 0 ) ;
} else {
// read old value
2009-01-30 16:33:00 +01:00
final Row . Entry v = table . get ( i , false ) ;
2008-01-30 22:58:30 +01:00
assert v ! = null ;
2008-01-17 13:12:52 +01:00
System . arraycopy ( row . getPrimaryKeyBytes ( ) , 0 , b , 0 , rowdef . primaryKeyLength ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , rowdef . objectsize - rowdef . primaryKeyLength ) ;
// write new value
table . set ( i , taildef . newEntry ( row . bytes ( ) , rowdef . primaryKeyLength , true ) ) ;
file . put ( i , row . bytes ( ) , 0 ) ;
}
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
// return old value
return rowdef . newEntry ( b ) ;
}
2009-03-11 21:23:19 +01:00
public synchronized void put ( final Entry row ) throws IOException {
2009-09-17 22:41:50 +02:00
assert file = = null | | file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2009-03-11 21:23:19 +01:00
assert row ! = null ;
assert row . bytes ( ) ! = null ;
2009-03-31 09:51:32 +02:00
if ( file = = null | | row = = null | | row . bytes ( ) = = null ) return ;
2009-06-07 23:48:01 +02:00
final int i = ( int ) index . get ( row . getPrimaryKeyBytes ( ) ) ;
2009-03-11 21:23:19 +01:00
if ( i = = - 1 ) {
addUnique ( row ) ;
return ;
}
if ( table = = null ) {
// write new value
file . put ( i , row . bytes ( ) , 0 ) ;
} else {
// write new value
table . set ( i , taildef . newEntry ( row . bytes ( ) , rowdef . primaryKeyLength , true ) ) ;
file . put ( i , row . bytes ( ) , 0 ) ;
}
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2009-03-11 21:23:19 +01:00
}
2008-01-17 13:12:52 +01:00
2008-08-02 14:12:04 +02:00
public synchronized Entry put ( final Entry row , final Date entryDate ) throws IOException {
2009-03-11 21:23:19 +01:00
return replace ( row ) ;
2008-01-17 13:12:52 +01:00
}
2009-07-29 21:43:11 +02:00
/ * *
* remove one entry from the file
* @param i an index position within the file ( not a byte position )
* @throws IOException
* /
2008-08-02 14:12:04 +02:00
private void removeInFile ( final int i ) throws IOException {
2008-01-20 02:22:46 +01:00
assert i > = 0 ;
2008-08-02 14:12:04 +02:00
final byte [ ] p = new byte [ rowdef . objectsize ] ;
2008-01-20 02:22:46 +01:00
if ( table = = null ) {
2008-01-22 22:40:25 +01:00
if ( i = = index . size ( ) - 1 ) {
2008-02-02 22:36:19 +01:00
file . cleanLast ( ) ;
2008-01-22 22:40:25 +01:00
} else {
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
2008-08-02 14:12:04 +02:00
final byte [ ] k = new byte [ rowdef . primaryKeyLength ] ;
2008-01-22 22:40:25 +01:00
System . arraycopy ( p , 0 , k , 0 , rowdef . primaryKeyLength ) ;
2009-03-08 22:37:17 +01:00
index . put ( k , i ) ;
2008-01-22 22:40:25 +01:00
}
2008-01-20 02:22:46 +01:00
} else {
if ( i = = index . size ( ) - 1 ) {
// special handling if the entry is the last entry in the file
table . removeRow ( i , false ) ;
2008-02-02 22:36:19 +01:00
file . cleanLast ( ) ;
2008-01-20 02:22:46 +01:00
} else {
// switch values
2009-01-30 16:33:00 +01:00
final Row . Entry te = table . removeOne ( ) ;
2008-01-20 02:22:46 +01:00
table . set ( i , te ) ;
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
2009-01-30 16:33:00 +01:00
final Row . Entry lr = rowdef . newEntry ( p ) ;
2009-03-08 22:37:17 +01:00
index . put ( lr . getPrimaryKeyBytes ( ) , i ) ;
2008-01-20 02:22:46 +01:00
}
}
}
2008-08-02 14:12:04 +02:00
public synchronized Entry remove ( final byte [ ] key ) throws IOException {
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-24 23:49:00 +01:00
assert key . length = = rowdef . primaryKeyLength ;
2009-06-07 23:48:01 +02:00
final int i = ( int ) index . get ( key ) ;
2008-01-17 13:12:52 +01:00
if ( i = = - 1 ) return null ; // nothing to do
// prepare result
2008-08-02 14:12:04 +02:00
final byte [ ] b = new byte [ rowdef . objectsize ] ;
final byte [ ] p = new byte [ rowdef . objectsize ] ;
final int sb = index . size ( ) ;
2008-02-02 22:36:19 +01:00
int ix ;
assert i < index . size ( ) ;
2008-01-17 13:12:52 +01:00
if ( table = = null ) {
2008-01-24 23:49:00 +01:00
if ( i = = index . size ( ) - 1 ) {
2009-06-16 08:38:26 +02:00
// element is at last entry position
2009-06-07 23:48:01 +02:00
ix = ( int ) index . remove ( key ) ;
2008-02-02 22:36:19 +01:00
assert ix = = i ;
file . cleanLast ( b , 0 ) ;
2008-01-24 23:49:00 +01:00
} else {
2009-06-16 08:38:26 +02:00
// remove entry from index
2008-02-02 22:36:19 +01:00
assert i < index . size ( ) - 1 ;
2009-06-07 23:48:01 +02:00
ix = ( int ) index . remove ( key ) ;
2008-02-02 22:36:19 +01:00
assert ix = = i ;
2009-06-16 08:38:26 +02:00
// read element that shall be removed
2008-01-24 23:49:00 +01:00
file . get ( i , b , 0 ) ;
2009-06-16 08:38:26 +02:00
// fill the gap with value from last entry in file
2008-01-24 23:49:00 +01:00
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
2008-08-02 14:12:04 +02:00
final byte [ ] k = new byte [ rowdef . primaryKeyLength ] ;
2008-01-24 23:49:00 +01:00
System . arraycopy ( p , 0 , k , 0 , rowdef . primaryKeyLength ) ;
2009-03-08 22:37:17 +01:00
index . put ( k , i ) ;
2008-01-24 23:49:00 +01:00
}
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
} else {
2008-01-24 23:49:00 +01:00
// get result value from the table copy, so we don't need to read it from the file
2009-01-30 16:33:00 +01:00
final Row . Entry v = table . get ( i , false ) ;
2008-01-17 13:12:52 +01:00
System . arraycopy ( key , 0 , b , 0 , key . length ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , taildef . objectsize ) ;
2008-01-24 23:49:00 +01:00
2008-01-17 13:12:52 +01:00
if ( i = = index . size ( ) - 1 ) {
// special handling if the entry is the last entry in the file
2009-06-07 23:48:01 +02:00
ix = ( int ) index . remove ( key ) ;
2008-02-03 03:23:04 +01:00
assert ix = = i ;
2008-01-17 13:12:52 +01:00
table . removeRow ( i , false ) ;
2008-02-02 22:36:19 +01:00
file . cleanLast ( ) ;
2008-01-17 13:12:52 +01:00
} else {
2009-06-16 08:38:26 +02:00
// remove entry from index
2009-06-07 23:48:01 +02:00
ix = ( int ) index . remove ( key ) ;
2008-02-03 03:23:04 +01:00
assert ix = = i ;
2009-06-16 08:38:26 +02:00
// switch values:
// remove last entry from the file copy to fill it in the gap
2009-01-30 16:33:00 +01:00
final Row . Entry te = table . removeOne ( ) ;
2009-06-16 08:38:26 +02:00
// fill the gap in file copy
2008-01-17 13:12:52 +01:00
table . set ( i , te ) ;
2009-06-16 08:38:26 +02:00
// move entry from last entry in file to gap position
2008-01-17 13:12:52 +01:00
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
2009-06-16 08:38:26 +02:00
// set new index for moved entry in index
2009-01-30 16:33:00 +01:00
final Row . Entry lr = rowdef . newEntry ( p ) ;
2009-03-08 22:37:17 +01:00
index . put ( lr . getPrimaryKeyBytes ( ) , i ) ;
2008-01-17 13:12:52 +01:00
}
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-24 23:49:00 +01:00
assert index . size ( ) + 1 = = sb : " index.size() = " + index . size ( ) + " , sb = " + sb ;
2008-01-17 13:12:52 +01:00
return rowdef . newEntry ( b ) ;
}
public synchronized Entry removeOne ( ) throws IOException {
2009-09-17 22:41:50 +02:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-08-02 14:12:04 +02:00
final byte [ ] le = new byte [ rowdef . objectsize ] ;
2009-07-16 01:08:35 +02:00
long fsb = file . size ( ) ;
assert fsb ! = 0 : " file.size() = " + fsb ;
2008-01-17 13:12:52 +01:00
file . cleanLast ( le , 0 ) ;
2009-07-16 01:08:35 +02:00
assert file . size ( ) < fsb : " file.size() = " + file . size ( ) ;
2009-01-30 16:33:00 +01:00
final Row . Entry lr = rowdef . newEntry ( le ) ;
2009-06-07 23:48:01 +02:00
final int i = ( int ) index . remove ( lr . getPrimaryKeyBytes ( ) ) ;
2008-01-17 13:12:52 +01:00
assert i > = 0 ;
2009-09-17 22:41:50 +02:00
if ( table ! = null ) table . removeOne ( ) ;
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
assert table = = null | | table . size ( ) = = index . size ( ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
return lr ;
}
2009-09-17 22:41:50 +02:00
public synchronized void clear ( ) throws IOException {
2008-08-02 14:12:04 +02:00
final File f = file . filename ( ) ;
2008-01-17 13:12:52 +01:00
file . close ( ) ;
2009-03-30 17:31:25 +02:00
FileUtils . deletedelete ( f ) ;
2008-01-17 13:12:52 +01:00
// make new file
FileOutputStream fos = null ;
try {
fos = new FileOutputStream ( f ) ;
2008-08-02 14:12:04 +02:00
} catch ( final FileNotFoundException e ) {
2008-01-17 13:12:52 +01:00
// should not happen
e . printStackTrace ( ) ;
}
2008-08-02 14:12:04 +02:00
if ( fos ! = null ) try { fos . close ( ) ; } catch ( final IOException e ) { }
2008-01-17 13:12:52 +01:00
// open an existing table file
try {
2009-09-09 22:49:16 +02:00
this . file = new BufferedRecords ( new Records ( f , rowdef . objectsize ) , this . buffersize ) ;
2008-08-02 14:12:04 +02:00
} catch ( final FileNotFoundException e ) {
2008-01-17 13:12:52 +01:00
// should never happen
e . printStackTrace ( ) ;
}
// initialize index and copy table
2009-01-30 16:33:00 +01:00
table = ( table = = null ) ? null : new RowSet ( taildef , 1 ) ;
2009-06-07 23:48:01 +02:00
index = new HandleMap ( rowdef . primaryKeyLength , rowdef . objectOrder , 4 , 1 , 100000 ) ;
2008-01-17 13:12:52 +01:00
}
2009-01-30 16:33:00 +01:00
public Row row ( ) {
2008-01-17 13:12:52 +01:00
return this . rowdef ;
}
public synchronized int size ( ) {
return index . size ( ) ;
}
2009-02-24 11:40:20 +01:00
public synchronized CloneableIterator < Entry > rows ( ) throws IOException {
return new rowIteratorNoOrder ( ) ;
}
public class rowIteratorNoOrder implements CloneableIterator < Entry > {
final Iterator < byte [ ] > ri ;
public rowIteratorNoOrder ( ) throws IOException {
ri = new ChunkIterator ( tablefile , rowdef . objectsize , rowdef . objectsize ) ;
}
public CloneableIterator < Entry > clone ( Object modifier ) {
try {
return new rowIteratorNoOrder ( ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
return null ;
}
}
public boolean hasNext ( ) {
return ri . hasNext ( ) ;
}
public Entry next ( ) {
byte [ ] r = ri . next ( ) ;
return rowdef . newEntry ( r ) ;
}
public void remove ( ) {
throw new UnsupportedOperationException ( " no remove in row iterator " ) ;
}
}
2008-01-17 13:12:52 +01:00
2009-01-30 23:08:08 +01:00
public synchronized CloneableIterator < Entry > rows ( final boolean up , final byte [ ] firstKey ) throws IOException {
2008-01-17 13:12:52 +01:00
return new rowIterator ( up , firstKey ) ;
}
2009-01-30 23:08:08 +01:00
public class rowIterator implements CloneableIterator < Entry > {
2008-01-17 13:12:52 +01:00
Iterator < byte [ ] > i ;
boolean up ;
byte [ ] fk ;
int c ;
2009-07-01 00:20:55 +02:00
public rowIterator ( final boolean up , final byte [ ] firstKey ) {
2008-01-17 13:12:52 +01:00
this . up = up ;
this . fk = firstKey ;
this . i = index . keys ( up , firstKey ) ;
this . c = - 1 ;
}
2009-01-30 23:08:08 +01:00
public CloneableIterator < Entry > clone ( final Object modifier ) {
2009-07-01 00:20:55 +02:00
return new rowIterator ( up , fk ) ;
2008-01-17 13:12:52 +01:00
}
public boolean hasNext ( ) {
return i . hasNext ( ) ;
}
public Entry next ( ) {
2008-08-02 14:12:04 +02:00
final byte [ ] k = i . next ( ) ;
2008-04-14 05:20:37 +02:00
assert k ! = null ;
if ( k = = null ) return null ;
2009-06-07 23:48:01 +02:00
this . c = ( int ) index . get ( k ) ;
2008-10-03 20:57:02 +02:00
if ( this . c < 0 ) throw new ConcurrentModificationException ( ) ; // this should only happen if the table was modified during the iteration
2008-08-02 14:12:04 +02:00
final byte [ ] b = new byte [ rowdef . objectsize ] ;
2008-01-17 13:12:52 +01:00
if ( table = = null ) {
// read from file
try {
file . get ( this . c , b , 0 ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-01-17 13:12:52 +01:00
e . printStackTrace ( ) ;
return null ;
}
} else {
// compose from table and key
2009-01-30 16:33:00 +01:00
final Row . Entry v = table . get ( this . c , false ) ;
2008-04-14 05:20:37 +02:00
assert v ! = null ;
if ( v = = null ) return null ;
2008-01-17 13:12:52 +01:00
System . arraycopy ( k , 0 , b , 0 , rowdef . primaryKeyLength ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , taildef . objectsize ) ;
}
return rowdef . newEntry ( b ) ;
}
public void remove ( ) {
2009-06-28 23:37:37 +02:00
throw new UnsupportedOperationException ( " no remove in TABLE " ) ;
2008-01-17 13:12:52 +01:00
}
}
2009-06-28 23:02:56 +02:00
private static byte [ ] testWord ( final char c ) {
return new byte [ ] { ( byte ) c , 32 , 32 , 32 } ;
}
private static String [ ] permutations ( final int letters ) {
String p = " " ;
for ( int i = 0 ; i < letters ; i + + ) p = p + ( ( char ) ( ( 'A' ) + i ) ) ;
return permutations ( p ) ;
}
private static String [ ] permutations ( final String source ) {
if ( source . length ( ) = = 0 ) return new String [ 0 ] ;
if ( source . length ( ) = = 1 ) return new String [ ] { source } ;
final char c = source . charAt ( 0 ) ;
final String [ ] recres = permutations ( source . substring ( 1 ) ) ;
final String [ ] result = new String [ source . length ( ) * recres . length ] ;
for ( int perm = 0 ; perm < recres . length ; perm + + ) {
result [ perm * source . length ( ) ] = c + recres [ perm ] ;
for ( int pos = 1 ; pos < source . length ( ) - 1 ; pos + + ) {
result [ perm * source . length ( ) + pos ] = recres [ perm ] . substring ( 0 , pos ) + c + recres [ perm ] . substring ( pos ) ;
}
result [ perm * source . length ( ) + source . length ( ) - 1 ] = recres [ perm ] + c ;
}
return result ;
}
2009-09-07 22:30:57 +02:00
private static ObjectIndex testTable ( final File f , final String testentities , final boolean useTailCache , final boolean exceed134217727 ) throws IOException {
2009-03-30 17:31:25 +02:00
if ( f . exists ( ) ) FileUtils . deletedelete ( f ) ;
2009-03-13 17:52:31 +01:00
final Row rowdef = new Row ( " byte[] a-4, byte[] b-4 " , NaturalOrder . naturalOrder ) ;
2009-09-07 22:30:57 +02:00
final ObjectIndex tt = new Table ( f , rowdef , 100 , 0 , useTailCache , exceed134217727 ) ;
2008-01-17 13:12:52 +01:00
byte [ ] b ;
2009-01-30 16:33:00 +01:00
final Row . Entry row = rowdef . newEntry ( ) ;
2008-01-17 13:12:52 +01:00
for ( int i = 0 ; i < testentities . length ( ) ; i + + ) {
2009-06-28 23:02:56 +02:00
b = testWord ( testentities . charAt ( i ) ) ;
2008-01-17 13:12:52 +01:00
row . setCol ( 0 , b ) ;
row . setCol ( 1 , b ) ;
tt . put ( row ) ;
}
return tt ;
}
2009-06-28 23:02:56 +02:00
private static int countElements ( final ObjectIndex t ) {
int count = 0 ;
try {
final Iterator < Row . Entry > iter = t . rows ( ) ;
Row . Entry row ;
while ( iter . hasNext ( ) ) {
count + + ;
row = iter . next ( ) ;
if ( row = = null ) System . out . println ( " ERROR! null element found " ) ;
// else System.out.println("counted element: " + new
// String(n.getKey()));
}
} catch ( final IOException e ) {
}
return count ;
}
2009-09-07 22:30:57 +02:00
public static void bigtest ( final int elements , final File testFile , final boolean useTailCache , final boolean exceed134217727 ) {
2008-01-17 13:12:52 +01:00
System . out . println ( " starting big test with " + elements + " elements: " ) ;
2008-08-02 14:12:04 +02:00
final long start = System . currentTimeMillis ( ) ;
2009-06-28 23:02:56 +02:00
final String [ ] s = permutations ( elements ) ;
2009-01-30 16:33:00 +01:00
ObjectIndex tt ;
2008-01-17 13:12:52 +01:00
try {
for ( int i = 0 ; i < s . length ; i + + ) {
System . out . println ( " *** probing tree " + i + " for permutation " + s [ i ] ) ;
// generate tree and delete elements
2009-09-07 22:30:57 +02:00
tt = testTable ( testFile , s [ i ] , useTailCache , exceed134217727 ) ;
2009-06-28 23:02:56 +02:00
if ( countElements ( tt ) ! = tt . size ( ) ) {
2008-01-17 13:12:52 +01:00
System . out . println ( " wrong size for " + s [ i ] ) ;
}
tt . close ( ) ;
for ( int j = 0 ; j < s . length ; j + + ) {
2009-09-07 22:30:57 +02:00
tt = testTable ( testFile , s [ i ] , useTailCache , exceed134217727 ) ;
2008-01-17 13:12:52 +01:00
// delete by permutation j
for ( int elt = 0 ; elt < s [ j ] . length ( ) ; elt + + ) {
2009-06-28 23:02:56 +02:00
tt . remove ( testWord ( s [ j ] . charAt ( elt ) ) ) ;
if ( countElements ( tt ) ! = tt . size ( ) ) {
2008-01-17 13:12:52 +01:00
System . out . println ( " ERROR! wrong size for probe tree " + s [ i ] + " ; probe delete " + s [ j ] + " ; position " + elt ) ;
}
}
tt . close ( ) ;
}
}
System . out . println ( " FINISHED test after " + ( ( System . currentTimeMillis ( ) - start ) / 1000 ) + " seconds. " ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2008-01-17 13:12:52 +01:00
e . printStackTrace ( ) ;
System . out . println ( " TERMINATED " ) ;
}
}
2008-12-18 01:08:17 +01:00
public void print ( ) throws IOException {
System . out . println ( " PRINTOUT of table, length= " + size ( ) ) ;
Entry row ;
byte [ ] key ;
2009-01-30 23:08:08 +01:00
CloneableIterator < byte [ ] > i = keys ( true , null ) ;
2008-12-18 01:08:17 +01:00
while ( i . hasNext ( ) ) {
System . out . print ( " row " + i + " : " ) ;
key = i . next ( ) ;
row = get ( key ) ;
System . out . println ( row . toString ( ) ) ;
}
System . out . println ( " EndOfTable " ) ;
}
2008-08-02 14:12:04 +02:00
public static void main ( final String [ ] args ) {
2008-01-17 13:12:52 +01:00
// open a file, add one entry and exit
2008-08-02 14:12:04 +02:00
final File f = new File ( args [ 0 ] ) ;
2008-01-19 13:23:56 +01:00
System . out . println ( " ========= Testcase: no tail cache: " ) ;
2009-09-07 22:30:57 +02:00
bigtest ( 5 , f , false , false ) ;
2008-01-19 13:23:56 +01:00
System . out . println ( " ========= Testcase: with tail cache: " ) ;
2009-09-07 22:30:57 +02:00
bigtest ( 5 , f , true , true ) ;
2008-01-17 13:12:52 +01:00
/ *
kelondroRow row = new kelondroRow ( " byte[] key-4, byte[] x-5 " , kelondroNaturalOrder . naturalOrder , 0 ) ;
try {
2009-06-28 23:37:37 +02:00
kelondroTABLE t = new kelondroTABLE ( f , row ) ;
2008-01-17 13:12:52 +01:00
kelondroRow . Entry entry = row . newEntry ( ) ;
entry . setCol ( 0 , " abcd " . getBytes ( ) ) ;
entry . setCol ( 1 , " dummy " . getBytes ( ) ) ;
t . put ( entry ) ;
t . close ( ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
}
* /
}
2008-09-12 13:51:48 +02:00
public void deleteOnExit ( ) {
this . file . deleteOnExit ( ) ;
}
2008-01-17 13:12:52 +01:00
}