2009-07-29 21:43:11 +02:00
// Table.java
2008-01-17 13:12:52 +01:00
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 14.01.2008 on http://yacy.net
//
2011-03-08 02:51:51 +01:00
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
2008-01-17 13:12:52 +01:00
//
// LICENSE
2011-06-01 21:31:56 +02:00
//
2008-01-17 13:12:52 +01:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2009-10-10 03:14:19 +02:00
package net.yacy.kelondro.table ;
2008-01-17 13:12:52 +01:00
import java.io.File ;
import java.io.FileNotFoundException ;
import java.io.FileOutputStream ;
import java.io.IOException ;
2008-01-20 02:22:46 +01:00
import java.util.ArrayList ;
2011-05-13 08:21:40 +02:00
import java.util.Collection ;
2008-10-03 20:57:02 +02:00
import java.util.ConcurrentModificationException ;
2008-01-17 22:48:08 +01:00
import java.util.HashMap ;
2008-01-17 13:12:52 +01:00
import java.util.Iterator ;
import java.util.List ;
2008-01-17 22:48:08 +01:00
import java.util.Map ;
2008-01-17 13:12:52 +01:00
import java.util.TreeMap ;
2008-01-20 02:22:46 +01:00
import java.util.TreeSet ;
2011-06-01 21:31:56 +02:00
2012-08-13 13:32:32 +02:00
import net.yacy.cora.document.ASCII ;
2011-12-16 23:59:29 +01:00
import net.yacy.cora.order.CloneableIterator ;
2012-09-21 16:46:57 +02:00
import net.yacy.cora.order.NaturalOrder ;
2012-07-27 12:13:53 +02:00
import net.yacy.cora.storage.HandleMap ;
import net.yacy.cora.util.SpaceExceededException ;
2009-10-10 01:32:08 +02:00
import net.yacy.kelondro.index.Column ;
2010-08-04 15:33:12 +02:00
import net.yacy.kelondro.index.Index ;
2009-10-10 01:32:08 +02:00
import net.yacy.kelondro.index.Row ;
2011-06-01 21:31:56 +02:00
import net.yacy.kelondro.index.Row.Entry ;
2009-10-10 01:32:08 +02:00
import net.yacy.kelondro.index.RowCollection ;
2012-07-27 12:13:53 +02:00
import net.yacy.kelondro.index.RowHandleMap ;
2009-10-10 01:32:08 +02:00
import net.yacy.kelondro.index.RowSet ;
2009-10-10 03:00:49 +02:00
import net.yacy.kelondro.io.BufferedRecords ;
import net.yacy.kelondro.io.Records ;
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.util.FileUtils ;
import net.yacy.kelondro.util.MemoryControl ;
import net.yacy.kelondro.util.kelondroException ;
2009-10-10 01:13:30 +02:00
2008-01-17 13:12:52 +01:00
/ *
2009-12-10 00:27:26 +01:00
* The Table builds upon the EcoFS and tries to reduce the number of IO requests that the
2008-01-17 13:12:52 +01:00
* EcoFS must do to a minimum . In best cases , no IO has to be done for read operations ( complete database shadow in RAM )
* and a rare number of write IO operations must be done for a large number of table - writings ( using the write buffer of EcoFS )
2009-12-10 00:27:26 +01:00
* To make the Table scalable in question of available RAM , there are two elements that must be scalable :
2008-01-17 13:12:52 +01:00
* - the access index can be either completely in RAM ( kelondroRAMIndex ) or it is file - based ( kelondroTree )
* - the content cache can be either a complete RAM - based shadow of the File , or empty .
* The content cache can also be deleted during run - time , if the available RAM gets too low .
* /
2010-08-04 15:33:12 +02:00
public class Table implements Index , Iterable < Row . Entry > {
2008-01-17 13:12:52 +01:00
// static tracker objects
2010-03-09 14:32:15 +01:00
private final static TreeMap < String , Table > tableTracker = new TreeMap < String , Table > ( ) ;
2012-06-09 09:05:47 +02:00
private final static long maxarraylength = 134217727L ; // (2^27-1) that may be the maximum size of array length in some JVMs
2011-06-01 21:31:56 +02:00
2010-04-21 13:29:27 +02:00
private final long minmemremaining ; // if less than this memory is remaininig, the memory copy of a table is abandoned
private final int buffersize ;
private final Row rowdef ;
private final Row taildef ;
private HandleMap index ;
private BufferedRecords file ;
private RowSet table ;
2011-06-01 21:31:56 +02:00
2009-09-07 22:30:57 +02:00
public Table (
final File tablefile ,
final Row rowdef ,
final int buffersize ,
final int initialSpace ,
2010-03-09 14:32:15 +01:00
boolean useTailCache ,
2011-09-15 12:01:21 +02:00
final boolean exceed134217727 ,
2012-07-27 12:13:53 +02:00
final boolean warmUp ) throws SpaceExceededException {
2011-06-01 21:31:56 +02:00
2008-01-17 13:12:52 +01:00
this . rowdef = rowdef ;
this . buffersize = buffersize ;
2012-06-08 12:48:25 +02:00
this . minmemremaining = Math . max ( 200L * 1024L * 1024L , MemoryControl . available ( ) / 10 ) ;
2008-10-03 20:57:02 +02:00
//this.fail = 0;
2008-01-17 13:12:52 +01:00
// define the taildef, a row like the rowdef but without the first column
2009-01-30 16:33:00 +01:00
final Column [ ] cols = new Column [ rowdef . columns ( ) - 1 ] ;
2008-01-17 13:12:52 +01:00
for ( int i = 0 ; i < cols . length ; i + + ) {
cols [ i ] = rowdef . column ( i + 1 ) ;
}
2009-03-13 17:52:31 +01:00
this . taildef = new Row ( cols , NaturalOrder . naturalOrder ) ;
2011-06-01 21:31:56 +02:00
2008-01-17 13:12:52 +01:00
// initialize table file
2009-02-16 17:13:48 +01:00
boolean freshFile = false ;
2008-01-17 13:12:52 +01:00
if ( ! tablefile . exists ( ) ) {
// make new file
2009-02-16 17:13:48 +01:00
freshFile = true ;
2008-01-17 13:12:52 +01:00
FileOutputStream fos = null ;
try {
fos = new FileOutputStream ( tablefile ) ;
2008-08-02 14:12:04 +02:00
} catch ( final FileNotFoundException e ) {
2008-01-17 13:12:52 +01:00
// should not happen
2009-09-29 23:52:17 +02:00
Log . logSevere ( " Table " , " " , e ) ;
2008-01-17 13:12:52 +01:00
}
2008-08-02 14:12:04 +02:00
if ( fos ! = null ) try { fos . close ( ) ; } catch ( final IOException e ) { }
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
2008-01-17 13:12:52 +01:00
try {
2008-01-17 22:48:08 +01:00
// open an existing table file
2011-06-01 21:31:56 +02:00
final int fileSize = ( int ) tableSize ( tablefile , rowdef . objectsize , true ) ;
2008-01-17 22:48:08 +01:00
// initialize index and copy table
2008-08-02 14:12:04 +02:00
final int records = Math . max ( fileSize , initialSpace ) ;
2012-06-09 09:05:47 +02:00
final long neededRAM4table = 200L * 1024L * 1024L + records * ( this . taildef . objectsize + rowdef . primaryKeyLength + 4L ) * 3L / 2L ;
2012-06-08 17:14:09 +02:00
this . table = null ;
try {
this . table = ( ( exceed134217727 | | neededRAM4table < maxarraylength ) & &
useTailCache & & MemoryControl . request ( neededRAM4table , true ) ) ? new RowSet ( this . taildef , records ) : null ;
2012-07-27 12:13:53 +02:00
} catch ( SpaceExceededException e ) {
2012-06-08 17:14:09 +02:00
this . table = null ;
} catch ( Throwable e ) {
this . table = null ;
}
2012-06-09 09:05:47 +02:00
Log . logInfo ( " TABLE " , " initialization of " + tablefile . getName ( ) + " . table copy: " + ( ( this . table = = null ) ? " no " : " yes " ) + " , available RAM: " + ( MemoryControl . available ( ) / 1024L / 1024L ) + " MB, needed: " + ( neededRAM4table / 1024L / 1024L ) + " MB, allocating space for " + records + " entries " ) ;
2012-09-11 22:28:10 +02:00
final long neededRAM4index = 100L * 1024L * 1024L + records * ( rowdef . primaryKeyLength + 4L ) * 3L / 2L ;
if ( records > 0 & & ! MemoryControl . request ( neededRAM4index , true ) ) {
2008-01-22 21:44:12 +01:00
// despite calculations seemed to show that there is enough memory for the table AND the index
// there is now not enough memory left for the index. So delete the table again to free the memory
// for the index
2012-06-09 09:05:47 +02:00
Log . logSevere ( " TABLE " , tablefile . getName ( ) + " : not enough RAM ( " + ( MemoryControl . available ( ) / 1024L / 1024L ) + " MB) left for index, deleting allocated table space to enable index space allocation (needed: " + ( neededRAM4index / 1024L / 1024L ) + " MB) " ) ;
2010-08-03 06:58:48 +02:00
this . table = null ; System . gc ( ) ;
2012-06-08 12:48:25 +02:00
Log . logSevere ( " TABLE " , tablefile . getName ( ) + " : RAM after releasing the table: " + ( MemoryControl . available ( ) / 1024L / 1024L ) + " MB " ) ;
2008-01-22 21:44:12 +01:00
}
2012-07-27 12:13:53 +02:00
this . index = new RowHandleMap ( rowdef . primaryKeyLength , rowdef . objectOrder , 4 , records , tablefile . getAbsolutePath ( ) ) ;
final RowHandleMap errors = new RowHandleMap ( rowdef . primaryKeyLength , NaturalOrder . naturalOrder , 4 , records , tablefile . getAbsolutePath ( ) + " .errors " ) ;
2011-06-01 21:31:56 +02:00
Log . logInfo ( " TABLE " , tablefile + " : TABLE " + tablefile . toString ( ) + " has table copy " + ( ( this . table = = null ) ? " DISABLED " : " ENABLED " ) ) ;
2008-01-20 22:42:35 +01:00
2008-01-17 22:48:08 +01:00
// read all elements from the file into the copy table
2009-06-28 23:37:37 +02:00
Log . logInfo ( " TABLE " , " initializing RAM index for TABLE " + tablefile . getName ( ) + " , please wait. " ) ;
2008-04-06 13:50:15 +02:00
int i = 0 ;
byte [ ] key ;
2010-08-03 06:58:48 +02:00
if ( this . table = = null ) {
2009-01-30 23:08:08 +01:00
final Iterator < byte [ ] > ki = new ChunkIterator ( tablefile , rowdef . objectsize , rowdef . primaryKeyLength ) ;
2008-04-06 13:50:15 +02:00
while ( ki . hasNext ( ) ) {
key = ki . next ( ) ;
// write the key into the index table
assert key ! = null ;
if ( key = = null ) { i + + ; continue ; }
2009-07-29 21:43:11 +02:00
if ( rowdef . objectOrder . wellformed ( key ) ) {
2010-08-03 06:58:48 +02:00
this . index . putUnique ( key , i + + ) ;
2009-07-29 21:43:11 +02:00
} else {
errors . putUnique ( key , i + + ) ;
}
2008-04-06 13:50:15 +02:00
}
} else {
byte [ ] record ;
key = new byte [ rowdef . primaryKeyLength ] ;
2009-01-30 23:08:08 +01:00
final Iterator < byte [ ] > ri = new ChunkIterator ( tablefile , rowdef . objectsize , rowdef . objectsize ) ;
2008-04-06 13:50:15 +02:00
while ( ri . hasNext ( ) ) {
record = ri . next ( ) ;
assert record ! = null ;
if ( record = = null ) { i + + ; continue ; }
System . arraycopy ( record , 0 , key , 0 , rowdef . primaryKeyLength ) ;
2011-06-01 21:31:56 +02:00
2008-04-06 13:50:15 +02:00
// write the key into the index table
2009-07-29 21:43:11 +02:00
if ( rowdef . objectOrder . wellformed ( key ) ) {
2011-06-01 21:31:56 +02:00
this . index . putUnique ( key , i + + ) ;
2009-07-29 21:43:11 +02:00
// write the tail into the table
2009-12-10 00:27:26 +01:00
try {
2011-06-01 21:31:56 +02:00
this . table . addUnique ( this . taildef . newEntry ( record , rowdef . primaryKeyLength , true ) ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2010-08-03 06:58:48 +02:00
this . table = null ;
2009-12-10 00:27:26 +01:00
break ;
}
2009-07-29 21:43:11 +02:00
} else {
errors . putUnique ( key , i + + ) ;
2008-04-06 13:50:15 +02:00
}
2008-01-24 23:49:00 +01:00
}
2012-06-08 12:48:25 +02:00
Runtime . getRuntime ( ) . gc ( ) ;
if ( abandonTable ( ) ) {
this . table = null ;
}
2008-01-17 22:48:08 +01:00
}
2012-06-06 18:23:18 +02:00
this . index . trim ( ) ;
2011-06-01 21:31:56 +02:00
2009-02-16 17:13:48 +01:00
// open the file
2009-09-09 22:49:16 +02:00
this . file = new BufferedRecords ( new Records ( tablefile , rowdef . objectsize ) , this . buffersize ) ;
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) + " , file = " + filename ( ) ;
2009-07-29 21:43:11 +02:00
// clean up the file by cleaning badly formed entries
2011-06-01 21:31:56 +02:00
final int errorc = errors . size ( ) ;
2009-07-29 21:43:11 +02:00
int errorcc = 0 ;
int idx ;
2012-07-27 12:13:53 +02:00
for ( final Map . Entry < byte [ ] , Long > entry : errors ) {
idx = ( int ) entry . getValue ( ) . longValue ( ) ;
2009-07-29 21:43:11 +02:00
removeInFile ( idx ) ;
2012-07-27 12:13:53 +02:00
key = entry . getKey ( ) ;
2011-11-14 13:32:15 +01:00
if ( key = = null ) continue ;
Log . logWarning ( " Table " , " removing not well-formed entry " + idx + " with key: " + NaturalOrder . arrayList ( key , 0 , key . length ) + " , " + errorcc + + + " / " + errorc ) ;
2009-07-29 21:43:11 +02:00
}
2010-08-03 06:58:48 +02:00
errors . close ( ) ;
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) + " , file = " + filename ( ) ;
2011-09-15 12:01:21 +02:00
// warm up
if ( ! freshFile & & warmUp ) { warmUp0 ( ) ; }
2008-08-02 14:12:04 +02:00
} catch ( final FileNotFoundException e ) {
2008-01-17 22:48:08 +01:00
// should never happen
2009-09-29 23:52:17 +02:00
Log . logSevere ( " Table " , " " , e ) ;
2008-01-17 22:48:08 +01:00
throw new kelondroException ( e . getMessage ( ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2009-09-29 23:52:17 +02:00
Log . logSevere ( " Table " , " " , e ) ;
2008-01-17 22:48:08 +01:00
throw new kelondroException ( e . getMessage ( ) ) ;
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
2008-01-17 13:12:52 +01:00
// track this table
tableTracker . put ( tablefile . toString ( ) , this ) ;
}
2011-06-01 21:31:56 +02:00
2011-09-15 12:01:21 +02:00
public synchronized void warmUp ( ) {
warmUp0 ( ) ;
}
private void warmUp0 ( ) {
// remove doubles
try {
final ArrayList < long [ ] > doubles = this . index . removeDoubles ( ) ;
//assert index.size() + doubles.size() == i;
//System.out.println(" -removed " + doubles.size() + " doubles- done.");
if ( doubles . isEmpty ( ) ) return ;
Log . logInfo ( " TABLE " , filename ( ) + " : WARNING - TABLE " + filename ( ) + " has " + doubles . size ( ) + " doubles " ) ;
// from all the doubles take one, put it back to the index and remove the others from the file
// first put back one element each
final byte [ ] record = new byte [ this . rowdef . objectsize ] ;
final byte [ ] key = new byte [ this . rowdef . primaryKeyLength ] ;
for ( final long [ ] ds : doubles ) {
this . file . get ( ( int ) ds [ 0 ] , record , 0 ) ;
System . arraycopy ( record , 0 , key , 0 , this . rowdef . primaryKeyLength ) ;
this . index . putUnique ( key , ( int ) ds [ 0 ] ) ;
}
// then remove the other doubles by removing them from the table, but do a re-indexing while doing that
// first aggregate all the delete positions because the elements from the top positions must be removed first
final TreeSet < Long > delpos = new TreeSet < Long > ( ) ;
for ( final long [ ] ds : doubles ) {
for ( int j = 1 ; j < ds . length ; j + + ) delpos . add ( ds [ j ] ) ;
}
// now remove the entries in a sorted way (top-down)
Long top ;
while ( ! delpos . isEmpty ( ) ) {
top = delpos . last ( ) ;
delpos . remove ( top ) ;
removeInFile ( top . intValue ( ) ) ;
}
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-09-15 12:01:21 +02:00
Log . logSevere ( " Table " , " " , e ) ;
} catch ( final IOException e ) {
Log . logSevere ( " Table " , " " , e ) ;
}
}
2012-01-10 03:03:12 +01:00
@Override
2010-08-03 06:58:48 +02:00
public long mem ( ) {
2011-06-01 21:31:56 +02:00
return this . index . mem ( ) + ( ( this . table = = null ) ? 0 : this . table . mem ( ) ) ;
2010-08-03 06:58:48 +02:00
}
2011-06-01 21:31:56 +02:00
2008-11-03 00:53:09 +01:00
private boolean abandonTable ( ) {
// check if not enough memory is there to maintain a memory copy of the table
2011-06-01 21:31:56 +02:00
return MemoryControl . shortStatus ( ) | | MemoryControl . available ( ) < this . minmemremaining ;
2008-11-03 00:53:09 +01:00
}
2011-06-01 21:31:56 +02:00
2012-01-10 03:03:12 +01:00
@Override
2010-04-19 18:42:37 +02:00
public byte [ ] smallestKey ( ) {
return this . index . smallestKey ( ) ;
}
2011-06-01 21:31:56 +02:00
2012-01-10 03:03:12 +01:00
@Override
2010-04-19 18:42:37 +02:00
public byte [ ] largestKey ( ) {
return this . index . largestKey ( ) ;
}
2011-06-01 21:31:56 +02:00
public static long tableSize ( final File tablefile , final int recordsize , final boolean fixIfCorrupted ) throws kelondroException {
2010-04-21 13:29:27 +02:00
try {
return Records . tableSize ( tablefile , recordsize ) ;
} catch ( final IOException e ) {
if ( ! fixIfCorrupted ) {
Log . logSevere ( " Table " , " table size broken for file " + tablefile . toString ( ) , e ) ;
throw new kelondroException ( e . getMessage ( ) ) ;
}
Log . logSevere ( " Table " , " table size broken, try to fix " + tablefile . toString ( ) ) ;
try {
Records . fixTableSize ( tablefile , recordsize ) ;
Log . logInfo ( " Table " , " successfully fixed table file " + tablefile . toString ( ) ) ;
return Records . tableSize ( tablefile , recordsize ) ;
} catch ( final IOException ee ) {
Log . logSevere ( " Table " , " table size fix did not work " , ee ) ;
throw new kelondroException ( e . getMessage ( ) ) ;
}
}
2008-01-17 13:12:52 +01:00
}
2008-01-17 22:48:08 +01:00
public static final Iterator < String > filenames ( ) {
// iterates string objects; all file names from record tracker
return tableTracker . keySet ( ) . iterator ( ) ;
}
2011-10-26 12:07:16 +02:00
public static final Map < StatKeys , String > memoryStats ( final String filename ) {
2008-01-17 22:48:08 +01:00
// returns a map for each file in the tracker;
// the map represents properties for each record objects,
// i.e. for cache memory allocation
2009-06-28 23:37:37 +02:00
final Table theTABLE = tableTracker . get ( filename ) ;
return theTABLE . memoryStats ( ) ;
2008-01-17 22:48:08 +01:00
}
2011-10-26 12:07:16 +02:00
public enum StatKeys {
tableSize , tableKeyChunkSize , tableKeyMem , tableValueChunkSize , tableValueMem
}
private final Map < StatKeys , String > memoryStats ( ) {
2008-01-17 22:48:08 +01:00
// returns statistical data about this object
2010-04-20 02:08:43 +02:00
synchronized ( this ) {
2011-06-01 21:31:56 +02:00
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
2010-04-19 18:42:37 +02:00
}
2011-10-26 12:07:16 +02:00
final HashMap < StatKeys , String > map = new HashMap < StatKeys , String > ( 8 ) ;
2011-06-01 21:31:56 +02:00
if ( this . index = = null ) return map ; // possibly closed or beeing closed
2011-10-26 12:07:16 +02:00
map . put ( StatKeys . tableSize , Integer . toString ( this . index . size ( ) ) ) ;
2012-07-27 12:13:53 +02:00
map . put ( StatKeys . tableKeyChunkSize , ( this . index instanceof RowHandleMap ) ? Integer . toString ( ( ( RowHandleMap ) this . index ) . row ( ) . objectsize ) : " -1 " ) ;
map . put ( StatKeys . tableKeyMem , ( this . index instanceof RowHandleMap ) ? Integer . toString ( ( ( RowHandleMap ) this . index ) . row ( ) . objectsize * this . index . size ( ) ) : " -1 " ) ;
2011-10-26 12:07:16 +02:00
map . put ( StatKeys . tableValueChunkSize , ( this . table = = null ) ? " 0 " : Integer . toString ( this . table . row ( ) . objectsize ) ) ;
map . put ( StatKeys . tableValueMem , ( this . table = = null ) ? " 0 " : Integer . toString ( this . table . row ( ) . objectsize * this . table . size ( ) ) ) ;
2008-01-17 22:48:08 +01:00
return map ;
}
2011-06-01 21:31:56 +02:00
2008-06-18 23:08:56 +02:00
public boolean usesFullCopy ( ) {
return this . table ! = null ;
}
2011-06-01 21:31:56 +02:00
2010-04-21 13:29:27 +02:00
public static long staticRAMIndexNeed ( final File f , final Row rowdef ) {
2011-06-01 21:31:56 +02:00
return ( ( ( rowdef . primaryKeyLength + 4 ) ) * tableSize ( f , rowdef . objectsize , true ) * RowCollection . growfactorLarge100 / 100L ) ;
2008-01-17 22:48:08 +01:00
}
2011-06-01 21:31:56 +02:00
2010-06-18 20:59:42 +02:00
public boolean consistencyCheck ( ) {
try {
2011-06-01 21:31:56 +02:00
return this . file . size ( ) = = this . index . size ( ) ;
} catch ( final IOException e ) {
2010-06-18 20:59:42 +02:00
Log . logException ( e ) ;
return false ;
}
}
2011-06-01 21:31:56 +02:00
2012-01-10 03:03:12 +01:00
@Override
2012-07-27 12:13:53 +02:00
public synchronized void addUnique ( final Entry row ) throws IOException , SpaceExceededException {
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
final int i = ( int ) this . file . size ( ) ;
2009-12-10 00:27:26 +01:00
try {
2011-06-01 21:31:56 +02:00
this . index . putUnique ( row . getPrimaryKeyBytes ( ) , i ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-06-01 21:31:56 +02:00
if ( this . table = = null ) throw e ; // in case the table is not used, there is no help here
this . table = null ;
2009-12-10 00:27:26 +01:00
// try again with less memory
2011-06-01 21:31:56 +02:00
this . index . putUnique ( row . getPrimaryKeyBytes ( ) , i ) ;
2009-12-10 00:27:26 +01:00
}
2011-09-15 13:17:02 +02:00
final byte [ ] rowbytes = row . bytes ( ) ;
2011-06-01 21:31:56 +02:00
if ( this . table ! = null ) {
assert this . table . size ( ) = = i ;
2009-12-10 00:27:26 +01:00
try {
2011-09-15 13:17:02 +02:00
this . table . addUnique ( this . taildef . newEntry ( rowbytes , this . rowdef . primaryKeyLength , true ) ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-06-01 21:31:56 +02:00
this . table = null ;
2009-12-10 00:27:26 +01:00
}
2011-06-01 21:31:56 +02:00
if ( abandonTable ( ) ) this . table = null ;
2008-01-17 13:12:52 +01:00
}
2011-09-15 13:17:02 +02:00
this . file . add ( rowbytes , 0 ) ;
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2012-07-27 12:13:53 +02:00
public synchronized void addUnique ( final List < Entry > rows ) throws IOException , SpaceExceededException {
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
for ( final Entry entry : rows ) {
2009-12-10 00:27:26 +01:00
try {
addUnique ( entry ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2009-12-10 00:27:26 +01:00
if ( this . table = = null ) throw e ;
2011-06-01 21:31:56 +02:00
this . table = null ;
2009-12-10 00:27:26 +01:00
addUnique ( entry ) ;
}
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2008-06-18 23:08:56 +02:00
/ * *
2012-07-27 12:13:53 +02:00
* @throws SpaceExceededException
2008-06-18 23:08:56 +02:00
* remove double - entries from the table
* this process calls the underlying removeDoubles ( ) method from the table index
2011-06-01 21:31:56 +02:00
* and
* @throws
2008-06-18 23:08:56 +02:00
* /
2012-01-10 03:03:12 +01:00
@Override
2012-07-27 12:13:53 +02:00
public synchronized List < RowCollection > removeDoubles ( ) throws IOException , SpaceExceededException {
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
2011-03-08 02:51:51 +01:00
final List < RowCollection > report = new ArrayList < RowCollection > ( ) ;
2009-01-30 16:33:00 +01:00
RowSet rows ;
2009-06-07 23:48:01 +02:00
final TreeSet < Long > d = new TreeSet < Long > ( ) ;
2011-06-01 21:31:56 +02:00
final byte [ ] b = new byte [ this . rowdef . objectsize ] ;
2009-01-30 16:33:00 +01:00
Row . Entry inconsistentEntry ;
2008-06-18 23:08:56 +02:00
// iterate over all entries that have inconsistent index references
2009-02-16 17:13:48 +01:00
long lastlog = System . currentTimeMillis ( ) ;
2011-03-08 02:51:51 +01:00
List < long [ ] > doubles ;
2009-12-10 00:27:26 +01:00
try {
2011-06-01 21:31:56 +02:00
doubles = this . index . removeDoubles ( ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2009-12-10 00:27:26 +01:00
if ( this . table = = null ) throw e ;
2011-06-01 21:31:56 +02:00
this . table = null ;
doubles = this . index . removeDoubles ( ) ;
2009-12-10 00:27:26 +01:00
}
2011-02-22 13:56:25 +01:00
for ( final long [ ] is : doubles ) {
2008-06-18 23:08:56 +02:00
// 'is' is the set of all indexes, that have the same reference
// we collect that entries now here
2009-01-30 16:33:00 +01:00
rows = new RowSet ( this . rowdef , is . length ) ;
2011-03-08 02:51:51 +01:00
for ( final long L : is ) {
2011-06-01 21:31:56 +02:00
assert ( int ) L < this . file . size ( ) : " L.intValue() = " + ( int ) L + " , file.size = " + this . file . size ( ) ; // prevent ooBounds Exception
2008-06-18 23:08:56 +02:00
d . add ( L ) ;
2011-06-01 21:31:56 +02:00
if ( ( int ) L > = this . file . size ( ) ) continue ; // prevent IndexOutOfBoundsException
this . file . get ( ( int ) L , b , 0 ) ; // TODO: fix IndexOutOfBoundsException here
inconsistentEntry = this . rowdef . newEntry ( b ) ;
2009-12-10 00:27:26 +01:00
try {
rows . addUnique ( inconsistentEntry ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2009-12-10 00:27:26 +01:00
if ( this . table = = null ) throw e ;
this . table = null ;
rows . addUnique ( inconsistentEntry ) ;
}
2008-01-20 02:22:46 +01:00
}
report . add ( rows ) ;
}
2008-05-02 00:40:42 +02:00
// finally delete the affected rows, but start with largest id first, otherwise we overwrite wrong entries
2009-06-07 23:48:01 +02:00
Long s ;
2009-12-02 01:37:59 +01:00
while ( ! d . isEmpty ( ) ) {
2008-01-20 02:22:46 +01:00
s = d . last ( ) ;
d . remove ( s ) ;
2011-06-01 21:31:56 +02:00
removeInFile ( s . intValue ( ) ) ;
2009-02-16 17:13:48 +01:00
if ( System . currentTimeMillis ( ) - lastlog > 30000 ) {
2011-06-01 21:31:56 +02:00
Log . logInfo ( " TABLE " , " removing " + d . size ( ) + " entries in " + filename ( ) ) ;
2009-02-16 17:21:12 +01:00
lastlog = System . currentTimeMillis ( ) ;
2009-02-16 17:13:48 +01:00
}
2008-01-20 02:22:46 +01:00
}
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
2008-01-20 02:22:46 +01:00
return report ;
}
2011-06-01 21:31:56 +02:00
2012-01-10 03:03:12 +01:00
@Override
2008-01-17 13:12:52 +01:00
public void close ( ) {
2011-01-04 00:06:33 +01:00
if ( this . file ! = null ) this . file . close ( ) ;
2008-12-29 13:22:13 +01:00
this . file = null ;
2010-08-03 06:58:48 +02:00
if ( this . table ! = null ) this . table . close ( ) ;
2008-12-29 13:22:13 +01:00
this . table = null ;
2010-08-03 06:58:48 +02:00
if ( this . index ! = null ) this . index . close ( ) ;
2008-12-29 13:22:13 +01:00
this . index = null ;
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
2011-03-08 02:51:51 +01:00
@Override
2008-08-06 21:43:12 +02:00
protected void finalize ( ) {
2011-06-01 21:31:56 +02:00
if ( this . file ! = null ) close ( ) ;
2008-01-17 13:12:52 +01:00
}
2012-01-10 03:03:12 +01:00
@Override
2008-01-17 13:12:52 +01:00
public String filename ( ) {
return this . file . filename ( ) . toString ( ) ;
}
2010-04-19 18:42:37 +02:00
2012-01-10 03:03:12 +01:00
@Override
2011-07-15 10:38:10 +02:00
public Entry get ( final byte [ ] key , final boolean _forcecopy ) throws IOException {
2011-06-01 21:31:56 +02:00
if ( this . file = = null | | this . index = = null ) return null ;
2010-04-19 18:42:37 +02:00
Entry e = get0 ( key ) ;
if ( e ! = null & & this . rowdef . objectOrder . equal ( key , e . getPrimaryKeyBytes ( ) ) ) return e ;
synchronized ( this ) {
2011-11-30 21:50:41 +01:00
//assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size() + ", file = " + filename();
2011-06-01 21:31:56 +02:00
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) + " , file = " + filename ( ) ;
2010-04-19 18:42:37 +02:00
e = get0 ( key ) ;
2012-08-13 13:32:32 +02:00
assert e = = null | | this . rowdef . objectOrder . equal ( key , e . getPrimaryKeyBytes ( ) ) : " key = " + ASCII . String ( key ) + " , e.k = " + ASCII . String ( e . getPrimaryKeyBytes ( ) ) ;
2010-04-19 18:42:37 +02:00
return e ;
}
}
2011-06-01 21:31:56 +02:00
2010-04-19 18:42:37 +02:00
private Entry get0 ( final byte [ ] key ) throws IOException {
2011-06-01 21:31:56 +02:00
if ( this . file = = null | | this . index = = null ) return null ;
final int i = ( int ) this . index . get ( key ) ;
2008-01-17 13:12:52 +01:00
if ( i = = - 1 ) return null ;
2011-06-01 21:31:56 +02:00
final byte [ ] b = new byte [ this . rowdef . objectsize ] ;
2011-06-22 01:10:50 +02:00
final Row . Entry cacherow ;
if ( this . table = = null | | ( cacherow = this . table . get ( i , false ) ) = = null ) {
2008-01-17 13:12:52 +01:00
// read row from the file
2011-11-03 15:41:38 +01:00
try {
this . file . get ( i , b , 0 ) ;
} catch ( final IndexOutOfBoundsException e ) {
// there must be a problem with the table index
Log . logSevere ( " Table " , " IndexOutOfBoundsException: " + e . getMessage ( ) , e ) ;
this . index . remove ( key ) ;
if ( this . table ! = null ) this . table . remove ( key ) ;
return null ;
}
2008-01-17 13:12:52 +01:00
} else {
// construct the row using the copy in RAM
2011-06-01 21:31:56 +02:00
assert key . length = = this . rowdef . primaryKeyLength ;
2008-01-17 13:12:52 +01:00
System . arraycopy ( key , 0 , b , 0 , key . length ) ;
2011-06-22 01:10:50 +02:00
System . arraycopy ( cacherow . bytes ( ) , 0 , b , this . rowdef . primaryKeyLength , this . rowdef . objectsize - this . rowdef . primaryKeyLength ) ;
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
return this . rowdef . newEntry ( b ) ;
2008-01-17 13:12:52 +01:00
}
2012-01-10 03:03:12 +01:00
@Override
2011-07-15 10:38:10 +02:00
public Map < byte [ ] , Row . Entry > get ( final Collection < byte [ ] > keys , final boolean forcecopy ) throws IOException , InterruptedException {
2011-06-01 21:31:56 +02:00
final Map < byte [ ] , Row . Entry > map = new TreeMap < byte [ ] , Row . Entry > ( row ( ) . objectOrder ) ;
2011-05-13 08:21:40 +02:00
Row . Entry entry ;
2011-06-01 21:31:56 +02:00
for ( final byte [ ] key : keys ) {
2011-07-15 10:38:10 +02:00
entry = get ( key , forcecopy ) ;
2011-05-13 08:21:40 +02:00
if ( entry ! = null ) map . put ( key , entry ) ;
}
return map ;
}
2012-01-10 03:03:12 +01:00
@Override
2010-04-19 18:42:37 +02:00
public boolean has ( final byte [ ] key ) {
2011-06-01 21:31:56 +02:00
if ( this . index = = null ) return false ;
return this . index . has ( key ) ;
2008-01-17 13:12:52 +01:00
}
2012-01-10 03:03:12 +01:00
@Override
2009-01-30 23:08:08 +01:00
public synchronized CloneableIterator < byte [ ] > keys ( final boolean up , final byte [ ] firstKey ) throws IOException {
2011-06-01 21:31:56 +02:00
return this . index . keys ( up , firstKey ) ;
2008-01-17 13:12:52 +01:00
}
2012-01-10 03:03:12 +01:00
@Override
2012-07-27 12:13:53 +02:00
public Entry replace ( final Entry row ) throws IOException , SpaceExceededException {
2008-01-25 12:44:27 +01:00
assert row ! = null ;
2011-09-15 13:17:02 +02:00
if ( this . file = = null | | row = = null ) return null ;
final byte [ ] rowb = row . bytes ( ) ;
assert rowb ! = null ;
if ( rowb = = null ) return null ;
final byte [ ] key = row . getPrimaryKeyBytes ( ) ;
synchronized ( this ) {
2011-11-25 12:23:52 +01:00
//assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
//assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
2011-09-15 13:17:02 +02:00
final int i = ( int ) this . index . get ( key ) ;
if ( i = = - 1 ) {
try {
addUnique ( row ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-09-15 13:17:02 +02:00
if ( this . table = = null ) throw e ;
this . table = null ;
addUnique ( row ) ;
}
return null ;
2009-12-10 00:27:26 +01:00
}
2011-06-01 21:31:56 +02:00
2011-09-15 13:17:02 +02:00
final byte [ ] b = new byte [ this . rowdef . objectsize ] ;
Row . Entry cacherow ;
if ( this . table = = null | | ( cacherow = this . table . get ( i , false ) ) = = null ) {
// read old value
this . file . get ( i , b , 0 ) ;
// write new value
this . file . put ( i , rowb , 0 ) ;
} else {
// read old value
assert cacherow ! = null ;
System . arraycopy ( key , 0 , b , 0 , this . rowdef . primaryKeyLength ) ;
System . arraycopy ( cacherow . bytes ( ) , 0 , b , this . rowdef . primaryKeyLength , this . rowdef . objectsize - this . rowdef . primaryKeyLength ) ;
// write new value
try {
this . table . set ( i , this . taildef . newEntry ( rowb , this . rowdef . primaryKeyLength , true ) ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-09-15 13:17:02 +02:00
this . table = null ;
}
if ( abandonTable ( ) ) this . table = null ;
this . file . put ( i , rowb , 0 ) ;
2009-12-10 00:27:26 +01:00
}
2011-09-15 13:17:02 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
// return old value
return this . rowdef . newEntry ( b ) ;
2008-01-17 13:12:52 +01:00
}
}
2011-06-01 21:31:56 +02:00
2010-09-15 12:43:14 +02:00
/ * *
* Adds the row to the index . The row is identified by the primary key of the row .
* @param row a index row
2011-06-01 21:31:56 +02:00
* @return true if this set did _not_ already contain the given row .
2010-09-15 12:43:14 +02:00
* @throws IOException
2012-07-27 12:13:53 +02:00
* @throws SpaceExceededException
2010-09-15 12:43:14 +02:00
* /
2012-06-06 18:23:18 +02:00
@Override
2012-07-27 12:13:53 +02:00
public boolean put ( final Entry row ) throws IOException , SpaceExceededException {
2009-03-11 21:23:19 +01:00
assert row ! = null ;
2011-09-15 13:17:02 +02:00
if ( this . file = = null | | row = = null ) return true ;
final byte [ ] rowb = row . bytes ( ) ;
assert rowb ! = null ;
if ( rowb = = null ) return true ;
final byte [ ] key = row . getPrimaryKeyBytes ( ) ;
synchronized ( this ) {
2011-11-25 12:23:52 +01:00
//assert this.file == null || this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size() + ", file = " + filename();
//assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size() + ", file = " + filename();
2011-09-15 13:17:02 +02:00
final int i = ( int ) this . index . get ( key ) ;
if ( i = = - 1 ) {
try {
addUnique ( row ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-09-15 13:17:02 +02:00
if ( this . table = = null ) throw e ;
this . table = null ;
addUnique ( row ) ;
}
return true ;
2009-12-10 00:27:26 +01:00
}
2011-06-01 21:31:56 +02:00
2011-09-15 13:17:02 +02:00
if ( this . table = = null ) {
// write new value
this . file . put ( i , rowb , 0 ) ;
} else {
// write new value
this . file . put ( i , rowb , 0 ) ;
if ( abandonTable ( ) ) this . table = null ; else try {
this . table . set ( i , this . taildef . newEntry ( rowb , this . rowdef . primaryKeyLength , true ) ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-09-15 13:17:02 +02:00
this . table = null ;
}
2009-12-10 00:27:26 +01:00
}
2011-09-15 13:17:02 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
return false ;
2009-03-11 21:23:19 +01:00
}
}
2008-01-17 13:12:52 +01:00
2009-07-29 21:43:11 +02:00
/ * *
* remove one entry from the file
* @param i an index position within the file ( not a byte position )
* @throws IOException
2012-07-27 12:13:53 +02:00
* @throws SpaceExceededException
2009-07-29 21:43:11 +02:00
* /
2012-07-27 12:13:53 +02:00
private void removeInFile ( final int i ) throws IOException , SpaceExceededException {
2008-01-20 02:22:46 +01:00
assert i > = 0 ;
2011-06-01 21:31:56 +02:00
2010-05-09 00:09:36 +02:00
final byte [ ] p = new byte [ this . rowdef . objectsize ] ;
if ( this . table = = null ) {
if ( i = = this . index . size ( ) - 1 ) {
this . file . cleanLast ( ) ;
2008-01-22 22:40:25 +01:00
} else {
2010-05-09 00:09:36 +02:00
while ( this . file . size ( ) > 0 ) {
this . file . cleanLast ( p , 0 ) ;
if ( ! ( this . rowdef . objectOrder . wellformed ( p , 0 , this . rowdef . primaryKeyLength ) ) ) {
continue ;
}
this . file . put ( i , p , 0 ) ;
final byte [ ] k = new byte [ this . rowdef . primaryKeyLength ] ;
System . arraycopy ( p , 0 , k , 0 , this . rowdef . primaryKeyLength ) ;
this . index . put ( k , i ) ;
break ;
}
2008-01-22 22:40:25 +01:00
}
2008-01-20 02:22:46 +01:00
} else {
2011-06-01 21:31:56 +02:00
if ( i = = this . index . size ( ) - 1 ) {
2008-01-20 02:22:46 +01:00
// special handling if the entry is the last entry in the file
2011-06-01 21:31:56 +02:00
this . table . removeRow ( i , false ) ;
this . file . cleanLast ( ) ;
2008-01-20 02:22:46 +01:00
} else {
// switch values
2011-06-01 21:31:56 +02:00
final Row . Entry te = this . table . removeOne ( ) ;
2009-12-10 00:27:26 +01:00
try {
2011-06-01 21:31:56 +02:00
this . table . set ( i , te ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-06-01 21:31:56 +02:00
this . table = null ;
2009-12-10 00:27:26 +01:00
}
2008-01-20 02:22:46 +01:00
2011-06-01 21:31:56 +02:00
while ( this . file . size ( ) > 0 ) {
this . file . cleanLast ( p , 0 ) ;
final Row . Entry lr = this . rowdef . newEntry ( p ) ;
2010-05-07 00:06:31 +02:00
if ( lr = = null ) {
// in case that p is not well-formed lr may be null
// drop table copy because that becomes too complicated here
2011-06-01 21:31:56 +02:00
this . table . clear ( ) ;
this . table = null ;
2010-05-07 00:06:31 +02:00
continue ;
}
2011-06-01 21:31:56 +02:00
this . file . put ( i , p , 0 ) ;
2012-05-11 20:46:50 +02:00
byte [ ] pk = lr . getPrimaryKeyBytes ( ) ;
2012-05-15 20:57:38 +02:00
if ( pk = = null ) {
// Table file might be corrupt
Log . logWarning ( " TABLE " , " Possible corruption found in table " + this . filename ( ) + " detected. i= " + i + " ,p= " + p ) ;
continue ;
}
2012-05-11 20:46:50 +02:00
this . index . put ( pk , i ) ;
2010-05-07 00:04:30 +02:00
break ;
}
2008-01-20 02:22:46 +01:00
}
}
}
2011-06-01 21:31:56 +02:00
2012-06-06 18:23:18 +02:00
@Override
2010-04-20 16:47:41 +02:00
public boolean delete ( final byte [ ] key ) throws IOException {
return remove ( key ) ! = null ;
}
2011-06-01 21:31:56 +02:00
2012-06-09 09:05:47 +02:00
@Override
2008-08-02 14:12:04 +02:00
public synchronized Entry remove ( final byte [ ] key ) throws IOException {
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert key . length = = this . rowdef . primaryKeyLength ;
final int i = ( int ) this . index . get ( key ) ;
2008-01-17 13:12:52 +01:00
if ( i = = - 1 ) return null ; // nothing to do
2011-06-01 21:31:56 +02:00
2008-01-17 13:12:52 +01:00
// prepare result
2011-06-01 21:31:56 +02:00
final byte [ ] b = new byte [ this . rowdef . objectsize ] ;
final byte [ ] p = new byte [ this . rowdef . objectsize ] ;
final int sb = this . index . size ( ) ;
2008-02-02 22:36:19 +01:00
int ix ;
2011-06-01 21:31:56 +02:00
assert i < this . index . size ( ) ;
2011-06-22 01:10:50 +02:00
final Row . Entry cacherow ;
if ( this . table = = null | | ( cacherow = this . table . get ( i , false ) ) = = null ) {
2011-06-01 21:31:56 +02:00
if ( i = = this . index . size ( ) - 1 ) {
2009-06-16 08:38:26 +02:00
// element is at last entry position
2011-06-01 21:31:56 +02:00
ix = ( int ) this . index . remove ( key ) ;
assert this . index . size ( ) < i + 1 : " index.size() = " + this . index . size ( ) + " , i = " + i ;
2008-02-02 22:36:19 +01:00
assert ix = = i ;
2011-06-01 21:31:56 +02:00
this . file . cleanLast ( b , 0 ) ;
2008-01-24 23:49:00 +01:00
} else {
2009-06-16 08:38:26 +02:00
// remove entry from index
2011-06-01 21:31:56 +02:00
assert i < this . index . size ( ) - 1 : " index.size() = " + this . index . size ( ) + " , i = " + i ;
ix = ( int ) this . index . remove ( key ) ;
assert i < this . index . size ( ) : " index.size() = " + this . index . size ( ) + " , i = " + i ;
2008-02-02 22:36:19 +01:00
assert ix = = i ;
2011-06-01 21:31:56 +02:00
2009-06-16 08:38:26 +02:00
// read element that shall be removed
2011-06-01 21:31:56 +02:00
this . file . get ( i , b , 0 ) ;
2009-06-16 08:38:26 +02:00
// fill the gap with value from last entry in file
2011-06-01 21:31:56 +02:00
this . file . cleanLast ( p , 0 ) ;
this . file . put ( i , p , 0 ) ;
final byte [ ] k = new byte [ this . rowdef . primaryKeyLength ] ;
System . arraycopy ( p , 0 , k , 0 , this . rowdef . primaryKeyLength ) ;
2009-12-10 00:27:26 +01:00
try {
2011-06-01 21:31:56 +02:00
this . index . put ( k , i ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2009-12-10 00:27:26 +01:00
Log . logException ( e ) ;
throw new IOException ( " RowSpaceExceededException: " + e . getMessage ( ) ) ;
}
2008-01-24 23:49:00 +01:00
}
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
2008-01-17 13:12:52 +01:00
} else {
2008-01-24 23:49:00 +01:00
// get result value from the table copy, so we don't need to read it from the file
2008-01-17 13:12:52 +01:00
System . arraycopy ( key , 0 , b , 0 , key . length ) ;
2011-06-22 01:10:50 +02:00
System . arraycopy ( cacherow . bytes ( ) , 0 , b , this . rowdef . primaryKeyLength , this . taildef . objectsize ) ;
2011-06-01 21:31:56 +02:00
if ( i = = this . index . size ( ) - 1 ) {
2008-01-17 13:12:52 +01:00
// special handling if the entry is the last entry in the file
2011-06-01 21:31:56 +02:00
ix = ( int ) this . index . remove ( key ) ;
assert this . index . size ( ) < i + 1 : " index.size() = " + this . index . size ( ) + " , i = " + i ;
2008-02-03 03:23:04 +01:00
assert ix = = i ;
2011-06-01 21:31:56 +02:00
this . table . removeRow ( i , false ) ;
this . file . cleanLast ( ) ;
2008-01-17 13:12:52 +01:00
} else {
2009-06-16 08:38:26 +02:00
// remove entry from index
2011-06-01 21:31:56 +02:00
ix = ( int ) this . index . remove ( key ) ;
assert i < this . index . size ( ) : " index.size() = " + this . index . size ( ) + " , i = " + i ;
2008-02-03 03:23:04 +01:00
assert ix = = i ;
2009-06-16 08:38:26 +02:00
// switch values:
// remove last entry from the file copy to fill it in the gap
2011-06-01 21:31:56 +02:00
final Row . Entry te = this . table . removeOne ( ) ;
2009-06-16 08:38:26 +02:00
// fill the gap in file copy
2009-12-10 00:27:26 +01:00
try {
2011-06-01 21:31:56 +02:00
this . table . set ( i , te ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2009-12-10 00:27:26 +01:00
Log . logException ( e ) ;
2011-06-01 21:31:56 +02:00
this . table = null ;
2009-12-10 00:27:26 +01:00
}
2008-01-17 13:12:52 +01:00
2009-06-16 08:38:26 +02:00
// move entry from last entry in file to gap position
2011-06-01 21:31:56 +02:00
this . file . cleanLast ( p , 0 ) ;
this . file . put ( i , p , 0 ) ;
2009-06-16 08:38:26 +02:00
// set new index for moved entry in index
2011-06-01 21:31:56 +02:00
final Row . Entry lr = this . rowdef . newEntry ( p ) ;
2009-12-10 00:27:26 +01:00
try {
2011-06-01 21:31:56 +02:00
this . index . put ( lr . getPrimaryKeyBytes ( ) , i ) ;
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2011-06-01 21:31:56 +02:00
this . table = null ;
2009-12-10 00:27:26 +01:00
throw new IOException ( " RowSpaceExceededException: " + e . getMessage ( ) ) ;
}
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . index . size ( ) + 1 = = sb : " index.size() = " + this . index . size ( ) + " , sb = " + sb ;
return this . rowdef . newEntry ( b ) ;
2008-01-17 13:12:52 +01:00
}
2012-07-02 15:40:40 +02:00
@Override
2008-01-17 13:12:52 +01:00
public synchronized Entry removeOne ( ) throws IOException {
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
final byte [ ] le = new byte [ this . rowdef . objectsize ] ;
final long fsb = this . file . size ( ) ;
2009-07-16 01:08:35 +02:00
assert fsb ! = 0 : " file.size() = " + fsb ;
2011-06-01 21:31:56 +02:00
this . file . cleanLast ( le , 0 ) ;
assert this . file . size ( ) < fsb : " file.size() = " + this . file . size ( ) ;
final Row . Entry lr = this . rowdef . newEntry ( le ) ;
2010-08-04 15:33:12 +02:00
assert lr ! = null ;
assert lr . getPrimaryKeyBytes ( ) ! = null ;
2011-06-01 21:31:56 +02:00
final int is = this . index . size ( ) ;
assert this . index . has ( lr . getPrimaryKeyBytes ( ) ) ;
final int i = ( int ) this . index . remove ( lr . getPrimaryKeyBytes ( ) ) ;
assert i < 0 | | this . index . size ( ) < is : " index.size() = " + this . index . size ( ) + " , is = " + is ;
2008-01-17 13:12:52 +01:00
assert i > = 0 ;
2011-06-01 21:31:56 +02:00
if ( this . table ! = null ) {
final int tsb = this . table . size ( ) ;
this . table . removeOne ( ) ;
assert this . table . size ( ) < tsb : " table.size() = " + this . table . size ( ) + " , tsb = " + tsb ;
2010-10-11 13:01:50 +02:00
}
2011-06-01 21:31:56 +02:00
assert this . file . size ( ) = = this . index . size ( ) : " file.size() = " + this . file . size ( ) + " , index.size() = " + this . index . size ( ) ;
assert this . table = = null | | this . table . size ( ) = = this . index . size ( ) : " table.size() = " + this . table . size ( ) + " , index.size() = " + this . index . size ( ) ;
2008-01-17 13:12:52 +01:00
return lr ;
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2010-05-31 02:27:00 +02:00
public List < Row . Entry > top ( int count ) throws IOException {
2011-06-01 21:31:56 +02:00
final ArrayList < Row . Entry > list = new ArrayList < Row . Entry > ( ) ;
if ( ( this . file = = null ) | | ( this . index = = null ) ) return list ;
long i = this . file . size ( ) - 1 ;
2010-05-31 02:27:00 +02:00
while ( count > 0 & & i > = 0 ) {
2011-06-01 21:31:56 +02:00
final byte [ ] b = new byte [ this . rowdef . objectsize ] ;
this . file . get ( i , b , 0 ) ;
list . add ( this . rowdef . newEntry ( b ) ) ;
2010-05-31 02:27:00 +02:00
i - - ;
count - - ;
}
return list ;
}
2008-01-17 13:12:52 +01:00
2012-07-02 15:40:40 +02:00
@Override
2009-09-17 22:41:50 +02:00
public synchronized void clear ( ) throws IOException {
2011-06-01 21:31:56 +02:00
final File f = this . file . filename ( ) ;
2011-01-04 00:06:33 +01:00
this . file . close ( ) ;
this . file = null ;
2009-03-30 17:31:25 +02:00
FileUtils . deletedelete ( f ) ;
2011-10-26 12:07:16 +02:00
tableTracker . remove ( f . getName ( ) ) ;
2011-06-01 21:31:56 +02:00
2008-01-17 13:12:52 +01:00
// make new file
FileOutputStream fos = null ;
try {
fos = new FileOutputStream ( f ) ;
2008-08-02 14:12:04 +02:00
} catch ( final FileNotFoundException e ) {
2008-01-17 13:12:52 +01:00
// should not happen
2009-09-29 23:52:17 +02:00
Log . logSevere ( " Table " , " " , e ) ;
2008-01-17 13:12:52 +01:00
}
2008-08-02 14:12:04 +02:00
if ( fos ! = null ) try { fos . close ( ) ; } catch ( final IOException e ) { }
2011-06-01 21:31:56 +02:00
this . file = new BufferedRecords ( new Records ( f , this . rowdef . objectsize ) , this . buffersize ) ;
2008-01-17 13:12:52 +01:00
// initialize index and copy table
2011-06-01 21:31:56 +02:00
this . table = ( this . table = = null ) ? null : new RowSet ( this . taildef ) ;
this . index . clear ( ) ;
2008-01-17 13:12:52 +01:00
}
2012-07-02 15:40:40 +02:00
@Override
2009-01-30 16:33:00 +01:00
public Row row ( ) {
2008-01-17 13:12:52 +01:00
return this . rowdef ;
}
2012-07-02 15:40:40 +02:00
@Override
2010-04-19 18:42:37 +02:00
public int size ( ) {
2011-11-08 12:49:04 +01:00
if ( this . index = = null ) return 0 ;
2011-06-01 21:31:56 +02:00
return this . index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2010-04-19 18:42:37 +02:00
public boolean isEmpty ( ) {
2011-06-01 21:31:56 +02:00
return this . index . isEmpty ( ) ;
2009-12-02 01:37:59 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2009-10-12 10:06:35 +02:00
public Iterator < Entry > iterator ( ) {
try {
return rows ( ) ;
2011-06-01 21:31:56 +02:00
} catch ( final IOException e ) {
2009-10-12 10:06:35 +02:00
return null ;
}
}
2008-01-17 13:12:52 +01:00
2012-07-02 15:40:40 +02:00
@Override
2009-02-24 11:40:20 +01:00
public synchronized CloneableIterator < Entry > rows ( ) throws IOException {
2009-10-12 10:06:35 +02:00
this . file . flushBuffer ( ) ;
2009-02-24 11:40:20 +01:00
return new rowIteratorNoOrder ( ) ;
}
2010-04-21 13:29:27 +02:00
private class rowIteratorNoOrder implements CloneableIterator < Entry > {
2012-07-27 12:13:53 +02:00
Iterator < Map . Entry < byte [ ] , Long > > i ;
long idx ;
2010-12-28 17:44:55 +01:00
byte [ ] key ;
2011-06-01 21:31:56 +02:00
2011-01-03 21:52:54 +01:00
public rowIteratorNoOrder ( ) {
2010-12-28 17:44:55 +01:00
// don't use the ChunkIterator here because it may create too many open files during string load
//ri = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.objectsize);
2011-06-01 21:31:56 +02:00
this . i = Table . this . index . iterator ( ) ;
2009-02-24 11:40:20 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2011-06-01 21:31:56 +02:00
public CloneableIterator < Entry > clone ( final Object modifier ) {
2011-01-03 21:52:54 +01:00
return new rowIteratorNoOrder ( ) ;
2009-02-24 11:40:20 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2009-02-24 11:40:20 +01:00
public boolean hasNext ( ) {
2011-06-01 21:31:56 +02:00
return this . i ! = null & & this . i . hasNext ( ) ;
2009-02-24 11:40:20 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2009-02-24 11:40:20 +01:00
public Entry next ( ) {
2012-07-27 12:13:53 +02:00
final Map . Entry < byte [ ] , Long > entry = this . i . next ( ) ;
2010-12-28 17:44:55 +01:00
if ( entry = = null ) return null ;
2012-07-27 12:13:53 +02:00
this . key = entry . getKey ( ) ;
2011-06-01 21:31:56 +02:00
if ( this . key = = null ) return null ;
2012-07-27 12:13:53 +02:00
this . idx = entry . getValue ( ) . longValue ( ) ;
2010-12-28 17:44:55 +01:00
try {
2011-07-15 10:38:10 +02:00
return get ( this . key , false ) ;
2011-06-01 21:31:56 +02:00
} catch ( final IOException e ) {
2010-12-28 17:44:55 +01:00
return null ;
}
2009-02-24 11:40:20 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2009-02-24 11:40:20 +01:00
public void remove ( ) {
2011-06-01 21:31:56 +02:00
if ( this . key ! = null ) {
2010-12-28 17:44:55 +01:00
try {
2012-07-27 12:13:53 +02:00
removeInFile ( ( int ) this . idx ) ;
2011-06-01 21:31:56 +02:00
} catch ( final IOException e ) {
2012-07-27 12:13:53 +02:00
} catch ( final SpaceExceededException e ) {
2010-12-28 17:44:55 +01:00
}
2011-06-01 21:31:56 +02:00
this . i . remove ( ) ;
2010-12-28 17:44:55 +01:00
}
2009-02-24 11:40:20 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
public void close ( ) {
if ( this . i instanceof CloneableIterator ) {
2012-07-27 12:13:53 +02:00
( ( CloneableIterator < Map . Entry < byte [ ] , Long > > ) this . i ) . close ( ) ;
2012-07-02 15:40:40 +02:00
}
}
2009-02-24 11:40:20 +01:00
}
2008-01-17 13:12:52 +01:00
2012-07-02 15:40:40 +02:00
@Override
2009-01-30 23:08:08 +01:00
public synchronized CloneableIterator < Entry > rows ( final boolean up , final byte [ ] firstKey ) throws IOException {
2008-01-17 13:12:52 +01:00
return new rowIterator ( up , firstKey ) ;
}
2010-04-21 13:29:27 +02:00
private class rowIterator implements CloneableIterator < Entry > {
2012-07-02 15:40:40 +02:00
private final CloneableIterator < byte [ ] > i ;
2011-06-01 21:31:56 +02:00
private final boolean up ;
private final byte [ ] fk ;
2010-04-21 13:29:27 +02:00
private int c ;
2011-06-01 21:31:56 +02:00
2010-04-21 13:29:27 +02:00
private rowIterator ( final boolean up , final byte [ ] firstKey ) {
2008-01-17 13:12:52 +01:00
this . up = up ;
this . fk = firstKey ;
2011-06-01 21:31:56 +02:00
this . i = Table . this . index . keys ( up , firstKey ) ;
2008-01-17 13:12:52 +01:00
this . c = - 1 ;
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
2009-01-30 23:08:08 +01:00
public CloneableIterator < Entry > clone ( final Object modifier ) {
2011-06-01 21:31:56 +02:00
return new rowIterator ( this . up , this . fk ) ;
2008-01-17 13:12:52 +01:00
}
2012-07-02 15:40:40 +02:00
@Override
2008-01-17 13:12:52 +01:00
public boolean hasNext ( ) {
2011-06-01 21:31:56 +02:00
return this . i . hasNext ( ) ;
2008-01-17 13:12:52 +01:00
}
2012-07-02 15:40:40 +02:00
@Override
2008-01-17 13:12:52 +01:00
public Entry next ( ) {
2011-06-01 21:31:56 +02:00
final byte [ ] k = this . i . next ( ) ;
2008-04-14 05:20:37 +02:00
assert k ! = null ;
if ( k = = null ) return null ;
2011-06-01 21:31:56 +02:00
this . c = ( int ) Table . this . index . get ( k ) ;
2008-10-03 20:57:02 +02:00
if ( this . c < 0 ) throw new ConcurrentModificationException ( ) ; // this should only happen if the table was modified during the iteration
2011-06-01 21:31:56 +02:00
final byte [ ] b = new byte [ Table . this . rowdef . objectsize ] ;
2011-06-22 01:10:50 +02:00
final Row . Entry cacherow ;
if ( Table . this . table = = null | | ( cacherow = Table . this . table . get ( this . c , false ) ) = = null ) {
2008-01-17 13:12:52 +01:00
// read from file
try {
2011-06-01 21:31:56 +02:00
Table . this . file . get ( this . c , b , 0 ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2009-09-29 23:52:17 +02:00
Log . logSevere ( " Table " , " " , e ) ;
2008-01-17 13:12:52 +01:00
return null ;
}
} else {
// compose from table and key
2011-06-01 21:31:56 +02:00
System . arraycopy ( k , 0 , b , 0 , Table . this . rowdef . primaryKeyLength ) ;
2011-06-22 01:10:50 +02:00
System . arraycopy ( cacherow . bytes ( ) , 0 , b , Table . this . rowdef . primaryKeyLength , Table . this . taildef . objectsize ) ;
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
return Table . this . rowdef . newEntry ( b ) ;
2008-01-17 13:12:52 +01:00
}
2012-07-02 15:40:40 +02:00
@Override
2008-01-17 13:12:52 +01:00
public void remove ( ) {
2010-05-19 14:53:09 +02:00
throw new UnsupportedOperationException ( " no remove in Table.rowIterator " ) ;
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
2012-07-02 15:40:40 +02:00
@Override
public void close ( ) {
this . i . close ( ) ;
}
2008-01-17 13:12:52 +01:00
}
2011-06-01 21:31:56 +02:00
2009-06-28 23:02:56 +02:00
private static byte [ ] testWord ( final char c ) {
return new byte [ ] { ( byte ) c , 32 , 32 , 32 } ;
}
private static String [ ] permutations ( final int letters ) {
String p = " " ;
for ( int i = 0 ; i < letters ; i + + ) p = p + ( ( char ) ( ( 'A' ) + i ) ) ;
return permutations ( p ) ;
}
2011-06-01 21:31:56 +02:00
2009-06-28 23:02:56 +02:00
private static String [ ] permutations ( final String source ) {
2012-07-10 22:59:03 +02:00
if ( source . isEmpty ( ) ) return new String [ 0 ] ;
2009-06-28 23:02:56 +02:00
if ( source . length ( ) = = 1 ) return new String [ ] { source } ;
final char c = source . charAt ( 0 ) ;
final String [ ] recres = permutations ( source . substring ( 1 ) ) ;
final String [ ] result = new String [ source . length ( ) * recres . length ] ;
for ( int perm = 0 ; perm < recres . length ; perm + + ) {
result [ perm * source . length ( ) ] = c + recres [ perm ] ;
for ( int pos = 1 ; pos < source . length ( ) - 1 ; pos + + ) {
result [ perm * source . length ( ) + pos ] = recres [ perm ] . substring ( 0 , pos ) + c + recres [ perm ] . substring ( pos ) ;
}
result [ perm * source . length ( ) + source . length ( ) - 1 ] = recres [ perm ] + c ;
}
return result ;
}
2012-07-27 12:13:53 +02:00
private static Table testTable ( final File f , final String testentities , final boolean useTailCache , final boolean exceed134217727 ) throws IOException , SpaceExceededException {
2009-03-30 17:31:25 +02:00
if ( f . exists ( ) ) FileUtils . deletedelete ( f ) ;
2009-03-13 17:52:31 +01:00
final Row rowdef = new Row ( " byte[] a-4, byte[] b-4 " , NaturalOrder . naturalOrder ) ;
2011-09-15 12:01:21 +02:00
final Table tt = new Table ( f , rowdef , 100 , 0 , useTailCache , exceed134217727 , true ) ;
2008-01-17 13:12:52 +01:00
byte [ ] b ;
2009-01-30 16:33:00 +01:00
final Row . Entry row = rowdef . newEntry ( ) ;
2008-01-17 13:12:52 +01:00
for ( int i = 0 ; i < testentities . length ( ) ; i + + ) {
2009-06-28 23:02:56 +02:00
b = testWord ( testentities . charAt ( i ) ) ;
2008-01-17 13:12:52 +01:00
row . setCol ( 0 , b ) ;
row . setCol ( 1 , b ) ;
tt . put ( row ) ;
}
return tt ;
}
2011-06-01 21:31:56 +02:00
2009-10-12 10:06:35 +02:00
private static int countElements ( final Table t ) {
2009-06-28 23:02:56 +02:00
int count = 0 ;
2011-06-01 21:31:56 +02:00
for ( final Row . Entry row : t ) {
2009-10-12 10:06:35 +02:00
count + + ;
if ( row = = null ) System . out . println ( " ERROR! null element found " ) ;
2011-06-01 21:31:56 +02:00
// else System.out.println("counted element: " + new
// String(n.getKey()));
2009-06-28 23:02:56 +02:00
}
return count ;
}
2011-06-01 21:31:56 +02:00
2009-09-07 22:30:57 +02:00
public static void bigtest ( final int elements , final File testFile , final boolean useTailCache , final boolean exceed134217727 ) {
2008-01-17 13:12:52 +01:00
System . out . println ( " starting big test with " + elements + " elements: " ) ;
2008-08-02 14:12:04 +02:00
final long start = System . currentTimeMillis ( ) ;
2009-06-28 23:02:56 +02:00
final String [ ] s = permutations ( elements ) ;
2009-10-12 10:06:35 +02:00
Table tt ;
int count ;
2008-01-17 13:12:52 +01:00
try {
for ( int i = 0 ; i < s . length ; i + + ) {
System . out . println ( " *** probing tree " + i + " for permutation " + s [ i ] ) ;
// generate tree and delete elements
2009-09-07 22:30:57 +02:00
tt = testTable ( testFile , s [ i ] , useTailCache , exceed134217727 ) ;
2009-10-12 10:06:35 +02:00
count = countElements ( tt ) ;
if ( count ! = tt . size ( ) ) {
System . out . println ( " wrong size for " + s [ i ] + " : count = " + count + " , size() = " + tt . size ( ) ) ;
2008-01-17 13:12:52 +01:00
}
tt . close ( ) ;
2011-06-01 21:31:56 +02:00
for ( final String element : s ) {
2009-09-07 22:30:57 +02:00
tt = testTable ( testFile , s [ i ] , useTailCache , exceed134217727 ) ;
2008-01-17 13:12:52 +01:00
// delete by permutation j
2011-06-01 21:31:56 +02:00
for ( int elt = 0 ; elt < element . length ( ) ; elt + + ) {
tt . remove ( testWord ( element . charAt ( elt ) ) ) ;
2009-10-12 10:06:35 +02:00
count = countElements ( tt ) ;
if ( count ! = tt . size ( ) ) {
2011-06-01 21:31:56 +02:00
System . out . println ( " ERROR! wrong size for probe tree " + s [ i ] + " ; probe delete " + element + " ; position " + elt + " ; count = " + count + " , size() = " + tt . size ( ) ) ;
2008-01-17 13:12:52 +01:00
}
}
tt . close ( ) ;
}
}
System . out . println ( " FINISHED test after " + ( ( System . currentTimeMillis ( ) - start ) / 1000 ) + " seconds. " ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-01-17 13:12:52 +01:00
System . out . println ( " TERMINATED " ) ;
}
}
2011-06-01 21:31:56 +02:00
2008-12-18 01:08:17 +01:00
public void print ( ) throws IOException {
System . out . println ( " PRINTOUT of table, length= " + size ( ) ) ;
Entry row ;
byte [ ] key ;
2011-06-01 21:31:56 +02:00
final CloneableIterator < byte [ ] > i = keys ( true , null ) ;
2008-12-18 01:08:17 +01:00
while ( i . hasNext ( ) ) {
System . out . print ( " row " + i + " : " ) ;
key = i . next ( ) ;
2011-07-15 10:38:10 +02:00
row = get ( key , false ) ;
2008-12-18 01:08:17 +01:00
System . out . println ( row . toString ( ) ) ;
}
System . out . println ( " EndOfTable " ) ;
}
2008-08-02 14:12:04 +02:00
public static void main ( final String [ ] args ) {
2008-01-17 13:12:52 +01:00
// open a file, add one entry and exit
2008-08-02 14:12:04 +02:00
final File f = new File ( args [ 0 ] ) ;
2008-01-19 13:23:56 +01:00
System . out . println ( " ========= Testcase: no tail cache: " ) ;
2009-09-07 22:30:57 +02:00
bigtest ( 5 , f , false , false ) ;
2008-01-19 13:23:56 +01:00
System . out . println ( " ========= Testcase: with tail cache: " ) ;
2009-09-07 22:30:57 +02:00
bigtest ( 5 , f , true , true ) ;
2008-01-17 13:12:52 +01:00
/ *
kelondroRow row = new kelondroRow ( " byte[] key-4, byte[] x-5 " , kelondroNaturalOrder . naturalOrder , 0 ) ;
try {
2009-06-28 23:37:37 +02:00
kelondroTABLE t = new kelondroTABLE ( f , row ) ;
2008-01-17 13:12:52 +01:00
kelondroRow . Entry entry = row . newEntry ( ) ;
entry . setCol ( 0 , " abcd " . getBytes ( ) ) ;
entry . setCol ( 1 , " dummy " . getBytes ( ) ) ;
t . put ( entry ) ;
t . close ( ) ;
} catch ( IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-01-17 13:12:52 +01:00
}
* /
}
2012-07-02 15:40:40 +02:00
@Override
2008-09-12 13:51:48 +02:00
public void deleteOnExit ( ) {
this . file . deleteOnExit ( ) ;
}
2008-01-17 13:12:52 +01:00
}