2008-01-17 13:12:52 +01:00
// kelondroEcoIndex.java
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 14.01.2008 on http://yacy.net
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro ;
import java.io.File ;
import java.io.FileNotFoundException ;
import java.io.FileOutputStream ;
import java.io.IOException ;
2008-01-20 02:22:46 +01:00
import java.util.ArrayList ;
2008-01-17 13:12:52 +01:00
import java.util.Date ;
2008-01-17 22:48:08 +01:00
import java.util.HashMap ;
2008-01-17 13:12:52 +01:00
import java.util.Iterator ;
import java.util.List ;
2008-01-17 22:48:08 +01:00
import java.util.Map ;
2008-01-17 13:12:52 +01:00
import java.util.TreeMap ;
2008-01-20 02:22:46 +01:00
import java.util.TreeSet ;
2008-01-17 13:12:52 +01:00
import de.anomic.kelondro.kelondroRow.Entry ;
import de.anomic.server.serverMemory ;
/ *
* The EcoIndex builts upon the EcoFS and tries to reduce the number of IO requests that the
* EcoFS must do to a minimum . In best cases , no IO has to be done for read operations ( complete database shadow in RAM )
* and a rare number of write IO operations must be done for a large number of table - writings ( using the write buffer of EcoFS )
* To make the EcoIndex scalable in question of available RAM , there are two elements that must be scalable :
* - the access index can be either completely in RAM ( kelondroRAMIndex ) or it is file - based ( kelondroTree )
* - the content cache can be either a complete RAM - based shadow of the File , or empty .
* The content cache can also be deleted during run - time , if the available RAM gets too low .
* /
public class kelondroEcoTable implements kelondroIndex {
// static tracker objects
private static TreeMap < String , kelondroEcoTable > tableTracker = new TreeMap < String , kelondroEcoTable > ( ) ;
2008-01-19 13:23:56 +01:00
public static final int tailCacheDenyUsage = 0 ;
public static final int tailCacheForceUsage = 1 ;
public static final int tailCacheUsageAuto = 2 ;
2008-02-23 00:46:27 +01:00
public static final long maxarraylength = 134217727L ; // that may be the maxmimum size of array length in some JVMs
2008-01-21 00:46:27 +01:00
2008-03-04 23:45:45 +01:00
kelondroRowSet table ;
kelondroBytesIntMap index ;
kelondroBufferedEcoFS file ;
kelondroRow rowdef ;
kelondroRow taildef ;
2008-01-17 13:12:52 +01:00
private int buffersize ;
2008-01-20 22:42:35 +01:00
public kelondroEcoTable ( File tablefile , kelondroRow rowdef , int useTailCache , int buffersize , int initialSpace ) {
2008-01-17 13:12:52 +01:00
this . rowdef = rowdef ;
this . buffersize = buffersize ;
assert rowdef . primaryKeyIndex = = 0 ;
// define the taildef, a row like the rowdef but without the first column
kelondroColumn [ ] cols = new kelondroColumn [ rowdef . columns ( ) - 1 ] ;
for ( int i = 0 ; i < cols . length ; i + + ) {
cols [ i ] = rowdef . column ( i + 1 ) ;
}
2008-01-22 21:02:20 +01:00
this . taildef = new kelondroRow ( cols , kelondroNaturalOrder . naturalOrder , - 1 ) ;
2008-01-17 13:12:52 +01:00
// initialize table file
if ( ! tablefile . exists ( ) ) {
// make new file
FileOutputStream fos = null ;
try {
fos = new FileOutputStream ( tablefile ) ;
} catch ( FileNotFoundException e ) {
// should not happen
e . printStackTrace ( ) ;
}
2008-03-04 23:45:45 +01:00
if ( fos ! = null ) try { fos . close ( ) ; } catch ( IOException e ) { }
2008-01-17 13:12:52 +01:00
}
try {
2008-01-17 22:48:08 +01:00
// open an existing table file
2008-01-17 13:12:52 +01:00
this . file = new kelondroBufferedEcoFS ( new kelondroEcoFS ( tablefile , rowdef . objectsize ) , this . buffersize ) ;
2008-01-17 22:48:08 +01:00
// initialize index and copy table
2008-01-21 00:46:27 +01:00
int records = ( int ) Math . max ( file . size ( ) , initialSpace ) ;
2008-02-23 00:46:27 +01:00
long neededRAM4table = ( ( long ) records ) * ( ( ( long ) rowdef . objectsize ) + 4L ) * 3L ;
2008-01-21 00:46:27 +01:00
table = ( ( neededRAM4table < maxarraylength ) & &
( ( useTailCache = = tailCacheForceUsage ) | |
2008-03-02 16:42:50 +01:00
( ( useTailCache = = tailCacheUsageAuto ) & & ( serverMemory . free ( ) > neededRAM4table + 200 * 1024 * 1024 ) ) ) ) ?
2008-01-20 22:42:35 +01:00
new kelondroRowSet ( taildef , records ) : null ;
2008-01-22 21:44:12 +01:00
System . out . println ( " *** DEBUG " + tablefile + " : available RAM: " + ( serverMemory . available ( ) / 1024 / 1024 ) + " MB, allocating space for " + records + " entries " ) ;
2008-02-04 11:33:45 +01:00
long neededRAM4index = 2 * 1024 * 1024 + records * ( rowdef . primaryKeyLength + 4 ) * 3 / 2 ;
2008-01-22 21:44:12 +01:00
if ( ! serverMemory . request ( neededRAM4index , false ) ) {
// despite calculations seemed to show that there is enough memory for the table AND the index
// there is now not enough memory left for the index. So delete the table again to free the memory
// for the index
System . out . println ( " *** DEBUG " + tablefile + " : not enough RAM ( " + ( serverMemory . available ( ) / 1024 / 1024 ) + " MB) left for index, deleting allocated table space to enable index space allocation (needed: " + ( neededRAM4index / 1024 / 1024 ) + " MB) " ) ;
table = null ; System . gc ( ) ;
System . out . println ( " *** DEBUG " + tablefile + " : RAM after releasing the table: " + ( serverMemory . available ( ) / 1024 / 1024 ) + " MB " ) ;
}
2008-01-20 22:42:35 +01:00
index = new kelondroBytesIntMap ( rowdef . primaryKeyLength , rowdef . objectOrder , records ) ;
2008-01-22 21:44:12 +01:00
System . out . println ( " *** DEBUG " + tablefile + " : EcoTable " + tablefile . toString ( ) + " has table copy " + ( ( table = = null ) ? " DISABLED " : " ENABLED " ) ) ;
2008-01-20 22:42:35 +01:00
2008-01-17 22:48:08 +01:00
// read all elements from the file into the copy table
byte [ ] record = new byte [ rowdef . objectsize ] ;
byte [ ] key = new byte [ rowdef . primaryKeyLength ] ;
2008-01-21 00:46:27 +01:00
int fs = ( int ) file . size ( ) ;
2008-01-25 12:44:27 +01:00
System . out . print ( " *** initializing RAM index for EcoTable " + tablefile . getName ( ) + " : " ) ;
2008-01-20 22:42:35 +01:00
for ( int i = 0 ; i < fs ; i + + ) {
2008-01-17 22:48:08 +01:00
// read entry
file . get ( i , record , 0 ) ;
2008-01-17 13:12:52 +01:00
2008-01-17 22:48:08 +01:00
// write the key into the index table
System . arraycopy ( record , 0 , key , 0 , rowdef . primaryKeyLength ) ;
index . addi ( key , i ) ;
2008-01-17 13:12:52 +01:00
2008-01-17 22:48:08 +01:00
// write the tail into the table
if ( table ! = null ) table . addUnique ( taildef . newEntry ( record , rowdef . primaryKeyLength , true ) ) ;
2008-01-24 23:49:00 +01:00
if ( ( i % 10000 ) = = 0 ) {
System . out . print ( '.' ) ;
System . out . flush ( ) ;
}
2008-01-17 22:48:08 +01:00
}
2008-01-24 23:49:00 +01:00
System . out . print ( " -ordering- .. " ) ;
System . out . flush ( ) ;
2008-01-20 22:42:35 +01:00
// check consistency
ArrayList < Integer [ ] > doubles = index . removeDoubles ( ) ;
2008-01-24 23:49:00 +01:00
System . out . println ( " -removed " + doubles . size ( ) + " doubles- done. " ) ;
2008-01-20 22:42:35 +01:00
if ( doubles . size ( ) > 0 ) {
2008-01-22 21:44:12 +01:00
System . out . println ( " DEBUG " + tablefile + " : WARNING - EcoTable " + tablefile + " has " + doubles . size ( ) + " doubles " ) ;
2008-01-22 22:40:25 +01:00
// from all the doubles take one, put it back to the index and remove the others from the file
Iterator < Integer [ ] > i = doubles . iterator ( ) ;
Integer [ ] ds ;
// first put back one element each
while ( i . hasNext ( ) ) {
ds = i . next ( ) ;
file . get ( ds [ 0 ] . longValue ( ) , record , 0 ) ;
System . arraycopy ( record , 0 , key , 0 , rowdef . primaryKeyLength ) ;
index . addi ( key , ds [ 0 ] . intValue ( ) ) ;
}
// then remove the other doubles by removing them from the table, but do a re-indexing while doing that
2008-01-23 12:13:39 +01:00
// first aggregate all the delete positions because the elements from the top positions must be removed first
2008-01-22 22:40:25 +01:00
i = doubles . iterator ( ) ;
2008-01-23 12:13:39 +01:00
TreeSet < Integer > delpos = new TreeSet < Integer > ( ) ;
2008-01-22 22:40:25 +01:00
while ( i . hasNext ( ) ) {
ds = i . next ( ) ;
for ( int j = 1 ; j < ds . length ; j + + ) {
2008-01-23 12:13:39 +01:00
delpos . add ( ds [ j ] ) ;
2008-01-22 22:40:25 +01:00
}
}
2008-01-23 12:13:39 +01:00
// now remove the entries in a sorted way (top-down)
Integer top ;
while ( delpos . size ( ) > 0 ) {
top = delpos . last ( ) ;
delpos . remove ( top ) ;
removeInFile ( top . intValue ( ) ) ;
}
2008-01-20 22:42:35 +01:00
}
2008-01-17 22:48:08 +01:00
} catch ( FileNotFoundException e ) {
// should never happen
e . printStackTrace ( ) ;
throw new kelondroException ( e . getMessage ( ) ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
throw new kelondroException ( e . getMessage ( ) ) ;
2008-01-17 13:12:52 +01:00
}
2008-01-19 13:23:56 +01:00
try {
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
}
2008-01-17 13:12:52 +01:00
// track this table
tableTracker . put ( tablefile . toString ( ) , this ) ;
}
public static long tableSize ( File tablefile , int recordsize ) {
// returns number of records in table
return kelondroEcoFS . tableSize ( tablefile , recordsize ) ;
}
2008-01-17 22:48:08 +01:00
public static final Iterator < String > filenames ( ) {
// iterates string objects; all file names from record tracker
return tableTracker . keySet ( ) . iterator ( ) ;
}
public static final Map < String , String > memoryStats ( String filename ) {
// returns a map for each file in the tracker;
// the map represents properties for each record objects,
// i.e. for cache memory allocation
kelondroEcoTable theEcoTable = tableTracker . get ( filename ) ;
return theEcoTable . memoryStats ( ) ;
}
private final Map < String , String > memoryStats ( ) {
// returns statistical data about this object
2008-01-20 02:51:02 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
2008-01-17 22:48:08 +01:00
HashMap < String , String > map = new HashMap < String , String > ( ) ;
2008-01-20 02:51:02 +01:00
map . put ( " tableSize " , Integer . toString ( index . size ( ) ) ) ;
map . put ( " tableKeyChunkSize " , Integer . toString ( index . row ( ) . objectsize ) ) ;
map . put ( " tableKeyMem " , Integer . toString ( ( int ) ( index . row ( ) . objectsize * index . size ( ) * kelondroRowCollection . growfactor ) ) ) ;
map . put ( " tableValueChunkSize " , ( table = = null ) ? " 0 " : Integer . toString ( table . row ( ) . objectsize ) ) ;
map . put ( " tableValueMem " , ( table = = null ) ? " 0 " : Integer . toString ( ( int ) ( table . row ( ) . objectsize * table . size ( ) * kelondroRowCollection . growfactor ) ) ) ;
2008-01-17 22:48:08 +01:00
return map ;
}
public static int staticRAMIndexNeed ( File f , kelondroRow rowdef ) {
2008-03-04 23:45:45 +01:00
return ( int ) ( ( rowdef . primaryKeyLength + 4 ) * tableSize ( f , rowdef . objectsize ) * kelondroRowCollection . growfactor ) ;
2008-01-17 22:48:08 +01:00
}
2008-01-17 13:12:52 +01:00
public synchronized void addUnique ( Entry row ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
2008-01-21 00:46:27 +01:00
int i = ( int ) file . size ( ) ;
2008-01-17 13:12:52 +01:00
index . addi ( row . getPrimaryKeyBytes ( ) , i ) ;
if ( table ! = null ) {
assert table . size ( ) = = i ;
table . addUnique ( taildef . newEntry ( row . bytes ( ) , rowdef . primaryKeyLength , true ) ) ;
}
file . put ( i , row . bytes ( ) , 0 ) ;
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
public synchronized void addUniqueMultiple ( List < Entry > rows ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
Iterator < Entry > i = rows . iterator ( ) ;
while ( i . hasNext ( ) ) {
addUnique ( i . next ( ) ) ;
}
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2008-01-20 02:22:46 +01:00
public synchronized ArrayList < kelondroRowSet > removeDoubles ( ) throws IOException {
ArrayList < Integer [ ] > indexreport = index . removeDoubles ( ) ;
ArrayList < kelondroRowSet > report = new ArrayList < kelondroRowSet > ( ) ;
Iterator < Integer [ ] > i = indexreport . iterator ( ) ;
Integer [ ] is ;
kelondroRowSet rows ;
TreeSet < Integer > d = new TreeSet < Integer > ( ) ;
byte [ ] b = new byte [ rowdef . objectsize ] ;
while ( i . hasNext ( ) ) {
is = i . next ( ) ;
rows = new kelondroRowSet ( this . rowdef , is . length ) ;
for ( int j = 0 ; j < is . length ; j + + ) {
d . add ( is [ j ] ) ;
file . get ( is [ j ] . intValue ( ) , b , 0 ) ;
rows . addUnique ( rowdef . newEntry ( b ) ) ;
}
report . add ( rows ) ;
}
// finally delete the affected rows, but start with largest id first, othervise we overwrite wrong entries
Integer s ;
while ( d . size ( ) > 0 ) {
s = d . last ( ) ;
d . remove ( s ) ;
this . removeInFile ( s . intValue ( ) ) ;
}
return report ;
}
2008-01-17 13:12:52 +01:00
public void close ( ) {
file . close ( ) ;
file = null ;
}
public void finalize ( ) {
if ( this . file ! = null ) this . close ( ) ;
}
public String filename ( ) {
return this . file . filename ( ) . toString ( ) ;
}
2008-01-20 02:22:46 +01:00
2008-01-17 13:12:52 +01:00
public synchronized Entry get ( byte [ ] key ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
int i = index . geti ( key ) ;
if ( i = = - 1 ) return null ;
byte [ ] b = new byte [ rowdef . objectsize ] ;
if ( table = = null ) {
// read row from the file
file . get ( i , b , 0 ) ;
} else {
// construct the row using the copy in RAM
kelondroRow . Entry v = table . get ( i ) ;
assert v ! = null ;
assert key . length = = rowdef . primaryKeyLength ;
System . arraycopy ( key , 0 , b , 0 , key . length ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , rowdef . objectsize - rowdef . primaryKeyLength ) ;
}
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
return rowdef . newEntry ( b ) ;
}
public synchronized boolean has ( byte [ ] key ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
return index . geti ( key ) > = 0 ;
}
public synchronized kelondroCloneableIterator < byte [ ] > keys ( boolean up , byte [ ] firstKey ) throws IOException {
return index . keys ( up , firstKey ) ;
}
public kelondroProfile profile ( ) {
return null ;
}
public synchronized Entry put ( Entry row ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
2008-01-25 12:44:27 +01:00
assert row ! = null ;
assert row . bytes ( ) ! = null ;
if ( ( row = = null ) | | ( row . bytes ( ) = = null ) ) return null ;
2008-01-17 13:12:52 +01:00
int i = index . geti ( row . getPrimaryKeyBytes ( ) ) ;
if ( i = = - 1 ) {
addUnique ( row ) ;
return null ;
}
byte [ ] b = new byte [ rowdef . objectsize ] ;
if ( table = = null ) {
// read old value
file . get ( i , b , 0 ) ;
// write new value
file . put ( i , row . bytes ( ) , 0 ) ;
} else {
// read old value
kelondroRow . Entry v = table . get ( i ) ;
2008-01-30 22:58:30 +01:00
assert v ! = null ;
2008-01-17 13:12:52 +01:00
System . arraycopy ( row . getPrimaryKeyBytes ( ) , 0 , b , 0 , rowdef . primaryKeyLength ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , rowdef . objectsize - rowdef . primaryKeyLength ) ;
// write new value
table . set ( i , taildef . newEntry ( row . bytes ( ) , rowdef . primaryKeyLength , true ) ) ;
file . put ( i , row . bytes ( ) , 0 ) ;
}
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
// return old value
return rowdef . newEntry ( b ) ;
}
public synchronized Entry put ( Entry row , Date entryDate ) throws IOException {
return put ( row ) ;
}
public synchronized void putMultiple ( List < Entry > rows ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
Iterator < Entry > i = rows . iterator ( ) ;
while ( i . hasNext ( ) ) {
put ( i . next ( ) ) ;
}
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2008-01-20 02:22:46 +01:00
private void removeInFile ( int i ) throws IOException {
assert i > = 0 ;
byte [ ] p = new byte [ rowdef . objectsize ] ;
if ( table = = null ) {
2008-01-22 22:40:25 +01:00
if ( i = = index . size ( ) - 1 ) {
2008-02-02 22:36:19 +01:00
file . cleanLast ( ) ;
2008-01-22 22:40:25 +01:00
} else {
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
byte [ ] k = new byte [ rowdef . primaryKeyLength ] ;
System . arraycopy ( p , 0 , k , 0 , rowdef . primaryKeyLength ) ;
index . puti ( k , i ) ;
}
2008-01-20 02:22:46 +01:00
} else {
if ( i = = index . size ( ) - 1 ) {
// special handling if the entry is the last entry in the file
table . removeRow ( i , false ) ;
2008-02-02 22:36:19 +01:00
file . cleanLast ( ) ;
2008-01-20 02:22:46 +01:00
} else {
// switch values
kelondroRow . Entry te = table . removeOne ( ) ;
table . set ( i , te ) ;
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
kelondroRow . Entry lr = rowdef . newEntry ( p ) ;
index . puti ( lr . getPrimaryKeyBytes ( ) , i ) ;
}
}
}
2008-01-17 13:12:52 +01:00
public synchronized Entry remove ( byte [ ] key , boolean keepOrder ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
assert keepOrder = = false ; // this class cannot keep the order during a remove
2008-01-24 23:49:00 +01:00
assert key . length = = rowdef . primaryKeyLength ;
2008-01-17 13:12:52 +01:00
int i = index . geti ( key ) ;
if ( i = = - 1 ) return null ; // nothing to do
// prepare result
byte [ ] b = new byte [ rowdef . objectsize ] ;
byte [ ] p = new byte [ rowdef . objectsize ] ;
2008-01-24 23:49:00 +01:00
int sb = index . size ( ) ;
2008-02-02 22:36:19 +01:00
int ix ;
assert i < index . size ( ) ;
2008-01-17 13:12:52 +01:00
if ( table = = null ) {
2008-01-24 23:49:00 +01:00
if ( i = = index . size ( ) - 1 ) {
2008-02-02 22:36:19 +01:00
ix = index . removei ( key ) ;
assert ix = = i ;
file . cleanLast ( b , 0 ) ;
2008-01-24 23:49:00 +01:00
} else {
2008-02-02 22:36:19 +01:00
assert i < index . size ( ) - 1 ;
ix = index . removei ( key ) ;
assert ix = = i ;
2008-01-24 23:49:00 +01:00
file . get ( i , b , 0 ) ;
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
byte [ ] k = new byte [ rowdef . primaryKeyLength ] ;
System . arraycopy ( p , 0 , k , 0 , rowdef . primaryKeyLength ) ;
index . puti ( k , i ) ;
}
2008-01-17 13:12:52 +01:00
assert ( file . size ( ) = = index . size ( ) ) ;
} else {
2008-01-24 23:49:00 +01:00
// get result value from the table copy, so we don't need to read it from the file
2008-01-17 13:12:52 +01:00
kelondroRow . Entry v = table . get ( i ) ;
System . arraycopy ( key , 0 , b , 0 , key . length ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , taildef . objectsize ) ;
2008-01-24 23:49:00 +01:00
2008-01-17 13:12:52 +01:00
if ( i = = index . size ( ) - 1 ) {
// special handling if the entry is the last entry in the file
2008-02-03 03:23:04 +01:00
ix = index . removei ( key ) ;
assert ix = = i ;
2008-01-17 13:12:52 +01:00
table . removeRow ( i , false ) ;
2008-02-02 22:36:19 +01:00
file . cleanLast ( ) ;
2008-01-17 13:12:52 +01:00
} else {
// switch values
2008-02-03 03:23:04 +01:00
ix = index . removei ( key ) ;
assert ix = = i ;
2008-01-24 23:49:00 +01:00
2008-01-17 13:12:52 +01:00
kelondroRow . Entry te = table . removeOne ( ) ;
table . set ( i , te ) ;
file . cleanLast ( p , 0 ) ;
file . put ( i , p , 0 ) ;
kelondroRow . Entry lr = rowdef . newEntry ( p ) ;
index . puti ( lr . getPrimaryKeyBytes ( ) , i ) ;
}
2008-01-24 23:49:00 +01:00
assert ( file . size ( ) = = index . size ( ) ) ;
assert ( table . size ( ) = = index . size ( ) ) : " table.size() = " + table . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
}
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
2008-01-24 23:49:00 +01:00
assert index . size ( ) + 1 = = sb : " index.size() = " + index . size ( ) + " , sb = " + sb ;
2008-01-17 13:12:52 +01:00
return rowdef . newEntry ( b ) ;
}
public synchronized Entry removeOne ( ) throws IOException {
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
assert ( ( table = = null ) | | ( table . size ( ) = = index . size ( ) ) ) ;
byte [ ] le = new byte [ rowdef . objectsize ] ;
file . cleanLast ( le , 0 ) ;
kelondroRow . Entry lr = rowdef . newEntry ( le ) ;
int i = index . removei ( lr . getPrimaryKeyBytes ( ) ) ;
assert i > = 0 ;
2008-01-24 23:49:00 +01:00
if ( table ! = null ) table . removeOne ( ) ;
2008-01-19 13:23:56 +01:00
assert file . size ( ) = = index . size ( ) : " file.size() = " + file . size ( ) + " , index.size() = " + index . size ( ) ;
2008-01-17 13:12:52 +01:00
return lr ;
}
public void reset ( ) throws IOException {
File f = file . filename ( ) ;
file . close ( ) ;
f . delete ( ) ;
// make new file
FileOutputStream fos = null ;
try {
fos = new FileOutputStream ( f ) ;
} catch ( FileNotFoundException e ) {
// should not happen
e . printStackTrace ( ) ;
}
2008-03-04 23:45:45 +01:00
if ( fos ! = null ) try { fos . close ( ) ; } catch ( IOException e ) { }
2008-01-17 13:12:52 +01:00
// open an existing table file
try {
this . file = new kelondroBufferedEcoFS ( new kelondroEcoFS ( f , rowdef . objectsize ) , this . buffersize ) ;
} catch ( FileNotFoundException e ) {
// should never happen
e . printStackTrace ( ) ;
}
// initialize index and copy table
table = new kelondroRowSet ( taildef , 1 ) ;
index = new kelondroBytesIntMap ( rowdef . primaryKeyLength , rowdef . objectOrder , 1 ) ;
}
public kelondroRow row ( ) {
return this . rowdef ;
}
public synchronized int size ( ) {
return index . size ( ) ;
}
public synchronized kelondroCloneableIterator < Entry > rows ( boolean up , byte [ ] firstKey ) throws IOException {
return new rowIterator ( up , firstKey ) ;
}
public class rowIterator implements kelondroCloneableIterator < Entry > {
Iterator < byte [ ] > i ;
boolean up ;
byte [ ] fk ;
int c ;
public rowIterator ( boolean up , byte [ ] firstKey ) throws IOException {
this . up = up ;
this . fk = firstKey ;
this . i = index . keys ( up , firstKey ) ;
this . c = - 1 ;
}
public kelondroCloneableIterator < Entry > clone ( Object modifier ) {
try {
return new rowIterator ( up , fk ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
return null ;
}
}
public boolean hasNext ( ) {
return i . hasNext ( ) ;
}
public Entry next ( ) {
byte [ ] k = i . next ( ) ;
try {
this . c = index . geti ( k ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
return null ;
}
byte [ ] b = new byte [ rowdef . objectsize ] ;
if ( table = = null ) {
// read from file
try {
file . get ( this . c , b , 0 ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
return null ;
}
} else {
// compose from table and key
kelondroRow . Entry v = table . get ( this . c ) ;
System . arraycopy ( k , 0 , b , 0 , rowdef . primaryKeyLength ) ;
System . arraycopy ( v . bytes ( ) , 0 , b , rowdef . primaryKeyLength , taildef . objectsize ) ;
}
return rowdef . newEntry ( b ) ;
}
public void remove ( ) {
throw new UnsupportedOperationException ( " no remove in EcoTable " ) ;
}
}
2008-01-19 13:23:56 +01:00
public static kelondroIndex testTable ( File f , String testentities , int testcase ) throws IOException {
2008-01-17 13:12:52 +01:00
if ( f . exists ( ) ) f . delete ( ) ;
kelondroRow rowdef = new kelondroRow ( " byte[] a-4, byte[] b-4 " , kelondroNaturalOrder . naturalOrder , 0 ) ;
2008-01-20 22:42:35 +01:00
kelondroIndex tt = new kelondroEcoTable ( f , rowdef , testcase , 100 , 0 ) ;
2008-01-17 13:12:52 +01:00
byte [ ] b ;
kelondroRow . Entry row = rowdef . newEntry ( ) ;
for ( int i = 0 ; i < testentities . length ( ) ; i + + ) {
b = kelondroTree . testWord ( testentities . charAt ( i ) ) ;
row . setCol ( 0 , b ) ;
row . setCol ( 1 , b ) ;
tt . put ( row ) ;
}
return tt ;
}
2008-01-19 13:23:56 +01:00
public static void bigtest ( int elements , File testFile , int testcase ) {
2008-01-17 13:12:52 +01:00
System . out . println ( " starting big test with " + elements + " elements: " ) ;
long start = System . currentTimeMillis ( ) ;
String [ ] s = kelondroTree . permutations ( elements ) ;
kelondroIndex tt ;
try {
for ( int i = 0 ; i < s . length ; i + + ) {
System . out . println ( " *** probing tree " + i + " for permutation " + s [ i ] ) ;
// generate tree and delete elements
2008-01-19 13:23:56 +01:00
tt = testTable ( testFile , s [ i ] , testcase ) ;
2008-01-17 13:12:52 +01:00
if ( kelondroTree . countElements ( tt ) ! = tt . size ( ) ) {
System . out . println ( " wrong size for " + s [ i ] ) ;
}
tt . close ( ) ;
for ( int j = 0 ; j < s . length ; j + + ) {
2008-01-19 13:23:56 +01:00
tt = testTable ( testFile , s [ i ] , testcase ) ;
2008-01-17 13:12:52 +01:00
// delete by permutation j
for ( int elt = 0 ; elt < s [ j ] . length ( ) ; elt + + ) {
tt . remove ( kelondroTree . testWord ( s [ j ] . charAt ( elt ) ) , false ) ;
if ( kelondroTree . countElements ( tt ) ! = tt . size ( ) ) {
System . out . println ( " ERROR! wrong size for probe tree " + s [ i ] + " ; probe delete " + s [ j ] + " ; position " + elt ) ;
}
}
tt . close ( ) ;
}
}
System . out . println ( " FINISHED test after " + ( ( System . currentTimeMillis ( ) - start ) / 1000 ) + " seconds. " ) ;
} catch ( Exception e ) {
e . printStackTrace ( ) ;
System . out . println ( " TERMINATED " ) ;
}
}
public static void main ( String [ ] args ) {
// open a file, add one entry and exit
File f = new File ( args [ 0 ] ) ;
2008-01-19 13:23:56 +01:00
System . out . println ( " ========= Testcase: no tail cache: " ) ;
bigtest ( 5 , f , tailCacheDenyUsage ) ;
System . out . println ( " ========= Testcase: with tail cache: " ) ;
bigtest ( 5 , f , tailCacheForceUsage ) ;
2008-01-17 13:12:52 +01:00
/ *
kelondroRow row = new kelondroRow ( " byte[] key-4, byte[] x-5 " , kelondroNaturalOrder . naturalOrder , 0 ) ;
try {
kelondroEcoTable t = new kelondroEcoTable ( f , row ) ;
kelondroRow . Entry entry = row . newEntry ( ) ;
entry . setCol ( 0 , " abcd " . getBytes ( ) ) ;
entry . setCol ( 1 , " dummy " . getBytes ( ) ) ;
t . put ( entry ) ;
t . close ( ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
}
* /
}
}