2006-03-01 01:25:02 +01:00
package de.anomic.kelondro ;
2006-08-05 01:04:03 +02:00
// a collectionIndex is an index to kelondroRowCollection objects
2006-03-01 01:25:02 +01:00
// such a collection ist defined by the following parameters
// - chunksize
// - chunkcount
// each of such a collection is stored in a byte[] which may or may not have space for more chunks
// than already exists in such an array. To store these arrays, we reserve entries in kelondroArray
// database files. There will be a set of array files for different sizes of the collection arrays.
// the 1st file has space for <loadfactor> chunks, the 2nd file for <loadfactor> * <loadfactor> chunks,
// the 3rd file for <loadfactor>^^3 chunks, and the n-th file for <loadfactor>^^n chunks.
// if the loadfactor is 4, then we have the following capacities:
// file 0: 4
// file 1: 16
// file 2: 64
// file 3: 256
// file 4: 1024
// file 5: 4096
// file 6:16384
// file 7:65536
// the maximum number of such files is called the partitions number.
// we don't want that these files grow too big, an kelondroOutOfLimitsException is throws if they
// are oversized.
// the collection arrays may be migration to another size during run-time, which means that not only the
// partitions as mentioned above are maintained, but also a set of "shadow-partitions", that represent old
// partitions and where data is read only and slowly migrated to the default partitions.
import java.io.File ;
import java.io.IOException ;
2006-08-05 01:04:03 +02:00
import java.util.HashMap ;
import java.util.Iterator ;
import java.util.Map ;
2006-08-05 21:18:33 +02:00
import java.util.Set ;
2006-03-01 01:25:02 +01:00
2006-08-11 05:20:44 +02:00
import de.anomic.server.serverFileUtils ;
2006-10-06 01:47:08 +02:00
import de.anomic.server.logging.serverLog ;
2006-08-11 05:20:44 +02:00
2006-03-01 01:25:02 +01:00
public class kelondroCollectionIndex {
2006-08-16 21:49:31 +02:00
protected kelondroIndex index ;
2006-10-06 01:47:08 +02:00
int keylength ;
2006-08-05 01:04:03 +02:00
private File path ;
private String filenameStub ;
private int loadfactor ;
private Map arrays ; // Map of (partitionNumber"-"chunksize)/kelondroFixedWidthArray - Objects
2006-08-16 21:49:31 +02:00
private kelondroRow playloadrow ; // definition of the payload (chunks inside the collections)
2006-08-05 01:04:03 +02:00
// private int partitions; // this is the maxmimum number of array files; yet not used
private static final int idx_col_key = 0 ; // the index
private static final int idx_col_chunksize = 1 ; // chunksize (number of bytes in a single chunk, needed for migration option)
private static final int idx_col_chunkcount = 2 ; // chunkcount (number of chunks in this collection) needed to identify array file that has the chunks
2006-10-06 01:47:08 +02:00
private static final int idx_col_clusteridx = 3 ; // selector for right cluster file, must be >= arrayIndex(chunkcount)
private static final int idx_col_flags = 4 ; // flags (for future use)
private static final int idx_col_indexpos = 5 ; // indexpos (position in index file)
private static final int idx_col_lastread = 6 ; // a time stamp, update time in days since 1.1.2000
private static final int idx_col_lastwrote = 7 ; // a time stamp, update time in days since 1.1.2000
2006-08-05 01:04:03 +02:00
2006-10-06 01:47:08 +02:00
private kelondroRow indexRow ( ) {
2006-08-05 01:04:03 +02:00
return new kelondroRow (
2006-10-06 01:47:08 +02:00
" byte[] key- " + keylength + " , " +
2006-08-05 01:04:03 +02:00
" int chunksize-4 {b256}, " +
" int chunkcount-4 {b256}, " +
2006-10-06 01:47:08 +02:00
" byte clusteridx-1 {b256}, " +
" byte flags-1 {b256}, " +
2006-08-05 01:04:03 +02:00
" int indexpos-4 {b256}, " +
2006-08-06 00:22:14 +02:00
" short lastread-2 {b256}, " +
2006-08-05 21:18:33 +02:00
" short lastwrote-2 {b256} "
2006-08-05 01:04:03 +02:00
) ;
}
2006-03-01 01:25:02 +01:00
2006-08-11 05:20:44 +02:00
private static String fillZ ( String s , int len ) {
while ( s . length ( ) < len ) s = " 0 " + s ;
return s ;
}
2006-08-05 21:18:33 +02:00
private static File arrayFile ( File path , String filenameStub , int loadfactor , int chunksize , int partitionNumber , int serialNumber ) {
2006-08-11 05:20:44 +02:00
String lf = fillZ ( Integer . toHexString ( loadfactor ) . toUpperCase ( ) , 2 ) ;
String cs = fillZ ( Integer . toHexString ( chunksize ) . toUpperCase ( ) , 4 ) ;
String pn = fillZ ( Integer . toHexString ( partitionNumber ) . toUpperCase ( ) , 2 ) ;
String sn = fillZ ( Integer . toHexString ( serialNumber ) . toUpperCase ( ) , 2 ) ;
2006-08-05 21:18:33 +02:00
return new File ( path , filenameStub + " . " + lf + " . " + cs + " . " + pn + " . " + sn + " .kca " ) ; // kelondro collection array
2006-03-01 01:25:02 +01:00
}
2006-10-06 01:47:08 +02:00
2006-08-11 05:20:44 +02:00
private static File propertyFile ( File path , String filenameStub , int loadfactor , int chunksize ) {
String lf = fillZ ( Integer . toHexString ( loadfactor ) . toUpperCase ( ) , 2 ) ;
String cs = fillZ ( Integer . toHexString ( chunksize ) . toUpperCase ( ) , 4 ) ;
2006-10-06 01:47:08 +02:00
return new File ( path , filenameStub + " . " + lf + " . " + cs + " .properties " ) ;
2006-08-11 05:20:44 +02:00
}
2006-07-04 01:57:33 +02:00
public kelondroCollectionIndex ( File path , String filenameStub , int keyLength , kelondroOrder indexOrder ,
long buffersize , long preloadTime ,
2006-08-05 01:04:03 +02:00
int loadfactor , kelondroRow rowdef ) throws IOException {
// the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree
2006-03-01 01:25:02 +01:00
this . path = path ;
this . filenameStub = filenameStub ;
2006-10-06 01:47:08 +02:00
this . keylength = keyLength ;
2006-08-16 21:49:31 +02:00
this . playloadrow = rowdef ;
2006-03-01 01:25:02 +01:00
this . loadfactor = loadfactor ;
2006-10-06 01:47:08 +02:00
boolean ramIndexGeneration = false ;
boolean fileIndexGeneration = ! ( new File ( path , filenameStub + " .index " ) . exists ( ) ) ;
if ( ramIndexGeneration ) index = new kelondroRAMIndex ( indexOrder , indexRow ( ) ) ;
if ( fileIndexGeneration ) index = new kelondroFlexTable ( path , filenameStub + " .index " , buffersize , preloadTime , indexRow ( ) , indexOrder ) ;
// open array files
this . arrays = new HashMap ( ) ; // all entries will be dynamically created with getArray()
if ( ( ( fileIndexGeneration ) | | ( ramIndexGeneration ) ) ) {
serverLog . logFine ( " STARTUP " , " STARTED MIGRATION OF OLD COLLECION INDEX TO NEW COLLECTION INDEX. THIS WILL TAKE SOME TIME " ) ;
openAllArrayFiles ( ( ( fileIndexGeneration ) | | ( ramIndexGeneration ) ) , indexOrder ) ;
}
// open/create index table
if ( index = = null ) index = openIndexFile ( path , filenameStub , indexOrder , buffersize , preloadTime , loadfactor , rowdef ) ;
}
private void openAllArrayFiles ( boolean indexGeneration , kelondroOrder indexOrder ) throws IOException {
String [ ] list = this . path . list ( ) ;
kelondroFixedWidthArray array ;
kelondroRow irow = indexRow ( ) ;
int t = kelondroRowCollection . daysSince2000 ( System . currentTimeMillis ( ) ) ;
for ( int i = 0 ; i < list . length ; i + + ) if ( list [ i ] . endsWith ( " .kca " ) ) {
// open array
int pos = list [ i ] . indexOf ( '.' ) ;
if ( pos < 0 ) continue ;
int chunksize = Integer . parseInt ( list [ i ] . substring ( pos + 4 , pos + 8 ) , 16 ) ;
int partitionNumber = Integer . parseInt ( list [ i ] . substring ( pos + 9 , pos + 11 ) , 16 ) ;
int serialNumber = Integer . parseInt ( list [ i ] . substring ( pos + 12 , pos + 14 ) , 16 ) ;
try {
array = openArrayFile ( partitionNumber , serialNumber , true ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
continue ;
}
// remember that we opened the array
arrays . put ( partitionNumber + " - " + chunksize , array ) ;
if ( ( index ! = null ) & & ( indexGeneration ) ) {
// loop over all elements in array and create index entry for each row
kelondroRow . Entry aentry , ientry ;
byte [ ] key ;
long start = System . currentTimeMillis ( ) ;
long lastlog = start ;
for ( int j = 0 ; j < array . size ( ) ; j + + ) {
aentry = array . get ( j ) ;
key = aentry . getColBytes ( 0 ) ;
if ( key = = null ) continue ; // skip deleted entries
kelondroRowSet indexrows = new kelondroRowSet ( this . playloadrow , aentry . getColBytes ( 1 ) ) ;
ientry = irow . newEntry ( ) ;
ientry . setCol ( idx_col_key , key ) ;
ientry . setCol ( idx_col_chunksize , chunksize ) ;
ientry . setCol ( idx_col_chunkcount , indexrows . size ( ) ) ;
ientry . setCol ( idx_col_clusteridx , ( byte ) partitionNumber ) ;
ientry . setCol ( idx_col_flags , ( byte ) 0 ) ;
ientry . setCol ( idx_col_indexpos , j ) ;
ientry . setCol ( idx_col_lastread , t ) ;
ientry . setCol ( idx_col_lastwrote , t ) ;
index . put ( ientry ) ;
// write a log
if ( System . currentTimeMillis ( ) - lastlog > 30000 ) {
serverLog . logFine ( " STARTUP " , " created " + j + " RWI index entries. " + ( ( ( System . currentTimeMillis ( ) - start ) * ( array . size ( ) - j ) / j ) / 60000 ) + " minutes remaining for this array " ) ;
lastlog = System . currentTimeMillis ( ) ;
}
}
}
}
}
private kelondroIndex openIndexFile ( File path , String filenameStub , kelondroOrder indexOrder ,
long buffersize , long preloadTime ,
int loadfactor , kelondroRow rowdef ) throws IOException {
// open/create index table
kelondroFlexTable theindex = new kelondroFlexTable ( path , filenameStub + " .index " , buffersize , preloadTime , indexRow ( ) , indexOrder ) ;
2006-08-05 01:04:03 +02:00
2006-08-11 05:20:44 +02:00
// save/check property file for this array
File propfile = propertyFile ( path , filenameStub , loadfactor , rowdef . objectsize ( ) ) ;
Map props = new HashMap ( ) ;
if ( propfile . exists ( ) ) {
props = serverFileUtils . loadHashMap ( propfile ) ;
String stored_rowdef = ( String ) props . get ( " rowdef " ) ;
if ( ( stored_rowdef = = null ) | | ( ! ( rowdef . subsumes ( new kelondroRow ( stored_rowdef ) ) ) ) ) {
System . out . println ( " FATAL ERROR: stored rowdef ' " + stored_rowdef + " ' does not match with new rowdef ' " +
rowdef + " ' for array cluster ' " + path + " / " + filenameStub + " ' " ) ;
System . exit ( - 1 ) ;
}
}
props . put ( " rowdef " , rowdef . toString ( ) ) ;
serverFileUtils . saveMap ( propfile , props , " CollectionIndex properties " ) ;
2006-10-06 01:47:08 +02:00
return theindex ;
2006-03-01 01:25:02 +01:00
}
2006-08-05 21:18:33 +02:00
private kelondroFixedWidthArray openArrayFile ( int partitionNumber , int serialNumber , boolean create ) throws IOException {
2006-08-16 21:49:31 +02:00
File f = arrayFile ( path , filenameStub , loadfactor , playloadrow . objectsize ( ) , partitionNumber , serialNumber ) ;
2006-08-12 17:59:14 +02:00
int load = arrayCapacity ( partitionNumber ) ;
kelondroRow rowdef = new kelondroRow (
2006-10-06 01:47:08 +02:00
" byte[] key- " + keylength + " , " +
2006-08-16 21:49:31 +02:00
" byte[] collection- " + ( kelondroRowCollection . exportOverheadSize + load * this . playloadrow . objectsize ( ) )
2006-08-12 17:59:14 +02:00
) ;
2006-08-24 04:19:25 +02:00
if ( ( ! ( f . exists ( ) ) ) & & ( ! create ) ) return null ;
2006-10-06 01:47:08 +02:00
kelondroFixedWidthArray a = new kelondroFixedWidthArray ( f , rowdef , 0 ) ;
serverLog . logFine ( " STARTUP " , " opened array file " + f + " with " + a . size ( ) + " RWIs " ) ;
return a ;
2006-03-01 01:25:02 +01:00
}
2006-08-05 21:18:33 +02:00
private kelondroFixedWidthArray getArray ( int partitionNumber , int serialNumber , int chunksize ) {
2006-08-05 01:04:03 +02:00
String accessKey = partitionNumber + " - " + chunksize ;
kelondroFixedWidthArray array = ( kelondroFixedWidthArray ) arrays . get ( accessKey ) ;
if ( array ! = null ) return array ;
try {
2006-08-05 21:18:33 +02:00
array = openArrayFile ( partitionNumber , serialNumber , true ) ;
2006-08-05 01:04:03 +02:00
} catch ( IOException e ) {
return null ;
}
arrays . put ( accessKey , array ) ;
return array ;
}
private int arrayCapacity ( int arrayCounter ) {
int load = this . loadfactor ;
for ( int i = 0 ; i < arrayCounter ; i + + ) load = load * this . loadfactor ;
return load ;
}
2006-03-01 01:25:02 +01:00
private int arrayIndex ( int requestedCapacity ) throws kelondroOutOfLimitsException {
// the requestedCapacity is the number of wanted chunks
2006-08-05 01:04:03 +02:00
int load = 1 , i = 0 ;
while ( true ) {
load = load * this . loadfactor ;
if ( load > = requestedCapacity ) return i ;
i + + ;
2006-03-01 01:25:02 +01:00
}
}
2006-07-03 17:14:54 +02:00
public int size ( ) throws IOException {
return index . size ( ) ;
}
2006-08-05 01:04:03 +02:00
2006-06-20 16:46:28 +02:00
public void put ( byte [ ] key , kelondroRowCollection collection ) throws IOException , kelondroOutOfLimitsException {
2006-08-05 01:04:03 +02:00
// this replaces an old collection by a new one
// this method is not approriate to extend an existing collection with another collection
2006-08-05 21:18:33 +02:00
putmergeremove ( key , collection , false , null , false ) ;
2006-08-05 01:04:03 +02:00
}
2006-08-05 21:18:33 +02:00
public void merge ( byte [ ] key , kelondroRowCollection collection ) throws IOException , kelondroOutOfLimitsException {
putmergeremove ( key , collection , true , null , false ) ;
2006-08-05 01:04:03 +02:00
}
2006-03-01 01:25:02 +01:00
2006-08-05 21:18:33 +02:00
public int remove ( byte [ ] key , Set removekeys , boolean deletecomplete ) throws IOException , kelondroOutOfLimitsException {
return putmergeremove ( key , null , false , removekeys , deletecomplete ) ;
}
private int putmergeremove ( byte [ ] key , kelondroRowCollection collection , boolean merge , Set removekeys , boolean deletecomplete ) throws IOException , kelondroOutOfLimitsException {
2006-08-05 01:04:03 +02:00
//if (collection.size() > maxChunks) throw new kelondroOutOfLimitsException(maxChunks, collection.size());
2006-08-06 00:22:14 +02:00
if ( ( ! merge ) & & ( removekeys ! = null ) & & ( collection ! = null ) & & ( collection . size ( ) = = 0 ) ) {
2006-08-05 01:04:03 +02:00
// this is not a replacement, it is a deletion
2006-08-05 21:18:33 +02:00
delete ( key ) ;
return 0 ;
2006-08-05 01:04:03 +02:00
}
2006-08-08 01:29:26 +02:00
synchronized ( index ) {
// first find an old entry, if one exists
kelondroRow . Entry oldindexrow = index . get ( key ) ;
2006-03-01 01:25:02 +01:00
2006-08-08 01:29:26 +02:00
if ( oldindexrow = = null ) {
if ( ( collection ! = null ) & & ( collection . size ( ) > 0 ) ) {
// the collection is new
overwrite ( key , collection ) ;
}
return 0 ;
2006-08-16 21:49:31 +02:00
}
2006-08-05 01:04:03 +02:00
2006-10-06 01:47:08 +02:00
// overwrite the old collection
// read old information
int oldchunksize = ( int ) oldindexrow . getColLong ( idx_col_chunksize ) ; // needed only for migration
int oldchunkcount = ( int ) oldindexrow . getColLong ( idx_col_chunkcount ) ;
int oldrownumber = ( int ) oldindexrow . getColLong ( idx_col_indexpos ) ;
int oldPartitionNumber = ( int ) oldindexrow . getColByte ( idx_col_clusteridx ) ;
assert ( oldPartitionNumber > = arrayIndex ( oldchunkcount ) ) ;
int oldSerialNumber = 0 ;
if ( merge ) {
// load the old collection and join it
kelondroRowSet oldcollection = getdelete ( oldindexrow , false , false ) ;
// join with new collection
oldcollection . addAll ( collection ) ;
collection = oldcollection ;
}
int removed = 0 ;
if ( removekeys ! = null ) {
// load the old collection and remove keys
kelondroRowSet oldcollection = getdelete ( oldindexrow , false , false ) ;
// remove the keys from the set
Iterator i = removekeys . iterator ( ) ;
Object k ;
while ( i . hasNext ( ) ) {
k = i . next ( ) ;
if ( ( k instanceof byte [ ] ) & & ( oldcollection . remove ( ( byte [ ] ) k ) ! = null ) ) removed + + ;
if ( ( k instanceof String ) & & ( oldcollection . remove ( ( ( String ) k ) . getBytes ( ) ) ! = null ) ) removed + + ;
2006-08-05 21:18:33 +02:00
}
2006-10-06 01:47:08 +02:00
oldcollection . shape ( ) ;
collection = oldcollection ;
}
if ( collection . size ( ) = = 0 ) {
if ( deletecomplete ) {
2006-08-08 01:29:26 +02:00
kelondroFixedWidthArray array = getArray ( oldPartitionNumber , oldSerialNumber , oldchunksize ) ;
array . remove ( oldrownumber ) ;
}
return removed ;
2006-10-06 01:47:08 +02:00
}
int newPartitionNumber = arrayIndex ( collection . size ( ) ) ;
int newSerialNumber = 0 ;
// see if we need new space or if we can overwrite the old space
if ( oldPartitionNumber = = newPartitionNumber ) {
// we don't need a new slot, just write into the old one
// find array file
kelondroFixedWidthArray array = getArray ( newPartitionNumber , newSerialNumber , this . playloadrow . objectsize ( ) ) ;
// define row
kelondroRow . Entry arrayEntry = array . row ( ) . newEntry ( ) ;
arrayEntry . setCol ( 0 , key ) ;
arrayEntry . setCol ( 1 , collection . exportCollection ( ) ) ;
// overwrite entry in this array
array . set ( oldrownumber , arrayEntry ) ;
// update the index entry
oldindexrow . setCol ( idx_col_chunkcount , collection . size ( ) ) ;
oldindexrow . setCol ( idx_col_clusteridx , ( byte ) newPartitionNumber ) ;
oldindexrow . setCol ( idx_col_flags , ( byte ) 0 ) ;
oldindexrow . setCol ( idx_col_lastwrote , kelondroRowCollection . daysSince2000 ( System . currentTimeMillis ( ) ) ) ;
index . put ( oldindexrow ) ;
} else {
// we need a new slot, that means we must first delete the old entry
// find array file
kelondroFixedWidthArray array = getArray ( oldPartitionNumber , oldSerialNumber , oldchunksize ) ;
// delete old entry
array . remove ( oldrownumber ) ;
// write a new entry in the other array
overwrite ( key , collection ) ;
}
return removed ;
2006-03-01 01:25:02 +01:00
}
}
2006-08-05 01:04:03 +02:00
private void overwrite ( byte [ ] key , kelondroRowCollection collection ) throws IOException {
2006-08-08 01:29:26 +02:00
// helper method, should not be called directly and only within a synchronized(index) environment
2006-08-05 01:04:03 +02:00
// simply store a collection without check if the collection existed before
// find array file
2006-10-06 01:47:08 +02:00
int clusteridx = arrayIndex ( collection . size ( ) ) ;
kelondroFixedWidthArray array = getArray ( clusteridx , 0 , this . playloadrow . objectsize ( ) ) ;
2006-08-05 01:04:03 +02:00
// define row
kelondroRow . Entry arrayEntry = array . row ( ) . newEntry ( ) ;
arrayEntry . setCol ( 0 , key ) ;
arrayEntry . setCol ( 1 , collection . exportCollection ( ) ) ;
// write a new entry in this array
int newRowNumber = array . add ( arrayEntry ) ;
// store the new row number in the index
kelondroRow . Entry indexEntry = index . row ( ) . newEntry ( ) ;
indexEntry . setCol ( idx_col_key , key ) ;
2006-08-16 21:49:31 +02:00
indexEntry . setCol ( idx_col_chunksize , this . playloadrow . objectsize ( ) ) ;
2006-08-11 05:20:44 +02:00
indexEntry . setCol ( idx_col_chunkcount , collection . size ( ) ) ;
2006-10-06 01:47:08 +02:00
indexEntry . setCol ( idx_col_clusteridx , ( byte ) clusteridx ) ;
indexEntry . setCol ( idx_col_flags , ( byte ) 0 ) ;
2006-08-11 05:20:44 +02:00
indexEntry . setCol ( idx_col_indexpos , ( long ) newRowNumber ) ;
indexEntry . setCol ( idx_col_lastread , kelondroRowCollection . daysSince2000 ( System . currentTimeMillis ( ) ) ) ;
indexEntry . setCol ( idx_col_lastwrote , kelondroRowCollection . daysSince2000 ( System . currentTimeMillis ( ) ) ) ;
2006-08-05 01:04:03 +02:00
index . put ( indexEntry ) ;
}
2006-03-01 01:25:02 +01:00
2006-08-11 18:01:18 +02:00
public int indexSize ( byte [ ] key ) throws IOException {
synchronized ( index ) {
kelondroRow . Entry indexrow = index . get ( key ) ;
if ( indexrow = = null ) return 0 ;
return ( int ) indexrow . getColLong ( idx_col_chunkcount ) ;
}
}
2006-08-05 21:18:33 +02:00
public kelondroRowSet get ( byte [ ] key , boolean deleteIfEmpty ) throws IOException {
2006-03-01 01:25:02 +01:00
// find an entry, if one exists
2006-08-08 01:29:26 +02:00
synchronized ( index ) {
kelondroRow . Entry indexrow = index . get ( key ) ;
if ( indexrow = = null ) return null ;
return getdelete ( indexrow , false , deleteIfEmpty ) ;
}
2006-08-05 21:18:33 +02:00
}
public kelondroRowSet delete ( byte [ ] key ) throws IOException {
// find an entry, if one exists
2006-08-08 01:29:26 +02:00
synchronized ( index ) {
kelondroRow . Entry indexrow = index . get ( key ) ;
if ( indexrow = = null ) return null ;
2006-10-11 02:46:45 +02:00
kelondroRowSet removedCollection = getdelete ( indexrow , true , false ) ;
index . remove ( key ) ;
return removedCollection ;
2006-08-08 01:29:26 +02:00
}
2006-08-05 21:18:33 +02:00
}
2006-08-16 21:49:31 +02:00
protected kelondroRowSet getdelete ( kelondroRow . Entry indexrow , boolean remove , boolean deleteIfEmpty ) throws IOException {
2006-08-08 01:29:26 +02:00
// call this only within a synchronized(index) environment
2006-03-01 01:25:02 +01:00
// read values
2006-10-06 01:47:08 +02:00
int chunksize = ( int ) indexrow . getColLong ( idx_col_chunksize ) ;
int chunkcount = ( int ) indexrow . getColLong ( idx_col_chunkcount ) ;
int rownumber = ( int ) indexrow . getColLong ( idx_col_indexpos ) ;
int partitionnumber = ( int ) indexrow . getColByte ( idx_col_clusteridx ) ;
assert ( partitionnumber > = arrayIndex ( chunkcount ) ) ;
2006-08-05 21:18:33 +02:00
int serialnumber = 0 ;
2006-08-05 01:04:03 +02:00
2006-03-01 01:25:02 +01:00
// open array entry
2006-08-05 21:18:33 +02:00
kelondroFixedWidthArray array = getArray ( partitionnumber , serialnumber , chunksize ) ;
2006-08-05 01:04:03 +02:00
kelondroRow . Entry arrayrow = array . get ( rownumber ) ;
2006-08-05 21:18:33 +02:00
if ( arrayrow = = null ) throw new kelondroException ( arrayFile ( this . path , this . filenameStub , this . loadfactor , chunksize , partitionnumber , serialnumber ) . toString ( ) , " array does not contain expected row " ) ;
2006-08-05 01:04:03 +02:00
2006-03-01 01:25:02 +01:00
// read the row and define a collection
2006-08-16 21:49:31 +02:00
kelondroRowSet collection = new kelondroRowSet ( this . playloadrow , arrayrow . getColBytes ( 1 ) ) ; // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
2006-08-08 01:29:26 +02:00
if ( index . order ( ) . compare ( arrayrow . getColBytes ( 0 ) , indexrow . getColBytes ( idx_col_key ) ) ! = 0 ) {
2006-08-08 02:52:04 +02:00
// check if we got the right row; this row is wrong. Fix it:
index . remove ( indexrow . getColBytes ( idx_col_key ) ) ; // the wrong row cannot be fixed
// store the row number in the index; this may be a double-entry, but better than nothing
kelondroRow . Entry indexEntry = index . row ( ) . newEntry ( ) ;
indexEntry . setCol ( idx_col_key , arrayrow . getColBytes ( 0 ) ) ;
2006-08-16 21:49:31 +02:00
indexEntry . setCol ( idx_col_chunksize , this . playloadrow . objectsize ( ) ) ;
2006-08-11 05:20:44 +02:00
indexEntry . setCol ( idx_col_chunkcount , collection . size ( ) ) ;
2006-10-06 01:47:08 +02:00
indexEntry . setCol ( idx_col_clusteridx , ( byte ) partitionnumber ) ;
indexEntry . setCol ( idx_col_flags , ( byte ) 0 ) ;
2006-08-11 05:20:44 +02:00
indexEntry . setCol ( idx_col_indexpos , ( long ) rownumber ) ;
indexEntry . setCol ( idx_col_lastread , kelondroRowCollection . daysSince2000 ( System . currentTimeMillis ( ) ) ) ;
indexEntry . setCol ( idx_col_lastwrote , kelondroRowCollection . daysSince2000 ( System . currentTimeMillis ( ) ) ) ;
2006-08-08 02:52:04 +02:00
index . put ( indexEntry ) ;
2006-10-13 01:14:41 +02:00
throw new kelondroException ( array . filename , " array contains wrong row ' " + new String ( arrayrow . getColBytes ( 0 ) ) + " ', expected is ' " + new String ( indexrow . getColBytes ( idx_col_key ) ) + " ', the row has been fixed " ) ;
2006-08-08 01:29:26 +02:00
}
2006-08-05 01:04:03 +02:00
int chunkcountInArray = collection . size ( ) ;
2006-08-08 01:29:26 +02:00
if ( chunkcountInArray ! = chunkcount ) {
// fix the entry in index
2006-08-11 05:20:44 +02:00
indexrow . setCol ( idx_col_chunkcount , chunkcountInArray ) ;
2006-08-08 01:29:26 +02:00
index . put ( indexrow ) ;
array . logFailure ( " INCONSISTENCY in " + arrayFile ( this . path , this . filenameStub , this . loadfactor , chunksize , partitionnumber , serialnumber ) . toString ( ) + " : array has different chunkcount than index: index = " + chunkcount + " , array = " + chunkcountInArray + " ; the index has been auto-fixed " ) ;
}
2006-08-05 21:18:33 +02:00
if ( ( remove ) | | ( ( chunkcountInArray = = 0 ) & & ( deleteIfEmpty ) ) ) array . remove ( rownumber ) ;
2006-08-05 01:04:03 +02:00
return collection ;
2006-03-01 01:25:02 +01:00
}
2006-08-05 21:18:33 +02:00
public Iterator keycollections ( byte [ ] startKey , boolean rot ) {
// returns an iteration of {byte[], kelondroRowSet} Objects
try {
return new keycollectionIterator ( startKey , rot ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
return null ;
}
}
2006-03-01 01:25:02 +01:00
2006-08-05 21:18:33 +02:00
public class keycollectionIterator implements Iterator {
2006-08-05 01:04:03 +02:00
2006-08-05 21:18:33 +02:00
Iterator indexRowIterator ;
2006-08-05 01:04:03 +02:00
2006-08-05 21:18:33 +02:00
public keycollectionIterator ( byte [ ] startKey , boolean rot ) throws IOException {
// iterator of {byte[], kelondroRowSet} Objects
indexRowIterator = index . rows ( true , rot , startKey ) ;
}
2006-08-05 01:04:03 +02:00
2006-08-05 21:18:33 +02:00
public boolean hasNext ( ) {
return indexRowIterator . hasNext ( ) ;
}
public Object next ( ) {
kelondroRow . Entry indexrow = ( kelondroRow . Entry ) indexRowIterator . next ( ) ;
if ( indexrow = = null ) return null ;
try {
return new Object [ ] { indexrow . getColBytes ( 0 ) , getdelete ( indexrow , false , false ) } ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
return null ;
}
}
public void remove ( ) {
indexRowIterator . remove ( ) ;
}
2006-08-05 01:04:03 +02:00
2006-03-01 01:25:02 +01:00
}
2006-08-05 21:18:33 +02:00
2006-08-05 01:04:03 +02:00
public void close ( ) throws IOException {
2006-08-25 01:39:52 +02:00
synchronized ( index ) {
this . index . close ( ) ;
Iterator i = arrays . values ( ) . iterator ( ) ;
while ( i . hasNext ( ) ) {
( ( kelondroFixedWidthArray ) i . next ( ) ) . close ( ) ;
}
2006-08-05 01:04:03 +02:00
}
}
2006-03-01 01:25:02 +01:00
public static void main ( String [ ] args ) {
2006-08-05 01:04:03 +02:00
// define payload structure
2006-08-05 21:18:33 +02:00
kelondroRow rowdef = new kelondroRow ( " byte[] a-10, byte[] b-80 " ) ;
2006-08-05 01:04:03 +02:00
File path = new File ( args [ 0 ] ) ;
String filenameStub = args [ 1 ] ;
long buffersize = 10000000 ;
long preloadTime = 10000 ;
try {
// initialize collection index
kelondroCollectionIndex collectionIndex = new kelondroCollectionIndex (
path , filenameStub , 9 /*keyLength*/ ,
kelondroNaturalOrder . naturalOrder , buffersize , preloadTime ,
4 /*loadfactor*/ , rowdef ) ;
// fill index with values
kelondroRowSet collection = new kelondroRowSet ( rowdef ) ;
collection . add ( rowdef . newEntry ( new byte [ ] [ ] { " abc " . getBytes ( ) , " efg " . getBytes ( ) } ) ) ;
collectionIndex . put ( " erstes " . getBytes ( ) , collection ) ;
for ( int i = 0 ; i < = 17 ; i + + ) {
collection = new kelondroRowSet ( rowdef ) ;
for ( int j = 0 ; j < i ; j + + ) {
collection . add ( rowdef . newEntry ( new byte [ ] [ ] { ( " abc " + j ) . getBytes ( ) , " xxx " . getBytes ( ) } ) ) ;
}
2006-08-05 21:18:33 +02:00
System . out . println ( " put key- " + i + " : " + collection . toString ( ) ) ;
2006-08-05 01:04:03 +02:00
collectionIndex . put ( ( " key- " + i ) . getBytes ( ) , collection ) ;
}
// extend collections with more values
for ( int i = 0 ; i < = 17 ; i + + ) {
collection = new kelondroRowSet ( rowdef ) ;
for ( int j = 0 ; j < i ; j + + ) {
collection . add ( rowdef . newEntry ( new byte [ ] [ ] { ( " def " + j ) . getBytes ( ) , " xxx " . getBytes ( ) } ) ) ;
}
2006-08-05 21:18:33 +02:00
collectionIndex . merge ( ( " key- " + i ) . getBytes ( ) , collection ) ;
2006-08-05 01:04:03 +02:00
}
// printout of index
2006-10-06 01:47:08 +02:00
kelondroFlexTable index = new kelondroFlexTable ( path , filenameStub + " .index " , buffersize , preloadTime , collectionIndex . indexRow ( ) , kelondroNaturalOrder . naturalOrder ) ;
collectionIndex . close ( ) ;
2006-08-05 01:04:03 +02:00
index . print ( ) ;
index . close ( ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
}
2006-03-01 01:25:02 +01:00
}
}