2006-06-13 17:05:11 +02:00
// kelondroFlexTable.java
2008-07-20 19:14:51 +02:00
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
2006-06-02 00:13:52 +02:00
// first published 01.06.2006 on http://www.anomic.de
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2006-06-01 18:18:27 +02:00
package de.anomic.kelondro ;
import java.io.File ;
import java.io.IOException ;
2007-02-27 16:54:02 +01:00
import java.util.ArrayList ;
2006-10-13 01:14:41 +02:00
import java.util.Date ;
2007-03-06 23:43:32 +01:00
import java.util.HashMap ;
2006-06-09 14:52:57 +02:00
import java.util.Iterator ;
2007-02-25 22:06:26 +01:00
import java.util.List ;
2007-02-27 16:54:02 +01:00
import java.util.Map ;
2007-02-25 22:06:26 +01:00
import java.util.TreeMap ;
2008-01-20 02:22:46 +01:00
import java.util.TreeSet ;
2006-06-01 18:18:27 +02:00
2007-03-06 23:43:32 +01:00
import de.anomic.server.serverMemory ;
2006-10-24 15:48:16 +02:00
import de.anomic.server.logging.serverLog ;
2006-06-01 18:18:27 +02:00
public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondroIndex {
2007-03-06 23:43:32 +01:00
// static tracker objects
2008-01-07 23:36:48 +01:00
private static TreeMap < String , kelondroFlexTable > tableTracker = new TreeMap < String , kelondroFlexTable > ( ) ;
2007-03-06 23:43:32 +01:00
// class objects
2007-04-19 15:37:02 +02:00
protected kelondroBytesIntMap index ;
2006-10-26 15:50:50 +02:00
private boolean RAMIndex ;
2008-08-02 14:12:04 +02:00
public kelondroFlexTable ( final File path , final String tablename , final kelondroRow rowdef , int minimumSpace , final boolean resetOnFail ) {
2006-11-05 22:30:53 +01:00
// the buffersize applies to a possible load of the ram-index
2007-12-11 16:07:03 +01:00
// the minimumSpace is a initial allocation space for the index; names the number of index slots
2006-11-05 22:30:53 +01:00
// if the ram is not sufficient, a tree file is generated
// if, and only if a tree file exists, the preload time is applied
2007-04-05 12:14:48 +02:00
super ( path , tablename , rowdef , resetOnFail ) ;
2007-04-05 21:08:59 +02:00
if ( ( super . col [ 0 ] . size ( ) < 0 ) & & ( resetOnFail ) ) try {
super . reset ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e2 ) {
2007-04-05 21:08:59 +02:00
e2 . printStackTrace ( ) ;
throw new kelondroException ( e2 . getMessage ( ) ) ;
}
2008-01-23 21:18:36 +01:00
minimumSpace = Math . max ( minimumSpace , super . size ( ) ) ;
2007-04-05 21:08:59 +02:00
try {
2008-08-02 14:12:04 +02:00
final long neededRAM = 10 * 1024 * 104 + ( long ) ( ( super . row ( ) . column ( 0 ) . cellwidth + 4 ) * minimumSpace * kelondroRowCollection . growfactor ) ;
2006-11-05 22:30:53 +01:00
2008-08-02 14:12:04 +02:00
final File newpath = new File ( path , tablename ) ;
final File indexfile = new File ( newpath , " col.000.index " ) ;
2007-04-05 12:14:48 +02:00
String description = " " ;
description = new String ( this . col [ 0 ] . getDescription ( ) ) ;
2008-08-02 14:12:04 +02:00
final int p = description . indexOf ( ';' , 4 ) ;
final long stt = ( p > 0 ) ? Long . parseLong ( description . substring ( 4 , p ) ) : 0 ;
2006-08-05 01:04:03 +02:00
System . out . println ( " *** Last Startup time: " + stt + " milliseconds " ) ;
2008-08-02 14:12:04 +02:00
final long start = System . currentTimeMillis ( ) ;
2006-06-30 14:54:19 +02:00
2008-01-23 21:18:36 +01:00
if ( serverMemory . request ( neededRAM , false ) ) {
2008-01-11 15:43:28 +01:00
// we can use a RAM index
if ( indexfile . exists ( ) ) {
// delete existing index file
System . out . println ( " *** Delete File index " + indexfile ) ;
indexfile . delete ( ) ;
}
// fill the index
2008-01-23 21:18:36 +01:00
System . out . print ( " *** Loading RAM index for " + size ( ) + " entries from " + newpath + " ; available RAM = " + ( serverMemory . available ( ) > > 20 ) + " MB, allocating " + ( neededRAM > > 20 ) + " MB for index. " ) ;
2008-01-11 15:43:28 +01:00
index = initializeRamIndex ( minimumSpace ) ;
System . out . println ( " -done- " ) ;
System . out . println ( index . size ( ) + " index entries initialized and sorted from " + super . col [ 0 ] . size ( ) + " keys. " ) ;
RAMIndex = true ;
tableTracker . put ( this . filename ( ) , this ) ;
} else {
// too less ram for a ram index
kelondroIndex ki ;
if ( indexfile . exists ( ) ) {
// use existing index file
System . out . println ( " *** Using File index " + indexfile ) ;
2008-02-19 10:14:07 +01:00
ki = new kelondroCache ( kelondroTree . open ( indexfile , true , 0 , treeIndexRow ( rowdef . width ( 0 ) , rowdef . objectOrder ) , 2 , 80 ) ) ;
2008-01-11 15:43:28 +01:00
RAMIndex = false ;
} else {
// generate new index file
System . out . println ( " *** Generating File index for " + size ( ) + " entries from " + indexfile ) ;
System . out . println ( " *** Cause: too less RAM ( " + serverMemory . available ( ) + " Bytes) configured. Assign at least " + ( neededRAM / 1024 / 1024 ) + " MB more RAM to enable a RAM index. " ) ;
2008-02-19 10:14:07 +01:00
ki = initializeTreeIndex ( indexfile , 0 , rowdef . objectOrder ) ;
2008-01-11 15:43:28 +01:00
System . out . println ( " -done- " ) ;
System . out . println ( ki . size ( ) + " entries indexed from " + super . col [ 0 ] . size ( ) + " keys. " ) ;
RAMIndex = false ;
}
index = new kelondroBytesIntMap ( ki ) ;
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
}
2008-01-20 22:42:35 +01:00
// check consistency
2008-08-02 14:12:04 +02:00
final ArrayList < Integer [ ] > doubles = index . removeDoubles ( ) ;
2008-01-20 22:42:35 +01:00
if ( doubles . size ( ) > 0 ) {
System . out . println ( " DEBUG: WARNING - FlexTable " + newpath . toString ( ) + " has " + doubles . size ( ) + " doubles " ) ;
}
2008-01-10 00:05:52 +01:00
2006-06-30 14:54:19 +02:00
// assign index to wrapper
description = " stt= " + Long . toString ( System . currentTimeMillis ( ) - start ) + " ; " ;
super . col [ 0 ] . setDescription ( description . getBytes ( ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-04-05 12:14:48 +02:00
if ( resetOnFail ) {
RAMIndex = true ;
2007-11-07 23:38:09 +01:00
index = new kelondroBytesIntMap ( super . row ( ) . column ( 0 ) . cellwidth , super . rowdef . objectOrder , 0 ) ;
2007-04-05 12:14:48 +02:00
} else {
throw new kelondroException ( e . getMessage ( ) ) ;
}
}
}
2008-05-24 14:30:50 +02:00
public void clear ( ) throws IOException {
2007-04-05 12:14:48 +02:00
super . reset ( ) ;
RAMIndex = true ;
2007-11-07 23:38:09 +01:00
index = new kelondroBytesIntMap ( super . row ( ) . column ( 0 ) . cellwidth , super . rowdef . objectOrder , 0 ) ;
2006-06-30 14:54:19 +02:00
}
2006-07-02 02:44:38 +02:00
2008-08-02 14:12:04 +02:00
public static int staticSize ( final File path , final String tablename ) {
2006-11-05 22:30:53 +01:00
return kelondroFlexWidthArray . staticsize ( path , tablename ) ;
}
2008-08-02 14:12:04 +02:00
public static int staticRAMIndexNeed ( final File path , final String tablename , final kelondroRow rowdef ) {
2008-03-04 23:45:45 +01:00
return ( int ) ( ( rowdef . column ( 0 ) . cellwidth + 4 ) * staticSize ( path , tablename ) * kelondroRowCollection . growfactor ) ;
2006-11-06 03:05:39 +01:00
}
2006-10-26 15:50:50 +02:00
public boolean hasRAMIndex ( ) {
return RAMIndex ;
}
2008-08-02 14:12:04 +02:00
public synchronized boolean has ( final byte [ ] key ) {
2006-10-26 15:50:50 +02:00
// it is not recommended to implement or use a has predicate unless
// it can be ensured that it causes no IO
2007-08-03 13:44:58 +02:00
if ( ( kelondroAbstractRecords . debugmode ) & & ( RAMIndex ! = true ) ) serverLog . logWarning ( " kelondroFlexTable " , " RAM index warning in file " + super . tablename ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2008-06-18 01:56:39 +02:00
return index . has ( key ) ;
2006-10-26 15:50:50 +02:00
}
2008-08-02 14:12:04 +02:00
private kelondroBytesIntMap initializeRamIndex ( final int initialSpace ) {
final int space = Math . max ( super . col [ 0 ] . size ( ) , initialSpace ) + 1 ;
2007-04-05 16:58:29 +02:00
if ( space < 0 ) throw new kelondroException ( " wrong space: " + space ) ;
2008-08-02 14:12:04 +02:00
final kelondroBytesIntMap ri = new kelondroBytesIntMap ( super . row ( ) . column ( 0 ) . cellwidth , super . rowdef . objectOrder , space ) ;
final Iterator < kelondroNode > content = super . col [ 0 ] . contentNodes ( - 1 ) ;
2007-08-03 13:44:58 +02:00
kelondroNode node ;
2006-06-09 14:52:57 +02:00
int i ;
2006-10-25 03:21:05 +02:00
byte [ ] key ;
2006-06-09 14:52:57 +02:00
while ( content . hasNext ( ) ) {
2008-01-07 23:36:48 +01:00
node = content . next ( ) ;
2006-06-09 14:52:57 +02:00
i = node . handle ( ) . hashCode ( ) ;
2006-10-25 03:21:05 +02:00
key = node . getKey ( ) ;
assert ( key ! = null ) : " DEBUG: empty key in initializeRamIndex " ; // should not happen; if it does, it is an error of the condentNodes iterator
2007-04-19 15:37:02 +02:00
//System.out.println("ENTRY: " + serverLog.arrayList(indexentry.bytes(), 0, indexentry.objectsize()));
2008-08-02 14:12:04 +02:00
try { ri . addi ( key , i ) ; } catch ( final IOException e ) { } // no IOException can happen here
2006-06-28 18:14:47 +02:00
if ( ( i % 10000 ) = = 0 ) {
System . out . print ( '.' ) ;
System . out . flush ( ) ;
}
2006-06-07 17:40:29 +02:00
}
2006-06-28 18:14:47 +02:00
System . out . print ( " -ordering- " ) ;
System . out . flush ( ) ;
2006-06-30 14:54:19 +02:00
return ri ;
2006-06-01 18:18:27 +02:00
}
2008-01-11 15:43:28 +01:00
2008-08-02 14:12:04 +02:00
private kelondroIndex initializeTreeIndex ( final File indexfile , final long preloadTime , final kelondroByteOrder objectOrder ) throws IOException {
final kelondroIndex treeindex = new kelondroCache ( new kelondroTree ( indexfile , true , preloadTime , treeIndexRow ( rowdef . primaryKeyLength , objectOrder ) , 2 , 80 ) ) ;
final Iterator < kelondroNode > content = super . col [ 0 ] . contentNodes ( - 1 ) ;
2008-01-11 15:43:28 +01:00
kelondroNode node ;
kelondroRow . Entry indexentry ;
2008-08-02 14:12:04 +02:00
int i , c = 0 ;
2008-08-02 15:57:00 +02:00
final int all = super . col [ 0 ] . size ( ) ;
2008-08-02 14:12:04 +02:00
final long start = System . currentTimeMillis ( ) ;
2008-01-11 15:43:28 +01:00
long last = start ;
while ( content . hasNext ( ) ) {
node = content . next ( ) ;
i = node . handle ( ) . hashCode ( ) ;
indexentry = treeindex . row ( ) . newEntry ( ) ;
indexentry . setCol ( 0 , node . getValueRow ( ) ) ;
indexentry . setCol ( 1 , i ) ;
treeindex . addUnique ( indexentry ) ;
c + + ;
if ( System . currentTimeMillis ( ) - last > 30000 ) {
System . out . println ( " .. generated " + c + " / " + all + " entries, " + ( ( System . currentTimeMillis ( ) - start ) / c * ( all - c ) / 60000 ) + " minutes remaining " ) ;
System . out . flush ( ) ;
last = System . currentTimeMillis ( ) ;
}
}
return treeindex ;
}
2008-08-02 14:12:04 +02:00
private static final kelondroRow treeIndexRow ( final int keywidth , final kelondroByteOrder objectOrder ) {
2008-01-11 15:43:28 +01:00
return new kelondroRow ( " byte[] key- " + keywidth + " , int reference-4 {b256} " , objectOrder , 0 ) ;
}
2008-08-02 14:12:04 +02:00
public synchronized kelondroRow . Entry get ( final byte [ ] key ) throws IOException {
2007-09-04 01:43:55 +02:00
if ( index = = null ) return null ; // case may happen during shutdown
2008-08-02 14:12:04 +02:00
final int pos = index . geti ( key ) ;
2008-01-07 23:36:48 +01:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2007-04-19 15:37:02 +02:00
if ( pos < 0 ) return null ;
2008-01-11 01:12:01 +01:00
// pos may be greater than this.size(), because this table may have deleted entries
2007-04-19 15:37:02 +02:00
// the deleted entries are subtracted from the 'real' tablesize,
// so the size may be smaller than an index to a row entry
2008-01-11 01:12:01 +01:00
/ * if ( kelondroAbstractRecords . debugmode ) {
kelondroRow . Entry result = super . get ( pos ) ;
assert result ! = null ;
assert rowdef . objectOrder . compare ( result . getPrimaryKeyBytes ( ) , key ) = = 0 : " key and row does not match; key = " + serverLog . arrayList ( key , 0 , key . length ) + " row.key = " + serverLog . arrayList ( result . getPrimaryKeyBytes ( ) , 0 , rowdef . primaryKeyLength ) ;
return result ;
} else { * /
// assume that the column for the primary key is 0,
// and the column 0 is stored in a file only for that column
// then we don't need to lookup from that file, because we already know the value (it's the key)
2008-08-02 14:12:04 +02:00
final kelondroRow . Entry result = super . getOmitCol0 ( pos , key ) ;
2008-01-11 01:12:01 +01:00
assert result ! = null ;
return result ;
//}
2007-04-19 15:37:02 +02:00
}
2007-02-25 22:06:26 +01:00
2008-08-02 14:12:04 +02:00
public synchronized void putMultiple ( final List < kelondroRow . Entry > rows ) throws IOException {
2007-02-25 22:06:26 +01:00
// put a list of entries in a ordered way.
// this should save R/W head positioning time
2008-08-02 14:12:04 +02:00
final Iterator < kelondroRow . Entry > i = rows . iterator ( ) ;
2007-02-25 22:06:26 +01:00
kelondroRow . Entry row ;
int pos ;
byte [ ] key ;
2008-08-02 14:12:04 +02:00
final TreeMap < Integer , kelondroRow . Entry > old_rows_ordered = new TreeMap < Integer , kelondroRow . Entry > ( ) ;
final ArrayList < kelondroRow . Entry > new_rows_sequential = new ArrayList < kelondroRow . Entry > ( ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2007-02-25 22:06:26 +01:00
while ( i . hasNext ( ) ) {
2008-01-07 23:36:48 +01:00
row = i . next ( ) ;
2007-02-25 22:06:26 +01:00
key = row . getColBytes ( 0 ) ;
2007-04-19 15:37:02 +02:00
pos = index . geti ( key ) ;
2007-02-25 22:06:26 +01:00
if ( pos < 0 ) {
2007-02-27 16:54:02 +01:00
new_rows_sequential . add ( row ) ;
2007-02-25 22:06:26 +01:00
} else {
2007-02-27 16:54:02 +01:00
old_rows_ordered . put ( new Integer ( pos ) , row ) ;
2007-02-25 22:06:26 +01:00
}
}
2007-02-27 16:54:02 +01:00
// overwrite existing entries in index
super . setMultiple ( old_rows_ordered ) ;
// write new entries to index
2007-05-16 12:48:26 +02:00
addUniqueMultiple ( new_rows_sequential ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2007-02-25 22:06:26 +01:00
}
2006-06-01 18:18:27 +02:00
2008-08-02 14:12:04 +02:00
public synchronized kelondroRow . Entry put ( final kelondroRow . Entry row , final Date entryDate ) throws IOException {
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
return put ( row ) ;
2006-10-13 01:14:41 +02:00
}
2008-08-02 14:12:04 +02:00
public synchronized kelondroRow . Entry put ( final kelondroRow . Entry row ) throws IOException {
2006-10-24 15:48:16 +02:00
assert ( row ! = null ) ;
assert ( ! ( serverLog . allZero ( row . getColBytes ( 0 ) ) ) ) ;
2007-04-03 14:10:12 +02:00
assert row . objectsize ( ) < = this . rowdef . objectsize ;
2008-08-02 14:12:04 +02:00
final byte [ ] key = row . getColBytes ( 0 ) ;
2007-09-04 01:43:55 +02:00
if ( index = = null ) return null ; // case may appear during shutdown
2007-04-19 15:37:02 +02:00
int pos = index . geti ( key ) ;
2007-03-08 17:15:40 +01:00
if ( pos < 0 ) {
2007-04-19 15:37:02 +02:00
pos = super . add ( row ) ;
index . puti ( key , pos ) ;
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2006-10-16 17:04:16 +02:00
return null ;
}
2007-04-19 15:37:02 +02:00
//System.out.println("row.key=" + serverLog.arrayList(row.bytes(), 0, row.objectsize()));
2008-08-02 14:12:04 +02:00
final kelondroRow . Entry oldentry = super . get ( pos ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
if ( oldentry = = null ) {
2008-08-02 15:57:00 +02:00
serverLog . logSevere ( " kelondroFlexTable " , " put(): index failure; the index pointed to a cell which is empty. content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ) ;
2007-04-19 15:37:02 +02:00
// patch bug ***** FIND CAUSE! (see also: remove)
2008-08-02 14:12:04 +02:00
final int oldindex = index . removei ( key ) ;
2007-04-19 15:37:02 +02:00
assert oldindex > = 0 ;
assert index . geti ( key ) = = - 1 ;
// here is this.size() > index.size() because of remove operation above
index . puti ( key , super . add ( row ) ) ;
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
return null ;
}
assert oldentry ! = null : " overwrite of empty position " + pos + " , index management must have failed before " ;
2007-11-07 23:38:09 +01:00
assert rowdef . objectOrder . compare ( oldentry . getPrimaryKeyBytes ( ) , key ) = = 0 : " key and row does not match; key = " + serverLog . arrayList ( key , 0 , key . length ) + " row.key = " + serverLog . arrayList ( oldentry . getPrimaryKeyBytes ( ) , 0 , rowdef . primaryKeyLength ) ;
2007-03-08 17:15:40 +01:00
super . set ( pos , row ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2007-02-20 09:35:51 +01:00
return oldentry ;
2006-06-01 18:18:27 +02:00
}
2008-08-02 14:12:04 +02:00
public synchronized boolean addUnique ( final kelondroRow . Entry row ) throws IOException {
2007-04-03 14:10:12 +02:00
assert row . objectsize ( ) = = this . rowdef . objectsize ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2008-04-08 13:55:59 +02:00
return index . addi ( row . getColBytes ( 0 ) , super . add ( row ) ) ;
2006-10-19 23:14:37 +02:00
}
2008-08-02 14:12:04 +02:00
public synchronized int addUniqueMultiple ( final List < kelondroRow . Entry > rows ) throws IOException {
2007-02-27 16:54:02 +01:00
// add a list of entries in a ordered way.
// this should save R/W head positioning time
2008-08-02 14:12:04 +02:00
final TreeMap < Integer , byte [ ] > indexed_result = super . addMultiple ( rows ) ;
2007-02-27 16:54:02 +01:00
// indexed_result is a Integer/byte[] relation
// that is used here to store the index
2008-08-02 14:12:04 +02:00
final Iterator < Map . Entry < Integer , byte [ ] > > i = indexed_result . entrySet ( ) . iterator ( ) ;
2008-01-07 23:36:48 +01:00
Map . Entry < Integer , byte [ ] > entry ;
2007-02-27 16:54:02 +01:00
while ( i . hasNext ( ) ) {
2008-01-07 23:36:48 +01:00
entry = i . next ( ) ;
index . puti ( entry . getValue ( ) , entry . getKey ( ) . intValue ( ) ) ;
2007-02-27 16:54:02 +01:00
}
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2008-04-08 13:55:59 +02:00
return indexed_result . size ( ) ;
2007-02-27 16:54:02 +01:00
}
2008-05-02 00:40:42 +02:00
public synchronized ArrayList < kelondroRowCollection > removeDoubles ( ) throws IOException {
2008-08-02 14:12:04 +02:00
final ArrayList < kelondroRowCollection > report = new ArrayList < kelondroRowCollection > ( ) ;
2008-01-20 02:22:46 +01:00
kelondroRowSet rows ;
2008-08-02 14:12:04 +02:00
final TreeSet < Integer > d = new TreeSet < Integer > ( ) ;
for ( final Integer [ ] is : index . removeDoubles ( ) ) {
2008-01-20 02:22:46 +01:00
rows = new kelondroRowSet ( this . rowdef , is . length ) ;
for ( int j = 0 ; j < is . length ; j + + ) {
d . add ( is [ j ] ) ;
rows . addUnique ( this . get ( is [ j ] . intValue ( ) ) ) ;
}
report . add ( rows ) ;
}
2008-05-02 00:40:42 +02:00
// finally delete the affected rows, but start with largest id first, otherwise we overwrite wrong entries
2008-01-20 02:22:46 +01:00
Integer s ;
while ( d . size ( ) > 0 ) {
s = d . last ( ) ;
d . remove ( s ) ;
this . remove ( s . intValue ( ) ) ;
}
return report ;
}
2008-08-02 14:12:04 +02:00
public synchronized kelondroRow . Entry remove ( final byte [ ] key ) throws IOException {
2008-07-05 02:35:20 +02:00
// the underlying data structure is a file, where the order cannot be maintained. Gaps are filled with new values.
2008-08-02 14:12:04 +02:00
final int i = index . removei ( key ) ;
2007-04-19 15:37:02 +02:00
assert ( index . geti ( key ) < 0 ) ; // must be deleted
if ( i < 0 ) {
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
return null ;
2007-04-17 17:15:47 +02:00
}
2008-08-02 14:12:04 +02:00
final kelondroRow . Entry r = super . getOmitCol0 ( i , key ) ;
2007-04-19 15:37:02 +02:00
if ( r = = null ) {
serverLog . logSevere ( " kelondroFlexTable " , " remove(): index failure; the index pointed to a cell which is empty. content.size() = " + this . size ( ) + " , index.size() = " + ( ( index = = null ) ? 0 : index . size ( ) ) ) ;
// patch bug ***** FIND CAUSE! (see also: put)
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
return null ;
}
assert r ! = null : " r == null " ; // should be avoided with path above
2007-11-07 23:38:09 +01:00
assert rowdef . objectOrder . compare ( r . getPrimaryKeyBytes ( ) , key ) = = 0 : " key and row does not match; key = " + serverLog . arrayList ( key , 0 , key . length ) + " row.key = " + serverLog . arrayList ( r . getPrimaryKeyBytes ( ) , 0 , rowdef . primaryKeyLength ) ;
2007-04-17 17:15:47 +02:00
super . remove ( i ) ;
assert super . get ( i ) = = null : " i = " + i + " , get(i) = " + serverLog . arrayList ( super . get ( i ) . bytes ( ) , 0 , 12 ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
return r ;
2006-06-01 18:18:27 +02:00
}
2006-10-16 17:04:16 +02:00
public synchronized kelondroRow . Entry removeOne ( ) throws IOException {
2008-08-02 14:12:04 +02:00
final int i = index . removeonei ( ) ;
2006-10-16 17:04:16 +02:00
if ( i < 0 ) return null ;
kelondroRow . Entry r ;
r = super . get ( i ) ;
2007-04-17 17:15:47 +02:00
super . remove ( i ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
return r ;
2006-10-16 17:04:16 +02:00
}
2008-08-02 14:12:04 +02:00
public synchronized kelondroCloneableIterator < byte [ ] > keys ( final boolean up , final byte [ ] firstKey ) throws IOException {
2007-12-27 18:56:59 +01:00
return index . keys ( up , firstKey ) ;
}
2008-08-02 14:12:04 +02:00
public synchronized kelondroCloneableIterator < kelondroRow . Entry > rows ( final boolean up , final byte [ ] firstKey ) throws IOException {
2008-03-04 23:45:45 +01:00
if ( index = = null ) return new rowIterator ( up , firstKey ) ;
2007-04-19 15:37:02 +02:00
assert this . size ( ) = = index . size ( ) : " content.size() = " + this . size ( ) + " , index.size() = " + index . size ( ) ;
2008-03-04 23:45:45 +01:00
return new rowIterator ( up , firstKey ) ;
2006-08-05 21:18:33 +02:00
}
2008-01-09 10:58:56 +01:00
public class rowIterator implements kelondroCloneableIterator < kelondroRow . Entry > {
2006-08-05 21:18:33 +02:00
2008-01-09 10:58:56 +01:00
kelondroCloneableIterator < kelondroRow . Entry > indexIterator ;
2007-03-08 23:07:17 +01:00
boolean up ;
2006-08-05 21:18:33 +02:00
2008-08-02 14:12:04 +02:00
public rowIterator ( final boolean up , final byte [ ] firstKey ) throws IOException {
2007-03-08 23:07:17 +01:00
this . up = up ;
2007-03-08 17:15:40 +01:00
indexIterator = index . rows ( up , firstKey ) ;
}
2008-08-02 14:12:04 +02:00
public rowIterator clone ( final Object modifier ) {
2007-03-08 23:07:17 +01:00
try {
2008-03-04 23:45:45 +01:00
return new rowIterator ( up , ( byte [ ] ) modifier ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-03-08 23:07:17 +01:00
return null ;
}
2006-08-05 21:18:33 +02:00
}
public boolean hasNext ( ) {
return indexIterator . hasNext ( ) ;
}
2008-01-09 10:58:56 +01:00
public kelondroRow . Entry next ( ) {
2007-02-04 05:39:47 +01:00
kelondroRow . Entry idxEntry = null ;
while ( ( indexIterator . hasNext ( ) ) & & ( idxEntry = = null ) ) {
2008-06-06 18:01:27 +02:00
idxEntry = indexIterator . next ( ) ;
2007-02-04 05:39:47 +01:00
}
if ( idxEntry = = null ) {
serverLog . logSevere ( " kelondroFlexTable.rowIterator: " + tablename , " indexIterator returned null " ) ;
return null ;
}
2008-08-02 14:12:04 +02:00
final int idx = ( int ) idxEntry . getColLong ( 1 ) ;
2006-08-05 21:18:33 +02:00
try {
return get ( idx ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2006-08-05 21:18:33 +02:00
e . printStackTrace ( ) ;
return null ;
}
}
public void remove ( ) {
indexIterator . remove ( ) ;
}
}
2006-10-16 17:04:16 +02:00
public kelondroProfile profile ( ) {
2007-04-19 15:37:02 +02:00
return index . profile ( ) ;
2006-10-16 17:04:16 +02:00
}
2008-01-07 23:36:48 +01:00
public static final Iterator < String > filenames ( ) {
2007-03-06 23:43:32 +01:00
// iterates string objects; all file names from record tracker
return tableTracker . keySet ( ) . iterator ( ) ;
2006-10-24 15:48:16 +02:00
}
2007-03-06 23:43:32 +01:00
2008-08-02 14:12:04 +02:00
public static final kelondroProfile profileStats ( final String filename ) {
2007-03-06 23:43:32 +01:00
// returns a map for each file in the tracker;
// the map represents properties for each record oobjects,
// i.e. for cache memory allocation
2008-08-02 14:12:04 +02:00
final kelondroFlexTable theFlexTable = tableTracker . get ( filename ) ;
2007-03-06 23:43:32 +01:00
return theFlexTable . profile ( ) ;
2006-10-24 15:48:16 +02:00
}
2008-08-02 14:12:04 +02:00
public static final Map < String , String > memoryStats ( final String filename ) {
2007-03-06 23:43:32 +01:00
// returns a map for each file in the tracker;
2008-01-06 20:23:38 +01:00
// the map represents properties for each record objects,
2007-03-06 23:43:32 +01:00
// i.e. for cache memory allocation
2008-08-02 14:12:04 +02:00
final kelondroFlexTable theFlexTable = tableTracker . get ( filename ) ;
2007-03-06 23:43:32 +01:00
return theFlexTable . memoryStats ( ) ;
2006-10-24 15:48:16 +02:00
}
2008-01-06 20:23:38 +01:00
private final Map < String , String > memoryStats ( ) {
2007-03-06 23:43:32 +01:00
// returns statistical data about this object
2008-08-02 14:12:04 +02:00
final HashMap < String , String > map = new HashMap < String , String > ( ) ;
2007-06-26 16:37:10 +02:00
map . put ( " tableIndexChunkSize " , ( ! RAMIndex ) ? " 0 " : Integer . toString ( index . row ( ) . objectsize ) ) ;
map . put ( " tableIndexCount " , ( ! RAMIndex ) ? " 0 " : Integer . toString ( index . size ( ) ) ) ;
map . put ( " tableIndexMem " , ( ! RAMIndex ) ? " 0 " : Integer . toString ( ( int ) ( index . row ( ) . objectsize * index . size ( ) * kelondroRowCollection . growfactor ) ) ) ;
2007-03-06 23:43:32 +01:00
return map ;
2006-10-24 15:48:16 +02:00
}
2007-03-09 09:48:47 +01:00
public synchronized void close ( ) {
2007-03-06 23:43:32 +01:00
if ( tableTracker . remove ( this . filename ) = = null ) {
serverLog . logWarning ( " kelondroFlexTable " , " close(): file ' " + this . filename + " ' was not tracked with record tracker. " ) ;
}
2007-04-19 15:37:02 +02:00
if ( ( index ! = null ) & & ( this . size ( ) ! = ( ( index = = null ) ? 0 : index . size ( ) ) ) ) {
2008-07-24 10:22:36 +02:00
serverLog . logSevere ( " kelondroFlexTable " , this . filename + " close(): inconsistent content/index size. content.size() = " + this . size ( ) + " , index.size() = " + ( ( index = = null ) ? 0 : index . size ( ) ) ) ;
2007-04-19 15:37:02 +02:00
}
if ( index ! = null ) { index . close ( ) ; index = null ; }
2006-10-24 15:48:16 +02:00
super . close ( ) ;
}
2008-08-02 14:12:04 +02:00
public static void main ( final String [ ] args ) {
2007-03-09 09:48:47 +01:00
// open a file, add one entry and exit
2008-08-02 14:12:04 +02:00
final File f = new File ( args [ 0 ] ) ;
final String name = args [ 1 ] ;
final kelondroRow row = new kelondroRow ( " Cardinal key-4 {b256}, byte[] x-64 " , kelondroNaturalOrder . naturalOrder , 0 ) ;
2007-03-09 09:48:47 +01:00
try {
2008-08-02 14:12:04 +02:00
final kelondroFlexTable t = new kelondroFlexTable ( f , name , row , 0 , true ) ;
final kelondroRow . Entry entry = row . newEntry ( ) ;
2007-03-09 09:48:47 +01:00
entry . setCol ( 0 , System . currentTimeMillis ( ) ) ;
entry . setCol ( 1 , " dummy " . getBytes ( ) ) ;
t . put ( entry ) ;
t . close ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-03-09 09:48:47 +01:00
e . printStackTrace ( ) ;
}
}
2006-06-01 18:18:27 +02:00
}