2009-10-10 01:32:08 +02:00
// RowCollection.java
2008-07-20 19:14:51 +02:00
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
2006-06-20 16:17:21 +02:00
// first published 12.01.2006 on http://www.anomic.de
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2009-10-10 01:32:08 +02:00
package net.yacy.kelondro.index ;
2006-06-20 16:17:21 +02:00
2007-03-03 01:55:51 +01:00
import java.io.File ;
2007-02-27 16:54:02 +01:00
import java.io.IOException ;
2008-01-20 02:22:46 +01:00
import java.util.ArrayList ;
2006-06-20 16:17:21 +02:00
import java.util.Iterator ;
2007-02-27 16:54:02 +01:00
import java.util.List ;
2007-11-09 01:51:38 +01:00
import java.util.Random ;
2006-09-11 00:36:47 +02:00
import java.util.Set ;
2008-03-20 00:47:24 +01:00
import java.util.concurrent.Callable ;
import java.util.concurrent.ExecutionException ;
2008-03-19 12:23:43 +01:00
import java.util.concurrent.ExecutorService ;
import java.util.concurrent.Executors ;
2008-03-20 00:47:24 +01:00
import java.util.concurrent.Future ;
2006-06-20 16:17:21 +02:00
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2009-10-10 01:22:22 +02:00
import net.yacy.kelondro.order.Base64Order ;
import net.yacy.kelondro.order.ByteOrder ;
import net.yacy.kelondro.order.NaturalOrder ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.util.FileUtils ;
import net.yacy.kelondro.util.MemoryControl ;
import net.yacy.kelondro.util.NamePrefixThreadFactory ;
import net.yacy.kelondro.util.kelondroException ;
2009-10-10 01:13:30 +02:00
2006-10-24 15:48:16 +02:00
2009-01-30 16:33:00 +01:00
public class RowCollection implements Iterable < Row . Entry > {
2006-06-20 16:17:21 +02:00
2009-10-27 16:25:48 +01:00
public static final long growfactorLarge100 = 140L ;
public static final long growfactorSmall100 = 120L ;
2007-11-09 16:34:11 +01:00
private static final int isortlimit = 20 ;
2009-10-10 01:32:08 +02:00
private static int availableCPU = Runtime . getRuntime ( ) . availableProcessors ( ) ;
2007-11-09 16:34:11 +01:00
2009-10-10 01:32:08 +02:00
public static final ExecutorService sortingthreadexecutor = ( availableCPU > 1 ) ? Executors . newCachedThreadPool ( new NamePrefixThreadFactory ( " sorting " ) ) : null ;
public static final ExecutorService partitionthreadexecutor = ( availableCPU > 1 ) ? Executors . newCachedThreadPool ( new NamePrefixThreadFactory ( " partition " ) ) : null ;
2007-03-06 23:43:32 +01:00
2009-10-12 19:37:12 +02:00
public final Row rowdef ;
protected byte [ ] chunkcache ;
protected int chunkcount ;
protected int sortBound ;
protected long lastTimeWrote ;
2006-08-05 01:04:03 +02:00
private static final int exp_chunkcount = 0 ;
private static final int exp_last_read = 1 ;
private static final int exp_last_wrote = 2 ;
private static final int exp_order_type = 3 ;
2009-01-01 23:31:16 +01:00
private static final int exp_order_bound = 4 ;
private static final int exp_collection = 5 ;
2006-06-20 16:17:21 +02:00
2009-01-30 16:33:00 +01:00
public RowCollection ( final RowCollection rc ) {
2006-08-05 21:18:33 +02:00
this . rowdef = rc . rowdef ;
this . chunkcache = rc . chunkcache ;
this . chunkcount = rc . chunkcount ;
this . sortBound = rc . sortBound ;
this . lastTimeWrote = rc . lastTimeWrote ;
}
2010-01-09 01:08:16 +01:00
public RowCollection ( final Row rowdef ) {
2006-06-20 16:17:21 +02:00
this . rowdef = rowdef ;
this . sortBound = 0 ;
2006-06-21 18:05:31 +02:00
this . lastTimeWrote = System . currentTimeMillis ( ) ;
2010-01-09 01:08:16 +01:00
this . chunkcache = new byte [ 0 ] ;
this . chunkcount = 0 ;
}
public RowCollection ( final Row rowdef , final int objectCount ) throws RowSpaceExceededException {
this ( rowdef ) ;
ensureSize ( objectCount ) ;
2006-06-20 16:17:21 +02:00
}
2007-04-05 12:14:48 +02:00
2009-01-30 16:33:00 +01:00
public RowCollection ( final Row rowdef , final int objectCount , final byte [ ] cache , final int sortBound ) {
2006-06-20 16:17:21 +02:00
this . rowdef = rowdef ;
this . chunkcache = cache ;
this . chunkcount = objectCount ;
2006-08-05 01:04:03 +02:00
this . sortBound = sortBound ;
2006-06-21 18:05:31 +02:00
this . lastTimeWrote = System . currentTimeMillis ( ) ;
2006-06-20 16:17:21 +02:00
}
2009-01-30 16:33:00 +01:00
public RowCollection ( final Row rowdef , final Row . Entry exportedCollectionRowEnvironment ) {
2009-01-01 23:31:16 +01:00
final int chunkcachelength = exportedCollectionRowEnvironment . cellwidth ( 1 ) - exportOverheadSize ;
2009-01-30 16:33:00 +01:00
final Row . Entry exportedCollection = exportRow ( chunkcachelength ) . newEntry ( exportedCollectionRowEnvironment , 1 ) ;
2009-01-01 23:31:16 +01:00
2006-08-05 01:04:03 +02:00
this . rowdef = rowdef ;
2006-08-11 05:20:44 +02:00
this . chunkcount = ( int ) exportedCollection . getColLong ( exp_chunkcount ) ;
2006-11-07 11:56:36 +01:00
if ( ( this . chunkcount > chunkcachelength / rowdef . objectsize ) ) {
2009-01-31 00:33:47 +01:00
Log . logWarning ( " RowCollection " , " corrected wrong chunkcount; chunkcount = " + this . chunkcount + " , chunkcachelength = " + chunkcachelength + " , rowdef.objectsize = " + rowdef . objectsize ) ;
2006-11-07 11:56:36 +01:00
this . chunkcount = chunkcachelength / rowdef . objectsize ; // patch problem
}
2006-08-11 05:20:44 +02:00
this . lastTimeWrote = ( exportedCollection . getColLong ( exp_last_wrote ) + 10957 ) * day ;
2008-08-02 14:12:04 +02:00
final String sortOrderKey = exportedCollection . getColString ( exp_order_type , null ) ;
2009-01-30 16:33:00 +01:00
ByteOrder oldOrder = null ;
2006-10-06 01:47:08 +02:00
if ( ( sortOrderKey = = null ) | | ( sortOrderKey . equals ( " __ " ) ) ) {
2006-12-06 04:02:57 +01:00
oldOrder = null ;
2006-08-05 01:04:03 +02:00
} else {
2009-01-30 16:33:00 +01:00
oldOrder = NaturalOrder . bySignature ( sortOrderKey ) ;
if ( oldOrder = = null ) oldOrder = Base64Order . bySignature ( sortOrderKey ) ;
2006-08-05 01:04:03 +02:00
}
2006-12-06 04:02:57 +01:00
if ( ( rowdef . objectOrder ! = null ) & & ( oldOrder ! = null ) & & ( ! ( rowdef . objectOrder . signature ( ) . equals ( oldOrder . signature ( ) ) ) ) )
2007-11-07 23:38:09 +01:00
throw new kelondroException ( " old collection order does not match with new order; objectOrder.signature = " + rowdef . objectOrder . signature ( ) + " , oldOrder.signature = " + oldOrder . signature ( ) ) ;
2006-08-11 05:20:44 +02:00
this . sortBound = ( int ) exportedCollection . getColLong ( exp_order_bound ) ;
2006-10-26 15:50:50 +02:00
if ( sortBound > chunkcount ) {
2009-01-31 00:33:47 +01:00
Log . logWarning ( " RowCollection " , " corrected wrong sortBound; sortBound = " + sortBound + " , chunkcount = " + chunkcount ) ;
2006-10-26 15:50:50 +02:00
this . sortBound = chunkcount ;
}
2006-08-05 01:04:03 +02:00
this . chunkcache = exportedCollection . getColBytes ( exp_collection ) ;
}
2007-04-05 12:14:48 +02:00
public void reset ( ) {
this . chunkcache = new byte [ 0 ] ;
this . chunkcount = 0 ;
this . sortBound = 0 ;
}
2009-01-30 16:33:00 +01:00
private static final Row exportMeasureRow = exportRow ( 0 /* no relevance */ ) ;
2007-04-03 14:10:12 +02:00
2009-01-30 16:33:00 +01:00
public static final int sizeOfExportedCollectionRows ( final Row . Entry exportedCollectionRowEnvironment , final int columnInEnvironment ) {
final Row . Entry exportedCollectionEntry = exportMeasureRow . newEntry ( exportedCollectionRowEnvironment , columnInEnvironment ) ;
2008-08-02 14:12:04 +02:00
final int chunkcount = ( int ) exportedCollectionEntry . getColLong ( exp_chunkcount ) ;
2007-03-27 10:21:03 +02:00
return chunkcount ;
}
2006-08-05 01:04:03 +02:00
private static final long day = 1000 * 60 * 60 * 24 ;
2008-08-02 14:12:04 +02:00
public static int daysSince2000 ( final long time ) {
2006-08-05 01:04:03 +02:00
return ( int ) ( time / day ) - 10957 ;
}
2009-08-28 15:28:11 +02:00
private static Column exportColumn0 , exportColumn1 , exportColumn2 , exportColumn3 , exportColumn4 ;
2009-01-30 16:33:00 +01:00
private static Row exportRow ( final int chunkcachelength ) {
2009-08-28 15:28:11 +02:00
/ *
2009-01-30 16:33:00 +01:00
return new Row (
2006-08-05 01:04:03 +02:00
" int size-4 {b256}, " +
" short lastread-2 {b256}, " + // as daysSince2000
" short lastwrote-2 {b256}, " + // as daysSince2000
" byte[] orderkey-2, " +
2009-01-01 23:31:16 +01:00
" int orderbound-4 {b256}, " +
2006-12-06 04:02:57 +01:00
" byte[] collection- " + chunkcachelength ,
2009-03-13 17:52:31 +01:00
NaturalOrder . naturalOrder
2006-08-05 01:04:03 +02:00
) ;
2009-08-28 15:28:11 +02:00
* /
if ( exportColumn0 = = null ) exportColumn0 = new Column ( " int size-4 {b256} " ) ;
if ( exportColumn1 = = null ) exportColumn1 = new Column ( " short lastread-2 {b256} " ) ;
if ( exportColumn2 = = null ) exportColumn2 = new Column ( " short lastwrote-2 {b256} " ) ;
if ( exportColumn3 = = null ) exportColumn3 = new Column ( " byte[] orderkey-2 " ) ;
if ( exportColumn4 = = null ) exportColumn4 = new Column ( " int orderbound-4 {b256} " ) ;
/ *
* because of a strange bug these objects cannot be initialized as normal
* static final . If I try that , they are not initialized and are assigned null . why ?
* /
return new Row ( new Column [ ] {
exportColumn0 , exportColumn1 , exportColumn2 , exportColumn3 , exportColumn4 ,
new Column ( " byte[] collection- " + chunkcachelength )
} ,
NaturalOrder . naturalOrder
) ;
2006-08-05 01:04:03 +02:00
}
public static final int exportOverheadSize = 14 ;
2006-11-19 21:05:25 +01:00
2007-03-14 09:55:05 +01:00
public synchronized byte [ ] exportCollection ( ) {
2006-08-05 01:04:03 +02:00
// returns null if the collection is empty
2007-03-08 17:15:40 +01:00
trim ( false ) ;
2009-01-06 14:51:59 +01:00
assert this . size ( ) * this . rowdef . objectsize = = this . chunkcache . length : " this.size() = " + this . size ( ) + " , objectsize = " + this . rowdef . objectsize + " , chunkcache.length = " + this . chunkcache . length ;
2009-01-30 16:33:00 +01:00
final Row row = exportRow ( chunkcache . length ) ;
final Row . Entry entry = row . newEntry ( ) ;
2006-10-26 15:50:50 +02:00
assert ( sortBound < = chunkcount ) : " sortBound = " + sortBound + " , chunkcount = " + chunkcount ;
2006-11-08 03:08:33 +01:00
assert ( this . chunkcount < = chunkcache . length / rowdef . objectsize ) : " chunkcount = " + this . chunkcount + " , chunkcache.length = " + chunkcache . length + " , rowdef.objectsize = " + rowdef . objectsize ;
2006-10-26 15:50:50 +02:00
entry . setCol ( exp_chunkcount , this . chunkcount ) ;
2009-03-18 23:19:08 +01:00
entry . setCol ( exp_last_read , daysSince2000 ( System . currentTimeMillis ( ) ) ) ;
2006-08-11 05:20:44 +02:00
entry . setCol ( exp_last_wrote , daysSince2000 ( this . lastTimeWrote ) ) ;
2006-12-06 04:02:57 +01:00
entry . setCol ( exp_order_type , ( this . rowdef . objectOrder = = null ) ? " __ " . getBytes ( ) : this . rowdef . objectOrder . signature ( ) . getBytes ( ) ) ;
2006-08-11 05:20:44 +02:00
entry . setCol ( exp_order_bound , this . sortBound ) ;
2007-03-03 01:55:51 +01:00
entry . setCol ( exp_collection , this . chunkcache ) ;
2006-08-05 01:04:03 +02:00
return entry . bytes ( ) ;
}
2007-03-03 01:55:51 +01:00
2008-08-02 14:12:04 +02:00
public void saveCollection ( final File file ) throws IOException {
2009-01-31 02:06:56 +01:00
FileUtils . copy ( exportCollection ( ) , file ) ;
2007-03-03 01:55:51 +01:00
}
2006-11-19 21:05:25 +01:00
2009-01-30 16:33:00 +01:00
public Row row ( ) {
2006-06-30 14:54:19 +02:00
return this . rowdef ;
}
2010-01-10 02:40:26 +01:00
protected final long neededSpaceForEnsuredSize ( final int elements , final boolean forcegc ) {
2009-08-28 15:28:11 +02:00
assert elements > 0 : " elements = " + elements ;
final long needed = elements * rowdef . objectsize ;
2009-10-27 16:25:48 +01:00
if ( chunkcache . length > = needed ) return 0 ;
2009-08-28 15:28:11 +02:00
assert needed > 0 : " needed = " + needed ;
2009-10-27 16:25:48 +01:00
long allocram = needed * growfactorLarge100 / 100L ;
assert allocram > 0 : " elements = " + elements + " , new = " + allocram ;
if ( MemoryControl . request ( allocram , false ) ) return allocram ;
allocram = needed * growfactorSmall100 / 100L ;
assert allocram > 0 : " elements = " + elements + " , new = " + allocram ;
if ( MemoryControl . request ( allocram , forcegc ) ) return allocram ;
return needed ;
}
2009-12-10 00:27:26 +01:00
protected final void ensureSize ( final int elements ) throws RowSpaceExceededException {
2010-01-09 01:08:16 +01:00
if ( elements = = 0 ) return ;
2010-01-10 02:40:26 +01:00
final long allocram = neededSpaceForEnsuredSize ( elements , true ) ;
2009-10-27 16:25:48 +01:00
if ( allocram = = 0 ) return ;
assert allocram > chunkcache . length : " wrong alloc computation: allocram = " + allocram + " , chunkcache.length = " + chunkcache . length ;
2009-12-10 00:27:26 +01:00
if ( ! MemoryControl . request ( allocram , true ) ) throw new RowSpaceExceededException ( allocram , " RowCollection grow " ) ;
try {
2010-01-10 02:40:26 +01:00
final byte [ ] newChunkcache = new byte [ ( int ) allocram ] ; // increase space
2009-12-10 00:27:26 +01:00
System . arraycopy ( chunkcache , 0 , newChunkcache , 0 , chunkcache . length ) ;
chunkcache = newChunkcache ;
} catch ( OutOfMemoryError e ) {
throw new RowSpaceExceededException ( allocram , " RowCollection grow after OutOfMemoryError " + e . getMessage ( ) ) ;
}
2007-05-16 19:52:11 +02:00
}
2009-02-16 00:35:59 +01:00
/ * *
* compute the needed memory in case of a cache extension . That is , if the cache is full and must
* be copied into a new cache which is larger . In such a case the Collection needs more than the double size
2009-08-28 15:28:11 +02:00
* than is necessary to store the data . This method computes the extra memory that is needed to perform this task .
2009-02-16 00:35:59 +01:00
* @return
* /
2007-03-21 14:26:18 +01:00
public final long memoryNeededForGrow ( ) {
2009-10-27 16:25:48 +01:00
return neededSpaceForEnsuredSize ( chunkcount + 1 , false ) ;
2007-03-21 14:26:18 +01:00
}
2008-08-02 14:12:04 +02:00
public synchronized void trim ( final boolean plusGrowFactor ) {
2007-11-09 01:51:38 +01:00
if ( chunkcache . length = = 0 ) return ;
2009-10-27 16:25:48 +01:00
long needed = chunkcount * rowdef . objectsize ;
if ( plusGrowFactor ) needed = neededSpaceForEnsuredSize ( chunkcount , false ) ;
2007-03-14 09:55:05 +01:00
if ( needed > = chunkcache . length )
return ; // in case that the growfactor causes that the cache would
// grow instead of shrink, simply ignore the growfactor
2009-01-30 16:33:00 +01:00
if ( MemoryControl . available ( ) + 1000 < needed )
2007-03-14 09:55:05 +01:00
return ; // if the swap buffer is not available, we must give up.
2008-04-24 17:09:06 +02:00
// This is not critical. Otherwise we provoke a serious
2007-03-14 09:55:05 +01:00
// problem with OOM
2010-01-10 02:40:26 +01:00
final byte [ ] newChunkcache = new byte [ ( int ) needed ] ;
2007-03-14 09:55:05 +01:00
System . arraycopy ( chunkcache , 0 , newChunkcache , 0 , Math . min (
chunkcache . length , newChunkcache . length ) ) ;
chunkcache = newChunkcache ;
2006-06-20 16:17:21 +02:00
}
2006-06-21 00:13:17 +02:00
public final long lastWrote ( ) {
2006-06-20 16:17:21 +02:00
return lastTimeWrote ;
}
2008-08-02 14:12:04 +02:00
public synchronized final byte [ ] getKey ( final int index ) {
2007-12-27 18:56:59 +01:00
assert ( index > = 0 ) : " get: access with index " + index + " is below zero " ;
assert ( index < chunkcount ) : " get: access with index " + index + " is above chunkcount " + chunkcount + " ; sortBound = " + sortBound ;
assert ( index * rowdef . objectsize < chunkcache . length ) ;
if ( ( chunkcache = = null ) | | ( rowdef = = null ) ) return null ; // case may appear during shutdown
if ( index > = chunkcount ) return null ;
2008-04-14 05:49:02 +02:00
if ( ( index + 1 ) * rowdef . objectsize > chunkcache . length ) return null ; // the whole chunk does not fit into the chunkcache
2008-08-02 14:12:04 +02:00
final byte [ ] b = new byte [ this . rowdef . width ( 0 ) ] ;
2007-12-27 18:56:59 +01:00
System . arraycopy ( chunkcache , index * rowdef . objectsize , b , 0 , b . length ) ;
return b ;
}
2009-01-30 16:33:00 +01:00
public synchronized final Row . Entry get ( final int index , final boolean clone ) {
2006-10-21 12:50:30 +02:00
assert ( index > = 0 ) : " get: access with index " + index + " is below zero " ;
2006-10-26 15:50:50 +02:00
assert ( index < chunkcount ) : " get: access with index " + index + " is above chunkcount " + chunkcount + " ; sortBound = " + sortBound ;
2008-08-20 10:37:39 +02:00
assert ( chunkcache ! = null & & index * rowdef . objectsize < chunkcache . length ) ;
2008-06-15 00:51:47 +02:00
assert sortBound < = chunkcount : " sortBound = " + sortBound + " , chunkcount = " + chunkcount ;
2007-09-04 01:43:55 +02:00
if ( ( chunkcache = = null ) | | ( rowdef = = null ) ) return null ; // case may appear during shutdown
2009-01-30 16:33:00 +01:00
Row . Entry entry ;
2008-08-02 14:12:04 +02:00
final int addr = index * rowdef . objectsize ;
2008-04-24 17:09:06 +02:00
synchronized ( this ) {
if ( index > = chunkcount ) return null ;
if ( addr + rowdef . objectsize > chunkcache . length ) return null ; // the whole chunk does not fit into the chunkcache
entry = rowdef . newEntry ( chunkcache , addr , clone ) ;
}
return entry ;
2006-06-20 16:17:21 +02:00
}
2009-12-10 00:27:26 +01:00
public synchronized final void set ( final int index , final Row . Entry a ) throws RowSpaceExceededException {
2007-09-04 01:43:55 +02:00
assert ( index > = 0 ) : " set: access with index " + index + " is below zero " ;
ensureSize ( index + 1 ) ;
2010-01-10 02:40:26 +01:00
final boolean sameKey = match ( a . bytes ( ) , 0 , a . cellwidth ( 0 ) , index ) ;
2009-01-29 17:42:01 +01:00
//if (sameKey) System.out.print("$");
2007-11-07 23:38:09 +01:00
a . writeToArray ( chunkcache , index * rowdef . objectsize ) ;
2009-01-01 23:31:16 +01:00
if ( index > = this . chunkcount ) this . chunkcount = index + 1 ;
2009-01-29 17:42:01 +01:00
if ( ! sameKey & & index < this . sortBound ) this . sortBound = index ;
2006-06-20 16:17:21 +02:00
this . lastTimeWrote = System . currentTimeMillis ( ) ;
}
2009-12-10 00:27:26 +01:00
public final void insertUnique ( final int index , final Row . Entry a ) throws RowSpaceExceededException {
2007-09-04 01:43:55 +02:00
assert ( a ! = null ) ;
if ( index < chunkcount ) {
// make room
ensureSize ( chunkcount + 1 ) ;
2007-11-07 23:38:09 +01:00
System . arraycopy ( chunkcache , rowdef . objectsize * index , chunkcache , rowdef . objectsize * ( index + 1 ) , ( chunkcount - index ) * rowdef . objectsize ) ;
2007-09-04 01:43:55 +02:00
chunkcount + + ;
}
// insert entry into gap
set ( index , a ) ;
}
2009-12-10 00:27:26 +01:00
public synchronized void addUnique ( final Row . Entry row ) throws RowSpaceExceededException {
2008-08-02 14:12:04 +02:00
final byte [ ] r = row . bytes ( ) ;
2008-11-24 00:55:08 +01:00
addUnique ( r , 0 , r . length ) ;
2006-06-20 16:17:21 +02:00
}
2006-10-19 23:14:37 +02:00
2009-12-10 00:27:26 +01:00
public synchronized void addUnique ( final List < Row . Entry > rows ) throws RowSpaceExceededException {
2007-05-16 12:48:26 +02:00
assert this . sortBound = = 0 : " sortBound = " + this . sortBound + " , chunkcount = " + this . chunkcount ;
2009-01-30 16:33:00 +01:00
final Iterator < Row . Entry > i = rows . iterator ( ) ;
2008-11-24 00:55:08 +01:00
while ( i . hasNext ( ) ) addUnique ( i . next ( ) ) ;
2007-02-27 16:54:02 +01:00
}
2009-12-10 00:27:26 +01:00
public synchronized void add ( final byte [ ] a ) throws RowSpaceExceededException {
2009-04-22 00:12:19 +02:00
assert a . length = = this . rowdef . objectsize : " a.length = " + a . length + " , objectsize = " + this . rowdef . objectsize ;
2006-10-30 03:39:39 +01:00
addUnique ( a , 0 , a . length ) ;
2006-06-21 01:47:51 +02:00
}
2009-12-10 00:27:26 +01:00
private final void addUnique ( final byte [ ] a , final int astart , final int alength ) throws RowSpaceExceededException {
2006-10-24 15:48:16 +02:00
assert ( a ! = null ) ;
2008-08-06 21:43:12 +02:00
assert ( astart > = 0 ) & & ( astart < a . length ) : " astart = " + astart ;
2009-01-31 00:33:47 +01:00
assert ( ! ( Log . allZero ( a , astart , alength ) ) ) : " a = " + NaturalOrder . arrayList ( a , astart , alength ) ;
2006-10-24 15:48:16 +02:00
assert ( alength > 0 ) ;
assert ( astart + alength < = a . length ) ;
2009-01-02 12:38:20 +01:00
assert alength = = rowdef . objectsize : " alength = " + alength + " , rowdef.objectsize = " + rowdef . objectsize ;
2008-08-02 14:12:04 +02:00
final int l = Math . min ( rowdef . objectsize , Math . min ( alength , a . length - astart ) ) ;
2007-03-14 09:55:05 +01:00
ensureSize ( chunkcount + 1 ) ;
2007-11-07 23:38:09 +01:00
System . arraycopy ( a , astart , chunkcache , rowdef . objectsize * chunkcount , l ) ;
2007-03-14 09:55:05 +01:00
chunkcount + + ;
2009-01-01 23:31:16 +01:00
// if possible, increase the sortbound value to suppress unnecessary sorting
if ( this . chunkcount = = 1 ) {
assert this . sortBound = = 0 ;
this . sortBound = 1 ;
} else if (
this . sortBound + 1 = = chunkcount & &
this . rowdef . objectOrder . compare ( chunkcache , rowdef . objectsize * ( chunkcount - 2 ) , rowdef . primaryKeyLength ,
chunkcache , rowdef . objectsize * ( chunkcount - 1 ) , rowdef . primaryKeyLength ) = = - 1 ) {
this . sortBound = chunkcount ;
}
2006-06-20 16:17:21 +02:00
this . lastTimeWrote = System . currentTimeMillis ( ) ;
}
2008-05-13 17:28:55 +02:00
2009-12-10 00:27:26 +01:00
protected final void addSorted ( final byte [ ] a , final int astart , final int alength ) throws RowSpaceExceededException {
2009-01-02 12:38:20 +01:00
assert ( a ! = null ) ;
assert ( astart > = 0 ) & & ( astart < a . length ) : " astart = " + astart ;
2009-01-31 00:33:47 +01:00
assert ( ! ( Log . allZero ( a , astart , alength ) ) ) : " a = " + NaturalOrder . arrayList ( a , astart , alength ) ;
2009-01-02 12:38:20 +01:00
assert ( alength > 0 ) ;
assert ( astart + alength < = a . length ) ;
assert alength = = rowdef . objectsize : " alength = " + alength + " , rowdef.objectsize = " + rowdef . objectsize ;
final int l = Math . min ( rowdef . objectsize , Math . min ( alength , a . length - astart ) ) ;
ensureSize ( chunkcount + 1 ) ;
System . arraycopy ( a , astart , chunkcache , rowdef . objectsize * chunkcount , l ) ;
this . chunkcount + + ;
this . sortBound = this . chunkcount ;
this . lastTimeWrote = System . currentTimeMillis ( ) ;
}
2009-12-10 00:27:26 +01:00
public synchronized final void addAllUnique ( final RowCollection c ) throws RowSpaceExceededException {
2006-12-07 03:40:57 +01:00
if ( c = = null ) return ;
2007-11-07 23:38:09 +01:00
assert ( rowdef . objectsize = = c . rowdef . objectsize ) ;
2007-03-14 09:55:05 +01:00
ensureSize ( chunkcount + c . size ( ) ) ;
2007-11-07 23:38:09 +01:00
System . arraycopy ( c . chunkcache , 0 , chunkcache , rowdef . objectsize * chunkcount , rowdef . objectsize * c . size ( ) ) ;
2007-03-14 09:55:05 +01:00
chunkcount + = c . size ( ) ;
2006-06-20 16:17:21 +02:00
}
2006-11-30 01:23:07 +01:00
2007-10-11 19:17:08 +02:00
/ * *
* This method removes the entry at position p ensuring the order of the remaining
* entries if specified by keepOrder .
* Note : Keeping the order is expensive . If you want to remove more than one element in
* a batch with this method , it ' d be better to do the removes without order keeping and doing
* the sort after all the removes are done .
*
* @param p element at this position will be removed
* @param keepOrder keep the order of remaining entries
* /
2009-01-30 16:33:00 +01:00
public synchronized final void removeRow ( final int p , final boolean keepOrder ) {
2007-04-19 15:37:02 +02:00
assert p > = 0 : " p = " + p ;
assert p < chunkcount : " p = " + p + " , chunkcount = " + chunkcount ;
assert chunkcount > 0 : " chunkcount = " + chunkcount ;
assert sortBound < = chunkcount : " sortBound = " + sortBound + " , chunkcount = " + chunkcount ;
2007-10-11 19:17:08 +02:00
if ( keepOrder & & ( p < sortBound ) ) {
// remove by shift (quite expensive for big collections)
2008-08-02 14:12:04 +02:00
final int addr = p * this . rowdef . objectsize ;
2007-10-11 19:17:08 +02:00
System . arraycopy (
2008-06-15 00:51:47 +02:00
chunkcache , addr + this . rowdef . objectsize ,
chunkcache , addr ,
2007-11-07 23:38:09 +01:00
( chunkcount - p - 1 ) * this . rowdef . objectsize ) ;
2008-06-15 00:51:47 +02:00
sortBound - - ; // this is only correct if p < sortBound, but this was already checked above
2007-03-14 09:55:05 +01:00
} else {
2007-10-11 19:17:08 +02:00
// remove by copying the top-element to the remove position
if ( p ! = chunkcount - 1 ) {
System . arraycopy (
2007-11-07 23:38:09 +01:00
chunkcache , ( chunkcount - 1 ) * this . rowdef . objectsize ,
chunkcache , p * this . rowdef . objectsize ,
this . rowdef . objectsize ) ;
2007-10-11 19:17:08 +02:00
}
// we moved the last element to the remove position: (p+1)st element
2007-10-22 01:26:22 +02:00
// only the first p elements keep their order (element p is already outside the order)
2008-06-15 00:51:47 +02:00
if ( sortBound > p ) sortBound = p ;
2006-06-20 16:17:21 +02:00
}
2007-03-14 09:55:05 +01:00
chunkcount - - ;
2006-06-20 16:17:21 +02:00
this . lastTimeWrote = System . currentTimeMillis ( ) ;
}
2008-10-24 15:58:26 +02:00
/ * *
* removes the last entry from the collection
* @return
* /
2009-01-30 16:33:00 +01:00
public synchronized Row . Entry removeOne ( ) {
2007-03-14 09:55:05 +01:00
if ( chunkcount = = 0 ) return null ;
2009-01-30 16:33:00 +01:00
final Row . Entry r = get ( chunkcount - 1 , true ) ;
2007-03-14 09:55:05 +01:00
if ( chunkcount = = sortBound ) sortBound - - ;
chunkcount - - ;
this . lastTimeWrote = System . currentTimeMillis ( ) ;
return r ;
2006-06-20 16:17:21 +02:00
}
2009-07-03 18:35:34 +02:00
public synchronized byte [ ] smallestKey ( ) {
if ( chunkcount = = 0 ) return null ;
this . sort ( ) ;
final Row . Entry r = get ( 0 , false ) ;
2010-01-10 02:40:26 +01:00
final byte [ ] b = r . getPrimaryKeyBytes ( ) ;
2009-07-03 18:35:34 +02:00
return b ;
}
public synchronized byte [ ] largestKey ( ) {
if ( chunkcount = = 0 ) return null ;
this . sort ( ) ;
final Row . Entry r = get ( chunkcount - 1 , false ) ;
2010-01-10 02:40:26 +01:00
final byte [ ] b = r . getPrimaryKeyBytes ( ) ;
2009-07-03 18:35:34 +02:00
return b ;
}
2007-03-14 09:55:05 +01:00
public synchronized void clear ( ) {
2007-05-16 12:48:26 +02:00
if ( this . chunkcache . length = = 0 ) return ;
2006-06-20 16:17:21 +02:00
this . chunkcache = new byte [ 0 ] ;
this . chunkcount = 0 ;
this . sortBound = 0 ;
2006-06-21 18:05:31 +02:00
this . lastTimeWrote = System . currentTimeMillis ( ) ;
2006-06-20 16:17:21 +02:00
}
2006-06-21 01:18:41 +02:00
public int size ( ) {
2009-01-02 12:38:20 +01:00
return this . chunkcount ;
}
2009-12-02 01:37:59 +01:00
public boolean isEmpty ( ) {
return this . chunkcount = = 0 ;
}
2009-01-02 12:38:20 +01:00
public int sorted ( ) {
return this . sortBound ;
2006-06-20 16:17:21 +02:00
}
2010-01-10 02:40:26 +01:00
public synchronized Iterator < byte [ ] > keys ( final boolean keepOrderWhenRemoving ) {
2007-12-27 18:56:59 +01:00
// iterates byte[] - type entries
2009-01-09 01:06:36 +01:00
return new keyIterator ( keepOrderWhenRemoving ) ;
2007-12-27 18:56:59 +01:00
}
/ * *
* Iterator for kelondroRowCollection .
* It supports remove ( ) though it doesn ' t contain the order of the underlying
* collection during removes .
*
* /
public class keyIterator implements Iterator < byte [ ] > {
private int p ;
2009-01-09 01:06:36 +01:00
private boolean keepOrderWhenRemoving ;
2007-12-27 18:56:59 +01:00
2010-01-10 02:40:26 +01:00
public keyIterator ( final boolean keepOrderWhenRemoving ) {
2009-01-09 01:06:36 +01:00
this . p = 0 ;
this . keepOrderWhenRemoving = keepOrderWhenRemoving ;
2007-12-27 18:56:59 +01:00
}
public boolean hasNext ( ) {
return p < chunkcount ;
}
public byte [ ] next ( ) {
return getKey ( p + + ) ;
}
public void remove ( ) {
p - - ;
2009-01-09 01:06:36 +01:00
removeRow ( p , keepOrderWhenRemoving ) ;
2007-12-27 18:56:59 +01:00
}
2009-01-02 12:38:20 +01:00
}
/ * *
* return an iterator for the row entries in this object
* /
2009-01-30 16:33:00 +01:00
public Iterator < Row . Entry > iterator ( ) {
2007-03-03 01:55:51 +01:00
// iterates kelondroRow.Entry - type entries
2006-07-04 16:47:27 +02:00
return new rowIterator ( ) ;
2006-06-20 16:17:21 +02:00
}
2007-10-11 19:17:08 +02:00
/ * *
* Iterator for kelondroRowCollection .
2009-01-09 01:06:36 +01:00
* It supports remove ( ) and keeps the order of the underlying
2007-10-11 19:17:08 +02:00
* collection during removes .
* /
2009-01-30 16:33:00 +01:00
public class rowIterator implements Iterator < Row . Entry > {
2006-06-20 16:17:21 +02:00
2006-07-04 16:47:27 +02:00
private int p ;
2006-06-20 16:17:21 +02:00
2006-07-04 16:47:27 +02:00
public rowIterator ( ) {
p = 0 ;
2006-06-20 16:17:21 +02:00
}
public boolean hasNext ( ) {
2006-07-04 16:47:27 +02:00
return p < chunkcount ;
2006-06-20 16:17:21 +02:00
}
2009-01-30 16:33:00 +01:00
public Row . Entry next ( ) {
2008-04-24 15:31:55 +02:00
return get ( p + + , true ) ;
2006-06-20 16:17:21 +02:00
}
2006-07-04 16:47:27 +02:00
2006-06-20 16:17:21 +02:00
public void remove ( ) {
2006-07-04 16:47:27 +02:00
p - - ;
2009-01-09 01:06:36 +01:00
removeRow ( p , true ) ;
2006-06-20 16:17:21 +02:00
}
2009-01-02 12:38:20 +01:00
2006-06-20 16:17:21 +02:00
}
2006-07-04 16:47:27 +02:00
2008-08-02 14:12:04 +02:00
public synchronized void select ( final Set < String > keys ) {
2006-09-11 00:36:47 +02:00
// removes all entries but the ones given by urlselection
2008-04-06 13:50:15 +02:00
if ( ( keys = = null ) | | ( keys . isEmpty ( ) ) ) return ;
2009-01-30 16:33:00 +01:00
final Iterator < Row . Entry > i = iterator ( ) ;
Row . Entry row ;
2007-03-14 09:55:05 +01:00
while ( i . hasNext ( ) ) {
2008-06-06 18:01:27 +02:00
row = i . next ( ) ;
2007-03-14 09:55:05 +01:00
if ( ! ( keys . contains ( row . getColString ( 0 , null ) ) ) ) i . remove ( ) ;
2006-09-11 00:36:47 +02:00
}
}
2006-07-04 16:47:27 +02:00
2007-03-14 09:55:05 +01:00
public synchronized final void sort ( ) {
2006-12-06 04:02:57 +01:00
assert ( this . rowdef . objectOrder ! = null ) ;
2006-06-20 16:17:21 +02:00
if ( this . sortBound = = this . chunkcount ) return ; // this is already sorted
2007-11-09 16:34:11 +01:00
if ( this . chunkcount < isortlimit ) {
isort ( 0 , this . chunkcount , new byte [ this . rowdef . objectsize ] ) ;
2007-11-13 03:24:10 +01:00
this . sortBound = this . chunkcount ;
2007-11-16 15:48:09 +01:00
assert this . isSorted ( ) ;
2007-11-09 16:34:11 +01:00
return ;
}
2008-08-02 14:12:04 +02:00
final byte [ ] swapspace = new byte [ this . rowdef . objectsize ] ;
final int p = partition ( 0 , this . chunkcount , this . sortBound , swapspace ) ;
2009-10-06 11:56:14 +02:00
if ( sortingthreadexecutor ! = null & &
! sortingthreadexecutor . isShutdown ( ) & &
2009-10-10 01:32:08 +02:00
availableCPU > 1 & &
2009-10-06 11:56:14 +02:00
this . chunkcount > 8000 & &
p > isortlimit * 5 & &
this . chunkcount - p > isortlimit * 5
) {
2008-02-27 16:16:47 +01:00
// sort this using multi-threading
2008-08-02 14:12:04 +02:00
final Future < Integer > part0 = partitionthreadexecutor . submit ( new partitionthread ( this , 0 , p , 0 ) ) ;
final Future < Integer > part1 = partitionthreadexecutor . submit ( new partitionthread ( this , p , this . chunkcount , p ) ) ;
2008-06-16 01:25:57 +02:00
try {
2008-08-02 14:12:04 +02:00
final int p0 = part0 . get ( ) . intValue ( ) ;
final Future < Object > sort0 = sortingthreadexecutor . submit ( new qsortthread ( this , 0 , p0 , 0 ) ) ;
final Future < Object > sort1 = sortingthreadexecutor . submit ( new qsortthread ( this , p0 , p , p0 ) ) ;
final int p1 = part1 . get ( ) . intValue ( ) ;
final Future < Object > sort2 = sortingthreadexecutor . submit ( new qsortthread ( this , p , p1 , p ) ) ;
final Future < Object > sort3 = sortingthreadexecutor . submit ( new qsortthread ( this , p1 , this . chunkcount , p1 ) ) ;
2008-06-16 01:25:57 +02:00
sort0 . get ( ) ;
sort1 . get ( ) ;
sort2 . get ( ) ;
sort3 . get ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final InterruptedException e ) {
2009-09-29 23:52:17 +02:00
Log . logSevere ( " RowCollection " , " " , e ) ;
2008-08-02 14:12:04 +02:00
} catch ( final ExecutionException e ) {
2009-09-29 23:52:17 +02:00
Log . logSevere ( " RowCollection " , " " , e ) ;
2008-06-16 01:25:57 +02:00
}
} else {
qsort ( 0 , p , 0 , swapspace ) ;
qsort ( p + 1 , this . chunkcount , 0 , swapspace ) ;
}
this . sortBound = this . chunkcount ;
//assert this.isSorted();
}
public synchronized final void sort2 ( ) {
assert ( this . rowdef . objectOrder ! = null ) ;
if ( this . sortBound = = this . chunkcount ) return ; // this is already sorted
if ( this . chunkcount < isortlimit ) {
isort ( 0 , this . chunkcount , new byte [ this . rowdef . objectsize ] ) ;
this . sortBound = this . chunkcount ;
assert this . isSorted ( ) ;
return ;
}
2008-08-02 14:12:04 +02:00
final byte [ ] swapspace = new byte [ this . rowdef . objectsize ] ;
final int p = partition ( 0 , this . chunkcount , this . sortBound , swapspace ) ;
2008-06-16 01:25:57 +02:00
if ( ( sortingthreadexecutor ! = null ) & &
( ! sortingthreadexecutor . isShutdown ( ) ) & &
2009-10-10 01:32:08 +02:00
( availableCPU > 1 ) & &
2008-06-16 01:25:57 +02:00
( this . chunkcount > 4000 ) ) {
// sort this using multi-threading
2008-08-02 14:12:04 +02:00
final Future < Object > part = sortingthreadexecutor . submit ( new qsortthread ( this , 0 , p , 0 ) ) ;
2008-03-21 00:11:04 +01:00
//CompletionService<Object> sortingthreadcompletion = new ExecutorCompletionService<Object>(sortingthreadexecutor);
//Future<Object> part = sortingthreadcompletion.submit(new qsortthread(this, 0, p, 0));
2008-06-16 01:25:57 +02:00
qsort ( p + 1 , this . chunkcount , 0 , swapspace ) ;
try {
2008-03-20 00:47:24 +01:00
part . get ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final InterruptedException e ) {
2009-09-29 23:52:17 +02:00
Log . logSevere ( " RowCollection " , " " , e ) ;
2008-08-02 14:12:04 +02:00
} catch ( final ExecutionException e ) {
2009-09-29 23:52:17 +02:00
Log . logSevere ( " RowCollection " , " " , e ) ;
2008-03-20 00:47:24 +01:00
}
2007-11-09 16:34:11 +01:00
} else {
2008-06-16 01:25:57 +02:00
qsort ( 0 , p , 0 , swapspace ) ;
qsort ( p + 1 , this . chunkcount , 0 , swapspace ) ;
2007-11-09 16:34:11 +01:00
}
2007-11-09 01:51:38 +01:00
this . sortBound = this . chunkcount ;
2007-11-12 02:14:51 +01:00
//assert this.isSorted();
2006-06-20 16:17:21 +02:00
}
2008-03-20 00:47:24 +01:00
public static class qsortthread implements Callable < Object > {
2009-01-30 16:33:00 +01:00
RowCollection rc ;
2008-03-19 12:23:43 +01:00
int L , R , S ;
2009-01-30 16:33:00 +01:00
public qsortthread ( final RowCollection rc , final int L , final int R , final int S ) {
2008-03-19 12:23:43 +01:00
this . rc = rc ;
this . L = L ;
this . R = R ;
this . S = S ;
2008-02-27 16:16:47 +01:00
}
2008-03-19 12:23:43 +01:00
2008-03-20 00:47:24 +01:00
public Object call ( ) throws Exception {
2008-03-19 12:23:43 +01:00
rc . qsort ( L , R , S , new byte [ rc . rowdef . objectsize ] ) ;
2008-03-20 00:47:24 +01:00
return null ;
}
2007-11-09 01:51:38 +01:00
}
2008-08-02 14:12:04 +02:00
final void qsort ( final int L , final int R , final int S , final byte [ ] swapspace ) {
2007-11-09 16:34:11 +01:00
if ( R - L < isortlimit ) {
2007-11-09 01:51:38 +01:00
isort ( L , R , swapspace ) ;
2007-11-09 16:34:11 +01:00
return ;
}
2008-03-23 00:15:28 +01:00
assert R > L : " L = " + L + " , R = " + R + " , S = " + S ;
2008-08-02 14:12:04 +02:00
final int p = partition ( L , R , S , swapspace ) ;
2008-03-23 00:15:28 +01:00
assert p > = L : " L = " + L + " , R = " + R + " , S = " + S + " , p = " + p ;
assert p < R : " L = " + L + " , R = " + R + " , S = " + S + " , p = " + p ;
2007-11-09 16:34:11 +01:00
qsort ( L , p , 0 , swapspace ) ;
2008-03-23 00:15:28 +01:00
qsort ( p + 1 , R , 0 , swapspace ) ;
2007-11-09 16:34:11 +01:00
}
2008-03-20 23:01:12 +01:00
2008-06-16 01:25:57 +02:00
public static class partitionthread implements Callable < Integer > {
2009-01-30 16:33:00 +01:00
RowCollection rc ;
2008-06-16 01:25:57 +02:00
int L , R , S ;
2009-01-30 16:33:00 +01:00
public partitionthread ( final RowCollection rc , final int L , final int R , final int S ) {
2008-06-16 01:25:57 +02:00
this . rc = rc ;
this . L = L ;
this . R = R ;
this . S = S ;
}
public Integer call ( ) throws Exception {
2008-08-06 21:43:12 +02:00
return Integer . valueOf ( rc . partition ( L , R , S , new byte [ rc . rowdef . objectsize ] ) ) ;
2008-06-16 01:25:57 +02:00
}
}
2008-10-24 15:58:26 +02:00
/ * *
* @param L is the first element in the sequence
* @param R is the right bound of the sequence , and outside of the sequence
* @param S is the bound of the sorted elements in the sequence
* @param swapspace
* @return
* /
2008-08-02 14:12:04 +02:00
final int partition ( final int L , final int R , int S , final byte [ ] swapspace ) {
2008-03-23 00:15:28 +01:00
assert ( L < R - 1 ) : " L = " + L + " , R = " + R + " , S = " + S ;
2009-06-02 00:45:28 +02:00
assert ( R - L > = isortlimit ) : " L = " + L + " , R = " + R + " , S = " + S + " , isortlimit = " + isortlimit ;
2006-06-20 16:17:21 +02:00
2007-11-09 01:51:38 +01:00
int p = L ;
int q = R - 1 ;
2009-06-30 11:27:46 +02:00
int pivot = pivot ( L , R , S ) ;
2009-01-30 16:33:00 +01:00
if ( this . rowdef . objectOrder instanceof Base64Order ) {
2007-11-09 01:51:38 +01:00
while ( p < = q ) {
2007-11-09 16:34:11 +01:00
// wenn pivot < S: pivot befindet sich in sortierter Sequenz von L bis S - 1
// d.h. alle Werte von L bis pivot sind kleiner als das pivot
2009-04-22 00:12:19 +02:00
// zu finden ist ein minimales p <= q so dass chunk[p] >= pivot
2007-11-09 16:34:11 +01:00
if ( ( pivot < S ) & & ( p < pivot ) ) {
//System.out.println("+++ saved " + (pivot - p) + " comparisments");
p = pivot ;
S = 0 ;
} else {
2009-04-22 00:12:19 +02:00
while ( ( p < R - 1 ) & & ( compare ( pivot , p ) > = 0 ) ) p + + ; // chunkAt[p] < pivot
2007-11-09 16:34:11 +01:00
}
// nun gilt chunkAt[p] >= pivot
2009-04-22 00:12:19 +02:00
while ( ( q > L ) & & ( compare ( pivot , q ) < = 0 ) ) q - - ; // chunkAt[q] > pivot
2007-11-09 01:51:38 +01:00
if ( p < = q ) {
pivot = swap ( p , q , pivot , swapspace ) ;
p + + ;
q - - ;
}
}
} else {
while ( p < = q ) {
2007-11-09 16:34:11 +01:00
if ( ( pivot < S ) & & ( p < pivot ) ) {
p = pivot ;
S = 0 ;
} else {
2008-03-23 00:15:28 +01:00
while ( ( p < R - 1 ) & & ( compare ( pivot , p ) > = 0 ) ) p + + ; // chunkAt[p] < pivot
2007-11-09 16:34:11 +01:00
}
2008-03-23 00:15:28 +01:00
while ( ( q > L ) & & ( compare ( pivot , q ) < = 0 ) ) q - - ; // chunkAt[q] > pivot
2007-11-09 01:51:38 +01:00
if ( p < = q ) {
pivot = swap ( p , q , pivot , swapspace ) ;
p + + ;
q - - ;
}
}
2006-06-20 16:17:21 +02:00
}
2008-03-23 00:15:28 +01:00
// now p is the beginning of the upper sequence
// finally, the pivot element should be exactly between the two sequences
// distinguish two cases: pivot in lower and upper sequence
// to do this it is sufficient to compare the index, not the entry content
if ( pivot < p ) {
// switch the pivot with the element _below_ p, the element in p belongs to the upper sequence
// and does not fit into the lower sequence
swap ( pivot , p - 1 , pivot , swapspace ) ;
return p - 1 ;
} else if ( pivot > p ) {
// switch the pivot with p, they are both in the same sequence
swap ( pivot , p , pivot , swapspace ) ;
return p ;
}
assert pivot = = p ;
2007-11-09 01:51:38 +01:00
return p ;
2006-06-20 16:17:21 +02:00
}
2007-11-09 01:51:38 +01:00
2009-06-30 11:27:46 +02:00
private final int pivot ( final int L , final int R , final int S ) {
2008-03-20 23:01:12 +01:00
if ( ( S = = 0 ) | | ( S < L ) ) {
// the collection has no ordering
// or
// the collection has an ordering, but this is not relevant for this pivot
// because the ordered zone is outside of ordering zone
2008-08-02 14:12:04 +02:00
final int m = picMiddle ( new int [ ] { L , ( 3 * L + R - 1 ) / 4 , ( L + R - 1 ) / 2 , ( L + 3 * R - 3 ) / 4 , R - 1 } , 5 ) ;
2008-03-20 23:01:12 +01:00
assert L < = m ;
assert m < R ;
return m ;
}
if ( S < R ) {
// the collection has an ordering
// and part of the ordered zone is inside the to-be-ordered zone
2008-08-02 14:12:04 +02:00
final int m = picMiddle ( new int [ ] { L , L + ( S - L ) / 3 , ( L + R - 1 ) / 2 , S , R - 1 } , 5 ) ;
2008-03-20 23:01:12 +01:00
assert L < = m ;
assert m < R ;
return m ;
}
// use the sorted set to find good pivot:
// the sort range is fully inside the sorted area:
// the middle element must be the best
2008-03-21 00:11:04 +01:00
// (however, it should be skipped because there is no point in sorting this)
2008-03-20 23:01:12 +01:00
return ( L + R - 1 ) / 2 ;
}
2008-08-02 14:12:04 +02:00
private final int picMiddle ( final int [ ] list , int len ) {
2008-08-06 21:43:12 +02:00
assert len % 2 ! = 0 ;
2008-03-20 23:01:12 +01:00
assert len < = list . length ;
2008-08-02 14:12:04 +02:00
final int cut = list . length / 2 ;
2008-03-20 23:01:12 +01:00
for ( int i = 0 ; i < cut ; i + + ) { remove ( list , len , min ( list , len ) ) ; len - - ; }
for ( int i = 0 ; i < cut ; i + + ) { remove ( list , len , max ( list , len ) ) ; len - - ; }
// the remaining element must be the middle element
assert len = = 1 ;
return list [ 0 ] ;
}
2008-08-02 14:12:04 +02:00
private final void remove ( final int [ ] list , final int len , final int idx ) {
2008-03-20 23:01:12 +01:00
if ( idx = = len - 1 ) return ;
list [ idx ] = list [ len - 1 ] ; // shift last element to front
}
2008-08-02 14:12:04 +02:00
private final int min ( final int [ ] list , int len ) {
2008-03-20 23:01:12 +01:00
assert len > 0 ;
int f = 0 ;
while ( len - - > 0 ) {
if ( compare ( list [ f ] , list [ len ] ) > 0 ) f = len ;
}
return f ;
}
2008-08-02 14:12:04 +02:00
private final int max ( final int [ ] list , int len ) {
2008-03-20 23:01:12 +01:00
assert len > 0 ;
int f = 0 ;
while ( len - - > 0 ) {
if ( compare ( list [ f ] , list [ len ] ) < 0 ) f = len ;
}
return f ;
}
2008-08-02 14:12:04 +02:00
private final void isort ( final int L , final int R , final byte [ ] swapspace ) {
2006-06-20 16:17:21 +02:00
for ( int i = L + 1 ; i < R ; i + + )
for ( int j = i ; j > L & & compare ( j - 1 , j ) > 0 ; j - - )
2007-11-09 01:51:38 +01:00
swap ( j , j - 1 , 0 , swapspace ) ;
2006-06-20 16:17:21 +02:00
}
2008-08-02 14:12:04 +02:00
private final int swap ( final int i , final int j , final int p , final byte [ ] swapspace ) {
2006-06-20 16:17:21 +02:00
if ( i = = j ) return p ;
2007-11-09 01:51:38 +01:00
System . arraycopy ( chunkcache , this . rowdef . objectsize * i , swapspace , 0 , this . rowdef . objectsize ) ;
System . arraycopy ( chunkcache , this . rowdef . objectsize * j , chunkcache , this . rowdef . objectsize * i , this . rowdef . objectsize ) ;
System . arraycopy ( swapspace , 0 , chunkcache , this . rowdef . objectsize * j , this . rowdef . objectsize ) ;
2006-06-20 16:17:21 +02:00
if ( i = = p ) return j ; else if ( j = = p ) return i ; else return p ;
}
2007-11-09 01:51:38 +01:00
public synchronized void uniq ( ) {
2006-12-06 04:02:57 +01:00
assert ( this . rowdef . objectOrder ! = null ) ;
2006-06-20 16:17:21 +02:00
// removes double-occurrences of chunks
// this works only if the collection was ordered with sort before
2007-04-20 09:53:58 +02:00
// if the collection is large and the number of deletions is also large,
// then this method may run a long time with 100% CPU load which is caused
2007-11-12 15:39:30 +01:00
// by the large number of memory movements.
2007-11-12 02:14:51 +01:00
if ( chunkcount < 2 ) return ;
int i = chunkcount - 2 ;
2008-08-02 14:12:04 +02:00
final long t = System . currentTimeMillis ( ) ; // for time-out
2007-11-12 15:39:30 +01:00
int d = 0 ;
try {
while ( i > = 0 ) {
2009-03-13 10:30:19 +01:00
if ( match ( i , i + 1 ) ) {
2009-03-13 11:07:04 +01:00
removeRow ( i + 1 , true ) ;
2007-11-12 15:39:30 +01:00
d + + ;
}
i - - ;
2009-03-13 10:30:19 +01:00
if ( System . currentTimeMillis ( ) - t > 60000 ) {
2009-05-16 01:07:10 +02:00
Log . logWarning ( " RowCollection " , " uniq() time-out at " + i + " (backwards) from " + chunkcount + " elements after " + ( System . currentTimeMillis ( ) - t ) + " milliseconds; " + d + " deletions so far " ) ;
return ;
2007-11-12 15:39:30 +01:00
}
2006-06-20 16:17:21 +02:00
}
2008-08-02 14:12:04 +02:00
} catch ( final RuntimeException e ) {
2009-05-16 01:07:10 +02:00
Log . logWarning ( " RowCollection " , e . getMessage ( ) , e ) ;
2006-06-20 16:17:21 +02:00
}
}
2009-12-10 00:27:26 +01:00
public synchronized ArrayList < RowCollection > removeDoubles ( ) throws RowSpaceExceededException {
2008-01-20 02:22:46 +01:00
assert ( this . rowdef . objectOrder ! = null ) ;
// removes double-occurrences of chunks
2008-05-02 00:40:42 +02:00
// in contrast to uniq() this removes also the remaining, non-double entry that had a double-occurrence to the others
2008-01-20 02:22:46 +01:00
// all removed chunks are returned in an array
this . sort ( ) ;
2009-01-30 16:33:00 +01:00
final ArrayList < RowCollection > report = new ArrayList < RowCollection > ( ) ;
2008-01-20 02:22:46 +01:00
if ( chunkcount < 2 ) return report ;
int i = chunkcount - 2 ;
boolean u = true ;
2009-01-30 16:33:00 +01:00
RowCollection collection = new RowCollection ( this . rowdef , 2 ) ;
2008-01-20 02:22:46 +01:00
try {
while ( i > = 0 ) {
2009-03-13 10:30:19 +01:00
if ( match ( i , i + 1 ) ) {
2008-04-24 15:31:55 +02:00
collection . addUnique ( get ( i + 1 , false ) ) ;
2008-01-20 02:22:46 +01:00
removeRow ( i + 1 , false ) ;
if ( i + 1 < chunkcount - 1 ) u = false ;
2009-12-02 01:37:59 +01:00
} else if ( ! collection . isEmpty ( ) ) {
2008-01-20 02:22:46 +01:00
// finish collection of double occurrences
2008-04-24 15:31:55 +02:00
collection . addUnique ( get ( i + 1 , false ) ) ;
2008-01-20 02:22:46 +01:00
removeRow ( i + 1 , false ) ;
if ( i + 1 < chunkcount - 1 ) u = false ;
collection . trim ( false ) ;
report . add ( collection ) ;
2009-01-30 16:33:00 +01:00
collection = new RowSet ( this . rowdef , 2 ) ;
2008-01-20 02:22:46 +01:00
}
i - - ;
}
2008-08-02 14:12:04 +02:00
} catch ( final RuntimeException e ) {
2009-01-31 00:33:47 +01:00
Log . logWarning ( " kelondroRowCollection " , e . getMessage ( ) , e ) ;
2008-01-20 02:22:46 +01:00
} finally {
if ( ! u ) this . sort ( ) ;
}
return report ;
}
2007-11-09 01:51:38 +01:00
public synchronized boolean isSorted ( ) {
assert ( this . rowdef . objectOrder ! = null ) ;
if ( chunkcount < = 1 ) return true ;
2007-11-16 15:48:09 +01:00
if ( chunkcount ! = this . sortBound ) return false ;
2009-01-09 01:06:36 +01:00
/ *
2007-11-09 01:51:38 +01:00
for ( int i = 0 ; i < chunkcount - 1 ; i + + ) {
//System.out.println("*" + new String(get(i).getColBytes(0)));
if ( compare ( i , i + 1 ) > 0 ) {
2008-04-24 15:31:55 +02:00
System . out . println ( " ? " + new String ( get ( i + 1 , false ) . getColBytes ( 0 ) ) ) ;
2007-11-09 01:51:38 +01:00
return false ;
}
}
2009-01-09 01:06:36 +01:00
* /
2007-11-09 01:51:38 +01:00
return true ;
}
2007-04-17 17:15:47 +02:00
public synchronized String toString ( ) {
2008-12-04 13:54:16 +01:00
final StringBuilder s = new StringBuilder ( ) ;
2009-01-30 16:33:00 +01:00
final Iterator < Row . Entry > i = iterator ( ) ;
2007-12-27 18:56:59 +01:00
if ( i . hasNext ( ) ) s . append ( i . next ( ) . toString ( ) ) ;
2008-06-06 18:01:27 +02:00
while ( i . hasNext ( ) ) s . append ( " , " + ( i . next ( ) ) . toString ( ) ) ;
2006-06-20 16:17:21 +02:00
return new String ( s ) ;
}
2008-08-02 14:12:04 +02:00
private final int compare ( final int i , final int j ) {
2007-11-07 23:38:09 +01:00
assert ( chunkcount * this . rowdef . objectsize < = chunkcache . length ) : " chunkcount = " + chunkcount + " , objsize = " + this . rowdef . objectsize + " , chunkcache.length = " + chunkcache . length ;
2006-10-23 02:59:55 +02:00
assert ( i > = 0 ) & & ( i < chunkcount ) : " i = " + i + " , chunkcount = " + chunkcount ;
assert ( j > = 0 ) & & ( j < chunkcount ) : " j = " + j + " , chunkcount = " + chunkcount ;
2006-12-06 04:02:57 +01:00
assert ( this . rowdef . objectOrder ! = null ) ;
2006-06-20 16:17:21 +02:00
if ( i = = j ) return 0 ;
2008-05-13 17:28:55 +02:00
//assert (!bugappearance(chunkcache, i * this.rowdef.objectsize + colstart, this.rowdef.primaryKeyLength));
//assert (!bugappearance(chunkcache, j * this.rowdef.objectsize + colstart, this.rowdef.primaryKeyLength));
2008-08-02 14:12:04 +02:00
final int c = this . rowdef . objectOrder . compare (
2006-06-20 16:17:21 +02:00
chunkcache ,
2009-03-13 17:52:31 +01:00
i * this . rowdef . objectsize ,
2007-11-07 23:38:09 +01:00
this . rowdef . primaryKeyLength ,
2006-06-20 16:17:21 +02:00
chunkcache ,
2009-03-13 17:52:31 +01:00
j * this . rowdef . objectsize ,
2007-11-07 23:38:09 +01:00
this . rowdef . primaryKeyLength ) ;
2006-06-20 16:17:21 +02:00
return c ;
}
2006-06-22 17:42:09 +02:00
2008-08-02 14:12:04 +02:00
protected synchronized int compare ( final byte [ ] a , final int astart , final int alength , final int chunknumber ) {
2007-03-14 09:55:05 +01:00
assert ( chunknumber < chunkcount ) ;
2008-08-02 14:12:04 +02:00
final int l = Math . min ( this . rowdef . primaryKeyLength , Math . min ( a . length - astart , alength ) ) ;
2009-03-13 17:52:31 +01:00
return rowdef . objectOrder . compare ( a , astart , l , chunkcache , chunknumber * this . rowdef . objectsize , this . rowdef . primaryKeyLength ) ;
2007-03-14 09:55:05 +01:00
}
2009-03-13 10:30:19 +01:00
protected final boolean match ( final int i , final int j ) {
assert ( chunkcount * this . rowdef . objectsize < = chunkcache . length ) : " chunkcount = " + chunkcount + " , objsize = " + this . rowdef . objectsize + " , chunkcache.length = " + chunkcache . length ;
assert ( i > = 0 ) & & ( i < chunkcount ) : " i = " + i + " , chunkcount = " + chunkcount ;
assert ( j > = 0 ) & & ( j < chunkcount ) : " j = " + j + " , chunkcount = " + chunkcount ;
if ( i > = chunkcount ) return false ;
if ( j > = chunkcount ) return false ;
assert ( this . rowdef . objectOrder ! = null ) ;
if ( i = = j ) return true ;
2009-03-13 17:52:31 +01:00
int astart = i * this . rowdef . objectsize ;
int bstart = j * this . rowdef . objectsize ;
2009-03-13 10:30:19 +01:00
int k = this . rowdef . primaryKeyLength ;
while ( k - - ! = 0 ) {
if ( chunkcache [ astart + + ] ! = chunkcache [ bstart + + ] ) return false ;
}
return true ;
}
protected synchronized boolean match ( final byte [ ] a , int astart , final int alength , final int chunknumber ) {
2007-03-14 09:55:05 +01:00
if ( chunknumber > = chunkcount ) return false ;
2009-03-13 17:52:31 +01:00
int p = chunknumber * this . rowdef . objectsize ;
2009-03-13 10:30:19 +01:00
int len = Math . min ( this . rowdef . primaryKeyLength , Math . min ( alength , a . length - astart ) ) ;
while ( len - - ! = 0 ) {
if ( a [ astart + + ] ! = chunkcache [ p + + ] ) return false ;
}
return true ;
2007-03-14 09:55:05 +01:00
}
2008-06-15 01:17:56 +02:00
2007-03-14 09:55:05 +01:00
public synchronized void close ( ) {
chunkcache = null ;
}
2008-08-02 14:12:04 +02:00
private static long d ( final long a , final long b ) {
2008-08-02 15:57:00 +02:00
if ( b = = 0 ) return a ;
return a / b ;
2007-11-10 09:57:00 +01:00
}
2008-05-03 11:06:00 +02:00
private static Random random = null ;
2007-11-11 01:28:22 +01:00
private static String randomHash ( ) {
return
2009-01-30 16:33:00 +01:00
Base64Order . enhancedCoder . encodeLong ( random . nextLong ( ) , 4 ) +
Base64Order . enhancedCoder . encodeLong ( random . nextLong ( ) , 4 ) +
Base64Order . enhancedCoder . encodeLong ( random . nextLong ( ) , 4 ) ;
2007-11-11 01:28:22 +01:00
}
2009-12-10 00:27:26 +01:00
public static void test ( final int testsize ) throws RowSpaceExceededException {
2009-01-30 16:33:00 +01:00
final Row r = new Row ( new Column [ ] {
2009-01-30 23:44:20 +01:00
new Column ( " hash " , Column . celltype_string , Column . encoder_bytes , 12 , " hash " ) } ,
2009-03-13 17:52:31 +01:00
Base64Order . enhancedCoder ) ;
2007-12-20 03:46:41 +01:00
2009-01-30 16:33:00 +01:00
RowCollection a = new RowCollection ( r , testsize ) ;
2008-01-20 02:22:46 +01:00
a . add ( " AAAAAAAAAAAA " . getBytes ( ) ) ;
a . add ( " BBBBBBBBBBBB " . getBytes ( ) ) ;
a . add ( " BBBBBBBBBBBB " . getBytes ( ) ) ;
a . add ( " BBBBBBBBBBBB " . getBytes ( ) ) ;
a . add ( " CCCCCCCCCCCC " . getBytes ( ) ) ;
2009-01-30 16:33:00 +01:00
final ArrayList < RowCollection > del = a . removeDoubles ( ) ;
2008-01-20 02:22:46 +01:00
System . out . println ( del + " rows double " ) ;
2009-01-30 16:33:00 +01:00
final Iterator < Row . Entry > j = a . iterator ( ) ;
2008-01-20 02:22:46 +01:00
while ( j . hasNext ( ) ) System . out . println ( new String ( j . next ( ) . bytes ( ) ) ) ;
2007-12-20 03:46:41 +01:00
System . out . println ( " kelondroRowCollection test with size = " + testsize ) ;
2009-01-30 16:33:00 +01:00
a = new RowCollection ( r , testsize ) ;
2008-06-16 01:25:57 +02:00
long t0 = System . nanoTime ( ) ;
2007-12-20 03:46:41 +01:00
random = new Random ( 0 ) ;
for ( int i = 0 ; i < testsize ; i + + ) a . add ( randomHash ( ) . getBytes ( ) ) ;
random = new Random ( 0 ) ;
for ( int i = 0 ; i < testsize ; i + + ) a . add ( randomHash ( ) . getBytes ( ) ) ;
a . sort ( ) ;
a . uniq ( ) ;
2008-03-21 00:11:04 +01:00
long t1 = System . nanoTime ( ) ;
System . out . println ( " create a : " + ( t1 - t0 ) + " nanoseconds, " + d ( testsize , ( t1 - t0 ) ) + " entries/nanoseconds; a.size() = " + a . size ( ) ) ;
2007-12-20 03:46:41 +01:00
2009-01-30 16:33:00 +01:00
final RowCollection c = new RowCollection ( r , testsize ) ;
2007-11-11 01:28:22 +01:00
random = new Random ( 0 ) ;
2008-03-21 00:11:04 +01:00
t0 = System . nanoTime ( ) ;
2007-11-09 01:51:38 +01:00
for ( int i = 0 ; i < testsize ; i + + ) {
2007-11-11 01:28:22 +01:00
c . add ( randomHash ( ) . getBytes ( ) ) ;
2007-11-09 01:51:38 +01:00
}
2008-03-21 00:11:04 +01:00
t1 = System . nanoTime ( ) ;
System . out . println ( " create c : " + ( t1 - t0 ) + " nanoseconds, " + d ( testsize , ( t1 - t0 ) ) + " entries/nanoseconds " ) ;
2009-01-30 16:33:00 +01:00
final RowCollection d = new RowCollection ( r , testsize ) ;
2007-11-09 01:51:38 +01:00
for ( int i = 0 ; i < testsize ; i + + ) {
2008-04-24 15:31:55 +02:00
d . add ( c . get ( i , false ) . getColBytes ( 0 ) ) ;
2007-11-09 01:51:38 +01:00
}
2008-08-02 14:12:04 +02:00
final long t2 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " copy c -> d: " + ( t2 - t1 ) + " nanoseconds, " + d ( testsize , ( t2 - t1 ) ) + " entries/nanoseconds " ) ;
2009-10-10 01:32:08 +02:00
availableCPU = 1 ;
2007-11-09 01:51:38 +01:00
c . sort ( ) ;
2008-08-02 14:12:04 +02:00
final long t3 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " sort c (1) : " + ( t3 - t2 ) + " nanoseconds, " + d ( testsize , ( t3 - t2 ) ) + " entries/nanoseconds " ) ;
2009-10-10 01:32:08 +02:00
availableCPU = 2 ;
2007-11-09 01:51:38 +01:00
d . sort ( ) ;
2008-08-02 14:12:04 +02:00
final long t4 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " sort d (2) : " + ( t4 - t3 ) + " nanoseconds, " + d ( testsize , ( t4 - t3 ) ) + " entries/nanoseconds " ) ;
2007-11-09 01:51:38 +01:00
c . uniq ( ) ;
2008-08-02 14:12:04 +02:00
final long t5 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " uniq c : " + ( t5 - t4 ) + " nanoseconds, " + d ( testsize , ( t5 - t4 ) ) + " entries/nanoseconds " ) ;
2007-11-09 01:51:38 +01:00
d . uniq ( ) ;
2008-08-02 14:12:04 +02:00
final long t6 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " uniq d : " + ( t6 - t5 ) + " nanoseconds, " + d ( testsize , ( t6 - t5 ) ) + " entries/nanoseconds " ) ;
2007-11-11 01:28:22 +01:00
random = new Random ( 0 ) ;
2009-01-30 16:33:00 +01:00
final RowSet e = new RowSet ( r , testsize ) ;
2008-06-15 00:51:47 +02:00
for ( int i = 0 ; i < testsize ; i + + ) {
e . put ( r . newEntry ( randomHash ( ) . getBytes ( ) ) ) ;
}
2008-08-02 14:12:04 +02:00
final long t7 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " create e : " + ( t7 - t6 ) + " nanoseconds, " + d ( testsize , ( t7 - t6 ) ) + " entries/nanoseconds " ) ;
2007-11-09 16:34:11 +01:00
e . sort ( ) ;
2008-08-02 14:12:04 +02:00
final long t8 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " sort e (2) : " + ( t8 - t7 ) + " nanoseconds, " + d ( testsize , ( t8 - t7 ) ) + " entries/nanoseconds " ) ;
2007-11-09 16:34:11 +01:00
e . uniq ( ) ;
2008-08-02 14:12:04 +02:00
final long t9 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " uniq e : " + ( t9 - t8 ) + " nanoseconds, " + d ( testsize , ( t9 - t8 ) ) + " entries/nanoseconds " ) ;
2008-08-02 14:12:04 +02:00
final boolean cis = c . isSorted ( ) ;
final long t10 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " c isSorted = " + ( ( cis ) ? " true " : " false " ) + " : " + ( t10 - t9 ) + " nanoseconds " ) ;
2008-08-02 14:12:04 +02:00
final boolean dis = d . isSorted ( ) ;
final long t11 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " d isSorted = " + ( ( dis ) ? " true " : " false " ) + " : " + ( t11 - t10 ) + " nanoseconds " ) ;
2008-08-02 14:12:04 +02:00
final boolean eis = e . isSorted ( ) ;
final long t12 = System . nanoTime ( ) ;
2008-03-21 00:11:04 +01:00
System . out . println ( " e isSorted = " + ( ( eis ) ? " true " : " false " ) + " : " + ( t12 - t11 ) + " nanoseconds " ) ;
2007-11-12 02:14:51 +01:00
random = new Random ( 0 ) ;
boolean allfound = true ;
for ( int i = 0 ; i < testsize ; i + + ) {
2008-08-02 14:12:04 +02:00
final String rh = randomHash ( ) ;
2008-06-15 00:51:47 +02:00
if ( e . get ( rh . getBytes ( ) ) = = null ) {
2007-11-12 02:14:51 +01:00
allfound = false ;
2008-06-15 00:51:47 +02:00
System . out . println ( " not found hash " + rh + " at attempt " + i ) ;
2007-11-12 02:14:51 +01:00
break ;
}
}
2008-08-02 14:12:04 +02:00
final long t13 = System . nanoTime ( ) ;
2008-06-15 00:51:47 +02:00
System . out . println ( " e allfound = " + ( ( allfound ) ? " true " : " false " ) + " : " + ( t13 - t12 ) + " nanoseconds " ) ;
2007-11-12 02:14:51 +01:00
boolean noghosts = true ;
for ( int i = 0 ; i < testsize ; i + + ) {
if ( e . get ( randomHash ( ) . getBytes ( ) ) ! = null ) {
noghosts = false ;
break ;
}
}
2008-08-02 14:12:04 +02:00
final long t14 = System . nanoTime ( ) ;
2008-06-15 00:51:47 +02:00
System . out . println ( " e noghosts = " + ( ( noghosts ) ? " true " : " false " ) + " : " + ( t14 - t13 ) + " nanoseconds " ) ;
2007-11-12 02:14:51 +01:00
System . out . println ( " Result size: c = " + c . size ( ) + " , d = " + d . size ( ) + " , e = " + e . size ( ) ) ;
2007-11-09 01:51:38 +01:00
System . out . println ( ) ;
2008-03-21 00:11:04 +01:00
if ( sortingthreadexecutor ! = null ) sortingthreadexecutor . shutdown ( ) ;
2007-11-09 01:51:38 +01:00
}
2008-08-02 14:12:04 +02:00
public static void main ( final String [ ] args ) {
2007-11-11 01:28:22 +01:00
//test(1000);
2009-12-10 00:27:26 +01:00
try {
test ( 50000 ) ;
} catch ( RowSpaceExceededException e ) {
e . printStackTrace ( ) ;
}
2008-03-21 00:11:04 +01:00
//test(100000);
2007-11-09 16:34:11 +01:00
//test(1000000);
2007-11-09 01:51:38 +01:00
/ *
2006-08-05 01:04:03 +02:00
System . out . println ( new java . util . Date ( 10957 * day ) ) ;
System . out . println ( new java . util . Date ( 0 ) ) ;
System . out . println ( daysSince2000 ( System . currentTimeMillis ( ) ) ) ;
2007-11-09 01:51:38 +01:00
* /
2006-08-05 01:04:03 +02:00
}
2008-04-19 09:54:44 +02:00
}