2009-03-16 01:18:37 +01:00
|
|
|
// AbstractIndex.java
|
|
|
|
// -----------------------------
|
|
|
|
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
|
|
|
// first published 15.3.2009 on http://yacy.net
|
|
|
|
//
|
|
|
|
// This is a part of YaCy, a peer-to-peer based web search engine
|
|
|
|
//
|
2011-03-08 02:51:51 +01:00
|
|
|
// $LastChangedDate$
|
|
|
|
// $LastChangedRevision$
|
|
|
|
// $LastChangedBy$
|
2009-03-16 01:18:37 +01:00
|
|
|
//
|
|
|
|
// LICENSE
|
2011-08-25 12:35:25 +02:00
|
|
|
//
|
2009-03-16 01:18:37 +01:00
|
|
|
// This program is free software; you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU General Public License as published by
|
|
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU General Public License
|
|
|
|
// along with this program; if not, write to the Free Software
|
|
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
|
2009-10-10 02:39:15 +02:00
|
|
|
package net.yacy.kelondro.rwi;
|
2009-03-16 01:18:37 +01:00
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.util.Iterator;
|
2010-04-25 23:37:36 +02:00
|
|
|
import java.util.TreeMap;
|
2009-03-16 17:24:53 +01:00
|
|
|
import java.util.TreeSet;
|
|
|
|
|
2012-09-21 16:46:57 +02:00
|
|
|
import net.yacy.cora.order.Base64Order;
|
2011-12-16 23:59:29 +01:00
|
|
|
import net.yacy.cora.order.Order;
|
2012-07-27 12:13:53 +02:00
|
|
|
import net.yacy.cora.storage.HandleSet;
|
|
|
|
import net.yacy.cora.util.SpaceExceededException;
|
2011-08-25 21:13:45 +02:00
|
|
|
import net.yacy.kelondro.index.Row;
|
2009-11-05 21:28:37 +01:00
|
|
|
import net.yacy.kelondro.logging.Log;
|
2009-10-10 01:22:22 +02:00
|
|
|
|
2009-04-15 08:34:27 +02:00
|
|
|
public abstract class AbstractIndex <ReferenceType extends Reference> implements Index<ReferenceType> {
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-04-15 08:34:27 +02:00
|
|
|
final protected ReferenceFactory<ReferenceType> factory;
|
|
|
|
|
|
|
|
public AbstractIndex(final ReferenceFactory<ReferenceType> factory) {
|
|
|
|
this.factory = factory;
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2011-05-18 16:26:28 +02:00
|
|
|
|
|
|
|
/**
|
|
|
|
* merge this index with another index
|
|
|
|
* @param otherIndex
|
2011-08-25 12:35:25 +02:00
|
|
|
* @throws IOException
|
2012-07-27 12:13:53 +02:00
|
|
|
* @throws SpaceExceededException
|
2011-05-18 16:26:28 +02:00
|
|
|
*/
|
2011-12-14 22:15:51 +01:00
|
|
|
@Override
|
2012-07-27 12:13:53 +02:00
|
|
|
public void merge(final Index<ReferenceType> otherIndex) throws IOException, SpaceExceededException {
|
2011-05-18 16:26:28 +02:00
|
|
|
byte[] term;
|
2011-08-25 12:35:25 +02:00
|
|
|
for (final ReferenceContainer<ReferenceType> otherContainer: otherIndex) {
|
2011-05-18 16:26:28 +02:00
|
|
|
term = otherContainer.getTermHash();
|
|
|
|
synchronized (this) {
|
2011-08-25 12:35:25 +02:00
|
|
|
final ReferenceContainer<ReferenceType> container = get(term, null);
|
2011-05-18 16:26:28 +02:00
|
|
|
if (container == null) {
|
|
|
|
this.add(otherContainer);
|
|
|
|
} else {
|
|
|
|
container.merge(otherContainer);
|
2011-08-25 12:35:25 +02:00
|
|
|
delete(term); // in some file-based environments we cannot just change the container
|
2011-05-18 16:26:28 +02:00
|
|
|
this.add(container);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2011-12-14 22:15:51 +01:00
|
|
|
@Override
|
2010-12-06 15:34:58 +01:00
|
|
|
public void removeDelayed(final HandleSet termHashes, final byte[] urlHashBytes) throws IOException {
|
|
|
|
// remove the same url hashes for multiple words
|
|
|
|
// this is mainly used when correcting a index after a search
|
|
|
|
final Iterator<byte[]> i = termHashes.iterator();
|
|
|
|
while (i.hasNext()) {
|
|
|
|
removeDelayed(i.next(), urlHashBytes);
|
|
|
|
}
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2011-12-14 22:15:51 +01:00
|
|
|
@Override
|
2010-04-15 15:22:59 +02:00
|
|
|
public int remove(final HandleSet termHashes, final byte[] urlHashBytes) throws IOException {
|
2009-03-16 01:18:37 +01:00
|
|
|
// remove the same url hashes for multiple words
|
|
|
|
// this is mainly used when correcting a index after a search
|
2009-04-16 17:29:00 +02:00
|
|
|
final Iterator<byte[]> i = termHashes.iterator();
|
2009-03-16 01:18:37 +01:00
|
|
|
int c = 0;
|
|
|
|
while (i.hasNext()) {
|
2010-03-26 19:33:20 +01:00
|
|
|
if (remove(i.next(), urlHashBytes)) c++;
|
2009-03-16 01:18:37 +01:00
|
|
|
}
|
|
|
|
return c;
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2012-06-05 12:06:26 +02:00
|
|
|
@Override
|
2011-12-14 22:15:51 +01:00
|
|
|
public synchronized TreeSet<ReferenceContainer<ReferenceType>> referenceContainer(final byte[] startHash, final boolean rot, final boolean excludePrivate, int count) throws IOException {
|
2009-03-16 17:24:53 +01:00
|
|
|
// creates a set of indexContainers
|
|
|
|
// this does not use the cache
|
2011-08-25 12:35:25 +02:00
|
|
|
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(this.factory, termKeyOrdering().clone());
|
|
|
|
final ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(this.factory, startHash);
|
2009-04-15 08:34:27 +02:00
|
|
|
containerOrder.rotate(emptyContainer);
|
|
|
|
final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder);
|
2011-12-14 22:15:51 +01:00
|
|
|
final Iterator<ReferenceContainer<ReferenceType>> i = referenceContainerIterator(startHash, rot, excludePrivate);
|
2009-03-16 17:24:53 +01:00
|
|
|
//if (ram) count = Math.min(size(), count);
|
2009-04-15 08:34:27 +02:00
|
|
|
ReferenceContainer<ReferenceType> container;
|
2009-03-16 17:24:53 +01:00
|
|
|
// this loop does not terminate using the i.hasNex() predicate when rot == true
|
|
|
|
// because then the underlying iterator is a rotating iterator without termination
|
|
|
|
// in this case a termination must be ensured with a counter
|
|
|
|
// It must also be ensured that the counter is in/decreased every loop
|
|
|
|
while ((count > 0) && (i.hasNext())) {
|
|
|
|
container = i.next();
|
2009-12-02 01:37:59 +01:00
|
|
|
if (container != null && !container.isEmpty()) {
|
2009-03-16 17:24:53 +01:00
|
|
|
containers.add(container);
|
|
|
|
}
|
|
|
|
count--; // decrease counter even if the container was null or empty to ensure termination
|
|
|
|
}
|
|
|
|
return containers; // this may return less containers as demanded
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
|
|
|
|
2009-05-28 01:51:34 +02:00
|
|
|
// methods to search in the index
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-05-28 01:51:34 +02:00
|
|
|
/**
|
2009-09-30 15:18:02 +02:00
|
|
|
* collect containers for given word hashes.
|
|
|
|
* This collection stops if a single container does not contain any references.
|
2009-05-28 01:51:34 +02:00
|
|
|
* In that case only a empty result is returned.
|
|
|
|
* @param wordHashes
|
|
|
|
* @param urlselection
|
|
|
|
* @return map of wordhash:indexContainer
|
|
|
|
*/
|
2011-12-14 22:15:51 +01:00
|
|
|
@Override
|
2010-04-25 23:37:36 +02:00
|
|
|
public TreeMap<byte[], ReferenceContainer<ReferenceType>> searchConjunction(final HandleSet wordHashes, final HandleSet urlselection) {
|
2009-05-28 01:51:34 +02:00
|
|
|
// first check if there is any entry that has no match; this uses only operations in ram
|
|
|
|
/*
|
|
|
|
Iterator<byte[]> i = wordHashes.iterator();
|
|
|
|
while (i.hasNext()) {
|
2010-04-25 23:37:36 +02:00
|
|
|
if (!this.has(i.next())); return new TreeMap<byte[], ReferenceContainer<ReferenceType>>(0);
|
2009-05-28 01:51:34 +02:00
|
|
|
}
|
|
|
|
*/
|
|
|
|
// retrieve entities that belong to the hashes
|
2010-04-25 23:37:36 +02:00
|
|
|
final TreeMap<byte[], ReferenceContainer<ReferenceType>> containers = new TreeMap<byte[], ReferenceContainer<ReferenceType>>(Base64Order.enhancedCoder);
|
2009-05-28 01:51:34 +02:00
|
|
|
byte[] singleHash;
|
|
|
|
ReferenceContainer<ReferenceType> singleContainer;
|
2010-01-10 02:40:26 +01:00
|
|
|
final Iterator<byte[]> i = wordHashes.iterator();
|
2009-05-28 01:51:34 +02:00
|
|
|
while (i.hasNext()) {
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-05-28 01:51:34 +02:00
|
|
|
// get next word hash:
|
|
|
|
singleHash = i.next();
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-05-28 01:51:34 +02:00
|
|
|
// retrieve index
|
|
|
|
try {
|
2011-08-25 12:35:25 +02:00
|
|
|
singleContainer = get(singleHash, urlselection);
|
|
|
|
} catch (final IOException e) {
|
2009-11-05 21:28:37 +01:00
|
|
|
Log.logException(e);
|
2009-05-28 01:51:34 +02:00
|
|
|
continue;
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-05-28 01:51:34 +02:00
|
|
|
// check result
|
2010-04-25 23:37:36 +02:00
|
|
|
if ((singleContainer == null || singleContainer.isEmpty())) return new TreeMap<byte[], ReferenceContainer<ReferenceType>>(Base64Order.enhancedCoder);
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-05-28 01:51:34 +02:00
|
|
|
containers.put(singleHash, singleContainer);
|
|
|
|
}
|
|
|
|
return containers;
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-09-29 10:16:24 +02:00
|
|
|
/**
|
|
|
|
* collect containers for given word hashes and join them as they are retrieved.
|
2009-09-30 15:18:02 +02:00
|
|
|
* This collection stops if a single container does not contain any references
|
|
|
|
* or the current result of the container join results in an empty container.
|
2009-09-29 10:16:24 +02:00
|
|
|
* In any fail case only a empty result container is returned.
|
|
|
|
* @param wordHashes
|
|
|
|
* @param urlselection
|
|
|
|
* @param maxDistance the maximum distance that the words in the result may have
|
|
|
|
* @return ReferenceContainer the join result
|
2012-07-27 12:13:53 +02:00
|
|
|
* @throws SpaceExceededException
|
2009-09-29 10:16:24 +02:00
|
|
|
*/
|
2012-07-27 12:13:53 +02:00
|
|
|
public ReferenceContainer<ReferenceType> searchJoin(final HandleSet wordHashes, final HandleSet urlselection, final int maxDistance) throws SpaceExceededException {
|
2009-09-30 15:18:02 +02:00
|
|
|
// first check if there is any entry that has no match;
|
|
|
|
// this uses only operations in ram
|
2011-08-25 12:35:25 +02:00
|
|
|
for (final byte[] wordHash: wordHashes) {
|
|
|
|
if (!has(wordHash)) return ReferenceContainer.emptyContainer(this.factory, null, 0);
|
2009-09-29 10:16:24 +02:00
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-09-29 10:16:24 +02:00
|
|
|
// retrieve entities that belong to the hashes
|
|
|
|
ReferenceContainer<ReferenceType> resultContainer = null;
|
|
|
|
ReferenceContainer<ReferenceType> singleContainer;
|
2011-08-25 12:35:25 +02:00
|
|
|
for (final byte[] wordHash: wordHashes) {
|
2009-09-29 10:16:24 +02:00
|
|
|
// retrieve index
|
|
|
|
try {
|
2011-08-25 12:35:25 +02:00
|
|
|
singleContainer = get(wordHash, urlselection);
|
|
|
|
} catch (final IOException e) {
|
2009-11-05 21:28:37 +01:00
|
|
|
Log.logException(e);
|
2009-09-29 10:16:24 +02:00
|
|
|
continue;
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-09-29 10:16:24 +02:00
|
|
|
// check result
|
2011-08-25 12:35:25 +02:00
|
|
|
if ((singleContainer == null || singleContainer.isEmpty())) return ReferenceContainer.emptyContainer(this.factory, null, 0);
|
2009-09-29 10:16:24 +02:00
|
|
|
if (resultContainer == null) resultContainer = singleContainer; else {
|
2011-08-25 12:35:25 +02:00
|
|
|
resultContainer = ReferenceContainer.joinConstructive(this.factory, resultContainer, singleContainer, maxDistance);
|
2009-09-29 10:16:24 +02:00
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-09-29 10:16:24 +02:00
|
|
|
// finish if the result is empty
|
2009-12-02 01:37:59 +01:00
|
|
|
if (resultContainer.isEmpty()) return resultContainer;
|
2009-09-29 10:16:24 +02:00
|
|
|
}
|
|
|
|
return resultContainer;
|
|
|
|
}
|
2011-08-25 12:35:25 +02:00
|
|
|
|
2009-06-03 10:49:54 +02:00
|
|
|
public TermSearch<ReferenceType> query(
|
2010-04-15 15:22:59 +02:00
|
|
|
final HandleSet queryHashes,
|
|
|
|
final HandleSet excludeHashes,
|
|
|
|
final HandleSet urlselection,
|
2010-01-10 02:40:26 +01:00
|
|
|
final ReferenceFactory<ReferenceType> termFactory,
|
2012-07-27 12:13:53 +02:00
|
|
|
final int maxDistance) throws SpaceExceededException {
|
2009-06-03 10:49:54 +02:00
|
|
|
return new TermSearch<ReferenceType>(this, queryHashes, excludeHashes, urlselection, termFactory, maxDistance);
|
2009-05-29 12:03:35 +02:00
|
|
|
}
|
2011-08-25 21:13:45 +02:00
|
|
|
|
2011-12-14 22:15:51 +01:00
|
|
|
@Override
|
2011-08-25 21:13:45 +02:00
|
|
|
public Row referenceRow() {
|
|
|
|
return this.factory.getRow();
|
|
|
|
}
|
2009-03-16 01:18:37 +01:00
|
|
|
}
|