yacy_search_server/source/net/yacy/kelondro/blob/MapHeap.java

// MapHeap.java
// -----------------------
// (C) 29.01.2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2004 as kelondroMap on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

package net.yacy.kelondro.blob;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.AbstractMap;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;

import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.order.ByteOrder;
import net.yacy.cora.order.CloneableIterator;
import net.yacy.cora.order.NaturalOrder;
import net.yacy.cora.storage.ARC;
import net.yacy.cora.storage.ConcurrentARC;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.LookAheadIterator;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.RotateIterator;

public class MapHeap implements Map<byte[], Map<String, String>> {

    private final BLOB blob;
    private final ARC<byte[], Map<String, String>> cache;
    private final char fillchar;


    public MapHeap(
            final File heapFile,
            final int keylength,
            final ByteOrder ordering,
            final int buffermax,
            final int cachesize,
            final char fillchar) throws IOException {
        this.blob = new Heap(heapFile, keylength, ordering, buffermax);
        this.cache = new ConcurrentARC<byte[], Map<String, String>>(cachesize, Math.min(32, 2 * Runtime.getRuntime().availableProcessors()), ordering);
        this.fillchar = fillchar;
    }

    /**
     * ask for the length of the primary key
     * @return the length of the key
     */
    public int keylength() {
        return this.blob.keylength();
    }

    /**
     * get the ordering of the primary keys
     * @return
     */
    public ByteOrder ordering() {
        return this.blob.ordering();
    }

    /**
     * clears the content of the database
     * @throws IOException
     */
    @Override
    public synchronized void clear() {
    	try {
            this.blob.clear();
        } catch (final IOException e) {
            ConcurrentLog.logException(e);
        }
        this.cache.clear();
    }

    private static String map2string(final Map<String, String> map, final String comment) {
        final StringBuilder bb = new StringBuilder(map.size() * 40);
        bb.append("# ").append(comment).append('\r').append('\n');
        for (final Map.Entry<String, String> entry: map.entrySet()) {
            if (entry.getValue() != null) {
                bb.append(entry.getKey());
                bb.append('=');
                bb.append(entry.getValue());
                bb.append('\r').append('\n');
             }
        }
        bb.append("# EOF\r\n");
        return bb.toString();
    }

    private static Map<String, String> bytes2map(final byte[] b) throws IOException, SpaceExceededException {
        final BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(b)));
        final Map<String, String> map = new ConcurrentHashMap<String, String>();
        String line;
        int pos;
        try {
        while ((line = br.readLine()) != null) { // very slow readLine????
            line = line.trim();
            if (line.equals("# EOF")) return map;
            if ((line.isEmpty()) || (line.charAt(0) == '#')) continue;
            pos = line.indexOf('=');
            if (pos < 0) continue;
            map.put(line.substring(0, pos), line.substring(pos + 1));
        }
        } catch (final OutOfMemoryError e) {
            throw new SpaceExceededException(0, "readLine probably uses too much RAM", e);
        } finally {
            br.close();
        }
        return map;
    }


    // use our own formatter to prevent concurrency locks with other processes
    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);

    /**
     * write a whole byte array as Map to the table
     * @param key  the primary key
     * @param newMap
     * @throws IOException
     * @throws SpaceExceededException
     */
    public void insert(byte[] key, final Map<String, String> newMap) throws IOException, SpaceExceededException {
        assert key != null;
        assert key.length > 0;
        assert newMap != null;
        key = normalizeKey(key);
        final String s = map2string(newMap, "W" + my_SHORT_SECOND_FORMATTER.format() + " ");
        assert s != null;
        final byte[] sb = UTF8.getBytes(s);
        if (this.cache == null) {
            // write entry
            if (this.blob != null) this.blob.insert(key, sb);
        } else {
            synchronized (this) {
                // write entry
                if (this.blob != null) this.blob.insert(key, sb);

                // write map to cache
                if (MemoryControl.shortStatus()) {
                    this.cache.clear();
                } else {
                    this.cache.insert(key, newMap);
                }
            }
        }
    }

    @Override
    public Map<String, String> put(final byte[] key, final Map<String, String> newMap) {
        Map<String, String> v = null;
        try {
            v = this.get(key);
            insert(key, newMap);
        } catch (final IOException e) {
            ConcurrentLog.logException(e);
        } catch (final SpaceExceededException e) {
            ConcurrentLog.logException(e);
        }
        return v;
    }

    /**
     * remove a Map
     * @param key  the primary key
     * @throws IOException
     */
    public void delete(byte[] key) throws IOException {
        // update elementCount
        if (key == null) return;
        key = normalizeKey(key);

        synchronized (this) {
            // remove from cache
            if (this.cache != null) this.cache.remove(key);

            // remove from file
            if (this.blob != null) this.blob.delete(key);
        }
    }

    @Override
    public Map<String, String> remove(final Object key)  {
        Map<String, String> v = null;
        try {
            v = this.get(key);
            delete((byte[]) key);
        } catch (final IOException e) {
            ConcurrentLog.logException(e);
        }
        return v;
    }

    /**
     * check if a specific key is in the database
     * @param key  the primary key
     * @return
     * @throws IOException
     */

    @Override
    public boolean containsKey(final Object k) {
        if (!(k instanceof byte[])) return false;
        assert k != null;
        if (this.cache == null) return false; // case may appear during shutdown
        final byte[] key = normalizeKey((byte[]) k);
        boolean h;
        synchronized (this) {
            h = this.cache.containsKey(key) || this.blob.containsKey(key);
        }
        return h;
    }

    /**
     * retrieve the whole Map from the table
     * @param key  the primary key
     * @return
     * @throws IOException
     */
    public Map<String, String> get(final byte[] key) throws IOException, SpaceExceededException {
        if (key == null) return null;
        return get(key, true);
    }

    @Override
    public Map<String, String> get(final Object key) {
        if (key == null) return null;
        try {
            if (key instanceof byte[]) return get((byte[]) key);
            if (key instanceof String) return get(UTF8.getBytes((String) key));
        } catch (final IOException e) {
            ConcurrentLog.logException(e);
        } catch (final SpaceExceededException e) {
            ConcurrentLog.logException(e);
        }
        return null;
    }

    private byte[] normalizeKey(final byte[] key) {
        if (this.blob == null || key == null) return key;
        if (key.length > this.blob.keylength()) {
            final byte[] b = new byte[this.blob.keylength()];
            System.arraycopy(key, 0, b, 0, this.blob.keylength());
            return b;
        }
        if (key.length < this.blob.keylength()) {
            final byte[] b = new byte[this.blob.keylength()];
            System.arraycopy(key, 0, b, 0, key.length);
            for (int i = key.length; i < b.length; i++) b[i] = (byte) this.fillchar;
            return b;
        }
        return key;
    }

    private byte[] removeFillchar(final byte[] key) {
        if (key == null) return key;
        int p = key.length - 1;
        while (p >= 0 && key[p] == this.fillchar) p--;
        if (p == key.length - 1) return key;
        // copy part of key into new byte[]
        final byte[] k = new byte[p + 1];
        System.arraycopy(key, 0, k, 0, k.length);
        return k;
    }

    protected Map<String, String> get(byte[] key, final boolean storeCache) throws IOException, SpaceExceededException {
        // load map from cache
        assert key != null;
        if (this.cache == null) return null; // case may appear during shutdown
        key = normalizeKey(key);

        if (MemoryControl.shortStatus()) {
            this.cache.clear();
        }

        // if we have the entry in the cache then just return that
        Map<String, String> map = this.cache.get(key);
        if (map != null) return map;

        // in all other cases we must look into the cache again within
        // a synchronization in case that the entry was not in the cache but stored
        // there while another process has taken it from the file system
        if (storeCache) {
            synchronized (this) {
                map = this.cache.get(key);
                if (map != null) return map;

                // read object
                final byte[] b = this.blob.get(key);
                if (b == null) return null;
                try {
                    map = bytes2map(b);
                } catch (final SpaceExceededException e) {
                    throw new IOException(e.getMessage());
                }

                // write map to cache
                this.cache.insert(key, map);

                // return value
                return map;
            }
        }
        byte[] b = null;
        synchronized (this) {
            map = this.cache.get(key);
            if (map != null) return map;
            b = this.blob.get(key);
        }
        if (b == null) return null;
        try {
            return bytes2map(b);
        } catch (final SpaceExceededException e) {
            throw new IOException(e.getMessage());
        }
    }

    /**
     * iterator over all keys
     * @param up
     * @param rotating
     * @return
     * @throws IOException
     */
    public synchronized CloneableIterator<byte[]> keys(final boolean up, final boolean rotating) throws IOException {
        // simple enumeration of key names without special ordering
        return new KeyIterator(up, rotating, null, null);
    }

    /**
     * return an iteration of the keys in the map
     * the keys in the map are de-normalized which means that the fill-character is removed
     * @param up
     * @param rotating
     * @param firstKey
     * @param secondKey
     * @return
     * @throws IOException
     */
    public synchronized CloneableIterator<byte[]> keys(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
        return new KeyIterator(up, rotating, firstKey, secondKey);
    }

    public synchronized CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException {
        return this.blob.keys(up, firstKey);
    }

    public class KeyIterator implements CloneableIterator<byte[]>, Iterator<byte[]> {

        private final boolean up, rotating;
        private final byte[] firstKey, secondKey;
        private Iterator<byte[]> iterator;
        final private CloneableIterator<byte[]> blobkeys;

        public KeyIterator(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
            this.up = up;
            this.rotating = rotating;
            this.firstKey = firstKey;
            this.secondKey = secondKey;
            this.blobkeys = MapHeap.this.blob.keys(up, firstKey);
            this.iterator = rotating ? new RotateIterator<byte[]>(this.blobkeys, secondKey, MapHeap.this.blob.size()) : this.blobkeys;
        }

        @Override
        public byte[] next() {
            assert this.iterator != null;
            if (this.iterator == null) return null;
            return removeFillchar(this.iterator.next());
        }

        @Override
        public boolean hasNext() {
            return this.iterator != null && this.iterator.hasNext();
        }

        @Override
        public void remove() {
            this.iterator.remove();
        }

        @Override
        public CloneableIterator<byte[]> clone(final Object modifier) {
            try {
                return new KeyIterator(this.up, this.rotating, this.firstKey, this.secondKey);
            } catch (final IOException e) {
                return null;
            }
        }

        @Override
        public void close() {
            this.blobkeys.close();
        }
    }

    public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final boolean rotating) throws IOException {
        return new FullMapIterator(keys(up, rotating));
    }

    public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
        return new FullMapIterator(keys(up, rotating, firstKey, secondKey));
    }

    /**
     * ask for the number of entries
     * @return the number of entries in the table
     */
    @Override
    public int size() {
        return (this.blob == null) ? 0 : this.blob.size();
    }

    @Override
    public boolean isEmpty() {
        return (this.blob == null) ? true : this.blob.isEmpty();
    }

    /**
     * close the Map table
     */
    public synchronized void close() {
        this.cache.clear();

        // close file
        if (this.blob != null) this.blob.close(true);
    }

    @Override
    public void finalize() {
        close();
    }

    protected class FullMapIterator extends LookAheadIterator<Map.Entry<byte[], Map<String, String>>> implements Iterator<Map.Entry<byte[], Map<String, String>>> {
        // enumerates Map-Type elements
        // the key is also included in every map that is returned; it's key is 'key'

        private final Iterator<byte[]> keyIterator;

        FullMapIterator(final Iterator<byte[]> keyIterator) {
            this.keyIterator = keyIterator;
        }

        @Override
        public Map.Entry<byte[], Map<String, String>> next0() {
            if (this.keyIterator == null) return null;
            byte[] nextKey;
            Map<String, String> map;
            while (this.keyIterator.hasNext()) {
                nextKey = this.keyIterator.next();
                try {
                    map = get(nextKey, false);
                } catch (final IOException e) {
                    ConcurrentLog.warn("MapDataMining", e.getMessage());
                    continue;
                } catch (final SpaceExceededException e) {
                    ConcurrentLog.logException(e);
                    continue;
                }
                if (map == null) continue; // circumvention of a modified exception
                // produce entry
                Map.Entry<byte[], Map<String, String>> entry = new AbstractMap.SimpleImmutableEntry<byte[], Map<String, String>>(nextKey, map);
                return entry;
            }
            return null;
        }
    } // class FullMapIterator


    @Override
    public void putAll(final Map<? extends byte[], ? extends Map<String, String>> map) {
        for (final Map.Entry<? extends byte[], ? extends Map<String, String>> me: map.entrySet()) {
            try {
                insert(me.getKey(), me.getValue());
            } catch (final SpaceExceededException e) {
                ConcurrentLog.logException(e);
            } catch (final IOException e) {
                ConcurrentLog.logException(e);
            }
        }
    }

    @Override
    public Set<byte[]> keySet() {
        final TreeSet<byte[]> set = new TreeSet<byte[]>(this.blob.ordering());
        try {
            final Iterator<byte[]> i = this.blob.keys(true, false);
            while (i.hasNext()) set.add(i.next());
        } catch (final IOException e) {}
        return set;
    }

    public final static byte[] POISON_QUEUE_ENTRY = "POISON".getBytes();
    public BlockingQueue<byte[]> keyQueue(final int size) {
        final ArrayBlockingQueue<byte[]> set = new ArrayBlockingQueue<byte[]>(size);
        (new Thread() {
            @Override
            public void run() {
                Thread.currentThread().setName("MapHeap.keyQueue:" + size);
                try {
                    final Iterator<byte[]> i = MapHeap.this.blob.keys(true, false);
                    while (i.hasNext())
                        try {
                            set.put(i.next());
                        } catch (final InterruptedException e) {
                            break;
                        }
                } catch (final IOException e) {}
                try {
                    set.put(MapHeap.POISON_QUEUE_ENTRY);
                } catch (final InterruptedException e) {
                }
            }}).start();
        return set;
    }

    @Override
    public Collection<Map<String, String>> values() {
        // this method shall not be used because it is not appropriate for this kind of data
        throw new UnsupportedOperationException();
    }

    @Override
    public Set<java.util.Map.Entry<byte[], Map<String, String>>> entrySet() {
        // this method shall not be used because it is not appropriate for this kind of data
        throw new UnsupportedOperationException();
    }

    @Override
    public boolean containsValue(final Object value) {
        // this method shall not be used because it is not appropriate for this kind of data
        throw new UnsupportedOperationException();
    }

    public static void main(final String[] args) {
        // test the class
        final File f = new File("maptest");
        if (f.exists()) FileUtils.deletedelete(f);
        try {
            // make map
            final MapHeap map = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 1024, 1024, '_');
            // put some values into the map
            final Map<String, String> m = new HashMap<String, String>();
            m.put("k", "000"); map.insert("123".getBytes(), m);
            m.put("k", "111"); map.insert("456".getBytes(), m);
            m.put("k", "222"); map.insert("789".getBytes(), m);
            // iterate over keys
            final Iterator<byte[]> i = map.keys(true, false);
            while (i.hasNext()) {
                System.out.println("key: " + UTF8.String(i.next()));
            }
            // clean up
            map.close();
        } catch (final IOException e) {
            ConcurrentLog.logException(e);
        } catch (final SpaceExceededException e) {
            ConcurrentLog.logException(e);
        }
    }
}