mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
161d2fd2ef
- better control to the cache by using combined request-header and content access methods - refactoring of many classes to comply to this new access method - make shure that the cache is always written if something was loaded - some redesign of the process how http response results are feeded into the new indexing queue - introduction of a cache read policy: * never use the cache * use the cache if entry exist * use the cache if the proxy freshness rule confirmes * use only the cache and go never online - added configuration options for the crawl profiles to use the new cache policies. There is not yet a input during crawl start to set the policy but this will be added in another step. - set the default policies for the existing crawl profiles. If you want them to appear in your default profiles you must delete the crawl profiles database; othervise the policy is 'proxy freshness rule' - enhanced some cache access methods in such a way that unnecessary retrievals are omitted (i.e. for size computation). That should reduce some IO but also a lot of CPU computation because sizes were computed after decompression of content after retrieval of the content from the disc. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6239 6c8d7289-2bf4-0310-a012-ef5d649a1542
60 lines
1.4 KiB
Java
60 lines
1.4 KiB
Java
/**
|
|
* MultiOutputStream.java
|
|
* @since 26.08.2008
|
|
*/
|
|
package de.anomic.http.client;
|
|
|
|
import java.io.IOException;
|
|
import java.io.OutputStream;
|
|
|
|
/**
|
|
* writes to multiple {link OutputStream}s (parallel)
|
|
*
|
|
* @author daniel
|
|
*
|
|
*/
|
|
public class MultiOutputStream extends OutputStream {
|
|
|
|
private final OutputStream[] streams;
|
|
|
|
/**
|
|
* creates a new MultiOutputStream
|
|
*
|
|
* @param streams
|
|
*/
|
|
public MultiOutputStream(final OutputStream[] streams) {
|
|
super();
|
|
// make a copy to avoid external modifications
|
|
this.streams = new OutputStream[streams.length];
|
|
System.arraycopy(streams, 0, this.streams, 0, streams.length);
|
|
}
|
|
|
|
/**
|
|
* writes the byte to each of the streams
|
|
*
|
|
* @see java.io.OutputStream#write(int)
|
|
*/
|
|
@Override
|
|
public void write(int b) throws IOException {
|
|
for (OutputStream stream: streams) {
|
|
stream.write(b);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* writes the byte[] to each of the streams
|
|
* overriding this high-level method causes less overhead
|
|
* than overriding only the low-level write method:
|
|
* it causes (a large number) less 'for' loops
|
|
*
|
|
* @see java.io.OutputStream#write(int)
|
|
*/
|
|
@Override
|
|
public void write(byte[] b, int start, int len) throws IOException {
|
|
for (OutputStream stream: streams) {
|
|
stream.write(b, start, len);
|
|
}
|
|
}
|
|
|
|
}
|