Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

reduced logging overhead (a bit)

Browse Source
This commit is contained in:
orbiter 2012-07-12 19:23:40 +02:00
parent e76159040b
commit 482afed07c
13 changed files with 86 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/net/yacy/document/AbstractParser.java
View File

@ -30,7 +30,7 @@ import net.yacy.kelondro.logging.Log;
public abstract class AbstractParser implements Parser {
protected final Log log = new Log("PARSER");
public final static Log log = new Log("PARSER");
protected final Set<String> SUPPORTED_MIME_TYPES = new HashSet<String>();
protected final Set<String> SUPPORTED_EXTENSIONS = new HashSet<String>();
private final String name;

30
source/net/yacy/document/TextParser.java
View File

@ -62,14 +62,12 @@ import net.yacy.document.parser.augment.AugmentParser;
import net.yacy.document.parser.images.genericImageParser;
import net.yacy.document.parser.rdfa.impl.RDFaParser;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
public final class TextParser {
private static final Log log = new Log("PARSER");
private static final Object v = new Object();
private static final Parser genericIdiom = new genericParser();
@ -129,13 +127,13 @@ public final class TextParser {
mime2parser.put(mimeType, p0);
}
p0.add(parser);
Log.logInfo("PARSER", "Parser for mime type '" + mimeType + "': " + parser.getName());
AbstractParser.log.logInfo("Parser for mime type '" + mimeType + "': " + parser.getName());
}
if (prototypeMime != null) for (String ext: parser.supportedExtensions()) {
ext = ext.toLowerCase();
final String s = ext2mime.get(ext);
if (s != null && !s.equals(prototypeMime)) Log.logInfo("PARSER", "Parser for extension '" + ext + "' was set to mime '" + s + "', overwriting with new mime '" + prototypeMime + "'.");
if (s != null && !s.equals(prototypeMime)) AbstractParser.log.logInfo("Parser for extension '" + ext + "' was set to mime '" + s + "', overwriting with new mime '" + prototypeMime + "'.");
ext2mime.put(ext, prototypeMime);
}
@ -148,7 +146,7 @@ public final class TextParser {
ext2parser.put(ext, p0);
}
p0.add(parser);
Log.logInfo("PARSER", "Parser for extension '" + ext + "': " + parser.getName());
AbstractParser.log.logInfo("Parser for extension '" + ext + "': " + parser.getName());
}
}
@ -162,10 +160,10 @@ public final class TextParser {
BufferedInputStream sourceStream = null;
Document[] docs = null;
try {
if (log.isFine()) log.logFine("Parsing '" + location + "' from file");
if (AbstractParser.log.isFine()) AbstractParser.log.logFine("Parsing '" + location + "' from file");
if (!sourceFile.exists() || !sourceFile.canRead() || sourceFile.length() == 0) {
final String errorMsg = sourceFile.exists() ? "Empty resource file." : "No resource content available (2).";
log.logInfo("Unable to parse '" + location + "'. " + errorMsg);
AbstractParser.log.logInfo("Unable to parse '" + location + "'. " + errorMsg);
throw new Parser.Failure(errorMsg, location);
}
sourceStream = new BufferedInputStream(new FileInputStream(sourceFile));
@ -173,7 +171,7 @@ public final class TextParser {
} catch (final Exception e) {
if (e instanceof InterruptedException) throw (InterruptedException) e;
if (e instanceof Parser.Failure) throw (Parser.Failure) e;
log.logSevere("Unexpected exception in parseSource from File: " + e.getMessage(), e);
AbstractParser.log.logSevere("Unexpected exception in parseSource from File: " + e.getMessage(), e);
throw new Parser.Failure("Unexpected exception: " + e.getMessage(), location);
} finally {
if (sourceStream != null) try { sourceStream.close(); } catch (final Exception ex) {}
@ -188,14 +186,14 @@ public final class TextParser {
final String charset,
final byte[] content
) throws Parser.Failure {
if (log.isFine()) log.logFine("Parsing '" + location + "' from byte-array");
if (AbstractParser.log.isFine()) AbstractParser.log.logFine("Parsing '" + location + "' from byte-array");
mimeType = normalizeMimeType(mimeType);
Set<Parser> idioms = null;
try {
idioms = parsers(location, mimeType);
} catch (final Parser.Failure e) {
final String errorMsg = "Parser Failure for extension '" + location.getFileExtension() + "' or mimetype '" + mimeType + "': " + e.getMessage();
log.logWarning(errorMsg);
AbstractParser.log.logWarning(errorMsg);
throw new Parser.Failure(errorMsg, location);
}
assert !idioms.isEmpty() : "no parsers applied for url " + location.toNormalform(true, false);
@ -212,14 +210,14 @@ public final class TextParser {
final long contentLength,
final InputStream sourceStream
) throws Parser.Failure {
if (log.isFine()) log.logFine("Parsing '" + location + "' from stream");
if (AbstractParser.log.isFine()) AbstractParser.log.logFine("Parsing '" + location + "' from stream");
mimeType = normalizeMimeType(mimeType);
Set<Parser> idioms = null;
try {
idioms = parsers(location, mimeType);
} catch (final Parser.Failure e) {
final String errorMsg = "Parser Failure for extension '" + location.getFileExtension() + "' or mimetype '" + mimeType + "': " + e.getMessage();
log.logWarning(errorMsg);
AbstractParser.log.logWarning(errorMsg);
throw new Parser.Failure(errorMsg, location);
}
assert !idioms.isEmpty() : "no parsers applied for url " + location.toNormalform(true, false);
@ -251,12 +249,12 @@ public final class TextParser {
final String charset,
final InputStream sourceStream
) throws Parser.Failure {
if (log.isFine()) log.logFine("Parsing '" + location + "' from stream");
if (AbstractParser.log.isFine()) AbstractParser.log.logFine("Parsing '" + location + "' from stream");
final String fileExt = location.getFileExtension();
final String documentCharset = htmlParser.patchCharsetEncoding(charset);
assert parser != null;
if (log.isFine()) log.logInfo("Parsing " + location + " with mimeType '" + mimeType + "' and file extension '" + fileExt + "'.");
if (AbstractParser.log.isFine()) AbstractParser.log.logFine("Parsing " + location + " with mimeType '" + mimeType + "' and file extension '" + fileExt + "'.");
try {
final Document[] docs = parser.parse(location, mimeType, documentCharset, sourceStream);
return docs;
@ -273,7 +271,7 @@ public final class TextParser {
final byte[] sourceArray
) throws Parser.Failure {
final String fileExt = location.getFileExtension();
if (log.isFine()) log.logInfo("Parsing " + location + " with mimeType '" + mimeType + "' and file extension '" + fileExt + "' from byte[]");
if (AbstractParser.log.isFine()) AbstractParser.log.logFine("Parsing " + location + " with mimeType '" + mimeType + "' and file extension '" + fileExt + "' from byte[]");
final String documentCharset = htmlParser.patchCharsetEncoding(charset);
assert !parsers.isEmpty();
@ -315,7 +313,7 @@ public final class TextParser {
}
String failedParsers = "";
for (final Map.Entry<Parser, Parser.Failure> error: failedParser.entrySet()) {
log.logWarning("tried parser '" + error.getKey().getName() + "' to parse " + location.toNormalform(true, false) + " but failed: " + error.getValue().getMessage(), error.getValue());
AbstractParser.log.logWarning("tried parser '" + error.getKey().getName() + "' to parse " + location.toNormalform(true, false) + " but failed: " + error.getValue().getMessage(), error.getValue());
failedParsers += error.getKey().getName() + " ";
}
throw new Parser.Failure("All parser failed: " + failedParsers, location);

4
source/net/yacy/document/parser/mmParser.java
View File

@ -93,9 +93,9 @@ public class mmParser extends AbstractParser implements Parser {
content = UTF8.getBytes(sb.toString());
} catch (SAXException ex) {
this.log.logWarning(ex.getMessage());
AbstractParser.log.logWarning(ex.getMessage());
} catch (IOException ex) {
this.log.logWarning(ex.getMessage());
AbstractParser.log.logWarning(ex.getMessage());
}
return new Document[]{new Document(

2
source/net/yacy/document/parser/pptParser.java
View File

@ -108,7 +108,7 @@ public class pptParser extends AbstractParser implements Parser {
*/
Log.logException(e);
final String errorMsg = "Unable to parse the ppt document '" + location + "':" + e.getMessage();
this.log.logSevere(errorMsg);
AbstractParser.log.logSevere(errorMsg);
throw new Parser.Failure(errorMsg, location);
}
}

4
source/net/yacy/document/parser/psParser.java
View File

@ -77,7 +77,7 @@ public class psParser extends AbstractParser implements Parser {
final int returnCode = ps2asciiProc.waitFor();
return (returnCode == 0);
} catch (final Exception e) {
if (this.log != null) this.log.logInfo("ps2ascii not found. Switching to java parser mode.");
if (AbstractParser.log != null) AbstractParser.log.logInfo("ps2ascii not found. Switching to java parser mode.");
return false;
}
}
@ -246,7 +246,7 @@ public class psParser extends AbstractParser implements Parser {
execCode = ps2asciiProc.waitFor();
} catch (final Exception e) {
final String errorMsg = "Unable to convert ps to ascii. " + e.getMessage();
this.log.logSevere(errorMsg);
AbstractParser.log.logSevere(errorMsg);
throw new Exception(errorMsg);
}

7
source/net/yacy/document/parser/sevenzipParser.java
View File

@ -75,15 +75,14 @@ public class sevenzipParser extends AbstractParser implements Parser {
null,
false);
Handler archive;
super.log.logFine("opening 7zip archive...");
AbstractParser.log.logFine("opening 7zip archive...");
try {
archive = new Handler(source);
} catch (final IOException e) {
throw new Parser.Failure("error opening 7zip archive: " + e.getMessage(), location);
}
final SZParserExtractCallback aec = new SZParserExtractCallback(super.log, archive,
doc, location.getFile());
super.log.logFine("processing archive contents...");
final SZParserExtractCallback aec = new SZParserExtractCallback(AbstractParser.log, archive, doc, location.getFile());
AbstractParser.log.logFine("processing archive contents...");
try {
archive.Extract(null, -1, 0, aec);
return doc;

2
source/net/yacy/document/parser/swfParser.java
View File

@ -130,7 +130,7 @@ public class swfParser extends AbstractParser implements Parser {
// if an unexpected error occures just log the error and raise a new Parser.Failure
final String errorMsg = "Unable to parse the swf document '" + location + "':" + e.getMessage();
this.log.logSevere(errorMsg);
AbstractParser.log.logSevere(errorMsg);
throw new Parser.Failure(errorMsg, location);
}
}

4
source/net/yacy/document/parser/tarParser.java
View File

@ -94,12 +94,12 @@ public class tarParser extends AbstractParser implements Parser {
if (subDocs == null) continue;
for (final Document d: subDocs) docacc.add(d);
} catch (final Parser.Failure e) {
this.log.logWarning("tar parser entry " + name + ": " + e.getMessage());
AbstractParser.log.logWarning("tar parser entry " + name + ": " + e.getMessage());
} finally {
if (tmp != null) FileUtils.deletedelete(tmp);
}
} catch (final IOException e) {
this.log.logWarning("tar parser:" + e.getMessage());
AbstractParser.log.logWarning("tar parser:" + e.getMessage());
break;
}
}

2
source/net/yacy/document/parser/vcfParser.java
View File

@ -198,7 +198,7 @@ public class vcfParser extends AbstractParser implements Parser {
}
} else {
if (this.log.isFinest()) this.log.logFinest("Invalid data in vcf file" +
if (AbstractParser.log.isFinest()) AbstractParser.log.logFinest("Invalid data in vcf file" +
"\n\tURL: " + url +
"\n\tLine: " + line +
"\n\tLine-Nr: " + lineNr);

4
source/net/yacy/document/parser/vsdParser.java
View File

@ -124,13 +124,13 @@ public class vsdParser extends AbstractParser implements Parser {
// if an unexpected error occures just log the error and raise a new ParserException
final String errorMsg = "Unable to parse the vsd document '" + location + "':" + e.getMessage();
this.log.logSevere(errorMsg);
AbstractParser.log.logSevere(errorMsg);
throw new Parser.Failure(errorMsg, location);
} finally {
if (theDoc == null) {
// if an unexpected error occures just log the error and raise a new Parser.Failure
final String errorMsg = "Unable to parse the vsd document '" + location + "': possibly out of memory";
this.log.logSevere(errorMsg);
AbstractParser.log.logSevere(errorMsg);
throw new Parser.Failure(errorMsg, location);
}
}

4
source/net/yacy/document/parser/zipParser.java
View File

@ -93,12 +93,12 @@ public class zipParser extends AbstractParser implements Parser {
if (docs == null) continue;
for (final Document d: docs) docacc.add(d);
} catch (final Parser.Failure e) {
this.log.logWarning("ZIP parser entry " + name + ": " + e.getMessage());
AbstractParser.log.logWarning("ZIP parser entry " + name + ": " + e.getMessage());
} finally {
if (tmp != null) FileUtils.deletedelete(tmp);
}
} catch (final IOException e) {
this.log.logWarning("ZIP parser:" + e.getMessage());
AbstractParser.log.logWarning("ZIP parser:" + e.getMessage());
break;
}
}

82
source/net/yacy/kelondro/blob/HeapReader.java
View File

@ -57,6 +57,8 @@ public class HeapReader {
//public final static long keepFreeMem = 20 * 1024 * 1024;
private final static Log log = new Log("HeapReader");
// input values
protected int keylength; // the length of the primary key
protected File heapFile; // the file of the heap
@ -104,10 +106,10 @@ public class HeapReader {
}
}
if (!ok) {
Log.logWarning("HeapReader", "verification of idx file for " + heapFile.toString() + " failed, re-building index");
log.logWarning("verification of idx file for " + heapFile.toString() + " failed, re-building index");
initIndexReadFromHeap();
} else {
Log.logInfo("HeapReader", "using a dump of the index of " + heapFile.toString() + ".");
log.logInfo("using a dump of the index of " + heapFile.toString() + ".");
}
} else {
// if we did not have a dump, create a new index
@ -158,7 +160,7 @@ public class HeapReader {
// if this is successful, return true; otherwise false
String fingerprint = fingerprintFileHash(this.heapFile);
if (fingerprint == null) {
Log.logSevere("HeapReader", "cannot generate a fingerprint for " + this.heapFile + ": null");
log.logSevere("cannot generate a fingerprint for " + this.heapFile + ": null");
return false;
}
this.fingerprintFileIdx = HeapWriter.fingerprintIndexFile(this.heapFile, fingerprint);
@ -184,13 +186,13 @@ public class HeapReader {
// check saturation
int[] saturation = this.index.saturation();
Log.logInfo("HeapReader", "saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * this.index.size() / 1024 / 1024) + " MB");
log.logInfo("saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * this.index.size() / 1024 / 1024) + " MB");
// read the gap file:
try {
this.free = new Gap(this.fingerprintFileGap);
} catch (IOException e) {
Log.logException(e);
Log.logException(e);
return false;
}
@ -242,7 +244,7 @@ public class HeapReader {
private void initIndexReadFromHeap() throws IOException {
// this initializes the this.index object by reading positions from the heap file
Log.logInfo("HeapReader", "generating index for " + this.heapFile.toString() + ", " + (this.file.length() / 1024 / 1024) + " MB. Please wait.");
log.logInfo("generating index for " + this.heapFile.toString() + ", " + (this.file.length() / 1024 / 1024) + " MB. Please wait.");
this.free = new Gap();
HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", this.keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
@ -261,7 +263,7 @@ public class HeapReader {
//assert reclen > 0 : " reclen == 0 at seek pos " + seek;
if (reclen == 0) {
// very bad file inconsistency
Log.logSevere("HeapReader", "reclen == 0 at seek pos " + seek + " in file " + this.heapFile);
log.logSevere("reclen == 0 at seek pos " + seek + " in file " + this.heapFile);
this.file.setLength(seek); // delete everything else at the remaining of the file :-(
break loop;
}
@ -288,7 +290,7 @@ public class HeapReader {
this.file.seek(seek + 4);
Arrays.fill(key, (byte) 0);
this.file.write(key); // mark the place as empty record
Log.logWarning("HeapReader", "BLOB " + this.heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek);
log.logWarning("BLOB " + this.heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek);
}
}
// new seek position
@ -301,11 +303,11 @@ public class HeapReader {
try {
this.index = indexready.result();
} catch (InterruptedException e) {
Log.logException(e);
Log.logException(e);
} catch (ExecutionException e) {
Log.logException(e);
Log.logException(e);
}
Log.logInfo("HeapReader", "finished index generation for " + this.heapFile.toString() + ", " + this.index.size() + " entries, " + this.free.size() + " gaps.");
log.logInfo("finished index generation for " + this.heapFile.toString() + ", " + this.index.size() + " entries, " + this.free.size() + " gaps.");
}
private void mergeFreeEntries() throws IOException {
@ -333,7 +335,7 @@ public class HeapReader {
lastFree = nextFree;
}
}
Log.logInfo("HeapReader", "BLOB " + this.heapFile.toString() + ": merged " + merged + " free records");
log.logInfo("BLOB " + this.heapFile.toString() + ": merged " + merged + " free records");
if (merged > 0) deleteFingerprint();
}
}
@ -353,7 +355,7 @@ public class HeapReader {
public int size() {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in size(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in size(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return 0;
}
synchronized (this.index) {
@ -364,7 +366,7 @@ public class HeapReader {
public boolean isEmpty() {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in isEmpty(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in isEmpty(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return true;
}
synchronized (this.index) {
@ -380,7 +382,7 @@ public class HeapReader {
public boolean containsKey(byte[] key) {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in containsKey(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in containsKey(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return false;
}
key = normalizeKey(key);
@ -405,7 +407,7 @@ public class HeapReader {
protected synchronized byte[] firstKey() throws IOException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in firstKey(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in firstKey(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return null;
}
synchronized (this.index) {
@ -423,7 +425,7 @@ public class HeapReader {
protected byte[] first() throws IOException, RowSpaceExceededException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in first(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in first(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return null;
}
synchronized (this.index) {
@ -443,7 +445,7 @@ public class HeapReader {
protected byte[] lastKey() throws IOException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in lastKey(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in lastKey(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return null;
}
if (this.index == null) return null;
@ -462,7 +464,7 @@ public class HeapReader {
protected byte[] last() throws IOException, RowSpaceExceededException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in last(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in last(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return null;
}
synchronized (this.index) {
@ -481,7 +483,7 @@ public class HeapReader {
public byte[] get(byte[] key) throws IOException, RowSpaceExceededException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in get(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in get(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return null;
}
key = normalizeKey(key);
@ -496,7 +498,7 @@ public class HeapReader {
final int len = this.file.readInt() - this.index.row().primaryKeyLength;
if (len < 0) {
// database file may be corrupted and should be deleted :-((
Log.logSevere("HeapReader", "file " + this.file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + this.index.row().primaryKeyLength);
log.logSevere("file " + this.file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + this.index.row().primaryKeyLength);
// to get lazy over that problem (who wants to tell the user to stop operation and delete the file???) we work on like the entry does not exist
this.index.remove(key);
return null;
@ -516,7 +518,7 @@ public class HeapReader {
this.file.readFully(keyf, 0, keyf.length);
if (!this.ordering.equal(key, keyf)) {
// verification of the indexed access failed. we must re-read the index
Log.logSevere("HeapReader", "indexed verification access failed for " + this.heapFile.toString());
log.logSevere("indexed verification access failed for " + this.heapFile.toString());
// this is a severe operation, it should never happen.
// remove entry from index because keeping that element in the index would not make sense
this.index.remove(key);
@ -545,9 +547,9 @@ public class HeapReader {
try {
return get((byte[]) key);
} catch (IOException e) {
Log.logException(e);
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
Log.logException(e);
}
return null;
}
@ -572,7 +574,7 @@ public class HeapReader {
public long length(byte[] key) throws IOException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in length(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in length(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return 0;
}
key = normalizeKey(key);
@ -598,7 +600,7 @@ public class HeapReader {
try {
this.file.close();
} catch (IOException e) {
Log.logException(e);
Log.logException(e);
}
this.file = null;
if (writeIDX && this.index != null && this.free != null && (this.index.size() > 3 || this.free.size() > 3)) {
@ -607,17 +609,17 @@ public class HeapReader {
try {
String fingerprint = fingerprintFileHash(this.heapFile);
if (fingerprint == null) {
Log.logSevere("HeapReader", "cannot write a dump for " + this.heapFile.getName()+ ": fingerprint is null");
log.logSevere("cannot write a dump for " + this.heapFile.getName()+ ": fingerprint is null");
} else {
File newFingerprintFileGap = HeapWriter.fingerprintGapFile(this.heapFile, fingerprint);
if (this.fingerprintFileGap != null &&
this.fingerprintFileGap.getName().equals(newFingerprintFileGap.getName()) &&
this.fingerprintFileGap.exists()) {
Log.logInfo("HeapReader", "using existing gap dump instead of writing a new one: " + this.fingerprintFileGap.getName());
log.logInfo("using existing gap dump instead of writing a new one: " + this.fingerprintFileGap.getName());
} else {
long start = System.currentTimeMillis();
this.free.dump(newFingerprintFileGap);
Log.logInfo("HeapReader", "wrote a dump for the " + this.free.size() + " gap entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
log.logInfo("wrote a dump for the " + this.free.size() + " gap entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
}
}
this.free.clear();
@ -627,17 +629,17 @@ public class HeapReader {
if (this.fingerprintFileIdx != null &&
this.fingerprintFileIdx.getName().equals(newFingerprintFileIdx.getName()) &&
this.fingerprintFileIdx.exists()) {
Log.logInfo("HeapReader", "using existing idx dump instead of writing a new one: " + this.fingerprintFileIdx.getName());
log.logInfo("using existing idx dump instead of writing a new one: " + this.fingerprintFileIdx.getName());
} else {
long start = System.currentTimeMillis();
this.index.dump(newFingerprintFileIdx);
Log.logInfo("HeapReader", "wrote a dump for the " + this.index.size() + " index entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
log.logInfo("wrote a dump for the " + this.index.size() + " index entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
}
}
this.index.close();
this.index = null;
} catch (IOException e) {
Log.logException(e);
Log.logException(e);
}
}
if (this.free != null) this.free.clear();
@ -645,7 +647,7 @@ public class HeapReader {
if (this.index != null) this.index.close();
this.index = null;
this.closeDate = new Date();
Log.logInfo("HeapReader", "close HeapFile " + this.heapFile.getName() + "; trace: " + Log.stackTrace());
log.logInfo("close HeapFile " + this.heapFile.getName() + "; trace: " + Log.stackTrace());
}
}
@ -676,7 +678,7 @@ public class HeapReader {
public CloneableIterator<byte[]> keys(final boolean up, final boolean rotating) throws IOException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in keys(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in keys(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return null;
}
synchronized (this.index) {
@ -694,7 +696,7 @@ public class HeapReader {
public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in keys(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in keys(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return null;
}
synchronized (this.index) {
@ -705,7 +707,7 @@ public class HeapReader {
public long length() {
assert (this.index != null) : "index == null; closeDate=" + this.closeDate + ", now=" + new Date();
if (this.index == null) {
Log.logSevere("HeapReader", "this.index == null in length(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
log.logSevere("this.index == null in length(); closeDate=" + this.closeDate + ", now=" + new Date() + this.heapFile == null ? "" : (" file = " + this.heapFile.toString()));
return 0;
}
synchronized (this.index) {
@ -745,7 +747,7 @@ public class HeapReader {
try {
return new entries(this.blobFile, this.keylen);
} catch (IOException e) {
Log.logException(e);
Log.logException(e);
return null;
}
}
@ -767,7 +769,7 @@ public class HeapReader {
// read some more bytes to consume the empty record
if (len > 1) {
if (len - 1 != this.is.skipBytes(len - 1)) { // all that is remaining
Log.logWarning("HeapReader", "problem skiping " + + len + " bytes in " + this.blobFile.getName());
log.logWarning("problem skiping " + + len + " bytes in " + this.blobFile.getName());
try {this.is.close();} catch (IOException e) {}
return null;
}
@ -798,7 +800,7 @@ public class HeapReader {
// the allocation of memory for the payload may fail
// this is bad because we must interrupt the iteration here but the
// process that uses the iteration may think that the iteraton has just been completed
Log.logSevere("HeapReader", "out of memory in LookAheadIterator.next0", ee);
log.logSevere("out of memory in LookAheadIterator.next0", ee);
try {this.is.close();} catch (IOException e) {}
return null;
}
@ -851,7 +853,7 @@ public class HeapReader {
System.out.println(ASCII.String(entry.getKey()) + ":" + UTF8.String(entry.getValue()));
}
} catch (IOException e) {
Log.logException(e);
Log.logException(e);
}
}

16
source/net/yacy/kelondro/logging/Log.java
View File

@ -31,8 +31,11 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.LogManager;
@ -344,14 +347,21 @@ public final class Log {
@Override
public void run() {
logEntry entry;
Map<String, Logger> loggerCache = new HashMap<String, Logger>();
//Map<String, AtomicInteger> loggerCounter = new HashMap<String, AtomicInteger>();
try {
while ((entry = logQueue.take()) != poison) {
if (entry.logger == null) {
assert entry.loggername != null;
//AtomicInteger i = loggerCounter.get(entry.loggername);
//if (i == null) {i = new AtomicInteger(0); loggerCounter.put(entry.loggername, i);}
//i.incrementAndGet();
Logger l = loggerCache.get(entry.loggername);
if (l == null) {l = Logger.getLogger(entry.loggername); loggerCache.put(entry.loggername, l);}
if (entry.thrown == null) {
Logger.getLogger(entry.loggername).log(entry.level, entry.message);
l.log(entry.level, entry.message);
} else {
Logger.getLogger(entry.loggername).log(entry.level, entry.message, entry.thrown);
l.log(entry.level, entry.message, entry.thrown);
}
} else {
assert entry.loggername == null;
@ -365,7 +375,7 @@ public final class Log {
} catch (final InterruptedException e) {
Log.logException(e);
}
//Logger.getLogger("Log").log(Level.INFO, "closing logRunner with cached loggers: " + loggerCounter.entrySet().toString());
}
}