Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added HT Cache basic statistics (hit rate)

Browse Source
This commit is contained in:
luccioman 2017-06-15 09:50:02 +02:00
parent 5fdd5d16b1
commit 9dd790087d
4 changed files with 72 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
htroot/ConfigHTCache_p.html
View File

@ -16,6 +16,8 @@
<form action="ConfigHTCache_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<fieldset><legend>HTCache Configuration</legend>
<dl>
<dt title="A cache hit occurs when the requested data can be found in a cache.">Cache hits</dt>
<dd>#[hitRate]# ( #[hits]# hits for a total of #[requests]# requests )</dd>
<dt><label for="HTCachePath">The path where the cache is stored</label></dt>
<dd><input name="HTCachePath" id="HTCachePath" type="text" size="20" maxlength="300" value="#[HTCachePath]#" /></dd>
<dt><label>The current size of the cache</label></dt>

8
htroot/ConfigHTCache_p.java
View File

@ -28,6 +28,7 @@
import java.io.File;
import java.io.IOException;
import java.text.NumberFormat;
import java.util.zip.Deflater;
import net.yacy.cora.protocol.RequestHeader;
@ -117,6 +118,13 @@ public class ConfigHTCache_p {
prop.put("actualCacheDocCount", Cache.getActualCacheDocCount());
prop.put("docSizeAverage", Cache.getActualCacheDocCount() == 0 ? 0 : Cache.getActualCacheSize() / Cache.getActualCacheDocCount() / 1024);
prop.put("maxCacheSize", env.getConfigLong(SwitchboardConstants.PROXY_CACHE_SIZE, 64));
/* Statistics */
final long hits = Cache.getHits();
final long totalRequests = Cache.getTotalRequests();
prop.put("hits", hits);
prop.put("requests", totalRequests);
prop.put("hitRate", NumberFormat.getPercentInstance().format(Cache.getHitRate()));
// return rewrite properties
return prop;
}

63
source/net/yacy/crawler/data/Cache.java
View File

@ -40,6 +40,7 @@ import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
@ -74,7 +75,14 @@ public final class Cache {
private static Compressor fileDB = null;
private static ArrayStack fileDBunbuffered = null;
private static long maxCacheSize = Long.MAX_VALUE;
private static volatile long maxCacheSize = Long.MAX_VALUE;
/** Total number of requests for cached response since last start/initialization or cache clear */
private static AtomicLong totalRequests = new AtomicLong(0);
/** Total number of cache hits since last start/initialization or cache clear */
private static AtomicLong hits = new AtomicLong(0);
private static File cachePath = null;
private static String prefix;
public static final ConcurrentLog log = new ConcurrentLog("HTCACHE");
@ -91,6 +99,8 @@ public final class Cache {
cachePath = htCachePath;
maxCacheSize = cacheSizeMax;
prefix = peerSalt;
totalRequests.set(0);
hits.set(0);
// set/make cache path
if (!htCachePath.exists()) {
@ -201,6 +211,9 @@ public final class Cache {
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
/* Clear statistics */
totalRequests.set(0);
hits.set(0);
}
/**
@ -287,13 +300,17 @@ public final class Cache {
* @return true if the content of the url is in the cache, false otherwise
*/
public static boolean has(final byte[] urlhash) {
totalRequests.incrementAndGet();
boolean headerExists;
boolean fileExists;
//synchronized (responseHeaderDB) {
headerExists = responseHeaderDB.containsKey(urlhash);
fileExists = fileDB.containsKey(urlhash);
//}
if (headerExists && fileExists) return true;
if (headerExists && fileExists) {
hits.incrementAndGet();
return true;
}
if (!headerExists && !fileExists) return false;
// if not both is there then we do a clean-up
if (headerExists) try {
@ -317,7 +334,7 @@ public final class Cache {
* info object couldn't be created
*/
public static ResponseHeader getResponseHeader(final byte[] hash) {
totalRequests.incrementAndGet();
// loading data from database
Map<String, String> hdb = null;
try {
@ -327,8 +344,11 @@ public final class Cache {
} catch (final SpaceExceededException e) {
return null;
}
if (hdb == null) return null;
if (hdb == null) {
return null;
}
hits.incrementAndGet();
return new ResponseHeader(hdb);
}
@ -341,10 +361,14 @@ public final class Cache {
* is returned.
*/
public static byte[] getContent(final byte[] hash) {
totalRequests.incrementAndGet();
// load the url as resource from the cache
try {
final byte[] b = fileDB.get(hash);
if (b == null) return null;
if (b == null) {
return null;
}
hits.incrementAndGet();
return b;
} catch (final UnsupportedEncodingException e) {
ConcurrentLog.logException(e);
@ -362,9 +386,14 @@ public final class Cache {
}
public static boolean hasContent(final byte[] hash) {
totalRequests.incrementAndGet();
// load the url as resource from the cache
try {
return fileDB.containsKey(hash);
boolean result = fileDB.containsKey(hash);
if(result) {
hits.incrementAndGet();
}
return result;
} catch (final OutOfMemoryError e) {
ConcurrentLog.logException(e);
return false;
@ -380,4 +409,26 @@ public final class Cache {
responseHeaderDB.delete(hash);
fileDB.delete(hash);
}
/**
* @return the total number of requests for cache content since last start/initialization or cache clear
*/
public static long getTotalRequests() {
return totalRequests.get();
}
/**
* @return the total number of cache hits (cached response found) since last start/initialization or cache clear
*/
public static long getHits() {
return hits.get();
}
/**
* @return the hit rate (proportion of hits over total requests)
*/
public static double getHitRate() {
final long total = totalRequests.get();
return total > 0 ? ((Cache.getHits() / ((double) total))) : 0.0 ;
}
}

33
test/java/net/yacy/crawler/data/CacheTest.java
View File

@ -29,6 +29,7 @@ import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.charset.StandardCharsets;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@ -266,12 +267,6 @@ public class CacheTest {
/** Sleep time (in milliseconds) between each cache operations */
private final long sleepTime;
/** Number of Cache.getContent() successes (not null returned) */
private int getContentSuccesses;
/** Number of Cache.getContent() misses (null returned) */
private int getContentMisses;
/** Number of Cache.store() failures */
private int storeFailures;
@ -333,8 +328,6 @@ public class CacheTest {
this.storeTime = 0;
this.maxStoreTime = 0;
this.getContentMisses = 0;
this.getContentSuccesses = 0;
this.storeFailures = 0;
this.getContentTime = 0;
this.maxGetContentTime = 0;
@ -366,11 +359,7 @@ public class CacheTest {
/* Measure content retrieval */
time = System.nanoTime();
if (Cache.getContent(urlHash) == null) {
this.getContentMisses++;
} else {
this.getContentSuccesses++;
}
Cache.getContent(urlHash);
time = (System.nanoTime() - time);
this.getContentTime += time;
this.maxGetContentTime = Math.max(time, this.maxGetContentTime);
@ -449,14 +438,6 @@ public class CacheTest {
return this.maxDeleteTime;
}
public int getGetContentMisses() {
return this.getContentMisses;
}
public int getGetContentSuccesses() {
return this.getContentSuccesses;
}
public int getStoreFailures() {
return this.storeFailures;
}
@ -543,8 +524,6 @@ public class CacheTest {
long maxStoreTime = 0;
long getContentTime = 0;
long maxGetContentTime = 0;
int getContentMisses = 0;
int getContentSuccesses = 0;
int storeFailures = 0;
long deleteTime = 0;
long maxDeleteTime = 0;
@ -554,8 +533,6 @@ public class CacheTest {
maxStoreTime = Math.max(task.getMaxStoreTime(), maxStoreTime);
getContentTime += task.getGetContentTime();
maxGetContentTime = Math.max(task.getMaxGetContentTime(), maxGetContentTime);
getContentMisses += task.getGetContentMisses();
getContentSuccesses += task.getGetContentSuccesses();
storeFailures += task.getStoreFailures();
deleteTime += task.getDeleteTime();
maxDeleteTime = Math.max(task.getMaxDeleteTime(), maxDeleteTime);
@ -573,9 +550,9 @@ public class CacheTest {
"Cache.getContent() maximum time (ms) : " + TimeUnit.NANOSECONDS.toMillis(maxGetContentTime));
System.out.println("Cache.getContent() mean time (ms) : "
+ TimeUnit.NANOSECONDS.toMillis(getContentTime / totalSteps));
System.out.println("Cache.getContent() : misses " + getContentMisses + " successes : "
+ getContentSuccesses + " ( miss rate : "
+ ((getContentMisses / ((double) getContentSuccesses + (double) getContentMisses)) * 100) + "% )");
System.out.println("Cache hits : " + Cache.getHits() + " total requests : "
+ Cache.getTotalRequests() + " ( hit rate : "
+ NumberFormat.getPercentInstance().format(Cache.getHitRate()) + " )");
System.out.println("");
System.out.println("Cache.delete() total time (ms) : " + TimeUnit.NANOSECONDS.toMillis(deleteTime));
System.out.println("Cache.delete() maximum time (ms) : " + TimeUnit.NANOSECONDS.toMillis(maxDeleteTime));