mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
*) attempt to find bug
See: http://www.yacy-forum.de/viewtopic.php?t=1121 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@650 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
db4670a3b4
commit
5194511e8e
|
@ -370,7 +370,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
|
|||
File cacheFile = cacheManager.getCachePath(url);
|
||||
String urlHash = plasmaURL.urlHash(url);
|
||||
httpHeader cachedResponseHeader = cacheManager.getCachedResponse(urlHash);
|
||||
boolean cacheExists = ((cacheFile.exists()) && (cacheFile.isFile()) && (cachedResponseHeader != null));
|
||||
boolean cacheExists = ((cacheFile.isFile()) && (cachedResponseHeader != null));
|
||||
|
||||
// why are files unzipped upon arrival? why not zip all files in cache?
|
||||
// This follows from the following premises
|
||||
|
@ -519,10 +519,11 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
|
|||
|
||||
// the cache does either not exist or is (supposed to be) stale
|
||||
long sizeBeforeDelete = -1;
|
||||
if ((cacheFile.exists()) && (cacheFile.isFile()) && (cachedResponseHeader != null)) {
|
||||
if ((cacheFile.isFile()) && (cachedResponseHeader != null)) {
|
||||
// delete the cache
|
||||
sizeBeforeDelete = cacheFile.length();
|
||||
cacheManager.deleteFile(url);
|
||||
cacheFile.delete();
|
||||
//cacheManager.deleteFile(url); // Hermes
|
||||
conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS");
|
||||
}
|
||||
|
||||
|
|
|
@ -353,9 +353,9 @@ public final class plasmaCrawlWorker extends Thread {
|
|||
remote.close();
|
||||
log.logInfo("REJECTED WRONG MIME/EXT TYPE " + res.responseHeader.mime() + " for URL " + url.toString());
|
||||
} else {
|
||||
if (cacheFile.isFile()) {
|
||||
cacheManager.deleteFile(url);
|
||||
}
|
||||
//if (cacheFile.isFile()) { // Hermes
|
||||
// cacheManager.deleteFile(url);
|
||||
//}
|
||||
// we write the new cache entry to file system directly
|
||||
cacheFile.getParentFile().mkdirs();
|
||||
FileOutputStream fos = null;
|
||||
|
|
|
@ -157,52 +157,6 @@ public final class plasmaHTCache {
|
|||
responseHeaderDB.set(urlHash, responseHeader);
|
||||
}
|
||||
|
||||
private boolean deleteFile(File file) {
|
||||
if (file.exists()) {
|
||||
long size = file.length();
|
||||
if (file.delete()) {
|
||||
currCacheSize -= size;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean deleteFileandDirs (File f, String msg) {
|
||||
if (deleteFile (f)) {
|
||||
log.logInfo("DELETED " + msg + " CACHE : " + f.toString());
|
||||
f = f.getParentFile();
|
||||
// If the has been emptied, remove it
|
||||
// Loop as long as we produce empty driectoriers, but stop at HTCACHE
|
||||
while ((!(f.equals(cachePath))) && (f.isDirectory()) && (f.list().length == 0)) {
|
||||
if (f.delete()) log.logInfo("DELETED EMPTY DIRECTORY : " + f.toString());
|
||||
f = f.getParentFile();
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean deleteURLfromCache (URL url, String msg) {
|
||||
if (deleteFileandDirs(getCachePath(url), msg)) {
|
||||
try {
|
||||
// As the file is gone, the entry in responseHeader.db is not needed anymore
|
||||
log.logFinest("Trying to remove responseHeader from URL: " + url.toString());
|
||||
responseHeaderDB.remove(plasmaURL.urlHash(url));
|
||||
} catch (IOException e) {
|
||||
log.logInfo("IOExeption removing response header from DB: " + e.getMessage(), e);
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean deleteFile(URL url) {
|
||||
return deleteURLfromCache(url, "FROM");
|
||||
}
|
||||
|
||||
public boolean writeFile(URL url, byte[] array) {
|
||||
if (array == null) return false;
|
||||
File file = getCachePath(url);
|
||||
|
@ -234,6 +188,48 @@ public final class plasmaHTCache {
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
private boolean deleteFile(File file) {
|
||||
if (file.exists()) {
|
||||
long size = file.length();
|
||||
if (file.delete()) {
|
||||
currCacheSize -= size;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean deleteFileandDirs (File f, String msg) {
|
||||
if (deleteFile (f)) {
|
||||
log.logInfo("DELETED " + msg + " CACHE : " + f.toString());
|
||||
f = f.getParentFile();
|
||||
// If the has been emptied, remove it
|
||||
// Loop as long as we produce empty driectoriers, but stop at HTCACHE
|
||||
while ((!(f.equals(cachePath))) && (f.isDirectory()) && (f.list().length == 0)) {
|
||||
if (f.delete()) log.logInfo("DELETED EMPTY DIRECTORY : " + f.toString());
|
||||
f = f.getParentFile();
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean deleteURLfromCache (URL url, String msg) {
|
||||
if (deleteFileandDirs(getCachePath(url), msg)) {
|
||||
try {
|
||||
// As the file is gone, the entry in responseHeader.db is not needed anymore
|
||||
log.logFinest("Trying to remove responseHeader from URL: " + url.toString());
|
||||
responseHeaderDB.remove(plasmaURL.urlHash(url));
|
||||
} catch (IOException e) {
|
||||
log.logInfo("IOExeption removing response header from DB: " + e.getMessage(), e);
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
private void cleanupDoIt(long newCacheSize) {
|
||||
File f;
|
||||
while ((currCacheSize >= newCacheSize) && (cacheAge.size() > 0)) {
|
||||
|
@ -254,11 +250,52 @@ public final class plasmaHTCache {
|
|||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
private boolean deleteFile(File file) {
|
||||
long size = file.length();
|
||||
if (file.exists()) {
|
||||
currCacheSize -= size;
|
||||
return file.delete();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean deleteFile(URL url) {
|
||||
return deleteFile(getCachePath(url));
|
||||
}
|
||||
|
||||
private void cleanupCache(long newCacheSize) {
|
||||
File object;
|
||||
long size;
|
||||
while (currCacheSize > maxCacheSize && cacheAge.size() > 0) {
|
||||
object = (File) cacheAge.remove(cacheAge.firstKey());
|
||||
if (object != null) {
|
||||
size = object.length();
|
||||
if (object.isFile() && object.delete()) {
|
||||
currCacheSize -= size;
|
||||
log.logInfo("DELETED OLD CACHE: " + object.toString());
|
||||
object = object.getParentFile();
|
||||
if (object.isDirectory() && object.list().length == 0) {
|
||||
if (object.delete()) {
|
||||
try {
|
||||
log.logInfo("DELETED EMPTY DIRECTORY: " + object.toString());
|
||||
responseHeaderDB.remove(plasmaURL.urlHash(getURL(cachePath , object)));
|
||||
} catch (IOException e) {
|
||||
log.logWarning("HTCACHE: IOExeption removing response header from DB: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanup() {
|
||||
// clean up cache to have 3% (enough) space for next entries
|
||||
// clean up cache to have 4% (enough) space for next entries
|
||||
if ((currCacheSize >= maxCacheSize) && (cacheAge.size() > 0)) {
|
||||
if (maxCacheSize > 0) cleanupDoIt(maxCacheSize - ((maxCacheSize / 100) * 3));
|
||||
if (maxCacheSize > 0) cleanupCache(maxCacheSize - ((maxCacheSize / 100) * 4));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user