Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

- added migration class to go from index collections to the index cell data structure.

Browse Source 
- added better control over file deletion, because this sometimes fails, especially on windows

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5756 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2009-03-30 15:31:25 +00:00
parent 9ab009b16b
commit 96eaecda3e
56 changed files with 559 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
htroot/ConfigAppearance_p.java
View File

@ -81,7 +81,7 @@ public class ConfigAppearance_p {
if (post.containsKey("delete_button")) {
// delete skin
final File skinfile = new File(skinPath, post.get("skin"));
skinfile.delete();
FileUtils.deletedelete(skinfile);
}
if (post.containsKey("install_button")) {

2
htroot/ConfigLanguage_p.java
View File

@ -71,7 +71,7 @@ public class ConfigLanguage_p {
//delete language file
}else if(post.containsKey("delete")){
final File langfile= new File(langPath, post.get("language"));
langfile.delete();
FileUtils.deletedelete(langfile);
//load language file from URL
} else if (post.containsKey("url")){

3
htroot/ConfigUpdate_p.java
View File

@ -31,6 +31,7 @@ import java.util.Iterator;
import java.util.TreeSet;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -77,7 +78,7 @@ public class ConfigUpdate_p {
final String release = post.get("releaseinstall", "");
if(release.length() > 0) {
try {
new File(sb.releasePath, release).delete();
FileUtils.deletedelete(new File(sb.releasePath, release));
} catch (final NullPointerException e) {
sb.getLog().logSevere("AUTO-UPDATE: could not delete release " + release + ": " + e.getMessage());
}

3
source/de/anomic/crawler/CrawlProfile.java
View File

@ -38,6 +38,7 @@ import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@ -73,7 +74,7 @@ public class CrawlProfile {
public void clear() {
// deletes the profile database and creates a new one
if (profileTable != null) profileTable.close();
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
FileUtils.deletedelete(profileTableFile);
profileTableFile.getParentFile().mkdirs();
BLOB dyn = null;
try {

3
source/de/anomic/crawler/CrawlQueues.java
View File

@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentHashMap;
import de.anomic.kelondro.table.FlexWidthArray;
import de.anomic.kelondro.text.Document;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
@ -77,7 +78,7 @@ public class CrawlQueues {
if (errorDBFile.exists()) {
// delete the error db to get a fresh each time on startup
// this is useful because there is currently no re-use of the data in this table.
if (errorDBFile.isDirectory()) FlexWidthArray.delete(plasmaPath, "urlError2.db"); else errorDBFile.delete();
if (errorDBFile.isDirectory()) FlexWidthArray.delete(plasmaPath, "urlError2.db"); else FileUtils.deletedelete(errorDBFile);
}
errorURL = new ZURL(plasmaPath, "urlError3.db", false);
delegatedURL = new ZURL(plasmaPath, "urlDelegated3.db", true);

3
source/de/anomic/crawler/NoticeURLImporter.java
View File

@ -6,6 +6,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaWordIndex;
public class NoticeURLImporter extends AbstractImporter implements Importer {
@ -76,7 +77,7 @@ public class NoticeURLImporter extends AbstractImporter implements Importer {
try {
this.importProfileDB = new CrawlProfile(profileDbFile);
} catch (IOException e) {
profileDbFile.delete();
FileUtils.deletedelete(profileDbFile);
try {
this.importProfileDB = new CrawlProfile(profileDbFile);
} catch (IOException e1) {

2
source/de/anomic/crawler/RobotsTxt.java
View File

@ -84,7 +84,7 @@ public class RobotsTxt {
private void resetDatabase() {
// deletes the robots.txt database and creates a new one
if (robotsTable != null) robotsTable.close();
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
FileUtils.deletedelete(robotsTableFile);
robotsTableFile.getParentFile().mkdirs();
BLOB blob = null;
try {

3
source/de/anomic/crawler/ZURL.java
View File

@ -38,6 +38,7 @@ import de.anomic.kelondro.index.ObjectIndex;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.table.FlexWidthArray;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@ -66,7 +67,7 @@ public class ZURL {
final File f = new File(cachePath, tablename);
if (startWithEmptyFile) {
if (f.exists()) {
if (f.isDirectory()) FlexWidthArray.delete(cachePath, tablename); else f.delete();
if (f.isDirectory()) FlexWidthArray.delete(cachePath, tablename); else FileUtils.deletedelete(f);
}
}
this.urlIndex = new EcoTable(f, rowdef, EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);

3
source/de/anomic/data/userDB.java
View File

@ -42,6 +42,7 @@ import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
public final class userDB {
@ -63,7 +64,7 @@ public final class userDB {
void resetDatabase() {
// deletes the database and creates a new one
if (userTable != null) userTable.close();
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
FileUtils.deletedelete(userTableFile);
userTableFile.getParentFile().mkdirs();
userTable = new MapView(new BLOBTree(userTableFile, true, true, 256, 512, '_', NaturalOrder.naturalOrder, true, false, false), 10);
}

11
source/de/anomic/kelondro/blob/BLOBArray.java
View File

@ -41,6 +41,7 @@ import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.order.MergeIterator;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils;
public class BLOBArray implements BLOB {
@ -172,6 +173,7 @@ public class BLOBArray implements BLOB {
}
blobItem b = this.blobs.remove(bestIndex);
b.blob.close(false);
b.blob = null;
return b.location;
}
@ -181,6 +183,7 @@ public class BLOBArray implements BLOB {
if (smallestFromFirst2 && this.blobs.get(1).location.length() < this.blobs.get(0).location.length()) idx = 1;
blobItem b = this.blobs.remove(idx);
b.blob.close(false);
b.blob = null;
return b.location;
}
@ -201,6 +204,7 @@ public class BLOBArray implements BLOB {
}
b = this.blobs.remove(bestIndex);
b.blob.close(false);
b.blob = null;
return b.location;
}
@ -245,7 +249,8 @@ public class BLOBArray implements BLOB {
// too old
blobItem oldestBLOB = blobs.remove(0);
oldestBLOB.blob.close(false);
if (!oldestBLOB.location.delete()) oldestBLOB.location.deleteOnExit();
oldestBLOB.blob = null;
FileUtils.deletedelete(oldestBLOB.location);
}
// size limit
@ -253,7 +258,7 @@ public class BLOBArray implements BLOB {
// too large
blobItem oldestBLOB = blobs.remove(0);
oldestBLOB.blob.close(false);
if (!oldestBLOB.location.delete()) oldestBLOB.location.deleteOnExit();
FileUtils.deletedelete(oldestBLOB.location);
}
}
@ -303,7 +308,7 @@ public class BLOBArray implements BLOB {
for (blobItem bi: blobs) {
bi.blob.clear();
bi.blob.close(false);
bi.location.delete();
FileUtils.deletedelete(bi.location);
}
blobs.clear();
}

1
source/de/anomic/kelondro/blob/BLOBHeap.java
View File

@ -241,6 +241,7 @@ public final class BLOBHeap extends BLOBHeapModifier implements BLOB {
}
this.buffer = null;
super.close();
assert file == null;
}
/**

16
source/de/anomic/kelondro/blob/BLOBHeapModifier.java
View File

@ -32,6 +32,7 @@ import java.util.SortedMap;
import de.anomic.kelondro.io.CachedRandomAccess;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.Log;
@ -90,12 +91,9 @@ public class BLOBHeapModifier extends HeapReader implements BLOB {
public synchronized void clear() throws IOException {
this.index.clear();
this.free.clear();
try {
this.file.close();
} catch (final IOException e) {
e.printStackTrace();
}
this.heapFile.delete();
this.file.close();
this.file = null;
FileUtils.deletedelete(this.heapFile);
this.file = new CachedRandomAccess(heapFile);
}
@ -105,11 +103,7 @@ public class BLOBHeapModifier extends HeapReader implements BLOB {
public synchronized void close(boolean writeIDX) {
shrinkWithGapsAtEnd();
if (file != null) {
try {
file.close();
} catch (final IOException e) {
e.printStackTrace();
}
file.close();
}
file = null;

6
source/de/anomic/kelondro/blob/BLOBTree.java
View File

@ -54,6 +54,7 @@ import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.table.FlexTable;
import de.anomic.kelondro.table.FlexWidthArray;
import de.anomic.kelondro.table.Tree;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
public class BLOBTree implements BLOB {
@ -86,7 +87,7 @@ public class BLOBTree implements BLOB {
} catch (final IOException e) {
e.printStackTrace();
if (resetOnFail) {
file.delete();
FileUtils.deletedelete(file);
try {
fbi = new Tree(file, useNodeCache, -1, rowdef, 1, 8);
} catch (final IOException e1) {
@ -145,8 +146,7 @@ public class BLOBTree implements BLOB {
public static final void delete(final File file) {
if (file.isFile()) {
file.delete();
if (file.exists()) file.deleteOnExit();
FileUtils.deletedelete(file);
} else {
FlexWidthArray.delete(file.getParentFile(), file.getName());
}

14
source/de/anomic/kelondro/blob/HeapReader.java
View File

@ -39,13 +39,14 @@ import de.anomic.kelondro.io.CachedRandomAccess;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.RotateIterator;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.Log;
public class HeapReader {
protected int keylength; // the length of the primary key
protected LongHandleIndex index; // key/seek relation for used records
protected LongHandleIndex index; // key/seek relation for used records
protected Gap free; // set of {seek, size} pairs denoting space and position of free records
protected final File heapFile; // the file of the heap
protected final ByteOrder ordering; // the ordering on keys
@ -112,7 +113,7 @@ public class HeapReader {
return false;
}
// an index file is a one-time throw-away object, so just delete it now
fif.delete();
FileUtils.deletedelete(fif);
// read the gap file:
try {
@ -122,7 +123,7 @@ public class HeapReader {
return false;
}
// same with gap file
fgf.delete();
FileUtils.deletedelete(fgf);
// everything is fine now
return this.index.size() > 0;
@ -277,13 +278,8 @@ public class HeapReader {
* close the BLOB table
*/
public synchronized void close() {
if (file != null) try {
file.close();
} catch (final IOException e) {
e.printStackTrace();
}
if (file != null) file.close();
file = null;
free.clear();
free = null;
index.close();

3
source/de/anomic/kelondro/blob/HeapWriter.java
View File

@ -33,6 +33,7 @@ import java.io.IOException;
import de.anomic.kelondro.index.LongHandleIndex;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
public final class HeapWriter {
@ -118,7 +119,7 @@ public final class HeapWriter {
String n = f.getName();
String[] l = d.list();
for (int i = 0; i < l.length; i++) {
if (l[i].startsWith(n) && (l[i].endsWith(".idx") || l[i].endsWith(".gap"))) new File(d, l[i]).delete();
if (l[i].startsWith(n) && (l[i].endsWith(".idx") || l[i].endsWith(".gap"))) FileUtils.deletedelete(new File(d, l[i]));
}
}

3
source/de/anomic/kelondro/blob/MapView.java
View File

@ -40,6 +40,7 @@ import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.order.RotateIterator;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.ScoreCluster;
import de.anomic.kelondro.util.kelondroException;
@ -339,7 +340,7 @@ public class MapView {
public static void main(String[] args) {
// test the class
File f = new File("maptest");
if (f.exists()) f.delete();
if (f.exists()) FileUtils.deletedelete(f);
try {
// make a blob
BLOB blob = new BLOBHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 1024);

8
source/de/anomic/kelondro/io/CachedRandomAccess.java
View File

@ -125,8 +125,12 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
RAFile.seek(pos);
}
public synchronized void close() throws IOException {
if (RAFile != null) RAFile.close();
public synchronized void close() {
if (RAFile != null) try {
RAFile.close();
} catch (IOException e) {
e.printStackTrace();
}
this.cache = null;
this.RAFile = null;
}

3
source/de/anomic/kelondro/io/EcoFS.java
View File

@ -30,6 +30,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.MemoryControl;
/**
@ -573,7 +574,7 @@ public class EcoFS {
public static void main(final String[] args) {
// open a file, add one entry and exit
final File f = new File(args[0]);
if (f.exists()) f.delete();
if (f.exists()) FileUtils.deletedelete(f);
try {
final EcoFS t = new EcoFS(f, 8);
final byte[] b = new byte[8];

3
source/de/anomic/kelondro/table/AbstractRecords.java
View File

@ -49,6 +49,7 @@ import de.anomic.kelondro.io.RandomAccessIOChunks;
import de.anomic.kelondro.io.RandomAccessRecords;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
@ -440,7 +441,7 @@ public abstract class AbstractRecords implements RandomAccessRecords {
final File f = ra.file();
assert f != null;
this.entryFile.close();
f.delete();
FileUtils.deletedelete(f);
ra = (useChannel) ? new ChannelRandomAccess(f) : new FileRandomAccess(f);
initNewFile(ra, this.HANDLES.length, this.TXTPROPS.length);
}

11
source/de/anomic/kelondro/table/EcoTable.java
View File

@ -49,6 +49,7 @@ import de.anomic.kelondro.io.BufferedEcoFS;
import de.anomic.kelondro.io.EcoFS;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
@ -416,11 +417,11 @@ public class EcoTable implements ObjectIndex {
}
public synchronized void put(final Entry row) throws IOException {
assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
assert file == null || file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert table == null || table.size() == index.size();
assert row != null;
assert row.bytes() != null;
if ((row == null) || (row.bytes() == null)) return;
if (file == null || table == null || row == null || row.bytes() == null) return;
final int i = index.get(row.getPrimaryKeyBytes());
if (i == -1) {
addUnique(row);
@ -562,7 +563,7 @@ public class EcoTable implements ObjectIndex {
public void clear() throws IOException {
final File f = file.filename();
file.close();
f.delete();
FileUtils.deletedelete(f);
// make new file
FileOutputStream fos = null;
@ -694,7 +695,7 @@ public class EcoTable implements ObjectIndex {
}
public static ObjectIndex testTable(final File f, final String testentities, final int testcase) throws IOException {
if (f.exists()) f.delete();
if (f.exists()) FileUtils.deletedelete(f);
final Row rowdef = new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder);
final ObjectIndex tt = new EcoTable(f, rowdef, testcase, 100, 0);
byte[] b;

3
source/de/anomic/kelondro/table/FixedWidthArray.java
View File

@ -37,6 +37,7 @@ import de.anomic.kelondro.index.ObjectArray;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.io.RandomAccessInterface;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
public class FixedWidthArray extends FullRecords implements ObjectArray {
@ -76,7 +77,7 @@ public class FixedWidthArray extends FullRecords implements ObjectArray {
try {
return new FixedWidthArray(file, rowdef, intprops);
} catch (final IOException e) {
file.delete();
FileUtils.deletedelete(file);
try {
return new FixedWidthArray(file, rowdef, intprops);
} catch (final IOException ee) {

3
source/de/anomic/kelondro/table/FlexTable.java
View File

@ -42,6 +42,7 @@ import de.anomic.kelondro.index.RowSet;
import de.anomic.kelondro.index.ObjectIndex;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
@ -88,7 +89,7 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
if (indexfile.exists()) {
// delete existing index file
System.out.println("*** Delete File index " + indexfile);
indexfile.delete();
FileUtils.deletedelete(indexfile);
}
// fill the index

7
source/de/anomic/kelondro/table/FlexWidthArray.java
View File

@ -36,6 +36,7 @@ import de.anomic.kelondro.index.ObjectArray;
import de.anomic.kelondro.index.Column;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
@ -174,16 +175,16 @@ public class FlexWidthArray implements ObjectArray {
final File tabledir = new File(path, tablename);
if (!(tabledir.exists())) return;
if ((!(tabledir.isDirectory()))) {
tabledir.delete();
FileUtils.deletedelete(tabledir);
return;
}
final String[] files = tabledir.list();
for (int i = 0; i < files.length; i++) {
new File(tabledir, files[i]).delete();
FileUtils.deletedelete(new File(tabledir, files[i]));
}
tabledir.delete();
FileUtils.deletedelete(tabledir);
}
public void reset() throws IOException {

3
source/de/anomic/kelondro/table/SplitTable.java
View File

@ -57,6 +57,7 @@ import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.order.MergeIterator;
import de.anomic.kelondro.order.Order;
import de.anomic.kelondro.order.StackIterator;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
import de.anomic.kelondro.util.NamePrefixThreadFactory;
@ -166,7 +167,7 @@ public class SplitTable implements ObjectIndex {
for (int i = 0; i < l.length; i++) {
if (l[i].startsWith(tablename)) {
final File f = new File(path, l[i]);
if (f.isDirectory()) FlexWidthArray.delete(path, l[i]); else f.delete();
if (f.isDirectory()) FlexWidthArray.delete(path, l[i]); else FileUtils.deletedelete(f);
}
}
init(true);

5
source/de/anomic/kelondro/table/Stack.java
View File

@ -36,6 +36,7 @@ import java.util.StringTokenizer;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
public final class Stack extends FullRecords {
@ -66,7 +67,7 @@ public final class Stack extends FullRecords {
try {
return new Stack(file, rowdef);
} catch (final IOException e) {
file.delete();
FileUtils.deletedelete(file);
try {
return new Stack(file, rowdef);
} catch (final IOException ee) {
@ -358,7 +359,7 @@ public final class Stack extends FullRecords {
if (args[0].equals("-c")) {
// create <keylen> <valuelen> <filename>
final File f = new File(args[3]);
if (f.exists()) f.delete();
if (f.exists()) FileUtils.deletedelete(f);
final Stack fm = new Stack(f, lens);
fm.close();
} else if (args[0].equals("-p")) {

11
source/de/anomic/kelondro/table/Tree.java
View File

@ -51,6 +51,7 @@ import de.anomic.kelondro.index.Row.Entry;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
@ -109,7 +110,7 @@ public class Tree extends CachedRecords implements ObjectIndex {
try {
return new Tree(file, useNodeCache, preloadTime, rowdef, txtProps, txtPropsWidth);
} catch (final IOException e) {
file.delete();
FileUtils.deletedelete(file);
try {
return new Tree(file, useNodeCache, preloadTime, rowdef, txtProps, txtPropsWidth);
} catch (final IOException ee) {
@ -1354,7 +1355,7 @@ public class Tree extends CachedRecords implements ObjectIndex {
if (args[0].equals("-c")) {
// create <keylen> <valuelen> <filename>
final File f = new File(args[3]);
if (f.exists()) f.delete();
if (f.exists()) FileUtils.deletedelete(f);
final Row lens = new Row("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]), NaturalOrder.naturalOrder);
final Tree fm = new Tree(f, true, 10, lens);
fm.close();
@ -1417,7 +1418,7 @@ public class Tree extends CachedRecords implements ObjectIndex {
try {
int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
if (testFile.exists()) FileUtils.deletedelete(testFile);
tt = new Tree(testFile, true, 10, new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder));
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
@ -1482,7 +1483,7 @@ public class Tree extends CachedRecords implements ObjectIndex {
public static void smalltest() {
final File f = new File("test.db");
if (f.exists()) f.delete();
if (f.exists()) FileUtils.deletedelete(f);
try {
final Tree tt = new Tree(f, true, 10, new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder));
byte[] b;
@ -1548,7 +1549,7 @@ public class Tree extends CachedRecords implements ObjectIndex {
*/
public static Tree testTree(final File f, final String testentities) throws IOException {
if (f.exists()) f.delete();
if (f.exists()) FileUtils.deletedelete(f);
final Tree tt = new Tree(f, false, 10, new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder));
byte[] b;
for (int i = 0; i < testentities.length(); i++) {

10
source/de/anomic/kelondro/text/BufferedIndexCollection.java
View File

@ -45,6 +45,7 @@ import de.anomic.kelondro.text.IndexCollection;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceContainerOrder;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.Log;
import de.anomic.server.serverProfiling;
@ -78,7 +79,7 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme
for (ReferenceContainer c: dhtInCache) {
this.buffer.add(c);
}
new File(textindexcache, "index.dhtin.blob").delete();
FileUtils.deletedelete(new File(textindexcache, "index.dhtin.blob"));
} else {
// read in new BLOB
this.buffer = new IndexBuffer(textindexcache, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
@ -328,16 +329,11 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme
public CloneableIterator<ReferenceContainer> references(String startWordHash, boolean rot) {
final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.buffer.ordering().clone());
return new MergeIterator<ReferenceContainer>(
this.buffer.references(startWordHash, rot),
new MergeIterator<ReferenceContainer>(
this.buffer.references(startWordHash, false),
this.collections.references(startWordHash, false),
containerOrder,
ReferenceContainer.containerMergeMethod,
true),
containerOrder,
ReferenceContainer.containerMergeMethod,
true);
true);
}
}

15
source/de/anomic/kelondro/text/IODispatcher.java
View File

@ -34,6 +34,7 @@ import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.text.ReferenceContainerCache.blobFileEntries;
import de.anomic.kelondro.util.FileUtils;
/**
* merger class for files from ReferenceContainerArray.
@ -173,7 +174,7 @@ public class IODispatcher extends Thread {
public void dump() {
try {
cache.dump(file, true);
array.mountBLOBContainer(file);
array.mountBLOBFile(file);
} catch (IOException e) {
e.printStackTrace();
}
@ -223,16 +224,16 @@ public class IODispatcher extends Thread {
CloneableIterator<ReferenceContainer> i2 = new blobFileEntries(f2, payloadrow);
if (!i1.hasNext()) {
if (i2.hasNext()) {
if (!f1.delete()) f1.deleteOnExit();
FileUtils.deletedelete(f1);
if (f2.renameTo(newFile)) return newFile;
return f2;
} else {
if (!f1.delete()) f1.deleteOnExit();
if (!f2.delete()) f2.deleteOnExit();
FileUtils.deletedelete(f1);
FileUtils.deletedelete(f2);
return null;
}
} else if (!i2.hasNext()) {
if (!f2.delete()) f2.deleteOnExit();
FileUtils.deletedelete(f2);
if (f1.renameTo(newFile)) return newFile;
return f1;
}
@ -242,8 +243,8 @@ public class IODispatcher extends Thread {
merge(i1, i2, array.ordering(), writer);
writer.close(true);
// we don't need the old files any more
if (!f1.delete()) f1.deleteOnExit();
if (!f2.delete()) f2.deleteOnExit();
FileUtils.deletedelete(f1);
FileUtils.deletedelete(f2);
return newFile;
}

10
source/de/anomic/kelondro/text/IndexCell.java
View File

@ -287,6 +287,16 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn
}
}
public File newContainerBLOBFile() {
// for migration of cache files
return this.array.newContainerBLOBFile();
}
public void mountBLOBFile(File blobFile) throws IOException {
// for migration of cache files
this.array.mountBLOBFile(blobFile);
}
public void cleanupBuffer(int time) {
// do nothing
}

4
source/de/anomic/kelondro/text/IndexCollection.java
View File

@ -106,7 +106,7 @@ public class IndexCollection extends AbstractIndex implements Index {
File cop = new File(path, filenameStub + "." + fillZ(Integer.toHexString(payloadrow.objectsize).toUpperCase(), 4) + ".commons");
this.commonsPath = (useCommons) ? cop : null;
if (this.commonsPath == null) {
FileUtils.deleteDirectory(cop);
FileUtils.deletedelete(cop);
} else {
this.commonsPath.mkdirs();
}
@ -1041,8 +1041,10 @@ public class IndexCollection extends AbstractIndex implements Index {
public synchronized void close() {
this.index.close();
this.index = null;
final Iterator<FixedWidthArray> i = arrays.values().iterator();
while (i.hasNext()) i.next().close();
this.arrays = null;
}
public static void main(final String[] args) {

331
source/de/anomic/kelondro/text/IndexCollectionMigration.java Normal file
View File

@ -0,0 +1,331 @@
// IndexCollectionMigration.java
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 30.03.2009 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-03-13 11:34:51 +0100 (Fr, 13 Mrz 2009) $
// $LastChangedRevision: 5709 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro.text;
import java.io.File;
import java.io.IOException;
import java.util.Set;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.MergeIterator;
import de.anomic.kelondro.order.Order;
import de.anomic.kelondro.order.RotateIterator;
import de.anomic.kelondro.text.Index;
import de.anomic.kelondro.text.IndexCollection;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceContainerOrder;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
public final class IndexCollectionMigration extends AbstractBufferedIndex implements Index, BufferedIndex {
private final IndexCell cell;
private IndexCollection collections;
private final IODispatcher merger;
public IndexCollectionMigration (
final File indexPrimaryTextLocation,
final ByteOrder wordOrdering,
final Row payloadrow,
final int entityCacheMaxSize,
final int maxCellArrayFiles,
final IODispatcher merger,
final Log log) throws IOException {
this.merger = merger;
final File celldir = new File(indexPrimaryTextLocation, "RICELL");
this.cell = new IndexCell(
celldir,
wordOrdering,
ReferenceRow.urlEntryRow,
entityCacheMaxSize, maxCellArrayFiles, this.merger);
final File textindexcache = new File(indexPrimaryTextLocation, "RICACHE");
if (textindexcache.exists()) {
// migrate the "index.dhtout.blob" into RICELL directory
File f = new File(textindexcache, "index.dhtout.blob");
if (f.exists()) {
File n = this.cell.newContainerBLOBFile();
f.renameTo(n);
this.cell.mountBLOBFile(n);
}
f = new File(textindexcache, "index.dhtin.blob");
if (f.exists()) {
File n = this.cell.newContainerBLOBFile();
f.renameTo(n);
this.cell.mountBLOBFile(n);
}
// delete everything else
String[] l = textindexcache.list();
for (String s: l) {
f = new File(textindexcache, s);
FileUtils.deletedelete(f);
}
FileUtils.deletedelete(textindexcache);
}
// open collections, this is for migration only.
final File textindexcollections = new File(indexPrimaryTextLocation, "RICOLLECTION");
if (textindexcollections.exists()) {
this.collections = new IndexCollection(
textindexcollections,
"collection",
12,
Base64Order.enhancedCoder,
BufferedIndexCollection.maxCollectionPartition,
ReferenceRow.urlEntryRow,
false);
if (this.collections.size() == 0) {
// delete everything here
this.collections.close();
this.collections = null;
String[] l = textindexcollections.list();
File f;
for (String s: l) {
f = new File(textindexcollections, s);
FileUtils.deletedelete(f);
}
FileUtils.deletedelete(textindexcollections);
}
} else {
this.collections = null;
}
}
/* methods for interface Index */
public void add(final ReferenceContainer entries) throws IOException {
assert (entries.row().objectsize == ReferenceRow.urlEntryRow.objectsize);
if (this.collections != null) {
ReferenceContainer e = this.collections.delete(entries.getWordHash());
if (e != null) {
e.merge(entries);
cell.add(e);
} else {
cell.add(entries);
}
} else {
cell.add(entries);
}
}
public void add(final String wordHash, final ReferenceRow entry) throws IOException {
if (this.collections != null) {
ReferenceContainer e = this.collections.delete(wordHash);
if (e != null) {
e.add(entry);
cell.add(e);
} else {
cell.add(wordHash, entry);
}
} else {
cell.add(wordHash, entry);
}
}
public boolean has(final String wordHash) {
if (this.collections != null) {
ReferenceContainer e = this.collections.delete(wordHash);
if (e != null) {
try {
cell.add(e);
} catch (IOException e1) {
e1.printStackTrace();
}
return true;
} else {
return cell.has(wordHash);
}
} else {
return cell.has(wordHash);
}
}
public int count(String wordHash) {
if (this.collections != null) {
ReferenceContainer e = this.collections.delete(wordHash);
if (e != null) {
try {
cell.add(e);
} catch (IOException e1) {
e1.printStackTrace();
}
return cell.count(wordHash);
} else {
return cell.count(wordHash);
}
} else {
return cell.count(wordHash);
}
}
public ReferenceContainer get(final String wordHash, final Set<String> urlselection) throws IOException {
if (wordHash == null) {
// wrong input
return null;
}
if (this.collections != null) {
ReferenceContainer e = this.collections.delete(wordHash);
if (e != null) cell.add(e);
}
return this.cell.get(wordHash, urlselection);
}
public ReferenceContainer delete(final String wordHash) throws IOException {
final ReferenceContainer c = new ReferenceContainer(
wordHash,
ReferenceRow.urlEntryRow,
cell.count(wordHash));
c.addAllUnique(cell.delete(wordHash));
if (this.collections != null) c.merge(collections.delete(wordHash));
return c;
}
public boolean remove(final String wordHash, final String urlHash) throws IOException {
if (this.collections != null) {
ReferenceContainer e = this.collections.delete(wordHash);
if (e != null) cell.add(e);
}
return cell.remove(wordHash, urlHash);
}
public int remove(final String wordHash, final Set<String> urlHashes) throws IOException {
if (this.collections != null) {
ReferenceContainer e = this.collections.delete(wordHash);
if (e != null) cell.add(e);
}
return cell.remove(wordHash, urlHashes);
}
public synchronized CloneableIterator<ReferenceContainer> references(final String startHash, final boolean rot, final boolean ram) throws IOException {
final CloneableIterator<ReferenceContainer> i = wordContainers(startHash, ram);
if (rot) {
return new RotateIterator<ReferenceContainer>(i, new String(Base64Order.zero(startHash.length())), cell.size() + ((ram) ? 0 : collections.size()));
}
return i;
}
private synchronized CloneableIterator<ReferenceContainer> wordContainers(final String startWordHash, final boolean ram) throws IOException {
final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(cell.ordering().clone());
containerOrder.rotate(ReferenceContainer.emptyContainer(startWordHash, 0));
if (ram) {
return cell.references(startWordHash, true);
}
if (collections == null) return cell.references(startWordHash, false);
return new MergeIterator<ReferenceContainer>(
cell.references(startWordHash, false),
collections.references(startWordHash, false),
containerOrder,
ReferenceContainer.containerMergeMethod,
true);
}
public void clear() {
try {
cell.clear();
} catch (IOException e1) {
e1.printStackTrace();
}
if (collections != null) try {
collections.clear();
} catch (IOException e) {
e.printStackTrace();
}
}
public void close() {
cell.close();
if (collections != null) collections.close();
}
public int size() {
return (collections == null) ? cell.size() : java.lang.Math.max(collections.size(), cell.size());
}
public int minMem() {
return 1024*1024 /* indexing overhead */ + cell.minMem() + ((collections == null) ? 0 : collections.minMem());
}
/*
* methods for cache management
*/
public int getBufferMaxReferences() {
return cell.getBufferMaxReferences();
}
public long getBufferMinAge() {
return cell.getBufferMinAge();
}
public long getBufferMaxAge() {
return cell.getBufferMaxAge();
}
public long getBufferSizeBytes() {
return cell.getBufferSizeBytes();
}
public void setBufferMaxWordCount(final int maxWords) {
cell.setBufferMaxWordCount(maxWords);
}
public int getBackendSize() {
return (collections == null) ? cell.getBackendSize() : collections.size();
}
public int getBufferSize() {
return cell.getBufferSize();
}
public ByteOrder ordering() {
return cell.ordering();
}
public CloneableIterator<ReferenceContainer> references(String startWordHash, boolean rot) {
final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.cell.ordering().clone());
if (this.collections == null) return this.cell.references(startWordHash, rot);
//else
return new MergeIterator<ReferenceContainer>(
this.cell.references(startWordHash, false),
this.collections.references(startWordHash, false),
containerOrder,
ReferenceContainer.containerMergeMethod,
true);
}
public void cleanupBuffer(int time) {
this.cell.cleanupBuffer(time);
}
}

7
source/de/anomic/kelondro/text/ReferenceContainerArray.java
View File

@ -36,6 +36,7 @@ import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.util.FileUtils;
public final class ReferenceContainerArray {
@ -88,7 +89,7 @@ public final class ReferenceContainerArray {
return this.array.newBLOB(new Date());
}
public void mountBLOBContainer(File location) throws IOException {
public void mountBLOBFile(File location) throws IOException {
this.array.mountBLOB(location);
}
@ -247,6 +248,10 @@ public final class ReferenceContainerArray {
if (this.array.entries() < 2) return false;
if (this.merger.queueLength() > 0) return false;
File f1 = this.array.unmountOldestBLOB(similar);
if (f1.length() == 0) {
FileUtils.deletedelete(f1);
return true;
}
File f2 = (similar) ? this.array.unmountSimilarSizeBLOB(f1.length()) : this.array.unmountOldestBLOB(false);
merger.merge(f1, f2, this.array, this.payloadrow, newContainerBLOBFile());
return true;

3
source/de/anomic/kelondro/text/ReferenceContainerCache.java
View File

@ -41,6 +41,7 @@ import de.anomic.kelondro.blob.HeapWriter;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
@ -112,7 +113,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
public void dump(final File heapFile, boolean writeIDX) throws IOException {
assert this.cache != null;
Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + cache.size() + " rwi's");
if (heapFile.exists()) heapFile.delete();
if (heapFile.exists()) FileUtils.deletedelete(heapFile);
HeapWriter dump = new HeapWriter(heapFile, payloadrow.primaryKeyLength, Base64Order.enhancedCoder);
final long startTime = System.currentTimeMillis();
long wordcount = 0, urlcount = 0;

49
source/de/anomic/kelondro/util/FileUtils.java
View File

@ -584,8 +584,7 @@ public final class FileUtils {
if(!(to.delete() && from.renameTo(to))) {
// do it manually
copy(from, to);
if(!from.delete())
from.deleteOnExit();
FileUtils.deletedelete(from);
}
}
@ -718,23 +717,41 @@ public final class FileUtils {
}
/**
* delete a directory
* if the directory is not empty, delete also everything inside
* @param directory
* delete files and directories
* if a directory is not empty, delete also everything inside
* because deletion sometimes fails on windows, there is also a windows exec included
* @param path
*/
public static void deleteDirectory(final File directory) {
final String[] list = directory.list();
if (list != null) {
File object;
for (int i = list.length - 1; i >= 0; i--) {
object = new File(directory, list[i]);
if (object.isFile()) {
object.delete();
} else {
deleteDirectory(object);
public static void deletedelete(final File path) {
if (!path.exists()) return;
// empty the directory first
if (path.isDirectory()) {
final String[] list = path.list();
if (list != null) {
for (String s: list) deletedelete(new File(path, s));
}
}
if (!path.delete()) path.deleteOnExit();
if (path.exists()) {
if (System.getProperties().getProperty("os.name","").toLowerCase().startsWith("windows")) {
// deleting files on windows sometimes does not work with java
try {
String p = path.getCanonicalPath();
String command = "cmd /C del /F /Q " + p;
Process r = Runtime.getRuntime().exec(command);
if (r == null) {
Log.logSevere("FileUtils", "cannot execute command: " + command);
} else {
byte[] response = read(r.getInputStream());
Log.logInfo("FileUtils", "deletedelete: " + new String(response));
}
if (path.exists()) Log.logSevere("FileUtils", "cannot delete file " + p);
} catch (IOException e) {
e.printStackTrace();
}
}
}
directory.delete();
}
}

2
source/de/anomic/kelondro/util/XMLTables.java
View File

@ -88,7 +88,7 @@ public class XMLTables {
xmlenc.close();
// delete old file and rename tmp-file to old file's name
this.propFile.delete();
FileUtils.deletedelete(this.propFile);
tmpFile.renameTo(this.propFile);
// set the new time stamp

3
source/de/anomic/plasma/parser/bzip/bzipParser.java
View File

@ -31,6 +31,7 @@ import java.util.Hashtable;
import org.apache.tools.bzip2.CBZip2InputStream;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.parser.AbstractParser;
@ -115,7 +116,7 @@ public class bzipParser extends AbstractParser implements Parser {
throw new ParserException("Unexpected error while parsing bzip file. " + e.getMessage(),location);
} finally {
if (tempFile != null) tempFile.delete();
if (tempFile != null) FileUtils.deletedelete(tempFile);
}
}

3
source/de/anomic/plasma/parser/gzip/gzipParser.java
View File

@ -30,6 +30,7 @@ import java.io.InputStream;
import java.util.Hashtable;
import java.util.zip.GZIPInputStream;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.parser.AbstractParser;
@ -98,7 +99,7 @@ public class gzipParser extends AbstractParser implements Parser {
throw new ParserException("Unexpected error while parsing gzip file. " + e.getMessage(),location);
} finally {
if (tempFile != null) tempFile.delete();
if (tempFile != null) FileUtils.deletedelete(tempFile);
}
}

2
source/de/anomic/plasma/parser/mimeType/mimeTypeParser.java
View File

@ -178,7 +178,7 @@ public class mimeTypeParser extends AbstractParser implements Parser {
} catch (final IOException e) {
throw new ParserException("Unexpected error while detect mimetype of resource. " + e.getMessage(),location);
} finally {
if (dstFile != null) {dstFile.delete();}
if (dstFile != null) FileUtils.deletedelete(dstFile);
}
}

2
source/de/anomic/plasma/parser/mimeType/odtDetector.java
View File

@ -66,7 +66,7 @@ public class odtDetector implements MagicDetector {
} catch (final IOException e) {
return null;
} finally {
if (dstFile != null) {dstFile.delete();}
if (dstFile != null) FileUtils.deletedelete(dstFile);
}
}

4
source/de/anomic/plasma/parser/odt/odtParser.java
View File

@ -197,7 +197,7 @@ public class odtParser extends AbstractParser implements Parser {
if (writer != null) try { writer.close(); } catch (final Exception ex) {/* ignore this */}
// delete the file
if (writerFile != null) try { writerFile.delete(); } catch (final Exception ex) {/* ignore this */}
if (writerFile != null) FileUtils.deletedelete(writerFile);
throw new ParserException("Unexpected error while parsing odt file. " + e.getMessage(),location);
}
@ -221,7 +221,7 @@ public class odtParser extends AbstractParser implements Parser {
throw new ParserException("Unexpected error while parsing odt file. " + e.getMessage(),location);
} finally {
if (dest != null) try { dest.delete(); } catch (final Exception e){/* ignore this */}
if (dest != null) FileUtils.deletedelete(dest);
}
}

2
source/de/anomic/plasma/parser/pdf/pdfParser.java
View File

@ -177,7 +177,7 @@ public class pdfParser extends AbstractParser implements Parser {
if (writer != null) try { writer.close(); } catch (final Exception ex) {/* ignore this */}
// delete the file
if (writerFile != null) try { writerFile.delete(); } catch (final Exception ex) {/* ignore this */}
if (writerFile != null) FileUtils.deletedelete(writerFile);
e.printStackTrace();
throw new ParserException("Unexpected error while parsing pdf file. " + e.getMessage(),location);

4
source/de/anomic/plasma/parser/ps/psParser.java
View File

@ -131,7 +131,7 @@ public class psParser extends AbstractParser implements Parser {
if (e instanceof ParserException) throw (ParserException) e;
// delete temp file
if (outputFile != null) outputFile.delete();
if (outputFile != null) FileUtils.deletedelete(outputFile);
// throw exception
throw new ParserException("Unexpected error while parsing ps file. " + e.getMessage(),location);
@ -285,7 +285,7 @@ public class psParser extends AbstractParser implements Parser {
throw new ParserException("Unable to parse the ps file. " + e.getMessage(),location, e);
} finally {
if (tempFile != null) try{ tempFile.delete(); }catch(final Exception e) {/* */}
if (tempFile != null) FileUtils.deletedelete(tempFile);
}
}

2
source/de/anomic/plasma/parser/rpm/rpmParser.java
View File

@ -85,7 +85,7 @@ public class rpmParser extends AbstractParser implements Parser {
} catch (final Exception e) {
return null;
} finally {
if (dstFile != null) {dstFile.delete();}
if (dstFile != null) FileUtils.deletedelete(dstFile);
}
}

4
source/de/anomic/plasma/parser/tar/tarParser.java
View File

@ -152,7 +152,7 @@ public class tarParser extends AbstractParser implements Parser {
} catch (final ParserException e) {
this.theLogger.logInfo("Unable to parse tar file entry '" + entryName + "'. " + e.getMessage());
} finally {
if (subDocTempFile != null) try {subDocTempFile.delete(); } catch(final Exception ex){/* ignore this */}
if (subDocTempFile != null) FileUtils.deletedelete(subDocTempFile);
}
if (subDoc == null) continue;
@ -224,7 +224,7 @@ public class tarParser extends AbstractParser implements Parser {
if (docText != null) try { docText.close(); } catch (final Exception ex) {/* ignore this */}
// delete the file
if (outputFile != null) try { outputFile.delete(); } catch (final Exception ex) {/* ignore this */}
if (outputFile != null) FileUtils.deletedelete(outputFile);
throw new ParserException("Unexpected error while parsing tar resource. " + e.getMessage(),location);
}

4
source/de/anomic/plasma/parser/zip/zipParser.java
View File

@ -135,7 +135,7 @@ public class zipParser extends AbstractParser implements Parser {
} catch (final ParserException e) {
this.theLogger.logInfo("Unable to parse zip file entry '" + entryName + "'. " + e.getMessage());
} finally {
if (subDocTempFile != null) try {subDocTempFile.delete(); } catch(final Exception ex){/* ignore this */}
if (subDocTempFile != null) FileUtils.deletedelete(subDocTempFile);
}
if (subDoc == null) continue;
@ -208,7 +208,7 @@ public class zipParser extends AbstractParser implements Parser {
if (docText != null) try { docText.close(); } catch (final Exception ex) {/* ignore this */}
// delete the file
if (outputFile != null) try { outputFile.delete(); } catch (final Exception ex) {/* ignore this */}
if (outputFile != null) FileUtils.deletedelete(outputFile);
throw new ParserException("Unexpected error while parsing zip resource. " + e.getClass().getName() + ": "+ e.getMessage(),location);
}

4
source/de/anomic/plasma/plasmaHTCache.java
View File

@ -98,7 +98,7 @@ public final class plasmaHTCache {
object.getName().equals("yacy") ||
object.getName().equals("https") ||
object.getName().equals("ftp")) {
FileUtils.deleteDirectory(cachePath);
FileUtils.deletedelete(cachePath);
}
}
}
@ -121,7 +121,7 @@ public final class plasmaHTCache {
log.logFine("reset responseHeader DB with "+ responseHeaderDB.size() +" entries");
if (responseHeaderDB != null) responseHeaderDB.close();
final File dbfile = new File(cachePath, RESPONSE_HEADER_DB_NAME);
if (dbfile.exists()) dbfile.delete();
if (dbfile.exists()) FileUtils.deletedelete(dbfile);
openDB();
}

2
source/de/anomic/plasma/plasmaParserDocument.java
View File

@ -470,7 +470,7 @@ dc_rights
// delete the temp file
if ((this.text != null) && (this.text instanceof File)) {
try {
((File)this.text).delete();
FileUtils.deletedelete((File) this.text);
} catch (final Exception e) {
/* ignore this */
} finally {

6
source/de/anomic/plasma/plasmaRankingCRProcess.java
View File

@ -298,7 +298,7 @@ public class plasmaRankingCRProcess {
// store the file
acc.toFile(tmp_file);
// since this was successful, we remove the old file and move the new file to it
to_file.delete();
FileUtils.deletedelete(to_file);
tmp_file.renameTo(to_file);
}
FileUtils.moveAll(tmp_dir, bkp_dir);
@ -313,7 +313,7 @@ public class plasmaRankingCRProcess {
public static int genrci(File cr_in, final File rci_out) throws IOException {
if (!(cr_in.exists())) return 0;
AttrSeq cr = new AttrSeq(cr_in, false);
//if (rci_out.exists()) rci_out.delete(); // we want only fresh rci here (during testing)
//if (rci_out.exists()) FileUtils.deletedelete(rci_out); // we want only fresh rci here (during testing)
if (!(rci_out.exists())) {
final AttrSeq rcix = new AttrSeq("Global Ranking Reverse Citation Index",
"<AnchorDom-6>,'='," +
@ -478,7 +478,7 @@ public class plasmaRankingCRProcess {
} catch (final IOException e) {
// there is something wrong with this file; delete it
System.out.println("file " + f.getName() + " is corrupted and deleted");
f.delete();
FileUtils.deletedelete(f);
}
}
final long seconds = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000);

2
source/de/anomic/plasma/plasmaRankingDistribution.java
View File

@ -179,7 +179,7 @@ public final class plasmaRankingDistribution {
result = yacyClient.transfer(address, crfile.getName(), b);
if (result == null) {
log.logInfo("RankingDistribution - transmitted file " + crfile + " to " + address + " successfully in " + ((System.currentTimeMillis() - starttime) / 1000) + " seconds");
crfile.delete(); // the file is not needed any more locally
FileUtils.deletedelete(crfile); // the file is not needed any more locally
} else {
log.logInfo("RankingDistribution - error transmitting file " + crfile + " to " + address + ": " + result);
}

65
source/de/anomic/plasma/plasmaWordIndex.java
View File

@ -45,6 +45,7 @@ import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.text.BufferedIndex;
import de.anomic.kelondro.text.BufferedIndexCollection;
import de.anomic.kelondro.text.IndexCell;
import de.anomic.kelondro.text.IndexCollectionMigration;
import de.anomic.kelondro.text.MetadataRowContainer;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.IODispatcher;
@ -52,6 +53,7 @@ import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.text.MetadataRepository;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.text.Blacklist;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
import de.anomic.tools.iso639;
@ -63,10 +65,11 @@ import de.anomic.yacy.yacyURL;
public final class plasmaWordIndex {
// environment constants
public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
public static final int wCacheMaxChunk = 800; // maximum number of references for each urlhash
public static final int lowcachedivisor = 900;
public static final int maxCollectionPartition = 7; // should be 7
public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
public static final int wCacheMaxChunk = 800; // maximum number of references for each urlhash
public static final int lowcachedivisor = 900;
public static final int maxCollectionPartition = 7; // should be 7
public static final int maxCellArrayFiles = 10;
public static final String CRAWL_PROFILE_PROXY = "proxy";
public static final String CRAWL_PROFILE_REMOTE = "remote";
@ -126,7 +129,7 @@ public final class plasmaWordIndex {
assert !indexPrimaryTextLocation.exists();
indexPrimaryTextLocation.mkdirs();
if (oldPrimaryTextLocation.renameTo(indexPrimaryTextLocation)) {
if (!oldPrimaryPath.delete()) oldPrimaryPath.deleteOnExit();
FileUtils.deletedelete(oldPrimaryPath);
} else {
indexPrimaryTextLocation = oldPrimaryTextLocation; // emergency case: stay with old directory
}
@ -134,20 +137,38 @@ public final class plasmaWordIndex {
}
this.merger = (useCell) ? new IODispatcher(1, 1) : null;
if (this.merger != null) this.merger.start();
this.index = (useCell) ?
new IndexCell(
new File(indexPrimaryTextLocation, "RICELL"),
wordOrder,
ReferenceRow.urlEntryRow,
entityCacheMaxSize, 10, this.merger) :
new BufferedIndexCollection(
indexPrimaryTextLocation,
wordOrder,
ReferenceRow.urlEntryRow,
entityCacheMaxSize,
useCommons,
redundancy,
log);
// check if the peer has migrated the index
if (new File(indexPrimaryTextLocation, "RICOLLECTION").exists()) {
this.index = (useCell) ?
new IndexCollectionMigration(
indexPrimaryTextLocation,
wordOrder,
ReferenceRow.urlEntryRow,
entityCacheMaxSize,
maxCellArrayFiles,
this.merger,
log)
:
new BufferedIndexCollection(
indexPrimaryTextLocation,
wordOrder,
ReferenceRow.urlEntryRow,
entityCacheMaxSize,
useCommons,
redundancy,
log);
} else {
this.index = new IndexCell(
new File(indexPrimaryTextLocation, "RICELL"),
wordOrder,
ReferenceRow.urlEntryRow,
entityCacheMaxSize,
maxCellArrayFiles,
this.merger);
}
// migrate LURL-db files into new subdirectory METADATA
File textdir = new File(this.secondaryRoot, "TEXT");
@ -173,7 +194,7 @@ public final class plasmaWordIndex {
try {
this.profilesActiveCrawls = new CrawlProfile(profilesActiveFile);
} catch (IOException e) {
profilesActiveFile.delete();
FileUtils.deletedelete(profilesActiveFile);
try {
this.profilesActiveCrawls = new CrawlProfile(profilesActiveFile);
} catch (IOException e1) {
@ -194,7 +215,7 @@ public final class plasmaWordIndex {
try {
this.profilesPassiveCrawls = new CrawlProfile(profilesPassiveFile);
} catch (IOException e) {
profilesPassiveFile.delete();
FileUtils.deletedelete(profilesPassiveFile);
try {
this.profilesPassiveCrawls = new CrawlProfile(profilesPassiveFile);
} catch (IOException e1) {
@ -323,7 +344,7 @@ public final class plasmaWordIndex {
private void resetProfiles() {
final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES);
if (pdb.exists()) pdb.delete();
if (pdb.exists()) FileUtils.deletedelete(pdb);
try {
profilesActiveCrawls = new CrawlProfile(pdb);
} catch (IOException e) {

2
source/de/anomic/server/serverSystem.java
View File

@ -387,7 +387,7 @@ public final class serverSystem {
} catch (final InterruptedException e) {
throw new IOException(e.getMessage());
}
starterFile.delete();
FileUtils.deletedelete(starterFile);
}
public static Vector<String> execSynchronous(final String command) throws IOException {

4
source/de/anomic/tools/enumerateFiles.java
View File

@ -26,6 +26,8 @@ import java.util.ArrayList;
import java.util.Enumeration;
import java.util.TreeSet;
import de.anomic.kelondro.util.FileUtils;
public class enumerateFiles implements Enumeration<File> {
// implements iterative search through recursively defined subdirectories
@ -83,7 +85,7 @@ public class enumerateFiles implements Enumeration<File> {
} else {
if (l.length == 0) {
if (delete_emptyFolders) {
f.delete();
FileUtils.deletedelete(f);
f = null;
} else {
if (!(return_folders)) f = null;

3
source/de/anomic/yacy/yacyNewsDB.java
View File

@ -54,6 +54,7 @@ import de.anomic.kelondro.index.ObjectIndex;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.server.serverCodings;
@ -70,7 +71,7 @@ public class yacyNewsDB {
private void resetDB() {
try {close();} catch (final Exception e) {}
if (path.exists()) path.delete();
if (path.exists()) FileUtils.deletedelete(path);
this.news = new EcoTable(path, yacyNewsRecord.rowdef, EcoTable.tailCacheUsageAuto, 10, 0);
}

3
source/de/anomic/yacy/yacyNewsQueue.java
View File

@ -55,6 +55,7 @@ import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.table.Stack;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils;
public class yacyNewsQueue {
@ -77,7 +78,7 @@ public class yacyNewsQueue {
private void resetDB() {
try {close();} catch (final Exception e) {}
if (path.exists()) path.delete();
if (path.exists()) FileUtils.deletedelete(path);
queueStack = Stack.open(path, rowdef);
}

7
source/de/anomic/yacy/yacySeedDB.java
View File

@ -267,7 +267,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
return new MapDataMining(new BLOBHeap(seedDBFile, commonHashLength, Base64Order.enhancedCoder, 1024 * 512), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (final Exception e) {
// try again
seedDBFile.delete();
FileUtils.deletedelete(seedDBFile);
try {
return new MapDataMining(new BLOBHeap(seedDBFile, commonHashLength, Base64Order.enhancedCoder, 1024 * 512), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (IOException e1) {
@ -283,7 +283,8 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
// seed.db is detected
yacyCore.log.logWarning("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
seedDB.close();
if(!seedDBFile.delete())
FileUtils.deletedelete(seedDBFile);
if (seedDBFile.exists())
Log.logWarning("yacySeedDB", "could not delete file "+ seedDBFile);
// create new seed database
seedDB = openSeedTable(seedDBFile);
@ -810,7 +811,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
} finally {
if (seedFile != null)
try {
seedFile.delete();
FileUtils.deletedelete(seedFile);
} catch (final Exception e) {
/* ignore this */
}

9
source/de/anomic/yacy/yacyVersion.java
View File

@ -369,9 +369,11 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed");
} catch (final IOException e) {
Log.logSevere("yacyVersion", "download of " + release.name + " failed: " + e.getMessage());
if (download != null && download.exists())
if(!download.delete())
if (download != null && download.exists()) {
FileUtils.deletedelete(download);
if (download.exists())
Log.logWarning("yacyVersion", "could not delete file "+ download);
}
download = null;
} finally {
if (res != null) {
@ -650,7 +652,8 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
final File downloadedFile = new File(filesPath + File.separator + filename);
if (now - downloadedFile.lastModified() > deleteAfterMillis) {
// delete file
if (!downloadedFile.delete()) {
FileUtils.deletedelete(downloadedFile);
if (downloadedFile.exists()) {
Log.logWarning("STARTUP", "cannot delete old release " + downloadedFile.getAbsolutePath());
}
}