mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
FTP: source cleanup (added finals, indention for easier diffs)
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4559 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
fa1090113d
commit
1a7870df0d
File diff suppressed because it is too large
Load Diff
|
@ -44,7 +44,6 @@
|
|||
// the intact and unchanged copyright notice.
|
||||
// Contributions and changes to the program code must be marked as such.
|
||||
|
||||
|
||||
package de.anomic.plasma.crawler;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
@ -65,48 +64,37 @@ import de.anomic.server.logging.serverLog;
|
|||
|
||||
public class plasmaFTPLoader {
|
||||
|
||||
private plasmaSwitchboard sb;
|
||||
private serverLog log;
|
||||
private final plasmaSwitchboard sb;
|
||||
private final serverLog log;
|
||||
|
||||
public plasmaFTPLoader(plasmaSwitchboard sb, serverLog log) {
|
||||
public plasmaFTPLoader(final plasmaSwitchboard sb, final serverLog log) {
|
||||
this.sb = sb;
|
||||
this.log = log;
|
||||
}
|
||||
|
||||
protected plasmaHTCache.Entry createCacheEntry(plasmaCrawlEntry entry, String mimeType, Date fileDate) {
|
||||
return plasmaHTCache.newEntry(
|
||||
new Date(),
|
||||
entry.depth(),
|
||||
entry.url(),
|
||||
entry.name(),
|
||||
"OK",
|
||||
new ResourceInfo(
|
||||
entry.url(),
|
||||
sb.getURL(entry.referrerhash()),
|
||||
mimeType,
|
||||
fileDate
|
||||
),
|
||||
entry.initiator(),
|
||||
sb.profilesActiveCrawls.getEntry(entry.profileHandle())
|
||||
);
|
||||
protected plasmaHTCache.Entry createCacheEntry(final plasmaCrawlEntry entry, final String mimeType,
|
||||
final Date fileDate) {
|
||||
return plasmaHTCache.newEntry(new Date(), entry.depth(), entry.url(), entry.name(), "OK", new ResourceInfo(
|
||||
entry.url(), sb.getURL(entry.referrerhash()), mimeType, fileDate), entry.initiator(),
|
||||
sb.profilesActiveCrawls.getEntry(entry.profileHandle()));
|
||||
}
|
||||
|
||||
public plasmaHTCache.Entry load(plasmaCrawlEntry entry) {
|
||||
public plasmaHTCache.Entry load(final plasmaCrawlEntry entry) {
|
||||
|
||||
ByteArrayOutputStream bout = new ByteArrayOutputStream();
|
||||
PrintStream out = new PrintStream(bout);
|
||||
final ByteArrayOutputStream bout = new ByteArrayOutputStream();
|
||||
final PrintStream out = new PrintStream(bout);
|
||||
|
||||
ByteArrayOutputStream berr = new ByteArrayOutputStream();
|
||||
PrintStream err = new PrintStream(berr);
|
||||
final ByteArrayOutputStream berr = new ByteArrayOutputStream();
|
||||
final PrintStream err = new PrintStream(berr);
|
||||
|
||||
// create a new ftp client
|
||||
ftpc ftpClient = new ftpc(System.in, out, err);
|
||||
final ftpc ftpClient = new ftpc(System.in, out, err);
|
||||
|
||||
// get username and password
|
||||
String userInfo = entry.url().getUserInfo();
|
||||
final String userInfo = entry.url().getUserInfo();
|
||||
String userName = "anonymous", userPwd = "anonymous";
|
||||
if (userInfo != null) {
|
||||
int pos = userInfo.indexOf(":");
|
||||
final int pos = userInfo.indexOf(":");
|
||||
if (pos != -1) {
|
||||
userName = userInfo.substring(0, pos);
|
||||
userPwd = userInfo.substring(pos + 1);
|
||||
|
@ -114,9 +102,9 @@ public class plasmaFTPLoader {
|
|||
}
|
||||
|
||||
// get server name, port and file path
|
||||
String host = entry.url().getHost();
|
||||
final String host = entry.url().getHost();
|
||||
String fullPath = entry.url().getPath();
|
||||
int port = entry.url().getPort();
|
||||
final int port = entry.url().getPort();
|
||||
|
||||
plasmaHTCache.Entry htCache = null;
|
||||
try {
|
||||
|
@ -139,7 +127,7 @@ public class plasmaFTPLoader {
|
|||
file = "";
|
||||
path = fullPath;
|
||||
} else {
|
||||
int pos = fullPath.lastIndexOf("/");
|
||||
final int pos = fullPath.lastIndexOf("/");
|
||||
if (pos == -1) {
|
||||
file = fullPath;
|
||||
path = "/";
|
||||
|
@ -154,7 +142,7 @@ public class plasmaFTPLoader {
|
|||
ftpClient.exec("cd \"" + path + "\"", false);
|
||||
|
||||
// testing if the current name is a directoy
|
||||
boolean isFolder = ftpClient.isFolder(file);
|
||||
final boolean isFolder = ftpClient.isFolder(file);
|
||||
if (isFolder) {
|
||||
fullPath = fullPath + "/";
|
||||
file = "";
|
||||
|
@ -162,7 +150,7 @@ public class plasmaFTPLoader {
|
|||
}
|
||||
|
||||
// creating a cache file object
|
||||
File cacheFile = plasmaHTCache.getCachePath(entry.url());
|
||||
final File cacheFile = plasmaHTCache.getCachePath(entry.url());
|
||||
|
||||
// TODO: aborting download if content is to long ...
|
||||
|
||||
|
@ -188,24 +176,27 @@ public class plasmaFTPLoader {
|
|||
htCache = createCacheEntry(entry, mimeType, fileDate);
|
||||
|
||||
// generate the dirlist
|
||||
StringBuffer dirList = ftpClient.dirhtml(fullPath);
|
||||
final StringBuffer dirList = ftpClient.dirhtml(fullPath);
|
||||
|
||||
if (dirList != null && dirList.length() > 0) try {
|
||||
if (dirList != null && dirList.length() > 0) {
|
||||
try {
|
||||
// write it into a file
|
||||
PrintWriter writer = new PrintWriter(new FileOutputStream(cacheFile),false);
|
||||
final PrintWriter writer = new PrintWriter(new FileOutputStream(cacheFile), false);
|
||||
writer.write(dirList.toString());
|
||||
writer.flush();
|
||||
writer.close();
|
||||
} catch (Exception e) {
|
||||
this.log.logInfo("Unable to write dirlist for URL " + entry.url().toString());
|
||||
} catch (final Exception e) {
|
||||
log.logInfo("Unable to write dirlist for URL " + entry.url().toString());
|
||||
htCache = null;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// determine the mimetype of the resource
|
||||
String extension = plasmaParser.getFileExt(entry.url());
|
||||
final String extension = plasmaParser.getFileExt(entry.url());
|
||||
mimeType = plasmaParser.getMimeTypeByFileExt(extension);
|
||||
|
||||
// if the mimetype and file extension is supported we start to download the file
|
||||
// if the mimetype and file extension is supported we start to
|
||||
// download the file
|
||||
if (plasmaParser.supportedContent(plasmaParser.PARSER_MODE_CRAWLER, entry.url(), mimeType)) {
|
||||
|
||||
// TODO: determine the real file date
|
||||
|
@ -220,9 +211,11 @@ public class plasmaFTPLoader {
|
|||
// download the remote file
|
||||
ftpClient.exec("get \"" + fullPath + "\" \"" + cacheFile.getAbsolutePath() + "\"", false);
|
||||
} else {
|
||||
// if the response has not the right file type then reject file
|
||||
this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + mimeType + " for URL " + entry.url().toString());
|
||||
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
|
||||
// if the response has not the right file type then reject
|
||||
// file
|
||||
log.logInfo("REJECTED WRONG MIME/EXT TYPE " + mimeType + " for URL " + entry.url().toString());
|
||||
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1,
|
||||
plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -230,11 +223,14 @@ public class plasmaFTPLoader {
|
|||
// pass the downloaded resource to the cache manager
|
||||
if (berr.size() > 0 || htCache == null) {
|
||||
// if the response has not the right file type then reject file
|
||||
this.log.logWarning("Unable to download URL " + entry.url().toString() + "\nErrorlog: " + berr.toString());
|
||||
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
|
||||
log.logWarning("Unable to download URL " + entry.url().toString() + "\nErrorlog: " + berr.toString());
|
||||
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1,
|
||||
plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
|
||||
|
||||
// an error has occured. cleanup
|
||||
if (cacheFile.exists()) cacheFile.delete();
|
||||
if (cacheFile.exists()) {
|
||||
cacheFile.delete();
|
||||
}
|
||||
} else {
|
||||
// announce the file
|
||||
plasmaHTCache.writeFileAnnouncement(cacheFile);
|
||||
|
@ -248,5 +244,4 @@ public class plasmaFTPLoader {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user