mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
*) adding missing calls for function close() to avoid "too many open file" bug*) adding
*) bugfix in plasma/plasmaParser.java: - parsers with missing dependencies wehre not ignored correctly *) passing a logger instance to the parsers modules which can be used for logging purposes by the parsers (not done yet) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@276 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
bd79c43b0b
commit
890e3f4d4a
|
@ -121,8 +121,9 @@ public class listManager {
|
|||
String line;
|
||||
Vector list = new Vector();
|
||||
int count = 0;
|
||||
BufferedReader br = null;
|
||||
try{
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile)));
|
||||
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile)));
|
||||
|
||||
while( (line = br.readLine()) != null){
|
||||
list.add(line);
|
||||
|
@ -131,19 +132,24 @@ public class listManager {
|
|||
br.close();
|
||||
}catch(IOException e){
|
||||
//list is empty
|
||||
} finally {
|
||||
if (br!=null)try{br.close();}catch(Exception e) {}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
//Writes the Liststring to a file
|
||||
public static boolean writeList(File listFile, String out){
|
||||
BufferedWriter bw = null;
|
||||
try{
|
||||
BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile)));
|
||||
bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile)));
|
||||
bw.write(out);
|
||||
bw.close();
|
||||
return true;
|
||||
}catch(IOException e){
|
||||
return false;
|
||||
} finally {
|
||||
if (bw!=null)try{bw.close();}catch(Exception e){}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -159,8 +165,9 @@ public class listManager {
|
|||
public static String getListString(String filename, boolean withcomments){
|
||||
String temp = "";
|
||||
String line = "";
|
||||
BufferedReader br = null;
|
||||
try{
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(listsPath ,filename))));
|
||||
br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(listsPath ,filename))));
|
||||
//Read the List
|
||||
while((line = br.readLine()) != null){
|
||||
if( (!line.startsWith("#") || withcomments) || (!line.equals("")) ){
|
||||
|
@ -168,7 +175,11 @@ public class listManager {
|
|||
}
|
||||
}
|
||||
br.close();
|
||||
}catch(IOException e){}
|
||||
} catch(IOException e){
|
||||
} finally {
|
||||
if (br!=null)try{br.close();}catch(Exception e){}
|
||||
}
|
||||
|
||||
return temp;
|
||||
}
|
||||
|
||||
|
|
|
@ -77,11 +77,14 @@ public class translator {
|
|||
|
||||
public static boolean translateFile(File sourceFile, File destFile, File translationFile){
|
||||
Properties translationList = new Properties();
|
||||
FileInputStream fileIn = null;
|
||||
try{
|
||||
translationList.load(new FileInputStream(translationFile));
|
||||
translationList.load(fileIn = new FileInputStream(translationFile));
|
||||
return translateFile(sourceFile, destFile, translationList);
|
||||
}catch(IOException e){
|
||||
return false;
|
||||
} finally {
|
||||
if (fileIn!=null)try{fileIn.close();}catch(Exception e){}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -89,22 +92,29 @@ public class translator {
|
|||
|
||||
String content = "";
|
||||
String line = "";
|
||||
BufferedReader br = null;
|
||||
try{
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile)));
|
||||
br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile)));
|
||||
while( (line = br.readLine()) != null){
|
||||
content += line + de.anomic.server.serverCore.crlfString;
|
||||
}
|
||||
br.close();
|
||||
}catch(IOException e){
|
||||
return false;
|
||||
} finally {
|
||||
if (br!=null)try{br.close();}catch(Exception e){}
|
||||
}
|
||||
|
||||
content = translate(content, translationList);
|
||||
BufferedWriter bw = null;
|
||||
try{
|
||||
BufferedWriter bw = new BufferedWriter(new FileWriter(destFile));
|
||||
bw = new BufferedWriter(new FileWriter(destFile));
|
||||
bw.write(content);
|
||||
bw.close();
|
||||
}catch(IOException e){
|
||||
return false;
|
||||
} finally {
|
||||
if(bw!=null)try{bw.close();}catch(Exception e){}
|
||||
}
|
||||
|
||||
return true;
|
||||
|
@ -112,11 +122,14 @@ public class translator {
|
|||
|
||||
public static boolean translateFiles(File sourceDir, File destDir, File translationFile, String extension){
|
||||
Properties translationList = new Properties();
|
||||
FileInputStream fileIn = null;
|
||||
try{
|
||||
translationList.load(new FileInputStream(translationFile));
|
||||
translationList.load(fileIn = new FileInputStream(translationFile));
|
||||
return translateFiles(sourceDir, destDir, translationList, extension);
|
||||
}catch(IOException e){
|
||||
return false;
|
||||
} finally {
|
||||
if (fileIn!=null)try{fileIn.close();}catch(Exception e){}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -349,8 +349,9 @@ public final class httpTemplate {
|
|||
filename= new String(replacePattern( prefix + filename.substring(1, filename.length()-1), pattern, dflt));
|
||||
}
|
||||
if ((!filename.equals("")) && (!filename.equals(dflt))) {
|
||||
BufferedReader br = null;
|
||||
try{
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream( new File("htroot", filename) )));
|
||||
br = new BufferedReader(new InputStreamReader(new FileInputStream( new File("htroot", filename) )));
|
||||
//Read the Include
|
||||
while( (line = br.readLine()) != null ){
|
||||
include+=line+de.anomic.server.serverCore.crlfString;
|
||||
|
@ -358,6 +359,8 @@ public final class httpTemplate {
|
|||
}catch(IOException e){
|
||||
//file not found?
|
||||
serverLog.logError("FILEHANDLER","Include Error with file: "+filename);
|
||||
} finally {
|
||||
if (br!=null) try{br.close(); br=null;}catch(Exception e){}
|
||||
}
|
||||
PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes()));
|
||||
writeTemplate(pis2, out, pattern, dflt, prefix);
|
||||
|
|
|
@ -93,9 +93,6 @@ import java.util.Map;
|
|||
import java.util.Properties;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import sun.security.provider.MD5;
|
||||
|
||||
import de.anomic.server.serverByteBuffer;
|
||||
import de.anomic.server.serverClassLoader;
|
||||
import de.anomic.server.serverCodings;
|
||||
import de.anomic.server.serverCore;
|
||||
|
@ -138,9 +135,10 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
|
|||
mimeTableInputStream = new FileInputStream(new File(switchboard.getRootPath(), mimeTablePath));
|
||||
this.mimeTable.load(mimeTableInputStream);
|
||||
} catch (Exception e) {
|
||||
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (Exception e1) {}
|
||||
serverLog.logError("HTTPDFiles", "ERROR: path to configuration file or configuration invalid\n" + e);
|
||||
System.exit(1);
|
||||
} finally {
|
||||
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (Exception e1) {}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -53,6 +53,7 @@ import java.io.InputStream;
|
|||
import java.net.URL;
|
||||
|
||||
import de.anomic.plasma.plasmaParserDocument;
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
||||
/**
|
||||
* New classes implementing the {@link de.anomic.plasma.parser.Parser} interface
|
||||
|
@ -67,6 +68,12 @@ public abstract class AbstractParser implements Parser{
|
|||
*/
|
||||
protected String[] libxDependencies = null;
|
||||
|
||||
/**
|
||||
* the logger class that should be used by the parser module for logging
|
||||
* purposes.
|
||||
*/
|
||||
protected serverLog theLogger = null;
|
||||
|
||||
/**
|
||||
* The Constructor of this class.
|
||||
*/
|
||||
|
@ -136,4 +143,11 @@ public abstract class AbstractParser implements Parser{
|
|||
return this.libxDependencies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setting the logger that should be used by this parser class ...
|
||||
*/
|
||||
public void setLogger(serverLog log) {
|
||||
this.theLogger = log;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -50,6 +50,7 @@ import java.net.URL;
|
|||
import java.util.Hashtable;
|
||||
|
||||
import de.anomic.plasma.plasmaParserDocument;
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
||||
/**
|
||||
* This interface defines a list of methods that needs to be implemented
|
||||
|
@ -115,4 +116,10 @@ public interface Parser {
|
|||
*/
|
||||
public String[] getLibxDependences();
|
||||
|
||||
/**
|
||||
* Can be used to set the logger that should be used by the parser module
|
||||
* @param log the {@link serverLog logger} that should be used
|
||||
*/
|
||||
public void setLogger(serverLog log);
|
||||
|
||||
}
|
||||
|
|
|
@ -49,8 +49,6 @@ import java.io.OutputStreamWriter;
|
|||
import java.net.URL;
|
||||
import java.util.Hashtable;
|
||||
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.pdfbox.pdfparser.PDFParser;
|
||||
import org.pdfbox.pdmodel.PDDocument;
|
||||
import org.pdfbox.pdmodel.PDDocumentInformation;
|
||||
|
@ -108,10 +106,9 @@ public class pdfParser extends AbstractParser implements Parser {
|
|||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
theDocument = parser.getPDDocument();
|
||||
|
||||
// extracting some metadata
|
||||
PDDocumentInformation theDocInfo = theDocument.getDocumentInformation();
|
||||
|
||||
if (theDocInfo != null)
|
||||
{
|
||||
if (theDocInfo != null) {
|
||||
docTitle = theDocInfo.getTitle();
|
||||
docSubject = theDocInfo.getSubject();
|
||||
docAuthor = theDocInfo.getAuthor();
|
||||
|
@ -157,7 +154,7 @@ public class pdfParser extends AbstractParser implements Parser {
|
|||
return theDoc;
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new ParserException("Unable to parse the pdf content. " + e.getMessage());
|
||||
throw new ParserException("Unable to parse the pdf content. " + e.getMessage(),e);
|
||||
} finally {
|
||||
if (theDocument != null) try { theDocument.close(); } catch (Exception e) {}
|
||||
if (writer != null) try { writer.close(); } catch (Exception e) {}
|
||||
|
|
|
@ -78,6 +78,7 @@ import org.apache.commons.pool.impl.GenericObjectPool;
|
|||
import de.anomic.htmlFilter.htmlFilterContentScraper;
|
||||
import de.anomic.htmlFilter.htmlFilterOutputStream;
|
||||
import de.anomic.plasma.parser.Parser;
|
||||
import de.anomic.plasma.parser.ParserException;
|
||||
import de.anomic.server.serverFileUtils;
|
||||
import de.anomic.server.logging.serverLog;
|
||||
import de.anomic.yacy.yacySeedUploader;
|
||||
|
@ -397,8 +398,7 @@ public final class plasmaParser {
|
|||
if (neededLibx != null) {
|
||||
for (int libxId=0; libxId < neededLibx.length; libxId++) {
|
||||
if (javaClassPath.indexOf(neededLibx[libxId]) == -1) {
|
||||
serverLog.logWarning("PARSER","Parser '" + className + "': issing dependency detected: '" + neededLibx[libxId] + "'. Parser will be ignored.");
|
||||
continue;
|
||||
throw new ParserException("Missing dependency detected: '" + neededLibx[libxId] + "'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -413,7 +413,7 @@ public final class plasmaParser {
|
|||
}
|
||||
|
||||
} catch (Exception e) { /* we can ignore this for the moment */
|
||||
serverLog.logWarning("PARSER", "Parser '" + className + "' doesn't work correctly and will be ignored. " + e.getClass().getName());
|
||||
serverLog.logWarning("PARSER", "Parser '" + className + "' doesn't work correctly and will be ignored.\n [" + e.getClass().getName() + "]: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -645,8 +645,19 @@ final class plasmaParserFactory implements KeyedPoolableObjectFactory {
|
|||
if (!(key instanceof String))
|
||||
throw new IllegalArgumentException("The object key must be of type string.");
|
||||
|
||||
// loading class by name
|
||||
Class moduleClass = Class.forName((String)key);
|
||||
return moduleClass.newInstance();
|
||||
|
||||
// instantiating class
|
||||
Parser theParser = (Parser) moduleClass.newInstance();
|
||||
|
||||
// setting logger that should by used
|
||||
String parserShortName = ((String)key).substring("de.anomic.plasma.parser.".length(),((String)key).lastIndexOf("."));
|
||||
|
||||
serverLog theLogger = new serverLog("PARSER." + parserShortName.toUpperCase());
|
||||
theParser.setLogger(theLogger);
|
||||
|
||||
return theParser;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -154,9 +154,15 @@ public final class serverLog {
|
|||
|
||||
public static final void configureLogging(String homePath) throws SecurityException, FileNotFoundException, IOException {
|
||||
|
||||
FileInputStream fileIn = null;
|
||||
try {
|
||||
File loggingConfigFile = new File(homePath, "yacy.logging");
|
||||
System.out.print("STARTUP: Trying to load logging configuration from file " + loggingConfigFile.toString());
|
||||
fileIn = new FileInputStream(loggingConfigFile);
|
||||
|
||||
// loading the logger configuration from file
|
||||
LogManager logManager = LogManager.getLogManager();
|
||||
logManager.readConfiguration(new FileInputStream(new File(homePath, "yacy.logging")));
|
||||
logManager.readConfiguration(fileIn);
|
||||
|
||||
// creating the logging directory
|
||||
File log = new File("./log/");
|
||||
|
@ -167,5 +173,8 @@ public final class serverLog {
|
|||
|
||||
// System.setOut(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDOUT"),Level.FINEST)));
|
||||
// System.setErr(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDERR"),Level.SEVERE)));
|
||||
} finally {
|
||||
if (fileIn != null) try {fileIn.close();}catch(Exception e){}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user