2010-06-29 21:20:45 +02:00
/ * *
* TextParser . java
* Copyright 2009 by Michael Peter Christen , mc @yacy.net , Frankfurt am Main , Germany
* First released 09 . 07 . 2009 at http : //yacy.net
*
* This library is free software ; you can redistribute it and / or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation ; either
* version 2 . 1 of the License , or ( at your option ) any later version .
2011-06-01 21:31:56 +02:00
*
2010-06-29 21:20:45 +02:00
* This library is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the GNU
* Lesser General Public License for more details .
2011-06-01 21:31:56 +02:00
*
2010-06-29 21:20:45 +02:00
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21 . txt
* If not , see < http : //www.gnu.org/licenses/>.
* /
2009-07-10 00:25:31 +02:00
2009-10-18 02:53:43 +02:00
package net.yacy.document ;
2009-07-10 00:25:31 +02:00
import java.io.BufferedInputStream ;
import java.io.ByteArrayInputStream ;
import java.io.File ;
import java.io.FileInputStream ;
2010-03-19 14:04:42 +01:00
import java.io.IOException ;
2009-07-10 00:25:31 +02:00
import java.io.InputStream ;
2009-10-21 17:12:34 +02:00
import java.util.ArrayList ;
2010-01-19 15:59:58 +01:00
import java.util.HashMap ;
2009-07-10 16:22:17 +02:00
import java.util.HashSet ;
2009-10-21 17:12:34 +02:00
import java.util.List ;
2009-07-10 16:22:17 +02:00
import java.util.Map ;
import java.util.Set ;
2010-04-21 15:46:02 +02:00
import java.util.concurrent.ConcurrentHashMap ;
2009-07-10 00:25:31 +02:00
2010-05-25 14:54:57 +02:00
import net.yacy.cora.document.MultiProtocolURI ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.bzipParser ;
2009-11-03 21:10:59 +01:00
import net.yacy.document.parser.csvParser ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.docParser ;
2010-11-30 17:13:55 +01:00
import net.yacy.document.parser.genericParser ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.gzipParser ;
import net.yacy.document.parser.htmlParser ;
2011-06-01 21:31:56 +02:00
import net.yacy.document.parser.mmParser ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.odtParser ;
import net.yacy.document.parser.ooxmlParser ;
import net.yacy.document.parser.pdfParser ;
import net.yacy.document.parser.pptParser ;
import net.yacy.document.parser.psParser ;
2010-08-20 13:30:02 +02:00
import net.yacy.document.parser.rssParser ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.rtfParser ;
import net.yacy.document.parser.sevenzipParser ;
2011-06-01 21:31:56 +02:00
import net.yacy.document.parser.sidAudioParser ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.swfParser ;
import net.yacy.document.parser.tarParser ;
2010-01-04 17:07:31 +01:00
import net.yacy.document.parser.torrentParser ;
2009-10-18 02:53:43 +02:00
import net.yacy.document.parser.vcfParser ;
import net.yacy.document.parser.vsdParser ;
import net.yacy.document.parser.xlsParser ;
import net.yacy.document.parser.zipParser ;
2009-10-20 00:34:44 +02:00
import net.yacy.document.parser.images.genericImageParser ;
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2010-03-19 14:04:42 +01:00
import net.yacy.kelondro.util.FileUtils ;
2011-06-01 21:31:56 +02:00
import net.yacy.kelondro.util.MemoryControl ;
2009-10-10 01:13:30 +02:00
2009-10-20 00:34:44 +02:00
public final class TextParser {
2009-07-10 00:25:31 +02:00
2009-07-10 16:22:17 +02:00
private static final Log log = new Log ( " PARSER " ) ;
2010-04-22 00:35:47 +02:00
private static final Object v = new Object ( ) ;
2009-10-21 17:12:34 +02:00
2010-11-30 17:13:55 +01:00
private static final Parser genericIdiom = new genericParser ( ) ;
2010-06-29 21:20:45 +02:00
private static final Map < String , Parser > mime2parser = new ConcurrentHashMap < String , Parser > ( ) ;
private static final Map < String , Parser > ext2parser = new ConcurrentHashMap < String , Parser > ( ) ;
2010-04-21 15:46:02 +02:00
private static final Map < String , String > ext2mime = new ConcurrentHashMap < String , String > ( ) ;
private static final Map < String , Object > denyMime = new ConcurrentHashMap < String , Object > ( ) ;
private static final Map < String , Object > denyExtensionx = new ConcurrentHashMap < String , Object > ( ) ;
2011-06-01 21:31:56 +02:00
2009-07-10 00:25:31 +02:00
static {
initParser ( new bzipParser ( ) ) ;
2009-11-03 21:10:59 +01:00
initParser ( new csvParser ( ) ) ;
2009-07-10 00:25:31 +02:00
initParser ( new docParser ( ) ) ;
initParser ( new gzipParser ( ) ) ;
2009-07-10 16:22:17 +02:00
initParser ( new htmlParser ( ) ) ;
2009-10-20 00:34:44 +02:00
initParser ( new genericImageParser ( ) ) ;
2010-12-27 21:13:31 +01:00
initParser ( new mmParser ( ) ) ;
2009-07-10 00:25:31 +02:00
initParser ( new odtParser ( ) ) ;
2009-08-08 17:34:41 +02:00
initParser ( new ooxmlParser ( ) ) ;
2009-07-10 00:25:31 +02:00
initParser ( new pdfParser ( ) ) ;
initParser ( new pptParser ( ) ) ;
initParser ( new psParser ( ) ) ;
2010-08-20 13:30:02 +02:00
initParser ( new rssParser ( ) ) ;
2009-07-10 00:25:31 +02:00
initParser ( new rtfParser ( ) ) ;
initParser ( new sevenzipParser ( ) ) ;
2010-12-28 13:06:04 +01:00
initParser ( new sidAudioParser ( ) ) ;
2009-07-10 00:25:31 +02:00
initParser ( new swfParser ( ) ) ;
initParser ( new tarParser ( ) ) ;
2010-01-04 17:07:31 +01:00
initParser ( new torrentParser ( ) ) ;
2009-07-10 00:25:31 +02:00
initParser ( new vcfParser ( ) ) ;
initParser ( new vsdParser ( ) ) ;
initParser ( new xlsParser ( ) ) ;
initParser ( new zipParser ( ) ) ;
}
2011-06-01 21:31:56 +02:00
2010-06-29 21:20:45 +02:00
public static Set < Parser > parsers ( ) {
2011-06-01 21:31:56 +02:00
final Set < Parser > c = new HashSet < Parser > ( ) ;
2010-01-04 17:07:31 +01:00
c . addAll ( ext2parser . values ( ) ) ;
2009-07-10 16:22:17 +02:00
c . addAll ( mime2parser . values ( ) ) ;
return c ;
}
2011-06-01 21:31:56 +02:00
private static void initParser ( final Parser parser ) {
2009-07-14 13:01:05 +02:00
String prototypeMime = null ;
2011-06-01 21:31:56 +02:00
for ( final String mime : parser . supportedMimeTypes ( ) ) {
2009-07-10 16:22:17 +02:00
// process the mime types
2009-07-14 13:01:05 +02:00
final String mimeType = normalizeMimeType ( mime ) ;
if ( prototypeMime = = null ) prototypeMime = mimeType ;
2011-06-01 21:31:56 +02:00
final Parser p0 = mime2parser . get ( mimeType ) ;
2009-07-10 18:15:33 +02:00
if ( p0 ! = null ) log . logSevere ( " parser for mime ' " + mimeType + " ' was set to ' " + p0 . getName ( ) + " ', overwriting with new parser ' " + parser . getName ( ) + " '. " ) ;
2009-07-10 16:22:17 +02:00
mime2parser . put ( mimeType , parser ) ;
Log . logInfo ( " PARSER " , " Parser for mime type ' " + mimeType + " ': " + parser . getName ( ) ) ;
2009-07-15 16:15:51 +02:00
}
2011-06-01 21:31:56 +02:00
2009-07-15 16:15:51 +02:00
if ( prototypeMime ! = null ) for ( String ext : parser . supportedExtensions ( ) ) {
2010-04-21 15:46:02 +02:00
ext = ext . toLowerCase ( ) ;
2011-06-01 21:31:56 +02:00
final String s = ext2mime . get ( ext ) ;
2009-07-15 16:15:51 +02:00
if ( s ! = null ) log . logSevere ( " parser for extension ' " + ext + " ' was set to mime ' " + s + " ', overwriting with new mime ' " + prototypeMime + " '. " ) ;
ext2mime . put ( ext , prototypeMime ) ;
2009-07-10 00:25:31 +02:00
}
2011-06-01 21:31:56 +02:00
2009-10-21 17:12:34 +02:00
for ( String ext : parser . supportedExtensions ( ) ) {
// process the extensions
2010-04-21 15:46:02 +02:00
ext = ext . toLowerCase ( ) ;
2011-06-01 21:31:56 +02:00
final Parser p0 = ext2parser . get ( ext ) ;
2009-10-21 17:12:34 +02:00
if ( p0 ! = null ) log . logSevere ( " parser for extension ' " + ext + " ' was set to ' " + p0 . getName ( ) + " ', overwriting with new parser ' " + parser . getName ( ) + " '. " ) ;
ext2parser . put ( ext , parser ) ;
Log . logInfo ( " PARSER " , " Parser for extension ' " + ext + " ': " + parser . getName ( ) ) ;
}
2009-07-10 00:25:31 +02:00
}
2011-06-01 21:31:56 +02:00
2010-06-29 21:20:45 +02:00
public static Document [ ] parseSource (
2010-05-25 14:54:57 +02:00
final MultiProtocolURI location ,
2009-07-14 13:01:05 +02:00
final String mimeType ,
final String charset ,
final File sourceFile
2010-06-29 21:20:45 +02:00
) throws InterruptedException , Parser . Failure {
2009-07-10 00:25:31 +02:00
BufferedInputStream sourceStream = null ;
2010-12-02 12:05:04 +01:00
Document [ ] docs = null ;
2009-07-10 00:25:31 +02:00
try {
2009-07-10 16:22:17 +02:00
if ( log . isFine ( ) ) log . logFine ( " Parsing ' " + location + " ' from file " ) ;
2009-10-05 22:11:41 +02:00
if ( ! sourceFile . exists ( ) | | ! sourceFile . canRead ( ) | | sourceFile . length ( ) = = 0 ) {
2009-07-10 00:25:31 +02:00
final String errorMsg = sourceFile . exists ( ) ? " Empty resource file. " : " No resource content available (2). " ;
2009-07-10 16:22:17 +02:00
log . logInfo ( " Unable to parse ' " + location + " '. " + errorMsg ) ;
2010-06-29 21:20:45 +02:00
throw new Parser . Failure ( errorMsg , location ) ;
2009-07-10 00:25:31 +02:00
}
sourceStream = new BufferedInputStream ( new FileInputStream ( sourceFile ) ) ;
2010-12-02 12:05:04 +01:00
docs = parseSource ( location , mimeType , charset , sourceFile . length ( ) , sourceStream ) ;
2009-07-10 00:25:31 +02:00
} catch ( final Exception e ) {
if ( e instanceof InterruptedException ) throw ( InterruptedException ) e ;
2010-06-29 21:20:45 +02:00
if ( e instanceof Parser . Failure ) throw ( Parser . Failure ) e ;
2009-07-10 16:22:17 +02:00
log . logSevere ( " Unexpected exception in parseSource from File: " + e . getMessage ( ) , e ) ;
2010-06-29 21:20:45 +02:00
throw new Parser . Failure ( " Unexpected exception: " + e . getMessage ( ) , location ) ;
2009-07-10 00:25:31 +02:00
} finally {
2010-12-02 12:05:04 +01:00
if ( sourceStream ! = null ) try { sourceStream . close ( ) ; } catch ( final Exception ex ) { }
2009-07-10 00:25:31 +02:00
}
2011-06-01 21:31:56 +02:00
for ( final Document d : docs ) { assert d . getText ( ) ! = null ; } // verify docs
2010-12-02 12:05:04 +01:00
return docs ;
2009-07-10 00:25:31 +02:00
}
2011-06-01 21:31:56 +02:00
2010-06-29 21:20:45 +02:00
public static Document [ ] parseSource (
2010-05-25 14:54:57 +02:00
final MultiProtocolURI location ,
2011-06-01 21:31:56 +02:00
final String mimeType ,
2010-03-19 14:04:42 +01:00
final String charset ,
final byte [ ] content
2010-06-29 21:20:45 +02:00
) throws Parser . Failure {
2010-03-19 14:04:42 +01:00
return parseSource ( location , mimeType , charset , content . length , new ByteArrayInputStream ( content ) ) ;
}
2011-06-01 21:31:56 +02:00
2010-06-29 21:20:45 +02:00
public static Document [ ] parseSource (
2010-05-25 14:54:57 +02:00
final MultiProtocolURI location ,
2009-07-14 13:01:05 +02:00
String mimeType ,
final String charset ,
final long contentLength ,
final InputStream sourceStream
2010-06-29 21:20:45 +02:00
) throws Parser . Failure {
2009-10-21 17:12:34 +02:00
if ( log . isFine ( ) ) log . logFine ( " Parsing ' " + location + " ' from stream " ) ;
mimeType = normalizeMimeType ( mimeType ) ;
2010-06-29 21:20:45 +02:00
List < Parser > idioms = null ;
2010-01-22 14:21:37 +01:00
try {
2010-06-29 21:20:45 +02:00
idioms = parsers ( location , mimeType ) ;
2011-06-01 21:31:56 +02:00
} catch ( final Parser . Failure e ) {
2010-01-22 14:21:37 +01:00
final String errorMsg = " Parser Failure for extension ' " + location . getFileExtension ( ) + " ' or mimetype ' " + mimeType + " ': " + e . getMessage ( ) ;
log . logWarning ( errorMsg ) ;
2010-06-29 21:20:45 +02:00
throw new Parser . Failure ( errorMsg , location ) ;
2009-10-21 17:12:34 +02:00
}
2010-09-02 21:24:22 +02:00
assert ! idioms . isEmpty ( ) : " no parsers applied for url " + location . toNormalform ( true , false ) ;
2011-06-01 21:31:56 +02:00
2010-03-19 14:04:42 +01:00
// if we do not have more than one parser or the content size is over MaxInt
// then we use only one stream-oriented parser.
if ( idioms . size ( ) = = 1 | | contentLength > Integer . MAX_VALUE ) {
// use a specific stream-oriented parser
2011-06-01 21:31:56 +02:00
final Document [ ] docs = parseSource ( location , mimeType , idioms . get ( 0 ) , charset , contentLength , sourceStream ) ;
for ( final Document d : docs ) { assert d . getText ( ) ! = null ; } // verify docs
2010-12-02 12:05:04 +01:00
return docs ;
2010-03-19 14:04:42 +01:00
}
2011-06-01 21:31:56 +02:00
2010-03-19 14:04:42 +01:00
// in case that we know more parsers we first transform the content into a byte[] and use that as base
// for a number of different parse attempts.
2010-11-30 17:13:55 +01:00
byte [ ] b = null ;
2010-03-19 14:04:42 +01:00
try {
2010-11-30 17:13:55 +01:00
b = FileUtils . read ( sourceStream , ( int ) contentLength ) ;
2011-06-01 21:31:56 +02:00
} catch ( final IOException e ) {
2010-06-29 21:20:45 +02:00
throw new Parser . Failure ( e . getMessage ( ) , location ) ;
2010-03-19 14:04:42 +01:00
}
2011-06-01 21:31:56 +02:00
final Document [ ] docs = parseSource ( location , mimeType , idioms , charset , b ) ;
for ( final Document d : docs ) { assert d . getText ( ) ! = null ; } // verify docs
2010-12-02 12:05:04 +01:00
return docs ;
2010-03-19 14:04:42 +01:00
}
2010-06-29 21:20:45 +02:00
private static Document [ ] parseSource (
2010-05-25 14:54:57 +02:00
final MultiProtocolURI location ,
2011-06-01 21:31:56 +02:00
final String mimeType ,
final Parser parser ,
2010-03-19 14:04:42 +01:00
final String charset ,
final long contentLength ,
final InputStream sourceStream
2010-06-29 21:20:45 +02:00
) throws Parser . Failure {
2010-03-19 14:04:42 +01:00
if ( log . isFine ( ) ) log . logFine ( " Parsing ' " + location + " ' from stream " ) ;
final String fileExt = location . getFileExtension ( ) ;
final String documentCharset = htmlParser . patchCharsetEncoding ( charset ) ;
2010-06-29 21:20:45 +02:00
assert parser ! = null ;
2010-03-19 14:04:42 +01:00
if ( log . isFine ( ) ) log . logInfo ( " Parsing " + location + " with mimeType ' " + mimeType + " ' and file extension ' " + fileExt + " '. " ) ;
try {
2011-06-01 21:31:56 +02:00
final Document [ ] docs = parser . parse ( location , mimeType , documentCharset , sourceStream ) ;
for ( final Document d : docs ) { assert d . getText ( ) ! = null ; } // verify docs
2010-12-02 12:05:04 +01:00
return docs ;
2011-06-01 21:31:56 +02:00
} catch ( final Exception e ) {
2010-06-29 21:20:45 +02:00
throw new Parser . Failure ( " parser failed: " + parser . getName ( ) , location ) ;
2010-03-19 14:04:42 +01:00
}
}
2010-06-29 21:20:45 +02:00
private static Document [ ] parseSource (
2010-05-25 14:54:57 +02:00
final MultiProtocolURI location ,
2010-06-29 21:20:45 +02:00
final String mimeType ,
final List < Parser > parsers ,
2010-03-19 14:04:42 +01:00
final String charset ,
final byte [ ] sourceArray
2010-06-29 21:20:45 +02:00
) throws Parser . Failure {
2010-03-19 14:04:42 +01:00
final String fileExt = location . getFileExtension ( ) ;
if ( log . isFine ( ) ) log . logInfo ( " Parsing " + location + " with mimeType ' " + mimeType + " ' and file extension ' " + fileExt + " ' from byte[] " ) ;
final String documentCharset = htmlParser . patchCharsetEncoding ( charset ) ;
2010-06-29 21:20:45 +02:00
assert ! parsers . isEmpty ( ) ;
2010-03-19 14:04:42 +01:00
2010-12-02 12:05:04 +01:00
Document [ ] docs = null ;
2011-06-01 21:31:56 +02:00
final HashMap < Parser , Parser . Failure > failedParser = new HashMap < Parser , Parser . Failure > ( ) ;
if ( MemoryControl . request ( sourceArray . length * 2 , false ) ) {
for ( final Parser parser : parsers ) {
try {
docs = parser . parse ( location , mimeType , documentCharset , new ByteArrayInputStream ( sourceArray ) ) ;
} catch ( final Parser . Failure e ) {
failedParser . put ( parser , e ) ;
//log.logWarning("tried parser '" + parser.getName() + "' to parse " + location.toNormalform(true, false) + " but failed: " + e.getMessage(), e);
} catch ( final Exception e ) {
failedParser . put ( parser , new Parser . Failure ( e . getMessage ( ) , location ) ) ;
//log.logWarning("tried parser '" + parser.getName() + "' to parse " + location.toNormalform(true, false) + " but failed: " + e.getMessage(), e);
}
if ( docs ! = null ) break ;
2009-07-10 00:25:31 +02:00
}
2009-10-21 17:12:34 +02:00
}
2011-06-01 21:31:56 +02:00
2010-12-02 12:05:04 +01:00
if ( docs = = null ) {
2010-12-27 21:13:31 +01:00
if ( failedParser . isEmpty ( ) ) {
2010-01-19 15:59:58 +01:00
final String errorMsg = " Parsing content with file extension ' " + location . getFileExtension ( ) + " ' and mimetype ' " + mimeType + " ' failed. " ;
//log.logWarning("Unable to parse '" + location + "'. " + errorMsg);
2010-06-29 21:20:45 +02:00
throw new Parser . Failure ( errorMsg , location ) ;
2010-01-19 15:59:58 +01:00
} else {
String failedParsers = " " ;
2011-06-01 21:31:56 +02:00
for ( final Map . Entry < Parser , Parser . Failure > error : failedParser . entrySet ( ) ) {
2010-01-19 15:59:58 +01:00
log . logWarning ( " tried parser ' " + error . getKey ( ) . getName ( ) + " ' to parse " + location . toNormalform ( true , false ) + " but failed: " + error . getValue ( ) . getMessage ( ) , error . getValue ( ) ) ;
failedParsers + = error . getKey ( ) . getName ( ) + " " ;
}
2010-06-29 21:20:45 +02:00
throw new Parser . Failure ( " All parser failed: " + failedParsers , location ) ;
2010-01-19 15:59:58 +01:00
}
2009-07-10 00:25:31 +02:00
}
2011-06-01 21:31:56 +02:00
for ( final Document d : docs ) { assert d . getText ( ) ! = null : " mimeType = " + mimeType ; } // verify docs
2010-12-02 12:05:04 +01:00
return docs ;
2009-07-10 00:25:31 +02:00
}
2011-06-01 21:31:56 +02:00
2009-07-14 13:01:05 +02:00
/ * *
* check if the parser supports the given content .
* @param url
* @param mimeType
2009-10-23 00:38:04 +02:00
* @return returns null if the content is supported . If the content is not supported , return a error string .
2009-07-14 13:01:05 +02:00
* /
2011-06-01 21:31:56 +02:00
public static String supports ( final MultiProtocolURI url , final String mimeType ) {
2009-07-14 13:01:05 +02:00
try {
// try to get a parser. If this works, we don't need the parser itself, we just return null to show that everything is ok.
2011-06-01 21:31:56 +02:00
final List < Parser > idioms = parsers ( url , mimeType ) ;
2010-12-11 01:31:57 +01:00
return ( idioms = = null | | idioms . isEmpty ( ) | | ( idioms . size ( ) = = 1 & & idioms . get ( 0 ) . getName ( ) . equals ( genericIdiom . getName ( ) ) ) ) ? " no parser found " : null ;
2011-06-01 21:31:56 +02:00
} catch ( final Parser . Failure e ) {
2009-07-14 13:01:05 +02:00
// in case that a parser is not available, return a error string describing the problem.
return e . getMessage ( ) ;
}
}
2011-06-01 21:31:56 +02:00
2009-10-21 17:12:34 +02:00
/ * *
* find a parser for a given url and mime type
* because mime types returned by web severs are sometimes wrong , we also compute the mime type again
* from the extension that can be extracted from the url path . That means that there are 3 criteria
* that can be used to select a parser :
* - the given extension
* - the given mime type
* - the mime type computed from the extension
* @param url the given url
* @param mimeType the given mime type
* @return a list of Idiom parsers that may be appropriate for the given criteria
2010-06-29 21:20:45 +02:00
* @throws Parser . Failure
2009-10-21 17:12:34 +02:00
* /
2010-06-29 21:20:45 +02:00
private static List < Parser > parsers ( final MultiProtocolURI url , String mimeType1 ) throws Parser . Failure {
2011-06-01 21:31:56 +02:00
final List < Parser > idioms = new ArrayList < Parser > ( 2 ) ;
2009-07-14 13:01:05 +02:00
// check extension
String ext = url . getFileExtension ( ) ;
2010-06-29 21:20:45 +02:00
Parser idiom ;
2009-10-21 17:12:34 +02:00
if ( ext ! = null & & ext . length ( ) > 0 ) {
2010-04-21 15:46:02 +02:00
ext = ext . toLowerCase ( ) ;
2010-06-29 21:20:45 +02:00
if ( denyExtensionx . containsKey ( ext ) ) throw new Parser . Failure ( " file extension ' " + ext + " ' is denied (1) " , url ) ;
2009-10-21 17:12:34 +02:00
idiom = ext2parser . get ( ext ) ;
if ( idiom ! = null ) idioms . add ( idiom ) ;
}
2011-06-01 21:31:56 +02:00
2009-10-21 17:12:34 +02:00
// check given mime type
if ( mimeType1 ! = null ) {
mimeType1 = normalizeMimeType ( mimeType1 ) ;
2010-06-29 21:20:45 +02:00
if ( denyMime . containsKey ( mimeType1 ) ) throw new Parser . Failure ( " mime type ' " + mimeType1 + " ' is denied (1) " , url ) ;
2009-10-21 17:12:34 +02:00
idiom = mime2parser . get ( mimeType1 ) ;
if ( idiom ! = null & & ! idioms . contains ( idiom ) ) idioms . add ( idiom ) ;
}
2011-06-01 21:31:56 +02:00
2009-10-21 17:12:34 +02:00
// check mime type computed from extension
2011-06-01 21:31:56 +02:00
final String mimeType2 = ext2mime . get ( ext ) ;
2010-12-11 01:31:57 +01:00
if ( mimeType2 ! = null & & ( idiom = mime2parser . get ( mimeType2 ) ) ! = null & & ! idioms . contains ( idiom ) ) idioms . add ( idiom ) ;
2011-06-01 21:31:56 +02:00
2010-11-30 17:13:55 +01:00
// always add the generic parser
idioms . add ( genericIdiom ) ;
//if (idioms.isEmpty()) throw new Parser.Failure("no parser found for extension '" + ext + "' and mime type '" + mimeType1 + "'", url);
2011-06-01 21:31:56 +02:00
2009-10-21 17:12:34 +02:00
return idioms ;
2009-07-14 13:01:05 +02:00
}
public static String supportsMime ( String mimeType ) {
if ( mimeType = = null ) return null ;
2009-07-10 18:15:33 +02:00
mimeType = normalizeMimeType ( mimeType ) ;
2010-04-21 15:46:02 +02:00
if ( denyMime . containsKey ( mimeType ) ) return " mime type ' " + mimeType + " ' is denied (2) " ;
2009-07-14 13:01:05 +02:00
if ( mime2parser . get ( mimeType ) = = null ) return " no parser for mime ' " + mimeType + " ' available " ;
return null ;
2009-07-10 16:22:17 +02:00
}
2010-12-02 12:05:04 +01:00
2010-05-25 14:54:57 +02:00
public static String supportsExtension ( final MultiProtocolURI url ) {
2011-06-01 21:31:56 +02:00
final String ext = url . getFileExtension ( ) . toLowerCase ( ) ;
2009-07-14 13:01:05 +02:00
if ( ext = = null | | ext . length ( ) = = 0 ) return null ;
2010-04-21 15:46:02 +02:00
if ( denyExtensionx . containsKey ( ext ) ) return " file extension ' " + ext + " ' is denied (2) " ;
2011-06-01 21:31:56 +02:00
final String mimeType = ext2mime . get ( ext ) ;
2009-07-14 13:01:05 +02:00
if ( mimeType = = null ) return " no parser available " ;
2011-06-01 21:31:56 +02:00
final Parser idiom = mime2parser . get ( mimeType ) ;
2009-07-14 13:01:05 +02:00
assert idiom ! = null ;
if ( idiom = = null ) return " no parser available (internal error!) " ;
return null ;
2009-07-10 16:22:17 +02:00
}
2011-06-01 21:31:56 +02:00
public static String mimeOf ( final MultiProtocolURI url ) {
2009-07-10 16:22:17 +02:00
return mimeOf ( url . getFileExtension ( ) ) ;
}
2011-06-01 21:31:56 +02:00
public static String mimeOf ( final String ext ) {
2010-04-21 15:46:02 +02:00
return ext2mime . get ( ext . toLowerCase ( ) ) ;
2009-07-10 16:22:17 +02:00
}
2011-06-01 21:31:56 +02:00
2009-07-10 16:22:17 +02:00
private static String normalizeMimeType ( String mimeType ) {
if ( mimeType = = null ) return " application/octet-stream " ;
2010-04-21 15:46:02 +02:00
mimeType = mimeType . toLowerCase ( ) ;
2009-07-10 16:22:17 +02:00
final int pos = mimeType . indexOf ( ';' ) ;
return ( ( pos < 0 ) ? mimeType . trim ( ) : mimeType . substring ( 0 , pos ) . trim ( ) ) ;
}
2011-06-01 21:31:56 +02:00
public static void setDenyMime ( final String denyList ) {
2009-07-10 16:22:17 +02:00
denyMime . clear ( ) ;
2009-11-11 16:49:53 +01:00
String n ;
2011-06-01 21:31:56 +02:00
for ( final String s : denyList . split ( " , " ) ) {
2009-11-11 16:49:53 +01:00
n = normalizeMimeType ( s ) ;
2010-04-22 00:35:47 +02:00
if ( n ! = null & & n . length ( ) > 0 ) denyMime . put ( n , v ) ;
2009-11-11 16:49:53 +01:00
}
2009-07-10 16:22:17 +02:00
}
2011-06-01 21:31:56 +02:00
2009-07-10 16:22:17 +02:00
public static String getDenyMime ( ) {
String s = " " ;
2011-06-01 21:31:56 +02:00
for ( final String d : denyMime . keySet ( ) ) s + = d + " , " ;
2010-02-03 20:33:59 +01:00
if ( s . length ( ) > 0 ) s = s . substring ( 0 , s . length ( ) - 1 ) ;
2009-07-10 16:22:17 +02:00
return s ;
}
2011-06-01 21:31:56 +02:00
public static void grantMime ( final String mime , final boolean grant ) {
final String n = normalizeMimeType ( mime ) ;
2009-11-11 16:49:53 +01:00
if ( n = = null | | n . length ( ) = = 0 ) return ;
2010-04-22 00:35:47 +02:00
if ( grant ) denyMime . remove ( n ) ; else denyMime . put ( n , v ) ;
2009-07-10 16:22:17 +02:00
}
2011-06-01 21:31:56 +02:00
public static void setDenyExtension ( final String denyList ) {
2010-04-21 15:46:02 +02:00
denyExtensionx . clear ( ) ;
2011-06-01 21:31:56 +02:00
for ( final String s : denyList . split ( " , " ) ) denyExtensionx . put ( s , v ) ;
2009-07-14 13:01:05 +02:00
}
2011-06-01 21:31:56 +02:00
2009-07-14 13:01:05 +02:00
public static String getDenyExtension ( ) {
String s = " " ;
2011-06-01 21:31:56 +02:00
for ( final String d : denyExtensionx . keySet ( ) ) s + = d + " , " ;
2009-07-14 13:01:05 +02:00
s = s . substring ( 0 , s . length ( ) - 1 ) ;
return s ;
}
2011-06-01 21:31:56 +02:00
public static void grantExtension ( final String ext , final boolean grant ) {
2010-04-22 00:35:47 +02:00
if ( grant ) denyExtensionx . remove ( ext ) ; else denyExtensionx . put ( ext , v ) ;
2009-07-14 13:01:05 +02:00
}
2010-06-29 21:20:45 +02:00
2009-07-10 00:25:31 +02:00
}