// wikiCode.java // ------------------------------------- // part of YACY // (C) by Michael Peter Christen; mc@anomic.de // first published on http://www.anomic.de // Frankfurt, Germany, 2004 // // This file ist contributed by Alexander Schier // last major change: 09.08.2004 // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // // Using this software in any meaning (reading, learning, copying, compiling, // running) means that you agree that the Author(s) is (are) not responsible // for cost, loss of data or any harm that may be caused directly or indirectly // by usage of this softare or this documentation. The usage of this software // is on your own risk. The installation and usage (starting/running) of this // software may allow other people or application to access your computer and // any attached devices and is highly dependent on the configuration of the // software which must be done by the user of the software; the author(s) is // (are) also not responsible for proper configuration and usage of the // software, even if provoked by documentation provided together with // the software. // // Any changes to this file according to the GPL as documented in the file // gpl.txt aside this file in the shipment you received can be done to the // lines that follows this copyright notice here, but changes must not be // done inside the copyright notive above. A re-distribution must contain // the intact and unchanged copyright notice. // Contributions and changes to the program code must be marked as such. // Contains contributions from Alexander Schier [AS] // Franz Brausse [FB] and Marc Nause [MN] package de.anomic.data; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; import de.anomic.yacy.yacyCore; /** This class provides methods to handle texts that have been posted in the yacyWiki. */ public class wikiCode { private String numListLevel=""; private String ListLevel=""; private String defListLevel=""; private plasmaSwitchboard sb; private boolean cellprocessing=false; // needed for prevention of double-execution of replaceHTML private boolean escape = false; //needed for escape private boolean escaped = false; //needed for
 not getting in the way
    private boolean escapeSpan = false;       //needed for escape symbols [= and =] spanning over several lines
    private boolean newrowstart=false;        // needed for the first row not to be empty
    private boolean preformatted = false;     //needed for preformatted text
    private boolean preformattedSpan = false; //needed for 
 and 
spanning over several lines private boolean replacedHTML = false; //indicates if method replaceHTML has been used with line already private boolean replacedCharacters = false; //indicates if method replaceCharachters has been used with line private boolean table=false; // needed for tables, because they reach over several lines private int preindented = 0; //needed for indented
s
    private int escindented = 0;              //needed for indented [=s

    /** Constructor of the class wikiCode */
    public wikiCode(plasmaSwitchboard switchboard){
        sb=switchboard;
    }

    public String transform(String content){
        try {
            return transform(content.getBytes("UTF-8"), sb);
        } catch (UnsupportedEncodingException e) {
            return transform(content.getBytes(), sb);
        }
    }

    public String transform(byte[] content){
        return transform(content, sb);
    }

    public String transform(byte[] content, plasmaSwitchboard switchboard) {
        ByteArrayInputStream bais = new ByteArrayInputStream(content);
        try {
            BufferedReader br = new BufferedReader(new InputStreamReader(bais,
                    "UTF-8"));
            String line;
            StringBuffer out = new StringBuffer(content.length);
            try {
                while ((line = br.readLine()) != null) {
                    out.append(transformLine(line, switchboard)).append(
                            serverCore.crlfString);
                }
                return out.toString();
            } catch (UnsupportedEncodingException e1) {
                // can not happen
                return null;
            }
        } catch (IOException e) {
            return "internal error: " + e.getMessage();
        }
    }

    /** Replaces special characters from a string. Avoids XSS attacks and ensures correct display of
      * special characters in non UTF-8 capable browsers.
      * @param text a string that possibly contains HTML
      * @return the string with all special characters encoded
      * @author Marc Nause, replaces code by Alexander Schier
      */
    public static String replaceHTML(String text) {
        text = replace(text, characters);
        text = replace(text, htmlentities);
        return text;
    }

    /** Replaces special characters from a string. Ensures correct display of
      * special characters in non UTF-8 capable browsers.
      * @param text a string that possibly contains special characters
      * @return the string with all special characters encoded
      * @author Marc Nause, replaces code by Alexander Schier
      */
    public static String replaceCharacters(String text) {
        text = replace(text, characters);
        return text;
    }

    /** Replaces special characters from a string. Avoids XSS attacks.
      * @param text a string that possibly contains HTML
      * @return the string without any HTML-tags that can be used for XSS
      * @author Marc Nause, replaces code by Alexander Schier
      */
    public static String replaceHTMLonly(String text) {
        text = replace(text, htmlentities);
        return text;
    }

    /** Replaces characters in a string with other characters defined in an array.
      * @param text a string that possibly contains special characters
      * @param entities array that contains characters to be replaced and characters it will be replaced by
      * @return the string with all characters replaced by the corresponding character from array
      * @author Franz Brausse, few changes by Marc Nause, replaces code by Alexander Schier
      */
    public static String replace(String text, String[] entities) {
        if (text==null) { return null; }
        for (int x=0;x<=entities.length-1;x=x+2) {
            int p=0;
            while ((p=text.indexOf(entities[x],p))>=0) {
                text=text.substring(0,p)+entities[x+1]+text.substring(p+entities[x].length());
                p+=entities[x+1].length();
            }
        }
        return text;
    }

    //This array contains codes (see http://mindprod.com/jgloss/unicode.html for details) 
    //that will be replaced. To add new codes or patterns, just put them at the end
    //of the list. Codes or patterns in this list can not be escaped with [= or 
    public static String[] htmlentities={
        // Ampersands _have_ to be replaced first. If they were replaced later,
        // other replaced characters containing ampersands would get messed up.
        "\u0026","&",      //ampersand
        "\"",""",         //quotation mark
        "\u003C","<",       //less than
        "\u003E",">",       //greater than
    };

    //This array contains codes (see http://mindprod.com/jgloss/unicode.html for details) and
    //patterns that will be replaced. To add new codes or patterns, just put them at the end
    //of the list. Codes or patterns in this list can not be escaped with [= or 
    public static String[] characters={
        "\u0026","&",      //ampersand
        "\u00A1","¡",    //inverted (spanish) exclamation mark
        "\u00A2","¢",     //cent
        "\u00A3","£",    //pound
        "\u00A4","¤",   //currency
        "\u00A5","¥",      //yen
        "\u00A6","¦",   //broken vertical bar
        "\u00A7","§",     //section sign
        "\u00A8","¨",      //diaeresis (umlaut)
        "\u00A9","©",     //copyright sign
        "\u00AA","ª",     //feminine ordinal indicator
        "\u00AB","«",    //left-pointing double angle quotation mark
        "\u00AC","¬",      //not sign
        "\u00AD","­",      //soft hyphen
        "\u00AE","®",      //registered sign
        "\u00AF","¯",     //macron
        "\u00B0","°",      //degree sign
        "\u00B1","±",   //plus-minus sign
        "\u00B2","²",     //superscript two
        "\u00B3","³",     //superscript three
        "\u00B4","´",    //acute accent
        "\u00B5","µ",    //micro sign
        "\u00B6","¶",     //paragraph sign
        "\u00B7","·",   //middle dot
        "\u00B8","¸",    //cedilla
        "\u00B9","¹",     //superscript one
        "\u00BA","º",     //masculine ordinal indicator
        "\u00BB","»",    //right-pointing double angle quotation mark
        "\u00BC","¼",   //fraction 1/4
        "\u00BD","½",   //fraction 1/2
        "\u00BE","¾",   //fraction 3/4
        "\u00BF","¿",   //inverted (spanisch) questionmark
        "\u00C0","À",
        "\u00C1","Á",
        "\u00C2","Â",
        "\u00C3","Ã",
        "\u00C4","Ä",
        "\u00C5","Å",
        "\u00C6","Æ",
        "\u00C7","Ç",
        "\u00C8","È",
        "\u00C9","É",
        "\u00CA","Ê",
        "\u00CB","Ë",
        "\u00CC","Ì",
        "\u00CD","Í",
        "\u00CE","Î",
        "\u00CF","Ï",
        "\u00D0","Ð",
        "\u00D1","Ñ",
        "\u00D2","Ò",
        "\u00D3","Ó",
        "\u00D4","Ô",
        "\u00D5","Õ",
        "\u00D6","Ö",
        "\u00D7","×",
        "\u00D8","Ø",
        "\u00D9","Ù",
        "\u00DA","Ú",
        "\u00DB","Û",
        "\u00DC","Ü",
        "\u00DD","Ý",
        "\u00DE","Þ",
        "\u00DF","ß",
        "\u00E0","à",
        "\u00E1","á",
        "\u00E2","â",
        "\u00E3","ã",
        "\u00E4","ä",
        "\u00E5","å",
        "\u00E6","æ",
        "\u00E7","ç",
        "\u00E8","è",
        "\u00E9","é",
        "\u00EA","ê",
        "\u00EB","ë",
        "\u00EC","ì",
        "\u00ED","í",
        "\u00EE","î",
        "\u00EF","ï",
        "\u00F0","ð",
        "\u00F1","ñ",
        "\u00F2","ò",
        "\u00F3","ó",
        "\u00F4","ô",
        "\u00F5","õ",
        "\u00F6","ö",
        "\u00F7","÷",
        "\u00F8","ø",
        "\u00F9","ù",
        "\u00FA","ú",
        "\u00FB","û",
        "\u00FC","ü",
        "\u00FD","ý",
        "\u00FE","þ",
        "\u00FF","ÿ",
        "(C)","©"
    };

    /** This method processes tables in the wiki code.
      * @param a string that might contain parts of a table
      * @return a string with wiki code of parts of table replaced by HTML code for table
      * @author Franz Brausse, changes by Marc Nause
      */
    private String processTable(String result, plasmaSwitchboard switchboard){
        String line="";
        if ((result.startsWith("{|")) && (!table)) {
            table=true;
            newrowstart=true;
            line="2) {
                line+=" "+result.substring(2).replaceAll(""","\"").trim();
            }
            line+=">";
            result=line;
        }
        else if (result.startsWith("|-") && (table)) {          // new row
            if (!newrowstart) {
                line+="\t\n";
            } else {
                newrowstart=false;
            }
            line=line+"\t";
            result=line;
        }
        else if ((result.startsWith("||")) && (table)) {
            result = replaceHTMLonly(result);
            line+="\t\t0)?(result.indexOf("||",2)):(result.length());
            int propEnd=(result.indexOf("|",2)>0)?(result.indexOf("|",2)):(cellEnd);
            // both point at same place => new line
            if (propEnd==cellEnd) {
                propEnd=1;
            } else {
                line+=" "+result.substring(2,propEnd).trim().replaceAll(""","\"");
            }
            table=false; cellprocessing=true;
            line+=">"+processTable(result.substring(propEnd+1,cellEnd).trim(), switchboard)+"";
            table=true; cellprocessing=false;
            if (cellEnd and 
) [MN] if((result.indexOf("[=")<0)&&(result.indexOf("=]")<0)&&(!escapeSpan)&& (result.indexOf("<pre>")<0)&&(result.indexOf("</pre>")<0)&&(!preformattedSpan)){ //tables first -> wiki-tags in cells can be treated after that result = this.processTable(result, switchboard); // format lines if (result.startsWith(" ")) result = "" + result + ""; if (result.startsWith("----")) result = "
"; // citings contributed by [MN] if(result.startsWith(":")){ String head = ""; String tail = ""; while(result.startsWith(":")){ head = head + "
"; tail = tail + "
"; result = result.substring(1); } result = head + result + tail; } // end contrib [MN] // format headers if ((p0 = result.indexOf("====")) >= 0) { p1 = result.indexOf("====", p0 + 4); if (p1 >= 0) result = result.substring(0, p0) + "

" + result.substring(p0 + 4, p1) + "

" + result.substring(p1 + 4); } if ((p0 = result.indexOf("===")) >= 0) { p1 = result.indexOf("===", p0 + 3); if (p1 >= 0) result = result.substring(0, p0) + "

" + result.substring(p0 + 3, p1) + "

" + result.substring(p1 + 3); } if ((p0 = result.indexOf("==")) >= 0) { p1 = result.indexOf("==", p0 + 2); if (p1 >= 0) result = result.substring(0, p0) + "

" + result.substring(p0 + 2, p1) + "

" + result.substring(p1 + 2); } if ((p0 = result.indexOf("'''''")) >= 0) { p1 = result.indexOf("'''''", p0 + 5); if (p1 >= 0) result = result.substring(0, p0) + "" + result.substring(p0 + 5, p1) + "" + result.substring(p1 + 5); } if ((p0 = result.indexOf("'''")) >= 0) { p1 = result.indexOf("'''", p0 + 3); if (p1 >= 0) result = result.substring(0, p0) + "" + result.substring(p0 + 3, p1) + "" + result.substring(p1 + 3); } if ((p0 = result.indexOf("''")) >= 0) { p1 = result.indexOf("''", p0 + 2); if (p1 >= 0) result = result.substring(0, p0) + "" + result.substring(p0 + 2, p1) + "" + result.substring(p1 + 2); } //* unorderd Lists contributed by [AS] //** Sublist if(result.startsWith(ListLevel + "*")){ //more stars p0 = result.indexOf(ListLevel); p1 = result.length(); result = "
    " + serverCore.crlfString + "
  • " + result.substring(ListLevel.length() + 1, p1) + "
  • "; ListLevel += "*"; }else if(ListLevel.length() > 0 && result.startsWith(ListLevel)){ //equal number of stars p0 = result.indexOf(ListLevel); p1 = result.length(); result = "
  • " + result.substring(ListLevel.length(), p1) + "
  • "; }else if(ListLevel.length() > 0){ //less stars int i = ListLevel.length(); String tmp = ""; while(! result.startsWith(ListLevel.substring(0,i)) ){ tmp += "
"; i--; } ListLevel = ListLevel.substring(0,i); p0 = ListLevel.length(); p1 = result.length(); if(ListLevel.length() > 0){ result = tmp + "
  • " + result.substring(p0, p1) + "
  • "; }else{ result = tmp + result.substring(p0, p1); } } //# sorted Lists contributed by [AS] //## Sublist if(result.startsWith(numListLevel + "#")){ //more # p0 = result.indexOf(numListLevel); p1 = result.length(); result = "
      " + serverCore.crlfString + "
    1. " + result.substring(numListLevel.length() + 1, p1) + "
    2. "; numListLevel += "#"; }else if(numListLevel.length() > 0 && result.startsWith(numListLevel)){ //equal number of # p0 = result.indexOf(numListLevel); p1 = result.length(); result = "
    3. " + result.substring(numListLevel.length(), p1) + "
    4. "; }else if(numListLevel.length() > 0){ //less # int i = numListLevel.length(); String tmp = ""; while(! result.startsWith(numListLevel.substring(0,i)) ){ tmp += "
    "; i--; } numListLevel = numListLevel.substring(0,i); p0 = numListLevel.length(); p1 = result.length(); if(numListLevel.length() > 0){ result = tmp + "
  • " + result.substring(p0, p1) + "
  • "; }else{ result = tmp + result.substring(p0, p1); } } // end contrib [AS] //* definition Lists contributed by [MN] based on unordered list code by [AS] if(result.startsWith(defListLevel + ";")){ //more semicolons String dt = ""; String dd = ""; p0 = result.indexOf(defListLevel); p1 = result.length(); String resultCopy = result.substring(defListLevel.length() + 1, p1); if((p0 = resultCopy.indexOf(":")) > 0){ dt = resultCopy.substring(0,p0); dd = resultCopy.substring(p0+1); result = "
    " + "
    " + dt + "
    " + "
    " + dd; defList = true; } defListLevel += ";"; }else if(defListLevel.length() > 0 && result.startsWith(defListLevel)){ //equal number of semicolons String dt = ""; String dd = ""; p0 = result.indexOf(defListLevel); p1 = result.length(); String resultCopy = result.substring(defListLevel.length(), p1); if((p0 = resultCopy.indexOf(":")) > 0){ dt = resultCopy.substring(0,p0); dd = resultCopy.substring(p0+1); result = "
    " + dt + "
    " + "
    " + dd; defList = true; } }else if(defListLevel.length() > 0){ //less semicolons String dt = ""; String dd = ""; int i = defListLevel.length(); String tmp = ""; while(! result.startsWith(defListLevel.substring(0,i)) ){ tmp += "
    "; i--; } defListLevel = defListLevel.substring(0,i); p0 = defListLevel.length(); p1 = result.length(); if(defListLevel.length() > 0){ String resultCopy = result.substring(p0, p1); if((p0 = resultCopy.indexOf(":")) > 0){ dt = resultCopy.substring(0,p0); dd = resultCopy.substring(p0+1); result = tmp + "
    " + dt + "
    " + "
    " + dd; defList = true; } }else{ result = tmp + result.substring(p0, p1); } } // end contrib [MN] // create links String kl, kv, alt, align; int p; // internal links and images while ((p0 = result.indexOf("[[")) >= 0) { p1 = result.indexOf("]]", p0 + 2); if (p1 <= p0) break; kl = result.substring(p0 + 2, p1); // this is the part of the code that's responsible for images // contributed by [MN] if (kl.startsWith("Image:")) { alt = ""; align = ""; kv = ""; kl = kl.substring(6); // are there any arguments for the image? if ((p = kl.indexOf("|")) > 0) { kv = kl.substring(p + 1); kl = kl.substring(0, p); // if there are 2 arguments, write them into ALIGN and ALT if ((p = kv.indexOf("|")) > 0) { align = " align=\"" + kv.substring(0, p) + "\""; alt = " alt=\"" + kv.substring(p + 1) + "\""; } // if there is just one, put it into ALT else alt = " alt=\"" + kv + "\""; } // replace incomplete URLs and make them point to http://peerip:port/... // with this feature you can access an image in DATA/HTDOCS/share/yacy.gif // using the wikicode [[Image:share/yacy.gif]] // or an image DATA/HTDOCS/grafics/kaskelix.jpg with [[Image:grafics/kaskelix.jpg]] // you are free to use other sub-paths of DATA/HTDOCS if (!((kl.indexOf("://"))>=0)) { kl = "http://" + yacyCore.seedDB.mySeed.getAddress().trim() + "/" + kl; } result = result.substring(0, p0) + "" + result.substring(p1 + 2); } // end contrib [MN] // if it's no image, it might be an internal link else { if ((p = kl.indexOf("|")) > 0) { kv = kl.substring(p + 1); kl = kl.substring(0, p); } else { kv = kl; } if (switchboard.wikiDB.read(kl) != null) result = result.substring(0, p0) + "" + kv + "" + result.substring(p1 + 2); else result = result.substring(0, p0) + "" + kv + "" + result.substring(p1 + 2); } } // external links while ((p0 = result.indexOf("[")) >= 0) { p1 = result.indexOf("]", p0 + 1); if (p1 <= p0) break; kl = result.substring(p0 + 1, p1); if ((p = kl.indexOf(" ")) > 0) { kv = kl.substring(p + 1); kl = kl.substring(0, p); } // No text for the link? -> http://www.url.com/ else { kv = kl; } // replace incomplete URLs and make them point to http://peerip:port/... // with this feature you can access a file at DATA/HTDOCS/share/page.html // using the wikicode [share/page.html] // or a file DATA/HTDOCS/www/page.html with [www/page.html] // you are free to use other sub-paths of DATA/HTDOCS if (!((kl.indexOf("://"))>=0)) { kl = "http://" + yacyCore.seedDB.mySeed.getAddress().trim() + "/" + kl; } result = result.substring(0, p0) + "" + kv + "" + result.substring(p1 + 1); } } //escape code ([=...=]) contributed by [MN] //both [= and =] in the same line else if(((p0 = result.indexOf("[="))>=0)&&((p1 = result.indexOf("=]"))>0)&&(!(preformatted))){ String escapeText = result.substring(p0+2,p1); //BUGS TO BE FIXED: [=[=text=]=] does not work properly: //[=[= undx=]x=] should resolve as [= undx=]x, but resolves as [= undxx=] //ALSO [=[= und =]=] [= und =] leads to an exception // //handlicg cases where the text inside [= and =] also contains //[= and =]. Example: [=[=...=]=] result = transformLine(result.substring(0,p0).replaceAll("!esc!", "!esc!!")+"!esc!txt!"+result.substring(p1+2).replaceAll("!esc!", "!esc!!"), switchboard); result = result.replaceAll("!esc!txt!", escapeText); result = result.replaceAll("!esc!!", "!esc!"); } //start [= else if(((p0 = result.indexOf("[="))>=0)&&(!escapeSpan)&&(!preformatted)){ escape = true; //prevent surplus line breaks escaped = true; //prevents
     being parsed
                String bq = "";   //gets filled with 
    s as needed String escapeText = result.substring(p0+2); //taking care of indented lines while(result.substring(escindented,p0).startsWith(":")){ escindented++; bq = bq + "
    "; } result = transformLine(result.substring(escindented,p0).replaceAll("!esc!", "!esc!!")+"!esc!txt!", switchboard); result = bq + result.replaceAll("!esc!txt!", escapeText); escape = false; escapeSpan = true; } //end =] else if(((p0 = result.indexOf("=]"))>=0)&&(escapeSpan)&&(!preformatted)){ escapeSpan = false; String bq = ""; //gets filled with
    s as neede String escapeText = result.substring(0,p0); //taking care of indented lines while(escindented > 0){ bq = bq + "
    "; escindented--; } result = transformLine("!esc!txt!"+result.substring(p0+2).replaceAll("!esc!", "!esc!!"), switchboard); result = result.replaceAll("!esc!txt!", escapeText) + bq; escaped = false; } //end contrib [MN] //preformatted code (
    ...
    ) contributed by [MN] //implementation very similar to escape code (see above) //both
     and 
    in the same line else if(((p0 = result.indexOf("<pre>"))>=0)&&((p1 = result.indexOf("</pre>"))>0)&&(!(escaped))){ String preformattedText = "
    "+result.substring(p0+11,p1)+"
    "; result = transformLine(result.substring(0,p0).replaceAll("!pre!", "!pre!!")+"!pre!txt!"+result.substring(p1+12).replaceAll("!pre!", "!pre!!"), switchboard); result = result.replaceAll("!pre!txt!", preformattedText); result = result.replaceAll("!pre!!", "!pre!"); } //start
            else if(((p0 = result.indexOf("<pre>"))>=0)&&(!preformattedSpan)&&(!escaped)){
                preformatted = true;    //prevent surplus line breaks
                String bq ="";  //gets filled with 
    s as needed String preformattedText = "
    "+result.substring(p0+11);
                //taking care of indented lines
                while(result.substring(preindented,p0).startsWith(":")){
                    preindented++;
                    bq = bq + "
    "; } result = transformLine(result.substring(preindented,p0).replaceAll("!pre!", "!pre!!")+"!pre!txt!", switchboard); result = bq + result.replaceAll("!pre!txt!", preformattedText); result = result.replaceAll("!pre!!", "!pre!"); preformattedSpan = true; } //end
    else if(((p0 = result.indexOf("</pre>"))>=0)&&(preformattedSpan)&&(!escaped)){ preformattedSpan = false; String bq = ""; //gets filled with
    s as needed String preformattedText = result.substring(0,p0)+"
    "; //taking care of indented lines while (preindented > 0){ bq = bq + ""; preindented--; } result = transformLine("!pre!txt!"+result.substring(p0+12).replaceAll("!pre!", "!pre!!"), switchboard); result = result.replaceAll("!pre!txt!", preformattedText) + bq; result = result.replaceAll("!pre!!", "!pre!"); preformatted = false; } //end contrib [MN] replacedHTML = false; replacedCharacters = false; if ((result.endsWith(""))||(defList)||(escape)||(preformatted)||(table)||(cellprocessing)) return result; return result + "
    "; } }