// yacyURL.java // (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 13.07.2006 on http://yacy.net // // $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ // $LastChangedRevision: 1986 $ // $LastChangedBy: orbiter $ // // LICENSE // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA package de.anomic.yacy; // this class exsist to provide a system-wide normal form representation of urls, // and to prevent that java.net.URL usage causes DNS queries which are used in java.net. import java.io.File; import java.net.InetAddress; import java.net.MalformedURLException; import java.util.HashMap; import java.util.Iterator; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.server.serverCodings; import de.anomic.server.serverDomains; public class yacyURL { // TLD separation in political and cultural parts // https://www.cia.gov/cia/publications/factbook/index.html // http://en.wikipedia.org/wiki/List_of_countries_by_continent private static final String[] TLD_NorthAmericaOceania={ // primary english-speaking countries // english-speaking countries from central america are also included // includes also dutch and french colonies in the caribbean sea // and US/English/Australian military bases in asia "EDU=US Educational", "GOV=US Government", "MIL=US Military", "NET=Network", "ORG=Non-Profit Organization", "AN=Netherlands Antilles", "AS=American Samoa", "AG=Antigua and Barbuda", "AI=Anguilla", "AU=Australia", "BB=Barbados", "BZ=Belize", "BM=Bermuda", "BS=Bahamas", "CA=Canada", "CC=Cocos (Keeling) Islands", "CK=Cook Islands", "CX=Christmas Island", // located in the Indian Ocean, but belongs to Australia "DM=Dominica", "FM=Micronesia", "FJ=Fiji", "GD=Grenada", "GP=Guadeloupe", "GS=South Georgia and the South Sandwich Islands", // south of south america, but administrated by british, has only a scientific base "GU=Guam", // strategical US basis close to Japan "HM=Heard and McDonald Islands", // uninhabited, sub-Antarctic island, owned by Australia "HT=Haiti", "IO=British Indian Ocean Territory", // UK-US naval support facility in the Indian Ocean "KI=Kiribati", // 33 coral atolls in the pacific, formerly owned by UK "KN=Saint Kitts and Nevis", // islands in the carribean see "KY=Cayman Islands", "LC=Saint Lucia", "MH=Marshall Islands", // formerly US atomic bomb test site, now a key installation in the US missile defense network "MP=Northern Mariana Islands", // US strategic location in the western Pacific Ocean "NC=New Caledonia", "NF=Norfolk Island", "NR=Nauru", // independent UN island "NU=Niue", // one of world's largest coral islands "NZ=New Zealand (Aotearoa)", "PG=Papua New Guinea", "PN=Pitcairn", // overseas territory of the UK "PR=Puerto Rico", // territory of the US with commonwealth status "PW=Palau", // was once governed by Micronesia "Sb=Solomon Islands", "TC=Turks and Caicos Islands", // overseas territory of the UK "TK=Tokelau", // group of three atolls in the South Pacific Ocean, british protectorat "TO=Tonga", "TT=Trinidad and Tobago", "TV=Tuvalu", // nine coral atolls in the South Pacific Ocean; in 2000, Tuvalu leased its TLD ".tv" for $50 million over a 12-year period "UM=US Minor Outlying Islands", // nine insular United States possessions in the Pacific Ocean and the Caribbean Sea "US=United States", "VC=Saint Vincent and the Grenadines", "VG=Virgin Islands (British)", "VI=Virgin Islands (U.S.)", "VU=Vanuatu", "WF=Wallis and Futuna Islands", "WS=Samoa" }; private static final String[] TLD_MiddleSouthAmerica = { // primary spanish and portugese-speaking "AR=Argentina", "AW=Aruba", "BR=Brazil", "BO=Bolivia", "CL=Chile", "CO=Colombia", "CR=Costa Rica", "CU=Cuba", "DO=Dominican Republic", "EC=Ecuador", "FK=Falkland Islands (Malvinas)", "GF=French Guiana", "GT=Guatemala", "GY=Guyana", "HN=Honduras", "JM=Jamaica", "MX=Mexico", "NI=Nicaragua", "PA=Panama", "PE=Peru", "PY=Paraguay", "SR=Suriname", "SV=El Salvador", "UY=Uruguay", "VE=Venezuela" }; private static final String[] TLD_EuropaRussia = { // includes also countries that are mainly french- dutch- speaking // and culturally close to europe "AD=Andorra", "AL=Albania", "AQ=Antarctica", "AT=Austria", "BA=Bosnia and Herzegovina", "BE=Belgium", "BG=Bulgaria", "BV=Bouvet Island", // this island is uninhabited and covered by ice, south of africa but governed by Norway "BY=Belarus", "CH=Switzerland", "CS=Czechoslovakia (former)", "CZ=Czech Republic", "CY=Cyprus", "DE=Germany", "DK=Denmark", "ES=Spain", "EE=Estonia", "FI=Finland", "FO=Faroe Islands", // Viking Settlers "FR=France", "FX=France, Metropolitan", "GB=Great Britain (UK)", "GI=Gibraltar", "GL=Greenland", "GR=Greece", "HR=Croatia (Hrvatska)", "HU=Hungary", "IE=Ireland", "IS=Iceland", "IT=Italy", "LI=Liechtenstein", "LT=Lithuania", "LU=Luxembourg", "LV=Latvia", "MD=Moldova", "MC=Monaco", "MK=Macedonia", "MN=Mongolia", "MS=Montserrat", // British island in the Caribbean Sea, almost not populated because of strong vulcanic activity "MT=Malta", "MQ=Martinique", // island in the eastern Caribbean Sea, overseas department of France "NATO=Nato field", "NL=Netherlands", "NO=Norway", "PF=French Polynesia", // French annexed Polynesian island in the South Pacific, French atomic bomb test site "PL=Poland", "PM=St. Pierre and Miquelon", // french-administrated colony close to canada, belongs to France "PT=Portugal", "RO=Romania", "RU=Russia", "SE=Sweden", "SI=Slovenia", "SJ=Svalbard and Jan Mayen Islands", // part of Norway "SM=San Marino", "SK=Slovak Republic", "SU=USSR (former)", "TF=French Southern Territories", // islands in the arctic see, no inhabitants "UK=United Kingdom", "UA=Ukraine", "VA=Vatican City State (Holy See)", "YU=Yugoslavia" }; private static final String[] TLD_MiddleEastWestAsia = { // states that are influenced by islamic culture and arabic language // includes also eurasia states and those that had been part of the former USSR and close to southwest asia "AE=United Arab Emirates", "AF=Afghanistan", "AM=Armenia", "AZ=Azerbaijan", "BH=Bahrain", "GE=Georgia", "IL=Israel", "IQ=Iraq", "IR=Iran", "JO=Jordan", "KG=Kyrgyzstan", "KZ=Kazakhstan", "KW=Kuwait", "LB=Lebanon", "OM=Oman", "QA=Qatar", "SA=Saudi Arabia", "SY=Syria", "TJ=Tajikistan", "TM=Turkmenistan", "PK=Pakistan", "TR=Turkey", "UZ=Uzbekistan", "YE=Yemen" }; private static final String[] TLD_SouthEastAsia = { "BD=Bangladesh", "BN=Brunei Darussalam", "BT=Bhutan", "CN=China", "HK=Hong Kong", "ID=Indonesia", "IN=India", "LA=Laos", "NP=Nepal", "JP=Japan", "KH=Cambodia", "KP=Korea (North)", "KR=Korea (South)", "LK=Sri Lanka", "MY=Malaysia", "MM=Myanmar", // formerly known as Burma "MO=Macau", // Portuguese settlement, part of China, but has some autonomy "MV=Maldives", // group of atolls in the Indian Ocean "PH=Philippines", "SG=Singapore", "TP=East Timor", "TH=Thailand", "TW=Taiwan", "VN=Viet Nam" }; private static final String[] TLD_Africa = { "AO=Angola", "BF=Burkina Faso", "BI=Burundi", "BJ=Benin", "BW=Botswana", "CF=Central African Republic", "CG=Congo", "CI=Cote D'Ivoire (Ivory Coast)", "CM=Cameroon", "CV=Cape Verde", "DJ=Djibouti", "DZ=Algeria", "EG=Egypt", "EH=Western Sahara", "ER=Eritrea", "ET=Ethiopia", "GA=Gabon", "GH=Ghana", "GM=Gambia", "GN=Guinea", "GQ=Equatorial Guinea", "GW=Guinea-Bissau", "KE=Kenya", "KM=Comoros", "LR=Liberia", "LS=Lesotho", "LY=Libya", "MA=Morocco", "MG=Madagascar", "ML=Mali", "MR=Mauritania", "MU=Mauritius", "MW=Malawi", "MZ=Mozambique", "NA=Namibia", "NE=Niger", "NG=Nigeria", "RE=Reunion", "RW=Rwanda", "SC=Seychelles", "SD=Sudan", "SH=St. Helena", "SL=Sierra Leone", "SN=Senegal", "SO=Somalia", "ST=Sao Tome and Principe", "SZ=Swaziland", "TD=Chad", "TG=Togo", "TN=Tunisia", "TZ=Tanzania", "UG=Uganda", "ZA=South Africa", "ZM=Zambia", "ZR=Zaire", "ZW=Zimbabwe", "YT=Mayotte" }; private static final String[] TLD_Generic = { "COM=US Commercial", "AERO=", "BIZ=", "COOP=", "INFO=", "MUSEUM=", "NAME=", "PRO=", "ARPA=", "INT=International", "ARPA=Arpanet", "NT=Neutral Zone" }; /* * TLDs: aero, biz, com, coop, edu, gov, info, int, mil, museum, name, net, * org, pro, arpa AC, AD, AE, AERO, AF, AG, AI, AL, AM, AN, AO, AQ, AR, * ARPA, AS, AT, AU, AW, AZ, BA, BB, BD, BE, BF, BG, BH, BI, BIZ, BJ, BM, * BN, BO, BR, BS, BT, BV, BW, BY, BZ, CA, CC, CD, CF, CG, CH, CI, CK, CL, * CM, CN, CO, COM, COOP, CR, CU, CV, CX, CY, CZ, DE, DJ, DK, DM, DO, DZ, * EC, EDU, EE, EG, ER, ES, ET, EU, FI, FJ, FK, FM, FO, FR, GA, GB, GD, GE, * GF, GG, GH, GI, GL, GM, GN, GOV, GP, GQ, GR, GS, GT, GU, GW, GY, HK, HM, * HN, HR, HT, HU, ID, IE, IL, IM, IN, INFO, INT, IO, IQ, IR, IS, IT, JE, * JM, JO, JOBS, JP, KE, KG, KH, KI, KM, KN, KR, KW, KY, KZ, LA, LB, LC, LI, * LK, LR, LS, LT, LU, LV, LY, MA, MC, MD, MG, MH, MIL, MK, ML, MM, MN, MO, * MOBI, MP, MQ, MR, MS, MT, MU, MUSEUM, MV, MW, MX, MY, MZ, NA, NAME, NC, * NE, NET, NF, NG, NI, NL, NO, NP, NR, NU, NZ, OM, ORG, PA, PE, PF, PG, PH, * PK, PL, PM, PN, PR, PRO, PS, PT, PW, PY, QA, RE, RO, RU, RW, SA, SB, SC, * SD, SE, SG, SH, SI, SJ, SK, SL, SM, SN, SO, SR, ST, SU, SV, SY, SZ, TC, * TD, TF, TG, TH, TJ, TK, TL, TM, TN, TO, TP, TR, TRAVEL, TT, TV, TW, TZ, * UA, UG, UK, UM, US, UY, UZ, VA, VC, VE, VG, VI, VN, VU, WF, WS, YE, YT, * YU, ZA, ZM, ZW */ public static String dummyHash; private static HashMap TLDID = new HashMap(); private static HashMap TLDName = new HashMap(); private static void insertTLDProps(String[] TLDList, int id) { int p; String tld, name; Integer ID = new Integer(id); for (int i = 0; i < TLDList.length; i++) { p = TLDList[i].indexOf('='); if (p > 0) { tld = TLDList[i].substring(0, p).toLowerCase(); name = TLDList[i].substring(p + 1); TLDID.put(tld, ID); TLDName.put(tld, name); } } } static { // create a dummy hash dummyHash = ""; for (int i = 0; i < yacySeedDB.commonHashLength; i++) dummyHash += "-"; // assign TLD-ids and names insertTLDProps(TLD_EuropaRussia, 0); insertTLDProps(TLD_MiddleSouthAmerica, 1); insertTLDProps(TLD_SouthEastAsia, 2); insertTLDProps(TLD_MiddleEastWestAsia, 3); insertTLDProps(TLD_NorthAmericaOceania, 4); insertTLDProps(TLD_Africa, 5); insertTLDProps(TLD_Generic, 6); // the id=7 is used to flag local addresses } // class variables private String protocol, host, userInfo, path, quest, ref, hash; private int port; public yacyURL(String url, String hash) throws MalformedURLException { if (url == null) throw new MalformedURLException("url string is null"); parseURLString(url); this.hash = hash; } private void parseURLString(String url) throws MalformedURLException { // identify protocol assert (url != null); url = url.trim(); int p = url.indexOf(':'); if (p < 0) throw new MalformedURLException("protocol is not given in '" + url + "'"); this.protocol = url.substring(0, p).toLowerCase().trim(); if (url.length() < p + 4) throw new MalformedURLException("URL not parseable: '" + url + "'"); if (url.substring(p + 1, p + 3).equals("//")) { // identify host, userInfo and file for http and ftp protocol int q = url.indexOf('/', p + 3); int r; if (q < 0) { if ((r = url.indexOf('@', p + 3)) < 0) { host = url.substring(p + 3); userInfo = null; } else { host = url.substring(r + 1); userInfo = url.substring(p + 3, r); } path = "/"; } else { host = url.substring(p + 3, q); if ((r = host.indexOf('@')) < 0) { userInfo = null; } else { userInfo = host.substring(0, r); host = host.substring(r + 1); } path = url.substring(q); } path = resolveBackpath(path); identPort(url, (protocol.equals("http") ? 80 : ((protocol.equals("https")) ? 443 : ((protocol.equals("ftp")) ? 21 : -1)))); identRef(); identQuest(); escape(); } else { // this is not a http or ftp url if (protocol.equals("mailto")) { // parse email url int q = url.indexOf('@', p + 3); if (q < 0) { throw new MalformedURLException("wrong email address: " + url); } else { userInfo = url.substring(p + 1, q); host = url.substring(q + 1); path = null; port = -1; quest = null; ref = null; } } else { throw new MalformedURLException("unknown protocol: " + url); } } } public yacyURL(File file) throws MalformedURLException { this("file", "", -1, file.getAbsolutePath()); } public static yacyURL newURL(String baseURL, String relPath) throws MalformedURLException { if ((baseURL == null) || (relPath.startsWith("http://")) || (relPath.startsWith("https://")) || (relPath.startsWith("ftp://")) || (relPath.startsWith("file://")) || (relPath.startsWith("smb://"))) { return new yacyURL(relPath, null); } else { return new yacyURL(new yacyURL(baseURL, null), relPath); } } public static yacyURL newURL(yacyURL baseURL, String relPath) throws MalformedURLException { if ((baseURL == null) || (relPath.startsWith("http://")) || (relPath.startsWith("https://")) || (relPath.startsWith("ftp://")) || (relPath.startsWith("file://")) || (relPath.startsWith("smb://"))) { return new yacyURL(relPath, null); } else { return new yacyURL(baseURL, relPath); } } private yacyURL(yacyURL baseURL, String relPath) throws MalformedURLException { if (baseURL == null) throw new MalformedURLException("base URL is null"); if (relPath == null) throw new MalformedURLException("relPath is null"); this.hash = null; this.protocol = baseURL.protocol; this.host = baseURL.host; this.port = baseURL.port; this.userInfo = baseURL.userInfo; if (relPath.toLowerCase().startsWith("javascript:")) { this.path = baseURL.path; } else if ( (relPath.startsWith("http://")) || (relPath.startsWith("https://")) || (relPath.startsWith("ftp://")) || (relPath.startsWith("file://")) || (relPath.startsWith("smb://"))) { this.path = baseURL.path; } else if (relPath.startsWith("/")) { this.path = relPath; } else if (baseURL.path.endsWith("/")) { if (relPath.startsWith("#") || relPath.startsWith("?")) { throw new MalformedURLException("relative path malformed: " + relPath); } else { this.path = baseURL.path + relPath; } } else { if (relPath.startsWith("#") || relPath.startsWith("?")) { this.path = baseURL.path + relPath; } else { int q = baseURL.path.lastIndexOf('/'); if (q < 0) { this.path = relPath; } else { this.path = baseURL.path.substring(0, q + 1) + relPath; } } } this.quest = baseURL.quest; this.ref = baseURL.ref; path = resolveBackpath(path); identRef(); identQuest(); escape(); } public yacyURL(String protocol, String host, int port, String path) throws MalformedURLException { if (protocol == null) throw new MalformedURLException("protocol is null"); this.protocol = protocol; this.host = host; this.port = port; this.path = path; this.hash = null; identRef(); identQuest(); escape(); } // resolve '..' String resolveBackpath(String path) /* throws MalformedURLException */ { /* original version by [MC] int p; while ((p = path.indexOf("/..")) >= 0) { String head = path.substring(0, p); int q = head.lastIndexOf('/'); if (q < 0) throw new MalformedURLException("backpath cannot be resolved in path = " + path); path = head.substring(0, q) + path.substring(p + 3); }*/ /* by [MT] */ if (path.length() == 0 || path.charAt(0) != '/') { path = "/" + path; } Pattern pathPattern = Pattern.compile("(/[^/]+(? *
  • path: see {@link #escape(String)}
    • *
  • ref: same as above
    • *
  • quest: same as above without the ampersand ("&") and the equals symbol
    • * */ private void escape() { if (path != null && path.indexOf('%') == -1) escapePath(); if (quest != null && quest.indexOf('%') == -1) escapeQuest(); if (ref != null && ref.indexOf('%') == -1) escapeRef(); } private void escapePath() { String[] pathp = path.split("/", -1); String ptmp = ""; for (int i = 0; i < pathp.length; i++) { ptmp += "/" + escape(pathp[i]); } path = ptmp.substring((ptmp.length() > 0) ? 1 : 0); } private void escapeRef() { ref = escape(ref); } private void escapeQuest() { String[] questp = quest.split("&", -1); String qtmp = ""; for (int i = 0; i < questp.length; i++) { if (questp[i].indexOf('=') != -1) { qtmp += "&" + escape(questp[i].substring(0, questp[i].indexOf('='))); qtmp += "=" + escape(questp[i].substring(questp[i].indexOf('=') + 1)); } else { qtmp += "&" + escape(questp[i]); } } quest = qtmp.substring((qtmp.length() > 0) ? 1 : 0); } private final static String[] hex = { "%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07", "%08", "%09", "%0A", "%0B", "%0C", "%0D", "%0E", "%0F", "%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17", "%18", "%19", "%1A", "%1B", "%1C", "%1D", "%1E", "%1F", "%20", "%21", "%22", "%23", "%24", "%25", "%26", "%27", "%28", "%29", "%2A", "%2B", "%2C", "%2D", "%2E", "%2F", "%30", "%31", "%32", "%33", "%34", "%35", "%36", "%37", "%38", "%39", "%3A", "%3B", "%3C", "%3D", "%3E", "%3F", "%40", "%41", "%42", "%43", "%44", "%45", "%46", "%47", "%48", "%49", "%4A", "%4B", "%4C", "%4D", "%4E", "%4F", "%50", "%51", "%52", "%53", "%54", "%55", "%56", "%57", "%58", "%59", "%5A", "%5B", "%5C", "%5D", "%5E", "%5F", "%60", "%61", "%62", "%63", "%64", "%65", "%66", "%67", "%68", "%69", "%6A", "%6B", "%6C", "%6D", "%6E", "%6F", "%70", "%71", "%72", "%73", "%74", "%75", "%76", "%77", "%78", "%79", "%7A", "%7B", "%7C", "%7D", "%7E", "%7F", "%80", "%81", "%82", "%83", "%84", "%85", "%86", "%87", "%88", "%89", "%8A", "%8B", "%8C", "%8D", "%8E", "%8F", "%90", "%91", "%92", "%93", "%94", "%95", "%96", "%97", "%98", "%99", "%9A", "%9B", "%9C", "%9D", "%9E", "%9F", "%A0", "%A1", "%A2", "%A3", "%A4", "%A5", "%A6", "%A7", "%A8", "%A9", "%AA", "%AB", "%AC", "%AD", "%AE", "%AF", "%B0", "%B1", "%B2", "%B3", "%B4", "%B5", "%B6", "%B7", "%B8", "%B9", "%BA", "%BB", "%BC", "%BD", "%BE", "%BF", "%C0", "%C1", "%C2", "%C3", "%C4", "%C5", "%C6", "%C7", "%C8", "%C9", "%CA", "%CB", "%CC", "%CD", "%CE", "%CF", "%D0", "%D1", "%D2", "%D3", "%D4", "%D5", "%D6", "%D7", "%D8", "%D9", "%DA", "%DB", "%DC", "%DD", "%DE", "%DF", "%E0", "%E1", "%E2", "%E3", "%E4", "%E5", "%E6", "%E7", "%E8", "%E9", "%EA", "%EB", "%EC", "%ED", "%EE", "%EF", "%F0", "%F1", "%F2", "%F3", "%F4", "%F5", "%F6", "%F7", "%F8", "%F9", "%FA", "%FB", "%FC", "%FD", "%FE", "%FF" }; /** * Encode a string to the "x-www-form-urlencoded" form, enhanced * with the UTF-8-in-URL proposal. This is what happens: * * * * @param s The string to be encoded * @return The encoded string */ // from: http://www.w3.org/International/URLUTF8Encoder.java public static String escape(String s) { StringBuffer sbuf = new StringBuffer(); int len = s.length(); for (int i = 0; i < len; i++) { int ch = s.charAt(i); if ('A' <= ch && ch <= 'Z') { // 'A'..'Z' sbuf.append((char)ch); } else if ('a' <= ch && ch <= 'z') { // 'a'..'z' sbuf.append((char)ch); } else if ('0' <= ch && ch <= '9') { // '0'..'9' sbuf.append((char)ch); } else if (ch == ' ') { // space sbuf.append("%20"); } else if (ch == '&' || ch == ':' // unreserved || ch == '-' || ch == '_' || ch == '.' || ch == '!' || ch == '~' || ch == '*' || ch == '\'' || ch == '(' || ch == ')' || ch == ';') { sbuf.append((char)ch); } else if (ch <= 0x007f) { // other ASCII sbuf.append(hex[ch]); } else if (ch <= 0x07FF) { // non-ASCII <= 0x7FF sbuf.append(hex[0xc0 | (ch >> 6)]); sbuf.append(hex[0x80 | (ch & 0x3F)]); } else { // 0x7FF < ch <= 0xFFFF sbuf.append(hex[0xe0 | (ch >> 12)]); sbuf.append(hex[0x80 | ((ch >> 6) & 0x3F)]); sbuf.append(hex[0x80 | (ch & 0x3F)]); } } return sbuf.toString(); } // from: http://www.w3.org/International/unescape.java public static String unescape(String s) { StringBuffer sbuf = new StringBuffer(); int l = s.length(); int ch = -1; int b, sumb = 0; for (int i = 0, more = -1; i < l; i++) { /* Get next byte b from URL segment s */ switch (ch = s.charAt(i)) { case '%': ch = s.charAt(++i) ; int hb = (Character.isDigit ((char) ch) ? ch - '0' : 10 + Character.toLowerCase((char) ch) - 'a') & 0xF; ch = s.charAt(++i) ; int lb = (Character.isDigit ((char) ch) ? ch - '0' : 10 + Character.toLowerCase ((char) ch) - 'a') & 0xF; b = (hb << 4) | lb; break; case '+': b = ' '; break; default: b = ch; } /* Decode byte b as UTF-8, sumb collects incomplete chars */ if ((b & 0xc0) == 0x80) { // 10xxxxxx (continuation byte) sumb = (sumb << 6) | (b & 0x3f) ; // Add 6 bits to sumb if (--more == 0) sbuf.append((char) sumb) ; // Add char to sbuf } else if ((b & 0x80) == 0x00) { // 0xxxxxxx (yields 7 bits) sbuf.append((char) b) ; // Store in sbuf } else if ((b & 0xe0) == 0xc0) { // 110xxxxx (yields 5 bits) sumb = b & 0x1f; more = 1; // Expect 1 more byte } else if ((b & 0xf0) == 0xe0) { // 1110xxxx (yields 4 bits) sumb = b & 0x0f; more = 2; // Expect 2 more bytes } else if ((b & 0xf8) == 0xf0) { // 11110xxx (yields 3 bits) sumb = b & 0x07; more = 3; // Expect 3 more bytes } else if ((b & 0xfc) == 0xf8) { // 111110xx (yields 2 bits) sumb = b & 0x03; more = 4; // Expect 4 more bytes } else /*if ((b & 0xfe) == 0xfc)*/ { // 1111110x (yields 1 bit) sumb = b & 0x01; more = 5; // Expect 5 more bytes } /* We don't test if the UTF-8 encoding is well-formed */ } return sbuf.toString(); } private void identPort(String inputURL, int dflt) throws MalformedURLException { // identify ref in file int r = this.host.indexOf(':'); if (r < 0) { this.port = dflt; } else { try { String portStr = this.host.substring(r + 1); if (portStr.trim().length() > 0) this.port = Integer.parseInt(portStr); else this.port = -1; this.host = this.host.substring(0, r); } catch (NumberFormatException e) { throw new MalformedURLException("wrong port in host fragment '" + this.host + "' of input url '" + inputURL + "'"); } } } private void identRef() { // identify ref in file int r = path.indexOf('#'); if (r < 0) { this.ref = null; } else { this.ref = path.substring(r + 1); this.path = path.substring(0, r); } } private void identQuest() { // identify quest in file int r = path.indexOf('?'); if (r < 0) { this.quest = null; } else { this.quest = path.substring(r + 1); this.path = path.substring(0, r); } } public String getFile() { return getFile(true); } public String getFile(boolean includeReference) { // this is the path plus quest plus ref // if there is no quest and no ref the result is identical to getPath // this is defined according to http://java.sun.com/j2se/1.4.2/docs/api/java/net/URL.html#getFile() if (quest != null) return ((includeReference) && (ref != null)) ? path + "?" + quest + "#" + ref : path + "?" + quest; return ((includeReference) && (ref != null)) ? path + "#" + ref : path; } public String getFileName() { // this is a method not defined in any sun api // it returns the last portion of a path without any reference int p = path.lastIndexOf('/'); if (p < 0) return path; if (p == path.length() - 1) return ""; // no file name, this is a path to a directory return path.substring(p + 1); // the 'real' file name } public String getPath() { return path; } public String getAuthority() { return ((port >= 0) && (host != null)) ? host + ":" + port : ((host != null) ? host : ""); } public String getHost() { return host; } public int getPort() { return port; } public String getProtocol() { return protocol; } public String getRef() { return ref; } public String getUserInfo() { return userInfo; } public String getQuery() { return quest; } public String toString() { return toNormalform(false, true); } public String toNormalform(boolean stripReference, boolean stripAmp) { if (stripAmp) return toNormalform(!stripReference).replaceAll("&", "&"); else return toNormalform(!stripReference); } private String toNormalform(boolean includeReference) { // generates a normal form of the URL boolean defaultPort = false; if (this.protocol.equals("mailto")) { return this.protocol + ":" + this.userInfo + "@" + this.host; } else if (this.protocol.equals("http")) { if (this.port < 0 || this.port == 80) { defaultPort = true; } } else if (this.protocol.equals("ftp")) { if (this.port < 0 || this.port == 21) { defaultPort = true; } } else if (this.protocol.equals("https")) { if (this.port < 0 || this.port == 443) { defaultPort = true; } } String path = resolveBackpath(this.getFile(includeReference)); if (defaultPort) { return this.protocol + "://" + ((this.userInfo != null) ? (this.userInfo + "@") : ("")) + this.getHost().toLowerCase() + path; } return this.protocol + "://" + ((this.userInfo != null) ? (this.userInfo + "@") : ("")) + this.getHost().toLowerCase() + ((defaultPort) ? ("") : (":" + this.port)) + path; } public boolean equals(yacyURL other) { return (((this.protocol == other.protocol) || (this.protocol.equals(other.protocol))) && ((this.host == other.host ) || (this.host.equals(other.host))) && ((this.userInfo == other.userInfo) || (this.userInfo.equals(other.userInfo))) && ((this.path == other.path ) || (this.path.equals(other.path))) && ((this.quest == other.quest ) || (this.quest.equals(other.quest))) && ((this.ref == other.ref ) || (this.ref.equals(other.ref))) && ((this.port == other.port ))); } public int hashCode() { return this.toString().hashCode(); } public int compareTo(Object h) { assert (h instanceof yacyURL); return this.toString().compareTo(((yacyURL) h).toString()); } // static methods from plasmaURL public static final int flagTypeID(String hash) { return (kelondroBase64Order.enhancedCoder.decodeByte(hash.charAt(11)) & 32) >> 5; } public static final int flagTLDID(String hash) { return (kelondroBase64Order.enhancedCoder.decodeByte(hash.charAt(11)) & 28) >> 2; } public static final int flagLengthID(String hash) { return (kelondroBase64Order.enhancedCoder.decodeByte(hash.charAt(11)) & 3); } public final String hash() { // in case that the object was initialized without a known url hash, compute it now if (this.hash == null) this.hash = urlHashComputation(); return this.hash; } private final String urlHashComputation() { // the url hash computation needs a DNS lookup to check if the addresses domain is local // that causes that this method may be very slow assert this.hash == null; // should only be called if the hash was not computed bevore int p = this.host.lastIndexOf('.'); String tld = "", dom = tld; if (p > 0) { tld = host.substring(p + 1); dom = host.substring(0, p); } Integer ID = (serverDomains.isLocal(tld)) ? null : (Integer) TLDID.get(tld); // identify local addresses int id = (ID == null) ? 7 : ID.intValue(); // local addresses are flagged with id=7 boolean isHTTP = this.protocol.equals("http"); p = dom.lastIndexOf('.'); // locate subdomain String subdom = ""; if (p > 0) { subdom = dom.substring(0, p); dom = dom.substring(p + 1); } // find rootpath String pathx = new String(this.path); if (pathx.startsWith("/")) pathx = pathx.substring(1); if (pathx.endsWith("/")) pathx = pathx.substring(0, pathx.length() - 1); p = pathx.indexOf('/'); String rootpath = ""; if (p > 0) { rootpath = pathx.substring(0, p); } // we collected enough information to compute the fragments that are // basis for hashes int l = dom.length(); int domlengthKey = (l <= 8) ? 0 : (l <= 12) ? 1 : (l <= 16) ? 2 : 3; byte flagbyte = (byte) (((isHTTP) ? 0 : 32) | (id << 2) | domlengthKey); // combine the attributes StringBuffer hash = new StringBuffer(12); // form the 'local' part of the hash hash.append(kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(toNormalform(true, true))).substring(0, 5)); // 5 chars hash.append(subdomPortPath(subdom, port, rootpath)); // 1 char // form the 'global' part of the hash hash.append(protocolHostPort(this.protocol, host, port)); // 5 chars hash.append(kelondroBase64Order.enhancedCoder.encodeByte(flagbyte)); // 1 char // return result hash return new String(hash); } private static char subdomPortPath(String subdom, int port, String rootpath) { return kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(subdom + ":" + port + ":" + rootpath)).charAt(0); } private static final char rootURLFlag = subdomPortPath("www", 80, ""); public static final boolean probablyRootURL(String urlHash) { return (urlHash.charAt(5) == rootURLFlag); } private static String protocolHostPort(String protocol, String host, int port) { return kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(protocol + ":" + host + ":" + port)).substring(0, 5); } private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" }; public static final yacyURL probablyWordURL(String urlHash, TreeSet words) { Iterator wi = words.iterator(); String word; while (wi.hasNext()) { word = (String) wi.next(); if ((word == null) || (word.length() == 0)) continue; String pattern = urlHash.substring(6, 11); for (int i = 0; i < testTLDs.length; i++) { if (pattern.equals(protocolHostPort("http", "www." + word.toLowerCase() + "." + testTLDs[i], 80))) try { return new yacyURL("http://www." + word.toLowerCase() + "." + testTLDs[i], null); } catch (MalformedURLException e) { return null; } } } return null; } public static final boolean isWordRootURL(String givenURLHash, TreeSet words) { if (!(probablyRootURL(givenURLHash))) return false; yacyURL wordURL = probablyWordURL(givenURLHash, words); if (wordURL == null) return false; if (wordURL.hash().equals(givenURLHash)) return true; return false; } public static final int domLengthEstimation(String urlHash) { // generates an estimation of the original domain length assert (urlHash != null); assert (urlHash.length() == 12) : "urlhash = " + urlHash; int flagbyte = kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)); int domLengthKey = flagbyte & 3; switch (domLengthKey) { case 0: return 4; case 1: return 10; case 2: return 14; case 3: return 20; } return 20; } public static int domLengthNormalized(String urlHash) { return 255 * domLengthEstimation(urlHash) / 30; } public static final int domDomain(String urlHash) { // returns the ID of the domain of the domain assert (urlHash != null); assert (urlHash.length() == 12) : "urlhash = " + urlHash; int flagbyte = kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)); return (flagbyte & 12) >> 2; } public static boolean isGlobalDomain(String urlhash) { return domDomain(urlhash) != 7; } // checks for local/global IP range and local IP public boolean isLocal() { InetAddress hostAddress = serverDomains.dnsResolve(this.host); // TODO: use a check with the hash first if (hostAddress == null) /* we are offline */ return false; // it is rare to be offline in intranets return hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress(); } // language calculation public static String language(yacyURL url) { String language = "uk"; String host = url.getHost(); int pos = host.lastIndexOf("."); if ((pos > 0) && (host.length() - pos == 3)) language = host.substring(pos + 1).toLowerCase(); return language; } public static void main(String[] args) { String[][] test = new String[][]{ new String[]{null, "http://www.anomic.de/home/test?x=1#home"}, new String[]{null, "http://www.anomic.de/home/test?x=1"}, new String[]{null, "http://www.anomic.de/home/test#home"}, new String[]{null, "ftp://ftp.anomic.de/home/test#home"}, new String[]{null, "http://www.anomic.de/home/../abc/"}, new String[]{null, "mailto:abcdefg@nomailnomail.com"}, new String[]{"http://www.anomic.de/home", "test"}, new String[]{"http://www.anomic.de/home", "test/"}, new String[]{"http://www.anomic.de/home/", "test"}, new String[]{"http://www.anomic.de/home/", "test/"}, new String[]{"http://www.anomic.de/home/index.html", "test.htm"}, new String[]{"http://www.anomic.de/home/index.html", "http://www.yacy.net/test"}, new String[]{"http://www.anomic.de/home/index.html", "ftp://ftp.yacy.net/test"}, new String[]{"http://www.anomic.de/home/index.html", "../test"}, new String[]{"http://www.anomic.de/home/index.html", "mailto:abcdefg@nomailnomail.com"}, new String[]{null, "news:de.test"}, new String[]{"http://www.anomic.de/home", "news:de.test"}, new String[]{"http://www.anomic.de/home", "ftp://ftp.anomic.de/src"}, new String[]{null, "ftp://ftp.delegate.org/"}, new String[]{"http://www.anomic.de/home", "ftp://ftp.delegate.org/"}, new String[]{"http://www.anomic.de","mailto:yacy@weltherrschaft.org"}, new String[]{"http://www.anomic.de","javascipt:temp"}, new String[]{null,"http://yacy-websuche.de/wiki/index.php?title=De:IntroInformationFreedom&action=history"}, new String[]{null, "http://diskusjion.no/index.php?s=5bad5f431a106d9a8355429b81bb0ca5&showuser=23585"}, new String[]{null, "http://diskusjion.no/index.php?s=5bad5f431a106d9a8355429b81bb0ca5&showuser=23585"} }; String environment, url; yacyURL aURL, aURL1; java.net.URL jURL; for (int i = 0; i < test.length; i++) { environment = test[i][0]; url = test[i][1]; try {aURL = yacyURL.newURL(environment, url);} catch (MalformedURLException e) {aURL = null;} if (environment == null) { try {jURL = new java.net.URL(url);} catch (MalformedURLException e) {jURL = null;} } else { try {jURL = new java.net.URL(new java.net.URL(environment), url);} catch (MalformedURLException e) {jURL = null;} } // check equality to java.net.URL if (((aURL == null) && (jURL != null)) || ((aURL != null) && (jURL == null)) || ((aURL != null) && (jURL != null) && (!(jURL.toString().equals(aURL.toString()))))) { System.out.println("Difference for environment=" + environment + ", url=" + url + ":"); System.out.println((jURL == null) ? "jURL rejected input" : "jURL=" + jURL.toString()); System.out.println((aURL == null) ? "aURL rejected input" : "aURL=" + aURL.toString()); } // check stability: the normalform of the normalform must be equal to the normalform if (aURL != null) try { aURL1 = new yacyURL(aURL.toNormalform(false, true), null); if (!(aURL1.toNormalform(false, true).equals(aURL.toNormalform(false, true)))) { System.out.println("no stability for url:"); System.out.println("aURL0=" + aURL.toString()); System.out.println("aURL1=" + aURL1.toString()); } } catch (MalformedURLException e) { System.out.println("no stability for url:"); System.out.println("aURL0=" + aURL.toString()); System.out.println("aURL1 cannot be computed:" + e.getMessage()); } } } }