This commit is contained in:
luc 2015-11-02 08:10:11 +01:00
commit 745e97a575
25 changed files with 115 additions and 54 deletions

View File

@ -100,7 +100,7 @@ public class ConfigBasic {
port = post.getLong("port", 8090);
ssl = post.getBoolean("withssl");
} else {
port = env.getLocalPort("port", 8090); //this allows a low port, but it will only get one, if the user edits the config himself.
port = env.getLocalPort(); //this allows a low port, but it will only get one, if the user edits the config himself.
ssl = env.getConfigBool("server.https", false);
}
if (ssl) prop.put("withsslenabled_sslport",env.getHttpServer().getSslPort());
@ -266,7 +266,7 @@ public class ConfigBasic {
// set default values
prop.putHTML("defaultName", sb.peers.mySeed().getName());
prop.put("defaultPort", env.getLocalPort("port", 8090));
prop.put("defaultPort", env.getLocalPort());
prop.put("withsslenabled", env.getConfigBool("server.https", false) ? 1 : 0);
lang = env.getConfig("locale.language", "default"); // re-assign lang, may have changed
prop.put("lang_de", "0");

View File

@ -224,7 +224,7 @@ public class ConfigPortal {
String myaddress = (sb.peers == null) || sb.peers.mySeed() == null || sb.peers.mySeed().getIP() == null ? null : sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP());
if (myaddress == null) {
myaddress = "localhost:" + sb.getLocalPort("port", 8090);
myaddress = "localhost:" + sb.getLocalPort();
}
prop.put("myaddress", myaddress);
return prop;

View File

@ -35,7 +35,7 @@ public class ConfigSearchBox {
final Switchboard sb = (Switchboard) env;
String myaddress = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP());
if (myaddress == null) myaddress = "localhost:" + sb.getLocalPort("port", 8090);
if (myaddress == null) myaddress = "localhost:" + sb.getLocalPort();
prop.put("myaddress", myaddress);
return prop;
}

View File

@ -214,7 +214,7 @@ public class CrawlStartScanner_p
path += "&crawlingURL=" + url.toNormalform(true);
WorkTables.execAPICall(
Domains.LOCALHOST,
sb.getLocalPort("port", 8090),
sb.getLocalPort(),
path,
pk,
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),
@ -261,7 +261,7 @@ public class CrawlStartScanner_p
path += "&crawlingURL=" + urlString;
WorkTables.execAPICall(
Domains.LOCALHOST,
sb.getLocalPort("port", 8090),
sb.getLocalPort(),
path,
u.hash(),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),

View File

@ -39,7 +39,7 @@ public class Load_MediawikiWiki {
// define visible variables
String a = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP());
if (a == null) a = "localhost:" + sb.getLocalPort("port", 8090);
if (a == null) a = "localhost:" + sb.getLocalPort();
final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet");
final String repository = "http://" + a + "/repository/";
prop.put("starturl", (intranet) ? repository : "http://");

View File

@ -39,7 +39,7 @@ public class Load_PHPBB3 {
// define visible variables
String a = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP());
if (a == null) a = "localhost:" + sb.getLocalPort("port", 8090);
if (a == null) a = "localhost:" + sb.getLocalPort();
final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet");
final String repository = "http://" + a + "/repository/";
prop.put("starturl", (intranet) ? repository : "http://");

View File

@ -99,7 +99,7 @@ public class SettingsAck_p {
/*
* display port info
*/
prop.put("info_port", env.getLocalPort("port", 8090));
prop.put("info_port", env.getLocalPort());
prop.put("info_restart", "0");
// read and process data
@ -494,7 +494,7 @@ public class SettingsAck_p {
// change https port
if (post.containsKey("port.ssl")) {
int port = post.getInt("port.ssl", 8443);
if (port > 0 && port != env.getLocalPort("port", 8090)) {
if (port > 0 && port != env.getConfigInt("port.ssl", 8443)) {
env.setConfig("port.ssl", port);
}
prop.put("info_port.ssl", port);

View File

@ -73,7 +73,7 @@ public final class Settings_p {
prop.put("settingsTables", "");
}
prop.put("port", env.getLocalPort("port", 8090));
prop.put("port", env.getLocalPort());
prop.putHTML("peerName", sb.peers.mySeed().getName());
prop.putHTML("staticIP", env.getConfig("staticIP", ""));

View File

@ -208,7 +208,7 @@ public class Table_API_p {
}
// now call the api URLs and store the result status
final Map<String, Integer> l = sb.tables.execAPICalls(Domains.LOCALHOST, sb.getLocalPort("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
final Map<String, Integer> l = sb.tables.execAPICalls(Domains.LOCALHOST, sb.getLocalPort(), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
// construct result table
prop.put("showexec", l.isEmpty() ? 0 : 1);

View File

@ -126,7 +126,7 @@ public class push_p {
Set<String> ips = Domains.myPublicIPs();
String address = ips.size() == 0 ? "127.0.0.1" : ips.iterator().next();
if (address == null) address = "127.0.0.1";
prop.put("mode_results_" + i + "_success_message", "http://" + address + ":" + sb.getLocalPort("port", 8090) + "/solr/select?q=sku:%22" + u + "%22");
prop.put("mode_results_" + i + "_success_message", "http://" + address + ":" + sb.getLocalPort() + "/solr/select?q=sku:%22" + u + "%22");
countsuccess++;
} catch (MalformedURLException e) {
e.printStackTrace();

View File

@ -40,7 +40,7 @@ public class opensearchdescription {
if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
String thisaddress = header.get("Host", Domains.LOCALHOST);
if (thisaddress.indexOf(':',0) == -1) thisaddress += ":" + env.getLocalPort("port", 8090);
if (thisaddress.indexOf(':',0) == -1) thisaddress += ":" + env.getLocalPort();
String thisprotocol = env.getConfigBool("server.https", false) ? "https" : "http";
final serverObjects prop = new serverObjects();

View File

@ -149,7 +149,7 @@ public class yacysearch {
// adding some additional properties needed for the rss feed
String hostName = header.get("Host", Domains.LOCALHOST);
if ( hostName.indexOf(':', 0) == -1 ) {
hostName += ":" + env.getLocalPort("port", 8090);
hostName += ":" + env.getLocalPort();
}
prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html");
prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.png");
@ -900,7 +900,7 @@ public class yacysearch {
// hostname and port (assume locahost if nothing helps)
final String hostIP = sb.peers.mySeed().getIP();
prop.put("myhost", hostIP != null ? hostIP : Domains.LOCALHOST);
prop.put("myport", Domains.LOCALHOST.equals(hostIP) ? sb.getLocalPort("port", 8090) : sb.getPublicPort("port", 8090));
prop.put("myport", Domains.LOCALHOST.equals(hostIP) ? sb.getLocalPort() : sb.getPublicPort("port", 8090));
// return rewrite properties
return prop;

View File

@ -96,7 +96,7 @@ public class yacysearch_location {
if (query.length() > 0 && (metatag || search_title || search_publisher || search_creator || search_subject)) try {
// get a queue of search results
final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getLocalPort("port", 8090) + "/yacysearch.rss";
final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getLocalPort() + "/yacysearch.rss";
final BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, lon == 0.0d && lat == 0.0d ? query : query + " /radius/" + lat + "/" + lon + "/" + radius, maximumTime, Integer.MAX_VALUE, null, false, ClientIdentification.yacyInternetCrawlerAgent);
@ -129,7 +129,7 @@ public class yacysearch_location {
String promoteSearchPageGreeting = env.getConfig(SwitchboardConstants.GREETING, "");
if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
String hostName = header.get("Host", Domains.LOCALHOST);
if (hostName.indexOf(':',0) == -1) hostName += ":" + env.getLocalPort("port", 8090);
if (hostName.indexOf(':',0) == -1) hostName += ":" + env.getLocalPort();
final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance
final boolean global = post.get("kml_resource", "local").equals("global");

View File

@ -52,7 +52,7 @@ all these locations into one search result.
== DEPENDENCIES? WHAT OTHER SOFTWARE DO I NEED? ==
You need java 1.6 or later to run YaCy, nothing else.
You need java 1.7 or later to run YaCy, nothing else.
Please download it from http://www.java.com
YaCy also runs on IcedTea6.

View File

@ -1024,6 +1024,11 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
}
/**
* Process line with geo coordinate metadata
* @param line of wiki text
* @return line with geo coordinate formatted to be recogizeable by parser
*/
private static String processMetadata(String line) {
int p, q, s = 0;
while ((p = line.indexOf(WIKI_OPEN_METADATA, s)) >= 0 && (q = line.indexOf(WIKI_CLOSE_METADATA, p + 1)) >= 0) {
@ -1041,7 +1046,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
// {{Coordinate |NS 45/37/43.0/N |EW. 07/58/41.0/E |type=landmark |region=IT-BI}} ## means: degree/minute/second
// {{Coordinate |NS 51.48994 |EW. 7.33249 |type=landmark |region=DE-NW}}
final String b[] = a.split("\\|");
float lon = 0.0f, lat = 0.0f;
float lon = Float.NaN, lat = Float.NaN;
float lonm = 0.0f, latm = 0.0f;
String lono = "E", lato = "N";
String name = "";
@ -1053,18 +1058,18 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
final String d[] = c.substring(3).split("/");
if (d.length == 1) {float l = Float.parseFloat(d[0]); if (l < 0) {lato = "S"; l = -l;} lat = (float) Math.floor(l); latm = 60.0f * (l - lat);}
else if (d.length == 2) {lat = Float.parseFloat(d[0]); latm = Float.parseFloat(d[1]);}
else if (d.length == 3) {lat = Float.parseFloat(d[0]); latm = Float.parseFloat(d[1]) + Float.parseFloat(d[2]) / 60.0f;}
if (d[d.length-1].toUpperCase().equals("S")) {}
else if (d.length >= 3) {lat = Float.parseFloat(d[0]); latm = Float.parseFloat(d[1]) + Float.parseFloat(d[2]) / 60.0f;}
if (d[d.length-1].toUpperCase().equals("S")) {lato = "S";}
}
if (c.toUpperCase().startsWith("EW=")) {
final String d[] = c.substring(3).split("/");
if (d.length == 1) {float l = Float.parseFloat(d[0]); if (l < 0) {lono = "W"; l = -l;} lon = (float) Math.floor(l); lonm = 60.0f * (l - lon);}
else if (d.length == 2) {lon = Float.parseFloat(d[0]); lonm = Float.parseFloat(d[1]);}
else if (d.length == 3) {lon = Float.parseFloat(d[0]); lonm = Float.parseFloat(d[1]) + Float.parseFloat(d[2]) / 60.0f;}
if (d[d.length-1].toUpperCase().equals("W")) {lon = -lon; lonm = -lonm;}
else if (d.length >= 3) {lon = Float.parseFloat(d[0]); lonm = Float.parseFloat(d[1]) + Float.parseFloat(d[2]) / 60.0f;}
if (d[d.length-1].toUpperCase().equals("W")) {lato = "W";}
}
}
if (lon != 0.0d && lat != 0.0d) {
if (!Float.isNaN(lon) && !Float.isNaN(lat)) {
// replace this with a format that the html parser can understand
line = line.substring(0, p) + (name.length() > 0 ? (" " + name) : "") + " <nobr> " + lato + " " + lat + "\u00B0 " + latm + "'</nobr><nobr>" + lono + " " + lon + "\u00B0 " + lonm + "'</nobr> " + line.substring(q + WIKI_CLOSE_METADATA.length());
s = p;

View File

@ -239,8 +239,8 @@ public final class Tray {
}
private String readyMessage() {
if (deutsch) return "YaCy laeuft unter http://localhost:" + sb.getLocalPort("port", 8090);
return "YaCy is running at http://localhost:" + sb.getLocalPort("port", 8090);
if (deutsch) return "YaCy laeuft unter http://localhost:" + sb.getLocalPort();
return "YaCy is running at http://localhost:" + sb.getLocalPort();
}
private String shutdownMessage() {

View File

@ -83,7 +83,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
final SSLContext sslContext = initSslContext(sb);
if (sslContext != null) {
int sslport = sb.getLocalPort("port.ssl", 8443);
int sslport = sb.getConfigInt("port.ssl", 8443);
sslContextFactory.setSslContext(sslContext);
// SSL HTTP Configuration
@ -289,8 +289,8 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
}
try { // reconnect with new settings (instead to stop/start server, just manipulate connectors
final Connector[] cons = server.getConnectors();
final int port = Switchboard.getSwitchboard().getLocalPort("port", 8090);
final int sslport = Switchboard.getSwitchboard().getLocalPort("port.ssl", 8443);
final int port = Switchboard.getSwitchboard().getLocalPort();
final int sslport = Switchboard.getSwitchboard().getConfigInt("port.ssl", 8443);
for (Connector con : cons) {
// check http connector
if (con.getName().startsWith("httpd") && ((ServerConnector)con).getPort() != port) {

View File

@ -959,7 +959,7 @@ public final class SeedDB implements AlternativeDomainNames {
if (this.mySeed == null) initMySeed();
if (seed == this.mySeed && !(seed.isOnline())) {
// take local ip instead of external
return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getLocalPort("port", 8090) + ((subdom == null) ? "" : ("/" + subdom));
return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getLocalPort() + ((subdom == null) ? "" : ("/" + subdom));
}
return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom));
} else {

View File

@ -304,7 +304,7 @@ public final class Switchboard extends serverSwitch {
super(dataPath, appPath, initPath, configPath);
sb = this;
// check if port is already occupied
final int port = getLocalPort("port", 8090);
final int port = getLocalPort();
if (TimeoutRequest.ping(Domains.LOCALHOST, port, 500)) {
throw new RuntimeException(
"a server is already running on the YaCy port "
@ -2200,7 +2200,7 @@ public final class Switchboard extends serverSwitch {
startupAction = false;
// execute api calls
final Map<String, Integer> callResult = this.tables.execAPICalls("localhost", getLocalPort("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
final Map<String, Integer> callResult = this.tables.execAPICalls("localhost", getLocalPort(), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
for ( final Map.Entry<String, Integer> call : callResult.entrySet() ) {
this.log.info("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey());
}

View File

@ -649,7 +649,7 @@ public final class Fulltext {
// format: 0=text, 1=html, 2=rss/xml
this.f = f;
this.pattern = filter == null ? null : Pattern.compile(filter);
this.query = query == null? "*:*" : query;
this.query = query == null? AbstractSolrConnector.CATCHALL_QUERY : query;
this.count = 0;
this.failure = null;
this.format = format;

View File

@ -330,34 +330,58 @@ public class QueryGoal {
for (final byte[] b: blues) this.include_hashes.remove(b);
}
/**
* Generate a Solr filter query to receive valid urls
*
* This filters out error-urls.
* On noimages=true a filter is added to exclude links to images
* using the content_type (as well as urls with common image file extension)
*
* @param noimages true if filter for images should be included
* @return Solr filter query
*/
public List<String> collectionTextFilterQuery(boolean noimages) {
final ArrayList<String> fqs = new ArrayList<>();
// add filter to prevent that results come from failed urls
fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200");
if (noimages) fqs.add("-" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(jpg OR png OR gif)");
if (noimages) {
fqs.add("-" + CollectionSchema.content_type.getSolrFieldName() + ":(image/*)");
fqs.add("-" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(jpg OR png OR gif)");
}
return fqs;
}
public StringBuilder collectionTextQuery() {
// parse special requests
if (isCatchall()) return new StringBuilder("*:*");
if (isCatchall()) return new StringBuilder(AbstractSolrConnector.CATCHALL_QUERY);
// add goal query
return getGoalQuery();
}
/**
* Generate a Solr filter query to receive valid image results.
*
* This filters error-urls out and includes urls with mime image/* as well
* as urls with links to images.
* We use the mime (image/*) only to find images as the parser assigned the
* best mime to index documents. This applies also to parsed file systems.
* This ensures that no text urls with image-fileextension is returned
* (as some large internet sites like to use such urls)
*
* @return Solr filter query for image urls
*/
public List<String> collectionImageFilterQuery() {
final ArrayList<String> fqs = new ArrayList<>();
// add filter to prevent that results come from failed urls
fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200");
fqs.add(
CollectionSchema.images_urlstub_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM + " OR " +
CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(jpg OR png OR gif) OR " +
CollectionSchema.content_type.getSolrFieldName() + ":(image/*)");
CollectionSchema.content_type.getSolrFieldName() + ":(image/*) OR " +
CollectionSchema.images_urlstub_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);
return fqs;
}
@ -365,7 +389,7 @@ public class QueryGoal {
final StringBuilder q = new StringBuilder(80);
// parse special requests
if (isCatchall()) return new StringBuilder("*:*");
if (isCatchall()) return new StringBuilder(AbstractSolrConnector.CATCHALL_QUERY);
// add goal query
StringBuilder w = getGoalQuery();

View File

@ -419,7 +419,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
final DigestURL digestURL = document.dc_source();
boolean allAttr = this.isEmpty();
String url = addURIAttributes(doc, allAttr, digestURL);
if (allAttr || contains(CollectionSchema.content_type)) add(doc, CollectionSchema.content_type, new String[]{document.dc_format()});
add(doc, CollectionSchema.content_type, new String[]{document.dc_format()}); // content_type (mime) is defined a schema field and we rely on it in some queries like imagequery (makes it mandatory, no need to check)
Set<ProcessType> processTypes = new LinkedHashSet<ProcessType>();
String host = digestURL.getHost();
@ -2028,9 +2028,8 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
final SolrInputDocument doc = new SolrInputDocument();
String url = configuration.addURIAttributes(doc, allAttr, this.getDigestURL());
if (allAttr || configuration.contains(CollectionSchema.content_type)) configuration.add(doc, CollectionSchema.content_type, new String[]{Classification.url2mime(this.digestURL)});
// content_type (mime) is defined a schema field and we rely on it in some queries like imagequery (makes it mandatory, no need to check)
CollectionSchema.content_type.add(doc, new String[]{Classification.url2mime(this.digestURL)});
if (allAttr || configuration.contains(CollectionSchema.load_date_dt)) configuration.add(doc, CollectionSchema.load_date_dt, getFailDate());
if (allAttr || configuration.contains(CollectionSchema.crawldepth_i)) configuration.add(doc, CollectionSchema.crawldepth_i, this.crawldepth);

View File

@ -222,16 +222,14 @@ public class serverSwitch {
* Wrapper for {@link #getConfigInt(String, int)} to have a more consistent
* API.
*
* @param key
* original key from config (for example "port" or "port.ssl")
* @param dflt
* default value which will be used if no value is found
* Default value 8090 will be used if no value is found
*
* @return the local port of this system
* @see #getPublicPort(String, int)
*/
public int getLocalPort(final String key, final int dflt) {
public int getLocalPort() {
return getConfigInt(key, dflt);
return getConfigInt("port", 8090);
}
// a logger for this switchboard

View File

@ -218,7 +218,7 @@ public final class yacy {
sb.setConfig("memoryTotalAfterStartup", startupMemTotal);
// start gui if wanted
if (gui) YaCyApp.start("localhost", sb.getLocalPort("port", 8090));
if (gui) YaCyApp.start("localhost", sb.getLocalPort());
// hardcoded, forced, temporary value-migration
sb.setConfig("htTemplatePath", "htroot/env/templates");
@ -293,7 +293,7 @@ public final class yacy {
HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
// start main threads
final int port = sb.getLocalPort("port", 8090);
final int port = sb.getLocalPort();
try {
// start http server
YaCyHttpServer httpServer;

View File

@ -0,0 +1,35 @@
package net.yacy.data.wiki;
import org.junit.Test;
import static org.junit.Assert.*;
public class WikiCodeTest {
/**
* test geo location metadata convert
*/
@Test
public void testProcessMetadata() {
String[] testmeta = new String[]{
"{{coordinate|NS=52.205944|EW=0.117593|region=GB-CAM|type=landmark}}", // decimal N-E location
"{{coordinate|NS=43/50/29/N|EW=73/23/17/W|type=landmark|region=US-NY}}", // N-W location
"{{Coordinate |text=DMS |NS=50/7/49/N |EW=6/8/09/E |type=landmark |region=BE-WLG |name=Monument des trois Frontières}}",
"{{Coordinate |text=DMS |NS= 49.047169|EW=7.899148|region=DE-RP |type=landmark |name=Europadenkmal (Rheinland-Pfalz)}}",
"{{coordinate|NS=0.00000|EW=0.117593}}", // testing equator coord
"{{coordinate|NS=-10.00000|EW=-10.10000}}" // testing S-E location
};
WikiCode wc = new WikiCode();
for (int i = 0; i < testmeta.length; i++) {
String result = wc.transform("http://wiki:8080",testmeta[i]);
System.out.println(testmeta[i] + " --> " + result);
// simply check if replacement took place, if no coordinate recognized original string is just html encoded
assertFalse(result.contains("#124;")); // simple check - result not containing char code for "{",
assertFalse(result.contains("#125;")); // simple check - result not containing char code for "}"
}
}
}