mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
replaced http links with https
Some checks failed
CI Script to build on self-hosted server / build (push) Has been cancelled
Some checks failed
CI Script to build on self-hosted server / build (push) Has been cancelled
This commit is contained in:
parent
fd45ccf76e
commit
910a496c9f
|
@ -12,7 +12,7 @@
|
|||
# ==========================================
|
||||
# This Software is Copyrighted
|
||||
# (C) by Michael Peter Christen; mc@yacy.net
|
||||
# first published on http://yacy.net
|
||||
# first published on https://yacy.net
|
||||
# Frankfurt, Germany, 2005-2009
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
@ -2,6 +2,6 @@
|
|||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
||||
<properties>
|
||||
<comment>Sources for OAI ListFriends entries. The Key is the URL to the ListFriends Source, the Value is the Path to the local file where the ListFriends file is stored</comment>
|
||||
<entry key="http://roar.eprints.org/cgi/roar_search/advanced/export_roar_ROAR::ListFriends.xml?_action_export=1&output=ROAR%3A%3AListFriends">DATA/DICTIONARIES/harvesting/export_roar_ROAR_ListFriends.xml</entry>
|
||||
<entry key="http://www.openarchives.org/Register/ListFriends">DATA/DICTIONARIES/harvesting/ListFriends.xml</entry>
|
||||
<entry key="https://roar.eprints.org/cgi/roar_search/advanced/export_roar_ROAR::ListFriends.xml?_action_export=1&output=ROAR%3A%3AListFriends">DATA/DICTIONARIES/harvesting/export_roar_ROAR_ListFriends.xml</entry>
|
||||
<entry key="https://www.openarchives.org/Register/ListFriends">DATA/DICTIONARIES/harvesting/ListFriends.xml</entry>
|
||||
</properties>
|
|
@ -597,7 +597,7 @@ proxyIndexingLocalMedia=true
|
|||
# proxy usage only for .yacy-Domains for autoconfig
|
||||
proxyYacyOnly=false
|
||||
|
||||
# enable proxy via url (/proxy.html?url=http://yacy.net)
|
||||
# enable proxy via url (/proxy.html?url=https://yacy.net)
|
||||
proxyURL=false
|
||||
proxyURL.access=127.0.0.1,0:0:0:0:0:0:0:1
|
||||
# which urls to rewrite to /proxy.html?url=x (values: all, domainlist)
|
||||
|
@ -1288,8 +1288,8 @@ content.phpbb3.dumpfile =
|
|||
|
||||
# search engine teaser: an about box in search results
|
||||
# this is only shown, if the about.body is filled
|
||||
about.headline=
|
||||
about.body=
|
||||
about.headline=Please Help Us
|
||||
about.body=<iframe src="/env/donate.html" style="border:none;"></iframe>
|
||||
|
||||
donation.iframesource=https://yacy.net/include/donate.html
|
||||
donation.iframetarget=env/donate.html
|
||||
|
@ -1405,7 +1405,7 @@ postprocessing.partialUpdate = true
|
|||
# To use this user agent option, you must define completely different names and strings
|
||||
# and remove the given example here, which will be ignored by default.
|
||||
crawler.userAgent.name = yacybot
|
||||
crawler.userAgent.string = yacybot ($$SYSTEM$$) http://yacy.net/bot.html
|
||||
crawler.userAgent.string = yacybot ($$SYSTEM$$) https://yacy.net/bot.html
|
||||
crawler.userAgent.minimumdelta = 500
|
||||
crawler.userAgent.clienttimeout = 10000
|
||||
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
* YaCySearchClient
|
||||
* an interface for Adaptive Replacement Caches
|
||||
* Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
|
||||
* First released 20.09.2010 at http://yacy.net
|
||||
* First released 20.09.2010 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
|
|
@ -6,7 +6,7 @@ this is a xml file with embedded dublin core properties
|
|||
you can validate it with http://www.stg.brown.edu/service/xmlvalid/
|
||||
-->
|
||||
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:yacy="http://yacy.net/" xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:yacy="https://yacy.net/" xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#">
|
||||
<record>
|
||||
<dc:title LANG="#[dc_language]#">#[dc_title]#</dc:title>
|
||||
<dc:creator>#[dc_creator]#</dc:creator>
|
||||
|
|
41
htroot/env/grafics/donate.html
vendored
Normal file
41
htroot/env/grafics/donate.html
vendored
Normal file
|
@ -0,0 +1,41 @@
|
|||
<html>
|
||||
<head></head>
|
||||
<body>
|
||||
<table border="0" cellpadding="0" cellspacing="0">
|
||||
|
||||
Please support our work on YaCy!
|
||||
|
||||
<!-- Patreon -->
|
||||
<tr height="24">
|
||||
<td valign="top"></td>
|
||||
<td valign="top"><a href="https://www.patreon.com/orbiterlab" target="_blank"><img src="/env/grafics/become_a_patron_button.png" width="162" height="38"></a></td>
|
||||
</tr>
|
||||
|
||||
<!-- Github Sponsors -->
|
||||
<tr height="32">
|
||||
<td valign="top"><a href="https://github.com/users/Orbiter/sponsorship" target="_blank"><img src="/env/grafics/logo-mona-100.png" width="32" height="32"></a></a></td>
|
||||
<td valign="top"><a href="https://github.com/users/Orbiter/sponsorship" target="_blank">Github Sponsors</a></td>
|
||||
</tr>
|
||||
|
||||
<!-- Paypal -->
|
||||
<tr height="40">
|
||||
<td valign="top"><img src="/env/grafics/paypal_logo.gif" width="85" height="23"></td>
|
||||
<td valign="top"><form target="_parent" action="https://www.paypal.com/cgi-bin/webscr" method="post" style="margin:0px; padding:0px;">
|
||||
<input type="hidden" name="cmd" value="_s-xclick">
|
||||
<input type="hidden" name="on0" value="Your contribution is received as a taxable, legal payment.">
|
||||
<select name="os0">
|
||||
<option value="beneficial">beneficial: 5 €</option>
|
||||
<option value="generous">generous: 25 €</option>
|
||||
<option value="gracious">gracious: 50 €</option>
|
||||
</select>
|
||||
<input type="hidden" name="currency_code" value="EUR">
|
||||
<input type="hidden" name="encrypted" value="-----BEGIN PKCS7-----MIIICQYJKoZIhvcNAQcEoIIH+jCCB/YCAQExggEwMIIBLAIBADCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwDQYJKoZIhvcNAQEBBQAEgYBdIJiffS8lpYJW7VZL45wo42eipolZjzta5yacL4KCWGpmy6zMPwp6KK+BZXXB2rgx3V16svK7WPr5LQfmfz1JqCFxJVSNrl+uLSrO3YE1OyF+XAwm2WfREKCQuL23ES3XjiR1JaTPMx6iJz0n2GeIzIHoVPXUqPJlnLbtvj4yhjELMAkGBSsOAwIaBQAwggGFBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECPQWs3gSmK/IgIIBYJh6P4ewR1+6WfPjFANryc+X8VIUm1aNfznDErqSQj2m+VQWa7leRgHKEqVoFWdOrJa7iTYsv7rGYJPyFSdDwWdMy2r4sMii96bginGuBNfEFVG/ub4vH3vu8N/xZUrBOFJp+kZfKvwjTf/2gu/6EWbcKmHCDF1uVSeGX7upMIKoWriENzLZPWVcA5MrL8YjvdmbC86KfLyf37KLLA6e9Gj45mZs+nGjyzQMpykatCAv2UU77Q/SBGeDhfFxHaxtdM70/oCYRJwiRmwxw1+l7LUtD3h0NSbN7xLu03iYwpHtCGVk8ZNz7YhML0wG63UgU0C6SA8zho5ea48dPhHftrKiPQaEAY/nSwOGu9O7f1bal40+EA/ZYiUF+I9ltD0Xc1Hoaj9xgy/p5+puEjxPReys7DFMyPmT0w8sQcWH8fi6JyCP7L5MSA8U7hRVV9h/C+50f5/u362B2wbXmsk+M3SgggOHMIIDgzCCAuygAwIBAgIBADANBgkqhkiG9w0BAQUFADCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20wHhcNMDQwMjEzMTAxMzE1WhcNMzUwMjEzMTAxMzE1WjCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMFHTt38RMxLXJyO2SmS+Ndl72T7oKJ4u4uw+6awntALWh03PewmIJuzbALScsTS4sZoS1fKciBGoh11gIfHzylvkdNe/hJl66/RGqrj5rFb08sAABNTzDTiqqNpJeBsYs/c2aiGozptX2RlnBktH+SUNpAajW724Nv2Wvhif6sFAgMBAAGjge4wgeswHQYDVR0OBBYEFJaffLvGbxe9WT9S1wob7BDWZJRrMIG7BgNVHSMEgbMwgbCAFJaffLvGbxe9WT9S1wob7BDWZJRroYGUpIGRMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbYIBADAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAIFfOlaagFrl71+jq6OKidbWFSE+Q4FqROvdgIONth+8kSK//Y/4ihuE4Ymvzn5ceE3S/iBSQQMjyvb+s2TWbQYDwcp129OPIbD9epdr4tJOUNiSojw7BHwYRiPh58S1xGlFgHFXwrEBb3dgNbMUa+u4qectsMAXpVHnD9wIyfmHMYIBmjCCAZYCAQEwgZQwgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEUMBIGA1UEChMLUGF5UGFsIEluYy4xEzARBgNVBAsUCmxpdmVfY2VydHMxETAPBgNVBAMUCGxpdmVfYXBpMRwwGgYJKoZIhvcNAQkBFg1yZUBwYXlwYWwuY29tAgEAMAkGBSsOAwIaBQCgXTAYBgkqhkiG9w0BCQMxCwYJKoZIhvcNAQcBMBwGCSqGSIb3DQEJBTEPFw0xMzAzMTExMzQ3MzZaMCMGCSqGSIb3DQEJBDEWBBTzsQu2wDz+GywI1AFeCT2Gn9SN+TANBgkqhkiG9w0BAQEFAASBgHXTveleRZAuT5Upn3yzP7mI1aKx2LeWPINN3aZ0IFLj7rzrT26YZsQNfdYPHtLjuahN9bhvRjQjvMQQzManec3SoCEpg6MDcwqB4TJY+kZeUKO02gVWFHM3m4dCss7WoXuhlcnQese4ZpQ63rCRMXAhXk7xYHpyghlNpFmJgl0W-----END PKCS7-----
|
||||
">
|
||||
<input type="submit" name="submit" value="Donate!" style:"float:left;">
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
|
@ -1,6 +1,6 @@
|
|||
/*
|
||||
* Copyright (C) 2005 - 2014 Alexander Schier, Michael Peter Christen,
|
||||
* and other YaCy developers (see http://yacy.net/en/Join.html)
|
||||
* and other YaCy developers (see https://yacy.net/en/Join.html)
|
||||
*
|
||||
* This file is part of YaCy.
|
||||
*
|
||||
|
@ -15,7 +15,7 @@
|
|||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with YaCy. If not, see <http://www.gnu.org/licenses/>.
|
||||
* along with YaCy. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
DELETE_STRING="delete";
|
||||
|
|
|
@ -50,7 +50,7 @@ $(document).ready(function() {
|
|||
theme : 'start',
|
||||
title : 'YaCy Search Widget',
|
||||
logo : yconf.url + '/yacy/ui/img/yacy-logo.png',
|
||||
link : 'http://yacy.net',
|
||||
link : 'https://yacy.net',
|
||||
width : 640,
|
||||
height : 640,
|
||||
position : [150,50],
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:yacy="http://www.yacy.net/"
|
||||
>
|
||||
<!-- YaCy Search Engine; http://yacy.net -->
|
||||
<!-- YaCy Search Engine; https://yacy.net -->
|
||||
|
||||
<title>#[promoteSearchPageGreeting]#: #[rss_query]#</title>
|
||||
<subtitle>Search for #[rss_query]#</subtitle>
|
||||
|
|
|
@ -9,7 +9,7 @@
|
|||
xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#"
|
||||
xmlns:ev="http://purl.org/rss/1.0/modules/event/"
|
||||
>
|
||||
<!-- YaCy Search Engine; http://yacy.net -->
|
||||
<!-- YaCy Search Engine; https://yacy.net -->
|
||||
<channel>
|
||||
<title>#[promoteSearchPageGreeting]#: #[rss_query]#</title>
|
||||
<description>Search for #[rss_query]#</description>
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
xmlns:yacy="http://www.yacy.net/"
|
||||
xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#"
|
||||
>
|
||||
<!-- YaCy Search Engine; http://yacy.net -->
|
||||
<!-- YaCy Search Engine; https://yacy.net -->
|
||||
<channel>
|
||||
<title>#[promoteSearchPageGreeting]#</title>
|
||||
<description>Search for #[rss_query]#</description>
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* OllamaClient
|
||||
* Copyright 2024 by Michael Peter Christen
|
||||
* First released 17.05.2024 at http://yacy.net
|
||||
* First released 17.05.2024 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -35,33 +35,33 @@ import org.json.JSONObject;
|
|||
public class OllamaClient {
|
||||
|
||||
public static String OLLAMA_API_HOST = "http://localhost:11434";
|
||||
|
||||
private String hoststub;
|
||||
|
||||
public OllamaClient(String hoststub) {
|
||||
|
||||
private final String hoststub;
|
||||
|
||||
public OllamaClient(final String hoststub) {
|
||||
this.hoststub = hoststub;
|
||||
}
|
||||
|
||||
public LinkedHashMap<String, Long> listOllamaModels() {
|
||||
LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();
|
||||
final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();
|
||||
try {
|
||||
String response = OpenAIClient.sendGetRequest(this.hoststub + "/api/tags");
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
JSONArray models = responseObject.getJSONArray("models");
|
||||
|
||||
List<Map.Entry<String, Long>> list = new ArrayList<>();
|
||||
final String response = OpenAIClient.sendGetRequest(this.hoststub + "/api/tags");
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final JSONArray models = responseObject.getJSONArray("models");
|
||||
|
||||
final List<Map.Entry<String, Long>> list = new ArrayList<>();
|
||||
for (int i = 0; i < models.length(); i++) {
|
||||
JSONObject model = models.getJSONObject(i);
|
||||
String name = model.optString("name", "");
|
||||
long size = model.optLong("size", 0);
|
||||
list.add(new AbstractMap.SimpleEntry<String, Long>(name, size));
|
||||
final JSONObject model = models.getJSONObject(i);
|
||||
final String name = model.optString("name", "");
|
||||
final long size = model.optLong("size", 0);
|
||||
list.add(new AbstractMap.SimpleEntry<>(name, size));
|
||||
}
|
||||
|
||||
// Sort the list in descending order based on the values
|
||||
list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));
|
||||
|
||||
|
||||
// Create a new LinkedHashMap and add the sorted entries
|
||||
for (Map.Entry<String, Long> entry : list) {
|
||||
for (final Map.Entry<String, Long> entry : list) {
|
||||
sortedMap.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
|
@ -70,8 +70,8 @@ public class OllamaClient {
|
|||
return sortedMap;
|
||||
}
|
||||
|
||||
public boolean ollamaModelExists(String name) {
|
||||
JSONObject data = new JSONObject();
|
||||
public boolean ollamaModelExists(final String name) {
|
||||
final JSONObject data = new JSONObject();
|
||||
try {
|
||||
data.put("name", name);
|
||||
OpenAIClient.sendPostRequest(this.hoststub + "/api/show", data);
|
||||
|
@ -81,37 +81,37 @@ public class OllamaClient {
|
|||
}
|
||||
}
|
||||
|
||||
public boolean pullOllamaModel(String name) {
|
||||
JSONObject data = new JSONObject();
|
||||
public boolean pullOllamaModel(final String name) {
|
||||
final JSONObject data = new JSONObject();
|
||||
try {
|
||||
data.put("name", name);
|
||||
data.put("stream", false);
|
||||
String response = OpenAIClient.sendPostRequest(this.hoststub + "/api/pull", data);
|
||||
final String response = OpenAIClient.sendPostRequest(this.hoststub + "/api/pull", data);
|
||||
// this sends {"status": "success"} in case of success
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
String status = responseObject.optString("status", "");
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final String status = responseObject.optString("status", "");
|
||||
return status.equals("success");
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
OllamaClient oc = new OllamaClient(OLLAMA_API_HOST);
|
||||
|
||||
LinkedHashMap<String, Long> models = oc.listOllamaModels();
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final OllamaClient oc = new OllamaClient(OLLAMA_API_HOST);
|
||||
|
||||
final LinkedHashMap<String, Long> models = oc.listOllamaModels();
|
||||
System.out.println(models.toString());
|
||||
|
||||
// check if model exists
|
||||
String model = "phi3:3.8b";
|
||||
final String model = "phi3:3.8b";
|
||||
if (oc.ollamaModelExists(model))
|
||||
System.out.println("model " + model + " exists");
|
||||
else
|
||||
System.out.println("model " + model + " does not exist");
|
||||
|
||||
|
||||
// pull a model
|
||||
boolean success = oc.pullOllamaModel(model);
|
||||
final boolean success = oc.pullOllamaModel(model);
|
||||
System.out.println("pulled model + " + model + ": " + success);
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* OpenAIClient
|
||||
* Copyright 2024 by Michael Peter Christen
|
||||
* First released 17.05.2024 at http://yacy.net
|
||||
* First released 17.05.2024 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -38,31 +38,31 @@ public class OpenAIClient {
|
|||
|
||||
private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
|
||||
|
||||
private String hoststub;
|
||||
|
||||
public OpenAIClient(String hoststub) {
|
||||
private final String hoststub;
|
||||
|
||||
public OpenAIClient(final String hoststub) {
|
||||
this.hoststub = hoststub;
|
||||
}
|
||||
|
||||
|
||||
// API Helper Methods
|
||||
|
||||
public static String sendPostRequest(String endpoint, JSONObject data) throws IOException, URISyntaxException {
|
||||
URL url = new URI(endpoint).toURL();
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
|
||||
public static String sendPostRequest(final String endpoint, final JSONObject data) throws IOException, URISyntaxException {
|
||||
final URL url = new URI(endpoint).toURL();
|
||||
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setRequestProperty("Content-Type", "application/json");
|
||||
conn.setDoOutput(true);
|
||||
|
||||
try (OutputStream os = conn.getOutputStream()) {
|
||||
byte[] input = data.toString().getBytes("utf-8");
|
||||
final byte[] input = data.toString().getBytes("utf-8");
|
||||
os.write(input, 0, input.length);
|
||||
}
|
||||
|
||||
int responseCode = conn.getResponseCode();
|
||||
final int responseCode = conn.getResponseCode();
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
|
||||
StringBuilder response = new StringBuilder();
|
||||
final StringBuilder response = new StringBuilder();
|
||||
String responseLine;
|
||||
while ((responseLine = br.readLine()) != null) {
|
||||
response.append(responseLine.trim());
|
||||
|
@ -74,15 +74,15 @@ public class OpenAIClient {
|
|||
}
|
||||
}
|
||||
|
||||
public static String sendGetRequest(String endpoint) throws IOException, URISyntaxException {
|
||||
URL url = new URI(endpoint).toURL();
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
public static String sendGetRequest(final String endpoint) throws IOException, URISyntaxException {
|
||||
final URL url = new URI(endpoint).toURL();
|
||||
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("GET");
|
||||
|
||||
int responseCode = conn.getResponseCode();
|
||||
final int responseCode = conn.getResponseCode();
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
|
||||
StringBuilder response = new StringBuilder();
|
||||
final StringBuilder response = new StringBuilder();
|
||||
String responseLine;
|
||||
while ((responseLine = br.readLine()) != null) {
|
||||
response.append(responseLine.trim());
|
||||
|
@ -93,14 +93,14 @@ public class OpenAIClient {
|
|||
throw new IOException("Request failed with response code " + responseCode);
|
||||
}
|
||||
}
|
||||
|
||||
// OpenAI chat client, works with llama.cpp and Ollama
|
||||
|
||||
public String chat(String model, String prompt, int max_tokens) throws IOException {
|
||||
JSONObject data = new JSONObject();
|
||||
JSONArray messages = new JSONArray();
|
||||
JSONObject systemPrompt = new JSONObject(true);
|
||||
JSONObject userPrompt = new JSONObject(true);
|
||||
|
||||
// OpenAI chat client, works with llama.cpp and Ollama
|
||||
|
||||
public String chat(final String model, final String prompt, final int max_tokens) throws IOException {
|
||||
final JSONObject data = new JSONObject();
|
||||
final JSONArray messages = new JSONArray();
|
||||
final JSONObject systemPrompt = new JSONObject(true);
|
||||
final JSONObject userPrompt = new JSONObject(true);
|
||||
messages.put(systemPrompt);
|
||||
messages.put(userPrompt);
|
||||
try {
|
||||
|
@ -114,52 +114,52 @@ public class OpenAIClient {
|
|||
data.put("messages", messages);
|
||||
data.put("stop", new JSONArray(STOPTOKENS));
|
||||
data.put("stream", false);
|
||||
String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
JSONArray choices = responseObject.getJSONArray("choices");
|
||||
JSONObject choice = choices.getJSONObject(0);
|
||||
JSONObject message = choice.getJSONObject("message");
|
||||
String content = message.optString("content", "");
|
||||
final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final JSONArray choices = responseObject.getJSONArray("choices");
|
||||
final JSONObject choice = choices.getJSONObject(0);
|
||||
final JSONObject message = choice.getJSONObject("message");
|
||||
final String content = message.optString("content", "");
|
||||
return content;
|
||||
} catch (JSONException | URISyntaxException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public static String[] stringsFromChat(String answer) {
|
||||
int p = answer.indexOf('[');
|
||||
int q = answer.indexOf(']');
|
||||
public static String[] stringsFromChat(final String answer) {
|
||||
final int p = answer.indexOf('[');
|
||||
final int q = answer.indexOf(']');
|
||||
if (p < 0 || q < 0 || q < p) return new String[0];
|
||||
try {
|
||||
JSONArray a = new JSONArray(answer.substring(p, q + 1));
|
||||
String[] arr = new String[a.length()];
|
||||
final JSONArray a = new JSONArray(answer.substring(p, q + 1));
|
||||
final String[] arr = new String[a.length()];
|
||||
for (int i = 0; i < a.length(); i++) arr[i] = a.getString(i);
|
||||
return arr;
|
||||
} catch (JSONException e) {
|
||||
} catch (final JSONException e) {
|
||||
return new String[0];
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
String model = "phi3:3.8b";
|
||||
OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST);
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final String model = "phi3:3.8b";
|
||||
final OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST);
|
||||
// make chat completion with model
|
||||
String question = "Who invented the wheel?";
|
||||
try {
|
||||
String answer = oaic.chat(model, question, 80);
|
||||
final String answer = oaic.chat(model, question, 80);
|
||||
System.out.println(answer);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
// try the json parser from chat results
|
||||
question = "Make a list of four names from Star Wars movies. Use a JSON Array.";
|
||||
try {
|
||||
String[] a = stringsFromChat(oaic.chat(model, question, 80));
|
||||
for (String s: a) System.out.println(s);
|
||||
} catch (IOException e) {
|
||||
final String[] a = stringsFromChat(oaic.chat(model, question, 80));
|
||||
for (final String s: a) System.out.println(s);
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AbstractFormatter
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 2.1.2011 at http://yacy.net
|
||||
* First released 2.1.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -11,12 +11,12 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -62,7 +62,7 @@ public abstract class AbstractFormatter implements DateFormatter {
|
|||
@Override
|
||||
public abstract String format();
|
||||
|
||||
private static final HashMap<Pattern, SimpleDateFormat> DATE_FORMAT_REGEXPS = new HashMap<Pattern, SimpleDateFormat>() {
|
||||
private static final HashMap<Pattern, SimpleDateFormat> DATE_FORMAT_REGEXPS = new HashMap<>() {
|
||||
private static final long serialVersionUID = 1321140786174228717L;
|
||||
{
|
||||
put(Pattern.compile("^\\d{8}$"), new SimpleDateFormat("yyyyMMdd", Locale.US));
|
||||
|
@ -92,16 +92,16 @@ public abstract class AbstractFormatter implements DateFormatter {
|
|||
}};
|
||||
|
||||
@Override
|
||||
public Date parse(String s) {
|
||||
public Date parse(final String s) {
|
||||
return parseAny(s);
|
||||
}
|
||||
|
||||
public static Date parseAny(String s) {
|
||||
for (Map.Entry<Pattern, SimpleDateFormat> ps: DATE_FORMAT_REGEXPS.entrySet()) {
|
||||
public static Date parseAny(final String s) {
|
||||
for (final Map.Entry<Pattern, SimpleDateFormat> ps: DATE_FORMAT_REGEXPS.entrySet()) {
|
||||
if (ps.getKey().matcher(s.toLowerCase()).matches()) {
|
||||
try {
|
||||
return ps.getValue().parse(s);
|
||||
} catch (ParseException e) {
|
||||
} catch (final ParseException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* DateFormatter
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 2.1.2011 at http://yacy.net
|
||||
* First released 2.1.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -11,12 +11,12 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -34,5 +34,5 @@ public interface DateFormatter {
|
|||
public Calendar parse(String s, int timezoneOffset) throws ParseException;
|
||||
public String format(final Date date);
|
||||
public String format();
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* GenericFormatter
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 2.1.2011 at http://yacy.net
|
||||
* First released 2.1.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -50,7 +50,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
public static final String PATTERN_RFC1123_SHORT = "EEE, dd MMM yyyy";
|
||||
public static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy";
|
||||
public static final String PATTERN_SIMPLE = "yyyy/MM/dd HH:mm:ss";
|
||||
|
||||
|
||||
/**
|
||||
* A regular expression matching the PATTERN_SIMPLE pattern (does not control
|
||||
* last day of month (30/31 or 28/29 for february). Can be used as a HTML5 input
|
||||
|
@ -74,7 +74,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
public static final DateTimeFormatter FORMAT_SHORT_MINUTE = DateTimeFormatter
|
||||
.ofPattern(PATTERN_SHORT_MINUTE.replace("yyyy", "uuuu"), Locale.US).withZone(ZoneId.systemDefault());
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* A thread-safe date formatter using the
|
||||
* {@link GenericFormatter#PATTERN_SHORT_SECOND} pattern with the US locale on
|
||||
|
@ -257,17 +257,17 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
@Override
|
||||
public Calendar parse(final String timeString, final int timezoneOffset) throws ParseException {
|
||||
synchronized (this.dateFormat) {
|
||||
Calendar cal = Calendar.getInstance(UTCtimeZone);
|
||||
final Calendar cal = Calendar.getInstance(UTCtimeZone);
|
||||
cal.setTime(this.dateFormat.parse(timeString));
|
||||
cal.add(Calendar.MINUTE, timezoneOffset); // add a correction; i.e. for UTC+1 -60 minutes is added to patch a time given in UTC+1 to the actual time at UTC
|
||||
return cal;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Like {@link #parseShortSecond(String)} using additional timezone information provided in an
|
||||
* offset String, like "+0100" for CET.
|
||||
* @throws ParseException
|
||||
* @throws ParseException
|
||||
*/
|
||||
public Calendar parse(final String timeString, final String UTCOffset) throws ParseException {
|
||||
if (timeString == null || timeString.isEmpty()) { return Calendar.getInstance(UTCtimeZone); }
|
||||
|
@ -292,7 +292,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
final int om = NumberTools.parseIntDecSubstring(diffString, 3);
|
||||
return ((ahead) ? 1 : -1) * (oh * 60 + om);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* get the difference of this servers time zone to UTC/GMT in milliseconds
|
||||
* @return
|
||||
|
@ -307,7 +307,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
return zoneOffsetHours + DSTOffsetHours;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static String UTCDiffString() {
|
||||
// we express the UTC Difference in 5 digits:
|
||||
// SHHMM
|
||||
|
@ -330,12 +330,12 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
}
|
||||
|
||||
private final static DecimalFormat D2 = new DecimalFormat("00");
|
||||
|
||||
|
||||
/**
|
||||
* Safely format the given time value using the given formatter. Fallback to
|
||||
* ISO-8601 representation or to raw time value without exception when the
|
||||
* format can not be applied.
|
||||
*
|
||||
*
|
||||
* @param time
|
||||
* a time value as millisecnods from Epoch (1970-01-01T00:00:00Z)
|
||||
* @param formatter
|
||||
|
@ -355,11 +355,11 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Safely format the given instant using the given formatter. Fallback to
|
||||
* ISO-8601 representation without exception when the format can not be applied.
|
||||
*
|
||||
*
|
||||
* @param instant
|
||||
* the instant to format
|
||||
* @param formatter
|
||||
|
@ -384,7 +384,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
|
|||
return res;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
public static void main(final String[] args) {
|
||||
System.out.println(UTCDiffString());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ISO8601
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 2.1.2011 at http://yacy.net
|
||||
* First released 2.1.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -11,12 +11,12 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -36,25 +36,25 @@ public class ISO8601Formatter extends AbstractFormatter implements DateFormatter
|
|||
|
||||
/** pattern for a W3C datetime variant of a non-localized ISO8601 date */
|
||||
private static final String PATTERN_ISO8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'";
|
||||
|
||||
|
||||
/** Date formatter/non-sloppy parser for W3C datetime (ISO8601) in GMT/UTC */
|
||||
private static final SimpleDateFormat FORMAT_ISO8601 = new SimpleDateFormat(PATTERN_ISO8601, Locale.US);
|
||||
|
||||
|
||||
static {
|
||||
FORMAT_ISO8601.setTimeZone(AbstractFormatter.UTCtimeZone);
|
||||
}
|
||||
|
||||
|
||||
public static final ISO8601Formatter FORMATTER = new ISO8601Formatter();
|
||||
|
||||
public ISO8601Formatter() {
|
||||
last_time = 0;
|
||||
last_format = "";
|
||||
this.last_time = 0;
|
||||
this.last_format = "";
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse dates as defined in <a href="http://www.w3.org/TR/NOTE-datetime">http://www.w3.org/TR/NOTE-datetime</a>.
|
||||
* This format (also specified in ISO8601) allows different "precisions".
|
||||
* The following lower precision versions for the complete date
|
||||
* The following lower precision versions for the complete date
|
||||
* "2007-12-19T10:20:30.567+0300" are allowed:<br>
|
||||
* "2007"<br>
|
||||
* "2007-12"<br>
|
||||
|
@ -66,7 +66,7 @@ public class ISO8601Formatter extends AbstractFormatter implements DateFormatter
|
|||
* Parsing is done in a fuzzy way. If there is an illegal character somewhere in
|
||||
* the String, the date parsed so far will be returned, e.g. the input
|
||||
* "2007-12-19FOO" would return a date that represents "2007-12-19".
|
||||
*
|
||||
*
|
||||
* @param s
|
||||
* @return
|
||||
* @throws ParseException
|
||||
|
@ -85,17 +85,17 @@ public class ISO8601Formatter extends AbstractFormatter implements DateFormatter
|
|||
int p = s.indexOf(';'); if (p >= 0) s = s.substring(0, p); // a semicolon may be used to separate two dates from each other; then we take the first
|
||||
p = s.indexOf(','); if (p >= 0) s = s.substring(0, p); // a comma may be used to separate two dates from each other; then we take the first
|
||||
while (!s.isEmpty() && s.endsWith("?")) s = s.substring(0, s.length() - 1); // sometimes used if write is not sure about date
|
||||
|
||||
|
||||
// no go for exact parsing
|
||||
final Calendar cal = Calendar.getInstance(AbstractFormatter.UTCtimeZone, Locale.US);
|
||||
cal.clear();
|
||||
|
||||
|
||||
// split 2007-12-19T10:20:30.789+0500 into its parts
|
||||
// correct: yyyy['-'MM['-'dd['T'HH':'MM[':'ss['.'SSS]]('Z'|ZZZZZ)]]]
|
||||
final StringTokenizer t = new StringTokenizer(s, "-T:.Z+", true);
|
||||
if (s == null || t.countTokens() == 0)
|
||||
throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0);
|
||||
|
||||
|
||||
try {
|
||||
// year
|
||||
cal.set(Calendar.YEAR, Integer.parseInt(t.nextToken()));
|
||||
|
@ -134,7 +134,7 @@ public class ISO8601Formatter extends AbstractFormatter implements DateFormatter
|
|||
token = t.nextToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// check for TZ data
|
||||
int offset;
|
||||
if (token.equals("Z")) {
|
||||
|
@ -158,13 +158,11 @@ public class ISO8601Formatter extends AbstractFormatter implements DateFormatter
|
|||
cal.set(Calendar.SECOND, sec);
|
||||
cal.set(Calendar.MILLISECOND, msec);
|
||||
}
|
||||
} catch (final NoSuchElementException e) {
|
||||
// ignore this as it is perfectly fine to have non-complete date in this format
|
||||
} catch (final Exception e) {
|
||||
// catch all Exceptions and return what we parsed so far
|
||||
//serverLog.logInfo("SERVER", "parseISO8601: DATE ERROR with: '" + s + "' got so far: '" + cal.toString());
|
||||
}
|
||||
|
||||
|
||||
// in case we couldn't even parse a year
|
||||
if (!cal.isSet(Calendar.YEAR))
|
||||
throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0);
|
||||
|
@ -176,29 +174,29 @@ public class ISO8601Formatter extends AbstractFormatter implements DateFormatter
|
|||
* Creates a String representation of a Date using the format defined
|
||||
* in ISO8601/W3C datetime
|
||||
* The result will be in UTC/GMT, e.g. "2007-12-19T10:20:30Z".
|
||||
*
|
||||
*
|
||||
* @param date The Date instance to transform.
|
||||
* @return A fixed width (20 chars) ISO8601 date String.
|
||||
*/
|
||||
@Override
|
||||
public final String format(final Date date) {
|
||||
if (date == null) return "";
|
||||
if (Math.abs(date.getTime() - last_time) < 1000) return last_format;
|
||||
if (Math.abs(date.getTime() - this.last_time) < 1000) return this.last_format;
|
||||
synchronized (FORMAT_ISO8601) {
|
||||
last_format = FORMAT_ISO8601.format(date);
|
||||
last_time = date.getTime();
|
||||
this.last_format = FORMAT_ISO8601.format(date);
|
||||
this.last_time = date.getTime();
|
||||
}
|
||||
return last_format;
|
||||
return this.last_format;
|
||||
}
|
||||
@Override
|
||||
public final String format() {
|
||||
long time = System.currentTimeMillis();
|
||||
if (Math.abs(time - last_time) < 1000) return last_format;
|
||||
final long time = System.currentTimeMillis();
|
||||
if (Math.abs(time - this.last_time) < 1000) return this.last_format;
|
||||
synchronized (FORMAT_ISO8601) {
|
||||
last_format = FORMAT_ISO8601.format(new Date(time));
|
||||
last_time = time;
|
||||
this.last_format = FORMAT_ISO8601.format(new Date(time));
|
||||
this.last_time = time;
|
||||
}
|
||||
return last_format;
|
||||
return this.last_format;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* MicroDate
|
||||
* Copyright 2008 by Michael Peter Christen
|
||||
* First released 3.7.2008 at http://yacy.net
|
||||
* First released 3.7.2008 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -29,11 +29,11 @@ public class MicroDate {
|
|||
|
||||
private static final long hour = 3600000L; // milliseconds of a hour
|
||||
private static final long day = 86400000L; // milliseconds of a day
|
||||
|
||||
|
||||
public static int microDateDays(final Date modified) {
|
||||
return microDateDays(modified.getTime());
|
||||
}
|
||||
|
||||
|
||||
public static int microDateDays(final long modified) {
|
||||
// this calculates a virtual age from a given date
|
||||
// the purpose is to have an age in days of a given modified date
|
||||
|
@ -42,20 +42,20 @@ public class MicroDate {
|
|||
// we take mod 64**3 = 262144, this is the mask of the storage
|
||||
return (int) ((modified / day) % 262144L);
|
||||
}
|
||||
|
||||
|
||||
public static int microDateHoursInt(final long time) {
|
||||
return (int) ((time / hour) % 262144L);
|
||||
}
|
||||
|
||||
|
||||
public static int microDateHoursAge(final String mdhs) {
|
||||
return microDateHoursInt(System.currentTimeMillis()) - (int) Base64Order.enhancedCoder.decodeLong(mdhs);
|
||||
}
|
||||
|
||||
|
||||
public static long reverseMicroDateDays(final long microDateDays) {
|
||||
return Math.min(System.currentTimeMillis(), microDateDays * day);
|
||||
}
|
||||
|
||||
|
||||
public static void main(final String[] args) {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* WordCache
|
||||
* Copyright 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
* first published 01.10.2009 on http://yacy.net
|
||||
* first published 01.10.2009 on https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -50,11 +50,11 @@ import net.yacy.cora.util.StringBuilderComparator;
|
|||
public class WordCache {
|
||||
|
||||
private final static ConcurrentLog log = new ConcurrentLog(WordCache.class.getName());
|
||||
|
||||
|
||||
// common word cache
|
||||
private static final int commonWordsMaxSize = 20000; // maximum size of common word cache
|
||||
private static final int commonWordsMinLength = 5; // words must have that length at minimum
|
||||
private static OrderedScoreMap<StringBuilder> commonWords = new OrderedScoreMap<StringBuilder>(StringBuilderComparator.CASE_INSENSITIVE_ORDER);
|
||||
private static OrderedScoreMap<StringBuilder> commonWords = new OrderedScoreMap<>(StringBuilderComparator.CASE_INSENSITIVE_ORDER);
|
||||
|
||||
// dictionaries
|
||||
private final File dictionaryPath;
|
||||
|
@ -67,8 +67,8 @@ public class WordCache {
|
|||
|
||||
public Dictionary(final File file) throws IOException {
|
||||
|
||||
this.dict = new TreeSet<StringBuilder>(StringBuilderComparator.CASE_INSENSITIVE_ORDER);
|
||||
this.tcid = new TreeSet<StringBuilder>(StringBuilderComparator.CASE_INSENSITIVE_ORDER);
|
||||
this.dict = new TreeSet<>(StringBuilderComparator.CASE_INSENSITIVE_ORDER);
|
||||
this.tcid = new TreeSet<>(StringBuilderComparator.CASE_INSENSITIVE_ORDER);
|
||||
|
||||
InputStream is = new FileInputStream(file);
|
||||
if (file.getName().endsWith(".gz")) {
|
||||
|
@ -103,7 +103,7 @@ public class WordCache {
|
|||
* @return set that contains all words that start or end with the input value
|
||||
*/
|
||||
public Set<StringBuilder> recommend(StringBuilder string) {
|
||||
final Set<StringBuilder> ret = new HashSet<StringBuilder>();
|
||||
final Set<StringBuilder> ret = new HashSet<>();
|
||||
SortedSet<StringBuilder> t = this.dict.tailSet(string);
|
||||
for (final StringBuilder r: t) {
|
||||
if (StringBuilderComparator.CASE_INSENSITIVE_ORDER.startsWith(r, string) && r.length() > string.length()) {
|
||||
|
@ -201,7 +201,7 @@ public class WordCache {
|
|||
*/
|
||||
public WordCache(final File dictionaryPath) {
|
||||
this.dictionaryPath = dictionaryPath;
|
||||
this.dictionaries = new ConcurrentHashMap<String, Dictionary>();
|
||||
this.dictionaries = new ConcurrentHashMap<>();
|
||||
reload();
|
||||
}
|
||||
|
||||
|
@ -222,8 +222,8 @@ public class WordCache {
|
|||
}
|
||||
}
|
||||
|
||||
public static void learn(Collection<String> wordset) {
|
||||
for (String s: wordset) {
|
||||
public static void learn(final Collection<String> wordset) {
|
||||
for (final String s: wordset) {
|
||||
learn(new StringBuilder(s));
|
||||
}
|
||||
}
|
||||
|
@ -239,7 +239,7 @@ public class WordCache {
|
|||
for (final String f: files) {
|
||||
if (f.endsWith(".words")) {
|
||||
try {
|
||||
Dictionary dict = new Dictionary(new File(this.dictionaryPath, f));
|
||||
final Dictionary dict = new Dictionary(new File(this.dictionaryPath, f));
|
||||
this.dictionaries.put(f.substring(0, f.length() - 6), dict);
|
||||
} catch (final IOException e) {
|
||||
log.warn(e);
|
||||
|
@ -262,9 +262,9 @@ public class WordCache {
|
|||
* @param string input value that is used to match recommendations
|
||||
* @return set that contains all words that start or end with the input value
|
||||
*/
|
||||
public Set<StringBuilder> recommend(StringBuilder string) {
|
||||
Set<StringBuilder> ret = new HashSet<StringBuilder>();
|
||||
for (Dictionary dict: this.dictionaries.values()) {
|
||||
public Set<StringBuilder> recommend(final StringBuilder string) {
|
||||
final Set<StringBuilder> ret = new HashSet<>();
|
||||
for (final Dictionary dict: this.dictionaries.values()) {
|
||||
ret.addAll(dict.recommend(string));
|
||||
}
|
||||
final SortedMap<StringBuilder, AtomicInteger> u = commonWords.tailMap(string);
|
||||
|
@ -290,7 +290,7 @@ public class WordCache {
|
|||
* @return true if the library contains the word
|
||||
*/
|
||||
public boolean contains(final StringBuilder s) {
|
||||
for (Dictionary dict: this.dictionaries.values()) {
|
||||
for (final Dictionary dict: this.dictionaries.values()) {
|
||||
if (dict.contains(s)) {
|
||||
return true;
|
||||
}
|
||||
|
@ -305,8 +305,8 @@ public class WordCache {
|
|||
* @param string the given word
|
||||
* @return true if the library supports the word
|
||||
*/
|
||||
public boolean supports(StringBuilder string) {
|
||||
for (Dictionary dict: this.dictionaries.values()) {
|
||||
public boolean supports(final StringBuilder string) {
|
||||
for (final Dictionary dict: this.dictionaries.values()) {
|
||||
if (dict.supports(string)) {
|
||||
return true;
|
||||
}
|
||||
|
@ -320,7 +320,7 @@ public class WordCache {
|
|||
*/
|
||||
public int size() {
|
||||
int size = 0;
|
||||
for (Dictionary dict: this.dictionaries.values()) {
|
||||
for (final Dictionary dict: this.dictionaries.values()) {
|
||||
size += dict.size();
|
||||
}
|
||||
return size;
|
||||
|
@ -329,7 +329,7 @@ public class WordCache {
|
|||
public static int sizeCommonWords() {
|
||||
return commonWords.size();
|
||||
}
|
||||
|
||||
|
||||
public static void clearCommonWords() {
|
||||
commonWords.clear();
|
||||
}
|
||||
|
@ -343,7 +343,7 @@ public class WordCache {
|
|||
* @return
|
||||
*/
|
||||
public boolean isRelevant(final int minimumWords) {
|
||||
for (Dictionary dict: this.dictionaries.values()) {
|
||||
for (final Dictionary dict: this.dictionaries.values()) {
|
||||
if (dict.isRelevant(minimumWords)) {
|
||||
return true;
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Classification.java
|
||||
* Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 09.07.2009 at http://yacy.net
|
||||
* First released 09.07.2009 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -34,13 +34,13 @@ import net.yacy.cora.util.CommonPattern;
|
|||
|
||||
public class Classification {
|
||||
|
||||
private static final Set<String> textExtSet = new HashSet<String>();
|
||||
private static final Set<String> mediaExtSet = new HashSet<String>();
|
||||
private static final Set<String> imageExtSet = new HashSet<String>();
|
||||
private static final Set<String> audioExtSet = new HashSet<String>();
|
||||
private static final Set<String> videoExtSet = new HashSet<String>();
|
||||
private static final Set<String> appsExtSet = new HashSet<String>();
|
||||
private static final Set<String> ctrlExtSet = new HashSet<String>();
|
||||
private static final Set<String> textExtSet = new HashSet<>();
|
||||
private static final Set<String> mediaExtSet = new HashSet<>();
|
||||
private static final Set<String> imageExtSet = new HashSet<>();
|
||||
private static final Set<String> audioExtSet = new HashSet<>();
|
||||
private static final Set<String> videoExtSet = new HashSet<>();
|
||||
private static final Set<String> appsExtSet = new HashSet<>();
|
||||
private static final Set<String> ctrlExtSet = new HashSet<>();
|
||||
|
||||
public enum ContentDomain {
|
||||
|
||||
|
@ -54,7 +54,7 @@ public class Classification {
|
|||
|
||||
private final int code;
|
||||
|
||||
ContentDomain(int code) {
|
||||
ContentDomain(final int code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
|
@ -104,17 +104,17 @@ public class Classification {
|
|||
addSet(mediaExtSet, apps + "," + audio + "," + video + "," + image); // all media formats
|
||||
}
|
||||
|
||||
private static void addSet(Set<String> set, final String extString) {
|
||||
private static void addSet(final Set<String> set, final String extString) {
|
||||
if ((extString == null) || (extString.isEmpty())) return;
|
||||
for (String s: CommonPattern.COMMA.split(extString, 0)) set.add(s.toLowerCase(Locale.ROOT).trim());
|
||||
for (final String s: CommonPattern.COMMA.split(extString, 0)) set.add(s.toLowerCase(Locale.ROOT).trim());
|
||||
}
|
||||
|
||||
public static boolean isTextExtension(String textExt) {
|
||||
public static boolean isTextExtension(final String textExt) {
|
||||
if (textExt == null) return false;
|
||||
return textExtSet.contains(textExt.trim().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
public static boolean isMediaExtension(String mediaExt) {
|
||||
public static boolean isMediaExtension(final String mediaExt) {
|
||||
if (mediaExt == null) return false;
|
||||
return mediaExtSet.contains(mediaExt.trim().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
@ -200,9 +200,9 @@ public class Classification {
|
|||
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (final Exception e1) {}
|
||||
}
|
||||
}
|
||||
for (Entry<Object, Object> entry: mimeTable.entrySet()) {
|
||||
String ext = (String) entry.getKey();
|
||||
String mime = (String) entry.getValue();
|
||||
for (final Entry<Object, Object> entry: mimeTable.entrySet()) {
|
||||
final String ext = (String) entry.getKey();
|
||||
final String mime = (String) entry.getValue();
|
||||
if (mime.startsWith("text/")) textExtSet.add(ext.toLowerCase(Locale.ROOT));
|
||||
if (mime.startsWith("audio/")) audioExtSet.add(ext.toLowerCase(Locale.ROOT));
|
||||
if (mime.startsWith("video/")) videoExtSet.add(ext.toLowerCase(Locale.ROOT));
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ASCIIComparator
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 25.2.2011 at http://yacy.net
|
||||
* First released 25.2.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -42,7 +42,7 @@ public class ASCII implements Comparator<String> {
|
|||
|
||||
public boolean insensitive;
|
||||
|
||||
public ASCII(boolean insensitive) {
|
||||
public ASCII(final boolean insensitive) {
|
||||
this.insensitive = insensitive;
|
||||
}
|
||||
|
||||
|
@ -52,13 +52,13 @@ public class ASCII implements Comparator<String> {
|
|||
}
|
||||
|
||||
@Override
|
||||
public int compare(String s0, String s1) {
|
||||
public int compare(final String s0, final String s1) {
|
||||
if (s0 == null && s1 == null) return 0;
|
||||
if (s0 == null) return -1;
|
||||
if (s1 == null) return 1;
|
||||
int i = 0;
|
||||
int l0 = s0.length(), l1 = s1.length();
|
||||
int lm = Math.min(l0, l1);
|
||||
final int l0 = s0.length(), l1 = s1.length();
|
||||
final int lm = Math.min(l0, l1);
|
||||
char c0, c1;
|
||||
while (i < lm) {
|
||||
c0 = s0.charAt(i);
|
||||
|
@ -74,13 +74,13 @@ public class ASCII implements Comparator<String> {
|
|||
return 0;
|
||||
}
|
||||
|
||||
public boolean equals(String s0, String s1) {
|
||||
public boolean equals(final String s0, final String s1) {
|
||||
if (s0 == null && s1 == null) return true;
|
||||
if (s0 == null) return false;
|
||||
if (s1 == null) return false;
|
||||
int i = 0;
|
||||
int l0 = s0.length(), l1 = s1.length();
|
||||
int lm = Math.min(l0, l1);
|
||||
final int l0 = s0.length(), l1 = s1.length();
|
||||
final int lm = Math.min(l0, l1);
|
||||
char c0, c1;
|
||||
while (i < lm) {
|
||||
c0 = s0.charAt(i);
|
||||
|
@ -95,7 +95,7 @@ public class ASCII implements Comparator<String> {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
public boolean equals(final Object obj) {
|
||||
return (obj == this);
|
||||
}
|
||||
|
||||
|
@ -112,7 +112,7 @@ public class ASCII implements Comparator<String> {
|
|||
}
|
||||
|
||||
public final static String String(final byte[] bytes, final int offset, final int length) {
|
||||
int l = Math.min(length, bytes.length - offset);
|
||||
final int l = Math.min(length, bytes.length - offset);
|
||||
final char[] c = new char[l];
|
||||
for (int i = 0; i < l; ++ i) {
|
||||
if (bytes[i + offset] < 0) throw new IllegalArgumentException();
|
||||
|
@ -124,7 +124,7 @@ public class ASCII implements Comparator<String> {
|
|||
public final static byte[] getBytes(final String s) {
|
||||
assert s != null;
|
||||
//assert s.length() < 3 || s.charAt(2) != '@';
|
||||
int count = s.length();
|
||||
final int count = s.length();
|
||||
final byte[] b = new byte[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
b[i] = (byte) s.charAt(i);
|
||||
|
@ -135,7 +135,7 @@ public class ASCII implements Comparator<String> {
|
|||
public final static byte[] getBytes(final String s, final int beginIndex, final int endIndex) {
|
||||
assert s != null;
|
||||
//assert s.length() < 3 || s.charAt(2) != '@';
|
||||
int count = endIndex - beginIndex;
|
||||
final int count = endIndex - beginIndex;
|
||||
final byte[] b = new byte[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
b[i] = (byte) s.charAt(i + beginIndex);
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* UTF8
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 25.2.2011 at http://yacy.net
|
||||
* First released 25.2.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -50,12 +50,12 @@ public class UTF8 implements Comparator<String> {
|
|||
|
||||
public boolean insensitive;
|
||||
|
||||
public UTF8(boolean insensitive) {
|
||||
public UTF8(final boolean insensitive) {
|
||||
this.insensitive = insensitive;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compare(String o0, String o1) {
|
||||
public int compare(final String o0, final String o1) {
|
||||
final int l0 = o0.length();
|
||||
final int l1 = o1.length();
|
||||
final int ml = Math.min(l0, l1);
|
||||
|
@ -96,7 +96,7 @@ public class UTF8 implements Comparator<String> {
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
public final static StringBody StringBody(final byte[] b) {
|
||||
return StringBody(UTF8.String(b));
|
||||
}
|
||||
|
@ -270,12 +270,12 @@ public class UTF8 implements Comparator<String> {
|
|||
|
||||
return (needToChange? sb.toString() : s);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Utility wrapper around the standard JDK {@link URLEncoder#encode(String)}
|
||||
* function, using the UTF-8 character set and catching
|
||||
* {@link UnsupportedEncodingException} exception.
|
||||
*
|
||||
*
|
||||
* @param str a string to encode. Must not be null.
|
||||
* @return str encoded in application/x-www-form-urlencoded format
|
||||
*/
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Channel
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 10.5.2010 at http://yacy.net
|
||||
* First released 10.5.2010 at https//yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -13,12 +13,12 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -30,18 +30,18 @@ package net.yacy.cora.document.feed;
|
|||
public interface Channel extends Iterable<Hit> {
|
||||
|
||||
public void setTitle(String title);
|
||||
|
||||
|
||||
public void setLink(String link);
|
||||
|
||||
|
||||
public void setDescription(String description);
|
||||
|
||||
|
||||
public void setImageURL(String imageUrl);
|
||||
|
||||
|
||||
public void setTotalResults(String totalResults);
|
||||
|
||||
|
||||
public void setStartIndex(String startIndex);
|
||||
|
||||
|
||||
public void setItemsPerPage(String itemsPerPage);
|
||||
|
||||
|
||||
public void setSearchTerms(String searchTerms);
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Hit
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 10.5.2010 at http://yacy.net
|
||||
* First released 10.5.2010 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -13,12 +13,12 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -32,68 +32,68 @@ import java.util.List;
|
|||
public interface Hit {
|
||||
|
||||
public void setAuthor(String title);
|
||||
|
||||
|
||||
public void setCopyright(String title);
|
||||
|
||||
|
||||
public void setCategory(String title);
|
||||
|
||||
|
||||
public void setTitle(String title);
|
||||
|
||||
|
||||
public void setLink(String link);
|
||||
|
||||
|
||||
public void setReferrer(String title);
|
||||
|
||||
|
||||
public void setLanguage(String title);
|
||||
|
||||
|
||||
public void setDescription(String description);
|
||||
|
||||
|
||||
public void setPubDate(Date pubdate);
|
||||
|
||||
|
||||
public void setGuid(String guid);
|
||||
|
||||
|
||||
public void setDocs(String docs);
|
||||
|
||||
public void setSubject(String[] tags);
|
||||
|
||||
|
||||
public void setSize(long size);
|
||||
|
||||
|
||||
/**
|
||||
* @param enclosure an URL describing a media object that is attached to a feed item
|
||||
*/
|
||||
public void setEnclosure(String enclosure);
|
||||
|
||||
|
||||
public String getAuthor();
|
||||
|
||||
|
||||
public String getCopyright();
|
||||
|
||||
|
||||
public String getCategory();
|
||||
|
||||
|
||||
public String getTitle();
|
||||
|
||||
|
||||
public String getLink();
|
||||
|
||||
|
||||
public String getReferrer();
|
||||
|
||||
|
||||
public String getLanguage();
|
||||
|
||||
|
||||
public List<String> getDescriptions();
|
||||
|
||||
|
||||
public Date getPubDate();
|
||||
|
||||
|
||||
public String getGuid();
|
||||
|
||||
|
||||
public String getDocs(); // A URL that points to the documentation for the format used in the RSS file.
|
||||
|
||||
public String[] getSubject();
|
||||
|
||||
|
||||
public long getSize();
|
||||
|
||||
|
||||
/**
|
||||
* @return an URL describing a media object that is attached to a feed item
|
||||
*/
|
||||
public String getEnclosure();
|
||||
|
||||
public double getLon();
|
||||
|
||||
|
||||
public double getLat();
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* RSSFeed
|
||||
* Copyright 2007 by Michael Peter Christen
|
||||
* First released 16.7.2007 at http://yacy.net
|
||||
* First released 16.7.2007 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -36,22 +36,22 @@ public class RSSFeed implements Iterable<RSSMessage> {
|
|||
public static final int DEFAULT_MAXSIZE = 10000;
|
||||
|
||||
// class variables
|
||||
|
||||
|
||||
/** Single required element see http://www.rssboard.org/rss-profile#element-channel */
|
||||
private RSSMessage channel = null;
|
||||
|
||||
|
||||
/** A guid:Item map */
|
||||
private final Map<String, RSSMessage> messages;
|
||||
private final int maxsize;
|
||||
|
||||
|
||||
/** Set to true when maxsize messages limit has been exceeded and exceeding messages have been discarded */
|
||||
private boolean maxSizeExceeded;
|
||||
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
final StringBuilder sb = new StringBuilder();
|
||||
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
|
||||
sb.append("<rss version=\"2.0\"\n");
|
||||
sb.append(" xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\"\n");
|
||||
|
@ -62,14 +62,14 @@ public class RSSFeed implements Iterable<RSSMessage> {
|
|||
sb.append("<opensearch:startIndex>0</opensearch:startIndex>\n");
|
||||
sb.append("<opensearch:itemsPerPage>" + this.size() + "</opensearch:itemsPerPage>\n");
|
||||
sb.append("<opensearch:totalResults>" + this.size() + "</opensearch:totalResults>\n");
|
||||
for (RSSMessage item: messages.values()) {
|
||||
for (final RSSMessage item: this.messages.values()) {
|
||||
sb.append(item.toString());
|
||||
}
|
||||
sb.append("</channel>\n");
|
||||
sb.append("</rss>\n");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
public RSSFeed(final int maxsize) {
|
||||
this.messages = Collections.synchronizedMap(new LinkedHashMap<String, RSSMessage>());
|
||||
this.channel = null;
|
||||
|
@ -83,11 +83,11 @@ public class RSSFeed implements Iterable<RSSMessage> {
|
|||
* @param links
|
||||
* @param source
|
||||
*/
|
||||
public RSSFeed(Set<MultiProtocolURL> links, String source) {
|
||||
public RSSFeed(final Set<MultiProtocolURL> links, final String source) {
|
||||
this(Integer.MAX_VALUE);
|
||||
String u;
|
||||
RSSMessage message;
|
||||
for (MultiProtocolURL uri: links) {
|
||||
for (final MultiProtocolURL uri: links) {
|
||||
u = uri.toNormalform(true);
|
||||
message = new RSSMessage(u, "", u);
|
||||
message.setAuthor(source);
|
||||
|
@ -112,8 +112,8 @@ public class RSSFeed implements Iterable<RSSMessage> {
|
|||
}
|
||||
|
||||
public Set<MultiProtocolURL> getLinks() {
|
||||
Set<MultiProtocolURL> links = new HashSet<MultiProtocolURL>();
|
||||
for (RSSMessage message: this.messages.values()) {
|
||||
final Set<MultiProtocolURL> links = new HashSet<>();
|
||||
for (final RSSMessage message: this.messages.values()) {
|
||||
try {links.add(new MultiProtocolURL(message.getLink()));} catch (final MalformedURLException e) {}
|
||||
}
|
||||
return links;
|
||||
|
@ -141,7 +141,7 @@ public class RSSFeed implements Iterable<RSSMessage> {
|
|||
public int size() {
|
||||
return this.messages.size();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return true when maxsize messages limit has been exceeded and exceeding messages have been discarded
|
||||
*/
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* RSSMessage
|
||||
* Copyright 2007 by Michael Peter Christen
|
||||
* First released 16.7.2007 at http://yacy.net
|
||||
* First released 16.7.2007 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -58,53 +58,53 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
|
||||
/** Human-readable title for an entry or a feed. */
|
||||
title(new String[]{"title","atom:title","rss:title",DublinCore.Title.getURIref()}),
|
||||
|
||||
|
||||
/** Reference from an entry or feed to a Web resource URL */
|
||||
link(new String[]{"link","atom:link","rss:link"}),
|
||||
|
||||
|
||||
/** Human-readable description or subtitle for an entry or a feed. */
|
||||
description(new String[]{"description","subtitle","atom:subtitle","rss:description", DublinCore.Description.getURIref()}),
|
||||
|
||||
|
||||
/** The publication date for content in a feed or for an antry. */
|
||||
pubDate(new String[]{"pubDate","lastBuildDate","updated","rss:lastBuildDate","rss:updated"}),
|
||||
|
||||
|
||||
/** Copyright notice for content in the channel. */
|
||||
copyright(new String[]{"copyright","publisher",DublinCore.Publisher.getURIref()}),
|
||||
|
||||
|
||||
/** The author of an item (Email address) */
|
||||
author(new String[]{"author","creator",DublinCore.Creator.getURIref()}),
|
||||
|
||||
|
||||
subject(new String[]{"subject",DublinCore.Subject.getURIref()}),
|
||||
|
||||
|
||||
/** One or more categories a channel or item belongs to */
|
||||
category(new String[]{"category"}),
|
||||
|
||||
|
||||
referrer(new String[]{"referrer","referer"}),
|
||||
|
||||
|
||||
/** The language the channel is written in. */
|
||||
language(new String[]{"language",DublinCore.Language.getURIref()}),
|
||||
|
||||
|
||||
/** A string that uniquely identifies an item (RSS 2.0) */
|
||||
guid(new String[]{"guid"}),
|
||||
|
||||
|
||||
/** URL describing a media object that is attached to a feed item */
|
||||
enclosure(new String[]{"enclosure"}),
|
||||
|
||||
|
||||
/** Time To Live : number of minutes that indicates how long a channel (RSS 2.0) can be cached before refreshing from the source. */
|
||||
ttl(new String[]{"ttl"}),
|
||||
|
||||
|
||||
/** URL to the documentation for the format used in the RSS file (for example http://www.rssboard.org/rss-specification) */
|
||||
docs(new String[]{"docs"}),
|
||||
|
||||
|
||||
size(new String[]{"size", "length", YaCyMetadata.size.getURIref()}),
|
||||
lon(new String[]{Geo.Long.getURIref(), "geo:lon"}), // include possible misspelling geo:lon (instead of geo:long)
|
||||
lat(new String[]{Geo.Lat.getURIref()});
|
||||
//point("gml:pos,georss:point,coordinates");
|
||||
|
||||
|
||||
private Set<String> keys;
|
||||
|
||||
private Token(final String[] keylist) {
|
||||
this.keys = new HashSet<String>();
|
||||
this.keys = new HashSet<>();
|
||||
this.keys.addAll(Arrays.asList(keylist));
|
||||
this.keys.add(this.name());
|
||||
}
|
||||
|
@ -127,23 +127,23 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
return this.keys.size() == 0 ? "" : this.keys.iterator().next();
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, Token> tokenNick2Token = new HashMap<String, Token>();
|
||||
|
||||
private static Map<String, Token> tokenNick2Token = new HashMap<>();
|
||||
static {
|
||||
for (Token t: Token.values()) {
|
||||
for (String nick: t.keys) tokenNick2Token.put(nick, t);
|
||||
for (final Token t: Token.values()) {
|
||||
for (final String nick: t.keys) tokenNick2Token.put(nick, t);
|
||||
}
|
||||
}
|
||||
|
||||
public static Token valueOfNick(String nick) {
|
||||
public static Token valueOfNick(final String nick) {
|
||||
return tokenNick2Token.get(nick);
|
||||
}
|
||||
|
||||
|
||||
private static String artificialGuidPrefix = "c0_";
|
||||
private static String calculatedGuidPrefix = "c1_";
|
||||
public static final RSSMessage POISON = new RSSMessage("", "", "");
|
||||
|
||||
public static final HashSet<String> tags = new HashSet<String>();
|
||||
public static final HashSet<String> tags = new HashSet<>();
|
||||
static {
|
||||
for (final Token token: Token.values()) {
|
||||
tags.addAll(token.keys());
|
||||
|
@ -153,7 +153,7 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
private final Map<String, String> map;
|
||||
|
||||
public RSSMessage(final String title, final String description, final String link) {
|
||||
this.map = new HashMap<String, String>();
|
||||
this.map = new HashMap<>();
|
||||
if (title.length() > 0) this.map.put(Token.title.name(), title);
|
||||
if (description.length() > 0) this.map.put(Token.description.name(), description);
|
||||
if (link.length() > 0) this.map.put(Token.link.name(), link);
|
||||
|
@ -162,7 +162,7 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
}
|
||||
|
||||
public RSSMessage(final String title, final String description, final MultiProtocolURL link, final String guid) {
|
||||
this.map = new HashMap<String, String>();
|
||||
this.map = new HashMap<>();
|
||||
if (title.length() > 0) this.map.put(Token.title.name(), title);
|
||||
if (description.length() > 0) this.map.put(Token.description.name(), description);
|
||||
this.map.put(Token.link.name(), link.toNormalform(true));
|
||||
|
@ -173,9 +173,9 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
}
|
||||
|
||||
public RSSMessage() {
|
||||
this.map = new HashMap<String, String>();
|
||||
this.map = new HashMap<>();
|
||||
}
|
||||
|
||||
|
||||
public void setValue(final Token token, final String value) {
|
||||
if (value.length() > 0) {
|
||||
this.map.put(token.name(), value);
|
||||
|
@ -214,8 +214,8 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
|
||||
@Override
|
||||
public List<String> getDescriptions() {
|
||||
List<String> ds = new ArrayList<String>();
|
||||
String d = Token.description.valueFrom(this.map, "");
|
||||
final List<String> ds = new ArrayList<>();
|
||||
final String d = Token.description.valueFrom(this.map, "");
|
||||
if (d.length() > 0) ds.add(d);
|
||||
return ds;
|
||||
}
|
||||
|
@ -285,12 +285,12 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
}
|
||||
return guid;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String getEnclosure() {
|
||||
return Token.enclosure.valueFrom(this.map, "");
|
||||
}
|
||||
|
||||
|
||||
public String getTTL() {
|
||||
return Token.ttl.valueFrom(this.map, "");
|
||||
}
|
||||
|
@ -330,9 +330,9 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
public String toString() {
|
||||
return this.toString(true);
|
||||
}
|
||||
|
||||
public String toString(boolean withItemTag) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
public String toString(final boolean withItemTag) {
|
||||
final StringBuilder sb = new StringBuilder();
|
||||
if (withItemTag) sb.append("<item>\n");
|
||||
if (this.map.containsKey(Token.title.name())) sb.append("<title>").append(this.map.get(Token.title.name())).append("</title>\n");
|
||||
if (this.map.containsKey(Token.link.name())) sb.append("<link>").append(this.map.get(Token.link.name())).append("</link>\n");
|
||||
|
@ -384,12 +384,12 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
|
|||
public void setGuid(final String guid) {
|
||||
setValue(Token.guid, guid);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void setEnclosure(final String enclosure) {
|
||||
setValue(Token.enclosure, enclosure);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void setLanguage(final String language) {
|
||||
setValue(Token.language, language);
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* RSSReader
|
||||
* Copyright 2007 by Michael Peter Christen
|
||||
* First released 16.7.2007 at http://yacy.net
|
||||
* First released 16.7.2007 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -50,7 +50,7 @@ public class RSSReader extends DefaultHandler {
|
|||
private boolean parsingChannel, parsingItem;
|
||||
private final RSSFeed theChannel;
|
||||
private Type type;
|
||||
|
||||
|
||||
/** When a parsing limit on instance construction has been exceeded */
|
||||
private boolean maxBytesExceeded;
|
||||
|
||||
|
@ -66,7 +66,7 @@ public class RSSReader extends DefaultHandler {
|
|||
this.maxBytesExceeded = false;
|
||||
}
|
||||
|
||||
private static final ThreadLocal<SAXParser> tlSax = new ThreadLocal<SAXParser>();
|
||||
private static final ThreadLocal<SAXParser> tlSax = new ThreadLocal<>();
|
||||
private static SAXParser getParser() throws SAXException {
|
||||
SAXParser parser = tlSax.get();
|
||||
if (parser == null) {
|
||||
|
@ -98,16 +98,16 @@ public class RSSReader extends DefaultHandler {
|
|||
throw new IOException (e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public RSSReader(final int maxsize, final long maxBytes, InputStream stream) throws IOException {
|
||||
this(maxsize);
|
||||
|
||||
|
||||
if (!(stream instanceof ByteArrayInputStream) && !(stream instanceof BufferedInputStream)) {
|
||||
stream = new BufferedInputStream(stream);
|
||||
}
|
||||
|
||||
StrictLimitInputStream limitedSource = new StrictLimitInputStream(stream, maxBytes);
|
||||
|
||||
|
||||
final StrictLimitInputStream limitedSource = new StrictLimitInputStream(stream, maxBytes);
|
||||
|
||||
try {
|
||||
final SAXParser saxParser = getParser();
|
||||
// do not look at external dtd - see: http://www.ibm.com/developerworks/xml/library/x-tipcfsx/index.html
|
||||
|
@ -121,7 +121,7 @@ public class RSSReader extends DefaultHandler {
|
|||
saxParser.parse(limitedSource, this);
|
||||
} catch (final SAXException e) {
|
||||
throw new IOException (e.getMessage());
|
||||
} catch(StreamLimitException e) {
|
||||
} catch(final StreamLimitException e) {
|
||||
this.maxBytesExceeded = true;
|
||||
}
|
||||
}
|
||||
|
@ -194,7 +194,7 @@ public class RSSReader extends DefaultHandler {
|
|||
if(StringUtils.isNotBlank(url)) {
|
||||
this.item.setEnclosure(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if ("rss".equals(tag)) {
|
||||
this.type = Type.rss;
|
||||
|
@ -236,7 +236,7 @@ public class RSSReader extends DefaultHandler {
|
|||
public RSSFeed getFeed() {
|
||||
return this.theChannel;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return true when a parsing limit on instance construction has been exceeded
|
||||
*/
|
||||
|
|
|
@ -1,18 +1,18 @@
|
|||
/**
|
||||
* Anchor
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* first published 15.09.2013 on http://yacy.net
|
||||
* first published 15.09.2013 on https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -40,7 +40,7 @@ public class AnchorURL extends DigestURL {
|
|||
this.imageURL = null;
|
||||
this.imageAlt = null;
|
||||
}
|
||||
|
||||
|
||||
public AnchorURL(final AnchorURL url) {
|
||||
super(url, url.hash());
|
||||
this.textBody = url.textBody;
|
||||
|
@ -76,7 +76,7 @@ public class AnchorURL extends DigestURL {
|
|||
this.imageURL = null;
|
||||
this.imageAlt = null;
|
||||
}
|
||||
|
||||
|
||||
public AnchorURL(final String protocol, final String host, final int port, final String path, final DigestURL imageURL, final String imageAlt) throws MalformedURLException {
|
||||
super(protocol, host, port, path);
|
||||
this.textBody = "";
|
||||
|
@ -104,42 +104,42 @@ public class AnchorURL extends DigestURL {
|
|||
}
|
||||
|
||||
public String getNameProperty() {
|
||||
return nameProperty;
|
||||
return this.nameProperty;
|
||||
}
|
||||
|
||||
public void setNameProperty(String name) {
|
||||
public void setNameProperty(final String name) {
|
||||
this.nameProperty = name;
|
||||
}
|
||||
|
||||
public String getTextProperty() {
|
||||
return textBody;
|
||||
return this.textBody;
|
||||
}
|
||||
|
||||
public void setTextProperty(String text) {
|
||||
public void setTextProperty(final String text) {
|
||||
this.textBody = text;
|
||||
}
|
||||
|
||||
public String getRelProperty() {
|
||||
return relProperty;
|
||||
return this.relProperty;
|
||||
}
|
||||
|
||||
public void setRelProperty(String rel) {
|
||||
public void setRelProperty(final String rel) {
|
||||
this.relProperty = rel;
|
||||
}
|
||||
|
||||
public DigestURL getImageURL() {
|
||||
return imageURL;
|
||||
return this.imageURL;
|
||||
}
|
||||
|
||||
public void setImageURL(DigestURL imageURL) {
|
||||
public void setImageURL(final DigestURL imageURL) {
|
||||
this.imageURL = imageURL;
|
||||
}
|
||||
|
||||
public String getImageAlt() {
|
||||
return imageAlt;
|
||||
return this.imageAlt;
|
||||
}
|
||||
|
||||
public void setImageAlt(String imageAlt) {
|
||||
public void setImageAlt(final String imageAlt) {
|
||||
this.imageAlt = imageAlt;
|
||||
}
|
||||
|
||||
|
@ -156,16 +156,16 @@ public class AnchorURL extends DigestURL {
|
|||
tagopts.setProperty("rel", this.relProperty);
|
||||
return tagopts;
|
||||
}
|
||||
|
||||
|
||||
public boolean attachedNofollow() {
|
||||
return this.relProperty.indexOf("nofollow") >= 0;
|
||||
}
|
||||
|
||||
|
||||
public String toHTML() {
|
||||
return "<a href=\"" + this.toNormalform(false) + "\"" +
|
||||
(this.nameProperty.length() > 0 ? (" name=\"" + this.nameProperty + "\"") : "") +
|
||||
(this.relProperty.length() > 0 ? (" rel=\"" + this.relProperty + "\"") : "") +
|
||||
">" + this.textBody + "</a>";
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -1,18 +1,18 @@
|
|||
/**
|
||||
* DigestURL
|
||||
* Copyright 2006 by Michael Peter Christen
|
||||
* first published 13.07.2006 on http://yacy.net
|
||||
* first published 13.07.2006 on https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -46,7 +46,7 @@ import net.yacy.cora.util.CommonPattern;
|
|||
public class DigestURL extends MultiProtocolURL implements Serializable {
|
||||
|
||||
public static final DigestURL POISON = new DigestURL(); // poison pill for concurrent link generators
|
||||
|
||||
|
||||
private static final long serialVersionUID = -1173233022912141885L;
|
||||
|
||||
// class variables
|
||||
|
@ -67,7 +67,7 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
else if (port > 999) h = "http://" + h + ":" + port;
|
||||
else h = "http://" + h;
|
||||
}
|
||||
DigestURL url = new DigestURL(h);
|
||||
final DigestURL url = new DigestURL(h);
|
||||
return (url == null) ? null : url.hosthash();
|
||||
}
|
||||
|
||||
|
@ -76,11 +76,11 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
* the list is separated by comma
|
||||
* @param hostlist
|
||||
* @return list of host hashes without separation
|
||||
* @throws MalformedURLException
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
public static String hosthashes(final String hostlist) throws MalformedURLException {
|
||||
String[] hs = CommonPattern.COMMA.split(hostlist);
|
||||
StringBuilder sb = new StringBuilder(hostlist.length());
|
||||
final String[] hs = CommonPattern.COMMA.split(hostlist);
|
||||
final StringBuilder sb = new StringBuilder(hostlist.length());
|
||||
for (String h: hs) {
|
||||
if (h == null) continue;
|
||||
h = h.trim();
|
||||
|
@ -92,9 +92,9 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
return sb.toString();
|
||||
}
|
||||
|
||||
public static Set<String> hosthashess(String hosthashes) {
|
||||
public static Set<String> hosthashess(final String hosthashes) {
|
||||
if (hosthashes == null || hosthashes.isEmpty()) return null;
|
||||
HashSet<String> h = new HashSet<String>();
|
||||
final HashSet<String> h = new HashSet<>();
|
||||
assert hosthashes.length() % 6 == 0;
|
||||
for (int i = 0; i < hosthashes.length(); i = i + 6) {
|
||||
h.add(hosthashes.substring(i, i + 6));
|
||||
|
@ -135,7 +135,7 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
super(url);
|
||||
this.hash = hash;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* DigestURI from general URI, hash already calculated
|
||||
* @param baseURL
|
||||
|
@ -172,9 +172,9 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
}
|
||||
return new DigestURL(baseURL, relPath);
|
||||
}
|
||||
|
||||
|
||||
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
|
||||
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
if (this.hashCache == Integer.MIN_VALUE) {
|
||||
|
@ -211,13 +211,13 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
|
||||
/**
|
||||
* <p>Extract a fragment of the url hash that can be used as a hash for the host name part of this url.</p>
|
||||
* <p><strong>WARNING : two URLs with the same host name but different protocols or ports will produce two different host hashes with this method!</strong></p>
|
||||
* <p><strong>WARNING : two URLs with the same host name but different protocols or ports will produce two different host hashes with this method!</strong></p>
|
||||
* @return a 6-byte hash fragment
|
||||
*/
|
||||
public String hosthash() {
|
||||
return ASCII.String(this.hash(), 6, 6);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* calculated YaCy-Hash of this URI
|
||||
*
|
||||
|
@ -306,7 +306,7 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
if (host == null) {
|
||||
return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(protocol)).substring(0, 5);
|
||||
}
|
||||
boolean isIPv6HostIP = host.indexOf(':') >= 0;
|
||||
final boolean isIPv6HostIP = host.indexOf(':') >= 0;
|
||||
final StringBuilder sb = new StringBuilder(host.length() + 15);
|
||||
sb.append(protocol).append(':');
|
||||
if (isIPv6HostIP) {sb.append('[').append(host).append(']');} else sb.append(host);
|
||||
|
@ -346,7 +346,7 @@ public class DigestURL extends MultiProtocolURL implements Serializable {
|
|||
public static final String hosthash6(final String host) {
|
||||
return hosthash6("http", host, 80);
|
||||
}
|
||||
|
||||
|
||||
//private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" };
|
||||
|
||||
public static final int domLengthEstimation(final byte[] urlHashBytes) {
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* MultiProtocolURI
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 25.5.2010 at http://yacy.net
|
||||
* First released 25.5.2010 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
@ -84,20 +84,20 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
|
||||
/** Register unreserved chars (never escaped in url) */
|
||||
private final static BitSet UNRESERVED_RFC1738 = new BitSet(128);
|
||||
|
||||
|
||||
/** Register unreserved chars for path part (not escaped in path) */
|
||||
private final static BitSet UNRESERVED_PATH = new BitSet(128);
|
||||
|
||||
|
||||
/**
|
||||
* Register regular expressions metacharacters used by the {@link Pattern}
|
||||
* class.
|
||||
*
|
||||
*
|
||||
* @see <a href=
|
||||
* "https://docs.oracle.com/javase/tutorial/essential/regex/literals.html">Regular
|
||||
* expressions string literals documentation</a>
|
||||
*/
|
||||
private static final BitSet PATTERN_METACHARACTERS = new BitSet(128);
|
||||
|
||||
|
||||
static {
|
||||
// unreserved characters (chars not to escape in url)
|
||||
for (int i = 'A'; i <= 'Z'; i++) { // hialpha RFC1738 Section 5
|
||||
|
@ -134,7 +134,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
UNRESERVED_PATH.set('@');
|
||||
UNRESERVED_PATH.set('&');
|
||||
UNRESERVED_PATH.set('=');
|
||||
|
||||
|
||||
/* Pattern metacharacters : <([{\^-=$!|]})?*+.> */
|
||||
PATTERN_METACHARACTERS.set('<');
|
||||
PATTERN_METACHARACTERS.set('(');
|
||||
|
@ -159,7 +159,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
|
||||
// session id handling
|
||||
private static final Object PRESENT = new Object();
|
||||
private static final ConcurrentHashMap<String, Object> sessionIDnames = new ConcurrentHashMap<String, Object>();
|
||||
private static final ConcurrentHashMap<String, Object> sessionIDnames = new ConcurrentHashMap<>();
|
||||
|
||||
public static final void initSessionIDNames(final Set<String> idNames) {
|
||||
for (String s: idNames) {
|
||||
|
@ -224,7 +224,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
|
||||
// identify protocol
|
||||
url = url.trim();
|
||||
|
||||
|
||||
if (url.startsWith("//")) {
|
||||
// patch for urls starting with "//" which can be found in the wild
|
||||
url = "http:" + url;
|
||||
|
@ -266,7 +266,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
}
|
||||
if (q < 0) { // check for www.test.com#fragment
|
||||
q = url.indexOf("#", p + 3);
|
||||
}
|
||||
}
|
||||
int r;
|
||||
if (q < 0) {
|
||||
if ((r = url.indexOf('@', p + 3)) < 0) {
|
||||
|
@ -331,14 +331,14 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
// wrong windows path, after the doublepoint there should be a backslash. Let's add a slash, as it will be slash in the normal form
|
||||
h = h.substring(0, 4) + '/' + h.substring(4);
|
||||
}
|
||||
int q = h.indexOf('/', 2);
|
||||
final int q = h.indexOf('/', 2);
|
||||
if (q < 0 || h.length() > 3 && h.charAt(3) == ':') {
|
||||
// Missing root slash such as "path" or "c:/path" accepted, but the path attribute must by after all start with it
|
||||
this.path = "/" + h.substring(2);
|
||||
this.path = "/" + h.substring(2);
|
||||
} else {
|
||||
this.host = h.substring(2, q ); // TODO: handle "c:" ?
|
||||
if (this.host.equalsIgnoreCase(Domains.LOCALHOST)) this.host = null;
|
||||
this.path = h.substring(q ); // "/path"
|
||||
this.path = h.substring(q ); // "/path"
|
||||
}
|
||||
} else if (h.startsWith("/")) { // "/host/path" or "/host/c:/path"
|
||||
this.path = h;
|
||||
|
@ -442,7 +442,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
}
|
||||
return new MultiProtocolURL(baseURL, relPath);
|
||||
}
|
||||
|
||||
|
||||
public MultiProtocolURL(final MultiProtocolURL baseURL, String relPath) throws MalformedURLException {
|
||||
if (baseURL == null) throw new MalformedURLException("base URL is null");
|
||||
if (relPath == null) throw new MalformedURLException("relPath is null");
|
||||
|
@ -470,12 +470,12 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
} else if (relPath.length() > 0 && relPath.charAt(0) == '/') {
|
||||
this.path = relPath;
|
||||
} else if (baseURL.path.endsWith("/")) {
|
||||
/* According to RFC 3986 example in Appendix B. (https://tools.ietf.org/html/rfc3986)
|
||||
/* According to RFC 3986 example in Appendix B. (https://tools.ietf.org/html/rfc3986)
|
||||
such an URL is valid : http://www.ics.uci.edu/pub/ietf/uri/#Related
|
||||
|
||||
We also find similar usages in the 2016 URL living standard (https://url.spec.whatwg.org/),
|
||||
for example : https://url.spec.whatwg.org/#syntax-url-absolute-with-fragment
|
||||
|
||||
|
||||
We also find similar usages in the 2016 URL living standard (https://url.spec.whatwg.org/),
|
||||
for example : https://url.spec.whatwg.org/#syntax-url-absolute-with-fragment
|
||||
|
||||
java.lang.URL constructor also accepts this form.*/
|
||||
if (relPath.startsWith("/")) this.path = baseURL.path + relPath.substring(1); else this.path = baseURL.path + relPath;
|
||||
} else {
|
||||
|
@ -518,7 +518,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
identSearchpart();
|
||||
escape();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param host the new host to apply to the copy
|
||||
* @return an exact copy of this URL instance but with a new host. The original instance remains unchanged.
|
||||
|
@ -528,22 +528,22 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
if(host == null || host.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Host parameter must not be null");
|
||||
}
|
||||
MultiProtocolURL copy = new MultiProtocolURL(this);
|
||||
|
||||
final MultiProtocolURL copy = new MultiProtocolURL(this);
|
||||
|
||||
if (host.indexOf(':') >= 0 && host.charAt(0) != '[') {
|
||||
copy.host = '[' + host + ']'; // IPv6 host must be enclosed in square brackets
|
||||
} else {
|
||||
copy.host = host;
|
||||
copy.host = host;
|
||||
}
|
||||
|
||||
|
||||
if (!Punycode.isBasic(this.host)) try {
|
||||
this.host = toPunycode(this.host);
|
||||
} catch (final PunycodeException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
|
||||
|
||||
return copy;
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -568,7 +568,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
p = matcher.replaceAll("");
|
||||
matcher.reset(p);
|
||||
}
|
||||
/* Let's remove any eventual remaining but inappropriate '..' segments at the beginning.
|
||||
/* Let's remove any eventual remaining but inappropriate '..' segments at the beginning.
|
||||
* See https://tools.ietf.org/html/rfc3986#section-5.2.4 -> parts 2.C and 2.D */
|
||||
while(p.startsWith("/../")) {
|
||||
p = p.substring(3);
|
||||
|
@ -600,7 +600,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
* (see RFC3986, and formerly RFC1738 & RFC2396). Uses UTF-8 character codes for
|
||||
* non-ASCII.</p>
|
||||
* <p>Important : already percent-encoded characters are not re-encoded</p>
|
||||
*
|
||||
*
|
||||
* @param pathToEscape the path part to escape.
|
||||
* @return an escaped path with only ASCII characters, or null when pathToEscape
|
||||
* is null.
|
||||
|
@ -612,13 +612,13 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
public static String escapePath(final String pathToEscape) {
|
||||
return escapePath(pathToEscape, false);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* <p>Percent-encode/escape an URL path regular expression according to the allowed
|
||||
* characters in an URL path (see RFC3986) and in the {@link Pattern} regular
|
||||
* expressions. Uses UTF-8 character codes for non-ASCII.</p>
|
||||
* <p>Important : already percent-encoded characters are not re-encoded</p>
|
||||
*
|
||||
*
|
||||
* @param pathPattern the URL path regular expression to escape.
|
||||
* @return an escaped path regular expression with only allowed ASCII
|
||||
* characters, or null when pathPattern is null.
|
||||
|
@ -630,7 +630,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
public static String escapePathPattern(final String pathPattern) {
|
||||
return escapePath(pathPattern, true);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Percent-encode/escape an URL path part according to the allowed characters
|
||||
|
@ -642,7 +642,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
* therefore meta-characters used by the {@link Pattern} class are not
|
||||
* percent-encoded.
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
* @param pathToEscape the path part to escape.
|
||||
* @param isPattern when true, regular meta-characters are not escaped
|
||||
* @return an escaped path regular expression with only allowed ASCII
|
||||
|
@ -661,7 +661,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
final int len = pathToEscape.length();
|
||||
int i = 0;
|
||||
while (i < len) {
|
||||
int ch = pathToEscape.charAt(i);
|
||||
final int ch = pathToEscape.charAt(i);
|
||||
if (ch == '%' && (i + 2) < len) {
|
||||
final char digit1 = pathToEscape.charAt(i + 1);
|
||||
final char digit2 = pathToEscape.charAt(i + 2);
|
||||
|
@ -707,7 +707,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
}
|
||||
return pathToEscape;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Decode UTF-8 percent-encoded characters eventually found in the given path.
|
||||
* <ul>
|
||||
|
@ -715,7 +715,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
* <li>the '+' character is not decoded to space character</li>
|
||||
* <li>no exception is thrown when invalid hexadecimal digits are found after a '%' character</li>
|
||||
* </ul>
|
||||
*
|
||||
*
|
||||
* @param path an URL path eventually escaped
|
||||
* @return return the unescaped path or null when path is null.
|
||||
*/
|
||||
|
@ -738,7 +738,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
utf8Bytes = ByteBuffer.allocate((len - i) / 3);
|
||||
}
|
||||
/* Percent-encoded character UTF-8 byte */
|
||||
int hexaValue = Integer.parseInt(escaped.substring(i + 1, i + 3), 16);
|
||||
final int hexaValue = Integer.parseInt(escaped.substring(i + 1, i + 3), 16);
|
||||
utf8Bytes.put((byte) hexaValue);
|
||||
modified = true;
|
||||
i += 2;
|
||||
|
@ -761,7 +761,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
|
||||
return modified ? unescaped.toString() : escaped;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param character a character to test
|
||||
* @return true when the character is a valid hexadecimal digit
|
||||
|
@ -770,7 +770,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f')
|
||||
|| (character >= 'A' && character <= 'F');
|
||||
}
|
||||
|
||||
|
||||
private void escapeSearchpart() {
|
||||
final StringBuilder qtmp = new StringBuilder(this.searchpart.length() + 10);
|
||||
for (final Map.Entry<String, String> element: getAttributes().entrySet()) {
|
||||
|
@ -827,7 +827,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
*
|
||||
* <li>The unreserved characters & : - _ . ! ~ * ' ( ) ; , = remain the same.
|
||||
* see RFC 1738 2.2 and RFC 3986 2.2
|
||||
*
|
||||
*
|
||||
* <li>All other ASCII characters are converted into the
|
||||
* 3-character string "%xy", where xy is
|
||||
* the two-digit hexadecimal representation of the character
|
||||
|
@ -849,24 +849,24 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
final int ch = s.charAt(i);
|
||||
if (ch == ' ') { // space
|
||||
sbuf.append("%20");
|
||||
} else if (ch == '%') {
|
||||
} else if (ch == '%') {
|
||||
if (i < len - 2 && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '9' && s.charAt(i + 2) >= '0' && s.charAt(i + 2) <= '9') {
|
||||
// TODO: actually 0..9 A..F a..f is allowed (or any of hex[] sequence)
|
||||
sbuf.append((char)ch); // lets consider this is used for encoding, leave it that way
|
||||
} else {
|
||||
sbuf.append("%25"); // '%' RFC 1738 2.2 unsafe char shall be encoded
|
||||
}
|
||||
} else if (ch == '&') {
|
||||
} else if (ch == '&') {
|
||||
if (i < len - 6 && "amp;".equals(s.substring(i + 1, i + 5).toLowerCase(Locale.ROOT))) {
|
||||
sbuf.append((char)ch); // leave it that way, it is used the right way
|
||||
} else {
|
||||
sbuf.append("%26"); // this must be urlencoded
|
||||
}
|
||||
} else if (ch == '#') { // RFC 1738 2.2 unsafe char is _not_ encoded because it may already be used for encoding
|
||||
} else if (ch == '#') { // RFC 1738 2.2 unsafe char is _not_ encoded because it may already be used for encoding
|
||||
sbuf.append((char)ch);
|
||||
} else if (ch == '!' || ch == ':' // unreserved
|
||||
|| ch == '-' || ch == '_'
|
||||
|| ch == '.' || ch == '~'
|
||||
|| ch == '.' || ch == '~'
|
||||
|| ch == '*' || ch == '\''
|
||||
|| ch == '(' || ch == ')'
|
||||
|| ch == '{' || ch == '}'
|
||||
|
@ -903,16 +903,16 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
public static String unescape(final String s) {
|
||||
try {
|
||||
return URLDecoder.decode(s, StandardCharsets.UTF_8.name());
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
} catch (final UnsupportedEncodingException e) {
|
||||
/* This should not happen */
|
||||
ConcurrentLog.logException(e);
|
||||
return s;
|
||||
} catch(Exception e) {
|
||||
} catch(final Exception e) {
|
||||
/*
|
||||
* URLDecode may throw an IllegalArgumentException (or any other
|
||||
* Exception in future implementations) when the string doesn't
|
||||
* match the application/x-www-form-urlencoded format: in that case
|
||||
* return the original string.
|
||||
* return the original string.
|
||||
* Example case : when the valid '%' character is used in a URL but without percent encoding purpose.
|
||||
*/
|
||||
return s;
|
||||
|
@ -1088,10 +1088,10 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
*/
|
||||
public String[] getPaths() {
|
||||
String s = (this.path == null || this.path.length() < 1) ? "" : this.path.charAt(0) == '/' ? this.path.substring(1) : this.path;
|
||||
int p = s.lastIndexOf('/');
|
||||
final int p = s.lastIndexOf('/');
|
||||
if (p < 0) return new String[0];
|
||||
s = s.substring(0, p); // the paths do not contain the last part, which is considered as the getFileName() part.
|
||||
String[] paths = CommonPattern.SLASH.split(s);
|
||||
final String[] paths = CommonPattern.SLASH.split(s);
|
||||
return paths;
|
||||
}
|
||||
|
||||
|
@ -1106,7 +1106,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
char c = this.path.charAt(1);
|
||||
if (c == ':') return new File(this.path);
|
||||
if (c == '|') return new File(this.path.charAt(0) + ":" + this.path.substring(2));
|
||||
|
||||
|
||||
if (this.path.length() > 1) { // prevent StringIndexOutOfBoundsException
|
||||
c = this.path.charAt(2);
|
||||
if (c == ':' || c == '|') return new File(this.path.charAt(1) + ":" + this.path.substring(3));
|
||||
|
@ -1124,12 +1124,12 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
public String getHost() {
|
||||
return this.host;
|
||||
}
|
||||
|
||||
|
||||
public String getOrganization() {
|
||||
String dnc = Domains.getDNC(host);
|
||||
String subdomOrga = host.length() - dnc.length() <= 0 ? "" : host.substring(0, host.length() - dnc.length() - 1);
|
||||
int p = subdomOrga.lastIndexOf('.');
|
||||
String orga = (p < 0) ? subdomOrga : subdomOrga.substring(p + 1);
|
||||
final String dnc = Domains.getDNC(this.host);
|
||||
final String subdomOrga = this.host.length() - dnc.length() <= 0 ? "" : this.host.substring(0, this.host.length() - dnc.length() - 1);
|
||||
final int p = subdomOrga.lastIndexOf('.');
|
||||
final String orga = (p < 0) ? subdomOrga : subdomOrga.substring(p + 1);
|
||||
return orga;
|
||||
}
|
||||
|
||||
|
@ -1138,7 +1138,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
*/
|
||||
public String getTLD() {
|
||||
if (this.host == null) return "";
|
||||
int p = this.host.lastIndexOf('.');
|
||||
final int p = this.host.lastIndexOf('.');
|
||||
if (p < 0) return "";
|
||||
return this.host.substring(p + 1);
|
||||
}
|
||||
|
@ -1161,7 +1161,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
/**
|
||||
* @return this URL fragment or null if has no fragment
|
||||
* @see <a href="https://url.spec.whatwg.org/#concept-url-fragment">URL fragment concept at WHATWG</a>
|
||||
* @see <a href="https://tools.ietf.org/html/rfc3986#section-3.5">URL fragment section in RFC 3986</a>
|
||||
* @see <a href="https://tools.ietf.org/html/rfc3986#section-3.5">URL fragment section in RFC 3986</a>
|
||||
*/
|
||||
public String getRef() {
|
||||
return this.anchor;
|
||||
|
@ -1194,10 +1194,10 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
public Map<String, String> getSearchpartMap() {
|
||||
if (this.searchpart == null) return null;
|
||||
this.searchpart = this.searchpart.replaceAll("&", "&");
|
||||
String[] parts = CommonPattern.AMP.split(this.searchpart);
|
||||
Map<String, String> map = new LinkedHashMap<String, String>();
|
||||
for (String part: parts) {
|
||||
int p = part.indexOf('=');
|
||||
final String[] parts = CommonPattern.AMP.split(this.searchpart);
|
||||
final Map<String, String> map = new LinkedHashMap<>();
|
||||
for (final String part: parts) {
|
||||
final int p = part.indexOf('=');
|
||||
if (p > 0) map.put(part.substring(0, p), part.substring(p + 1)); else map.put(part, "");
|
||||
}
|
||||
return map;
|
||||
|
@ -1210,7 +1210,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
|
||||
/**
|
||||
* Tokenizes url as string (without the protocol).
|
||||
* For example "http://host.com/path/file.txt" returns "host com path file ext"
|
||||
* For example "http://host.com/path/file.txt" returns "host com path file ext"
|
||||
* @return url tokens as one string
|
||||
*/
|
||||
public String toTokens() {
|
||||
|
@ -1233,7 +1233,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
|
||||
// split the string into tokens and add all camel-case splitting
|
||||
final String[] u = CommonPattern.SPACES.split(sb);
|
||||
final Set<String> token = new LinkedHashSet<String>();
|
||||
final Set<String> token = new LinkedHashSet<>();
|
||||
for (final String r: u) token.add(r);
|
||||
for (final String r: u) token.addAll(parseCamelCase(r));
|
||||
|
||||
|
@ -1246,7 +1246,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
public static enum CharType { low, high, number; }
|
||||
|
||||
private static Set<String> parseCamelCase(String s) {
|
||||
final Set<String> token = new LinkedHashSet<String>();
|
||||
final Set<String> token = new LinkedHashSet<>();
|
||||
if (s.isEmpty()) return token;
|
||||
int p = 0;
|
||||
CharType type = charType(s.charAt(0)), nct = type;
|
||||
|
@ -1272,18 +1272,18 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
/**
|
||||
* Evaluates url search part and returns attribute '=' value pairs
|
||||
* the returned values are in clear text (without urlencoding).
|
||||
*
|
||||
*
|
||||
* To get the parameter map as (url-encoded key and values)
|
||||
* @see getSearchpartMap()
|
||||
*
|
||||
* @return map key=attribue name, value=string after '='
|
||||
*/
|
||||
public Map<String, String> getAttributes() {
|
||||
Map<String, String > map = new LinkedHashMap<>();
|
||||
final Map<String, String > map = new LinkedHashMap<>();
|
||||
if (this.searchpart == null) return map;
|
||||
final String[] questp = CommonPattern.AMP.split(this.searchpart, -1);
|
||||
for (final String element : questp) {
|
||||
int p = element.indexOf('=');
|
||||
final int p = element.indexOf('=');
|
||||
if (p != -1) {
|
||||
map.put(unescape(element.substring(0, p)), unescape(element.substring(p + 1)));
|
||||
} else {
|
||||
|
@ -1292,13 +1292,13 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
private static CharType charType(final char c) {
|
||||
if (Character.isLowerCase(c)) return CharType.low;
|
||||
if (Character.isDigit(c)) return CharType.number;
|
||||
return CharType.high;
|
||||
}
|
||||
|
||||
|
||||
public String toNormalform(final boolean excludeAnchor) {
|
||||
return toNormalform(excludeAnchor, false);
|
||||
}
|
||||
|
@ -1326,7 +1326,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
defaultPort = true;
|
||||
}
|
||||
String urlPath = this.getFile(excludeAnchor, removeSessionID);
|
||||
String h = getHost();
|
||||
final String h = getHost();
|
||||
final StringBuilder u = new StringBuilder(20 + (urlPath == null ? 0 : urlPath.length()) + ((h == null) ? 0 : h.length()));
|
||||
u.append(this.protocol);
|
||||
u.append("://");
|
||||
|
@ -1345,8 +1345,8 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
urlPath = urlPath.replace('\\', '/');
|
||||
}
|
||||
u.append(urlPath);
|
||||
String result = u.toString();
|
||||
|
||||
final String result = u.toString();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
@ -1375,8 +1375,8 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
} else if (isFile()) {
|
||||
defaultPort = true;
|
||||
}
|
||||
String urlPath = this.getFile(excludeAnchor, removeSessionID);
|
||||
String h = getHost();
|
||||
final String urlPath = this.getFile(excludeAnchor, removeSessionID);
|
||||
final String h = getHost();
|
||||
final StringBuilder u = new StringBuilder(20 + urlPath.length() + ((h == null) ? 0 : h.length()));
|
||||
if (h != null) {
|
||||
if (this.userInfo != null && !(this.isFTP() && this.userInfo.startsWith(FTPClient.ANONYMOUS))) {
|
||||
|
@ -1390,8 +1390,8 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
u.append(this.port);
|
||||
}
|
||||
u.append(urlPath);
|
||||
String result = u.toString();
|
||||
|
||||
final String result = u.toString();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
@ -1491,12 +1491,12 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
String language = "en";
|
||||
if (this.host == null) return language;
|
||||
final int pos = this.host.lastIndexOf('.');
|
||||
String host_tld = this.host.substring(pos + 1).toLowerCase(Locale.ROOT);
|
||||
final String host_tld = this.host.substring(pos + 1).toLowerCase(Locale.ROOT);
|
||||
if (pos == 0) return language;
|
||||
int length = this.host.length() - pos - 1;
|
||||
final int length = this.host.length() - pos - 1;
|
||||
switch (length) {
|
||||
case 2:
|
||||
char firstletter = host_tld.charAt(0);
|
||||
final char firstletter = host_tld.charAt(0);
|
||||
switch (firstletter) {//speed-up
|
||||
case 'a':
|
||||
if (host_tld.equals("au")) {//Australia /91,000,000
|
||||
|
@ -2500,7 +2500,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
/**
|
||||
* Get directory listing of file or smb url
|
||||
* respects the hidden attribute of a directory (return null if hidden)
|
||||
*
|
||||
*
|
||||
* @return names of files and directories or null
|
||||
* @throws IOException
|
||||
*/
|
||||
|
@ -2554,11 +2554,11 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
|
||||
public byte[] get(final ClientIdentification.Agent agent, final String username, final String pass) throws IOException {
|
||||
if (isFile()) {
|
||||
byte[] b = read(new FileInputStream(getFSFile()));
|
||||
final byte[] b = read(new FileInputStream(getFSFile()));
|
||||
return b;
|
||||
}
|
||||
if (isSMB()) {
|
||||
byte[] b = read(new SmbFileInputStream(getSmbFile()));
|
||||
final byte[] b = read(new SmbFileInputStream(getSmbFile()));
|
||||
return b;
|
||||
}
|
||||
if (isFTP()) {
|
||||
|
@ -2589,19 +2589,19 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
if (isFTP()) {
|
||||
final FTPClient client = new FTPClient();
|
||||
client.open(this.host, this.port < 0 ? 21 : this.port);
|
||||
return client.fileSize(path) > 0;
|
||||
return client.fileSize(this.path) > 0;
|
||||
}
|
||||
if (isHTTP() || isHTTPS()) {
|
||||
final HTTPClient client = new HTTPClient(agent);
|
||||
client.setHost(getHost());
|
||||
org.apache.http.HttpResponse response = client.HEADResponse(this, true);
|
||||
final org.apache.http.HttpResponse response = client.HEADResponse(this, true);
|
||||
client.close();
|
||||
if (response == null) return false;
|
||||
int status = response.getStatusLine().getStatusCode();
|
||||
final int status = response.getStatusLine().getStatusCode();
|
||||
return status == 200 || status == 301 || status == 302;
|
||||
}
|
||||
return false;
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
if (e.getMessage().contains("Circular redirect to")) return true; // exception; this is a 302 which the client actually accepts
|
||||
//e.printStackTrace();
|
||||
return false;
|
||||
|
@ -2626,7 +2626,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
} finally {
|
||||
try {
|
||||
source.close();
|
||||
} catch(IOException ignored) {
|
||||
} catch(final IOException ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -2654,7 +2654,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
// TODO lowering case in a locale sensitive manner makes sense here, but the used language locale should not dependant on the default system locale
|
||||
return splitpattern.split(normalizedURL.toLowerCase()); // word components of the url
|
||||
}
|
||||
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final String[][] test = new String[][]{
|
||||
new String[]{null, "file://y:/"},
|
||||
|
@ -2717,7 +2717,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
|
|||
String environment, url;
|
||||
MultiProtocolURL aURL, aURL1;
|
||||
java.net.URL jURL;
|
||||
for (String[] element : test) {
|
||||
for (final String[] element : test) {
|
||||
environment = element[0];
|
||||
url = element[1];
|
||||
try {aURL = MultiProtocolURL.newURL(environment, url);} catch (final MalformedURLException e) {e.printStackTrace(); aURL = null;}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AbstractFederateSearchConnector.java
|
||||
* Copyright 2015 by Burkhard Buelte
|
||||
* First released 19.01.2015 at http://yacy.net
|
||||
* First released 19.01.2015 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or modify it under
|
||||
* the terms of the GNU Lesser General Public License as published by the Free
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* FederateSearchConnector.java
|
||||
* Copyright 2015 by Burkhard Buelte
|
||||
* First released 19.01.2015 at http://yacy.net
|
||||
* First released 19.01.2015 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or modify it under
|
||||
* the terms of the GNU Lesser General Public License as published by the Free
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* FederateSearchManager.java
|
||||
* Copyright 2015 by Burkhard Buelte
|
||||
* First released 19.01.2015 at http://yacy.net
|
||||
* First released 19.01.2015 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or modify it under
|
||||
* the terms of the GNU Lesser General Public License as published by the Free
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SolrFederateSearchConnector.java
|
||||
* Copyright 2015 by Burkhard Buelte
|
||||
* First released 19.01.2015 at http://yacy.net
|
||||
* First released 19.01.2015 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or modify it under
|
||||
* the terms of the GNU Lesser General Public License as published by the Free
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* OpenSearchConnector
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 03.11.2012 at http://yacy.net
|
||||
* First released 03.11.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -60,13 +60,13 @@ import net.yacy.search.schema.CollectionSchema;
|
|||
* configured systems until number of needed results are available.
|
||||
*/
|
||||
public class OpenSearchConnector extends AbstractFederateSearchConnector implements FederateSearchConnector {
|
||||
|
||||
/**
|
||||
|
||||
/**
|
||||
* HTML mapping properties used to retrieve result from HTML when the results
|
||||
* are not provided as a standard RSS/Atom feed but as simple HTML.
|
||||
*/
|
||||
private Properties htmlMapping;
|
||||
|
||||
private final Properties htmlMapping;
|
||||
|
||||
/**
|
||||
* @param instanceName open search instance name
|
||||
* @return the html mapping configuration file name derived from the instance name
|
||||
|
@ -74,16 +74,16 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
public static String htmlMappingFileName(final String instanceName) {
|
||||
return instanceName + ".html.map.properties";
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param urlTemplate OpenSearch URL template
|
||||
* @param urlTemplate OpenSearch URL template
|
||||
*/
|
||||
public OpenSearchConnector(final String urlTemplate) {
|
||||
super();
|
||||
this.baseurl = urlTemplate;
|
||||
this.htmlMapping = new Properties();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean init(final String name, final String cfgFileName) {
|
||||
this.instancename = name;
|
||||
|
@ -94,13 +94,13 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
try {
|
||||
cfgFileStream = new BufferedInputStream(new FileInputStream(cfgFileName));
|
||||
this.htmlMapping.load(cfgFileStream);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.config("OpenSearchConnector." + this.instancename, "Error reading html mapping file : " + cfgFileName, e);
|
||||
} finally {
|
||||
if (cfgFileStream != null) {
|
||||
try {
|
||||
cfgFileStream.close();
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.config("OpenSearchConnector." + this.instancename, "Error closing html mapping file : " + cfgFileName, e);
|
||||
}
|
||||
}
|
||||
|
@ -108,11 +108,11 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* replace Opensearchdescription search template parameter with actual values
|
||||
*/
|
||||
private String parseSearchTemplate(String searchurltemplate, String query, int start, int rows) {
|
||||
private String parseSearchTemplate(final String searchurltemplate, final String query, final int start, final int rows) {
|
||||
String tmps = searchurltemplate.replaceAll("\\?}", "}"); // some optional parameters may include question mark '{param?}='
|
||||
tmps = tmps.replace("{startIndex}", Integer.toString(start));
|
||||
tmps = tmps.replace("{startPage}", "");
|
||||
|
@ -127,42 +127,42 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
* @param linkElement html link result node. Must not be null.
|
||||
* @return and {@link URIMetadataNode} instance from the html link element or null when minimum required information is missing or malformed
|
||||
*/
|
||||
protected URIMetadataNode htmlLinkToMetadataNode(Element linkElement) {
|
||||
protected URIMetadataNode htmlLinkToMetadataNode(final Element linkElement) {
|
||||
URIMetadataNode doc = null;
|
||||
String absoluteURL = linkElement.absUrl("href");
|
||||
final String absoluteURL = linkElement.absUrl("href");
|
||||
try {
|
||||
if (!absoluteURL.isEmpty()) {
|
||||
DigestURL uri = new DigestURL(absoluteURL);
|
||||
final DigestURL uri = new DigestURL(absoluteURL);
|
||||
|
||||
doc = new URIMetadataNode(uri);
|
||||
|
||||
|
||||
if(linkElement.hasText() && !this.htmlMapping.containsKey("title")) {
|
||||
/* Let's use the link text as default title when no mapping is defined.*/
|
||||
doc.setField(CollectionSchema.title.getSolrFieldName(), linkElement.text());
|
||||
}
|
||||
|
||||
String targetLang = linkElement.attr("hreflang");
|
||||
|
||||
final String targetLang = linkElement.attr("hreflang");
|
||||
if(targetLang != null && !targetLang.isEmpty()) {
|
||||
doc.setField(CollectionSchema.language_s.getSolrFieldName(), targetLang);
|
||||
}
|
||||
|
||||
|
||||
final String mime = TextParser.mimeOf(uri);
|
||||
if (mime != null) {
|
||||
doc.setField(CollectionSchema.content_type.getSolrFieldName(), mime);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* add collection "dht" which is used to differentiate metadata
|
||||
* from full crawl data in the index
|
||||
*/
|
||||
doc.setField(CollectionSchema.collection_sxt.getSolrFieldName(), "dht");
|
||||
}
|
||||
} catch (MalformedURLException e) {
|
||||
} catch (final MalformedURLException e) {
|
||||
ConcurrentLog.fine("OpenSearchConnector." + this.instancename, "Malformed url : " + absoluteURL);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Extract results from the HTML result stream, using the html mapping properties.
|
||||
* Important : it is the responsibility of the caller to close the stream.
|
||||
|
@ -171,33 +171,33 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
* @return a list of URI nodes, eventually empty.
|
||||
* @throws IOException when a read/write exception occurred
|
||||
*/
|
||||
protected List<URIMetadataNode> parseHTMLResult(InputStream resultStream, String charsetName) throws IOException {
|
||||
List<URIMetadataNode> docs = new ArrayList<>();
|
||||
String resultSelector = this.htmlMapping.getProperty("_result");
|
||||
String skuSelector = this.htmlMapping.getProperty("_sku");
|
||||
protected List<URIMetadataNode> parseHTMLResult(final InputStream resultStream, final String charsetName) throws IOException {
|
||||
final List<URIMetadataNode> docs = new ArrayList<>();
|
||||
final String resultSelector = this.htmlMapping.getProperty("_result");
|
||||
final String skuSelector = this.htmlMapping.getProperty("_sku");
|
||||
if (resultSelector == null || skuSelector == null) {
|
||||
ConcurrentLog.warn("OpenSearchConnector." + this.instancename, "HTML mapping is incomplete!");
|
||||
return docs;
|
||||
}
|
||||
|
||||
Document jsoupDoc = Jsoup.parse(resultStream, charsetName, this.baseurl);
|
||||
Elements results = jsoupDoc.select(resultSelector);
|
||||
final Document jsoupDoc = Jsoup.parse(resultStream, charsetName, this.baseurl);
|
||||
final Elements results = jsoupDoc.select(resultSelector);
|
||||
|
||||
for (Element result : results) {
|
||||
Elements skuNodes = result.select(skuSelector);
|
||||
for (final Element result : results) {
|
||||
final Elements skuNodes = result.select(skuSelector);
|
||||
if (!skuNodes.isEmpty()) {
|
||||
Element skuNode = skuNodes.first();
|
||||
if (!"a".equals(skuNode.tagName())) {
|
||||
/*
|
||||
* The selector may refer to a node with link(s) inside
|
||||
*/
|
||||
Elements links = skuNode.select("a[href]");
|
||||
final Elements links = skuNode.select("a[href]");
|
||||
if (!links.isEmpty()) {
|
||||
skuNode = links.first();
|
||||
}
|
||||
}
|
||||
if (skuNode.hasAttr("href")) {
|
||||
URIMetadataNode newDoc = htmlLinkToMetadataNode(skuNode);
|
||||
final URIMetadataNode newDoc = htmlLinkToMetadataNode(skuNode);
|
||||
if (newDoc != null) {
|
||||
/* Let's handle other field mappings */
|
||||
htmlResultToFields(result, newDoc);
|
||||
|
@ -214,32 +214,32 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
* @param resultNode html single result node
|
||||
* @param newdoc result document to fill
|
||||
*/
|
||||
private void htmlResultToFields(Element resultNode, URIMetadataNode newdoc) {
|
||||
for (Entry<Object, Object> entry : this.htmlMapping.entrySet()) {
|
||||
private void htmlResultToFields(final Element resultNode, final URIMetadataNode newdoc) {
|
||||
for (final Entry<Object, Object> entry : this.htmlMapping.entrySet()) {
|
||||
if (entry.getKey() instanceof String && entry.getValue() instanceof String) {
|
||||
String yacyFieldName = (String) entry.getKey();
|
||||
String selector = (String) entry.getValue();
|
||||
|
||||
final String yacyFieldName = (String) entry.getKey();
|
||||
final String selector = (String) entry.getValue();
|
||||
|
||||
if (!yacyFieldName.startsWith("_")) {
|
||||
/* If Switchboard environment is set, check the index configuration has this field enabled */
|
||||
if (Switchboard.getSwitchboard() == null || Switchboard.getSwitchboard().index == null
|
||||
|| Switchboard.getSwitchboard().index.fulltext().getDefaultConfiguration()
|
||||
.contains(yacyFieldName)) {
|
||||
|
||||
Elements nodes = resultNode.select(selector);
|
||||
final Elements nodes = resultNode.select(selector);
|
||||
|
||||
SchemaDeclaration est;
|
||||
try {
|
||||
est = CollectionSchema.valueOf(yacyFieldName);
|
||||
} catch(IllegalArgumentException e) {
|
||||
} catch(final IllegalArgumentException e) {
|
||||
ConcurrentLog.config("OpenSearchConnector." + this.instancename,
|
||||
"Ignored " + yacyFieldName + " field mapping : not a field of this schema.");
|
||||
continue;
|
||||
}
|
||||
if (est.isMultiValued()) {
|
||||
if (!nodes.isEmpty()) {
|
||||
for (Element node : nodes) {
|
||||
String value = node.text();
|
||||
for (final Element node : nodes) {
|
||||
final String value = node.text();
|
||||
if (!value.isEmpty()) {
|
||||
newdoc.addField(yacyFieldName, value);
|
||||
}
|
||||
|
@ -247,8 +247,8 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
}
|
||||
} else {
|
||||
if (!nodes.isEmpty()) {
|
||||
Element node = nodes.first();
|
||||
String value = node.text();
|
||||
final Element node = nodes.first();
|
||||
final String value = node.text();
|
||||
if (!value.isEmpty()) {
|
||||
/* Perform eventual type conversion */
|
||||
try {
|
||||
|
@ -257,7 +257,7 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
} else {
|
||||
newdoc.setField(yacyFieldName, value);
|
||||
}
|
||||
} catch (NumberFormatException ex) {
|
||||
} catch (final NumberFormatException ex) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
@ -278,11 +278,11 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
* @return query results (metadata) with fields according to YaCy schema
|
||||
*/
|
||||
@Override
|
||||
public List<URIMetadataNode> query(QueryParams query) {
|
||||
public List<URIMetadataNode> query(final QueryParams query) {
|
||||
|
||||
return query(query.getQueryGoal().getQueryString(false), 0, query.itemsPerPage);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Query the remote system at baseurl with the specified search terms
|
||||
* @param searchTerms search terms
|
||||
|
@ -291,17 +291,17 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
* @return a result list eventually empty when no results where found or when an error occured
|
||||
*/
|
||||
public List<URIMetadataNode> query(final String searchTerms, final int startIndex, final int count) {
|
||||
List<URIMetadataNode> docs = new ArrayList<URIMetadataNode>();
|
||||
|
||||
List<URIMetadataNode> docs = new ArrayList<>();
|
||||
|
||||
// see http://www.loc.gov/standards/sru/
|
||||
String searchurl = this.parseSearchTemplate(baseurl, searchTerms, startIndex, count);
|
||||
final String searchurl = this.parseSearchTemplate(this.baseurl, searchTerms, startIndex, count);
|
||||
try {
|
||||
DigestURL aurl = new DigestURL(searchurl);
|
||||
final DigestURL aurl = new DigestURL(searchurl);
|
||||
try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
|
||||
this.lastaccesstime = System.currentTimeMillis();
|
||||
|
||||
byte[] result = httpClient.GETbytes(aurl, null, null, false);
|
||||
|
||||
|
||||
final byte[] result = httpClient.GETbytes(aurl, null, null, false);
|
||||
|
||||
if(result == null) {
|
||||
String details;
|
||||
if(httpClient.getHttpResponse() != null && httpClient.getHttpResponse().getStatusLine() != null) {
|
||||
|
@ -324,15 +324,15 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
}
|
||||
} else {
|
||||
/* Other mime types or unknown : let's try to parse the result as RSS or Atom Feed */
|
||||
RSSReader rssReader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
|
||||
final RSSReader rssReader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
|
||||
if (rssReader != null) {
|
||||
final RSSFeed feed = rssReader.getFeed();
|
||||
if (feed != null) {
|
||||
for (final RSSMessage item : feed) {
|
||||
try {
|
||||
DigestURL uri = new DigestURL(item.getLink());
|
||||
final DigestURL uri = new DigestURL(item.getLink());
|
||||
|
||||
URIMetadataNode doc = new URIMetadataNode(uri);
|
||||
final URIMetadataNode doc = new URIMetadataNode(uri);
|
||||
doc.setField(CollectionSchema.charset_s.getSolrFieldName(), StandardCharsets.UTF_8.name());
|
||||
doc.setField(CollectionSchema.author.getSolrFieldName(), item.getAuthor());
|
||||
doc.setField(CollectionSchema.title.getSolrFieldName(), item.getTitle());
|
||||
|
@ -369,16 +369,16 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
}
|
||||
}
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
} catch (final IOException ex) {
|
||||
ConcurrentLog.logException(ex);
|
||||
ConcurrentLog.info("OpenSearchConnector." + this.instancename, "no connection to " + searchurl);
|
||||
}
|
||||
} catch (MalformedURLException ee) {
|
||||
} catch (final MalformedURLException ee) {
|
||||
ConcurrentLog.warn("OpenSearchConnector." + this.instancename, "malformed url " + searchurl);
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Main procedure : can be used to test results retrieval from an open search system
|
||||
* @param args main arguments list:
|
||||
|
@ -388,14 +388,14 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
* <li>Html mapping file path (optional)</li>
|
||||
* </ol>
|
||||
*/
|
||||
public static void main(String args[]) {
|
||||
public static void main(final String args[]) {
|
||||
try {
|
||||
if (args.length < 2) {
|
||||
System.out.println("Usage : java " + OpenSearchConnector.class.getCanonicalName()
|
||||
+ " <templateURL> <\"searchTerms\"> [htmlMappingFile]");
|
||||
return;
|
||||
}
|
||||
OpenSearchConnector connector = new OpenSearchConnector(args[0]);
|
||||
final OpenSearchConnector connector = new OpenSearchConnector(args[0]);
|
||||
String htmlMappingFile;
|
||||
if (args.length > 2) {
|
||||
htmlMappingFile = args[2];
|
||||
|
@ -407,12 +407,12 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
|
|||
if(searchTerms.length() > 2 && searchTerms.startsWith("\"") && searchTerms.endsWith("\"")) {
|
||||
searchTerms = searchTerms.substring(1, searchTerms.length() - 1);
|
||||
}
|
||||
List<URIMetadataNode> docs = connector.query(searchTerms, 0, 20);
|
||||
final List<URIMetadataNode> docs = connector.query(searchTerms, 0, 20);
|
||||
if (docs.isEmpty()) {
|
||||
System.out.println("No results");
|
||||
} else {
|
||||
|
||||
for (URIMetadataNode doc : docs) {
|
||||
for (final URIMetadataNode doc : docs) {
|
||||
System.out.println("title : " + doc.getFieldValue(CollectionSchema.title.getSolrFieldName()));
|
||||
System.out.println("sku : " + doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
|
||||
System.out.println(
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AccumulateSRURSS
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 06.01.2011 at http://yacy.net
|
||||
* First released 06.01.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
@ -25,6 +25,8 @@ import java.net.MalformedURLException;
|
|||
import java.util.LinkedHashMap;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
|
||||
import org.apache.http.entity.mime.content.ContentBody;
|
||||
|
||||
import net.yacy.cora.document.encoding.UTF8;
|
||||
import net.yacy.cora.document.feed.RSSFeed;
|
||||
import net.yacy.cora.document.feed.RSSMessage;
|
||||
|
@ -34,8 +36,6 @@ import net.yacy.cora.federate.yacy.CacheStrategy;
|
|||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.protocol.http.HTTPClient;
|
||||
|
||||
import org.apache.http.entity.mime.content.ContentBody;
|
||||
|
||||
public class SRURSSConnector {
|
||||
|
||||
private final static int recordsPerSession = 250;
|
||||
|
@ -114,7 +114,7 @@ public class SRURSSConnector {
|
|||
// send request
|
||||
byte[] result = new byte[0];
|
||||
try {
|
||||
final LinkedHashMap<String,ContentBody> parts = new LinkedHashMap<String,ContentBody>();
|
||||
final LinkedHashMap<String,ContentBody> parts = new LinkedHashMap<>();
|
||||
parts.put("query", UTF8.StringBody(query));
|
||||
parts.put("startRecord", UTF8.StringBody(Integer.toString(startRecord)));
|
||||
parts.put("maximumRecords", UTF8.StringBody(Long.toString(maximumRecords)));
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* FailCategory
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 17.10.2013 at http://yacy.net
|
||||
* First released 17.10.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* FailType
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 23.11.2012 at http://yacy.net
|
||||
* First released 23.11.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ProcessType
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 02.01.2013 at http://yacy.net
|
||||
* First released 02.01.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Ranking
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 12.03.2013 at http://yacy.net
|
||||
* First released 12.03.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SchemaConfiguration
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
|
||||
* First released 29.06.2011 at http://yacy.net
|
||||
* First released 29.06.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SchemaDeclaration
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 14.04.2011 at http://yacy.net
|
||||
* First released 14.04.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SolrType
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 14.04.2011 at http://yacy.net
|
||||
* First released 14.04.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AbstractSolrConnector
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 27.06.2012 at http://yacy.net
|
||||
* First released 27.06.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* EmbeddedSolrConnector
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 21.06.2012 at http://yacy.net
|
||||
* First released 21.06.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* MirrorSolrConnector
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 18.02.2012 at http://yacy.net
|
||||
* First released 18.02.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SolrSingleConnector
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 14.04.2011 at http://yacy.net
|
||||
* First released 14.04.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SolrChardingSelection
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 25.05.2011 at http://yacy.net
|
||||
* First released 25.05.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SolrConnector
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 13.09.2011 at http://yacy.net
|
||||
* First released 13.09.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SolrServerConnector
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 21.06.2012 at http://yacy.net
|
||||
* First released 21.06.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* EmbeddedInstance
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 13.02.2013 at http://yacy.net
|
||||
* First released 13.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* InstanceMirror
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 18.02.2013 at http://yacy.net
|
||||
* First released 18.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* RemoteInstance
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 13.02.2013 at http://yacy.net
|
||||
* First released 13.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ResponseAccumulator
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 13.02.2013 at http://yacy.net
|
||||
* First released 13.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ServerMirror
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 18.02.2013 at http://yacy.net
|
||||
* First released 18.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ServerShard
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 13.02.2013 at http://yacy.net
|
||||
* First released 13.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ShardInstance
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 13.02.2013 at http://yacy.net
|
||||
* First released 13.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SolrInstance
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 13.02.2013 at http://yacy.net
|
||||
* First released 13.02.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AbstractTerm
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 03.08.2014 at http://yacy.net
|
||||
* First released 03.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* BooleanLiteral
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 24.10.2014 at http://yacy.net
|
||||
* First released 24.10.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* CatchallLiteral
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 24.10.2014 at http://yacy.net
|
||||
* First released 24.10.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Conjunction
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 03.08.2014 at http://yacy.net
|
||||
* First released 03.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Disjunction
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 03.08.2014 at http://yacy.net
|
||||
* First released 03.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Literal
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 03.08.2014 at http://yacy.net
|
||||
* First released 03.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* LongLiteral
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 24.10.2014 at http://yacy.net
|
||||
* First released 24.10.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Negation
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 03.08.2014 at http://yacy.net
|
||||
* First released 03.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* StringLiteral
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 03.08.2014 at http://yacy.net
|
||||
* First released 03.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Term
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 03.08.2014 at http://yacy.net
|
||||
* First released 03.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* TermTools
|
||||
* Copyright 2014 by Michael Peter Christen
|
||||
* First released 04.08.2014 at http://yacy.net
|
||||
* First released 04.08.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* FastXMLResponseWriter
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 06.08.2012 at http://yacy.net
|
||||
* First released 06.08.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* FlatJSONResponseWriter
|
||||
* Copyright 2017 by Michael Peter Christen
|
||||
* First released 30.03.2017 at http://yacy.net
|
||||
* First released 30.03.2017 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* GrepHTMLResponseWriter
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 09.06.2013 at http://yacy.net
|
||||
* First released 09.06.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* HTMLResponseWriter
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 09.06.2013 at http://yacy.net
|
||||
* First released 09.06.2013 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* OpensearchResponseWriter
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 06.08.2012 at http://yacy.net
|
||||
* First released 06.08.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* JsonResponseWriter
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 10.09.2012 at http://yacy.net
|
||||
* First released 10.09.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* CacheStrategy
|
||||
* Copyright 2011 by Michael Peter Christen
|
||||
* First released 2011 at http://yacy.net
|
||||
* First released 2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* VerticalPartition
|
||||
* Copyright 2009 by Michael Peter Christen
|
||||
* First released 28.01.2009 at http://yacy.net
|
||||
* First released 28.01.2009 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Peer
|
||||
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 21.09.2012 at http://yacy.net
|
||||
* First released 21.09.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Peers
|
||||
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 21.09.2012 at http://yacy.net
|
||||
* First released 21.09.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Peers
|
||||
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 21.09.2012 at http://yacy.net
|
||||
* First released 21.09.2012 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Phonetic
|
||||
* Copyright 201 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
|
||||
* First released 13.12.2011 at http://yacy.net
|
||||
* First released 13.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Literal
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 18.12.2011 at http://yacy.net
|
||||
* First released 18.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AbstractScoreMap
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Syntax
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
* First released 17.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Vocabulary
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AnnoteaA
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AnnoteaB
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* CreativeCommons
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
* First released 17.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* DMOZ
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* DublinCore
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Foaf
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
* First released 17.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Geo
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* HttpHeader
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Owl
|
||||
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 11.06.2011 at http://yacy.net
|
||||
* First released 11.06.2011 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Rdf
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
* First released 17.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* YaCyMetadata
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
* First released 16.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* AbstractOrder
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 25.08.2011 at http://yacy.net
|
||||
* First released 25.08.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Order
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 25.08.2011 at http://yacy.net
|
||||
* First released 25.08.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* RatingOrder.java
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 25.08.2011 at http://yacy.net
|
||||
* First released 25.08.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ClassProvider
|
||||
* Copyright 201 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
|
||||
* First released 13.12.2011 at http://yacy.net
|
||||
* First released 13.12.2011 at https://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user