mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
d95b288f19
Upgraded to InetAccessHandler. Added InetPathAccessHandler extension to InetAccessHandler to maintain path patterns capability previously available in IPAccessHandler but lost in InetAccessHandler. Filtering on IPv6 addresses is now supported. Support for deprecated pattern formats such as "192.168." and "192.168.1.1/path" has been removed, but startup automated migration should convert such patterns eventually present in serverClient.
906 lines
45 KiB
Java
906 lines
45 KiB
Java
// yacy.java
|
|
// -----------------------
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
// first published on http://www.yacy.net
|
|
// Frankfurt, Germany, 2004, 2005
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
package net.yacy;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.io.FileNotFoundException;
|
|
import java.io.FileOutputStream;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.io.PrintWriter;
|
|
import java.io.RandomAccessFile;
|
|
import java.nio.channels.FileChannel;
|
|
import java.nio.channels.FileLock;
|
|
import java.util.ArrayList;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.Properties;
|
|
import java.util.concurrent.Semaphore;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import org.apache.http.Header;
|
|
import org.apache.http.HttpStatus;
|
|
import org.apache.http.entity.mime.content.ContentBody;
|
|
|
|
import com.google.common.io.Files;
|
|
|
|
import net.yacy.cora.date.GenericFormatter;
|
|
import net.yacy.cora.document.encoding.UTF8;
|
|
import net.yacy.cora.document.id.DigestURL;
|
|
import net.yacy.cora.document.id.MultiProtocolURL;
|
|
import net.yacy.cora.federate.yacy.CacheStrategy;
|
|
import net.yacy.cora.order.Digest;
|
|
import net.yacy.cora.protocol.ClientIdentification;
|
|
import net.yacy.cora.protocol.ConnectionInfo;
|
|
import net.yacy.cora.protocol.HeaderFramework;
|
|
import net.yacy.cora.protocol.TimeoutRequest;
|
|
import net.yacy.cora.protocol.http.HTTPClient;
|
|
import net.yacy.cora.sorting.Array;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.crawler.retrieval.Response;
|
|
import net.yacy.data.TransactionManager;
|
|
import net.yacy.data.Translator;
|
|
import net.yacy.document.Document;
|
|
import net.yacy.gui.YaCyApp;
|
|
import net.yacy.gui.framework.Browser;
|
|
import net.yacy.http.Jetty9HttpServerImpl;
|
|
import net.yacy.http.YaCyHttpServer;
|
|
import net.yacy.kelondro.util.FileUtils;
|
|
import net.yacy.kelondro.util.Formatter;
|
|
import net.yacy.kelondro.util.MemoryControl;
|
|
import net.yacy.kelondro.util.OS;
|
|
import net.yacy.peers.Seed;
|
|
import net.yacy.peers.operation.yacyBuildProperties;
|
|
import net.yacy.peers.operation.yacyRelease;
|
|
import net.yacy.search.Switchboard;
|
|
import net.yacy.search.SwitchboardConstants;
|
|
import net.yacy.server.serverSwitch;
|
|
import net.yacy.utils.translation.TranslatorXliff;
|
|
|
|
|
|
/**
|
|
* This is the main class of YaCy. Several threads are started from here:
|
|
* <ul>
|
|
* <li>one single instance of the plasmaSwitchboard is generated, which itself
|
|
* starts a thread with a plasmaHTMLCache object. This object simply counts
|
|
* files sizes in the cache and terminates them. It also generates a
|
|
* plasmaCrawlerLoader object, which may itself start some more httpc-calling
|
|
* threads to load web pages. They terminate automatically when a page has
|
|
* loaded.
|
|
* <li>one serverCore - thread is started, which implements a multi-threaded
|
|
* server. The process may start itself many more processes that handle
|
|
* connections.lo
|
|
* <li>finally, all idle-dependent processes are written in a queue in
|
|
* plasmaSwitchboard which are worked off inside an idle-sensitive loop of the
|
|
* main process. (here)
|
|
* </ul>
|
|
*
|
|
* On termination, the following must be done:
|
|
* <ul>
|
|
* <li>stop feeding of the crawling process because it otherwise fills the
|
|
* indexing queue.
|
|
* <li>say goodbye to connected peers and disable new connections. Don't wait for
|
|
* success.
|
|
* <li>first terminate the serverCore thread. This prevents that new cache
|
|
* objects are queued.
|
|
* <li>wait that the plasmaHTMLCache terminates (it should be normal that this
|
|
* process already has terminated).
|
|
* <li>then wait for termination of all loader process of the
|
|
* plasmaCrawlerLoader.
|
|
* <li>work off the indexing and cache storage queue. These values are inside a
|
|
* RAM cache and would be lost otherwise.
|
|
* <li>write all settings.
|
|
* <li>terminate.
|
|
* </ul>
|
|
*/
|
|
|
|
public final class yacy {
|
|
|
|
// static objects
|
|
public static final String vString = yacyBuildProperties.getVersion();
|
|
|
|
public static final String vDATE = yacyBuildProperties.getBuildDate();
|
|
public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
|
|
public static final String hline = "-------------------------------------------------------------------------------";
|
|
public static final Semaphore shutdownSemaphore = new Semaphore(0);
|
|
|
|
public static File htDocsPath = null;
|
|
public static File shareDefaultPath = null;
|
|
public static File shareDumpDefaultPath = null;
|
|
|
|
/**
|
|
* a reference to the {@link Switchboard} created by the
|
|
* {@link yacy#startup(String, long, long)} method.
|
|
*/
|
|
private static Switchboard sb = null;
|
|
|
|
/**
|
|
* Starts up the whole application. Sets up all datastructures and starts
|
|
* the main threads.
|
|
*
|
|
* @param homePath Root-path where all information is to be found.
|
|
* @param startupFree free memory at startup time, to be used later for statistics
|
|
*/
|
|
private static void startup(final File dataHome, final File appHome, final long startupMemFree, final long startupMemTotal, final boolean gui) {
|
|
String tmpdir=null;
|
|
try {
|
|
// start up
|
|
System.out.println(copyright);
|
|
System.out.println(hline);
|
|
|
|
// ensure that there is a DATA directory, if not, create one and if that fails warn and die
|
|
mkdirsIfNeseccary(dataHome);
|
|
mkdirsIfNeseccary(appHome);
|
|
File f = new File(dataHome, "DATA/");
|
|
mkdirsIfNeseccary(f);
|
|
if (!(f.exists())) {
|
|
System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate.");
|
|
System.exit(-1);
|
|
}
|
|
|
|
// setting up logging
|
|
f = new File(dataHome, "DATA/LOG/");
|
|
mkdirsIfNeseccary(f);
|
|
f = new File(dataHome, "DATA/LOG/yacy.logging");
|
|
final File f0 = new File(appHome, "defaults/yacy.logging");
|
|
if (!f.exists() || f0.lastModified() > f.lastModified()) try {
|
|
Files.copy(f0, f);
|
|
} catch (final IOException e){
|
|
System.out.println("could not copy yacy.logging");
|
|
}
|
|
try{
|
|
ConcurrentLog.configureLogging(dataHome, appHome, new File(dataHome, "DATA/LOG/yacy.logging"));
|
|
} catch (final IOException e) {
|
|
System.out.println("could not find logging properties in homePath=" + dataHome);
|
|
ConcurrentLog.logException(e);
|
|
}
|
|
ConcurrentLog.config("STARTUP", "YaCy version: " + yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision());
|
|
ConcurrentLog.config("STARTUP", "Java version: " + System.getProperty("java.version", "no-java-version"));
|
|
ConcurrentLog.config("STARTUP", "Operation system: " + System.getProperty("os.name","unknown"));
|
|
ConcurrentLog.config("STARTUP", "Application root-path: " + appHome);
|
|
ConcurrentLog.config("STARTUP", "Data root-path: " + dataHome);
|
|
ConcurrentLog.config("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis());
|
|
ConcurrentLog.config("STARTUP", "Maximum file system path length: " + OS.maxPathLength);
|
|
|
|
f = new File(dataHome, "DATA/yacy.running");
|
|
final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
|
|
if (!f.createNewFile()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be created!");
|
|
try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID
|
|
f.deleteOnExit();
|
|
FileChannel channel = null;
|
|
FileLock lock = null;
|
|
try {
|
|
channel = new RandomAccessFile(f,"rw").getChannel();
|
|
lock = channel.tryLock(); // lock yacy.running
|
|
} catch (final Exception e) { }
|
|
|
|
// set jvm tmpdir to a subdir for easy cleanup (as extensive use file.deleteonexit waists memory during long runs, as todelete files names are collected and never cleaned up during runtime)
|
|
try {
|
|
tmpdir = java.nio.file.Files.createTempDirectory("yacy-tmp-").toString(); // creates sub dir in jvm's temp (see System.property "java.io.tempdir")
|
|
System.setProperty("java.io.tmpdir", tmpdir);
|
|
} catch (IOException ex) { }
|
|
|
|
try {
|
|
sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf);
|
|
} catch (final RuntimeException e) {
|
|
ConcurrentLog.severe("STARTUP", "YaCy cannot start: " + e.getMessage(), e);
|
|
System.exit(-1);
|
|
}
|
|
//sbSync.V(); // signal that the sb reference was set
|
|
|
|
// switch the memory strategy
|
|
MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true));
|
|
|
|
// save information about available memory at startup time
|
|
sb.setConfig("memoryFreeAfterStartup", startupMemFree);
|
|
sb.setConfig("memoryTotalAfterStartup", startupMemTotal);
|
|
|
|
// start gui if wanted
|
|
if (gui) YaCyApp.start("localhost", sb.getLocalPort());
|
|
|
|
// hardcoded, forced, temporary value-migration
|
|
sb.setConfig("htTemplatePath", "htroot/env/templates");
|
|
|
|
double oldVer;
|
|
try {
|
|
String tmpversion = sb.getConfig(Seed.VERSION, "");
|
|
if (tmpversion.isEmpty()) { // before 1.83009737 only the svnRevision nr was in config (like 9737)
|
|
tmpversion = yacyBuildProperties.getVersion();
|
|
int oldRev = Integer.parseInt(sb.getConfig("svnRevision", "0"));
|
|
if (oldRev > 1) {
|
|
oldVer = Double.parseDouble(tmpversion) + oldRev / 100000000.0;
|
|
} else {
|
|
oldVer = Double.parseDouble(yacyBuildProperties.getLongVersion()); // failsafe (assume current version = no migration)
|
|
}
|
|
} else {
|
|
oldVer = Double.parseDouble(tmpversion);
|
|
}
|
|
} catch (final NumberFormatException e) {
|
|
oldVer = 0.0d;
|
|
}
|
|
final double newRev = Double.parseDouble(yacyBuildProperties.getLongVersion());
|
|
sb.setConfig(Seed.VERSION, yacyBuildProperties.getLongVersion());
|
|
sb.setConfig("applicationRoot", appHome.toString());
|
|
sb.setConfig("dataRoot", dataHome.toString());
|
|
|
|
// create some directories
|
|
final File htRootPath = new File(appHome, sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
|
|
mkdirIfNeseccary(htRootPath);
|
|
htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
|
|
mkdirIfNeseccary(htDocsPath);
|
|
//final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
|
|
|
|
// copy the donate iframe (better to copy this once here instead of doing this in an actual iframe in the search result)
|
|
importDonationIFrame(sb, htDocsPath);
|
|
|
|
// create default notifier picture
|
|
File notifierFile = new File(htDocsPath, "notifier.gif");
|
|
if (!notifierFile.exists()) try {Files.copy(new File(htRootPath, "env/grafics/empty.gif"), notifierFile);} catch (final IOException e) {}
|
|
|
|
final File htdocsReadme = new File(htDocsPath, "readme.txt");
|
|
if (!(htdocsReadme.exists())) try {FileUtils.copy((
|
|
"This is your root directory for individual Web Content\r\n" +
|
|
"\r\n" +
|
|
"Please place your html files into the www subdirectory.\r\n" +
|
|
"The URL of that path is either\r\n" +
|
|
"http://www.<your-peer-name>.yacy or\r\n" +
|
|
"http://<your-ip>:<your-port>/www\r\n" +
|
|
"\r\n" +
|
|
"Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
|
|
"This directory shares it's content with the applications htroot path, so you\r\n" +
|
|
"may access your yacy search page with\r\n" +
|
|
"http://<your-peer-name>.yacy/\r\n" +
|
|
"\r\n").getBytes(), htdocsReadme);} catch (final IOException e) {
|
|
System.out.println("Error creating htdocs readme: " + e.getMessage());
|
|
}
|
|
|
|
shareDefaultPath = new File(htDocsPath, "share");
|
|
mkdirIfNeseccary(shareDefaultPath);
|
|
shareDumpDefaultPath = new File(shareDefaultPath, "dump");
|
|
mkdirIfNeseccary(shareDumpDefaultPath);
|
|
|
|
migration.migrate(sb, oldVer, newRev);
|
|
|
|
// delete old release files
|
|
final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
|
|
yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
|
|
|
|
// set user-agent
|
|
HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
|
|
|
|
// start main threads
|
|
final int port = sb.getLocalPort();
|
|
try {
|
|
// start http server
|
|
YaCyHttpServer httpServer;
|
|
httpServer = new Jetty9HttpServerImpl(port);
|
|
httpServer.startupServer();
|
|
sb.setHttpServer(httpServer);
|
|
// TODO: this has no effect on Jetty (but needed to reflect configured value and limit is still used)
|
|
ConnectionInfo.setServerMaxcount(sb.getConfigInt("connectionsMax", ConnectionInfo.getMaxcount()));
|
|
|
|
ConcurrentLog.info("STARTUP",httpServer.getVersion());
|
|
|
|
// open the browser window
|
|
final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
|
|
if (browserPopUpTrigger) try {
|
|
final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
|
|
//boolean properPW = (sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT, "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
|
|
//if (!properPW) browserPopUpPage = "ConfigBasic.html";
|
|
/* YaCy main startup process must not hang because browser opening is long or fails.
|
|
* Let's open try opening the browser in a separate thread */
|
|
new Thread("Browser opening") {
|
|
@Override
|
|
public void run() {
|
|
Browser.openBrowser(("http://localhost:"+port) + "/" + browserPopUpPage);
|
|
}
|
|
}.start();
|
|
// Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage);
|
|
} catch (final Throwable e) {
|
|
// cannot open browser. This may be normal in headless environments
|
|
//Log.logException(e);
|
|
}
|
|
|
|
// enable browser popup, http server is ready now
|
|
sb.tray.setReady();
|
|
|
|
//regenerate Locales from Translationlist, if needed
|
|
final File locale_source = sb.getAppPath("locale.source", "locales");
|
|
final String lang = sb.getConfig("locale.language", "");
|
|
// on lang=browser all active translation should be checked (because any could be requested by client)
|
|
List<String> langlist;
|
|
if (lang.endsWith("browser"))
|
|
langlist = Translator.activeTranslations(); // get all translated languages
|
|
else {
|
|
langlist = new ArrayList<String>();
|
|
langlist.add(lang);
|
|
}
|
|
for (String tmplang : langlist) {
|
|
if (!tmplang.equals("") && !tmplang.equals("default") && !tmplang.equals("browser")) { //locale is used
|
|
String currentRev = null;
|
|
BufferedReader br = null;
|
|
try {
|
|
br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang + "/version"))));
|
|
currentRev = br.readLine(); // may return null
|
|
} catch (final IOException e) {
|
|
//Error
|
|
} finally {
|
|
try {
|
|
br.close();
|
|
} catch(IOException ioe) {
|
|
ConcurrentLog.warn("STARTUP", "Could not close " + tmplang + " version file");
|
|
}
|
|
}
|
|
|
|
if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) {
|
|
try { //is this another version?!
|
|
final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
|
|
final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang);
|
|
if (new TranslatorXliff().translateFilesRecursive(sourceDir, destDir, new File(locale_source, tmplang + ".lng"), "html,template,inc", "locale")) { //translate it
|
|
//write the new Versionnumber
|
|
final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
|
|
bw.write(sb.getConfig(Seed.VERSION, "Error getting Version"));
|
|
bw.close();
|
|
}
|
|
} catch (final IOException e) {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// initialize number formatter with this locale
|
|
if (!lang.equals("browser")) // "default" is handled by .setLocale()
|
|
Formatter.setLocale(lang);
|
|
|
|
// registering shutdown hook
|
|
ConcurrentLog.config("STARTUP", "Registering Shutdown Hook");
|
|
final Runtime run = Runtime.getRuntime();
|
|
run.addShutdownHook(new shutdownHookThread(sb, shutdownSemaphore));
|
|
|
|
// save information about available memory after all initializations
|
|
//try {
|
|
sb.setConfig("memoryFreeAfterInitBGC", MemoryControl.free());
|
|
sb.setConfig("memoryTotalAfterInitBGC", MemoryControl.total());
|
|
System.gc();
|
|
sb.setConfig("memoryFreeAfterInitAGC", MemoryControl.free());
|
|
sb.setConfig("memoryTotalAfterInitAGC", MemoryControl.total());
|
|
//} catch (final ConcurrentModificationException e) {}
|
|
|
|
// wait for server shutdown
|
|
try {
|
|
sb.waitForShutdown();
|
|
} catch (final Exception e) {
|
|
ConcurrentLog.severe("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
|
|
}
|
|
// shut down
|
|
Array.terminate();
|
|
ConcurrentLog.config("SHUTDOWN", "caught termination signal");
|
|
httpServer.stop();
|
|
|
|
ConcurrentLog.config("SHUTDOWN", "server has terminated");
|
|
sb.close();
|
|
} catch (final Exception e) {
|
|
ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
|
|
//System.exit(1);
|
|
}
|
|
if(lock != null) lock.release();
|
|
if(channel != null) channel.close();
|
|
} catch (final Exception ee) {
|
|
ConcurrentLog.severe("STARTUP", "FATAL ERROR: " + ee.getMessage(),ee);
|
|
} finally {
|
|
}
|
|
|
|
if (tmpdir != null) FileUtils.deletedelete(new File(tmpdir)); // clean up temp dir (set at startup as subdir of system.propery "java.io.tmpdir")
|
|
|
|
ConcurrentLog.config("SHUTDOWN", "goodbye. (this is the last line)");
|
|
ConcurrentLog.shutdown();
|
|
shutdownSemaphore.release(1000);
|
|
try {
|
|
System.exit(0);
|
|
} catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
|
|
}
|
|
|
|
/**
|
|
* Concurrently import the donation iframe content to serve it directly from this peer.
|
|
* @param switchBoard the SwitchBoard instance. Must not be null.
|
|
* @param htDocsDirectory the custom htdocs directory. Must not be null.
|
|
*/
|
|
private static void importDonationIFrame(final Switchboard switchBoard, final File htDocsDirectory) {
|
|
final File wwwEnvPath = new File(htDocsDirectory, "env");
|
|
mkdirIfNeseccary(wwwEnvPath);
|
|
final String iframesource = switchBoard.getConfig("donation.iframesource", "");
|
|
final String iframetarget = switchBoard.getConfig("donation.iframetarget", "");
|
|
final File iframefile = new File(htDocsDirectory, iframetarget);
|
|
if (!iframefile.exists()) new Thread("yacy.importDonationIFrame") {
|
|
@Override
|
|
public void run() {
|
|
final ClientIdentification.Agent agent = ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName);
|
|
Response documentResponse;
|
|
try {
|
|
/* Load the donation html frame content */
|
|
documentResponse = switchBoard.loader == null ? null : switchBoard.loader.load(switchBoard.loader.request(new DigestURL(iframesource), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent);
|
|
if (documentResponse != null) {
|
|
Document[] documents = documentResponse.parse();
|
|
if(documents != null && documents.length > 0 && documents[0] != null) {
|
|
Document donateDocument = documents[0];
|
|
String donateDocContent = new String(documentResponse.getContent(), donateDocument.getCharset());
|
|
/* Load image resources contained in the page */
|
|
if(donateDocument.getImages() != null) {
|
|
for(DigestURL imgURL : donateDocument.getImages().keySet()) {
|
|
try {
|
|
Response response = switchBoard.loader.load(switchBoard.loader.request(imgURL, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent);
|
|
if (response != null) {
|
|
String imgFileName = imgURL.getFileName();
|
|
/* Store each image in the same directory as the iframe target file */
|
|
FileUtils.copy(response.getContent(), new File(iframefile.getParentFile(), imgFileName));
|
|
|
|
/* Transform the original image URL to a relative one */
|
|
donateDocContent = donateDocContent.replace(imgURL.getURL().toString(), imgFileName);
|
|
}
|
|
} catch(IOException e) {
|
|
/* Failing to load one image should not stop the whole task */
|
|
ConcurrentLog.warn("STARTUP", "Donation frame retrieval : could not get an image resource.", e);
|
|
}
|
|
}
|
|
}
|
|
FileUtils.copy(donateDocContent.getBytes(donateDocument.getCharset()), iframefile);
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
ConcurrentLog.warn("STARTUP", "Could not retrieve donation frame content.", e);
|
|
}
|
|
}
|
|
}.start();
|
|
}
|
|
|
|
/**
|
|
* @param f
|
|
*/
|
|
private static void delete(final File f) {
|
|
if(!f.delete())
|
|
ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be deleted!");
|
|
}
|
|
|
|
/**
|
|
* @see File#mkdir()
|
|
* @param path
|
|
*/
|
|
private static void mkdirIfNeseccary(final File path) {
|
|
if (!(path.exists()))
|
|
if(!path.mkdir())
|
|
ConcurrentLog.warn("STARTUP", "could not create directory "+ path.toString());
|
|
}
|
|
|
|
/**
|
|
* @see File#mkdirs()
|
|
* @param path
|
|
*/
|
|
public static void mkdirsIfNeseccary(final File path) {
|
|
if (!(path.exists()))
|
|
if(!path.mkdirs())
|
|
ConcurrentLog.warn("STARTUP", "could not create directories "+ path.toString());
|
|
}
|
|
|
|
/**
|
|
* Loads the configuration from the data-folder.
|
|
* FIXME: Why is this called over and over again from every method, instead
|
|
* of setting the configurationdata once for this class in main?
|
|
*
|
|
* @param mes Where are we called from, so that the errormessages can be
|
|
* more descriptive.
|
|
* @param homePath Root-path where all the information is to be found.
|
|
* @return Properties read from the configurationfile.
|
|
*/
|
|
private static Properties configuration(final String mes, final File homePath) {
|
|
ConcurrentLog.config(mes, "Application Root Path: " + homePath.toString());
|
|
|
|
// read data folder
|
|
final File dataFolder = new File(homePath, "DATA");
|
|
if (!(dataFolder.exists())) {
|
|
ConcurrentLog.severe(mes, "Application was never started or root path wrong.");
|
|
System.exit(-1);
|
|
}
|
|
|
|
final Properties config = new Properties();
|
|
FileInputStream fis = null;
|
|
try {
|
|
fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf"));
|
|
config.load(fis);
|
|
} catch (final FileNotFoundException e) {
|
|
ConcurrentLog.severe(mes, "could not find configuration file.");
|
|
System.exit(-1);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe(mes, "could not read configuration file.");
|
|
System.exit(-1);
|
|
} finally {
|
|
if(fis != null) {
|
|
try {
|
|
fis.close();
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.logException(e);
|
|
}
|
|
}
|
|
}
|
|
|
|
return config;
|
|
}
|
|
|
|
/**
|
|
* Call the shutdown-page of YaCy to tell it to shut down. This method is
|
|
* called if you start yacy with the argument -shutdown.
|
|
*
|
|
* @param homePath Root-path where all the information is to be found.
|
|
*/
|
|
public static void shutdown(final File homePath) {
|
|
// start up
|
|
System.out.println(copyright);
|
|
System.out.println(hline);
|
|
|
|
final LinkedHashMap<String,ContentBody> post = new LinkedHashMap<String,ContentBody>();
|
|
post.put("shutdown", UTF8.StringBody(""));
|
|
submitPostURL(homePath, "Steering.html", "Terminate YaCy", post);
|
|
}
|
|
|
|
public static void update(final File homePath) {
|
|
// start up
|
|
System.out.println(copyright);
|
|
System.out.println(hline);
|
|
submitURL(homePath, "ConfigUpdate_p.html?autoUpdate=", "Update YaCy to most recent version");
|
|
}
|
|
|
|
/**
|
|
* Submits post data to the local peer URL, authenticating as administrator
|
|
* @param homePath directory containing YaCy DATA folder
|
|
* @param path url relative path part
|
|
* @param processdescription description of the operation for logging purpose
|
|
* @param post data to post
|
|
*/
|
|
private static void submitPostURL(final File homePath, final String path, final String processdescription, final Map<String, ContentBody> post) {
|
|
final Properties config = configuration("COMMAND-STEERING", homePath);
|
|
|
|
// read port
|
|
final int port = Integer.parseInt(config.getProperty(SwitchboardConstants.SERVER_PORT, "8090"));
|
|
|
|
// read password
|
|
final String encodedPassword = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");
|
|
final String adminUser = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
|
|
|
|
// send 'wget' to web interface
|
|
final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
|
|
// con.setHeader(requestHeader.entrySet());
|
|
try {
|
|
/* First get a valid transaction token using HTTP GET */
|
|
con.GETbytes("http://localhost:"+ port +"/" + path, adminUser, encodedPassword, false);
|
|
|
|
if (con.getStatusCode() != HttpStatus.SC_OK) {
|
|
throw new IOException("Error response from YACY socket: " + con.getHttpResponse().getStatusLine());
|
|
}
|
|
|
|
final Header transactionTokenHeader = con.getHttpResponse().getFirstHeader(HeaderFramework.X_YACY_TRANSACTION_TOKEN);
|
|
if(transactionTokenHeader == null) {
|
|
throw new IOException("Could not retrieve a valid transaction token");
|
|
}
|
|
|
|
/* Then POST the request */
|
|
post.put(TransactionManager.TRANSACTION_TOKEN_PARAM, UTF8.StringBody(transactionTokenHeader.getValue()));
|
|
con.POSTbytes(new MultiProtocolURL("http://localhost:"+ port +"/" + path), null, post, adminUser, encodedPassword, false, false);
|
|
if (con.getStatusCode() >= HttpStatus.SC_OK && con.getStatusCode() < HttpStatus.SC_MULTIPLE_CHOICES) {
|
|
ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
|
|
} else {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e1) {
|
|
e1.printStackTrace();
|
|
}
|
|
|
|
System.exit(-1);
|
|
}
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e1) {
|
|
e1.printStackTrace();
|
|
}
|
|
|
|
System.exit(-1);
|
|
}
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
// finished
|
|
ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
|
|
}
|
|
|
|
private static void submitURL(final File homePath, final String path, final String processdescription) {
|
|
final Properties config = configuration("COMMAND-STEERING", homePath);
|
|
|
|
// read port
|
|
final int port = Integer.parseInt(config.getProperty(SwitchboardConstants.SERVER_PORT, "8090"));
|
|
|
|
// read password
|
|
String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5);
|
|
if (encodedPassword == null) encodedPassword = ""; // not defined
|
|
|
|
// send 'wget' to web interface
|
|
final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
|
|
// con.setHeader(requestHeader.entrySet());
|
|
try {
|
|
con.GETbytes("http://localhost:"+ port +"/" + path, config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword, false);
|
|
if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
|
|
ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
|
|
|
|
} else {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
|
|
System.exit(-1);
|
|
}
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
|
|
System.exit(-1);
|
|
}
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
// finished
|
|
ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
|
|
}
|
|
|
|
/**
|
|
* read saved config file and perform action which need to be done before main task starts
|
|
* - like check on system alrady running etc.
|
|
*
|
|
* @param dataHome data directory
|
|
*/
|
|
static private void preReadSavedConfigandInit(File dataHome) {
|
|
File lockFile = new File(dataHome, "DATA/yacy.running");
|
|
final String conf = "DATA/SETTINGS/yacy.conf";
|
|
|
|
// If YaCy is actually running, then we check if the server port is open.
|
|
// If yes, then we consider that a restart is a user mistake and then we just respond
|
|
// as the user expects and tell the browser to open the start page.
|
|
// That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply
|
|
// open the web interface. (They don't think of 'servers' they just want to get to the search page).
|
|
// We need to parse the configuration file for that to get the host port
|
|
File configFile = new File(dataHome, conf);
|
|
|
|
if (configFile.exists()) {
|
|
Properties p = new Properties();
|
|
FileInputStream fis = null;
|
|
try {
|
|
fis = new FileInputStream(configFile);
|
|
p.load(fis);
|
|
|
|
// test for yacy already running
|
|
if (lockFile.exists()) { // another instance running? VM crash? User will have to care about this
|
|
//standard log system not up yet - use simply stdout
|
|
// prevents also creation of a log file while just opening browser
|
|
System.out.println("WARNING: the file " + lockFile + " exists, this usually means that a YaCy instance is still running. If you want to restart YaCy, try first ./stopYACY.sh, then ./startYACY.sh. If ./stopYACY.sh fails, try ./killYACY.sh");
|
|
|
|
int port = Integer.parseInt(p.getProperty(SwitchboardConstants.SERVER_PORT, "8090"));
|
|
if (TimeoutRequest.ping("127.0.0.1", port, 1000)) {
|
|
Browser.openBrowser("http://localhost:" + port + "/" + p.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"));
|
|
// Thats it; YaCy was running, the user is happy, we can stop now.
|
|
System.out.println("WARNING: YaCy instance was still running; just opening the browser and exit.");
|
|
System.exit(0);
|
|
} else {
|
|
// YaCy is not running; thus delete the file an go on as nothing was wrong.
|
|
System.err.println("INFO: delete old yacy.running file; likely previous YaCy session was not orderly shutdown!");
|
|
delete(lockFile);
|
|
}
|
|
}
|
|
} catch (IOException ex) {
|
|
System.err.println("ERROR: config file seems to be corrupt");
|
|
System.err.println("ERROR: if problem persists, delete file");
|
|
System.err.println(configFile.getAbsolutePath());
|
|
ConcurrentLog.logException(ex);
|
|
ConcurrentLog.severe("Startup", "cannot read " + configFile.toString() + ", please delete the corrupted file if problem persits");
|
|
} finally {
|
|
try {
|
|
fis.close();
|
|
} catch (IOException e) {
|
|
ConcurrentLog.warn("Startup", "Could not close file " + configFile);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Main-method which is started by java. Checks for special arguments or
|
|
* starts up the application.
|
|
*
|
|
* @param args
|
|
* Given arguments from the command line.
|
|
*/
|
|
public static void main(String args[]) {
|
|
|
|
try {
|
|
// check assertion status
|
|
//ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
|
|
boolean assertionenabled = false;
|
|
assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement"
|
|
if (assertionenabled) System.out.println("Asserts are enabled");
|
|
|
|
// check memory amount
|
|
System.gc();
|
|
final long startupMemFree = MemoryControl.free();
|
|
final long startupMemTotal = MemoryControl.total();
|
|
|
|
// maybe go into headless awt mode: we have three cases depending on OS and one exception:
|
|
// windows : better do not go into headless mode
|
|
// mac : go into headless mode because an application is shown in gui which may not be wanted
|
|
// linux : go into headless mode because this does not need any head operation
|
|
// exception : if the -gui option is used then do not go into headless mode since that uses a gui
|
|
boolean headless = true;
|
|
if (OS.isWindows) headless = false;
|
|
if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) headless = false;
|
|
System.setProperty("java.awt.headless", headless ? "true" : "false");
|
|
|
|
String s = ""; for (final String a: args) s += a + " ";
|
|
yacyRelease.startParameter = s.trim();
|
|
|
|
File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
|
|
File dataRoot = applicationRoot;
|
|
//System.out.println("args.length=" + args.length);
|
|
//System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
|
|
if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-startup") || args[0].equals("-start"))) {
|
|
// normal start-up of yacy
|
|
if (args.length > 1) {
|
|
dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
|
|
}
|
|
preReadSavedConfigandInit(dataRoot);
|
|
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
|
|
} else if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) {
|
|
// start-up of yacy with gui
|
|
if (args.length > 1) {
|
|
dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
|
|
}
|
|
preReadSavedConfigandInit(dataRoot);
|
|
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true);
|
|
} else if ((args.length >= 1) && ((args[0].toLowerCase(Locale.ROOT).equals("-shutdown")) || (args[0].equals("-stop")))) {
|
|
// normal shutdown of yacy
|
|
if (args.length == 2) applicationRoot= new File(args[1]);
|
|
shutdown(applicationRoot);
|
|
} else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-update"))) {
|
|
// aut-update yacy
|
|
if (args.length == 2) applicationRoot= new File(args[1]);
|
|
update(applicationRoot);
|
|
} else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-version"))) {
|
|
// show yacy version
|
|
System.out.println(copyright);
|
|
} else if ((args.length > 1) && (args[0].toLowerCase(Locale.ROOT).equals("-config"))) {
|
|
// set config parameter. Special handling of adminAccount=user:pwd (generates md5 encoded password)
|
|
// on Windows parameter should be enclosed in doublequotes to accept = sign (e.g. -config "port=8090" "port.ssl=8043")
|
|
File f = new File (dataRoot,"DATA/SETTINGS/");
|
|
if (!f.exists()) {
|
|
mkdirsIfNeseccary(f);
|
|
} else {
|
|
if (new File(dataRoot, "DATA/yacy.running").exists()) {
|
|
System.out.println("please restart YaCy");
|
|
}
|
|
}
|
|
// use serverSwitch to read config properties (including init values from yacy.init
|
|
serverSwitch ss = new serverSwitch(dataRoot,applicationRoot,"defaults/yacy.init","DATA/SETTINGS/yacy.conf");
|
|
|
|
for (int icnt=1; icnt < args.length ; icnt++) {
|
|
String cfg = args[icnt];
|
|
int pos = cfg.indexOf('=');
|
|
if (pos > 0) {
|
|
String cmd = cfg.substring(0, pos);
|
|
String val = cfg.substring(pos + 1);
|
|
|
|
if (!val.isEmpty()) {
|
|
if (cmd.equalsIgnoreCase(SwitchboardConstants.ADMIN_ACCOUNT)) { // special command to set adminusername and md5-pwd
|
|
int cpos = val.indexOf(':'); //format adminAccount=adminname:adminpwd
|
|
if (cpos >= 0) {
|
|
String username = val.substring(0, cpos);
|
|
String pwdtxt = val.substring(cpos + 1);
|
|
if (!username.isEmpty()) {
|
|
ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, username);
|
|
System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME + " = " + username);
|
|
} else {
|
|
username = ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
|
|
}
|
|
ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "MD5:" + Digest.encodeMD5Hex(username + ":" + ss.getConfig(SwitchboardConstants.ADMIN_REALM, "YaCy") + ":" + pwdtxt));
|
|
System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_B64MD5 + " = " + ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
|
|
}
|
|
} else {
|
|
ss.setConfig(cmd, val);
|
|
System.out.println("Set property " + cmd + " = " + val);
|
|
}
|
|
}
|
|
} else {
|
|
System.out.println("skip parameter " + cfg + " (equal sign missing, put parameter in doublequotes)");
|
|
}
|
|
System.out.println();
|
|
}
|
|
} else {
|
|
if (args.length == 1) {
|
|
applicationRoot= new File(args[0]);
|
|
}
|
|
preReadSavedConfigandInit(dataRoot);
|
|
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
|
|
}
|
|
} finally {
|
|
ConcurrentLog.shutdown();
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This class is a helper class whose instance is started, when the java virtual
|
|
* machine shuts down. Signals the plasmaSwitchboard to shut down.
|
|
*/
|
|
class shutdownHookThread extends Thread {
|
|
private final Switchboard sb;
|
|
private final Semaphore shutdownSemaphore;
|
|
|
|
public shutdownHookThread(final Switchboard sb, final Semaphore shutdownSemaphore) {
|
|
super("yacy.shutdownHookThread");
|
|
this.sb = sb;
|
|
this.shutdownSemaphore = shutdownSemaphore;
|
|
}
|
|
|
|
@Override
|
|
public void run() {
|
|
try {
|
|
if (!this.sb.isTerminated()) {
|
|
ConcurrentLog.config("SHUTDOWN","Shutdown via shutdown hook.");
|
|
/* Print also to the standard output, as the LogManager is likely to have
|
|
* been concurrently reset by its own shutdown hook thread */
|
|
System.out.println("SHUTDOWN Starting shutdown via shutdown hook.");
|
|
|
|
// sending the yacy main thread a shutdown signal
|
|
ConcurrentLog.fine("SHUTDOWN","Signaling shutdown to the switchboard.");
|
|
this.sb.terminate("shutdown hook");
|
|
|
|
// waiting for the yacy thread to finish execution
|
|
ConcurrentLog.fine("SHUTDOWN","Waiting for main thread to finish.");
|
|
|
|
/* Main thread will release the shutdownSemaphore once completely terminated.
|
|
* We do not wait indefinitely as the application is supposed here to quickly terminate */
|
|
this.shutdownSemaphore.tryAcquire(30, TimeUnit.SECONDS);
|
|
}
|
|
} catch (final Exception e) {
|
|
ConcurrentLog.severe("SHUTDOWN","Unexpected error. " + e.getClass().getName(),e);
|
|
}
|
|
}
|
|
}
|