2013-09-22 02:21:24 +02:00
// yacy.java
// -----------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.yacy.net
// Frankfurt, Germany, 2004, 2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy ;
import java.io.BufferedReader ;
import java.io.BufferedWriter ;
import java.io.File ;
import java.io.FileInputStream ;
import java.io.FileNotFoundException ;
import java.io.FileOutputStream ;
import java.io.FileWriter ;
import java.io.IOException ;
import java.io.InputStreamReader ;
import java.io.PrintWriter ;
import java.io.RandomAccessFile ;
import java.nio.channels.FileChannel ;
import java.nio.channels.FileLock ;
2016-08-23 00:04:24 +02:00
import java.util.ArrayList ;
import java.util.List ;
2013-09-22 02:21:24 +02:00
import java.util.Properties ;
2014-05-11 01:53:11 +02:00
import java.util.concurrent.Semaphore ;
2016-12-28 09:47:27 +01:00
import java.util.concurrent.TimeUnit ;
2016-08-17 15:15:31 +02:00
import com.google.common.io.Files ;
2013-09-22 02:21:24 +02:00
import net.yacy.cora.date.GenericFormatter ;
2016-08-17 15:15:31 +02:00
import net.yacy.cora.document.id.DigestURL ;
import net.yacy.cora.federate.yacy.CacheStrategy ;
import net.yacy.cora.order.Digest ;
2013-09-22 02:21:24 +02:00
import net.yacy.cora.protocol.ClientIdentification ;
2016-08-17 15:15:31 +02:00
import net.yacy.cora.protocol.ConnectionInfo ;
2013-09-22 02:21:24 +02:00
import net.yacy.cora.protocol.TimeoutRequest ;
import net.yacy.cora.protocol.http.HTTPClient ;
import net.yacy.cora.sorting.Array ;
import net.yacy.cora.util.ConcurrentLog ;
2016-08-17 15:15:31 +02:00
import net.yacy.crawler.retrieval.Response ;
2016-08-23 00:04:24 +02:00
import net.yacy.data.Translator ;
2016-08-17 15:15:31 +02:00
import net.yacy.document.Document ;
2013-09-22 02:21:24 +02:00
import net.yacy.gui.YaCyApp ;
import net.yacy.gui.framework.Browser ;
2014-05-11 01:53:11 +02:00
import net.yacy.http.Jetty9HttpServerImpl ;
2013-10-09 00:40:48 +02:00
import net.yacy.http.YaCyHttpServer ;
2013-09-22 02:21:24 +02:00
import net.yacy.kelondro.util.FileUtils ;
import net.yacy.kelondro.util.Formatter ;
import net.yacy.kelondro.util.MemoryControl ;
import net.yacy.kelondro.util.OS ;
2016-08-17 15:15:31 +02:00
import net.yacy.peers.Seed ;
2013-09-22 02:21:24 +02:00
import net.yacy.peers.operation.yacyBuildProperties ;
import net.yacy.peers.operation.yacyRelease ;
import net.yacy.search.Switchboard ;
import net.yacy.search.SwitchboardConstants ;
2014-02-24 05:16:31 +01:00
import net.yacy.server.serverSwitch ;
2016-06-03 01:46:30 +02:00
import net.yacy.utils.translation.TranslatorXliff ;
2013-09-22 02:21:24 +02:00
/ * *
* This is the main class of YaCy . Several threads are started from here :
* < ul >
* < li > one single instance of the plasmaSwitchboard is generated , which itself
* starts a thread with a plasmaHTMLCache object . This object simply counts
* files sizes in the cache and terminates them . It also generates a
* plasmaCrawlerLoader object , which may itself start some more httpc - calling
* threads to load web pages . They terminate automatically when a page has
* loaded .
* < li > one serverCore - thread is started , which implements a multi - threaded
* server . The process may start itself many more processes that handle
* connections . lo
* < li > finally , all idle - dependent processes are written in a queue in
* plasmaSwitchboard which are worked off inside an idle - sensitive loop of the
* main process . ( here )
* < / ul >
*
* On termination , the following must be done :
* < ul >
* < li > stop feeding of the crawling process because it otherwise fills the
* indexing queue .
* < li > say goodbye to connected peers and disable new connections . Don ' t wait for
* success .
* < li > first terminate the serverCore thread . This prevents that new cache
* objects are queued .
* < li > wait that the plasmaHTMLCache terminates ( it should be normal that this
* process already has terminated ) .
* < li > then wait for termination of all loader process of the
* plasmaCrawlerLoader .
* < li > work off the indexing and cache storage queue . These values are inside a
* RAM cache and would be lost otherwise .
* < li > write all settings .
* < li > terminate .
* < / ul >
* /
public final class yacy {
// static objects
public static final String vString = yacyBuildProperties . getVersion ( ) ;
public static final String vDATE = yacyBuildProperties . getBuildDate ( ) ;
public static final String copyright = " [ YaCy v " + vString + " , build " + vDATE + " by Michael Christen / www.yacy.net ] " ;
public static final String hline = " ------------------------------------------------------------------------------- " ;
public static final Semaphore shutdownSemaphore = new Semaphore ( 0 ) ;
2016-03-11 18:12:13 +01:00
public static File htDocsPath = null ;
public static File shareDefaultPath = null ;
public static File shareDumpDefaultPath = null ;
2013-09-22 02:21:24 +02:00
/ * *
* a reference to the { @link Switchboard } created by the
* { @link yacy # startup ( String , long , long ) } method .
* /
private static Switchboard sb = null ;
/ * *
* Starts up the whole application . Sets up all datastructures and starts
* the main threads .
*
* @param homePath Root - path where all information is to be found .
* @param startupFree free memory at startup time , to be used later for statistics
* /
private static void startup ( final File dataHome , final File appHome , final long startupMemFree , final long startupMemTotal , final boolean gui ) {
2015-11-17 22:27:07 +01:00
String tmpdir = null ;
2013-09-22 02:21:24 +02:00
try {
// start up
System . out . println ( copyright ) ;
System . out . println ( hline ) ;
// ensure that there is a DATA directory, if not, create one and if that fails warn and die
mkdirsIfNeseccary ( dataHome ) ;
mkdirsIfNeseccary ( appHome ) ;
File f = new File ( dataHome , " DATA/ " ) ;
mkdirsIfNeseccary ( f ) ;
if ( ! ( f . exists ( ) ) ) {
System . err . println ( " Error creating DATA-directory in " + dataHome . toString ( ) + " . Please check your write-permission for this folder. YaCy will now terminate. " ) ;
System . exit ( - 1 ) ;
}
// setting up logging
f = new File ( dataHome , " DATA/LOG/ " ) ;
mkdirsIfNeseccary ( f ) ;
f = new File ( dataHome , " DATA/LOG/yacy.logging " ) ;
final File f0 = new File ( appHome , " defaults/yacy.logging " ) ;
if ( ! f . exists ( ) | | f0 . lastModified ( ) > f . lastModified ( ) ) try {
Files . copy ( f0 , f ) ;
} catch ( final IOException e ) {
System . out . println ( " could not copy yacy.logging " ) ;
}
try {
ConcurrentLog . configureLogging ( dataHome , appHome , new File ( dataHome , " DATA/LOG/yacy.logging " ) ) ;
} catch ( final IOException e ) {
System . out . println ( " could not find logging properties in homePath= " + dataHome ) ;
ConcurrentLog . logException ( e ) ;
}
ConcurrentLog . config ( " STARTUP " , " YaCy version: " + yacyBuildProperties . getVersion ( ) + " / " + yacyBuildProperties . getSVNRevision ( ) ) ;
ConcurrentLog . config ( " STARTUP " , " Java version: " + System . getProperty ( " java.version " , " no-java-version " ) ) ;
ConcurrentLog . config ( " STARTUP " , " Operation system: " + System . getProperty ( " os.name " , " unknown " ) ) ;
ConcurrentLog . config ( " STARTUP " , " Application root-path: " + appHome ) ;
ConcurrentLog . config ( " STARTUP " , " Data root-path: " + dataHome ) ;
ConcurrentLog . config ( " STARTUP " , " Time zone: UTC " + GenericFormatter . UTCDiffString ( ) + " ; UTC+0000 is " + System . currentTimeMillis ( ) ) ;
ConcurrentLog . config ( " STARTUP " , " Maximum file system path length: " + OS . maxPathLength ) ;
f = new File ( dataHome , " DATA/yacy.running " ) ;
final String conf = " DATA/SETTINGS/yacy.conf " . replace ( " / " , File . separator ) ;
if ( ! f . createNewFile ( ) ) ConcurrentLog . severe ( " STARTUP " , " WARNING: the file " + f + " can not be created! " ) ;
try { new FileOutputStream ( f ) . write ( Integer . toString ( OS . getPID ( ) ) . getBytes ( ) ) ; } catch ( final Exception e ) { } // write PID
f . deleteOnExit ( ) ;
FileChannel channel = null ;
FileLock lock = null ;
try {
channel = new RandomAccessFile ( f , " rw " ) . getChannel ( ) ;
lock = channel . tryLock ( ) ; // lock yacy.running
} catch ( final Exception e ) { }
2015-11-17 22:27:07 +01:00
// set jvm tmpdir to a subdir for easy cleanup (as extensive use file.deleteonexit waists memory during long runs, as todelete files names are collected and never cleaned up during runtime)
try {
tmpdir = java . nio . file . Files . createTempDirectory ( " yacy-tmp- " ) . toString ( ) ; // creates sub dir in jvm's temp (see System.property "java.io.tempdir")
System . setProperty ( " java.io.tmpdir " , tmpdir ) ;
} catch ( IOException ex ) { }
2013-09-22 02:21:24 +02:00
try {
sb = new Switchboard ( dataHome , appHome , " defaults/yacy.init " . replace ( " / " , File . separator ) , conf ) ;
} catch ( final RuntimeException e ) {
ConcurrentLog . severe ( " STARTUP " , " YaCy cannot start: " + e . getMessage ( ) , e ) ;
System . exit ( - 1 ) ;
}
//sbSync.V(); // signal that the sb reference was set
// switch the memory strategy
MemoryControl . setStandardStrategy ( sb . getConfigBool ( " memory.standardStrategy " , true ) ) ;
// save information about available memory at startup time
sb . setConfig ( " memoryFreeAfterStartup " , startupMemFree ) ;
sb . setConfig ( " memoryTotalAfterStartup " , startupMemTotal ) ;
// start gui if wanted
2015-10-31 23:09:03 +01:00
if ( gui ) YaCyApp . start ( " localhost " , sb . getLocalPort ( ) ) ;
2013-09-22 02:21:24 +02:00
// hardcoded, forced, temporary value-migration
sb . setConfig ( " htTemplatePath " , " htroot/env/templates " ) ;
2016-03-20 03:34:28 +01:00
double oldVer ;
2013-09-22 02:21:24 +02:00
try {
2016-03-20 03:34:28 +01:00
String tmpversion = sb . getConfig ( Seed . VERSION , " " ) ;
if ( tmpversion . isEmpty ( ) ) { // before 1.83009737 only the svnRevision nr was in config (like 9737)
tmpversion = yacyBuildProperties . getVersion ( ) ;
int oldRev = Integer . parseInt ( sb . getConfig ( " svnRevision " , " 0 " ) ) ;
if ( oldRev > 1 ) {
oldVer = Double . parseDouble ( tmpversion ) + oldRev / 100000000 . 0 ;
} else {
oldVer = Double . parseDouble ( yacyBuildProperties . getLongVersion ( ) ) ; // failsafe (assume current version = no migration)
}
} else {
oldVer = Double . parseDouble ( tmpversion ) ;
}
2013-09-22 02:21:24 +02:00
} catch ( final NumberFormatException e ) {
2016-03-20 03:34:28 +01:00
oldVer = 0 . 0d ;
2013-09-22 02:21:24 +02:00
}
2016-03-20 03:34:28 +01:00
final double newRev = Double . parseDouble ( yacyBuildProperties . getLongVersion ( ) ) ;
sb . setConfig ( Seed . VERSION , yacyBuildProperties . getLongVersion ( ) ) ;
2013-09-22 02:21:24 +02:00
sb . setConfig ( " applicationRoot " , appHome . toString ( ) ) ;
sb . setConfig ( " dataRoot " , dataHome . toString ( ) ) ;
// create some directories
2014-02-27 00:23:34 +01:00
final File htRootPath = new File ( appHome , sb . getConfig ( SwitchboardConstants . HTROOT_PATH , SwitchboardConstants . HTROOT_PATH_DEFAULT ) ) ;
mkdirIfNeseccary ( htRootPath ) ;
2016-03-11 18:12:13 +01:00
htDocsPath = sb . getDataPath ( SwitchboardConstants . HTDOCS_PATH , SwitchboardConstants . HTDOCS_PATH_DEFAULT ) ;
2013-09-22 02:21:24 +02:00
mkdirIfNeseccary ( htDocsPath ) ;
//final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
2013-10-27 15:12:17 +01:00
// copy the donate iframe (better to copy this once here instead of doing this in an actual iframe in the search result)
2016-08-17 15:15:31 +02:00
importDonationIFrame ( sb , htDocsPath ) ;
2013-10-27 15:12:17 +01:00
2013-09-22 02:21:24 +02:00
// create default notifier picture
2013-10-27 15:12:17 +01:00
File notifierFile = new File ( htDocsPath , " notifier.gif " ) ;
if ( ! notifierFile . exists ( ) ) try { Files . copy ( new File ( htRootPath , " env/grafics/empty.gif " ) , notifierFile ) ; } catch ( final IOException e ) { }
2013-09-22 02:21:24 +02:00
final File htdocsReadme = new File ( htDocsPath , " readme.txt " ) ;
if ( ! ( htdocsReadme . exists ( ) ) ) try { FileUtils . copy ( (
" This is your root directory for individual Web Content \ r \ n " +
" \ r \ n " +
" Please place your html files into the www subdirectory. \ r \ n " +
" The URL of that path is either \ r \ n " +
" http://www.<your-peer-name>.yacy or \ r \ n " +
" http://<your-ip>:<your-port>/www \ r \ n " +
" \ r \ n " +
" Other subdirectories may be created; they map to corresponding sub-domains. \ r \ n " +
" This directory shares it's content with the applications htroot path, so you \ r \ n " +
" may access your yacy search page with \ r \ n " +
" http://<your-peer-name>.yacy/ \ r \ n " +
" \ r \ n " ) . getBytes ( ) , htdocsReadme ) ; } catch ( final IOException e ) {
System . out . println ( " Error creating htdocs readme: " + e . getMessage ( ) ) ;
}
2016-03-11 18:12:13 +01:00
shareDefaultPath = new File ( htDocsPath , " share " ) ;
2013-09-22 02:21:24 +02:00
mkdirIfNeseccary ( shareDefaultPath ) ;
2016-03-11 18:12:13 +01:00
shareDumpDefaultPath = new File ( shareDefaultPath , " dump " ) ;
mkdirIfNeseccary ( shareDumpDefaultPath ) ;
2013-09-22 02:21:24 +02:00
2016-03-20 03:34:28 +01:00
migration . migrate ( sb , oldVer , newRev ) ;
2013-09-22 02:21:24 +02:00
// delete old release files
final int deleteOldDownloadsAfterDays = ( int ) sb . getConfigLong ( " update.deleteOld " , 30 ) ;
yacyRelease . deleteOldDownloads ( sb . releasePath , deleteOldDownloadsAfterDays ) ;
// set user-agent
HTTPClient . setDefaultUserAgent ( ClientIdentification . yacyInternetCrawlerAgent . userAgent ) ;
// start main threads
2015-10-31 23:09:03 +01:00
final int port = sb . getLocalPort ( ) ;
2013-09-22 02:21:24 +02:00
try {
2013-10-17 03:34:22 +02:00
// start http server
YaCyHttpServer httpServer ;
2014-05-11 01:53:11 +02:00
httpServer = new Jetty9HttpServerImpl ( port ) ;
2013-10-17 03:34:22 +02:00
httpServer . startupServer ( ) ;
2013-10-09 00:40:48 +02:00
sb . setHttpServer ( httpServer ) ;
2015-03-29 07:12:23 +02:00
// TODO: this has no effect on Jetty (but needed to reflect configured value and limit is still used)
ConnectionInfo . setServerMaxcount ( sb . getConfigInt ( " connectionsMax " , ConnectionInfo . getMaxcount ( ) ) ) ;
2013-10-09 00:40:48 +02:00
ConcurrentLog . info ( " STARTUP " , httpServer . getVersion ( ) ) ;
2013-09-22 02:21:24 +02:00
// open the browser window
final boolean browserPopUpTrigger = sb . getConfig ( SwitchboardConstants . BROWSER_POP_UP_TRIGGER , " true " ) . equals ( " true " ) ;
if ( browserPopUpTrigger ) try {
final String browserPopUpPage = sb . getConfig ( SwitchboardConstants . BROWSER_POP_UP_PAGE , " ConfigBasic.html " ) ;
2014-01-05 04:23:44 +01:00
//boolean properPW = (sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT, "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
2013-09-22 02:21:24 +02:00
//if (!properPW) browserPopUpPage = "ConfigBasic.html";
2016-09-28 14:52:30 +02:00
/ * YaCy main startup process must not hang because browser opening is long or fails .
* Let ' s open try opening the browser in a separate thread * /
new Thread ( " Browser opening " ) {
@Override
public void run ( ) {
Browser . openBrowser ( ( " http://localhost: " + port ) + " / " + browserPopUpPage ) ;
}
} . start ( ) ;
2013-09-22 02:21:24 +02:00
// Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage);
} catch ( final Throwable e ) {
// cannot open browser. This may be normal in headless environments
//Log.logException(e);
}
// enable browser popup, http server is ready now
sb . tray . setReady ( ) ;
//regenerate Locales from Translationlist, if needed
final File locale_source = sb . getAppPath ( " locale.source " , " locales " ) ;
final String lang = sb . getConfig ( " locale.language " , " " ) ;
2016-08-23 00:04:24 +02:00
// on lang=browser all active translation should be checked (because any could be requested by client)
List < String > langlist ;
if ( lang . endsWith ( " browser " ) )
langlist = Translator . activeTranslations ( ) ; // get all translated languages
else {
langlist = new ArrayList < String > ( ) ;
langlist . add ( lang ) ;
}
for ( String tmplang : langlist ) {
if ( ! tmplang . equals ( " " ) & & ! tmplang . equals ( " default " ) & & ! tmplang . equals ( " browser " ) ) { //locale is used
String currentRev = null ;
try {
final BufferedReader br = new BufferedReader ( new InputStreamReader ( new FileInputStream ( new File ( sb . getDataPath ( " locale.translated_html " , " DATA/LOCALE/htroot " ) , tmplang + " /version " ) ) ) ) ;
currentRev = br . readLine ( ) ; // may return null
br . close ( ) ;
} catch ( final IOException e ) {
//Error
}
2013-09-22 02:21:24 +02:00
2016-08-23 00:04:24 +02:00
if ( currentRev = = null | | ! currentRev . equals ( sb . getConfig ( Seed . VERSION , " " ) ) ) {
try { //is this another version?!
final File sourceDir = new File ( sb . getConfig ( SwitchboardConstants . HTROOT_PATH , SwitchboardConstants . HTROOT_PATH_DEFAULT ) ) ;
final File destDir = new File ( sb . getDataPath ( " locale.translated_html " , " DATA/LOCALE/htroot " ) , tmplang ) ;
if ( new TranslatorXliff ( ) . translateFilesRecursive ( sourceDir , destDir , new File ( locale_source , tmplang + " .lng " ) , " html,template,inc " , " locale " ) ) { //translate it
//write the new Versionnumber
final BufferedWriter bw = new BufferedWriter ( new PrintWriter ( new FileWriter ( new File ( destDir , " version " ) ) ) ) ;
bw . write ( sb . getConfig ( Seed . VERSION , " Error getting Version " ) ) ;
bw . close ( ) ;
}
} catch ( final IOException e ) {
}
2013-09-22 02:21:24 +02:00
}
2016-08-23 00:04:24 +02:00
}
2013-09-22 02:21:24 +02:00
}
// initialize number formatter with this locale
2016-08-23 00:04:24 +02:00
if ( ! lang . equals ( " browser " ) ) // "default" is handled by .setLocale()
Formatter . setLocale ( lang ) ;
2013-09-22 02:21:24 +02:00
// registering shutdown hook
ConcurrentLog . config ( " STARTUP " , " Registering Shutdown Hook " ) ;
final Runtime run = Runtime . getRuntime ( ) ;
2016-12-28 09:47:27 +01:00
run . addShutdownHook ( new shutdownHookThread ( sb , shutdownSemaphore ) ) ;
2013-09-22 02:21:24 +02:00
// save information about available memory after all initializations
//try {
sb . setConfig ( " memoryFreeAfterInitBGC " , MemoryControl . free ( ) ) ;
sb . setConfig ( " memoryTotalAfterInitBGC " , MemoryControl . total ( ) ) ;
System . gc ( ) ;
sb . setConfig ( " memoryFreeAfterInitAGC " , MemoryControl . free ( ) ) ;
sb . setConfig ( " memoryTotalAfterInitAGC " , MemoryControl . total ( ) ) ;
//} catch (final ConcurrentModificationException e) {}
// wait for server shutdown
try {
sb . waitForShutdown ( ) ;
} catch ( final Exception e ) {
ConcurrentLog . severe ( " MAIN CONTROL LOOP " , " PANIC: " + e . getMessage ( ) , e ) ;
}
// shut down
Array . terminate ( ) ;
ConcurrentLog . config ( " SHUTDOWN " , " caught termination signal " ) ;
httpServer . stop ( ) ;
ConcurrentLog . config ( " SHUTDOWN " , " server has terminated " ) ;
sb . close ( ) ;
} catch ( final Exception e ) {
ConcurrentLog . severe ( " STARTUP " , " Unexpected Error: " + e . getClass ( ) . getName ( ) , e ) ;
//System.exit(1);
}
if ( lock ! = null ) lock . release ( ) ;
if ( channel ! = null ) channel . close ( ) ;
} catch ( final Exception ee ) {
ConcurrentLog . severe ( " STARTUP " , " FATAL ERROR: " + ee . getMessage ( ) , ee ) ;
} finally {
}
2015-11-17 22:27:07 +01:00
if ( tmpdir ! = null ) FileUtils . deletedelete ( new File ( tmpdir ) ) ; // clean up temp dir (set at startup as subdir of system.propery "java.io.tmpdir")
2013-09-22 02:21:24 +02:00
ConcurrentLog . config ( " SHUTDOWN " , " goodbye. (this is the last line) " ) ;
ConcurrentLog . shutdown ( ) ;
shutdownSemaphore . release ( 1000 ) ;
try {
System . exit ( 0 ) ;
} catch ( final Exception e ) { } // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
}
2016-08-17 15:15:31 +02:00
/ * *
* Concurrently import the donation iframe content to serve it directly from this peer .
* @param switchBoard the SwitchBoard instance . Must not be null .
* @param htDocsDirectory the custom htdocs directory . Must not be null .
* /
private static void importDonationIFrame ( final Switchboard switchBoard , final File htDocsDirectory ) {
final File wwwEnvPath = new File ( htDocsDirectory , " env " ) ;
mkdirIfNeseccary ( wwwEnvPath ) ;
final String iframesource = switchBoard . getConfig ( " donation.iframesource " , " " ) ;
final String iframetarget = switchBoard . getConfig ( " donation.iframetarget " , " " ) ;
final File iframefile = new File ( htDocsDirectory , iframetarget ) ;
2016-10-22 17:17:21 +02:00
if ( ! iframefile . exists ( ) ) new Thread ( " yacy.importDonationIFrame " ) {
2016-08-17 15:15:31 +02:00
@Override
public void run ( ) {
final ClientIdentification . Agent agent = ClientIdentification . getAgent ( ClientIdentification . yacyInternetCrawlerAgentName ) ;
Response documentResponse ;
try {
/* Load the donation html frame content */
documentResponse = switchBoard . loader = = null ? null : switchBoard . loader . load ( switchBoard . loader . request ( new DigestURL ( iframesource ) , false , true ) , CacheStrategy . NOCACHE , Integer . MAX_VALUE , null , agent ) ;
if ( documentResponse ! = null ) {
Document [ ] documents = documentResponse . parse ( ) ;
if ( documents ! = null & & documents . length > 0 & & documents [ 0 ] ! = null ) {
Document donateDocument = documents [ 0 ] ;
String donateDocContent = new String ( documentResponse . getContent ( ) , donateDocument . getCharset ( ) ) ;
/* Load image resources contained in the page */
if ( donateDocument . getImages ( ) ! = null ) {
for ( DigestURL imgURL : donateDocument . getImages ( ) . keySet ( ) ) {
try {
Response response = switchBoard . loader . load ( switchBoard . loader . request ( imgURL , false , true ) , CacheStrategy . NOCACHE , Integer . MAX_VALUE , null , agent ) ;
if ( response ! = null ) {
String imgFileName = imgURL . getFileName ( ) ;
/* Store each image in the same directory as the iframe target file */
FileUtils . copy ( response . getContent ( ) , new File ( iframefile . getParentFile ( ) , imgFileName ) ) ;
/* Transform the original image URL to a relative one */
donateDocContent = donateDocContent . replace ( imgURL . getURL ( ) . toString ( ) , imgFileName ) ;
}
} catch ( IOException e ) {
/* Failing to load one image should not stop the whole task */
ConcurrentLog . warn ( " STARTUP " , " Donation frame retrieval : could not get an image resource. " , e ) ;
}
}
}
FileUtils . copy ( donateDocContent . getBytes ( donateDocument . getCharset ( ) ) , iframefile ) ;
}
}
} catch ( Exception e ) {
ConcurrentLog . warn ( " STARTUP " , " Could not retrieve donation frame content. " , e ) ;
}
}
} . start ( ) ;
}
2013-09-22 02:21:24 +02:00
/ * *
* @param f
* /
private static void delete ( final File f ) {
if ( ! f . delete ( ) )
ConcurrentLog . severe ( " STARTUP " , " WARNING: the file " + f + " can not be deleted! " ) ;
}
/ * *
* @see File # mkdir ( )
* @param path
* /
private static void mkdirIfNeseccary ( final File path ) {
if ( ! ( path . exists ( ) ) )
if ( ! path . mkdir ( ) )
ConcurrentLog . warn ( " STARTUP " , " could not create directory " + path . toString ( ) ) ;
}
/ * *
* @see File # mkdirs ( )
* @param path
* /
public static void mkdirsIfNeseccary ( final File path ) {
if ( ! ( path . exists ( ) ) )
if ( ! path . mkdirs ( ) )
ConcurrentLog . warn ( " STARTUP " , " could not create directories " + path . toString ( ) ) ;
}
/ * *
* Loads the configuration from the data - folder .
* FIXME : Why is this called over and over again from every method , instead
* of setting the configurationdata once for this class in main ?
*
* @param mes Where are we called from , so that the errormessages can be
* more descriptive .
* @param homePath Root - path where all the information is to be found .
* @return Properties read from the configurationfile .
* /
private static Properties configuration ( final String mes , final File homePath ) {
ConcurrentLog . config ( mes , " Application Root Path: " + homePath . toString ( ) ) ;
// read data folder
final File dataFolder = new File ( homePath , " DATA " ) ;
if ( ! ( dataFolder . exists ( ) ) ) {
ConcurrentLog . severe ( mes , " Application was never started or root path wrong. " ) ;
System . exit ( - 1 ) ;
}
final Properties config = new Properties ( ) ;
FileInputStream fis = null ;
try {
fis = new FileInputStream ( new File ( homePath , " DATA/SETTINGS/yacy.conf " ) ) ;
config . load ( fis ) ;
} catch ( final FileNotFoundException e ) {
ConcurrentLog . severe ( mes , " could not find configuration file. " ) ;
System . exit ( - 1 ) ;
} catch ( final IOException e ) {
ConcurrentLog . severe ( mes , " could not read configuration file. " ) ;
System . exit ( - 1 ) ;
} finally {
if ( fis ! = null ) {
try {
fis . close ( ) ;
} catch ( final IOException e ) {
ConcurrentLog . logException ( e ) ;
}
}
}
return config ;
}
/ * *
* Call the shutdown - page of YaCy to tell it to shut down . This method is
* called if you start yacy with the argument - shutdown .
*
* @param homePath Root - path where all the information is to be found .
* /
public static void shutdown ( final File homePath ) {
// start up
System . out . println ( copyright ) ;
System . out . println ( hline ) ;
submitURL ( homePath , " Steering.html?shutdown= " , " Terminate YaCy " ) ;
}
public static void update ( final File homePath ) {
// start up
System . out . println ( copyright ) ;
System . out . println ( hline ) ;
submitURL ( homePath , " ConfigUpdate_p.html?autoUpdate= " , " Update YaCy to most recent version " ) ;
}
private static void submitURL ( final File homePath , final String path , final String processdescription ) {
final Properties config = configuration ( " COMMAND-STEERING " , homePath ) ;
// read port
2013-12-29 02:00:44 +01:00
final int port = Integer . parseInt ( config . getProperty ( " port " , " 8090 " ) ) ;
2013-09-22 02:21:24 +02:00
// read password
String encodedPassword = ( String ) config . get ( SwitchboardConstants . ADMIN_ACCOUNT_B64MD5 ) ;
if ( encodedPassword = = null ) encodedPassword = " " ; // not defined
// send 'wget' to web interface
final HTTPClient con = new HTTPClient ( ClientIdentification . yacyInternetCrawlerAgent ) ;
2016-12-18 02:38:43 +01:00
// con.setHeader(requestHeader.entrySet());
2013-09-22 02:21:24 +02:00
try {
2014-01-19 15:21:23 +01:00
con . GETbytes ( " http://localhost: " + port + " / " + path , config . getProperty ( SwitchboardConstants . ADMIN_ACCOUNT_USER_NAME , " admin " ) , encodedPassword , false ) ;
2013-09-22 02:21:24 +02:00
if ( con . getStatusCode ( ) > 199 & & con . getStatusCode ( ) < 300 ) {
ConcurrentLog . config ( " COMMAND-STEERING " , " YACY accepted steering command: " + processdescription ) ;
2014-01-06 01:48:54 +01:00
2013-09-22 02:21:24 +02:00
} else {
ConcurrentLog . severe ( " COMMAND-STEERING " , " error response from YACY socket: " + con . getHttpResponse ( ) . getStatusLine ( ) ) ;
System . exit ( - 1 ) ;
}
} catch ( final IOException e ) {
ConcurrentLog . severe ( " COMMAND-STEERING " , " could not establish connection to YACY socket: " + e . getMessage ( ) ) ;
System . exit ( - 1 ) ;
}
try {
HTTPClient . closeConnectionManager ( ) ;
} catch ( final InterruptedException e ) {
e . printStackTrace ( ) ;
}
// finished
ConcurrentLog . config ( " COMMAND-STEERING " , " SUCCESSFULLY FINISHED COMMAND: " + processdescription ) ;
}
2014-01-06 07:00:16 +01:00
/ * *
* read saved config file and perform action which need to be done before main task starts
* - like check on system alrady running etc .
*
* @param dataHome data directory
* /
static private void preReadSavedConfigandInit ( File dataHome ) {
File lockFile = new File ( dataHome , " DATA/yacy.running " ) ;
final String conf = " DATA/SETTINGS/yacy.conf " ;
// If YaCy is actually running, then we check if the server port is open.
// If yes, then we consider that a restart is a user mistake and then we just respond
// as the user expects and tell the browser to open the start page.
// That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply
// open the web interface. (They don't think of 'servers' they just want to get to the search page).
// We need to parse the configuration file for that to get the host port
File configFile = new File ( dataHome , conf ) ;
if ( configFile . exists ( ) ) {
Properties p = new Properties ( ) ;
try {
2014-03-24 02:32:17 +01:00
FileInputStream fis = new FileInputStream ( configFile ) ;
p . load ( fis ) ;
fis . close ( ) ;
2014-01-06 07:00:16 +01:00
// Test for server access restriction (is implemented using Jetty IPaccessHandler which does not support IPv6
2014-09-30 14:53:52 +02:00
// try to disable IPv6
2014-01-06 07:00:16 +01:00
String teststr = p . getProperty ( " serverClient " , " * " ) ;
if ( ! teststr . equals ( " * " ) ) {
// testing on Win-8 showed this property has to be set befor Switchboard starts
// and seems to be sensitive (or time critical) if other code had been executed before this (don't know why ... ?)
System . setProperty ( " java.net.preferIPv6Addresses " , " false " ) ;
System . setProperty ( " java.net.preferIPv4Stack " , " true " ) ; // DO NOT PREFER IPv6, i.e. freifunk uses ipv6 only and host resolving does not work
teststr = System . getProperty ( " java.net.preferIPv4Stack " ) ;
2014-01-06 11:28:05 +01:00
System . out . println ( " set system property java.net.preferIP4Stack= " + teststr ) ;
}
// test for yacy already running
if ( lockFile . exists ( ) ) { // another instance running? VM crash? User will have to care about this
//standard log system not up yet - use simply stdout
// prevents also creation of a log file while just opening browser
System . out . println ( " WARNING: the file " + lockFile + " exists, this usually means that a YaCy instance is still running. If you want to restart YaCy, try first ./stopYACY.sh, then ./startYACY.sh. If ./stopYACY.sh fails, try ./killYACY.sh " ) ;
int port = Integer . parseInt ( p . getProperty ( " port " , " 8090 " ) ) ;
2014-09-16 14:53:30 +02:00
if ( TimeoutRequest . ping ( " 127.0.0.1 " , port , 1000 ) ) {
Browser . openBrowser ( " http://localhost: " + port + " / " + p . getProperty ( SwitchboardConstants . BROWSER_POP_UP_PAGE , " index.html " ) ) ;
// Thats it; YaCy was running, the user is happy, we can stop now.
System . out . println ( " WARNING: YaCy instance was still running; just opening the browser and exit. " ) ;
System . exit ( 0 ) ;
} else {
// YaCy is not running; thus delete the file an go on as nothing was wrong.
System . err . println ( " INFO: delete old yacy.running file; likely previous YaCy session was not orderly shutdown! " ) ;
delete ( lockFile ) ;
}
2014-01-06 07:00:16 +01:00
}
2014-09-16 23:14:13 +02:00
} catch ( IOException ex ) {
System . err . println ( " ERROR: config file seems to be corrupt " ) ;
System . err . println ( " ERROR: if problem persists, delete file " ) ;
System . err . println ( configFile . getAbsolutePath ( ) ) ;
2014-09-16 14:53:30 +02:00
ConcurrentLog . logException ( ex ) ;
2014-09-16 23:14:13 +02:00
ConcurrentLog . severe ( " Startup " , " cannot read " + configFile . toString ( ) + " , please delete the corrupted file if problem persits " ) ;
2014-09-16 14:53:30 +02:00
}
2014-01-06 07:00:16 +01:00
}
2014-01-06 11:28:05 +01:00
}
2013-09-22 02:21:24 +02:00
/ * *
* Main - method which is started by java . Checks for special arguments or
* starts up the application .
*
* @param args
* Given arguments from the command line .
* /
public static void main ( String args [ ] ) {
try {
// check assertion status
//ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
boolean assertionenabled = false ;
assert ( assertionenabled = true ) = = true ; // compare to true to remove warning: "Possible accidental assignement"
if ( assertionenabled ) System . out . println ( " Asserts are enabled " ) ;
// check memory amount
System . gc ( ) ;
final long startupMemFree = MemoryControl . free ( ) ;
final long startupMemTotal = MemoryControl . total ( ) ;
// maybe go into headless awt mode: we have three cases depending on OS and one exception:
// windows : better do not go into headless mode
// mac : go into headless mode because an application is shown in gui which may not be wanted
// linux : go into headless mode because this does not need any head operation
// exception : if the -gui option is used then do not go into headless mode since that uses a gui
boolean headless = true ;
if ( OS . isWindows ) headless = false ;
if ( args . length > = 1 & & args [ 0 ] . toLowerCase ( ) . equals ( " -gui " ) ) headless = false ;
System . setProperty ( " java.awt.headless " , headless ? " true " : " false " ) ;
2015-11-17 22:27:07 +01:00
2013-09-22 02:21:24 +02:00
String s = " " ; for ( final String a : args ) s + = a + " " ;
yacyRelease . startParameter = s . trim ( ) ;
File applicationRoot = new File ( System . getProperty ( " user.dir " ) . replace ( '\\' , '/' ) ) ;
File dataRoot = applicationRoot ;
//System.out.println("args.length=" + args.length);
//System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
if ( ( args . length > = 1 ) & & ( args [ 0 ] . toLowerCase ( ) . equals ( " -startup " ) | | args [ 0 ] . equals ( " -start " ) ) ) {
// normal start-up of yacy
2016-09-02 11:23:02 +02:00
if ( args . length > 1 ) {
dataRoot = new File ( System . getProperty ( " user.home " ) . replace ( '\\' , '/' ) , args [ 1 ] ) ;
}
preReadSavedConfigandInit ( dataRoot ) ;
2013-09-22 02:21:24 +02:00
startup ( dataRoot , applicationRoot , startupMemFree , startupMemTotal , false ) ;
} else if ( args . length > = 1 & & args [ 0 ] . toLowerCase ( ) . equals ( " -gui " ) ) {
// start-up of yacy with gui
2016-09-02 11:23:02 +02:00
if ( args . length > 1 ) {
dataRoot = new File ( System . getProperty ( " user.home " ) . replace ( '\\' , '/' ) , args [ 1 ] ) ;
}
preReadSavedConfigandInit ( dataRoot ) ;
2013-09-22 02:21:24 +02:00
startup ( dataRoot , applicationRoot , startupMemFree , startupMemTotal , true ) ;
} else if ( ( args . length > = 1 ) & & ( ( args [ 0 ] . toLowerCase ( ) . equals ( " -shutdown " ) ) | | ( args [ 0 ] . equals ( " -stop " ) ) ) ) {
// normal shutdown of yacy
if ( args . length = = 2 ) applicationRoot = new File ( args [ 1 ] ) ;
shutdown ( applicationRoot ) ;
} else if ( ( args . length > = 1 ) & & ( args [ 0 ] . toLowerCase ( ) . equals ( " -update " ) ) ) {
// aut-update yacy
if ( args . length = = 2 ) applicationRoot = new File ( args [ 1 ] ) ;
update ( applicationRoot ) ;
} else if ( ( args . length > = 1 ) & & ( args [ 0 ] . toLowerCase ( ) . equals ( " -version " ) ) ) {
// show yacy version
System . out . println ( copyright ) ;
2016-09-02 11:23:02 +02:00
} else if ( ( args . length > 1 ) & & ( args [ 0 ] . toLowerCase ( ) . equals ( " -config " ) ) ) {
2014-02-24 05:16:31 +01:00
// set config parameter. Special handling of adminAccount=user:pwd (generates md5 encoded password)
// on Windows parameter should be enclosed in doublequotes to accept = sign (e.g. -config "port=8090" "port.ssl=8043")
File f = new File ( dataRoot , " DATA/SETTINGS/ " ) ;
if ( ! f . exists ( ) ) {
mkdirsIfNeseccary ( f ) ;
} else {
if ( new File ( dataRoot , " DATA/yacy.running " ) . exists ( ) ) {
System . out . println ( " please restart YaCy " ) ;
}
}
// use serverSwitch to read config properties (including init values from yacy.init
serverSwitch ss = new serverSwitch ( dataRoot , applicationRoot , " defaults/yacy.init " , " DATA/SETTINGS/yacy.conf " ) ;
for ( int icnt = 1 ; icnt < args . length ; icnt + + ) {
String cfg = args [ icnt ] ;
int pos = cfg . indexOf ( '=' ) ;
if ( pos > 0 ) {
String cmd = cfg . substring ( 0 , pos ) ;
String val = cfg . substring ( pos + 1 ) ;
if ( ! val . isEmpty ( ) ) {
if ( cmd . equalsIgnoreCase ( SwitchboardConstants . ADMIN_ACCOUNT ) ) { // special command to set adminusername and md5-pwd
int cpos = val . indexOf ( ':' ) ; //format adminAccount=adminname:adminpwd
if ( cpos > = 0 ) {
String username = val . substring ( 0 , cpos ) ;
String pwdtxt = val . substring ( cpos + 1 ) ;
if ( ! username . isEmpty ( ) ) {
ss . setConfig ( SwitchboardConstants . ADMIN_ACCOUNT_USER_NAME , username ) ;
System . out . println ( " Set property " + SwitchboardConstants . ADMIN_ACCOUNT_USER_NAME + " = " + username ) ;
} else {
username = ss . getConfig ( SwitchboardConstants . ADMIN_ACCOUNT_USER_NAME , " admin " ) ;
}
ss . setConfig ( SwitchboardConstants . ADMIN_ACCOUNT_B64MD5 , " MD5: " + Digest . encodeMD5Hex ( username + " : " + ss . getConfig ( SwitchboardConstants . ADMIN_REALM , " YaCy " ) + " : " + pwdtxt ) ) ;
System . out . println ( " Set property " + SwitchboardConstants . ADMIN_ACCOUNT_B64MD5 + " = " + ss . getConfig ( SwitchboardConstants . ADMIN_ACCOUNT_B64MD5 , " " ) ) ;
}
} else {
ss . setConfig ( cmd , val ) ;
System . out . println ( " Set property " + cmd + " = " + val ) ;
}
}
} else {
System . out . println ( " skip parameter " + cfg + " (equal sign missing, put parameter in doublequotes) " ) ;
}
System . out . println ( ) ;
}
2016-09-02 11:23:02 +02:00
} else {
if ( args . length = = 1 ) {
applicationRoot = new File ( args [ 0 ] ) ;
}
preReadSavedConfigandInit ( dataRoot ) ;
2013-09-22 02:21:24 +02:00
startup ( dataRoot , applicationRoot , startupMemFree , startupMemTotal , false ) ;
}
} finally {
ConcurrentLog . shutdown ( ) ;
}
}
}
/ * *
* This class is a helper class whose instance is started , when the java virtual
* machine shuts down . Signals the plasmaSwitchboard to shut down .
* /
class shutdownHookThread extends Thread {
private final Switchboard sb ;
2016-12-28 09:47:27 +01:00
private final Semaphore shutdownSemaphore ;
2013-09-22 02:21:24 +02:00
2016-12-28 09:47:27 +01:00
public shutdownHookThread ( final Switchboard sb , final Semaphore shutdownSemaphore ) {
2016-10-22 17:17:21 +02:00
super ( " yacy.shutdownHookThread " ) ;
2013-09-22 02:21:24 +02:00
this . sb = sb ;
2016-12-28 09:47:27 +01:00
this . shutdownSemaphore = shutdownSemaphore ;
2013-09-22 02:21:24 +02:00
}
@Override
public void run ( ) {
try {
if ( ! this . sb . isTerminated ( ) ) {
ConcurrentLog . config ( " SHUTDOWN " , " Shutdown via shutdown hook. " ) ;
2016-12-28 09:47:27 +01:00
/ * Print also to the standard output , as the LogManager is likely to have
* been concurrently reset by its own shutdown hook thread * /
System . out . println ( " SHUTDOWN Starting shutdown via shutdown hook. " ) ;
2013-09-22 02:21:24 +02:00
// sending the yacy main thread a shutdown signal
ConcurrentLog . fine ( " SHUTDOWN " , " Signaling shutdown to the switchboard. " ) ;
this . sb . terminate ( " shutdown hook " ) ;
// waiting for the yacy thread to finish execution
ConcurrentLog . fine ( " SHUTDOWN " , " Waiting for main thread to finish. " ) ;
2016-12-28 09:47:27 +01:00
/ * Main thread will release the shutdownSemaphore once completely terminated .
* We do not wait indefinitely as the application is supposed here to quickly terminate * /
this . shutdownSemaphore . tryAcquire ( 30 , TimeUnit . SECONDS ) ;
2013-09-22 02:21:24 +02:00
}
} catch ( final Exception e ) {
ConcurrentLog . severe ( " SHUTDOWN " , " Unexpected error. " + e . getClass ( ) . getName ( ) , e ) ;
}
}
}