2011-06-13 23:44:03 +02:00
// yacyRelease.java
2009-05-02 14:12:22 +02:00
// ----------------
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 27.04.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
2010-05-18 23:09:41 +02:00
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
2009-05-02 14:12:22 +02:00
//
// LICENSE
2011-06-13 23:44:03 +02:00
//
2009-05-02 14:12:22 +02:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2011-09-25 18:59:06 +02:00
package net.yacy.peers.operation ;
2009-05-02 14:12:22 +02:00
import java.io.BufferedOutputStream ;
import java.io.File ;
import java.io.FileInputStream ;
import java.io.FileOutputStream ;
import java.io.IOException ;
import java.security.InvalidKeyException ;
import java.security.NoSuchAlgorithmException ;
import java.security.PublicKey ;
import java.security.SignatureException ;
import java.util.ArrayList ;
2010-05-18 23:31:59 +02:00
import java.util.List ;
2009-05-02 14:12:22 +02:00
import java.util.Map ;
2011-04-21 15:58:49 +02:00
import java.util.Properties ;
2009-05-02 14:12:22 +02:00
import java.util.SortedSet ;
import java.util.TreeSet ;
2010-06-01 15:02:11 +02:00
import java.util.concurrent.ConcurrentHashMap ;
2009-05-02 14:12:22 +02:00
2010-05-25 14:54:57 +02:00
import net.yacy.cora.document.MultiProtocolURI ;
2011-03-07 21:36:40 +01:00
import net.yacy.cora.document.UTF8 ;
2012-09-25 21:20:03 +02:00
import net.yacy.cora.federate.yacy.CacheStrategy ;
2012-09-21 16:46:57 +02:00
import net.yacy.cora.order.Base64Order ;
2011-04-26 15:35:29 +02:00
import net.yacy.cora.protocol.ClientIdentification ;
2010-08-23 14:32:02 +02:00
import net.yacy.cora.protocol.ResponseHeader ;
2010-08-23 00:32:39 +02:00
import net.yacy.cora.protocol.http.HTTPClient ;
2012-06-22 11:39:17 +02:00
import net.yacy.cora.storage.Files ;
2012-09-21 15:48:16 +02:00
import net.yacy.crawler.data.CrawlQueues ;
2012-01-23 17:27:29 +01:00
import net.yacy.document.Document ;
2012-04-16 09:50:55 +02:00
import net.yacy.document.parser.tarParser ;
2011-09-14 22:11:27 +02:00
import net.yacy.kelondro.data.meta.DigestURI ;
2009-10-18 02:53:43 +02:00
import net.yacy.kelondro.io.CharBuffer ;
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.util.FileUtils ;
2009-10-20 00:34:44 +02:00
import net.yacy.kelondro.util.OS ;
2011-10-04 11:06:24 +02:00
import net.yacy.peers.Network ;
2011-09-25 18:59:06 +02:00
import net.yacy.search.Switchboard ;
2012-09-21 15:48:16 +02:00
import net.yacy.server.serverCore ;
import net.yacy.utils.CryptoLib ;
import net.yacy.utils.SignatureOutputStream ;
import net.yacy.utils.tarTools ;
2009-05-02 14:12:22 +02:00
public final class yacyRelease extends yacyVersion {
// information about latest release, retrieved from download pages
// this static information should be overwritten by network-specific locations
// for details see defaults/yacy.network.freeworld.unit
2010-06-01 15:02:11 +02:00
private static Map < yacyUpdateLocation , DevAndMainVersions > latestReleases = new ConcurrentHashMap < yacyUpdateLocation , DevAndMainVersions > ( ) ;
2010-05-18 23:31:59 +02:00
public final static List < yacyUpdateLocation > latestReleaseLocations = new ArrayList < yacyUpdateLocation > ( ) ; // will be initialized with value in defaults/yacy.network.freeworld.unit
2010-09-04 01:08:43 +02:00
public static String startParameter = " " ;
2011-06-13 23:44:03 +02:00
2010-05-25 14:54:57 +02:00
private MultiProtocolURI url ;
2009-05-02 14:12:22 +02:00
private File releaseFile ;
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
private PublicKey publicKey ;
2011-06-13 23:44:03 +02:00
2010-05-25 14:54:57 +02:00
public yacyRelease ( final MultiProtocolURI url ) {
2010-10-24 23:43:01 +02:00
super ( url . getFileName ( ) , url . getHost ( ) ) ;
2009-05-02 14:12:22 +02:00
this . url = url ;
}
2011-06-13 23:44:03 +02:00
2011-12-04 08:22:13 +01:00
private yacyRelease ( final MultiProtocolURI url , final PublicKey publicKey ) {
2009-05-02 14:12:22 +02:00
this ( url ) ;
this . publicKey = publicKey ;
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public yacyRelease ( final File releaseFile ) {
2010-10-24 23:43:01 +02:00
super ( releaseFile . getName ( ) , null ) ;
2010-05-18 23:31:59 +02:00
this . releaseFile = releaseFile ;
2009-05-02 14:12:22 +02:00
}
2010-05-25 14:54:57 +02:00
public MultiProtocolURI getUrl ( ) {
2011-06-13 23:44:03 +02:00
return this . url ;
2009-05-02 14:12:22 +02:00
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public static final yacyRelease rulebasedUpdateInfo ( final boolean manual ) {
// according to update properties, decide if we should retrieve update information
// if true, the release that can be obtained is returned.
// if false, null is returned
2009-07-19 22:37:44 +02:00
final Switchboard sb = Switchboard . getSwitchboard ( ) ;
2011-06-13 23:44:03 +02:00
2009-07-11 19:03:22 +02:00
// check if release was installed by packagemanager
if ( yacyBuildProperties . isPkgManager ( ) ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo ( " rulebasedUpdateInfo: package manager is used for update " ) ;
2009-07-11 19:03:22 +02:00
return null ;
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
// check if update process allows update retrieve
final String process = sb . getConfig ( " update.process " , " manual " ) ;
if ( ( ! manual ) & & ( ! process . equals ( " auto " ) ) ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo ( " rulebasedUpdateInfo: not an automatic update selected " ) ;
2009-05-02 14:12:22 +02:00
return null ; // no, its a manual or guided process
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
// check if the last retrieve time is a minimum time ago
final long cycle = Math . max ( 1 , sb . getConfigLong ( " update.cycle " , 168 ) ) * 60 * 60 * 1000 ; // update.cycle is hours
final long timeLookup = sb . getConfigLong ( " update.time.lookup " , System . currentTimeMillis ( ) ) ;
if ( ( ! manual ) & & ( timeLookup + cycle > System . currentTimeMillis ( ) ) ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo ( " rulebasedUpdateInfo: too early for a lookup for a new release (timeLookup = " + timeLookup + " , cycle = " + cycle + " , now = " + System . currentTimeMillis ( ) + " ) " ) ;
2009-05-02 14:12:22 +02:00
return null ; // no we have recently made a lookup
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
// check if we know that there is a release that is more recent than that which we are using
final DevAndMainVersions releases = yacyRelease . allReleases ( true , sb . getConfig ( " update.onlySignedFiles " , " 1 " ) . equals ( " 1 " ) ) ;
2009-12-02 01:37:59 +01:00
final yacyRelease latestmain = ( releases . main . isEmpty ( ) ) ? null : releases . main . last ( ) ;
final yacyRelease latestdev = ( releases . dev . isEmpty ( ) ) ? null : releases . dev . last ( ) ;
2009-05-02 14:12:22 +02:00
final String concept = sb . getConfig ( " update.concept " , " any " ) ;
String blacklist = sb . getConfig ( " update.blacklist " , " ...[123] " ) ;
if ( blacklist . equals ( " ....[123] " ) ) {
2009-06-20 22:46:58 +02:00
// patch the blacklist because of a release strategy change from 0.7 and up
blacklist = " ...[123] " ;
sb . setConfig ( " update.blacklist " , blacklist ) ;
2009-05-02 14:12:22 +02:00
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
if ( ( manual ) | | ( concept . equals ( " any " ) ) ) {
// return a dev-release or a main-release
if ( ( latestdev ! = null ) & &
( ( latestmain = = null ) | | ( latestdev . compareTo ( latestmain ) > 0 ) ) & &
2012-05-30 15:28:20 +02:00
( ! ( Double . toString ( latestdev . getReleaseNr ( ) ) . matches ( blacklist ) ) ) ) {
2009-05-02 14:12:22 +02:00
// consider a dev-release
if ( latestdev . compareTo ( thisVersion ( ) ) < = 0 ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo (
2009-05-02 14:12:22 +02:00
" rulebasedUpdateInfo: latest dev " + latestdev . getName ( ) +
" is not more recent than installed release " + thisVersion ( ) . getName ( ) ) ;
return null ;
}
return latestdev ;
}
if ( latestmain ! = null ) {
// consider a main release
2012-05-30 15:28:20 +02:00
if ( ( Double . toString ( latestmain . getReleaseNr ( ) ) . matches ( blacklist ) ) ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo (
2009-05-02 14:12:22 +02:00
" rulebasedUpdateInfo: latest dev " + ( latestdev = = null ? " null " : latestdev . getName ( ) ) +
" matches with blacklist ' " + blacklist + " ' " ) ;
return null ;
}
if ( latestmain . compareTo ( thisVersion ( ) ) < = 0 ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo (
2009-05-02 14:12:22 +02:00
" rulebasedUpdateInfo: latest main " + latestmain . getName ( ) +
" is not more recent than installed release (1) " + thisVersion ( ) . getName ( ) ) ;
return null ;
}
return latestmain ;
}
}
if ( ( concept . equals ( " main " ) ) & & ( latestmain ! = null ) ) {
// return a main-release
2012-05-30 15:28:20 +02:00
if ( ( Double . toString ( latestmain . getReleaseNr ( ) ) . matches ( blacklist ) ) ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo (
2009-05-02 14:12:22 +02:00
" rulebasedUpdateInfo: latest main " + latestmain . getName ( ) +
" matches with blacklist' " + blacklist + " ' " ) ;
return null ;
}
if ( latestmain . compareTo ( thisVersion ( ) ) < = 0 ) {
2011-10-04 11:06:24 +02:00
Network . log . logInfo (
2009-05-02 14:12:22 +02:00
" rulebasedUpdateInfo: latest main " + latestmain . getName ( ) +
" is not more recent than installed release (2) " + thisVersion ( ) . getName ( ) ) ;
2011-06-13 23:44:03 +02:00
return null ;
2009-05-02 14:12:22 +02:00
}
return latestmain ;
}
2011-10-04 11:06:24 +02:00
Network . log . logInfo ( " rulebasedUpdateInfo: failed to find more recent release " ) ;
2009-05-02 14:12:22 +02:00
return null ;
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public static DevAndMainVersions allReleases ( final boolean force , final boolean onlySigned ) {
// join the release infos
final TreeSet < yacyRelease > alldev = new TreeSet < yacyRelease > ( ) ;
final TreeSet < yacyRelease > allmain = new TreeSet < yacyRelease > ( ) ;
2011-06-13 23:44:03 +02:00
for ( final yacyUpdateLocation updateLocation : latestReleaseLocations ) {
2009-05-02 14:12:22 +02:00
if ( ! onlySigned | | updateLocation . getPublicKey ( ) ! = null ) {
2011-06-13 23:44:03 +02:00
final DevAndMainVersions versions = getReleases ( updateLocation , force ) ;
2010-06-16 20:43:45 +02:00
if ( versions ! = null & & versions . dev ! = null ) alldev . addAll ( versions . dev ) ;
if ( versions ! = null & & versions . main ! = null ) allmain . addAll ( versions . main ) ;
2009-05-02 14:12:22 +02:00
}
}
return new DevAndMainVersions ( alldev , allmain ) ;
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
/ * *
2011-06-13 23:44:03 +02:00
* get all Releases from update location using cache
2009-05-02 14:12:22 +02:00
* @param location Update location
* @param force when true , don ' t fetch from cache
* @return
* /
private static DevAndMainVersions getReleases ( final yacyUpdateLocation location , final boolean force ) {
// get release info from a Internet resource
2009-06-20 22:46:58 +02:00
DevAndMainVersions locLatestRelease = latestReleases . get ( location ) ;
2009-05-02 14:12:22 +02:00
if ( force | |
( locLatestRelease = = null ) / * | |
2009-12-02 01:37:59 +01:00
( ( latestRelease [ 0 ] . isEmpty ( ) ) & &
( latestRelease [ 1 ] . isEmpty ( ) ) & &
( latestRelease [ 2 ] . isEmpty ( ) ) & &
( latestRelease [ 3 ] . isEmpty ( ) ) ) * / ) {
2009-05-02 14:12:22 +02:00
locLatestRelease = allReleaseFrom ( location ) ;
2010-06-16 20:43:45 +02:00
if ( locLatestRelease ! = null ) latestReleases . put ( location , locLatestRelease ) ;
2009-05-02 14:12:22 +02:00
}
return locLatestRelease ;
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
/ * *
* get all releases from update location
* @param location
* @return
* /
2011-06-13 23:44:03 +02:00
private static DevAndMainVersions allReleaseFrom ( final yacyUpdateLocation location ) {
2009-05-02 14:12:22 +02:00
// retrieves the latest info about releases
// this is done by contacting a release location,
// parsing the content and filtering+parsing links
// returns the version info if successful, null otherwise
2012-01-23 17:27:29 +01:00
Document scraper ;
2009-05-02 14:12:22 +02:00
try {
2011-09-14 22:11:27 +02:00
final DigestURI uri = location . getLocationURL ( ) ;
Thread . currentThread ( ) . setName ( " allReleaseFrom - host " + uri . getHost ( ) ) ; // makes it more easy to see which release blocks process in thread dump
2013-05-20 22:05:28 +02:00
scraper = Switchboard . getSwitchboard ( ) . loader . loadDocument ( uri , CacheStrategy . NOCACHE , null , CrawlQueues . queuedMinLoadDelay , ClientIdentification . DEFAULT_TIMEOUT ) ;
2009-05-02 14:12:22 +02:00
} catch ( final IOException e ) {
return null ;
}
2011-06-13 23:44:03 +02:00
2010-06-22 14:28:53 +02:00
// analyze links in scraper resource, and find link to latest release in it
2013-02-22 15:45:15 +01:00
final Map < DigestURI , Properties > anchors = scraper . getAnchors ( ) ; // a url (String) / name (String) relation
2009-05-02 14:12:22 +02:00
final TreeSet < yacyRelease > mainReleases = new TreeSet < yacyRelease > ( ) ;
final TreeSet < yacyRelease > devReleases = new TreeSet < yacyRelease > ( ) ;
2013-02-22 15:45:15 +01:00
for ( final DigestURI url : anchors . keySet ( ) ) {
2009-05-02 14:12:22 +02:00
try {
2011-06-13 23:44:03 +02:00
final yacyRelease release = new yacyRelease ( url , location . getPublicKey ( ) ) ;
2009-05-02 14:12:22 +02:00
//System.out.println("r " + release.toAnchor());
2010-10-24 23:43:01 +02:00
if ( release . isMainRelease ( ) ) {
2009-05-02 14:12:22 +02:00
mainReleases . add ( release ) ;
} else {
devReleases . add ( release ) ;
}
} catch ( final RuntimeException e ) {
// the release string was not well-formed.
// that might have been another link
// just don't care
continue ;
}
}
2009-07-19 22:37:44 +02:00
Switchboard . getSwitchboard ( ) . setConfig ( " update.time.lookup " , System . currentTimeMillis ( ) ) ;
2009-05-02 14:12:22 +02:00
return new DevAndMainVersions ( devReleases , mainReleases ) ;
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public static final class DevAndMainVersions {
public TreeSet < yacyRelease > dev , main ;
public DevAndMainVersions ( final TreeSet < yacyRelease > dev , final TreeSet < yacyRelease > main ) {
this . dev = dev ;
this . main = main ;
}
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
/ * *
* < p > download this release and if public key is know , download signature and check it .
* < p > The signature is named $releaseurl . sig and contains the base64 encoded signature
* ( @see de . anomic . tools . CryptoLib )
* @return file object of release file , null in case of failure
* /
public File downloadRelease ( ) {
2009-07-19 22:37:44 +02:00
final File storagePath = Switchboard . getSwitchboard ( ) . releasePath ;
2009-05-02 14:12:22 +02:00
File download = null ;
2011-06-13 23:44:03 +02:00
final String name = getUrl ( ) . getFileName ( ) ;
2009-06-20 22:46:58 +02:00
byte [ ] signatureBytes = null ;
2012-10-28 19:56:02 +01:00
final HTTPClient client = new HTTPClient ( ClientIdentification . getUserAgent ( ) , ClientIdentification . DEFAULT_TIMEOUT ) ;
2011-06-13 23:44:03 +02:00
2009-06-20 22:46:58 +02:00
// download signature first, if public key is available
2010-05-18 23:31:59 +02:00
try {
2010-05-26 02:01:16 +02:00
if ( this . publicKey ! = null ) {
2011-06-13 23:44:03 +02:00
final byte [ ] signatureData = client . GETbytes ( getUrl ( ) . toString ( ) + " .sig " ) ;
2010-05-26 02:01:16 +02:00
if ( signatureData = = null ) {
2011-06-13 23:44:03 +02:00
Log . logWarning ( " yacyVersion " , " download of signature " + getUrl ( ) . toString ( ) + " failed. ignoring signature file. " ) ;
2010-05-26 02:01:16 +02:00
}
2011-03-07 21:36:40 +01:00
else signatureBytes = Base64Order . standardCoder . decode ( UTF8 . String ( signatureData ) . trim ( ) ) ;
2010-05-26 02:01:16 +02:00
}
2010-08-02 00:35:11 +02:00
client . setTimout ( 120000 ) ;
2011-06-13 23:44:03 +02:00
client . GET ( getUrl ( ) . toString ( ) ) ;
2012-06-25 18:17:31 +02:00
int statusCode = client . getHttpResponse ( ) . getStatusLine ( ) . getStatusCode ( ) ;
final ResponseHeader header = new ResponseHeader ( statusCode , client . getHttpResponse ( ) . getAllHeaders ( ) ) ;
2010-05-18 23:31:59 +02:00
2010-08-02 00:35:11 +02:00
final boolean unzipped = header . gzip ( ) & & ( header . mime ( ) . toLowerCase ( ) . equals ( " application/x-tar " ) ) ; // if true, then the httpc has unzipped the file
2012-04-16 09:50:55 +02:00
if ( unzipped & & name . endsWith ( " .tar.gz " ) ) {
2010-05-18 23:31:59 +02:00
download = new File ( storagePath , name . substring ( 0 , name . length ( ) - 3 ) ) ;
} else {
download = new File ( storagePath , name ) ;
}
if ( this . publicKey ! = null & & signatureBytes ! = null ) {
// copy to file and check signature
SignatureOutputStream verifyOutput = null ;
try {
2011-06-13 23:44:03 +02:00
verifyOutput = new SignatureOutputStream ( new FileOutputStream ( download ) , CryptoLib . signAlgorithm , this . publicKey ) ;
2010-08-02 00:35:11 +02:00
client . writeTo ( new BufferedOutputStream ( verifyOutput ) ) ;
2010-05-18 23:31:59 +02:00
if ( ! verifyOutput . verify ( signatureBytes ) ) throw new IOException ( " Bad Signature! " ) ;
2011-06-13 23:44:03 +02:00
} catch ( final NoSuchAlgorithmException e ) {
2010-05-18 23:31:59 +02:00
throw new IOException ( " No such algorithm " ) ;
2011-06-13 23:44:03 +02:00
} catch ( final SignatureException e ) {
2010-05-18 23:31:59 +02:00
throw new IOException ( " Signature exception " ) ;
} finally {
if ( verifyOutput ! = null )
verifyOutput . close ( ) ;
}
// Save signature
2011-06-13 23:44:03 +02:00
final File signatureFile = new File ( download . getAbsoluteFile ( ) + " .sig " ) ;
2011-03-11 00:25:07 +01:00
FileUtils . copy ( UTF8 . getBytes ( Base64Order . standardCoder . encode ( signatureBytes ) ) , signatureFile ) ;
2010-05-18 23:31:59 +02:00
if ( ( ! signatureFile . exists ( ) ) | | ( signatureFile . length ( ) = = 0 ) ) throw new IOException ( " create signature file failed " ) ;
} else {
// just copy into file
2010-08-02 00:35:11 +02:00
client . writeTo ( new BufferedOutputStream ( new FileOutputStream ( download ) ) ) ;
2010-05-18 23:31:59 +02:00
}
2011-06-13 23:44:03 +02:00
if ( ( ! download . exists ( ) ) | | ( download . length ( ) = = 0 ) ) throw new IOException ( " wget of url " + getUrl ( ) + " failed " ) ;
2012-04-16 09:50:55 +02:00
// check again if this is actually a tar.gz or tar file since the httpc may have decompressed it
if ( download . getName ( ) . endsWith ( " tar.gz " ) & & tarParser . isTar ( download ) ) {
String ts = download . getAbsoluteFile ( ) . toString ( ) ;
File tar = new File ( ts . substring ( 0 , ts . length ( ) - 3 ) ) ;
download . renameTo ( tar ) ;
download = tar ;
}
2010-05-18 23:31:59 +02:00
} catch ( final IOException e ) {
// Saving file failed, abort download
2011-06-13 23:44:03 +02:00
Log . logSevere ( " yacyVersion " , " download of " + getName ( ) + " failed: " + e . getMessage ( ) ) ;
2010-05-18 23:31:59 +02:00
if ( download ! = null & & download . exists ( ) ) {
FileUtils . deletedelete ( download ) ;
if ( download . exists ( ) ) Log . logWarning ( " yacyVersion " , " could not delete file " + download ) ;
}
download = null ;
} finally {
2010-08-02 00:35:11 +02:00
try {
client . finish ( ) ;
2011-06-13 23:44:03 +02:00
} catch ( final IOException e ) {
Log . logSevere ( " yacyVersion " , " finish of " + getName ( ) + " failed: " + e . getMessage ( ) ) ;
2010-08-02 00:35:11 +02:00
}
2010-05-18 23:31:59 +02:00
}
2009-05-02 14:12:22 +02:00
this . releaseFile = download ;
2009-07-19 22:37:44 +02:00
Switchboard . getSwitchboard ( ) . setConfig ( " update.time.download " , System . currentTimeMillis ( ) ) ;
2009-05-02 14:12:22 +02:00
return this . releaseFile ;
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public boolean checkSignature ( ) {
2011-06-13 23:44:03 +02:00
if ( this . releaseFile ! = null ) {
2009-06-20 22:46:58 +02:00
try {
2010-05-18 23:31:59 +02:00
final CharBuffer signBuffer = new CharBuffer ( getSignatureFile ( ) ) ;
final byte [ ] signByteBuffer = Base64Order . standardCoder . decode ( signBuffer . toString ( ) . trim ( ) ) ;
2012-02-02 09:55:27 +01:00
signBuffer . close ( ) ;
2010-05-18 23:31:59 +02:00
final CryptoLib cl = new CryptoLib ( ) ;
for ( final yacyUpdateLocation updateLocation : latestReleaseLocations ) {
try {
if ( cl . verifySignature ( updateLocation . getPublicKey ( ) ,
2011-06-13 23:44:03 +02:00
new FileInputStream ( this . releaseFile ) , signByteBuffer ) ) {
2010-05-18 23:31:59 +02:00
return true ;
}
2011-06-13 23:44:03 +02:00
} catch ( final InvalidKeyException e ) {
} catch ( final SignatureException e ) {
2010-05-18 23:31:59 +02:00
}
}
2011-06-13 23:44:03 +02:00
} catch ( final IOException e1 ) {
} catch ( final NoSuchAlgorithmException e ) {
2009-06-20 22:46:58 +02:00
}
2009-05-02 14:12:22 +02:00
2010-05-18 23:31:59 +02:00
}
return false ;
2009-05-02 14:12:22 +02:00
}
/ * *
* restart yacy by stopping yacy and previously running a batch
* script , which waits until yacy is terminated and starts it again
* /
public static void restart ( ) {
2010-05-18 23:31:59 +02:00
final Switchboard sb = Switchboard . getSwitchboard ( ) ;
if ( OS . isWindows ) {
2010-09-02 21:24:22 +02:00
final File startType = new File ( sb . getDataPath ( ) , " DATA/yacy.noconsole " . replace ( " / " , File . separator ) ) ;
2010-05-18 23:31:59 +02:00
String starterFile = " startYACY_debug.bat " ;
if ( startType . exists ( ) ) starterFile = " startYACY.bat " ; // startType noconsole
2010-09-04 01:08:43 +02:00
if ( startParameter . startsWith ( " -gui " ) ) starterFile + = " " + startParameter ;
2010-05-18 23:31:59 +02:00
try {
Log . logInfo ( " RESTART " , " INITIATED " ) ;
final String script =
" @echo off " + serverCore . LF_STRING +
" title YaCy restarter " + serverCore . LF_STRING +
" set loading=YACY RESTARTER " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" cd \" " + sb . getDataPath ( ) . toString ( ) + " /DATA/RELEASE/ " . replace ( " / " , File . separator ) + " \" " + serverCore . LF_STRING +
2010-05-18 23:31:59 +02:00
" :WAIT " + serverCore . LF_STRING +
" set loading=%loading%. " + serverCore . LF_STRING +
" cls " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
" ping -n 2 127.0.0.1 >nul " + serverCore . LF_STRING +
" IF exist .. \\ yacy.running goto WAIT " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" cd \" " + sb . getAppPath ( ) . toString ( ) + " \" " + serverCore . LF_STRING +
2010-05-18 23:31:59 +02:00
" start /MIN CMD /C " + starterFile + serverCore . LF_STRING ;
2010-09-02 21:24:22 +02:00
final File scriptFile = new File ( sb . getDataPath ( ) , " DATA/RELEASE/restart.bat " . replace ( " / " , File . separator ) ) ;
2010-05-18 23:31:59 +02:00
OS . deployScript ( scriptFile , script ) ;
Log . logInfo ( " RESTART " , " wrote restart-script to " + scriptFile . getAbsolutePath ( ) ) ;
OS . execAsynchronous ( scriptFile ) ;
Log . logInfo ( " RESTART " , " script is running " ) ;
2010-09-14 11:13:28 +02:00
sb . terminate ( 10 , " windows restart " ) ;
2010-05-18 23:31:59 +02:00
} catch ( final IOException e ) {
Log . logSevere ( " RESTART " , " restart failed " , e ) ;
2009-05-02 14:12:22 +02:00
}
2010-05-18 23:31:59 +02:00
}
if ( yacyBuildProperties . isPkgManager ( ) ) {
// start a re-start daemon
try {
Log . logInfo ( " RESTART " , " INITIATED " ) ;
final String script =
" #!/bin/sh " + serverCore . LF_STRING +
yacyBuildProperties . getRestartCmd ( ) + " >/var/lib/yacy/RELEASE/log " + serverCore . LF_STRING ;
2010-09-02 21:24:22 +02:00
final File scriptFile = new File ( sb . getDataPath ( ) , " DATA/RELEASE/restart.sh " ) ;
2010-05-18 23:31:59 +02:00
OS . deployScript ( scriptFile , script ) ;
Log . logInfo ( " RESTART " , " wrote restart-script to " + scriptFile . getAbsolutePath ( ) ) ;
OS . execAsynchronous ( scriptFile ) ;
Log . logInfo ( " RESTART " , " script is running " ) ;
} catch ( final IOException e ) {
Log . logSevere ( " RESTART " , " restart failed " , e ) ;
}
} else if ( OS . canExecUnix ) {
// start a re-start daemon
try {
Log . logInfo ( " RESTART " , " INITIATED " ) ;
final String script =
" #!/bin/sh " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" cd " + sb . getDataPath ( ) + " /DATA/RELEASE/ " + serverCore . LF_STRING +
2010-05-18 23:31:59 +02:00
" while [ -f ../yacy.running ]; do " + serverCore . LF_STRING +
" sleep 1 " + serverCore . LF_STRING +
" done " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
//"cd ../../" + serverCore.LF_STRING +
" cd " + sb . getAppPath ( ) + serverCore . LF_STRING +
2010-09-04 01:08:43 +02:00
" nohup ./startYACY.sh " + ( startParameter . startsWith ( " -gui " ) ? startParameter : " " ) + " > /dev/null " + serverCore . LF_STRING ;
2010-09-02 21:24:22 +02:00
final File scriptFile = new File ( sb . getDataPath ( ) , " DATA/RELEASE/restart.sh " ) ;
2010-05-18 23:31:59 +02:00
OS . deployScript ( scriptFile , script ) ;
Log . logInfo ( " RESTART " , " wrote restart-script to " + scriptFile . getAbsolutePath ( ) ) ;
OS . execAsynchronous ( scriptFile ) ;
Log . logInfo ( " RESTART " , " script is running " ) ;
2010-09-14 11:13:28 +02:00
sb . terminate ( 10 , " unix restart " ) ;
2010-05-18 23:31:59 +02:00
} catch ( final IOException e ) {
Log . logSevere ( " RESTART " , " restart failed " , e ) ;
2009-05-02 14:12:22 +02:00
}
}
2010-05-18 23:31:59 +02:00
}
2009-05-02 14:12:22 +02:00
/ * *
* stop yacy and run a batch script , applies a new release and restarts yacy
* @param releaseFile
* /
public static void deployRelease ( final File releaseFile ) {
2010-05-18 23:31:59 +02:00
if ( yacyBuildProperties . isPkgManager ( ) ) {
2009-07-11 19:03:22 +02:00
return ;
}
2009-05-02 14:12:22 +02:00
try {
2009-07-19 22:37:44 +02:00
final Switchboard sb = Switchboard . getSwitchboard ( ) ;
2009-05-02 14:12:22 +02:00
Log . logInfo ( " UPDATE " , " INITIATED " ) ;
try {
2010-09-02 21:24:22 +02:00
tarTools . unTar ( tarTools . getInputStream ( releaseFile ) , sb . getDataPath ( ) + " /DATA/RELEASE/ " . replace ( " / " , File . separator ) ) ;
2009-05-02 14:12:22 +02:00
} catch ( final Exception e ) {
2009-06-20 22:46:58 +02:00
Log . logSevere ( " UNTAR " , " failed " , e ) ;
2009-05-02 14:12:22 +02:00
}
String script = null ;
String scriptFileName = null ;
2010-09-10 11:48:09 +02:00
if ( OS . isMacArchitecture ) {
// overwrite Info.plist for Mac Applications (this holds the class paths and can be seen as the start script)
2011-06-13 23:44:03 +02:00
final File InfoPlistSource = new File ( sb . getDataPath ( ) , " DATA/RELEASE/yacy/addon/YaCy.app/Contents/Info.plist " ) ;
final File InfoPlistDestination = new File ( sb . getAppPath ( ) , " addon/YaCy.app/Contents/Info.plist " ) ;
2010-09-10 11:48:09 +02:00
if ( InfoPlistSource . exists ( ) & & InfoPlistDestination . exists ( ) ) {
2012-06-22 11:39:17 +02:00
Files . copy ( InfoPlistSource , InfoPlistDestination ) ;
2010-09-10 11:48:09 +02:00
Log . logInfo ( " UPDATE " , " replaced Info.plist " ) ;
}
}
2009-10-20 00:34:44 +02:00
if ( OS . isWindows ) {
2010-09-02 21:24:22 +02:00
final File startType = new File ( sb . getDataPath ( ) , " DATA/yacy.noconsole " . replace ( " / " , File . separator ) ) ;
2009-06-20 22:46:58 +02:00
String starterFile = " startYACY_debug.bat " ;
if ( startType . exists ( ) ) starterFile = " startYACY.bat " ; // startType noconsole
2010-09-04 01:08:43 +02:00
if ( startParameter . startsWith ( " -gui " ) ) starterFile + = " " + startParameter ;
2011-06-13 23:44:03 +02:00
script =
2009-06-20 22:46:58 +02:00
" @echo off " + serverCore . LF_STRING +
" title YaCy updater " + serverCore . LF_STRING +
" set loading=YACY UPDATER " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" cd \" " + sb . getDataPath ( ) . toString ( ) + " /DATA/RELEASE/ " . replace ( " / " , File . separator ) + " \" " + serverCore . LF_STRING +
2011-06-13 23:44:03 +02:00
2009-06-20 22:46:58 +02:00
" :WAIT " + serverCore . LF_STRING +
" set loading=%loading%. " + serverCore . LF_STRING +
" cls " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
" ping -n 2 127.0.0.1 >nul " + serverCore . LF_STRING +
" IF exist .. \\ yacy.running goto WAIT " + serverCore . LF_STRING +
" IF not exist yacy goto NODATA " + serverCore . LF_STRING +
2009-05-02 14:12:22 +02:00
2009-06-20 22:46:58 +02:00
" cd yacy " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" del /Q \" " + sb . getAppPath ( ) . toString ( ) + " \\ lib \\ * \" >nul " + serverCore . LF_STRING +
" xcopy *.* \" " + sb . getAppPath ( ) . toString ( ) + " \" /E /Y >nul " + serverCore . LF_STRING +
2009-06-20 22:46:58 +02:00
// /E - all subdirectories
// /Y - don't ask
" cd .. " + serverCore . LF_STRING +
" rd yacy /S /Q " + serverCore . LF_STRING +
// /S delete tree
// /Q don't ask
" goto END " + serverCore . LF_STRING +
2011-06-13 23:44:03 +02:00
2009-06-20 22:46:58 +02:00
" :NODATA " + serverCore . LF_STRING +
" echo YACY UPDATER ERROR: NO UPDATE SOURCE FILES ON FILESYSTEM " + serverCore . LF_STRING +
" pause " + serverCore . LF_STRING +
2011-06-13 23:44:03 +02:00
2009-06-20 22:46:58 +02:00
" :END " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" cd \" " + sb . getAppPath ( ) . toString ( ) + " \" " + serverCore . LF_STRING +
2009-06-20 22:46:58 +02:00
" start /MIN CMD /C " + starterFile + serverCore . LF_STRING ;
scriptFileName = " update.bat " ;
2009-05-02 14:12:22 +02:00
} else { // unix/linux
2009-06-20 22:46:58 +02:00
script =
" #!/bin/sh " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" cd " + sb . getDataPath ( ) + " /DATA/RELEASE/ " + serverCore . LF_STRING +
2009-06-20 22:46:58 +02:00
" while [ -f ../yacy.running ]; do " + serverCore . LF_STRING +
" sleep 1 " + serverCore . LF_STRING +
" done " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" rm " + sb . getAppPath ( ) . toString ( ) + " /lib/* " + serverCore . LF_STRING +
" cp -Rf yacy/* " + sb . getAppPath ( ) . toString ( ) + serverCore . LF_STRING +
2009-06-20 22:46:58 +02:00
" rm -Rf yacy " + serverCore . LF_STRING +
2010-09-02 21:24:22 +02:00
" cd " + sb . getAppPath ( ) . toString ( ) + serverCore . LF_STRING +
2010-03-25 10:51:01 +01:00
" chmod 755 *.sh " + serverCore . LF_STRING + // tarTools does not keep access/execute right
" chmod 755 bin/*.sh " + serverCore . LF_STRING +
2010-09-04 01:08:43 +02:00
" nohup ./startYACY.sh " + ( startParameter . startsWith ( " -gui " ) ? startParameter : " " ) + " > /dev/null " + serverCore . LF_STRING ;
2009-06-20 22:46:58 +02:00
scriptFileName = " update.sh " ;
2009-05-02 14:12:22 +02:00
}
2011-06-13 23:44:03 +02:00
final File scriptFile = new File ( sb . getDataPath ( ) , " DATA/RELEASE/ " . replace ( " / " , File . separator ) + scriptFileName ) ;
2009-10-20 00:34:44 +02:00
OS . deployScript ( scriptFile , script ) ;
2009-05-02 14:12:22 +02:00
Log . logInfo ( " UPDATE " , " wrote update-script to " + scriptFile . getAbsolutePath ( ) ) ;
2009-10-20 00:34:44 +02:00
OS . execAsynchronous ( scriptFile ) ;
2009-05-02 14:12:22 +02:00
Log . logInfo ( " UPDATE " , " script is running " ) ;
sb . setConfig ( " update.time.deploy " , System . currentTimeMillis ( ) ) ;
2010-09-14 11:13:28 +02:00
sb . terminate ( 10 , " auto-deploy for " + releaseFile . getName ( ) ) ;
2009-05-02 14:12:22 +02:00
} catch ( final IOException e ) {
Log . logSevere ( " UPDATE " , " update failed " , e ) ;
}
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public static void main ( final String [ ] args ) {
System . out . println ( thisVersion ( ) ) ;
final float base = ( float ) 0 . 53 ;
final String blacklist = " ....[123] " ;
String test ;
for ( int i = 0 ; i < 20 ; i + + ) {
test = Float . toString ( base + ( ( ( float ) i ) / 1000 ) ) ;
System . out . println ( test + " is " + ( ( test . matches ( blacklist ) ) ? " blacklisted " : " not blacklisted " ) ) ;
}
}
/ * *
* keep only releases of last month ( minimum latest and 1 main ( maybe the same ) )
2011-06-13 23:44:03 +02:00
*
2009-05-02 14:12:22 +02:00
* @param filesPath where all downloaded files reside
2011-06-13 23:44:03 +02:00
* @param deleteAfterDays
2009-05-02 14:12:22 +02:00
* /
public static void deleteOldDownloads ( final File filesPath , final int deleteAfterDays ) {
// list downloaded releases
yacyVersion release ;
final String [ ] downloaded = filesPath . list ( ) ;
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
// parse all filenames and put them in a sorted set
final SortedSet < yacyVersion > downloadedreleases = new TreeSet < yacyVersion > ( ) ;
2011-06-13 23:44:03 +02:00
for ( final String element : downloaded ) {
2009-05-02 14:12:22 +02:00
try {
2011-06-13 23:44:03 +02:00
release = new yacyVersion ( element , null ) ;
2009-05-02 14:12:22 +02:00
downloadedreleases . add ( release ) ;
} catch ( final RuntimeException e ) {
// not a valid release
}
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
// if we have some files
2009-12-02 01:37:59 +01:00
if ( ! downloadedreleases . isEmpty ( ) ) {
2009-05-02 14:12:22 +02:00
Log . logFine ( " STARTUP " , " deleting downloaded releases older than " + deleteAfterDays + " days " ) ;
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
// keep latest version
final yacyVersion latest = downloadedreleases . last ( ) ;
downloadedreleases . remove ( latest ) ;
// if latest is a developer release, we also keep a main release
final boolean keepMain = ! latest . isMainRelease ( ) ;
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
// remove old files
final long now = System . currentTimeMillis ( ) ;
final long deleteAfterMillis = deleteAfterDays * 24L * 60 * 60000 ;
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
String lastMain = null ;
String filename ;
for ( final yacyVersion aVersion : downloadedreleases ) {
filename = aVersion . getName ( ) ;
if ( keepMain & & aVersion . isMainRelease ( ) ) {
// keep this one, delete last remembered main release file
if ( lastMain ! = null ) {
filename = lastMain ;
}
lastMain = aVersion . getName ( ) ;
}
// check file age
final File downloadedFile = new File ( filesPath + File . separator + filename ) ;
if ( now - downloadedFile . lastModified ( ) > deleteAfterMillis ) {
// delete file
FileUtils . deletedelete ( downloadedFile ) ;
2011-03-20 18:59:58 +01:00
FileUtils . deletedelete ( new File ( downloadedFile . getAbsolutePath ( ) + " .sig " ) ) ;
2009-05-02 14:12:22 +02:00
if ( downloadedFile . exists ( ) ) {
Log . logWarning ( " STARTUP " , " cannot delete old release " + downloadedFile . getAbsolutePath ( ) ) ;
}
}
}
}
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public File getReleaseFile ( ) {
2011-06-13 23:44:03 +02:00
return this . releaseFile ;
2009-05-02 14:12:22 +02:00
}
2011-06-13 23:44:03 +02:00
2009-05-02 14:12:22 +02:00
public File getSignatureFile ( ) {
2011-06-13 23:44:03 +02:00
return new File ( this . releaseFile . getAbsoluteFile ( ) + " .sig " ) ;
2009-05-02 14:12:22 +02:00
}
public PublicKey getPublicKey ( ) {
2011-06-13 23:44:03 +02:00
return this . publicKey ;
2009-05-02 14:12:22 +02:00
}
}