2006-02-21 01:30:38 +01:00
// yacyVersion.java
2007-04-27 11:23:44 +02:00
// ----------------
2008-07-20 19:14:51 +02:00
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
2007-04-27 11:23:44 +02:00
// first published 27.04.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
2006-02-21 01:30:38 +01:00
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
2007-04-27 11:23:44 +02:00
// LICENSE
//
2006-02-21 01:30:38 +01:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2005-09-22 12:30:55 +02:00
package de.anomic.yacy ;
2008-04-11 00:47:05 +02:00
import java.io.BufferedInputStream ;
2008-04-10 11:53:07 +02:00
import java.io.BufferedOutputStream ;
2007-06-28 16:52:26 +02:00
import java.io.File ;
2008-04-05 15:17:16 +02:00
import java.io.FileOutputStream ;
2007-04-27 11:23:44 +02:00
import java.io.IOException ;
2007-07-02 17:16:05 +02:00
import java.util.ArrayList ;
2007-04-27 11:23:44 +02:00
import java.util.Comparator ;
2007-07-02 17:16:05 +02:00
import java.util.HashMap ;
2007-04-27 11:23:44 +02:00
import java.util.Iterator ;
import java.util.Map ;
2008-04-26 03:08:59 +02:00
import java.util.SortedSet ;
2007-04-27 11:23:44 +02:00
import java.util.TreeSet ;
2007-07-12 18:23:33 +02:00
import java.util.regex.Matcher ;
import java.util.regex.Pattern ;
2007-04-27 11:23:44 +02:00
2008-07-04 13:03:03 +02:00
import de.anomic.crawler.HTTPLoader ;
2007-04-27 11:23:44 +02:00
import de.anomic.htmlFilter.htmlFilterContentScraper ;
2008-04-10 01:35:20 +02:00
import de.anomic.http.JakartaCommonsHttpClient ;
2008-04-11 00:47:05 +02:00
import de.anomic.http.JakartaCommonsHttpResponse ;
2008-08-25 20:11:47 +02:00
import de.anomic.http.httpResponseHeader ;
import de.anomic.http.httpRequestHeader ;
2007-06-28 16:52:26 +02:00
import de.anomic.plasma.plasmaSwitchboard ;
2007-07-02 00:43:08 +02:00
import de.anomic.server.serverCore ;
2008-04-11 00:47:05 +02:00
import de.anomic.server.serverFileUtils ;
2007-06-29 15:56:28 +02:00
import de.anomic.server.serverSystem ;
import de.anomic.server.logging.serverLog ;
2008-05-21 17:23:56 +02:00
import de.anomic.tools.tarTools ;
2007-04-27 11:23:44 +02:00
2008-01-11 15:13:08 +01:00
public final class yacyVersion implements Comparator < yacyVersion > , Comparable < yacyVersion > {
2007-04-27 11:23:44 +02:00
// general release info
2005-09-22 12:30:55 +02:00
public static final float YACY_SUPPORTS_PORT_FORWARDING = ( float ) 0 . 383 ;
2008-05-04 12:53:04 +02:00
public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED = ( float ) 0 . 58204761 ;
2005-11-11 00:48:20 +01:00
public static final float YACY_ACCEPTS_RANKING_TRANSMISSION = ( float ) 0 . 414 ;
2006-11-20 03:46:53 +01:00
public static final float YACY_HANDLES_COLLECTION_INDEX = ( float ) 0 . 486 ;
2007-11-29 03:07:37 +01:00
public static final float YACY_POVIDES_REMOTECRAWL_LISTS = ( float ) 0 . 550 ;
2007-12-20 03:46:41 +01:00
public static final float YACY_STANDARDREL_IS_PRO = ( float ) 0 . 557 ;
2007-04-27 11:23:44 +02:00
// information about latest release, retrieved by other peers release version
public static double latestRelease = 0 . 1 ; // this value is overwritten when a peer with later version appears
// information about latest release, retrieved from download pages
2007-06-28 16:52:26 +02:00
// this static information should be overwritten by network-specific locations
2008-03-16 23:31:54 +01:00
// for details see defaults/yacy.network.freeworld.unit
2008-01-12 01:43:33 +01:00
private static HashMap < yacyURL , DevMain > latestReleases = new HashMap < yacyURL , DevMain > ( ) ;
2008-08-02 15:57:00 +02:00
public final static ArrayList < yacyURL > latestReleaseLocations = new ArrayList < yacyURL > ( ) ; // will be initialized with value in defaults/yacy.network.freeworld.unit
2007-04-27 11:23:44 +02:00
2007-07-11 01:56:25 +02:00
// private static release info about this release; is generated only once and can be retrieved by thisVersion()
private static yacyVersion thisVersion = null ;
2007-04-27 11:23:44 +02:00
// class variables
public float releaseNr ;
public String dateStamp ;
public int svn ;
2008-01-12 01:43:33 +01:00
public boolean mainRelease ;
2007-09-05 11:01:35 +02:00
public yacyURL url ;
2007-06-28 16:52:26 +02:00
public String name ;
2007-04-27 11:23:44 +02:00
2008-08-02 14:12:04 +02:00
public yacyVersion ( final yacyURL url ) {
2007-04-27 11:23:44 +02:00
this ( url . getFileName ( ) ) ;
this . url = url ;
}
public yacyVersion ( String release ) {
// parse a release file name
// the have the following form:
// yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
// yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
// i.e. yacy_v0.51_20070321_3501.tar.gz
this . url = null ;
2007-06-28 16:52:26 +02:00
this . name = release ;
2008-04-11 14:55:43 +02:00
if ( ( release = = null ) | | ( ! ( ( release . endsWith ( " .tar.gz " ) | | ( release . endsWith ( " .tar " ) ) ) ) ) ) {
2007-04-27 11:23:44 +02:00
throw new RuntimeException ( " release file name ' " + release + " ' is not valid, no tar.gz " ) ;
}
// cut off tail
2008-04-11 14:55:43 +02:00
release = release . substring ( 0 , release . length ( ) - ( ( release . endsWith ( " .gz " ) ) ? 7 : 4 ) ) ;
2007-06-28 16:52:26 +02:00
if ( release . startsWith ( " yacy_pro_v " ) ) {
2007-12-20 03:46:41 +01:00
release = release . substring ( 10 ) ;
} else if ( release . startsWith ( " yacy_emb_v " ) ) {
2008-01-12 01:43:33 +01:00
throw new RuntimeException ( " release file name ' " + release + " ' is not valid, no support for emb " ) ;
2007-04-27 11:23:44 +02:00
} else if ( release . startsWith ( " yacy_v " ) ) {
release = release . substring ( 6 ) ;
} else {
throw new RuntimeException ( " release file name ' " + release + " ' is not valid, wrong prefix " ) ;
}
// now all release names have the form
// ${releaseVersion}_${DSTAMP}_${releaseNr}
2008-08-02 14:12:04 +02:00
final String [ ] comp = release . split ( " _ " ) ; // should be 3 parts
2007-04-27 11:23:44 +02:00
if ( comp . length ! = 3 ) {
throw new RuntimeException ( " release file name ' " + release + " ' is not valid, 3 information parts expected " ) ;
}
try {
this . releaseNr = Float . parseFloat ( comp [ 0 ] ) ;
2008-08-02 14:12:04 +02:00
} catch ( final NumberFormatException e ) {
2007-04-27 11:23:44 +02:00
throw new RuntimeException ( " release file name ' " + release + " ' is not valid, ' " + comp [ 0 ] + " ' should be a float number " ) ;
}
2008-06-06 18:01:27 +02:00
this . mainRelease = ( ( int ) ( this . releaseNr * 1000 ) ) % 10 = = 0 ;
2007-06-28 16:52:26 +02:00
//System.out.println("Release version " + this.releaseNr + " is " + ((this.mainRelease) ? "main" : "std"));
2007-04-27 11:23:44 +02:00
this . dateStamp = comp [ 1 ] ;
if ( this . dateStamp . length ( ) ! = 8 ) {
throw new RuntimeException ( " release file name ' " + release + " ' is not valid, ' " + comp [ 1 ] + " ' should be a 8-digit date string " ) ;
}
try {
this . svn = Integer . parseInt ( comp [ 2 ] ) ;
2008-08-02 14:12:04 +02:00
} catch ( final NumberFormatException e ) {
2007-04-27 11:23:44 +02:00
throw new RuntimeException ( " release file name ' " + release + " ' is not valid, ' " + comp [ 2 ] + " ' should be a integer number " ) ;
}
// finished! we parsed a relase string
}
2008-01-12 01:43:33 +01:00
public static final class DevMain {
public TreeSet < yacyVersion > dev , main ;
2008-08-02 14:12:04 +02:00
public DevMain ( final TreeSet < yacyVersion > dev , final TreeSet < yacyVersion > main ) {
2008-01-12 01:43:33 +01:00
this . dev = dev ;
this . main = main ;
}
}
2008-08-02 14:12:04 +02:00
public int compareTo ( final yacyVersion obj ) {
2007-07-11 01:56:25 +02:00
// returns 0 if this object is equal to the obj, -1 if this is smaller than obj and 1 if this is greater than obj
2008-01-11 15:13:08 +01:00
return compare ( this , obj ) ;
2007-04-27 11:23:44 +02:00
}
2008-08-02 14:12:04 +02:00
public int compare ( final yacyVersion v0 , final yacyVersion v1 ) {
2007-04-27 11:23:44 +02:00
// compare-operator for two yacyVersion objects
// must be implemented to make it possible to put this object into
// a ordered structure, like TreeSet or TreeMap
2008-08-06 21:43:12 +02:00
return ( Integer . valueOf ( v0 . svn ) ) . compareTo ( Integer . valueOf ( v1 . svn ) ) ;
2007-04-27 11:23:44 +02:00
}
2008-08-02 14:12:04 +02:00
public boolean equals ( final Object obj ) {
2008-05-03 11:06:00 +02:00
if ( obj instanceof yacyVersion ) {
2008-08-02 14:12:04 +02:00
final yacyVersion v = ( yacyVersion ) obj ;
2008-05-03 11:06:00 +02:00
return ( this . svn = = v . svn ) & & ( this . url . toNormalform ( true , true ) . equals ( v . url . toNormalform ( true , true ) ) ) ;
}
2008-08-02 15:57:00 +02:00
return false ;
2007-04-27 11:23:44 +02:00
}
public int hashCode ( ) {
2007-07-19 17:32:10 +02:00
return this . url . toNormalform ( true , true ) . hashCode ( ) ;
2007-04-27 11:23:44 +02:00
}
public String toAnchor ( ) {
// generates an anchor string that can be used to embed in an html for direct download
2008-01-12 01:43:33 +01:00
return " <a href= " + this . url . toNormalform ( true , true ) + " >YaCy " + ( ( this . mainRelease ) ? " main release " : " dev release " ) + " v " + this . releaseNr + " , SVN " + this . svn + " </a> " ;
2007-06-28 16:52:26 +02:00
}
2007-04-27 11:23:44 +02:00
2007-07-11 01:56:25 +02:00
// static methods:
public static final yacyVersion thisVersion ( ) {
// construct a virtual release name for this release
if ( thisVersion = = null ) {
2008-08-02 14:12:04 +02:00
final plasmaSwitchboard sb = plasmaSwitchboard . getSwitchboard ( ) ;
2008-04-10 00:59:17 +02:00
if ( sb = = null ) return null ;
2008-08-02 14:12:04 +02:00
final boolean full = new File ( sb . getRootPath ( ) , " libx " ) . exists ( ) ;
2007-07-11 01:56:25 +02:00
thisVersion = new yacyVersion (
2007-12-20 03:46:41 +01:00
" yacy " + ( ( full ) ? " " : " _emb " ) +
2007-07-11 01:56:25 +02:00
" _v " + sb . getConfig ( " version " , " 0.1 " ) + " _ " +
sb . getConfig ( " vdate " , " 19700101 " ) + " _ " +
sb . getConfig ( " svnRevision " , " 0 " ) + " .tar.gz " ) ;
}
return thisVersion ;
}
2008-08-02 14:12:04 +02:00
public static final yacyVersion rulebasedUpdateInfo ( final boolean manual ) {
2007-07-11 01:56:25 +02:00
// according to update properties, decide if we should retrieve update information
// if true, the release that can be obtained is returned.
// if false, null is returned
2008-08-02 14:12:04 +02:00
final plasmaSwitchboard sb = plasmaSwitchboard . getSwitchboard ( ) ;
2007-07-11 01:56:25 +02:00
2007-12-20 03:46:41 +01:00
// check if update process allows update retrieve
2008-08-02 14:12:04 +02:00
final String process = sb . getConfig ( " update.process " , " manual " ) ;
2007-07-17 01:47:21 +02:00
if ( ( ! manual ) & & ( ! process . equals ( " auto " ) ) ) {
2007-07-18 15:03:20 +02:00
yacyCore . log . logInfo ( " rulebasedUpdateInfo: not an automatic update selected " ) ;
2007-07-17 01:47:21 +02:00
return null ; // no, its a manual or guided process
}
2007-07-11 01:56:25 +02:00
2007-07-12 18:23:33 +02:00
// check if the last retrieve time is a minimum time ago
2008-08-02 14:12:04 +02:00
final long cycle = Math . max ( 1 , sb . getConfigLong ( " update.cycle " , 168 ) ) * 60 * 60 * 1000 ; // update.cycle is hours
final long timeLookup = sb . getConfigLong ( " update.time.lookup " , System . currentTimeMillis ( ) ) ;
2007-07-17 01:47:21 +02:00
if ( ( ! manual ) & & ( timeLookup + cycle > System . currentTimeMillis ( ) ) ) {
2007-07-18 15:03:20 +02:00
yacyCore . log . logInfo ( " rulebasedUpdateInfo: too early for a lookup for a new release (timeLookup = " + timeLookup + " , cycle = " + cycle + " , now = " + System . currentTimeMillis ( ) + " ) " ) ;
2007-07-17 01:47:21 +02:00
return null ; // no we have recently made a lookup
}
2007-07-11 01:56:25 +02:00
2007-07-12 18:23:33 +02:00
// check if we know that there is a release that is more recent than that which we are using
2008-08-02 14:12:04 +02:00
final DevMain releasess = yacyVersion . allReleases ( true ) ;
final yacyVersion latestmain = ( releasess . main . size ( ) = = 0 ) ? null : releasess . main . last ( ) ;
final yacyVersion latestdev = ( releasess . dev . size ( ) = = 0 ) ? null : releasess . dev . last ( ) ;
final String concept = sb . getConfig ( " update.concept " , " any " ) ;
final String blacklist = sb . getConfig ( " update.blacklist " , " . \\ ...[123] " ) ;
2007-07-19 17:32:10 +02:00
2007-07-17 01:47:21 +02:00
if ( ( manual ) | | ( concept . equals ( " any " ) ) ) {
2007-07-11 01:56:25 +02:00
// return a dev-release or a main-release
2007-07-19 17:32:10 +02:00
if ( ( latestdev ! = null ) & &
( ( latestmain = = null ) | | ( latestdev . compareTo ( latestmain ) > 0 ) ) & &
( ! ( Float . toString ( latestdev . releaseNr ) . matches ( blacklist ) ) ) ) {
// consider a dev-release
2008-08-02 15:57:00 +02:00
if ( latestdev . compareTo ( thisVersion ( ) ) < = 0 ) {
2007-07-19 17:32:10 +02:00
yacyCore . log . logInfo (
" rulebasedUpdateInfo: latest dev " + latestdev . name +
" is not more recent than installed release " + thisVersion ( ) . name ) ;
2007-07-17 01:47:21 +02:00
return null ;
}
2008-08-02 15:57:00 +02:00
return latestdev ;
2007-07-19 17:32:10 +02:00
}
if ( latestmain ! = null ) {
// consider a main release
2007-07-17 01:47:21 +02:00
if ( ( Float . toString ( latestmain . releaseNr ) . matches ( blacklist ) ) ) {
2007-07-19 17:32:10 +02:00
yacyCore . log . logInfo (
2008-06-06 18:01:27 +02:00
" rulebasedUpdateInfo: latest dev " + ( latestdev = = null ? " null " : latestdev . name ) +
2007-07-19 17:32:10 +02:00
" matches with blacklist ' " + blacklist + " ' " ) ;
2007-07-17 01:47:21 +02:00
return null ;
}
2008-08-02 15:57:00 +02:00
if ( latestmain . compareTo ( thisVersion ( ) ) < = 0 ) {
2007-07-19 17:32:10 +02:00
yacyCore . log . logInfo (
" rulebasedUpdateInfo: latest main " + latestmain . name +
" is not more recent than installed release (1) " + thisVersion ( ) . name ) ;
2007-07-17 01:47:21 +02:00
return null ;
}
2008-08-02 15:57:00 +02:00
return latestmain ;
2007-07-11 01:56:25 +02:00
}
}
2007-07-19 17:32:10 +02:00
if ( ( concept . equals ( " main " ) ) & & ( latestmain ! = null ) ) {
2007-07-11 01:56:25 +02:00
// return a main-release
2007-07-17 01:47:21 +02:00
if ( ( Float . toString ( latestmain . releaseNr ) . matches ( blacklist ) ) ) {
2007-07-19 17:32:10 +02:00
yacyCore . log . logInfo (
" rulebasedUpdateInfo: latest main " + latestmain . name +
" matches with blacklist' " + blacklist + " ' " ) ;
2007-07-17 01:47:21 +02:00
return null ;
}
2008-08-02 15:57:00 +02:00
if ( latestmain . compareTo ( thisVersion ( ) ) < = 0 ) {
2007-07-19 17:32:10 +02:00
yacyCore . log . logInfo (
" rulebasedUpdateInfo: latest main " + latestmain . name +
" is not more recent than installed release (2) " + thisVersion ( ) . name ) ;
2007-07-17 01:47:21 +02:00
return null ;
}
2008-08-02 15:57:00 +02:00
return latestmain ;
2007-07-11 01:56:25 +02:00
}
2007-07-17 01:47:21 +02:00
yacyCore . log . logInfo ( " rulebasedUpdateInfo: failed to find more recent release " ) ;
2007-07-11 01:56:25 +02:00
return null ;
}
2008-08-02 14:12:04 +02:00
public static DevMain allReleases ( final boolean force ) {
2007-06-28 16:52:26 +02:00
// join the release infos
2008-08-02 14:12:04 +02:00
final DevMain [ ] a = new DevMain [ latestReleaseLocations . size ( ) ] ;
2007-07-02 17:16:05 +02:00
for ( int j = 0 ; j < latestReleaseLocations . size ( ) ; j + + ) {
2008-01-12 01:43:33 +01:00
a [ j ] = getReleases ( latestReleaseLocations . get ( j ) , force ) ;
2007-04-27 11:23:44 +02:00
}
2008-08-02 14:12:04 +02:00
final TreeSet < yacyVersion > alldev = new TreeSet < yacyVersion > ( ) ;
final TreeSet < yacyVersion > allmain = new TreeSet < yacyVersion > ( ) ;
2008-01-12 01:43:33 +01:00
for ( int j = 0 ; j < a . length ; j + + ) if ( ( a [ j ] ! = null ) & & ( a [ j ] . dev ! = null ) ) alldev . addAll ( a [ j ] . dev ) ;
for ( int j = 0 ; j < a . length ; j + + ) if ( ( a [ j ] ! = null ) & & ( a [ j ] . main ! = null ) ) allmain . addAll ( a [ j ] . main ) ;
return new DevMain ( alldev , allmain ) ;
2007-04-27 11:23:44 +02:00
}
2008-08-02 14:12:04 +02:00
private static DevMain getReleases ( final yacyURL location , final boolean force ) {
2007-06-28 16:52:26 +02:00
// get release info from a internet resource
2008-06-06 18:01:27 +02:00
DevMain locLatestRelease = latestReleases . get ( location ) ;
2007-07-11 01:56:25 +02:00
if ( force | |
2008-06-06 18:01:27 +02:00
( locLatestRelease = = null ) / * | |
2007-07-02 17:16:05 +02:00
( ( latestRelease [ 0 ] . size ( ) = = 0 ) & &
( latestRelease [ 1 ] . size ( ) = = 0 ) & &
( latestRelease [ 2 ] . size ( ) = = 0 ) & &
2007-07-12 18:23:33 +02:00
( latestRelease [ 3 ] . size ( ) = = 0 ) ) * / ) {
2008-06-06 18:01:27 +02:00
locLatestRelease = allReleaseFrom ( location ) ;
latestReleases . put ( location , locLatestRelease ) ;
2007-04-27 11:23:44 +02:00
}
2008-06-06 18:01:27 +02:00
return locLatestRelease ;
2007-04-27 11:23:44 +02:00
}
2008-01-12 01:43:33 +01:00
private static DevMain allReleaseFrom ( yacyURL url ) {
2007-04-27 11:23:44 +02:00
// retrieves the latest info about releases
// this is done by contacting a release location,
// parsing the content and filtering+parsing links
// returns the version info if successful, null otherwise
htmlFilterContentScraper scraper ;
try {
scraper = htmlFilterContentScraper . parseResource ( url ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-04-27 11:23:44 +02:00
return null ;
}
// analyse links in scraper resource, and find link to latest release in it
2008-08-02 14:12:04 +02:00
final Map < yacyURL , String > anchors = scraper . getAnchors ( ) ; // a url (String) / name (String) relation
final Iterator < yacyURL > i = anchors . keySet ( ) . iterator ( ) ;
final TreeSet < yacyVersion > devreleases = new TreeSet < yacyVersion > ( ) ;
final TreeSet < yacyVersion > mainreleases = new TreeSet < yacyVersion > ( ) ;
2007-04-27 11:23:44 +02:00
yacyVersion release ;
while ( i . hasNext ( ) ) {
2008-01-22 12:51:43 +01:00
url = i . next ( ) ;
2007-04-27 11:23:44 +02:00
try {
release = new yacyVersion ( url ) ;
//System.out.println("r " + release.toAnchor());
2008-01-12 01:43:33 +01:00
if ( release . mainRelease ) mainreleases . add ( release ) ;
if ( ! release . mainRelease ) devreleases . add ( release ) ;
2008-08-02 14:12:04 +02:00
} catch ( final RuntimeException e ) {
2007-04-27 11:23:44 +02:00
// the release string was not well-formed.
// that might have been another link
// just dont care
continue ;
}
}
2007-07-11 01:56:25 +02:00
plasmaSwitchboard . getSwitchboard ( ) . setConfig ( " update.time.lookup " , System . currentTimeMillis ( ) ) ;
2008-01-12 01:43:33 +01:00
return new DevMain ( devreleases , mainreleases ) ;
2007-06-28 16:52:26 +02:00
}
2008-08-02 14:12:04 +02:00
public static File downloadRelease ( final yacyVersion release ) {
final File storagePath = plasmaSwitchboard . getSwitchboard ( ) . releasePath ;
2007-06-28 16:52:26 +02:00
// load file
2008-04-11 14:37:17 +02:00
File download = null ;
2008-08-25 20:11:47 +02:00
final httpRequestHeader header = new httpRequestHeader ( ) ;
header . put ( httpResponseHeader . USER_AGENT , HTTPLoader . yacyUserAgent ) ;
2008-08-17 12:16:32 +02:00
final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient ( 120000 , header ) ;
2008-04-11 00:47:05 +02:00
JakartaCommonsHttpResponse res = null ;
2008-08-02 14:12:04 +02:00
final String name = release . url . getFileName ( ) ;
2008-04-05 15:17:16 +02:00
try {
res = client . GET ( release . url . toString ( ) ) ;
2008-08-02 14:12:04 +02:00
final boolean unzipped = res . getResponseHeader ( ) . gzip ( ) & & ( res . getResponseHeader ( ) . mime ( ) . toLowerCase ( ) . equals ( " application/x-tar " ) ) ; // if true, then the httpc has unzipped the file
2008-04-11 14:37:17 +02:00
if ( ( unzipped ) & & ( name . endsWith ( " .tar.gz " ) ) ) {
download = new File ( storagePath , name . substring ( 0 , name . length ( ) - 3 ) ) ;
} else {
download = new File ( storagePath , name ) ;
}
2008-04-11 00:47:05 +02:00
try {
serverFileUtils . copyToStream ( new BufferedInputStream ( res . getDataAsStream ( ) ) , new BufferedOutputStream ( new FileOutputStream ( download ) ) ) ;
} finally {
res . closeStream ( ) ;
}
2008-04-08 23:36:33 +02:00
if ( ( ! download . exists ( ) ) | | ( download . length ( ) = = 0 ) ) throw new IOException ( " wget of url " + release . url + " failed " ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-04-08 23:36:33 +02:00
serverLog . logSevere ( " yacyVersion " , " download of " + release . name + " failed: " + e . getMessage ( ) ) ;
2008-08-06 21:43:12 +02:00
if ( download ! = null & & download . exists ( ) )
if ( ! download . delete ( ) )
serverLog . logWarning ( " yacyVersion " , " could not delete file " + download ) ;
2008-04-11 14:37:17 +02:00
download = null ;
2008-04-05 15:17:16 +02:00
} finally {
2008-04-08 23:36:33 +02:00
if ( res ! = null ) {
2008-04-05 15:17:16 +02:00
// release connection
res . closeStream ( ) ;
}
}
2007-07-11 01:56:25 +02:00
plasmaSwitchboard . getSwitchboard ( ) . setConfig ( " update.time.download " , System . currentTimeMillis ( ) ) ;
2008-04-11 14:55:43 +02:00
return ( ( download ! = null ) & & ( download . exists ( ) ) ) ? download : null ;
2007-04-27 11:23:44 +02:00
}
2007-06-28 22:36:15 +02:00
2007-06-29 15:56:28 +02:00
public static void restart ( ) {
2008-08-02 14:12:04 +02:00
final plasmaSwitchboard sb = plasmaSwitchboard . getSwitchboard ( ) ;
final String apphome = sb . getRootPath ( ) . toString ( ) ;
2007-07-11 01:56:25 +02:00
2008-05-21 17:23:56 +02:00
if ( serverSystem . isWindows ) {
2008-08-02 14:12:04 +02:00
final File startType = new File ( sb . getRootPath ( ) , " DATA/yacy.noconsole " . replace ( " / " , File . separator ) ) ;
2008-07-14 20:41:11 +02:00
String starterFile = " startYACY_debug.bat " ;
if ( startType . exists ( ) ) starterFile = " startYACY.bat " ; // startType noconsole
2008-05-22 17:48:51 +02:00
try {
serverLog . logInfo ( " RESTART " , " INITIATED " ) ;
2008-08-02 14:12:04 +02:00
final String script =
2008-05-22 17:48:51 +02:00
" @echo off " + serverCore . LF_STRING +
" title YaCy restarter " + serverCore . LF_STRING +
2008-09-03 20:49:02 +02:00
" set loading=YACY RESTARTER " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
" cd \" " + apphome + " /DATA/RELEASE/ " . replace ( " / " , File . separator ) + " \" " + serverCore . LF_STRING +
2008-05-22 17:48:51 +02:00
" :WAIT " + serverCore . LF_STRING +
2008-09-03 20:49:02 +02:00
" set loading=%loading%. " + serverCore . LF_STRING +
" cls " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
2008-05-22 17:48:51 +02:00
" ping -n 2 127.0.0.1 >nul " + serverCore . LF_STRING +
" IF exist .. \\ yacy.running goto WAIT " + serverCore . LF_STRING +
2008-09-03 20:49:02 +02:00
" cd \" " + apphome + " \" " + serverCore . LF_STRING +
2008-05-22 17:48:51 +02:00
" start /MIN CMD /C " + starterFile + serverCore . LF_STRING ;
2008-08-02 14:12:04 +02:00
final File scriptFile = new File ( sb . getRootPath ( ) , " DATA/RELEASE/restart.bat " . replace ( " / " , File . separator ) ) ;
2008-05-22 17:48:51 +02:00
serverSystem . deployScript ( scriptFile , script ) ;
serverLog . logInfo ( " RESTART " , " wrote restart-script to " + scriptFile . getAbsolutePath ( ) ) ;
serverSystem . execAsynchronous ( scriptFile ) ;
serverLog . logInfo ( " RESTART " , " script is running " ) ;
sb . terminate ( 5000 ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-05-22 17:48:51 +02:00
serverLog . logSevere ( " RESTART " , " restart failed " , e ) ;
}
2007-06-29 15:56:28 +02:00
// create yacy.restart file which is used in Windows startscript
2008-05-22 17:48:51 +02:00
/ * final File yacyRestart = new File ( sb . getRootPath ( ) , " DATA/yacy.restart " ) ;
2007-06-29 15:56:28 +02:00
if ( ! yacyRestart . exists ( ) ) {
try {
yacyRestart . createNewFile ( ) ;
plasmaSwitchboard . getSwitchboard ( ) . terminate ( 5000 ) ;
} catch ( IOException e ) {
serverLog . logSevere ( " SHUTDOWN " , " restart failed " , e ) ;
}
2008-05-22 17:48:51 +02:00
} * /
2007-06-29 15:56:28 +02:00
}
2007-06-29 23:36:18 +02:00
if ( serverSystem . canExecUnix ) {
// start a re-start daemon
try {
serverLog . logInfo ( " RESTART " , " INITIATED " ) ;
2008-08-02 14:12:04 +02:00
final String script =
2007-12-14 20:17:54 +01:00
" #!/bin/sh " + serverCore . LF_STRING +
" cd " + sb . getRootPath ( ) + " /DATA/RELEASE/ " + serverCore . LF_STRING +
" while [ -f ../yacy.running ]; do " + serverCore . LF_STRING +
" sleep 1 " + serverCore . LF_STRING +
" done " + serverCore . LF_STRING +
" cd ../../ " + serverCore . LF_STRING +
" nohup ./startYACY.sh > /dev/null " + serverCore . LF_STRING ;
2008-08-02 14:12:04 +02:00
final File scriptFile = new File ( sb . getRootPath ( ) , " DATA/RELEASE/restart.sh " ) ;
2007-06-29 23:36:18 +02:00
serverSystem . deployScript ( scriptFile , script ) ;
serverLog . logInfo ( " RESTART " , " wrote restart-script to " + scriptFile . getAbsolutePath ( ) ) ;
serverSystem . execAsynchronous ( scriptFile ) ;
serverLog . logInfo ( " RESTART " , " script is running " ) ;
2007-07-11 01:56:25 +02:00
sb . terminate ( 5000 ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-06-29 23:36:18 +02:00
serverLog . logSevere ( " RESTART " , " restart failed " , e ) ;
}
}
2007-06-29 15:56:28 +02:00
}
2008-08-02 14:12:04 +02:00
public static void deployRelease ( final File releaseFile ) {
2007-06-29 15:56:28 +02:00
//byte[] script = ("cd " + plasmaSwitchboard.getSwitchboard().getRootPath() + ";while [ -e ../yacy.running ]; do sleep 1;done;tar xfz " + release + ";cp -Rf yacy/* ../../;rm -Rf yacy;cd ../../;startYACY.sh").getBytes();
2007-07-02 17:16:05 +02:00
try {
2008-08-02 14:12:04 +02:00
final plasmaSwitchboard sb = plasmaSwitchboard . getSwitchboard ( ) ;
final String apphome = sb . getRootPath ( ) . toString ( ) ;
2007-07-02 17:16:05 +02:00
serverLog . logInfo ( " UPDATE " , " INITIATED " ) ;
2008-05-21 17:23:56 +02:00
try {
tarTools . unTar ( tarTools . getInputStream ( releaseFile ) , sb . getRootPath ( ) + " /DATA/RELEASE/ " . replace ( " / " , File . separator ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2008-05-21 17:23:56 +02:00
serverLog . logSevere ( " UNTAR " , " failed " , e ) ;
}
String script = null ;
String scriptFileName = null ;
if ( serverSystem . isWindows ) {
2008-08-02 14:12:04 +02:00
final File startType = new File ( sb . getRootPath ( ) , " DATA/yacy.noconsole " . replace ( " / " , File . separator ) ) ;
2008-07-14 20:41:11 +02:00
String starterFile = " startYACY_debug.bat " ;
if ( startType . exists ( ) ) starterFile = " startYACY.bat " ; // startType noconsole
2008-05-21 17:23:56 +02:00
script =
" @echo off " + serverCore . LF_STRING +
" title YaCy updater " + serverCore . LF_STRING +
2008-09-03 20:49:02 +02:00
" set loading=YACY UPDATER " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
" cd \" " + apphome + " /DATA/RELEASE/ " . replace ( " / " , File . separator ) + " \" " + serverCore . LF_STRING +
2008-05-21 17:23:56 +02:00
" :WAIT " + serverCore . LF_STRING +
2008-09-03 20:49:02 +02:00
" set loading=%loading%. " + serverCore . LF_STRING +
" cls " + serverCore . LF_STRING +
" echo %loading% " + serverCore . LF_STRING +
2008-05-21 17:23:56 +02:00
" ping -n 2 127.0.0.1 >nul " + serverCore . LF_STRING +
" IF exist .. \\ yacy.running goto WAIT " + serverCore . LF_STRING +
" IF not exist yacy goto NODATA " + serverCore . LF_STRING +
2008-05-22 17:48:51 +02:00
2008-05-21 17:23:56 +02:00
" cd yacy " + serverCore . LF_STRING +
2008-09-03 20:49:02 +02:00
" xcopy *.* \" " + apphome + " \" /E /Y >nul " + serverCore . LF_STRING +
2008-05-21 17:23:56 +02:00
// /E - all subdirectories
// /Y - don't ask
" cd .. " + serverCore . LF_STRING +
" rd yacy /S /Q " + serverCore . LF_STRING +
// /S delete tree
// /Q don't ask
" goto END " + serverCore . LF_STRING +
" :NODATA " + serverCore . LF_STRING +
" echo YACY UPDATER ERROR: NO UPDATE SOURCE FILES ON FILESYSTEM " + serverCore . LF_STRING +
" pause " + serverCore . LF_STRING +
" :END " + serverCore . LF_STRING +
2008-09-03 20:49:02 +02:00
" cd \" " + apphome + " \" " + serverCore . LF_STRING +
2008-05-24 22:03:11 +02:00
" start /MIN CMD /C " + starterFile + serverCore . LF_STRING ;
2008-05-21 17:23:56 +02:00
scriptFileName = " update.bat " ;
} else { // unix/linux
script =
" #!/bin/sh " + serverCore . LF_STRING +
" cd " + sb . getRootPath ( ) + " /DATA/RELEASE/ " + serverCore . LF_STRING +
/ * ( ( releaseFile . getName ( ) . endsWith ( " .gz " ) ) ?
// test gz-file for integrity and tar xfz then
( " if gunzip -t " + releaseFile . getAbsolutePath ( ) + serverCore . LF_STRING +
" then " + serverCore . LF_STRING +
" gunzip -c " + releaseFile . getAbsolutePath ( ) + " | tar xf - " + serverCore . LF_STRING ) :
// just tar xf the file, no integrity test possible?
( " tar xf " + releaseFile . getAbsolutePath ( ) + serverCore . LF_STRING )
) + * /
" while [ -f ../yacy.running ]; do " + serverCore . LF_STRING +
" sleep 1 " + serverCore . LF_STRING +
" done " + serverCore . LF_STRING +
" cp -Rf yacy/* " + apphome + serverCore . LF_STRING +
" rm -Rf yacy " + serverCore . LF_STRING +
/ * ( ( releaseFile . getName ( ) . endsWith ( " .gz " ) ) ?
// else-case of gunzip -t test: if failed, just restart
( " else " + serverCore . LF_STRING +
" while [ -f ../yacy.running ]; do " + serverCore . LF_STRING +
" sleep 1 " + serverCore . LF_STRING +
" done " + serverCore . LF_STRING +
" fi " + serverCore . LF_STRING ) :
// in case that we did not made the integrity test, there is no else case
" "
) + * /
" cd " + apphome + serverCore . LF_STRING +
" nohup ./startYACY.sh > /dev/null " + serverCore . LF_STRING ;
scriptFileName = " update.sh " ;
}
2008-08-02 14:12:04 +02:00
final File scriptFile = new File ( sb . getRootPath ( ) , " DATA/RELEASE/ " . replace ( " / " , File . separator ) + scriptFileName ) ;
2007-07-02 17:16:05 +02:00
serverSystem . deployScript ( scriptFile , script ) ;
serverLog . logInfo ( " UPDATE " , " wrote update-script to " + scriptFile . getAbsolutePath ( ) ) ;
serverSystem . execAsynchronous ( scriptFile ) ;
serverLog . logInfo ( " UPDATE " , " script is running " ) ;
2007-07-11 01:56:25 +02:00
sb . setConfig ( " update.time.deploy " , System . currentTimeMillis ( ) ) ;
sb . terminate ( 5000 ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-07-02 17:16:05 +02:00
serverLog . logSevere ( " UPDATE " , " update failed " , e ) ;
}
2007-06-28 22:36:15 +02:00
}
2007-07-12 18:23:33 +02:00
/ * *
* Converts combined version - string to a pretty string , e . g . " 0.435/01818 " or " dev/01818 " ( development version ) or " dev/00000 " ( in case of wrong input )
*
* @param ver Combined version string matching regular expression : " \ A( \ d+ \ . \ d{3})( \ d{4}| \ d{5}) \ z " < br >
* ( i . e . : start of input , 1 or more digits in front of decimal point , decimal point followed by 3 digits as major version , 4 or 5 digits for SVN - Version , end of input )
* @return If the major version is & lt ; 0 . 11 - major version is separated from SVN - version by '/' , e . g . " 0.435/01818 " < br >
* If the major version is & gt ; = 0 . 11 - major version is replaced by " dev " and separated SVN - version by '/' , e . g . " dev/01818 " < br >
* " dev/00000 " - If the input does not matcht the regular expression above
* /
2008-08-02 14:12:04 +02:00
public static String combined2prettyVersion ( final String ver ) {
2007-07-12 18:23:33 +02:00
return combined2prettyVersion ( ver , " " ) ;
}
2008-08-02 14:12:04 +02:00
public static String combined2prettyVersion ( final String ver , final String computerName ) {
2007-07-12 18:23:33 +02:00
final Matcher matcher = Pattern . compile ( " \\ A( \\ d+ \\ . \\ d{1,3})( \\ d{0,5}) \\ z " ) . matcher ( ver ) ;
if ( ! matcher . find ( ) ) {
serverLog . logWarning ( " STARTUP " , " Peer ' " + computerName + " ': wrong format of version-string: ' " + ver + " '. Using default string 'dev/00000' instead " ) ;
return " dev/00000 " ;
}
2008-08-02 14:12:04 +02:00
final String mainversion = ( Double . parseDouble ( matcher . group ( 1 ) ) < 0 . 11 ? " dev " : matcher . group ( 1 ) ) ;
2007-07-12 18:23:33 +02:00
String revision = matcher . group ( 2 ) ;
for ( int i = revision . length ( ) ; i < 5 ; + + i ) revision + = " 0 " ;
return mainversion + " / " + revision ;
}
/ * *
* Combines the version of YaCy with the versionnumber from SVN to a
* combined version
*
* @param version Current given version .
* @param svn Current version given from SVN .
* @return String with the combined version .
* /
2008-08-02 14:12:04 +02:00
public static double versvn2combinedVersion ( final double version , final int svn ) {
2008-06-06 18:01:27 +02:00
return ( Math . rint ( ( version * 100000000 . 0 ) + ( svn ) ) / 100000000 ) ;
2007-07-12 18:23:33 +02:00
}
2008-08-02 14:12:04 +02:00
public static void main ( final String [ ] args ) {
2008-04-05 15:17:16 +02:00
System . out . println ( thisVersion ( ) ) ;
2008-08-02 14:12:04 +02:00
final float base = ( float ) 0 . 53 ;
final String blacklist = " ....[123] " ;
2007-07-12 18:23:33 +02:00
String test ;
for ( int i = 0 ; i < 20 ; i + + ) {
test = Float . toString ( base + ( ( ( float ) i ) / 1000 ) ) ;
System . out . println ( test + " is " + ( ( test . matches ( blacklist ) ) ? " blacklisted " : " not blacklisted " ) ) ;
}
}
2008-04-22 18:53:53 +02:00
/ * *
* keep only releases of last month ( minimum latest and 1 main ( maybe the same ) )
*
* @param filesPath where all downloaded files reside
* @param deleteAfterDays
* /
2008-08-02 14:12:04 +02:00
public static void deleteOldDownloads ( final File filesPath , final int deleteAfterDays ) {
2008-04-22 18:53:53 +02:00
// list downloaded releases
yacyVersion release ;
2008-08-02 14:12:04 +02:00
final String [ ] downloaded = filesPath . list ( ) ;
2008-04-22 18:53:53 +02:00
// parse all filenames and put them in a sorted set
2008-08-02 14:12:04 +02:00
final SortedSet < yacyVersion > downloadedreleases = new TreeSet < yacyVersion > ( ) ;
2008-04-22 18:53:53 +02:00
for ( int j = 0 ; j < downloaded . length ; j + + ) {
try {
release = new yacyVersion ( downloaded [ j ] ) ;
downloadedreleases . add ( release ) ;
2008-08-02 14:12:04 +02:00
} catch ( final RuntimeException e ) {
2008-04-22 18:53:53 +02:00
// not a valid release
}
}
2008-04-22 23:36:52 +02:00
// if we have some files
if ( downloadedreleases . size ( ) > 0 ) {
serverLog . logFine ( " STARTUP " , " deleting downloaded releases older than " + deleteAfterDays + " days " ) ;
2008-04-22 18:53:53 +02:00
2008-04-22 23:36:52 +02:00
// keep latest version
final yacyVersion latest = downloadedreleases . last ( ) ;
downloadedreleases . remove ( latest ) ;
// if latest is a developer release, we also keep a main release
2008-08-02 14:12:04 +02:00
final boolean keepMain = ! latest . mainRelease ;
2008-04-22 18:53:53 +02:00
2008-04-22 23:36:52 +02:00
// remove old files
2008-08-02 14:12:04 +02:00
final long now = System . currentTimeMillis ( ) ;
2008-08-06 21:43:12 +02:00
final long deleteAfterMillis = deleteAfterDays * 24L * 60 * 60000 ;
2008-04-26 03:08:59 +02:00
String lastMain = null ;
String filename ;
for ( final yacyVersion aRelease : downloadedreleases ) {
filename = aRelease . name ;
if ( keepMain & & aRelease . mainRelease ) {
// keep this one, delete last remembered main release file
if ( lastMain ! = null ) {
filename = lastMain ;
}
lastMain = aRelease . name ;
2008-04-22 23:36:52 +02:00
}
2008-04-26 03:08:59 +02:00
2008-04-22 23:36:52 +02:00
// check file age
2008-08-02 14:12:04 +02:00
final File downloadedFile = new File ( filesPath + File . separator + filename ) ;
2008-04-26 03:08:59 +02:00
if ( now - downloadedFile . lastModified ( ) > deleteAfterMillis ) {
2008-04-22 23:36:52 +02:00
// delete file
2008-04-26 03:08:59 +02:00
if ( ! downloadedFile . delete ( ) ) {
serverLog . logWarning ( " STARTUP " , " cannot delete old release " + downloadedFile . getAbsolutePath ( ) ) ;
2008-04-22 23:36:52 +02:00
}
2008-04-22 18:53:53 +02:00
}
}
}
}
2007-07-12 18:23:33 +02:00
2005-09-22 12:30:55 +02:00
}