2015-10-26 23:46:48 +01:00
#!/usr/bin/env python
2013-02-10 03:55:04 +01:00
# Script by Ben Limmer
# https://github.com/l1m5
#
2015-11-21 04:10:49 +01:00
# This Python script will combine all the host files you provide
2013-02-10 03:55:04 +01:00
# as sources into one, unique host file to keep you internet browsing happy.
2016-02-08 02:09:42 +01:00
# pylint: disable=invalid-name
# pylint: disable=bad-whitespace
2015-10-26 23:46:48 +01:00
# Making Python 2 compatible with Python 3
from __future__ import absolute_import , division , print_function , unicode_literals
2013-02-10 03:55:04 +01:00
import os
2013-07-13 23:57:11 +02:00
import platform
2013-02-17 21:51:49 +01:00
import re
2016-02-15 00:15:22 +01:00
import shutil
2013-02-10 03:55:04 +01:00
import string
2013-07-13 23:57:11 +02:00
import subprocess
2013-02-10 03:55:04 +01:00
import sys
import tempfile
2016-02-15 00:15:22 +01:00
import time
2015-11-05 13:38:08 +01:00
import glob
2016-02-11 04:35:09 +01:00
import argparse
2016-03-18 21:10:36 +01:00
import socket
2016-03-23 05:36:01 +01:00
import json
2016-03-24 04:36:47 +01:00
2015-10-26 23:16:55 +01:00
# zip files are not used actually, support deleted
# StringIO is not needed in Python 3
# Python 3 works differently with urlopen
2015-10-26 23:46:48 +01:00
2016-05-02 10:59:54 +02:00
try : # Python 3
2016-02-08 01:15:05 +01:00
from urllib . parse import urlparse , urlencode
from urllib . request import urlopen , Request
from urllib . error import HTTPError
2016-05-02 10:59:54 +02:00
except ImportError : # Python 2
2016-02-08 01:15:05 +01:00
from urlparse import urlparse
from urllib import urlencode
from urllib2 import urlopen , Request , HTTPError
2015-10-26 23:46:48 +01:00
2016-05-02 10:59:54 +02:00
try : # Python 2
raw_input
except NameError : # Python 3
raw_input = input
2016-03-24 04:36:47 +01:00
# Detecting Python 3 for version-dependent implementations
Python3 = sys . version_info > = ( 3 , 0 )
2015-10-26 23:46:48 +01:00
# This function handles both Python 2 and Python 3
def getFileByUrl ( url ) :
2016-02-08 01:15:05 +01:00
try :
f = urlopen ( url )
2016-02-08 01:18:16 +01:00
return f . read ( ) . decode ( " UTF-8 " )
2016-02-08 01:15:05 +01:00
except :
2016-02-08 01:28:31 +01:00
print ( " Problem getting file: " , url )
2016-02-08 01:15:05 +01:00
# raise
2015-10-26 23:46:48 +01:00
2015-10-26 23:16:55 +01:00
# In Python 3 "print" is a function, braces are added everywhere
2013-02-10 03:55:04 +01:00
2015-10-26 23:46:48 +01:00
# Cross-python writing function
2016-02-08 01:18:16 +01:00
def writeData ( f , data ) :
2016-02-08 01:15:05 +01:00
if Python3 :
2016-03-28 19:27:17 +02:00
f . write ( bytes ( data , " UTF-8 " ) )
2016-02-08 01:15:05 +01:00
else :
2016-03-28 19:27:17 +02:00
f . write ( str ( data ) . encode ( " UTF-8 " ) )
2015-10-29 00:33:16 +01:00
2015-11-05 13:38:08 +01:00
# This function doesn't list hidden files
2016-02-08 01:18:16 +01:00
def listdir_nohidden ( path ) :
2016-03-28 19:27:17 +02:00
return glob . glob ( os . path . join ( path , " * " ) )
2015-10-26 23:46:48 +01:00
2013-02-17 21:51:49 +01:00
# Project Settings
2016-03-26 04:44:54 +01:00
BASEDIR_PATH = os . path . dirname ( os . path . realpath ( __file__ ) )
2016-03-24 06:10:13 +01:00
defaults = {
" numberofrules " : 0 ,
" datapath " : os . path . join ( BASEDIR_PATH , " data " ) ,
" freshen " : True ,
" replace " : False ,
2016-04-04 03:29:47 +02:00
" backup " : False ,
2016-04-19 21:42:59 +02:00
" skipstatichosts " : False ,
2016-03-24 06:10:13 +01:00
" extensionspath " : os . path . join ( BASEDIR_PATH , " extensions " ) ,
" extensions " : [ ] ,
" outputsubfolder " : " " ,
" datafilenames " : " hosts " ,
" targetip " : " 0.0.0.0 " ,
" updateurlfilename " : " update.info " ,
" readmefilename " : " readme.md " ,
" readmetemplate " : os . path . join ( BASEDIR_PATH , " readme_template.md " ) ,
" readmedata " : { } ,
2016-03-31 14:35:42 +02:00
" readmedatafilename " : os . path . join ( BASEDIR_PATH , " readmeData.json " ) ,
2016-03-24 06:10:13 +01:00
" exclusionpattern " : " ([a-zA-Z \ d-]+ \ .) { 0,} " ,
" exclusionregexs " : [ ] ,
" exclusions " : [ ] ,
" commonexclusions " : [ " hulu.com " ] ,
" whitelistfile " : os . path . join ( BASEDIR_PATH , " whitelist " ) }
2013-02-10 03:55:04 +01:00
def main ( ) :
2016-02-15 23:06:38 +01:00
2016-02-28 02:16:41 +01:00
parser = argparse . ArgumentParser ( description = " Creates a unified hosts file from hosts stored in data subfolders. " )
2016-03-28 19:27:17 +02:00
parser . add_argument ( " --auto " , " -a " , dest = " auto " , default = False , action = " store_true " , help = " Run without prompting. " )
2016-04-04 03:30:46 +02:00
parser . add_argument ( " --backup " , " -b " , dest = " backup " , default = False , action = " store_true " , help = " Backup the hosts files before they are overridden. " )
2016-03-28 19:27:17 +02:00
parser . add_argument ( " --extensions " , " -e " , dest = " extensions " , default = [ ] , nargs = " * " , help = " Host extensions to include in the final hosts file. " )
2016-04-04 03:30:46 +02:00
parser . add_argument ( " --ip " , " -i " , dest = " targetip " , default = " 0.0.0.0 " , help = " Target IP address. Default is 0.0.0.0. " )
2016-03-28 19:27:17 +02:00
parser . add_argument ( " --noupdate " , " -n " , dest = " noupdate " , default = False , action = " store_true " , help = " Don ' t update from host data sources. " )
2016-04-19 21:42:59 +02:00
parser . add_argument ( " --skipstatichosts " , " -s " , dest = " skipstatichosts " , default = False , action = " store_true " , help = " Skip static localhost entries in the final hosts file. " )
2016-04-04 03:30:46 +02:00
parser . add_argument ( " --output " , " -o " , dest = " outputsubfolder " , default = " " , help = " Output subfolder for generated hosts file. " )
parser . add_argument ( " --replace " , " -r " , dest = " replace " , default = False , action = " store_true " , help = " Replace your active hosts file with this new hosts file. " )
2016-06-12 14:04:38 +02:00
parser . add_argument ( " --flush-dns-cache " , " -f " , dest = " flushdnscache " , default = False , action = " store_true " , help = " Attempt to flush DNS cache after replacing the hosts file. " )
2016-03-24 06:10:13 +01:00
2016-03-26 04:45:18 +01:00
global settings
2016-03-24 06:10:13 +01:00
options = vars ( parser . parse_args ( ) )
2016-03-13 05:44:49 +01:00
2016-03-24 06:10:13 +01:00
options [ " outputpath " ] = os . path . join ( BASEDIR_PATH , options [ " outputsubfolder " ] )
options [ " freshen " ] = not options [ " noupdate " ]
settings = { }
settings . update ( defaults )
settings . update ( options )
settings [ " sources " ] = listdir_nohidden ( settings [ " datapath " ] )
2016-03-29 05:04:34 +02:00
settings [ " extensionsources " ] = listdir_nohidden ( settings [ " extensionspath " ] )
2016-02-11 04:35:09 +01:00
2016-02-22 16:22:26 +01:00
# All our extensions folders...
2016-03-24 06:10:13 +01:00
settings [ " extensions " ] = [ os . path . basename ( item ) for item in listdir_nohidden ( settings [ " extensionspath " ] ) ]
2016-03-13 07:47:56 +01:00
# ... intersected with the extensions passed-in as arguments, then sorted.
2016-03-24 06:10:13 +01:00
settings [ " extensions " ] = sorted ( list ( set ( options [ " extensions " ] ) . intersection ( settings [ " extensions " ] ) ) )
2016-02-22 16:22:26 +01:00
2016-03-28 19:27:17 +02:00
with open ( settings [ " readmedatafilename " ] , " r " ) as f :
2016-03-24 06:10:13 +01:00
settings [ " readmedata " ] = json . load ( f )
2016-03-23 05:36:01 +01:00
2016-02-08 01:15:05 +01:00
promptForUpdate ( )
promptForExclusions ( )
mergeFile = createInitialFile ( )
removeOldHostsFile ( )
2016-02-08 01:18:16 +01:00
finalFile = removeDupsAndExcl ( mergeFile )
finalizeFile ( finalFile )
2016-03-24 06:10:13 +01:00
updateReadmeData ( )
2016-03-28 19:27:17 +02:00
printSuccess ( " Success! The hosts file has been saved in folder " + settings [ " outputsubfolder " ] + " \n It contains " +
" {:,} " . format ( settings [ " numberofrules " ] ) + " unique entries. " )
2015-10-18 02:23:33 +02:00
2016-02-08 01:18:16 +01:00
promptForMove ( finalFile )
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
# Prompt the User
2013-02-10 03:55:04 +01:00
def promptForUpdate ( ) :
2016-02-08 01:15:05 +01:00
# Create hosts file if it doesn't exists
2016-03-28 19:27:17 +02:00
if not os . path . isfile ( os . path . join ( BASEDIR_PATH , " hosts " ) ) :
2016-02-08 01:15:05 +01:00
try :
2016-03-28 19:27:17 +02:00
open ( os . path . join ( BASEDIR_PATH , " hosts " ) , " w+ " ) . close ( )
2016-02-08 01:15:05 +01:00
except :
2016-02-08 01:18:16 +01:00
printFailure ( " ERROR: No ' hosts ' file in the folder, try creating one manually " )
2016-02-08 01:15:05 +01:00
2016-03-24 06:10:13 +01:00
if not settings [ " freshen " ] :
2016-03-18 21:27:09 +01:00
return
2016-03-24 06:10:13 +01:00
response = " yes " if settings [ " auto " ] else query_yes_no ( " Do you want to update all data sources? " )
2016-03-18 21:27:09 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
updateAllSources ( )
else :
2016-03-24 06:10:13 +01:00
if not settings [ " auto " ] :
2016-03-28 19:27:17 +02:00
print ( " OK, we ' ll stick with what we ' ve got locally. " )
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
def promptForExclusions ( ) :
2016-03-24 06:10:13 +01:00
response = " no " if settings [ " auto " ] else query_yes_no ( " Do you want to exclude any domains? \n " +
2016-02-08 01:15:05 +01:00
" For example, hulu.com video streaming must be able to access " +
2016-02-08 01:18:16 +01:00
" its tracking and ad servers in order to play video. " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
displayExclusionOptions ( )
else :
2016-03-24 06:10:13 +01:00
if not settings [ " auto " ] :
2016-03-28 19:27:17 +02:00
print ( " OK, we ' ll only exclude domains in the whitelist. " )
2013-02-17 22:39:40 +01:00
2016-04-26 15:46:14 +02:00
def promptForMoreCustomExclusions ( question = " Do you have more domains you want to enter? " ) :
return query_yes_no ( question ) == " yes "
2013-07-30 18:59:31 +02:00
2016-06-12 14:04:38 +02:00
def promptForFlushDnsCache ( ) :
2016-06-13 02:13:36 +02:00
if settings [ ' auto ' ] :
if settings [ ' flushdnscache ' ] :
flushDnsCache ( )
else :
if settings [ ' flushdnscache ' ] or query_yes_no ( " Attempt to flush the DNS cache? " ) :
flushDnsCache ( )
2016-06-12 14:04:38 +02:00
2016-02-08 01:18:16 +01:00
def promptForMove ( finalFile ) :
2016-02-23 03:26:52 +01:00
2016-04-19 21:42:59 +02:00
if settings [ " replace " ] and not settings [ " skipstatichosts " ] :
2016-02-23 03:26:52 +01:00
response = " yes "
else :
2016-04-19 21:42:59 +02:00
response = " no " if settings [ " auto " ] or settings [ " skipstatichosts " ] else query_yes_no ( " Do you want to replace your existing hosts file " +
2016-02-08 01:25:32 +01:00
" with the newly generated file? " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:18:16 +01:00
moveHostsFileIntoPlace ( finalFile )
2016-06-12 14:04:38 +02:00
promptForFlushDnsCache ( )
2016-02-08 01:15:05 +01:00
else :
return False
2013-02-17 21:51:49 +01:00
# End Prompt the User
# Exclusion logic
def displayExclusionOptions ( ) :
2016-04-04 03:19:41 +02:00
for exclusionOption in settings [ " commonexclusions " ] :
2016-02-08 01:18:16 +01:00
response = query_yes_no ( " Do you want to exclude the domain " + exclusionOption + " ? " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
excludeDomain ( exclusionOption )
else :
continue
2016-02-08 01:18:16 +01:00
response = query_yes_no ( " Do you want to exclude any other domains? " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
gatherCustomExclusions ( )
2015-10-29 00:33:16 +01:00
2013-02-17 21:51:49 +01:00
def gatherCustomExclusions ( ) :
2016-02-08 01:15:05 +01:00
while True :
# Cross-python Input
2016-05-02 10:59:54 +02:00
domainFromUser = raw_input ( " Enter the domain you want to exclude (e.g. facebook.com): " )
2016-02-08 01:23:40 +01:00
if isValidDomainFormat ( domainFromUser ) :
2016-02-08 01:18:16 +01:00
excludeDomain ( domainFromUser )
2016-04-26 15:46:14 +02:00
if not promptForMoreCustomExclusions ( ) :
2016-02-08 01:15:05 +01:00
return
2013-02-17 21:51:49 +01:00
2016-02-08 01:18:16 +01:00
def excludeDomain ( domain ) :
2016-03-24 06:10:13 +01:00
settings [ " exclusionregexs " ] . append ( re . compile ( settings [ " exclusionpattern " ] + domain ) )
2013-02-17 21:51:49 +01:00
2016-02-08 01:18:16 +01:00
def matchesExclusions ( strippedRule ) :
2016-02-08 01:15:05 +01:00
strippedDomain = strippedRule . split ( ) [ 1 ]
2016-03-24 06:10:13 +01:00
for exclusionRegex in settings [ " exclusionregexs " ] :
2016-02-08 01:18:16 +01:00
if exclusionRegex . search ( strippedDomain ) :
2016-02-08 01:15:05 +01:00
return True
return False
2013-02-17 21:51:49 +01:00
# End Exclusion Logic
# Update Logic
2013-02-10 03:55:04 +01:00
def updateAllSources ( ) :
2016-03-29 05:04:34 +02:00
allsources = list ( set ( settings [ " sources " ] ) | set ( settings [ " extensionsources " ] ) )
2016-02-23 01:20:20 +01:00
for source in allsources :
2016-02-23 15:50:20 +01:00
if os . path . isdir ( source ) :
2016-04-26 15:46:14 +02:00
for updateURL in getUpdateURLsFromFile ( source ) :
2016-03-06 02:28:32 +01:00
print ( " Updating source " + os . path . basename ( source ) + " from " + updateURL )
# Cross-python call
updatedFile = getFileByUrl ( updateURL )
try :
2016-03-28 19:27:17 +02:00
updatedFile = updatedFile . replace ( " \r " , " " ) #get rid of carriage-return symbols
2016-03-06 02:28:32 +01:00
# This is cross-python code
2016-03-28 19:27:17 +02:00
dataFile = open ( os . path . join ( settings [ " datapath " ] , source , settings [ " datafilenames " ] ) , " wb " )
2016-03-06 02:28:32 +01:00
writeData ( dataFile , updatedFile )
dataFile . close ( )
except :
print ( " Skipping. " )
def getUpdateURLsFromFile ( source ) :
2016-03-24 06:10:13 +01:00
pathToUpdateFile = os . path . join ( settings [ " datapath " ] , source , settings [ " updateurlfilename " ] )
2016-03-06 02:28:32 +01:00
if os . path . exists ( pathToUpdateFile ) :
2016-03-28 19:27:17 +02:00
updateFile = open ( pathToUpdateFile , " r " )
2016-03-06 02:28:32 +01:00
retURLs = updateFile . readlines ( )
# .strip()
updateFile . close ( )
else :
retURL = None
2016-03-28 19:27:17 +02:00
printFailure ( " Warning: Can ' t find the update file for source " + source + " \n " +
" Make sure that there ' s a file at " + pathToUpdateFile )
2016-03-06 02:28:32 +01:00
return retURLs
# End Update Logic
2016-01-03 00:57:23 +01:00
2016-02-08 01:18:16 +01:00
def getUpdateURLFromFile ( source ) :
2016-03-24 06:10:13 +01:00
pathToUpdateFile = os . path . join ( settings [ " datapath " ] , source , settings [ " updateurlfilename " ] )
2016-02-08 01:18:16 +01:00
if os . path . exists ( pathToUpdateFile ) :
2016-04-26 15:46:14 +02:00
with open ( pathToUpdateFile , " r " ) as updateFile :
return updateFile . readline ( ) . strip ( )
printFailure ( " Warning: Can ' t find the update file for source " + source + " \n " +
" Make sure that there ' s a file at " + pathToUpdateFile )
return None
2013-02-17 21:51:49 +01:00
# End Update Logic
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
# File Logic
2013-02-10 03:55:04 +01:00
def createInitialFile ( ) :
2016-02-08 01:15:05 +01:00
mergeFile = tempfile . NamedTemporaryFile ( )
2016-03-24 06:10:13 +01:00
for source in settings [ " sources " ] :
2016-04-26 15:46:14 +02:00
filename = os . path . join ( settings [ " datapath " ] , source , settings [ " datafilenames " ] )
2016-04-26 15:49:31 +02:00
with open ( filename , " r " ) as curFile :
2016-04-26 15:46:14 +02:00
#Done in a cross-python way
writeData ( mergeFile , curFile . read ( ) )
2015-10-29 00:33:16 +01:00
2016-03-24 06:10:13 +01:00
for source in settings [ " extensions " ] :
2016-04-26 15:46:14 +02:00
filename = os . path . join ( settings [ " extensionspath " ] , source , settings [ " datafilenames " ] )
with open ( filename , " r " ) as curFile :
#Done in a cross-python way
writeData ( mergeFile , curFile . read ( ) )
2016-02-23 01:20:20 +01:00
2016-02-08 01:15:05 +01:00
return mergeFile
2013-02-10 03:55:04 +01:00
2016-02-08 01:18:16 +01:00
def removeDupsAndExcl ( mergeFile ) :
2016-03-24 06:10:13 +01:00
numberOfRules = settings [ " numberofrules " ]
if os . path . isfile ( settings [ " whitelistfile " ] ) :
with open ( settings [ " whitelistfile " ] , " r " ) as ins :
2016-02-08 01:15:05 +01:00
for line in ins :
2016-04-02 05:49:22 +02:00
line = line . strip ( " \t \n \r " )
if line and not line . startswith ( " # " ) :
2016-03-24 06:10:13 +01:00
settings [ " exclusions " ] . append ( line )
2013-07-13 22:59:29 +02:00
2016-03-24 06:10:13 +01:00
if not os . path . exists ( settings [ " outputpath " ] ) :
os . makedirs ( settings [ " outputpath " ] )
2016-03-13 05:17:37 +01:00
2015-10-29 00:33:16 +01:00
# Another mode is required to read and write the file in Python 3
2016-04-26 15:46:14 +02:00
finalFile = open ( os . path . join ( settings [ " outputpath " ] , " hosts " ) ,
" w+b " if Python3 else " w+ " )
2013-02-10 03:55:04 +01:00
2016-03-13 16:59:25 +01:00
mergeFile . seek ( 0 ) # reset file pointer
2016-05-02 10:59:54 +02:00
hostnames = set ( [ " localhost " , " localhost.localdomain " , " local " , " broadcasthost " ] )
2016-03-24 06:10:13 +01:00
exclusions = settings [ " exclusions " ]
2016-02-08 01:15:05 +01:00
for line in mergeFile . readlines ( ) :
2016-03-28 19:27:17 +02:00
write = " true "
2015-10-26 23:16:55 +01:00
# Explicit encoding
2016-02-08 01:18:16 +01:00
line = line . decode ( " UTF-8 " )
2016-02-15 18:36:38 +01:00
# replace tabs with space
2016-03-28 19:27:17 +02:00
line = line . replace ( " \t + " , " " )
2016-03-24 23:32:46 +01:00
# Trim trailing whitespace
2016-03-25 04:34:28 +01:00
line = line . rstrip ( ) + " \n "
2016-02-08 01:15:05 +01:00
# Testing the first character doesn't require startswith
2016-03-28 19:27:17 +02:00
if line [ 0 ] == " # " or re . match ( r ' ^ \ s*$ ' , line [ 0 ] ) :
2016-02-08 01:15:05 +01:00
# Cross-python write
2016-02-08 01:18:16 +01:00
writeData ( finalFile , line )
2016-02-08 01:15:05 +01:00
continue
2016-03-28 19:27:17 +02:00
if " ::1 " in line :
2016-02-08 01:15:05 +01:00
continue
2016-02-08 01:18:16 +01:00
strippedRule = stripRule ( line ) #strip comments
2016-04-26 15:46:14 +02:00
if not strippedRule or matchesExclusions ( strippedRule ) :
2016-02-08 01:15:05 +01:00
continue
2016-02-08 01:18:16 +01:00
hostname , normalizedRule = normalizeRule ( strippedRule ) # normalize rule
2016-03-24 06:10:13 +01:00
for exclude in exclusions :
2016-02-08 01:32:00 +01:00
if exclude in line :
2016-03-28 19:27:17 +02:00
write = " false "
2016-02-08 01:15:05 +01:00
break
2016-03-28 19:27:17 +02:00
if normalizedRule and ( hostname not in hostnames ) and ( write == " true " ) :
2016-02-08 01:18:16 +01:00
writeData ( finalFile , normalizedRule )
hostnames . add ( hostname )
2016-02-08 01:15:05 +01:00
numberOfRules + = 1
2016-03-24 06:10:13 +01:00
settings [ " numberofrules " ] = numberOfRules
2016-02-08 01:15:05 +01:00
mergeFile . close ( )
return finalFile
2013-02-10 03:55:04 +01:00
2014-05-16 14:13:11 +02:00
def normalizeRule ( rule ) :
2016-02-08 01:18:16 +01:00
result = re . search ( r ' ^[ \ t]*( \ d+ \ . \ d+ \ . \ d+ \ . \ d+) \ s+([ \ w \ .-]+)(.*) ' , rule )
2016-02-08 01:15:05 +01:00
if result :
2016-02-08 02:25:33 +01:00
hostname , suffix = result . group ( 2 , 3 )
2016-02-15 18:29:02 +01:00
hostname = hostname . lower ( ) . strip ( ) # explicitly lowercase and trim the hostname
2016-04-26 15:46:14 +02:00
if suffix :
2016-02-08 01:15:05 +01:00
# add suffix as comment only, not as a separate host
2016-03-24 06:10:13 +01:00
return hostname , " %s %s # %s \n " % ( settings [ " targetip " ] , hostname , suffix )
2016-02-08 01:15:05 +01:00
else :
2016-03-24 06:10:13 +01:00
return hostname , " %s %s \n " % ( settings [ " targetip " ] , hostname )
2016-02-08 01:37:27 +01:00
print ( " ==> %s <== " % rule )
2016-02-08 01:15:05 +01:00
return None , None
2014-05-16 14:13:11 +02:00
2016-02-08 01:18:16 +01:00
def finalizeFile ( finalFile ) :
writeOpeningHeader ( finalFile )
2016-02-08 01:15:05 +01:00
finalFile . close ( )
2013-02-10 03:55:04 +01:00
# Some sources put comments around their rules, for accuracy we need to strip them
# the comments are preserved in the output hosts file
2016-02-08 01:18:16 +01:00
def stripRule ( line ) :
2016-02-08 01:15:05 +01:00
splitLine = line . split ( )
2016-02-08 01:32:00 +01:00
if len ( splitLine ) < 2 :
2016-02-08 01:15:05 +01:00
# just return blank
2016-03-28 19:27:17 +02:00
return " "
2016-02-08 01:15:05 +01:00
else :
2016-03-28 19:27:17 +02:00
return splitLine [ 0 ] + " " + splitLine [ 1 ]
2013-02-10 03:55:04 +01:00
def writeOpeningHeader ( finalFile ) :
2016-02-08 01:18:16 +01:00
finalFile . seek ( 0 ) #reset file pointer
2016-02-08 01:28:31 +01:00
fileContents = finalFile . read ( ) #save content
2016-02-08 01:18:16 +01:00
finalFile . seek ( 0 ) #write at the top
2016-03-28 19:27:17 +02:00
writeData ( finalFile , " # This hosts file is a merged collection of hosts from reputable sources, \n " )
writeData ( finalFile , " # with a dash of crowd sourcing via Github \n # \n " )
2016-04-26 17:52:00 +02:00
writeData ( finalFile , " # Date: " + time . strftime ( " % B %d % Y " , time . gmtime ( ) ) + " \n " )
2016-03-24 15:29:51 +01:00
if settings [ " extensions " ] :
2016-03-28 19:27:17 +02:00
writeData ( finalFile , " # Extensions added to this file: " + " , " . join ( settings [ " extensions " ] ) + " \n " )
2016-04-26 16:11:51 +02:00
writeData ( finalFile , " # Number of unique domains: " + " {:,} \n # \n " . format ( settings [ " numberofrules " ] ) )
2016-03-28 19:27:17 +02:00
writeData ( finalFile , " # Fetch the latest version of this file: https://raw.githubusercontent.com/StevenBlack/hosts/master/ " + os . path . join ( settings [ " outputsubfolder " ] , " " ) + " hosts \n " )
writeData ( finalFile , " # Project home page: https://github.com/StevenBlack/hosts \n # \n " )
writeData ( finalFile , " # =============================================================== \n " )
writeData ( finalFile , " \n " )
2016-04-19 21:42:59 +02:00
if not settings [ " skipstatichosts " ] :
writeData ( finalFile , " 127.0.0.1 localhost \n " )
writeData ( finalFile , " 127.0.0.1 localhost.localdomain \n " )
writeData ( finalFile , " 127.0.0.1 local \n " )
writeData ( finalFile , " 255.255.255.255 broadcasthost \n " )
writeData ( finalFile , " ::1 localhost \n " )
writeData ( finalFile , " fe80::1 %lo 0 localhost \n " )
if platform . system ( ) == " Linux " :
writeData ( finalFile , " 127.0.1.1 " + socket . gethostname ( ) + " \n " )
writeData ( finalFile , " \n " )
2016-02-08 01:18:16 +01:00
2016-02-08 01:28:31 +01:00
preamble = os . path . join ( BASEDIR_PATH , " myhosts " )
2016-02-08 01:18:16 +01:00
if os . path . isfile ( preamble ) :
with open ( preamble , " r " ) as f :
2016-02-08 01:28:31 +01:00
writeData ( finalFile , f . read ( ) )
2016-02-08 01:18:16 +01:00
finalFile . write ( fileContents )
2016-03-24 06:10:13 +01:00
def updateReadmeData ( ) :
2016-03-23 05:36:01 +01:00
extensionsKey = " base "
hostsLocation = " "
2016-03-24 06:10:13 +01:00
if settings [ " extensions " ] :
extensionsKey = " - " . join ( settings [ " extensions " ] )
2016-03-23 05:36:01 +01:00
2016-04-26 16:03:36 +02:00
generationData = { " location " : os . path . join ( settings [ " outputsubfolder " ] , " " ) ,
2016-04-26 15:46:14 +02:00
" entries " : settings [ " numberofrules " ] }
2016-03-24 06:10:13 +01:00
settings [ " readmedata " ] [ extensionsKey ] = generationData
2016-03-28 19:27:17 +02:00
with open ( settings [ " readmedatafilename " ] , " w " ) as f :
2016-03-24 06:10:13 +01:00
json . dump ( settings [ " readmedata " ] , f )
2016-03-23 05:36:01 +01:00
2016-06-12 14:04:38 +02:00
2016-02-08 01:18:16 +01:00
def moveHostsFileIntoPlace ( finalFile ) :
2016-03-28 19:27:17 +02:00
if os . name == " posix " :
2016-02-08 01:37:27 +01:00
print ( " Moving the file requires administrative privileges. " +
" You might need to enter your password. " )
2016-02-08 01:32:00 +01:00
if subprocess . call ( [ " /usr/bin/sudo " , " cp " , os . path . abspath ( finalFile . name ) , " /etc/hosts " ] ) :
2016-02-08 01:18:16 +01:00
printFailure ( " Moving the file failed. " )
2016-03-28 19:27:17 +02:00
elif os . name == " nt " :
2016-06-12 14:04:38 +02:00
print ( " Automatically moving the hosts file in place is not yet supported. " )
print ( " Please move the generated file to % SystemRoot % \ system32 \ drivers \ etc \ hosts " )
def flushDnsCache ( ) :
print ( " Flushing the DNS cache to utilize new hosts file... " )
print ( " Flushing the DNS cache requires administrative privileges. " +
" You might need to enter your password. " )
dnsCacheFound = False
if platform . system ( ) == " Darwin " :
if subprocess . call ( [ " /usr/bin/sudo " , " killall " , " -HUP " , " mDNSResponder " ] ) :
printFailure ( " Flushing the DNS cache failed. " )
else :
if os . path . isfile ( " /etc/rc.d/init.d/nscd " ) :
dnsCacheFound = True
if subprocess . call ( [ " /usr/bin/sudo " , " /etc/rc.d/init.d/nscd " , " restart " ] ) :
printFailure ( " Flushing the DNS cache failed. " )
else :
printSuccess ( " Flushing DNS by restarting nscd succeeded " )
if os . path . isfile ( " /usr/lib/systemd/system/NetworkManager.service " ) :
dnsCacheFound = True
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " NetworkManager.service " ] ) :
printFailure ( " Flushing the DNS cache failed. " )
else :
printSuccess ( " Flushing DNS by restarting NetworkManager succeeded " )
if os . path . isfile ( " /usr/lib/systemd/system/wicd.service " ) :
dnsCacheFound = True
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " wicd.service " ] ) :
printFailure ( " Flushing the DNS cache failed. " )
else :
printSuccess ( " Flushing DNS by restarting wicd succeeded " )
if os . path . isfile ( " /usr/lib/systemd/system/dnsmasq.service " ) :
dnsCacheFound = True
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " dnsmasq.service " ] ) :
printFailure ( " Flushing the DNS cache failed. " )
else :
printSuccess ( " Flushing DNS by restarting dnsmasq succeeded " )
if os . path . isfile ( " /usr/lib/systemd/system/networking.service " ) :
dnsCacheFound = True
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " networking.service " ] ) :
printFailure ( " Flushing the DNS cache failed. " )
else :
printSuccess ( " Flushing DNS by restarting networking.service succeeded " )
if not dnsCacheFound :
printFailure ( " Unable to determine DNS management tool. " )
2016-02-08 01:15:05 +01:00
def removeOldHostsFile ( ) : # hotfix since merging with an already existing hosts file leads to artefacts and duplicates
2016-03-28 19:27:17 +02:00
oldFilePath = os . path . join ( BASEDIR_PATH , " hosts " )
open ( oldFilePath , " a " ) . close ( ) # create if already removed, so remove wont raise an error
2016-04-04 03:29:47 +02:00
if settings [ " backup " ] :
2016-04-26 16:11:51 +02:00
backupFilePath = os . path . join ( BASEDIR_PATH , " hosts- {} " . format ( time . strftime ( " % Y- % m- %d - % H- % M- % S " ) ) )
2016-04-04 03:29:47 +02:00
shutil . copy ( oldFilePath , backupFilePath ) # make a backup copy, marking the date in which the list was updated
2016-02-08 01:28:31 +01:00
os . remove ( oldFilePath )
2016-03-28 19:27:17 +02:00
open ( oldFilePath , " a " ) . close ( ) # create new empty hostsfile
2013-07-14 00:08:31 +02:00
2013-02-17 21:51:49 +01:00
# End File Logic
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
# Helper Functions
2013-02-10 03:55:04 +01:00
## {{{ http://code.activestate.com/recipes/577058/ (r2)
2016-02-08 01:18:16 +01:00
def query_yes_no ( question , default = " yes " ) :
2013-02-10 03:55:04 +01:00
""" Ask a yes/no question via raw_input() and return their answer.
2015-01-02 03:35:11 +01:00
2013-02-10 03:55:04 +01:00
" question " is a string that is presented to the user .
" default " is the presumed answer if the user just hits < Enter > .
It must be " yes " ( the default ) , " no " or None ( meaning
an answer is required of the user ) .
The " answer " return value is one of " yes " or " no " .
"""
2016-02-08 01:15:05 +01:00
valid = { " yes " : " yes " , " y " : " yes " , " ye " : " yes " ,
" no " : " no " , " n " : " no " }
2016-05-02 10:59:54 +02:00
prompt = { None : " [y/n] " ,
" yes " : " [Y/n] " ,
" no " : " [y/N] " } . get ( default , None )
if not prompt :
2016-02-08 01:18:16 +01:00
raise ValueError ( " invalid default answer: ' %s ' " % default )
2013-02-10 03:55:04 +01:00
while 1 :
2016-02-08 01:18:16 +01:00
sys . stdout . write ( colorize ( question , colors . PROMPT ) + prompt )
2015-10-26 23:46:48 +01:00
# Changed to be cross-python
2016-05-02 10:59:54 +02:00
choice = raw_input ( ) . lower ( )
if default and not choice :
2013-02-10 03:55:04 +01:00
return default
2016-05-02 10:59:54 +02:00
elif choice in valid :
2013-02-10 03:55:04 +01:00
return valid [ choice ]
else :
2016-05-02 10:59:54 +02:00
printFailure (
" Please respond with ' yes ' or ' no ' (or ' y ' or ' n ' ). \n " )
2013-02-10 03:55:04 +01:00
## end of http://code.activestate.com/recipes/577058/ }}}
2016-02-08 01:18:16 +01:00
def isValidDomainFormat ( domain ) :
2016-03-28 19:27:17 +02:00
if domain == " " :
print ( " You didn ' t enter a domain. Try again. " )
2016-02-08 01:15:05 +01:00
return False
2016-02-08 01:18:16 +01:00
domainRegex = re . compile ( " www \ d { 0,3}[.]|https? " )
2016-02-08 01:32:00 +01:00
if domainRegex . match ( domain ) :
2016-02-08 01:34:27 +01:00
print ( " The domain " + domain + " is not valid. " +
" Do not include www.domain.com or http(s)://domain.com. Try again. " )
2016-02-08 01:15:05 +01:00
return False
else :
return True
2013-02-17 22:49:16 +01:00
# Colors
class colors :
2016-03-28 19:27:17 +02:00
PROMPT = " \033 [94m "
SUCCESS = " \033 [92m "
FAIL = " \033 [91m "
ENDC = " \033 [0m "
2013-02-17 22:49:16 +01:00
2016-02-08 01:18:16 +01:00
def colorize ( text , color ) :
2016-02-08 01:15:05 +01:00
return color + text + colors . ENDC
2013-02-17 22:49:16 +01:00
2016-02-08 01:18:16 +01:00
def printSuccess ( text ) :
print ( colorize ( text , colors . SUCCESS ) )
2013-02-17 22:49:16 +01:00
2016-02-08 01:18:16 +01:00
def printFailure ( text ) :
print ( colorize ( text , colors . FAIL ) )
2013-02-17 21:51:49 +01:00
# End Helper Functions
2013-02-10 03:55:04 +01:00
if __name__ == " __main__ " :
2016-02-08 01:15:05 +01:00
main ( )