2015-10-26 23:46:48 +01:00
#!/usr/bin/env python
2013-02-10 03:55:04 +01:00
# Script by Ben Limmer
# https://github.com/l1m5
#
2015-11-21 04:10:49 +01:00
# This Python script will combine all the host files you provide
2013-02-10 03:55:04 +01:00
# as sources into one, unique host file to keep you internet browsing happy.
2016-02-08 02:09:42 +01:00
# pylint: disable=invalid-name
# pylint: disable=bad-whitespace
2015-10-26 23:46:48 +01:00
# Making Python 2 compatible with Python 3
from __future__ import absolute_import , division , print_function , unicode_literals
2013-02-10 03:55:04 +01:00
import os
2013-07-13 23:57:11 +02:00
import platform
2013-02-17 21:51:49 +01:00
import re
2016-02-15 00:15:22 +01:00
import shutil
2013-02-10 03:55:04 +01:00
import string
2013-07-13 23:57:11 +02:00
import subprocess
2013-02-10 03:55:04 +01:00
import sys
import tempfile
2016-02-15 00:15:22 +01:00
import time
2015-11-05 13:38:08 +01:00
import glob
2016-02-11 04:35:09 +01:00
import argparse
2016-03-18 21:10:36 +01:00
import socket
2016-03-23 05:36:01 +01:00
import json
2016-03-24 04:36:47 +01:00
2015-10-26 23:16:55 +01:00
# zip files are not used actually, support deleted
# StringIO is not needed in Python 3
# Python 3 works differently with urlopen
2015-10-26 23:46:48 +01:00
# Supporting urlopen in Python 2 and Python 3
try :
2016-02-08 01:15:05 +01:00
from urllib . parse import urlparse , urlencode
from urllib . request import urlopen , Request
from urllib . error import HTTPError
2015-10-26 23:46:48 +01:00
except ImportError :
2016-02-08 01:15:05 +01:00
from urlparse import urlparse
from urllib import urlencode
from urllib2 import urlopen , Request , HTTPError
2015-10-26 23:46:48 +01:00
2016-03-24 04:36:47 +01:00
# Detecting Python 3 for version-dependent implementations
Python3 = sys . version_info > = ( 3 , 0 )
2015-10-26 23:46:48 +01:00
# This function handles both Python 2 and Python 3
def getFileByUrl ( url ) :
2016-02-08 01:15:05 +01:00
try :
f = urlopen ( url )
2016-02-08 01:18:16 +01:00
return f . read ( ) . decode ( " UTF-8 " )
2016-02-08 01:15:05 +01:00
except :
2016-02-08 01:28:31 +01:00
print ( " Problem getting file: " , url )
2016-02-08 01:15:05 +01:00
# raise
2015-10-26 23:46:48 +01:00
2015-10-26 23:16:55 +01:00
# In Python 3 "print" is a function, braces are added everywhere
2013-02-10 03:55:04 +01:00
2015-10-26 23:46:48 +01:00
# This function works in both Python 2 and Python 3
2016-02-08 01:18:16 +01:00
def myInput ( msg = " " ) :
2016-02-08 01:15:05 +01:00
if Python3 :
2016-02-08 01:28:31 +01:00
return input ( msg )
2016-02-08 01:15:05 +01:00
else :
2016-02-08 01:28:31 +01:00
return raw_input ( msg )
2015-10-26 23:46:48 +01:00
# Cross-python writing function
2016-02-08 01:18:16 +01:00
def writeData ( f , data ) :
2016-02-08 01:15:05 +01:00
if Python3 :
2016-02-08 01:18:16 +01:00
f . write ( bytes ( data , ' UTF-8 ' ) )
2016-02-08 01:15:05 +01:00
else :
2016-02-08 01:18:16 +01:00
f . write ( str ( data ) . encode ( ' UTF-8 ' ) )
2015-10-29 00:33:16 +01:00
2015-11-05 13:38:08 +01:00
# This function doesn't list hidden files
2016-02-08 01:18:16 +01:00
def listdir_nohidden ( path ) :
return glob . glob ( os . path . join ( path , ' * ' ) )
2015-10-26 23:46:48 +01:00
2013-02-17 21:51:49 +01:00
# Project Settings
2016-03-23 05:36:01 +01:00
BASEDIR_PATH = os . path . dirname ( os . path . realpath ( __file__ ) )
2016-03-24 06:10:13 +01:00
defaults = {
" numberofrules " : 0 ,
" datapath " : os . path . join ( BASEDIR_PATH , " data " ) ,
" freshen " : True ,
" replace " : False ,
" extensionspath " : os . path . join ( BASEDIR_PATH , " extensions " ) ,
" extensions " : [ ] ,
" outputsubfolder " : " " ,
" datafilenames " : " hosts " ,
" targetip " : " 0.0.0.0 " ,
" updateurlfilename " : " update.info " ,
" readmefilename " : " readme.md " ,
" readmetemplate " : os . path . join ( BASEDIR_PATH , " readme_template.md " ) ,
" readmedata " : { } ,
" readmedatafilename " : " readmeData.json " ,
" exclusionpattern " : " ([a-zA-Z \ d-]+ \ .) { 0,} " ,
" exclusionregexs " : [ ] ,
" exclusions " : [ ] ,
" commonexclusions " : [ " hulu.com " ] ,
" whitelistfile " : os . path . join ( BASEDIR_PATH , " whitelist " ) }
options = { }
settings = { }
2016-02-11 04:35:09 +01:00
2013-02-10 03:55:04 +01:00
def main ( ) :
2016-02-15 23:06:38 +01:00
2016-02-28 02:16:41 +01:00
parser = argparse . ArgumentParser ( description = " Creates a unified hosts file from hosts stored in data subfolders. " )
2016-02-11 04:35:09 +01:00
parser . add_argument ( " --auto " , " -a " , dest = " auto " , default = False , action = ' store_true ' , help = " Run without prompting. " )
2016-02-23 03:26:52 +01:00
parser . add_argument ( " --replace " , " -r " , dest = " replace " , default = False , action = ' store_true ' , help = " Replace your active hosts file with this new hosts file. " )
2016-03-24 06:10:13 +01:00
parser . add_argument ( " --ip " , " -i " , dest = " targetip " , default = " 0.0.0.0 " , help = " Target IP address. Default is 0.0.0.0. " )
2016-02-22 16:22:26 +01:00
parser . add_argument ( " --extensions " , " -e " , dest = " extensions " , default = [ ] , nargs = ' * ' , help = " Host extensions to include in the final hosts file. " )
2016-03-24 06:10:13 +01:00
parser . add_argument ( " --output " , " -o " , dest = " outputsubfolder " , default = " " , help = " Output subfolder for generated hosts file. " )
parser . add_argument ( " --noupdate " , " -n " , dest = " noupdate " , default = False , action = ' store_true ' , help = " Don ' t update from host data sources. " )
global defaults , options , settings
options = vars ( parser . parse_args ( ) )
2016-03-13 05:44:49 +01:00
2016-03-24 06:10:13 +01:00
options [ " outputpath " ] = os . path . join ( BASEDIR_PATH , options [ " outputsubfolder " ] )
options [ " freshen " ] = not options [ " noupdate " ]
settings = { }
settings . update ( defaults )
settings . update ( options )
settings [ " sources " ] = listdir_nohidden ( settings [ " datapath " ] )
2016-02-11 04:35:09 +01:00
2016-02-22 16:22:26 +01:00
# All our extensions folders...
2016-03-24 06:10:13 +01:00
settings [ " extensions " ] = [ os . path . basename ( item ) for item in listdir_nohidden ( settings [ " extensionspath " ] ) ]
2016-03-13 07:47:56 +01:00
# ... intersected with the extensions passed-in as arguments, then sorted.
2016-03-24 06:10:13 +01:00
settings [ " extensions " ] = sorted ( list ( set ( options [ " extensions " ] ) . intersection ( settings [ " extensions " ] ) ) )
2016-02-22 16:22:26 +01:00
2016-03-24 06:10:13 +01:00
with open ( settings [ " readmedatafilename " ] , ' r ' ) as f :
settings [ " readmedata " ] = json . load ( f )
2016-03-23 05:36:01 +01:00
2016-02-08 01:15:05 +01:00
promptForUpdate ( )
promptForExclusions ( )
mergeFile = createInitialFile ( )
removeOldHostsFile ( )
2016-02-08 01:18:16 +01:00
finalFile = removeDupsAndExcl ( mergeFile )
finalizeFile ( finalFile )
2016-03-24 06:10:13 +01:00
updateReadmeData ( )
printSuccess ( ' Success! The hosts file has been saved in folder ' + settings [ " outputsubfolder " ] + ' \n It contains ' +
" {:,} " . format ( settings [ " numberofrules " ] ) + ' unique entries. ' )
2015-10-18 02:23:33 +02:00
2016-02-08 01:18:16 +01:00
promptForMove ( finalFile )
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
# Prompt the User
2013-02-10 03:55:04 +01:00
def promptForUpdate ( ) :
2016-02-08 01:15:05 +01:00
# Create hosts file if it doesn't exists
2016-02-08 01:18:16 +01:00
if not os . path . isfile ( os . path . join ( BASEDIR_PATH , ' hosts ' ) ) :
2016-02-08 01:15:05 +01:00
try :
2016-02-08 02:25:33 +01:00
open ( os . path . join ( BASEDIR_PATH , ' hosts ' ) , ' w+ ' ) . close ( )
2016-02-08 01:15:05 +01:00
except :
2016-02-08 01:18:16 +01:00
printFailure ( " ERROR: No ' hosts ' file in the folder, try creating one manually " )
2016-02-08 01:15:05 +01:00
2016-03-24 06:10:13 +01:00
if not settings [ " freshen " ] :
2016-03-18 21:27:09 +01:00
return
2016-03-24 06:10:13 +01:00
response = " yes " if settings [ " auto " ] else query_yes_no ( " Do you want to update all data sources? " )
2016-03-18 21:27:09 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
updateAllSources ( )
else :
2016-03-24 06:10:13 +01:00
if not settings [ " auto " ] :
2016-02-14 03:06:37 +01:00
print ( " OK, we \' ll stick with what we \' ve got locally. " )
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
def promptForExclusions ( ) :
2016-03-24 06:10:13 +01:00
response = " no " if settings [ " auto " ] else query_yes_no ( " Do you want to exclude any domains? \n " +
2016-02-08 01:15:05 +01:00
" For example, hulu.com video streaming must be able to access " +
2016-02-08 01:18:16 +01:00
" its tracking and ad servers in order to play video. " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
displayExclusionOptions ( )
else :
2016-03-24 06:10:13 +01:00
if not settings [ " auto " ] :
2016-02-14 03:06:37 +01:00
print ( " OK, we \' ll only exclude domains in the whitelist. " )
2013-02-17 22:39:40 +01:00
def promptForMoreCustomExclusions ( ) :
2016-02-08 01:18:16 +01:00
response = query_yes_no ( " Do you have more domains you want to enter? " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
return True
else :
return False
2013-07-30 18:59:31 +02:00
2016-02-08 01:18:16 +01:00
def promptForMove ( finalFile ) :
2016-02-23 03:26:52 +01:00
2016-03-24 06:10:13 +01:00
if settings [ " replace " ] :
2016-02-23 03:26:52 +01:00
response = " yes "
else :
2016-03-24 06:10:13 +01:00
response = " no " if settings [ " auto " ] else query_yes_no ( " Do you want to replace your existing hosts file " +
2016-02-08 01:25:32 +01:00
" with the newly generated file? " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:18:16 +01:00
moveHostsFileIntoPlace ( finalFile )
2016-02-08 01:15:05 +01:00
else :
return False
2013-02-17 21:51:49 +01:00
# End Prompt the User
# Exclusion logic
def displayExclusionOptions ( ) :
2016-03-24 06:10:13 +01:00
for exclusionOption in settings [ " common_exclusions " ] :
2016-02-08 01:18:16 +01:00
response = query_yes_no ( " Do you want to exclude the domain " + exclusionOption + " ? " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
excludeDomain ( exclusionOption )
else :
continue
2016-02-08 01:18:16 +01:00
response = query_yes_no ( " Do you want to exclude any other domains? " )
2016-02-08 01:23:40 +01:00
if response == " yes " :
2016-02-08 01:15:05 +01:00
gatherCustomExclusions ( )
2015-10-29 00:33:16 +01:00
2013-02-17 21:51:49 +01:00
def gatherCustomExclusions ( ) :
2016-02-08 01:15:05 +01:00
while True :
# Cross-python Input
2016-02-08 01:18:16 +01:00
domainFromUser = myInput ( " Enter the domain you want to exclude (e.g. facebook.com): " )
2016-02-08 01:23:40 +01:00
if isValidDomainFormat ( domainFromUser ) :
2016-02-08 01:18:16 +01:00
excludeDomain ( domainFromUser )
2016-02-08 01:41:44 +01:00
if promptForMoreCustomExclusions ( ) is False :
2016-02-08 01:15:05 +01:00
return
2013-02-17 21:51:49 +01:00
2016-02-08 01:18:16 +01:00
def excludeDomain ( domain ) :
2016-03-24 06:10:13 +01:00
settings [ " exclusionregexs " ] . append ( re . compile ( settings [ " exclusionpattern " ] + domain ) )
2013-02-17 21:51:49 +01:00
2016-02-08 01:18:16 +01:00
def matchesExclusions ( strippedRule ) :
2016-02-08 01:15:05 +01:00
strippedDomain = strippedRule . split ( ) [ 1 ]
2016-03-24 06:10:13 +01:00
for exclusionRegex in settings [ " exclusionregexs " ] :
2016-02-08 01:18:16 +01:00
if exclusionRegex . search ( strippedDomain ) :
2016-02-08 01:15:05 +01:00
return True
return False
2013-02-17 21:51:49 +01:00
# End Exclusion Logic
# Update Logic
2013-02-10 03:55:04 +01:00
def updateAllSources ( ) :
2016-03-24 06:10:13 +01:00
allsources = list ( set ( settings [ " sources " ] ) | set ( settings [ " extensions " ] ) )
2016-02-23 01:20:20 +01:00
for source in allsources :
2016-02-23 15:50:20 +01:00
if os . path . isdir ( source ) :
2016-03-06 02:28:32 +01:00
updateURLs = getUpdateURLsFromFile ( source )
if not len ( updateURLs ) :
2016-02-23 15:50:20 +01:00
continue
2016-03-06 02:28:32 +01:00
for updateURL in updateURLs :
print ( " Updating source " + os . path . basename ( source ) + " from " + updateURL )
# Cross-python call
updatedFile = getFileByUrl ( updateURL )
try :
updatedFile = updatedFile . replace ( ' \r ' , ' ' ) #get rid of carriage-return symbols
# This is cross-python code
2016-03-24 06:10:13 +01:00
dataFile = open ( os . path . join ( settings [ " datapath " ] , source , settings [ " datafilenames " ] ) , ' wb ' )
2016-03-06 02:28:32 +01:00
writeData ( dataFile , updatedFile )
dataFile . close ( )
except :
print ( " Skipping. " )
def getUpdateURLsFromFile ( source ) :
2016-03-24 06:10:13 +01:00
pathToUpdateFile = os . path . join ( settings [ " datapath " ] , source , settings [ " updateurlfilename " ] )
2016-03-06 02:28:32 +01:00
if os . path . exists ( pathToUpdateFile ) :
updateFile = open ( pathToUpdateFile , ' r ' )
retURLs = updateFile . readlines ( )
# .strip()
updateFile . close ( )
else :
retURL = None
printFailure ( ' Warning: Can \' t find the update file for source ' + source + ' \n ' +
' Make sure that there \' s a file at ' + pathToUpdateFile )
return retURLs
# End Update Logic
2016-01-03 00:57:23 +01:00
2016-02-08 01:18:16 +01:00
def getUpdateURLFromFile ( source ) :
2016-03-24 06:10:13 +01:00
pathToUpdateFile = os . path . join ( settings [ " datapath " ] , source , settings [ " updateurlfilename " ] )
2016-02-08 01:18:16 +01:00
if os . path . exists ( pathToUpdateFile ) :
updateFile = open ( pathToUpdateFile , ' r ' )
2016-02-08 01:15:05 +01:00
retURL = updateFile . readline ( ) . strip ( )
updateFile . close ( )
else :
retURL = None
2016-02-08 01:18:16 +01:00
printFailure ( ' Warning: Can \' t find the update file for source ' + source + ' \n ' +
' Make sure that there \' s a file at ' + pathToUpdateFile )
2016-02-08 01:15:05 +01:00
return retURL
2013-02-17 21:51:49 +01:00
# End Update Logic
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
# File Logic
2013-02-10 03:55:04 +01:00
def createInitialFile ( ) :
2016-02-08 01:15:05 +01:00
mergeFile = tempfile . NamedTemporaryFile ( )
2016-03-24 06:10:13 +01:00
for source in settings [ " sources " ] :
curFile = open ( os . path . join ( settings [ " datapath " ] , source , settings [ " datafilenames " ] ) , ' r ' )
2016-02-08 01:15:05 +01:00
#Done in a cross-python way
2016-02-08 01:18:16 +01:00
writeData ( mergeFile , curFile . read ( ) )
2015-10-29 00:33:16 +01:00
2016-03-24 06:10:13 +01:00
for source in settings [ " extensions " ] :
curFile = open ( os . path . join ( settings [ " extensionspath " ] , source , settings [ " datafilenames " ] ) , ' r ' )
2016-02-23 01:20:20 +01:00
#Done in a cross-python way
writeData ( mergeFile , curFile . read ( ) )
2016-02-08 01:15:05 +01:00
return mergeFile
2013-02-10 03:55:04 +01:00
2016-02-08 01:18:16 +01:00
def removeDupsAndExcl ( mergeFile ) :
2016-03-24 06:10:13 +01:00
numberOfRules = settings [ " numberofrules " ]
if os . path . isfile ( settings [ " whitelistfile " ] ) :
with open ( settings [ " whitelistfile " ] , " r " ) as ins :
2016-02-08 01:15:05 +01:00
for line in ins :
2016-02-23 03:48:02 +01:00
if line . rstrip ( ) :
2016-03-24 06:10:13 +01:00
settings [ " exclusions " ] . append ( line )
2013-07-13 22:59:29 +02:00
2016-03-24 06:10:13 +01:00
if not os . path . exists ( settings [ " outputpath " ] ) :
os . makedirs ( settings [ " outputpath " ] )
2016-03-13 05:17:37 +01:00
2015-10-29 00:33:16 +01:00
# Another mode is required to read and write the file in Python 3
2016-03-13 16:59:25 +01:00
if Python3 :
2016-03-24 06:10:13 +01:00
finalFile = open ( os . path . join ( settings [ " outputpath " ] , ' hosts ' ) , ' w+b ' )
2016-03-13 16:59:25 +01:00
else :
2016-03-24 06:10:13 +01:00
finalFile = open ( os . path . join ( settings [ " outputpath " ] , ' hosts ' ) , ' w+ ' )
2013-02-10 03:55:04 +01:00
2016-03-13 16:59:25 +01:00
mergeFile . seek ( 0 ) # reset file pointer
2016-02-08 01:15:05 +01:00
hostnames = set ( )
2016-02-08 01:18:16 +01:00
hostnames . add ( " localhost " )
2016-02-19 14:10:03 +01:00
hostnames . add ( " localhost.localdomain " )
2016-02-19 16:12:26 +01:00
hostnames . add ( " local " )
2016-02-19 14:10:03 +01:00
hostnames . add ( " broadcasthost " )
2016-03-24 06:10:13 +01:00
exclusions = settings [ " exclusions " ]
2016-02-08 01:15:05 +01:00
for line in mergeFile . readlines ( ) :
write = ' true '
2015-10-26 23:16:55 +01:00
# Explicit encoding
2016-02-08 01:18:16 +01:00
line = line . decode ( " UTF-8 " )
2016-02-15 18:36:38 +01:00
# replace tabs with space
line = line . replace ( ' \t + ' , ' ' )
2016-03-24 23:32:46 +01:00
# Trim trailing whitespace
2016-03-25 04:34:28 +01:00
line = line . rstrip ( ) + " \n "
2016-02-08 01:15:05 +01:00
# Testing the first character doesn't require startswith
if line [ 0 ] == ' # ' or re . match ( r ' ^ \ s*$ ' , line [ 0 ] ) :
# Cross-python write
2016-02-08 01:18:16 +01:00
writeData ( finalFile , line )
2016-02-08 01:15:05 +01:00
continue
if ' ::1 ' in line :
continue
2016-02-08 01:18:16 +01:00
strippedRule = stripRule ( line ) #strip comments
if len ( strippedRule ) == 0 :
2016-02-08 01:15:05 +01:00
continue
2016-02-08 01:18:16 +01:00
if matchesExclusions ( strippedRule ) :
2016-02-08 01:15:05 +01:00
continue
2016-02-08 01:18:16 +01:00
hostname , normalizedRule = normalizeRule ( strippedRule ) # normalize rule
2016-03-24 06:10:13 +01:00
for exclude in exclusions :
2016-02-08 01:32:00 +01:00
if exclude in line :
2016-02-08 01:15:05 +01:00
write = ' false '
break
2016-02-08 01:18:16 +01:00
if normalizedRule and ( hostname not in hostnames ) and ( write == ' true ' ) :
writeData ( finalFile , normalizedRule )
hostnames . add ( hostname )
2016-02-08 01:15:05 +01:00
numberOfRules + = 1
2016-03-24 06:10:13 +01:00
settings [ " numberofrules " ] = numberOfRules
2016-02-08 01:15:05 +01:00
mergeFile . close ( )
return finalFile
2013-02-10 03:55:04 +01:00
2014-05-16 14:13:11 +02:00
def normalizeRule ( rule ) :
2016-02-08 01:18:16 +01:00
result = re . search ( r ' ^[ \ t]*( \ d+ \ . \ d+ \ . \ d+ \ . \ d+) \ s+([ \ w \ .-]+)(.*) ' , rule )
2016-02-08 01:15:05 +01:00
if result :
2016-02-08 02:25:33 +01:00
hostname , suffix = result . group ( 2 , 3 )
2016-02-15 18:29:02 +01:00
hostname = hostname . lower ( ) . strip ( ) # explicitly lowercase and trim the hostname
2016-02-08 01:15:05 +01:00
if suffix is not ' ' :
# add suffix as comment only, not as a separate host
2016-03-24 06:10:13 +01:00
return hostname , " %s %s # %s \n " % ( settings [ " targetip " ] , hostname , suffix )
2016-02-08 01:15:05 +01:00
else :
2016-03-24 06:10:13 +01:00
return hostname , " %s %s \n " % ( settings [ " targetip " ] , hostname )
2016-02-08 01:37:27 +01:00
print ( " ==> %s <== " % rule )
2016-02-08 01:15:05 +01:00
return None , None
2014-05-16 14:13:11 +02:00
2016-02-08 01:18:16 +01:00
def finalizeFile ( finalFile ) :
writeOpeningHeader ( finalFile )
2016-02-08 01:15:05 +01:00
finalFile . close ( )
2013-02-10 03:55:04 +01:00
# Some sources put comments around their rules, for accuracy we need to strip them
# the comments are preserved in the output hosts file
2016-02-08 01:18:16 +01:00
def stripRule ( line ) :
2016-02-08 01:15:05 +01:00
splitLine = line . split ( )
2016-02-08 01:32:00 +01:00
if len ( splitLine ) < 2 :
2016-02-08 01:15:05 +01:00
# just return blank
return ' '
else :
return splitLine [ 0 ] + ' ' + splitLine [ 1 ]
2013-02-10 03:55:04 +01:00
def writeOpeningHeader ( finalFile ) :
2016-02-08 01:18:16 +01:00
finalFile . seek ( 0 ) #reset file pointer
2016-02-08 01:28:31 +01:00
fileContents = finalFile . read ( ) #save content
2016-02-08 01:18:16 +01:00
finalFile . seek ( 0 ) #write at the top
2016-02-27 04:55:07 +01:00
writeData ( finalFile , ' # This hosts file is a merged collection of hosts from reputable sources, \n ' )
2016-02-08 01:18:16 +01:00
writeData ( finalFile , ' # with a dash of crowd sourcing via Github \n # \n ' )
2016-03-16 05:33:49 +01:00
writeData ( finalFile , ' # Date: ' + time . strftime ( " % B %d % Y " , time . gmtime ( ) ) + ' \n ' )
2016-03-24 15:29:51 +01:00
if settings [ " extensions " ] :
writeData ( finalFile , ' # Extensions added to this file: ' + " , " . join ( settings [ " extensions " ] ) + ' \n ' )
2016-03-24 06:10:13 +01:00
writeData ( finalFile , ' # Number of unique domains: ' + " {:,} " . format ( settings [ " numberofrules " ] ) + ' \n # \n ' )
2016-03-24 15:29:51 +01:00
writeData ( finalFile , ' # Fetch the latest version of this file: https://raw.githubusercontent.com/StevenBlack/hosts/master/ ' + os . path . join ( settings [ " outputsubfolder " ] , " " ) + ' hosts \n ' )
2016-02-08 01:18:16 +01:00
writeData ( finalFile , ' # Project home page: https://github.com/StevenBlack/hosts \n # \n ' )
writeData ( finalFile , ' # =============================================================== \n ' )
writeData ( finalFile , ' \n ' )
writeData ( finalFile , ' 127.0.0.1 localhost \n ' )
2016-02-19 14:10:03 +01:00
writeData ( finalFile , ' 127.0.0.1 localhost.localdomain \n ' )
2016-03-24 10:54:23 +01:00
writeData ( finalFile , ' 127.0.0.1 local \n ' )
2016-02-19 14:10:03 +01:00
writeData ( finalFile , ' 255.255.255.255 broadcasthost \n ' )
2016-02-08 01:18:16 +01:00
writeData ( finalFile , ' ::1 localhost \n ' )
2016-02-21 01:33:29 +01:00
writeData ( finalFile , ' fe80::1 %lo 0 localhost \n ' )
2016-03-18 21:10:36 +01:00
if platform . system ( ) == ' Linux ' :
writeData ( finalFile , ' 127.0.1.1 ' + socket . gethostname ( ) + ' \n ' )
2016-02-08 01:18:16 +01:00
writeData ( finalFile , ' \n ' )
2016-02-08 01:28:31 +01:00
preamble = os . path . join ( BASEDIR_PATH , " myhosts " )
2016-02-08 01:18:16 +01:00
if os . path . isfile ( preamble ) :
with open ( preamble , " r " ) as f :
2016-02-08 01:28:31 +01:00
writeData ( finalFile , f . read ( ) )
2016-02-08 01:18:16 +01:00
finalFile . write ( fileContents )
2016-03-24 06:10:13 +01:00
def updateReadmeData ( ) :
2016-03-23 05:36:01 +01:00
extensionsKey = " base "
hostsLocation = " "
2016-03-24 06:10:13 +01:00
if settings [ " extensions " ] :
extensionsKey = " - " . join ( settings [ " extensions " ] )
2016-03-23 05:36:01 +01:00
generationData = { }
2016-03-24 06:10:13 +01:00
generationData [ " location " ] = os . path . join ( settings [ " outputsubfolder " ] , ' ' )
generationData [ " entries " ] = settings [ " numberofrules " ]
2016-03-23 05:36:01 +01:00
2016-03-24 06:10:13 +01:00
settings [ " readmedata " ] [ extensionsKey ] = generationData
with open ( settings [ " readmedatafilename " ] , ' w ' ) as f :
json . dump ( settings [ " readmedata " ] , f )
2016-03-23 05:36:01 +01:00
2016-02-08 01:18:16 +01:00
def moveHostsFileIntoPlace ( finalFile ) :
2016-02-08 01:32:00 +01:00
if os . name == ' posix ' :
2016-03-19 02:35:22 +01:00
dnsCacheFound = False
2016-02-08 01:37:27 +01:00
print ( " Moving the file requires administrative privileges. " +
" You might need to enter your password. " )
2016-02-08 01:32:00 +01:00
if subprocess . call ( [ " /usr/bin/sudo " , " cp " , os . path . abspath ( finalFile . name ) , " /etc/hosts " ] ) :
2016-02-08 01:18:16 +01:00
printFailure ( " Moving the file failed. " )
2016-02-08 01:37:27 +01:00
print ( " Flushing the DNS Cache to utilize new hosts file... " )
2016-02-08 01:32:00 +01:00
if platform . system ( ) == ' Darwin ' :
2016-03-19 02:35:22 +01:00
dnsCacheFound = True
2016-02-08 01:32:00 +01:00
if subprocess . call ( [ " /usr/bin/sudo " , " killall " , " -HUP " , " mDNSResponder " ] ) :
2016-02-08 01:18:16 +01:00
printFailure ( " Flushing the DNS Cache failed. " )
2016-02-08 01:15:05 +01:00
else :
2016-02-08 01:18:16 +01:00
if os . path . isfile ( " /etc/rc.d/init.d/nscd " ) :
2016-03-19 02:35:22 +01:00
dnsCacheFound = True
2016-02-08 01:32:00 +01:00
if subprocess . call ( [ " /usr/bin/sudo " , " /etc/rc.d/init.d/nscd " , " restart " ] ) :
2016-02-08 01:18:16 +01:00
printFailure ( " Flushing the DNS Cache failed. " )
2016-03-18 08:08:45 +01:00
else :
printSuccess ( " Flushing DNS by restarting nscd succeeded " )
2016-02-08 01:18:16 +01:00
if os . path . isfile ( " /usr/lib/systemd/system/NetworkManager.service " ) :
2016-03-19 02:35:22 +01:00
dnsCacheFound = True
2016-02-08 01:32:00 +01:00
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " NetworkManager.service " ] ) :
2016-02-08 01:18:16 +01:00
printFailure ( " Flushing the DNS Cache failed. " )
2016-03-18 08:08:45 +01:00
else :
printSuccess ( " Flushing DNS by restarting NetworkManager succeeded " )
2016-02-15 19:21:23 +01:00
if os . path . isfile ( " /usr/lib/systemd/system/wicd.service " ) :
2016-03-19 02:35:22 +01:00
dnsCacheFound = True
2016-02-15 19:21:23 +01:00
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " wicd.service " ] ) :
printFailure ( " Flushing the DNS Cache failed. " )
2016-03-18 08:08:45 +01:00
else :
printSuccess ( " Flushing DNS by restarting wicd succeeded " )
if os . path . isfile ( " /usr/lib/systemd/system/dnsmasq.service " ) :
2016-03-19 02:35:22 +01:00
dnsCacheFound = True
2016-03-18 08:08:45 +01:00
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " dnsmasq.service " ] ) :
printFailure ( " Flushing the DNS Cache failed. " )
else :
printSuccess ( " Flushing DNS by restarting dnsmasq succeeded " )
2016-03-24 23:07:17 +01:00
if os . path . isfile ( " /usr/lib/systemd/system/networking.service " ) :
dnsCacheFound = True
if subprocess . call ( [ " /usr/bin/sudo " , " /usr/bin/systemctl " , " restart " , " networking.service " ] ) :
printFailure ( " Flushing the DNS Cache failed. " )
else :
printSuccess ( " Flushing DNS by restarting networking.service succeeded " )
2016-03-19 02:35:22 +01:00
if not dnsCacheFound :
2016-03-18 08:08:45 +01:00
printFailure ( " Unable to determine DNS management tool. " )
2016-02-08 01:32:00 +01:00
elif os . name == ' nt ' :
2016-02-08 01:37:27 +01:00
print ( " Automatically moving the hosts file in place is not yet supported. " )
print ( " Please move the generated file to % SystemRoot % \ system32 \ drivers \ etc \ hosts " )
2016-02-08 01:15:05 +01:00
def removeOldHostsFile ( ) : # hotfix since merging with an already existing hosts file leads to artefacts and duplicates
2016-02-08 01:18:16 +01:00
oldFilePath = os . path . join ( BASEDIR_PATH , ' hosts ' )
open ( oldFilePath , ' a ' ) . close ( ) # create if already removed, so remove wont raise an error
2016-02-22 02:55:38 +01:00
backupFilePath = os . path . join ( BASEDIR_PATH , ' hosts- {0} ' . format ( time . strftime ( " % Y- % m- %d - % H- % M- % S " ) ) )
2016-02-21 17:37:03 +01:00
shutil . copy ( oldFilePath , backupFilePath ) # make a backup copy, marking the date in which the list was updated
2016-02-08 01:28:31 +01:00
os . remove ( oldFilePath )
2016-02-08 01:18:16 +01:00
open ( oldFilePath , ' a ' ) . close ( ) # create new empty hostsfile
2013-07-14 00:08:31 +02:00
2013-02-17 21:51:49 +01:00
# End File Logic
2013-02-10 03:55:04 +01:00
2013-02-17 21:51:49 +01:00
# Helper Functions
2013-02-10 03:55:04 +01:00
## {{{ http://code.activestate.com/recipes/577058/ (r2)
2016-02-08 01:18:16 +01:00
def query_yes_no ( question , default = " yes " ) :
2013-02-10 03:55:04 +01:00
""" Ask a yes/no question via raw_input() and return their answer.
2015-01-02 03:35:11 +01:00
2013-02-10 03:55:04 +01:00
" question " is a string that is presented to the user .
" default " is the presumed answer if the user just hits < Enter > .
It must be " yes " ( the default ) , " no " or None ( meaning
an answer is required of the user ) .
The " answer " return value is one of " yes " or " no " .
"""
2016-02-08 01:15:05 +01:00
valid = { " yes " : " yes " , " y " : " yes " , " ye " : " yes " ,
" no " : " no " , " n " : " no " }
2016-02-08 01:41:44 +01:00
if default is None :
2013-02-10 03:55:04 +01:00
prompt = " [y/n] "
elif default == " yes " :
prompt = " [Y/n] "
elif default == " no " :
prompt = " [y/N] "
else :
2016-02-08 01:18:16 +01:00
raise ValueError ( " invalid default answer: ' %s ' " % default )
2013-02-10 03:55:04 +01:00
while 1 :
2016-02-08 01:18:16 +01:00
sys . stdout . write ( colorize ( question , colors . PROMPT ) + prompt )
2015-10-26 23:46:48 +01:00
# Changed to be cross-python
choice = myInput ( ) . lower ( )
2013-02-10 03:55:04 +01:00
if default is not None and choice == ' ' :
return default
elif choice in valid . keys ( ) :
return valid [ choice ]
else :
2016-02-08 01:18:16 +01:00
printFailure ( " Please respond with ' yes ' or ' no ' " \
" (or ' y ' or ' n ' ). \n " )
2013-02-10 03:55:04 +01:00
## end of http://code.activestate.com/recipes/577058/ }}}
2016-02-08 01:18:16 +01:00
def isValidDomainFormat ( domain ) :
2016-02-08 01:32:00 +01:00
if domain == ' ' :
2016-02-08 01:18:16 +01:00
print ( " You didn \' t enter a domain. Try again. " )
2016-02-08 01:15:05 +01:00
return False
2016-02-08 01:18:16 +01:00
domainRegex = re . compile ( " www \ d { 0,3}[.]|https? " )
2016-02-08 01:32:00 +01:00
if domainRegex . match ( domain ) :
2016-02-08 01:34:27 +01:00
print ( " The domain " + domain + " is not valid. " +
" Do not include www.domain.com or http(s)://domain.com. Try again. " )
2016-02-08 01:15:05 +01:00
return False
else :
return True
2013-02-17 22:49:16 +01:00
# Colors
class colors :
2016-02-08 01:15:05 +01:00
PROMPT = ' \033 [94m '
2013-02-17 22:49:16 +01:00
SUCCESS = ' \033 [92m '
2016-02-08 01:15:05 +01:00
FAIL = ' \033 [91m '
ENDC = ' \033 [0m '
2013-02-17 22:49:16 +01:00
2016-02-08 01:18:16 +01:00
def colorize ( text , color ) :
2016-02-08 01:15:05 +01:00
return color + text + colors . ENDC
2013-02-17 22:49:16 +01:00
2016-02-08 01:18:16 +01:00
def printSuccess ( text ) :
print ( colorize ( text , colors . SUCCESS ) )
2013-02-17 22:49:16 +01:00
2016-02-08 01:18:16 +01:00
def printFailure ( text ) :
print ( colorize ( text , colors . FAIL ) )
2013-02-17 21:51:49 +01:00
# End Helper Functions
2013-02-10 03:55:04 +01:00
if __name__ == " __main__ " :
2016-02-08 01:15:05 +01:00
main ( )