Adds first working version

cmd/jitsilg/main.go

@@ -1,11 +1,28 @@
 package main
 
 import (
+	"log"
 	"os"
+	"regexp"
 
 	jitsilg "git.jugendhacker.de/j.r/jitsilg/internal"
 )
 
 func main() {
-	jitsilg.ParseFile(os.Args[1])
+	var badStunStrings = []string{
+		`stun[0-9]*\..*\.google.com`,
+	}
+
+	var badStuns []*regexp.Regexp
+	for _, stun := range badStunStrings {
+		regex, err := regexp.Compile(stun)
+		if err != nil {
+			log.Fatalf("Error compiling regex %s: %e", stun, err)
+		}
+		badStuns = append(badStuns, regex)
+	}
+
+	domains := jitsilg.ParseFile(os.Args[1])
+	output := jitsilg.ProcessDomains(domains, badStuns)
+	jitsilg.OutputMarkdown(output)
 }

go.mod

@@ -1,3 +1,8 @@
 module git.jugendhacker.de/j.r/jitsilg
 
 go 1.16
+
+require (
+	github.com/dop251/goja v0.0.0-20210406175830-1b11a6af686d // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+)

go.sum

@@ -0,0 +1,20 @@
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dop251/goja v0.0.0-20210406175830-1b11a6af686d h1:eyoriwRl4YlfXy64RCAiMyo3oX/UtA3eeje+qJk+fQA=
+github.com/dop251/goja v0.0.0-20210406175830-1b11a6af686d/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

internal/models/config.go

@@ -0,0 +1,26 @@
+package models
+
+type ConfigJS struct {
+	P2P            P2P          `mapstructure:"p2p"`
+	P2PStunServers []StunServer `mapstructure:"p2pStunServers"`
+	Hosts          Hosts        `mapstructure:"hosts"`
+	Analytics      Analytics    `mapstructure:"analytics"`
+}
+
+type P2P struct {
+	StunServers []StunServer `mapstructure:"stunServers"`
+}
+
+type StunServer struct {
+	Urls string `mapstructure:"urls"`
+}
+
+type Hosts struct {
+	Anonymousdomain string `mapstructure:"anonymousdomain"`
+}
+
+type Analytics struct {
+	GoogleTrackingID string   `mapstructure:"googleAnalyticsTrackingId"`
+	AmplitudeKey     string   `mapstructure:"amplitudeAPPKey"`
+	ScriptURLs       []string `mapstructure:"scriptURLs"`
+}

internal/models/geoip.go

@@ -0,0 +1,9 @@
+package models
+
+type GeoIPModel struct {
+	Status      string `json:"status"`
+	CountryCode string `json:"countryCode"`
+	ISP         string `json:"isp"`
+	AS          string `json:"as"`
+	Query       string `json:"query"`
+}

internal/output.go

@@ -0,0 +1,46 @@
+package jitsilg
+
+import (
+	"fmt"
+	"sort"
+)
+
+func OutputMarkdown(domains []Domain) {
+	sort.Slice(domains, func(i, j int) bool {
+		return domains[i].Name < domains[j].Name
+	})
+
+	var availabel, unavailable []Domain
+	for _, domain := range domains {
+		if domain.Available {
+			availabel = append(availabel, domain)
+		} else {
+			unavailable = append(unavailable, domain)
+		}
+	}
+
+	fmt.Println("# Jitsi Instanzen")
+
+	for _, domain := range availabel {
+		fmt.Printf("[%s](https://%s) %s (Hoster: %s, Version: %s)", domain.Name, domain.Name, domain.CountryCode, domain.HosterName, domain.Version)
+		if !domain.UnwantedStun {
+			fmt.Print(" \u2705")
+		}
+		if domain.AuthEnabled {
+			fmt.Print(" \U0001F510")
+		}
+		if domain.AnalyticsEnabled {
+			fmt.Print(" \U0001F50D")
+		}
+		if domain.UnwantedHoster {
+			fmt.Print(" \u274c")
+		}
+		fmt.Print("\n")
+	}
+
+	fmt.Println("\n## Aktuell nicht erreichbar")
+
+	for _, domain := range unavailable {
+		fmt.Printf("[%s](https://%s)\n", domain.Name, domain.Name)
+	}
+}

internal/parser.go

@@ -2,12 +2,11 @@ package jitsilg
 
 import (
 	"bufio"
-	"fmt"
 	"log"
 	"os"
 )
 
-func ParseFile(path string) []string {
+func ParseFile(path string) (domains []string) {
 	file, err := os.Open(path)
 	if err != nil {
 		log.Fatal(err)
@@ -16,7 +15,16 @@ func ParseFile(path string) []string {
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		fmt.Println(scanner.Text())
+		var domain string
+		domainFound := false
+		for _, char := range scanner.Text() {
+			if char == '"' {
+				domainFound = !domainFound
+			} else if domainFound {
+				domain += string(char)
+			}
+		}
+		domains = append(domains, domain)
 	}
-	return nil
+	return
 }

internal/processor.go

@@ -0,0 +1,77 @@
+package jitsilg
+
+import (
+	"net"
+	"regexp"
+	"sync"
+)
+
+type Domain struct {
+	Name             string
+	IP               net.IP
+	Available        bool
+	UnwantedHoster   bool
+	HosterName       string
+	CountryCode      string
+	UnwantedStun     bool
+	Version          string
+	AuthEnabled      bool
+	AnalyticsEnabled bool
+}
+
+type Timeout struct {
+	WaitSeconds int
+	sync.Mutex
+}
+
+func ProcessDomains(domains []string, badStuns []*regexp.Regexp) []Domain {
+	apiQueue := make(chan Domain)
+	pageParsingQueue := make(chan Domain)
+	outputQueue := make(chan Domain)
+
+	go func() {
+		var wg sync.WaitGroup
+		for _, domain := range domains {
+			var domainObj Domain
+			domainObj.Name = domain
+			wg.Add(1)
+			go LookupDomain(domainObj, apiQueue, outputQueue, &wg)
+		}
+		wg.Wait()
+		close(apiQueue)
+	}()
+
+	go func() {
+		var wg sync.WaitGroup
+		var timeout Timeout
+		var apiDomains []Domain
+		for domain := range apiQueue {
+			apiDomains = append(apiDomains, domain)
+			if len(apiDomains) == 100 {
+				wg.Add(1)
+				go CheckGeoIP(apiDomains, pageParsingQueue, &wg, &timeout)
+				apiDomains = []Domain{}
+			}
+		}
+		wg.Add(1)
+		go CheckGeoIP(apiDomains, pageParsingQueue, &wg, &timeout)
+		wg.Wait()
+		close(pageParsingQueue)
+	}()
+
+	go func() {
+		var wg sync.WaitGroup
+		for domain := range pageParsingQueue {
+			wg.Add(1)
+			go CheckPage(domain, badStuns, outputQueue, &wg)
+		}
+		wg.Wait()
+		close(outputQueue)
+	}()
+
+	var output []Domain
+	for domain := range outputQueue {
+		output = append(output, domain)
+	}
+	return output
+}

internal/worker.go

@@ -0,0 +1,140 @@
+package jitsilg
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+	"sync"
+	"time"
+
+	"git.jugendhacker.de/j.r/jitsilg/internal/models"
+	"github.com/dop251/goja"
+	"github.com/mitchellh/mapstructure"
+)
+
+func LookupDomain(domain Domain, outputQueue chan Domain, dropQueue chan Domain, wg *sync.WaitGroup) {
+	ips, _ := net.LookupIP(domain.Name)
+	if len(ips) > 0 {
+		domain.IP = ips[0]
+		outputQueue <- domain
+	} else {
+		domain.Available = false
+		dropQueue <- domain
+	}
+	wg.Done()
+}
+
+func CheckGeoIP(domains []Domain, outputQueue chan Domain, wg *sync.WaitGroup, timeout *Timeout) {
+	var ips []net.IP
+	for _, domain := range domains {
+		ips = append(ips, domain.IP)
+	}
+
+	var requestBody bytes.Buffer
+	encoder := json.NewEncoder(&requestBody)
+	encoder.Encode(ips)
+	timeout.Lock()
+	log.Printf("Waiting for %v secs", timeout.WaitSeconds)
+	time.Sleep(time.Duration(timeout.WaitSeconds) * time.Second)
+	timeout.WaitSeconds = 0
+	timeout.Unlock()
+	resp, err := http.Post("http://ip-api.com/batch?fields=status,countryCode,isp,as,query", "application/json", &requestBody)
+	if err != nil {
+		log.Fatalf("An error occured sending GeoIP request: %e", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.Header["X-Rl"][0] == "0" {
+		timeout.Lock()
+		timeout.WaitSeconds, _ = strconv.Atoi(resp.Header["X-Ttl"][0])
+		timeout.Unlock()
+	}
+
+	var response []models.GeoIPModel
+	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		log.Fatalf("An error occured deconding API response as JSON: %e", err)
+	}
+
+	for i := 0; i < len(domains); i++ {
+		domain := &domains[i]
+		resp := &response[i]
+		domain.HosterName = resp.ISP
+		domain.CountryCode = resp.CountryCode
+	}
+
+	for _, domain := range domains {
+		outputQueue <- domain
+	}
+
+	wg.Done()
+}
+
+func CheckPage(domain Domain, badStuns []*regexp.Regexp, outputQueue chan Domain, wg *sync.WaitGroup) {
+	reqURL := url.URL{
+		Scheme: "https",
+		Host:   domain.Name,
+		Path:   "config.js",
+	}
+	resp, err := http.Get(reqURL.String())
+	if _, ok := err.(*url.Error); ok {
+		domain.Available = false
+		outputQueue <- domain
+		wg.Done()
+		return
+	}
+	if err != nil {
+		log.Fatalf("An error occured getting config.js: %v, error Type: %T", err, err)
+	}
+
+	domain.Available = true
+
+	if resp.Header.Get("Content-Type") == "application/javascript" {
+
+		log.Printf("processing config.js for: %v", domain.Name)
+		respData, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			log.Fatalf("Could not read body: %v", err)
+		}
+
+		vm := goja.New()
+
+		_, err = vm.RunScript("config.js", string(respData))
+		if err != nil {
+			log.Fatalf("Error running config.js for %v: %v", domain.Name, err)
+		}
+
+		config := vm.Get("config")
+
+		var configJS models.ConfigJS
+		mapstructure.Decode(config.Export(), &configJS)
+
+	Stuns:
+		for _, stun := range append(configJS.P2P.StunServers, configJS.P2PStunServers...) {
+			for _, badStun := range badStuns {
+				if badStun.MatchString(stun.Urls) {
+					domain.UnwantedStun = true
+					break Stuns
+				}
+			}
+		}
+
+		if configJS.Hosts.Anonymousdomain != "" {
+			domain.AuthEnabled = true
+		}
+
+		if configJS.Analytics.AmplitudeKey != "" || configJS.Analytics.GoogleTrackingID != "" || len(configJS.Analytics.ScriptURLs) > 0 {
+			domain.AnalyticsEnabled = true
+		}
+	} else {
+		domain.Available = false
+	}
+
+	outputQueue <- domain
+	wg.Done()
+}