jitsilg/internal/worker.go

/*
Copyright (C) 2021 j.r <j.r@jugendhacker.de>

jitsilg is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package jitsilg

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"sync"
	"time"

	"git.sr.ht/~j-r/jitsilg/internal/models"
	"github.com/dop251/goja"
	"github.com/mitchellh/mapstructure"
)

func LookupDomain(domain Domain, outputQueue chan Domain, dropQueue chan Domain, wg *sync.WaitGroup) {
	ips, _ := net.LookupIP(domain.Name)
	if len(ips) > 0 {
		domain.IP = ips[0]
		outputQueue <- domain
	} else {
		domain.Available = false
		dropQueue <- domain
	}
	wg.Done()
}

func CheckGeoIP(domains []Domain, outputQueue chan Domain, wg *sync.WaitGroup, timeout *Timeout) {
	var ips []net.IP
	for _, domain := range domains {
		ips = append(ips, domain.IP)
	}

	var requestBody bytes.Buffer
	encoder := json.NewEncoder(&requestBody)
	encoder.Encode(ips)
	timeout.Lock()
	log.Printf("Waiting for %v secs", timeout.WaitSeconds)
	time.Sleep(time.Duration(timeout.WaitSeconds) * time.Second)
	timeout.WaitSeconds = 0
	timeout.Unlock()
	resp, err := http.Post("http://ip-api.com/batch?fields=status,countryCode,isp,as,query", "application/json", &requestBody)
	if err != nil {
		log.Fatalf("An error occured sending GeoIP request: %e", err)
	}
	defer resp.Body.Close()

	if resp.Header["X-Rl"][0] == "0" {
		timeout.Lock()
		timeout.WaitSeconds, _ = strconv.Atoi(resp.Header["X-Ttl"][0])
		timeout.Unlock()
	}

	var response []models.GeoIPModel
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		log.Fatalf("An error occured deconding API response as JSON: %e", err)
	}

	for i := 0; i < len(domains); i++ {
		domain := &domains[i]
		resp := &response[i]
		domain.HosterName = resp.ISP
		domain.CountryCode = resp.CountryCode
	}

	for _, domain := range domains {
		outputQueue <- domain
	}

	wg.Done()
}

func CheckPage(domain Domain, badStuns []*regexp.Regexp, outputQueue chan Domain, wg *sync.WaitGroup) {
	reqURL := url.URL{
		Scheme: "https",
		Host:   domain.Name,
		Path:   "config.js",
	}
	resp, err := http.Get(reqURL.String())
	if _, ok := err.(*url.Error); ok {
		domain.Available = false
		outputQueue <- domain
		wg.Done()
		return
	}
	if err != nil {
		log.Fatalf("An error occured getting config.js: %v, error Type: %T", err, err)
	}

	domain.Available = true

	if resp.Header.Get("Content-Type") == "application/javascript" {

		log.Printf("processing config.js for: %v", domain.Name)
		respData, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatalf("Could not read body: %v", err)
		}

		vm := goja.New()

		_, err = vm.RunScript("config.js", string(respData))
		if err != nil {
			log.Fatalf("Error running config.js for %v: %v", domain.Name, err)
		}

		config := vm.Get("config")

		var configJS models.ConfigJS
		mapstructure.Decode(config.Export(), &configJS)

	Stuns:
		for _, stun := range append(configJS.P2P.StunServers, configJS.P2PStunServers...) {
			for _, badStun := range badStuns {
				if badStun.MatchString(stun.Urls) {
					domain.UnwantedStun = true
					break Stuns
				}
			}
		}

		if configJS.Hosts.Anonymousdomain != "" {
			domain.AuthEnabled = true
		}

		if configJS.Analytics.AmplitudeKey != "" || configJS.Analytics.GoogleTrackingID != "" || len(configJS.Analytics.ScriptURLs) > 0 {
			domain.AnalyticsEnabled = true
		}
	} else {
		domain.Available = false
	}

	outputQueue <- domain
	wg.Done()
}