From e0fce7b1dfe10825177d277669e4480bcd78c81e Mon Sep 17 00:00:00 2001
From: Martin Wimpress <martin@wimpress.org>
Date: Fri, 10 Nov 2023 13:48:31 +0000
Subject: [PATCH] feat: restore automated Windows .iso downloads from Microsoft
 servers

This implementation is based on Mido:

- https://github.com/ElliotKillick/Mido

More useful failure messages are presented and if the download is blocked clear steps to manually complete the setup are presented. Windows 8 is re-instated as a supported release, although without install automation or driver optimisation.
---
 README.md |   9 ++-
 quickget  | 234 +++++++++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 192 insertions(+), 51 deletions(-)

diff --git a/README.md b/README.md
index 7d4c949..9a46737 100644
--- a/README.md
+++ b/README.md
@@ -482,7 +482,7 @@ sudo rm /Library/Preferences/SystemConfiguration/NetworkInterfaces.plist
 
 Now reboot, and the App Store should work.
 
-## Windows 10 & 11 Guests
+## Windows 8, 10 & 11 Guests
 
 `quickget` can download
 [Windows10](https://www.microsoft.com/software-download/windows10) and
@@ -491,9 +491,12 @@ automatically and create an optimised virtual machine configuration.
 This configuration also includes the [VirtIO drivers for
 Windows](https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/).
 
+Windows 8.1 is also supported but doesn't feature any automated installation or
+driver optimisation.
+
 ``` bash
 quickget windows 11
-quickemu --vm windows-11-22H2.conf
+quickemu --vm windows-11.conf
 ```
 
 -   Complete the installation as you normally would.
@@ -511,7 +514,7 @@ disk_img="windows-11/disk.qcow2"
 iso="windows-11/windows-11.iso"
 fixed_iso="windows-11/virtio-win.iso"
 tpm="on"
-secureboot="on"
+secureboot="off"
 ```
 
 -   `guest_os="windows"` instructs `quickemu` to optimise for Windows.
diff --git a/quickget b/quickget
index b382df0..9b3382e 100755
--- a/quickget
+++ b/quickget
@@ -893,7 +893,7 @@ function releases_vxlinux() {
 }
 
 function releases_windows() {
-    echo 10 11
+    echo 8 10 11
 }
 
 function releases_xerolinux() {
@@ -1157,13 +1157,7 @@ EOF
         # Enable TPM for Windows 11
         if [ "${OS}" == "windows" ] && [ "${RELEASE}" -ge 11 ]; then
             echo "tpm=\"on\"" >> "${CONF_FILE}"
-            # Only force SecureBoot on for non-Debian/Ubuntu distros.
-            if [ -e "/usr/share/OVMF/OVMF_CODE_4M.fd" ] && [ -e "/usr/share/OVMF/OVMF_VARS_4M.fd" ]; then
-              SEC_BOOT="off"
-            else
-              SEC_BOOT="on"
-            fi
-            echo "secureboot=\"${SEC_BOOT}\"" >> "${CONF_FILE}"
+            echo "secureboot=\"off\"" >> "${CONF_FILE}"
         fi
     fi
     echo
@@ -2477,48 +2471,191 @@ function unattended_windows() {
 EOF
 }
 
+handle_curl_error() {
+    local error_code="$1"
 
-function get_windows() {
-    # Use the iTechtics CDN to download the ISOs.
-    # - https://www.itechtics.com/windows-10-download-iso/
-    # - https://www.itechtics.com/windows-11-download-iso/
-    #  0 : Prompt for a manual ISO download
-    #  1 : Download automatically
-    local AUTO_DOWNLOAD=1
-    local DOWNLOAD_URL=""
+    local fatal_error_action=2
 
-    if [ ${AUTO_DOWNLOAD} -eq 1 ]; then
-        # Ignore the most recent Windows 10 release for now.
-        case ${RELEASE} in
-            10) DOWNLOAD_URL="https://www.itechtics.com/?dl_id=173"
-                EDITION="22H2"
-                FILE_NAME="Win${RELEASE}_${EDITION}_EnglishInternational_x64v1.iso"
-                FILE_HASH="dc3982ad27e2d4e03b680630c28f824cb78bcd47"
-                ;;
-            11) DOWNLOAD_URL="https://www.itechtics.com/?dl_id=168"
-                EDITION="22H2"
-                FILE_NAME="Win${RELEASE}_${EDITION}_English_x64v1.iso"
-                FILE_HASH="c5341ba26e420684468fa4d4ab434823c9d1b61f"
-                ;;
-        esac
+    case "$error_code" in
+        6)
+            echo "Failed to resolve Microsoft servers! Is there an Internet connection? Exiting..."
+            return "$fatal_error_action"
+            ;;
+        7)
+            echo "Failed to contact Microsoft servers! Is there an Internet connection or is the server down?"
+            ;;
+        23)
+            echo "Failed at writing Windows media to disk! Out of disk space or permission error? Exiting..."
+            return "$fatal_error_action"
+            ;;
+        26)
+            echo "Ran out of memory during download! Exiting..."
+            return "$fatal_error_action"
+            ;;
+        36)
+            echo "Failed to continue earlier download!"
+            ;;
+        22)
+            echo "Microsoft servers returned failing HTTP status code!"
+            ;;
+        # POSIX defines exit statuses 1-125 as usable by us
+        # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
+        $((error_code <= 125)))
+            # Must be some other server error (possibly with this specific request/file)
+            # This is when accounting for all possible errors in the curl manual assuming a correctly formed curl command and HTTP(S) request, using only the curl features we're using, and a sane build
+            echo "Server returned an error status!"
+            ;;
+        126 | 127)
+            echo "Curl command not found! Please install curl and try again. Exiting..."
+            return "$fatal_error_action"
+            ;;
+        # Exit statuses are undefined by POSIX beyond this point
+        *)
+            case "$(kill -l "$error_code")" in
+                # Signals defined to exist by POSIX:
+                # https://pubs.opengroup.org/onlinepubs/009695399/basedefs/signal.h.html
+                INT)
+                    echo "Curl was interrupted!"
+                    ;;
+                # There could be other signals but these are most common
+                SEGV | ABRT)
+                    echo "Curl crashed! Failed exploitation attempt? Please report any core dumps to curl developers. Exiting..."
+                    return "$fatal_error_action"
+                    ;;
+                *)
+                    echo "Curl terminated due to a fatal signal!"
+                    ;;
+            esac
+    esac
 
-        echo "Downloading Windows ${RELEASE} ${EDITION}..."
-        web_get "${DOWNLOAD_URL}" "${VM_PATH}" "${FILE_NAME}"
+    return 1
+}
 
-        check_hash "${FILE_NAME}" "${FILE_HASH}"
-    else
-        case ${RELEASE} in
-            10) WINDOWS_ISO_URL="https://www.microsoft.com/software-download/windows10";;
-            11) WINDOWS_ISO_URL="https://www.microsoft.com/software-download/windows11";;
-        esac
-        echo "######################################################################"
-        echo "# Download a Windows ${RELEASE} .iso image from:"
-        echo "#  - ${WINDOWS_ISO_URL}"
-        echo "# Put the .iso image in the ${VM_PATH} directory and rename"
-        echo "# it to windows-${RELEASE}.iso."
-        echo "######################################################################"
+function download_windows() {
+    # Download newer consumer Windows versions from behind gated Microsoft API
+    # This function aims to precisely emulate what Fido does down to the URL requests and HTTP headers (exceptions: updated user agent and referer adapts to Windows version instead of always being "windows11") but written in POSIX sh (with coreutils) and curl instead of PowerShell (also simplified to greatly reduce attack surface)
+    # However, differences such as the order of HTTP headers and TLS stacks (could be used to do TLS fingerprinting) still exist
+    #
+    # Command translated: ./Fido -Win 10 -Lang English -Verbose
+    # "English" = "English (United States)" (as opposed to the default "English (International)")
+    # For testing Fido, replace all "https://" with "http://" and remove all instances of "-MaximumRedirection 0" (to allow redirection of HTTP traffic to HTTPS) so HTTP requests can easily be inspected in Wireshark
+    # Fido (command-line only) works under PowerShell for Linux if that makes it easier for you
+    # UPDATE: Fido v1.4.2+ no longer works without being edited on Linux due to these issues on the Fido GitHub repo (and possibly others after these): #56 and #58
+    #
+    # If this function in Mido fails to work for you then please test with the Fido script before creating an issue because we basically just copy what Fido does exactly:
+    # https://github.com/pbatard/Fido
+
+    # Either 8, 10, or 11
+    local windows_version="$1"
+
+    local url="https://www.microsoft.com/en-us/software-download/windows$windows_version"
+    case "$windows_version" in
+        8 | 10) url="${url}ISO";;
+    esac
+
+    local user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0"
+    # uuidgen: For MacOS (installed by default) and other systems (e.g. with no /proc) that don't have a kernel interface for generating random UUIDs
+    local session_id="$(cat /proc/sys/kernel/random/uuid 2> /dev/null || uuidgen --random)"
+
+    # Get product edition ID for latest release of given Windows version
+    # Product edition ID: This specifies both the Windows release (e.g. 22H2) and edition ("multi-edition" is default, either Home/Pro/Edu/etc., we select "Pro" in the answer files) in one number
+    # This is the *only* request we make that Fido doesn't. Fido manually maintains a list of all the Windows release/edition product edition IDs in its script (see: $WindowsVersions array). This is helpful for downloading older releases (e.g. Windows 10 1909, 21H1, etc.) but we always want to get the newest release which is why we get this value dynamically
+    # Also, keeping a "$WindowsVersions" array like Fido does would be way too much of a maintenance burden
+    # Remove "Accept" header that curl sends by default
+    local iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || {
+        handle_curl_error $?
+        return $?
+    }
+
+    # Limit untrusted size for input validation
+    iso_download_page_html="$(echo "$iso_download_page_html" | head -c 102400)"
+    # tr: Filter for only numerics to prevent HTTP parameter injection
+    # head -c was recently added to POSIX: https://austingroupbugs.net/view.php?id=407
+    local product_edition_id="$(echo "$iso_download_page_html" | grep -Eo '<option value="[0-9]+">Windows' | cut -d '"' -f 2 | head -n 1 | tr -cd '0-9' | head -c 16)"
+    echo " - Product edition ID: $product_edition_id"
+
+    # Permit Session ID
+    # "org_id" is always the same value
+    curl --silent --output /dev/null --user-agent "$user_agent" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "https://vlscppe.microsoft.com/tags?org_id=y6jn8c31&session_id=$session_id" || {
+        # This should only happen if there's been some change to how this API works (copy whatever fix Fido implements)
+        handle_curl_error $?
+        return $?
+    }
+
+    # Extract everything after the last slash
+    local url_segment_parameter="${url##*/}"
+
+    # Get language -> skuID association table
+    # SKU ID: This specifies the language of the ISO. We always use "English (United States)", however, the SKU for this changes with each Windows release
+    # We must make this request so our next one will be allowed
+    # --data "" is required otherwise no "Content-Length" header will be sent causing HTTP response "411 Length Required"
+    local language_skuid_table_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=a8f8f489-4c7f-463a-9ca6-5cff94d8d041&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=getskuinformationbyproductedition&sessionId=$session_id&productEditionId=$product_edition_id&sdVersion=2")" || {
+        handle_curl_error $?
+        return $?
+    }
+
+    # Limit untrusted size for input validation
+    language_skuid_table_html="$(echo "$language_skuid_table_html" | head -c 10240)"
+    # tr: Filter for only alphanumerics or "-" to prevent HTTP parameter injection
+    local sku_id="$(echo "$language_skuid_table_html" | grep "English (United States)" | sed 's/&quot;//g' | cut -d ',' -f 1  | cut -d ':' -f 2 | tr -cd '[:alnum:]-' | head -c 16)"
+    echo " - SKU ID: $sku_id"
+
+    # Get ISO download link
+    # If any request is going to be blocked by Microsoft it's always this last one (the previous requests always seem to succeed)
+    # --referer: Required by Microsoft servers to allow request
+    local iso_download_link_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --referer "$url" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=6e2a1789-ef16-4f27-a296-74ef7ef5d96b&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=GetProductDownloadLinksBySku&sessionId=$session_id&skuId=$sku_id&language=English&sdVersion=2")" || {
+        # This should only happen if there's been some change to how this API works
+        handle_curl_error $?
+        return $?
+    }
+
+    local failed=0
+
+    # Limit untrusted size for input validation
+    iso_download_link_html="$(echo "$iso_download_link_html" | head -c 4096)"
+
+    if ! [ "$iso_download_link_html" ]; then
+        # This should only happen if there's been some change to how this API works
+        echo " - Microsoft servers gave us an empty response to our request for an automated download."
+        failed=1
     fi
 
+    if echo "$iso_download_link_html" | grep -q "We are unable to complete your request at this time."; then
+        echo " - Microsoft blocked the automated download request based on your IP address."
+        failed=1
+    fi
+
+    if [ ${failed} -eq 1 ]; then
+        echo " - Manually download the Windows ${windows_version} ISO using a web browser from: ${url}"
+        echo " - Save the downloaded ISO to: $(realpath ${VM_PATH})"
+        echo " - Update the config file to reference the downloaded ISO: ./${VM_PATH}.conf"
+        echo " - Continuing with the VM creation process..."
+        return 1
+    fi
+
+    # Filter for 64-bit ISO download URL
+    # sed: HTML decode "&" character
+    # tr: Filter for only alphanumerics or punctuation
+    local iso_download_link="$(echo "$iso_download_link_html" | grep -o "https://software.download.prss.microsoft.com.*IsoX64" | cut -d '"' -f 1 | sed 's/&amp;/\&/g' | tr -cd '[:alnum:][:punct:]' | head -c 512)"
+
+    if ! [ "$iso_download_link" ]; then
+        # This should only happen if there's been some change to the download endpoint web address
+        echo " - Microsoft servers gave us no download link to our request for an automated download. Please manually download this ISO in a web browser: $url"
+        return 1
+    fi
+
+    echo " - Got latest ISO download link (valid for 24 hours): $iso_download_link"
+
+    # Download ISO
+    FILE_NAME="$(echo "$iso_download_link" | cut -d'?' -f1 | cut -d'/' -f5)"
+    web_get "$iso_download_link" "${VM_PATH}" "${FILE_NAME}"
+}
+
+function get_windows() {
+    echo "Downloading Windows ${RELEASE}..."
+    download_windows "${RELEASE}"
+
+    echo "Downloading VirtIO drivers..."
     web_get "https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso" "${VM_PATH}"
 
     rm -f "${VM_PATH}/unattended.iso"
@@ -2534,10 +2671,11 @@ function get_windows() {
             ;;
     esac
 
-    case "${AUTO_DOWNLOAD}" in
-        0) make_vm_config "windows-${RELEASE}.iso" "virtio-win.iso";;
-        1) make_vm_config "${FILE_NAME}" "virtio-win.iso";;
-    esac
+    if [ -n "${FILE_NAME}" ]; then
+        make_vm_config "${FILE_NAME}" "virtio-win.iso"
+    else
+        make_vm_config "windows-${RELEASE}.iso" "virtio-win.iso"
+    fi
 }
 
 open_url() {