chore: demo

.github/workflows/crawl.yml

@@ -1,5 +1,3 @@
-name: crawler
-
 on:
   # TODO enable scheduled execution
   # schedule:
@@ -10,7 +8,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/web_build.yml

@@ -1,25 +1,21 @@
-name: Web build
-
 on:
   push:
     branches:
       - master
 
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
 jobs:
   build:
     runs-on: ubuntu-20.04
 
-    strategy:
-      matrix:
-        node-version: [12.x]
-
     steps:
       - uses: actions/checkout@v2
 
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+      - uses: actions/setup-node@v2
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: '12'
 
       - name: Install dependencies & build
         run: |

crawl.sh

@@ -4,5 +4,12 @@ SITES=`cat sites.csv | cut -d ',' -f 1`
 for site in $SITES; do
   # see https://github.com/mozilla/http-observatory/blob/master/httpobs/docs/api.md
   echo "Scanning $site using Mozilla HTTP Observatory API"
-  curl -s -X POST "https://http-observatory.security.mozilla.org/api/v1/analyze?host=$site&rescan=true" > output/${site//./¡}.json
-done
+  curl -s -X POST "https://http-observatory.security.mozilla.org/api/v1/analyze?host=$site&rescan=true"
+done
+
+sleep 60
+
+for site in $SITES; do
+  echo "Requesting $site scan results"
+  curl -s -X GET "https://http-observatory.security.mozilla.org/api/v1/analyze?host=$site" > output/${site//./¡}.json
+done

output/www.auvasa.es

@@ -1 +0,0 @@
-{"algorithm_version":2,"end_time":"Tue, 12 Jan 2021 19:42:27 GMT","grade":"F","hidden":true,"likelihood_indicator":"MEDIUM","response_headers":{"Cache-Control":"private","Content-Encoding":"gzip","Content-Length":"26418","Content-Type":"text/html","Date":"Tue, 12 Jan 2021 19:42:25 GMT","Server":"Microsoft-IIS/10.0","Set-Cookie":"ASPSESSIONIDCWRSRCRA=CHHOMFDCJPJEHDPFMMGMDAOF; secure; path=/","Vary":"Accept-Encoding","X-Powered-By":"ASP.NET","X-Powered-By-Plesk":"PleskWin"},"scan_id":17249600,"score":0,"start_time":"Tue, 12 Jan 2021 19:42:21 GMT","state":"FINISHED","status_code":200,"tests_failed":8,"tests_passed":4,"tests_quantity":12}

output/www.casadelaindia.org

@@ -1 +0,0 @@
-{"algorithm_version":2,"end_time":"Wed, 13 Jan 2021 16:48:37 GMT","grade":"F","hidden":false,"likelihood_indicator":"MEDIUM","response_headers":{"Accept-Ranges":"bytes","Connection":"Keep-Alive","Content-Length":"151","Content-Type":"text/html; charset=UTF-8","Date":"Wed, 13 Jan 2021 16:48:36 GMT","ETag":"\"6e010f-97-4ef62ba51a239\"","Keep-Alive":"timeout=5, max=30","Last-Modified":"Tue, 07 Jan 2014 15:06:58 GMT","Server":"Apache/2.2.22 (@RELEASE@) DAV/2 mod_fcgid/2.3.9 mod_ssl/2.2.15 OpenSSL/1.0.1e-fips"},"scan_id":17260088,"score":0,"start_time":"Wed, 13 Jan 2021 16:48:32 GMT","state":"FINISHED","status_code":200,"tests_failed":6,"tests_passed":6,"tests_quantity":12}

output/www.museocienciavalladolid.es

@@ -1 +0,0 @@
-{"algorithm_version":2,"end_time":"Wed, 13 Jan 2021 16:52:51 GMT","grade":"F","hidden":false,"likelihood_indicator":"MEDIUM","response_headers":{"Accept-Ranges":"bytes","Connection":"Keep-Alive","Content-Encoding":"gzip","Content-Length":"260","Content-Type":"text/html","Date":"Wed, 13 Jan 2021 16:52:50 GMT","ETag":"\"16e-5b4c91cab27ab-gzip\"","Keep-Alive":"timeout=60","Last-Modified":"Mon, 23 Nov 2020 16:59:05 GMT","Server":"Apache/2.4.25 (Debian)","Vary":"Accept-Encoding"},"scan_id":17260126,"score":0,"start_time":"Wed, 13 Jan 2021 16:52:46 GMT","state":"FINISHED","status_code":200,"tests_failed":6,"tests_passed":6,"tests_quantity":12}

output/www.uva.es

@@ -1 +0,0 @@
-{"algorithm_version":2,"end_time":"Wed, 13 Jan 2021 16:52:53 GMT","grade":"C-","hidden":false,"likelihood_indicator":"MEDIUM","response_headers":{"Cache-Control":"max-age=900, private","Connection":"keep-alive","Content-Encoding":"gzip","Content-Type":"text/html","Date":"Wed, 13 Jan 2021 16:52:50 GMT","ETag":"W/\"13667-1610361035000\"","Expires":"Wed, 13 Jan 2021 17:07:50 GMT","Last-Modified":"Mon, 11 Jan 2021 10:30:35 GMT","Server":"nginx","Transfer-Encoding":"chunked","Vary":"Accept-Encoding","X-Cache-Status":"HIT","X-Frame-Options":"SAMEORIGIN","X-XSS-Protection":"1; mode=block"},"scan_id":17260124,"score":45,"start_time":"Wed, 13 Jan 2021 16:52:46 GMT","state":"FINISHED","status_code":200,"tests_failed":4,"tests_passed":8,"tests_quantity":12}

output/www.valladolid.es

@@ -1 +0,0 @@
-{"algorithm_version":2,"end_time":"Wed, 13 Jan 2021 16:52:59 GMT","grade":"D","hidden":false,"likelihood_indicator":"MEDIUM","response_headers":{"Accept-Ranges":"bytes","Access-Control-Allow-Origin":"http://www.valladolid.es","Cache-Control":"must-revalidate","Connection":"Keep-Alive","Content-Encoding":"gzip","Content-Length":"19376","Content-Security-Policy":"frame-ancestors 'self' http://www.valladolid.es https://www.valladolid.es http://www.valladolid.gob.es https://www.valladolid.gob.es","Content-Type":"text/html;charset=UTF-8","Date":"Wed, 13 Jan 2021 16:52:51 GMT","ETag":"\"66BAB4763AF6D0FB223055F80F27D8BB1610552173000\"","Keep-Alive":"timeout=15","ProxiaInstanceId":"836883674:1312654243:0","Server":"Apache","Set-Cookie":"_consul_session-consul=; Max-Age=0; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Domain=valladolid.es; Path=/; Secure","Strict-Transport-Security":"max-age=63072000; includeSubdomains;","X-Content-Type-Options":"nosniff","X-XSS-Protection":"1; mode=block"},"scan_id":17260125,"score":30,"start_time":"Wed, 13 Jan 2021 16:52:46 GMT","state":"FINISHED","status_code":200,"tests_failed":2,"tests_passed":10,"tests_quantity":12}