websegura/crawler/results.js

/**
 * Solicita los resultados (estado actual + histórico) de los sitios web dados de alta en el proyecto.
 * El análisis se hace utilizando el API de Mozilla.
 *
 * Más información en:
 * https://github.com/mozilla/http-observatory/blob/master/httpobs/docs/api.md
 */
const fs = require("fs/promises");
const axios = require("axios");
const Bottleneck = require("bottleneck");
const parser = require("./sites-parser");

const MOZILLA_API_BASE_URL =
  "https://http-observatory.security.mozilla.org/api/v1";
const MAX_CONCURRENT_REQUESTS = 20;

// TODO use the limiter (see analyze)
const limiter = new Bottleneck({ maxConcurrent: MAX_CONCURRENT_REQUESTS });

async function results() {
  const sites = await parser.parse();

  for (const site of sites) {
    const fileName = site.replace(/\./g, "!") + ".json";

    try {
      console.log(`Request ${site} scan history`);
      const history = await axios.get(
        `${MOZILLA_API_BASE_URL}/getHostHistory?host=${site}&rescan=true`
      );
      await fs.writeFile(
        `_data/results/history/${fileName}`,
        JSON.stringify(history.data, null, 2)
      );
    } catch (e) {
      if (e.response) {
        console.error(
          `Failed request (${e.response.status}): ${e.response.statusText}`
        );
      } else {
        console.error(e);
      }
    }

    try {
      console.log(`Request ${site} scan results`);
      const result = await axios.get(
        `${MOZILLA_API_BASE_URL}/analyze?host=${site}`
      );
      if (result.data.state === "FINISHED") {
        await fs.writeFile(
          `_data/results/${fileName}`,
          JSON.stringify(result.data, null, 2)
        );
      }
    } catch (e) {
      if (e.response) {
        console.error(
          `Failed (${e.response.status}): ${e.response.statusText}`
        );
      } else {
        console.error(e);
      }
    }
  }
}

results().catch((err) => {
  console.error(err);
  process.exit(1);
});