Run wget on quiet mode
Some checks reported errors
continuous-integration/drone/push Build was killed
Some checks reported errors
continuous-integration/drone/push Build was killed
This commit is contained in:
parent
c9f0af7dce
commit
3c10992282
|
@ -13,21 +13,21 @@ APP=(wget jq tar gzip)
|
||||||
function getCookies() {
|
function getCookies() {
|
||||||
echo "Getting cookies..."
|
echo "Getting cookies..."
|
||||||
|
|
||||||
wget -qO- --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
|
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
function fetchConcessions() {
|
function fetchCalls() {
|
||||||
local outputFile="page_$(printf %03d "$1").json"
|
local outputFile="page_$(printf %03d "$1").json"
|
||||||
|
|
||||||
echo "Downloading page $1..."
|
echo "Downloading page $1 of $2..."
|
||||||
|
|
||||||
if [ -z ${COOKIE+x} ]; then
|
if [ -z ${COOKIE+x} ]; then
|
||||||
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc";
|
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc";
|
||||||
else
|
else
|
||||||
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc"
|
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
export -f fetchConcessions
|
export -f fetchCalls
|
||||||
|
|
||||||
function getNumPages() {
|
function getNumPages() {
|
||||||
local page=1
|
local page=1
|
||||||
|
@ -35,7 +35,7 @@ function getNumPages() {
|
||||||
|
|
||||||
echo "Getting number of pages..."
|
echo "Getting number of pages..."
|
||||||
|
|
||||||
fetchConcessions $page
|
fetchCalls $page $PAGES
|
||||||
|
|
||||||
PAGES=$(jq '.total' "$outputFile")
|
PAGES=$(jq '.total' "$outputFile")
|
||||||
local records=$(jq '.records' "$outputFile")
|
local records=$(jq '.records' "$outputFile")
|
||||||
|
@ -51,7 +51,7 @@ function getNumPages() {
|
||||||
function getAllCalls() {
|
function getAllCalls() {
|
||||||
echo "Downloading all calls..."
|
echo "Downloading all calls..."
|
||||||
|
|
||||||
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchConcessions page"
|
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchCalls page $PAGES"
|
||||||
}
|
}
|
||||||
|
|
||||||
function convertJson2Csv() {
|
function convertJson2Csv() {
|
||||||
|
|
|
@ -14,18 +14,18 @@ APP=(wget jq csvgrep tar gzip)
|
||||||
function getCookies() {
|
function getCookies() {
|
||||||
echo "Getting cookies..."
|
echo "Getting cookies..."
|
||||||
|
|
||||||
wget -qO- --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
|
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
function fetchConcessions() {
|
function fetchConcessions() {
|
||||||
local outputFile="page_$(printf %03d "$1").json"
|
local outputFile="page_$(printf %03d "$1").json"
|
||||||
|
|
||||||
echo "Downloading page $1..."
|
echo "Downloading page $1 of $2..."
|
||||||
|
|
||||||
if [ -z ${COOKIE+x} ]; then
|
if [ -z ${COOKIE+x} ]; then
|
||||||
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc";
|
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc";
|
||||||
else
|
else
|
||||||
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc"
|
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
export -f fetchConcessions
|
export -f fetchConcessions
|
||||||
|
@ -36,7 +36,7 @@ function getNumPages() {
|
||||||
|
|
||||||
echo "Getting number of pages..."
|
echo "Getting number of pages..."
|
||||||
|
|
||||||
fetchConcessions $page
|
fetchConcessions $page $PAGES
|
||||||
|
|
||||||
PAGES=$(jq '.total' "$outputFile")
|
PAGES=$(jq '.total' "$outputFile")
|
||||||
local records=$(jq '.records' "$outputFile")
|
local records=$(jq '.records' "$outputFile")
|
||||||
|
@ -52,7 +52,7 @@ function getNumPages() {
|
||||||
function getAllConcessions() {
|
function getAllConcessions() {
|
||||||
echo "Downloading all concessions..."
|
echo "Downloading all concessions..."
|
||||||
|
|
||||||
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchConcessions page"
|
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchConcessions page $PAGES"
|
||||||
}
|
}
|
||||||
|
|
||||||
function convertJson2Csv() {
|
function convertJson2Csv() {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user