Marown/subvenciones
Marown
/
subvenciones
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Run wget on quiet mode
continuous-integration/drone/push Build was killed Details

This commit is contained in:
Amab 2023-01-21 16:22:23 +01:00
parent c9f0af7dce
commit 3c10992282
2 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scripts/calls.sh
View File

@ -13,21 +13,21 @@ APP=(wget jq tar gzip)
function getCookies() {
echo "Getting cookies..."
wget -qO- --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
}
function fetchConcessions() {
function fetchCalls() {
local outputFile="page_$(printf %03d "$1").json"
echo "Downloading page $1..."
echo "Downloading page $1 of $2..."
if [ -z ${COOKIE+x} ]; then
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc";
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc";
else
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc"
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=convs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=4&sord=asc"
fi
}
export -f fetchConcessions
export -f fetchCalls
function getNumPages() {
local page=1
@ -35,7 +35,7 @@ function getNumPages() {
echo "Getting number of pages..."
fetchConcessions $page
fetchCalls $page $PAGES
PAGES=$(jq '.total' "$outputFile")
local records=$(jq '.records' "$outputFile")
@ -51,7 +51,7 @@ function getNumPages() {
function getAllCalls() {
echo "Downloading all calls..."
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchConcessions page"
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchCalls page $PAGES"
}
function convertJson2Csv() {

12
scripts/concessions.sh
View File

@ -14,18 +14,18 @@ APP=(wget jq csvgrep tar gzip)
function getCookies() {
echo "Getting cookies..."
wget -qO- --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --save-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" 'https://www.infosubvenciones.es/' &> /dev/null
}
function fetchConcessions() {
local outputFile="page_$(printf %03d "$1").json"
echo "Downloading page $1..."
echo "Downloading page $1 of $2..."
if [ -z ${COOKIE+x} ]; then
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc";
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --load-cookies $F_COOKIES --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc";
else
wget --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc"
wget -q --ca-certificate=AC_Componentes_Informaticos_SHA256.crt --keep-session-cookies --header "Cookie: $COOKIE" --header "User-Agent: $USER_AGENT" -O "$outputFile" "https://www.infosubvenciones.es/bdnstrans/busqueda?type=concs&_search=false&nd=$ND&rows=$ROWS&page=$1&sidx=8&sord=asc"
fi
}
export -f fetchConcessions
@ -36,7 +36,7 @@ function getNumPages() {
echo "Getting number of pages..."
fetchConcessions $page
fetchConcessions $page $PAGES
PAGES=$(jq '.total' "$outputFile")
local records=$(jq '.records' "$outputFile")
@ -52,7 +52,7 @@ function getNumPages() {
function getAllConcessions() {
echo "Downloading all concessions..."
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchConcessions page"
seq 2 "$PAGES" | xargs -I page -n 1 -P 5 bash -c "fetchConcessions page $PAGES"
}
function convertJson2Csv() {