increase timeout

This commit is contained in:
Nemo 2021-06-11 19:41:20 +05:30
parent 1fc2c76aaf
commit e726415cb3
1 changed files with 4 additions and 4 deletions

View File

@ -9,9 +9,9 @@ function fetch_page() {
# for each row # for each row
pup '#nsdl-tables tr json{}' | \ pup '#nsdl-tables tr json{}' | \
# generate 6 lines (second column has a link, so parse that) with raw output # generate 6 lines (second column has a link, so parse that) with raw output
jq --raw-output '.[] | [.children[0].text, .children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \ jq --raw-output '.[] | [.children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \
# and create a CSV from every 6 lines # and create a CSV from every 5 lines
paste -d, - - - - - - | \ paste -d, - - - - - | \
# and we don't need the first row # and we don't need the first row
tail -n +2 >> "$3" tail -n +2 >> "$3"
} }
@ -26,7 +26,7 @@ export -f fetch_page
function fetch_class() { function fetch_class() {
for i in $(seq 1 $2); do for i in $(seq 1 $2); do
sem -j 10 --timeout 300% fetch_page $1 $i "$1.csv" sem -j 10 --timeout 500% fetch_page $1 $i "$1.csv"
done done
} }