increase timeout
This commit is contained in:
parent
1fc2c76aaf
commit
e726415cb3
8
fetch.sh
8
fetch.sh
|
@ -9,9 +9,9 @@ function fetch_page() {
|
||||||
# for each row
|
# for each row
|
||||||
pup '#nsdl-tables tr json{}' | \
|
pup '#nsdl-tables tr json{}' | \
|
||||||
# generate 6 lines (second column has a link, so parse that) with raw output
|
# generate 6 lines (second column has a link, so parse that) with raw output
|
||||||
jq --raw-output '.[] | [.children[0].text, .children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \
|
jq --raw-output '.[] | [.children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \
|
||||||
# and create a CSV from every 6 lines
|
# and create a CSV from every 5 lines
|
||||||
paste -d, - - - - - - | \
|
paste -d, - - - - - | \
|
||||||
# and we don't need the first row
|
# and we don't need the first row
|
||||||
tail -n +2 >> "$3"
|
tail -n +2 >> "$3"
|
||||||
}
|
}
|
||||||
|
@ -26,7 +26,7 @@ export -f fetch_page
|
||||||
|
|
||||||
function fetch_class() {
|
function fetch_class() {
|
||||||
for i in $(seq 1 $2); do
|
for i in $(seq 1 $2); do
|
||||||
sem -j 10 --timeout 300% fetch_page $1 $i "$1.csv"
|
sem -j 10 --timeout 500% fetch_page $1 $i "$1.csv"
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue