diff --git a/fetch.sh b/fetch.sh index 39e5717..71d890d 100644 --- a/fetch.sh +++ b/fetch.sh @@ -9,9 +9,9 @@ function fetch_page() { # for each row pup '#nsdl-tables tr json{}' | \ # generate 6 lines (second column has a link, so parse that) with raw output - jq --raw-output '.[] | [.children[0].text, .children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \ - # and create a CSV from every 6 lines - paste -d, - - - - - - | \ + jq --raw-output '.[] | [.children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \ + # and create a CSV from every 5 lines + paste -d, - - - - - | \ # and we don't need the first row tail -n +2 >> "$3" } @@ -26,7 +26,7 @@ export -f fetch_page function fetch_class() { for i in $(seq 1 $2); do - sem -j 10 --timeout 300% fetch_page $1 $i "$1.csv" + sem -j 10 --timeout 500% fetch_page $1 $i "$1.csv" done }