nemo
/
bengaluru-food-census
1
0
Fork 0
Code Issues 1 Pull Requests Releases 4 Wiki Activity

Run parallel and only download missing files

This commit is contained in:
Nemo 2020-07-29 22:46:39 +05:30
parent 1e20e93cc7
commit efba5ac0cf
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bootstrap.sh
View File

@ -9,7 +9,7 @@ mkdir -p html/restaurants
function dl_z() {
echo "[+] $2"
curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
[ ! -f "html/$2" ] && curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
}
export -f dl_z
@ -26,9 +26,9 @@ done <<< $(bundle exec ruby parse_dir.rb)
while read -r LINK; do
FILENAME="$(basename $LINK).html"
echo $FILENAME
# sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
done <<< $(bundle exec ruby parse_listing.rb)
# bundle exec ruby parse_restaurant.rb
bundle exec ruby parse_restaurant.rb
# sem --wait
sem --wait