Browse Source

Run parallel and only download missing files

master
Nemo 7 months ago
parent
commit
efba5ac0cf
1 changed files with 4 additions and 4 deletions
  1. +4
    -4
      bootstrap.sh

+ 4
- 4
bootstrap.sh View File

@ -9,7 +9,7 @@ mkdir -p html/restaurants
function dl_z() {
echo "[+] $2"
curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
[ ! -f "html/$2" ] && curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
}
export -f dl_z
@ -26,9 +26,9 @@ done <<< $(bundle exec ruby parse_dir.rb)
while read -r LINK; do
FILENAME="$(basename $LINK).html"
echo $FILENAME
# sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
done <<< $(bundle exec ruby parse_listing.rb)
# bundle exec ruby parse_restaurant.rb
bundle exec ruby parse_restaurant.rb
# sem --wait
sem --wait

Loading…
Cancel
Save