diff --git a/bootstrap.sh b/bootstrap.sh index 8eafb2e..4f6671b 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -9,7 +9,7 @@ mkdir -p html/restaurants function dl_z() { echo "[+] $2" - curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2" + [ ! -f "html/$2" ] && curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2" } export -f dl_z @@ -26,9 +26,9 @@ done <<< $(bundle exec ruby parse_dir.rb) while read -r LINK; do FILENAME="$(basename $LINK).html" echo $FILENAME - # sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME" + sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME" done <<< $(bundle exec ruby parse_listing.rb) -# bundle exec ruby parse_restaurant.rb +bundle exec ruby parse_restaurant.rb -# sem --wait \ No newline at end of file +sem --wait \ No newline at end of file