Run parallel and only download missing files

bootstrap.sh

@@ -9,7 +9,7 @@ mkdir -p html/restaurants
 
 function dl_z() {
     echo "[+] $2"
-    curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
+    [ ! -f "html/$2" ] && curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
 }
 
 export -f dl_z
@@ -26,9 +26,9 @@ done <<< $(bundle exec ruby parse_dir.rb)
 while read -r LINK; do
     FILENAME="$(basename $LINK).html"
     echo $FILENAME
-    # sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
+    sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
 done <<< $(bundle exec ruby parse_listing.rb)
 
-# bundle exec ruby parse_restaurant.rb
+bundle exec ruby parse_restaurant.rb
 
-# sem --wait
+sem --wait