|
|
@ -9,7 +9,7 @@ mkdir -p html/restaurants |
|
|
|
|
|
|
|
function dl_z() { |
|
|
|
echo "[+] $2" |
|
|
|
curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2" |
|
|
|
[ ! -f "html/$2" ] && curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2" |
|
|
|
} |
|
|
|
|
|
|
|
export -f dl_z |
|
|
@ -26,9 +26,9 @@ done <<< $(bundle exec ruby parse_dir.rb) |
|
|
|
while read -r LINK; do |
|
|
|
FILENAME="$(basename $LINK).html" |
|
|
|
echo $FILENAME |
|
|
|
# sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME" |
|
|
|
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME" |
|
|
|
done <<< $(bundle exec ruby parse_listing.rb) |
|
|
|
|
|
|
|
# bundle exec ruby parse_restaurant.rb |
|
|
|
bundle exec ruby parse_restaurant.rb |
|
|
|
|
|
|
|
# sem --wait |
|
|
|
sem --wait |