Run parallel and only download missing files
This commit is contained in:
parent
1e20e93cc7
commit
efba5ac0cf
|
@ -9,7 +9,7 @@ mkdir -p html/restaurants
|
||||||
|
|
||||||
function dl_z() {
|
function dl_z() {
|
||||||
echo "[+] $2"
|
echo "[+] $2"
|
||||||
curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
|
[ ! -f "html/$2" ] && curl -sS --http2-prior-knowledge --compressed -H "User-Agent: $USER_AGENT" $1 > "html/$2"
|
||||||
}
|
}
|
||||||
|
|
||||||
export -f dl_z
|
export -f dl_z
|
||||||
|
@ -26,9 +26,9 @@ done <<< $(bundle exec ruby parse_dir.rb)
|
||||||
while read -r LINK; do
|
while read -r LINK; do
|
||||||
FILENAME="$(basename $LINK).html"
|
FILENAME="$(basename $LINK).html"
|
||||||
echo $FILENAME
|
echo $FILENAME
|
||||||
# sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
|
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
|
||||||
done <<< $(bundle exec ruby parse_listing.rb)
|
done <<< $(bundle exec ruby parse_listing.rb)
|
||||||
|
|
||||||
# bundle exec ruby parse_restaurant.rb
|
bundle exec ruby parse_restaurant.rb
|
||||||
|
|
||||||
# sem --wait
|
sem --wait
|
Loading…
Reference in New Issue