Minor changes
This commit is contained in:
parent
42cb68468c
commit
92b3175ce0
|
@ -16,14 +16,18 @@ export -f dl_z
|
|||
|
||||
dl "$DIRECTORY_URL" "directory.html"
|
||||
|
||||
# Download all the listing pages
|
||||
while read -r LINK; do
|
||||
FILENAME="$(basename $LINK).html"
|
||||
dl "$LINK" "$FILENAME"
|
||||
done <<< $(bundle exec ruby parse_dir.rb)
|
||||
|
||||
# Download all the restaurant pages (~15k)
|
||||
while read -r LINK; do
|
||||
FILENAME="$(basename $LINK).html"
|
||||
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
|
||||
done <<< $(bundle exec ruby parse_restaurant.rb)
|
||||
done <<< $(bundle exec ruby parse_listing.rb)
|
||||
|
||||
bundle exec ruby parse_restaurant.rb
|
||||
|
||||
sem --wait
|
|
@ -0,0 +1,4 @@
|
|||
require 'nokogiri'
|
||||
require "csv"
|
||||
|
||||
restaurants = []
|
Loading…
Reference in New Issue