Minor changes
This commit is contained in:
parent
42cb68468c
commit
92b3175ce0
|
@ -16,14 +16,18 @@ export -f dl_z
|
||||||
|
|
||||||
dl "$DIRECTORY_URL" "directory.html"
|
dl "$DIRECTORY_URL" "directory.html"
|
||||||
|
|
||||||
|
# Download all the listing pages
|
||||||
while read -r LINK; do
|
while read -r LINK; do
|
||||||
FILENAME="$(basename $LINK).html"
|
FILENAME="$(basename $LINK).html"
|
||||||
dl "$LINK" "$FILENAME"
|
dl "$LINK" "$FILENAME"
|
||||||
done <<< $(bundle exec ruby parse_dir.rb)
|
done <<< $(bundle exec ruby parse_dir.rb)
|
||||||
|
|
||||||
|
# Download all the restaurant pages (~15k)
|
||||||
while read -r LINK; do
|
while read -r LINK; do
|
||||||
FILENAME="$(basename $LINK).html"
|
FILENAME="$(basename $LINK).html"
|
||||||
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
|
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
|
||||||
done <<< $(bundle exec ruby parse_restaurant.rb)
|
done <<< $(bundle exec ruby parse_listing.rb)
|
||||||
|
|
||||||
|
bundle exec ruby parse_restaurant.rb
|
||||||
|
|
||||||
sem --wait
|
sem --wait
|
|
@ -0,0 +1,4 @@
|
||||||
|
require 'nokogiri'
|
||||||
|
require "csv"
|
||||||
|
|
||||||
|
restaurants = []
|
Loading…
Reference in New Issue