Minor changes

This commit is contained in:
Nemo 2018-01-06 19:37:40 +05:30
parent 42cb68468c
commit 92b3175ce0
3 changed files with 9 additions and 1 deletions

View File

@ -16,14 +16,18 @@ export -f dl_z
dl "$DIRECTORY_URL" "directory.html"
# Download all the listing pages
while read -r LINK; do
FILENAME="$(basename $LINK).html"
dl "$LINK" "$FILENAME"
done <<< $(bundle exec ruby parse_dir.rb)
# Download all the restaurant pages (~15k)
while read -r LINK; do
FILENAME="$(basename $LINK).html"
sem -j 30 --timeout 300% dl_z "$LINK" "restaurants/$FILENAME"
done <<< $(bundle exec ruby parse_restaurant.rb)
done <<< $(bundle exec ruby parse_listing.rb)
bundle exec ruby parse_restaurant.rb
sem --wait

4
parse_restaurants.rb Normal file
View File

@ -0,0 +1,4 @@
require 'nokogiri'
require "csv"
restaurants = []