mirror of https://github.com/captn3m0/muse-dl.git
Add 60s timeout to downloads. Do backoff for all errors.
This commit is contained in:
parent
762164e223
commit
4e435dd3ab
16
src/fetch.cr
16
src/fetch.cr
|
@ -5,6 +5,7 @@ require "myhtml"
|
||||||
module Muse::Dl
|
module Muse::Dl
|
||||||
class Fetch
|
class Fetch
|
||||||
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
|
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
|
||||||
|
DOWNLOAD_TIMEOUT_SECS = 60
|
||||||
|
|
||||||
HEADERS = {
|
HEADERS = {
|
||||||
"User-Agent" => USER_AGENT,
|
"User-Agent" => USER_AGENT,
|
||||||
|
@ -33,6 +34,11 @@ module Muse::Dl
|
||||||
|
|
||||||
# TODO: Remove this hardcoding, and make this more generic by generating it within the Book class
|
# TODO: Remove this hardcoding, and make this more generic by generating it within the Book class
|
||||||
url = "https://muse.jhu.edu/chapter/#{chapter_id}/pdf"
|
url = "https://muse.jhu.edu/chapter/#{chapter_id}/pdf"
|
||||||
|
uri = URI.parse(url)
|
||||||
|
http_client = HTTP::Client.new(uri)
|
||||||
|
# Raise a IO::TimeoutError after 60 seconds.
|
||||||
|
http_client.read_timeout = DOWNLOAD_TIMEOUT_SECS
|
||||||
|
|
||||||
headers = HEADERS.merge({
|
headers = HEADERS.merge({
|
||||||
"Referer" => "https://muse.jhu.edu/verify?url=%2Fchapter%2F#{chapter_id}%2Fpdf",
|
"Referer" => "https://muse.jhu.edu/verify?url=%2Fchapter%2F#{chapter_id}%2Fpdf",
|
||||||
})
|
})
|
||||||
|
@ -41,8 +47,15 @@ module Muse::Dl
|
||||||
headers["Cookie"] = cookie
|
headers["Cookie"] = cookie
|
||||||
end
|
end
|
||||||
|
|
||||||
|
request = Crest::Request.new(:get, url, headers: headers, max_redirects: 0, handle_errors: false)
|
||||||
|
|
||||||
|
begin
|
||||||
|
response = request.execute
|
||||||
|
rescue ex : IO::TimeoutError
|
||||||
|
raise Muse::Dl::Errors::DownloadError.new("Error downloading chapter. Download took longer than #{DOWNLOAD_TIMEOUT_SECS} seconds.")
|
||||||
|
end
|
||||||
|
|
||||||
# TODO: Add validation for the downloaded file (should be PDF)
|
# TODO: Add validation for the downloaded file (should be PDF)
|
||||||
Crest.get(url, max_redirects: 0, handle_errors: false, headers: headers) do |response|
|
|
||||||
if !response.success?
|
if !response.success?
|
||||||
raise Muse::Dl::Errors::DownloadError.new("Error downloading chapter. HTTP response code: #{response.status}")
|
raise Muse::Dl::Errors::DownloadError.new("Error downloading chapter. HTTP response code: #{response.status}")
|
||||||
end
|
end
|
||||||
|
@ -67,7 +80,6 @@ module Muse::Dl
|
||||||
raise Muse::Dl::Errors::DownloadError.new("Error: downloaded chapter file size is zero. Response Content-Length header was #{headers["Content-Length"]}")
|
raise Muse::Dl::Errors::DownloadError.new("Error: downloaded chapter file size is zero. Response Content-Length header was #{headers["Content-Length"]}")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
|
||||||
|
|
||||||
pdftk = Muse::Dl::Pdftk.new tmp_path
|
pdftk = Muse::Dl::Pdftk.new tmp_path
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue