require "crest" require "./errors/*" module Muse::Dl class Fetch USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" HEADERS = { "User-Agent" => USER_AGENT, "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Language" => "en-US,en;q=0.5", "DNT" => "1", "Cookie" => "session=124.123.104.8.1585420925750331; session=25719682.5a1ef8cb90ec8", "Connection" => "keep-alive", "Upgrade-Insecure-Requests" => "1", "Cache-Control" => "max-age=0", } def self.chapter_file_name(id : String, tmp_path : String) "#{tmp_path}/chapter-#{id}.pdf" end def self.save_chapter(tmp_path : String, chapter_id : String, chapter_title : String, add_bookmark = true) final_pdf_file = chapter_file_name chapter_id, tmp_path tmp_pdf_file = "#{final_pdf_file}.tmp" if File.exists? final_pdf_file puts "#{chapter_id} already downloaded" return end url = "https://muse.jhu.edu/chapter/#{chapter_id}" headers = HEADERS.merge({ "Referer" => "https://muse.jhu.edu/verify?url=%2Fchapter%2F#{chapter_id}%2Fpdf", }) Crest.get(url, max_redirects: 0, handle_errors: false, headers: headers) do |response| File.open(tmp_pdf_file, "w") do |file| IO.copy(response.body_io, file) end end pdftk = Muse::Dl::Pdftk.new tmp_path pdftk.strip_first_page tmp_pdf_file if add_bookmark # Run pdftk and add the bookmark to the file pdftk.add_bookmark tmp_pdf_file, chapter_title end # Now we can move the file to the proper PDF filename File.rename tmp_pdf_file, final_pdf_file puts "Downloaded #{chapter_id}" end def self.get_info(url : String) : Muse::Dl::Thing | Nil match = /https:\/\/muse.jhu.edu\/(book|journal)\/(\d+)/.match url if match begin response = Crest.get(url).to_s case match[1] when "book" return Muse::Dl::Book.new response when "journal" return Muse::Dl::Journal.new response end rescue ex : Crest::NotFound raise Muse::Dl::Errors::InvalidLink.new end else raise Muse::Dl::Errors::InvalidLink.new end end end end