mirror of https://github.com/captn3m0/muse-dl.git
Adds support for final journal downloads
This commit is contained in:
parent
3a2d45fb6e
commit
03fccde754
|
@ -27,6 +27,11 @@ module Muse::Dl
|
|||
File.delete(fns) if File.exists?(fns)
|
||||
end
|
||||
|
||||
def self.cleanup_articles(tmp_path : String, id : String)
|
||||
fns = article_file_name(id, tmp_path)
|
||||
File.delete(fns) if File.exists?(fns)
|
||||
end
|
||||
|
||||
def self.save_url(url : String, referer : String, file_name : String, tmp_path : String, cookie : String | Nil = nil, bookmark_title : String | Nil = nil, strip_first_page = true)
|
||||
tmp_pdf_file = "#{file_name}.tmp"
|
||||
if File.exists? file_name
|
||||
|
|
|
@ -42,6 +42,10 @@ module Muse::Dl
|
|||
end
|
||||
end
|
||||
|
||||
def self.journal_title(myhtml : Myhtml::Parser)
|
||||
myhtml.css("#journal_about_info .title").map(&.inner_text).to_a[0].strip
|
||||
end
|
||||
|
||||
def self.author(myhtml : Myhtml::Parser)
|
||||
myhtml.css("#book_about_info .author").map(&.inner_text).to_a[0].strip.gsub("<BR>", ", ").gsub("\n", " ")
|
||||
end
|
||||
|
|
18
src/issue.cr
18
src/issue.cr
|
@ -16,6 +16,8 @@ module Muse::Dl
|
|||
date : String | Nil,
|
||||
journal_title : String | Nil
|
||||
|
||||
setter :journal_title
|
||||
|
||||
def initialize(id : String, response : String | Nil = nil)
|
||||
@id = id
|
||||
@url = "https://muse.jhu.edu/issue/#{id}"
|
||||
|
@ -31,6 +33,11 @@ module Muse::Dl
|
|||
false
|
||||
end
|
||||
|
||||
def parse
|
||||
html = Crest.get(@url).to_s
|
||||
parse(html)
|
||||
end
|
||||
|
||||
def parse(html : String)
|
||||
h = Myhtml::Parser.new html
|
||||
@info = InfoParser.infobox(h)
|
||||
|
@ -47,26 +54,31 @@ module Muse::Dl
|
|||
@volume = /Volume (\d+)/.match(t).try &.[1]
|
||||
@number = /Number (\d+)/.match(t).try &.[1]
|
||||
@number = /Issue (\d+)/.match(t).try &.[1] unless @number
|
||||
@date = /((January|February|March|April|May|June|July|August|September|October|November|December) (\d+))/.match(t).try &.[1]
|
||||
@date = /((January|February|March|April|May|June|July|August|September|October|November|December|Sring|Winter|Fall|Summer) (\d+))/.match(t).try &.[1]
|
||||
@date = /(\d{4})/.match(t).try &.[1] unless @date
|
||||
end
|
||||
end
|
||||
|
||||
def parse_contents(myhtml : Myhtml::Parser)
|
||||
unless @journal_title
|
||||
journal_title_a = myhtml.css("#journal_banner_title a").first
|
||||
if journal_title_a
|
||||
@journal_title = journal_title_a.inner_text
|
||||
end
|
||||
end
|
||||
myhtml.css(".articles_list_text ol").each do |ol|
|
||||
link = ol.css("li.title a").first
|
||||
title = link.inner_text
|
||||
|
||||
pages = ol.css("li.pg").first.try &.inner_text
|
||||
matches = /(\d+)-(\d+)/.match pages
|
||||
pages = ol.css("li.pg")
|
||||
if pages.size > 0
|
||||
p = pages.first.try &.inner_text
|
||||
matches = /(\d+)-(\d+)/.match p
|
||||
if matches
|
||||
start_page = matches[1].to_i
|
||||
end_page = matches[2].to_i
|
||||
end
|
||||
end
|
||||
|
||||
ol.css("a").each do |l|
|
||||
url = l.attribute_by("href").to_s
|
||||
|
|
|
@ -3,11 +3,12 @@ require "./issue.cr"
|
|||
|
||||
module Muse::Dl
|
||||
class Journal
|
||||
getter :info, :summary, :publisher, :issues
|
||||
getter :info, :summary, :publisher, :issues, :title
|
||||
@info = Hash(String, String).new
|
||||
@summary : String
|
||||
@publisher : String
|
||||
@issues = [] of Muse::Dl::Issue
|
||||
@title : String
|
||||
|
||||
private getter :h
|
||||
|
||||
|
@ -16,6 +17,7 @@ module Muse::Dl
|
|||
@info = InfoParser.infobox(h)
|
||||
@summary = InfoParser.summary(h)
|
||||
@publisher = InfoParser.journal_publisher(h)
|
||||
@title = InfoParser.journal_title(h)
|
||||
parse_volumes(h)
|
||||
end
|
||||
|
||||
|
@ -32,7 +34,9 @@ module Muse::Dl
|
|||
|
||||
matches = /\/issue\/(\d+)/.match link
|
||||
if matches
|
||||
@issues.push Muse::Dl::Issue.new matches[1]
|
||||
issue = Muse::Dl::Issue.new matches[1]
|
||||
issue.journal_title = @title
|
||||
@issues.push issue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -12,6 +12,7 @@ module Muse::Dl
|
|||
class Main
|
||||
def self.dl(parser : Parser)
|
||||
url = parser.url
|
||||
puts "Downloading #{url}"
|
||||
thing = Fetch.get_info(url) if url
|
||||
return unless thing
|
||||
|
||||
|
@ -78,7 +79,7 @@ module Muse::Dl
|
|||
FileUtils.rm source if parser.cleanup
|
||||
elsif thing.is_a? Muse::Dl::Issue
|
||||
# Will have no effect if parser has a custom title
|
||||
parser.output = Util.slug_filename "#{thing.journal_title} - #{thing.title}.pdf"
|
||||
parser.force_set_output Util.slug_filename "#{thing.journal_title} - #{thing.title}.pdf"
|
||||
|
||||
# If file exists and we can't clobber
|
||||
if File.exists?(parser.output) && parser.clobber == false
|
||||
|
@ -88,7 +89,6 @@ module Muse::Dl
|
|||
temp_stitched_file = nil
|
||||
pdf_builder = Pdftk.new(parser.tmp)
|
||||
|
||||
# ## TODO till 111
|
||||
thing.articles.each do |article|
|
||||
begin
|
||||
Fetch.save_article(parser.tmp, article.id, parser.cookie, article.title, parser.strip_first)
|
||||
|
@ -101,21 +101,25 @@ module Muse::Dl
|
|||
|
||||
# Stitch the PDFs together
|
||||
temp_stitched_file = pdf_builder.stitch_articles article_ids
|
||||
# TODO: Add metadata for each Issue
|
||||
pdf_builder.add_metadata(temp_stitched_file, parser.output, thing)
|
||||
|
||||
# temp_stitched_file.delete if temp_stitched_file
|
||||
puts "--dont-strip-first-page was on. Please validate PDF file for any errors." unless parser.strip_first
|
||||
puts "DL: #{url}. Saved final output to #{parser.output}"
|
||||
|
||||
# Cleanup the chapter files
|
||||
# TODO
|
||||
# if parser.cleanup
|
||||
# thing.articles.each do |c|
|
||||
# Fetch.cleanup(parser.tmp, c[0])
|
||||
# end
|
||||
# end
|
||||
####
|
||||
# Cleanup the issue files
|
||||
if parser.cleanup
|
||||
thing.articles.each do |a|
|
||||
Fetch.cleanup_articles(parser.tmp, a.id)
|
||||
end
|
||||
end
|
||||
elsif thing.is_a? Muse::Dl::Journal
|
||||
thing.issues.each do |issue|
|
||||
# Update the issue
|
||||
issue.parse
|
||||
parser.url = issue.url
|
||||
Main.dl parser
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -26,6 +26,10 @@ module Muse::Dl
|
|||
@output = output_file unless @output != DEFAULT_FILE_NAME
|
||||
end
|
||||
|
||||
def force_set_output(output_file : String)
|
||||
@output = output_file
|
||||
end
|
||||
|
||||
def reset_output_file
|
||||
@output = DEFAULT_FILE_NAME
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue