mirror of https://github.com/captn3m0/muse-dl.git
commit
60b8581b73
|
@ -0,0 +1,8 @@
|
||||||
|
.git
|
||||||
|
LICENSE
|
||||||
|
Dockerfile
|
||||||
|
spec/
|
||||||
|
bin/
|
||||||
|
Makefile
|
||||||
|
.dockerignore
|
||||||
|
*.Dockerfile
|
10
.travis.yml
10
.travis.yml
|
@ -1,8 +1,18 @@
|
||||||
language: crystal
|
language: crystal
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Path to 'hadolint' binary
|
||||||
|
HADOLINT: "${HOME}/hadolint"
|
||||||
|
|
||||||
|
install:
|
||||||
|
# Download hadolint binary and set it as executable
|
||||||
|
- curl -sL -o ${HADOLINT} "https://github.com/hadolint/hadolint/releases/download/v1.17.5/hadolint-$(uname -s)-$(uname -m)"
|
||||||
|
&& chmod 700 ${HADOLINT}
|
||||||
|
- shards install
|
||||||
script:
|
script:
|
||||||
- crystal spec
|
- crystal spec
|
||||||
- crystal tool format --check
|
- crystal tool format --check
|
||||||
|
- git ls-files --exclude='Dockerfile*' --ignored | xargs --max-lines=1 ${HADOLINT}
|
||||||
|
|
||||||
addons:
|
addons:
|
||||||
apt:
|
apt:
|
||||||
|
|
|
@ -0,0 +1,43 @@
|
||||||
|
FROM debian:10-slim
|
||||||
|
|
||||||
|
WORKDIR /build
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Add the key for the crystal debian repo
|
||||||
|
ADD https://keybase.io/crystal/pgp_keys.asc /tmp/crystal.gpg
|
||||||
|
|
||||||
|
# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 for why mkdir is needed
|
||||||
|
RUN mkdir -p /usr/share/man/man1 && \
|
||||||
|
apt-get update && \
|
||||||
|
apt-get install --yes --no-install-recommends \
|
||||||
|
# Install gnupg for the apt-key operation
|
||||||
|
gnupg=2.2.12-1+deb10u1 \
|
||||||
|
# libssl for faster TLS in Crystal
|
||||||
|
libssl-dev=1.1.1d-0+deb10u2 \
|
||||||
|
# pdftk as a dependency for muse-dl
|
||||||
|
pdftk=2.02-5 \
|
||||||
|
# ca-certificates for talking to crystal-lang.org
|
||||||
|
ca-certificates=20190110 \
|
||||||
|
# git to let shards install happen
|
||||||
|
git=1:2.20.1-2+deb10u1 \
|
||||||
|
# build --release
|
||||||
|
zlib1g-dev=1:1.2.11.dfsg-1 && \
|
||||||
|
# See https://crystal-lang.org/install/
|
||||||
|
apt-key add /tmp/crystal.gpg && \
|
||||||
|
echo "deb https://dist.crystal-lang.org/apt crystal main" > /etc/apt/sources.list.d/crystal.list && \
|
||||||
|
apt-get update && \
|
||||||
|
apt-get install --no-install-recommends --yes crystal=0.33.0-1 && \
|
||||||
|
# Cleanup
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN shards install && shards build --release && \
|
||||||
|
ln /build/bin/muse-dl /usr/bin/muse-dl
|
||||||
|
|
||||||
|
RUN apt-get --yes remove git gnupg
|
||||||
|
|
||||||
|
WORKDIR /data
|
||||||
|
VOLUME /data
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/bin/muse-dl"]
|
|
@ -0,0 +1,10 @@
|
||||||
|
mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
|
||||||
|
current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path))))
|
||||||
|
|
||||||
|
release:
|
||||||
|
# Build a static binary and save it in muse-dl-static
|
||||||
|
docker build --tag muse-dl-static --file static.Dockerfile .
|
||||||
|
# Then extract the image | extract the layer.tar file (we only have one layer) | extract the muse-dl-static file
|
||||||
|
docker image save muse-dl-static | tar xf - --wildcards "*/layer.tar" -O | tar xf - "muse-dl-static"
|
||||||
|
# And move it to the bin/ directory
|
||||||
|
mv -f muse-dl-static bin/
|
23
README.md
23
README.md
|
@ -8,6 +8,8 @@ Any downloads you perform with this tool are for your own usage. I personally ha
|
||||||
|
|
||||||
# Installation
|
# Installation
|
||||||
|
|
||||||
|
## Linux / Build
|
||||||
|
|
||||||
```
|
```
|
||||||
git clone https://github.com/captn3m0/muse-dl.git
|
git clone https://github.com/captn3m0/muse-dl.git
|
||||||
cd muse-dl
|
cd muse-dl
|
||||||
|
@ -16,9 +18,28 @@ shards build
|
||||||
./bin/muse-dl --help
|
./bin/muse-dl --help
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Linux / Download
|
||||||
|
|
||||||
|
A linux x86_64 static build is available in the latest release: <https://github.com/captn3m0/muse-dl/releases/latest>. Save the file as `muse-dl` and remember to mark it as executable (`chmod +x`).
|
||||||
|
|
||||||
|
## Docker
|
||||||
|
|
||||||
|
A docker image is available at `captn3m0/muse-dl` on Docker Hub. The working directory for the image is set as `/data`, so you'll need to mount your output-directory as `/data` for it to work. Sample invocations;
|
||||||
|
|
||||||
|
```
|
||||||
|
# Download the book, and put it in your Downloads directory
|
||||||
|
docker run -it /home/nemo/Downloads:/data captn3m0/muse-dl https://muse.jhu.edu/book/875
|
||||||
|
|
||||||
|
# If you have a list.txt file in your Downloads directory, then you can run
|
||||||
|
docker run -it /home/nemo/Downloads:/data captn3m0/muse-dl /data/list.txt
|
||||||
|
|
||||||
|
# If you want to keep the temporary files with your host, and not delete them
|
||||||
|
docker run -it /home/nemo/Downloads:/data /tmp:/musetmp --tmp-dir /musetmp --no-cleanup https://muse.jhu.edu/book/875
|
||||||
|
```
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
Please ensure you have `pdftk` installed, and run the `muse-dl` binary. To build the binary, please run the steps in Installation.
|
Please ensure you have `pdftk` installed, unless you're running via docker.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
require "./spec_helper"
|
||||||
|
# require "errors/muse_corrupt_pdf.cr"
|
||||||
|
|
||||||
|
describe Muse::Dl::Book do
|
||||||
|
it "should notice the unable to construct chapter PDF error" do
|
||||||
|
f = "/tmp/chapter-2379787.pdf"
|
||||||
|
File.delete(f) if File.exists? f
|
||||||
|
expect_raises Muse::Dl::Errors::MuseCorruptPDF do
|
||||||
|
Muse::Dl::Fetch.save_chapter("/tmp", "2379787", "NA")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,4 @@
|
||||||
|
module Muse::Dl::Errors
|
||||||
|
class MuseCorruptPDF < Exception
|
||||||
|
end
|
||||||
|
end
|
13
src/fetch.cr
13
src/fetch.cr
|
@ -1,5 +1,6 @@
|
||||||
require "crest"
|
require "crest"
|
||||||
require "./errors/*"
|
require "./errors/*"
|
||||||
|
require "myhtml"
|
||||||
|
|
||||||
module Muse::Dl
|
module Muse::Dl
|
||||||
class Fetch
|
class Fetch
|
||||||
|
@ -42,6 +43,18 @@ module Muse::Dl
|
||||||
|
|
||||||
# TODO: Add validation for the downloaded file (should be PDF)
|
# TODO: Add validation for the downloaded file (should be PDF)
|
||||||
Crest.get(url, max_redirects: 0, handle_errors: false, headers: headers) do |response|
|
Crest.get(url, max_redirects: 0, handle_errors: false, headers: headers) do |response|
|
||||||
|
# puts response.headers["Content-Type"]
|
||||||
|
content_type = response.headers["Content-Type"]
|
||||||
|
if content_type.is_a? String
|
||||||
|
if /html/.match content_type
|
||||||
|
puts response
|
||||||
|
response.body_io.each_line do |line|
|
||||||
|
if /Unable to construct chapter PDF/.match line
|
||||||
|
raise Muse::Dl::Errors::MuseCorruptPDF.new
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
File.open(tmp_pdf_file, "w") do |file|
|
File.open(tmp_pdf_file, "w") do |file|
|
||||||
IO.copy(response.body_io, file)
|
IO.copy(response.body_io, file)
|
||||||
end
|
end
|
||||||
|
|
|
@ -33,7 +33,12 @@ module Muse::Dl
|
||||||
unless parser.input_pdf
|
unless parser.input_pdf
|
||||||
# Save each chapter
|
# Save each chapter
|
||||||
thing.chapters.each do |chapter|
|
thing.chapters.each do |chapter|
|
||||||
Fetch.save_chapter(parser.tmp, chapter[0], chapter[1], parser.cookie, parser.bookmarks)
|
begin
|
||||||
|
Fetch.save_chapter(parser.tmp, chapter[0], chapter[1], parser.cookie, parser.bookmarks)
|
||||||
|
rescue e : Muse::Dl::Errors::MuseCorruptPDF
|
||||||
|
STDERR.puts "Got a 'Unable to construct chapter PDF' error from MUSE, skipping: #{url}"
|
||||||
|
return
|
||||||
|
end
|
||||||
end
|
end
|
||||||
chapter_ids = thing.chapters.map { |c| c[0] }
|
chapter_ids = thing.chapters.map { |c| c[0] }
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
FROM crystallang/crystal:latest as builder
|
||||||
|
|
||||||
|
WORKDIR /build
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN shards install && \
|
||||||
|
shards build --release --static
|
||||||
|
|
||||||
|
FROM scratch
|
||||||
|
|
||||||
|
COPY --from=builder /build/bin/muse-dl /muse-dl-static
|
Loading…
Reference in New Issue