diff --git a/README.md b/README.md index 2b5e4ba..344d898 100644 --- a/README.md +++ b/README.md @@ -2,102 +2,113 @@ ![Nebula header image](https://cdn.spacetelescope.org/archives/images/thumb700x/heic0707a.jpg) ->Where stars are born. +> Where stars are born. Manages the local infrastructure of my home server. I'm also doing blog posts around the same: -1. [Part 1, Hardware](https://captnemo.in/blog/2017/09/17/home-server-build/) -2. [Part 2, Terraform/Docker](https://captnemo.in/blog/2017/11/09/home-server-update/) -3. [Part 3, Learnings](https://captnemo.in/blog/2017/12/18/home-server-learnings/) -4. [Part 4, Migrating from Google (and more)](https://captnemo.in/blog/2017/12/31/migrating-from-google/) -5. [Part 5, Networking](https://captnemo.in/blog/2018/04/22/home-server-networking/) +1. [Part 1, Hardware](https://captnemo.in/blog/2017/09/17/home-server-build/) +2. [Part 2, Terraform/Docker](https://captnemo.in/blog/2017/11/09/home-server-update/) +3. [Part 3, Learnings](https://captnemo.in/blog/2017/12/18/home-server-learnings/) +4. [Part 4, Migrating from Google (and more)](https://captnemo.in/blog/2017/12/31/migrating-from-google/) +5. [Part 5, Networking](https://captnemo.in/blog/2018/04/22/home-server-networking/) The canonical URL for this repo is https://git.captnemo.in/nemo/nebula/. A mirror is maintained on GitHub at # modules -1. docker: to actually run the services. Catch-all for miscellaneous containers -2. cloudflare: to manage the DNS. -3. mysql: to create mysql users and databases. -4. media: Media related containers (Jackett, Lidarr, Radarr, Sonarr) -5. Monitoring: Monitoring related resources (Cadvisor, Grafana, NodeExporter, Prometheus, Transmission-Exporter) -6. Gitea: Just git.captnemo.in -7. tt-rss: Tiny-Tiny RSS Web reader -8. Radicale: CardDav/CalDav webserver +1. docker: to actually run the services. Catch-all for miscellaneous containers +2. cloudflare: to manage the DNS. +3. mysql: to create mysql users and databases. +4. media: Media related containers (Jackett, Lidarr, Radarr, Sonarr) +5. Monitoring: Monitoring related resources (Cadvisor, Grafana, NodeExporter, Prometheus, Transmission-Exporter) +6. Gitea: Just git.captnemo.in +7. tt-rss: Tiny-Tiny RSS Web reader +8. Radicale: CardDav/CalDav webserver Self-learning project for terraform/docker. # Planned -1. ~Setup DigitalOcean~ -2. Add DO infrastructure via ansible -3. ~Add traefik for proper proxying~ -4. Maybe add docker swarm (or k8s?) across both the servers. Might setup the k8s API on the Raspberry Pi. +1. ~Setup DigitalOcean~ +2. Add DO infrastructure via ansible +3. ~Add traefik for proper proxying~ +4. Maybe add docker swarm (or k8s?) across both the servers. Might setup the k8s API on the Raspberry Pi. # Service List Currently running the following (all links are to the `store.docker.com` links for the docker images that I'm using: -| image | tag | size | category/module | -|--------------------------------|---------|------|-----------------| -| prom/node-exporter | v0.15.2 | 22.8 | monitoring | -| redis | alpine | 27.8 | gitea | -| linuxserver/transmission | latest | 43.9 | media | -| traefik | 1.6 | 51.8 | docker | -| google/cadvisor | latest | 62.2 | monitoring | -| odarriba/timemachine | latest | 77.2 | backup | -| gitea/gitea | 1.4 | 77.4 | gitea | -| linuxserver/heimdall | latest | 101 | general | -| linuxserver/tt-rss | latest | 108 | tt-rss | -| prom/prometheus | latest | 113 | monitoring | -| linuxserver/ubooquity | latest | 114 | docker | -| captn3m0/speedtest-exporter | alpine | 115 | monitoring | -| tomsquest/docker-radicale | latest | 130 | radicale | -| linuxserver/lychee | latest | 154 | lychee | -| linuxserver/resilio-sync | latest | 167 | resilio | -| emby/embyserver | latest | 202 | media | -| linuxserver/airsonic | latest | 239 | media | -| grafana/grafana | latest | 301 | monitoring | -| requarks/wiki | latest | 317 | wiki | -| percona/percona-server-mongodb | latest | 321 | wiki | -| mariadb | 10.3 | 402 | db | -| linuxserver/jackett | latest | 556 | media | -| linuxserver/sonarr | latest | 562 | media | -| linuxserver/radarr | latest | 566 | media | -| linuxserver/lidarr | latest | 574 | media | +| image | tag | module/link | +| -------------------------------- | ---------- | ---------------------------------------------------- | +| bleenco/abstruse | latest | ci | +| captn3m0/opml-gen | latest | https://opml.bb8.fun | +| captn3m0/prometheus-act-exporter | latest | https://git.captnemo.in/nemo/prometheus-act-exporter | +| captn3m0/rss-bridge | latest | https://github.com/RSS-Bridge/rss-bridge | +| captn3m0/speedtest-exporter | alpine | https://github.com/stefanwalther/speedtest-exporter | +| emby/embyserver | latest | https://emby.media | +| gitea/gitea | 1.5.0-rc1 | services | +| google/cadvisor | latest | monitoring | +| grafana/grafana | latest | monitoring | +| jankysolutions/requestbin | latest | tools | +| linuxserver/airsonic | latest | media | +| linuxserver/heimdall | latest | tools | +| linuxserver/jackett | latest | media | +| linuxserver/lidarr | latest | media | +| linuxserver/lychee | latest | media | +| linuxserver/radarr | latest | media | +| linuxserver/resilio-sync | latest | sync | +| linuxserver/sonarr | latest | media | +| linuxserver/transmission | latest | media | +| linuxserver/tt-rss | latest | tools | +| linuxserver/ubooquity | latest | media | +| miniflux/miniflux | 2.0.9 | tools | +| monicahq/monicahq | latest | services | +| odarriba/timemachine | latest | tools | +| percona/percona-server-mongodb | 3.4 | database | +| postgres | 10-alpine | database | +| prom/node-exporter | v0.15.2 | monitoring | +| prom/prometheus | latest | monitoring | +| requarks/wiki | latest | services | +| serjs/go-socks5-proxy | latest | tools | +| tocttou/gotviz | latest | na | +| tomsquest/docker-radicale | latest | services | +| traefik | 1.6-alpine | plumbing | ## Docker Notes + - Lots of the above images are from the excellent [LinuxServer.io](https://www.linuxserver.io), and they're doing great work :+1: - Most images are running the latest beta (if available) or stable versions. - Traefik is running with wildcard certificates. ## Upstream -Issues I've faced/reported as a result of this project: +I've been using this as a contributing opportunity and reporting/fixing issues upstream: -1. Airsonic HTTPS proxying is broken. Reported: https://github.com/airsonic/airsonic/issues/641. Turned out to be a known issue: https://github.com/airsonic/airsonic/issues/594. Now fixed. -2. Traefik docker backend security headers were broken with dashes. I [reported it here](https://github.com/containous/traefik/issues/2493), and fixed by https://github.com/containous/traefik/pull/2496 :white_check_mark: -3. Headphones dies repeatedly with no error logs. Yet-to-report. (Already reported, fails due to classical artists) -4. Terraform doesn't parse mariadb version numbers. Report: https://github.com/terraform-providers/terraform-provider-mysql/issues/6. Filed a [PR to fix](https://github.com/hashicorp/go-version/pull/34) and [to bump the go-version dependency](https://github.com/terraform-providers/terraform-provider-mysql/pull/27) :white_check_mark: -5. `elibsrv` didn't support ebook-convert, only mobigen. PR is at https://github.com/captn3m0/elibsrv/pull/1. Merged to `elibsrv` trunk, will be part of next release. -6. `ubooquity` docker container doesn't let you set admin password: https://github.com/linuxserver/docker-ubooquity/issues/17. (Couldn't reproduce, closed) :white_check_mark: -7. Traefik customresponseheaders can't contain colons on the docker backend: https://github.com/containous/traefik/issues/2517. Fixed with https://github.com/containous/traefik/pull/2509 :white_check_mark: -8. Traefik Security headers don't overwrite upstream headers: https://github.com/containous/traefik/issues/2618 :white_check_mark: -9. Transmission exporter broke with different data types while unmarshalling JSON in go. I filed a PR https://github.com/metalmatze/transmission-exporter/pull/2 :white_check_mark: +1. Airsonic HTTPS proxying is broken. Reported: https://github.com/airsonic/airsonic/issues/641. Turned out to be a known issue: https://github.com/airsonic/airsonic/issues/594. Now fixed. +2. Traefik docker backend security headers were broken with dashes. I [reported it here](https://github.com/containous/traefik/issues/2493), and fixed by https://github.com/containous/traefik/pull/2496 :white_check_mark: +3. Headphones dies repeatedly with no error logs. Yet-to-report. (Already reported, fails due to classical artists) +4. Terraform doesn't parse mariadb version numbers. Report: https://github.com/terraform-providers/terraform-provider-mysql/issues/6. Filed a [PR to fix](https://github.com/hashicorp/go-version/pull/34) and [to bump the go-version dependency](https://github.com/terraform-providers/terraform-provider-mysql/pull/27) :white_check_mark: +5. `elibsrv` didn't support ebook-convert, only mobigen. PR is at https://github.com/captn3m0/elibsrv/pull/1. Merged to `elibsrv` trunk, will be part of next release. +6. `ubooquity` docker container doesn't let you set admin password: https://github.com/linuxserver/docker-ubooquity/issues/17. (Couldn't reproduce, closed) :white_check_mark: +7. Traefik customresponseheaders can't contain colons on the docker backend: https://github.com/containous/traefik/issues/2517. Fixed with https://github.com/containous/traefik/pull/2509 :white_check_mark: +8. Traefik Security headers don't overwrite upstream headers: https://github.com/containous/traefik/issues/2618 :white_check_mark: +9. Transmission exporter broke with different data types while unmarshalling JSON in go. I filed a PR https://github.com/metalmatze/transmission-exporter/pull/2 :white_check_mark: 10. Radarr official docker container was [running a very old `mediainfo`](https://github.com/Radarr/Radarr/issues/2668#issuecomment-376310514). [Filed a fix to upgrade `mediainfo` on the official radarr image](https://github.com/linuxserver/docker-baseimage-mono/pull/3) :white_check_mark: 11. Patched the [speedtest-exporter](https://github.com/stefanwalther/speedtest-exporter/pull/7) to use Alpine and upgraded Node.JS for a smaller updated build. 12. Faced (4) above again because mariadb decided to add `:` in the version response. [Workaround was to force set `--version=10.3-mariadb`](https://git.captnemo.in/nemo/nebula/commit/5f47a08bb55eea2c708c41668657ac1efa84c72a) 13. Reported [2 critical security issues in Abstruse CI](https://github.com/bleenco/abstruse/issues/363). :white_check_mark: +14. Faced (13) above again with postgres, thankfully [someone already fixed version parsing](https://github.com/terraform-providers/terraform-provider-postgresql/pull/31) :white_check_mark: +15. RSS Bridge was missing an official Docker Image. [I Filed a PR](https://github.com/RSS-Bridge/rss-bridge/pull/720) :white_check_mark: # Plumbing Their is a lot of additional infrastructure that is _not-yet_ part of this repo. This includes: -1. The Digital Ocean droplet running DNSCrypt and simpleproxy to proxy over a openvpn connection to this box. -2. openbox, kodi configuration to run on boot along with the Steam Controller for the HTPC setup -3. Docker main configuration with half-baked CA setup -4. btrfs-backed subvolumes and snapshotting for most things in /mnt/xwing/ (in-progress) -5. User-creation on the main server. (I'm using a common user for media applications and specific users for other applications) +1. The Digital Ocean droplet running DNSCrypt and simpleproxy to proxy over a openvpn connection to this box. +2. openbox, kodi configuration to run on boot along with the Steam Controller for the HTPC setup +3. Docker main configuration with half-baked CA setup +4. btrfs-backed subvolumes and snapshotting for most things in /mnt/xwing/ (in-progress) +5. User-creation on the main server. (I'm using a common user for media applications and specific users for other applications) # License diff --git a/db/mariadb.tf b/db/mariadb.tf deleted file mode 100644 index 787f562..0000000 --- a/db/mariadb.tf +++ /dev/null @@ -1,49 +0,0 @@ -resource "docker_container" "mariadb" { - name = "mariadb" - image = "${docker_image.mariadb.latest}" - - volumes { - volume_name = "${docker_volume.mariadb_volume.name}" - container_path = "/var/lib/mysql" - host_path = "${docker_volume.mariadb_volume.mountpoint}" - } - - // This is so that other host-only services can share this - ports { - internal = 3306 - external = 3306 - ip = "${var.ips["eth0"]}" - } - - // This is a not-so-great idea - // TODO: Figure out a better way to make terraform SSH and then connect to localhost - ports { - internal = 3306 - external = 3306 - ip = "${var.ips["tun0"]}" - } - - memory = 512 - restart = "unless-stopped" - destroy_grace_seconds = 10 - must_run = true - - env = [ - "MYSQL_ROOT_PASSWORD=${var.mysql_root_password}", - ] - - command = [ - "--version=${var.mariadb-version}-MariaDB", - ] - - networks = ["${docker_network.mariadb.id}"] -} - -resource "docker_image" "mariadb" { - name = "${data.docker_registry_image.mariadb.name}" - pull_triggers = ["${data.docker_registry_image.mariadb.sha256_digest}"] -} - -data "docker_registry_image" "mariadb" { - name = "mariadb:${var.mariadb-version}" -} diff --git a/db/network.tf b/db/network.tf index b7aee32..caa4588 100644 --- a/db/network.tf +++ b/db/network.tf @@ -1,14 +1,3 @@ -resource "docker_network" "mariadb" { - name = "mariadb" - driver = "bridge" - internal = true - - ipam_config { - subnet = "172.19.0.0/28" - gateway = "172.19.0.1" - } -} - resource "docker_network" "mongorocks" { name = "mongorocks" driver = "bridge" diff --git a/db/outputs.tf b/db/outputs.tf index 44e6640..99b38ce 100644 --- a/db/outputs.tf +++ b/db/outputs.tf @@ -1,7 +1,3 @@ -output "names-mariadb" { - value = "${docker_container.mariadb.name}" -} - output "networks-mongorocks" { value = "${docker_network.mongorocks.name}" } diff --git a/db/variables.tf b/db/variables.tf index 555c9fc..e57033a 100644 --- a/db/variables.tf +++ b/db/variables.tf @@ -1,8 +1,3 @@ -variable "mariadb-version" { - description = "mariadb version to use for fetching the docker image" - default = "10.2.14" -} - variable "postgres-version" { description = "postgres version to use for fetching the docker image" default = "10-alpine" @@ -12,5 +7,4 @@ variable "ips" { type = "map" } -variable "mysql_root_password" {} variable "postgres-root-password" {} diff --git a/db/volumes.tf b/db/volumes.tf index 2ad7e2a..677a4a2 100644 --- a/db/volumes.tf +++ b/db/volumes.tf @@ -1,7 +1,3 @@ -resource "docker_volume" "mariadb_volume" { - name = "mariadb_volume" -} - resource "docker_volume" "postgres_volume" { name = "postgres_volume" } diff --git a/docker/data.tf b/docker/data.tf index 7569914..b2315f0 100644 --- a/docker/data.tf +++ b/docker/data.tf @@ -1,7 +1,7 @@ data "docker_registry_image" "traefik" { # Critical and I like upgrading it # for updating config for new features - name = "traefik:1.6-alpine" + name = "traefik:1.7-alpine" } data "docker_registry_image" "wikijs" { diff --git a/docker/images.tf b/docker/images.tf index 07c4ad6..22d72cd 100644 --- a/docker/images.tf +++ b/docker/images.tf @@ -1,4 +1,4 @@ -resource "docker_image" "traefik16" { +resource "docker_image" "traefik17" { name = "${data.docker_registry_image.traefik.name}" pull_triggers = ["${data.docker_registry_image.traefik.sha256_digest}"] } diff --git a/docker/traefik.tf b/docker/traefik.tf index bab8e33..1417926 100644 --- a/docker/traefik.tf +++ b/docker/traefik.tf @@ -1,6 +1,6 @@ resource "docker_container" "traefik" { name = "traefik" - image = "${docker_image.traefik16.latest}" + image = "${docker_image.traefik17.latest}" # Admin Backend ports { diff --git a/main.tf b/main.tf index 2c00b5d..14760dc 100644 --- a/main.tf +++ b/main.tf @@ -4,15 +4,6 @@ module "cloudflare" { ips = "${var.ips}" } -# module "mysql" { -# source = "mysql" -# mysql_root_password = "${var.mysql_root_password}" -# mysql_lychee_password = "${var.mysql_lychee_password}" -# mysql_airsonic_password = "${var.mysql_airsonic_password}" -# mysql_kodi_password = "${var.mysql_kodi_password}" -# lychee_ip = "${module.docker.lychee-ip}" -# } - module "docker" { source = "docker" web_username = "${var.web_username}" @@ -27,7 +18,6 @@ module "docker" { module "db" { source = "db" - mysql_root_password = "${var.mysql_root_password}" postgres-root-password = "${var.postgres-root-password}" ips = "${var.ips}" } @@ -120,13 +110,10 @@ module "heimdall" { } module "media" { - source = "media" - domain = "bb8.fun" - - # links-mariadb = "${module.db.names-mariadb}" + source = "media" + domain = "bb8.fun" traefik-labels = "${var.traefik-common-labels}" airsonic-smtp-password = "${var.airsonic-smtp-password}" - airsonic-db-password = "${var.mysql_airsonic_password}" ips = "${var.ips}" traefik-network-id = "${module.docker.traefik-network-id}" } diff --git a/media/variables.tf b/media/variables.tf index 89d4cca..0fdd605 100644 --- a/media/variables.tf +++ b/media/variables.tf @@ -2,11 +2,8 @@ variable "domain" { type = "string" } -# variable "links-mariadb" {} variable "airsonic-smtp-password" {} -variable "airsonic-db-password" {} - variable "traefik-labels" { type = "map" } diff --git a/mysql/airsonic.tf b/mysql/airsonic.tf deleted file mode 100644 index c1515e9..0000000 --- a/mysql/airsonic.tf +++ /dev/null @@ -1,16 +0,0 @@ -resource "mysql_database" "airsonic" { - name = "airsonic" -} - -resource "mysql_user" "airsonic" { - user = "airsonic" - host = "%" - plaintext_password = "${var.mysql_airsonic_password}" -} - -resource "mysql_grant" "airsonic" { - user = "${mysql_user.airsonic.user}" - host = "${mysql_user.airsonic.host}" - database = "${mysql_database.airsonic.name}" - privileges = ["ALL"] -} diff --git a/mysql/lychee.tf b/mysql/lychee.tf deleted file mode 100644 index dfc9744..0000000 --- a/mysql/lychee.tf +++ /dev/null @@ -1,16 +0,0 @@ -resource "mysql_database" "lychee" { - name = "lychee" -} - -resource "mysql_user" "lychee" { - user = "lychee" - host = "%" - plaintext_password = "${var.mysql_lychee_password}" -} - -resource "mysql_grant" "lychee" { - user = "${mysql_user.lychee.user}" - host = "${mysql_user.lychee.host}" - database = "${mysql_database.lychee.name}" - privileges = ["ALL"] -} diff --git a/mysql/variables.tf b/mysql/variables.tf deleted file mode 100644 index a8bd97f..0000000 --- a/mysql/variables.tf +++ /dev/null @@ -1,17 +0,0 @@ -variable "mysql_root_password" { - type = "string" -} - -variable "mysql_lychee_password" { - type = "string" -} - -variable "mysql_airsonic_password" { - type = "string" -} - -variable "mysql_kodi_password" { - type = "string" -} - -variable "lychee_ip" {} diff --git a/variables.tf b/variables.tf index 09c754e..1493235 100644 --- a/variables.tf +++ b/variables.tf @@ -11,20 +11,10 @@ variable "web_password" { type = "string" } -variable "mysql_root_password" { - type = "string" -} - variable "postgres-root-password" { type = "string" } -variable "mysql_lychee_password" {} - -variable "mysql_airsonic_password" {} - -variable "mysql_kodi_password" {} - variable "mysql-ttrss-password" {} variable "gitea-mysql-password" {}