From 3ab14e79e54d28b260f2d896f11b315cabcfac3b Mon Sep 17 00:00:00 2001
From: Nemo <me@captnemo.in>
Date: Wed, 18 Jul 2018 18:17:57 +0530
Subject: [PATCH] Upgrades and kill mysql everywhere
---
README.md | 133 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------
main.tf | 17 ++---------------
variables.tf | 10 ----------
db/mariadb.tf | 49 -------------------------------------------------
db/network.tf | 11 -----------
db/outputs.tf | 4 ----
db/variables.tf | 6 ------
db/volumes.tf | 4 ----
docker/data.tf | 2 +-
docker/images.tf | 2 +-
docker/traefik.tf | 2 +-
media/variables.tf | 3 ---
mysql/airsonic.tf | 16 ----------------
mysql/lychee.tf | 16 ----------------
mysql/variables.tf | 17 -----------------
15 files changed, 77 insertions(+), 215 deletions(-)
diff --git a/README.md b/README.md
index 2b5e4ba..344d898 100644
--- a/README.md
+++ a/README.md
@@ -1,103 +1,114 @@
# nebula
->Where stars are born.
+> Where stars are born.
Manages the local infrastructure of my home server. I'm also doing blog posts around the same:
-1. [Part 1, Hardware](https://captnemo.in/blog/2017/09/17/home-server-build/)
-2. [Part 2, Terraform/Docker](https://captnemo.in/blog/2017/11/09/home-server-update/)
-3. [Part 3, Learnings](https://captnemo.in/blog/2017/12/18/home-server-learnings/)
-4. [Part 4, Migrating from Google (and more)](https://captnemo.in/blog/2017/12/31/migrating-from-google/)
-5. [Part 5, Networking](https://captnemo.in/blog/2018/04/22/home-server-networking/)
+1. [Part 1, Hardware](https://captnemo.in/blog/2017/09/17/home-server-build/)
+2. [Part 2, Terraform/Docker](https://captnemo.in/blog/2017/11/09/home-server-update/)
+3. [Part 3, Learnings](https://captnemo.in/blog/2017/12/18/home-server-learnings/)
+4. [Part 4, Migrating from Google (and more)](https://captnemo.in/blog/2017/12/31/migrating-from-google/)
+5. [Part 5, Networking](https://captnemo.in/blog/2018/04/22/home-server-networking/)
The canonical URL for this repo is https://git.captnemo.in/nemo/nebula/. A mirror is maintained on GitHub at <https://github.com/captn3m0/nebula>
# modules
-1. docker: to actually run the services. Catch-all for miscellaneous containers
-2. cloudflare: to manage the DNS.
-3. mysql: to create mysql users and databases.
-4. media: Media related containers (Jackett, Lidarr, Radarr, Sonarr)
-5. Monitoring: Monitoring related resources (Cadvisor, Grafana, NodeExporter, Prometheus, Transmission-Exporter)
-6. Gitea: Just git.captnemo.in
-7. tt-rss: Tiny-Tiny RSS Web reader
-8. Radicale: CardDav/CalDav webserver
+1. docker: to actually run the services. Catch-all for miscellaneous containers
+2. cloudflare: to manage the DNS.
+3. mysql: to create mysql users and databases.
+4. media: Media related containers (Jackett, Lidarr, Radarr, Sonarr)
+5. Monitoring: Monitoring related resources (Cadvisor, Grafana, NodeExporter, Prometheus, Transmission-Exporter)
+6. Gitea: Just git.captnemo.in
+7. tt-rss: Tiny-Tiny RSS Web reader
+8. Radicale: CardDav/CalDav webserver
Self-learning project for terraform/docker.
# Planned
-1. ~Setup DigitalOcean~
-2. Add DO infrastructure via ansible
-3. ~Add traefik for proper proxying~
-4. Maybe add docker swarm (or k8s?) across both the servers. Might setup the k8s API on the Raspberry Pi.
+1. ~Setup DigitalOcean~
+2. Add DO infrastructure via ansible
+3. ~Add traefik for proper proxying~
+4. Maybe add docker swarm (or k8s?) across both the servers. Might setup the k8s API on the Raspberry Pi.
# Service List
Currently running the following (all links are to the `store.docker.com` links for the docker images that I'm using:
-| image | tag | size | category/module |
-|--------------------------------|---------|------|-----------------|
-| prom/node-exporter | v0.15.2 | 22.8 | monitoring |
-| redis | alpine | 27.8 | gitea |
-| linuxserver/transmission | latest | 43.9 | media |
-| traefik | 1.6 | 51.8 | docker |
-| google/cadvisor | latest | 62.2 | monitoring |
-| odarriba/timemachine | latest | 77.2 | backup |
-| gitea/gitea | 1.4 | 77.4 | gitea |
-| linuxserver/heimdall | latest | 101 | general |
-| linuxserver/tt-rss | latest | 108 | tt-rss |
-| prom/prometheus | latest | 113 | monitoring |
-| linuxserver/ubooquity | latest | 114 | docker |
-| captn3m0/speedtest-exporter | alpine | 115 | monitoring |
-| tomsquest/docker-radicale | latest | 130 | radicale |
-| linuxserver/lychee | latest | 154 | lychee |
-| linuxserver/resilio-sync | latest | 167 | resilio |
-| emby/embyserver | latest | 202 | media |
-| linuxserver/airsonic | latest | 239 | media |
-| grafana/grafana | latest | 301 | monitoring |
-| requarks/wiki | latest | 317 | wiki |
-| percona/percona-server-mongodb | latest | 321 | wiki |
-| mariadb | 10.3 | 402 | db |
-| linuxserver/jackett | latest | 556 | media |
-| linuxserver/sonarr | latest | 562 | media |
-| linuxserver/radarr | latest | 566 | media |
-| linuxserver/lidarr | latest | 574 | media |
+| image | tag | module/link |
+| -------------------------------- | ---------- | ---------------------------------------------------- |
+| bleenco/abstruse | latest | ci |
+| captn3m0/opml-gen | latest | https://opml.bb8.fun |
+| captn3m0/prometheus-act-exporter | latest | https://git.captnemo.in/nemo/prometheus-act-exporter |
+| captn3m0/rss-bridge | latest | https://github.com/RSS-Bridge/rss-bridge |
+| captn3m0/speedtest-exporter | alpine | https://github.com/stefanwalther/speedtest-exporter |
+| emby/embyserver | latest | https://emby.media |
+| gitea/gitea | 1.5.0-rc1 | services |
+| google/cadvisor | latest | monitoring |
+| grafana/grafana | latest | monitoring |
+| jankysolutions/requestbin | latest | tools |
+| linuxserver/airsonic | latest | media |
+| linuxserver/heimdall | latest | tools |
+| linuxserver/jackett | latest | media |
+| linuxserver/lidarr | latest | media |
+| linuxserver/lychee | latest | media |
+| linuxserver/radarr | latest | media |
+| linuxserver/resilio-sync | latest | sync |
+| linuxserver/sonarr | latest | media |
+| linuxserver/transmission | latest | media |
+| linuxserver/tt-rss | latest | tools |
+| linuxserver/ubooquity | latest | media |
+| miniflux/miniflux | 2.0.9 | tools |
+| monicahq/monicahq | latest | services |
+| odarriba/timemachine | latest | tools |
+| percona/percona-server-mongodb | 3.4 | database |
+| postgres | 10-alpine | database |
+| prom/node-exporter | v0.15.2 | monitoring |
+| prom/prometheus | latest | monitoring |
+| requarks/wiki | latest | services |
+| serjs/go-socks5-proxy | latest | tools |
+| tocttou/gotviz | latest | na |
+| tomsquest/docker-radicale | latest | services |
+| traefik | 1.6-alpine | plumbing |
## Docker Notes
+
- Lots of the above images are from the excellent [LinuxServer.io](https://www.linuxserver.io), and they're doing great work :+1:
- Most images are running the latest beta (if available) or stable versions.
- Traefik is running with wildcard certificates.
## Upstream
-Issues I've faced/reported as a result of this project:
-
-1. Airsonic HTTPS proxying is broken. Reported: https://github.com/airsonic/airsonic/issues/641. Turned out to be a known issue: https://github.com/airsonic/airsonic/issues/594. Now fixed.
-2. Traefik docker backend security headers were broken with dashes. I [reported it here](https://github.com/containous/traefik/issues/2493), and fixed by https://github.com/containous/traefik/pull/2496 :white_check_mark:
-3. Headphones dies repeatedly with no error logs. Yet-to-report. (Already reported, fails due to classical artists)
-4. Terraform doesn't parse mariadb version numbers. Report: https://github.com/terraform-providers/terraform-provider-mysql/issues/6. Filed a [PR to fix](https://github.com/hashicorp/go-version/pull/34) and [to bump the go-version dependency](https://github.com/terraform-providers/terraform-provider-mysql/pull/27) :white_check_mark:
-5. `elibsrv` didn't support ebook-convert, only mobigen. PR is at https://github.com/captn3m0/elibsrv/pull/1. Merged to `elibsrv` trunk, will be part of next release.
-6. `ubooquity` docker container doesn't let you set admin password: https://github.com/linuxserver/docker-ubooquity/issues/17. (Couldn't reproduce, closed) :white_check_mark:
-7. Traefik customresponseheaders can't contain colons on the docker backend: https://github.com/containous/traefik/issues/2517. Fixed with https://github.com/containous/traefik/pull/2509 :white_check_mark:
-8. Traefik Security headers don't overwrite upstream headers: https://github.com/containous/traefik/issues/2618 :white_check_mark:
-9. Transmission exporter broke with different data types while unmarshalling JSON in go. I filed a PR https://github.com/metalmatze/transmission-exporter/pull/2 :white_check_mark:
+I've been using this as a contributing opportunity and reporting/fixing issues upstream:
+
+1. Airsonic HTTPS proxying is broken. Reported: https://github.com/airsonic/airsonic/issues/641. Turned out to be a known issue: https://github.com/airsonic/airsonic/issues/594. Now fixed.
+2. Traefik docker backend security headers were broken with dashes. I [reported it here](https://github.com/containous/traefik/issues/2493), and fixed by https://github.com/containous/traefik/pull/2496 :white_check_mark:
+3. Headphones dies repeatedly with no error logs. Yet-to-report. (Already reported, fails due to classical artists)
+4. Terraform doesn't parse mariadb version numbers. Report: https://github.com/terraform-providers/terraform-provider-mysql/issues/6. Filed a [PR to fix](https://github.com/hashicorp/go-version/pull/34) and [to bump the go-version dependency](https://github.com/terraform-providers/terraform-provider-mysql/pull/27) :white_check_mark:
+5. `elibsrv` didn't support ebook-convert, only mobigen. PR is at https://github.com/captn3m0/elibsrv/pull/1. Merged to `elibsrv` trunk, will be part of next release.
+6. `ubooquity` docker container doesn't let you set admin password: https://github.com/linuxserver/docker-ubooquity/issues/17. (Couldn't reproduce, closed) :white_check_mark:
+7. Traefik customresponseheaders can't contain colons on the docker backend: https://github.com/containous/traefik/issues/2517. Fixed with https://github.com/containous/traefik/pull/2509 :white_check_mark:
+8. Traefik Security headers don't overwrite upstream headers: https://github.com/containous/traefik/issues/2618 :white_check_mark:
+9. Transmission exporter broke with different data types while unmarshalling JSON in go. I filed a PR https://github.com/metalmatze/transmission-exporter/pull/2 :white_check_mark:
10. Radarr official docker container was [running a very old `mediainfo`](https://github.com/Radarr/Radarr/issues/2668#issuecomment-376310514). [Filed a fix to upgrade `mediainfo` on the official radarr image](https://github.com/linuxserver/docker-baseimage-mono/pull/3) :white_check_mark:
11. Patched the [speedtest-exporter](https://github.com/stefanwalther/speedtest-exporter/pull/7) to use Alpine and upgraded Node.JS for a smaller updated build.
12. Faced (4) above again because mariadb decided to add `:` in the version response. [Workaround was to force set `--version=10.3-mariadb`](https://git.captnemo.in/nemo/nebula/commit/5f47a08bb55eea2c708c41668657ac1efa84c72a)
13. Reported [2 critical security issues in Abstruse CI](https://github.com/bleenco/abstruse/issues/363). :white_check_mark:
+14. Faced (13) above again with postgres, thankfully [someone already fixed version parsing](https://github.com/terraform-providers/terraform-provider-postgresql/pull/31) :white_check_mark:
+15. RSS Bridge was missing an official Docker Image. [I Filed a PR](https://github.com/RSS-Bridge/rss-bridge/pull/720) :white_check_mark:
# Plumbing
Their is a lot of additional infrastructure that is _not-yet_ part of this repo. This includes:
-1. The Digital Ocean droplet running DNSCrypt and simpleproxy to proxy over a openvpn connection to this box.
-2. openbox, kodi configuration to run on boot along with the Steam Controller for the HTPC setup
-3. Docker main configuration with half-baked CA setup
-4. btrfs-backed subvolumes and snapshotting for most things in /mnt/xwing/ (in-progress)
-5. User-creation on the main server. (I'm using a common user for media applications and specific users for other applications)
+1. The Digital Ocean droplet running DNSCrypt and simpleproxy to proxy over a openvpn connection to this box.
+2. openbox, kodi configuration to run on boot along with the Steam Controller for the HTPC setup
+3. Docker main configuration with half-baked CA setup
+4. btrfs-backed subvolumes and snapshotting for most things in /mnt/xwing/ (in-progress)
+5. User-creation on the main server. (I'm using a common user for media applications and specific users for other applications)
# License
diff --git a/main.tf b/main.tf
index 2c00b5d..14760dc 100644
--- a/main.tf
+++ a/main.tf
@@ -1,18 +1,9 @@
module "cloudflare" {
source = "cloudflare"
domain = "bb8.fun"
ips = "${var.ips}"
}
-# module "mysql" {
-# source = "mysql"
-# mysql_root_password = "${var.mysql_root_password}"
-# mysql_lychee_password = "${var.mysql_lychee_password}"
-# mysql_airsonic_password = "${var.mysql_airsonic_password}"
-# mysql_kodi_password = "${var.mysql_kodi_password}"
-# lychee_ip = "${module.docker.lychee-ip}"
-# }
-
module "docker" {
source = "docker"
web_username = "${var.web_username}"
@@ -27,7 +18,6 @@
module "db" {
source = "db"
- mysql_root_password = "${var.mysql_root_password}"
postgres-root-password = "${var.postgres-root-password}"
ips = "${var.ips}"
}
@@ -120,13 +110,10 @@
}
module "media" {
- source = "media"
- domain = "bb8.fun"
-
- # links-mariadb = "${module.db.names-mariadb}"
+ source = "media"
+ domain = "bb8.fun"
traefik-labels = "${var.traefik-common-labels}"
airsonic-smtp-password = "${var.airsonic-smtp-password}"
- airsonic-db-password = "${var.mysql_airsonic_password}"
ips = "${var.ips}"
traefik-network-id = "${module.docker.traefik-network-id}"
}
diff --git a/variables.tf b/variables.tf
index 09c754e..1493235 100644
--- a/variables.tf
+++ a/variables.tf
@@ -11,19 +11,9 @@
type = "string"
}
-variable "mysql_root_password" {
- type = "string"
-}
-
variable "postgres-root-password" {
type = "string"
}
-
-variable "mysql_lychee_password" {}
-
-variable "mysql_airsonic_password" {}
-
-variable "mysql_kodi_password" {}
variable "mysql-ttrss-password" {}
variable "gitea-mysql-password" {}
diff --git a/db/mariadb.tf b/db/mariadb.tf
deleted file mode 100644
index 787f562..0000000 100644
--- a/db/mariadb.tf
+++ /dev/null
@@ -1,49 +1,0 @@
-resource "docker_container" "mariadb" {
- name = "mariadb"
- image = "${docker_image.mariadb.latest}"
-
- volumes {
- volume_name = "${docker_volume.mariadb_volume.name}"
- container_path = "/var/lib/mysql"
- host_path = "${docker_volume.mariadb_volume.mountpoint}"
- }
-
- // This is so that other host-only services can share this
- ports {
- internal = 3306
- external = 3306
- ip = "${var.ips["eth0"]}"
- }
-
- // This is a not-so-great idea
- // TODO: Figure out a better way to make terraform SSH and then connect to localhost
- ports {
- internal = 3306
- external = 3306
- ip = "${var.ips["tun0"]}"
- }
-
- memory = 512
- restart = "unless-stopped"
- destroy_grace_seconds = 10
- must_run = true
-
- env = [
- "MYSQL_ROOT_PASSWORD=${var.mysql_root_password}",
- ]
-
- command = [
- "--version=${var.mariadb-version}-MariaDB",
- ]
-
- networks = ["${docker_network.mariadb.id}"]
-}
-
-resource "docker_image" "mariadb" {
- name = "${data.docker_registry_image.mariadb.name}"
- pull_triggers = ["${data.docker_registry_image.mariadb.sha256_digest}"]
-}
-
-data "docker_registry_image" "mariadb" {
- name = "mariadb:${var.mariadb-version}"
-}
diff --git a/db/network.tf b/db/network.tf
index b7aee32..caa4588 100644
--- a/db/network.tf
+++ a/db/network.tf
@@ -1,14 +1,3 @@
-resource "docker_network" "mariadb" {
- name = "mariadb"
- driver = "bridge"
- internal = true
-
- ipam_config {
- subnet = "172.19.0.0/28"
- gateway = "172.19.0.1"
- }
-}
-
resource "docker_network" "mongorocks" {
name = "mongorocks"
driver = "bridge"
diff --git a/db/outputs.tf b/db/outputs.tf
index 44e6640..99b38ce 100644
--- a/db/outputs.tf
+++ a/db/outputs.tf
@@ -1,7 +1,3 @@
-output "names-mariadb" {
- value = "${docker_container.mariadb.name}"
-}
-
output "networks-mongorocks" {
value = "${docker_network.mongorocks.name}"
}
diff --git a/db/variables.tf b/db/variables.tf
index 555c9fc..e57033a 100644
--- a/db/variables.tf
+++ a/db/variables.tf
@@ -1,8 +1,3 @@
-variable "mariadb-version" {
- description = "mariadb version to use for fetching the docker image"
- default = "10.2.14"
-}
-
variable "postgres-version" {
description = "postgres version to use for fetching the docker image"
default = "10-alpine"
@@ -12,5 +7,4 @@
type = "map"
}
-variable "mysql_root_password" {}
variable "postgres-root-password" {}
diff --git a/db/volumes.tf b/db/volumes.tf
index 2ad7e2a..677a4a2 100644
--- a/db/volumes.tf
+++ a/db/volumes.tf
@@ -1,7 +1,3 @@
-resource "docker_volume" "mariadb_volume" {
- name = "mariadb_volume"
-}
-
resource "docker_volume" "postgres_volume" {
name = "postgres_volume"
}
diff --git a/docker/data.tf b/docker/data.tf
index 7569914..b2315f0 100644
--- a/docker/data.tf
+++ a/docker/data.tf
@@ -1,7 +1,7 @@
data "docker_registry_image" "traefik" {
# Critical and I like upgrading it
# for updating config for new features
- name = "traefik:1.6-alpine"
+ name = "traefik:1.7-alpine"
}
data "docker_registry_image" "wikijs" {
diff --git a/docker/images.tf b/docker/images.tf
index 07c4ad6..22d72cd 100644
--- a/docker/images.tf
+++ a/docker/images.tf
@@ -1,4 +1,4 @@
-resource "docker_image" "traefik16" {
+resource "docker_image" "traefik17" {
name = "${data.docker_registry_image.traefik.name}"
pull_triggers = ["${data.docker_registry_image.traefik.sha256_digest}"]
}
diff --git a/docker/traefik.tf b/docker/traefik.tf
index bab8e33..1417926 100644
--- a/docker/traefik.tf
+++ a/docker/traefik.tf
@@ -1,6 +1,6 @@
resource "docker_container" "traefik" {
name = "traefik"
- image = "${docker_image.traefik16.latest}"
+ image = "${docker_image.traefik17.latest}"
# Admin Backend
ports {
diff --git a/media/variables.tf b/media/variables.tf
index 89d4cca..0fdd605 100644
--- a/media/variables.tf
+++ a/media/variables.tf
@@ -1,11 +1,8 @@
variable "domain" {
type = "string"
}
-# variable "links-mariadb" {}
variable "airsonic-smtp-password" {}
-
-variable "airsonic-db-password" {}
variable "traefik-labels" {
type = "map"
diff --git a/mysql/airsonic.tf b/mysql/airsonic.tf
deleted file mode 100644
index c1515e9..0000000 100644
--- a/mysql/airsonic.tf
+++ /dev/null
@@ -1,16 +1,0 @@
-resource "mysql_database" "airsonic" {
- name = "airsonic"
-}
-
-resource "mysql_user" "airsonic" {
- user = "airsonic"
- host = "%"
- plaintext_password = "${var.mysql_airsonic_password}"
-}
-
-resource "mysql_grant" "airsonic" {
- user = "${mysql_user.airsonic.user}"
- host = "${mysql_user.airsonic.host}"
- database = "${mysql_database.airsonic.name}"
- privileges = ["ALL"]
-}
diff --git a/mysql/lychee.tf b/mysql/lychee.tf
deleted file mode 100644
index dfc9744..0000000 100644
--- a/mysql/lychee.tf
+++ /dev/null
@@ -1,16 +1,0 @@
-resource "mysql_database" "lychee" {
- name = "lychee"
-}
-
-resource "mysql_user" "lychee" {
- user = "lychee"
- host = "%"
- plaintext_password = "${var.mysql_lychee_password}"
-}
-
-resource "mysql_grant" "lychee" {
- user = "${mysql_user.lychee.user}"
- host = "${mysql_user.lychee.host}"
- database = "${mysql_database.lychee.name}"
- privileges = ["ALL"]
-}
diff --git a/mysql/variables.tf b/mysql/variables.tf
deleted file mode 100644
index a8bd97f..0000000 100644
--- a/mysql/variables.tf
+++ /dev/null
@@ -1,17 +1,0 @@
-variable "mysql_root_password" {
- type = "string"
-}
-
-variable "mysql_lychee_password" {
- type = "string"
-}
-
-variable "mysql_airsonic_password" {
- type = "string"
-}
-
-variable "mysql_kodi_password" {
- type = "string"
-}
-
-variable "lychee_ip" {}
--
rgit 0.1.5