From 9b40bfd3417e112527e22835af510c14b13d99a8 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sat, 12 Jan 2019 23:03:39 +0530 Subject: [PATCH 01/22] [k8s] S01E01 Control Plane: etcd This brings up etcd using the terraform docker provider to my Digital Ocean VPN Server. The listen address is set to the VPN Address (10.8.0.1 = openvpn master server, also running on the same server). /mnt/disk is a Digital Ocean Volume attached to the instance. --- cloudflare/main.tf | 19 ++++++++++++++++ kubernetes.tf | 9 ++++++++ media/airsonic.tf | 13 ++++++++--- media/jackett.tf | 2 -- media/radarr.tf | 2 -- modules/container/main.tf | 3 ++- modules/container/vars.tf | 6 +++++ modules/etcd/main.tf | 48 +++++++++++++++++++++++++++++++++++++++ modules/etcd/variables.tf | 15 ++++++++++++ variables.tf | 1 + 10 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 kubernetes.tf create mode 100644 modules/etcd/main.tf create mode 100644 modules/etcd/variables.tf diff --git a/cloudflare/main.tf b/cloudflare/main.tf index 40e71f6..dd3fb0a 100644 --- a/cloudflare/main.tf +++ b/cloudflare/main.tf @@ -64,6 +64,25 @@ resource "cloudflare_record" "vpn_wildcard" { ttl = 3600 } +/** + * vpn.bb8.fun + * *.vpn.bb8.fun + */ +resource "cloudflare_record" "dovpn" { + domain = "${var.domain}" + name = "dovpn" + value = "${var.ips["dovpn"]}" + type = "A" +} + +resource "cloudflare_record" "dovpn_wildcard" { + domain = "${var.domain}" + name = "*.dovpn.${var.domain}" + value = "${cloudflare_record.dovpn.hostname}" + type = "CNAME" + ttl = 3600 +} + ######################## ## Mailgun Mailing Lists ######################## diff --git a/kubernetes.tf b/kubernetes.tf new file mode 100644 index 0000000..8ab7a7e --- /dev/null +++ b/kubernetes.tf @@ -0,0 +1,9 @@ +module "etcd" { + source = "modules/etcd" + host_ip = "${var.ips["dovpn"]}" + data_dir = "/mnt/xwing/etcd" + + providers = { + docker = "docker.sydney" + } +} diff --git a/media/airsonic.tf b/media/airsonic.tf index 74986bf..695e5a0 100644 --- a/media/airsonic.tf +++ b/media/airsonic.tf @@ -4,7 +4,7 @@ module "airsonic" { name = "airsonic" resource { - memory = "256" + memory = "1024" } web { @@ -13,8 +13,6 @@ module "airsonic" { expose = true } - user = "lounge:audio" - env = [ "PUID=1004", "PGID=1003", @@ -22,6 +20,11 @@ module "airsonic" { "JAVA_OPTS=-Xmx512m -Dserver.use-forward-headers=true -Dserver.context-path=/", ] + devices = [{ + host_path = "/dev/snd" + container_path = "/dev/snd" + }] + # files = [ # "/usr/lib/jvm/java-1.8-openjdk/jre/lib/airsonic.properties", # "/usr/lib/jvm/java-1.8-openjdk/jre/lib/sound.properties", @@ -50,6 +53,10 @@ module "airsonic" { host_path = "/mnt/xwing/config/airsonic/podcasts" container_path = "/podcasts" }, + { + host_path = "/mnt/xwing/config/airsonic/jre" + container_path = "/usr/lib/jvm/java-1.8-openjdk/jre/lib/" + }, ] } diff --git a/media/jackett.tf b/media/jackett.tf index e036c62..3b54030 100644 --- a/media/jackett.tf +++ b/media/jackett.tf @@ -9,8 +9,6 @@ module "jackett" { host = "jackett.${var.domain}" } - networks = ["${docker_network.media.id}", "${var.traefik-network-id}"] - volumes = [{ host_path = "/mnt/xwing/config/jackett" container_path = "/config" diff --git a/media/radarr.tf b/media/radarr.tf index d656a10..06ef78f 100644 --- a/media/radarr.tf +++ b/media/radarr.tf @@ -14,8 +14,6 @@ module "radarr" { memory_swap = 1024 } - networks = ["${docker_network.media.id}", "${var.traefik-network-id}"] - volumes = [ { host_path = "/mnt/xwing/config/radarr" diff --git a/modules/container/main.tf b/modules/container/main.tf index 2c9d59c..248bb55 100644 --- a/modules/container/main.tf +++ b/modules/container/main.tf @@ -3,7 +3,7 @@ data "docker_registry_image" "image" { } resource "docker_image" "image" { - name = "${data.docker_registry_image.image.name}" + name = "${var.image}" pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] } @@ -31,6 +31,7 @@ resource "docker_container" "container" { memory_swap = "${local.resource["memory_swap"]}" volumes = ["${var.volumes}"] + devices = ["${var.devices}"] # Look at this monstrosity # And then https://github.com/hashicorp/terraform/issues/12453#issuecomment-365569618 diff --git a/modules/container/vars.tf b/modules/container/vars.tf index 0540ab0..b77fda6 100644 --- a/modules/container/vars.tf +++ b/modules/container/vars.tf @@ -88,3 +88,9 @@ variable "volumes" { type = "list" default = [] } + +variable "devices" { + description = "volumes" + type = "list" + default = [] +} diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf new file mode 100644 index 0000000..fb22601 --- /dev/null +++ b/modules/etcd/main.tf @@ -0,0 +1,48 @@ +module "container" { + source = "../container" + image = "captn3m0/etcd:v3.3.11" + name = "etcd" + + web = { + expose = false + host = "" + } + + networks = [] + + volumes = [ + { + host_path = "/usr/share/ca-certificates/" + container_path = "/etc/ssl/certs" + }, + { + host_path = "${var.data_dir}" + container_path = "/etcd-data" + }, + ] + + ports = [ + { + internal = 2379 + external = 2379 + ip = "${var.host_ip}" + }, + { + internal = 2380 + external = 2380 + ip = "${var.host_ip}" + }, + ] + + command = [ + "/usr/local/bin/etcd", + "--data-dir=/etcd-data", + "--name=${var.node_name}", + "--advertise-client-urls=http://${var.host_ip}:2379", + "--initial-advertise-peer-urls=http://${var.host_ip}:2380", + "--initial-cluster=${var.node_name}=http://${var.host_ip}:2380", + ] + + # "--listen-client-urls=http://0.0.0.0:2379", + # "--listen-peer-urls=http://0.0.0.0:2380", +} diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf new file mode 100644 index 0000000..dbaef83 --- /dev/null +++ b/modules/etcd/variables.tf @@ -0,0 +1,15 @@ +variable "host_ip" { + description = "Host IP Address to bind etcd to" + type = "string" + default = "0.0.0.0" +} + +variable "data_dir" { + description = "Directory on host to mount to /etcd-data" + type = "string" +} + +variable "node_name" { + description = "name of the etcd node" + default = "master" +} diff --git a/variables.tf b/variables.tf index 1f50bdc..7a4dce8 100644 --- a/variables.tf +++ b/variables.tf @@ -27,6 +27,7 @@ variable "ips" { default = { eth0 = "192.168.1.111" tun0 = "10.8.0.14" + dovpn = "10.8.0.1" static = "139.59.48.222" } } From 86c0613d28d96caeb4b156ae18dfbbd3d1bec946 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 00:39:08 +0530 Subject: [PATCH 02/22] [k8s] S01E02 Control Plane: bootkube-render --- kubernetes.tf | 16 ++++++++++++++++ modules/bootkube/data.tf | 1 + modules/bootkube/main.tf | 30 ++++++++++++++++++++++++++++++ modules/bootkube/variables.tf | 29 +++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 modules/bootkube/data.tf create mode 100644 modules/bootkube/main.tf create mode 100644 modules/bootkube/variables.tf diff --git a/kubernetes.tf b/kubernetes.tf index 8ab7a7e..ad5972b 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -7,3 +7,19 @@ module "etcd" { docker = "docker.sydney" } } + +# module "kubelet" { +# source = "modules/kubelet" +# listen_ip = "${var.ips["dovpn"]}" +# } + +module "bootkube-render" { + source = "modules/bootkube" + mode = "render" + host_ip = "${var.ips["dovpn"]}" + k8s_host = "k8s.${var.root-domain}" + + providers = { + docker = "docker.sydney" + } +} diff --git a/modules/bootkube/data.tf b/modules/bootkube/data.tf new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/modules/bootkube/data.tf @@ -0,0 +1 @@ + diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf new file mode 100644 index 0000000..d4ac1a7 --- /dev/null +++ b/modules/bootkube/main.tf @@ -0,0 +1,30 @@ +resource "docker_container" "bootkube" { + image = "${docker_image.image.latest}" + name = "bootkube-render" + + volumes { + container_path = "/home/.bootkube" + volume_name = "${var.asset_dir_volume_name}" + } + + command = [ + "bootkube", + "${var.mode}", + "--asset-dir=/home/.bootkube", + "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}", + "--pod-cidr=${var.pod_cidr}", + ] + + # "--service-cidr=${var.service_cidr}", + restart = "on-failure" + max_retry_count = 5 +} + +data "docker_registry_image" "image" { + name = "captn3m0/bootkube:v${var.version}" +} + +resource "docker_image" "image" { + name = "${data.docker_registry_image.image.name}" + pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] +} diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf new file mode 100644 index 0000000..075bd5e --- /dev/null +++ b/modules/bootkube/variables.tf @@ -0,0 +1,29 @@ +// Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube + +variable "asset_dir_volume_name" { + default = "k8s-assets" +} + +variable "k8s_host" { + description = "kubenetes hostname" +} + +variable "host_port" { + default = "8443" +} + +variable "host_ip" {} + +variable "pod_cidr" { + default = "10.25.0.0/16" +} + +variable "service_cidr" { + default = "10.96.0.0/16" +} + +variable "mode" {} + +variable "version" { + default = "0.14.0" +} From 789c9c5d3fc7a53784827cca6e716e4d034eb8b2 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 01:22:04 +0530 Subject: [PATCH 03/22] [k8s] S01E03 Control Plane: bootkube-start --- kubernetes.tf | 11 +++++++++++ modules/bootkube/main.tf | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/kubernetes.tf b/kubernetes.tf index ad5972b..7abb2f1 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -23,3 +23,14 @@ module "bootkube-render" { docker = "docker.sydney" } } + +module "bootkube-start" { + source = "modules/bootkube" + mode = "start" + host_ip = "${var.ips["dovpn"]}" + k8s_host = "k8s.${var.root-domain}" + + providers = { + docker = "docker.sydney" + } +} diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index d4ac1a7..5fb147d 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -1,4 +1,5 @@ -resource "docker_container" "bootkube" { +resource "docker_container" "render" { + count = "${var.mode == "render" ? 1 : 0}" image = "${docker_image.image.latest}" name = "bootkube-render" @@ -9,13 +10,43 @@ resource "docker_container" "bootkube" { command = [ "bootkube", - "${var.mode}", + "render", "--asset-dir=/home/.bootkube", "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}", "--pod-cidr=${var.pod_cidr}", ] - # "--service-cidr=${var.service_cidr}", + network_mode = "host" + restart = "on-failure" + max_retry_count = 5 +} + +resource "docker_container" "start" { + count = "${var.mode == "start" ? 1 : 0}" + image = "${docker_image.image.latest}" + name = "bootkube-${var.mode}" + + volumes { + container_path = "/home/.bootkube" + volume_name = "${var.asset_dir_volume_name}" + read_only = true + } + + volumes { + container_path = "/etc/kubernetes/manifests" + host_path = "/etc/kubernetes/manifests" + } + + # "There is no war within the container. Here we are safe. Here we are free." + # - Docker Li agent brainwashing Nemo + command = [ + "bootkube", + "start", + "--asset-dir=/home/.bootkube", + "--pod-manifest-path=/etc/kubernetes/manifests", + ] + + network_mode = "host" restart = "on-failure" max_retry_count = 5 } From 97ef9179e4d5cfac9369c55e5f532a1722bac10f Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 01:40:20 +0530 Subject: [PATCH 04/22] [k8s] Make bootkube start depend on bootkube render --- kubernetes.tf | 9 +++++---- modules/bootkube/outputs.tf | 13 +++++++++++++ modules/bootkube/variables.tf | 6 ++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 modules/bootkube/outputs.tf diff --git a/kubernetes.tf b/kubernetes.tf index 7abb2f1..47e3220 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -25,10 +25,11 @@ module "bootkube-render" { } module "bootkube-start" { - source = "modules/bootkube" - mode = "start" - host_ip = "${var.ips["dovpn"]}" - k8s_host = "k8s.${var.root-domain}" + depends_on = "${module.bootkube-render.image}" + source = "modules/bootkube" + mode = "start" + host_ip = "${var.ips["dovpn"]}" + k8s_host = "k8s.${var.root-domain}" providers = { docker = "docker.sydney" diff --git a/modules/bootkube/outputs.tf b/modules/bootkube/outputs.tf new file mode 100644 index 0000000..29077f3 --- /dev/null +++ b/modules/bootkube/outputs.tf @@ -0,0 +1,13 @@ +# output "exit_code" { +# # TODO: Pick correct exit code +# # value = "${coalesce(formatlist("%s", docker_container.render.*.exit_code))}" +# # See https://github.com/hashicorp/terraform/issues/15165 +# value = "${var.mode == "render" ? +# "${element(concat(docker_container.render.*.exit_code, list("")), 0)}" : +# "${element(concat(docker_container.start.*.exit_code, list("")), 0)}" +# }" +# } + +output "image" { + value = "${docker_image.image.latest}" +} diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index 075bd5e..6098aa6 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -27,3 +27,9 @@ variable "mode" {} variable "version" { default = "0.14.0" } + +variable "depends_on" { + default = [] + + type = "list" +} From 7214355a89d4eadfd813579a53e6f119294b532e Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 04:01:14 +0530 Subject: [PATCH 05/22] [k8s] Adds kubelet, start stitching things together Challenges: 1. etcd booting before bootkube meant I missed certs 2. etcd can run without certs, but managing docker network over static pod manifests might be tricky :fingers_crossed: --- kubernetes.tf | 18 ++++-- modules/bootkube/main.tf | 13 ++-- modules/bootkube/variables.tf | 8 +-- modules/etcd/main.tf | 32 ++++------ modules/etcd/variables.tf | 12 ++++ modules/kubelet/main.tf | 116 ++++++++++++++++++++++++++++++++++ modules/kubelet/variables.tf | 19 ++++++ 7 files changed, 182 insertions(+), 36 deletions(-) create mode 100644 modules/kubelet/main.tf create mode 100644 modules/kubelet/variables.tf diff --git a/kubernetes.tf b/kubernetes.tf index 47e3220..ed5d3f8 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -3,16 +3,24 @@ module "etcd" { host_ip = "${var.ips["dovpn"]}" data_dir = "/mnt/xwing/etcd" + bootkube_asset_dir = "/etc/kube-assets" + + providers = { + docker = "docker.sydney" + } + + depends_on = "${module.bootkube-start.image}" +} + +module "kubelet-master" { + source = "modules/kubelet" + depends_on = "${module.bootkube-start.image}" + providers = { docker = "docker.sydney" } } -# module "kubelet" { -# source = "modules/kubelet" -# listen_ip = "${var.ips["dovpn"]}" -# } - module "bootkube-render" { source = "modules/bootkube" mode = "render" diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 5fb147d..cb9d95f 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -5,15 +5,17 @@ resource "docker_container" "render" { volumes { container_path = "/home/.bootkube" - volume_name = "${var.asset_dir_volume_name}" + volume_name = "/etc/kube-assets" } command = [ "bootkube", "render", + "--etcd-servers=http://${host_ip}:2379", "--asset-dir=/home/.bootkube", - "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}", + "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", "--pod-cidr=${var.pod_cidr}", + "--network-provider=${var.network_provider}", ] network_mode = "host" @@ -28,13 +30,13 @@ resource "docker_container" "start" { volumes { container_path = "/home/.bootkube" - volume_name = "${var.asset_dir_volume_name}" + volume_name = "/etc/kube-assets" read_only = true } volumes { - container_path = "/etc/kubernetes/manifests" - host_path = "/etc/kubernetes/manifests" + container_path = "/etc/kubernetes" + host_path = "/etc/kubernetes" } # "There is no war within the container. Here we are safe. Here we are free." @@ -43,7 +45,6 @@ resource "docker_container" "start" { "bootkube", "start", "--asset-dir=/home/.bootkube", - "--pod-manifest-path=/etc/kubernetes/manifests", ] network_mode = "host" diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index 6098aa6..cf04247 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -1,9 +1,5 @@ // Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube -variable "asset_dir_volume_name" { - default = "k8s-assets" -} - variable "k8s_host" { description = "kubenetes hostname" } @@ -12,6 +8,10 @@ variable "host_port" { default = "8443" } +variable "network_provider" { + default = "flannel" +} + variable "host_ip" {} variable "pod_cidr" { diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf index fb22601..26aa193 100644 --- a/modules/etcd/main.tf +++ b/modules/etcd/main.tf @@ -8,32 +8,15 @@ module "container" { host = "" } - networks = [] + networks = ["${docker_network.etcd.id}"] volumes = [ - { - host_path = "/usr/share/ca-certificates/" - container_path = "/etc/ssl/certs" - }, { host_path = "${var.data_dir}" container_path = "/etcd-data" }, ] - ports = [ - { - internal = 2379 - external = 2379 - ip = "${var.host_ip}" - }, - { - internal = 2380 - external = 2380 - ip = "${var.host_ip}" - }, - ] - command = [ "/usr/local/bin/etcd", "--data-dir=/etcd-data", @@ -42,7 +25,14 @@ module "container" { "--initial-advertise-peer-urls=http://${var.host_ip}:2380", "--initial-cluster=${var.node_name}=http://${var.host_ip}:2380", ] - - # "--listen-client-urls=http://0.0.0.0:2379", - # "--listen-peer-urls=http://0.0.0.0:2380", +} + +resource "docker_network" "etcd" { + name = "etcd" + driver = "bridge" + + ipam_config { + subnet = "10.10.10.0/25" + gateway = "10.10.10.1" + } } diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf index dbaef83..6babce5 100644 --- a/modules/etcd/variables.tf +++ b/modules/etcd/variables.tf @@ -9,7 +9,19 @@ variable "data_dir" { type = "string" } +variable "bootkube_asset_dir" { + description = "bootkube render is run against this directory" + type = "string" + default = "/etc/kube-assets" +} + variable "node_name" { description = "name of the etcd node" default = "master" } + +variable "depends_on" { + default = [] + + type = "list" +} diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf new file mode 100644 index 0000000..2ca7012 --- /dev/null +++ b/modules/kubelet/main.tf @@ -0,0 +1,116 @@ +// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper +resource "docker_container" "kubelet" { + image = "${docker_image.image.latest}" + name = "kubelet-static" + + volumes { + container_path = "/etc/kubernetes" + host_path = "/etc/kubernetes" + } + + volumes { + container_path = "/etc/kubernetes/kubeconfig" + host_path = "/etc/kube-assets/auth/kubeconfig-kubelet" + } + + volumes { + container_path = "/etc/kubernetes/kubeconfig-admin" + host_path = "/etc/kube-assets/auth/kubeconfig" + } + + volumes { + container_path = "/etc/kubernetes/ca.crt" + host_path = "/etc/kube-assets/tls/ca.crt" + } + + volumes { + container_path = "/etc/ssl/certs" + host_path = "/etc/ssl/certs" + read_only = true + } + + volumes { + container_path = "/usr/share/ca-certificates" + host_path = "/usr/share/ca-certificates" + read_only = true + } + + volumes { + container_path = "/var/lib/docker" + host_path = "/var/lib/docker" + } + + volumes { + container_path = "/var/lib/kubelet" + host_path = "/var/lib/kubelet" + } + + volumes { + container_path = "/var/log" + host_path = "/var/log" + } + + volumes { + container_path = "/run" + host_path = "/run" + } + + volumes { + container_path = "/lib/modules" + host_path = "/lib/modules" + read_only = true + } + + volumes { + container_path = "/etc/os-release" + host_path = "/usr/lib/os-release" + read_only = true + } + + volumes { + container_path = "/etc/machine-id" + host_path = "/etc/machine-id" + read_only = true + } + + // Deviates from kubelet-wrapper + + volumes { + container_path = "/var/lib/cni" + host_path = "/var/lib/cni" + } + command = [ + "kubelet", + "--kubeconfig=/etc/kubernetes/kubeconfig", + "--client-ca-file=/etc/kubernetes/ca.crt", + "--anonymous-auth=false", + "--cni-conf-dir=/etc/kubernetes/cni/net.d", + "--network-plugin=cni", + "--lock-file=/var/run/lock/kubelet.lock", + "--exit-on-lock-contention", + "--pod-manifest-path=/etc/kubernetes/manifests", + "--allow-privileged", + "--minimum-container-ttl-duration=10m0s", + "--cluster_dns=10.25.0.10", + "--cluster_domain=k8s.bb8.fun", + ] + + # TODO + # "--register-with-taints=${var.node_taints}", + # "--node-labels=${var.node_label}", + + network_mode = "host" + privileged = true + restart = "no" + must_run = false + max_retry_count = 1 +} + +data "docker_registry_image" "image" { + name = "gcr.io/google_containers/hyperkube:v${var.version}" +} + +resource "docker_image" "image" { + name = "${data.docker_registry_image.image.name}" + pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] +} diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf new file mode 100644 index 0000000..b754c86 --- /dev/null +++ b/modules/kubelet/variables.tf @@ -0,0 +1,19 @@ +variable "version" { + description = "kubelet version" + default = "1.13.2" +} + +variable "node_label" { + description = "kubelet version" + default = "node.kubernetes.io/master" +} + +variable "depends_on" { + default = [] + + type = "list" +} + +variable "asset_dir_volume_name" { + default = "k8s-assets" +} From 524949c8e5ef1641ac73e87157888dfa674f8004 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 04:15:12 +0530 Subject: [PATCH 06/22] [k8s] Run etcd with bootkube generated certs --- modules/etcd/main.tf | 56 ++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf index 26aa193..af677c2 100644 --- a/modules/etcd/main.tf +++ b/modules/etcd/main.tf @@ -8,31 +8,57 @@ module "container" { host = "" } - networks = ["${docker_network.etcd.id}"] - volumes = [ { host_path = "${var.data_dir}" container_path = "/etcd-data" }, + { + host_path = "${var.bootkube_asset_dir}/tls/etcd-client.crt" + container_path = "/etc/etcd-client.crt" + }, + { + host_path = "${var.bootkube_asset_dir}/tls/etcd-client.key" + container_path = "/etc/etcd-client.key" + }, + { + host_path = "${var.bootkube_asset_dir}/tls/etcd-client-ca.crt" + container_path = "/etc/etcd-client-ca.crt" + }, + { + host_path = "${var.bootkube_asset_dir}/tls/etcd" + container_path = "/etc/ssl/certs/etcd" + }, + ] + + ports = [ + { + internal = 2379 + external = 2379 + ip = "${var.host_ip}" + }, + { + internal = 2380 + external = 2380 + ip = "${var.host_ip}" + }, ] command = [ "/usr/local/bin/etcd", "--data-dir=/etcd-data", "--name=${var.node_name}", - "--advertise-client-urls=http://${var.host_ip}:2379", - "--initial-advertise-peer-urls=http://${var.host_ip}:2380", - "--initial-cluster=${var.node_name}=http://${var.host_ip}:2380", + "--advertise-client-urls=https://${var.host_ip}:2379", + "--initial-advertise-peer-urls=https://${var.host_ip}:2380", + "--initial-cluster=${var.node_name}=https://${var.host_ip}:2380", + "--listen-client-urls=https://0.0.0.0:2379", + "--listen-peer-urls=https://0.0.0.0:2380", + "--trusted-ca-file=/etc/ssl/certs/etcd/server-ca.crt", + "--cert-file=/etc/ssl/certs/etcd/server.crt", + "--key-file=/etc/ssl/certs/etcd/server.key", + "--client-cert-auth=true", + "--peer-trusted-ca-file=/etc/ssl/certs/etcd/peer-ca.crt", + "--peer-cert-file=/etc/ssl/certs/etcd/peer.crt", + "--peer-key-file=/etc/ssl/certs/etcd/peer.key", ] } - -resource "docker_network" "etcd" { - name = "etcd" - driver = "bridge" - - ipam_config { - subnet = "10.10.10.0/25" - gateway = "10.10.10.1" - } -} From 6deddebe48c6506de85a92a5cbebdb0f548c0257 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 05:24:50 +0530 Subject: [PATCH 07/22] Resolve DNS issues by bypassing DNS for now --- kubernetes.tf | 5 +++- main.tf | 11 ++++----- media/airsonic.tf | 17 ++++++++------ media/variables.tf | 2 +- modules/bootkube/main.tf | 2 +- modules/kubelet/main.tf | 44 ++++++++++++++++++++++++------------ modules/kubelet/variables.tf | 6 +++++ 7 files changed, 57 insertions(+), 30 deletions(-) diff --git a/kubernetes.tf b/kubernetes.tf index ed5d3f8..0011d03 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -13,7 +13,10 @@ module "etcd" { } module "kubelet-master" { - source = "modules/kubelet" + source = "modules/kubelet" + host_ip = "${var.ips["dovpn"]}" + k8s_host = "k8s.${var.root-domain}" + depends_on = "${module.bootkube-start.image}" providers = { diff --git a/main.tf b/main.tf index 1b3d938..1329e2f 100644 --- a/main.tf +++ b/main.tf @@ -67,12 +67,11 @@ module "resilio" { } module "media" { - source = "media" - domain = "bb8.fun" - traefik-labels = "${var.traefik-common-labels}" - airsonic-smtp-password = "${var.airsonic-smtp-password}" - ips = "${var.ips}" - traefik-network-id = "${module.docker.traefik-network-id}" + source = "media" + domain = "bb8.fun" + traefik-labels = "${var.traefik-common-labels}" + ips = "${var.ips}" + traefik-network-id = "${module.docker.traefik-network-id}" } module "monitoring" { diff --git a/media/airsonic.tf b/media/airsonic.tf index 695e5a0..8de3640 100644 --- a/media/airsonic.tf +++ b/media/airsonic.tf @@ -60,12 +60,15 @@ module "airsonic" { ] } -data "template_file" "airsonic-properties-file" { - template = "${file("${path.module}/conf/airsonic.properties.tpl")}" +# data "template_file" "airsonic-properties-file" { +# template = "${file("${path.module}/conf/airsonic.properties.tpl")}" - vars { - smtp-password = "${var.airsonic-smtp-password}" - # db-password = "${var.airsonic-db-password}" - } -} +# vars { +# smtp-password = "${var.airsonic-smtp-password}" + + +# # db-password = "${var.airsonic-db-password}" +# } +# } + diff --git a/media/variables.tf b/media/variables.tf index 0fdd605..1d66838 100644 --- a/media/variables.tf +++ b/media/variables.tf @@ -2,7 +2,7 @@ variable "domain" { type = "string" } -variable "airsonic-smtp-password" {} +# variable "airsonic-smtp-password" {} variable "traefik-labels" { type = "map" diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index cb9d95f..403796f 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -11,7 +11,7 @@ resource "docker_container" "render" { command = [ "bootkube", "render", - "--etcd-servers=http://${host_ip}:2379", + "--etcd-servers=http://${var.host_ip}:2379", "--asset-dir=/home/.bootkube", "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", "--pod-cidr=${var.pod_cidr}", diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index 2ca7012..76d918c 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -81,29 +81,45 @@ resource "docker_container" "kubelet" { } command = [ "kubelet", - "--kubeconfig=/etc/kubernetes/kubeconfig", - "--client-ca-file=/etc/kubernetes/ca.crt", - "--anonymous-auth=false", - "--cni-conf-dir=/etc/kubernetes/cni/net.d", - "--network-plugin=cni", - "--lock-file=/var/run/lock/kubelet.lock", - "--exit-on-lock-contention", - "--pod-manifest-path=/etc/kubernetes/manifests", "--allow-privileged", + "--anonymous-auth=false", + "--cert-dir=/var/lib/kubelet/pki", + "--client-ca-file=/etc/kubernetes/ca.crt", + "--cni-conf-dir=/etc/kubernetes/cni/net.d", + "--exit-on-lock-contention=true", + "--hostname-override=${var.host_ip}", + "--kubeconfig=/etc/kubernetes/kubeconfig", + "--lock-file=/var/run/lock/kubelet.lock", "--minimum-container-ttl-duration=10m0s", + "--network-plugin=cni", + "--node-labels=node-role.kubernetes.io/master", + "--pod-manifest-path=/etc/kubernetes/manifests", + "--rotate-certificates", + + // TODO: Change to var "--cluster_dns=10.25.0.10", - "--cluster_domain=k8s.bb8.fun", + + "--cluster_domain=${var.k8s_host}", ] + host { + host = "kubernetes.default" + ip = "${var.host_ip}" + } + host { + host = "${var.k8s_host}" + ip = "${var.host_ip}" + } # TODO # "--register-with-taints=${var.node_taints}", # "--node-labels=${var.node_label}", - network_mode = "host" - privileged = true - restart = "no" - must_run = false - max_retry_count = 1 + network_mode = "host" + privileged = true + restart = "no" + must_run = false + + # max_retry_count = 1 } data "docker_registry_image" "image" { diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf index b754c86..ca3025d 100644 --- a/modules/kubelet/variables.tf +++ b/modules/kubelet/variables.tf @@ -17,3 +17,9 @@ variable "depends_on" { variable "asset_dir_volume_name" { default = "k8s-assets" } + +variable "host_ip" {} + +variable "k8s_host" { + description = "kubenetes hostname" +} From 2a12c17948fb749e75e2da1b2c023219127d8a28 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 12:55:46 +0530 Subject: [PATCH 08/22] Switch to official images --- modules/bootkube/main.tf | 4 ++-- modules/kubelet/main.tf | 9 ++++----- modules/kubelet/variables.tf | 4 ++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 403796f..bd4a860 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -11,7 +11,7 @@ resource "docker_container" "render" { command = [ "bootkube", "render", - "--etcd-servers=http://${var.host_ip}:2379", + "--etcd-servers=https://${var.host_ip}:2379", "--asset-dir=/home/.bootkube", "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", "--pod-cidr=${var.pod_cidr}", @@ -53,7 +53,7 @@ resource "docker_container" "start" { } data "docker_registry_image" "image" { - name = "captn3m0/bootkube:v${var.version}" + name = "quay.io/coreos/bootkube:v${var.version}" } resource "docker_image" "image" { diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index 76d918c..130b0b8 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -83,9 +83,11 @@ resource "docker_container" "kubelet" { "kubelet", "--allow-privileged", "--anonymous-auth=false", + "--authentication-token-webhook", + "--authorization-mode=Webhook", "--cert-dir=/var/lib/kubelet/pki", "--client-ca-file=/etc/kubernetes/ca.crt", - "--cni-conf-dir=/etc/kubernetes/cni/net.d", + "--cluster_dns=${var.dns_ip}", "--exit-on-lock-contention=true", "--hostname-override=${var.host_ip}", "--kubeconfig=/etc/kubernetes/kubeconfig", @@ -94,11 +96,8 @@ resource "docker_container" "kubelet" { "--network-plugin=cni", "--node-labels=node-role.kubernetes.io/master", "--pod-manifest-path=/etc/kubernetes/manifests", + "--read-only-port=0", "--rotate-certificates", - - // TODO: Change to var - "--cluster_dns=10.25.0.10", - "--cluster_domain=${var.k8s_host}", ] host { diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf index ca3025d..d68cf21 100644 --- a/modules/kubelet/variables.tf +++ b/modules/kubelet/variables.tf @@ -20,6 +20,10 @@ variable "asset_dir_volume_name" { variable "host_ip" {} +variable "dns_ip" { + default = "10.25.0.10" +} + variable "k8s_host" { description = "kubenetes hostname" } From 6f6c4f974edb57f486f6dfd9d6ba0444af8ddb8b Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 14:14:19 +0530 Subject: [PATCH 09/22] [k8s] Run kubelet using --containerized (or maybe not) --- modules/bootkube/main.tf | 6 +++--- modules/kubelet/main.tf | 16 +++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index bd4a860..00b019c 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -9,11 +9,11 @@ resource "docker_container" "render" { } command = [ - "bootkube", + "/bootkube", "render", "--etcd-servers=https://${var.host_ip}:2379", "--asset-dir=/home/.bootkube", - "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", + "--api-servers=https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", "--pod-cidr=${var.pod_cidr}", "--network-provider=${var.network_provider}", ] @@ -42,7 +42,7 @@ resource "docker_container" "start" { # "There is no war within the container. Here we are safe. Here we are free." # - Docker Li agent brainwashing Nemo command = [ - "bootkube", + "/bootkube", "start", "--asset-dir=/home/.bootkube", ] diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index 130b0b8..ea2eed8 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -73,6 +73,13 @@ resource "docker_container" "kubelet" { read_only = true } + volumes { + container_path = "/rootfs" + host_path = "/" + read_only = true + read_only = true + } + // Deviates from kubelet-wrapper volumes { @@ -88,7 +95,11 @@ resource "docker_container" "kubelet" { "--cert-dir=/var/lib/kubelet/pki", "--client-ca-file=/etc/kubernetes/ca.crt", "--cluster_dns=${var.dns_ip}", + "--cluster_domain=${var.k8s_host}", + + # "--containerized", "--exit-on-lock-contention=true", + "--hostname-override=${var.host_ip}", "--kubeconfig=/etc/kubernetes/kubeconfig", "--lock-file=/var/run/lock/kubelet.lock", @@ -98,12 +109,7 @@ resource "docker_container" "kubelet" { "--pod-manifest-path=/etc/kubernetes/manifests", "--read-only-port=0", "--rotate-certificates", - "--cluster_domain=${var.k8s_host}", ] - host { - host = "kubernetes.default" - ip = "${var.host_ip}" - } host { host = "${var.k8s_host}" ip = "${var.host_ip}" From 5949a9448ada116bea955559640a0bbb9fd95761 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 17:32:01 +0530 Subject: [PATCH 10/22] [k8s] Use terraform-render-bootkube to bring up etcd - Stop running render on the server --- cloudflare/main.tf | 7 +++ kubernetes.tf | 42 ++++++++----- modules/etcd/main.tf | 123 +++++++++++++++++++++----------------- modules/etcd/variables.tf | 26 ++++---- 4 files changed, 118 insertions(+), 80 deletions(-) diff --git a/cloudflare/main.tf b/cloudflare/main.tf index dd3fb0a..32f3969 100644 --- a/cloudflare/main.tf +++ b/cloudflare/main.tf @@ -83,6 +83,13 @@ resource "cloudflare_record" "dovpn_wildcard" { ttl = 3600 } +resource "cloudflare_record" "etcd" { + domain = "${var.domain}" + name = "etcd" + value = "${var.ips["dovpn"]}" + type = "A" +} + ######################## ## Mailgun Mailing Lists ######################## diff --git a/kubernetes.tf b/kubernetes.tf index 0011d03..59e375b 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -1,15 +1,27 @@ module "etcd" { - source = "modules/etcd" - host_ip = "${var.ips["dovpn"]}" - data_dir = "/mnt/xwing/etcd" + source = "modules/etcd" + data_dir = "/mnt/disk/etcd" + host_bind_ip = "10.8.0.1" + domain = "etcd.bb8.fun" - bootkube_asset_dir = "/etc/kube-assets" + pki = { + /** + * client_cert = "${module.bootkube.etcd_client_cert}" + * client_key = "${module.bootkube.etcd_client_key}" + */ + + ca_cert = "${module.bootkube.etcd_ca_cert}" + server_cert = "${module.bootkube.etcd_server_cert}" + server_key = "${module.bootkube.etcd_server_key}" + peer_cert = "${module.bootkube.etcd_peer_cert}" + peer_key = "${module.bootkube.etcd_peer_key}" + } providers = { docker = "docker.sydney" } - depends_on = "${module.bootkube-start.image}" + depends_on = "${module.bootkube.id}" } module "kubelet-master" { @@ -24,9 +36,9 @@ module "kubelet-master" { } } -module "bootkube-render" { +module "bootkube-start" { source = "modules/bootkube" - mode = "render" + mode = "start" host_ip = "${var.ips["dovpn"]}" k8s_host = "k8s.${var.root-domain}" @@ -35,14 +47,12 @@ module "bootkube-render" { } } -module "bootkube-start" { - depends_on = "${module.bootkube-render.image}" - source = "modules/bootkube" - mode = "start" - host_ip = "${var.ips["dovpn"]}" - k8s_host = "k8s.${var.root-domain}" +module "bootkube" { + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcbdddd8d07c99ab88b2e9ebfb662de4c104de0a" - providers = { - docker = "docker.sydney" - } + cluster_name = "k8s.bb8.fun" + api_servers = ["10.8.0.1", "k8s.bb8.fun"] + cluster_domain_suffix = "k8s.bb8.fun" + etcd_servers = ["etcd.bb8.fun"] + asset_dir = "./k8s" } diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf index af677c2..595215a 100644 --- a/modules/etcd/main.tf +++ b/modules/etcd/main.tf @@ -1,64 +1,79 @@ -module "container" { - source = "../container" - image = "captn3m0/etcd:v3.3.11" - name = "etcd" +resource "docker_container" "etcd" { + name = "etcd" + image = "${docker_image.image.latest}" - web = { - expose = false - host = "" + volumes { + host_path = "${var.data_dir}" + container_path = "/etcd-data" } - volumes = [ - { - host_path = "${var.data_dir}" - container_path = "/etcd-data" - }, - { - host_path = "${var.bootkube_asset_dir}/tls/etcd-client.crt" - container_path = "/etc/etcd-client.crt" - }, - { - host_path = "${var.bootkube_asset_dir}/tls/etcd-client.key" - container_path = "/etc/etcd-client.key" - }, - { - host_path = "${var.bootkube_asset_dir}/tls/etcd-client-ca.crt" - container_path = "/etc/etcd-client-ca.crt" - }, - { - host_path = "${var.bootkube_asset_dir}/tls/etcd" - container_path = "/etc/ssl/certs/etcd" - }, - ] + ports { + internal = 2379 + external = 2379 + ip = "${var.host_bind_ip}" + } - ports = [ - { - internal = 2379 - external = 2379 - ip = "${var.host_ip}" - }, - { - internal = 2380 - external = 2380 - ip = "${var.host_ip}" - }, + ports { + internal = 2380 + external = 2380 + ip = "${var.host_bind_ip}" + } + + upload { + content = "${var.pki["ca_cert"]}" + file = "/etc/ssl/ca_cert.pem" + } + + upload { + content = "${var.pki["server_cert"]}" + file = "/etc/ssl/server_cert.pem" + } + + upload { + content = "${var.pki["server_key"]}" + file = "/etc/ssl/server_key.pem" + } + + upload { + content = "${var.pki["peer_cert"]}" + file = "/etc/ssl/peer_cert.pem" + } + + upload { + content = "${var.pki["peer_key"]}" + file = "/etc/ssl/peer_key.pem" + } + + env = [ + "ETCD_NAME=${var.node_name}", + "ETCD_DATA_DIR=/var/lib/etcd", + "ETCD_ADVERTISE_CLIENT_URLS=https://${var.domain}:2379", + "ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${var.domain}:2380", + "ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379", + "ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380", + "ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381", + "ETCD_CLIENT_CERT_AUTH=true", + "ETCD_INITIAL_CLUSTER=${var.node_name}=https://${var.domain}:2380", + "ETCD_STRICT_RECONFIG_CHECK=true", + "ETCD_CERT_FILE=/etc/ssl/server_cert.pem", + "ETCD_KEY_FILE=/etc/ssl/server_key.pem", + "ETCD_TRUSTED_CA_FILE=/etc/ssl/ca_cert.pem", + "ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/ca_cert.pem", + "ETCD_PEER_CERT_FILE=/etc/ssl/peer_cert.pem", + "ETCD_PEER_KEY_FILE=/etc/ssl/peer_key.pem", + "ETCD_PEER_CLIENT_CERT_AUTH=true", ] command = [ "/usr/local/bin/etcd", - "--data-dir=/etcd-data", - "--name=${var.node_name}", - "--advertise-client-urls=https://${var.host_ip}:2379", - "--initial-advertise-peer-urls=https://${var.host_ip}:2380", - "--initial-cluster=${var.node_name}=https://${var.host_ip}:2380", - "--listen-client-urls=https://0.0.0.0:2379", - "--listen-peer-urls=https://0.0.0.0:2380", - "--trusted-ca-file=/etc/ssl/certs/etcd/server-ca.crt", - "--cert-file=/etc/ssl/certs/etcd/server.crt", - "--key-file=/etc/ssl/certs/etcd/server.key", - "--client-cert-auth=true", - "--peer-trusted-ca-file=/etc/ssl/certs/etcd/peer-ca.crt", - "--peer-cert-file=/etc/ssl/certs/etcd/peer.crt", - "--peer-key-file=/etc/ssl/certs/etcd/peer.key", ] } + +data "docker_registry_image" "image" { + name = "quay.io/coreos/etcd:v${var.version}" +} + +resource "docker_image" "image" { + name = "${data.docker_registry_image.image.name}" + pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] +} diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf index 6babce5..d47db7e 100644 --- a/modules/etcd/variables.tf +++ b/modules/etcd/variables.tf @@ -1,7 +1,6 @@ -variable "host_ip" { - description = "Host IP Address to bind etcd to" +variable "domain" { + description = "Host name to advertise" type = "string" - default = "0.0.0.0" } variable "data_dir" { @@ -9,15 +8,9 @@ variable "data_dir" { type = "string" } -variable "bootkube_asset_dir" { - description = "bootkube render is run against this directory" - type = "string" - default = "/etc/kube-assets" -} - variable "node_name" { description = "name of the etcd node" - default = "master" + default = "controller" } variable "depends_on" { @@ -25,3 +18,16 @@ variable "depends_on" { type = "list" } + +variable "pki" { + type = "map" +} + +variable "version" { + description = "etcd version" + default = "3.3.11" +} + +variable "host_bind_ip" { + description = "IP address to expose the ports on host" +} From 94f9a23b4f61ba871d32ffeeb9d35ef49e85259f Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 14 Jan 2019 15:31:06 +0530 Subject: [PATCH 11/22] Remove unused variables --- kubernetes.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kubernetes.tf b/kubernetes.tf index 59e375b..9792867 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -5,11 +5,6 @@ module "etcd" { domain = "etcd.bb8.fun" pki = { - /** - * client_cert = "${module.bootkube.etcd_client_cert}" - * client_key = "${module.bootkube.etcd_client_key}" - */ - ca_cert = "${module.bootkube.etcd_ca_cert}" server_cert = "${module.bootkube.etcd_server_cert}" server_key = "${module.bootkube.etcd_server_key}" From a3dec142add5ec62e1599182aeda85b3fae10d3c Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 27 Jan 2019 04:02:59 +0530 Subject: [PATCH 12/22] [k8s] Upload all assets using upload{} inside docker_container --- kubernetes.tf | 27 ++++- modules/bootkube/data.tf | 1 - modules/bootkube/main.tf | 213 ++++++++++++++++++++++++++++------ modules/bootkube/variables.tf | 6 + modules/kubelet/main.tf | 33 +++--- modules/kubelet/variables.tf | 4 + providers.tf | 2 +- 7 files changed, 230 insertions(+), 56 deletions(-) delete mode 100644 modules/bootkube/data.tf diff --git a/kubernetes.tf b/kubernetes.tf index 9792867..c253da0 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -24,6 +24,13 @@ module "kubelet-master" { host_ip = "${var.ips["dovpn"]}" k8s_host = "k8s.${var.root-domain}" + assets = { + kubeconfig = "${module.bootkube.kubeconfig-kubelet}" + ca_cert = "${base64decode(module.bootkube.ca_cert)}" + kubelet_cert = "${base64decode(module.bootkube.kubelet_cert)}" + kubelet_key = "${base64decode(module.bootkube.kubelet_key)}" + } + depends_on = "${module.bootkube-start.image}" providers = { @@ -32,10 +39,22 @@ module "kubelet-master" { } module "bootkube-start" { - source = "modules/bootkube" - mode = "start" - host_ip = "${var.ips["dovpn"]}" - k8s_host = "k8s.${var.root-domain}" + source = "modules/bootkube" + mode = "start" + host_ip = "${var.ips["dovpn"]}" + k8s_host = "k8s.${var.root-domain}" + asset-dir = "${path.root}/k8s" + + assets = { + kubeconfig-kubelet = "${module.bootkube.kubeconfig-kubelet}" + etcd_ca_cert = "${module.bootkube.etcd_ca_cert}" + etcd_client_cert = "${module.bootkube.etcd_client_cert}" + etcd_client_key = "${module.bootkube.etcd_client_key}" + etcd_server_cert = "${module.bootkube.etcd_server_cert}" + etcd_server_key = "${module.bootkube.etcd_server_key}" + etcd_peer_cert = "${module.bootkube.etcd_peer_cert}" + etcd_peer_key = "${module.bootkube.etcd_peer_key}" + } providers = { docker = "docker.sydney" diff --git a/modules/bootkube/data.tf b/modules/bootkube/data.tf deleted file mode 100644 index 8b13789..0000000 --- a/modules/bootkube/data.tf +++ /dev/null @@ -1 +0,0 @@ - diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 00b019c..6c0d6b8 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -1,52 +1,193 @@ -resource "docker_container" "render" { - count = "${var.mode == "render" ? 1 : 0}" +resource "docker_container" "bootkube" { image = "${docker_image.image.latest}" - name = "bootkube-render" + name = "bootkube" volumes { - container_path = "/home/.bootkube" - volume_name = "/etc/kube-assets" + container_path = "/etc/kubernetes/manifests" + host_path = "/etc/kubernetes/manifests" } - command = [ - "/bootkube", - "render", - "--etcd-servers=https://${var.host_ip}:2379", - "--asset-dir=/home/.bootkube", - "--api-servers=https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", - "--pod-cidr=${var.pod_cidr}", - "--network-provider=${var.network_provider}", - ] + # bootstrap manifests - network_mode = "host" - restart = "on-failure" - max_retry_count = 5 -} - -resource "docker_container" "start" { - count = "${var.mode == "start" ? 1 : 0}" - image = "${docker_image.image.latest}" - name = "bootkube-${var.mode}" - - volumes { - container_path = "/home/.bootkube" - volume_name = "/etc/kube-assets" - read_only = true + upload { + content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-apiserver.yaml")}" + file = "/home/.bootkube/bootstra-manifests/bootstrap-apiserver.yaml" } - - volumes { - container_path = "/etc/kubernetes" - host_path = "/etc/kubernetes" + upload { + content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-controller-manager.yaml")}" + file = "/home/.bootkube/bootstra-manifests/bootstrap-controller-manager.yaml" + } + upload { + content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-scheduler.yaml")}" + file = "/home/.bootkube/bootstra-manifests/bootstrap-scheduler.yaml" + } + # Cluster Networking + upload { + content = "${file("${var.asset-dir}/manifests-networking/cluster-role-binding.yaml")}" + file = "/home/.bootkube/manifests-networking/cluster-role-binding.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networking/cluster-role.yaml")}" + file = "/home/.bootkube/manifests-networking/cluster-role.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networking/config.yaml")}" + file = "/home/.bootkube/manifests-networking/config.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networking/daemonset.yaml")}" + file = "/home/.bootkube/manifests-networking/daemonset.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networkingservice-account.yaml")}" + file = "/home/.bootkube/manifests-networking/service-account.yaml" + } + # TLS + upload { + file = "/home/.bootkube/tls/service-account.pub" + content = "${file("${var.asset-dir}/tls/service-account.pub")}" + } + upload { + content = "${file("${var.asset-dir}/tls/ca.key")}" + file = "/home/.bootkube/tls/ca.key" + } + upload { + content = "${file("${var.asset-dir}/tls/ca.crt")}" + file = "/home/.bootkube/tls/ca.crt" + } + upload { + content = "${file("${var.asset-dir}/tls/apiserver.key")}" + file = "/home/.bootkube/tls/apiserver.key" + } + upload { + content = "${file("${var.asset-dir}/tls/apiserver.crt")}" + file = "/home/.bootkube/tls/apiserver.crt" + } + upload { + content = "${var.assets["kubelet_cert"]}" + file = "/home/.bootkube/tls/kubelet.crt" + } + upload { + content = "${var.assets["kubelet_key"]}" + file = "/home/.bootkube/tls/kubelet.key" + } + # TODO: Generate Filenames Dynamically + # TODO: Check if this is needed at all + upload { + content = "${file("${var.asset-dir}/auth/k8s.bb8.fun-config")}" + file = "/home/.bootkube/auth/k8s.bb8.fun-config" + } + # auth/kubeconfig-kubelet + upload { + content = "${var.assets["kubeconfig-kubelet"]}" + file = "/home/.bootkube/auth/kubeconfig-kubelet" + } + # Manifests Directory + upload { + file = "/home/.bootkube/manifests/kube-apiserver-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-apiserver-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-apiserver-secret.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver-secret.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-apiserver.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kubeconfig-in-cluster.yaml" + content = "${file("${var.asset-dir}/manifests/kubeconfig-in-cluster.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-disruption.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-disruption.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-secret.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-secret.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kubelet-nodes-cluster-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kubelet-nodes-cluster-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-proxy-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-proxy-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-proxy-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-proxy-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-proxy.yaml" + content = "${file("${var.asset-dir}/manifests/kube-proxy.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-disruption.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-disruption.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-volume-scheduler-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-volume-scheduler-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-role.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-sa.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer.yaml")}" } - - # "There is no war within the container. Here we are safe. Here we are free." - # - Docker Li agent brainwashing Nemo command = [ "/bootkube", "start", "--asset-dir=/home/.bootkube", ] - network_mode = "host" restart = "on-failure" max_retry_count = 5 diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index cf04247..1325b72 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -33,3 +33,9 @@ variable "depends_on" { type = "list" } + +variable "assets" { + type = "map" +} + +variable "asset-dir" {} diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index ea2eed8..06f9e96 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -3,24 +3,20 @@ resource "docker_container" "kubelet" { image = "${docker_image.image.latest}" name = "kubelet-static" - volumes { - container_path = "/etc/kubernetes" - host_path = "/etc/kubernetes" + upload { + file = "/etc/kubernetes/kubeconfig" + content = "${var.assets["kubeconfig"]}" } - volumes { - container_path = "/etc/kubernetes/kubeconfig" - host_path = "/etc/kube-assets/auth/kubeconfig-kubelet" + upload { + file = "/etc/kubernetes/ca.crt" + content = "${var.assets["ca_cert"]}" } - volumes { - container_path = "/etc/kubernetes/kubeconfig-admin" - host_path = "/etc/kube-assets/auth/kubeconfig" - } - - volumes { - container_path = "/etc/kubernetes/ca.crt" - host_path = "/etc/kube-assets/tls/ca.crt" + # Make sure that the manifests directory exists + upload { + file = "/etc/kubernetes/manifests/.empty" + content = "" } volumes { @@ -40,6 +36,11 @@ resource "docker_container" "kubelet" { host_path = "/var/lib/docker" } + volumes { + container_path = "/etc/kubernetes" + host_path = "/etc/kubernetes" + } + volumes { container_path = "/var/lib/kubelet" host_path = "/var/lib/kubelet" @@ -86,6 +87,10 @@ resource "docker_container" "kubelet" { container_path = "/var/lib/cni" host_path = "/var/lib/cni" } + # + # "There is no war within the container. Here we are safe. Here we are free." + # - Docker Li agent brainwashing Nemo + # command = [ "kubelet", "--allow-privileged", diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf index d68cf21..0426c4d 100644 --- a/modules/kubelet/variables.tf +++ b/modules/kubelet/variables.tf @@ -27,3 +27,7 @@ variable "dns_ip" { variable "k8s_host" { description = "kubenetes hostname" } + +variable "assets" { + type = "map" +} diff --git a/providers.tf b/providers.tf index 95afb20..1939c9a 100644 --- a/providers.tf +++ b/providers.tf @@ -5,7 +5,7 @@ provider "docker" { } provider "docker" { - host = "tcp://dovpn.vpn.bb8.fun:2376" + host = "tcp://docker.dovpn.bb8.fun:2376" cert_path = "./secrets/sydney" alias = "sydney" version = "~> 2.0.0" From 0956877ac7d79bef460dbcab19cbd4db6d575442 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 27 Jan 2019 04:16:32 +0530 Subject: [PATCH 13/22] asset copy fixes --- kubernetes.tf | 19 ++++++++++++------- modules/bootkube/main.tf | 20 +++++++++++++------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/kubernetes.tf b/kubernetes.tf index c253da0..5c44a68 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -46,14 +46,19 @@ module "bootkube-start" { asset-dir = "${path.root}/k8s" assets = { + kubeconfig = "${module.bootkube.kubeconfig-kubelet}" + ca_cert = "${base64decode(module.bootkube.ca_cert)}" + kubelet_cert = "${base64decode(module.bootkube.kubelet_cert)}" + kubelet_key = "${base64decode(module.bootkube.kubelet_key)}" kubeconfig-kubelet = "${module.bootkube.kubeconfig-kubelet}" - etcd_ca_cert = "${module.bootkube.etcd_ca_cert}" - etcd_client_cert = "${module.bootkube.etcd_client_cert}" - etcd_client_key = "${module.bootkube.etcd_client_key}" - etcd_server_cert = "${module.bootkube.etcd_server_cert}" - etcd_server_key = "${module.bootkube.etcd_server_key}" - etcd_peer_cert = "${module.bootkube.etcd_peer_cert}" - etcd_peer_key = "${module.bootkube.etcd_peer_key}" + + # etcd_ca_cert = "${module.bootkube.etcd_ca_cert}" + # etcd_client_cert = "${module.bootkube.etcd_client_cert}" + # etcd_client_key = "${module.bootkube.etcd_client_key}" + # etcd_server_cert = "${module.bootkube.etcd_server_cert}" + # etcd_server_key = "${module.bootkube.etcd_server_key}" + # etcd_peer_cert = "${module.bootkube.etcd_peer_cert}" + # etcd_peer_key = "${module.bootkube.etcd_peer_key}" } providers = { diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 6c0d6b8..4e8896a 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -10,16 +10,16 @@ resource "docker_container" "bootkube" { # bootstrap manifests upload { - content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-apiserver.yaml")}" - file = "/home/.bootkube/bootstra-manifests/bootstrap-apiserver.yaml" + content = "${file("${var.asset-dir}/bootstrap-manifests/bootstrap-apiserver.yaml")}" + file = "/home/.bootkube/bootstrap-manifests/bootstrap-apiserver.yaml" } upload { - content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-controller-manager.yaml")}" - file = "/home/.bootkube/bootstra-manifests/bootstrap-controller-manager.yaml" + content = "${file("${var.asset-dir}/bootstrap-manifests/bootstrap-controller-manager.yaml")}" + file = "/home/.bootkube/bootstrap-manifests/bootstrap-controller-manager.yaml" } upload { - content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-scheduler.yaml")}" - file = "/home/.bootkube/bootstra-manifests/bootstrap-scheduler.yaml" + content = "${file("${var.asset-dir}/bootstrap-manifests/bootstrap-scheduler.yaml")}" + file = "/home/.bootkube/bootstrap-manifests/bootstrap-scheduler.yaml" } # Cluster Networking upload { @@ -39,7 +39,7 @@ resource "docker_container" "bootkube" { file = "/home/.bootkube/manifests-networking/daemonset.yaml" } upload { - content = "${file("${var.asset-dir}/manifests-networkingservice-account.yaml")}" + content = "${file("${var.asset-dir}/manifests-networking/service-account.yaml")}" file = "/home/.bootkube/manifests-networking/service-account.yaml" } # TLS @@ -82,6 +82,12 @@ resource "docker_container" "bootkube" { content = "${var.assets["kubeconfig-kubelet"]}" file = "/home/.bootkube/auth/kubeconfig-kubelet" } + # TODO: Move to a module read instead of file + # auth/kubeconfig + upload { + file = "/home/.bootkube/auth/kubeconfig" + content = "${file("${var.asset-dir}/auth/kubeconfig")}" + } # Manifests Directory upload { file = "/home/.bootkube/manifests/kube-apiserver-role-binding.yaml" From 6586244fa87948723ab4c4345fcc0e0c7b7d2f2c Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 27 Jan 2019 18:56:12 +0530 Subject: [PATCH 14/22] Adds etcd secrets to bootkube-start --- cloudflare/main.tf | 6 +++--- kubernetes.tf | 2 +- modules/bootkube/main.tf | 18 ++++++++++++++++-- modules/kubelet/main.tf | 8 ++++++-- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/cloudflare/main.tf b/cloudflare/main.tf index 32f3969..9fb6ab6 100644 --- a/cloudflare/main.tf +++ b/cloudflare/main.tf @@ -124,10 +124,10 @@ resource "cloudflare_record" "mailgun-mxb" { priority = 20 } -resource "cloudflare_record" "k8s-talk" { +resource "cloudflare_record" "k8s" { domain = "${var.domain}" name = "k8s" - value = "lightsaber.captnemo.in" - type = "CNAME" + value = "10.8.0.1" + type = "A" ttl = 3600 } diff --git a/kubernetes.tf b/kubernetes.tf index 5c44a68..db86c00 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -70,7 +70,7 @@ module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcbdddd8d07c99ab88b2e9ebfb662de4c104de0a" cluster_name = "k8s.bb8.fun" - api_servers = ["10.8.0.1", "k8s.bb8.fun"] + api_servers = ["k8s.bb8.fun"] cluster_domain_suffix = "k8s.bb8.fun" etcd_servers = ["etcd.bb8.fun"] asset_dir = "./k8s" diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 4e8896a..710506e 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -3,8 +3,8 @@ resource "docker_container" "bootkube" { name = "bootkube" volumes { - container_path = "/etc/kubernetes/manifests" - host_path = "/etc/kubernetes/manifests" + container_path = "/etc/kubernetes" + host_path = "/etc/kubernetes" } # bootstrap manifests @@ -21,6 +21,20 @@ resource "docker_container" "bootkube" { content = "${file("${var.asset-dir}/bootstrap-manifests/bootstrap-scheduler.yaml")}" file = "/home/.bootkube/bootstrap-manifests/bootstrap-scheduler.yaml" } + # etcd secrets + # + upload { + file = "/home/.bootkube/tls/etcd-client-ca.crt" + content = "${file("${var.asset-dir}/tls/etcd-client-ca.crt")}" + } + upload { + file = "/home/.bootkube/tls/etcd-client.crt" + content = "${file("${var.asset-dir}/tls/etcd-client.crt")}" + } + upload { + file = "/home/.bootkube/tls/etcd-client.key" + content = "${file("${var.asset-dir}/tls/etcd-client.key")}" + } # Cluster Networking upload { content = "${file("${var.asset-dir}/manifests-networking/cluster-role-binding.yaml")}" diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index 06f9e96..b85ca93 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -84,8 +84,12 @@ resource "docker_container" "kubelet" { // Deviates from kubelet-wrapper volumes { - container_path = "/var/lib/cni" - host_path = "/var/lib/cni" + container_path = "/opt/cni/bin" + host_path = "/opt/cni/bin" + } + volumes { + container_path = "/etc/cni/net.d" + host_path = "/etc/cni/net.d" } # # "There is no war within the container. Here we are safe. Here we are free." From ff8efd3139c893f845bce7e4b1a5158aac6b3bbd Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 27 Jan 2019 20:07:52 +0530 Subject: [PATCH 15/22] fix etcd, c-m, and node label/taints for kubelet --- modules/bootkube/main.tf | 4 ++++ modules/etcd/main.tf | 2 +- modules/kubelet/main.tf | 8 +++++--- modules/kubelet/variables.tf | 6 +++++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 710506e..d26ca7f 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -61,6 +61,10 @@ resource "docker_container" "bootkube" { file = "/home/.bootkube/tls/service-account.pub" content = "${file("${var.asset-dir}/tls/service-account.pub")}" } + upload { + file = "/home/.bootkube/tls/service-account.key" + content = "${file("${var.asset-dir}/tls/service-account.key")}" + } upload { content = "${file("${var.asset-dir}/tls/ca.key")}" file = "/home/.bootkube/tls/ca.key" diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf index 595215a..6001622 100644 --- a/modules/etcd/main.tf +++ b/modules/etcd/main.tf @@ -46,7 +46,7 @@ resource "docker_container" "etcd" { env = [ "ETCD_NAME=${var.node_name}", - "ETCD_DATA_DIR=/var/lib/etcd", + "ETCD_DATA_DIR=/etcd-data", "ETCD_ADVERTISE_CLIENT_URLS=https://${var.domain}:2379", "ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${var.domain}:2380", "ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379", diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index b85ca93..62cc7af 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -101,8 +101,10 @@ resource "docker_container" "kubelet" { "--anonymous-auth=false", "--authentication-token-webhook", "--authorization-mode=Webhook", - "--cert-dir=/var/lib/kubelet/pki", + + # "--cert-dir=/var/lib/kubelet/pki", "--client-ca-file=/etc/kubernetes/ca.crt", + "--cluster_dns=${var.dns_ip}", "--cluster_domain=${var.k8s_host}", @@ -117,6 +119,8 @@ resource "docker_container" "kubelet" { "--node-labels=node-role.kubernetes.io/master", "--pod-manifest-path=/etc/kubernetes/manifests", "--read-only-port=0", + "--register-with-taints=${var.node_taints}", + "--node-labels=${var.node_label}", "--rotate-certificates", ] host { @@ -125,8 +129,6 @@ resource "docker_container" "kubelet" { } # TODO - # "--register-with-taints=${var.node_taints}", - # "--node-labels=${var.node_label}", network_mode = "host" privileged = true diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf index 0426c4d..788f03f 100644 --- a/modules/kubelet/variables.tf +++ b/modules/kubelet/variables.tf @@ -5,7 +5,11 @@ variable "version" { variable "node_label" { description = "kubelet version" - default = "node.kubernetes.io/master" + default = "node-role.kubernetes.io/master" +} + +variable "node_taints" { + default = "node-role.kubernetes.io/master=:NoSchedule" } variable "depends_on" { From 23cf15b8a9b844e0ca523b4d1339d7d69c50a05e Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 28 Jan 2019 02:01:46 +0530 Subject: [PATCH 16/22] minor fixes --- modules/bootkube/main.tf | 10 +++++----- modules/kubelet/main.tf | 38 +++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index d26ca7f..654da70 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -38,23 +38,23 @@ resource "docker_container" "bootkube" { # Cluster Networking upload { content = "${file("${var.asset-dir}/manifests-networking/cluster-role-binding.yaml")}" - file = "/home/.bootkube/manifests-networking/cluster-role-binding.yaml" + file = "/home/.bootkube/manifests/networking-cluster-role-binding.yaml" } upload { content = "${file("${var.asset-dir}/manifests-networking/cluster-role.yaml")}" - file = "/home/.bootkube/manifests-networking/cluster-role.yaml" + file = "/home/.bootkube/manifests/networking-cluster-role.yaml" } upload { content = "${file("${var.asset-dir}/manifests-networking/config.yaml")}" - file = "/home/.bootkube/manifests-networking/config.yaml" + file = "/home/.bootkube/manifests/networking-config.yaml" } upload { content = "${file("${var.asset-dir}/manifests-networking/daemonset.yaml")}" - file = "/home/.bootkube/manifests-networking/daemonset.yaml" + file = "/home/.bootkube/manifests/networking-daemonset.yaml" } upload { content = "${file("${var.asset-dir}/manifests-networking/service-account.yaml")}" - file = "/home/.bootkube/manifests-networking/service-account.yaml" + file = "/home/.bootkube/manifests/networking-service-account.yaml" } # TLS upload { diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index 62cc7af..34d2399 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -26,59 +26,69 @@ resource "docker_container" "kubelet" { } volumes { - container_path = "/usr/share/ca-certificates" - host_path = "/usr/share/ca-certificates" + container_path = "/sys" + host_path = "/sys" read_only = true } + volumes { + container_path = "/dev" + host_path = "/dev" + } + + # volumes { + # container_path = "/usr" + # host_path = "/usr" + # } + + # volumes { + # container_path = "/lib64" + # host_path = "/lib64" + # } + volumes { + container_path = "/usr/share/ca-certificates" + host_path = "/usr/share/ca-certificates" + read_only = true + } volumes { container_path = "/var/lib/docker" host_path = "/var/lib/docker" } - volumes { container_path = "/etc/kubernetes" host_path = "/etc/kubernetes" } - volumes { container_path = "/var/lib/kubelet" host_path = "/var/lib/kubelet" } - volumes { container_path = "/var/log" host_path = "/var/log" } - volumes { container_path = "/run" host_path = "/run" } - volumes { container_path = "/lib/modules" host_path = "/lib/modules" read_only = true } - volumes { container_path = "/etc/os-release" host_path = "/usr/lib/os-release" read_only = true } - volumes { container_path = "/etc/machine-id" host_path = "/etc/machine-id" read_only = true } - volumes { container_path = "/rootfs" host_path = "/" read_only = true - read_only = true } // Deviates from kubelet-wrapper @@ -89,7 +99,7 @@ resource "docker_container" "kubelet" { } volumes { container_path = "/etc/cni/net.d" - host_path = "/etc/cni/net.d" + host_path = "/etc/kubernetes/cni/net.d" } # # "There is no war within the container. Here we are safe. Here we are free." @@ -101,10 +111,8 @@ resource "docker_container" "kubelet" { "--anonymous-auth=false", "--authentication-token-webhook", "--authorization-mode=Webhook", - - # "--cert-dir=/var/lib/kubelet/pki", + "--cert-dir=/var/lib/kubelet/pki", "--client-ca-file=/etc/kubernetes/ca.crt", - "--cluster_dns=${var.dns_ip}", "--cluster_domain=${var.k8s_host}", From 86f2edc11287b9344b888f08b20570371158d952 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sat, 2 Feb 2019 22:55:55 +0530 Subject: [PATCH 17/22] Get cluster up and running --- modules/kubelet/main.tf | 53 ++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index 34d2399..86415ff 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -13,12 +13,6 @@ resource "docker_container" "kubelet" { content = "${var.assets["ca_cert"]}" } - # Make sure that the manifests directory exists - upload { - file = "/etc/kubernetes/manifests/.empty" - content = "" - } - volumes { container_path = "/etc/ssl/certs" host_path = "/etc/ssl/certs" @@ -36,55 +30,72 @@ resource "docker_container" "kubelet" { host_path = "/dev" } - # volumes { - # container_path = "/usr" - # host_path = "/usr" - # } - - # volumes { - # container_path = "/lib64" - # host_path = "/lib64" - # } volumes { container_path = "/usr/share/ca-certificates" host_path = "/usr/share/ca-certificates" read_only = true } + volumes { container_path = "/var/lib/docker" host_path = "/var/lib/docker" } + + // TODO: Test with this + // It technically only needs the /etc/kubernetes/manifests + // Make sure that the manifests directory exists + upload { + file = "/etc/kubernetes/manifests/.empty" + content = "" + } + volumes { container_path = "/etc/kubernetes" host_path = "/etc/kubernetes" } + + // See https://github.com/kubernetes/kubernetes/issues/4869#issuecomment-193316593 volumes { container_path = "/var/lib/kubelet" host_path = "/var/lib/kubelet" + shared = true } + volumes { container_path = "/var/log" host_path = "/var/log" } + volumes { container_path = "/run" host_path = "/run" } + + volumes { + container_path = "/var/run" + host_path = "/var/run" + } + volumes { container_path = "/lib/modules" host_path = "/lib/modules" read_only = true } + volumes { container_path = "/etc/os-release" host_path = "/usr/lib/os-release" read_only = true } + volumes { container_path = "/etc/machine-id" host_path = "/etc/machine-id" read_only = true } + + // Don't think this is needed anymore + volumes { container_path = "/rootfs" host_path = "/" @@ -103,10 +114,11 @@ resource "docker_container" "kubelet" { } # # "There is no war within the container. Here we are safe. Here we are free." - # - Docker Li agent brainwashing Nemo + # - Docker Li agent brainwashing the author # command = [ "kubelet", + "--address=${var.host_ip}", "--allow-privileged", "--anonymous-auth=false", "--authentication-token-webhook", @@ -115,10 +127,7 @@ resource "docker_container" "kubelet" { "--client-ca-file=/etc/kubernetes/ca.crt", "--cluster_dns=${var.dns_ip}", "--cluster_domain=${var.k8s_host}", - - # "--containerized", "--exit-on-lock-contention=true", - "--hostname-override=${var.host_ip}", "--kubeconfig=/etc/kubernetes/kubeconfig", "--lock-file=/var/run/lock/kubelet.lock", @@ -135,15 +144,11 @@ resource "docker_container" "kubelet" { host = "${var.k8s_host}" ip = "${var.host_ip}" } - - # TODO - network_mode = "host" + pid_mode = "host" privileged = true restart = "no" must_run = false - - # max_retry_count = 1 } data "docker_registry_image" "image" { From 80ce34d52f515d3730bb22735942122c8ee3a577 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sat, 2 Feb 2019 23:21:49 +0530 Subject: [PATCH 18/22] Bring up a sample pod --- kube-test.tf | 22 ++++++++++++++++++++++ providers.tf | 7 +++++++ 2 files changed, 29 insertions(+) create mode 100644 kube-test.tf diff --git a/kube-test.tf b/kube-test.tf new file mode 100644 index 0000000..fae4802 --- /dev/null +++ b/kube-test.tf @@ -0,0 +1,22 @@ +// Bring up a simple test container +// In the controller node + +resource "kubernetes_pod" "nginx" { + metadata { + name = "terraform-example" + namespace = "default" + } + + spec { + toleration { + key = "node-role.kubernetes.io/master" + operator = "Exists" + effect = "NoSchedule" + } + + container { + image = "nginx:latest" + name = "nginx" + } + } +} diff --git a/providers.tf b/providers.tf index 1939c9a..7d4ce7b 100644 --- a/providers.tf +++ b/providers.tf @@ -11,6 +11,13 @@ provider "docker" { version = "~> 2.0.0" } +provider "kubernetes" { + version = "1.3.0-custom" + host = "https://k8s.bb8.fun:6443" + + config_path = "${path.root}/k8s/auth/kubeconfig" +} + provider "cloudflare" { email = "bb8@captnemo.in" token = "${var.cloudflare_key}" From 97300459fd9d4b06f578262716fdb79f18653430 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 3 Feb 2019 18:39:10 +0530 Subject: [PATCH 19/22] General Updates --- .gitignore | 1 + .terraform-version | 2 +- kubernetes.tf | 79 ++++------------------------------- modules/bootkube/main.tf | 6 --- modules/bootkube/outputs.tf | 10 ----- modules/bootkube/variables.tf | 2 - modules/etcd/variables.tf | 1 + modules/kubelet/main.tf | 30 +++---------- modules/kubelet/variables.tf | 3 +- providers.tf | 7 ++++ 10 files changed, 25 insertions(+), 116 deletions(-) diff --git a/.gitignore b/.gitignore index d2ef326..e1e2930 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ *.backup secrets k8s/ +k8s2/ diff --git a/.terraform-version b/.terraform-version index 1ee43fc..e6adeaa 100644 --- a/.terraform-version +++ b/.terraform-version @@ -1 +1 @@ -0.11.8 +0.11.12-beta1 diff --git a/kubernetes.tf b/kubernetes.tf index db86c00..1a7cf98 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -1,77 +1,12 @@ -module "etcd" { - source = "modules/etcd" - data_dir = "/mnt/disk/etcd" - host_bind_ip = "10.8.0.1" - domain = "etcd.bb8.fun" - - pki = { - ca_cert = "${module.bootkube.etcd_ca_cert}" - server_cert = "${module.bootkube.etcd_server_cert}" - server_key = "${module.bootkube.etcd_server_key}" - peer_cert = "${module.bootkube.etcd_peer_cert}" - peer_key = "${module.bootkube.etcd_peer_key}" - } - - providers = { - docker = "docker.sydney" - } - - depends_on = "${module.bootkube.id}" -} - -module "kubelet-master" { - source = "modules/kubelet" - host_ip = "${var.ips["dovpn"]}" - k8s_host = "k8s.${var.root-domain}" - - assets = { - kubeconfig = "${module.bootkube.kubeconfig-kubelet}" - ca_cert = "${base64decode(module.bootkube.ca_cert)}" - kubelet_cert = "${base64decode(module.bootkube.kubelet_cert)}" - kubelet_key = "${base64decode(module.bootkube.kubelet_key)}" - } - - depends_on = "${module.bootkube-start.image}" +module "k8s" { + source = "modules/k8s" + cluster_name = "k8s.${var.root-domain}" + etcd_domain = "etcd.${var.root-domain}" + etcd_data_dir = "/mnt/disk/etcd" + asset_dir = "${path.root}/k8s2" + host_ip = "${var.ips["dovpn"]}" providers = { docker = "docker.sydney" } } - -module "bootkube-start" { - source = "modules/bootkube" - mode = "start" - host_ip = "${var.ips["dovpn"]}" - k8s_host = "k8s.${var.root-domain}" - asset-dir = "${path.root}/k8s" - - assets = { - kubeconfig = "${module.bootkube.kubeconfig-kubelet}" - ca_cert = "${base64decode(module.bootkube.ca_cert)}" - kubelet_cert = "${base64decode(module.bootkube.kubelet_cert)}" - kubelet_key = "${base64decode(module.bootkube.kubelet_key)}" - kubeconfig-kubelet = "${module.bootkube.kubeconfig-kubelet}" - - # etcd_ca_cert = "${module.bootkube.etcd_ca_cert}" - # etcd_client_cert = "${module.bootkube.etcd_client_cert}" - # etcd_client_key = "${module.bootkube.etcd_client_key}" - # etcd_server_cert = "${module.bootkube.etcd_server_cert}" - # etcd_server_key = "${module.bootkube.etcd_server_key}" - # etcd_peer_cert = "${module.bootkube.etcd_peer_cert}" - # etcd_peer_key = "${module.bootkube.etcd_peer_key}" - } - - providers = { - docker = "docker.sydney" - } -} - -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcbdddd8d07c99ab88b2e9ebfb662de4c104de0a" - - cluster_name = "k8s.bb8.fun" - api_servers = ["k8s.bb8.fun"] - cluster_domain_suffix = "k8s.bb8.fun" - etcd_servers = ["etcd.bb8.fun"] - asset_dir = "./k8s" -} diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 654da70..188a0ec 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -89,12 +89,6 @@ resource "docker_container" "bootkube" { content = "${var.assets["kubelet_key"]}" file = "/home/.bootkube/tls/kubelet.key" } - # TODO: Generate Filenames Dynamically - # TODO: Check if this is needed at all - upload { - content = "${file("${var.asset-dir}/auth/k8s.bb8.fun-config")}" - file = "/home/.bootkube/auth/k8s.bb8.fun-config" - } # auth/kubeconfig-kubelet upload { content = "${var.assets["kubeconfig-kubelet"]}" diff --git a/modules/bootkube/outputs.tf b/modules/bootkube/outputs.tf index 29077f3..acc0ef3 100644 --- a/modules/bootkube/outputs.tf +++ b/modules/bootkube/outputs.tf @@ -1,13 +1,3 @@ -# output "exit_code" { -# # TODO: Pick correct exit code -# # value = "${coalesce(formatlist("%s", docker_container.render.*.exit_code))}" -# # See https://github.com/hashicorp/terraform/issues/15165 -# value = "${var.mode == "render" ? -# "${element(concat(docker_container.render.*.exit_code, list("")), 0)}" : -# "${element(concat(docker_container.start.*.exit_code, list("")), 0)}" -# }" -# } - output "image" { value = "${docker_image.image.latest}" } diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index 1325b72..45f8246 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -22,8 +22,6 @@ variable "service_cidr" { default = "10.96.0.0/16" } -variable "mode" {} - variable "version" { default = "0.14.0" } diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf index d47db7e..6b8c90a 100644 --- a/modules/etcd/variables.tf +++ b/modules/etcd/variables.tf @@ -30,4 +30,5 @@ variable "version" { variable "host_bind_ip" { description = "IP address to expose the ports on host" + default = "0.0.0.0" } diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index 86415ff..6903f4b 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -1,15 +1,15 @@ // This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper resource "docker_container" "kubelet" { image = "${docker_image.image.latest}" - name = "kubelet-static" + name = "kubelet" upload { - file = "/etc/kubernetes/kubeconfig" + file = "/etc/kubeconfig" content = "${var.assets["kubeconfig"]}" } upload { - file = "/etc/kubernetes/ca.crt" + file = "/etc/kubeca.crt" content = "${var.assets["ca_cert"]}" } @@ -41,14 +41,6 @@ resource "docker_container" "kubelet" { host_path = "/var/lib/docker" } - // TODO: Test with this - // It technically only needs the /etc/kubernetes/manifests - // Make sure that the manifests directory exists - upload { - file = "/etc/kubernetes/manifests/.empty" - content = "" - } - volumes { container_path = "/etc/kubernetes" host_path = "/etc/kubernetes" @@ -94,14 +86,6 @@ resource "docker_container" "kubelet" { read_only = true } - // Don't think this is needed anymore - - volumes { - container_path = "/rootfs" - host_path = "/" - read_only = true - } - // Deviates from kubelet-wrapper volumes { @@ -123,21 +107,19 @@ resource "docker_container" "kubelet" { "--anonymous-auth=false", "--authentication-token-webhook", "--authorization-mode=Webhook", - "--cert-dir=/var/lib/kubelet/pki", - "--client-ca-file=/etc/kubernetes/ca.crt", + "--client-ca-file=/etc/kubeca.crt", "--cluster_dns=${var.dns_ip}", "--cluster_domain=${var.k8s_host}", "--exit-on-lock-contention=true", "--hostname-override=${var.host_ip}", - "--kubeconfig=/etc/kubernetes/kubeconfig", + "--kubeconfig=/etc/kubeconfig", "--lock-file=/var/run/lock/kubelet.lock", "--minimum-container-ttl-duration=10m0s", "--network-plugin=cni", - "--node-labels=node-role.kubernetes.io/master", + "--node-labels=${var.node_label}", "--pod-manifest-path=/etc/kubernetes/manifests", "--read-only-port=0", "--register-with-taints=${var.node_taints}", - "--node-labels=${var.node_label}", "--rotate-certificates", ] host { diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf index 788f03f..24e643f 100644 --- a/modules/kubelet/variables.tf +++ b/modules/kubelet/variables.tf @@ -9,7 +9,8 @@ variable "node_label" { } variable "node_taints" { - default = "node-role.kubernetes.io/master=:NoSchedule" + description = "node taints" + default = "node-role.kubernetes.io/master=:NoSchedule" } variable "depends_on" { diff --git a/providers.tf b/providers.tf index 7d4ce7b..e4d7417 100644 --- a/providers.tf +++ b/providers.tf @@ -11,6 +11,13 @@ provider "docker" { version = "~> 2.0.0" } +provider "docker" { + host = "tcp://docker.captnemo.in:4243" + cert_path = "./secrets/nautilus" + alias = "nautilus" + version = "~> 2.0.0" +} + provider "kubernetes" { version = "1.3.0-custom" host = "https://k8s.bb8.fun:6443" From f85692da9e42dc5f65a5e0b379081508c967e637 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 10 Feb 2019 23:14:10 +0530 Subject: [PATCH 20/22] Switch to a remote state --- state.tf | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 state.tf diff --git a/state.tf b/state.tf new file mode 100644 index 0000000..91d84e1 --- /dev/null +++ b/state.tf @@ -0,0 +1,8 @@ +terraform { + backend "s3" { + bucket = "rmx-nemo" + key = "terraform/nebula.tfstate" + region = "ap-south-1" + profile = "nebula" + } +} From 40b967edce494dd202159d2674702dcf55289a41 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 10 Feb 2019 23:14:21 +0530 Subject: [PATCH 21/22] Migrate to kayak --- kayak.tf | 31 +++++ kubernetes.tf | 12 -- modules/bootkube/main.tf | 221 ---------------------------------- modules/bootkube/outputs.tf | 3 - modules/bootkube/variables.tf | 39 ------ modules/etcd/main.tf | 79 ------------ modules/etcd/variables.tf | 34 ------ modules/kubelet/main.tf | 143 ---------------------- modules/kubelet/variables.tf | 38 ------ providers.tf | 14 --- 10 files changed, 31 insertions(+), 583 deletions(-) create mode 100644 kayak.tf delete mode 100644 kubernetes.tf delete mode 100644 modules/bootkube/main.tf delete mode 100644 modules/bootkube/outputs.tf delete mode 100644 modules/bootkube/variables.tf delete mode 100644 modules/etcd/main.tf delete mode 100644 modules/etcd/variables.tf delete mode 100644 modules/kubelet/main.tf delete mode 100644 modules/kubelet/variables.tf diff --git a/kayak.tf b/kayak.tf new file mode 100644 index 0000000..8bdb0cf --- /dev/null +++ b/kayak.tf @@ -0,0 +1,31 @@ +// Points to the local working directory instead of +// the published version +module "kayak" { + source = "../terraform-digitalocean-kayak" + cert_path = "${path.root}/secrets/kayak" + domain = "kayak.${var.root-domain}" + ssh_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD0Getey8585AqdgIl9mqQ3SH9w6z7NZUW4HXdOqZwC7sYEaDrLOBV014gtFS8h8ymm4dcw6xEGUkaavcHC8W9ChTLKBMK4N1/sUS/umLy+Wi/K//g13y0VHSdvcc+gMQ27b9n/DwDY4ZKkaf6t+4HWyFWNh6gp0cT1WCyLNlsER55KUdy+C1lCOpv1SMepOaYc7uyBlC9FfgewJho/OfxnoTztQV6QeSGfr2Xr94Ip1FUPoLoBLLilh4ZbCe6F6bqn0kNgVBTkrVwWJv5Z0jCJpUjER69cqjASRao9KCHkyPtybzKKhCLZIlB3QMggEv0xnlHMpeeuDWcGrBVPKI8V" + + asset_dir = "${path.root}/k8s" + + providers { + docker = "docker.kayak" + } +} + +provider "docker" { + host = "tcp://${cloudflare_record.kayak-docker.hostname}:2376" + version = "~> 2.0.0" + alias = "kayak" + ca_material = "${module.kayak.docker_ca_cert}" + cert_material = "${module.kayak.docker_client_cert}" + key_material = "${module.kayak.docker_client_key}" +} + +resource "cloudflare_record" "kayak-docker" { + name = "docker.kayak" + value = "${module.kayak.droplet_ipv4}" + domain = "${var.root-domain}" + type = "A" + ttl = 3600 +} diff --git a/kubernetes.tf b/kubernetes.tf deleted file mode 100644 index 1a7cf98..0000000 --- a/kubernetes.tf +++ /dev/null @@ -1,12 +0,0 @@ -module "k8s" { - source = "modules/k8s" - cluster_name = "k8s.${var.root-domain}" - etcd_domain = "etcd.${var.root-domain}" - etcd_data_dir = "/mnt/disk/etcd" - asset_dir = "${path.root}/k8s2" - host_ip = "${var.ips["dovpn"]}" - - providers = { - docker = "docker.sydney" - } -} diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf deleted file mode 100644 index 188a0ec..0000000 --- a/modules/bootkube/main.tf +++ /dev/null @@ -1,221 +0,0 @@ -resource "docker_container" "bootkube" { - image = "${docker_image.image.latest}" - name = "bootkube" - - volumes { - container_path = "/etc/kubernetes" - host_path = "/etc/kubernetes" - } - - # bootstrap manifests - - upload { - content = "${file("${var.asset-dir}/bootstrap-manifests/bootstrap-apiserver.yaml")}" - file = "/home/.bootkube/bootstrap-manifests/bootstrap-apiserver.yaml" - } - upload { - content = "${file("${var.asset-dir}/bootstrap-manifests/bootstrap-controller-manager.yaml")}" - file = "/home/.bootkube/bootstrap-manifests/bootstrap-controller-manager.yaml" - } - upload { - content = "${file("${var.asset-dir}/bootstrap-manifests/bootstrap-scheduler.yaml")}" - file = "/home/.bootkube/bootstrap-manifests/bootstrap-scheduler.yaml" - } - # etcd secrets - # - upload { - file = "/home/.bootkube/tls/etcd-client-ca.crt" - content = "${file("${var.asset-dir}/tls/etcd-client-ca.crt")}" - } - upload { - file = "/home/.bootkube/tls/etcd-client.crt" - content = "${file("${var.asset-dir}/tls/etcd-client.crt")}" - } - upload { - file = "/home/.bootkube/tls/etcd-client.key" - content = "${file("${var.asset-dir}/tls/etcd-client.key")}" - } - # Cluster Networking - upload { - content = "${file("${var.asset-dir}/manifests-networking/cluster-role-binding.yaml")}" - file = "/home/.bootkube/manifests/networking-cluster-role-binding.yaml" - } - upload { - content = "${file("${var.asset-dir}/manifests-networking/cluster-role.yaml")}" - file = "/home/.bootkube/manifests/networking-cluster-role.yaml" - } - upload { - content = "${file("${var.asset-dir}/manifests-networking/config.yaml")}" - file = "/home/.bootkube/manifests/networking-config.yaml" - } - upload { - content = "${file("${var.asset-dir}/manifests-networking/daemonset.yaml")}" - file = "/home/.bootkube/manifests/networking-daemonset.yaml" - } - upload { - content = "${file("${var.asset-dir}/manifests-networking/service-account.yaml")}" - file = "/home/.bootkube/manifests/networking-service-account.yaml" - } - # TLS - upload { - file = "/home/.bootkube/tls/service-account.pub" - content = "${file("${var.asset-dir}/tls/service-account.pub")}" - } - upload { - file = "/home/.bootkube/tls/service-account.key" - content = "${file("${var.asset-dir}/tls/service-account.key")}" - } - upload { - content = "${file("${var.asset-dir}/tls/ca.key")}" - file = "/home/.bootkube/tls/ca.key" - } - upload { - content = "${file("${var.asset-dir}/tls/ca.crt")}" - file = "/home/.bootkube/tls/ca.crt" - } - upload { - content = "${file("${var.asset-dir}/tls/apiserver.key")}" - file = "/home/.bootkube/tls/apiserver.key" - } - upload { - content = "${file("${var.asset-dir}/tls/apiserver.crt")}" - file = "/home/.bootkube/tls/apiserver.crt" - } - upload { - content = "${var.assets["kubelet_cert"]}" - file = "/home/.bootkube/tls/kubelet.crt" - } - upload { - content = "${var.assets["kubelet_key"]}" - file = "/home/.bootkube/tls/kubelet.key" - } - # auth/kubeconfig-kubelet - upload { - content = "${var.assets["kubeconfig-kubelet"]}" - file = "/home/.bootkube/auth/kubeconfig-kubelet" - } - # TODO: Move to a module read instead of file - # auth/kubeconfig - upload { - file = "/home/.bootkube/auth/kubeconfig" - content = "${file("${var.asset-dir}/auth/kubeconfig")}" - } - # Manifests Directory - upload { - file = "/home/.bootkube/manifests/kube-apiserver-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/kube-apiserver-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-apiserver-sa.yaml" - content = "${file("${var.asset-dir}/manifests/kube-apiserver-sa.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-apiserver-secret.yaml" - content = "${file("${var.asset-dir}/manifests/kube-apiserver-secret.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-apiserver.yaml" - content = "${file("${var.asset-dir}/manifests/kube-apiserver.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kubeconfig-in-cluster.yaml" - content = "${file("${var.asset-dir}/manifests/kubeconfig-in-cluster.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-controller-manager-disruption.yaml" - content = "${file("${var.asset-dir}/manifests/kube-controller-manager-disruption.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-controller-manager-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/kube-controller-manager-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-controller-manager-sa.yaml" - content = "${file("${var.asset-dir}/manifests/kube-controller-manager-sa.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-controller-manager-secret.yaml" - content = "${file("${var.asset-dir}/manifests/kube-controller-manager-secret.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-controller-manager.yaml" - content = "${file("${var.asset-dir}/manifests/kube-controller-manager.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kubelet-nodes-cluster-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/kubelet-nodes-cluster-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-proxy-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/kube-proxy-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-proxy-sa.yaml" - content = "${file("${var.asset-dir}/manifests/kube-proxy-sa.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-proxy.yaml" - content = "${file("${var.asset-dir}/manifests/kube-proxy.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-scheduler-disruption.yaml" - content = "${file("${var.asset-dir}/manifests/kube-scheduler-disruption.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-scheduler-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/kube-scheduler-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-scheduler-sa.yaml" - content = "${file("${var.asset-dir}/manifests/kube-scheduler-sa.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-scheduler-volume-scheduler-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/kube-scheduler-volume-scheduler-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/kube-scheduler.yaml" - content = "${file("${var.asset-dir}/manifests/kube-scheduler.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role.yaml" - content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/pod-checkpointer-role-binding.yaml" - content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role-binding.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/pod-checkpointer-role.yaml" - content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/pod-checkpointer-sa.yaml" - content = "${file("${var.asset-dir}/manifests/pod-checkpointer-sa.yaml")}" - } - upload { - file = "/home/.bootkube/manifests/pod-checkpointer.yaml" - content = "${file("${var.asset-dir}/manifests/pod-checkpointer.yaml")}" - } - command = [ - "/bootkube", - "start", - "--asset-dir=/home/.bootkube", - ] - network_mode = "host" - restart = "on-failure" - max_retry_count = 5 -} - -data "docker_registry_image" "image" { - name = "quay.io/coreos/bootkube:v${var.version}" -} - -resource "docker_image" "image" { - name = "${data.docker_registry_image.image.name}" - pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] -} diff --git a/modules/bootkube/outputs.tf b/modules/bootkube/outputs.tf deleted file mode 100644 index acc0ef3..0000000 --- a/modules/bootkube/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "image" { - value = "${docker_image.image.latest}" -} diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf deleted file mode 100644 index 45f8246..0000000 --- a/modules/bootkube/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -// Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube - -variable "k8s_host" { - description = "kubenetes hostname" -} - -variable "host_port" { - default = "8443" -} - -variable "network_provider" { - default = "flannel" -} - -variable "host_ip" {} - -variable "pod_cidr" { - default = "10.25.0.0/16" -} - -variable "service_cidr" { - default = "10.96.0.0/16" -} - -variable "version" { - default = "0.14.0" -} - -variable "depends_on" { - default = [] - - type = "list" -} - -variable "assets" { - type = "map" -} - -variable "asset-dir" {} diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf deleted file mode 100644 index 6001622..0000000 --- a/modules/etcd/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -resource "docker_container" "etcd" { - name = "etcd" - image = "${docker_image.image.latest}" - - volumes { - host_path = "${var.data_dir}" - container_path = "/etcd-data" - } - - ports { - internal = 2379 - external = 2379 - ip = "${var.host_bind_ip}" - } - - ports { - internal = 2380 - external = 2380 - ip = "${var.host_bind_ip}" - } - - upload { - content = "${var.pki["ca_cert"]}" - file = "/etc/ssl/ca_cert.pem" - } - - upload { - content = "${var.pki["server_cert"]}" - file = "/etc/ssl/server_cert.pem" - } - - upload { - content = "${var.pki["server_key"]}" - file = "/etc/ssl/server_key.pem" - } - - upload { - content = "${var.pki["peer_cert"]}" - file = "/etc/ssl/peer_cert.pem" - } - - upload { - content = "${var.pki["peer_key"]}" - file = "/etc/ssl/peer_key.pem" - } - - env = [ - "ETCD_NAME=${var.node_name}", - "ETCD_DATA_DIR=/etcd-data", - "ETCD_ADVERTISE_CLIENT_URLS=https://${var.domain}:2379", - "ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${var.domain}:2380", - "ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379", - "ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380", - "ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381", - "ETCD_CLIENT_CERT_AUTH=true", - "ETCD_INITIAL_CLUSTER=${var.node_name}=https://${var.domain}:2380", - "ETCD_STRICT_RECONFIG_CHECK=true", - "ETCD_CERT_FILE=/etc/ssl/server_cert.pem", - "ETCD_KEY_FILE=/etc/ssl/server_key.pem", - "ETCD_TRUSTED_CA_FILE=/etc/ssl/ca_cert.pem", - "ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/ca_cert.pem", - "ETCD_PEER_CERT_FILE=/etc/ssl/peer_cert.pem", - "ETCD_PEER_KEY_FILE=/etc/ssl/peer_key.pem", - "ETCD_PEER_CLIENT_CERT_AUTH=true", - ] - - command = [ - "/usr/local/bin/etcd", - ] -} - -data "docker_registry_image" "image" { - name = "quay.io/coreos/etcd:v${var.version}" -} - -resource "docker_image" "image" { - name = "${data.docker_registry_image.image.name}" - pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] -} diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf deleted file mode 100644 index 6b8c90a..0000000 --- a/modules/etcd/variables.tf +++ /dev/null @@ -1,34 +0,0 @@ -variable "domain" { - description = "Host name to advertise" - type = "string" -} - -variable "data_dir" { - description = "Directory on host to mount to /etcd-data" - type = "string" -} - -variable "node_name" { - description = "name of the etcd node" - default = "controller" -} - -variable "depends_on" { - default = [] - - type = "list" -} - -variable "pki" { - type = "map" -} - -variable "version" { - description = "etcd version" - default = "3.3.11" -} - -variable "host_bind_ip" { - description = "IP address to expose the ports on host" - default = "0.0.0.0" -} diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf deleted file mode 100644 index 6903f4b..0000000 --- a/modules/kubelet/main.tf +++ /dev/null @@ -1,143 +0,0 @@ -// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper -resource "docker_container" "kubelet" { - image = "${docker_image.image.latest}" - name = "kubelet" - - upload { - file = "/etc/kubeconfig" - content = "${var.assets["kubeconfig"]}" - } - - upload { - file = "/etc/kubeca.crt" - content = "${var.assets["ca_cert"]}" - } - - volumes { - container_path = "/etc/ssl/certs" - host_path = "/etc/ssl/certs" - read_only = true - } - - volumes { - container_path = "/sys" - host_path = "/sys" - read_only = true - } - - volumes { - container_path = "/dev" - host_path = "/dev" - } - - volumes { - container_path = "/usr/share/ca-certificates" - host_path = "/usr/share/ca-certificates" - read_only = true - } - - volumes { - container_path = "/var/lib/docker" - host_path = "/var/lib/docker" - } - - volumes { - container_path = "/etc/kubernetes" - host_path = "/etc/kubernetes" - } - - // See https://github.com/kubernetes/kubernetes/issues/4869#issuecomment-193316593 - volumes { - container_path = "/var/lib/kubelet" - host_path = "/var/lib/kubelet" - shared = true - } - - volumes { - container_path = "/var/log" - host_path = "/var/log" - } - - volumes { - container_path = "/run" - host_path = "/run" - } - - volumes { - container_path = "/var/run" - host_path = "/var/run" - } - - volumes { - container_path = "/lib/modules" - host_path = "/lib/modules" - read_only = true - } - - volumes { - container_path = "/etc/os-release" - host_path = "/usr/lib/os-release" - read_only = true - } - - volumes { - container_path = "/etc/machine-id" - host_path = "/etc/machine-id" - read_only = true - } - - // Deviates from kubelet-wrapper - - volumes { - container_path = "/opt/cni/bin" - host_path = "/opt/cni/bin" - } - volumes { - container_path = "/etc/cni/net.d" - host_path = "/etc/kubernetes/cni/net.d" - } - # - # "There is no war within the container. Here we are safe. Here we are free." - # - Docker Li agent brainwashing the author - # - command = [ - "kubelet", - "--address=${var.host_ip}", - "--allow-privileged", - "--anonymous-auth=false", - "--authentication-token-webhook", - "--authorization-mode=Webhook", - "--client-ca-file=/etc/kubeca.crt", - "--cluster_dns=${var.dns_ip}", - "--cluster_domain=${var.k8s_host}", - "--exit-on-lock-contention=true", - "--hostname-override=${var.host_ip}", - "--kubeconfig=/etc/kubeconfig", - "--lock-file=/var/run/lock/kubelet.lock", - "--minimum-container-ttl-duration=10m0s", - "--network-plugin=cni", - "--node-labels=${var.node_label}", - "--pod-manifest-path=/etc/kubernetes/manifests", - "--read-only-port=0", - "--register-with-taints=${var.node_taints}", - "--rotate-certificates", - ] - host { - host = "${var.k8s_host}" - ip = "${var.host_ip}" - } - network_mode = "host" - pid_mode = "host" - privileged = true - restart = "no" - must_run = false -} - -data "docker_registry_image" "image" { - name = "gcr.io/google_containers/hyperkube:v${var.version}" -} - -resource "docker_image" "image" { - name = "${data.docker_registry_image.image.name}" - pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] -} diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf deleted file mode 100644 index 24e643f..0000000 --- a/modules/kubelet/variables.tf +++ /dev/null @@ -1,38 +0,0 @@ -variable "version" { - description = "kubelet version" - default = "1.13.2" -} - -variable "node_label" { - description = "kubelet version" - default = "node-role.kubernetes.io/master" -} - -variable "node_taints" { - description = "node taints" - default = "node-role.kubernetes.io/master=:NoSchedule" -} - -variable "depends_on" { - default = [] - - type = "list" -} - -variable "asset_dir_volume_name" { - default = "k8s-assets" -} - -variable "host_ip" {} - -variable "dns_ip" { - default = "10.25.0.10" -} - -variable "k8s_host" { - description = "kubenetes hostname" -} - -variable "assets" { - type = "map" -} diff --git a/providers.tf b/providers.tf index e4d7417..c9eedb8 100644 --- a/providers.tf +++ b/providers.tf @@ -4,20 +4,6 @@ provider "docker" { version = "~> 2.0.0" } -provider "docker" { - host = "tcp://docker.dovpn.bb8.fun:2376" - cert_path = "./secrets/sydney" - alias = "sydney" - version = "~> 2.0.0" -} - -provider "docker" { - host = "tcp://docker.captnemo.in:4243" - cert_path = "./secrets/nautilus" - alias = "nautilus" - version = "~> 2.0.0" -} - provider "kubernetes" { version = "1.3.0-custom" host = "https://k8s.bb8.fun:6443" From 83eb97c8dbc4ea2fa375b7fb7a58f92c6a8684d0 Mon Sep 17 00:00:00 2001 From: Nemo Date: Wed, 13 Feb 2019 20:37:38 +0530 Subject: [PATCH 22/22] Create etcd dns entry --- kayak.tf | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/kayak.tf b/kayak.tf index 8bdb0cf..a15ea2d 100644 --- a/kayak.tf +++ b/kayak.tf @@ -27,5 +27,21 @@ resource "cloudflare_record" "kayak-docker" { value = "${module.kayak.droplet_ipv4}" domain = "${var.root-domain}" type = "A" - ttl = 3600 + ttl = 120 +} + +resource "cloudflare_record" "kayak" { + name = "kayak" + value = "${module.kayak.droplet_ipv4}" + domain = "${var.root-domain}" + type = "A" + ttl = 120 +} + +resource "cloudflare_record" "kayak-etcd" { + name = "etcd.kayak" + value = "${module.kayak.droplet_ipv4_private}" + domain = "${var.root-domain}" + type = "A" + ttl = 120 }