[k8s] Use terraform-render-bootkube to bring up etcd

- Stop running render on the server

cloudflare/main.tf

@@ -83,6 +83,13 @@ resource "cloudflare_record" "dovpn_wildcard" {
   ttl    = 3600
 }
 
+resource "cloudflare_record" "etcd" {
+  domain = "${var.domain}"
+  name   = "etcd"
+  value  = "${var.ips["dovpn"]}"
+  type   = "A"
+}
+
 ########################
 ## Mailgun Mailing Lists
 ########################

kubernetes.tf

@@ -1,15 +1,27 @@
 module "etcd" {
-  source   = "modules/etcd"
-  host_ip  = "${var.ips["dovpn"]}"
-  data_dir = "/mnt/xwing/etcd"
+  source       = "modules/etcd"
+  data_dir     = "/mnt/disk/etcd"
+  host_bind_ip = "10.8.0.1"
+  domain       = "etcd.bb8.fun"
 
-  bootkube_asset_dir = "/etc/kube-assets"
+  pki = {
+    /**
+     * client_cert = "${module.bootkube.etcd_client_cert}"
+     * client_key  = "${module.bootkube.etcd_client_key}"
+     */
+
+    ca_cert     = "${module.bootkube.etcd_ca_cert}"
+    server_cert = "${module.bootkube.etcd_server_cert}"
+    server_key  = "${module.bootkube.etcd_server_key}"
+    peer_cert   = "${module.bootkube.etcd_peer_cert}"
+    peer_key    = "${module.bootkube.etcd_peer_key}"
+  }
 
   providers = {
     docker = "docker.sydney"
   }
 
-  depends_on = "${module.bootkube-start.image}"
+  depends_on = "${module.bootkube.id}"
 }
 
 module "kubelet-master" {
@@ -24,9 +36,9 @@ module "kubelet-master" {
   }
 }
 
-module "bootkube-render" {
+module "bootkube-start" {
   source   = "modules/bootkube"
-  mode     = "render"
+  mode     = "start"
   host_ip  = "${var.ips["dovpn"]}"
   k8s_host = "k8s.${var.root-domain}"
 
@@ -35,14 +47,12 @@ module "bootkube-render" {
   }
 }
 
-module "bootkube-start" {
-  depends_on = "${module.bootkube-render.image}"
-  source     = "modules/bootkube"
-  mode       = "start"
-  host_ip    = "${var.ips["dovpn"]}"
-  k8s_host   = "k8s.${var.root-domain}"
+module "bootkube" {
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcbdddd8d07c99ab88b2e9ebfb662de4c104de0a"
 
-  providers = {
-    docker = "docker.sydney"
-  }
+  cluster_name          = "k8s.bb8.fun"
+  api_servers           = ["10.8.0.1", "k8s.bb8.fun"]
+  cluster_domain_suffix = "k8s.bb8.fun"
+  etcd_servers          = ["etcd.bb8.fun"]
+  asset_dir             = "./k8s"
 }

modules/etcd/main.tf

@@ -1,64 +1,79 @@
-module "container" {
-  source = "../container"
-  image  = "captn3m0/etcd:v3.3.11"
-  name   = "etcd"
+resource "docker_container" "etcd" {
+  name  = "etcd"
+  image = "${docker_image.image.latest}"
 
-  web = {
-    expose = false
-    host   = ""
+  volumes {
+    host_path      = "${var.data_dir}"
+    container_path = "/etcd-data"
   }
 
-  volumes = [
-    {
-      host_path      = "${var.data_dir}"
-      container_path = "/etcd-data"
-    },
-    {
-      host_path      = "${var.bootkube_asset_dir}/tls/etcd-client.crt"
-      container_path = "/etc/etcd-client.crt"
-    },
-    {
-      host_path      = "${var.bootkube_asset_dir}/tls/etcd-client.key"
-      container_path = "/etc/etcd-client.key"
-    },
-    {
-      host_path      = "${var.bootkube_asset_dir}/tls/etcd-client-ca.crt"
-      container_path = "/etc/etcd-client-ca.crt"
-    },
-    {
-      host_path      = "${var.bootkube_asset_dir}/tls/etcd"
-      container_path = "/etc/ssl/certs/etcd"
-    },
-  ]
+  ports {
+    internal = 2379
+    external = 2379
+    ip       = "${var.host_bind_ip}"
+  }
 
-  ports = [
-    {
-      internal = 2379
-      external = 2379
-      ip       = "${var.host_ip}"
-    },
-    {
-      internal = 2380
-      external = 2380
-      ip       = "${var.host_ip}"
-    },
+  ports {
+    internal = 2380
+    external = 2380
+    ip       = "${var.host_bind_ip}"
+  }
+
+  upload {
+    content = "${var.pki["ca_cert"]}"
+    file    = "/etc/ssl/ca_cert.pem"
+  }
+
+  upload {
+    content = "${var.pki["server_cert"]}"
+    file    = "/etc/ssl/server_cert.pem"
+  }
+
+  upload {
+    content = "${var.pki["server_key"]}"
+    file    = "/etc/ssl/server_key.pem"
+  }
+
+  upload {
+    content = "${var.pki["peer_cert"]}"
+    file    = "/etc/ssl/peer_cert.pem"
+  }
+
+  upload {
+    content = "${var.pki["peer_key"]}"
+    file    = "/etc/ssl/peer_key.pem"
+  }
+
+  env = [
+    "ETCD_NAME=${var.node_name}",
+    "ETCD_DATA_DIR=/var/lib/etcd",
+    "ETCD_ADVERTISE_CLIENT_URLS=https://${var.domain}:2379",
+    "ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${var.domain}:2380",
+    "ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379",
+    "ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380",
+    "ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381",
+    "ETCD_CLIENT_CERT_AUTH=true",
+    "ETCD_INITIAL_CLUSTER=${var.node_name}=https://${var.domain}:2380",
+    "ETCD_STRICT_RECONFIG_CHECK=true",
+    "ETCD_CERT_FILE=/etc/ssl/server_cert.pem",
+    "ETCD_KEY_FILE=/etc/ssl/server_key.pem",
+    "ETCD_TRUSTED_CA_FILE=/etc/ssl/ca_cert.pem",
+    "ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/ca_cert.pem",
+    "ETCD_PEER_CERT_FILE=/etc/ssl/peer_cert.pem",
+    "ETCD_PEER_KEY_FILE=/etc/ssl/peer_key.pem",
+    "ETCD_PEER_CLIENT_CERT_AUTH=true",
   ]
 
   command = [
     "/usr/local/bin/etcd",
-    "--data-dir=/etcd-data",
-    "--name=${var.node_name}",
-    "--advertise-client-urls=https://${var.host_ip}:2379",
-    "--initial-advertise-peer-urls=https://${var.host_ip}:2380",
-    "--initial-cluster=${var.node_name}=https://${var.host_ip}:2380",
-    "--listen-client-urls=https://0.0.0.0:2379",
-    "--listen-peer-urls=https://0.0.0.0:2380",
-    "--trusted-ca-file=/etc/ssl/certs/etcd/server-ca.crt",
-    "--cert-file=/etc/ssl/certs/etcd/server.crt",
-    "--key-file=/etc/ssl/certs/etcd/server.key",
-    "--client-cert-auth=true",
-    "--peer-trusted-ca-file=/etc/ssl/certs/etcd/peer-ca.crt",
-    "--peer-cert-file=/etc/ssl/certs/etcd/peer.crt",
-    "--peer-key-file=/etc/ssl/certs/etcd/peer.key",
   ]
 }
+
+data "docker_registry_image" "image" {
+  name = "quay.io/coreos/etcd:v${var.version}"
+}
+
+resource "docker_image" "image" {
+  name          = "${data.docker_registry_image.image.name}"
+  pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"]
+}

modules/etcd/variables.tf

@@ -1,7 +1,6 @@
-variable "host_ip" {
-  description = "Host IP Address to bind etcd to"
+variable "domain" {
+  description = "Host name to advertise"
   type        = "string"
-  default     = "0.0.0.0"
 }
 
 variable "data_dir" {
@@ -9,15 +8,9 @@ variable "data_dir" {
   type        = "string"
 }
 
-variable "bootkube_asset_dir" {
-  description = "bootkube render is run against this directory"
-  type        = "string"
-  default     = "/etc/kube-assets"
-}
-
 variable "node_name" {
   description = "name of the etcd node"
-  default     = "master"
+  default     = "controller"
 }
 
 variable "depends_on" {
@@ -25,3 +18,16 @@ variable "depends_on" {
 
   type = "list"
 }
+
+variable "pki" {
+  type = "map"
+}
+
+variable "version" {
+  description = "etcd version"
+  default     = "3.3.11"
+}
+
+variable "host_bind_ip" {
+  description = "IP address to expose the ports on host"
+}