[k8s] Adds kubelet, start stitching things together

Challenges:

1. etcd booting before bootkube meant I missed certs
2. etcd can run without certs, but managing docker network
   over static pod manifests might be tricky :fingers_crossed:

kubernetes.tf

@@ -3,16 +3,24 @@ module "etcd" {
   host_ip  = "${var.ips["dovpn"]}"
   data_dir = "/mnt/xwing/etcd"
 
+  bootkube_asset_dir = "/etc/kube-assets"
+
+  providers = {
+    docker = "docker.sydney"
+  }
+
+  depends_on = "${module.bootkube-start.image}"
+}
+
+module "kubelet-master" {
+  source     = "modules/kubelet"
+  depends_on = "${module.bootkube-start.image}"
+
   providers = {
     docker = "docker.sydney"
   }
 }
 
-# module "kubelet" {
-#   source = "modules/kubelet"
-#   listen_ip =  "${var.ips["dovpn"]}"
-# }
-
 module "bootkube-render" {
   source   = "modules/bootkube"
   mode     = "render"

modules/bootkube/main.tf

@@ -5,15 +5,17 @@ resource "docker_container" "render" {
 
   volumes {
     container_path = "/home/.bootkube"
-    volume_name    = "${var.asset_dir_volume_name}"
+    volume_name    = "/etc/kube-assets"
   }
 
   command = [
     "bootkube",
     "render",
+    "--etcd-servers=http://${host_ip}:2379",
     "--asset-dir=/home/.bootkube",
-    "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}",
+    "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}",
     "--pod-cidr=${var.pod_cidr}",
+    "--network-provider=${var.network_provider}",
   ]
 
   network_mode    = "host"
@@ -28,13 +30,13 @@ resource "docker_container" "start" {
 
   volumes {
     container_path = "/home/.bootkube"
-    volume_name    = "${var.asset_dir_volume_name}"
+    volume_name    = "/etc/kube-assets"
     read_only      = true
   }
 
   volumes {
-    container_path = "/etc/kubernetes/manifests"
-    host_path      = "/etc/kubernetes/manifests"
+    container_path = "/etc/kubernetes"
+    host_path      = "/etc/kubernetes"
   }
 
   # "There is no war within the container. Here we are safe. Here we are free."
@@ -43,7 +45,6 @@ resource "docker_container" "start" {
     "bootkube",
     "start",
     "--asset-dir=/home/.bootkube",
-    "--pod-manifest-path=/etc/kubernetes/manifests",
   ]
 
   network_mode    = "host"

modules/bootkube/variables.tf

@@ -1,9 +1,5 @@
 // Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube
 
-variable "asset_dir_volume_name" {
-  default = "k8s-assets"
-}
-
 variable "k8s_host" {
   description = "kubenetes hostname"
 }
@@ -12,6 +8,10 @@ variable "host_port" {
   default = "8443"
 }
 
+variable "network_provider" {
+  default = "flannel"
+}
+
 variable "host_ip" {}
 
 variable "pod_cidr" {

modules/etcd/main.tf

@@ -8,32 +8,15 @@ module "container" {
     host   = ""
   }
 
-  networks = []
+  networks = ["${docker_network.etcd.id}"]
 
   volumes = [
-    {
-      host_path      = "/usr/share/ca-certificates/"
-      container_path = "/etc/ssl/certs"
-    },
     {
       host_path      = "${var.data_dir}"
       container_path = "/etcd-data"
     },
   ]
 
-  ports = [
-    {
-      internal = 2379
-      external = 2379
-      ip       = "${var.host_ip}"
-    },
-    {
-      internal = 2380
-      external = 2380
-      ip       = "${var.host_ip}"
-    },
-  ]
-
   command = [
     "/usr/local/bin/etcd",
     "--data-dir=/etcd-data",
@@ -42,7 +25,14 @@ module "container" {
     "--initial-advertise-peer-urls=http://${var.host_ip}:2380",
     "--initial-cluster=${var.node_name}=http://${var.host_ip}:2380",
   ]
-
-  # "--listen-client-urls=http://0.0.0.0:2379",
-  # "--listen-peer-urls=http://0.0.0.0:2380",
+}
+
+resource "docker_network" "etcd" {
+  name   = "etcd"
+  driver = "bridge"
+
+  ipam_config {
+    subnet  = "10.10.10.0/25"
+    gateway = "10.10.10.1"
+  }
 }

modules/etcd/variables.tf

@@ -9,7 +9,19 @@ variable "data_dir" {
   type        = "string"
 }
 
+variable "bootkube_asset_dir" {
+  description = "bootkube render is run against this directory"
+  type        = "string"
+  default     = "/etc/kube-assets"
+}
+
 variable "node_name" {
   description = "name of the etcd node"
   default     = "master"
 }
+
+variable "depends_on" {
+  default = []
+
+  type = "list"
+}

modules/kubelet/main.tf

@@ -0,0 +1,116 @@
+// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper
+resource "docker_container" "kubelet" {
+  image = "${docker_image.image.latest}"
+  name  = "kubelet-static"
+
+  volumes {
+    container_path = "/etc/kubernetes"
+    host_path      = "/etc/kubernetes"
+  }
+
+  volumes {
+    container_path = "/etc/kubernetes/kubeconfig"
+    host_path      = "/etc/kube-assets/auth/kubeconfig-kubelet"
+  }
+
+  volumes {
+    container_path = "/etc/kubernetes/kubeconfig-admin"
+    host_path      = "/etc/kube-assets/auth/kubeconfig"
+  }
+
+  volumes {
+    container_path = "/etc/kubernetes/ca.crt"
+    host_path      = "/etc/kube-assets/tls/ca.crt"
+  }
+
+  volumes {
+    container_path = "/etc/ssl/certs"
+    host_path      = "/etc/ssl/certs"
+    read_only      = true
+  }
+
+  volumes {
+    container_path = "/usr/share/ca-certificates"
+    host_path      = "/usr/share/ca-certificates"
+    read_only      = true
+  }
+
+  volumes {
+    container_path = "/var/lib/docker"
+    host_path      = "/var/lib/docker"
+  }
+
+  volumes {
+    container_path = "/var/lib/kubelet"
+    host_path      = "/var/lib/kubelet"
+  }
+
+  volumes {
+    container_path = "/var/log"
+    host_path      = "/var/log"
+  }
+
+  volumes {
+    container_path = "/run"
+    host_path      = "/run"
+  }
+
+  volumes {
+    container_path = "/lib/modules"
+    host_path      = "/lib/modules"
+    read_only      = true
+  }
+
+  volumes {
+    container_path = "/etc/os-release"
+    host_path      = "/usr/lib/os-release"
+    read_only      = true
+  }
+
+  volumes {
+    container_path = "/etc/machine-id"
+    host_path      = "/etc/machine-id"
+    read_only      = true
+  }
+
+  // Deviates from kubelet-wrapper
+
+  volumes {
+    container_path = "/var/lib/cni"
+    host_path      = "/var/lib/cni"
+  }
+  command = [
+    "kubelet",
+    "--kubeconfig=/etc/kubernetes/kubeconfig",
+    "--client-ca-file=/etc/kubernetes/ca.crt",
+    "--anonymous-auth=false",
+    "--cni-conf-dir=/etc/kubernetes/cni/net.d",
+    "--network-plugin=cni",
+    "--lock-file=/var/run/lock/kubelet.lock",
+    "--exit-on-lock-contention",
+    "--pod-manifest-path=/etc/kubernetes/manifests",
+    "--allow-privileged",
+    "--minimum-container-ttl-duration=10m0s",
+    "--cluster_dns=10.25.0.10",
+    "--cluster_domain=k8s.bb8.fun",
+  ]
+
+  # TODO
+  # "--register-with-taints=${var.node_taints}",
+  # "--node-labels=${var.node_label}",
+
+  network_mode    = "host"
+  privileged      = true
+  restart         = "no"
+  must_run        = false
+  max_retry_count = 1
+}
+
+data "docker_registry_image" "image" {
+  name = "gcr.io/google_containers/hyperkube:v${var.version}"
+}
+
+resource "docker_image" "image" {
+  name          = "${data.docker_registry_image.image.name}"
+  pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"]
+}

modules/kubelet/variables.tf

@@ -0,0 +1,19 @@
+variable "version" {
+  description = "kubelet version"
+  default     = "1.13.2"
+}
+
+variable "node_label" {
+  description = "kubelet version"
+  default     = "node.kubernetes.io/master"
+}
+
+variable "depends_on" {
+  default = []
+
+  type = "list"
+}
+
+variable "asset_dir_volume_name" {
+  default = "k8s-assets"
+}