diff --git a/kubernetes.tf b/kubernetes.tf index 9792867..c253da0 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -24,6 +24,13 @@ module "kubelet-master" { host_ip = "${var.ips["dovpn"]}" k8s_host = "k8s.${var.root-domain}" + assets = { + kubeconfig = "${module.bootkube.kubeconfig-kubelet}" + ca_cert = "${base64decode(module.bootkube.ca_cert)}" + kubelet_cert = "${base64decode(module.bootkube.kubelet_cert)}" + kubelet_key = "${base64decode(module.bootkube.kubelet_key)}" + } + depends_on = "${module.bootkube-start.image}" providers = { @@ -32,10 +39,22 @@ module "kubelet-master" { } module "bootkube-start" { - source = "modules/bootkube" - mode = "start" - host_ip = "${var.ips["dovpn"]}" - k8s_host = "k8s.${var.root-domain}" + source = "modules/bootkube" + mode = "start" + host_ip = "${var.ips["dovpn"]}" + k8s_host = "k8s.${var.root-domain}" + asset-dir = "${path.root}/k8s" + + assets = { + kubeconfig-kubelet = "${module.bootkube.kubeconfig-kubelet}" + etcd_ca_cert = "${module.bootkube.etcd_ca_cert}" + etcd_client_cert = "${module.bootkube.etcd_client_cert}" + etcd_client_key = "${module.bootkube.etcd_client_key}" + etcd_server_cert = "${module.bootkube.etcd_server_cert}" + etcd_server_key = "${module.bootkube.etcd_server_key}" + etcd_peer_cert = "${module.bootkube.etcd_peer_cert}" + etcd_peer_key = "${module.bootkube.etcd_peer_key}" + } providers = { docker = "docker.sydney" diff --git a/modules/bootkube/data.tf b/modules/bootkube/data.tf deleted file mode 100644 index 8b13789..0000000 --- a/modules/bootkube/data.tf +++ /dev/null @@ -1 +0,0 @@ - diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 00b019c..6c0d6b8 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -1,52 +1,193 @@ -resource "docker_container" "render" { - count = "${var.mode == "render" ? 1 : 0}" +resource "docker_container" "bootkube" { image = "${docker_image.image.latest}" - name = "bootkube-render" + name = "bootkube" volumes { - container_path = "/home/.bootkube" - volume_name = "/etc/kube-assets" + container_path = "/etc/kubernetes/manifests" + host_path = "/etc/kubernetes/manifests" } - command = [ - "/bootkube", - "render", - "--etcd-servers=https://${var.host_ip}:2379", - "--asset-dir=/home/.bootkube", - "--api-servers=https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", - "--pod-cidr=${var.pod_cidr}", - "--network-provider=${var.network_provider}", - ] + # bootstrap manifests - network_mode = "host" - restart = "on-failure" - max_retry_count = 5 -} - -resource "docker_container" "start" { - count = "${var.mode == "start" ? 1 : 0}" - image = "${docker_image.image.latest}" - name = "bootkube-${var.mode}" - - volumes { - container_path = "/home/.bootkube" - volume_name = "/etc/kube-assets" - read_only = true + upload { + content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-apiserver.yaml")}" + file = "/home/.bootkube/bootstra-manifests/bootstrap-apiserver.yaml" } - - volumes { - container_path = "/etc/kubernetes" - host_path = "/etc/kubernetes" + upload { + content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-controller-manager.yaml")}" + file = "/home/.bootkube/bootstra-manifests/bootstrap-controller-manager.yaml" + } + upload { + content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-scheduler.yaml")}" + file = "/home/.bootkube/bootstra-manifests/bootstrap-scheduler.yaml" + } + # Cluster Networking + upload { + content = "${file("${var.asset-dir}/manifests-networking/cluster-role-binding.yaml")}" + file = "/home/.bootkube/manifests-networking/cluster-role-binding.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networking/cluster-role.yaml")}" + file = "/home/.bootkube/manifests-networking/cluster-role.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networking/config.yaml")}" + file = "/home/.bootkube/manifests-networking/config.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networking/daemonset.yaml")}" + file = "/home/.bootkube/manifests-networking/daemonset.yaml" + } + upload { + content = "${file("${var.asset-dir}/manifests-networkingservice-account.yaml")}" + file = "/home/.bootkube/manifests-networking/service-account.yaml" + } + # TLS + upload { + file = "/home/.bootkube/tls/service-account.pub" + content = "${file("${var.asset-dir}/tls/service-account.pub")}" + } + upload { + content = "${file("${var.asset-dir}/tls/ca.key")}" + file = "/home/.bootkube/tls/ca.key" + } + upload { + content = "${file("${var.asset-dir}/tls/ca.crt")}" + file = "/home/.bootkube/tls/ca.crt" + } + upload { + content = "${file("${var.asset-dir}/tls/apiserver.key")}" + file = "/home/.bootkube/tls/apiserver.key" + } + upload { + content = "${file("${var.asset-dir}/tls/apiserver.crt")}" + file = "/home/.bootkube/tls/apiserver.crt" + } + upload { + content = "${var.assets["kubelet_cert"]}" + file = "/home/.bootkube/tls/kubelet.crt" + } + upload { + content = "${var.assets["kubelet_key"]}" + file = "/home/.bootkube/tls/kubelet.key" + } + # TODO: Generate Filenames Dynamically + # TODO: Check if this is needed at all + upload { + content = "${file("${var.asset-dir}/auth/k8s.bb8.fun-config")}" + file = "/home/.bootkube/auth/k8s.bb8.fun-config" + } + # auth/kubeconfig-kubelet + upload { + content = "${var.assets["kubeconfig-kubelet"]}" + file = "/home/.bootkube/auth/kubeconfig-kubelet" + } + # Manifests Directory + upload { + file = "/home/.bootkube/manifests/kube-apiserver-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-apiserver-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-apiserver-secret.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver-secret.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-apiserver.yaml" + content = "${file("${var.asset-dir}/manifests/kube-apiserver.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kubeconfig-in-cluster.yaml" + content = "${file("${var.asset-dir}/manifests/kubeconfig-in-cluster.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-disruption.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-disruption.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager-secret.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager-secret.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-controller-manager.yaml" + content = "${file("${var.asset-dir}/manifests/kube-controller-manager.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kubelet-nodes-cluster-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kubelet-nodes-cluster-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-proxy-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-proxy-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-proxy-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-proxy-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-proxy.yaml" + content = "${file("${var.asset-dir}/manifests/kube-proxy.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-disruption.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-disruption.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-sa.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler-volume-scheduler-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler-volume-scheduler-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/kube-scheduler.yaml" + content = "${file("${var.asset-dir}/manifests/kube-scheduler.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-role-binding.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role-binding.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-role.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer-sa.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer-sa.yaml")}" + } + upload { + file = "/home/.bootkube/manifests/pod-checkpointer.yaml" + content = "${file("${var.asset-dir}/manifests/pod-checkpointer.yaml")}" } - - # "There is no war within the container. Here we are safe. Here we are free." - # - Docker Li agent brainwashing Nemo command = [ "/bootkube", "start", "--asset-dir=/home/.bootkube", ] - network_mode = "host" restart = "on-failure" max_retry_count = 5 diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index cf04247..1325b72 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -33,3 +33,9 @@ variable "depends_on" { type = "list" } + +variable "assets" { + type = "map" +} + +variable "asset-dir" {} diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf index ea2eed8..06f9e96 100644 --- a/modules/kubelet/main.tf +++ b/modules/kubelet/main.tf @@ -3,24 +3,20 @@ resource "docker_container" "kubelet" { image = "${docker_image.image.latest}" name = "kubelet-static" - volumes { - container_path = "/etc/kubernetes" - host_path = "/etc/kubernetes" + upload { + file = "/etc/kubernetes/kubeconfig" + content = "${var.assets["kubeconfig"]}" } - volumes { - container_path = "/etc/kubernetes/kubeconfig" - host_path = "/etc/kube-assets/auth/kubeconfig-kubelet" + upload { + file = "/etc/kubernetes/ca.crt" + content = "${var.assets["ca_cert"]}" } - volumes { - container_path = "/etc/kubernetes/kubeconfig-admin" - host_path = "/etc/kube-assets/auth/kubeconfig" - } - - volumes { - container_path = "/etc/kubernetes/ca.crt" - host_path = "/etc/kube-assets/tls/ca.crt" + # Make sure that the manifests directory exists + upload { + file = "/etc/kubernetes/manifests/.empty" + content = "" } volumes { @@ -40,6 +36,11 @@ resource "docker_container" "kubelet" { host_path = "/var/lib/docker" } + volumes { + container_path = "/etc/kubernetes" + host_path = "/etc/kubernetes" + } + volumes { container_path = "/var/lib/kubelet" host_path = "/var/lib/kubelet" @@ -86,6 +87,10 @@ resource "docker_container" "kubelet" { container_path = "/var/lib/cni" host_path = "/var/lib/cni" } + # + # "There is no war within the container. Here we are safe. Here we are free." + # - Docker Li agent brainwashing Nemo + # command = [ "kubelet", "--allow-privileged", diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf index d68cf21..0426c4d 100644 --- a/modules/kubelet/variables.tf +++ b/modules/kubelet/variables.tf @@ -27,3 +27,7 @@ variable "dns_ip" { variable "k8s_host" { description = "kubenetes hostname" } + +variable "assets" { + type = "map" +} diff --git a/providers.tf b/providers.tf index 95afb20..1939c9a 100644 --- a/providers.tf +++ b/providers.tf @@ -5,7 +5,7 @@ provider "docker" { } provider "docker" { - host = "tcp://dovpn.vpn.bb8.fun:2376" + host = "tcp://docker.dovpn.bb8.fun:2376" cert_path = "./secrets/sydney" alias = "sydney" version = "~> 2.0.0"