From a3dec142add5ec62e1599182aeda85b3fae10d3c Mon Sep 17 00:00:00 2001
From: Nemo <me@captnemo.in>
Date: Sun, 27 Jan 2019 04:02:59 +0530
Subject: [PATCH] [k8s] Upload all assets using upload{} inside docker_container
---
kubernetes.tf | 27 +++++++++++++++++++++++++--
providers.tf | 2 +-
modules/bootkube/data.tf | 1 -
modules/bootkube/main.tf | 213 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------
modules/bootkube/variables.tf | 6 ++++++
modules/kubelet/main.tf | 33 +++++++++++++++++++++------------
modules/kubelet/variables.tf | 4 ++++
7 files changed, 230 insertions(+), 56 deletions(-)
diff --git a/kubernetes.tf b/kubernetes.tf
index 9792867..c253da0 100644
--- a/kubernetes.tf
+++ a/kubernetes.tf
@@ -24,6 +24,13 @@
host_ip = "${var.ips["dovpn"]}"
k8s_host = "k8s.${var.root-domain}"
+ assets = {
+ kubeconfig = "${module.bootkube.kubeconfig-kubelet}"
+ ca_cert = "${base64decode(module.bootkube.ca_cert)}"
+ kubelet_cert = "${base64decode(module.bootkube.kubelet_cert)}"
+ kubelet_key = "${base64decode(module.bootkube.kubelet_key)}"
+ }
+
depends_on = "${module.bootkube-start.image}"
providers = {
@@ -32,10 +39,22 @@
}
module "bootkube-start" {
- source = "modules/bootkube"
- mode = "start"
- host_ip = "${var.ips["dovpn"]}"
- k8s_host = "k8s.${var.root-domain}"
+ source = "modules/bootkube"
+ mode = "start"
+ host_ip = "${var.ips["dovpn"]}"
+ k8s_host = "k8s.${var.root-domain}"
+ asset-dir = "${path.root}/k8s"
+
+ assets = {
+ kubeconfig-kubelet = "${module.bootkube.kubeconfig-kubelet}"
+ etcd_ca_cert = "${module.bootkube.etcd_ca_cert}"
+ etcd_client_cert = "${module.bootkube.etcd_client_cert}"
+ etcd_client_key = "${module.bootkube.etcd_client_key}"
+ etcd_server_cert = "${module.bootkube.etcd_server_cert}"
+ etcd_server_key = "${module.bootkube.etcd_server_key}"
+ etcd_peer_cert = "${module.bootkube.etcd_peer_cert}"
+ etcd_peer_key = "${module.bootkube.etcd_peer_key}"
+ }
providers = {
docker = "docker.sydney"
diff --git a/providers.tf b/providers.tf
index 95afb20..1939c9a 100644
--- a/providers.tf
+++ a/providers.tf
@@ -5,7 +5,7 @@
}
provider "docker" {
- host = "tcp://dovpn.vpn.bb8.fun:2376"
+ host = "tcp://docker.dovpn.bb8.fun:2376"
cert_path = "./secrets/sydney"
alias = "sydney"
version = "~> 2.0.0"
diff --git a/modules/bootkube/data.tf b/modules/bootkube/data.tf
deleted file mode 100644
index 8b13789..0000000 100644
--- a/modules/bootkube/data.tf
+++ /dev/null
@@ -1,1 +1,0 @@
-
diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf
index 00b019c..6c0d6b8 100644
--- a/modules/bootkube/main.tf
+++ a/modules/bootkube/main.tf
@@ -1,52 +1,193 @@
-resource "docker_container" "render" {
- count = "${var.mode == "render" ? 1 : 0}"
+resource "docker_container" "bootkube" {
image = "${docker_image.image.latest}"
- name = "bootkube-render"
+ name = "bootkube"
volumes {
- container_path = "/home/.bootkube"
- volume_name = "/etc/kube-assets"
+ container_path = "/etc/kubernetes/manifests"
+ host_path = "/etc/kubernetes/manifests"
}
- command = [
- "/bootkube",
- "render",
- "--etcd-servers=https://${var.host_ip}:2379",
- "--asset-dir=/home/.bootkube",
- "--api-servers=https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}",
- "--pod-cidr=${var.pod_cidr}",
- "--network-provider=${var.network_provider}",
- ]
-
- network_mode = "host"
- restart = "on-failure"
- max_retry_count = 5
-}
-
-resource "docker_container" "start" {
- count = "${var.mode == "start" ? 1 : 0}"
- image = "${docker_image.image.latest}"
- name = "bootkube-${var.mode}"
+ # bootstrap manifests
- volumes {
- container_path = "/home/.bootkube"
- volume_name = "/etc/kube-assets"
- read_only = true
+ upload {
+ content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-apiserver.yaml")}"
+ file = "/home/.bootkube/bootstra-manifests/bootstrap-apiserver.yaml"
}
-
- volumes {
- container_path = "/etc/kubernetes"
- host_path = "/etc/kubernetes"
+ upload {
+ content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-controller-manager.yaml")}"
+ file = "/home/.bootkube/bootstra-manifests/bootstrap-controller-manager.yaml"
}
-
- # "There is no war within the container. Here we are safe. Here we are free."
- # - Docker Li agent brainwashing Nemo
+ upload {
+ content = "${file("${var.asset-dir}/bootstra-manifests/bootstrap-scheduler.yaml")}"
+ file = "/home/.bootkube/bootstra-manifests/bootstrap-scheduler.yaml"
+ }
+ # Cluster Networking
+ upload {
+ content = "${file("${var.asset-dir}/manifests-networking/cluster-role-binding.yaml")}"
+ file = "/home/.bootkube/manifests-networking/cluster-role-binding.yaml"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/manifests-networking/cluster-role.yaml")}"
+ file = "/home/.bootkube/manifests-networking/cluster-role.yaml"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/manifests-networking/config.yaml")}"
+ file = "/home/.bootkube/manifests-networking/config.yaml"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/manifests-networking/daemonset.yaml")}"
+ file = "/home/.bootkube/manifests-networking/daemonset.yaml"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/manifests-networkingservice-account.yaml")}"
+ file = "/home/.bootkube/manifests-networking/service-account.yaml"
+ }
+ # TLS
+ upload {
+ file = "/home/.bootkube/tls/service-account.pub"
+ content = "${file("${var.asset-dir}/tls/service-account.pub")}"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/tls/ca.key")}"
+ file = "/home/.bootkube/tls/ca.key"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/tls/ca.crt")}"
+ file = "/home/.bootkube/tls/ca.crt"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/tls/apiserver.key")}"
+ file = "/home/.bootkube/tls/apiserver.key"
+ }
+ upload {
+ content = "${file("${var.asset-dir}/tls/apiserver.crt")}"
+ file = "/home/.bootkube/tls/apiserver.crt"
+ }
+ upload {
+ content = "${var.assets["kubelet_cert"]}"
+ file = "/home/.bootkube/tls/kubelet.crt"
+ }
+ upload {
+ content = "${var.assets["kubelet_key"]}"
+ file = "/home/.bootkube/tls/kubelet.key"
+ }
+ # TODO: Generate Filenames Dynamically
+ # TODO: Check if this is needed at all
+ upload {
+ content = "${file("${var.asset-dir}/auth/k8s.bb8.fun-config")}"
+ file = "/home/.bootkube/auth/k8s.bb8.fun-config"
+ }
+ # auth/kubeconfig-kubelet
+ upload {
+ content = "${var.assets["kubeconfig-kubelet"]}"
+ file = "/home/.bootkube/auth/kubeconfig-kubelet"
+ }
+ # Manifests Directory
+ upload {
+ file = "/home/.bootkube/manifests/kube-apiserver-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-apiserver-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-apiserver-sa.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-apiserver-sa.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-apiserver-secret.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-apiserver-secret.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-apiserver.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-apiserver.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kubeconfig-in-cluster.yaml"
+ content = "${file("${var.asset-dir}/manifests/kubeconfig-in-cluster.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-controller-manager-disruption.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-controller-manager-disruption.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-controller-manager-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-controller-manager-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-controller-manager-sa.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-controller-manager-sa.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-controller-manager-secret.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-controller-manager-secret.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-controller-manager.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-controller-manager.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kubelet-nodes-cluster-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/kubelet-nodes-cluster-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-proxy-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-proxy-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-proxy-sa.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-proxy-sa.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-proxy.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-proxy.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-scheduler-disruption.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-scheduler-disruption.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-scheduler-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-scheduler-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-scheduler-sa.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-scheduler-sa.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-scheduler-volume-scheduler-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-scheduler-volume-scheduler-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/kube-scheduler.yaml"
+ content = "${file("${var.asset-dir}/manifests/kube-scheduler.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/pod-checkpointer-cluster-role.yaml"
+ content = "${file("${var.asset-dir}/manifests/pod-checkpointer-cluster-role.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/pod-checkpointer-role-binding.yaml"
+ content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role-binding.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/pod-checkpointer-role.yaml"
+ content = "${file("${var.asset-dir}/manifests/pod-checkpointer-role.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/pod-checkpointer-sa.yaml"
+ content = "${file("${var.asset-dir}/manifests/pod-checkpointer-sa.yaml")}"
+ }
+ upload {
+ file = "/home/.bootkube/manifests/pod-checkpointer.yaml"
+ content = "${file("${var.asset-dir}/manifests/pod-checkpointer.yaml")}"
+ }
command = [
"/bootkube",
"start",
"--asset-dir=/home/.bootkube",
]
-
network_mode = "host"
restart = "on-failure"
max_retry_count = 5
diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf
index cf04247..1325b72 100644
--- a/modules/bootkube/variables.tf
+++ a/modules/bootkube/variables.tf
@@ -33,3 +33,9 @@
type = "list"
}
+
+variable "assets" {
+ type = "map"
+}
+
+variable "asset-dir" {}
diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf
index ea2eed8..06f9e96 100644
--- a/modules/kubelet/main.tf
+++ a/modules/kubelet/main.tf
@@ -1,26 +1,22 @@
// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper
resource "docker_container" "kubelet" {
image = "${docker_image.image.latest}"
name = "kubelet-static"
- volumes {
- container_path = "/etc/kubernetes"
- host_path = "/etc/kubernetes"
- }
-
- volumes {
- container_path = "/etc/kubernetes/kubeconfig"
- host_path = "/etc/kube-assets/auth/kubeconfig-kubelet"
+ upload {
+ file = "/etc/kubernetes/kubeconfig"
+ content = "${var.assets["kubeconfig"]}"
}
- volumes {
- container_path = "/etc/kubernetes/kubeconfig-admin"
- host_path = "/etc/kube-assets/auth/kubeconfig"
+ upload {
+ file = "/etc/kubernetes/ca.crt"
+ content = "${var.assets["ca_cert"]}"
}
- volumes {
- container_path = "/etc/kubernetes/ca.crt"
- host_path = "/etc/kube-assets/tls/ca.crt"
+ # Make sure that the manifests directory exists
+ upload {
+ file = "/etc/kubernetes/manifests/.empty"
+ content = ""
}
volumes {
@@ -38,6 +34,11 @@
volumes {
container_path = "/var/lib/docker"
host_path = "/var/lib/docker"
+ }
+
+ volumes {
+ container_path = "/etc/kubernetes"
+ host_path = "/etc/kubernetes"
}
volumes {
@@ -86,6 +87,10 @@
container_path = "/var/lib/cni"
host_path = "/var/lib/cni"
}
+ #
+ # "There is no war within the container. Here we are safe. Here we are free."
+ # - Docker Li agent brainwashing Nemo
+ #
command = [
"kubelet",
"--allow-privileged",
diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf
index d68cf21..0426c4d 100644
--- a/modules/kubelet/variables.tf
+++ a/modules/kubelet/variables.tf
@@ -27,3 +27,7 @@
variable "k8s_host" {
description = "kubenetes hostname"
}
+
+variable "assets" {
+ type = "map"
+}
--
rgit 0.1.5