From 7214355a89d4eadfd813579a53e6f119294b532e Mon Sep 17 00:00:00 2001
From: Nemo <me@captnemo.in>
Date: Sun, 13 Jan 2019 04:01:14 +0530
Subject: [PATCH] [k8s] Adds kubelet, start stitching things together
Challenges:
1. etcd booting before bootkube meant I missed certs
2. etcd can run without certs, but managing docker network
over static pod manifests might be tricky :fingers_crossed:
---
kubernetes.tf | 16 ++++++++++++----
modules/bootkube/main.tf | 13 +++++++------
modules/bootkube/variables.tf | 8 ++++----
modules/etcd/main.tf | 32 ++++++++++++--------------------
modules/etcd/variables.tf | 12 ++++++++++++
modules/kubelet/main.tf | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
modules/kubelet/variables.tf | 19 +++++++++++++++++++
7 files changed, 181 insertions(+), 35 deletions(-)
diff --git a/kubernetes.tf b/kubernetes.tf
index 47e3220..ed5d3f8 100644
--- a/kubernetes.tf
+++ a/kubernetes.tf
@@ -1,17 +1,25 @@
module "etcd" {
source = "modules/etcd"
host_ip = "${var.ips["dovpn"]}"
data_dir = "/mnt/xwing/etcd"
+ bootkube_asset_dir = "/etc/kube-assets"
+
providers = {
docker = "docker.sydney"
}
+
+ depends_on = "${module.bootkube-start.image}"
}
+
+module "kubelet-master" {
+ source = "modules/kubelet"
+ depends_on = "${module.bootkube-start.image}"
-# module "kubelet" {
-# source = "modules/kubelet"
-# listen_ip = "${var.ips["dovpn"]}"
-# }
+ providers = {
+ docker = "docker.sydney"
+ }
+}
module "bootkube-render" {
source = "modules/bootkube"
diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf
index 5fb147d..cb9d95f 100644
--- a/modules/bootkube/main.tf
+++ a/modules/bootkube/main.tf
@@ -5,15 +5,17 @@
volumes {
container_path = "/home/.bootkube"
- volume_name = "${var.asset_dir_volume_name}"
+ volume_name = "/etc/kube-assets"
}
command = [
"bootkube",
"render",
+ "--etcd-servers=http://${host_ip}:2379",
"--asset-dir=/home/.bootkube",
- "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}",
+ "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}",
"--pod-cidr=${var.pod_cidr}",
+ "--network-provider=${var.network_provider}",
]
network_mode = "host"
@@ -28,13 +30,13 @@
volumes {
container_path = "/home/.bootkube"
- volume_name = "${var.asset_dir_volume_name}"
+ volume_name = "/etc/kube-assets"
read_only = true
}
volumes {
- container_path = "/etc/kubernetes/manifests"
- host_path = "/etc/kubernetes/manifests"
+ container_path = "/etc/kubernetes"
+ host_path = "/etc/kubernetes"
}
# "There is no war within the container. Here we are safe. Here we are free."
@@ -43,7 +45,6 @@
"bootkube",
"start",
"--asset-dir=/home/.bootkube",
- "--pod-manifest-path=/etc/kubernetes/manifests",
]
network_mode = "host"
diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf
index 6098aa6..cf04247 100644
--- a/modules/bootkube/variables.tf
+++ a/modules/bootkube/variables.tf
@@ -1,15 +1,15 @@
// Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube
-variable "asset_dir_volume_name" {
- default = "k8s-assets"
-}
-
variable "k8s_host" {
description = "kubenetes hostname"
}
variable "host_port" {
default = "8443"
+}
+
+variable "network_provider" {
+ default = "flannel"
}
variable "host_ip" {}
diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf
index fb22601..26aa193 100644
--- a/modules/etcd/main.tf
+++ a/modules/etcd/main.tf
@@ -8,30 +8,13 @@
host = ""
}
- networks = []
+ networks = ["${docker_network.etcd.id}"]
volumes = [
{
- host_path = "/usr/share/ca-certificates/"
- container_path = "/etc/ssl/certs"
- },
- {
host_path = "${var.data_dir}"
container_path = "/etcd-data"
- },
- ]
-
- ports = [
- {
- internal = 2379
- external = 2379
- ip = "${var.host_ip}"
},
- {
- internal = 2380
- external = 2380
- ip = "${var.host_ip}"
- },
]
command = [
@@ -42,7 +25,14 @@
"--initial-advertise-peer-urls=http://${var.host_ip}:2380",
"--initial-cluster=${var.node_name}=http://${var.host_ip}:2380",
]
-
- # "--listen-client-urls=http://0.0.0.0:2379",
- # "--listen-peer-urls=http://0.0.0.0:2380",
+}
+
+resource "docker_network" "etcd" {
+ name = "etcd"
+ driver = "bridge"
+
+ ipam_config {
+ subnet = "10.10.10.0/25"
+ gateway = "10.10.10.1"
+ }
}
diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf
index dbaef83..6babce5 100644
--- a/modules/etcd/variables.tf
+++ a/modules/etcd/variables.tf
@@ -9,7 +9,19 @@
type = "string"
}
+variable "bootkube_asset_dir" {
+ description = "bootkube render is run against this directory"
+ type = "string"
+ default = "/etc/kube-assets"
+}
+
variable "node_name" {
description = "name of the etcd node"
default = "master"
+}
+
+variable "depends_on" {
+ default = []
+
+ type = "list"
}
diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf
new file mode 100644
index 0000000..2ca7012 100644
--- /dev/null
+++ a/modules/kubelet/main.tf
@@ -1,0 +1,116 @@
+// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper
+resource "docker_container" "kubelet" {
+ image = "${docker_image.image.latest}"
+ name = "kubelet-static"
+
+ volumes {
+ container_path = "/etc/kubernetes"
+ host_path = "/etc/kubernetes"
+ }
+
+ volumes {
+ container_path = "/etc/kubernetes/kubeconfig"
+ host_path = "/etc/kube-assets/auth/kubeconfig-kubelet"
+ }
+
+ volumes {
+ container_path = "/etc/kubernetes/kubeconfig-admin"
+ host_path = "/etc/kube-assets/auth/kubeconfig"
+ }
+
+ volumes {
+ container_path = "/etc/kubernetes/ca.crt"
+ host_path = "/etc/kube-assets/tls/ca.crt"
+ }
+
+ volumes {
+ container_path = "/etc/ssl/certs"
+ host_path = "/etc/ssl/certs"
+ read_only = true
+ }
+
+ volumes {
+ container_path = "/usr/share/ca-certificates"
+ host_path = "/usr/share/ca-certificates"
+ read_only = true
+ }
+
+ volumes {
+ container_path = "/var/lib/docker"
+ host_path = "/var/lib/docker"
+ }
+
+ volumes {
+ container_path = "/var/lib/kubelet"
+ host_path = "/var/lib/kubelet"
+ }
+
+ volumes {
+ container_path = "/var/log"
+ host_path = "/var/log"
+ }
+
+ volumes {
+ container_path = "/run"
+ host_path = "/run"
+ }
+
+ volumes {
+ container_path = "/lib/modules"
+ host_path = "/lib/modules"
+ read_only = true
+ }
+
+ volumes {
+ container_path = "/etc/os-release"
+ host_path = "/usr/lib/os-release"
+ read_only = true
+ }
+
+ volumes {
+ container_path = "/etc/machine-id"
+ host_path = "/etc/machine-id"
+ read_only = true
+ }
+
+ // Deviates from kubelet-wrapper
+
+ volumes {
+ container_path = "/var/lib/cni"
+ host_path = "/var/lib/cni"
+ }
+ command = [
+ "kubelet",
+ "--kubeconfig=/etc/kubernetes/kubeconfig",
+ "--client-ca-file=/etc/kubernetes/ca.crt",
+ "--anonymous-auth=false",
+ "--cni-conf-dir=/etc/kubernetes/cni/net.d",
+ "--network-plugin=cni",
+ "--lock-file=/var/run/lock/kubelet.lock",
+ "--exit-on-lock-contention",
+ "--pod-manifest-path=/etc/kubernetes/manifests",
+ "--allow-privileged",
+ "--minimum-container-ttl-duration=10m0s",
+ "--cluster_dns=10.25.0.10",
+ "--cluster_domain=k8s.bb8.fun",
+ ]
+
+ # TODO
+ # "--register-with-taints=${var.node_taints}",
+ # "--node-labels=${var.node_label}",
+
+ network_mode = "host"
+ privileged = true
+ restart = "no"
+ must_run = false
+ max_retry_count = 1
+}
+
+data "docker_registry_image" "image" {
+ name = "gcr.io/google_containers/hyperkube:v${var.version}"
+}
+
+resource "docker_image" "image" {
+ name = "${data.docker_registry_image.image.name}"
+ pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"]
+}
diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf
new file mode 100644
index 0000000..b754c86 100644
--- /dev/null
+++ a/modules/kubelet/variables.tf
@@ -1,0 +1,19 @@
+variable "version" {
+ description = "kubelet version"
+ default = "1.13.2"
+}
+
+variable "node_label" {
+ description = "kubelet version"
+ default = "node.kubernetes.io/master"
+}
+
+variable "depends_on" {
+ default = []
+
+ type = "list"
+}
+
+variable "asset_dir_volume_name" {
+ default = "k8s-assets"
+}
--
rgit 0.1.5