From 7214355a89d4eadfd813579a53e6f119294b532e Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 13 Jan 2019 04:01:14 +0530 Subject: [PATCH] [k8s] Adds kubelet, start stitching things together Challenges: 1. etcd booting before bootkube meant I missed certs 2. etcd can run without certs, but managing docker network over static pod manifests might be tricky :fingers_crossed: --- kubernetes.tf | 18 ++++-- modules/bootkube/main.tf | 13 ++-- modules/bootkube/variables.tf | 8 +-- modules/etcd/main.tf | 32 ++++------ modules/etcd/variables.tf | 12 ++++ modules/kubelet/main.tf | 116 ++++++++++++++++++++++++++++++++++ modules/kubelet/variables.tf | 19 ++++++ 7 files changed, 182 insertions(+), 36 deletions(-) create mode 100644 modules/kubelet/main.tf create mode 100644 modules/kubelet/variables.tf diff --git a/kubernetes.tf b/kubernetes.tf index 47e3220..ed5d3f8 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -3,16 +3,24 @@ module "etcd" { host_ip = "${var.ips["dovpn"]}" data_dir = "/mnt/xwing/etcd" + bootkube_asset_dir = "/etc/kube-assets" + + providers = { + docker = "docker.sydney" + } + + depends_on = "${module.bootkube-start.image}" +} + +module "kubelet-master" { + source = "modules/kubelet" + depends_on = "${module.bootkube-start.image}" + providers = { docker = "docker.sydney" } } -# module "kubelet" { -# source = "modules/kubelet" -# listen_ip = "${var.ips["dovpn"]}" -# } - module "bootkube-render" { source = "modules/bootkube" mode = "render" diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf index 5fb147d..cb9d95f 100644 --- a/modules/bootkube/main.tf +++ b/modules/bootkube/main.tf @@ -5,15 +5,17 @@ resource "docker_container" "render" { volumes { container_path = "/home/.bootkube" - volume_name = "${var.asset_dir_volume_name}" + volume_name = "/etc/kube-assets" } command = [ "bootkube", "render", + "--etcd-servers=http://${host_ip}:2379", "--asset-dir=/home/.bootkube", - "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}", + "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}", "--pod-cidr=${var.pod_cidr}", + "--network-provider=${var.network_provider}", ] network_mode = "host" @@ -28,13 +30,13 @@ resource "docker_container" "start" { volumes { container_path = "/home/.bootkube" - volume_name = "${var.asset_dir_volume_name}" + volume_name = "/etc/kube-assets" read_only = true } volumes { - container_path = "/etc/kubernetes/manifests" - host_path = "/etc/kubernetes/manifests" + container_path = "/etc/kubernetes" + host_path = "/etc/kubernetes" } # "There is no war within the container. Here we are safe. Here we are free." @@ -43,7 +45,6 @@ resource "docker_container" "start" { "bootkube", "start", "--asset-dir=/home/.bootkube", - "--pod-manifest-path=/etc/kubernetes/manifests", ] network_mode = "host" diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index 6098aa6..cf04247 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -1,9 +1,5 @@ // Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube -variable "asset_dir_volume_name" { - default = "k8s-assets" -} - variable "k8s_host" { description = "kubenetes hostname" } @@ -12,6 +8,10 @@ variable "host_port" { default = "8443" } +variable "network_provider" { + default = "flannel" +} + variable "host_ip" {} variable "pod_cidr" { diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf index fb22601..26aa193 100644 --- a/modules/etcd/main.tf +++ b/modules/etcd/main.tf @@ -8,32 +8,15 @@ module "container" { host = "" } - networks = [] + networks = ["${docker_network.etcd.id}"] volumes = [ - { - host_path = "/usr/share/ca-certificates/" - container_path = "/etc/ssl/certs" - }, { host_path = "${var.data_dir}" container_path = "/etcd-data" }, ] - ports = [ - { - internal = 2379 - external = 2379 - ip = "${var.host_ip}" - }, - { - internal = 2380 - external = 2380 - ip = "${var.host_ip}" - }, - ] - command = [ "/usr/local/bin/etcd", "--data-dir=/etcd-data", @@ -42,7 +25,14 @@ module "container" { "--initial-advertise-peer-urls=http://${var.host_ip}:2380", "--initial-cluster=${var.node_name}=http://${var.host_ip}:2380", ] - - # "--listen-client-urls=http://0.0.0.0:2379", - # "--listen-peer-urls=http://0.0.0.0:2380", +} + +resource "docker_network" "etcd" { + name = "etcd" + driver = "bridge" + + ipam_config { + subnet = "10.10.10.0/25" + gateway = "10.10.10.1" + } } diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf index dbaef83..6babce5 100644 --- a/modules/etcd/variables.tf +++ b/modules/etcd/variables.tf @@ -9,7 +9,19 @@ variable "data_dir" { type = "string" } +variable "bootkube_asset_dir" { + description = "bootkube render is run against this directory" + type = "string" + default = "/etc/kube-assets" +} + variable "node_name" { description = "name of the etcd node" default = "master" } + +variable "depends_on" { + default = [] + + type = "list" +} diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf new file mode 100644 index 0000000..2ca7012 --- /dev/null +++ b/modules/kubelet/main.tf @@ -0,0 +1,116 @@ +// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper +resource "docker_container" "kubelet" { + image = "${docker_image.image.latest}" + name = "kubelet-static" + + volumes { + container_path = "/etc/kubernetes" + host_path = "/etc/kubernetes" + } + + volumes { + container_path = "/etc/kubernetes/kubeconfig" + host_path = "/etc/kube-assets/auth/kubeconfig-kubelet" + } + + volumes { + container_path = "/etc/kubernetes/kubeconfig-admin" + host_path = "/etc/kube-assets/auth/kubeconfig" + } + + volumes { + container_path = "/etc/kubernetes/ca.crt" + host_path = "/etc/kube-assets/tls/ca.crt" + } + + volumes { + container_path = "/etc/ssl/certs" + host_path = "/etc/ssl/certs" + read_only = true + } + + volumes { + container_path = "/usr/share/ca-certificates" + host_path = "/usr/share/ca-certificates" + read_only = true + } + + volumes { + container_path = "/var/lib/docker" + host_path = "/var/lib/docker" + } + + volumes { + container_path = "/var/lib/kubelet" + host_path = "/var/lib/kubelet" + } + + volumes { + container_path = "/var/log" + host_path = "/var/log" + } + + volumes { + container_path = "/run" + host_path = "/run" + } + + volumes { + container_path = "/lib/modules" + host_path = "/lib/modules" + read_only = true + } + + volumes { + container_path = "/etc/os-release" + host_path = "/usr/lib/os-release" + read_only = true + } + + volumes { + container_path = "/etc/machine-id" + host_path = "/etc/machine-id" + read_only = true + } + + // Deviates from kubelet-wrapper + + volumes { + container_path = "/var/lib/cni" + host_path = "/var/lib/cni" + } + command = [ + "kubelet", + "--kubeconfig=/etc/kubernetes/kubeconfig", + "--client-ca-file=/etc/kubernetes/ca.crt", + "--anonymous-auth=false", + "--cni-conf-dir=/etc/kubernetes/cni/net.d", + "--network-plugin=cni", + "--lock-file=/var/run/lock/kubelet.lock", + "--exit-on-lock-contention", + "--pod-manifest-path=/etc/kubernetes/manifests", + "--allow-privileged", + "--minimum-container-ttl-duration=10m0s", + "--cluster_dns=10.25.0.10", + "--cluster_domain=k8s.bb8.fun", + ] + + # TODO + # "--register-with-taints=${var.node_taints}", + # "--node-labels=${var.node_label}", + + network_mode = "host" + privileged = true + restart = "no" + must_run = false + max_retry_count = 1 +} + +data "docker_registry_image" "image" { + name = "gcr.io/google_containers/hyperkube:v${var.version}" +} + +resource "docker_image" "image" { + name = "${data.docker_registry_image.image.name}" + pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"] +} diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf new file mode 100644 index 0000000..b754c86 --- /dev/null +++ b/modules/kubelet/variables.tf @@ -0,0 +1,19 @@ +variable "version" { + description = "kubelet version" + default = "1.13.2" +} + +variable "node_label" { + description = "kubelet version" + default = "node.kubernetes.io/master" +} + +variable "depends_on" { + default = [] + + type = "list" +} + +variable "asset_dir_volume_name" { + default = "k8s-assets" +}