🏡 index : github.com/captn3m0/nebula.git

7214355a89d4eadfd813579a53e6f119294b532e [patch]

617f3d71ed7aec9690646bbb5ff08abd0884812a

97ef9179e4d5cfac9369c55e5f532a1722bac10f

7214355a89d4eadfd813579a53e6f119294b532e.tar.gz

[k8s] Adds kubelet, start stitching things together

Challenges:

1. etcd booting before bootkube meant I missed certs
2. etcd can run without certs, but managing docker network
   over static pod manifests might be tricky :fingers_crossed:

Diff

 kubernetes.tf                 |  16 ++++++++++++----
 modules/bootkube/main.tf      |  13 +++++++------
 modules/bootkube/variables.tf |   8 ++++----
 modules/etcd/main.tf          |  32 ++++++++++++--------------------
 modules/etcd/variables.tf     |  12 ++++++++++++
 modules/kubelet/main.tf       | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 modules/kubelet/variables.tf  |  19 +++++++++++++++++++
 7 files changed, 181 insertions(+), 35 deletions(-)

diff --git a/kubernetes.tf b/kubernetes.tf
index 47e3220..ed5d3f8 100644
--- a/kubernetes.tf
+++ a/kubernetes.tf
@@ -1,17 +1,25 @@
module "etcd" {

  source   = "modules/etcd"
  host_ip  = "${var.ips["dovpn"]}"
  data_dir = "/mnt/xwing/etcd"

  bootkube_asset_dir = "/etc/kube-assets"

  providers = {

    docker = "docker.sydney"
  }

  depends_on = "${module.bootkube-start.image}"
}

module "kubelet-master" {

  source     = "modules/kubelet"
  depends_on = "${module.bootkube-start.image}"

# module "kubelet" {
#   source = "modules/kubelet"
#   listen_ip =  "${var.ips["dovpn"]}"
# }
  providers = {

    docker = "docker.sydney"
  }
}

module "bootkube-render" {

  source   = "modules/bootkube"
diff --git a/modules/bootkube/main.tf b/modules/bootkube/main.tf
index 5fb147d..cb9d95f 100644
--- a/modules/bootkube/main.tf
+++ a/modules/bootkube/main.tf
@@ -5,15 +5,17 @@

  volumes {

    container_path = "/home/.bootkube"
    volume_name    = "${var.asset_dir_volume_name}"
    volume_name    = "/etc/kube-assets"
  }

  command = [

    "bootkube",
    "render",
    "--etcd-servers=http://${host_ip}:2379",
    "--asset-dir=/home/.bootkube",
    "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}",
    "--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}",
    "--pod-cidr=${var.pod_cidr}",
    "--network-provider=${var.network_provider}",
  ]

  network_mode    = "host"
@@ -28,13 +30,13 @@

  volumes {

    container_path = "/home/.bootkube"
    volume_name    = "${var.asset_dir_volume_name}"
    volume_name    = "/etc/kube-assets"
    read_only      = true
  }

  volumes {

    container_path = "/etc/kubernetes/manifests"
    host_path      = "/etc/kubernetes/manifests"
    container_path = "/etc/kubernetes"
    host_path      = "/etc/kubernetes"
  }

  # "There is no war within the container. Here we are safe. Here we are free."
@@ -43,7 +45,6 @@
    "bootkube",
    "start",
    "--asset-dir=/home/.bootkube",
    "--pod-manifest-path=/etc/kubernetes/manifests",
  ]

  network_mode    = "host"
diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf
index 6098aa6..cf04247 100644
--- a/modules/bootkube/variables.tf
+++ a/modules/bootkube/variables.tf
@@ -1,15 +1,15 @@
// Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube

variable "asset_dir_volume_name" {

  default = "k8s-assets"
}

variable "k8s_host" {

  description = "kubenetes hostname"
}

variable "host_port" {

  default = "8443"
}

variable "network_provider" {

  default = "flannel"
}

variable "host_ip" {}
diff --git a/modules/etcd/main.tf b/modules/etcd/main.tf
index fb22601..26aa193 100644
--- a/modules/etcd/main.tf
+++ a/modules/etcd/main.tf
@@ -8,30 +8,13 @@
    host   = ""
  }

  networks = []
  networks = ["${docker_network.etcd.id}"]

  volumes = [

    {
      host_path      = "/usr/share/ca-certificates/"
      container_path = "/etc/ssl/certs"
    },
    {
      host_path      = "${var.data_dir}"
      container_path = "/etcd-data"
    },
  ]

  ports = [

    {
      internal = 2379
      external = 2379
      ip       = "${var.host_ip}"
    },
    {
      internal = 2380
      external = 2380
      ip       = "${var.host_ip}"
    },
  ]

  command = [

@@ -42,7 +25,14 @@
    "--initial-advertise-peer-urls=http://${var.host_ip}:2380",
    "--initial-cluster=${var.node_name}=http://${var.host_ip}:2380",
  ]

  # "--listen-client-urls=http://0.0.0.0:2379",
  # "--listen-peer-urls=http://0.0.0.0:2380",
}

resource "docker_network" "etcd" {

  name   = "etcd"
  driver = "bridge"

  ipam_config {

    subnet  = "10.10.10.0/25"
    gateway = "10.10.10.1"
  }
}
diff --git a/modules/etcd/variables.tf b/modules/etcd/variables.tf
index dbaef83..6babce5 100644
--- a/modules/etcd/variables.tf
+++ a/modules/etcd/variables.tf
@@ -9,7 +9,19 @@
  type        = "string"
}

variable "bootkube_asset_dir" {

  description = "bootkube render is run against this directory"
  type        = "string"
  default     = "/etc/kube-assets"
}

variable "node_name" {

  description = "name of the etcd node"
  default     = "master"
}

variable "depends_on" {

  default = []

  type = "list"
}
diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf
new file mode 100644
index 0000000..2ca7012 100644
--- /dev/null
+++ a/modules/kubelet/main.tf
@@ -1,0 +1,116 @@
// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper
resource "docker_container" "kubelet" {

  image = "${docker_image.image.latest}"
  name  = "kubelet-static"

  volumes {

    container_path = "/etc/kubernetes"
    host_path      = "/etc/kubernetes"
  }

  volumes {

    container_path = "/etc/kubernetes/kubeconfig"
    host_path      = "/etc/kube-assets/auth/kubeconfig-kubelet"
  }

  volumes {

    container_path = "/etc/kubernetes/kubeconfig-admin"
    host_path      = "/etc/kube-assets/auth/kubeconfig"
  }

  volumes {

    container_path = "/etc/kubernetes/ca.crt"
    host_path      = "/etc/kube-assets/tls/ca.crt"
  }

  volumes {

    container_path = "/etc/ssl/certs"
    host_path      = "/etc/ssl/certs"
    read_only      = true
  }

  volumes {

    container_path = "/usr/share/ca-certificates"
    host_path      = "/usr/share/ca-certificates"
    read_only      = true
  }

  volumes {

    container_path = "/var/lib/docker"
    host_path      = "/var/lib/docker"
  }

  volumes {

    container_path = "/var/lib/kubelet"
    host_path      = "/var/lib/kubelet"
  }

  volumes {

    container_path = "/var/log"
    host_path      = "/var/log"
  }

  volumes {

    container_path = "/run"
    host_path      = "/run"
  }

  volumes {

    container_path = "/lib/modules"
    host_path      = "/lib/modules"
    read_only      = true
  }

  volumes {

    container_path = "/etc/os-release"
    host_path      = "/usr/lib/os-release"
    read_only      = true
  }

  volumes {

    container_path = "/etc/machine-id"
    host_path      = "/etc/machine-id"
    read_only      = true
  }

  // Deviates from kubelet-wrapper

  volumes {

    container_path = "/var/lib/cni"
    host_path      = "/var/lib/cni"
  }
  command = [

    "kubelet",
    "--kubeconfig=/etc/kubernetes/kubeconfig",
    "--client-ca-file=/etc/kubernetes/ca.crt",
    "--anonymous-auth=false",
    "--cni-conf-dir=/etc/kubernetes/cni/net.d",
    "--network-plugin=cni",
    "--lock-file=/var/run/lock/kubelet.lock",
    "--exit-on-lock-contention",
    "--pod-manifest-path=/etc/kubernetes/manifests",
    "--allow-privileged",
    "--minimum-container-ttl-duration=10m0s",
    "--cluster_dns=10.25.0.10",
    "--cluster_domain=k8s.bb8.fun",
  ]

  # TODO
  # "--register-with-taints=${var.node_taints}",
  # "--node-labels=${var.node_label}",

  network_mode    = "host"
  privileged      = true
  restart         = "no"
  must_run        = false
  max_retry_count = 1
}

data "docker_registry_image" "image" {

  name = "gcr.io/google_containers/hyperkube:v${var.version}"
}

resource "docker_image" "image" {

  name          = "${data.docker_registry_image.image.name}"
  pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"]
}
diff --git a/modules/kubelet/variables.tf b/modules/kubelet/variables.tf
new file mode 100644
index 0000000..b754c86 100644
--- /dev/null
+++ a/modules/kubelet/variables.tf
@@ -1,0 +1,19 @@
variable "version" {

  description = "kubelet version"
  default     = "1.13.2"
}

variable "node_label" {

  description = "kubelet version"
  default     = "node.kubernetes.io/master"
}

variable "depends_on" {

  default = []

  type = "list"
}

variable "asset_dir_volume_name" {

  default = "k8s-assets"
}