[k8s] Adds kubelet, start stitching things together
Challenges: 1. etcd booting before bootkube meant I missed certs 2. etcd can run without certs, but managing docker network over static pod manifests might be tricky :fingers_crossed:
This commit is contained in:
parent
97ef9179e4
commit
7214355a89
|
@ -3,16 +3,24 @@ module "etcd" {
|
||||||
host_ip = "${var.ips["dovpn"]}"
|
host_ip = "${var.ips["dovpn"]}"
|
||||||
data_dir = "/mnt/xwing/etcd"
|
data_dir = "/mnt/xwing/etcd"
|
||||||
|
|
||||||
|
bootkube_asset_dir = "/etc/kube-assets"
|
||||||
|
|
||||||
|
providers = {
|
||||||
|
docker = "docker.sydney"
|
||||||
|
}
|
||||||
|
|
||||||
|
depends_on = "${module.bootkube-start.image}"
|
||||||
|
}
|
||||||
|
|
||||||
|
module "kubelet-master" {
|
||||||
|
source = "modules/kubelet"
|
||||||
|
depends_on = "${module.bootkube-start.image}"
|
||||||
|
|
||||||
providers = {
|
providers = {
|
||||||
docker = "docker.sydney"
|
docker = "docker.sydney"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# module "kubelet" {
|
|
||||||
# source = "modules/kubelet"
|
|
||||||
# listen_ip = "${var.ips["dovpn"]}"
|
|
||||||
# }
|
|
||||||
|
|
||||||
module "bootkube-render" {
|
module "bootkube-render" {
|
||||||
source = "modules/bootkube"
|
source = "modules/bootkube"
|
||||||
mode = "render"
|
mode = "render"
|
||||||
|
|
|
@ -5,15 +5,17 @@ resource "docker_container" "render" {
|
||||||
|
|
||||||
volumes {
|
volumes {
|
||||||
container_path = "/home/.bootkube"
|
container_path = "/home/.bootkube"
|
||||||
volume_name = "${var.asset_dir_volume_name}"
|
volume_name = "/etc/kube-assets"
|
||||||
}
|
}
|
||||||
|
|
||||||
command = [
|
command = [
|
||||||
"bootkube",
|
"bootkube",
|
||||||
"render",
|
"render",
|
||||||
|
"--etcd-servers=http://${host_ip}:2379",
|
||||||
"--asset-dir=/home/.bootkube",
|
"--asset-dir=/home/.bootkube",
|
||||||
"--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host},https://${var.host_ip}:${var.host_port}",
|
"--api-servers=https://kubernetes.default:${var.host_port},https://${var.k8s_host}:${var.host_port},https://${var.host_ip}:${var.host_port}",
|
||||||
"--pod-cidr=${var.pod_cidr}",
|
"--pod-cidr=${var.pod_cidr}",
|
||||||
|
"--network-provider=${var.network_provider}",
|
||||||
]
|
]
|
||||||
|
|
||||||
network_mode = "host"
|
network_mode = "host"
|
||||||
|
@ -28,13 +30,13 @@ resource "docker_container" "start" {
|
||||||
|
|
||||||
volumes {
|
volumes {
|
||||||
container_path = "/home/.bootkube"
|
container_path = "/home/.bootkube"
|
||||||
volume_name = "${var.asset_dir_volume_name}"
|
volume_name = "/etc/kube-assets"
|
||||||
read_only = true
|
read_only = true
|
||||||
}
|
}
|
||||||
|
|
||||||
volumes {
|
volumes {
|
||||||
container_path = "/etc/kubernetes/manifests"
|
container_path = "/etc/kubernetes"
|
||||||
host_path = "/etc/kubernetes/manifests"
|
host_path = "/etc/kubernetes"
|
||||||
}
|
}
|
||||||
|
|
||||||
# "There is no war within the container. Here we are safe. Here we are free."
|
# "There is no war within the container. Here we are safe. Here we are free."
|
||||||
|
@ -43,7 +45,6 @@ resource "docker_container" "start" {
|
||||||
"bootkube",
|
"bootkube",
|
||||||
"start",
|
"start",
|
||||||
"--asset-dir=/home/.bootkube",
|
"--asset-dir=/home/.bootkube",
|
||||||
"--pod-manifest-path=/etc/kubernetes/manifests",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
network_mode = "host"
|
network_mode = "host"
|
||||||
|
|
|
@ -1,9 +1,5 @@
|
||||||
// Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube
|
// Based on https://github.com/v1k0d3n/dockerfiles/tree/master/bootkube
|
||||||
|
|
||||||
variable "asset_dir_volume_name" {
|
|
||||||
default = "k8s-assets"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "k8s_host" {
|
variable "k8s_host" {
|
||||||
description = "kubenetes hostname"
|
description = "kubenetes hostname"
|
||||||
}
|
}
|
||||||
|
@ -12,6 +8,10 @@ variable "host_port" {
|
||||||
default = "8443"
|
default = "8443"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "network_provider" {
|
||||||
|
default = "flannel"
|
||||||
|
}
|
||||||
|
|
||||||
variable "host_ip" {}
|
variable "host_ip" {}
|
||||||
|
|
||||||
variable "pod_cidr" {
|
variable "pod_cidr" {
|
||||||
|
|
|
@ -8,32 +8,15 @@ module "container" {
|
||||||
host = ""
|
host = ""
|
||||||
}
|
}
|
||||||
|
|
||||||
networks = []
|
networks = ["${docker_network.etcd.id}"]
|
||||||
|
|
||||||
volumes = [
|
volumes = [
|
||||||
{
|
|
||||||
host_path = "/usr/share/ca-certificates/"
|
|
||||||
container_path = "/etc/ssl/certs"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
host_path = "${var.data_dir}"
|
host_path = "${var.data_dir}"
|
||||||
container_path = "/etcd-data"
|
container_path = "/etcd-data"
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
ports = [
|
|
||||||
{
|
|
||||||
internal = 2379
|
|
||||||
external = 2379
|
|
||||||
ip = "${var.host_ip}"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
internal = 2380
|
|
||||||
external = 2380
|
|
||||||
ip = "${var.host_ip}"
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
command = [
|
command = [
|
||||||
"/usr/local/bin/etcd",
|
"/usr/local/bin/etcd",
|
||||||
"--data-dir=/etcd-data",
|
"--data-dir=/etcd-data",
|
||||||
|
@ -42,7 +25,14 @@ module "container" {
|
||||||
"--initial-advertise-peer-urls=http://${var.host_ip}:2380",
|
"--initial-advertise-peer-urls=http://${var.host_ip}:2380",
|
||||||
"--initial-cluster=${var.node_name}=http://${var.host_ip}:2380",
|
"--initial-cluster=${var.node_name}=http://${var.host_ip}:2380",
|
||||||
]
|
]
|
||||||
|
}
|
||||||
# "--listen-client-urls=http://0.0.0.0:2379",
|
|
||||||
# "--listen-peer-urls=http://0.0.0.0:2380",
|
resource "docker_network" "etcd" {
|
||||||
|
name = "etcd"
|
||||||
|
driver = "bridge"
|
||||||
|
|
||||||
|
ipam_config {
|
||||||
|
subnet = "10.10.10.0/25"
|
||||||
|
gateway = "10.10.10.1"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,19 @@ variable "data_dir" {
|
||||||
type = "string"
|
type = "string"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "bootkube_asset_dir" {
|
||||||
|
description = "bootkube render is run against this directory"
|
||||||
|
type = "string"
|
||||||
|
default = "/etc/kube-assets"
|
||||||
|
}
|
||||||
|
|
||||||
variable "node_name" {
|
variable "node_name" {
|
||||||
description = "name of the etcd node"
|
description = "name of the etcd node"
|
||||||
default = "master"
|
default = "master"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "depends_on" {
|
||||||
|
default = []
|
||||||
|
|
||||||
|
type = "list"
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,116 @@
|
||||||
|
// This is primarily based on https://github.com/coreos/coreos-overlay/blob/master/app-admin/kubelet-wrapper/files/kubelet-wrapper
|
||||||
|
resource "docker_container" "kubelet" {
|
||||||
|
image = "${docker_image.image.latest}"
|
||||||
|
name = "kubelet-static"
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/etc/kubernetes"
|
||||||
|
host_path = "/etc/kubernetes"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/etc/kubernetes/kubeconfig"
|
||||||
|
host_path = "/etc/kube-assets/auth/kubeconfig-kubelet"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/etc/kubernetes/kubeconfig-admin"
|
||||||
|
host_path = "/etc/kube-assets/auth/kubeconfig"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/etc/kubernetes/ca.crt"
|
||||||
|
host_path = "/etc/kube-assets/tls/ca.crt"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/etc/ssl/certs"
|
||||||
|
host_path = "/etc/ssl/certs"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/usr/share/ca-certificates"
|
||||||
|
host_path = "/usr/share/ca-certificates"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/var/lib/docker"
|
||||||
|
host_path = "/var/lib/docker"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/var/lib/kubelet"
|
||||||
|
host_path = "/var/lib/kubelet"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/var/log"
|
||||||
|
host_path = "/var/log"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/run"
|
||||||
|
host_path = "/run"
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/lib/modules"
|
||||||
|
host_path = "/lib/modules"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/etc/os-release"
|
||||||
|
host_path = "/usr/lib/os-release"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/etc/machine-id"
|
||||||
|
host_path = "/etc/machine-id"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deviates from kubelet-wrapper
|
||||||
|
|
||||||
|
volumes {
|
||||||
|
container_path = "/var/lib/cni"
|
||||||
|
host_path = "/var/lib/cni"
|
||||||
|
}
|
||||||
|
command = [
|
||||||
|
"kubelet",
|
||||||
|
"--kubeconfig=/etc/kubernetes/kubeconfig",
|
||||||
|
"--client-ca-file=/etc/kubernetes/ca.crt",
|
||||||
|
"--anonymous-auth=false",
|
||||||
|
"--cni-conf-dir=/etc/kubernetes/cni/net.d",
|
||||||
|
"--network-plugin=cni",
|
||||||
|
"--lock-file=/var/run/lock/kubelet.lock",
|
||||||
|
"--exit-on-lock-contention",
|
||||||
|
"--pod-manifest-path=/etc/kubernetes/manifests",
|
||||||
|
"--allow-privileged",
|
||||||
|
"--minimum-container-ttl-duration=10m0s",
|
||||||
|
"--cluster_dns=10.25.0.10",
|
||||||
|
"--cluster_domain=k8s.bb8.fun",
|
||||||
|
]
|
||||||
|
|
||||||
|
# TODO
|
||||||
|
# "--register-with-taints=${var.node_taints}",
|
||||||
|
# "--node-labels=${var.node_label}",
|
||||||
|
|
||||||
|
network_mode = "host"
|
||||||
|
privileged = true
|
||||||
|
restart = "no"
|
||||||
|
must_run = false
|
||||||
|
max_retry_count = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
data "docker_registry_image" "image" {
|
||||||
|
name = "gcr.io/google_containers/hyperkube:v${var.version}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "docker_image" "image" {
|
||||||
|
name = "${data.docker_registry_image.image.name}"
|
||||||
|
pull_triggers = ["${data.docker_registry_image.image.sha256_digest}"]
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
variable "version" {
|
||||||
|
description = "kubelet version"
|
||||||
|
default = "1.13.2"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "node_label" {
|
||||||
|
description = "kubelet version"
|
||||||
|
default = "node.kubernetes.io/master"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "depends_on" {
|
||||||
|
default = []
|
||||||
|
|
||||||
|
type = "list"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "asset_dir_volume_name" {
|
||||||
|
default = "k8s-assets"
|
||||||
|
}
|
Loading…
Reference in New Issue