feat: Bootstrap Talos k8s cluster
This commit is contained in:
parent
9ca0f7c431
commit
7d3c60325f
SSH key fingerprint:
SHA256:B5tYaxBExaDm74r1px9iVeZ6F/ZDiyiy9SbBqfZYrvg
7 changed files with 336 additions and 0 deletions
|
|
@ -140,3 +140,41 @@ module "workers" {
|
|||
worker_addresses = local.worker_addresses
|
||||
}
|
||||
}
|
||||
|
||||
module "talos-bootstrap" {
|
||||
depends_on = [
|
||||
module.controlplanes,
|
||||
module.workers
|
||||
]
|
||||
|
||||
source = "./modules/talos-bootstrap"
|
||||
|
||||
node_config = {
|
||||
ipv4_gateway = local.ipv4_gateway
|
||||
talos_version = local.talos_version
|
||||
cilium_version = local.cilium_version
|
||||
cluster_name = local.cluster_name
|
||||
kubernetes_version = local.kubernetes_version
|
||||
cluster_endpoint = local.cluster_endpoint_ip
|
||||
proxmox_node = local.proxmox_node
|
||||
controlplane_addresses = local.controlplane_addresses
|
||||
worker_addresses = local.worker_addresses
|
||||
}
|
||||
|
||||
talos_nodes = concat(
|
||||
[
|
||||
for node in local.controlplanes.nodes : {
|
||||
name = format("k8s-wheatley-%s", node.name)
|
||||
type = "controlplane"
|
||||
ipv4_address = node.ip_address
|
||||
}
|
||||
],
|
||||
[
|
||||
for node in local.workers.nodes : {
|
||||
name = format("k8s-wheatley-%s", node.name)
|
||||
type = "worker"
|
||||
ipv4_address = node.ip_address
|
||||
}
|
||||
]
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
talos = {
|
||||
source = "siderolabs/talos"
|
||||
version = "0.9.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
locals {
|
||||
cluster_endpoint_full = format("https://%s:6443", var.node_config.cluster_endpoint)
|
||||
}
|
||||
|
||||
resource "talos_machine_secrets" "machine_secrets" {}
|
||||
|
||||
data "talos_machine_configuration" "machineconfig-cp" {
|
||||
cluster_name = var.node_config.cluster_name
|
||||
machine_type = "controlplane"
|
||||
cluster_endpoint = local.cluster_endpoint_full
|
||||
kubernetes_version = var.node_config.kubernetes_version
|
||||
talos_version = talos_machine_secrets.machine_secrets.talos_version
|
||||
machine_secrets = talos_machine_secrets.machine_secrets.machine_secrets
|
||||
}
|
||||
|
||||
data "talos_machine_configuration" "machineconfig-worker" {
|
||||
cluster_name = var.node_config.cluster_name
|
||||
machine_type = "worker"
|
||||
cluster_endpoint = local.cluster_endpoint_full
|
||||
kubernetes_version = var.node_config.kubernetes_version
|
||||
talos_version = talos_machine_secrets.machine_secrets.talos_version
|
||||
machine_secrets = talos_machine_secrets.machine_secrets.machine_secrets
|
||||
}
|
||||
|
||||
data "talos_client_configuration" "talosconfig" {
|
||||
cluster_name = var.node_config.cluster_name
|
||||
endpoints = var.node_config.controlplane_addresses
|
||||
client_configuration = talos_machine_secrets.machine_secrets.client_configuration
|
||||
nodes = concat(var.node_config.controlplane_addresses, var.node_config.worker_addresses)
|
||||
}
|
||||
|
||||
resource "talos_machine_configuration_apply" "config_apply_cp" {
|
||||
for_each = {
|
||||
for talos_node in var.talos_nodes : talos_node.name => talos_node
|
||||
if talos_node.type == "controlplane"
|
||||
}
|
||||
|
||||
client_configuration = talos_machine_secrets.machine_secrets.client_configuration
|
||||
machine_configuration_input = data.talos_machine_configuration.machineconfig-cp.machine_configuration
|
||||
endpoint = each.value.ipv4_address
|
||||
node = each.value.ipv4_address
|
||||
config_patches = [
|
||||
templatefile("${path.module}/templates/machineconfig-cp.yaml.tmpl", {
|
||||
hostname = each.value.name
|
||||
cluster_name = var.node_config.cluster_name
|
||||
vip_address = var.node_config.cluster_endpoint
|
||||
node_name = format("%s.wheatley.in", var.node_config.proxmox_node)
|
||||
machine_type = each.value.type
|
||||
talos_version = var.node_config.talos_version
|
||||
cilium_version = var.node_config.cilium_version
|
||||
cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl")
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
resource "talos_machine_configuration_apply" "config_apply_worker" {
|
||||
for_each = {
|
||||
for talos_node in var.talos_nodes : talos_node.name => talos_node
|
||||
if talos_node.type == "worker"
|
||||
}
|
||||
|
||||
client_configuration = talos_machine_secrets.machine_secrets.client_configuration
|
||||
machine_configuration_input = data.talos_machine_configuration.machineconfig-worker.machine_configuration
|
||||
endpoint = each.value.ipv4_address
|
||||
node = each.value.ipv4_address
|
||||
config_patches = [
|
||||
templatefile("${path.module}/templates/machineconfig-worker.yaml.tmpl", {
|
||||
hostname = each.value.name
|
||||
cluster_name = var.node_config.cluster_name
|
||||
vip_address = var.node_config.cluster_endpoint
|
||||
node_name = format("%s.wheatley.in", var.node_config.proxmox_node)
|
||||
machine_type = each.value.type
|
||||
talos_version = var.node_config.talos_version
|
||||
cilium_version = var.node_config.cilium_version
|
||||
cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl")
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
resource "talos_machine_bootstrap" "talos_machine_bootstrap" {
|
||||
depends_on = [
|
||||
talos_machine_configuration_apply.config_apply_cp,
|
||||
talos_machine_configuration_apply.config_apply_worker
|
||||
]
|
||||
client_configuration = talos_machine_secrets.machine_secrets.client_configuration
|
||||
node = var.talos_nodes[0].ipv4_address
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: cilium-install
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: cluster-admin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: cilium-install
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: cilium-install
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: cilium-install
|
||||
namespace: kube-system
|
||||
spec:
|
||||
backoffLimit: 10
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: cilium-install
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
- effect: NoSchedule
|
||||
operator: Exists
|
||||
- effect: NoExecute
|
||||
operator: Exists
|
||||
- effect: PreferNoSchedule
|
||||
operator: Exists
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoExecute
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: PreferNoSchedule
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
serviceAccount: cilium-install
|
||||
serviceAccountName: cilium-install
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- name: cilium-install
|
||||
image: quay.io/cilium/cilium-cli:latest
|
||||
env:
|
||||
- name: KUBERNETES_SERVICE_HOST
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: status.podIP
|
||||
- name: KUBERNETES_SERVICE_PORT
|
||||
value: "6443"
|
||||
command:
|
||||
- cilium
|
||||
- install
|
||||
- --set
|
||||
- ipam.mode=kubernetes
|
||||
- --set
|
||||
- kubeProxyReplacement=true
|
||||
- --set
|
||||
- securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}
|
||||
- --set
|
||||
- securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}
|
||||
- --set
|
||||
- cgroup.autoMount.enabled=false
|
||||
- --set
|
||||
- cgroup.hostRoot=/sys/fs/cgroup
|
||||
- --set
|
||||
- k8sServiceHost=10.13.38.11
|
||||
- --set
|
||||
- k8sServicePort=6443
|
||||
- --set
|
||||
- gatewayAPI.enabled=true
|
||||
- --set
|
||||
- gatewayAPI.enableAlpn=true
|
||||
- --set
|
||||
- gatewayAPI.enableAppProtocol=true
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
debug: false
|
||||
machine:
|
||||
type: ${machine_type}
|
||||
install:
|
||||
disk: /dev/vda
|
||||
image: factory.talos.dev/nocloud-installer-secureboot/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v${talos_version}
|
||||
network:
|
||||
hostname: ${hostname}
|
||||
nameservers:
|
||||
- 10.13.37.2
|
||||
%{ if machine_type == "controlplane" }
|
||||
interfaces:
|
||||
- interface: eth0
|
||||
dhcp: false
|
||||
vip:
|
||||
ip: ${vip_address}
|
||||
%{ endif }
|
||||
kubelet:
|
||||
extraArgs:
|
||||
pod-max-pids: 1000
|
||||
extraConfig:
|
||||
imageGCHighThresholdPercent: 75
|
||||
imageGCLowThresholdPercent: 70
|
||||
nodeLabels:
|
||||
topology.kubernetes.io/region: ${cluster_name}
|
||||
topology.kubernetes.io/zone: ${node_name}
|
||||
|
||||
cluster:
|
||||
apiServer:
|
||||
auditPolicy:
|
||||
apiVersion: audit.k8s.io/v1
|
||||
kind: Policy
|
||||
rules:
|
||||
- level: Metadata
|
||||
network:
|
||||
cni:
|
||||
name: none
|
||||
proxy:
|
||||
disabled: true
|
||||
inlineManifests:
|
||||
- name: cilium-bootstrap
|
||||
contents: |
|
||||
${indent(6, cilium_install_file)}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
debug: false
|
||||
machine:
|
||||
type: ${machine_type}
|
||||
install:
|
||||
disk: /dev/vda
|
||||
image: factory.talos.dev/nocloud-installer-secureboot/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v${talos_version}
|
||||
network:
|
||||
hostname: ${hostname}
|
||||
nameservers:
|
||||
- 10.13.37.2
|
||||
kubelet:
|
||||
extraArgs:
|
||||
pod-max-pids: 1000
|
||||
extraConfig:
|
||||
imageGCHighThresholdPercent: 75
|
||||
imageGCLowThresholdPercent: 70
|
||||
nodeLabels:
|
||||
topology.kubernetes.io/region: ${cluster_name}
|
||||
topology.kubernetes.io/zone: ${node_name}
|
||||
|
||||
cluster:
|
||||
apiServer:
|
||||
auditPolicy:
|
||||
apiVersion: audit.k8s.io/v1
|
||||
kind: Policy
|
||||
rules:
|
||||
- level: Metadata
|
||||
network:
|
||||
cni:
|
||||
name: none
|
||||
proxy:
|
||||
disabled: true
|
||||
inlineManifests:
|
||||
- name: cilium-bootstrap
|
||||
contents: |
|
||||
${indent(6, cilium_install_file)}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
variable "node_config" {
|
||||
description = "Talos node configuration"
|
||||
type = object({
|
||||
ipv4_gateway = string
|
||||
talos_version = string
|
||||
cilium_version = string
|
||||
cluster_name = string
|
||||
kubernetes_version = string
|
||||
cluster_endpoint = string
|
||||
proxmox_node = string
|
||||
controlplane_addresses = list(string)
|
||||
worker_addresses = list(string)
|
||||
})
|
||||
}
|
||||
|
||||
variable "talos_nodes" {
|
||||
description = "List of Talos nodes to bootstrap"
|
||||
type = list(object({
|
||||
name = string
|
||||
type = string
|
||||
ipv4_address = string
|
||||
}))
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue