feat: Bootstrap Talos k8s cluster

.gitignore

@@ -37,3 +37,8 @@ override.tf.json
 .terraformrc
 terraform.rc
 
+# ---> End Terraform
+
+.idea/
+.vscode/
+.DS_Store

pve01.wheatley.in/k8s-wheatley/main.tf

@@ -140,3 +140,41 @@ module "workers" {
     worker_addresses       = local.worker_addresses
   }
 }
+
+module "talos-bootstrap" {
+  depends_on = [
+    module.controlplanes,
+    module.workers
+  ]
+
+  source = "./modules/talos-bootstrap"
+
+  node_config = {
+    ipv4_gateway           = local.ipv4_gateway
+    talos_version          = local.talos_version
+    cilium_version         = local.cilium_version
+    cluster_name           = local.cluster_name
+    kubernetes_version     = local.kubernetes_version
+    cluster_endpoint       = local.cluster_endpoint_ip
+    proxmox_node           = local.proxmox_node
+    controlplane_addresses = local.controlplane_addresses
+    worker_addresses       = local.worker_addresses
+  }
+
+  talos_nodes = concat(
+    [
+      for node in local.controlplanes.nodes : {
+        name         = format("k8s-wheatley-%s", node.name)
+        type         = "controlplane"
+        ipv4_address = node.ip_address
+      }
+    ],
+    [
+      for node in local.workers.nodes : {
+        name         = format("k8s-wheatley-%s", node.name)
+        type         = "worker"
+        ipv4_address = node.ip_address
+      }
+    ]
+  )
+}

pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf

@@ -0,0 +1,95 @@
+terraform {
+  required_providers {
+    talos = {
+      source  = "siderolabs/talos"
+      version = "0.9.0"
+    }
+  }
+}
+locals {
+  cluster_endpoint_full = format("https://%s:6443", var.node_config.cluster_endpoint)
+}
+
+resource "talos_machine_secrets" "machine_secrets" {}
+
+data "talos_machine_configuration" "machineconfig-cp" {
+  cluster_name       = var.node_config.cluster_name
+  machine_type       = "controlplane"
+  cluster_endpoint   = local.cluster_endpoint_full
+  kubernetes_version = var.node_config.kubernetes_version
+  talos_version      = talos_machine_secrets.machine_secrets.talos_version
+  machine_secrets    = talos_machine_secrets.machine_secrets.machine_secrets
+}
+
+data "talos_machine_configuration" "machineconfig-worker" {
+  cluster_name       = var.node_config.cluster_name
+  machine_type       = "worker"
+  cluster_endpoint   = local.cluster_endpoint_full
+  kubernetes_version = var.node_config.kubernetes_version
+  talos_version      = talos_machine_secrets.machine_secrets.talos_version
+  machine_secrets    = talos_machine_secrets.machine_secrets.machine_secrets
+}
+
+data "talos_client_configuration" "talosconfig" {
+  cluster_name         = var.node_config.cluster_name
+  endpoints            = var.node_config.controlplane_addresses
+  client_configuration = talos_machine_secrets.machine_secrets.client_configuration
+  nodes                = concat(var.node_config.controlplane_addresses, var.node_config.worker_addresses)
+}
+
+resource "talos_machine_configuration_apply" "config_apply_cp" {
+  for_each = {
+    for talos_node in var.talos_nodes : talos_node.name => talos_node
+    if talos_node.type == "controlplane"
+  }
+
+  client_configuration        = talos_machine_secrets.machine_secrets.client_configuration
+  machine_configuration_input = data.talos_machine_configuration.machineconfig-cp.machine_configuration
+  endpoint                    = each.value.ipv4_address
+  node                        = each.value.ipv4_address
+  config_patches = [
+    templatefile("${path.module}/templates/machineconfig-cp.yaml.tmpl", {
+      hostname            = each.value.name
+      cluster_name        = var.node_config.cluster_name
+      vip_address         = var.node_config.cluster_endpoint
+      node_name           = format("%s.wheatley.in", var.node_config.proxmox_node)
+      machine_type        = each.value.type
+      talos_version       = var.node_config.talos_version
+      cilium_version      = var.node_config.cilium_version
+      cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl")
+    })
+  ]
+}
+
+resource "talos_machine_configuration_apply" "config_apply_worker" {
+  for_each = {
+    for talos_node in var.talos_nodes : talos_node.name => talos_node
+    if talos_node.type == "worker"
+  }
+
+  client_configuration        = talos_machine_secrets.machine_secrets.client_configuration
+  machine_configuration_input = data.talos_machine_configuration.machineconfig-worker.machine_configuration
+  endpoint                    = each.value.ipv4_address
+  node                        = each.value.ipv4_address
+  config_patches = [
+    templatefile("${path.module}/templates/machineconfig-worker.yaml.tmpl", {
+      hostname            = each.value.name
+      cluster_name        = var.node_config.cluster_name
+      vip_address         = var.node_config.cluster_endpoint
+      node_name           = format("%s.wheatley.in", var.node_config.proxmox_node)
+      machine_type        = each.value.type
+      talos_version       = var.node_config.talos_version
+      cilium_version      = var.node_config.cilium_version
+      cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl")
+    })
+  ]
+}
+
+resource "talos_machine_bootstrap" "talos_machine_bootstrap" {
+  depends_on = [
+    talos_machine_configuration_apply.config_apply_cp,
+    talos_machine_configuration_apply.config_apply_worker
+  ]
+  client_configuration = talos_machine_secrets.machine_secrets.client_configuration
+  node                 = var.talos_nodes[0].ipv4_address
+}

pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/cilium-install.yaml.tmpl

@@ -0,0 +1,96 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-install
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: cilium-install
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-install
+  namespace: kube-system
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cilium-install
+  namespace: kube-system
+spec:
+  backoffLimit: 10
+  template:
+    metadata:
+      labels:
+        app: cilium-install
+    spec:
+      restartPolicy: OnFailure
+      tolerations:
+        - operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - effect: PreferNoSchedule
+          operator: Exists
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoExecute
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: PreferNoSchedule
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+      serviceAccount: cilium-install
+      serviceAccountName: cilium-install
+      hostNetwork: true
+      containers:
+      - name: cilium-install
+        image: quay.io/cilium/cilium-cli:latest
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: status.podIP
+        - name: KUBERNETES_SERVICE_PORT
+          value: "6443"
+        command:
+          - cilium
+          - install
+          - --set
+          - ipam.mode=kubernetes
+          - --set
+          - kubeProxyReplacement=true
+          - --set
+          - securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}
+          - --set
+          - securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}
+          - --set
+          - cgroup.autoMount.enabled=false
+          - --set
+          - cgroup.hostRoot=/sys/fs/cgroup
+          - --set
+          - k8sServiceHost=10.13.38.11
+          - --set
+          - k8sServicePort=6443
+          - --set
+          - gatewayAPI.enabled=true
+          - --set
+          - gatewayAPI.enableAlpn=true 
+          - --set
+          - gatewayAPI.enableAppProtocol=true

pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl

@@ -0,0 +1,43 @@
+debug: false
+machine:
+  type: ${machine_type}
+  install:
+    disk: /dev/vda
+    image: factory.talos.dev/nocloud-installer-secureboot/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v${talos_version}
+  network:
+    hostname: ${hostname}
+    nameservers:
+      - 10.13.37.2
+    %{ if machine_type == "controlplane" }
+    interfaces:
+      - interface: eth0
+        dhcp: false
+        vip:
+          ip: ${vip_address}
+    %{ endif }
+  kubelet:
+    extraArgs:
+      pod-max-pids: 1000
+    extraConfig:
+      imageGCHighThresholdPercent: 75
+      imageGCLowThresholdPercent: 70
+  nodeLabels:
+    topology.kubernetes.io/region: ${cluster_name}
+    topology.kubernetes.io/zone: ${node_name}
+
+cluster:
+  apiServer:
+    auditPolicy:
+      apiVersion: audit.k8s.io/v1
+      kind: Policy
+      rules:
+        - level: Metadata
+  network:
+    cni:
+      name: none
+  proxy:
+    disabled: true
+  inlineManifests:
+  - name: cilium-bootstrap
+    contents: |
+      ${indent(6, cilium_install_file)}

pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl

@@ -0,0 +1,36 @@
+debug: false
+machine:
+  type: ${machine_type}
+  install:
+    disk: /dev/vda
+    image: factory.talos.dev/nocloud-installer-secureboot/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v${talos_version}
+  network:
+    hostname: ${hostname}
+    nameservers:
+      - 10.13.37.2
+  kubelet:
+    extraArgs:
+      pod-max-pids: 1000
+    extraConfig:
+      imageGCHighThresholdPercent: 75
+      imageGCLowThresholdPercent: 70
+  nodeLabels:
+    topology.kubernetes.io/region: ${cluster_name}
+    topology.kubernetes.io/zone: ${node_name}
+
+cluster:
+  apiServer:
+    auditPolicy:
+      apiVersion: audit.k8s.io/v1
+      kind: Policy
+      rules:
+        - level: Metadata
+  network:
+    cni:
+      name: none
+  proxy:
+    disabled: true
+  inlineManifests:
+  - name: cilium-bootstrap
+    contents: |
+      ${indent(6, cilium_install_file)}

pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf

@@ -0,0 +1,23 @@
+variable "node_config" {
+  description = "Talos node configuration"
+  type = object({
+    ipv4_gateway           = string
+    talos_version          = string
+    cilium_version         = string
+    cluster_name           = string
+    kubernetes_version     = string
+    cluster_endpoint       = string
+    proxmox_node           = string
+    controlplane_addresses = list(string)
+    worker_addresses       = list(string)
+  })
+}
+
+variable "talos_nodes" {
+  description = "List of Talos nodes to bootstrap"
+  type = list(object({
+    name         = string
+    type         = string
+    ipv4_address = string
+  }))
+}