diff --git a/.gitignore b/.gitignore index 146575e..3dcbe85 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,8 @@ override.tf.json .terraformrc terraform.rc +# ---> End Terraform + +.idea/ +.vscode/ +.DS_Store diff --git a/pve01.wheatley.in/k8s-wheatley/main.tf b/pve01.wheatley.in/k8s-wheatley/main.tf index fcf6794..e97ae2e 100644 --- a/pve01.wheatley.in/k8s-wheatley/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/main.tf @@ -140,3 +140,41 @@ module "workers" { worker_addresses = local.worker_addresses } } + +module "talos-bootstrap" { + depends_on = [ + module.controlplanes, + module.workers + ] + + source = "./modules/talos-bootstrap" + + node_config = { + ipv4_gateway = local.ipv4_gateway + talos_version = local.talos_version + cilium_version = local.cilium_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + cluster_endpoint = local.cluster_endpoint_ip + proxmox_node = local.proxmox_node + controlplane_addresses = local.controlplane_addresses + worker_addresses = local.worker_addresses + } + + talos_nodes = concat( + [ + for node in local.controlplanes.nodes : { + name = format("k8s-wheatley-%s", node.name) + type = "controlplane" + ipv4_address = node.ip_address + } + ], + [ + for node in local.workers.nodes : { + name = format("k8s-wheatley-%s", node.name) + type = "worker" + ipv4_address = node.ip_address + } + ] + ) +} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf new file mode 100644 index 0000000..2097b86 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf @@ -0,0 +1,95 @@ +terraform { + required_providers { + talos = { + source = "siderolabs/talos" + version = "0.9.0" + } + } +} +locals { + cluster_endpoint_full = format("https://%s:6443", var.node_config.cluster_endpoint) +} + +resource "talos_machine_secrets" "machine_secrets" {} + +data "talos_machine_configuration" "machineconfig-cp" { + cluster_name = var.node_config.cluster_name + machine_type = "controlplane" + cluster_endpoint = local.cluster_endpoint_full + kubernetes_version = var.node_config.kubernetes_version + talos_version = talos_machine_secrets.machine_secrets.talos_version + machine_secrets = talos_machine_secrets.machine_secrets.machine_secrets +} + +data "talos_machine_configuration" "machineconfig-worker" { + cluster_name = var.node_config.cluster_name + machine_type = "worker" + cluster_endpoint = local.cluster_endpoint_full + kubernetes_version = var.node_config.kubernetes_version + talos_version = talos_machine_secrets.machine_secrets.talos_version + machine_secrets = talos_machine_secrets.machine_secrets.machine_secrets +} + +data "talos_client_configuration" "talosconfig" { + cluster_name = var.node_config.cluster_name + endpoints = var.node_config.controlplane_addresses + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + nodes = concat(var.node_config.controlplane_addresses, var.node_config.worker_addresses) +} + +resource "talos_machine_configuration_apply" "config_apply_cp" { + for_each = { + for talos_node in var.talos_nodes : talos_node.name => talos_node + if talos_node.type == "controlplane" + } + + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + machine_configuration_input = data.talos_machine_configuration.machineconfig-cp.machine_configuration + endpoint = each.value.ipv4_address + node = each.value.ipv4_address + config_patches = [ + templatefile("${path.module}/templates/machineconfig-cp.yaml.tmpl", { + hostname = each.value.name + cluster_name = var.node_config.cluster_name + vip_address = var.node_config.cluster_endpoint + node_name = format("%s.wheatley.in", var.node_config.proxmox_node) + machine_type = each.value.type + talos_version = var.node_config.talos_version + cilium_version = var.node_config.cilium_version + cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl") + }) + ] +} + +resource "talos_machine_configuration_apply" "config_apply_worker" { + for_each = { + for talos_node in var.talos_nodes : talos_node.name => talos_node + if talos_node.type == "worker" + } + + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + machine_configuration_input = data.talos_machine_configuration.machineconfig-worker.machine_configuration + endpoint = each.value.ipv4_address + node = each.value.ipv4_address + config_patches = [ + templatefile("${path.module}/templates/machineconfig-worker.yaml.tmpl", { + hostname = each.value.name + cluster_name = var.node_config.cluster_name + vip_address = var.node_config.cluster_endpoint + node_name = format("%s.wheatley.in", var.node_config.proxmox_node) + machine_type = each.value.type + talos_version = var.node_config.talos_version + cilium_version = var.node_config.cilium_version + cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl") + }) + ] +} + +resource "talos_machine_bootstrap" "talos_machine_bootstrap" { + depends_on = [ + talos_machine_configuration_apply.config_apply_cp, + talos_machine_configuration_apply.config_apply_worker + ] + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + node = var.talos_nodes[0].ipv4_address +} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/cilium-install.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/cilium-install.yaml.tmpl new file mode 100644 index 0000000..3db0a24 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/cilium-install.yaml.tmpl @@ -0,0 +1,96 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-install +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: cilium-install + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium-install + namespace: kube-system +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: cilium-install + namespace: kube-system +spec: + backoffLimit: 10 + template: + metadata: + labels: + app: cilium-install + spec: + restartPolicy: OnFailure + tolerations: + - operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - effect: PreferNoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoExecute + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: PreferNoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccount: cilium-install + serviceAccountName: cilium-install + hostNetwork: true + containers: + - name: cilium-install + image: quay.io/cilium/cilium-cli:latest + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + command: + - cilium + - install + - --set + - ipam.mode=kubernetes + - --set + - kubeProxyReplacement=true + - --set + - securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID} + - --set + - securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE} + - --set + - cgroup.autoMount.enabled=false + - --set + - cgroup.hostRoot=/sys/fs/cgroup + - --set + - k8sServiceHost=10.13.38.11 + - --set + - k8sServicePort=6443 + - --set + - gatewayAPI.enabled=true + - --set + - gatewayAPI.enableAlpn=true + - --set + - gatewayAPI.enableAppProtocol=true diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl new file mode 100644 index 0000000..c177e20 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl @@ -0,0 +1,43 @@ +debug: false +machine: + type: ${machine_type} + install: + disk: /dev/vda + image: factory.talos.dev/nocloud-installer-secureboot/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v${talos_version} + network: + hostname: ${hostname} + nameservers: + - 10.13.37.2 + %{ if machine_type == "controlplane" } + interfaces: + - interface: eth0 + dhcp: false + vip: + ip: ${vip_address} + %{ endif } + kubelet: + extraArgs: + pod-max-pids: 1000 + extraConfig: + imageGCHighThresholdPercent: 75 + imageGCLowThresholdPercent: 70 + nodeLabels: + topology.kubernetes.io/region: ${cluster_name} + topology.kubernetes.io/zone: ${node_name} + +cluster: + apiServer: + auditPolicy: + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + - level: Metadata + network: + cni: + name: none + proxy: + disabled: true + inlineManifests: + - name: cilium-bootstrap + contents: | + ${indent(6, cilium_install_file)} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl new file mode 100644 index 0000000..4227892 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl @@ -0,0 +1,36 @@ +debug: false +machine: + type: ${machine_type} + install: + disk: /dev/vda + image: factory.talos.dev/nocloud-installer-secureboot/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v${talos_version} + network: + hostname: ${hostname} + nameservers: + - 10.13.37.2 + kubelet: + extraArgs: + pod-max-pids: 1000 + extraConfig: + imageGCHighThresholdPercent: 75 + imageGCLowThresholdPercent: 70 + nodeLabels: + topology.kubernetes.io/region: ${cluster_name} + topology.kubernetes.io/zone: ${node_name} + +cluster: + apiServer: + auditPolicy: + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + - level: Metadata + network: + cni: + name: none + proxy: + disabled: true + inlineManifests: + - name: cilium-bootstrap + contents: | + ${indent(6, cilium_install_file)} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf new file mode 100644 index 0000000..525a53a --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf @@ -0,0 +1,23 @@ +variable "node_config" { + description = "Talos node configuration" + type = object({ + ipv4_gateway = string + talos_version = string + cilium_version = string + cluster_name = string + kubernetes_version = string + cluster_endpoint = string + proxmox_node = string + controlplane_addresses = list(string) + worker_addresses = list(string) + }) +} + +variable "talos_nodes" { + description = "List of Talos nodes to bootstrap" + type = list(object({ + name = string + type = string + ipv4_address = string + })) +}