diff --git a/pve01.wheatley.in/k8s-wheatley/main.tf b/pve01.wheatley.in/k8s-wheatley/main.tf index 0c06bee..e0079a3 100644 --- a/pve01.wheatley.in/k8s-wheatley/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/main.tf @@ -5,56 +5,109 @@ provider "proxmox" { } locals { - talos_version = "1.11.3" -} + cluster_name = "k8s-wheatley" + kubernetes_version = "1.34.1" + talos_version = "1.11.3" + ipv4_gateway = "10.13.38.1" + ipv4_cidr = "/24" + cluster_endpoint_ip = "10.13.38.20" -module "controlplanes" { - source = "./modules/controlplane" controlplanes = { - cpu = 4 - memory = 4 - disk = "40G" - storagepool = "nvme-fastpool" - talos_version = local.talos_version + cpu = 4 + memory = 4 + disk = 40 + storagepool = "local-zfs" + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + ipv4_gateway = local.ipv4_gateway nodes = [ { name = "cp01" - ip_address = "10.13.38.20" - }, - { - name = "cp02" ip_address = "10.13.38.21" }, { - name = "cp03" + name = "cp02" ip_address = "10.13.38.22" - }] + }, + { + name = "cp03" + ip_address = "10.13.38.23" + } + ] } -} - -module "workers" { - source = "./modules/worker" workers = { - cpu = 4 - memory = 4 - disk = "40G" - storagepool = "nvme-fastpool" - talos_version = local.talos_version + cpu = 4 + memory = 4 + disk = 40 + storagepool = "local-zfs" + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + ipv4_gateway = local.ipv4_gateway nodes = [ { name = "worker01" - ip_address = "10.13.38.30" + ip_address = "10.13.38.20" }, { name = "worker02" - ip_address = "10.13.38.31" - }] + ip_address = "10.13.38.21" + }, + { + name = "worker03" + ip_address = "10.13.38.22" + } + ] + } +} + module "talos-image" { source = "./modules/talos-image" talos_version = local.talos_version } + +module "controlplanes" { + depends_on = [module.talos-image] + + source = "./modules/controlplane" + for_each = { for node in local.controlplanes.nodes : node.name => node } + + controlplane = { + cpu = local.controlplanes.cpu + memory = local.controlplanes.memory + disk = local.controlplanes.disk + storagepool = local.controlplanes.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + node_name = format("k8s-wheatley-%s", each.value.name) + cluster_endpoint = format("https://%s:6443", local.cluster_endpoint_ip) + node_ipv4_address = format("%s%s", each.value.ip_address, local.ipv4_cidr) + ipv4_gateway = local.ipv4_gateway + } +} + +module "workers" { + depends_on = [module.controlplanes] + + source = "./modules/worker" + for_each = { for node in local.workers.nodes : node.name => node } + + worker = { + cpu = local.workers.cpu + memory = local.workers.memory + disk = local.workers.disk + storagepool = local.workers.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + node_name = format("k8s-wheatley-%s", each.value.name) + cluster_endpoint = format("https://%s:6443", local.cluster_endpoint_ip) + node_ipv4_address = format("%s%s", each.value.ip_address, local.ipv4_cidr) + ipv4_gateway = local.ipv4_gateway } } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl new file mode 100644 index 0000000..a1da4bb --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl @@ -0,0 +1,34 @@ +debug: false +machine: + install: + disk: ${install_disk} + network: + hostname: ${hostname} + nameservers: + - 10.13.37.2 + interfaces: + - interface: eth0 + dhcp: false + kubelet: + extraArgs: + pod-max-pids: 1000 + extraConfig: + imageGCHighThresholdPercent: 75 + imageGCLowThresholdPercent: 70 +cluster: + apiServer: + auditPolicy: + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + # Log only metadata in audit logs + - level: Metadata + network: + hostname: ${hostname} + cni: + name: none + nodeLabels: + topology.kubernetes.io/region: ${cluster_name} + topology.kubernetes.io/zone: ${node_name} + proxy: + disabled: true diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf index c64dd2b..8cb028b 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf @@ -1,44 +1,93 @@ terraform { required_providers { proxmox = { - source = "telmate/proxmox" - version = ">= 3.0.2-rc05" + source = "bpg/proxmox" + version = "0.85.1" + } + talos = { + source = "siderolabs/talos" + version = "0.9.0" } } } -resource "proxmox_vm_qemu" "controlplane" { - for_each = { for node in var.controlplanes.nodes : node.name => node } - name = format("k8s-wheatley-%s", each.value.name) - target_node = "pve01" - tags = "k8s-wheatley,controlplane" - onboot = true - bios = "ovmf" - boot = "order=virtio0;net0" - clone = format("talos-%s", var.controlplanes.talos_version) - scsihw = "virtio-scsi-pci" +resource "proxmox_virtual_environment_vm" "controlplane" { - disk { - size = var.controlplanes.disk - storage = var.controlplanes.storagepool - type = "disk" - slot = "virtio0" - format = "raw" + name = var.controlplane.node_name + node_name = "pve01" + tags = ["tofu"] + bios = "ovmf" + on_boot = true + machine = "q35" + stop_on_destroy = true + + operating_system { + type = "l26" + } + agent { + enabled = true } cpu { - cores = var.controlplanes.cpu + cores = var.controlplane.cpu sockets = 1 + type = "x86-64-v2-AES" } - memory = var.controlplanes.memory * 1024 + memory { + dedicated = var.controlplane.memory * 1024 + } - network { - id = 0 - model = "virtio" + disk { + datastore_id = var.controlplane.storagepool + interface = "virtio0" + aio = "io_uring" + size = var.controlplane.disk * 1024 + file_format = "raw" + } + + cdrom { + file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.controlplane.talos_version) + } + + efi_disk { + datastore_id = var.controlplane.storagepool + file_format = "raw" + type = "4m" + } + + tpm_state { + datastore_id = var.controlplane.storagepool + version = "v2.0" + } + + initialization { + datastore_id = var.controlplane.storagepool + ip_config { + ipv4 { + address = var.controlplane.node_ipv4_address + gateway = var.controlplane.ipv4_gateway + } + } + dns { + servers = ["10.13.37.2"] + } + } + + network_device { bridge = "vmbr1" } - ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address) - skip_ipv6 = true + } + +# resource "talos_machine_secrets" "controlplane" { +# talos_version = var.controlplane.talos_version +# } +# +# data "talos_client_configuration" "controlplane" { +# cluster_name = var.controlplane.cluster_name +# client_configuration = talos_machine_secrets.controlplane.client_configuration +# nodes = [for k, v in var.controlplane : v.ip] +# endpoints = [var.controlplane.cluster_endpoint] +# } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf index 4b0fffe..7e1916a 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf @@ -1,15 +1,17 @@ -variable "controlplanes" { +variable "controlplane" { description = "Control plane node configuration" type = object({ - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - nodes = list(object({ - name = string - ip_address = string - })) + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + cluster_name = string + kubernetes_version = string + node_name = string + node_ipv4_address = string + cluster_endpoint = string + ipv4_gateway = string }) } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl new file mode 100644 index 0000000..a1da4bb --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl @@ -0,0 +1,34 @@ +debug: false +machine: + install: + disk: ${install_disk} + network: + hostname: ${hostname} + nameservers: + - 10.13.37.2 + interfaces: + - interface: eth0 + dhcp: false + kubelet: + extraArgs: + pod-max-pids: 1000 + extraConfig: + imageGCHighThresholdPercent: 75 + imageGCLowThresholdPercent: 70 +cluster: + apiServer: + auditPolicy: + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + # Log only metadata in audit logs + - level: Metadata + network: + hostname: ${hostname} + cni: + name: none + nodeLabels: + topology.kubernetes.io/region: ${cluster_name} + topology.kubernetes.io/zone: ${node_name} + proxy: + disabled: true diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf index 69928f8..2510623 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf @@ -1,44 +1,93 @@ terraform { required_providers { proxmox = { - source = "telmate/proxmox" - version = ">= 3.0.2-rc05" + source = "bpg/proxmox" + version = "0.85.1" + } + talos = { + source = "siderolabs/talos" + version = "0.9.0" } } } -resource "proxmox_vm_qemu" "worker" { - for_each = { for node in var.workers.nodes : node.name => node } - name = format("k8s-wheatley-%s", each.value.name) - target_node = "pve01" - tags = "k8s-wheatley,worker" - onboot = true - bios = "ovmf" - boot = "order=virtio0;net0" - clone = format("talos-%s", var.workers.talos_version) - scsihw = "virtio-scsi-pci" +resource "proxmox_virtual_environment_vm" "worker" { - disk { - size = var.workers.disk - storage = var.workers.storagepool - type = "disk" - slot = "virtio0" - format = "raw" + name = var.worker.node_name + node_name = "pve01" + tags = ["tofu"] + bios = "ovmf" + on_boot = true + machine = "q35" + stop_on_destroy = true + + operating_system { + type = "l26" + } + agent { + enabled = true } cpu { - cores = var.workers.cpu + cores = var.worker.cpu sockets = 1 + type = "x86-64-v2-AES" } - memory = var.workers.memory * 1024 + memory { + dedicated = var.worker.memory * 1024 + } - network { - id = 0 - model = "virtio" + disk { + datastore_id = var.worker.storagepool + interface = "virtio0" + aio = "io_uring" + size = var.worker.disk * 1024 + file_format = "raw" + } + + cdrom { + file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.worker.talos_version) + } + + efi_disk { + datastore_id = var.worker.storagepool + file_format = "raw" + type = "4m" + } + + tpm_state { + datastore_id = var.worker.storagepool + version = "v2.0" + } + + initialization { + datastore_id = var.worker.storagepool + ip_config { + ipv4 { + address = var.worker.node_ipv4_address + gateway = var.worker.ipv4_gateway + } + } + dns { + servers = ["10.13.37.2"] + } + } + + network_device { bridge = "vmbr1" } - ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address) - skip_ipv6 = true + } + +# resource "talos_machine_secrets" "controlplane" { +# talos_version = var.controlplane.talos_version +# } +# +# data "talos_client_configuration" "controlplane" { +# cluster_name = var.controlplane.cluster_name +# client_configuration = talos_machine_secrets.controlplane.client_configuration +# nodes = [for k, v in var.controlplane : v.ip] +# endpoints = [var.controlplane.cluster_endpoint] +# } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf index e08e802..627f900 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf @@ -1,15 +1,17 @@ -variable "workers" { +variable "worker" { description = "Worker node configuration" type = object({ - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - nodes = list(object({ - name = string - ip_address = string - })) + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + cluster_name = string + kubernetes_version = string + node_name = string + node_ipv4_address = string + cluster_endpoint = string + ipv4_gateway = string }) }