chore: Refactor using new Proxmox provider

pve01.wheatley.in/k8s-wheatley/main.tf

@@ -5,56 +5,109 @@ provider "proxmox" {
 }
 
 locals {
+  cluster_name        = "k8s-wheatley"
+  kubernetes_version  = "1.34.1"
   talos_version       = "1.11.3"
-}
+  ipv4_gateway        = "10.13.38.1"
+  ipv4_cidr           = "/24"
+  cluster_endpoint_ip = "10.13.38.20"
 
-module "controlplanes" {
-  source = "./modules/controlplane"
 
   controlplanes = {
     cpu                = 4
     memory             = 4
-    disk          = "40G"
+    disk               = 40
-    storagepool   = "nvme-fastpool"
+    storagepool        = "local-zfs"
     talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    ipv4_gateway       = local.ipv4_gateway
     nodes = [
       {
         name       = "cp01"
-        ip_address = "10.13.38.20"
-      },
-      {
-        name       = "cp02"
         ip_address = "10.13.38.21"
       },
       {
-        name       = "cp03"
+        name       = "cp02"
         ip_address = "10.13.38.22"
-    }]
+      },
+      {
+        name       = "cp03"
+        ip_address = "10.13.38.23"
+      }
+    ]
   }
-}
-
-module "workers" {
-  source = "./modules/worker"
 
   workers = {
     cpu                = 4
     memory             = 4
-    disk          = "40G"
+    disk               = 40
-    storagepool   = "nvme-fastpool"
+    storagepool        = "local-zfs"
     talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    ipv4_gateway       = local.ipv4_gateway
     nodes = [
       {
         name       = "worker01"
-        ip_address = "10.13.38.30"
+        ip_address = "10.13.38.20"
       },
       {
         name       = "worker02"
-        ip_address = "10.13.38.31"
+        ip_address = "10.13.38.21"
-      }]
+      },
+      {
+        name       = "worker03"
+        ip_address = "10.13.38.22"
+      }
+    ]
+  }
+}
+
 module "talos-image" {
   source = "./modules/talos-image"
 
   talos_version = local.talos_version
 }
+
+module "controlplanes" {
+  depends_on  = [module.talos-image]
+
+  source   = "./modules/controlplane"
+  for_each = { for node in local.controlplanes.nodes : node.name => node }
+
+  controlplane = {
+    cpu                = local.controlplanes.cpu
+    memory             = local.controlplanes.memory
+    disk               = local.controlplanes.disk
+    storagepool        = local.controlplanes.storagepool
+    talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    node_name          = format("k8s-wheatley-%s", each.value.name)
+    cluster_endpoint   = format("https://%s:6443", local.cluster_endpoint_ip)
+    node_ipv4_address  = format("%s%s", each.value.ip_address, local.ipv4_cidr)
+    ipv4_gateway       = local.ipv4_gateway
+  }
+}
+
+module "workers" {
+  depends_on = [module.controlplanes]
+
+  source   = "./modules/worker"
+  for_each = { for node in local.workers.nodes : node.name => node }
+
+  worker = {
+    cpu                = local.workers.cpu
+    memory             = local.workers.memory
+    disk               = local.workers.disk
+    storagepool        = local.workers.storagepool
+    talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    node_name          = format("k8s-wheatley-%s", each.value.name)
+    cluster_endpoint   = format("https://%s:6443", local.cluster_endpoint_ip)
+    node_ipv4_address  = format("%s%s", each.value.ip_address, local.ipv4_cidr)
+    ipv4_gateway       = local.ipv4_gateway
   }
 }

pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl

@@ -0,0 +1,34 @@
+debug: false
+machine:
+  install:
+    disk: ${install_disk}
+  network:
+    hostname: ${hostname}
+    nameservers:
+      - 10.13.37.2
+    interfaces:
+      - interface: eth0
+        dhcp: false
+  kubelet:
+    extraArgs:
+      pod-max-pids: 1000
+    extraConfig:
+      imageGCHighThresholdPercent: 75
+      imageGCLowThresholdPercent: 70
+cluster:
+  apiServer:
+    auditPolicy:
+      apiVersion: audit.k8s.io/v1
+      kind: Policy
+      rules:
+        # Log only metadata in audit logs
+        - level: Metadata
+  network:
+    hostname: ${hostname}
+    cni:
+      name: none
+  nodeLabels:
+    topology.kubernetes.io/region: ${cluster_name}
+    topology.kubernetes.io/zone: ${node_name}
+  proxy:
+    disabled: true

pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf

@@ -1,44 +1,93 @@
 terraform {
   required_providers {
     proxmox = {
-      source  = "telmate/proxmox"
+      source  = "bpg/proxmox"
-      version = ">= 3.0.2-rc05"
+      version = "0.85.1"
+    }
+    talos = {
+      source  = "siderolabs/talos"
+      version = "0.9.0"
     }
   }
 }
 
-resource "proxmox_vm_qemu" "controlplane" {
-  for_each = { for node in var.controlplanes.nodes : node.name => node }
 
-  name        = format("k8s-wheatley-%s", each.value.name)
+resource "proxmox_virtual_environment_vm" "controlplane" {
-  target_node = "pve01"
+
-  tags        = "k8s-wheatley,controlplane"
+  name      = var.controlplane.node_name
-  onboot      = true
+  node_name = "pve01"
+  tags      = ["tofu"]
   bios      = "ovmf"
-  boot        = "order=virtio0;net0"
+  on_boot   = true
-  clone       = format("talos-%s", var.controlplanes.talos_version)
+  machine   = "q35"
-  scsihw      = "virtio-scsi-pci"
+  stop_on_destroy = true
 
-  disk {
+  operating_system {
-    size    = var.controlplanes.disk
+    type = "l26"
-    storage = var.controlplanes.storagepool
+  }
-    type    = "disk"
+  agent {
-    slot    = "virtio0"
+    enabled = true
-    format  = "raw"
   }
 
   cpu {
-    cores   = var.controlplanes.cpu
+    cores   = var.controlplane.cpu
     sockets = 1
+    type    = "x86-64-v2-AES"
   }
 
-  memory = var.controlplanes.memory * 1024
+  memory {
+    dedicated = var.controlplane.memory * 1024
+  }
 
-  network {
+  disk {
-    id     = 0
+    datastore_id = var.controlplane.storagepool
-    model  = "virtio"
+    interface    = "virtio0"
+    aio          = "io_uring"
+    size         = var.controlplane.disk * 1024
+    file_format  = "raw"
+  }
+
+  cdrom {
+    file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.controlplane.talos_version)
+  }
+
+  efi_disk {
+    datastore_id = var.controlplane.storagepool
+    file_format  = "raw"
+    type         = "4m"
+  }
+
+  tpm_state {
+    datastore_id = var.controlplane.storagepool
+    version = "v2.0"
+  }
+
+  initialization {
+    datastore_id = var.controlplane.storagepool
+    ip_config {
+      ipv4 {
+        address = var.controlplane.node_ipv4_address
+        gateway = var.controlplane.ipv4_gateway
+      }
+    }
+    dns {
+      servers = ["10.13.37.2"]
+    }
+  }
+
+  network_device {
     bridge = "vmbr1"
   }
-  ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address)
+
-  skip_ipv6 = true
 }
+
+# resource "talos_machine_secrets" "controlplane" {
+#   talos_version = var.controlplane.talos_version
+# }
+#
+# data "talos_client_configuration" "controlplane" {
+#   cluster_name         = var.controlplane.cluster_name
+#   client_configuration = talos_machine_secrets.controlplane.client_configuration
+#   nodes                = [for k, v in var.controlplane : v.ip]
+#   endpoints            = [var.controlplane.cluster_endpoint]
+# }

pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf

@@ -1,4 +1,4 @@
-variable "controlplanes" {
+variable "controlplane" {
   description = "Control plane node configuration"
   type = object({
     cpu                = number
@@ -6,10 +6,12 @@ variable "controlplanes" {
     disk               = string
     storagepool        = string
     talos_version      = string
-    nodes = list(object({
+    cluster_name       = string
-      name       = string
+    kubernetes_version = string
-      ip_address = string
+    node_name          = string
-    }))
+    node_ipv4_address  = string
+    cluster_endpoint   = string
+    ipv4_gateway       = string
   })
 }
 

pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl

@@ -0,0 +1,34 @@
+debug: false
+machine:
+  install:
+    disk: ${install_disk}
+  network:
+    hostname: ${hostname}
+    nameservers:
+      - 10.13.37.2
+    interfaces:
+      - interface: eth0
+        dhcp: false
+  kubelet:
+    extraArgs:
+      pod-max-pids: 1000
+    extraConfig:
+      imageGCHighThresholdPercent: 75
+      imageGCLowThresholdPercent: 70
+cluster:
+  apiServer:
+    auditPolicy:
+      apiVersion: audit.k8s.io/v1
+      kind: Policy
+      rules:
+        # Log only metadata in audit logs
+        - level: Metadata
+  network:
+    hostname: ${hostname}
+    cni:
+      name: none
+  nodeLabels:
+    topology.kubernetes.io/region: ${cluster_name}
+    topology.kubernetes.io/zone: ${node_name}
+  proxy:
+    disabled: true

pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf

@@ -1,44 +1,93 @@
 terraform {
   required_providers {
     proxmox = {
-      source  = "telmate/proxmox"
+      source  = "bpg/proxmox"
-      version = ">= 3.0.2-rc05"
+      version = "0.85.1"
+    }
+    talos = {
+      source  = "siderolabs/talos"
+      version = "0.9.0"
     }
   }
 }
 
-resource "proxmox_vm_qemu" "worker" {
-  for_each = { for node in var.workers.nodes : node.name => node }
 
-  name        = format("k8s-wheatley-%s", each.value.name)
+resource "proxmox_virtual_environment_vm" "worker" {
-  target_node = "pve01"
+
-  tags        = "k8s-wheatley,worker"
+  name      = var.worker.node_name
-  onboot      = true
+  node_name = "pve01"
+  tags      = ["tofu"]
   bios      = "ovmf"
-  boot        = "order=virtio0;net0"
+  on_boot   = true
-  clone       = format("talos-%s", var.workers.talos_version)
+  machine   = "q35"
-  scsihw      = "virtio-scsi-pci"
+  stop_on_destroy = true
 
-  disk {
+  operating_system {
-    size    = var.workers.disk
+    type = "l26"
-    storage = var.workers.storagepool
+  }
-    type    = "disk"
+  agent {
-    slot    = "virtio0"
+    enabled = true
-    format  = "raw"
   }
 
   cpu {
-    cores   = var.workers.cpu
+    cores   = var.worker.cpu
     sockets = 1
+    type    = "x86-64-v2-AES"
   }
 
-  memory = var.workers.memory * 1024
+  memory {
+    dedicated = var.worker.memory * 1024
+  }
 
-  network {
+  disk {
-    id     = 0
+    datastore_id = var.worker.storagepool
-    model  = "virtio"
+    interface    = "virtio0"
+    aio          = "io_uring"
+    size         = var.worker.disk * 1024
+    file_format  = "raw"
+  }
+
+  cdrom {
+    file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.worker.talos_version)
+  }
+
+  efi_disk {
+    datastore_id = var.worker.storagepool
+    file_format  = "raw"
+    type         = "4m"
+  }
+
+  tpm_state {
+    datastore_id = var.worker.storagepool
+    version = "v2.0"
+  }
+
+  initialization {
+    datastore_id = var.worker.storagepool
+    ip_config {
+      ipv4 {
+        address = var.worker.node_ipv4_address
+        gateway = var.worker.ipv4_gateway
+      }
+    }
+    dns {
+      servers = ["10.13.37.2"]
+    }
+  }
+
+  network_device {
     bridge = "vmbr1"
   }
-  ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address)
+
-  skip_ipv6 = true
 }
+
+# resource "talos_machine_secrets" "controlplane" {
+#   talos_version = var.controlplane.talos_version
+# }
+#
+# data "talos_client_configuration" "controlplane" {
+#   cluster_name         = var.controlplane.cluster_name
+#   client_configuration = talos_machine_secrets.controlplane.client_configuration
+#   nodes                = [for k, v in var.controlplane : v.ip]
+#   endpoints            = [var.controlplane.cluster_endpoint]
+# }

pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf

@@ -1,4 +1,4 @@
-variable "workers" {
+variable "worker" {
   description = "Worker node configuration"
   type = object({
     cpu                = number
@@ -6,10 +6,12 @@ variable "workers" {
     disk               = string
     storagepool        = string
     talos_version      = string
-    nodes = list(object({
+    cluster_name       = string
-      name       = string
+    kubernetes_version = string
-      ip_address = string
+    node_name          = string
-    }))
+    node_ipv4_address  = string
+    cluster_endpoint   = string
+    ipv4_gateway       = string
   })
 }