chore: Refactor using new Proxmox provider

pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl

@@ -1,6 +1,46 @@
 # This file is maintained automatically by "tofu init".
 # Manual edits may be lost in future updates.
 
+provider "registry.opentofu.org/bpg/proxmox" {
+  version     = "0.85.1"
+  constraints = "0.85.1"
+  hashes = [
+    "h1:VWZI8z0O7flk66jBrWK8rltUKEU8zCjljZvoQ54wwf0=",
+    "zh:0866c44f3bc222d01fb6cc998eadef102cda2434464b929466998719c98ef77f",
+    "zh:1db527d7a8b12c9fc99151798469352e756d9af0ba2fbfb155dea935df3b423a",
+    "zh:2e9dbbc846028cd7fdf21fda884c766bf3d81b7d8a2e5f9924cc48759805fe74",
+    "zh:32817963c835c7e4528c6e13147f183647b201fd17d189f102a41c478d3994e7",
+    "zh:3ef2915dee3a0c4fea4f205afde7d3318401e91989df99ee560a42302ff0d4d7",
+    "zh:5829f4d0b9731057febf6209d854a5a424d5787297ae9fb6b7f5106fc4271c04",
+    "zh:69f932cb1c5b4594914fc4bba0c110311cc5de79c56c75cf9ff6e85faf8ddd38",
+    "zh:87840842b229796c34ef07bd7c01016f19b11a83646cc6ff19c1ce3aeec010f7",
+    "zh:9299c9aeeca2afc836066e971f587facff2287125c52658b9031c97e1b62eb24",
+    "zh:ce3bbb9375518a3ad8a7a4b8434ff72bfe2d6543309f419e4128961b73d6d84e",
+    "zh:d0b5fa229ae989350f47e42244dfcf91f149c27c1f229fe5b8ad9828f21a896e",
+    "zh:d80cc976b0cc4726d18d6cac094b520c0a670898494c7f8cdb5a72903ba7b57e",
+    "zh:f1d3919f53f195e9163ca6d85f5906fc3f40d9aac4c320cbc04aee0103780242",
+    "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
+    "zh:ff1ae863a11fcbfb5850dc8f0b652c17c0e6ec0a1a4f4877e95143023426a2f7",
+  ]
+}
+
+provider "registry.opentofu.org/hashicorp/http" {
+  version = "3.5.0"
+  hashes = [
+    "h1:eClUBisXme48lqiUl3U2+H2a2mzDawS9biqfkd9synw=",
+    "zh:0a2b33494eec6a91a183629cf217e073be063624c5d3f70870456ddb478308e9",
+    "zh:180f40124fa01b98b3d2f79128646b151818e09d6a1a9ca08e0b032a0b1e9cb1",
+    "zh:3e29e1de149dc10bf78620526c7cb8c62cd76087f5630dfaba0e93cda1f3aa7b",
+    "zh:4420950200cf86042ec940d0e2c9b7c89966bf556bf8038ba36217eae663bca5",
+    "zh:5d1f7d02109b2e2dca7ec626e5563ee765583792d0fd64081286f16f9433bd0d",
+    "zh:8500b138d338b1994c4206aa577b5c44e1d7260825babcf43245a7075bfa52a5",
+    "zh:b42165a6c4cfb22825938272d12b676e4a6946ac4e750f85df870c947685df2d",
+    "zh:b919bf3ee8e3b01051a0da3433b443a925e272893d3724ee8fc0f666ec7012c9",
+    "zh:d13b81ea6755cae785b3e11634936cdff2dc1ec009dc9610d8e3c7eb32f42e69",
+    "zh:f1c9d2eb1a6b618ae77ad86649679241bd8d6aacec06d0a68d86f748687f4eb3",
+  ]
+}
+
 provider "registry.opentofu.org/siderolabs/talos" {
   version     = "0.9.0"
   constraints = "0.9.0"
@@ -23,25 +63,3 @@ provider "registry.opentofu.org/siderolabs/talos" {
     "zh:feb941cc673692c6858f96590f3ca082683a62aa2d2842b9bfffcd327eb840d3",
   ]
 }
-
-provider "registry.opentofu.org/telmate/proxmox" {
-  version     = "3.0.2-rc05"
-  constraints = ">= 3.0.2-rc05, 3.0.2-rc05"
-  hashes = [
-    "h1:PG5/Mu1UUP6/WIdtytJKZ8NxQJg8OIGygFwJOoPdmZw=",
-    "zh:042d748367f33aaf440698644be4f2a2875f9db31915c1ef84616f176fc6174f",
-    "zh:1488781da1920d60d933c8ce926c34b5e989ffae58e3fbe437973d2b1d2faafc",
-    "zh:283dd6f74627f1d1d75d616b31f8ced3f97fd5277a07c9535e85cfa765d7a321",
-    "zh:378f1c2da21aeea083ac2e632db274a02c7a01e2486a40d3c813d05a21142db3",
-    "zh:38d63d0961f8c32273392caaace30f50cff8ab06e5dda17f67a8827ebffeba98",
-    "zh:52159782df101ec98f20faff81e8f2d9d92cb4ec903314fcddcc57ec16cdaacb",
-    "zh:6ca47b90c66b1d2706cb3cbb05da8b3f90a202c4865010202b2962e2b64d217e",
-    "zh:6e7b85cb2380e4dc0be694dd0e4a24927f7f66df41960eca3cfe907443d4f0b9",
-    "zh:758775f733673ab5c196db6a33648458037746f94d4bef7ce148cb01474efe2d",
-    "zh:7c31a3ca6d52db39da2bdd60be37af71d59d808fc206de50fe661535ea436da3",
-    "zh:af16984350a2f4d77c21f66a479007801e2527543310567c99cd82eb421e249e",
-    "zh:c1f965d3f96cf3f87af2c12ab9d4bde42f8ef660f8dc34ba3cfc9b20435a7269",
-    "zh:c2b9022a31103919a5ffbac6ee8d7feb6c4f5f580c1766f769569c2e8e4ce7f1",
-    "zh:e90162c42f1237323291e3d0de0c62701b3f89350fae18246da06702f41a6123",
-  ]
-}

pve01.wheatley.in/k8s-wheatley/main.tf

@@ -1,68 +1,113 @@
-terraform {
-  required_providers {
-    proxmox = {
-      source  = "Telmate/proxmox"
-      version = "3.0.2-rc05"
-    }
-    talos = {
-      source  = "siderolabs/talos"
-      version = "0.9.0"
-    }
-  }
-}
 provider "proxmox" {
-  pm_api_url          = "https://10.167.84.10:8006/api2/json"
-  pm_api_token_id     = ""
-  pm_api_token_secret = ""
-  pm_tls_insecure     = true
+  endpoint  = var.proxmox_endpoint
+  api_token = var.proxmox_api_token
+  insecure  = true
 }
 
 locals {
-  talos_version = "1.11.3"
-}
+  cluster_name        = "k8s-wheatley"
+  kubernetes_version  = "1.34.1"
+  talos_version       = "1.11.3"
+  ipv4_gateway        = "10.13.38.1"
+  ipv4_cidr           = "/24"
+  cluster_endpoint_ip = "10.13.38.20"
 
-module "controlplanes" {
-  source = "./modules/controlplane"
 
   controlplanes = {
-    cpu           = 4
-    memory        = 4
-    disk          = "40G"
-    storagepool   = "nvme-fastpool"
-    talos_version = local.talos_version
+    cpu                = 4
+    memory             = 4
+    disk               = 40
+    storagepool        = "local-zfs"
+    talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    ipv4_gateway       = local.ipv4_gateway
     nodes = [
       {
         name       = "cp01"
-        ip_address = "10.13.38.20"
-      },
-      {
-        name       = "cp02"
         ip_address = "10.13.38.21"
       },
       {
-        name       = "cp03"
+        name       = "cp02"
         ip_address = "10.13.38.22"
-    }]
+      },
+      {
+        name       = "cp03"
+        ip_address = "10.13.38.23"
+      }
+    ]
+  }
+
+  workers = {
+    cpu                = 4
+    memory             = 4
+    disk               = 40
+    storagepool        = "local-zfs"
+    talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    ipv4_gateway       = local.ipv4_gateway
+    nodes = [
+      {
+        name       = "worker01"
+        ip_address = "10.13.38.20"
+      },
+      {
+        name       = "worker02"
+        ip_address = "10.13.38.21"
+      },
+      {
+        name       = "worker03"
+        ip_address = "10.13.38.22"
+      }
+    ]
+  }
+}
+
+module "talos-image" {
+  source = "./modules/talos-image"
+
+  talos_version = local.talos_version
+}
+
+module "controlplanes" {
+  depends_on  = [module.talos-image]
+
+  source   = "./modules/controlplane"
+  for_each = { for node in local.controlplanes.nodes : node.name => node }
+
+  controlplane = {
+    cpu                = local.controlplanes.cpu
+    memory             = local.controlplanes.memory
+    disk               = local.controlplanes.disk
+    storagepool        = local.controlplanes.storagepool
+    talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    node_name          = format("k8s-wheatley-%s", each.value.name)
+    cluster_endpoint   = format("https://%s:6443", local.cluster_endpoint_ip)
+    node_ipv4_address  = format("%s%s", each.value.ip_address, local.ipv4_cidr)
+    ipv4_gateway       = local.ipv4_gateway
   }
 }
 
 module "workers" {
-  source = "./modules/worker"
+  depends_on = [module.controlplanes]
 
-  workers = {
-    cpu           = 4
-    memory        = 4
-    disk          = "40G"
-    storagepool   = "nvme-fastpool"
-    talos_version = local.talos_version
-    nodes = [
-      {
-        name       = "worker01"
-        ip_address = "10.13.38.30"
-      },
-      {
-        name       = "worker02"
-        ip_address = "10.13.38.31"
-      }]
+  source   = "./modules/worker"
+  for_each = { for node in local.workers.nodes : node.name => node }
+
+  worker = {
+    cpu                = local.workers.cpu
+    memory             = local.workers.memory
+    disk               = local.workers.disk
+    storagepool        = local.workers.storagepool
+    talos_version      = local.talos_version
+    cluster_name       = local.cluster_name
+    kubernetes_version = local.kubernetes_version
+    node_name          = format("k8s-wheatley-%s", each.value.name)
+    cluster_endpoint   = format("https://%s:6443", local.cluster_endpoint_ip)
+    node_ipv4_address  = format("%s%s", each.value.ip_address, local.ipv4_cidr)
+    ipv4_gateway       = local.ipv4_gateway
   }
 }

pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl

@@ -0,0 +1,34 @@
+debug: false
+machine:
+  install:
+    disk: ${install_disk}
+  network:
+    hostname: ${hostname}
+    nameservers:
+      - 10.13.37.2
+    interfaces:
+      - interface: eth0
+        dhcp: false
+  kubelet:
+    extraArgs:
+      pod-max-pids: 1000
+    extraConfig:
+      imageGCHighThresholdPercent: 75
+      imageGCLowThresholdPercent: 70
+cluster:
+  apiServer:
+    auditPolicy:
+      apiVersion: audit.k8s.io/v1
+      kind: Policy
+      rules:
+        # Log only metadata in audit logs
+        - level: Metadata
+  network:
+    hostname: ${hostname}
+    cni:
+      name: none
+  nodeLabels:
+    topology.kubernetes.io/region: ${cluster_name}
+    topology.kubernetes.io/zone: ${node_name}
+  proxy:
+    disabled: true

pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf

@@ -1,44 +1,93 @@
 terraform {
   required_providers {
     proxmox = {
-      source  = "telmate/proxmox"
-      version = ">= 3.0.2-rc05"
+      source  = "bpg/proxmox"
+      version = "0.85.1"
+    }
+    talos = {
+      source  = "siderolabs/talos"
+      version = "0.9.0"
     }
   }
 }
 
-resource "proxmox_vm_qemu" "controlplane" {
-  for_each = { for node in var.controlplanes.nodes : node.name => node }
 
-  name        = format("k8s-wheatley-%s", each.value.name)
-  target_node = "pve01"
-  tags        = "k8s-wheatley,controlplane"
-  onboot      = true
-  bios        = "ovmf"
-  boot        = "order=virtio0;net0"
-  clone       = format("talos-%s", var.controlplanes.talos_version)
-  scsihw      = "virtio-scsi-pci"
+resource "proxmox_virtual_environment_vm" "controlplane" {
 
-  disk {
-    size    = var.controlplanes.disk
-    storage = var.controlplanes.storagepool
-    type    = "disk"
-    slot    = "virtio0"
-    format  = "raw"
+  name      = var.controlplane.node_name
+  node_name = "pve01"
+  tags      = ["tofu"]
+  bios      = "ovmf"
+  on_boot   = true
+  machine   = "q35"
+  stop_on_destroy = true
+
+  operating_system {
+    type = "l26"
+  }
+  agent {
+    enabled = true
   }
 
   cpu {
-    cores   = var.controlplanes.cpu
+    cores   = var.controlplane.cpu
     sockets = 1
+    type    = "x86-64-v2-AES"
   }
 
-  memory = var.controlplanes.memory * 1024
+  memory {
+    dedicated = var.controlplane.memory * 1024
+  }
 
-  network {
-    id     = 0
-    model  = "virtio"
+  disk {
+    datastore_id = var.controlplane.storagepool
+    interface    = "virtio0"
+    aio          = "io_uring"
+    size         = var.controlplane.disk * 1024
+    file_format  = "raw"
+  }
+
+  cdrom {
+    file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.controlplane.talos_version)
+  }
+
+  efi_disk {
+    datastore_id = var.controlplane.storagepool
+    file_format  = "raw"
+    type         = "4m"
+  }
+
+  tpm_state {
+    datastore_id = var.controlplane.storagepool
+    version = "v2.0"
+  }
+
+  initialization {
+    datastore_id = var.controlplane.storagepool
+    ip_config {
+      ipv4 {
+        address = var.controlplane.node_ipv4_address
+        gateway = var.controlplane.ipv4_gateway
+      }
+    }
+    dns {
+      servers = ["10.13.37.2"]
+    }
+  }
+
+  network_device {
     bridge = "vmbr1"
   }
-  ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address)
-  skip_ipv6 = true
+
 }
+
+# resource "talos_machine_secrets" "controlplane" {
+#   talos_version = var.controlplane.talos_version
+# }
+#
+# data "talos_client_configuration" "controlplane" {
+#   cluster_name         = var.controlplane.cluster_name
+#   client_configuration = talos_machine_secrets.controlplane.client_configuration
+#   nodes                = [for k, v in var.controlplane : v.ip]
+#   endpoints            = [var.controlplane.cluster_endpoint]
+# }

pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf

@@ -1,15 +1,17 @@
-variable "controlplanes" {
+variable "controlplane" {
   description = "Control plane node configuration"
   type = object({
-    cpu           = number
-    memory        = number
-    disk          = string
-    storagepool   = string
-    talos_version = string
-    nodes = list(object({
-      name       = string
-      ip_address = string
-    }))
+    cpu                = number
+    memory             = number
+    disk               = string
+    storagepool        = string
+    talos_version      = string
+    cluster_name       = string
+    kubernetes_version = string
+    node_name          = string
+    node_ipv4_address  = string
+    cluster_endpoint   = string
+    ipv4_gateway       = string
   })
 }
 

pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf

@@ -0,0 +1,34 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "0.85.1"
+    }
+  }
+}
+
+data "http" "schematic" {
+  url          = "https://factory.talos.dev/schematics"
+  method       = "POST"
+  request_body = file("${path.module}/schematic.yaml")
+  request_headers = {
+    "Content-Type" = "application/x-yaml"
+  }
+}
+
+locals {
+  schematic_id = jsondecode(data.http.schematic.response_body)["id"]
+  iso_url      = "${"https://factory.talos.dev/image"}/${local.schematic_id}/v${var.talos_version}/nocloud-amd64-secureboot.iso"
+  iso_file     = "talos-${var.talos_version}-nocloud-amd64-secureboot.iso"
+}
+
+resource "proxmox_virtual_environment_download_file" "talos-image" {
+  node_name               = "pve01"
+  content_type            = "iso"
+  datastore_id            = "local"
+  overwrite               = true
+
+  url       = local.iso_url
+  file_name = local.iso_file
+
+}

pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml

@@ -0,0 +1,4 @@
+customization:
+  systemExtensions:
+    officialExtensions:
+      - siderolabs/qemu-guest-agent

pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf

@@ -0,0 +1,4 @@
+variable "talos_version" {
+  description = "Talos version to download"
+  type        = string
+}

pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl

@@ -0,0 +1,34 @@
+debug: false
+machine:
+  install:
+    disk: ${install_disk}
+  network:
+    hostname: ${hostname}
+    nameservers:
+      - 10.13.37.2
+    interfaces:
+      - interface: eth0
+        dhcp: false
+  kubelet:
+    extraArgs:
+      pod-max-pids: 1000
+    extraConfig:
+      imageGCHighThresholdPercent: 75
+      imageGCLowThresholdPercent: 70
+cluster:
+  apiServer:
+    auditPolicy:
+      apiVersion: audit.k8s.io/v1
+      kind: Policy
+      rules:
+        # Log only metadata in audit logs
+        - level: Metadata
+  network:
+    hostname: ${hostname}
+    cni:
+      name: none
+  nodeLabels:
+    topology.kubernetes.io/region: ${cluster_name}
+    topology.kubernetes.io/zone: ${node_name}
+  proxy:
+    disabled: true

pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf

@@ -1,44 +1,93 @@
 terraform {
   required_providers {
     proxmox = {
-      source  = "telmate/proxmox"
-      version = ">= 3.0.2-rc05"
+      source  = "bpg/proxmox"
+      version = "0.85.1"
+    }
+    talos = {
+      source  = "siderolabs/talos"
+      version = "0.9.0"
     }
   }
 }
 
-resource "proxmox_vm_qemu" "worker" {
-  for_each = { for node in var.workers.nodes : node.name => node }
 
-  name        = format("k8s-wheatley-%s", each.value.name)
-  target_node = "pve01"
-  tags        = "k8s-wheatley,worker"
-  onboot      = true
-  bios        = "ovmf"
-  boot        = "order=virtio0;net0"
-  clone       = format("talos-%s", var.workers.talos_version)
-  scsihw      = "virtio-scsi-pci"
+resource "proxmox_virtual_environment_vm" "worker" {
 
-  disk {
-    size    = var.workers.disk
-    storage = var.workers.storagepool
-    type    = "disk"
-    slot    = "virtio0"
-    format  = "raw"
+  name      = var.worker.node_name
+  node_name = "pve01"
+  tags      = ["tofu"]
+  bios      = "ovmf"
+  on_boot   = true
+  machine   = "q35"
+  stop_on_destroy = true
+
+  operating_system {
+    type = "l26"
+  }
+  agent {
+    enabled = true
   }
 
   cpu {
-    cores   = var.workers.cpu
+    cores   = var.worker.cpu
     sockets = 1
+    type    = "x86-64-v2-AES"
   }
 
-  memory = var.workers.memory * 1024
+  memory {
+    dedicated = var.worker.memory * 1024
+  }
 
-  network {
-    id     = 0
-    model  = "virtio"
+  disk {
+    datastore_id = var.worker.storagepool
+    interface    = "virtio0"
+    aio          = "io_uring"
+    size         = var.worker.disk * 1024
+    file_format  = "raw"
+  }
+
+  cdrom {
+    file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.worker.talos_version)
+  }
+
+  efi_disk {
+    datastore_id = var.worker.storagepool
+    file_format  = "raw"
+    type         = "4m"
+  }
+
+  tpm_state {
+    datastore_id = var.worker.storagepool
+    version = "v2.0"
+  }
+
+  initialization {
+    datastore_id = var.worker.storagepool
+    ip_config {
+      ipv4 {
+        address = var.worker.node_ipv4_address
+        gateway = var.worker.ipv4_gateway
+      }
+    }
+    dns {
+      servers = ["10.13.37.2"]
+    }
+  }
+
+  network_device {
     bridge = "vmbr1"
   }
-  ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address)
-  skip_ipv6 = true
+
 }
+
+# resource "talos_machine_secrets" "controlplane" {
+#   talos_version = var.controlplane.talos_version
+# }
+#
+# data "talos_client_configuration" "controlplane" {
+#   cluster_name         = var.controlplane.cluster_name
+#   client_configuration = talos_machine_secrets.controlplane.client_configuration
+#   nodes                = [for k, v in var.controlplane : v.ip]
+#   endpoints            = [var.controlplane.cluster_endpoint]
+# }

pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf

@@ -1,15 +1,17 @@
-variable "workers" {
+variable "worker" {
   description = "Worker node configuration"
   type = object({
-    cpu           = number
-    memory        = number
-    disk          = string
-    storagepool   = string
-    talos_version = string
-    nodes = list(object({
-      name       = string
-      ip_address = string
-    }))
+    cpu                = number
+    memory             = number
+    disk               = string
+    storagepool        = string
+    talos_version      = string
+    cluster_name       = string
+    kubernetes_version = string
+    node_name          = string
+    node_ipv4_address  = string
+    cluster_endpoint   = string
+    ipv4_gateway       = string
   })
 }
 

pve01.wheatley.in/k8s-wheatley/providers.tf

@@ -0,0 +1,12 @@
+terraform {
+  required_providers {
+    talos = {
+      source  = "siderolabs/talos"
+      version = "0.9.0"
+    }
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "0.85.1"
+    }
+  }
+}

pve01.wheatley.in/k8s-wheatley/variables.tf

@@ -0,0 +1,8 @@
+variable "proxmox_endpoint" {
+  description = "Proxmox API endpoint"
+  type        = string
+}
+variable "proxmox_api_token" {
+  description = "Proxmox API token"
+  type        = string
+}