diff --git a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl index 8b2a866..bd5e235 100644 --- a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl +++ b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl @@ -1,46 +1,6 @@ # This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. -provider "registry.opentofu.org/bpg/proxmox" { - version = "0.85.1" - constraints = "0.85.1" - hashes = [ - "h1:VWZI8z0O7flk66jBrWK8rltUKEU8zCjljZvoQ54wwf0=", - "zh:0866c44f3bc222d01fb6cc998eadef102cda2434464b929466998719c98ef77f", - "zh:1db527d7a8b12c9fc99151798469352e756d9af0ba2fbfb155dea935df3b423a", - "zh:2e9dbbc846028cd7fdf21fda884c766bf3d81b7d8a2e5f9924cc48759805fe74", - "zh:32817963c835c7e4528c6e13147f183647b201fd17d189f102a41c478d3994e7", - "zh:3ef2915dee3a0c4fea4f205afde7d3318401e91989df99ee560a42302ff0d4d7", - "zh:5829f4d0b9731057febf6209d854a5a424d5787297ae9fb6b7f5106fc4271c04", - "zh:69f932cb1c5b4594914fc4bba0c110311cc5de79c56c75cf9ff6e85faf8ddd38", - "zh:87840842b229796c34ef07bd7c01016f19b11a83646cc6ff19c1ce3aeec010f7", - "zh:9299c9aeeca2afc836066e971f587facff2287125c52658b9031c97e1b62eb24", - "zh:ce3bbb9375518a3ad8a7a4b8434ff72bfe2d6543309f419e4128961b73d6d84e", - "zh:d0b5fa229ae989350f47e42244dfcf91f149c27c1f229fe5b8ad9828f21a896e", - "zh:d80cc976b0cc4726d18d6cac094b520c0a670898494c7f8cdb5a72903ba7b57e", - "zh:f1d3919f53f195e9163ca6d85f5906fc3f40d9aac4c320cbc04aee0103780242", - "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", - "zh:ff1ae863a11fcbfb5850dc8f0b652c17c0e6ec0a1a4f4877e95143023426a2f7", - ] -} - -provider "registry.opentofu.org/hashicorp/http" { - version = "3.5.0" - hashes = [ - "h1:eClUBisXme48lqiUl3U2+H2a2mzDawS9biqfkd9synw=", - "zh:0a2b33494eec6a91a183629cf217e073be063624c5d3f70870456ddb478308e9", - "zh:180f40124fa01b98b3d2f79128646b151818e09d6a1a9ca08e0b032a0b1e9cb1", - "zh:3e29e1de149dc10bf78620526c7cb8c62cd76087f5630dfaba0e93cda1f3aa7b", - "zh:4420950200cf86042ec940d0e2c9b7c89966bf556bf8038ba36217eae663bca5", - "zh:5d1f7d02109b2e2dca7ec626e5563ee765583792d0fd64081286f16f9433bd0d", - "zh:8500b138d338b1994c4206aa577b5c44e1d7260825babcf43245a7075bfa52a5", - "zh:b42165a6c4cfb22825938272d12b676e4a6946ac4e750f85df870c947685df2d", - "zh:b919bf3ee8e3b01051a0da3433b443a925e272893d3724ee8fc0f666ec7012c9", - "zh:d13b81ea6755cae785b3e11634936cdff2dc1ec009dc9610d8e3c7eb32f42e69", - "zh:f1c9d2eb1a6b618ae77ad86649679241bd8d6aacec06d0a68d86f748687f4eb3", - ] -} - provider "registry.opentofu.org/siderolabs/talos" { version = "0.9.0" constraints = "0.9.0" @@ -63,3 +23,25 @@ provider "registry.opentofu.org/siderolabs/talos" { "zh:feb941cc673692c6858f96590f3ca082683a62aa2d2842b9bfffcd327eb840d3", ] } + +provider "registry.opentofu.org/telmate/proxmox" { + version = "3.0.2-rc05" + constraints = ">= 3.0.2-rc05, 3.0.2-rc05" + hashes = [ + "h1:PG5/Mu1UUP6/WIdtytJKZ8NxQJg8OIGygFwJOoPdmZw=", + "zh:042d748367f33aaf440698644be4f2a2875f9db31915c1ef84616f176fc6174f", + "zh:1488781da1920d60d933c8ce926c34b5e989ffae58e3fbe437973d2b1d2faafc", + "zh:283dd6f74627f1d1d75d616b31f8ced3f97fd5277a07c9535e85cfa765d7a321", + "zh:378f1c2da21aeea083ac2e632db274a02c7a01e2486a40d3c813d05a21142db3", + "zh:38d63d0961f8c32273392caaace30f50cff8ab06e5dda17f67a8827ebffeba98", + "zh:52159782df101ec98f20faff81e8f2d9d92cb4ec903314fcddcc57ec16cdaacb", + "zh:6ca47b90c66b1d2706cb3cbb05da8b3f90a202c4865010202b2962e2b64d217e", + "zh:6e7b85cb2380e4dc0be694dd0e4a24927f7f66df41960eca3cfe907443d4f0b9", + "zh:758775f733673ab5c196db6a33648458037746f94d4bef7ce148cb01474efe2d", + "zh:7c31a3ca6d52db39da2bdd60be37af71d59d808fc206de50fe661535ea436da3", + "zh:af16984350a2f4d77c21f66a479007801e2527543310567c99cd82eb421e249e", + "zh:c1f965d3f96cf3f87af2c12ab9d4bde42f8ef660f8dc34ba3cfc9b20435a7269", + "zh:c2b9022a31103919a5ffbac6ee8d7feb6c4f5f580c1766f769569c2e8e4ce7f1", + "zh:e90162c42f1237323291e3d0de0c62701b3f89350fae18246da06702f41a6123", + ] +} diff --git a/pve01.wheatley.in/k8s-wheatley/main.tf b/pve01.wheatley.in/k8s-wheatley/main.tf index e0079a3..d387f48 100644 --- a/pve01.wheatley.in/k8s-wheatley/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/main.tf @@ -1,113 +1,68 @@ +terraform { + required_providers { + proxmox = { + source = "Telmate/proxmox" + version = "3.0.2-rc05" + } + talos = { + source = "siderolabs/talos" + version = "0.9.0" + } + } +} provider "proxmox" { - endpoint = var.proxmox_endpoint - api_token = var.proxmox_api_token - insecure = true + pm_api_url = "https://10.167.84.10:8006/api2/json" + pm_api_token_id = "" + pm_api_token_secret = "" + pm_tls_insecure = true } locals { - cluster_name = "k8s-wheatley" - kubernetes_version = "1.34.1" - talos_version = "1.11.3" - ipv4_gateway = "10.13.38.1" - ipv4_cidr = "/24" - cluster_endpoint_ip = "10.13.38.20" - - - controlplanes = { - cpu = 4 - memory = 4 - disk = 40 - storagepool = "local-zfs" - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - ipv4_gateway = local.ipv4_gateway - nodes = [ - { - name = "cp01" - ip_address = "10.13.38.21" - }, - { - name = "cp02" - ip_address = "10.13.38.22" - }, - { - name = "cp03" - ip_address = "10.13.38.23" - } - ] - } - - workers = { - cpu = 4 - memory = 4 - disk = 40 - storagepool = "local-zfs" - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - ipv4_gateway = local.ipv4_gateway - nodes = [ - { - name = "worker01" - ip_address = "10.13.38.20" - }, - { - name = "worker02" - ip_address = "10.13.38.21" - }, - { - name = "worker03" - ip_address = "10.13.38.22" - } - ] - } -} - -module "talos-image" { - source = "./modules/talos-image" - - talos_version = local.talos_version + talos_version = "1.11.3" } module "controlplanes" { - depends_on = [module.talos-image] + source = "./modules/controlplane" - source = "./modules/controlplane" - for_each = { for node in local.controlplanes.nodes : node.name => node } - - controlplane = { - cpu = local.controlplanes.cpu - memory = local.controlplanes.memory - disk = local.controlplanes.disk - storagepool = local.controlplanes.storagepool - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - node_name = format("k8s-wheatley-%s", each.value.name) - cluster_endpoint = format("https://%s:6443", local.cluster_endpoint_ip) - node_ipv4_address = format("%s%s", each.value.ip_address, local.ipv4_cidr) - ipv4_gateway = local.ipv4_gateway + controlplanes = { + cpu = 4 + memory = 4 + disk = "40G" + storagepool = "nvme-fastpool" + talos_version = local.talos_version + nodes = [ + { + name = "cp01" + ip_address = "10.13.38.20" + }, + { + name = "cp02" + ip_address = "10.13.38.21" + }, + { + name = "cp03" + ip_address = "10.13.38.22" + }] } } module "workers" { - depends_on = [module.controlplanes] + source = "./modules/worker" - source = "./modules/worker" - for_each = { for node in local.workers.nodes : node.name => node } - - worker = { - cpu = local.workers.cpu - memory = local.workers.memory - disk = local.workers.disk - storagepool = local.workers.storagepool - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - node_name = format("k8s-wheatley-%s", each.value.name) - cluster_endpoint = format("https://%s:6443", local.cluster_endpoint_ip) - node_ipv4_address = format("%s%s", each.value.ip_address, local.ipv4_cidr) - ipv4_gateway = local.ipv4_gateway + workers = { + cpu = 4 + memory = 4 + disk = "40G" + storagepool = "nvme-fastpool" + talos_version = local.talos_version + nodes = [ + { + name = "worker01" + ip_address = "10.13.38.30" + }, + { + name = "worker02" + ip_address = "10.13.38.31" + }] } } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl deleted file mode 100644 index a1da4bb..0000000 --- a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl +++ /dev/null @@ -1,34 +0,0 @@ -debug: false -machine: - install: - disk: ${install_disk} - network: - hostname: ${hostname} - nameservers: - - 10.13.37.2 - interfaces: - - interface: eth0 - dhcp: false - kubelet: - extraArgs: - pod-max-pids: 1000 - extraConfig: - imageGCHighThresholdPercent: 75 - imageGCLowThresholdPercent: 70 -cluster: - apiServer: - auditPolicy: - apiVersion: audit.k8s.io/v1 - kind: Policy - rules: - # Log only metadata in audit logs - - level: Metadata - network: - hostname: ${hostname} - cni: - name: none - nodeLabels: - topology.kubernetes.io/region: ${cluster_name} - topology.kubernetes.io/zone: ${node_name} - proxy: - disabled: true diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf index 8cb028b..c64dd2b 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf @@ -1,93 +1,44 @@ terraform { required_providers { proxmox = { - source = "bpg/proxmox" - version = "0.85.1" - } - talos = { - source = "siderolabs/talos" - version = "0.9.0" + source = "telmate/proxmox" + version = ">= 3.0.2-rc05" } } } +resource "proxmox_vm_qemu" "controlplane" { + for_each = { for node in var.controlplanes.nodes : node.name => node } -resource "proxmox_virtual_environment_vm" "controlplane" { + name = format("k8s-wheatley-%s", each.value.name) + target_node = "pve01" + tags = "k8s-wheatley,controlplane" + onboot = true + bios = "ovmf" + boot = "order=virtio0;net0" + clone = format("talos-%s", var.controlplanes.talos_version) + scsihw = "virtio-scsi-pci" - name = var.controlplane.node_name - node_name = "pve01" - tags = ["tofu"] - bios = "ovmf" - on_boot = true - machine = "q35" - stop_on_destroy = true - - operating_system { - type = "l26" - } - agent { - enabled = true + disk { + size = var.controlplanes.disk + storage = var.controlplanes.storagepool + type = "disk" + slot = "virtio0" + format = "raw" } cpu { - cores = var.controlplane.cpu + cores = var.controlplanes.cpu sockets = 1 - type = "x86-64-v2-AES" } - memory { - dedicated = var.controlplane.memory * 1024 - } + memory = var.controlplanes.memory * 1024 - disk { - datastore_id = var.controlplane.storagepool - interface = "virtio0" - aio = "io_uring" - size = var.controlplane.disk * 1024 - file_format = "raw" - } - - cdrom { - file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.controlplane.talos_version) - } - - efi_disk { - datastore_id = var.controlplane.storagepool - file_format = "raw" - type = "4m" - } - - tpm_state { - datastore_id = var.controlplane.storagepool - version = "v2.0" - } - - initialization { - datastore_id = var.controlplane.storagepool - ip_config { - ipv4 { - address = var.controlplane.node_ipv4_address - gateway = var.controlplane.ipv4_gateway - } - } - dns { - servers = ["10.13.37.2"] - } - } - - network_device { + network { + id = 0 + model = "virtio" bridge = "vmbr1" } - + ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address) + skip_ipv6 = true } - -# resource "talos_machine_secrets" "controlplane" { -# talos_version = var.controlplane.talos_version -# } -# -# data "talos_client_configuration" "controlplane" { -# cluster_name = var.controlplane.cluster_name -# client_configuration = talos_machine_secrets.controlplane.client_configuration -# nodes = [for k, v in var.controlplane : v.ip] -# endpoints = [var.controlplane.cluster_endpoint] -# } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf index 7e1916a..4b0fffe 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf @@ -1,17 +1,15 @@ -variable "controlplane" { +variable "controlplanes" { description = "Control plane node configuration" type = object({ - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - cluster_name = string - kubernetes_version = string - node_name = string - node_ipv4_address = string - cluster_endpoint = string - ipv4_gateway = string + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + nodes = list(object({ + name = string + ip_address = string + })) }) } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf deleted file mode 100644 index 610ecd4..0000000 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf +++ /dev/null @@ -1,34 +0,0 @@ -terraform { - required_providers { - proxmox = { - source = "bpg/proxmox" - version = "0.85.1" - } - } -} - -data "http" "schematic" { - url = "https://factory.talos.dev/schematics" - method = "POST" - request_body = file("${path.module}/schematic.yaml") - request_headers = { - "Content-Type" = "application/x-yaml" - } -} - -locals { - schematic_id = jsondecode(data.http.schematic.response_body)["id"] - iso_url = "${"https://factory.talos.dev/image"}/${local.schematic_id}/v${var.talos_version}/nocloud-amd64-secureboot.iso" - iso_file = "talos-${var.talos_version}-nocloud-amd64-secureboot.iso" -} - -resource "proxmox_virtual_environment_download_file" "talos-image" { - node_name = "pve01" - content_type = "iso" - datastore_id = "local" - overwrite = true - - url = local.iso_url - file_name = local.iso_file - -} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml deleted file mode 100644 index f27af6e..0000000 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml +++ /dev/null @@ -1,4 +0,0 @@ -customization: - systemExtensions: - officialExtensions: - - siderolabs/qemu-guest-agent diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf deleted file mode 100644 index 99ad2b8..0000000 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf +++ /dev/null @@ -1,4 +0,0 @@ -variable "talos_version" { - description = "Talos version to download" - type = string -} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl deleted file mode 100644 index a1da4bb..0000000 --- a/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl +++ /dev/null @@ -1,34 +0,0 @@ -debug: false -machine: - install: - disk: ${install_disk} - network: - hostname: ${hostname} - nameservers: - - 10.13.37.2 - interfaces: - - interface: eth0 - dhcp: false - kubelet: - extraArgs: - pod-max-pids: 1000 - extraConfig: - imageGCHighThresholdPercent: 75 - imageGCLowThresholdPercent: 70 -cluster: - apiServer: - auditPolicy: - apiVersion: audit.k8s.io/v1 - kind: Policy - rules: - # Log only metadata in audit logs - - level: Metadata - network: - hostname: ${hostname} - cni: - name: none - nodeLabels: - topology.kubernetes.io/region: ${cluster_name} - topology.kubernetes.io/zone: ${node_name} - proxy: - disabled: true diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf index 2510623..69928f8 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf @@ -1,93 +1,44 @@ terraform { required_providers { proxmox = { - source = "bpg/proxmox" - version = "0.85.1" - } - talos = { - source = "siderolabs/talos" - version = "0.9.0" + source = "telmate/proxmox" + version = ">= 3.0.2-rc05" } } } +resource "proxmox_vm_qemu" "worker" { + for_each = { for node in var.workers.nodes : node.name => node } -resource "proxmox_virtual_environment_vm" "worker" { + name = format("k8s-wheatley-%s", each.value.name) + target_node = "pve01" + tags = "k8s-wheatley,worker" + onboot = true + bios = "ovmf" + boot = "order=virtio0;net0" + clone = format("talos-%s", var.workers.talos_version) + scsihw = "virtio-scsi-pci" - name = var.worker.node_name - node_name = "pve01" - tags = ["tofu"] - bios = "ovmf" - on_boot = true - machine = "q35" - stop_on_destroy = true - - operating_system { - type = "l26" - } - agent { - enabled = true + disk { + size = var.workers.disk + storage = var.workers.storagepool + type = "disk" + slot = "virtio0" + format = "raw" } cpu { - cores = var.worker.cpu + cores = var.workers.cpu sockets = 1 - type = "x86-64-v2-AES" } - memory { - dedicated = var.worker.memory * 1024 - } + memory = var.workers.memory * 1024 - disk { - datastore_id = var.worker.storagepool - interface = "virtio0" - aio = "io_uring" - size = var.worker.disk * 1024 - file_format = "raw" - } - - cdrom { - file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.worker.talos_version) - } - - efi_disk { - datastore_id = var.worker.storagepool - file_format = "raw" - type = "4m" - } - - tpm_state { - datastore_id = var.worker.storagepool - version = "v2.0" - } - - initialization { - datastore_id = var.worker.storagepool - ip_config { - ipv4 { - address = var.worker.node_ipv4_address - gateway = var.worker.ipv4_gateway - } - } - dns { - servers = ["10.13.37.2"] - } - } - - network_device { + network { + id = 0 + model = "virtio" bridge = "vmbr1" } - + ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address) + skip_ipv6 = true } - -# resource "talos_machine_secrets" "controlplane" { -# talos_version = var.controlplane.talos_version -# } -# -# data "talos_client_configuration" "controlplane" { -# cluster_name = var.controlplane.cluster_name -# client_configuration = talos_machine_secrets.controlplane.client_configuration -# nodes = [for k, v in var.controlplane : v.ip] -# endpoints = [var.controlplane.cluster_endpoint] -# } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf index 627f900..e08e802 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf @@ -1,17 +1,15 @@ -variable "worker" { +variable "workers" { description = "Worker node configuration" type = object({ - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - cluster_name = string - kubernetes_version = string - node_name = string - node_ipv4_address = string - cluster_endpoint = string - ipv4_gateway = string + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + nodes = list(object({ + name = string + ip_address = string + })) }) } diff --git a/pve01.wheatley.in/k8s-wheatley/providers.tf b/pve01.wheatley.in/k8s-wheatley/providers.tf deleted file mode 100644 index 240a9d3..0000000 --- a/pve01.wheatley.in/k8s-wheatley/providers.tf +++ /dev/null @@ -1,12 +0,0 @@ -terraform { - required_providers { - talos = { - source = "siderolabs/talos" - version = "0.9.0" - } - proxmox = { - source = "bpg/proxmox" - version = "0.85.1" - } - } -} diff --git a/pve01.wheatley.in/k8s-wheatley/variables.tf b/pve01.wheatley.in/k8s-wheatley/variables.tf deleted file mode 100644 index 086f9ff..0000000 --- a/pve01.wheatley.in/k8s-wheatley/variables.tf +++ /dev/null @@ -1,8 +0,0 @@ -variable "proxmox_endpoint" { - description = "Proxmox API endpoint" - type = string -} -variable "proxmox_api_token" { - description = "Proxmox API token" - type = string -}