diff --git a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl index bd5e235..8b2a866 100644 --- a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl +++ b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl @@ -1,6 +1,46 @@ # This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. +provider "registry.opentofu.org/bpg/proxmox" { + version = "0.85.1" + constraints = "0.85.1" + hashes = [ + "h1:VWZI8z0O7flk66jBrWK8rltUKEU8zCjljZvoQ54wwf0=", + "zh:0866c44f3bc222d01fb6cc998eadef102cda2434464b929466998719c98ef77f", + "zh:1db527d7a8b12c9fc99151798469352e756d9af0ba2fbfb155dea935df3b423a", + "zh:2e9dbbc846028cd7fdf21fda884c766bf3d81b7d8a2e5f9924cc48759805fe74", + "zh:32817963c835c7e4528c6e13147f183647b201fd17d189f102a41c478d3994e7", + "zh:3ef2915dee3a0c4fea4f205afde7d3318401e91989df99ee560a42302ff0d4d7", + "zh:5829f4d0b9731057febf6209d854a5a424d5787297ae9fb6b7f5106fc4271c04", + "zh:69f932cb1c5b4594914fc4bba0c110311cc5de79c56c75cf9ff6e85faf8ddd38", + "zh:87840842b229796c34ef07bd7c01016f19b11a83646cc6ff19c1ce3aeec010f7", + "zh:9299c9aeeca2afc836066e971f587facff2287125c52658b9031c97e1b62eb24", + "zh:ce3bbb9375518a3ad8a7a4b8434ff72bfe2d6543309f419e4128961b73d6d84e", + "zh:d0b5fa229ae989350f47e42244dfcf91f149c27c1f229fe5b8ad9828f21a896e", + "zh:d80cc976b0cc4726d18d6cac094b520c0a670898494c7f8cdb5a72903ba7b57e", + "zh:f1d3919f53f195e9163ca6d85f5906fc3f40d9aac4c320cbc04aee0103780242", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + "zh:ff1ae863a11fcbfb5850dc8f0b652c17c0e6ec0a1a4f4877e95143023426a2f7", + ] +} + +provider "registry.opentofu.org/hashicorp/http" { + version = "3.5.0" + hashes = [ + "h1:eClUBisXme48lqiUl3U2+H2a2mzDawS9biqfkd9synw=", + "zh:0a2b33494eec6a91a183629cf217e073be063624c5d3f70870456ddb478308e9", + "zh:180f40124fa01b98b3d2f79128646b151818e09d6a1a9ca08e0b032a0b1e9cb1", + "zh:3e29e1de149dc10bf78620526c7cb8c62cd76087f5630dfaba0e93cda1f3aa7b", + "zh:4420950200cf86042ec940d0e2c9b7c89966bf556bf8038ba36217eae663bca5", + "zh:5d1f7d02109b2e2dca7ec626e5563ee765583792d0fd64081286f16f9433bd0d", + "zh:8500b138d338b1994c4206aa577b5c44e1d7260825babcf43245a7075bfa52a5", + "zh:b42165a6c4cfb22825938272d12b676e4a6946ac4e750f85df870c947685df2d", + "zh:b919bf3ee8e3b01051a0da3433b443a925e272893d3724ee8fc0f666ec7012c9", + "zh:d13b81ea6755cae785b3e11634936cdff2dc1ec009dc9610d8e3c7eb32f42e69", + "zh:f1c9d2eb1a6b618ae77ad86649679241bd8d6aacec06d0a68d86f748687f4eb3", + ] +} + provider "registry.opentofu.org/siderolabs/talos" { version = "0.9.0" constraints = "0.9.0" @@ -23,25 +63,3 @@ provider "registry.opentofu.org/siderolabs/talos" { "zh:feb941cc673692c6858f96590f3ca082683a62aa2d2842b9bfffcd327eb840d3", ] } - -provider "registry.opentofu.org/telmate/proxmox" { - version = "3.0.2-rc05" - constraints = ">= 3.0.2-rc05, 3.0.2-rc05" - hashes = [ - "h1:PG5/Mu1UUP6/WIdtytJKZ8NxQJg8OIGygFwJOoPdmZw=", - "zh:042d748367f33aaf440698644be4f2a2875f9db31915c1ef84616f176fc6174f", - "zh:1488781da1920d60d933c8ce926c34b5e989ffae58e3fbe437973d2b1d2faafc", - "zh:283dd6f74627f1d1d75d616b31f8ced3f97fd5277a07c9535e85cfa765d7a321", - "zh:378f1c2da21aeea083ac2e632db274a02c7a01e2486a40d3c813d05a21142db3", - "zh:38d63d0961f8c32273392caaace30f50cff8ab06e5dda17f67a8827ebffeba98", - "zh:52159782df101ec98f20faff81e8f2d9d92cb4ec903314fcddcc57ec16cdaacb", - "zh:6ca47b90c66b1d2706cb3cbb05da8b3f90a202c4865010202b2962e2b64d217e", - "zh:6e7b85cb2380e4dc0be694dd0e4a24927f7f66df41960eca3cfe907443d4f0b9", - "zh:758775f733673ab5c196db6a33648458037746f94d4bef7ce148cb01474efe2d", - "zh:7c31a3ca6d52db39da2bdd60be37af71d59d808fc206de50fe661535ea436da3", - "zh:af16984350a2f4d77c21f66a479007801e2527543310567c99cd82eb421e249e", - "zh:c1f965d3f96cf3f87af2c12ab9d4bde42f8ef660f8dc34ba3cfc9b20435a7269", - "zh:c2b9022a31103919a5ffbac6ee8d7feb6c4f5f580c1766f769569c2e8e4ce7f1", - "zh:e90162c42f1237323291e3d0de0c62701b3f89350fae18246da06702f41a6123", - ] -} diff --git a/pve01.wheatley.in/k8s-wheatley/main.tf b/pve01.wheatley.in/k8s-wheatley/main.tf index d387f48..e0079a3 100644 --- a/pve01.wheatley.in/k8s-wheatley/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/main.tf @@ -1,68 +1,113 @@ -terraform { - required_providers { - proxmox = { - source = "Telmate/proxmox" - version = "3.0.2-rc05" - } - talos = { - source = "siderolabs/talos" - version = "0.9.0" - } - } -} provider "proxmox" { - pm_api_url = "https://10.167.84.10:8006/api2/json" - pm_api_token_id = "" - pm_api_token_secret = "" - pm_tls_insecure = true + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + insecure = true } locals { - talos_version = "1.11.3" -} + cluster_name = "k8s-wheatley" + kubernetes_version = "1.34.1" + talos_version = "1.11.3" + ipv4_gateway = "10.13.38.1" + ipv4_cidr = "/24" + cluster_endpoint_ip = "10.13.38.20" -module "controlplanes" { - source = "./modules/controlplane" controlplanes = { - cpu = 4 - memory = 4 - disk = "40G" - storagepool = "nvme-fastpool" - talos_version = local.talos_version + cpu = 4 + memory = 4 + disk = 40 + storagepool = "local-zfs" + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + ipv4_gateway = local.ipv4_gateway nodes = [ { name = "cp01" - ip_address = "10.13.38.20" - }, - { - name = "cp02" ip_address = "10.13.38.21" }, { - name = "cp03" + name = "cp02" ip_address = "10.13.38.22" - }] + }, + { + name = "cp03" + ip_address = "10.13.38.23" + } + ] + } + + workers = { + cpu = 4 + memory = 4 + disk = 40 + storagepool = "local-zfs" + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + ipv4_gateway = local.ipv4_gateway + nodes = [ + { + name = "worker01" + ip_address = "10.13.38.20" + }, + { + name = "worker02" + ip_address = "10.13.38.21" + }, + { + name = "worker03" + ip_address = "10.13.38.22" + } + ] + } +} + +module "talos-image" { + source = "./modules/talos-image" + + talos_version = local.talos_version +} + +module "controlplanes" { + depends_on = [module.talos-image] + + source = "./modules/controlplane" + for_each = { for node in local.controlplanes.nodes : node.name => node } + + controlplane = { + cpu = local.controlplanes.cpu + memory = local.controlplanes.memory + disk = local.controlplanes.disk + storagepool = local.controlplanes.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + node_name = format("k8s-wheatley-%s", each.value.name) + cluster_endpoint = format("https://%s:6443", local.cluster_endpoint_ip) + node_ipv4_address = format("%s%s", each.value.ip_address, local.ipv4_cidr) + ipv4_gateway = local.ipv4_gateway } } module "workers" { - source = "./modules/worker" + depends_on = [module.controlplanes] - workers = { - cpu = 4 - memory = 4 - disk = "40G" - storagepool = "nvme-fastpool" - talos_version = local.talos_version - nodes = [ - { - name = "worker01" - ip_address = "10.13.38.30" - }, - { - name = "worker02" - ip_address = "10.13.38.31" - }] + source = "./modules/worker" + for_each = { for node in local.workers.nodes : node.name => node } + + worker = { + cpu = local.workers.cpu + memory = local.workers.memory + disk = local.workers.disk + storagepool = local.workers.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + node_name = format("k8s-wheatley-%s", each.value.name) + cluster_endpoint = format("https://%s:6443", local.cluster_endpoint_ip) + node_ipv4_address = format("%s%s", each.value.ip_address, local.ipv4_cidr) + ipv4_gateway = local.ipv4_gateway } } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl new file mode 100644 index 0000000..a1da4bb --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/machineconfig.yaml.tmpl @@ -0,0 +1,34 @@ +debug: false +machine: + install: + disk: ${install_disk} + network: + hostname: ${hostname} + nameservers: + - 10.13.37.2 + interfaces: + - interface: eth0 + dhcp: false + kubelet: + extraArgs: + pod-max-pids: 1000 + extraConfig: + imageGCHighThresholdPercent: 75 + imageGCLowThresholdPercent: 70 +cluster: + apiServer: + auditPolicy: + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + # Log only metadata in audit logs + - level: Metadata + network: + hostname: ${hostname} + cni: + name: none + nodeLabels: + topology.kubernetes.io/region: ${cluster_name} + topology.kubernetes.io/zone: ${node_name} + proxy: + disabled: true diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf index c64dd2b..8cb028b 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/main.tf @@ -1,44 +1,93 @@ terraform { required_providers { proxmox = { - source = "telmate/proxmox" - version = ">= 3.0.2-rc05" + source = "bpg/proxmox" + version = "0.85.1" + } + talos = { + source = "siderolabs/talos" + version = "0.9.0" } } } -resource "proxmox_vm_qemu" "controlplane" { - for_each = { for node in var.controlplanes.nodes : node.name => node } - name = format("k8s-wheatley-%s", each.value.name) - target_node = "pve01" - tags = "k8s-wheatley,controlplane" - onboot = true - bios = "ovmf" - boot = "order=virtio0;net0" - clone = format("talos-%s", var.controlplanes.talos_version) - scsihw = "virtio-scsi-pci" +resource "proxmox_virtual_environment_vm" "controlplane" { - disk { - size = var.controlplanes.disk - storage = var.controlplanes.storagepool - type = "disk" - slot = "virtio0" - format = "raw" + name = var.controlplane.node_name + node_name = "pve01" + tags = ["tofu"] + bios = "ovmf" + on_boot = true + machine = "q35" + stop_on_destroy = true + + operating_system { + type = "l26" + } + agent { + enabled = true } cpu { - cores = var.controlplanes.cpu + cores = var.controlplane.cpu sockets = 1 + type = "x86-64-v2-AES" } - memory = var.controlplanes.memory * 1024 + memory { + dedicated = var.controlplane.memory * 1024 + } - network { - id = 0 - model = "virtio" + disk { + datastore_id = var.controlplane.storagepool + interface = "virtio0" + aio = "io_uring" + size = var.controlplane.disk * 1024 + file_format = "raw" + } + + cdrom { + file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.controlplane.talos_version) + } + + efi_disk { + datastore_id = var.controlplane.storagepool + file_format = "raw" + type = "4m" + } + + tpm_state { + datastore_id = var.controlplane.storagepool + version = "v2.0" + } + + initialization { + datastore_id = var.controlplane.storagepool + ip_config { + ipv4 { + address = var.controlplane.node_ipv4_address + gateway = var.controlplane.ipv4_gateway + } + } + dns { + servers = ["10.13.37.2"] + } + } + + network_device { bridge = "vmbr1" } - ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address) - skip_ipv6 = true + } + +# resource "talos_machine_secrets" "controlplane" { +# talos_version = var.controlplane.talos_version +# } +# +# data "talos_client_configuration" "controlplane" { +# cluster_name = var.controlplane.cluster_name +# client_configuration = talos_machine_secrets.controlplane.client_configuration +# nodes = [for k, v in var.controlplane : v.ip] +# endpoints = [var.controlplane.cluster_endpoint] +# } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf index 4b0fffe..7e1916a 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/controlplane/variables.tf @@ -1,15 +1,17 @@ -variable "controlplanes" { +variable "controlplane" { description = "Control plane node configuration" type = object({ - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - nodes = list(object({ - name = string - ip_address = string - })) + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + cluster_name = string + kubernetes_version = string + node_name = string + node_ipv4_address = string + cluster_endpoint = string + ipv4_gateway = string }) } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf new file mode 100644 index 0000000..610ecd4 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf @@ -0,0 +1,34 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "0.85.1" + } + } +} + +data "http" "schematic" { + url = "https://factory.talos.dev/schematics" + method = "POST" + request_body = file("${path.module}/schematic.yaml") + request_headers = { + "Content-Type" = "application/x-yaml" + } +} + +locals { + schematic_id = jsondecode(data.http.schematic.response_body)["id"] + iso_url = "${"https://factory.talos.dev/image"}/${local.schematic_id}/v${var.talos_version}/nocloud-amd64-secureboot.iso" + iso_file = "talos-${var.talos_version}-nocloud-amd64-secureboot.iso" +} + +resource "proxmox_virtual_environment_download_file" "talos-image" { + node_name = "pve01" + content_type = "iso" + datastore_id = "local" + overwrite = true + + url = local.iso_url + file_name = local.iso_file + +} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml new file mode 100644 index 0000000..f27af6e --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml @@ -0,0 +1,4 @@ +customization: + systemExtensions: + officialExtensions: + - siderolabs/qemu-guest-agent diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf new file mode 100644 index 0000000..99ad2b8 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf @@ -0,0 +1,4 @@ +variable "talos_version" { + description = "Talos version to download" + type = string +} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl new file mode 100644 index 0000000..a1da4bb --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/machineconfig.yaml.tmpl @@ -0,0 +1,34 @@ +debug: false +machine: + install: + disk: ${install_disk} + network: + hostname: ${hostname} + nameservers: + - 10.13.37.2 + interfaces: + - interface: eth0 + dhcp: false + kubelet: + extraArgs: + pod-max-pids: 1000 + extraConfig: + imageGCHighThresholdPercent: 75 + imageGCLowThresholdPercent: 70 +cluster: + apiServer: + auditPolicy: + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + # Log only metadata in audit logs + - level: Metadata + network: + hostname: ${hostname} + cni: + name: none + nodeLabels: + topology.kubernetes.io/region: ${cluster_name} + topology.kubernetes.io/zone: ${node_name} + proxy: + disabled: true diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf index 69928f8..2510623 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/main.tf @@ -1,44 +1,93 @@ terraform { required_providers { proxmox = { - source = "telmate/proxmox" - version = ">= 3.0.2-rc05" + source = "bpg/proxmox" + version = "0.85.1" + } + talos = { + source = "siderolabs/talos" + version = "0.9.0" } } } -resource "proxmox_vm_qemu" "worker" { - for_each = { for node in var.workers.nodes : node.name => node } - name = format("k8s-wheatley-%s", each.value.name) - target_node = "pve01" - tags = "k8s-wheatley,worker" - onboot = true - bios = "ovmf" - boot = "order=virtio0;net0" - clone = format("talos-%s", var.workers.talos_version) - scsihw = "virtio-scsi-pci" +resource "proxmox_virtual_environment_vm" "worker" { - disk { - size = var.workers.disk - storage = var.workers.storagepool - type = "disk" - slot = "virtio0" - format = "raw" + name = var.worker.node_name + node_name = "pve01" + tags = ["tofu"] + bios = "ovmf" + on_boot = true + machine = "q35" + stop_on_destroy = true + + operating_system { + type = "l26" + } + agent { + enabled = true } cpu { - cores = var.workers.cpu + cores = var.worker.cpu sockets = 1 + type = "x86-64-v2-AES" } - memory = var.workers.memory * 1024 + memory { + dedicated = var.worker.memory * 1024 + } - network { - id = 0 - model = "virtio" + disk { + datastore_id = var.worker.storagepool + interface = "virtio0" + aio = "io_uring" + size = var.worker.disk * 1024 + file_format = "raw" + } + + cdrom { + file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.worker.talos_version) + } + + efi_disk { + datastore_id = var.worker.storagepool + file_format = "raw" + type = "4m" + } + + tpm_state { + datastore_id = var.worker.storagepool + version = "v2.0" + } + + initialization { + datastore_id = var.worker.storagepool + ip_config { + ipv4 { + address = var.worker.node_ipv4_address + gateway = var.worker.ipv4_gateway + } + } + dns { + servers = ["10.13.37.2"] + } + } + + network_device { bridge = "vmbr1" } - ipconfig0 = format("ip=%s/24,gw=10.13.38.1", each.value.ip_address) - skip_ipv6 = true + } + +# resource "talos_machine_secrets" "controlplane" { +# talos_version = var.controlplane.talos_version +# } +# +# data "talos_client_configuration" "controlplane" { +# cluster_name = var.controlplane.cluster_name +# client_configuration = talos_machine_secrets.controlplane.client_configuration +# nodes = [for k, v in var.controlplane : v.ip] +# endpoints = [var.controlplane.cluster_endpoint] +# } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf index e08e802..627f900 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/worker/variables.tf @@ -1,15 +1,17 @@ -variable "workers" { +variable "worker" { description = "Worker node configuration" type = object({ - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - nodes = list(object({ - name = string - ip_address = string - })) + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + cluster_name = string + kubernetes_version = string + node_name = string + node_ipv4_address = string + cluster_endpoint = string + ipv4_gateway = string }) } diff --git a/pve01.wheatley.in/k8s-wheatley/providers.tf b/pve01.wheatley.in/k8s-wheatley/providers.tf new file mode 100644 index 0000000..240a9d3 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/providers.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + talos = { + source = "siderolabs/talos" + version = "0.9.0" + } + proxmox = { + source = "bpg/proxmox" + version = "0.85.1" + } + } +} diff --git a/pve01.wheatley.in/k8s-wheatley/variables.tf b/pve01.wheatley.in/k8s-wheatley/variables.tf new file mode 100644 index 0000000..086f9ff --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/variables.tf @@ -0,0 +1,8 @@ +variable "proxmox_endpoint" { + description = "Proxmox API endpoint" + type = string +} +variable "proxmox_api_token" { + description = "Proxmox API token" + type = string +}