diff --git a/.gitignore b/.gitignore index 3dcbe85..bfde7dd 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ terraform.rc .idea/ .vscode/ .DS_Store +**/.envrc diff --git a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl index 8b2a866..b040ddb 100644 --- a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl +++ b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl @@ -2,25 +2,25 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/bpg/proxmox" { - version = "0.85.1" - constraints = "0.85.1" + version = "0.86.0" + constraints = "0.86.0" hashes = [ - "h1:VWZI8z0O7flk66jBrWK8rltUKEU8zCjljZvoQ54wwf0=", - "zh:0866c44f3bc222d01fb6cc998eadef102cda2434464b929466998719c98ef77f", - "zh:1db527d7a8b12c9fc99151798469352e756d9af0ba2fbfb155dea935df3b423a", - "zh:2e9dbbc846028cd7fdf21fda884c766bf3d81b7d8a2e5f9924cc48759805fe74", - "zh:32817963c835c7e4528c6e13147f183647b201fd17d189f102a41c478d3994e7", - "zh:3ef2915dee3a0c4fea4f205afde7d3318401e91989df99ee560a42302ff0d4d7", - "zh:5829f4d0b9731057febf6209d854a5a424d5787297ae9fb6b7f5106fc4271c04", - "zh:69f932cb1c5b4594914fc4bba0c110311cc5de79c56c75cf9ff6e85faf8ddd38", - "zh:87840842b229796c34ef07bd7c01016f19b11a83646cc6ff19c1ce3aeec010f7", - "zh:9299c9aeeca2afc836066e971f587facff2287125c52658b9031c97e1b62eb24", - "zh:ce3bbb9375518a3ad8a7a4b8434ff72bfe2d6543309f419e4128961b73d6d84e", - "zh:d0b5fa229ae989350f47e42244dfcf91f149c27c1f229fe5b8ad9828f21a896e", - "zh:d80cc976b0cc4726d18d6cac094b520c0a670898494c7f8cdb5a72903ba7b57e", - "zh:f1d3919f53f195e9163ca6d85f5906fc3f40d9aac4c320cbc04aee0103780242", + "h1:brQxwkXGxNw5N2gNwwefF9XxzrudYgSyjHrRhdvAdjo=", + "zh:09b627b92a59848769fadfc3d8103eebf070a3800144bf03cb93f44472327f44", + "zh:0e19eb7f1047d541e50b97d7ac440ea73685d0c28ed2dbe64217cbe2f0b353e0", + "zh:20f1e70091ff3056876618c93afd79527c8995f955d153993e8fbb10fa42593b", + "zh:3920315be565976f5a9da0803f8f1a108221282f1bc9e21160669d793af4e0c8", + "zh:5133b2a2027428d3926eaa3bcdc0ab65a75305d54f6cbc7c54cce746dfddbc8e", + "zh:514c588b04738d55c9e6b1c5a4e3fb1ef4041dfb809d2268f14d29839ecfba59", + "zh:55916034025b4833bd6a93bb5948dfb7d00830a772ef74fa70898c6f7de0da0b", + "zh:58b485a4b0bde56ca7032fca0ac09cb4c6ff2579e06cf4f2a311bb695baa0df1", + "zh:75ebe44e6da4108af5fe02a9cd99ed0189985b486a2a56594952098d161ceb3d", + "zh:a8c870bfb5958a3d49d639db3c2761cfb453c6a6f95e5e241890922b11c8a4d8", + "zh:c2df2748b9be47a6c3e613667c64874d5cb1d3fbb5b985d6eb9c3af5af298454", + "zh:c3059668f4f81e450e555a47310e7042044b335f131643262fd51f9ba96f2214", + "zh:ddbbb23910666f70cf4a9587ba57b45f5f58c53a1f8d7cee1d6f90a3d3ef38ef", + "zh:e430138b897edcd3b64e4309db34ac872526187782626aa074d8d1647a0abfa8", "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", - "zh:ff1ae863a11fcbfb5850dc8f0b652c17c0e6ec0a1a4f4877e95143023426a2f7", ] } @@ -41,6 +41,23 @@ provider "registry.opentofu.org/hashicorp/http" { ] } +provider "registry.opentofu.org/hashicorp/null" { + version = "3.2.4" + hashes = [ + "h1:i+WKhUHL2REY5EGmiHjfUljJB8UKZ9QdhdM5uTeUhC4=", + "zh:1769783386610bed8bb1e861a119fe25058be41895e3996d9216dd6bb8a7aee3", + "zh:32c62a9387ad0b861b5262b41c5e9ed6e940eda729c2a0e58100e6629af27ddb", + "zh:339bf8c2f9733fce068eb6d5612701144c752425cebeafab36563a16be460fb2", + "zh:36731f23343aee12a7e078067a98644c0126714c4fe9ac930eecb0f2361788c4", + "zh:3d106c7e32a929e2843f732625a582e562ff09120021e510a51a6f5d01175b8d", + "zh:74bcb3567708171ad83b234b92c9d63ab441ef882b770b0210c2b14fdbe3b1b6", + "zh:90b55bdbffa35df9204282251059e62c178b0ac7035958b93a647839643c0072", + "zh:ae24c0e5adc692b8f94cb23a000f91a316070fdc19418578dcf2134ff57cf447", + "zh:b5c10d4ad860c4c21273203d1de6d2f0286845edf1c64319fa2362df526b5f58", + "zh:e05bbd88e82e1d6234988c85db62fd66f11502645838fff594a2ec25352ecd80", + ] +} + provider "registry.opentofu.org/siderolabs/talos" { version = "0.9.0" constraints = "0.9.0" diff --git a/pve01.wheatley.in/k8s-wheatley/main.tf b/pve01.wheatley.in/k8s-wheatley/main.tf index aa392ad..209af38 100644 --- a/pve01.wheatley.in/k8s-wheatley/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/main.tf @@ -8,9 +8,9 @@ locals { cluster_name = "k8s-wheatley" vm-id_prefix = 200 kubernetes_version = "1.34.1" - talos_version = "1.11.3" - gateway_api_version = "1.4.0" - cilium_version = "1.14.1" + talos_version = "1.11.5" + gateway_api_version = "1.13.0" + cilium_version = "1.18.4" ipv4_gateway = "10.13.38.1" cluster_endpoint_ip = "10.13.38.10" proxmox_node = "pve01" @@ -18,7 +18,7 @@ locals { controlplanes = { cpu = 4 - memory = 4 + memory = 8 disk = 40 storagepool = "nvme-fastpool" talos_version = local.talos_version @@ -46,14 +46,16 @@ locals { ] workers = { - cpu = 4 - memory = 4 + cpu = 8 + memory = 8 disk = 40 storagepool = "nvme-fastpool" talos_version = local.talos_version cluster_name = local.cluster_name kubernetes_version = local.kubernetes_version ipv4_gateway = local.ipv4_gateway + pvc_disks = [100] + nodes = [ { name = "worker01" @@ -83,60 +85,47 @@ locals { ] } -module "talos-image" { - source = "./modules/talos-image" - - talos_version = local.talos_version -} - module "controlplanes" { - depends_on = [module.talos-image] - source = "./modules/talos-node" for_each = { for node in local.controlplanes.nodes : node.name => node } node = { - id = local.vm-id_prefix + 10 + tonumber(replace(each.key, "cp", "")) - name = format("k8s-wheatley-%s", each.value.name) - ipv4_address = each.value.ip_address - cpu = local.controlplanes.cpu - memory = local.controlplanes.memory - disk = local.controlplanes.disk - storagepool = local.controlplanes.storagepool - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - cluster_endpoint = local.cluster_endpoint_ip - ipv4_gateway = local.ipv4_gateway - proxmox_node = local.proxmox_node - controlplane_addresses = local.controlplane_addresses - worker_addresses = local.worker_addresses + id = local.vm-id_prefix + 10 + tonumber(replace(each.key, "cp", "")) + name = format("k8s-wheatley-%s", each.value.name) + ipv4_address = each.value.ip_address + cpu = local.controlplanes.cpu + memory = local.controlplanes.memory + disk = local.controlplanes.disk + storagepool = local.controlplanes.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + cluster_endpoint = local.cluster_endpoint_ip + ipv4_gateway = local.ipv4_gateway + proxmox_node = local.proxmox_node } } module "workers" { - depends_on = [module.talos-image] - source = "./modules/talos-node" for_each = { for node in local.workers.nodes : node.name => node } node = { - id = local.vm-id_prefix + 20 + tonumber(replace(each.key, "worker", "")) - name = format("k8s-wheatley-%s", each.value.name) - ipv4_address = each.value.ip_address - cpu = local.workers.cpu - memory = local.workers.memory - disk = local.workers.disk - storagepool = local.workers.storagepool - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - cluster_endpoint = local.cluster_endpoint_ip - ipv4_gateway = local.ipv4_gateway - proxmox_node = local.proxmox_node - controlplane_addresses = local.controlplane_addresses - worker_addresses = local.worker_addresses + id = local.vm-id_prefix + 20 + tonumber(replace(each.key, "worker", "")) + name = format("k8s-wheatley-%s", each.value.name) + ipv4_address = each.value.ip_address + cpu = local.workers.cpu + memory = local.workers.memory + disk = local.workers.disk + storagepool = local.workers.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + cluster_endpoint = local.cluster_endpoint_ip + ipv4_gateway = local.ipv4_gateway + proxmox_node = local.proxmox_node } + pvc_disks = local.workers.pvc_disks } module "talos-bootstrap" { diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl index 052c621..54a4d15 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl @@ -7,7 +7,7 @@ machine: network: hostname: ${hostname} nameservers: - - 10.13.37.2 + - 192.168.1.2 interfaces: - interface: eth0 dhcp: false diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl index 423f1f5..66e8e8a 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl @@ -7,7 +7,7 @@ machine: network: hostname: ${hostname} nameservers: - - 10.13.37.2 + - 192.168.1.2 kubelet: extraArgs: pod-max-pids: 1000 diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf deleted file mode 100644 index d5fb8d8..0000000 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf +++ /dev/null @@ -1,34 +0,0 @@ -terraform { - required_providers { - proxmox = { - source = "bpg/proxmox" - version = "0.85.1" - } - } -} - -data "http" "schematic" { - url = "https://factory.talos.dev/schematics" - method = "POST" - request_body = file("${path.module}/schematic.yaml") - request_headers = { - "Content-Type" = "application/x-yaml" - } -} - -locals { - schematic_id = jsondecode(data.http.schematic.response_body)["id"] - iso_url = "${"https://factory.talos.dev/image"}/${local.schematic_id}/v${var.talos_version}/nocloud-amd64-secureboot.iso" - iso_file = "talos-${var.talos_version}-nocloud-amd64-secureboot.iso" -} - -resource "proxmox_virtual_environment_download_file" "talos-image" { - node_name = "pve01" - content_type = "iso" - datastore_id = "local" - overwrite = true - - url = local.iso_url - file_name = local.iso_file - -} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf deleted file mode 100644 index 99ad2b8..0000000 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf +++ /dev/null @@ -1,4 +0,0 @@ -variable "talos_version" { - description = "Talos version to download" - type = string -} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf index 3d07db2..9ac4b7c 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { proxmox = { source = "bpg/proxmox" - version = "0.85.1" + version = "0.86.0" } } } @@ -12,16 +12,9 @@ resource "proxmox_virtual_environment_vm" "talos-node" { name = var.node.name node_name = var.node.proxmox_node tags = ["tofu"] - bios = "ovmf" - on_boot = true - machine = "q35" - stop_on_destroy = true - operating_system { - type = "l26" - } - agent { - enabled = true + clone { + vm_id = 10000 + tonumber(replace(var.node.talos_version, ".", "")) } cpu { @@ -37,26 +30,18 @@ resource "proxmox_virtual_environment_vm" "talos-node" { disk { datastore_id = var.node.storagepool interface = "virtio0" - aio = "io_uring" size = var.node.disk file_format = "raw" } - cdrom { - file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.node.talos_version) - } - - efi_disk { - datastore_id = var.node.storagepool - file_format = "raw" - type = "4m" - } - - boot_order = ["virtio0", "ide3", "net0"] - - tpm_state { - datastore_id = var.node.storagepool - version = "v2.0" + dynamic "disk" { + for_each = tolist(var.pvc_disks) + content { + datastore_id = "nvme-fastpool" + interface = "virtio${disk.key + 1}" + size = disk.value + file_format = "raw" + } } initialization { @@ -68,11 +53,7 @@ resource "proxmox_virtual_environment_vm" "talos-node" { } } dns { - servers = ["10.13.37.2"] + servers = ["192.168.1.2"] } } - - network_device { - bridge = "vmbr1" - } } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf index d1bfea6..389306f 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf @@ -1,20 +1,23 @@ variable "node" { - description = "Virtual node configuration" + description = "Basic configuration for the Talos node" type = object({ - id = number - name = string - ipv4_address = string - ipv4_gateway = string - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - cluster_name = string - kubernetes_version = string - cluster_endpoint = string - proxmox_node = string - controlplane_addresses = list(string) - worker_addresses = list(string) + id = number + name = string + ipv4_address = string + ipv4_gateway = string + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + cluster_name = string + kubernetes_version = string + cluster_endpoint = string + proxmox_node = string }) } +variable "pvc_disks" { + description = "List of extra disks to attach to the node" + type = list(number) + default = [] +} diff --git a/pve01.wheatley.in/k8s-wheatley/providers.tf b/pve01.wheatley.in/k8s-wheatley/providers.tf index 240a9d3..77de528 100644 --- a/pve01.wheatley.in/k8s-wheatley/providers.tf +++ b/pve01.wheatley.in/k8s-wheatley/providers.tf @@ -6,7 +6,7 @@ terraform { } proxmox = { source = "bpg/proxmox" - version = "0.85.1" + version = "0.86.0" } } } diff --git a/pve01.wheatley.in/templates/talos/.terraform.lock.hcl b/pve01.wheatley.in/templates/talos/.terraform.lock.hcl new file mode 100644 index 0000000..9b6f319 --- /dev/null +++ b/pve01.wheatley.in/templates/talos/.terraform.lock.hcl @@ -0,0 +1,59 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/bpg/proxmox" { + version = "0.86.0" + constraints = "0.86.0" + hashes = [ + "h1:brQxwkXGxNw5N2gNwwefF9XxzrudYgSyjHrRhdvAdjo=", + "zh:09b627b92a59848769fadfc3d8103eebf070a3800144bf03cb93f44472327f44", + "zh:0e19eb7f1047d541e50b97d7ac440ea73685d0c28ed2dbe64217cbe2f0b353e0", + "zh:20f1e70091ff3056876618c93afd79527c8995f955d153993e8fbb10fa42593b", + "zh:3920315be565976f5a9da0803f8f1a108221282f1bc9e21160669d793af4e0c8", + "zh:5133b2a2027428d3926eaa3bcdc0ab65a75305d54f6cbc7c54cce746dfddbc8e", + "zh:514c588b04738d55c9e6b1c5a4e3fb1ef4041dfb809d2268f14d29839ecfba59", + "zh:55916034025b4833bd6a93bb5948dfb7d00830a772ef74fa70898c6f7de0da0b", + "zh:58b485a4b0bde56ca7032fca0ac09cb4c6ff2579e06cf4f2a311bb695baa0df1", + "zh:75ebe44e6da4108af5fe02a9cd99ed0189985b486a2a56594952098d161ceb3d", + "zh:a8c870bfb5958a3d49d639db3c2761cfb453c6a6f95e5e241890922b11c8a4d8", + "zh:c2df2748b9be47a6c3e613667c64874d5cb1d3fbb5b985d6eb9c3af5af298454", + "zh:c3059668f4f81e450e555a47310e7042044b335f131643262fd51f9ba96f2214", + "zh:ddbbb23910666f70cf4a9587ba57b45f5f58c53a1f8d7cee1d6f90a3d3ef38ef", + "zh:e430138b897edcd3b64e4309db34ac872526187782626aa074d8d1647a0abfa8", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + ] +} + +provider "registry.opentofu.org/hashicorp/http" { + version = "3.5.0" + hashes = [ + "h1:eClUBisXme48lqiUl3U2+H2a2mzDawS9biqfkd9synw=", + "zh:0a2b33494eec6a91a183629cf217e073be063624c5d3f70870456ddb478308e9", + "zh:180f40124fa01b98b3d2f79128646b151818e09d6a1a9ca08e0b032a0b1e9cb1", + "zh:3e29e1de149dc10bf78620526c7cb8c62cd76087f5630dfaba0e93cda1f3aa7b", + "zh:4420950200cf86042ec940d0e2c9b7c89966bf556bf8038ba36217eae663bca5", + "zh:5d1f7d02109b2e2dca7ec626e5563ee765583792d0fd64081286f16f9433bd0d", + "zh:8500b138d338b1994c4206aa577b5c44e1d7260825babcf43245a7075bfa52a5", + "zh:b42165a6c4cfb22825938272d12b676e4a6946ac4e750f85df870c947685df2d", + "zh:b919bf3ee8e3b01051a0da3433b443a925e272893d3724ee8fc0f666ec7012c9", + "zh:d13b81ea6755cae785b3e11634936cdff2dc1ec009dc9610d8e3c7eb32f42e69", + "zh:f1c9d2eb1a6b618ae77ad86649679241bd8d6aacec06d0a68d86f748687f4eb3", + ] +} + +provider "registry.opentofu.org/hashicorp/null" { + version = "3.2.4" + hashes = [ + "h1:i+WKhUHL2REY5EGmiHjfUljJB8UKZ9QdhdM5uTeUhC4=", + "zh:1769783386610bed8bb1e861a119fe25058be41895e3996d9216dd6bb8a7aee3", + "zh:32c62a9387ad0b861b5262b41c5e9ed6e940eda729c2a0e58100e6629af27ddb", + "zh:339bf8c2f9733fce068eb6d5612701144c752425cebeafab36563a16be460fb2", + "zh:36731f23343aee12a7e078067a98644c0126714c4fe9ac930eecb0f2361788c4", + "zh:3d106c7e32a929e2843f732625a582e562ff09120021e510a51a6f5d01175b8d", + "zh:74bcb3567708171ad83b234b92c9d63ab441ef882b770b0210c2b14fdbe3b1b6", + "zh:90b55bdbffa35df9204282251059e62c178b0ac7035958b93a647839643c0072", + "zh:ae24c0e5adc692b8f94cb23a000f91a316070fdc19418578dcf2134ff57cf447", + "zh:b5c10d4ad860c4c21273203d1de6d2f0286845edf1c64319fa2362df526b5f58", + "zh:e05bbd88e82e1d6234988c85db62fd66f11502645838fff594a2ec25352ecd80", + ] +} diff --git a/pve01.wheatley.in/templates/talos/main.tf b/pve01.wheatley.in/templates/talos/main.tf new file mode 100644 index 0000000..cff4fae --- /dev/null +++ b/pve01.wheatley.in/templates/talos/main.tf @@ -0,0 +1,96 @@ +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + insecure = true +} + +locals { + talos_versions = ["1.11.4", "1.11.5"] +} + +data "http" "schematic" { + url = "https://factory.talos.dev/schematics" + method = "POST" + request_body = file("${path.module}/schematic.yaml") + request_headers = { + "Content-Type" = "application/x-yaml" + } +} + +locals { + schematic_id = jsondecode(data.http.schematic.response_body)["id"] + + storagepool = "nvme-fastpool" +} + +resource "null_resource" "local_download_talos_image" { + for_each = toset(local.talos_versions) + + provisioner "local-exec" { + command = <