diff --git a/.gitignore b/.gitignore index bfde7dd..3dcbe85 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,3 @@ terraform.rc .idea/ .vscode/ .DS_Store -**/.envrc diff --git a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl index b040ddb..8b2a866 100644 --- a/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl +++ b/pve01.wheatley.in/k8s-wheatley/.terraform.lock.hcl @@ -2,25 +2,25 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/bpg/proxmox" { - version = "0.86.0" - constraints = "0.86.0" + version = "0.85.1" + constraints = "0.85.1" hashes = [ - "h1:brQxwkXGxNw5N2gNwwefF9XxzrudYgSyjHrRhdvAdjo=", - "zh:09b627b92a59848769fadfc3d8103eebf070a3800144bf03cb93f44472327f44", - "zh:0e19eb7f1047d541e50b97d7ac440ea73685d0c28ed2dbe64217cbe2f0b353e0", - "zh:20f1e70091ff3056876618c93afd79527c8995f955d153993e8fbb10fa42593b", - "zh:3920315be565976f5a9da0803f8f1a108221282f1bc9e21160669d793af4e0c8", - "zh:5133b2a2027428d3926eaa3bcdc0ab65a75305d54f6cbc7c54cce746dfddbc8e", - "zh:514c588b04738d55c9e6b1c5a4e3fb1ef4041dfb809d2268f14d29839ecfba59", - "zh:55916034025b4833bd6a93bb5948dfb7d00830a772ef74fa70898c6f7de0da0b", - "zh:58b485a4b0bde56ca7032fca0ac09cb4c6ff2579e06cf4f2a311bb695baa0df1", - "zh:75ebe44e6da4108af5fe02a9cd99ed0189985b486a2a56594952098d161ceb3d", - "zh:a8c870bfb5958a3d49d639db3c2761cfb453c6a6f95e5e241890922b11c8a4d8", - "zh:c2df2748b9be47a6c3e613667c64874d5cb1d3fbb5b985d6eb9c3af5af298454", - "zh:c3059668f4f81e450e555a47310e7042044b335f131643262fd51f9ba96f2214", - "zh:ddbbb23910666f70cf4a9587ba57b45f5f58c53a1f8d7cee1d6f90a3d3ef38ef", - "zh:e430138b897edcd3b64e4309db34ac872526187782626aa074d8d1647a0abfa8", + "h1:VWZI8z0O7flk66jBrWK8rltUKEU8zCjljZvoQ54wwf0=", + "zh:0866c44f3bc222d01fb6cc998eadef102cda2434464b929466998719c98ef77f", + "zh:1db527d7a8b12c9fc99151798469352e756d9af0ba2fbfb155dea935df3b423a", + "zh:2e9dbbc846028cd7fdf21fda884c766bf3d81b7d8a2e5f9924cc48759805fe74", + "zh:32817963c835c7e4528c6e13147f183647b201fd17d189f102a41c478d3994e7", + "zh:3ef2915dee3a0c4fea4f205afde7d3318401e91989df99ee560a42302ff0d4d7", + "zh:5829f4d0b9731057febf6209d854a5a424d5787297ae9fb6b7f5106fc4271c04", + "zh:69f932cb1c5b4594914fc4bba0c110311cc5de79c56c75cf9ff6e85faf8ddd38", + "zh:87840842b229796c34ef07bd7c01016f19b11a83646cc6ff19c1ce3aeec010f7", + "zh:9299c9aeeca2afc836066e971f587facff2287125c52658b9031c97e1b62eb24", + "zh:ce3bbb9375518a3ad8a7a4b8434ff72bfe2d6543309f419e4128961b73d6d84e", + "zh:d0b5fa229ae989350f47e42244dfcf91f149c27c1f229fe5b8ad9828f21a896e", + "zh:d80cc976b0cc4726d18d6cac094b520c0a670898494c7f8cdb5a72903ba7b57e", + "zh:f1d3919f53f195e9163ca6d85f5906fc3f40d9aac4c320cbc04aee0103780242", "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + "zh:ff1ae863a11fcbfb5850dc8f0b652c17c0e6ec0a1a4f4877e95143023426a2f7", ] } @@ -41,23 +41,6 @@ provider "registry.opentofu.org/hashicorp/http" { ] } -provider "registry.opentofu.org/hashicorp/null" { - version = "3.2.4" - hashes = [ - "h1:i+WKhUHL2REY5EGmiHjfUljJB8UKZ9QdhdM5uTeUhC4=", - "zh:1769783386610bed8bb1e861a119fe25058be41895e3996d9216dd6bb8a7aee3", - "zh:32c62a9387ad0b861b5262b41c5e9ed6e940eda729c2a0e58100e6629af27ddb", - "zh:339bf8c2f9733fce068eb6d5612701144c752425cebeafab36563a16be460fb2", - "zh:36731f23343aee12a7e078067a98644c0126714c4fe9ac930eecb0f2361788c4", - "zh:3d106c7e32a929e2843f732625a582e562ff09120021e510a51a6f5d01175b8d", - "zh:74bcb3567708171ad83b234b92c9d63ab441ef882b770b0210c2b14fdbe3b1b6", - "zh:90b55bdbffa35df9204282251059e62c178b0ac7035958b93a647839643c0072", - "zh:ae24c0e5adc692b8f94cb23a000f91a316070fdc19418578dcf2134ff57cf447", - "zh:b5c10d4ad860c4c21273203d1de6d2f0286845edf1c64319fa2362df526b5f58", - "zh:e05bbd88e82e1d6234988c85db62fd66f11502645838fff594a2ec25352ecd80", - ] -} - provider "registry.opentofu.org/siderolabs/talos" { version = "0.9.0" constraints = "0.9.0" diff --git a/pve01.wheatley.in/k8s-wheatley/main.tf b/pve01.wheatley.in/k8s-wheatley/main.tf index 209af38..aa392ad 100644 --- a/pve01.wheatley.in/k8s-wheatley/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/main.tf @@ -8,9 +8,9 @@ locals { cluster_name = "k8s-wheatley" vm-id_prefix = 200 kubernetes_version = "1.34.1" - talos_version = "1.11.5" - gateway_api_version = "1.13.0" - cilium_version = "1.18.4" + talos_version = "1.11.3" + gateway_api_version = "1.4.0" + cilium_version = "1.14.1" ipv4_gateway = "10.13.38.1" cluster_endpoint_ip = "10.13.38.10" proxmox_node = "pve01" @@ -18,7 +18,7 @@ locals { controlplanes = { cpu = 4 - memory = 8 + memory = 4 disk = 40 storagepool = "nvme-fastpool" talos_version = local.talos_version @@ -46,16 +46,14 @@ locals { ] workers = { - cpu = 8 - memory = 8 + cpu = 4 + memory = 4 disk = 40 storagepool = "nvme-fastpool" talos_version = local.talos_version cluster_name = local.cluster_name kubernetes_version = local.kubernetes_version ipv4_gateway = local.ipv4_gateway - pvc_disks = [100] - nodes = [ { name = "worker01" @@ -85,47 +83,60 @@ locals { ] } +module "talos-image" { + source = "./modules/talos-image" + + talos_version = local.talos_version +} + module "controlplanes" { + depends_on = [module.talos-image] + source = "./modules/talos-node" for_each = { for node in local.controlplanes.nodes : node.name => node } node = { - id = local.vm-id_prefix + 10 + tonumber(replace(each.key, "cp", "")) - name = format("k8s-wheatley-%s", each.value.name) - ipv4_address = each.value.ip_address - cpu = local.controlplanes.cpu - memory = local.controlplanes.memory - disk = local.controlplanes.disk - storagepool = local.controlplanes.storagepool - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - cluster_endpoint = local.cluster_endpoint_ip - ipv4_gateway = local.ipv4_gateway - proxmox_node = local.proxmox_node + id = local.vm-id_prefix + 10 + tonumber(replace(each.key, "cp", "")) + name = format("k8s-wheatley-%s", each.value.name) + ipv4_address = each.value.ip_address + cpu = local.controlplanes.cpu + memory = local.controlplanes.memory + disk = local.controlplanes.disk + storagepool = local.controlplanes.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + cluster_endpoint = local.cluster_endpoint_ip + ipv4_gateway = local.ipv4_gateway + proxmox_node = local.proxmox_node + controlplane_addresses = local.controlplane_addresses + worker_addresses = local.worker_addresses } } module "workers" { + depends_on = [module.talos-image] + source = "./modules/talos-node" for_each = { for node in local.workers.nodes : node.name => node } node = { - id = local.vm-id_prefix + 20 + tonumber(replace(each.key, "worker", "")) - name = format("k8s-wheatley-%s", each.value.name) - ipv4_address = each.value.ip_address - cpu = local.workers.cpu - memory = local.workers.memory - disk = local.workers.disk - storagepool = local.workers.storagepool - talos_version = local.talos_version - cluster_name = local.cluster_name - kubernetes_version = local.kubernetes_version - cluster_endpoint = local.cluster_endpoint_ip - ipv4_gateway = local.ipv4_gateway - proxmox_node = local.proxmox_node + id = local.vm-id_prefix + 20 + tonumber(replace(each.key, "worker", "")) + name = format("k8s-wheatley-%s", each.value.name) + ipv4_address = each.value.ip_address + cpu = local.workers.cpu + memory = local.workers.memory + disk = local.workers.disk + storagepool = local.workers.storagepool + talos_version = local.talos_version + cluster_name = local.cluster_name + kubernetes_version = local.kubernetes_version + cluster_endpoint = local.cluster_endpoint_ip + ipv4_gateway = local.ipv4_gateway + proxmox_node = local.proxmox_node + controlplane_addresses = local.controlplane_addresses + worker_addresses = local.worker_addresses } - pvc_disks = local.workers.pvc_disks } module "talos-bootstrap" { diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl index 54a4d15..052c621 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl @@ -7,7 +7,7 @@ machine: network: hostname: ${hostname} nameservers: - - 192.168.1.2 + - 10.13.37.2 interfaces: - interface: eth0 dhcp: false diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl index 66e8e8a..423f1f5 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl @@ -7,7 +7,7 @@ machine: network: hostname: ${hostname} nameservers: - - 192.168.1.2 + - 10.13.37.2 kubelet: extraArgs: pod-max-pids: 1000 diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf new file mode 100644 index 0000000..d5fb8d8 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/main.tf @@ -0,0 +1,34 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "0.85.1" + } + } +} + +data "http" "schematic" { + url = "https://factory.talos.dev/schematics" + method = "POST" + request_body = file("${path.module}/schematic.yaml") + request_headers = { + "Content-Type" = "application/x-yaml" + } +} + +locals { + schematic_id = jsondecode(data.http.schematic.response_body)["id"] + iso_url = "${"https://factory.talos.dev/image"}/${local.schematic_id}/v${var.talos_version}/nocloud-amd64-secureboot.iso" + iso_file = "talos-${var.talos_version}-nocloud-amd64-secureboot.iso" +} + +resource "proxmox_virtual_environment_download_file" "talos-image" { + node_name = "pve01" + content_type = "iso" + datastore_id = "local" + overwrite = true + + url = local.iso_url + file_name = local.iso_file + +} diff --git a/pve01.wheatley.in/templates/talos/schematic.yaml b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml similarity index 100% rename from pve01.wheatley.in/templates/talos/schematic.yaml rename to pve01.wheatley.in/k8s-wheatley/modules/talos-image/schematic.yaml diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf new file mode 100644 index 0000000..99ad2b8 --- /dev/null +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-image/variables.tf @@ -0,0 +1,4 @@ +variable "talos_version" { + description = "Talos version to download" + type = string +} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf index 9ac4b7c..3d07db2 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { proxmox = { source = "bpg/proxmox" - version = "0.86.0" + version = "0.85.1" } } } @@ -12,9 +12,16 @@ resource "proxmox_virtual_environment_vm" "talos-node" { name = var.node.name node_name = var.node.proxmox_node tags = ["tofu"] + bios = "ovmf" + on_boot = true + machine = "q35" + stop_on_destroy = true - clone { - vm_id = 10000 + tonumber(replace(var.node.talos_version, ".", "")) + operating_system { + type = "l26" + } + agent { + enabled = true } cpu { @@ -30,18 +37,26 @@ resource "proxmox_virtual_environment_vm" "talos-node" { disk { datastore_id = var.node.storagepool interface = "virtio0" + aio = "io_uring" size = var.node.disk file_format = "raw" } - dynamic "disk" { - for_each = tolist(var.pvc_disks) - content { - datastore_id = "nvme-fastpool" - interface = "virtio${disk.key + 1}" - size = disk.value - file_format = "raw" - } + cdrom { + file_id = format("local:iso/talos-%s-nocloud-amd64-secureboot.iso", var.node.talos_version) + } + + efi_disk { + datastore_id = var.node.storagepool + file_format = "raw" + type = "4m" + } + + boot_order = ["virtio0", "ide3", "net0"] + + tpm_state { + datastore_id = var.node.storagepool + version = "v2.0" } initialization { @@ -53,7 +68,11 @@ resource "proxmox_virtual_environment_vm" "talos-node" { } } dns { - servers = ["192.168.1.2"] + servers = ["10.13.37.2"] } } + + network_device { + bridge = "vmbr1" + } } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf index 389306f..d1bfea6 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf @@ -1,23 +1,20 @@ variable "node" { - description = "Basic configuration for the Talos node" + description = "Virtual node configuration" type = object({ - id = number - name = string - ipv4_address = string - ipv4_gateway = string - cpu = number - memory = number - disk = string - storagepool = string - talos_version = string - cluster_name = string - kubernetes_version = string - cluster_endpoint = string - proxmox_node = string + id = number + name = string + ipv4_address = string + ipv4_gateway = string + cpu = number + memory = number + disk = string + storagepool = string + talos_version = string + cluster_name = string + kubernetes_version = string + cluster_endpoint = string + proxmox_node = string + controlplane_addresses = list(string) + worker_addresses = list(string) }) } -variable "pvc_disks" { - description = "List of extra disks to attach to the node" - type = list(number) - default = [] -} diff --git a/pve01.wheatley.in/k8s-wheatley/providers.tf b/pve01.wheatley.in/k8s-wheatley/providers.tf index 77de528..240a9d3 100644 --- a/pve01.wheatley.in/k8s-wheatley/providers.tf +++ b/pve01.wheatley.in/k8s-wheatley/providers.tf @@ -6,7 +6,7 @@ terraform { } proxmox = { source = "bpg/proxmox" - version = "0.86.0" + version = "0.85.1" } } } diff --git a/pve01.wheatley.in/templates/talos/.terraform.lock.hcl b/pve01.wheatley.in/templates/talos/.terraform.lock.hcl deleted file mode 100644 index 9b6f319..0000000 --- a/pve01.wheatley.in/templates/talos/.terraform.lock.hcl +++ /dev/null @@ -1,59 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/bpg/proxmox" { - version = "0.86.0" - constraints = "0.86.0" - hashes = [ - "h1:brQxwkXGxNw5N2gNwwefF9XxzrudYgSyjHrRhdvAdjo=", - "zh:09b627b92a59848769fadfc3d8103eebf070a3800144bf03cb93f44472327f44", - "zh:0e19eb7f1047d541e50b97d7ac440ea73685d0c28ed2dbe64217cbe2f0b353e0", - "zh:20f1e70091ff3056876618c93afd79527c8995f955d153993e8fbb10fa42593b", - "zh:3920315be565976f5a9da0803f8f1a108221282f1bc9e21160669d793af4e0c8", - "zh:5133b2a2027428d3926eaa3bcdc0ab65a75305d54f6cbc7c54cce746dfddbc8e", - "zh:514c588b04738d55c9e6b1c5a4e3fb1ef4041dfb809d2268f14d29839ecfba59", - "zh:55916034025b4833bd6a93bb5948dfb7d00830a772ef74fa70898c6f7de0da0b", - "zh:58b485a4b0bde56ca7032fca0ac09cb4c6ff2579e06cf4f2a311bb695baa0df1", - "zh:75ebe44e6da4108af5fe02a9cd99ed0189985b486a2a56594952098d161ceb3d", - "zh:a8c870bfb5958a3d49d639db3c2761cfb453c6a6f95e5e241890922b11c8a4d8", - "zh:c2df2748b9be47a6c3e613667c64874d5cb1d3fbb5b985d6eb9c3af5af298454", - "zh:c3059668f4f81e450e555a47310e7042044b335f131643262fd51f9ba96f2214", - "zh:ddbbb23910666f70cf4a9587ba57b45f5f58c53a1f8d7cee1d6f90a3d3ef38ef", - "zh:e430138b897edcd3b64e4309db34ac872526187782626aa074d8d1647a0abfa8", - "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", - ] -} - -provider "registry.opentofu.org/hashicorp/http" { - version = "3.5.0" - hashes = [ - "h1:eClUBisXme48lqiUl3U2+H2a2mzDawS9biqfkd9synw=", - "zh:0a2b33494eec6a91a183629cf217e073be063624c5d3f70870456ddb478308e9", - "zh:180f40124fa01b98b3d2f79128646b151818e09d6a1a9ca08e0b032a0b1e9cb1", - "zh:3e29e1de149dc10bf78620526c7cb8c62cd76087f5630dfaba0e93cda1f3aa7b", - "zh:4420950200cf86042ec940d0e2c9b7c89966bf556bf8038ba36217eae663bca5", - "zh:5d1f7d02109b2e2dca7ec626e5563ee765583792d0fd64081286f16f9433bd0d", - "zh:8500b138d338b1994c4206aa577b5c44e1d7260825babcf43245a7075bfa52a5", - "zh:b42165a6c4cfb22825938272d12b676e4a6946ac4e750f85df870c947685df2d", - "zh:b919bf3ee8e3b01051a0da3433b443a925e272893d3724ee8fc0f666ec7012c9", - "zh:d13b81ea6755cae785b3e11634936cdff2dc1ec009dc9610d8e3c7eb32f42e69", - "zh:f1c9d2eb1a6b618ae77ad86649679241bd8d6aacec06d0a68d86f748687f4eb3", - ] -} - -provider "registry.opentofu.org/hashicorp/null" { - version = "3.2.4" - hashes = [ - "h1:i+WKhUHL2REY5EGmiHjfUljJB8UKZ9QdhdM5uTeUhC4=", - "zh:1769783386610bed8bb1e861a119fe25058be41895e3996d9216dd6bb8a7aee3", - "zh:32c62a9387ad0b861b5262b41c5e9ed6e940eda729c2a0e58100e6629af27ddb", - "zh:339bf8c2f9733fce068eb6d5612701144c752425cebeafab36563a16be460fb2", - "zh:36731f23343aee12a7e078067a98644c0126714c4fe9ac930eecb0f2361788c4", - "zh:3d106c7e32a929e2843f732625a582e562ff09120021e510a51a6f5d01175b8d", - "zh:74bcb3567708171ad83b234b92c9d63ab441ef882b770b0210c2b14fdbe3b1b6", - "zh:90b55bdbffa35df9204282251059e62c178b0ac7035958b93a647839643c0072", - "zh:ae24c0e5adc692b8f94cb23a000f91a316070fdc19418578dcf2134ff57cf447", - "zh:b5c10d4ad860c4c21273203d1de6d2f0286845edf1c64319fa2362df526b5f58", - "zh:e05bbd88e82e1d6234988c85db62fd66f11502645838fff594a2ec25352ecd80", - ] -} diff --git a/pve01.wheatley.in/templates/talos/main.tf b/pve01.wheatley.in/templates/talos/main.tf deleted file mode 100644 index cff4fae..0000000 --- a/pve01.wheatley.in/templates/talos/main.tf +++ /dev/null @@ -1,96 +0,0 @@ -provider "proxmox" { - endpoint = var.proxmox_endpoint - api_token = var.proxmox_api_token - insecure = true -} - -locals { - talos_versions = ["1.11.4", "1.11.5"] -} - -data "http" "schematic" { - url = "https://factory.talos.dev/schematics" - method = "POST" - request_body = file("${path.module}/schematic.yaml") - request_headers = { - "Content-Type" = "application/x-yaml" - } -} - -locals { - schematic_id = jsondecode(data.http.schematic.response_body)["id"] - - storagepool = "nvme-fastpool" -} - -resource "null_resource" "local_download_talos_image" { - for_each = toset(local.talos_versions) - - provisioner "local-exec" { - command = <