diff --git a/pve01.wheatley.in/k8s-wheatley/main.tf b/pve01.wheatley.in/k8s-wheatley/main.tf index 6e63247..f8e2fe3 100644 --- a/pve01.wheatley.in/k8s-wheatley/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/main.tf @@ -71,16 +71,19 @@ locals { name = "worker01" ip_address = "10.13.37.21" storage_address = "10.0.69.21" + gpu_enabled = false }, { name = "worker02" ip_address = "10.13.37.22" storage_address = "10.0.69.22" + gpu_enabled = false }, { name = "worker03" ip_address = "10.13.37.23" storage_address = "10.0.69.23" + gpu_enabled = false }, ] } @@ -122,6 +125,7 @@ module "workers" { name = format("k8s-wheatley-%s", each.value.name) ipv4_address = each.value.ip_address cpu = local.workers.cpu + gpu = each.value.gpu_enabled memory = local.workers.memory disk = local.workers.disk storagepool = local.workers.storagepool @@ -171,6 +175,7 @@ module "talos-bootstrap" { ipv4_address = node.ip_address storage_address = node.storage_address pvc_disks = local.workers.pvc_disks + gpu_enabled = node.gpu_enabled } ] ) diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf index 117a5ae..55625c7 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/main.tf @@ -54,7 +54,6 @@ resource "talos_machine_configuration_apply" "config_apply_cp" { vip_address = var.node_config.cluster_endpoint node_name = format("%s.wheatley.in", var.node_config.proxmox_node) machine_type = each.value.type - talos_version = var.node_config.talos_version gateway-api_version = var.node_config.gateway_api_version cilium_version = var.node_config.cilium_version cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl") @@ -84,6 +83,7 @@ resource "talos_machine_configuration_apply" "config_apply_worker" { cilium_version = var.node_config.cilium_version cilium_install_file = file("${path.module}/templates/cilium-install.yaml.tmpl") pvc_disks = each.value.pvc_disks + gpu_enabled = each.value.gpu_enabled }) ] } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl index 978019b..53ce1f8 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-cp.yaml.tmpl @@ -3,7 +3,6 @@ machine: type: ${machine_type} install: disk: /dev/vda - image: factory.talos.dev/nocloud-installer-secureboot/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v${talos_version} network: hostname: ${hostname} nameservers: diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl index 4300403..7692f7d 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/templates/machineconfig-worker.yaml.tmpl @@ -3,7 +3,9 @@ machine: type: ${machine_type} install: disk: /dev/vda - image: factory.talos.dev/nocloud-installer-secureboot/6adc7e7fba27948460e2231e5272e88b85159da3f3db980551976bf9898ff64b:v${talos_version} +%{ if gpu_enabled != false ~} + image: factory.talos.dev/nocloud-installer-secureboot/29d29d87bf3b88fe13caf73c20c7a8b6a4355d8177d0d690d5c9f85d4ddb67b7:v${talos_version} +%{ endif ~} network: hostname: ${hostname} nameservers: @@ -32,6 +34,9 @@ machine: - usermode_helper=disabled - name: drbd_transport_tcp - name: dm_thin_pool +%{ if gpu_enabled != false ~} + - name: i915 +%{ endif ~} nodeLabels: topology.kubernetes.io/region: ${cluster_name} topology.kubernetes.io/zone: ${node_name} diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf index 9fb3342..095e6c6 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-bootstrap/variables.tf @@ -21,6 +21,7 @@ variable "talos_nodes" { type = string ipv4_address = string storage_address = optional(string) - pvc_disks = optional(list(number)), + pvc_disks = optional(list(number)) + gpu_enabled = optional(bool, false) })) } diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf index deae941..c312e77 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/main.tf @@ -61,6 +61,16 @@ resource "proxmox_virtual_environment_vm" "talos-node" { } } + dynamic "hostpci" { + for_each = var.node.gpu == true ? [1] : [] + content { + device = "hostpci0" + mapping = "A380_GPU" + pcie = true + rombar = true + } + } + initialization { datastore_id = var.node.storagepool ip_config { diff --git a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf index 84c1013..26b7451 100644 --- a/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf +++ b/pve01.wheatley.in/k8s-wheatley/modules/talos-node/variables.tf @@ -7,6 +7,7 @@ variable "node" { ipv4_address = string ipv4_gateway = string cpu = number + gpu = optional(bool, false) memory = number disk = string storagepool = string