From d4e93de226216f5643c13292e5e07c09562b7325 Mon Sep 17 00:00:00 2001
From: pgijsbertsen <117165507+pgijsbertsen@users.noreply.github.com>
Date: Fri, 30 Jan 2026 23:22:20 +0100
Subject: [PATCH] chore(alloy): Use known config on k8s-wheatley

---
 k8s-wheatley/alloy/configmap.yaml | 645 +++++++++++++++++++++++++-----
 1 file changed, 542 insertions(+), 103 deletions(-)

diff --git a/k8s-wheatley/alloy/configmap.yaml b/k8s-wheatley/alloy/configmap.yaml
index acc193b..bc7adcf 100644
--- a/k8s-wheatley/alloy/configmap.yaml
+++ b/k8s-wheatley/alloy/configmap.yaml
@@ -7,53 +7,408 @@ data:
   config.alloy: |-
     prometheus.exporter.unix "node" {}
 
-    prometheus.relabel "cadvisor" {
-      rule { 
-        action = "keep" 
-        source_labels = ["__name__"] 
-        regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes" 
-        }
-
-      forward_to = [prometheus.relabel.global_filter.receiver]
-    }
-
-    prometheus.relabel "global_filter" {
-      rule { 
-        action = "drop" 
-        source_labels = ["__name__"] 
-        regex = "go_.*|process_.*"
-      }
-      rule { 
-        action = "drop" 
-        source_labels = ["__name__"] 
-        regex = "kube_.*_labels|kube_.*_annotations"
-      }
-      rule { 
-        action = "labeldrop" 
-        regex = "pod_uid|container_id|image_id|container_hash|endpoint|id" 
-      }
-
-      forward_to = [prometheus.remote_write.default.receiver]
-    }
-
-    discovery.kubernetes "nodes" {
-      role = "node"
-    }
-
-    discovery.kubernetes "nodes_cadvisor" {
-      role = "node"
-    }
-
-    discovery.kubernetes "pods" {
-      role = "pod"
-    }
-
-    discovery.kubernetes "apiservers" {
+    discovery.kubernetes "kubernetes_apiservers" {
       role = "endpoints"
     }
 
+    discovery.kubernetes "kubernetes_nodes" {
+      role = "node"
+    }
+
+    discovery.kubernetes "kubernetes_nodes_cadvisor" {
+      role = "node"
+    }
+
+    discovery.kubernetes "kubernetes_service_endpoints" {
+      role = "endpoints"
+    }
+
+    discovery.kubernetes "kubernetes_service_endpoints_slow" {
+      role = "endpoints"
+    }
+
+    discovery.kubernetes "prometheus_pushgateway" {
+      role = "service"
+    }
+
+    discovery.kubernetes "kubernetes_services" {
+      role = "service"
+    }
+
+    discovery.kubernetes "kubernetes_pods" {
+      role = "pod"
+    }
+
+    discovery.kubernetes "kubernetes_pods_slow" {
+      role = "pod"
+    }
+
+    discovery.kubernetes "pod_logs" {
+      role = "pod"
+    }
+
+    discovery.relabel "kubernetes_apiservers" {
+      targets = discovery.kubernetes.kubernetes_apiservers.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"]
+        regex         = "default;kubernetes;https"
+        action        = "keep"
+      }
+    }
+
+    discovery.relabel "kubernetes_nodes" {
+      targets = discovery.kubernetes.kubernetes_nodes.targets
+
+      rule {
+        regex  = "__meta_kubernetes_node_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        target_label = "__address__"
+        replacement  = "kubernetes.default.svc:443"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+        replacement   = "/api/v1/nodes/$1/proxy/metrics"
+      }
+    }
+
+    discovery.relabel "kubernetes_nodes_cadvisor" {
+      targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets
+
+      rule {
+        regex  = "__meta_kubernetes_node_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        target_label = "__address__"
+        replacement  = "kubernetes.default.svc:443"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+        replacement   = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
+      }
+    }
+
+    discovery.relabel "kubernetes_service_endpoints" {
+      targets = discovery.kubernetes.kubernetes_service_endpoints.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
+        regex         = "(.+?)(?::\\d+)?;(\\d+)"
+        target_label  = "__address__"
+        replacement   = "$1:$2"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_service_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_name"]
+        target_label  = "service"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
+    discovery.relabel "kubernetes_service_endpoints_slow" {
+      targets = discovery.kubernetes.kubernetes_service_endpoints_slow.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
+        regex         = "(.+?)(?::\\d+)?;(\\d+)"
+        target_label  = "__address__"
+        replacement   = "$1:$2"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_service_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_name"]
+        target_label  = "service"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
+    discovery.relabel "prometheus_pushgateway" {
+      targets = discovery.kubernetes.prometheus_pushgateway.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
+        regex         = "pushgateway"
+        action        = "keep"
+      }
+    }
+
+    discovery.relabel "kubernetes_services" {
+      targets = discovery.kubernetes.kubernetes_services.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__address__"]
+        target_label  = "__param_target"
+      }
+
+      rule {
+        target_label = "__address__"
+        replacement  = "blackbox"
+      }
+
+      rule {
+        source_labels = ["__param_target"]
+        target_label  = "instance"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_service_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_name"]
+        target_label  = "service"
+      }
+    }
+
+    discovery.relabel "kubernetes_pods" {
+      targets = discovery.kubernetes.kubernetes_pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
+        target_label  = "__address__"
+        replacement   = "[$2]:$1"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);((([0-9]+?)(\\.|$)){4})"
+        target_label  = "__address__"
+        replacement   = "$2:$1"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_pod_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_phase"]
+        regex         = "Pending|Succeeded|Failed|Completed"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
+    discovery.relabel "kubernetes_pods_slow" {
+      targets = discovery.kubernetes.kubernetes_pods_slow.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
+        target_label  = "__address__"
+        replacement   = "[$2]:$1"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);((([0-9]+?)(\\.|$)){4})"
+        target_label  = "__address__"
+        replacement   = "$2:$1"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_pod_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_phase"]
+        regex         = "Pending|Succeeded|Failed|Completed"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
     discovery.relabel "pod_logs" {
-      targets = discovery.kubernetes.pods.targets
+      targets = discovery.kubernetes.pod_logs.targets
 
       rule {
         source_labels = ["__meta_kubernetes_pod_node_name"]
@@ -101,80 +456,164 @@ data:
       }
     }
 
-    prometheus.scrape "node_exporter" {
-      targets  = prometheus.exporter.unix.node.targets
-      job_name = "node-exporter"
-
-      forward_to = [prometheus.relabel.global_filter.receiver]
-    }
-
-    prometheus.scrape "kubernetes_nodes_cadvisor" {
-      targets  = discovery.kubernetes.nodes_cadvisor.targets
-      job_name = "kubernetes-nodes-cadvisor"
-
-      scheme = "https"
-      authorization {
-        type             = "Bearer"
-        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-      }
-
-      tls_config {
-        ca_file             = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-        insecure_skip_verify = true
-      }
-
-      forward_to = [prometheus.relabel.cadvisor.receiver]
-    }
-
-    prometheus.scrape "kubernetes_pods" {
-      targets  = discovery.kubernetes.pods.targets
-      job_name = "kubernetes-pods"
-      honor_labels = true
-      clustering {
-        enabled = true
-      }
-
-      forward_to = [prometheus.relabel.global_filter.receiver]
-    }
-
-    prometheus.scrape "kubernetes_apiservers" {
-      targets  = discovery.kubernetes.apiservers.targets
-      job_name = "kubernetes-apiservers"
-      clustering {
-        enabled = true
-      }
-
-      scheme = "https"
-      authorization {
-        type             = "Bearer"
-        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-      }
-
-      tls_config {
-        ca_file             = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-        insecure_skip_verify = true
-      }
-
-      forward_to = [prometheus.relabel.global_filter.receiver]
-    }
-
     local.file_match "pod_logs" {
       path_targets = discovery.relabel.pod_logs.output
     }
 
+    prometheus.scrape "prometheus" {
+      targets = [{
+        __address__ = "localhost:9090",
+      }]
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "prometheus"
+    }
+
+    prometheus.scrape "node_exporter" {
+      targets    = prometheus.exporter.unix.node.targets
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "node-exporter"
+    }
+
+    prometheus.scrape "kubernetes_apiservers" {
+      targets    = discovery.relabel.kubernetes_apiservers.output
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "kubernetes-apiservers"
+      scheme     = "https"
+      clustering {
+        enabled = true 
+      }
+
+      authorization {
+        type             = "Bearer"
+        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      }
+
+      tls_config {
+        ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        insecure_skip_verify = true
+      }
+    }
+
+    prometheus.scrape "kubernetes_nodes" {
+      targets    = discovery.relabel.kubernetes_nodes.output
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "kubernetes-nodes"
+      scheme     = "https"
+      clustering {
+        enabled = true 
+      }
+
+      authorization {
+        type             = "Bearer"
+        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      }
+
+      tls_config {
+        ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        insecure_skip_verify = true
+      }
+    }
+
+    prometheus.scrape "kubernetes_nodes_cadvisor" {
+      targets    = discovery.relabel.kubernetes_nodes_cadvisor.output
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "kubernetes-nodes-cadvisor"
+      scheme     = "https"
+      clustering {
+        enabled = true 
+      }
+
+      authorization {
+        type             = "Bearer"
+        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      }
+
+      tls_config {
+        ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        insecure_skip_verify = true
+      }
+    }
+
+    prometheus.scrape "kubernetes_service_endpoints" {
+      targets      = discovery.relabel.kubernetes_service_endpoints.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "kubernetes-service-endpoints"
+      honor_labels = true
+      clustering {
+        enabled = true 
+      }
+    }
+
+    prometheus.scrape "kubernetes_service_endpoints_slow" {
+      targets         = discovery.relabel.kubernetes_service_endpoints_slow.output
+      forward_to      = [prometheus.remote_write.default.receiver]
+      job_name        = "kubernetes-service-endpoints-slow"
+      honor_labels    = true
+      scrape_interval = "5m0s"
+      scrape_timeout  = "30s"
+      clustering {
+        enabled = true 
+      }
+    }
+
+    prometheus.scrape "prometheus_pushgateway" {
+      targets      = discovery.relabel.prometheus_pushgateway.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "prometheus-pushgateway"
+      honor_labels = true
+      clustering {
+        enabled = true 
+      }
+    }
+
+    prometheus.scrape "kubernetes_services" {
+      targets      = discovery.relabel.kubernetes_services.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "kubernetes-services"
+      honor_labels = true
+      clustering {
+        enabled = true 
+      }
+      params       = {
+        module = ["http_2xx"],
+      }
+      metrics_path = "/probe"
+    }
+
+    prometheus.scrape "kubernetes_pods" {
+      targets      = discovery.relabel.kubernetes_pods.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "kubernetes-pods"
+      honor_labels = true
+      clustering {
+        enabled = true 
+      }
+    }
+
+    prometheus.scrape "kubernetes_pods_slow" {
+      targets         = discovery.relabel.kubernetes_pods_slow.output
+      forward_to      = [prometheus.remote_write.default.receiver]
+      job_name        = "kubernetes-pods-slow"
+      honor_labels    = true
+      scrape_interval = "5m0s"
+      scrape_timeout  = "30s"
+      clustering {
+        enabled = true 
+      }
+    }
 
     loki.process "pod_logs" {
       forward_to = [loki.write.default.receiver]
 
       stage.static_labels {
         values = {
-          cluster = "k8s-wheatley",
+          cluster = "k8s-peterg",
         }
       }
 
       stage.template {
         source   = "merged_cluster-namespace-container_string"
-        template = "k8s-wheatley;{{`{{.namespace}};{{.container}}`}}"
+        template = "k8s-peterg;{{`{{.namespace}};{{.container}}`}}"
       }
     }
 
@@ -207,7 +646,7 @@ data:
 
     prometheus.remote_write "default" {
       external_labels = {
-        cluster = "k8s-wheatley",
+        cluster = "k8s-peterg",
         node    = env("HOSTNAME"),
       }