From 5bb06eaac8d895cf2fa8b0864bf3ba62653f651c Mon Sep 17 00:00:00 2001 From: pgijsbertsen <117165507+pgijsbertsen@users.noreply.github.com> Date: Tue, 20 Jan 2026 21:55:44 +0100 Subject: [PATCH] chore(alloy): Improve config --- k8s-wheatley/alloy/configmap.yaml | 127 +++++++----------------------- 1 file changed, 28 insertions(+), 99 deletions(-) diff --git a/k8s-wheatley/alloy/configmap.yaml b/k8s-wheatley/alloy/configmap.yaml index d406cc2..c70b747 100644 --- a/k8s-wheatley/alloy/configmap.yaml +++ b/k8s-wheatley/alloy/configmap.yaml @@ -5,31 +5,35 @@ metadata: name: alloy-config data: config.alloy: |- + prometheus.exporter.unix "node" { + } + prometheus.relabel "global_metric_filter" { rule { action = "drop" source_labels = ["__name__"] regex = "container_(fs|network|tasks_state|threads|processes|open_fds|max_fds|file_descriptors|file_descriptor|memory_failures_total|spec_.*|cpu_cfs_.*|last_seen|start_time_seconds)" } rule { action = "drop" source_labels = ["__name__"] regex = "go_.*|process_.*" } rule { action = "drop" source_labels = ["__name__"] regex = "kube_(pod|deployment|replicaset|daemonset|job|cronjob|endpoint|service|ingress)_labels" } rule { action = "drop" source_labels = ["__name__"] regex = "kube_(pod|replicaset|endpoint|service)_annotations" } rule { action = "labeldrop" regex = "pod_uid|container_id|image_id|container_hash|endpoint|id" } + + forward_to = [prometheus.remote_write.default.receiver] } prometheus.relabel "cadvisor_minimal" { rule { action = "keep" source_labels = ["__name__"] regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes" } + + forward_to = [prometheus.relabel.global_metric_filter.receiver] } prometheus.relabel "node_exporter_minimal" { rule { action = "drop" source_labels = ["__name__"] regex = "node_(filesystem|network|disk|vmstat|pressure|entropy|time|schedstat|timex|sockstat|nfs)_.*" } + + forward_to = [prometheus.relabel.global_metric_filter.receiver] } - prometheus.exporter.unix "node" {} + prometheus.exporter.node "node" {} - discovery.kubernetes "kubernetes_apiservers" { role = "endpoints" } discovery.kubernetes "kubernetes_nodes" { role = "node" } discovery.kubernetes "kubernetes_nodes_cadvisor" { role = "node" } - discovery.kubernetes "kubernetes_service_endpoints" { role = "endpoints" } - discovery.kubernetes "kubernetes_service_endpoints_slow" { role = "endpoints" } - discovery.kubernetes "prometheus_pushgateway" { role = "service" } - discovery.kubernetes "kubernetes_services" { role = "service" } discovery.kubernetes "kubernetes_pods" { role = "pod" } discovery.kubernetes "kubernetes_pods_slow" { role = "pod" } discovery.kubernetes "pod_logs" { role = "pod" } @@ -462,116 +466,41 @@ data: } prometheus.scrape "node_exporter" { - targets = prometheus.exporter.unix.node.targets - forward_to = [ - prometheus.relabel.node_exporter_minimal.receiver, - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] + targets = prometheus.exporter.node.targets job_name = "node-exporter" - } - prometheus.scrape "kubernetes_apiservers" { - targets = discovery.relabel.kubernetes_apiservers.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] - job_name = "kubernetes-apiservers" - scheme = "https" - authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } - tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } - } - - prometheus.scrape "kubernetes_nodes" { - targets = discovery.relabel.kubernetes_nodes.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] - job_name = "kubernetes-nodes" - scheme = "https" - authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } - tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } + forward_to = [prometheus.relabel.node_exporter_minimal.receiver] } prometheus.scrape "kubernetes_nodes_cadvisor" { - targets = discovery.relabel.kubernetes_nodes_cadvisor.output - forward_to = [ - prometheus.relabel.cadvisor_minimal.receiver, - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] + targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets job_name = "kubernetes-nodes-cadvisor" + scheme = "https" authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } - } - prometheus.scrape "kubernetes_service_endpoints" { - targets = discovery.relabel.kubernetes_service_endpoints.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] - job_name = "kubernetes-service-endpoints" - honor_labels = true - } - - prometheus.scrape "kubernetes_service_endpoints_slow" { - targets = discovery.relabel.kubernetes_service_endpoints_slow.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] - job_name = "kubernetes-service-endpoints-slow" - honor_labels = true - scrape_interval = "5m" - scrape_timeout = "30s" - } - - prometheus.scrape "prometheus_pushgateway" { - targets = discovery.relabel.prometheus_pushgateway.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] - job_name = "prometheus-pushgateway" - honor_labels = true - } - - prometheus.scrape "kubernetes_services" { - targets = discovery.relabel.kubernetes_services.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] - job_name = "kubernetes-services" - honor_labels = true - params = { module = ["http_2xx"] } - metrics_path = "/probe" + forward_to = [prometheus.relabel.cadvisor_minimal.receiver] } prometheus.scrape "kubernetes_pods" { - targets = discovery.relabel.kubernetes_pods.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] + targets = discovery.kubernetes.kubernetes_pods.targets job_name = "kubernetes-pods" honor_labels = true + + forward_to = [prometheus.relabel.global_metric_filter.receiver] } - prometheus.scrape "kubernetes_pods_slow" { - targets = discovery.relabel.kubernetes_pods_slow.output - forward_to = [ - prometheus.relabel.global_metric_filter.receiver, - prometheus.remote_write.default.receiver - ] - job_name = "kubernetes-pods-slow" - honor_labels = true - scrape_interval = "5m" - scrape_timeout = "30s" + prometheus.remote_write "default" { + external_labels = { + cluster = "k8s-wheatley", + node = env("HOSTNAME"), + } + + endpoint { + url = "https://mimir.peterg.nl/api/v1/push" + headers = { "X-Scope-OrgID" = "wheatley" } + } } loki.process "pod_logs" {