chore(alloy): Use known config on k8s-wheatley

This commit is contained in:
Peter 2026-01-30 23:22:20 +01:00
parent c17d1e7193
commit d4e93de226
Signed by: Peter
SSH key fingerprint: SHA256:B5tYaxBExaDm74r1px9iVeZ6F/ZDiyiy9SbBqfZYrvg

View file

@ -7,53 +7,408 @@ data:
config.alloy: |-
prometheus.exporter.unix "node" {}
prometheus.relabel "cadvisor" {
rule {
action = "keep"
source_labels = ["__name__"]
regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes"
}
forward_to = [prometheus.relabel.global_filter.receiver]
}
prometheus.relabel "global_filter" {
rule {
action = "drop"
source_labels = ["__name__"]
regex = "go_.*|process_.*"
}
rule {
action = "drop"
source_labels = ["__name__"]
regex = "kube_.*_labels|kube_.*_annotations"
}
rule {
action = "labeldrop"
regex = "pod_uid|container_id|image_id|container_hash|endpoint|id"
}
forward_to = [prometheus.remote_write.default.receiver]
}
discovery.kubernetes "nodes" {
role = "node"
}
discovery.kubernetes "nodes_cadvisor" {
role = "node"
}
discovery.kubernetes "pods" {
role = "pod"
}
discovery.kubernetes "apiservers" {
discovery.kubernetes "kubernetes_apiservers" {
role = "endpoints"
}
discovery.kubernetes "kubernetes_nodes" {
role = "node"
}
discovery.kubernetes "kubernetes_nodes_cadvisor" {
role = "node"
}
discovery.kubernetes "kubernetes_service_endpoints" {
role = "endpoints"
}
discovery.kubernetes "kubernetes_service_endpoints_slow" {
role = "endpoints"
}
discovery.kubernetes "prometheus_pushgateway" {
role = "service"
}
discovery.kubernetes "kubernetes_services" {
role = "service"
}
discovery.kubernetes "kubernetes_pods" {
role = "pod"
}
discovery.kubernetes "kubernetes_pods_slow" {
role = "pod"
}
discovery.kubernetes "pod_logs" {
role = "pod"
}
discovery.relabel "kubernetes_apiservers" {
targets = discovery.kubernetes.kubernetes_apiservers.targets
rule {
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"]
regex = "default;kubernetes;https"
action = "keep"
}
}
discovery.relabel "kubernetes_nodes" {
targets = discovery.kubernetes.kubernetes_nodes.targets
rule {
regex = "__meta_kubernetes_node_label_(.+)"
action = "labelmap"
}
rule {
target_label = "__address__"
replacement = "kubernetes.default.svc:443"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "__metrics_path__"
replacement = "/api/v1/nodes/$1/proxy/metrics"
}
}
discovery.relabel "kubernetes_nodes_cadvisor" {
targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets
rule {
regex = "__meta_kubernetes_node_label_(.+)"
action = "labelmap"
}
rule {
target_label = "__address__"
replacement = "kubernetes.default.svc:443"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "__metrics_path__"
replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
}
}
discovery.relabel "kubernetes_service_endpoints" {
targets = discovery.kubernetes.kubernetes_service_endpoints.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
regex = "(.+?)(?::\\d+)?;(\\d+)"
target_label = "__address__"
replacement = "$1:$2"
}
rule {
regex = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_service_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_service_name"]
target_label = "service"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
discovery.relabel "kubernetes_service_endpoints_slow" {
targets = discovery.kubernetes.kubernetes_service_endpoints_slow.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
regex = "(.+?)(?::\\d+)?;(\\d+)"
target_label = "__address__"
replacement = "$1:$2"
}
rule {
regex = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_service_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_service_name"]
target_label = "service"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
discovery.relabel "prometheus_pushgateway" {
targets = discovery.kubernetes.prometheus_pushgateway.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
regex = "pushgateway"
action = "keep"
}
}
discovery.relabel "kubernetes_services" {
targets = discovery.kubernetes.kubernetes_services.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__address__"]
target_label = "__param_target"
}
rule {
target_label = "__address__"
replacement = "blackbox"
}
rule {
source_labels = ["__param_target"]
target_label = "instance"
}
rule {
regex = "__meta_kubernetes_service_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_service_name"]
target_label = "service"
}
}
discovery.relabel "kubernetes_pods" {
targets = discovery.kubernetes.kubernetes_pods.targets
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
target_label = "__address__"
replacement = "[$2]:$1"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);((([0-9]+?)(\\.|$)){4})"
target_label = "__address__"
replacement = "$2:$1"
}
rule {
regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_pod_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_phase"]
regex = "Pending|Succeeded|Failed|Completed"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
discovery.relabel "kubernetes_pods_slow" {
targets = discovery.kubernetes.kubernetes_pods_slow.targets
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
target_label = "__address__"
replacement = "[$2]:$1"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);((([0-9]+?)(\\.|$)){4})"
target_label = "__address__"
replacement = "$2:$1"
}
rule {
regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_pod_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_phase"]
regex = "Pending|Succeeded|Failed|Completed"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
discovery.relabel "pod_logs" {
targets = discovery.kubernetes.pods.targets
targets = discovery.kubernetes.pod_logs.targets
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
@ -101,18 +456,33 @@ data:
}
}
prometheus.scrape "node_exporter" {
targets = prometheus.exporter.unix.node.targets
job_name = "node-exporter"
forward_to = [prometheus.relabel.global_filter.receiver]
local.file_match "pod_logs" {
path_targets = discovery.relabel.pod_logs.output
}
prometheus.scrape "kubernetes_nodes_cadvisor" {
targets = discovery.kubernetes.nodes_cadvisor.targets
job_name = "kubernetes-nodes-cadvisor"
prometheus.scrape "prometheus" {
targets = [{
__address__ = "localhost:9090",
}]
forward_to = [prometheus.remote_write.default.receiver]
job_name = "prometheus"
}
prometheus.scrape "node_exporter" {
targets = prometheus.exporter.unix.node.targets
forward_to = [prometheus.remote_write.default.receiver]
job_name = "node-exporter"
}
prometheus.scrape "kubernetes_apiservers" {
targets = discovery.relabel.kubernetes_apiservers.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-apiservers"
scheme = "https"
clustering {
enabled = true
}
authorization {
type = "Bearer"
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
@ -122,59 +492,128 @@ data:
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
}
forward_to = [prometheus.relabel.cadvisor.receiver]
prometheus.scrape "kubernetes_nodes" {
targets = discovery.relabel.kubernetes_nodes.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-nodes"
scheme = "https"
clustering {
enabled = true
}
authorization {
type = "Bearer"
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
}
prometheus.scrape "kubernetes_nodes_cadvisor" {
targets = discovery.relabel.kubernetes_nodes_cadvisor.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-nodes-cadvisor"
scheme = "https"
clustering {
enabled = true
}
authorization {
type = "Bearer"
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
}
prometheus.scrape "kubernetes_service_endpoints" {
targets = discovery.relabel.kubernetes_service_endpoints.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-service-endpoints"
honor_labels = true
clustering {
enabled = true
}
}
prometheus.scrape "kubernetes_service_endpoints_slow" {
targets = discovery.relabel.kubernetes_service_endpoints_slow.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-service-endpoints-slow"
honor_labels = true
scrape_interval = "5m0s"
scrape_timeout = "30s"
clustering {
enabled = true
}
}
prometheus.scrape "prometheus_pushgateway" {
targets = discovery.relabel.prometheus_pushgateway.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "prometheus-pushgateway"
honor_labels = true
clustering {
enabled = true
}
}
prometheus.scrape "kubernetes_services" {
targets = discovery.relabel.kubernetes_services.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-services"
honor_labels = true
clustering {
enabled = true
}
params = {
module = ["http_2xx"],
}
metrics_path = "/probe"
}
prometheus.scrape "kubernetes_pods" {
targets = discovery.kubernetes.pods.targets
targets = discovery.relabel.kubernetes_pods.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-pods"
honor_labels = true
clustering {
enabled = true
}
forward_to = [prometheus.relabel.global_filter.receiver]
}
prometheus.scrape "kubernetes_apiservers" {
targets = discovery.kubernetes.apiservers.targets
job_name = "kubernetes-apiservers"
prometheus.scrape "kubernetes_pods_slow" {
targets = discovery.relabel.kubernetes_pods_slow.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-pods-slow"
honor_labels = true
scrape_interval = "5m0s"
scrape_timeout = "30s"
clustering {
enabled = true
}
scheme = "https"
authorization {
type = "Bearer"
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
forward_to = [prometheus.relabel.global_filter.receiver]
}
local.file_match "pod_logs" {
path_targets = discovery.relabel.pod_logs.output
}
loki.process "pod_logs" {
forward_to = [loki.write.default.receiver]
stage.static_labels {
values = {
cluster = "k8s-wheatley",
cluster = "k8s-peterg",
}
}
stage.template {
source = "merged_cluster-namespace-container_string"
template = "k8s-wheatley;{{`{{.namespace}};{{.container}}`}}"
template = "k8s-peterg;{{`{{.namespace}};{{.container}}`}}"
}
}
@ -207,7 +646,7 @@ data:
prometheus.remote_write "default" {
external_labels = {
cluster = "k8s-wheatley",
cluster = "k8s-peterg",
node = env("HOSTNAME"),
}