kubernetes/k8s-wheatley/alloy/configmap.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: alloy-config
data:
  config.alloy: |-
    prometheus.relabel "global_metric_filter" {
      rule { action = "drop" source_labels = ["__name__"] regex = "container_(fs|network|tasks_state|threads|processes|open_fds|max_fds|file_descriptors|file_descriptor|memory_failures_total|spec_.*|cpu_cfs_.*|last_seen|start_time_seconds)" }
      rule { action = "drop" source_labels = ["__name__"] regex = "go_.*|process_.*" }
      rule { action = "drop" source_labels = ["__name__"] regex = "kube_(pod|deployment|replicaset|daemonset|job|cronjob|endpoint|service|ingress)_labels" }
      rule { action = "drop" source_labels = ["__name__"] regex = "kube_(pod|replicaset|endpoint|service)_annotations" }
      rule { action = "labeldrop" regex = "pod_uid|container_id|image_id|container_hash|endpoint|id" }
    }

    prometheus.relabel "cadvisor_minimal" {
      rule { action = "keep" source_labels = ["__name__"] regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes" }
    }

    prometheus.relabel "node_exporter_minimal" {
      rule { action = "drop" source_labels = ["__name__"] regex = "node_(filesystem|network|disk|vmstat|pressure|entropy|time|schedstat|timex|sockstat|nfs)_.*" }
    }

    prometheus.exporter.unix "node" {}

    discovery.kubernetes "kubernetes_apiservers" { role = "endpoints" }
    discovery.kubernetes "kubernetes_nodes" { role = "node" }
    discovery.kubernetes "kubernetes_nodes_cadvisor" { role = "node" }
    discovery.kubernetes "kubernetes_service_endpoints" { role = "endpoints" }
    discovery.kubernetes "kubernetes_service_endpoints_slow" { role = "endpoints" }
    discovery.kubernetes "prometheus_pushgateway" { role = "service" }
    discovery.kubernetes "kubernetes_services" { role = "service" }
    discovery.kubernetes "kubernetes_pods" { role = "pod" }
    discovery.kubernetes "kubernetes_pods_slow" { role = "pod" }
    discovery.kubernetes "pod_logs" { role = "pod" }

    discovery.relabel "kubernetes_apiservers" {
      targets = discovery.kubernetes.kubernetes_apiservers.targets

      rule {
        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"]
        regex         = "default;kubernetes;https"
        action        = "keep"
      }
    }

    discovery.relabel "kubernetes_nodes" {
      targets = discovery.kubernetes.kubernetes_nodes.targets

      rule {
        regex  = "__meta_kubernetes_node_label_(.+)"
        action = "labelmap"
      }

      rule {
        target_label = "__address__"
        replacement  = "kubernetes.default.svc:443"
      }

      rule {
        source_labels = ["__meta_kubernetes_node_name"]
        regex         = "(.+)"
        target_label  = "__metrics_path__"
        replacement   = "/api/v1/nodes/$1/proxy/metrics"
      }
    }

    discovery.relabel "kubernetes_nodes_cadvisor" {
      targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets

      rule {
        regex  = "__meta_kubernetes_node_label_(.+)"
        action = "labelmap"
      }

      rule {
        target_label = "__address__"
        replacement  = "kubernetes.default.svc:443"
      }

      rule {
        source_labels = ["__meta_kubernetes_node_name"]
        regex         = "(.+)"
        target_label  = "__metrics_path__"
        replacement   = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
      }
    }

    discovery.relabel "kubernetes_service_endpoints" {
      targets = discovery.kubernetes.kubernetes_service_endpoints.targets

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"]
        regex         = "true"
        action        = "keep"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
        regex         = "true"
        action        = "drop"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
        regex         = "(https?)"
        target_label  = "__scheme__"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
        regex         = "(.+)"
        target_label  = "__metrics_path__"
      }

      rule {
        source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
        regex         = "(.+?)(?::\\d+)?;(\\d+)"
        target_label  = "__address__"
        replacement   = "$1:$2"
      }

      rule {
        regex       = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
        replacement = "__param_$1"
        action      = "labelmap"
      }

      rule {
        regex  = "__meta_kubernetes_service_label_(.+)"
        action = "labelmap"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        target_label  = "namespace"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_name"]
        target_label  = "service"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_node_name"]
        target_label  = "node"
      }
    }

    discovery.relabel "kubernetes_service_endpoints_slow" {
      targets = discovery.kubernetes.kubernetes_service_endpoints_slow.targets

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
        regex         = "true"
        action        = "keep"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
        regex         = "(https?)"
        target_label  = "__scheme__"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
        regex         = "(.+)"
        target_label  = "__metrics_path__"
      }

      rule {
        source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
        regex         = "(.+?)(?::\\d+)?;(\\d+)"
        target_label  = "__address__"
        replacement   = "$1:$2"
      }

      rule {
        regex       = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
        replacement = "__param_$1"
        action      = "labelmap"
      }

      rule {
        regex  = "__meta_kubernetes_service_label_(.+)"
        action = "labelmap"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        target_label  = "namespace"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_name"]
        target_label  = "service"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_node_name"]
        target_label  = "node"
      }
    }

    discovery.relabel "prometheus_pushgateway" {
      targets = discovery.kubernetes.prometheus_pushgateway.targets

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
        regex         = "pushgateway"
        action        = "keep"
      }
    }

    discovery.relabel "kubernetes_services" {
      targets = discovery.kubernetes.kubernetes_services.targets

      rule {
        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
        regex         = "true"
        action        = "keep"
      }

      rule {
        source_labels = ["__address__"]
        target_label  = "__param_target"
      }

      rule {
        target_label = "__address__"
        replacement  = "blackbox"
      }

      rule {
        source_labels = ["__param_target"]
        target_label  = "instance"
      }

      rule {
        regex  = "__meta_kubernetes_service_label_(.+)"
        action = "labelmap"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        target_label  = "namespace"
      }

      rule {
        source_labels = ["__meta_kubernetes_service_name"]
        target_label  = "service"
      }
    }

    discovery.relabel "kubernetes_pods" {
      targets = discovery.kubernetes.kubernetes_pods.targets

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
        regex         = "true"
        action        = "keep"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
        regex         = "true"
        action        = "drop"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
        regex         = "(https?)"
        target_label  = "__scheme__"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
        regex         = "(.+)"
        target_label  = "__metrics_path__"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
        regex         = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
        target_label  = "__address__"
        replacement   = "[$2]:$1"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
        regex         = "(\\d+);((([0-9]+?)(\\.|$)){4})"
        target_label  = "__address__"
        replacement   = "$2:$1"
      }

      rule {
        regex       = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
        replacement = "__param_$1"
        action      = "labelmap"
      }

      rule {
        regex  = "__meta_kubernetes_pod_label_(.+)"
        action = "labelmap"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        target_label  = "namespace"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_name"]
        target_label  = "pod"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_phase"]
        regex         = "Pending|Succeeded|Failed|Completed"
        action        = "drop"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_node_name"]
        target_label  = "node"
      }
    }

    discovery.relabel "kubernetes_pods_slow" {
      targets = discovery.kubernetes.kubernetes_pods_slow.targets

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
        regex         = "true"
        action        = "keep"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
        regex         = "(https?)"
        target_label  = "__scheme__"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
        regex         = "(.+)"
        target_label  = "__metrics_path__"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
        regex         = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
        target_label  = "__address__"
        replacement   = "[$2]:$1"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
        regex         = "(\\d+);((([0-9]+?)(\\.|$)){4})"
        target_label  = "__address__"
        replacement   = "$2:$1"
      }

      rule {
        regex       = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
        replacement = "__param_$1"
        action      = "labelmap"
      }

      rule {
        regex  = "__meta_kubernetes_pod_label_(.+)"
        action = "labelmap"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        target_label  = "namespace"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_name"]
        target_label  = "pod"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_phase"]
        regex         = "Pending|Succeeded|Failed|Completed"
        action        = "drop"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_node_name"]
        target_label  = "node"
      }
    }

    discovery.relabel "pod_logs" {
      targets = discovery.kubernetes.pod_logs.targets

      rule {
        source_labels = ["__meta_kubernetes_pod_node_name"]
        action        = "replace"
        target_label  = "__host__"
      }

      rule {
        regex  = "__meta_kubernetes_pod_label_(.+)"
        action = "labelmap"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
        action        = "replace"
        separator     = "/"
        target_label  = "job"
        replacement   = "$1"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        action        = "replace"
        target_label  = "namespace"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_name"]
        action        = "replace"
        target_label  = "pod"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_container_name"]
        action        = "replace"
        target_label  = "container"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
        action        = "replace"
        separator     = "/"
        target_label  = "__path__"
        replacement   = "/var/log/pods/*$1/*.log"
      }
    }

    prometheus.scrape "alloy_self" {
      targets = [{ __address__ = "localhost:12345" }]
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "alloy"
    }

    prometheus.scrape "prometheus" {
      targets = [{ __address__ = "localhost:9090" }]
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "prometheus"
    }

    prometheus.scrape "node_exporter" {
      targets = prometheus.exporter.unix.node.targets
      forward_to = [
        prometheus.relabel.node_exporter_minimal.receiver,
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "node-exporter"
    }

    prometheus.scrape "kubernetes_apiservers" {
      targets = discovery.relabel.kubernetes_apiservers.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-apiservers"
      scheme = "https"
      authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" }
      tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true }
    }

    prometheus.scrape "kubernetes_nodes" {
      targets = discovery.relabel.kubernetes_nodes.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-nodes"
      scheme = "https"
      authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" }
      tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true }
    }

    prometheus.scrape "kubernetes_nodes_cadvisor" {
      targets = discovery.relabel.kubernetes_nodes_cadvisor.output
      forward_to = [
        prometheus.relabel.cadvisor_minimal.receiver,
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-nodes-cadvisor"
      scheme = "https"
      authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" }
      tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true }
    }

    prometheus.scrape "kubernetes_service_endpoints" {
      targets = discovery.relabel.kubernetes_service_endpoints.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-service-endpoints"
      honor_labels = true
    }

    prometheus.scrape "kubernetes_service_endpoints_slow" {
      targets = discovery.relabel.kubernetes_service_endpoints_slow.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-service-endpoints-slow"
      honor_labels = true
      scrape_interval = "5m"
      scrape_timeout  = "30s"
    }

    prometheus.scrape "prometheus_pushgateway" {
      targets = discovery.relabel.prometheus_pushgateway.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "prometheus-pushgateway"
      honor_labels = true
    }

    prometheus.scrape "kubernetes_services" {
      targets = discovery.relabel.kubernetes_services.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-services"
      honor_labels = true
      params = { module = ["http_2xx"] }
      metrics_path = "/probe"
    }

    prometheus.scrape "kubernetes_pods" {
      targets = discovery.relabel.kubernetes_pods.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-pods"
      honor_labels = true
    }

    prometheus.scrape "kubernetes_pods_slow" {
      targets = discovery.relabel.kubernetes_pods_slow.output
      forward_to = [
        prometheus.relabel.global_metric_filter.receiver,
        prometheus.remote_write.default.receiver
      ]
      job_name = "kubernetes-pods-slow"
      honor_labels = true
      scrape_interval = "5m"
      scrape_timeout  = "30s"
    }

    loki.process "pod_logs" {
      forward_to = [loki.write.default.receiver]

      stage.static_labels {
        values = {
          cluster = "k8s-wheatley",
        }
      }

      stage.template {
        source   = "merged_cluster-namespace-container_string"
        template = "k8s-wheatley;{{`{{.namespace}};{{.container}}`}}"
      }
    }

    loki.source.file "pod_logs" {
      targets               = local.file_match.pod_logs.targets
      forward_to            = [loki.process.pod_logs.receiver]
      legacy_positions_file = "/tmp/positions.yaml"
    }

    local.file_match "auditlogs" {
      path_targets = [{
        __address__ = "localhost",
        __path__    = "/var/log/audit/kube/kube-apiserver.log",
        host        = env("HOSTNAME"),
        logtype     = "audit",
      }]
    }

    loki.source.file "auditlogs" {
      targets     = local.file_match.auditlogs.targets
      forward_to  = [loki.write.default.receiver]
    }

    loki.write "default" {
      endpoint {
        url       = "https://loki.peterg.nl/loki/api/v1/push"
        tenant_id = "wheatley"
      }
    }

    prometheus.remote_write "default" {
      external_labels = {
        cluster = "k8s-wheatley",
        node    = env("HOSTNAME"),
      }

      endpoint {
        url = "https://mimir.peterg.nl/api/v1/push"

        headers = {
          "X-Scope-OrgID" = "wheatley",
        }
      }
    }