kubernetes/k8s-wheatley/alloy/configmap.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: alloy-config
data:
  config.alloy: |-
    prometheus.exporter.unix "node" {}

    prometheus.relabel "cadvisor" {
      rule { 
        action = "keep" 
        source_labels = ["__name__"] 
        regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes" 
        }

      forward_to = [prometheus.relabel.global_filter.receiver]
    }

    prometheus.relabel "global_filter" {
      rule { 
        action = "drop" 
        source_labels = ["__name__"] 
        regex = "go_.*|process_.*"
      }
      rule { 
        action = "drop" 
        source_labels = ["__name__"] 
        regex = "kube_.*_labels|kube_.*_annotations"
      }
      rule { 
        action = "labeldrop" 
        regex = "pod_uid|container_id|image_id|container_hash|endpoint|id" 
      }

      forward_to = [prometheus.remote_write.default.receiver]
    }

    discovery.kubernetes "nodes" {
      role = "node"
    }

    discovery.kubernetes "nodes_cadvisor" {
      role = "node"
    }

    discovery.kubernetes "pods" {
      role = "pod"
    }

    discovery.kubernetes "apiservers" {
      role = "endpoints"
    }

    discovery.relabel "pod_logs" {
      targets = discovery.kubernetes.pods.targets

      rule {
        source_labels = ["__meta_kubernetes_pod_node_name"]
        action        = "replace"
        target_label  = "__host__"
      }

      rule {
        regex  = "__meta_kubernetes_pod_label_(.+)"
        action = "labelmap"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
        action        = "replace"
        separator     = "/"
        target_label  = "job"
        replacement   = "$1"
      }

      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        action        = "replace"
        target_label  = "namespace"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_name"]
        action        = "replace"
        target_label  = "pod"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_container_name"]
        action        = "replace"
        target_label  = "container"
      }

      rule {
        source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
        action        = "replace"
        separator     = "/"
        target_label  = "__path__"
        replacement   = "/var/log/pods/*$1/*.log"
      }
    }

    prometheus.scrape "node_exporter" {
      targets  = prometheus.exporter.unix.node.targets
      job_name = "node-exporter"

      forward_to = [prometheus.relabel.global_filter.receiver]
    }

    prometheus.scrape "kubernetes_nodes_cadvisor" {
      targets  = discovery.kubernetes.nodes_cadvisor.targets
      job_name = "kubernetes-nodes-cadvisor"

      scheme = "https"
      authorization {
        type             = "Bearer"
        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
      }

      tls_config {
        ca_file             = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
        insecure_skip_verify = true
      }

      forward_to = [prometheus.relabel.cadvisor.receiver]
    }

    prometheus.scrape "kubernetes_pods" {
      targets  = discovery.kubernetes.pods.targets
      job_name = "kubernetes-pods"
      honor_labels = true
      clustering {
        enabled = true
      }

      forward_to = [prometheus.relabel.global_filter.receiver]
    }

    prometheus.scrape "kubernetes_apiservers" {
      targets  = discovery.kubernetes.apiservers.targets
      job_name = "kubernetes-apiservers"
      clustering {
        enabled = true
      }

      scheme = "https"
      authorization {
        type             = "Bearer"
        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
      }

      tls_config {
        ca_file             = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
        insecure_skip_verify = true
      }

      forward_to = [prometheus.relabel.global_filter.receiver]
    }

    local.file_match "pod_logs" {
      path_targets = discovery.relabel.pod_logs.output
    }


    loki.process "pod_logs" {
      forward_to = [loki.write.default.receiver]

      stage.static_labels {
        values = {
          cluster = "k8s-wheatley",
        }
      }

      stage.template {
        source   = "merged_cluster-namespace-container_string"
        template = "k8s-wheatley;{{`{{.namespace}};{{.container}}`}}"
      }
    }

    loki.source.file "pod_logs" {
      targets               = local.file_match.pod_logs.targets
      forward_to            = [loki.process.pod_logs.receiver]
      legacy_positions_file = "/tmp/positions.yaml"
    }

    local.file_match "auditlogs" {
      path_targets = [{
        __address__ = "localhost",
        __path__    = "/var/log/audit/kube/kube-apiserver.log",
        host        = env("HOSTNAME"),
        logtype     = "audit",
      }]
    }

    loki.source.file "auditlogs" {
      targets     = local.file_match.auditlogs.targets
      forward_to  = [loki.write.default.receiver]
    }

    loki.write "default" {
      endpoint {
        url       = "https://loki.peterg.nl/loki/api/v1/push"
        tenant_id = "wheatley"
      }
    }

    prometheus.remote_write "default" {
      external_labels = {
        cluster = "k8s-wheatley",
        node    = env("HOSTNAME"),
      }

      endpoint {
        url = "https://mimir.peterg.nl/api/v1/push"

        headers = {
          "X-Scope-OrgID" = "wheatley",
        }
      }
    }