feat: Deploy Alloy on k8s-wheatley

k8s-peterg/argocd/applications-wheatley.yaml

@@ -60,6 +60,26 @@ spec:
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
+metadata:
+  name: alloy-wheatley
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://code.peterg.nl/wheatley/kubernetes.git
+    path: k8s-wheatley/alloy
+    targetRevision: HEAD
+  destination:
+    server: https://10.13.37.10:6443
+    namespace: alloy
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
 metadata:
   name: cloudnative-pg
   namespace: argocd

k8s-wheatley/alloy/configmap.yaml

@@ -0,0 +1,634 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-config
+data:
+  config.alloy: |-
+    prometheus.exporter.unix "node" {
+    }
+
+    discovery.kubernetes "kubernetes_apiservers" {
+      role = "endpoints"
+    }
+
+    discovery.kubernetes "kubernetes_nodes" {
+      role = "node"
+    }
+
+    discovery.kubernetes "kubernetes_nodes_cadvisor" {
+      role = "node"
+    }
+
+    discovery.kubernetes "kubernetes_service_endpoints" {
+      role = "endpoints"
+    }
+
+    discovery.kubernetes "kubernetes_service_endpoints_slow" {
+      role = "endpoints"
+    }
+
+    discovery.kubernetes "prometheus_pushgateway" {
+      role = "service"
+    }
+
+    discovery.kubernetes "kubernetes_services" {
+      role = "service"
+    }
+
+    discovery.kubernetes "kubernetes_pods" {
+      role = "pod"
+    }
+
+    discovery.kubernetes "kubernetes_pods_slow" {
+      role = "pod"
+    }
+
+    discovery.kubernetes "pod_logs" {
+      role = "pod"
+    }
+
+    discovery.relabel "kubernetes_apiservers" {
+      targets = discovery.kubernetes.kubernetes_apiservers.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"]
+        regex         = "default;kubernetes;https"
+        action        = "keep"
+      }
+    }
+
+    discovery.relabel "kubernetes_nodes" {
+      targets = discovery.kubernetes.kubernetes_nodes.targets
+
+      rule {
+        regex  = "__meta_kubernetes_node_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        target_label = "__address__"
+        replacement  = "kubernetes.default.svc:443"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+        replacement   = "/api/v1/nodes/$1/proxy/metrics"
+      }
+    }
+
+    discovery.relabel "kubernetes_nodes_cadvisor" {
+      targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets
+
+      rule {
+        regex  = "__meta_kubernetes_node_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        target_label = "__address__"
+        replacement  = "kubernetes.default.svc:443"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+        replacement   = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
+      }
+    }
+
+    discovery.relabel "kubernetes_service_endpoints" {
+      targets = discovery.kubernetes.kubernetes_service_endpoints.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
+        regex         = "(.+?)(?::\\d+)?;(\\d+)"
+        target_label  = "__address__"
+        replacement   = "$1:$2"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_service_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_name"]
+        target_label  = "service"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
+    discovery.relabel "kubernetes_service_endpoints_slow" {
+      targets = discovery.kubernetes.kubernetes_service_endpoints_slow.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
+        regex         = "(.+?)(?::\\d+)?;(\\d+)"
+        target_label  = "__address__"
+        replacement   = "$1:$2"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_service_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_name"]
+        target_label  = "service"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
+    discovery.relabel "prometheus_pushgateway" {
+      targets = discovery.kubernetes.prometheus_pushgateway.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
+        regex         = "pushgateway"
+        action        = "keep"
+      }
+    }
+
+    discovery.relabel "kubernetes_services" {
+      targets = discovery.kubernetes.kubernetes_services.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__address__"]
+        target_label  = "__param_target"
+      }
+
+      rule {
+        target_label = "__address__"
+        replacement  = "blackbox"
+      }
+
+      rule {
+        source_labels = ["__param_target"]
+        target_label  = "instance"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_service_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_service_name"]
+        target_label  = "service"
+      }
+    }
+
+    discovery.relabel "kubernetes_pods" {
+      targets = discovery.kubernetes.kubernetes_pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
+        target_label  = "__address__"
+        replacement   = "[$2]:$1"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);((([0-9]+?)(\\.|$)){4})"
+        target_label  = "__address__"
+        replacement   = "$2:$1"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_pod_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_phase"]
+        regex         = "Pending|Succeeded|Failed|Completed"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
+    discovery.relabel "kubernetes_pods_slow" {
+      targets = discovery.kubernetes.kubernetes_pods_slow.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
+        regex         = "true"
+        action        = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
+        regex         = "(https?)"
+        target_label  = "__scheme__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
+        regex         = "(.+)"
+        target_label  = "__metrics_path__"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
+        target_label  = "__address__"
+        replacement   = "[$2]:$1"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
+        regex         = "(\\d+);((([0-9]+?)(\\.|$)){4})"
+        target_label  = "__address__"
+        replacement   = "$2:$1"
+      }
+
+      rule {
+        regex       = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
+        replacement = "__param_$1"
+        action      = "labelmap"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_pod_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_phase"]
+        regex         = "Pending|Succeeded|Failed|Completed"
+        action        = "drop"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label  = "node"
+      }
+    }
+
+    discovery.relabel "pod_logs" {
+      targets = discovery.kubernetes.pod_logs.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        action        = "replace"
+        target_label  = "__host__"
+      }
+
+      rule {
+        regex  = "__meta_kubernetes_pod_label_(.+)"
+        action = "labelmap"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
+        action        = "replace"
+        separator     = "/"
+        target_label  = "job"
+        replacement   = "$1"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        action        = "replace"
+        target_label  = "namespace"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        action        = "replace"
+        target_label  = "pod"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        action        = "replace"
+        target_label  = "container"
+      }
+
+      rule {
+        source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+        action        = "replace"
+        separator     = "/"
+        target_label  = "__path__"
+        replacement   = "/var/log/pods/*$1/*.log"
+      }
+    }
+
+    local.file_match "pod_logs" {
+      path_targets = discovery.relabel.pod_logs.output
+    }
+
+    prometheus.scrape "prometheus" {
+      targets = [{
+        __address__ = "localhost:9090",
+      }]
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "prometheus"
+    }
+
+    prometheus.scrape "node_exporter" {
+      targets    = prometheus.exporter.unix.node.targets
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "node-exporter"
+    }
+
+    prometheus.scrape "kubernetes_apiservers" {
+      targets    = discovery.relabel.kubernetes_apiservers.output
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "kubernetes-apiservers"
+      scheme     = "https"
+
+      authorization {
+        type             = "Bearer"
+        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      }
+
+      tls_config {
+        ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        insecure_skip_verify = true
+      }
+    }
+
+    prometheus.scrape "kubernetes_nodes" {
+      targets    = discovery.relabel.kubernetes_nodes.output
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "kubernetes-nodes"
+      scheme     = "https"
+
+      authorization {
+        type             = "Bearer"
+        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      }
+
+      tls_config {
+        ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        insecure_skip_verify = true
+      }
+    }
+
+    prometheus.scrape "kubernetes_nodes_cadvisor" {
+      targets    = discovery.relabel.kubernetes_nodes_cadvisor.output
+      forward_to = [prometheus.remote_write.default.receiver]
+      job_name   = "kubernetes-nodes-cadvisor"
+      scheme     = "https"
+
+      authorization {
+        type             = "Bearer"
+        credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      }
+
+      tls_config {
+        ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        insecure_skip_verify = true
+      }
+    }
+
+    prometheus.scrape "kubernetes_service_endpoints" {
+      targets      = discovery.relabel.kubernetes_service_endpoints.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "kubernetes-service-endpoints"
+      honor_labels = true
+    }
+
+    prometheus.scrape "kubernetes_service_endpoints_slow" {
+      targets         = discovery.relabel.kubernetes_service_endpoints_slow.output
+      forward_to      = [prometheus.remote_write.default.receiver]
+      job_name        = "kubernetes-service-endpoints-slow"
+      honor_labels    = true
+      scrape_interval = "5m0s"
+      scrape_timeout  = "30s"
+    }
+
+    prometheus.scrape "prometheus_pushgateway" {
+      targets      = discovery.relabel.prometheus_pushgateway.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "prometheus-pushgateway"
+      honor_labels = true
+    }
+
+    prometheus.scrape "kubernetes_services" {
+      targets      = discovery.relabel.kubernetes_services.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "kubernetes-services"
+      honor_labels = true
+      params       = {
+        module = ["http_2xx"],
+      }
+      metrics_path = "/probe"
+    }
+
+    prometheus.scrape "kubernetes_pods" {
+      targets      = discovery.relabel.kubernetes_pods.output
+      forward_to   = [prometheus.remote_write.default.receiver]
+      job_name     = "kubernetes-pods"
+      honor_labels = true
+    }
+
+    prometheus.scrape "kubernetes_pods_slow" {
+      targets         = discovery.relabel.kubernetes_pods_slow.output
+      forward_to      = [prometheus.remote_write.default.receiver]
+      job_name        = "kubernetes-pods-slow"
+      honor_labels    = true
+      scrape_interval = "5m0s"
+      scrape_timeout  = "30s"
+    }
+
+    loki.process "pod_logs" {
+      forward_to = [loki.write.default.receiver]
+
+      stage.static_labels {
+        values = {
+          cluster = "k8s-wheatley",
+        }
+      }
+
+      stage.template {
+        source   = "merged_cluster-namespace-container_string"
+        template = "k8s-wheatley;{{`{{.namespace}};{{.container}}`}}"
+      }
+    }
+
+    loki.source.file "pod_logs" {
+      targets               = local.file_match.pod_logs.targets
+      forward_to            = [loki.process.pod_logs.receiver]
+      legacy_positions_file = "/tmp/positions.yaml"
+    }
+
+    local.file_match "auditlogs" {
+      path_targets = [{
+        __address__ = "localhost",
+        __path__    = "/var/log/audit/kube/kube-apiserver.log",
+        host        = env("HOSTNAME"),
+        logtype     = "audit",
+      }]
+    }
+
+    loki.source.file "auditlogs" {
+      targets     = local.file_match.auditlogs.targets
+      forward_to  = [loki.write.default.receiver]
+    }
+
+    loki.write "default" {
+      endpoint {
+        url       = "https://loki.peterg.nl/loki/api/v1/push"
+        tenant_id = "wheatley"
+      }
+    }
+
+    prometheus.remote_write "default" {
+      external_labels = {
+        cluster = "k8s-wheatley",
+        node    = env("HOSTNAME"),
+      }
+
+      endpoint {
+        url = "https://mimir.peterg.nl/api/v1/push"
+
+        headers = {
+          "X-Scope-OrgID" = "wheatley",
+        }
+      }
+    }

k8s-wheatley/alloy/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../kustomize-bases/alloy
+  - configmap.yaml