diff --git a/k8s-wheatley/alloy/configmap.yaml b/k8s-wheatley/alloy/configmap.yaml index 6e4faa6..d406cc2 100644 --- a/k8s-wheatley/alloy/configmap.yaml +++ b/k8s-wheatley/alloy/configmap.yaml @@ -5,48 +5,34 @@ metadata: name: alloy-config data: config.alloy: |- - prometheus.exporter.unix "node" { + prometheus.relabel "global_metric_filter" { + rule { action = "drop" source_labels = ["__name__"] regex = "container_(fs|network|tasks_state|threads|processes|open_fds|max_fds|file_descriptors|file_descriptor|memory_failures_total|spec_.*|cpu_cfs_.*|last_seen|start_time_seconds)" } + rule { action = "drop" source_labels = ["__name__"] regex = "go_.*|process_.*" } + rule { action = "drop" source_labels = ["__name__"] regex = "kube_(pod|deployment|replicaset|daemonset|job|cronjob|endpoint|service|ingress)_labels" } + rule { action = "drop" source_labels = ["__name__"] regex = "kube_(pod|replicaset|endpoint|service)_annotations" } + rule { action = "labeldrop" regex = "pod_uid|container_id|image_id|container_hash|endpoint|id" } } - discovery.kubernetes "kubernetes_apiservers" { - role = "endpoints" + prometheus.relabel "cadvisor_minimal" { + rule { action = "keep" source_labels = ["__name__"] regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes" } } - discovery.kubernetes "kubernetes_nodes" { - role = "node" + prometheus.relabel "node_exporter_minimal" { + rule { action = "drop" source_labels = ["__name__"] regex = "node_(filesystem|network|disk|vmstat|pressure|entropy|time|schedstat|timex|sockstat|nfs)_.*" } } - discovery.kubernetes "kubernetes_nodes_cadvisor" { - role = "node" - } + prometheus.exporter.unix "node" {} - discovery.kubernetes "kubernetes_service_endpoints" { - role = "endpoints" - } - - discovery.kubernetes "kubernetes_service_endpoints_slow" { - role = "endpoints" - } - - discovery.kubernetes "prometheus_pushgateway" { - role = "service" - } - - discovery.kubernetes "kubernetes_services" { - role = "service" - } - - discovery.kubernetes "kubernetes_pods" { - role = "pod" - } - - discovery.kubernetes "kubernetes_pods_slow" { - role = "pod" - } - - discovery.kubernetes "pod_logs" { - role = "pod" - } + discovery.kubernetes "kubernetes_apiservers" { role = "endpoints" } + discovery.kubernetes "kubernetes_nodes" { role = "node" } + discovery.kubernetes "kubernetes_nodes_cadvisor" { role = "node" } + discovery.kubernetes "kubernetes_service_endpoints" { role = "endpoints" } + discovery.kubernetes "kubernetes_service_endpoints_slow" { role = "endpoints" } + discovery.kubernetes "prometheus_pushgateway" { role = "service" } + discovery.kubernetes "kubernetes_services" { role = "service" } + discovery.kubernetes "kubernetes_pods" { role = "pod" } + discovery.kubernetes "kubernetes_pods_slow" { role = "pod" } + discovery.kubernetes "pod_logs" { role = "pod" } discovery.relabel "kubernetes_apiservers" { targets = discovery.kubernetes.kubernetes_apiservers.targets @@ -457,122 +443,134 @@ data: } } - local.file_match "pod_logs" { - path_targets = discovery.relabel.pod_logs.output + prometheus.scrape "alloy_self" { + targets = [{ __address__ = "localhost:12345" }] + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "alloy" } prometheus.scrape "prometheus" { - targets = [{ - __address__ = "localhost:9090", - }] - forward_to = [prometheus.remote_write.default.receiver] - job_name = "prometheus" + targets = [{ __address__ = "localhost:9090" }] + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "prometheus" } prometheus.scrape "node_exporter" { - targets = prometheus.exporter.unix.node.targets - forward_to = [prometheus.remote_write.default.receiver] - job_name = "node-exporter" + targets = prometheus.exporter.unix.node.targets + forward_to = [ + prometheus.relabel.node_exporter_minimal.receiver, + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "node-exporter" } prometheus.scrape "kubernetes_apiservers" { - targets = discovery.relabel.kubernetes_apiservers.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-apiservers" - scheme = "https" - - authorization { - type = "Bearer" - credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - } - - tls_config { - ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify = true - } + targets = discovery.relabel.kubernetes_apiservers.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-apiservers" + scheme = "https" + authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } + tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } } prometheus.scrape "kubernetes_nodes" { - targets = discovery.relabel.kubernetes_nodes.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-nodes" - scheme = "https" - - authorization { - type = "Bearer" - credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - } - - tls_config { - ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify = true - } + targets = discovery.relabel.kubernetes_nodes.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-nodes" + scheme = "https" + authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } + tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } } prometheus.scrape "kubernetes_nodes_cadvisor" { - targets = discovery.relabel.kubernetes_nodes_cadvisor.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-nodes-cadvisor" - scheme = "https" - - authorization { - type = "Bearer" - credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - } - - tls_config { - ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify = true - } + targets = discovery.relabel.kubernetes_nodes_cadvisor.output + forward_to = [ + prometheus.relabel.cadvisor_minimal.receiver, + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-nodes-cadvisor" + scheme = "https" + authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } + tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } } prometheus.scrape "kubernetes_service_endpoints" { - targets = discovery.relabel.kubernetes_service_endpoints.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-service-endpoints" + targets = discovery.relabel.kubernetes_service_endpoints.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-service-endpoints" honor_labels = true } prometheus.scrape "kubernetes_service_endpoints_slow" { - targets = discovery.relabel.kubernetes_service_endpoints_slow.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-service-endpoints-slow" - honor_labels = true - scrape_interval = "5m0s" + targets = discovery.relabel.kubernetes_service_endpoints_slow.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-service-endpoints-slow" + honor_labels = true + scrape_interval = "5m" scrape_timeout = "30s" } prometheus.scrape "prometheus_pushgateway" { - targets = discovery.relabel.prometheus_pushgateway.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "prometheus-pushgateway" + targets = discovery.relabel.prometheus_pushgateway.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "prometheus-pushgateway" honor_labels = true } prometheus.scrape "kubernetes_services" { - targets = discovery.relabel.kubernetes_services.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-services" + targets = discovery.relabel.kubernetes_services.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-services" honor_labels = true - params = { - module = ["http_2xx"], - } + params = { module = ["http_2xx"] } metrics_path = "/probe" } prometheus.scrape "kubernetes_pods" { - targets = discovery.relabel.kubernetes_pods.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-pods" + targets = discovery.relabel.kubernetes_pods.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-pods" honor_labels = true } prometheus.scrape "kubernetes_pods_slow" { - targets = discovery.relabel.kubernetes_pods_slow.output - forward_to = [prometheus.remote_write.default.receiver] - job_name = "kubernetes-pods-slow" - honor_labels = true - scrape_interval = "5m0s" + targets = discovery.relabel.kubernetes_pods_slow.output + forward_to = [ + prometheus.relabel.global_metric_filter.receiver, + prometheus.remote_write.default.receiver + ] + job_name = "kubernetes-pods-slow" + honor_labels = true + scrape_interval = "5m" scrape_timeout = "30s" }