484 lines
13 KiB
YAML
484 lines
13 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: alloy-config
|
|
data:
|
|
config.alloy: |-
|
|
prometheus.exporter.unix "node" {
|
|
set_collectors = [
|
|
"cpu", "diskstats", "filesystem", "loadavg",
|
|
"meminfo", "netdev", "netstat", "os",
|
|
"pressure", "processes", "stat", "uname", "vmstat",
|
|
]
|
|
}
|
|
|
|
discovery.kubernetes "kubernetes_apiservers" {
|
|
role = "endpoints"
|
|
}
|
|
|
|
discovery.kubernetes "kubernetes_nodes" {
|
|
role = "node"
|
|
}
|
|
|
|
discovery.kubernetes "kubernetes_nodes_cadvisor" {
|
|
role = "node"
|
|
}
|
|
|
|
discovery.kubernetes "kubernetes_service_endpoints" {
|
|
role = "endpoints"
|
|
}
|
|
|
|
discovery.kubernetes "kubernetes_services" {
|
|
role = "service"
|
|
}
|
|
|
|
discovery.kubernetes "kubernetes_pods" {
|
|
role = "pod"
|
|
}
|
|
|
|
discovery.kubernetes "pod_logs" {
|
|
role = "pod"
|
|
}
|
|
|
|
discovery.relabel "kubernetes_apiservers" {
|
|
targets = discovery.kubernetes.kubernetes_apiservers.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"]
|
|
regex = "default;kubernetes;https"
|
|
action = "keep"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "kubernetes_nodes" {
|
|
targets = discovery.kubernetes.kubernetes_nodes.targets
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_node_label_(.+)"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
target_label = "__address__"
|
|
replacement = "kubernetes.default.svc:443"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_node_name"]
|
|
regex = "(.+)"
|
|
target_label = "__metrics_path__"
|
|
replacement = "/api/v1/nodes/$1/proxy/metrics"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "kubernetes_nodes_cadvisor" {
|
|
targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_node_label_(.+)"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
target_label = "__address__"
|
|
replacement = "kubernetes.default.svc:443"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_node_name"]
|
|
regex = "(.+)"
|
|
target_label = "__metrics_path__"
|
|
replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "kubernetes_service_endpoints" {
|
|
targets = discovery.kubernetes.kubernetes_service_endpoints.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"]
|
|
regex = "true"
|
|
action = "keep"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
|
|
regex = "(https?)"
|
|
target_label = "__scheme__"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
|
|
regex = "(.+)"
|
|
target_label = "__metrics_path__"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
|
|
regex = "(.+?)(?::\\d+)?;(\\d+)"
|
|
target_label = "__address__"
|
|
replacement = "$1:$2"
|
|
}
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
|
|
replacement = "__param_$1"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_service_label_(.+)"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace"]
|
|
target_label = "namespace"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_name"]
|
|
target_label = "service"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
|
target_label = "node"
|
|
}
|
|
|
|
rule {
|
|
regex = "pod_template_hash|controller_revision_hash|deployment_kubernetes_io_revision"
|
|
action = "labeldrop"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "kubernetes_services" {
|
|
targets = discovery.kubernetes.kubernetes_services.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
|
|
regex = "true"
|
|
action = "keep"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__address__"]
|
|
target_label = "__param_target"
|
|
}
|
|
|
|
rule {
|
|
target_label = "__address__"
|
|
replacement = "blackbox"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__param_target"]
|
|
target_label = "instance"
|
|
}
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_service_label_(.+)"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace"]
|
|
target_label = "namespace"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_name"]
|
|
target_label = "service"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "kubernetes_pods" {
|
|
targets = discovery.kubernetes.kubernetes_pods.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
|
|
regex = "true"
|
|
action = "keep"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
|
|
regex = "(https?)"
|
|
target_label = "__scheme__"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
|
|
regex = "(.+)"
|
|
target_label = "__metrics_path__"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
|
|
regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
|
|
target_label = "__address__"
|
|
replacement = "[$2]:$1"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
|
|
regex = "(\\d+);((([0-9]+?)(\\.|$)){4})"
|
|
target_label = "__address__"
|
|
replacement = "$2:$1"
|
|
}
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
|
|
replacement = "__param_$1"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_pod_label_(.+)"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace"]
|
|
target_label = "namespace"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_name"]
|
|
target_label = "pod"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_phase"]
|
|
regex = "Pending|Succeeded|Failed|Completed"
|
|
action = "drop"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
|
target_label = "node"
|
|
}
|
|
|
|
rule {
|
|
regex = "pod_template_hash|controller_revision_hash|deployment_kubernetes_io_revision"
|
|
action = "labeldrop"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "pod_logs" {
|
|
targets = discovery.kubernetes.pod_logs.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
|
action = "replace"
|
|
target_label = "__host__"
|
|
}
|
|
|
|
rule {
|
|
regex = "__meta_kubernetes_pod_label_(.+)"
|
|
action = "labelmap"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
|
|
action = "replace"
|
|
separator = "/"
|
|
target_label = "job"
|
|
replacement = "$1"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace"]
|
|
action = "replace"
|
|
target_label = "namespace"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_name"]
|
|
action = "replace"
|
|
target_label = "pod"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_container_name"]
|
|
action = "replace"
|
|
target_label = "container"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
|
|
action = "replace"
|
|
separator = "/"
|
|
target_label = "__path__"
|
|
replacement = "/var/log/pods/*$1/*.log"
|
|
}
|
|
}
|
|
|
|
local.file_match "pod_logs" {
|
|
path_targets = discovery.relabel.pod_logs.output
|
|
}
|
|
|
|
prometheus.scrape "alloy" {
|
|
targets = [{
|
|
__address__ = "localhost:12345",
|
|
}]
|
|
job_name = "alloy"
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
}
|
|
|
|
prometheus.scrape "node_exporter" {
|
|
targets = prometheus.exporter.unix.node.targets
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
job_name = "node-exporter"
|
|
}
|
|
|
|
prometheus.scrape "kubernetes_apiservers" {
|
|
targets = discovery.relabel.kubernetes_apiservers.output
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
job_name = "kubernetes-apiservers"
|
|
scheme = "https"
|
|
|
|
authorization {
|
|
type = "Bearer"
|
|
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
}
|
|
|
|
tls_config {
|
|
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
insecure_skip_verify = true
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "kubernetes_nodes" {
|
|
targets = discovery.relabel.kubernetes_nodes.output
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
job_name = "kubernetes-nodes"
|
|
scheme = "https"
|
|
|
|
authorization {
|
|
type = "Bearer"
|
|
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
}
|
|
|
|
tls_config {
|
|
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
insecure_skip_verify = true
|
|
}
|
|
}
|
|
|
|
prometheus.relabel "cadvisor" {
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
|
|
rule {
|
|
source_labels = ["__name__"]
|
|
regex = "container_(cpu_usage_seconds_total|memory_usage_bytes|memory_working_set_bytes|memory_rss|memory_cache|memory_swap|network_receive_bytes_total|network_transmit_bytes_total|network_receive_packets_total|network_transmit_packets_total|fs_reads_bytes_total|fs_writes_bytes_total|spec_cpu_quota|spec_cpu_period|spec_memory_limit_bytes|last_seen)"
|
|
action = "keep"
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "kubernetes_nodes_cadvisor" {
|
|
targets = discovery.relabel.kubernetes_nodes_cadvisor.output
|
|
forward_to = [prometheus.relabel.cadvisor.receiver]
|
|
job_name = "kubernetes-nodes-cadvisor"
|
|
scheme = "https"
|
|
|
|
authorization {
|
|
type = "Bearer"
|
|
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
}
|
|
|
|
tls_config {
|
|
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
insecure_skip_verify = true
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "kubernetes_service_endpoints" {
|
|
targets = discovery.relabel.kubernetes_service_endpoints.output
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
job_name = "kubernetes-service-endpoints"
|
|
honor_labels = true
|
|
}
|
|
|
|
prometheus.scrape "kubernetes_services" {
|
|
targets = discovery.relabel.kubernetes_services.output
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
job_name = "kubernetes-services"
|
|
honor_labels = true
|
|
params = {
|
|
module = ["http_2xx"],
|
|
}
|
|
metrics_path = "/probe"
|
|
}
|
|
|
|
prometheus.scrape "kubernetes_pods" {
|
|
targets = discovery.relabel.kubernetes_pods.output
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
job_name = "kubernetes-pods"
|
|
honor_labels = true
|
|
}
|
|
|
|
loki.process "pod_logs" {
|
|
forward_to = [loki.write.default.receiver]
|
|
|
|
stage.cri {}
|
|
stage.timestamp {
|
|
source = "timestamp"
|
|
format = "RFC3339Nano"
|
|
location = "UTC"
|
|
}
|
|
|
|
stage.static_labels {
|
|
values = {
|
|
cluster = "k8s-peterg",
|
|
}
|
|
}
|
|
|
|
stage.template {
|
|
source = "merged_cluster-namespace-container_string"
|
|
template = "k8s-peterg;{{`{{.namespace}};{{.container}}`}}"
|
|
}
|
|
}
|
|
|
|
loki.source.file "pod_logs" {
|
|
targets = local.file_match.pod_logs.targets
|
|
forward_to = [loki.process.pod_logs.receiver]
|
|
legacy_positions_file = "/tmp/positions.yaml"
|
|
}
|
|
|
|
local.file_match "auditlogs" {
|
|
path_targets = [{
|
|
__address__ = "localhost",
|
|
__path__ = "/var/log/audit/kube/kube-apiserver.log",
|
|
host = env("HOSTNAME"),
|
|
logtype = "audit",
|
|
}]
|
|
}
|
|
|
|
loki.source.file "auditlogs" {
|
|
targets = local.file_match.auditlogs.targets
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.write "default" {
|
|
endpoint {
|
|
url = "https://loki.peterg.nl/loki/api/v1/push"
|
|
tenant_id = "wheatley"
|
|
}
|
|
}
|
|
|
|
prometheus.remote_write "default" {
|
|
external_labels = {
|
|
cluster = "k8s-peterg",
|
|
node = env("HOSTNAME"),
|
|
}
|
|
|
|
endpoint {
|
|
url = "https://mimir.peterg.nl/api/v1/push"
|
|
|
|
headers = {
|
|
"X-Scope-OrgID" = "wheatley",
|
|
}
|
|
}
|
|
}
|