feat: Set up Alloy

This commit is contained in:
Peter 2025-12-14 15:33:14 +01:00
parent 572ec08b9a
commit bae2fc8c1e
Signed by: Peter
SSH key fingerprint: SHA256:B5tYaxBExaDm74r1px9iVeZ6F/ZDiyiy9SbBqfZYrvg
5 changed files with 667 additions and 0 deletions

View file

@ -0,0 +1,526 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: alloy-config
data:
config.alloy: |-
discovery.kubernetes "kubernetes_apiservers" {
role = "endpoints"
}
discovery.kubernetes "kubernetes_nodes" {
role = "node"
}
discovery.kubernetes "kubernetes_nodes_cadvisor" {
role = "node"
}
discovery.kubernetes "kubernetes_service_endpoints" {
role = "endpoints"
}
discovery.kubernetes "kubernetes_service_endpoints_slow" {
role = "endpoints"
}
discovery.kubernetes "prometheus_pushgateway" {
role = "service"
}
discovery.kubernetes "kubernetes_services" {
role = "service"
}
discovery.kubernetes "kubernetes_pods" {
role = "pod"
}
discovery.kubernetes "kubernetes_pods_slow" {
role = "pod"
}
discovery.relabel "kubernetes_apiservers" {
targets = discovery.kubernetes.kubernetes_apiservers.targets
rule {
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"]
regex = "default;kubernetes;https"
action = "keep"
}
}
discovery.relabel "kubernetes_nodes" {
targets = discovery.kubernetes.kubernetes_nodes.targets
rule {
regex = "__meta_kubernetes_node_label_(.+)"
action = "labelmap"
}
rule {
target_label = "__address__"
replacement = "kubernetes.default.svc:443"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "__metrics_path__"
replacement = "/api/v1/nodes/$1/proxy/metrics"
}
}
discovery.relabel "kubernetes_nodes_cadvisor" {
targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets
rule {
regex = "__meta_kubernetes_node_label_(.+)"
action = "labelmap"
}
rule {
target_label = "__address__"
replacement = "kubernetes.default.svc:443"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "__metrics_path__"
replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
}
}
discovery.relabel "kubernetes_service_endpoints" {
targets = discovery.kubernetes.kubernetes_service_endpoints.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
regex = "(.+?)(?::\\d+)?;(\\d+)"
target_label = "__address__"
replacement = "$1:$2"
}
rule {
regex = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_service_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_service_name"]
target_label = "service"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
discovery.relabel "kubernetes_service_endpoints_slow" {
targets = discovery.kubernetes.kubernetes_service_endpoints_slow.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
regex = "(.+?)(?::\\d+)?;(\\d+)"
target_label = "__address__"
replacement = "$1:$2"
}
rule {
regex = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_service_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_service_name"]
target_label = "service"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
discovery.relabel "prometheus_pushgateway" {
targets = discovery.kubernetes.prometheus_pushgateway.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
regex = "pushgateway"
action = "keep"
}
}
discovery.relabel "kubernetes_services" {
targets = discovery.kubernetes.kubernetes_services.targets
rule {
source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__address__"]
target_label = "__param_target"
}
rule {
target_label = "__address__"
replacement = "blackbox"
}
rule {
source_labels = ["__param_target"]
target_label = "instance"
}
rule {
regex = "__meta_kubernetes_service_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_service_name"]
target_label = "service"
}
}
discovery.relabel "kubernetes_pods" {
targets = discovery.kubernetes.kubernetes_pods.targets
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
target_label = "__address__"
replacement = "[$2]:$1"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);((([0-9]+?)(\\.|$)){4})"
target_label = "__address__"
replacement = "$2:$1"
}
rule {
regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_pod_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_phase"]
regex = "Pending|Succeeded|Failed|Completed"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
discovery.relabel "kubernetes_pods_slow" {
targets = discovery.kubernetes.kubernetes_pods_slow.targets
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"]
regex = "true"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"]
regex = "(https?)"
target_label = "__scheme__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
target_label = "__address__"
replacement = "[$2]:$1"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"]
regex = "(\\d+);((([0-9]+?)(\\.|$)){4})"
target_label = "__address__"
replacement = "$2:$1"
}
rule {
regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)"
replacement = "__param_$1"
action = "labelmap"
}
rule {
regex = "__meta_kubernetes_pod_label_(.+)"
action = "labelmap"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_phase"]
regex = "Pending|Succeeded|Failed|Completed"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
}
prometheus.scrape "prometheus" {
targets = [{
__address__ = "localhost:9090",
}]
forward_to = [prometheus.remote_write.default.receiver]
job_name = "prometheus"
}
prometheus.scrape "kubernetes_apiservers" {
targets = discovery.relabel.kubernetes_apiservers.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-apiservers"
scheme = "https"
authorization {
type = "Bearer"
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
}
prometheus.scrape "kubernetes_nodes" {
targets = discovery.relabel.kubernetes_nodes.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-nodes"
scheme = "https"
authorization {
type = "Bearer"
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
}
prometheus.scrape "kubernetes_nodes_cadvisor" {
targets = discovery.relabel.kubernetes_nodes_cadvisor.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-nodes-cadvisor"
scheme = "https"
authorization {
type = "Bearer"
credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
}
prometheus.scrape "kubernetes_service_endpoints" {
targets = discovery.relabel.kubernetes_service_endpoints.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-service-endpoints"
honor_labels = true
}
prometheus.scrape "kubernetes_service_endpoints_slow" {
targets = discovery.relabel.kubernetes_service_endpoints_slow.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-service-endpoints-slow"
honor_labels = true
scrape_interval = "5m0s"
scrape_timeout = "30s"
}
prometheus.scrape "prometheus_pushgateway" {
targets = discovery.relabel.prometheus_pushgateway.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "prometheus-pushgateway"
honor_labels = true
}
prometheus.scrape "kubernetes_services" {
targets = discovery.relabel.kubernetes_services.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-services"
honor_labels = true
params = {
module = ["http_2xx"],
}
metrics_path = "/probe"
}
prometheus.scrape "kubernetes_pods" {
targets = discovery.relabel.kubernetes_pods.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-pods"
honor_labels = true
}
prometheus.scrape "kubernetes_pods_slow" {
targets = discovery.relabel.kubernetes_pods_slow.output
forward_to = [prometheus.remote_write.default.receiver]
job_name = "kubernetes-pods-slow"
honor_labels = true
scrape_interval = "5m0s"
scrape_timeout = "30s"
}
prometheus.remote_write "default" {
external_labels = {
cluster = "k8s-peterg",
}
endpoint {
url = "https://mimir.peterg.nl/api/v1/push"
headers = {
"X-Scope-OrgID" = "wheatley",
}
}
}

View file

@ -0,0 +1,7 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../kustomize-bases/alloy
- configmap.yaml

View file

@ -0,0 +1,14 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: alloy
resources:
- namespace.yaml
helmCharts:
- name: alloy
repo: https://grafana.github.io/helm-charts
version: "1.4.0"
releaseName: alloy
valuesFile: values.yaml

View file

@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: alloy

View file

@ -0,0 +1,115 @@
crds:
# -- Whether to install CRDs for monitoring.
create: true
## Various Alloy settings. For backwards compatibility with the grafana-agent
## chart, this field may also be called "agent". Naming this field "agent" is
## deprecated and will be removed in a future release.
alloy:
configMap:
# -- Create a new ConfigMap for the config file.
create: false
# -- Name of existing ConfigMap to use. Used when create is false.
name: alloy-config
# -- Key in ConfigMap to get config from.
key: config.alloy
rbac:
# -- Whether to create RBAC resources for Alloy.
create: true
# -- The rules to create for the ClusterRole or Role objects.
rules:
# -- Rules required for the `discovery.kubernetes` component.
- apiGroups: ["", "discovery.k8s.io", "networking.k8s.io"]
resources: ["endpoints", "endpointslices", "ingresses", "pods", "services"]
verbs: ["get", "list", "watch"]
# -- Rules required for the `loki.source.kubernetes` component.
- apiGroups: [""]
resources: ["pods", "pods/log", "namespaces"]
verbs: ["get", "list", "watch"]
# -- Rules required for the `loki.source.podlogs` component.
- apiGroups: ["monitoring.grafana.com"]
resources: ["podlogs"]
verbs: ["get", "list", "watch"]
# -- Rules required for the `mimir.rules.kubernetes` component.
- apiGroups: ["monitoring.coreos.com"]
resources: ["prometheusrules"]
verbs: ["get", "list", "watch"]
# -- Rules required for the `prometheus.operator.*` components.
- apiGroups: ["monitoring.coreos.com"]
resources: ["podmonitors", "servicemonitors", "probes", "scrapeconfigs"]
verbs: ["get", "list", "watch"]
# -- Rules required for the `loki.source.kubernetes_events` component.
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch"]
# -- Rules required for the `remote.kubernetes.*` components.
- apiGroups: [""]
resources: ["configmaps", "secrets"]
verbs: ["get", "list", "watch"]
# -- Rules required for the `otelcol.processor.k8sattributes` component.
- apiGroups: ["apps", "extensions"]
resources: ["replicasets"]
verbs: ["get", "list", "watch"]
# -- The rules to create for the ClusterRole objects.
clusterRules:
# -- Rules required for the `discovery.kubernetes` component.
- apiGroups: [""]
resources: ["nodes", "nodes/proxy", "nodes/metrics"]
verbs: ["get", "list", "watch"]
# -- Rules required for accessing metrics endpoint.
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
serviceAccount:
# -- Whether to create a service account for the Grafana Alloy deployment.
create: true
# -- Additional labels to add to the created service account.
additionalLabels: {}
# -- Annotations to add to the created service account.
annotations: {}
# -- The name of the existing service account to use when
# serviceAccount.create is false.
name: null
# Whether the Alloy pod should automatically mount the service account token.
automountServiceAccountToken: true
# Options for the extra controller used for config reloading.
configReloader:
# -- Enables automatically reloading when the Alloy config changes.
enabled: true
image:
# -- Config reloader image registry (defaults to docker.io)
registry: "quay.io"
# -- Repository to get config reloader image from.
repository: prometheus-operator/prometheus-config-reloader
# -- Tag of image to use for config reloading.
tag: v0.81.0
# -- SHA256 digest of image to use for config reloading (either in format "sha256:XYZ" or "XYZ"). When set, will override `configReloader.image.tag`
digest: ""
# -- Override the args passed to the container.
customArgs: []
# -- Resource requests and limits to apply to the config reloader container.
resources:
requests:
cpu: "10m"
memory: "50Mi"
# -- Security context to apply to the Grafana configReloader container.
securityContext: {}
service:
# -- Creates a Service for the controller's pods.
enabled: true
# -- Service type
type: ClusterIP
# -- NodePort port. Only takes effect when `service.type: NodePort`
nodePort: 31128
# -- Cluster IP, can be set to None, empty "" or an IP address
clusterIP: ''
# -- Value for internal traffic policy. 'Cluster' or 'Local'
internalTrafficPolicy: Cluster
annotations: {}
# cloud.google.com/load-balancer-type: Internal