diff --git a/k8s-peterg/alloy/configmap.yaml b/k8s-peterg/alloy/configmap.yaml new file mode 100644 index 0000000..66d60c5 --- /dev/null +++ b/k8s-peterg/alloy/configmap.yaml @@ -0,0 +1,526 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: alloy-config +data: + config.alloy: |- + discovery.kubernetes "kubernetes_apiservers" { + role = "endpoints" + } + + discovery.kubernetes "kubernetes_nodes" { + role = "node" + } + + discovery.kubernetes "kubernetes_nodes_cadvisor" { + role = "node" + } + + discovery.kubernetes "kubernetes_service_endpoints" { + role = "endpoints" + } + + discovery.kubernetes "kubernetes_service_endpoints_slow" { + role = "endpoints" + } + + discovery.kubernetes "prometheus_pushgateway" { + role = "service" + } + + discovery.kubernetes "kubernetes_services" { + role = "service" + } + + discovery.kubernetes "kubernetes_pods" { + role = "pod" + } + + discovery.kubernetes "kubernetes_pods_slow" { + role = "pod" + } + + discovery.relabel "kubernetes_apiservers" { + targets = discovery.kubernetes.kubernetes_apiservers.targets + + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"] + regex = "default;kubernetes;https" + action = "keep" + } + } + + discovery.relabel "kubernetes_nodes" { + targets = discovery.kubernetes.kubernetes_nodes.targets + + rule { + regex = "__meta_kubernetes_node_label_(.+)" + action = "labelmap" + } + + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc:443" + } + + rule { + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + target_label = "__metrics_path__" + replacement = "/api/v1/nodes/$1/proxy/metrics" + } + } + + discovery.relabel "kubernetes_nodes_cadvisor" { + targets = discovery.kubernetes.kubernetes_nodes_cadvisor.targets + + rule { + regex = "__meta_kubernetes_node_label_(.+)" + action = "labelmap" + } + + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc:443" + } + + rule { + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + target_label = "__metrics_path__" + replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" + } + } + + discovery.relabel "kubernetes_service_endpoints" { + targets = discovery.kubernetes.kubernetes_service_endpoints.targets + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"] + regex = "true" + action = "keep" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"] + regex = "true" + action = "drop" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"] + regex = "(https?)" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"] + regex = "(.+)" + target_label = "__metrics_path__" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"] + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + + rule { + regex = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)" + replacement = "__param_$1" + action = "labelmap" + } + + rule { + regex = "__meta_kubernetes_service_label_(.+)" + action = "labelmap" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_service_name"] + target_label = "service" + } + + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + } + + discovery.relabel "kubernetes_service_endpoints_slow" { + targets = discovery.kubernetes.kubernetes_service_endpoints_slow.targets + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape_slow"] + regex = "true" + action = "keep" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"] + regex = "(https?)" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"] + regex = "(.+)" + target_label = "__metrics_path__" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"] + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + + rule { + regex = "__meta_kubernetes_service_annotation_prometheus_io_param_(.+)" + replacement = "__param_$1" + action = "labelmap" + } + + rule { + regex = "__meta_kubernetes_service_label_(.+)" + action = "labelmap" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_service_name"] + target_label = "service" + } + + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + } + + discovery.relabel "prometheus_pushgateway" { + targets = discovery.kubernetes.prometheus_pushgateway.targets + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"] + regex = "pushgateway" + action = "keep" + } + } + + discovery.relabel "kubernetes_services" { + targets = discovery.kubernetes.kubernetes_services.targets + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_probe"] + regex = "true" + action = "keep" + } + + rule { + source_labels = ["__address__"] + target_label = "__param_target" + } + + rule { + target_label = "__address__" + replacement = "blackbox" + } + + rule { + source_labels = ["__param_target"] + target_label = "instance" + } + + rule { + regex = "__meta_kubernetes_service_label_(.+)" + action = "labelmap" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_service_name"] + target_label = "service" + } + } + + discovery.relabel "kubernetes_pods" { + targets = discovery.kubernetes.kubernetes_pods.targets + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"] + regex = "true" + action = "keep" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"] + regex = "true" + action = "drop" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"] + regex = "(https?)" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"] + regex = "(.+)" + target_label = "__metrics_path__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" + target_label = "__address__" + replacement = "[$2]:$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" + target_label = "__address__" + replacement = "$2:$1" + } + + rule { + regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)" + replacement = "__param_$1" + action = "labelmap" + } + + rule { + regex = "__meta_kubernetes_pod_label_(.+)" + action = "labelmap" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Completed" + action = "drop" + } + + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + } + + discovery.relabel "kubernetes_pods_slow" { + targets = discovery.kubernetes.kubernetes_pods_slow.targets + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow"] + regex = "true" + action = "keep" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"] + regex = "(https?)" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"] + regex = "(.+)" + target_label = "__metrics_path__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" + target_label = "__address__" + replacement = "[$2]:$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" + target_label = "__address__" + replacement = "$2:$1" + } + + rule { + regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)" + replacement = "__param_$1" + action = "labelmap" + } + + rule { + regex = "__meta_kubernetes_pod_label_(.+)" + action = "labelmap" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Completed" + action = "drop" + } + + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + } + + prometheus.scrape "prometheus" { + targets = [{ + __address__ = "localhost:9090", + }] + forward_to = [prometheus.remote_write.default.receiver] + job_name = "prometheus" + } + + prometheus.scrape "kubernetes_apiservers" { + targets = discovery.relabel.kubernetes_apiservers.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-apiservers" + scheme = "https" + + authorization { + type = "Bearer" + credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + } + + prometheus.scrape "kubernetes_nodes" { + targets = discovery.relabel.kubernetes_nodes.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-nodes" + scheme = "https" + + authorization { + type = "Bearer" + credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + } + + prometheus.scrape "kubernetes_nodes_cadvisor" { + targets = discovery.relabel.kubernetes_nodes_cadvisor.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-nodes-cadvisor" + scheme = "https" + + authorization { + type = "Bearer" + credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + } + + prometheus.scrape "kubernetes_service_endpoints" { + targets = discovery.relabel.kubernetes_service_endpoints.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-service-endpoints" + honor_labels = true + } + + prometheus.scrape "kubernetes_service_endpoints_slow" { + targets = discovery.relabel.kubernetes_service_endpoints_slow.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-service-endpoints-slow" + honor_labels = true + scrape_interval = "5m0s" + scrape_timeout = "30s" + } + + prometheus.scrape "prometheus_pushgateway" { + targets = discovery.relabel.prometheus_pushgateway.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "prometheus-pushgateway" + honor_labels = true + } + + prometheus.scrape "kubernetes_services" { + targets = discovery.relabel.kubernetes_services.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-services" + honor_labels = true + params = { + module = ["http_2xx"], + } + metrics_path = "/probe" + } + + prometheus.scrape "kubernetes_pods" { + targets = discovery.relabel.kubernetes_pods.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-pods" + honor_labels = true + } + + prometheus.scrape "kubernetes_pods_slow" { + targets = discovery.relabel.kubernetes_pods_slow.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "kubernetes-pods-slow" + honor_labels = true + scrape_interval = "5m0s" + scrape_timeout = "30s" + } + + prometheus.remote_write "default" { + external_labels = { + cluster = "k8s-peterg", + } + + endpoint { + url = "https://mimir.peterg.nl/api/v1/push" + + headers = { + "X-Scope-OrgID" = "wheatley", + } + } + } + diff --git a/k8s-peterg/alloy/kustomization.yaml b/k8s-peterg/alloy/kustomization.yaml new file mode 100644 index 0000000..03bf1a0 --- /dev/null +++ b/k8s-peterg/alloy/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../kustomize-bases/alloy + - configmap.yaml diff --git a/kustomize-bases/alloy/kustomization.yaml b/kustomize-bases/alloy/kustomization.yaml new file mode 100644 index 0000000..fa87798 --- /dev/null +++ b/kustomize-bases/alloy/kustomization.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: alloy + +resources: + - namespace.yaml + +helmCharts: + - name: alloy + repo: https://grafana.github.io/helm-charts + version: "1.4.0" + releaseName: alloy + valuesFile: values.yaml diff --git a/kustomize-bases/alloy/namespace.yaml b/kustomize-bases/alloy/namespace.yaml new file mode 100644 index 0000000..ef44880 --- /dev/null +++ b/kustomize-bases/alloy/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: alloy diff --git a/kustomize-bases/alloy/values.yaml b/kustomize-bases/alloy/values.yaml new file mode 100644 index 0000000..59e3dae --- /dev/null +++ b/kustomize-bases/alloy/values.yaml @@ -0,0 +1,115 @@ +crds: + # -- Whether to install CRDs for monitoring. + create: true + +## Various Alloy settings. For backwards compatibility with the grafana-agent +## chart, this field may also be called "agent". Naming this field "agent" is +## deprecated and will be removed in a future release. +alloy: + configMap: + # -- Create a new ConfigMap for the config file. + create: false + + # -- Name of existing ConfigMap to use. Used when create is false. + name: alloy-config + # -- Key in ConfigMap to get config from. + key: config.alloy + +rbac: + # -- Whether to create RBAC resources for Alloy. + create: true + + # -- The rules to create for the ClusterRole or Role objects. + rules: + # -- Rules required for the `discovery.kubernetes` component. + - apiGroups: ["", "discovery.k8s.io", "networking.k8s.io"] + resources: ["endpoints", "endpointslices", "ingresses", "pods", "services"] + verbs: ["get", "list", "watch"] + # -- Rules required for the `loki.source.kubernetes` component. + - apiGroups: [""] + resources: ["pods", "pods/log", "namespaces"] + verbs: ["get", "list", "watch"] + # -- Rules required for the `loki.source.podlogs` component. + - apiGroups: ["monitoring.grafana.com"] + resources: ["podlogs"] + verbs: ["get", "list", "watch"] + # -- Rules required for the `mimir.rules.kubernetes` component. + - apiGroups: ["monitoring.coreos.com"] + resources: ["prometheusrules"] + verbs: ["get", "list", "watch"] + # -- Rules required for the `prometheus.operator.*` components. + - apiGroups: ["monitoring.coreos.com"] + resources: ["podmonitors", "servicemonitors", "probes", "scrapeconfigs"] + verbs: ["get", "list", "watch"] + # -- Rules required for the `loki.source.kubernetes_events` component. + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch"] + # -- Rules required for the `remote.kubernetes.*` components. + - apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get", "list", "watch"] + # -- Rules required for the `otelcol.processor.k8sattributes` component. + - apiGroups: ["apps", "extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] + + # -- The rules to create for the ClusterRole objects. + clusterRules: + # -- Rules required for the `discovery.kubernetes` component. + - apiGroups: [""] + resources: ["nodes", "nodes/proxy", "nodes/metrics"] + verbs: ["get", "list", "watch"] + # -- Rules required for accessing metrics endpoint. + - nonResourceURLs: ["/metrics"] + verbs: ["get"] + +serviceAccount: + # -- Whether to create a service account for the Grafana Alloy deployment. + create: true + # -- Additional labels to add to the created service account. + additionalLabels: {} + # -- Annotations to add to the created service account. + annotations: {} + # -- The name of the existing service account to use when + # serviceAccount.create is false. + name: null + # Whether the Alloy pod should automatically mount the service account token. + automountServiceAccountToken: true + +# Options for the extra controller used for config reloading. +configReloader: + # -- Enables automatically reloading when the Alloy config changes. + enabled: true + image: + # -- Config reloader image registry (defaults to docker.io) + registry: "quay.io" + # -- Repository to get config reloader image from. + repository: prometheus-operator/prometheus-config-reloader + # -- Tag of image to use for config reloading. + tag: v0.81.0 + # -- SHA256 digest of image to use for config reloading (either in format "sha256:XYZ" or "XYZ"). When set, will override `configReloader.image.tag` + digest: "" + # -- Override the args passed to the container. + customArgs: [] + # -- Resource requests and limits to apply to the config reloader container. + resources: + requests: + cpu: "10m" + memory: "50Mi" + # -- Security context to apply to the Grafana configReloader container. + securityContext: {} + +service: + # -- Creates a Service for the controller's pods. + enabled: true + # -- Service type + type: ClusterIP + # -- NodePort port. Only takes effect when `service.type: NodePort` + nodePort: 31128 + # -- Cluster IP, can be set to None, empty "" or an IP address + clusterIP: '' + # -- Value for internal traffic policy. 'Cluster' or 'Local' + internalTrafficPolicy: Cluster + annotations: {} + # cloud.google.com/load-balancer-type: Internal