from __future__ import annotations import os import tempfile from ._services_common import _LogFn, _detect_mode, _helm, _kubectl, _log, _namespace, _to_bool def _monitoring_namespace(namespace: str | None, env: dict | None) -> str: if env: explicit = str(env.get("MONITORING_NAMESPACE") or "").strip() if explicit: return explicit return _namespace(namespace, env, default="monitoring") def _release_name(env: dict | None) -> str: return str((env or {}).get("MONITORING_RELEASE") or "prometheus") def initialize( *, namespace: str | None = None, env: dict | None = None, log: _LogFn | None = None, mode: str | None = None, ) -> None: update(namespace=namespace, env=env, log=log, mode=mode) def start( *, namespace: str | None = None, env: dict | None = None, log: _LogFn | None = None, mode: str | None = None, ) -> None: update(namespace=namespace, env=env, log=log, mode=mode) def _values_yaml_k8s(grafana_password: str) -> str: """Helm values for GKE / cloud-managed Kubernetes (no local PV affinity).""" return ( "prometheus:\n" " prometheusSpec:\n" " storageSpec:\n" " volumeClaimTemplate:\n" " spec:\n" " storageClassName: standard\n" " accessModes: [ReadWriteOnce]\n" " resources:\n" " requests:\n" " storage: 10Gi\n" "alertmanager:\n" " alertmanagerSpec:\n" " storage:\n" " volumeClaimTemplate:\n" " spec:\n" " storageClassName: standard\n" " accessModes: [ReadWriteOnce]\n" " resources:\n" " requests:\n" " storage: 5Gi\n" "grafana:\n" " adminUser: admin\n" f" adminPassword: {grafana_password}\n" " persistence:\n" " type: sts\n" " enabled: true\n" " storageClassName: standard\n" " accessModes: [ReadWriteOnce]\n" " size: 5Gi\n" ) def _values_yaml_k3s(grafana_password: str, env: dict | None) -> str: """Helm values for k3s homelab (local iSCSI PVs pinned to merlin.prole.org).""" data_dir = str((env or {}).get("PROLE_MONITORING_DATA_DIR") or "/synology/d004").rstrip("/") volume_id = os.path.basename(data_dir) # e.g. "d004" sc_prom = f"merlin-local-iscsi-{volume_id}-prometheus" sc_alert = f"merlin-local-iscsi-{volume_id}-alertmanager" sc_grafana = f"merlin-local-iscsi-{volume_id}-grafana" monitoring_node = str((env or {}).get("MONITORING_PRIMARY_NODE") or "merlin.prole.org") excluded_nodes = str((env or {}).get("MONITORING_NODE_EXPORTER_EXCLUDE_NODES") or "pi.prole.org") excluded_list = [n.strip() for n in excluded_nodes.split(",") if n.strip()] node_affinity_yaml = ( " affinity:\n" " nodeAffinity:\n" " requiredDuringSchedulingIgnoredDuringExecution:\n" " nodeSelectorTerms:\n" " - matchExpressions:\n" " - key: kubernetes.io/hostname\n" " operator: In\n" " values:\n" f" - {monitoring_node}\n" ) values = "prometheus-node-exporter:\n" if excluded_list: values += ( " affinity:\n" " nodeAffinity:\n" " requiredDuringSchedulingIgnoredDuringExecution:\n" " nodeSelectorTerms:\n" " - matchExpressions:\n" " - key: kubernetes.io/hostname\n" " operator: NotIn\n" " values:\n" ) for node in excluded_list: values += f" - {node}\n" values += ( "prometheus:\n" " prometheusSpec:\n" + node_affinity_yaml + " storageSpec:\n" " volumeClaimTemplate:\n" " spec:\n" f" storageClassName: {sc_prom}\n" " accessModes: [ReadWriteOnce]\n" " resources:\n" " requests:\n" " storage: 30Gi\n" ) values += ( "alertmanager:\n" " alertmanagerSpec:\n" + node_affinity_yaml + " storage:\n" " volumeClaimTemplate:\n" " spec:\n" f" storageClassName: {sc_alert}\n" " accessModes: [ReadWriteOnce]\n" " resources:\n" " requests:\n" " storage: 5Gi\n" ) values += ( "grafana:\n" " adminUser: admin\n" f" adminPassword: {grafana_password}\n" " persistence:\n" " type: sts\n" " enabled: true\n" f" storageClassName: {sc_grafana}\n" " accessModes: [ReadWriteOnce]\n" " size: 10Gi\n" ) return values def update( *, namespace: str | None = None, env: dict | None = None, log: _LogFn | None = None, mode: str | None = None, ) -> None: effective_mode = _detect_mode(mode, env) ns = _monitoring_namespace(namespace, env) release = _release_name(env) chart = str((env or {}).get("MONITORING_CHART") or "prometheus-community/kube-prometheus-stack") grafana_password = str((env or {}).get("GRAFANA_ADMIN_PASSWORD") or "prole") _log(log, "[MONITORING] Ensuring helm repos") _helm(["repo", "add", "prometheus-community", "https://prometheus-community.github.io/helm-charts"], env=env) _helm(["repo", "update"], env=env) _kubectl(["create", "namespace", ns], env=env, timeout=60) if effective_mode == "k8s": values_yaml = _values_yaml_k8s(grafana_password) else: # k3s / k3d: homelab local-PV setup values_yaml = _values_yaml_k3s(grafana_password, env) tmp_values = tempfile.NamedTemporaryFile( mode="w", suffix=".yaml", prefix="monitoring-values-", delete=False ) try: tmp_values.write(values_yaml) tmp_values.flush() tmp_values.close() _log(log, f"[MONITORING] Deploying {release} in namespace {ns}") _helm( [ "upgrade", "--install", release, chart, "--namespace", ns, "-f", tmp_values.name, "--wait", ], env=env, timeout=600, check=True, ) # Cross-cluster: install Prometheus Operator CRDs on the DB cluster so that # CNPG (running on knoe-cnpg-0) can create PodMonitor resources without # the reconciler looping on "PodMonitor CRD not present". db_ctx = str((env or {}).get("DB_CLUSTER_KUBECONTEXT") or "").strip() if db_ctx and effective_mode == "k8s": _log(log, f"[MONITORING] Installing Prometheus Operator CRDs on DB cluster ({db_ctx})") # helm show crds is client-side — clear KUBECONTEXT so no --kube-context flag crd_env = {**(env or {}), "KUBECONTEXT": ""} crd_res = _helm(["show", "crds", chart], env=crd_env, timeout=60) if crd_res.returncode == 0 and crd_res.stdout.strip(): db_env = {**(env or {}), "KUBECONTEXT": db_ctx} _kubectl( ["apply", "--server-side", "-f", "-"], env=db_env, input_text=crd_res.stdout, timeout=120, ) _log(log, "[MONITORING] Prometheus Operator CRDs installed on DB cluster.") else: _log(log, "[MONITORING] WARNING: could not fetch CRDs from chart — skipping DB cluster CRD install.") finally: os.unlink(tmp_values.name) def restart( *, namespace: str | None = None, env: dict | None = None, log: _LogFn | None = None, ) -> None: ns = _monitoring_namespace(namespace, env) _log(log, f"[MONITORING] Restarting grafana deployment in namespace {ns}") _kubectl(["-n", ns, "rollout", "restart", "deployment/prometheus-grafana"], env=env, timeout=180) def stop( *, namespace: str | None = None, env: dict | None = None, log: _LogFn | None = None, ) -> None: ns = _monitoring_namespace(namespace, env) release = _release_name(env) _log(log, f"[MONITORING] Uninstalling release {release} in namespace {ns}") _helm(["uninstall", release, "--namespace", ns], env=env, timeout=180) def status( *, namespace: str | None = None, env: dict | None = None, ) -> bool: ns = _monitoring_namespace(namespace, env) dep = _kubectl( [ "-n", ns, "get", "deployment", "prometheus-grafana", "-o", "jsonpath={.status.readyReplicas}", ], env=env, timeout=20, ) return _to_bool(dep.returncode == 0 and (dep.stdout or "0").strip() not in {"", "0"})