prole/knoe/core/ops/monitoring.py

from __future__ import annotations

import os
import tempfile

from ._services_common import _LogFn, _detect_mode, _helm, _kubectl, _log, _namespace, _to_bool


def _monitoring_namespace(namespace: str | None, env: dict | None) -> str:
    if env:
        explicit = str(env.get("MONITORING_NAMESPACE") or "").strip()
        if explicit:
            return explicit
    return _namespace(namespace, env, default="monitoring")


def _release_name(env: dict | None) -> str:
    return str((env or {}).get("MONITORING_RELEASE") or "prometheus")


def initialize(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
    mode: str | None = None,
) -> None:
    update(namespace=namespace, env=env, log=log, mode=mode)


def start(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
    mode: str | None = None,
) -> None:
    update(namespace=namespace, env=env, log=log, mode=mode)


def update(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
    mode: str | None = None,
) -> None:
    _detect_mode(mode, env)  # mode retained for parity with other owners
    ns = _monitoring_namespace(namespace, env)
    release = _release_name(env)
    chart = str((env or {}).get("MONITORING_CHART") or "prometheus-community/kube-prometheus-stack")
    grafana_password = str((env or {}).get("GRAFANA_ADMIN_PASSWORD") or "prole")

    _log(log, "[MONITORING] Ensuring helm repos")
    _helm(["repo", "add", "prometheus-community", "https://prometheus-community.github.io/helm-charts"], env=env)
    _helm(["repo", "update"], env=env)

    _kubectl(["create", "namespace", ns], env=env, timeout=60)

    # Nodes with broken kubelet (e.g. pi.prole.org returning 502) must be excluded
    # from the node-exporter DaemonSet so helm --wait can succeed.
    excluded_nodes = str((env or {}).get("MONITORING_NODE_EXPORTER_EXCLUDE_NODES") or "pi.prole.org")
    excluded_list = [n.strip() for n in excluded_nodes.split(",") if n.strip()]

    values_yaml = "prometheus-node-exporter:\n"
    if excluded_list:
        values_yaml += (
            "  affinity:\n"
            "    nodeAffinity:\n"
            "      requiredDuringSchedulingIgnoredDuringExecution:\n"
            "        nodeSelectorTerms:\n"
            "        - matchExpressions:\n"
            "          - key: kubernetes.io/hostname\n"
            "            operator: NotIn\n"
            "            values:\n"
        )
        for node in excluded_list:
            values_yaml += f"            - {node}\n"

    tmp_values = tempfile.NamedTemporaryFile(
        mode="w", suffix=".yaml", prefix="monitoring-values-", delete=False
    )
    try:
        tmp_values.write(values_yaml)
        tmp_values.flush()
        tmp_values.close()

        _log(log, f"[MONITORING] Deploying {release} in namespace {ns}")
        _helm(
            [
                "upgrade",
                "--install",
                release,
                chart,
                "--namespace",
                ns,
                "--set",
                "grafana.adminUser=admin",
                "--set",
                f"grafana.adminPassword={grafana_password}",
                "-f",
                tmp_values.name,
                "--wait",
            ],
            env=env,
            timeout=600,
            check=True,
        )
    finally:
        os.unlink(tmp_values.name)


def restart(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
) -> None:
    ns = _monitoring_namespace(namespace, env)
    _log(log, f"[MONITORING] Restarting grafana deployment in namespace {ns}")
    _kubectl(["-n", ns, "rollout", "restart", "deployment/prometheus-grafana"], env=env, timeout=180)


def stop(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
) -> None:
    ns = _monitoring_namespace(namespace, env)
    release = _release_name(env)
    _log(log, f"[MONITORING] Uninstalling release {release} in namespace {ns}")
    _helm(["uninstall", release, "--namespace", ns], env=env, timeout=180)


def status(
    *,
    namespace: str | None = None,
    env: dict | None = None,
) -> bool:
    ns = _monitoring_namespace(namespace, env)
    dep = _kubectl(
        [
            "-n",
            ns,
            "get",
            "deployment",
            "prometheus-grafana",
            "-o",
            "jsonpath={.status.readyReplicas}",
        ],
        env=env,
        timeout=20,
    )
    return _to_bool(dep.returncode == 0 and (dep.stdout or "0").strip() not in {"", "0"})