prole/knoe/core/ops/monitoring.py

from __future__ import annotations

import os
import tempfile

from ._services_common import _LogFn, _detect_mode, _helm, _kubectl, _log, _namespace, _to_bool


def _monitoring_namespace(namespace: str | None, env: dict | None) -> str:
    if env:
        explicit = str(env.get("MONITORING_NAMESPACE") or "").strip()
        if explicit:
            return explicit
    return _namespace(namespace, env, default="monitoring")


def _release_name(env: dict | None) -> str:
    return str((env or {}).get("MONITORING_RELEASE") or "prometheus")


def initialize(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
    mode: str | None = None,
) -> None:
    update(namespace=namespace, env=env, log=log, mode=mode)


def start(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
    mode: str | None = None,
) -> None:
    update(namespace=namespace, env=env, log=log, mode=mode)


def update(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
    mode: str | None = None,
) -> None:
    _detect_mode(mode, env)  # mode retained for parity with other owners
    ns = _monitoring_namespace(namespace, env)
    release = _release_name(env)
    chart = str((env or {}).get("MONITORING_CHART") or "prometheus-community/kube-prometheus-stack")
    grafana_password = str((env or {}).get("GRAFANA_ADMIN_PASSWORD") or "prole")

    # Derive storage class names from PROLE_MONITORING_DATA_DIR (same convention as init_monitoring.sh)
    data_dir = str((env or {}).get("PROLE_MONITORING_DATA_DIR") or "/synology/d004").rstrip("/")
    volume_id = os.path.basename(data_dir)  # e.g. "d004"
    sc_prom = f"merlin-local-iscsi-{volume_id}-prometheus"
    sc_alert = f"merlin-local-iscsi-{volume_id}-alertmanager"
    sc_grafana = f"merlin-local-iscsi-{volume_id}-grafana"

    # Primary monitoring node: prometheus and alertmanager must schedule here to bind local PVs
    monitoring_node = str((env or {}).get("MONITORING_PRIMARY_NODE") or "merlin.prole.org")

    _log(log, "[MONITORING] Ensuring helm repos")
    _helm(["repo", "add", "prometheus-community", "https://prometheus-community.github.io/helm-charts"], env=env)
    _helm(["repo", "update"], env=env)

    _kubectl(["create", "namespace", ns], env=env, timeout=60)

    # Nodes with broken kubelet (e.g. pi.prole.org returning 502) must be excluded
    # from the node-exporter DaemonSet so helm --wait can succeed.
    excluded_nodes = str((env or {}).get("MONITORING_NODE_EXPORTER_EXCLUDE_NODES") or "pi.prole.org")
    excluded_list = [n.strip() for n in excluded_nodes.split(",") if n.strip()]

    # Node affinity block for components that must land on the monitoring node (local PV binding)
    node_affinity_yaml = (
        "    affinity:\n"
        "      nodeAffinity:\n"
        "        requiredDuringSchedulingIgnoredDuringExecution:\n"
        "          nodeSelectorTerms:\n"
        "          - matchExpressions:\n"
        "            - key: kubernetes.io/hostname\n"
        "              operator: In\n"
        "              values:\n"
        f"              - {monitoring_node}\n"
    )

    # Node-exporter: exclude broken nodes
    values_yaml = "prometheus-node-exporter:\n"
    if excluded_list:
        values_yaml += (
            "  affinity:\n"
            "    nodeAffinity:\n"
            "      requiredDuringSchedulingIgnoredDuringExecution:\n"
            "        nodeSelectorTerms:\n"
            "        - matchExpressions:\n"
            "          - key: kubernetes.io/hostname\n"
            "            operator: NotIn\n"
            "            values:\n"
        )
        for node in excluded_list:
            values_yaml += f"            - {node}\n"

    # Prometheus: pin to monitoring node + persistent storage
    values_yaml += (
        "prometheus:\n"
        "  prometheusSpec:\n"
        + node_affinity_yaml
        + "    storageSpec:\n"
        "      volumeClaimTemplate:\n"
        "        spec:\n"
        f"          storageClassName: {sc_prom}\n"
        "          accessModes: [ReadWriteOnce]\n"
        "          resources:\n"
        "            requests:\n"
        "              storage: 30Gi\n"
    )

    # Alertmanager: pin to monitoring node + persistent storage
    values_yaml += (
        "alertmanager:\n"
        "  alertmanagerSpec:\n"
        + node_affinity_yaml
        + "    storage:\n"
        "      volumeClaimTemplate:\n"
        "        spec:\n"
        f"          storageClassName: {sc_alert}\n"
        "          accessModes: [ReadWriteOnce]\n"
        "          resources:\n"
        "            requests:\n"
        "              storage: 5Gi\n"
    )

    # Grafana: persistent storage (Deployment can float; local PV affinity will pull it to merlin)
    values_yaml += (
        "grafana:\n"
        "  adminUser: admin\n"
        f"  adminPassword: {grafana_password}\n"
        "  persistence:\n"
        "    type: sts\n"
        "    enabled: true\n"
        f"    storageClassName: {sc_grafana}\n"
        "    accessModes: [ReadWriteOnce]\n"
        "    size: 10Gi\n"
    )

    tmp_values = tempfile.NamedTemporaryFile(
        mode="w", suffix=".yaml", prefix="monitoring-values-", delete=False
    )
    try:
        tmp_values.write(values_yaml)
        tmp_values.flush()
        tmp_values.close()

        _log(log, f"[MONITORING] Deploying {release} in namespace {ns}")
        _helm(
            [
                "upgrade",
                "--install",
                release,
                chart,
                "--namespace",
                ns,
                "-f",
                tmp_values.name,
                "--wait",
            ],
            env=env,
            timeout=600,
            check=True,
        )
    finally:
        os.unlink(tmp_values.name)


def restart(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
) -> None:
    ns = _monitoring_namespace(namespace, env)
    _log(log, f"[MONITORING] Restarting grafana deployment in namespace {ns}")
    _kubectl(["-n", ns, "rollout", "restart", "deployment/prometheus-grafana"], env=env, timeout=180)


def stop(
    *,
    namespace: str | None = None,
    env: dict | None = None,
    log: _LogFn | None = None,
) -> None:
    ns = _monitoring_namespace(namespace, env)
    release = _release_name(env)
    _log(log, f"[MONITORING] Uninstalling release {release} in namespace {ns}")
    _helm(["uninstall", release, "--namespace", ns], env=env, timeout=180)


def status(
    *,
    namespace: str | None = None,
    env: dict | None = None,
) -> bool:
    ns = _monitoring_namespace(namespace, env)
    dep = _kubectl(
        [
            "-n",
            ns,
            "get",
            "deployment",
            "prometheus-grafana",
            "-o",
            "jsonpath={.status.readyReplicas}",
        ],
        env=env,
        timeout=20,
    )
    return _to_bool(dep.returncode == 0 and (dep.stdout or "0").strip() not in {"", "0"})