prole/tests/test_reconcile_blocked_cluster_state.py

from __future__ import annotations

from types import SimpleNamespace

import pytest

import knoe.core.actions as actions
from knoe.core.actions import KnoeConsoleInstaller
from knoe.core.controller import KnoeController


def _mk_installer(tmp_path) -> KnoeConsoleInstaller:
    c = KnoeController(tmp_path)
    return KnoeConsoleInstaller(c)


def test_repair_flow_classifies_blockers_attempts_reclaim_and_avoids_blind_health_wait(
    tmp_path, monkeypatch, capsys
):
    installer = _mk_installer(tmp_path)
    installer.inputs["init_cluster.cluster_env"] = "service"

    # Avoid actually invoking any shell-based status/init scripts.
    def run_script(*_a, **_k):
        raise AssertionError("_run_script should not be called when hard blockers are present")

    monkeypatch.setattr(installer, "_run_script", run_script)
    monkeypatch.setattr(installer, "_ensure_namespace_ready", lambda *_a, **_k: None)
    monkeypatch.setattr(
        installer,
        "_script_env_for_namespace",
        lambda ns: {"SERVICE_NAMESPACE": ns, "DB_NAMESPACE": ns, "NAMESPACE": ns},
    )

    service_ns = installer._service_namespace()

    # Model the real-world unschedulable pattern: affinity + taint + PV binding.
    unsched_msg = (
        "0/3 nodes are available: "
        "1 node(s) didn't match Pod's node affinity/selector, "
        "1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, "
        "1 node(s) didn't find available persistent volumes to bind."
    )

    def fake_kubectl_get_json(_env, args):
        if args[:3] == ["get", "pvc", "-A"]:
            return {
                "items": [
                    {
                        "metadata": {
                            "name": "data-garage-0",
                            "namespace": service_ns,
                            "uid": "new-garage-uid",
                        },
                        "spec": {
                            "storageClassName": "synology-iscsi",
                            "accessModes": ["ReadWriteOnce"],
                            "resources": {"requests": {"storage": "20Gi"}},
                        },
                        "status": {"phase": "Pending"},
                    },
                    {
                        "metadata": {
                            "name": "data-openbao-0",
                            "namespace": service_ns,
                            "uid": "new-bao-uid",
                        },
                        "spec": {
                            "storageClassName": "synology-iscsi",
                            "accessModes": ["ReadWriteOnce"],
                            "resources": {"requests": {"storage": "20Gi"}},
                        },
                        "status": {"phase": "Pending"},
                    },
                ]
            }
        if args[:2] == ["get", "pv"]:
            return {
                "items": [
                    {
                        "metadata": {"name": "synology-iscsi-d001-garage"},
                        "spec": {
                            "storageClassName": "synology-iscsi",
                            "capacity": {"storage": "20Gi"},
                            "accessModes": ["ReadWriteOnce"],
                            "claimRef": {
                                "name": "data-garage-0",
                                "namespace": service_ns,
                                "uid": "old-garage-uid",
                            },
                        },
                        "status": {"phase": "Released"},
                    },
                    {
                        "metadata": {"name": "synology-iscsi-d001-openbao"},
                        "spec": {
                            "storageClassName": "synology-iscsi",
                            "capacity": {"storage": "20Gi"},
                            "accessModes": ["ReadWriteOnce"],
                            "claimRef": {
                                "name": "data-openbao-0",
                                "namespace": service_ns,
                                "uid": "old-bao-uid",
                            },
                        },
                        "status": {"phase": "Released"},
                    },
                ]
            }
        if args[:3] == ["get", "pod", "-A"]:
            return {
                "items": [
                    {
                        "metadata": {"name": "garage-0", "namespace": service_ns},
                        "status": {
                            "phase": "Pending",
                            "conditions": [
                                {
                                    "type": "PodScheduled",
                                    "status": "False",
                                    "reason": "Unschedulable",
                                    "message": unsched_msg,
                                }
                            ],
                        },
                    },
                    {
                        "metadata": {"name": "openbao-0", "namespace": service_ns},
                        "status": {
                            "phase": "Pending",
                            "conditions": [
                                {
                                    "type": "PodScheduled",
                                    "status": "False",
                                    "reason": "Unschedulable",
                                    "message": unsched_msg,
                                }
                            ],
                        },
                    },
                    {
                        "metadata": {"name": "opentofu-abc", "namespace": service_ns},
                        "status": {
                            "phase": "Pending",
                            "containerStatuses": [
                                {
                                    "name": "opentofu",
                                    "state": {"waiting": {"reason": "ContainerCreating"}},
                                }
                            ],
                        },
                    },
                ]
            }
        return {"items": []}

    monkeypatch.setattr(installer, "_kubectl_get_json", fake_kubectl_get_json)

    calls: list[list[str]] = []

    def fake_run(cmd, **kwargs):
        calls.append(list(cmd))

        # Missing downstream Kong resources.
        if cmd[:4] == ["kubectl", "-n", service_ns, "get"] and cmd[4:6] in (
            ["svc", "knoe-svc-kong"],
            ["deploy", "knoe-svc-kong"],
        ):
            return SimpleNamespace(returncode=1, stdout="", stderr="not found")

        # PV reclaim (claimRef clearing)
        if cmd[:3] == ["kubectl", "patch", "pv"]:
            return SimpleNamespace(returncode=0, stdout="ok", stderr="")

        return SimpleNamespace(returncode=0, stdout="ok", stderr="")

    monkeypatch.setattr(actions.subprocess, "run", fake_run)

    with pytest.raises(Exception) as exc:
        installer._deploy_common_services(env_key="service")

    msg = str(exc.value)
    assert "Cluster prerequisites are blocked" in msg

    # Reclaim was attempted for matching stale Released PVs.
    patched = [c for c in calls if c[:3] == ["kubectl", "patch", "pv"]]
    assert any("synology-iscsi-d001-garage" in c for c in patched)
    assert any("synology-iscsi-d001-openbao" in c for c in patched)

    # Explicit scheduling/storage classification is surfaced.
    assert "Storage blockers" in msg
    assert "unschedulable_due_to_pv_binding" in msg
    assert "unschedulable_due_to_node_affinity" in msg
    assert "unschedulable_due_to_untolerated_taint" in msg
    assert "container_creating" in msg

    # Downstream missingness is present, but labeled as downstream.
    assert "Downstream missing resources" in msg
    assert "knoe-svc-kong" in msg

    # Strict policy: never delete PVs.
    assert not any(c[:3] == ["kubectl", "delete", "pv"] for c in calls)


def test_classify_blocked_pod_storage_first_when_multiple_unschedulable_reasons(tmp_path):
    installer = _mk_installer(tmp_path)

    msg = (
        "0/3 nodes are available: "
        "1 node(s) didn't match Pod's node affinity/selector, "
        "1 node(s) had untolerated taint {x: y}, "
        "1 node(s) didn't find available persistent volumes to bind."
    )
    pod = {
        "metadata": {"name": "garage-0", "namespace": "knoe-system"},
        "status": {
            "phase": "Pending",
            "conditions": [
                {
                    "type": "PodScheduled",
                    "status": "False",
                    "reason": "Unschedulable",
                    "message": msg,
                }
            ],
        },
    }

    classified = installer._classify_blocked_pod(env={}, pod=pod)
    assert classified["type"] == "unschedulable_due_to_pv_binding"
    assert set(classified["types"]) >= {
        "unschedulable_due_to_pv_binding",
        "unschedulable_due_to_node_affinity",
        "unschedulable_due_to_untolerated_taint",
    }