from __future__ import annotations from types import SimpleNamespace import pytest import knoe.core.actions as actions from knoe.core.actions import KnoeConsoleInstaller from knoe.core.controller import KnoeController def _mk_installer(tmp_path) -> KnoeConsoleInstaller: c = KnoeController(tmp_path) return KnoeConsoleInstaller(c) def test_repair_flow_classifies_blockers_attempts_reclaim_and_avoids_blind_health_wait( tmp_path, monkeypatch, capsys ): installer = _mk_installer(tmp_path) installer.inputs["init_cluster.cluster_env"] = "service" # Avoid actually invoking any shell-based status/init scripts. def run_script(*_a, **_k): raise AssertionError("_run_script should not be called when hard blockers are present") monkeypatch.setattr(installer, "_run_script", run_script) monkeypatch.setattr(installer, "_ensure_namespace_ready", lambda *_a, **_k: None) monkeypatch.setattr( installer, "_script_env_for_namespace", lambda ns: {"SERVICE_NAMESPACE": ns, "DB_NAMESPACE": ns, "NAMESPACE": ns}, ) service_ns = installer._service_namespace() # Model the real-world unschedulable pattern: affinity + taint + PV binding. unsched_msg = ( "0/3 nodes are available: " "1 node(s) didn't match Pod's node affinity/selector, " "1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, " "1 node(s) didn't find available persistent volumes to bind." ) def fake_kubectl_get_json(_env, args): if args[:3] == ["get", "pvc", "-A"]: return { "items": [ { "metadata": { "name": "data-garage-0", "namespace": service_ns, "uid": "new-garage-uid", }, "spec": { "storageClassName": "synology-iscsi", "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "20Gi"}}, }, "status": {"phase": "Pending"}, }, { "metadata": { "name": "data-openbao-0", "namespace": service_ns, "uid": "new-bao-uid", }, "spec": { "storageClassName": "synology-iscsi", "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "20Gi"}}, }, "status": {"phase": "Pending"}, }, ] } if args[:2] == ["get", "pv"]: return { "items": [ { "metadata": {"name": "synology-iscsi-d001-garage"}, "spec": { "storageClassName": "synology-iscsi", "capacity": {"storage": "20Gi"}, "accessModes": ["ReadWriteOnce"], "claimRef": { "name": "data-garage-0", "namespace": service_ns, "uid": "old-garage-uid", }, }, "status": {"phase": "Released"}, }, { "metadata": {"name": "synology-iscsi-d001-openbao"}, "spec": { "storageClassName": "synology-iscsi", "capacity": {"storage": "20Gi"}, "accessModes": ["ReadWriteOnce"], "claimRef": { "name": "data-openbao-0", "namespace": service_ns, "uid": "old-bao-uid", }, }, "status": {"phase": "Released"}, }, ] } if args[:3] == ["get", "pod", "-A"]: return { "items": [ { "metadata": {"name": "garage-0", "namespace": service_ns}, "status": { "phase": "Pending", "conditions": [ { "type": "PodScheduled", "status": "False", "reason": "Unschedulable", "message": unsched_msg, } ], }, }, { "metadata": {"name": "openbao-0", "namespace": service_ns}, "status": { "phase": "Pending", "conditions": [ { "type": "PodScheduled", "status": "False", "reason": "Unschedulable", "message": unsched_msg, } ], }, }, { "metadata": {"name": "opentofu-abc", "namespace": service_ns}, "status": { "phase": "Pending", "containerStatuses": [ { "name": "opentofu", "state": {"waiting": {"reason": "ContainerCreating"}}, } ], }, }, ] } return {"items": []} monkeypatch.setattr(installer, "_kubectl_get_json", fake_kubectl_get_json) calls: list[list[str]] = [] def fake_run(cmd, **kwargs): calls.append(list(cmd)) # Missing downstream Kong resources. if cmd[:4] == ["kubectl", "-n", service_ns, "get"] and cmd[4:6] in ( ["svc", "prole-svc-kong"], ["deploy", "prole-svc-kong"], ): return SimpleNamespace(returncode=1, stdout="", stderr="not found") # PV reclaim (claimRef clearing) if cmd[:3] == ["kubectl", "patch", "pv"]: return SimpleNamespace(returncode=0, stdout="ok", stderr="") return SimpleNamespace(returncode=0, stdout="ok", stderr="") monkeypatch.setattr(actions.subprocess, "run", fake_run) with pytest.raises(Exception) as exc: installer._deploy_common_services(env_key="service") msg = str(exc.value) assert "Cluster prerequisites are blocked" in msg # Reclaim was attempted for matching stale Released PVs. patched = [c for c in calls if c[:3] == ["kubectl", "patch", "pv"]] assert any("synology-iscsi-d001-garage" in c for c in patched) assert any("synology-iscsi-d001-openbao" in c for c in patched) # Explicit scheduling/storage classification is surfaced. assert "Storage blockers" in msg assert "unschedulable_due_to_pv_binding" in msg assert "unschedulable_due_to_node_affinity" in msg assert "unschedulable_due_to_untolerated_taint" in msg assert "container_creating" in msg # Downstream missingness is present, but labeled as downstream. assert "Downstream missing resources" in msg assert "prole-svc-kong" in msg # Strict policy: never delete PVs. assert not any(c[:3] == ["kubectl", "delete", "pv"] for c in calls) def test_classify_blocked_pod_storage_first_when_multiple_unschedulable_reasons(tmp_path): installer = _mk_installer(tmp_path) msg = ( "0/3 nodes are available: " "1 node(s) didn't match Pod's node affinity/selector, " "1 node(s) had untolerated taint {x: y}, " "1 node(s) didn't find available persistent volumes to bind." ) pod = { "metadata": {"name": "garage-0", "namespace": "knoe-system"}, "status": { "phase": "Pending", "conditions": [ { "type": "PodScheduled", "status": "False", "reason": "Unschedulable", "message": msg, } ], }, } classified = installer._classify_blocked_pod(env={}, pod=pod) assert classified["type"] == "unschedulable_due_to_pv_binding" assert set(classified["types"]) >= { "unschedulable_due_to_pv_binding", "unschedulable_due_to_node_affinity", "unschedulable_due_to_untolerated_taint", }