mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 10:13:58 +00:00
c0a7d0c5dc
- improve installer/action/controller flow and shell-variable expansion handling across screens\n- adjust Supabase Helm rendering and storage deployment templates\n- align monitoring, cloudnative-pg and repair pipeline behavior with updated config paths\n- refresh and expand installer/core regression tests around milestones, navigation and repair logic Co-authored-by: Junie <junie@jetbrains.com>
435 lines
14 KiB
Python
435 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
from contextlib import contextmanager
|
|
import json
|
|
import os
|
|
from types import SimpleNamespace
|
|
|
|
import knoe.core.actions as actions
|
|
from knoe.core.actions import KnoeConsoleInstaller
|
|
from knoe.core.controller import KnoeController
|
|
|
|
|
|
@contextmanager
|
|
def _preserve_namespace_env():
|
|
"""Preserve/restore any pre-existing namespace-related env vars.
|
|
|
|
Tests must not assume a global/default namespace and must not leak env
|
|
mutations across the suite.
|
|
"""
|
|
|
|
keys = ("SERVICE_NAMESPACE", "DB_NAMESPACE", "NAMESPACE")
|
|
before = {k: os.environ.get(k) for k in keys}
|
|
try:
|
|
yield
|
|
finally:
|
|
for k, v in before.items():
|
|
if v is None:
|
|
os.environ.pop(k, None)
|
|
else:
|
|
os.environ[k] = v
|
|
|
|
|
|
def _mk_installer(tmp_path) -> KnoeConsoleInstaller:
|
|
c = KnoeController(tmp_path)
|
|
return KnoeConsoleInstaller(c)
|
|
|
|
|
|
def test_match_stale_released_pv_happy_path(tmp_path):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
pvc = {
|
|
"metadata": {"name": "data-openbao-0", "namespace": "knoe-system", "uid": "new-uid"},
|
|
"spec": {
|
|
"storageClassName": "local-path",
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"resources": {"requests": {"storage": "10Gi"}},
|
|
},
|
|
"status": {"phase": "Pending"},
|
|
}
|
|
pv = {
|
|
"metadata": {"name": "pvc-123"},
|
|
"spec": {
|
|
"storageClassName": "local-path",
|
|
"capacity": {"storage": "20Gi"},
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"claimRef": {
|
|
"name": "data-openbao-0",
|
|
"namespace": "knoe-system",
|
|
"uid": "old-uid",
|
|
},
|
|
},
|
|
"status": {"phase": "Released"},
|
|
}
|
|
|
|
assert installer._match_stale_released_pv(pvc, [pv]) == pv
|
|
|
|
|
|
def test_match_stale_released_pv_rejects_storage_class_mismatch(tmp_path):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
pvc = {
|
|
"metadata": {"name": "x", "namespace": "ns", "uid": "new"},
|
|
"spec": {
|
|
"storageClassName": "sc-a",
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"resources": {"requests": {"storage": "1Gi"}},
|
|
},
|
|
"status": {"phase": "Pending"},
|
|
}
|
|
pv = {
|
|
"metadata": {"name": "pv"},
|
|
"spec": {
|
|
"storageClassName": "sc-b",
|
|
"capacity": {"storage": "10Gi"},
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"claimRef": {"name": "x", "namespace": "ns", "uid": "old"},
|
|
},
|
|
"status": {"phase": "Released"},
|
|
}
|
|
|
|
assert installer._match_stale_released_pv(pvc, [pv]) is None
|
|
|
|
|
|
def test_match_stale_released_pv_rejects_size_too_small(tmp_path):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
pvc = {
|
|
"metadata": {"name": "x", "namespace": "ns", "uid": "new"},
|
|
"spec": {
|
|
"storageClassName": "sc",
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"resources": {"requests": {"storage": "11Gi"}},
|
|
},
|
|
"status": {"phase": "Pending"},
|
|
}
|
|
pv = {
|
|
"metadata": {"name": "pv"},
|
|
"spec": {
|
|
"storageClassName": "sc",
|
|
"capacity": {"storage": "10Gi"},
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"claimRef": {"name": "x", "namespace": "ns", "uid": "old"},
|
|
},
|
|
"status": {"phase": "Released"},
|
|
}
|
|
|
|
assert installer._match_stale_released_pv(pvc, [pv]) is None
|
|
|
|
|
|
def test_clear_pv_claim_ref_patches_null_claimref(tmp_path, monkeypatch):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
calls: list[list[str]] = []
|
|
|
|
def fake_run(cmd, **kwargs):
|
|
calls.append(list(cmd))
|
|
return SimpleNamespace(returncode=0, stdout="ok", stderr="")
|
|
|
|
monkeypatch.setattr(actions.subprocess, "run", fake_run)
|
|
|
|
assert installer._clear_pv_claim_ref(env={}, pv_name="pv-1") is True
|
|
|
|
assert calls
|
|
assert calls[0][:3] == ["kubectl", "patch", "pv"]
|
|
assert "pv-1" in calls[0]
|
|
assert "-p" in calls[0]
|
|
assert '{"spec":{"claimRef":null}}' in calls[0]
|
|
|
|
|
|
def test_reset_reclaim_stale_released_pvs_patches_concrete_garage_and_openbao(
|
|
tmp_path, monkeypatch
|
|
):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
# Do not assume a global/default namespace; preserve and restore prior env.
|
|
with _preserve_namespace_env():
|
|
os.environ["SERVICE_NAMESPACE"] = "knoe-system"
|
|
os.environ["DB_NAMESPACE"] = "knoe-system"
|
|
|
|
pending_pvcs = [
|
|
{
|
|
"metadata": {
|
|
"name": "data-garage-0",
|
|
"namespace": "knoe-system",
|
|
"uid": "new-garage-uid",
|
|
},
|
|
"spec": {
|
|
"storageClassName": "synology-iscsi",
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"resources": {"requests": {"storage": "10Gi"}},
|
|
},
|
|
"status": {"phase": "Pending"},
|
|
},
|
|
{
|
|
"metadata": {
|
|
"name": "data-openbao-0",
|
|
"namespace": "knoe-system",
|
|
"uid": "new-bao-uid",
|
|
},
|
|
"spec": {
|
|
"storageClassName": "synology-iscsi",
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"resources": {"requests": {"storage": "10Gi"}},
|
|
},
|
|
"status": {"phase": "Pending"},
|
|
},
|
|
]
|
|
|
|
released_pvs = [
|
|
{
|
|
"metadata": {"name": "synology-iscsi-d001-garage"},
|
|
"spec": {
|
|
"storageClassName": "synology-iscsi",
|
|
"capacity": {"storage": "20Gi"},
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"claimRef": {
|
|
"name": "data-garage-0",
|
|
"namespace": "knoe-system",
|
|
"uid": "old-garage-uid",
|
|
},
|
|
},
|
|
"status": {"phase": "Released"},
|
|
},
|
|
{
|
|
"metadata": {"name": "synology-iscsi-d001-openbao"},
|
|
"spec": {
|
|
"storageClassName": "synology-iscsi",
|
|
"capacity": {"storage": "20Gi"},
|
|
"accessModes": ["ReadWriteOnce"],
|
|
"claimRef": {
|
|
"name": "data-openbao-0",
|
|
"namespace": "knoe-system",
|
|
"uid": "old-bao-uid",
|
|
},
|
|
},
|
|
"status": {"phase": "Released"},
|
|
},
|
|
]
|
|
|
|
actions._list_pending_pvcs = lambda _env, namespaces=None: pending_pvcs # type: ignore[method-assign]
|
|
actions._list_released_pvs = lambda _env, storage_class=None: released_pvs # type: ignore[method-assign]
|
|
|
|
calls: list[list[str]] = []
|
|
|
|
def fake_run(cmd, **kwargs):
|
|
cmd = list(cmd)
|
|
calls.append(cmd)
|
|
|
|
if cmd[:4] == ["kubectl", "get", "pvc", "-A"] and "-o" in cmd and "json" in cmd:
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=json.dumps({"items": pending_pvcs}),
|
|
stderr="",
|
|
)
|
|
if cmd[:3] == ["kubectl", "get", "pv"] and "-o" in cmd and "json" in cmd:
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=json.dumps({"items": released_pvs}),
|
|
stderr="",
|
|
)
|
|
|
|
return SimpleNamespace(returncode=0, stdout="ok", stderr="")
|
|
|
|
monkeypatch.setattr(actions.subprocess, "run", fake_run)
|
|
|
|
changed = installer._repair_stale_released_pvs(env={}, reset=True)
|
|
assert changed is True
|
|
|
|
patched = [c for c in calls if c[:3] == ["kubectl", "patch", "pv"]]
|
|
assert any("synology-iscsi-d001-garage" in c for c in patched)
|
|
assert any("synology-iscsi-d001-openbao" in c for c in patched)
|
|
assert all('{"spec":{"claimRef":null}}' in c for c in patched)
|
|
|
|
# Reset policy: never delete PVs.
|
|
assert not any(c[:3] == ["kubectl", "delete", "pv"] for c in calls)
|
|
|
|
|
|
def test_step_cluster_runs_reset_reclaim_before_openbao_initialize(tmp_path, monkeypatch):
|
|
installer = _mk_installer(tmp_path)
|
|
installer.inputs["init_cluster.cluster_env"] = "service"
|
|
installer._reset_reclaim_pvs_requested = True
|
|
|
|
calls: list[str] = []
|
|
|
|
def fake_run_script(script, args=None, **_kwargs):
|
|
if script == "init_openbao.sh" and (args or [])[:1] == ["initialize"]:
|
|
calls.append("openbao-initialize")
|
|
return 0
|
|
|
|
def fake_reset_reclaim(_env):
|
|
calls.append("reset-reclaim")
|
|
return True
|
|
|
|
monkeypatch.setattr(installer, "_run_script", fake_run_script)
|
|
monkeypatch.setattr(installer, "_script_env_for_namespace", lambda _ns: {})
|
|
monkeypatch.setattr(installer, "_reset_reclaim_stale_pvs", fake_reset_reclaim)
|
|
monkeypatch.setattr(installer, "_maybe_run_repair_pipeline", lambda *_a, **_k: None)
|
|
monkeypatch.setattr(installer, "_deploy_common_services", lambda *_a, **_k: None)
|
|
monkeypatch.setattr(installer, "_fetch_k3s_kubeconfig", lambda: None)
|
|
|
|
# Make the cluster reachability check succeed.
|
|
monkeypatch.setattr(
|
|
actions.subprocess,
|
|
"run",
|
|
lambda *_a, **_k: SimpleNamespace(returncode=0, stdout="ok", stderr=""),
|
|
)
|
|
|
|
installer._openbao_init_env = {"x": "y"}
|
|
installer._openbao_init_password = "pw"
|
|
|
|
installer._step_cluster()
|
|
|
|
assert "reset-reclaim" in calls
|
|
assert "openbao-initialize" in calls
|
|
assert calls.index("reset-reclaim") < calls.index("openbao-initialize")
|
|
|
|
|
|
def test_maybe_run_repair_pipeline_runs_stale_repair_first(tmp_path):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
calls: list[str] = []
|
|
|
|
def stale(env):
|
|
calls.append("stale")
|
|
return False
|
|
|
|
def dash(env):
|
|
calls.append("dash")
|
|
return False
|
|
|
|
def auth():
|
|
calls.append("auth")
|
|
return False
|
|
|
|
def pipeline(env, anomalies):
|
|
calls.append("pipeline")
|
|
|
|
def ensure_ns(env, name):
|
|
calls.append("ensure-ns")
|
|
|
|
installer._repair_stale_released_pvs = stale # type: ignore[method-assign]
|
|
installer._ensure_namespace_ready = ensure_ns # type: ignore[method-assign]
|
|
installer._dashboard_kong_missing = dash # type: ignore[method-assign]
|
|
installer._authority_context_missing = auth # type: ignore[method-assign]
|
|
installer._run_cluster_repair_pipeline = pipeline # type: ignore[method-assign]
|
|
|
|
installer._maybe_run_repair_pipeline(env={})
|
|
|
|
assert calls[:4] == ["stale", "ensure-ns", "dash", "auth"]
|
|
assert "pipeline" not in calls
|
|
|
|
|
|
def test_maybe_run_repair_pipeline_invokes_python_pipeline_when_anomalies(tmp_path):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
calls: list[str] = []
|
|
|
|
def stale(env):
|
|
calls.append("stale")
|
|
return False
|
|
|
|
def dash(env):
|
|
calls.append("dash")
|
|
return True
|
|
|
|
def auth():
|
|
calls.append("auth")
|
|
return True
|
|
|
|
def pipeline(env, anomalies):
|
|
calls.append("pipeline")
|
|
assert set(anomalies) == {"dashboard", "authority"}
|
|
|
|
def ensure_ns(env, name):
|
|
calls.append("ensure-ns")
|
|
|
|
installer._repair_stale_released_pvs = stale # type: ignore[method-assign]
|
|
installer._ensure_namespace_ready = ensure_ns # type: ignore[method-assign]
|
|
installer._dashboard_kong_missing = dash # type: ignore[method-assign]
|
|
installer._authority_context_missing = auth # type: ignore[method-assign]
|
|
installer._run_cluster_repair_pipeline = pipeline # type: ignore[method-assign]
|
|
|
|
installer._maybe_run_repair_pipeline(env={})
|
|
|
|
assert calls == ["stale", "ensure-ns", "dash", "auth", "pipeline"]
|
|
|
|
|
|
def test_ensure_namespace_ready_recreates_after_termination(tmp_path, monkeypatch):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
# Simulate: first call sees terminating namespace, then it disappears.
|
|
seq = [
|
|
{"metadata": {"name": "knoe-system", "deletionTimestamp": "2026-03-07T12:00:00Z"}, "status": {"phase": "Terminating"}},
|
|
None,
|
|
None,
|
|
]
|
|
|
|
def fake_get(env, name):
|
|
return seq.pop(0) if seq else None
|
|
|
|
created: list[list[str]] = []
|
|
|
|
def fake_run_cmd(cmd, **kwargs):
|
|
created.append(list(cmd))
|
|
return 0
|
|
|
|
# Avoid real sleeping in tests.
|
|
monkeypatch.setattr(actions.time, "sleep", lambda *_a, **_k: None)
|
|
|
|
installer._kubectl_get_namespace_json = fake_get # type: ignore[method-assign]
|
|
installer._run_cmd = fake_run_cmd # type: ignore[method-assign]
|
|
|
|
installer._ensure_namespace_ready(env={}, name="knoe-system", timeout_s=0)
|
|
|
|
assert created
|
|
assert created[0] == ["kubectl", "create", "namespace", "knoe-system"]
|
|
|
|
|
|
def test_ensure_namespace_ready_raises_if_stuck_terminating(tmp_path, monkeypatch):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
def fake_get(env, name):
|
|
return {
|
|
"metadata": {"name": name, "deletionTimestamp": "2026-03-07T12:00:00Z"},
|
|
"status": {"phase": "Terminating"},
|
|
}
|
|
|
|
monkeypatch.setattr(actions.time, "sleep", lambda *_a, **_k: None)
|
|
installer._kubectl_get_namespace_json = fake_get # type: ignore[method-assign]
|
|
|
|
try:
|
|
installer._ensure_namespace_ready(env={}, name="knoe-system", timeout_s=0)
|
|
assert False, "expected exception"
|
|
except Exception as e:
|
|
assert "still terminating" in str(e)
|
|
|
|
|
|
def test_deploy_common_services_runs_namespace_preflight_first(tmp_path):
|
|
installer = _mk_installer(tmp_path)
|
|
|
|
calls: list[str] = []
|
|
|
|
def ensure(env, name):
|
|
calls.append("ensure")
|
|
|
|
def run_script(script_name, args=None, env=None, **_kw):
|
|
calls.append(script_name)
|
|
if script_name == "status_common_services.sh":
|
|
# First status check fails to trigger deploy; second passes.
|
|
return 1 if calls.count("status_common_services.sh") == 1 else 0
|
|
if script_name == "init_common_services.sh":
|
|
return 0
|
|
return 0
|
|
|
|
installer._ensure_namespace_ready = ensure # type: ignore[method-assign]
|
|
installer._run_script = run_script # type: ignore[method-assign]
|
|
installer._service_namespace = lambda: "knoe-system" # type: ignore[method-assign]
|
|
installer._script_env_for_namespace = lambda _ns: {} # type: ignore[method-assign]
|
|
installer._get_input_bool = lambda *_a, **_k: False # type: ignore[method-assign]
|
|
|
|
installer._deploy_common_services(env_key="service")
|
|
|
|
assert calls[0] == "ensure"
|
|
assert "init_common_services.sh" in calls
|