installer: stabilize retries + improve k3s inventory detection

- Reuse namespace from existing prole.cfg unless env overrides

- Support k3s_hosts children groups and prefer k3s_servers as default

- Add headless pytest tkinter stubs and blocked-cluster reconciliation tests

- Update registry mirror endpoint, port mappings, and prole-db manifests

conf/port-mapping.cfg

@@ -1,3 +1,11 @@
 # Port mappings for Prole Tools (generated).
 # Format: key: local=... remote=... ns=... svc=... address=...
 
+argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0
+garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0
+openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0
+opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0
+dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1
+postgres: local=5432 remote=5432 ns=prole-db-chrisfu-4ac8c0 svc=prole-db-rw address=0.0.0.0
+prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1
+grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0

conf/postgresql/.version

@@ -0,0 +1 @@
+18

conf/prole.cfg

@@ -1,16 +1,16 @@
 ; Prole Master Configuration File
-; Generated by install.py on 2026-03-07 10:32:42
+; Generated by install.py on 2026-03-07 23:56:48
 ; This file is used as input for Ansible deployment and k8s cluster creation.
 
 [User]
 ; User-editable values; derived values below reference these by default.
-NAMESPACE = prole-db-chrisfu-4ac8c0
+NAMESPACE = knoe-db
 PROLE_CONF = ${PROLE_HOME}/conf
 PROLE_DATA = /Users/chrisfu/.prole/data
 PROLE_HOME = /Users/chrisfu/dev/prole
 PROLE_LOGS = /opt/prole/logs/chrisfu
 PROLE_SERVICE = ${PROLE_HOME}/etc
-SERVICE_NAMESPACE = knoe-system
+SERVICE_NAMESPACE = knoe-dbnoe-system
 
 [Inputs]
 ; Screen-scoped inputs used for unattended replays (-S)
@@ -54,8 +54,8 @@ init_cluster.at_rest_encryption_enabled = true
 init_cluster.cluster_env = service
 init_cluster.deployment_target = prole-service-cluster
 init_cluster.gitops_enabled = false
-init_cluster.k3s_server_url = 
+init_cluster.k3s_server_url = https://myrddin.prole.org:6443
-init_cluster.k3s_token = ${PROLE_SECRET:v1:38WOjNObQUt_S1W_:q7cZ6C5FcsiNPfxtmUI5RtsuJ4iaBAdvLbw_1hpOx87Cvv2KL-HjbdE7s5J4aPegFw02IrwWuPP5WXe5nGkg7uTXreCsvg5uWYXc1vxjWHUzKEwVfcP2nPEKBjJrRuai1VxD8WaPbwcUMQfXEtqNrFrkBSIICYltM1hmmA==}
+init_cluster.k3s_token = ${PROLE_SECRET:v1:i1FXxxDV0iTGUprA:Esd5SwMPbXNiWB1FeAPJqDVMhaIwg6TeR7aPH-IbF3HURxH6qs89IPQWhAAOmuFYLNCG_Fc8e7y6knoe-dbTbIrV-md-fOVOZ8PbRHwGp_6u2w7O5WUf_QIdhxF30H-UQZlFZHOAHnW2l970O2zQQPGRJknoe-dbqQq6VzoK2_P6tiJ8uA==}
 init_cluster.kerberos_enabled = true
 init_cluster.mode = k3s
 init_cluster.start_cluster = true
@@ -65,14 +65,14 @@ init_cnpg_deploy.run_deploy = true
 init_db_build.run_build = true
 init_password.db_host_port = 5432
 init_password.db_namespace = ${NAMESPACE}
-init_password.db_password = 
+init_password.db_password = ${PROLE_SECRET:v1:eRSi-Qb0odxbdYGH:KedZCSfHSVX6cmxyqrZG2VwA6T0aH-n1m0hSZcpCJ3T-pobLBKtb}
-init_password.db_password_confirm = 
+init_password.db_password_confirm = ${PROLE_SECRET:v1:eRSi-Qb0odxbdYGH:KedZCSfHSVX6cmxyqrZG2VwA6T0aH-n1m0hSZcpCJ3T-pobLBKtb}
 init_password.db_username = root
 init_password.generate_ssh_key = true
 init_scripts.run_scripts = true
 kerberos_config.enabled = true
 kerberos_config.kdc = 10.0.0.3
-kerberos_config.password = ${PROLE_SECRET:v1:q3AVwS1x8Xr8V6n9:IRDXFhRi8uBOZvDmLoT-zKVjwGsiiDQvB4c430CqJS1PWz3X}
+kerberos_config.password = ${PROLE_SECRET:v1:yXgcBJEzbAwNJ9E_:Aq6ppY9xJzN22l2FYAdVWWHyuAssQdyZKZMYm_m2SDgT6peM}
 kerberos_config.realm = PROLE.ORG
 kerberos_config.test_connection = false
 kerberos_config.user = administrator
@@ -85,16 +85,17 @@ ollama_config.server_port = 11434
 ; Variables used by name in more than one place or assumed global scope
 CLUSTER_ENV = service
 DB_HOST_PORT = 5432
-DB_PASSWORD = 
+DB_PASSWORD = ${PROLE_SECRET:v1:JSPYBCmth7xOrNlE:dH6t-0yLoc5qvt-sBFoQw7KRWo84LXPgUU4EIpGIRJyBMcTVFcjC}
 DEPLOYMENT_MODE = k3s
 DEPLOYMENT_TARGET = prole-service-cluster
+DOCKER_IMPORT_DIR = 
 NAMESPACE = ${NAMESPACE}
 PROLE_DB_USER = root
 PROLE_HOME = ${PROLE_HOME}
-PROLE_K3S_SERVER = 
+PROLE_K3S_SERVER = https://myrddin.prole.org:6443
-PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:38WOjNObQUt_S1W_:q7cZ6C5FcsiNPfxtmUI5RtsuJ4iaBAdvLbw_1hpOx87Cvv2KL-HjbdE7s5J4aPegFw02IrwWuPP5WXe5nGkg7uTXreCsvg5uWYXc1vxjWHUzKEwVfcP2nPEKBjJrRuai1VxD8WaPbwcUMQfXEtqNrFrkBSIICYltM1hmmA==}
+PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:i1FXxxDV0iTGUprA:Esd5SwMPbXNiWB1FeAPJqDVMhaIwg6TeR7aPH-IbF3HURxH6qs89IPQWhAAOmuFYLNCG_Fc8e7y6knoe-dbTbIrV-md-fOVOZ8PbRHwGp_6u2w7O5WUf_QIdhxF30H-UQZlFZHOAHnW2l970O2zQQPGRJknoe-dbqQq6VzoK2_P6tiJ8uA==}
 PROLE_OPENTOFU_URL = http://127.0.0.1:8080
-SERVICE_NAMESPACE = knoe-system
+SERVICE_NAMESPACE = knoe-dbnoe-system
 
 [Welcome]
 ; No configuration values captured yet for this section.
@@ -109,7 +110,7 @@ ANSIBLE_DOMAIN = prole.org
 ANSIBLE_INFRASTRUCTURE = ${PROLE_HOME}/infrastructure
 ANSIBLE_INVENTORY = ${PROLE_HOME}/infrastructure/inventory
 ANSIBLE_REALM = PROLE.ORG
-ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"","server_host":"","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
+ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
 KDC_ANSIBLE_DETECTED = 10.0.0.3
 KDC_AUTO_DETECTED = 10.0.0.3
 KERBEROS_AUTO_ENABLED = True
@@ -147,20 +148,20 @@ NAMESPACE = ${NAMESPACE}
 
 [Initialize Cluster]
 ENVIRONMENT = service
-K3S_SERVER_URL = 
+K3S_SERVER_URL = https://myrddin.prole.org:6443
-K3S_TOKEN = ${PROLE_SECRET:v1:38WOjNObQUt_S1W_:q7cZ6C5FcsiNPfxtmUI5RtsuJ4iaBAdvLbw_1hpOx87Cvv2KL-HjbdE7s5J4aPegFw02IrwWuPP5WXe5nGkg7uTXreCsvg5uWYXc1vxjWHUzKEwVfcP2nPEKBjJrRuai1VxD8WaPbwcUMQfXEtqNrFrkBSIICYltM1hmmA==}
+K3S_TOKEN = ${PROLE_SECRET:v1:i1FXxxDV0iTGUprA:Esd5SwMPbXNiWB1FeAPJqDVMhaIwg6TeR7aPH-IbF3HURxH6qs89IPQWhAAOmuFYLNCG_Fc8e7y6kTbIrV-md-fOVOZ8PbRHwGp_6u2w7O5WUf_QIdhxF30H-UQZlFZHOAHnW2l970O2zQQPGRJkqQq6VzoK2_P6tiJ8uA==}
 
 [Dev Cluster (k3d)]
 CLUSTER_ENV = dev
 DISPLAY_NAME = knoe-dev-cluster
-KUBECTL_CONTEXT = prole-k3s
+KUBECTL_CONTEXT = prole-knoe-db3s
 MODE = k3d
 
 [Service Cluster (k3s)]
 CLUSTER_ENV = prole-service-cluster
 DISPLAY_NAME = prole-service-cluster
-K3S_SERVER_URL = 
+K3S_SERVER_URL = https://myrddin.prole.org:6443
-K3S_TOKEN = ${PROLE_SECRET:v1:38WOjNObQUt_S1W_:q7cZ6C5FcsiNPfxtmUI5RtsuJ4iaBAdvLbw_1hpOx87Cvv2KL-HjbdE7s5J4aPegFw02IrwWuPP5WXe5nGkg7uTXreCsvg5uWYXc1vxjWHUzKEwVfcP2nPEKBjJrRuai1VxD8WaPbwcUMQfXEtqNrFrkBSIICYltM1hmmA==}
+K3S_TOKEN = ${PROLE_SECRET:v1:i1FXxxDV0iTGUprA:Esd5SwMPbXNiWB1FeAPJqDVMhaIwg6TeR7aPH-IbF3HURxH6qs89IPQWhAAOmuFYLNCG_Fc8e7y6knoe-dbTbIrV-md-fOVOZ8PbRHwGp_6u2w7O5WUf_QIdhxF30H-UQZlFZHOAHnW2l970O2zQQPGRJknoe-dbqQq6VzoK2_P6tiJ8uA==}
 MODE = k3s
 PIPELINE_URL = http://127.0.0.1:8080
 
@@ -172,7 +173,8 @@ MODE = k8s
 PIPELINE_URL = http://127.0.0.1:8080
 
 [Docker Build]
-; No configuration values captured yet for this section.
+LOCAL_REGISTRY = localhost:5000
+LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000
 
 [Initialization Scripts]
 ; No configuration values captured yet for this section.

deploy/opentofu/k3s/manifests/prole/prole-db.yaml

@@ -3,7 +3,7 @@ kind: Cluster
 metadata:
   name: prole-db
 spec:
-  instances: 3
+  instances: 2
   imageName: myrddin.prole.org:5000/prole-db:17.7-059
   postgresUID: 100
   postgresGID: 101

etc/init_k3s_registry.sh

@@ -45,7 +45,7 @@ mirrors:
       - "http://${host}:${port}"
   "registry.${ns}.svc.cluster.local:${port}":
     endpoint:
-      - "http://registry.${ns}.svc.cluster.local:${port}"
+      - "http://${host}:${port}"
 configs:
   "${host}:${port}":
     tls:

installer/core/actions.py

@@ -522,7 +522,48 @@ class ProleInstallerBase:
             return "prole-db"
         return cleaned
 
+    def _read_existing_cfg_namespace(self) -> str | None:
+        """Best-effort read of NAMESPACE from the existing prole.cfg.
+
+        Silent retries should reuse the namespace stored in prole.cfg unless the
+        user explicitly overrides via env vars.
+        """
+
+        cfg_path = getattr(self, "cfg_path", None)
+        if not cfg_path:
+            return None
+        try:
+            p = Path(str(cfg_path)).expanduser()
+            if p.is_dir():
+                p = p / "prole.cfg"
+            if not p.exists():
+                return None
+
+            cfg = configparser.ConfigParser(interpolation=None)
+            cfg.optionxform = str
+            cfg.read(p)
+
+            # Current canonical location
+            for section, key in (
+                ("User", "NAMESPACE"),
+                ("User", "PROLE_NAMESPACE"),
+                # Legacy fallbacks
+                ("Database Creation", "NAMESPACE"),
+                ("Database Creation", "DB_NAME"),
+            ):
+                if cfg.has_option(section, key):
+                    ns = (cfg.get(section, key, fallback="") or "").strip()
+                    if ns:
+                        return ns
+        except Exception:
+            return None
+        return None
+
     def _initial_namespace(self) -> str:
+        # Explicit overrides must win.
+        ns = os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE")
+        if ns:
+            return self._ensure_namespace_prefix(ns)
         try:
             existing = self._read_existing_env()
             ns = existing.get("NAMESPACE") or existing.get("PROLE_NAMESPACE")
@@ -530,7 +571,9 @@ class ProleInstallerBase:
                 return self._ensure_namespace_prefix(ns)
         except Exception:
             pass
-        ns = os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE")
+
+        # Prefer the namespace already stored in prole.cfg to make retries stable.
+        ns = self._read_existing_cfg_namespace()
         if ns:
             return self._ensure_namespace_prefix(ns)
         return self._ensure_namespace_prefix(self._generate_namespace_name())

installer/core/env.py

@@ -1047,8 +1047,31 @@ def _detect_ansible_k3s_settings(
     inventory_path: Path, groups: dict, domain: str, ip_map: dict
 ) -> dict:
     k3s_hosts = groups.get("k3s_hosts") or []
+    # Support inventories that declare hosts via a children group, e.g.:
+    #   [k3s_hosts:children]
+    #   k3s_servers
+    #   k3s_agents
+    if not k3s_hosts:
+        expanded: list[str] = []
+        for child in (groups.get("k3s_hosts:children") or []):
+            expanded.extend(groups.get(child) or [])
+        if not expanded:
+            expanded.extend(groups.get("k3s_servers") or [])
+            expanded.extend(groups.get("k3s_agents") or [])
+        # Deduplicate while preserving order.
+        seen: set[str] = set()
+        k3s_hosts = []
+        for h in expanded:
+            if not h or h in seen:
+                continue
+            seen.add(h)
+            k3s_hosts.append(h)
     server_url = ""
     server_host = ""
+    # If an explicit servers group exists, prefer it as the default server host.
+    servers = groups.get("k3s_servers") or []
+    if servers:
+        server_host = servers[0]
     for host in k3s_hosts:
         host_vars_path = inventory_path / "host_vars" / f"{host}.yml"
         vals = _parse_yaml_scalar_values(
@@ -1062,6 +1085,10 @@ def _detect_ansible_k3s_settings(
                 or vals.get("k3s_role") == "server"
             ):
                 server_host = host
+    if not server_host and servers:
+        server_host = servers[0]
+    if not server_host and k3s_hosts:
+        server_host = k3s_hosts[0]
     if not server_url and server_host:
         host_for_url = server_host
         if domain and "." not in host_for_url:

k8s/prole/prole-db-recovery.yaml.tpl

@@ -5,7 +5,7 @@ metadata:
   name: prole-db
 spec:
   instances: 3
-  imageName: myrddin.prole.org:5000/prole-db:18-105
+  imageName: myrddin.prole.org:5000/prole-db:17.7-106
   postgresUID: 100
   postgresGID: 101
   maxSyncReplicas: 1

k8s/prole/prole-db.yaml

@@ -3,8 +3,8 @@ kind: Cluster
 metadata:
   name: prole-db
 spec:
-  instances: 3
+  instances: 2
-  imageName: myrddin.prole.org:5000/prole-db:18-105
+  imageName: myrddin.prole.org:5000/prole-db:17.7-106
   postgresUID: 100
   postgresGID: 101
   maxSyncReplicas: 1

mock_val/init_k3s_registry.sh

@@ -45,7 +45,7 @@ mirrors:
       - "http://${host}:${port}"
   "registry.${ns}.svc.cluster.local:${port}":
     endpoint:
-      - "http://registry.${ns}.svc.cluster.local:${port}"
+      - "http://${host}:${port}"
 configs:
   "${host}:${port}":
     tls:

prole-db/.version

@@ -1 +1 @@
-106
+107

tests/conftest.py

@@ -0,0 +1,149 @@
+"""Global pytest configuration.
+
+CI/autobuild must run unattended and non-interactive.
+
+We do **not** globally replace the whole `tkinter` module because some tests
+exercise real widget layout, but we *do* stub popup-oriented modules
+(`tkinter.messagebox`, `tkinter.filedialog`) so no test can block on a dialog.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from unittest.mock import MagicMock
+
+import pytest
+
+try:
+    import tkinter as tk
+except Exception:  # pragma: no cover
+    tk = None  # type: ignore[assignment]
+
+
+def _configure_messagebox(msg: MagicMock) -> None:
+    # Confirmation-style prompts: auto-accept.
+    for fn in (
+        "askyesno",
+        "askokcancel",
+        "askretrycancel",
+        "askyesnocancel",
+    ):
+        getattr(msg, fn).return_value = True
+
+    # Informational prompts: no-op.
+    for fn in (
+        "showinfo",
+        "showwarning",
+        "showerror",
+    ):
+        getattr(msg, fn).return_value = None
+def _install_popup_stubs() -> dict[str, MagicMock]:
+    messagebox = MagicMock(name="tkinter.messagebox")
+    filedialog = MagicMock(name="tkinter.filedialog")
+    _configure_messagebox(messagebox)
+
+    # Seed sys.modules so `from tkinter import messagebox, filedialog` picks up
+    # these stubs without requiring a real UI.
+    sys.modules["tkinter.messagebox"] = messagebox
+    sys.modules["tkinter.filedialog"] = filedialog
+
+    # If tkinter is already imported, ensure it points at our stub submodules.
+    tk_mod = sys.modules.get("tkinter")
+    if tk_mod is not None:
+        try:
+            setattr(tk_mod, "messagebox", messagebox)
+            setattr(tk_mod, "filedialog", filedialog)
+        except Exception:
+            pass
+
+    return {
+        "tkinter.messagebox": messagebox,
+        "tkinter.filedialog": filedialog,
+    }
+
+
+_POPUP_STUBS = _install_popup_stubs()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _tk_default_root_session():
+    """Ensure a default Tkinter root exists for tests that create variables.
+
+    Some UI logic uses `tkinter.StringVar`/`BooleanVar` without explicitly
+    providing a master, which requires a default root. In headless contexts we
+    prefer a `Tcl()` interpreter fallback.
+    """
+
+    if tk is None:
+        yield
+        return
+
+    existing = getattr(tk, "_default_root", None)
+    if existing is not None:
+        yield
+        return
+
+    root = None
+    try:
+        root = tk.Tk()
+        try:
+            root.withdraw()
+        except Exception:
+            pass
+    except Exception:
+        try:
+            root = tk.Tcl()
+            try:
+                tk._default_root = root  # type: ignore[attr-defined]
+            except Exception:
+                pass
+        except Exception:
+            root = None
+
+    try:
+        yield
+    finally:
+        try:
+            if root is not None and hasattr(root, "destroy"):
+                root.destroy()
+        except Exception:
+            pass
+        try:
+            tk._default_root = None  # type: ignore[attr-defined]
+        except Exception:
+            pass
+
+
+def _stub_module(name: str, *, is_package: bool = False) -> types.ModuleType:
+    m = types.ModuleType(name)
+    if is_package:
+        # Mark as a package so submodule imports can succeed.
+        m.__path__ = []  # type: ignore[attr-defined]
+    return m
+
+
+def _ensure_importable_or_stub(name: str, *, is_package: bool = False) -> None:
+    if name in sys.modules:
+        return
+    try:
+        __import__(name)
+    except Exception:
+        sys.modules[name] = _stub_module(name, is_package=is_package)
+
+
+# Optional UI/OS modules: only stub when unavailable (keeps real deps when present).
+_ensure_importable_or_stub("Foundation")
+_ensure_importable_or_stub("AppKit")
+_ensure_importable_or_stub("PIL", is_package=True)
+_ensure_importable_or_stub("PIL.Image")
+_ensure_importable_or_stub("PIL.ImageTk")
+
+
+@pytest.fixture(autouse=True)
+def _reset_tkinter_stubs_between_tests():
+    # Avoid test-order coupling by resetting call history between tests.
+    for mod in _POPUP_STUBS.values():
+        mod.reset_mock()
+    _configure_messagebox(_POPUP_STUBS["tkinter.messagebox"])
+    yield

tests/installer/test_env_helpers.py

@@ -20,6 +20,7 @@ from installer.core.env import (
     _clean_yaml_value,
     _cluster_env_radio_value,
     _collect_images_from_files,
+    _detect_ansible_k3s_settings,
     _deployment_mode_from_env,
     _deployment_target_label,
     _expand_cfg_value,
@@ -457,6 +458,28 @@ class TestParseInventory:
         assert groups["grp"] == ["host1"]
 
 
+class TestDetectAnsibleK3sSettings:
+    def test_children_groups_fallback_to_servers(self, tmp_path):
+        inventory_path = tmp_path / "inventory"
+        (inventory_path / "host_vars").mkdir(parents=True)
+
+        # Host vars are intentionally empty; the detection should still infer a
+        # server host from the k3s_servers group.
+        (inventory_path / "host_vars" / "myrddin.prole.org.yml").write_text("\n")
+        (inventory_path / "host_vars" / "pi.prole.org.yml").write_text("\n")
+
+        groups = {
+            "k3s_hosts:children": ["k3s_servers", "k3s_agents"],
+            "k3s_servers": ["myrddin.prole.org"],
+            "k3s_agents": ["pi.prole.org"],
+        }
+        res = _detect_ansible_k3s_settings(
+            inventory_path=inventory_path, groups=groups, domain="prole.org", ip_map={}
+        )
+        assert res["server_host"] == "myrddin.prole.org"
+        assert res["server_url"] == "https://myrddin.prole.org:6443"
+
+
 # ===== _resolve_host_ip =====
 
 

tests/test_reconcile_blocked_cluster_state.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+import installer.core.actions as actions
+from installer.core.actions import ProleSilentInstaller
+from installer.core.controller import ProleController
+
+
+def _mk_installer(tmp_path) -> ProleSilentInstaller:
+    c = ProleController(tmp_path)
+    return ProleSilentInstaller(c)
+
+
+def test_repair_flow_classifies_blockers_attempts_reclaim_and_avoids_blind_health_wait(
+    tmp_path, monkeypatch, capsys
+):
+    installer = _mk_installer(tmp_path)
+    installer.inputs["init_cluster.cluster_env"] = "service"
+
+    # Avoid actually invoking any shell-based status/init scripts.
+    def run_script(*_a, **_k):
+        raise AssertionError("_run_script should not be called when hard blockers are present")
+
+    monkeypatch.setattr(installer, "_run_script", run_script)
+    monkeypatch.setattr(installer, "_ensure_namespace_ready", lambda *_a, **_k: None)
+    monkeypatch.setattr(
+        installer,
+        "_script_env_for_namespace",
+        lambda ns: {"SERVICE_NAMESPACE": ns, "DB_NAMESPACE": ns, "NAMESPACE": ns},
+    )
+
+    service_ns = installer._service_namespace()
+
+    # Model the real-world unschedulable pattern: affinity + taint + PV binding.
+    unsched_msg = (
+        "0/3 nodes are available: "
+        "1 node(s) didn't match Pod's node affinity/selector, "
+        "1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, "
+        "1 node(s) didn't find available persistent volumes to bind."
+    )
+
+    def fake_kubectl_get_json(_env, args):
+        if args[:3] == ["get", "pvc", "-A"]:
+            return {
+                "items": [
+                    {
+                        "metadata": {
+                            "name": "data-garage-0",
+                            "namespace": service_ns,
+                            "uid": "new-garage-uid",
+                        },
+                        "spec": {
+                            "storageClassName": "prole-iscsi",
+                            "accessModes": ["ReadWriteOnce"],
+                            "resources": {"requests": {"storage": "20Gi"}},
+                        },
+                        "status": {"phase": "Pending"},
+                    },
+                    {
+                        "metadata": {
+                            "name": "data-openbao-0",
+                            "namespace": service_ns,
+                            "uid": "new-bao-uid",
+                        },
+                        "spec": {
+                            "storageClassName": "prole-iscsi",
+                            "accessModes": ["ReadWriteOnce"],
+                            "resources": {"requests": {"storage": "20Gi"}},
+                        },
+                        "status": {"phase": "Pending"},
+                    },
+                ]
+            }
+        if args[:2] == ["get", "pv"]:
+            return {
+                "items": [
+                    {
+                        "metadata": {"name": "prole-iscsi-d001-garage"},
+                        "spec": {
+                            "storageClassName": "prole-iscsi",
+                            "capacity": {"storage": "20Gi"},
+                            "accessModes": ["ReadWriteOnce"],
+                            "claimRef": {
+                                "name": "data-garage-0",
+                                "namespace": service_ns,
+                                "uid": "old-garage-uid",
+                            },
+                        },
+                        "status": {"phase": "Released"},
+                    },
+                    {
+                        "metadata": {"name": "prole-iscsi-d001-openbao"},
+                        "spec": {
+                            "storageClassName": "prole-iscsi",
+                            "capacity": {"storage": "20Gi"},
+                            "accessModes": ["ReadWriteOnce"],
+                            "claimRef": {
+                                "name": "data-openbao-0",
+                                "namespace": service_ns,
+                                "uid": "old-bao-uid",
+                            },
+                        },
+                        "status": {"phase": "Released"},
+                    },
+                ]
+            }
+        if args[:3] == ["get", "pod", "-A"]:
+            return {
+                "items": [
+                    {
+                        "metadata": {"name": "garage-0", "namespace": service_ns},
+                        "status": {
+                            "phase": "Pending",
+                            "conditions": [
+                                {
+                                    "type": "PodScheduled",
+                                    "status": "False",
+                                    "reason": "Unschedulable",
+                                    "message": unsched_msg,
+                                }
+                            ],
+                        },
+                    },
+                    {
+                        "metadata": {"name": "openbao-0", "namespace": service_ns},
+                        "status": {
+                            "phase": "Pending",
+                            "conditions": [
+                                {
+                                    "type": "PodScheduled",
+                                    "status": "False",
+                                    "reason": "Unschedulable",
+                                    "message": unsched_msg,
+                                }
+                            ],
+                        },
+                    },
+                    {
+                        "metadata": {"name": "opentofu-abc", "namespace": service_ns},
+                        "status": {
+                            "phase": "Pending",
+                            "containerStatuses": [
+                                {
+                                    "name": "opentofu",
+                                    "state": {"waiting": {"reason": "ContainerCreating"}},
+                                }
+                            ],
+                        },
+                    },
+                ]
+            }
+        return {"items": []}
+
+    monkeypatch.setattr(installer, "_kubectl_get_json", fake_kubectl_get_json)
+
+    calls: list[list[str]] = []
+
+    def fake_run(cmd, **kwargs):
+        calls.append(list(cmd))
+
+        # Missing downstream Kong resources.
+        if cmd[:4] == ["kubectl", "-n", service_ns, "get"] and cmd[4:6] in (
+            ["svc", "prole-svc-kong"],
+            ["deploy", "prole-svc-kong"],
+        ):
+            return SimpleNamespace(returncode=1, stdout="", stderr="not found")
+
+        # PV reclaim (claimRef clearing)
+        if cmd[:3] == ["kubectl", "patch", "pv"]:
+            return SimpleNamespace(returncode=0, stdout="ok", stderr="")
+
+        return SimpleNamespace(returncode=0, stdout="ok", stderr="")
+
+    monkeypatch.setattr(actions.subprocess, "run", fake_run)
+
+    with pytest.raises(Exception) as exc:
+        installer._deploy_common_services(env_key="service")
+
+    msg = str(exc.value)
+    assert "Cluster prerequisites are blocked" in msg
+
+    # Reclaim was attempted for matching stale Released PVs.
+    patched = [c for c in calls if c[:3] == ["kubectl", "patch", "pv"]]
+    assert any("prole-iscsi-d001-garage" in c for c in patched)
+    assert any("prole-iscsi-d001-openbao" in c for c in patched)
+
+    # Explicit scheduling/storage classification is surfaced.
+    assert "Storage blockers" in msg
+    assert "unschedulable_due_to_pv_binding" in msg
+    assert "unschedulable_due_to_node_affinity" in msg
+    assert "unschedulable_due_to_untolerated_taint" in msg
+    assert "container_creating" in msg
+
+    # Downstream missingness is present, but labeled as downstream.
+    assert "Downstream missing resources" in msg
+    assert "prole-svc-kong" in msg
+
+    # Strict policy: never delete PVs.
+    assert not any(c[:3] == ["kubectl", "delete", "pv"] for c in calls)
+
+
+def test_classify_blocked_pod_storage_first_when_multiple_unschedulable_reasons(tmp_path):
+    installer = _mk_installer(tmp_path)
+
+    msg = (
+        "0/3 nodes are available: "
+        "1 node(s) didn't match Pod's node affinity/selector, "
+        "1 node(s) had untolerated taint {x: y}, "
+        "1 node(s) didn't find available persistent volumes to bind."
+    )
+    pod = {
+        "metadata": {"name": "garage-0", "namespace": "knoe-system"},
+        "status": {
+            "phase": "Pending",
+            "conditions": [
+                {
+                    "type": "PodScheduled",
+                    "status": "False",
+                    "reason": "Unschedulable",
+                    "message": msg,
+                }
+            ],
+        },
+    }
+
+    classified = installer._classify_blocked_pod(env={}, pod=pod)
+    assert classified["type"] == "unschedulable_due_to_pv_binding"
+    assert set(classified["types"]) >= {
+        "unschedulable_due_to_pv_binding",
+        "unschedulable_due_to_node_affinity",
+        "unschedulable_due_to_untolerated_taint",
+    }