""" Installer actions and unattended workflow helpers. """ from __future__ import annotations import json import logging import secrets import shlex import socket import string import sys import threading import uuid from typing import Callable, Sequence from knoe.config import ( _expand_path, _collect_cfg_vars, _expand_cfg_value, _parse_bool, _is_openbao_ref, _is_prole_secret, _decrypt_prole_secret, _encrypt_prole_secret, _write_k3s_kubeconfig, _encrypt_cfg_secret, _merge_kubeconfig, ) from knoe import prole_conf as prole_conf_mgr from knoe.core.controller import ProleController from knoe.core.env import * # noqa: F401,F403 from knoe.core.env import ( _parse_ollama_host, _bool_str, _deployment_mode_from_env, _openbao_placeholder, _looks_like_k8s_bearer_token, _find_kubeconfig_file, _normalize_cluster_env, _pf_upsert_mapping, _build_required_port_forwards, _local_registry_enabled, _collect_images_from_files, _http_ping_registry, _registry_image_ref_exists, _resolve_supabase_home, _push_docker_image, _detect_ansible_topology, _detect_ansible_storage_mounts, _detect_ansible_node_labels, _deployment_target_label, _default_opentofu_pipeline_url, _format_ollama_host, _host_from_url, _render_prole_cfg, _k3d_prole_data_volume_args, _sync_opentofu_pipeline, _normalize_k3s_token, _read_k3s_cfg, _safe_str, _expand_cfg_value as _expand_env_cfg_value, _resolve_k3s_connection as _resolve_k3s_connection_fn, _kubectl_base_cmd_for_k3s as _kubectl_base_cmd_for_k3s_fn, _verify_k3s_services_status, ) from knoe.core.milestones import ( DependenciesMilestone, NetworkScanMilestone, EnvSetupMilestone, SecretManagementMilestone, DatabaseCreationMilestone, DockerBuildMilestone, ClusterLifecycleMilestone, InitializationScriptsMilestone, KerberosMilestone, DeploymentMilestone, GitOpsMilestone, SupabaseImagePreloadMilestone, SupabaseMilestone, ) from knoe.core.stream_exec import run_streaming_cmd from knoe.core.build_context import copy_build_context_dir from knoe.core.cnpg_placement import ( load_cnpg_placement_plan, plan_cnpg_placement, save_cnpg_placement_plan, ) from knoe.core.policy import ( POLICY_CFG_KEY, OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES, evaluate_optional_workloads_allowed, ) def _configure_unbuffered_io(): os.environ.setdefault("PYTHONUNBUFFERED", "1") for stream in (sys.stdout, sys.stderr): try: stream.reconfigure(line_buffering=True, write_through=True) except Exception: try: stream.flush() except Exception: pass def _kubeconfig_cluster_info_ok(kubeconfig: Path, timeout_s: int = 8) -> bool: try: res = subprocess.run( ["kubectl", "--kubeconfig", str(kubeconfig), "cluster-info"], capture_output=True, text=True, timeout=timeout_s, ) return res.returncode == 0 except Exception: return False def _select_existing_kubeconfig(candidates: Sequence[Path]) -> Path | None: """Return the first kubeconfig that exists and can reach/auth the cluster. If none of the existing candidates pass `kubectl cluster-info` (e.g. cluster not yet reachable), fall back to the first existing candidate to preserve previous behavior. """ first_existing: Path | None = None for kc in candidates: try: if not kc.exists(): continue except Exception: continue if first_existing is None: first_existing = kc if _kubeconfig_cluster_info_ok(kc): return kc return first_existing class ProleInstaller: """Shared business logic for both silent and interactive installer modes. Subclasses must provide a :meth:`_get_input` implementation that bridges their respective data-access layers (``dict`` for silent, tk ``StringVar`` for the GUI). All common helpers – namespace, env, secrets, port-forwards, script-env, logging – live here so they are defined exactly once. """ # Subclasses should set this to ``True`` for headless operation. silent: bool = False # ------------------------------------------------------------------ init def _init_shared_state(self): """Initialise state common to every installer flavour.""" self.prole_cfg_data = { "Global": {}, "Welcome": {}, "Dependencies": {}, "Network": {}, "Storage": {}, "Port Forwards": {}, "System Environment": {}, "Monitoring": {}, "Kerberos Authentication": {}, "Ollama": {}, "Optional Features": {}, "Database Options": {}, "Database Creation": {}, "Docker Build": {}, "Initialize Cluster": {}, "Initialization Scripts": {}, "GitOps": {}, "Deployment": {}, "Dev Cluster (k3d)": {}, "Service Cluster (k3s)": {}, "Prod Cluster (k8s)": {}, "Install": {}, } self._cfg_secret_cache: dict = {} self._secrets_finalized: bool = False self._managed_kubeconfig: str | None = None self._repair_ran: bool = False self._optional_workloads_policy_cache: tuple[bool, int, str] | None = None def _optional_workloads_policy(self, env: dict) -> tuple[bool, int, str]: """Evaluate and cache the optional-workloads policy. Central rule: `optional_workloads_allowed = ready_schedulable_nodes >= min_required` `min_required` is loaded from `prole.cfg` Global key `OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES`. """ if self._optional_workloads_policy_cache is not None: return self._optional_workloads_policy_cache raw = ( (self.prole_cfg_data.get("Global", {}) or {}).get(POLICY_CFG_KEY, "") ) try: min_nodes = int(str(raw).strip() or str(OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES)) except Exception: min_nodes = OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES allowed, ready_count, reason = evaluate_optional_workloads_allowed( env=env, min_nodes=min_nodes ) self._optional_workloads_policy_cache = (allowed, ready_count, reason) self.log(f"[POLICY] {reason}") return self._optional_workloads_policy_cache # --------------------------------------------------------- data access def _get_input(self, key: str, default: str | None = None) -> str: """Return user input for *key*. Subclasses **must** override.""" raise NotImplementedError def _get_input_bool(self, key: str, default: bool = False) -> bool: return _parse_bool(self._get_input(key, None), default=default) # --------------------------------------------------------- logging def log(self, msg: str): print(msg, flush=True) logging.info(msg) def err(self, msg: str): print(msg, file=sys.stderr, flush=True) logging.error(msg) # ------------------------------------------------ deployment helpers def _deployment_mode(self) -> str: return _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) def _secret_namespace(self) -> str: ns = (self._get_input("init_password.db_namespace", "") or "").strip() if not ns: ns = (self._get_input("env_setup.NAMESPACE", "") or "").strip() if not ns: ns = ((self.prole_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "") or "").strip() if ns.startswith("${") and ns.endswith("}"): ns = "" return ns or "default" def _service_namespace(self) -> str: ns = ( (self.prole_cfg_data.get("Global", {}) or {}) .get("SERVICE_NAMESPACE", "") .strip() ) if ns.startswith("${") and ns.endswith("}"): ns = "" if not ns: ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip() if ns: return ns # In k3s service clusters, core services live in knoe-system by default. try: if self._deployment_mode() == "k3s": return "knoe-system" except Exception: pass return "default" # ------------------------------------------ k3s connection helpers def _read_k3s_cfg_values( self, cfg_path: "Path | None" = None ) -> tuple[str, str, str]: """Return (cluster_env, server_url, token) from prole.cfg. Delegates to the module-level ``_read_k3s_cfg`` so that every presentation layer (Tk, ncurses, silent) uses the same logic. """ if cfg_path is None: cfg_path = getattr(self, "cfg_path", None) return _read_k3s_cfg(cfg_path) def _resolve_k3s_connection(self) -> tuple[str, str]: """Resolve k3s server URL and token from all sources. Delegates to the module-level ``_resolve_k3s_connection`` so that every presentation layer uses identical resolution order. """ cfg_path = getattr(self, "cfg_path", None) root = getattr(self, "project_root", None) or PROJECT_ROOT topology = getattr(self, "ansible_topology", None) return _resolve_k3s_connection_fn( project_root=root, cfg_path=cfg_path, ansible_topology=topology, ) def _kubectl_base_cmd_for_k3s(self) -> list[str]: """Build a kubectl command prefix suitable for the k3s service cluster. Delegates to the module-level ``_kubectl_base_cmd_for_k3s``. """ managed = getattr(self, "_managed_kubeconfig", None) root = getattr(self, "project_root", None) or PROJECT_ROOT return _kubectl_base_cmd_for_k3s_fn( managed_kubeconfig=managed, project_root=root, ) def _kubectl_base_cmd(self, mode: str | None = None) -> list[str]: """Build a consistent kubectl command based on the current cluster environment. Prefers environment-specific base commands (k3s/k8s) or KUBECONFIG, and ensures the selected context is used if provided. """ if not mode: mode = self._deployment_mode() # 1. Start with environment-specific base command if mode == "k3s": cmd = self._kubectl_base_cmd_for_k3s() elif mode == "k8s": kubeconfig = _find_kubeconfig_file() if kubeconfig: cmd = ["kubectl", "--kubeconfig", kubeconfig] else: cmd = ["kubectl"] elif mode == "k3d": # For k3d, check if we have a managed or merged config kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip() if kubeconfig_env: cmd = ["kubectl", "--kubeconfig", kubeconfig_env] else: cmd = ["kubectl"] else: cmd = ["kubectl"] # 2. Add context if explicitly selected (and not already in the command) ctx = self._get_input("init_cluster.selected_kubectx", "").strip() if ctx and cmd[0] == "kubectl" and "--server" not in cmd and "--context" not in cmd: cmd.extend(["--context", ctx]) return cmd def ensure_db_k8s_secrets( self, namespace: str, db_password: str, *, log_fn=None, ) -> None: """Ensure DB/CNPG secrets exist in Kubernetes for *namespace*. `etc/init_cloudnative_pg.sh` expects the following secrets: - `knoe-db-user` with `username` and `password` - `knoe-db-superuser` with `username` and `password` - `cnpg-admin-key` with `admin.key` and `admin.pub` This uses the provided DB password (user-entered) and a locally generated ssh keypair (`~/.ssh/id_prole_ed25519`) to bootstrap the admin key files in `PROLE_SERVICE/secrets/`. Notes: - `knoe-db-user` and `knoe-db-superuser` are always applied/updated. - `cnpg-admin-key` is created only if missing (no silent rotation). - Secret manifests are never logged (to avoid leaking credentials). """ ns = (namespace or "").strip() if not ns: raise Exception("Database namespace cannot be empty.") pw = (db_password or "").strip() if pw.startswith("${"): resolved = self._resolve_secret_value(pw) if resolved and not resolved.startswith("${"): pw = resolved if not pw or pw.startswith("${"): raise Exception("Database password is required to ensure Kubernetes secrets.") log = log_fn or getattr(self, "log", None) or (lambda *_: None) env = self._script_env_for_namespace(ns) def _run( cmd: list[str], *, input_text: str | None = None, log_output: bool = True, ) -> subprocess.CompletedProcess: res = subprocess.run( cmd, input=input_text, text=True, capture_output=True, env=env, ) if log_output: out = (res.stdout or "").strip() err = (res.stderr or "").strip() if out: log(out) if err: log(err) return res def _ensure_namespace() -> None: chk = _run(["kubectl", "get", "namespace", ns], log_output=False) if chk.returncode == 0: return rc = _run(["kubectl", "create", "namespace", ns], log_output=True) if rc.returncode != 0: raise Exception(f"Failed to create namespace '{ns}'") def _apply_secret_from_create(create_cmd: list[str]) -> None: # 1) render YAML (do not log; contains secret material) rendered = _run(create_cmd, log_output=False) if rendered.returncode != 0 or not (rendered.stdout or "").strip(): raise Exception("Failed to render Kubernetes Secret manifest.") # 2) apply it (safe to log) applied = _run( ["kubectl", "apply", "-n", ns, "-f", "-"], input_text=rendered.stdout, log_output=True, ) if applied.returncode != 0: raise Exception("Failed to apply Kubernetes Secret manifest.") def _apply_literal_secret(name: str, data: dict[str, str]) -> None: args = ["kubectl", "create", "secret", "generic", name, "-n", ns] for k, v in data.items(): args.append(f"--from-literal={k}={v}") args.extend(["--dry-run=client", "-o", "yaml"]) _apply_secret_from_create(args) def _secret_exists(name: str) -> bool: res = _run(["kubectl", "-n", ns, "get", "secret", name], log_output=False) return res.returncode == 0 log(f"==> Ensuring DB secrets in namespace '{ns}'") _ensure_namespace() # Ensure expected admin key files exist on disk for CNPG tooling. service_dir = Path((env.get("PROLE_SERVICE") or "").strip()) secrets_dir = service_dir / "secrets" if service_dir else None if secrets_dir: secrets_dir.mkdir(parents=True, exist_ok=True) ssh_priv = Path.home() / ".ssh" / "id_prole_ed25519" ssh_pub = Path.home() / ".ssh" / "id_prole_ed25519.pub" if not (ssh_priv.exists() and ssh_pub.exists()): # Best-effort: generate if missing. try: self._generate_ssh_key(self._get_input("init_password.db_username", "prole").strip() or "prole") except Exception: pass if secrets_dir and ssh_priv.exists() and ssh_pub.exists(): # Write both generic and legacy filenames expected by scripts. try: admin_priv_generic = secrets_dir / "admin.key" admin_pub_generic = secrets_dir / "admin.pub" admin_priv_legacy = secrets_dir / "admin_ed25519" admin_pub_legacy = secrets_dir / "admin_ed25519.pub" if not admin_priv_generic.exists(): shutil.copyfile(ssh_priv, admin_priv_generic) os.chmod(admin_priv_generic, 0o600) if not admin_pub_generic.exists(): shutil.copyfile(ssh_pub, admin_pub_generic) if not admin_priv_legacy.exists(): shutil.copyfile(ssh_priv, admin_priv_legacy) os.chmod(admin_priv_legacy, 0o600) if not admin_pub_legacy.exists(): shutil.copyfile(ssh_pub, admin_pub_legacy) except Exception: pass # DB credentials secrets expected by CNPG init script. _apply_literal_secret("knoe-db-user", {"username": "prole", "password": pw}) _apply_literal_secret( "knoe-db-superuser", {"username": "postgres", "password": pw} ) # Admin key secret only if missing (avoid rotation). if not _secret_exists("cnpg-admin-key"): if not secrets_dir: raise Exception("PROLE_SERVICE is not set; cannot locate secrets directory for CNPG admin key.") admin_priv = secrets_dir / "admin.key" admin_pub = secrets_dir / "admin.pub" if not (admin_priv.exists() and admin_pub.exists()): raise Exception( "CNPG admin key files are missing (admin.key/admin.pub); cannot create cnpg-admin-key secret." ) create = [ "kubectl", "create", "secret", "generic", "cnpg-admin-key", "-n", ns, f"--from-file=admin.key={admin_priv}", f"--from-file=admin.pub={admin_pub}", "--dry-run=client", "-o", "yaml", ] _apply_secret_from_create(create) def _verify_k3s_services_core(self) -> dict[str, str]: """Check registry, OpenBao and OpenTofu on a remote k3s cluster. Delegates to the module-level ``_verify_k3s_services_status``. """ managed = getattr(self, "_managed_kubeconfig", None) root = getattr(self, "project_root", None) or PROJECT_ROOT return _verify_k3s_services_status( project_root=root, managed_kubeconfig=managed, ) def _deploy_k3s_services_core(self, controller=None, log_fn=None): """Deploy registry, OpenBao and OpenTofu to a remote k3s cluster. *controller* must expose ``run_script(name, args, env, on_line)``. Falls back to ``self.controller`` when not provided. """ controller = controller or getattr(self, "controller", None) if controller is None: raise RuntimeError("No controller available for script execution") server, token = self._resolve_k3s_connection() if not server or not token: if log_fn: log_fn("Missing k3s server URL or token.\n") return ns = getattr(self, "db_namespace", None) or "default" if hasattr(ns, "get"): ns = ns.get() ns = (ns or "default").strip() db_pw = getattr(self, "db_password", "") or "" if hasattr(db_pw, "get"): db_pw = db_pw.get() env = os.environ.copy() root = getattr(self, "project_root", None) or PROJECT_ROOT env["PROLE_HOME"] = str(root) env["PROLE_SERVICE"] = str(root) env["DB_PASSWORD"] = db_pw env["OPENTOFU_ADMIN_PASSWORD"] = db_pw env["PROLE_MODE"] = "k3s" # Resolve KUBECONFIG through the same path the Tk UI and silent # installer use: prefer existing cert-based file, fall back to # token-generated kubeconfig. kubeconfig = _find_kubeconfig_file(env) if not kubeconfig: try: kubeconfig = str(_write_k3s_kubeconfig(server, token)) except Exception: pass if kubeconfig: env["KUBECONFIG"] = kubeconfig try: _merge_kubeconfig(kubeconfig) except Exception: pass argocd_ns = env.get("ARGOCD_NAMESPACE") or "argocd" registry_ns = env.get("REGISTRY_NAMESPACE") or "default" env["ARGOCD_NAMESPACE"] = argocd_ns env["REGISTRY_NAMESPACE"] = registry_ns mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) try: controller.run_script( "init_registry.sh", args=[ "update", "--mode", mode, "-n", argocd_ns, "--registry-namespace", registry_ns, "--config", str(controller.cfg_path), ], env=env, on_line=log_fn, ) controller.run_script( "init_openbao.sh", args=[ "update", "--mode", mode, "-n", ns, "--config", str(controller.cfg_path), ], env=env, on_line=log_fn, ) controller.run_script( "init_opentofu.sh", args=[ "update", "--mode", mode, "-n", ns, "--config", str(controller.cfg_path), ], env=env, on_line=log_fn, ) except Exception as e: if log_fn: log_fn(f"Deploy error: {e}\n") # ------------------------------------------------- secret helpers def _secret_cfg_value( self, section: str, key: str, plaintext: str, leaf: str, bao_key: str ) -> str: if self._secrets_finalized: return _openbao_placeholder(self._secret_namespace(), leaf, bao_key) if not plaintext: return "" if _is_openbao_ref(plaintext) or _is_prole_secret(plaintext): return plaintext existing = (self.prole_cfg_data.get(section, {}) or {}).get(key) if existing and _is_prole_secret(existing): try: if _decrypt_prole_secret(existing) == plaintext: return existing except Exception: pass try: return _encrypt_prole_secret(plaintext) except Exception: return plaintext def _resolve_openbao_ref(self, value: str) -> str: if not _is_openbao_ref(value): return value inner = value[len(OPENBAO_PREFIX) : -len(OPENBAO_SUFFIX)] path, key = (inner.split("#", 1) + [""])[:2] if not path or not key: return "" mount = "kv" secret_path = path if "/" in path: maybe_mount, rest = path.split("/", 1) if maybe_mount: mount = maybe_mount secret_path = rest token = os.environ.get("OPENBAO_ROOT_TOKEN", "") if not token: prole_service = os.environ.get("PROLE_SERVICE") if prole_service: token_path = Path(prole_service) / "secrets" / "openbao-root-token" if token_path.exists(): token = token_path.read_text().strip() if not token: return value url = os.environ.get("PROLE_OPENBAO_URL") if not url: for p in ["8200", "18200"]: try: import urllib.request with urllib.request.urlopen( f"http://127.0.0.1:{p}/v1/sys/health", timeout=0.5 ) as r: if r.getcode() == 200: url = f"http://127.0.0.1:{p}" break except Exception: pass if not url: url = "http://127.0.0.1:8200" url = url.rstrip("/") try: req = urllib.request.Request(f"{url}/v1/{mount}/data/{secret_path}") req.add_header("X-Vault-Token", token) with urllib.request.urlopen(req, timeout=4) as resp: payload = json.loads(resp.read().decode("utf-8")) return payload.get("data", {}).get("data", {}).get(key, "") or "" except Exception: return value def _resolve_secret_value(self, value: str) -> str: if _is_prole_secret(value): return _decrypt_prole_secret(value) if _is_openbao_ref(value): return self._resolve_openbao_ref(value) return value # --------------------------------------------- cfg sanitisation def _sanitize_sections_for_cfg(self, sections: dict) -> dict: sanitized = {k: dict(v) for k, v in sections.items()} # Persist fully expanded values in prole.cfg (no $VAR / ${VAR} expansions). # Downstream tools treat these values as plain strings. env_map = dict(os.environ) try: env_map.update( {k: str(v or "") for k, v in (self._read_existing_env() or {}).items()} ) except Exception: pass for k in ( "PROLE_HOME", "PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE", "NAMESPACE", "SERVICE_NAMESPACE", ): val = (self._get_input(f"env_setup.{k}", "") or "").strip() if val: try: env_map[k] = self._expand_shell_path(val, env=env_map) except Exception: env_map[k] = val for section_name, section in sanitized.items(): for k, v in list(section.items()): if not isinstance(v, str) or not v.strip(): continue if _is_prole_secret(v) or _is_openbao_ref(v): continue try: section[k] = self._expand_shell_path(v, env=env_map) except Exception: section[k] = v if "Kerberos Authentication" in sanitized: val = sanitized["Kerberos Authentication"].get("PASSWORD", "") if val or ("Kerberos Authentication", "PASSWORD") in self._cfg_secret_cache: sanitized["Kerberos Authentication"]["PASSWORD"] = ( self._secret_cfg_value( "Kerberos Authentication", "PASSWORD", val, "kerberos", "password", ) ) if "Monitoring" in sanitized: val = sanitized["Monitoring"].get("GRAFANA_ADMIN_PASSWORD", "") if ( val or ("Monitoring", "GRAFANA_ADMIN_PASSWORD") in self._cfg_secret_cache ): sanitized["Monitoring"]["GRAFANA_ADMIN_PASSWORD"] = ( self._secret_cfg_value( "Monitoring", "GRAFANA_ADMIN_PASSWORD", val, "monitoring", "grafana_admin_password", ) ) return sanitized # ------------------------------------------- namespace helpers def _get_local_owner(self) -> str: try: return getpass.getuser() except Exception: try: return os.getlogin() except Exception: return "knoe" def _sanitize_namespace(self, name: str) -> str: cleaned = re.sub(r"[^a-z0-9-]+", "-", (name or "").lower()) cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-") if not cleaned: cleaned = "knoe-db" if len(cleaned) > 63: cleaned = cleaned[:63].rstrip("-") return cleaned def _generate_namespace_name(self) -> str: owner = self._sanitize_namespace(self._get_local_owner()) suffix = uuid.uuid4().hex[:6] base = f"knoe-db-{owner}-{suffix}" return self._sanitize_namespace(base) def _ensure_namespace_prefix(self, name: str) -> str: cleaned = (name or "").strip() if not cleaned: return "knoe-db" return cleaned def _read_existing_cfg_namespace(self) -> str | None: """Best-effort read of NAMESPACE from the existing prole.cfg. Silent retries should reuse the namespace stored in prole.cfg unless the user explicitly overrides via env vars. """ cfg_path = getattr(self, "cfg_path", None) if not cfg_path: return None try: p = Path(str(cfg_path)).expanduser() if p.is_dir(): p = p / "prole.cfg" if not p.exists(): return None cfg = configparser.ConfigParser(interpolation=None) cfg.optionxform = str cfg.read(p) # Current canonical location for section, key in ( ("User", "NAMESPACE"), ("User", "PROLE_NAMESPACE"), # Legacy fallbacks ("Database Creation", "NAMESPACE"), ("Database Creation", "DB_NAME"), ): if cfg.has_option(section, key): ns = (cfg.get(section, key, fallback="") or "").strip() if ns: return ns except Exception: return None return None def _initial_namespace(self) -> str: # Explicit overrides must win. ns = os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE") if ns: return self._ensure_namespace_prefix(ns) try: existing = self._read_existing_env() ns = existing.get("NAMESPACE") or existing.get("PROLE_NAMESPACE") if ns: return self._ensure_namespace_prefix(ns) except Exception: pass # Prefer the namespace already stored in prole.cfg to make retries stable. ns = self._read_existing_cfg_namespace() if ns: return self._ensure_namespace_prefix(ns) return self._ensure_namespace_prefix(self._generate_namespace_name()) def _initial_db_namespace(self) -> str: """Initial namespace suggestion for CNPG/Postgres. Unlike `_initial_namespace()`, this default is version-aware to encourage parallel installs (e.g. `knoe-db-18-142`). """ # Explicit overrides must win. base = ( os.environ.get("KNOE_DB_NAMESPACE") or os.environ.get("DB_NAMESPACE") or os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE") or "" ).strip() if not base: try: existing = self._read_existing_env() base = ( (existing.get("NAMESPACE") or "") or (existing.get("PROLE_NAMESPACE") or "") ).strip() except Exception: base = "" if not base: base = (self._read_existing_cfg_namespace() or "").strip() or "knoe-db" base = self._sanitize_namespace(self._ensure_namespace_prefix(base)) version = "" try: controller = getattr(self, "controller", None) if controller is not None: version = (controller.get_knoe_db_version() or "").strip() except Exception: version = "" if not version: return base version_ns = self._sanitize_namespace(version.replace(".", "-")) if base.endswith(f"-{version_ns}"): return base combined = f"{base}-{version_ns}" if len(combined) > 63: # Keep the version suffix intact and trim the base to fit. max_base_len = 63 - len(version_ns) - 1 if max_base_len < 1: return self._sanitize_namespace(f"knoe-db-{version_ns}") base_trim = base[:max_base_len].rstrip("-") combined = f"{base_trim}-{version_ns}" return self._sanitize_namespace(combined) def _is_valid_namespace(self, name: str) -> bool: if not name or len(name) > 63: return False if "db" not in name.lower(): return False return re.match(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", name) is not None def _generate_db_password(self, length: int = 24) -> str: alphabet = string.ascii_letters + string.digits return "".join(secrets.choice(alphabet) for _ in range(length)) # --------------------------------------------------- env helpers _FOREIGN_HOME_PREFIX_RE = re.compile(r"^/(Users|home)/([^/]+)(/.*)?$") def _rewrite_foreign_home_prefix(self, value: str) -> str: """Rewrite foreign home prefixes to the current user's home. Example: running on Linux with HOME=/home/, rewrite `/Users//dev/prole` -> `/home//dev/prole`. This helps prevent stale macOS paths (often stored in `~/.prole/env.sh`) from being persisted into `prole.cfg` on Linux. """ raw = (value or "").strip() if not raw or "$" in raw: return raw m = self._FOREIGN_HOME_PREFIX_RE.match(raw) if not m: return raw foreign_root = m.group(1) foreign_user = m.group(2) suffix = m.group(3) or "" try: current_home = Path.home() except Exception: return raw # Only rewrite when the path refers to the current username. if foreign_user != current_home.name: return raw system = platform.system() if system != "Darwin" and foreign_root == "Users": return str(current_home) + suffix if system == "Darwin" and foreign_root == "home": return str(current_home) + suffix return raw def _shellify_home_path(self, value: str) -> str: """Represent paths under the current home directory using `$HOME`. Keeps values unchanged if they already contain shell variables. """ raw = (value or "").strip() if not raw or "$" in raw: return raw try: p = Path(raw).expanduser() home = Path.home() rel = p.relative_to(home) except Exception: return raw rel_str = rel.as_posix() if not rel_str or rel_str == ".": return "$HOME" return f"$HOME/{rel_str}" def _expand_shell_path(self, value: str, env: dict[str, str] | None = None) -> str: raw = (value or "").strip() if not raw: return "" env_map = env or os.environ expanded = _expand_env_cfg_value(raw, env=env_map, max_depth=10) return os.path.expanduser(expanded) def _cfgify_home_path(self, value: str) -> str: """Represent paths under the current home directory using `${HOME}`. Also rewrites stale foreign-home paths like `/Users//...` when the username matches the current user. """ raw = (value or "").strip() if not raw or "$" in raw: return raw raw = self._rewrite_foreign_home_prefix(raw) try: p = Path(raw).expanduser() home = Path.home() rel = p.relative_to(home) except Exception: return raw rel_str = rel.as_posix() if not rel_str or rel_str == ".": return "${HOME}" return f"${{HOME}}/{rel_str}" def _env_defaults(self, namespace: str | None = None) -> dict: # Default install layout: ~/.prole # NOTE: UI may present this as $HOME/.prole, but internally we keep # resolved literal paths for filesystem operations. default_home = resolve_prole_home(env={}) resolved_home = self._resolve_env_value("PROLE_HOME", str(default_home)) or str( default_home ) env_map = dict(os.environ) env_map["PROLE_HOME"] = resolved_home expanded_home = self._expand_shell_path(resolved_home, env=env_map) or str(default_home) home = Path(expanded_home) env_map["PROLE_HOME"] = str(home) if namespace is None: namespace = ( self._get_input("init_password.db_namespace", "") or "" ).strip() if not namespace: namespace = os.environ.get("NAMESPACE", "") return { "PROLE_HOME": str(home), "PROLE_CONF": self._expand_shell_path( self._resolve_env_value("PROLE_CONF", str(home / "conf")) or str(home / "conf"), env=env_map, ), "PROLE_DATA": self._expand_shell_path( self._resolve_env_value("PROLE_DATA", str(home / "data")) or str(home / "data"), env=env_map, ), "PROLE_LOGS": self._expand_shell_path( self._resolve_env_value("PROLE_LOGS", str(home / "logs")) or str(home / "logs"), env=env_map, ), "PROLE_SERVICE": self._expand_shell_path( self._resolve_env_value("PROLE_SERVICE", str(home / "etc")) or str(home / "etc"), env=env_map, ), "NAMESPACE": namespace or "", } def _resolve_env_value(self, key: str, fallback: str | None = None) -> str | None: val = os.environ.get(key) if val: return val try: env = self._read_existing_env() val = env.get(key) if val: return val except Exception: pass return fallback def _resolve_env_dir(self, key: str, default_suffix: str) -> Path: input_key = f"env_setup.{key}" val = (self._get_input(input_key, "") or "").strip() env_map = dict(os.environ) try: env_map.update({k: str(v or "") for k, v in (self._read_existing_env() or {}).items()}) except Exception: pass if val: try: return Path(self._expand_shell_path(val, env=env_map)) except Exception: pass val = self._resolve_env_value(key) if val: try: return Path(self._expand_shell_path(val, env=env_map)) except Exception: pass return Path.home() / "dev" / "prole" / default_suffix def _read_existing_env(self) -> dict: env = {} prole_home = os.environ.get("PROLE_HOME") candidates = [] if prole_home: try: candidates.append(Path(self._expand_shell_path(prole_home, env=os.environ)) / "env.sh") except Exception: candidates.append(Path(prole_home) / "env.sh") candidates.append(resolve_prole_home(env={}) / "env.sh") for p in candidates: try: if p.exists(): for line in p.read_text().splitlines(): line = line.strip() if not line or line.startswith("#"): continue if line.startswith("export "): line = line[len("export ") :] if "=" in line: k, v = line.split("=", 1) k = k.strip() v = v.strip().strip('"') if k in { "PROLE_HOME", "PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE", }: v = self._rewrite_foreign_home_prefix(v) env[k] = v break except Exception: pass return env def _save_env_to_file(self, values: dict): cluster_env = self._get_input("init_cluster.cluster_env", "dev") env_key = _normalize_cluster_env(cluster_env) env_map = dict(os.environ) try: env_map.update({str(k): str(v) for k, v in (values or {}).items() if v is not None}) except Exception: pass expanded_home = self._expand_shell_path(values.get("PROLE_HOME", ""), env=env_map) expanded_home = os.path.expanduser(expanded_home) if expanded_home else "" if expanded_home: env_map["PROLE_HOME"] = expanded_home home = Path(expanded_home) else: home = resolve_prole_home(env={}) if env_key == "dev": home.mkdir(parents=True, exist_ok=True) for key in ("PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE"): try: raw = values.get(key, "") expanded = self._expand_shell_path(raw, env=env_map) expanded = os.path.expanduser(expanded) if expanded else "" if expanded: env_map[key] = expanded Path(expanded).mkdir(parents=True, exist_ok=True) except Exception: pass content = [] content.append("#!/usr/bin/env bash") content.append("# Prole environment configuration") content.append( "# This file is generated by the installer. Source it in new shells, or execute as a wrapper:" ) content.append('# "$PROLE_HOME/env.sh" [args…]') content.append("# shellcheck shell=bash") for k in ( "PROLE_HOME", "PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE", ): content.append(f'export {k}="{self._shellify_home_path(values[k])}"') # NAMESPACE/PROLE_NAMESPACE must never be written to env.sh — only prole.cfg is the source of truth content.append("") content.append("# Ensure PATH works for GUI-launched shells (Docker, etc.)") content.append( '_prole_add_path() { case ":${PATH}:" in *":$1:"*) ;; *) PATH="$1:${PATH:-}" ;; esac; }' ) content.append('_prole_add_path "$PROLE_HOME/bin"') content.append('_prole_add_path "/opt/homebrew/bin"') content.append('_prole_add_path "/usr/local/bin"') content.append('_prole_add_path "/usr/bin"') content.append('_prole_add_path "/bin"') content.append('_prole_add_path "/usr/sbin"') content.append('_prole_add_path "/sbin"') content.append("export PATH") content.append("") content.append("# Add custom paths below if needed (examples):") content.append("") content.append( "# If executed with arguments (and not sourced), run them under this environment" ) content.append('if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [ "$#" -gt 0 ]; then') content.append(' exec "$@"') content.append("fi") new_content = "\n".join(content) + "\n" out = home / "env.sh" if env_key != "dev": out = resolve_prole_home(env={}) / "env.sh" out.parent.mkdir(parents=True, exist_ok=True) should_write = True if out.exists(): if out.read_text() == new_content: should_write = False self.log(f"[OK] {out} is up to date.") if should_write: tmp = out.parent / "env.sh.tmp" tmp.write_text(new_content) tmp.replace(out) try: os.chmod(out, 0o755) except Exception: pass self.log(f"[OK] Wrote {out}") self._deploy_env_resources(values) def _deploy_env_resources(self, values: dict): """Copy init scripts and etc directory into PROLE_HOME / PROLE_SERVICE.""" try: env_map = dict(os.environ) try: env_map.update({str(k): str(v) for k, v in (values or {}).items() if v is not None}) except Exception: pass expanded_home = self._expand_shell_path(values.get("PROLE_HOME", ""), env=env_map) expanded_home = os.path.expanduser(expanded_home) if expanded_home else "" if expanded_home: env_map["PROLE_HOME"] = expanded_home expanded_service = self._expand_shell_path(values.get("PROLE_SERVICE", ""), env=env_map) expanded_service = os.path.expanduser(expanded_service) if expanded_service else "" prole_home = Path(expanded_home) if expanded_home else resolve_prole_home(env={}) prole_service = Path(expanded_service or (prole_home / "etc")) init_pf_src_candidates = [ self.project_root / "src" / "prole" / "etc" / "init-port-forward.sh", self.project_root / "etc" / "init-port-forward.sh", ] init_pf_src = next((p for p in init_pf_src_candidates if p.exists()), None) if init_pf_src is not None: init_pf_dst = prole_home / "init-port-forward.sh" try: data = init_pf_src.read_bytes() init_pf_dst.write_bytes(data) os.chmod(init_pf_dst, 0o755) except Exception: pass etc_src_candidates = [ self.project_root / "src" / "prole" / "etc", self.project_root / "etc", ] etc_src = next((p for p in etc_src_candidates if p.exists()), None) if etc_src is not None: try: prole_service.mkdir(parents=True, exist_ok=True) if etc_src.resolve() != prole_service.resolve(): if prole_service.exists(): shutil.rmtree(prole_service, ignore_errors=True) shutil.copytree(etc_src, prole_service) except Exception: pass except Exception: pass def reload_env_from_shell(self) -> None: raw_home = self._get_input( "env_setup.PROLE_HOME", str(resolve_prole_home(env={})) ) expanded_home = self._expand_shell_path(raw_home, env=os.environ) expanded_home = os.path.expanduser(expanded_home) if expanded_home else "" home = Path(expanded_home) if expanded_home else resolve_prole_home(env={}) env_file = home / "env.sh" cmd = ( f"export PROLE_HOME={shlex.quote(str(home))}; " f"source {shlex.quote(str(env_file))}; " "env -0" ) try: out = subprocess.check_output(["bash", "-lc", cmd]) except Exception as e: raise Exception(f"Failed to reload environment: {e}") try: for raw in out.split(b"\x00"): if not raw: continue kv = raw.decode("utf-8", errors="ignore") if "=" not in kv: continue k, v = kv.split("=", 1) if k in ("PYTHONPATH", "PYTHONHOME"): continue os.environ[k] = v except Exception: pass def _update_env_namespace(self, namespace: str): # Namespace is persisted to prole.cfg only — never to env.sh or os.environ. # This method is intentionally a no-op for environment mutation. pass # --------------------------------------------- port-forward helpers def _add_port_mapping(self, mapping_str): mode = self._deployment_mode() prefix = ( "PORT_FORWARD_K3S_MAPPING_" if mode == "k3s" else "PORT_FORWARD_K3D_MAPPING_" ) pf_section = self.prole_cfg_data.get("Port Forwards", {}) if pf_section is None: pf_section = {} if not _pf_upsert_mapping(pf_section, prefix, mapping_str): return self.prole_cfg_data["Port Forwards"] = pf_section if hasattr(self, "_save_prole_cfg"): self._save_prole_cfg() elif hasattr(self, "_write_cfg"): self._write_cfg() def _process_script_output_line(self, line): if "GRAFANA_ADMIN_PASSWORD=" in line: pwd = line.split("GRAFANA_ADMIN_PASSWORD=")[1].strip() if pwd: mon = self.prole_cfg_data.get("Monitoring", {}) if mon is None: mon = {} mon["GRAFANA_ADMIN_PASSWORD"] = pwd self.prole_cfg_data["Monitoring"] = mon if hasattr(self, "_save_prole_cfg"): self._save_prole_cfg() elif hasattr(self, "_write_cfg"): self._write_cfg() if "PORT_FORWARD_MAPPING:" in line: mapping = line.split("PORT_FORWARD_MAPPING:")[1].strip() if mapping: self._add_port_mapping(mapping) def _sync_port_forward_mappings(self): mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if not mode: mode = "k3d" prefix = ( "PORT_FORWARD_K3S_MAPPING_" if mode == "k3s" else "PORT_FORWARD_K3D_MAPPING_" ) pf_section = self.prole_cfg_data.get("Port Forwards", {}) if pf_section is None: pf_section = {} for k, v in list(pf_section.items()): if isinstance(v, str) and ( "id=registry;" in v or v.strip() == "id=registry" ): del pf_section[k] service_ns = (self._service_namespace() or "").strip() or "default" db_ns = (self._get_input("init_password.db_namespace", "") or "").strip() if not db_ns: db_ns = ( (self.prole_cfg_data.get("Global", {}) or {}) .get("NAMESPACE", "") .strip() ) if not db_ns: db_ns = (os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE") or "").strip() if not db_ns: db_ns = "default" argocd_ns = ( (self.prole_cfg_data.get("Global", {}) or {}) .get("ARGOCD_NAMESPACE", "") .strip() ) if not argocd_ns: argocd_ns = (os.environ.get("ARGOCD_NAMESPACE") or "").strip() if not argocd_ns: argocd_ns = "argocd" db_host_port = (self._get_input("init_password.db_host_port", "") or "").strip() if not db_host_port: db_host_port = ( (self.prole_cfg_data.get("Global", {}) or {}) .get("DB_HOST_PORT", "") .strip() ) if not db_host_port: db_host_port = "5432" supabase_enabled = self._get_input_bool("init_cluster.supabase_enabled", False) supabase_ns = (os.environ.get("SUPABASE_NAMESPACE") or "").strip() if not supabase_ns: supabase_ns = ( (self.prole_cfg_data.get("Supabase", {}) or {}) .get("NAMESPACE", "") .strip() ) if not supabase_ns: supabase_ns = "supabase" gitops_enabled = self._get_input_bool("init_cluster.gitops_enabled", False) gitops_ns = (os.environ.get("GITEA_NAMESPACE") or "").strip() if not gitops_ns: gitops_ns = ( (self.prole_cfg_data.get("GitOps", {}) or {}) .get("GITOPS_NAMESPACE", "") .strip() ) if not gitops_ns: gitops_ns = (self._get_input("gitops.namespace", "") or "").strip() if not gitops_ns: gitops_ns = "gitea" mappings = _build_required_port_forwards( mode=mode, service_ns=service_ns, argocd_ns=argocd_ns, db_ns=db_ns, db_host_port=db_host_port, supabase_enabled=supabase_enabled, supabase_namespace=supabase_ns, gitops_enabled=gitops_enabled, gitops_namespace=gitops_ns, ) for mapping in mappings: _pf_upsert_mapping(pf_section, prefix, mapping) self.prole_cfg_data["Port Forwards"] = pf_section # --------------------------------------------- script env def _script_env_for_namespace(self, namespace: str) -> dict: namespace = _safe_str(namespace) env = os.environ.copy() root = getattr(self, "project_root", None) or PROJECT_ROOT env["PROLE_HOME"] = str(root) env["PROLE_SERVICE"] = str(root) env["NAMESPACE"] = namespace service_ns = self._service_namespace() env["PROLE_CONF"] = self._get_input( "env_setup.PROLE_CONF", str(root / "conf") ) env["PROLE_DATA"] = self._get_input( "env_setup.PROLE_DATA", str(root / "knoe-db" / "data") ) env["PROLE_LOGS"] = self._get_input( "env_setup.PROLE_LOGS", str(root / "logs") ) env["PROLE_SERVICE"] = self._get_input( "env_setup.PROLE_SERVICE", str(root / "etc") ) env["KNOE_DB_USER"] = self._get_input("init_password.db_username", "").strip() # Make the DB/root master password available to shell scripts. # OpenTofu bootstrap uses it to create the initial admin secret when missing. db_pw = (self._get_input("init_password.db_password", "") or "").strip() if db_pw: env.setdefault("DB_PASSWORD", db_pw) env.setdefault("OPENTOFU_ADMIN_PASSWORD", db_pw) mode = self._deployment_mode() if mode: env["PROLE_MODE"] = mode env["DEPLOYMENT_MODE"] = mode cluster_env = self._get_input("init_cluster.cluster_env", "dev") env["DEPLOYMENT_TARGET"] = _deployment_target_label(cluster_env) if mode == "k3s": # 1. Prefer an existing kubeconfig that already exists on disk # (e.g. Ansible-fetched with client-certificate auth). kc = _find_kubeconfig_file(env) if kc: env["KUBECONFIG"] = kc else: # 2. Try fetching from k3s via Ansible or generate token-based fetched = self._fetch_k3s_kubeconfig() if fetched: env["KUBECONFIG"] = str(fetched) else: server, token = self._resolve_k3s_connection() if server and token and _looks_like_k8s_bearer_token(token): try: managed = _write_k3s_kubeconfig(server, token) self._managed_kubeconfig = str(managed) env["KUBECONFIG"] = self._managed_kubeconfig _merge_kubeconfig(self._managed_kubeconfig) except Exception: pass elif server and token: self.err( "[WARN] k3s token does not look like a Kubernetes bearer token; cannot generate kubeconfig. Ensure a cert-based prole-k3s.kubeconfig exists (Ansible fetch)." ) raw_server, raw_token = self._resolve_k3s_connection() if raw_server: env["PROLE_K3S_SERVER"] = raw_server if raw_token: env["PROLE_K3S_TOKEN"] = raw_token elif mode == "k3d": # Ensure k3d dev clusters have a resolvable KUBECONFIG. cluster_name = self._get_input("init_cluster.deployment_target", "knoe-dev-cluster") if not cluster_name or cluster_name == "knoe-dev-cluster": # Fallback to check if we have a more specific one from UI if available cluster_name = getattr(self, "selected_k3d_cluster", None) if hasattr(cluster_name, "get"): cluster_name = cluster_name.get() cluster_name = cluster_name or "knoe-dev-cluster" try: subprocess.run( [ "k3d", "kubeconfig", "merge", cluster_name, "--kubeconfig-switch-context", ], capture_output=True, text=True, timeout=10, ) except Exception: pass default_kube = str(Path.home() / ".kube" / "config") if Path(default_kube).exists(): env["KUBECONFIG"] = default_kube else: env.pop("KUBECONFIG", None) env["SERVICE_NAMESPACE"] = service_ns env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) db_pw = self._get_input("init_password.db_password", "").strip() if db_pw: db_pw = self._resolve_secret_value(db_pw) env["DB_PASSWORD"] = db_pw env["OPENTOFU_ADMIN_PASSWORD"] = db_pw grafana_pw = ( self.prole_cfg_data.get("Monitoring", {}) .get("GRAFANA_ADMIN_PASSWORD", "") .strip() ) resolved_grafana = self._resolve_secret_value(grafana_pw) if grafana_pw else "" if resolved_grafana and not ( _is_openbao_ref(resolved_grafana) or _is_prole_secret(resolved_grafana) ): env["GRAFANA_ADMIN_PASSWORD"] = resolved_grafana elif db_pw: env["GRAFANA_ADMIN_PASSWORD"] = db_pw realm = self._get_input("kerberos_config.realm", "").strip() if realm: env["KRB5_REALM"] = realm env["REALM"] = realm env["DOMAIN"] = realm.lower() kdc = self._get_input("kerberos_config.kdc", "").strip() if kdc: env["KRB5_KDC"] = kdc env["KRB5_ADMIN"] = kdc env["KRB5_USER"] = self._get_input("kerberos_config.user", "") krb_pw = self._get_input("kerberos_config.password", "") if krb_pw: env["KRB5_PASSWORD"] = self._resolve_secret_value(krb_pw) else: env["KRB5_PASSWORD"] = "" # Registry settings reg_host = self.prole_cfg_data.get("Docker Build", {}).get("LOCAL_REGISTRY") if reg_host: env["LOCAL_REGISTRY"] = reg_host reg_internal = self.prole_cfg_data.get("Docker Build", {}).get( "LOCAL_REGISTRY_INTERNAL" ) if reg_internal: env["LOCAL_REGISTRY_INTERNAL"] = reg_internal try: placement_plan, placement_path = self._resolve_cnpg_placement_plan(namespace) eligible_nodes = placement_plan.get("eligible_nodes") or [] assignments = placement_plan.get("assignments") or {} assignment_pairs = [] for ordinal in sorted(assignments.keys(), key=lambda k: int(str(k))): assignment_pairs.append(f"{ordinal}:{assignments[ordinal]}") env["CNPG_PLACEMENT_PLAN_FILE"] = str(placement_path) env["CNPG_PLACEMENT_PLAN_ID"] = str(placement_plan.get("plan_id") or "") env["CNPG_PLACEMENT_PLAN_HASH"] = str( placement_plan.get("plan_hash") or "" ) env["CNPG_PLACEMENT_ELIGIBLE_NODES"] = ",".join(eligible_nodes) env["CNPG_PLACEMENT_ASSIGNMENTS"] = ",".join(assignment_pairs) env["CNPG_PLACEMENT_REUSED"] = _bool_str( bool((placement_plan.get("metadata") or {}).get("reused", False)) ) env.setdefault( "CNPG_CLUSTER_NAME", str(placement_plan.get("cluster_name") or "knoe-db"), ) env.setdefault( "CNPG_INSTANCES", str(placement_plan.get("desired_instances") or "3"), ) # Compatibility bridge for existing shell topology flow. if eligible_nodes: env.setdefault("CNPG_STAGE1_NODE", str(eligible_nodes[0])) if len(eligible_nodes) == 1: env.setdefault( "CNPG_DB_NODE_SELECTOR", f"kubernetes.io/hostname={eligible_nodes[0]}", ) except Exception as e: self.err(f"[WARN] CNPG placement planning failed; using fallback topology: {e}") return env def _cnpg_cluster_name(self) -> str: glob = self.prole_cfg_data.get("Global", {}) or {} cluster_name = ( os.environ.get("CNPG_CLUSTER_NAME") or str(glob.get("CNPG_CLUSTER_NAME") or "") ).strip() return cluster_name or "knoe-db" def _cnpg_desired_instances(self) -> int: glob = self.prole_cfg_data.get("Global", {}) or {} raw = (os.environ.get("CNPG_INSTANCES") or str(glob.get("CNPG_INSTANCES") or "")).strip() try: val = int(raw) if raw else 3 except Exception: val = 3 return max(1, val) def _cnpg_rebalance_requested(self) -> bool: glob = self.prole_cfg_data.get("Global", {}) or {} raw = ( os.environ.get("CNPG_REBALANCE") or str(glob.get("CNPG_REBALANCE") or "") ) return _parse_bool(raw, default=False) def _cnpg_candidate_nodes(self) -> list[str]: glob = self.prole_cfg_data.get("Global", {}) or {} nodes: set[str] = set() explicit = ( os.environ.get("CNPG_ELIGIBLE_NODES") or str(glob.get("CNPG_ELIGIBLE_NODES") or "") ) if explicit: for part in explicit.split(","): node = str(part or "").strip() if node: nodes.add(node) mounts_raw = str( (self.prole_cfg_data.get("Storage", {}) or {}).get("ANSIBLE_ISCSI_MOUNTS", "") ).strip() if mounts_raw: try: mounts = json.loads(mounts_raw) except Exception: mounts = {} if isinstance(mounts, dict): for details in mounts.values(): if not isinstance(details, dict): continue node = str(details.get("host") or "").strip() if node: nodes.add(node) stage1 = str(glob.get("CNPG_STAGE1_NODE") or "").strip() if stage1: nodes.add(stage1) return sorted(nodes) def _cnpg_plan_path(self, namespace: str, cluster_name: str) -> Path: conf_root = self._get_input( "env_setup.PROLE_CONF", str((getattr(self, "project_root", None) or PROJECT_ROOT) / "conf"), ) conf_dir = Path(conf_root) def _safe_segment(raw: str) -> str: s = str(raw or "").strip() if not s: return "default" return "".join(ch if ch.isalnum() or ch in "-_." else "-" for ch in s) ns_seg = _safe_segment(namespace) cluster_seg = _safe_segment(cluster_name) return conf_dir / "cnpg-placement" / f"{ns_seg}-{cluster_seg}.json" def _resolve_cnpg_placement_plan(self, namespace: str) -> tuple[dict, Path]: cluster_name = self._cnpg_cluster_name() desired_instances = self._cnpg_desired_instances() candidate_nodes = self._cnpg_candidate_nodes() rebalance = self._cnpg_rebalance_requested() plan_path = self._cnpg_plan_path(namespace, cluster_name) prior_plan = load_cnpg_placement_plan(plan_path) placement_plan = plan_cnpg_placement( cluster_name=cluster_name, desired_instances=desired_instances, candidate_nodes=candidate_nodes, prior_plan=prior_plan, rebalance=rebalance, ) if not isinstance(prior_plan, dict) or prior_plan != placement_plan: save_cnpg_placement_plan(plan_path, placement_plan) glob = self.prole_cfg_data.setdefault("Global", {}) glob["CNPG_PLACEMENT_PLAN_FILE"] = str(plan_path) glob["CNPG_PLACEMENT_PLAN_ID"] = str(placement_plan.get("plan_id") or "") glob["CNPG_PLACEMENT_PLAN_HASH"] = str(placement_plan.get("plan_hash") or "") return placement_plan, plan_path # --------------------------------------------- authority / repair def _authority_context_missing(self) -> bool: enabled = self._get_input_bool( "kerberos_config.enabled", False ) or self._get_input_bool("init_cluster.kerberos_enabled", False) if not enabled: return False candidates = [] env_home = (os.environ.get("PROLE_HOME") or "").strip() if env_home: candidates.append(Path(env_home)) candidates.append(self.project_root) for base in candidates: try: if (base / "authority").is_dir(): return False if (base / "prole" / "authority").is_dir(): return False except Exception: continue return True def _dashboard_kong_missing(self, env: dict) -> bool: ns = "kubernetes-dashboard" try: res_ns = subprocess.run( ["kubectl", "get", "ns", ns, "-o", "name"], env=env, capture_output=True, text=True, timeout=6, ) if res_ns.returncode != 0 or ns not in (res_ns.stdout or ""): return True res = subprocess.run( ["kubectl", "-n", ns, "get", "pods", "--no-headers"], env=env, capture_output=True, text=True, timeout=6, ) if res.returncode != 0: return True kong_lines = [ l for l in (res.stdout or "").splitlines() if "kong" in l.lower() ] if not kong_lines: return True for line in kong_lines: parts = line.split() if len(parts) < 3: return True if parts[2] != "Running": return True return False except Exception: return True def _parse_k8s_quantity_bytes(self, raw: str | None) -> int | None: """Parse a Kubernetes resource quantity (e.g. 10Gi, 500Mi) into bytes. Conservative: returns None when parsing fails. """ s = (raw or "").strip() if not s: return None # Common forms: 10Gi, 500Mi, 1Ti, 1000, 1.5Gi num = "" suf = "" for ch in s: if (ch.isdigit() or ch in (".", "+", "-")) and not suf: num += ch else: suf += ch try: value = float(num) except Exception: return None suf = suf.strip() if not suf: try: return int(value) except Exception: return None binary = { "Ki": 1024, "Mi": 1024**2, "Gi": 1024**3, "Ti": 1024**4, "Pi": 1024**5, "Ei": 1024**6, } decimal = { "K": 1000, "M": 1000**2, "G": 1000**3, "T": 1000**4, "P": 1000**5, "E": 1000**6, } if suf in binary: return int(value * binary[suf]) if suf in decimal: return int(value * decimal[suf]) # Be conservative: unknown suffix (e.g. m) => do not parse. return None def _kubectl_get_json(self, env: dict, args: list[str]) -> dict | None: try: res = subprocess.run( ["kubectl"] + list(args), env=env, capture_output=True, text=True, timeout=30, ) if res.returncode != 0: err = (res.stderr or "").strip() or (res.stdout or "").strip() if err: self.err(f"[WARN] kubectl {' '.join(args)} failed: {err}") return None out = (res.stdout or "").strip() if not out: return None return json.loads(out) except Exception as e: self.err(f"[WARN] kubectl {' '.join(args)} failed: {e}") return None def _kubectl_get_namespace_json(self, env: dict, name: str) -> dict | None: n = (name or "").strip() if not n: return None return self._kubectl_get_json(env, ["get", "namespace", n, "-o", "json"]) def _namespace_is_terminating(self, ns: dict | None) -> bool: if not ns: return False md = ns.get("metadata") or {} if (md.get("deletionTimestamp") or "").strip(): return True phase = ((ns.get("status") or {}).get("phase") or "").strip() return phase == "Terminating" # ------------------------------------------ blocked cluster reconciliation def _classify_blocked_pod(self, env: dict, pod: dict) -> dict: """Classify common pod blockers into stable, testable categories. Returns a dict with: - `type`: primary classification (storage-first) - `types`: list of all detected classification tags """ _ = env # reserved for future use meta = pod.get("metadata") or {} status = pod.get("status") or {} name = (meta.get("name") or "").strip() ns = (meta.get("namespace") or "").strip() phase = (status.get("phase") or "").strip() types: list[str] = [] # ContainerCreating / ImagePullBackOff etc. for cs in status.get("containerStatuses") or []: waiting = ((cs.get("state") or {}).get("waiting") or {}) reason = (waiting.get("reason") or "").strip() if reason == "ContainerCreating": types.append("container_creating") elif reason: types.append(f"container_waiting_{reason.lower()}") # Unschedulable reasons. for cond in status.get("conditions") or []: try: if (cond.get("type") or "").strip() != "PodScheduled": continue if (cond.get("status") or "").strip() != "False": continue if (cond.get("reason") or "").strip() != "Unschedulable": continue msg = ((cond.get("message") or "").strip() or "").lower() if "persistent volumes" in msg and "bind" in msg: types.append("unschedulable_due_to_pv_binding") if "didn't match pod's node affinity" in msg or "node affinity/selector" in msg: types.append("unschedulable_due_to_node_affinity") if "had untolerated taint" in msg: types.append("unschedulable_due_to_untolerated_taint") except Exception: continue # Primary type: prefer storage/root-cause categories. primary = "ok" for preferred in ( "unschedulable_due_to_pv_binding", "unschedulable_due_to_node_affinity", "unschedulable_due_to_untolerated_taint", "container_creating", ): if preferred in types: primary = preferred break if primary == "ok" and phase in ("Pending", "Unknown") and types: primary = types[0] # Dedupe while keeping order. seen: set[str] = set() deduped: list[str] = [] for t in types: if t not in seen: seen.add(t) deduped.append(t) return { "pod": f"{ns}/{name}" if ns and name else (name or ns or ""), "namespace": ns, "name": name, "phase": phase, "type": primary, "types": deduped, } def _reconcile_blocked_cluster_state(self, env: dict, service_ns: str) -> None: """Detect hard cluster blockers and raise before running deployment scripts. This is intentionally conservative: it attempts safe PV claimRef repair and then surfaces clear diagnostics for scheduling/storage blockers. """ svc_ns = (service_ns or "").strip() or self._service_namespace() # Always attempt safe stale Released PV claimRef repair first. try: self._repair_stale_released_pvs(env, reset=False) except Exception as e: self.err(f"[WARN] Stale PV claimRef repair failed: {e}") pods_data = self._kubectl_get_json(env, ["get", "pod", "-A", "-o", "json"]) or {} pods = pods_data.get("items") or [] classified = [self._classify_blocked_pod(env, p) for p in pods] blocked = [c for c in classified if (c.get("type") or "") != "ok"] # If we can't positively identify blocked pods, do not block deployment. # This keeps behavior conservative when `kubectl` is unavailable. if not blocked: return # Downstream missingness: surface but treat separately. missing: list[str] = [] for kind in ("svc", "deploy"): try: res = subprocess.run( ["kubectl", "-n", svc_ns, "get", kind, "prole-svc-kong"], env=env, capture_output=True, text=True, timeout=10, ) if res.returncode != 0: missing.append(f"{kind}/prole-svc-kong") except FileNotFoundError: # If kubectl isn't available, don't guess missing resources. pass except Exception: pass all_types: set[str] = set() for b in blocked: for t in b.get("types") or []: all_types.add(t) # Group summary for easier scanning. storage_types = [t for t in sorted(all_types) if "pv_binding" in t] sched_types = [ t for t in sorted(all_types) if "node_affinity" in t or "untolerated_taint" in t ] container_types = [t for t in sorted(all_types) if t.startswith("container_")] lines: list[str] = ["Cluster prerequisites are blocked"] if storage_types or any("pv_binding" in (b.get("type") or "") for b in blocked): lines.append("\nStorage blockers:") for t in storage_types or ["unschedulable_due_to_pv_binding"]: lines.append(f"- {t}") if sched_types: lines.append("\nScheduling blockers:") for t in sched_types: lines.append(f"- {t}") if container_types: lines.append("\nContainer blockers:") for t in container_types: lines.append(f"- {t}") if blocked: lines.append("\nBlocked pods:") for b in blocked: lines.append( f"- {b.get('pod')} type={b.get('type')} types={','.join(b.get('types') or [])}" ) if missing: lines.append("\nDownstream missing resources:") for m in sorted(set(missing)): lines.append(f"- {m}") raise Exception("\n".join(lines)) def _ensure_namespace_ready(self, env: dict, name: str, timeout_s: int = 90) -> None: """Ensure a namespace is usable for deployments. If the namespace is in `Terminating`, wait for deletion to complete. If it disappears, recreate it. If it remains terminating after the timeout, raise with an actionable message. """ n = (name or "").strip() if not n: return ns = self._kubectl_get_namespace_json(env, n) if self._namespace_is_terminating(ns): self.err( f"[WARN] Namespace '{n}' is terminating; waiting up to {timeout_s}s for deletion to complete..." ) deadline = time.time() + max(0, int(timeout_s)) while time.time() < deadline: time.sleep(3) ns = self._kubectl_get_namespace_json(env, n) if not ns: break if not self._namespace_is_terminating(ns): break ns = self._kubectl_get_namespace_json(env, n) if self._namespace_is_terminating(ns): raise Exception( f"Namespace '{n}' is still terminating. " f"Wait for it to finish deleting (kubectl get namespace {n}) or force-delete it, " f"then retry the installer." ) # Create namespace if missing. if not self._kubectl_get_namespace_json(env, n): self.log(f"[INFO] Creating namespace '{n}' ...") rc = self._run_cmd(["kubectl", "create", "namespace", n], env=env) if rc != 0: # Be idempotent: a concurrent creator may have raced us. self.err(f"[WARN] Failed to create namespace '{n}' (code {rc}); continuing") def _list_pending_pvcs( self, env: dict, namespaces: list[str] | None = None ) -> list[dict]: data = self._kubectl_get_json(env, ["get", "pvc", "-A", "-o", "json"]) items = (data or {}).get("items") or [] ns_filter = {n.strip() for n in (namespaces or []) if (n or "").strip()} out: list[dict] = [] for pvc in items: try: phase = ((pvc.get("status") or {}).get("phase") or "").strip() if phase != "Pending": continue ns = ((pvc.get("metadata") or {}).get("namespace") or "").strip() if ns_filter and ns not in ns_filter: continue out.append(pvc) except Exception: continue return out def _list_released_pvs(self, env: dict, storage_class: str | None = None) -> list[dict]: data = self._kubectl_get_json(env, ["get", "pv", "-o", "json"]) items = (data or {}).get("items") or [] sc = (storage_class or "").strip() out: list[dict] = [] for pv in items: try: phase = ((pv.get("status") or {}).get("phase") or "").strip() if phase != "Released": continue pv_sc = ((pv.get("spec") or {}).get("storageClassName") or "").strip() if sc and pv_sc != sc: continue out.append(pv) except Exception: continue return out def _match_stale_released_pv(self, pvc: dict, pvs: list[dict]) -> dict | None: pvc_meta = pvc.get("metadata") or {} pvc_spec = pvc.get("spec") or {} pvc_name = (pvc_meta.get("name") or "").strip() pvc_ns = (pvc_meta.get("namespace") or "").strip() pvc_uid = (pvc_meta.get("uid") or "").strip() pvc_sc = (pvc_spec.get("storageClassName") or "").strip() pvc_volume_name = (pvc_spec.get("volumeName") or "").strip() pvc_modes = pvc_spec.get("accessModes") or [] pvc_mode = (pvc_spec.get("volumeMode") or "").strip() req_storage = ( (((pvc_spec.get("resources") or {}).get("requests") or {}).get("storage") or "").strip() ) req_bytes = self._parse_k8s_quantity_bytes(req_storage) if not pvc_name or not pvc_ns: return None candidates: list[tuple[int, dict]] = [] for pv in pvs: pv_meta = pv.get("metadata") or {} pv_spec = pv.get("spec") or {} pv_status = pv.get("status") or {} pv_name = (pv_meta.get("name") or "").strip() if not pv_name: continue if (pv_status.get("phase") or "").strip() != "Released": continue if pvc_volume_name and pvc_volume_name != pv_name: continue pv_sc = (pv_spec.get("storageClassName") or "").strip() if pvc_sc and pv_sc and pvc_sc != pv_sc: continue if pvc_sc and not pv_sc: continue claim_ref = pv_spec.get("claimRef") or {} if not claim_ref: continue if (claim_ref.get("name") or "").strip() != pvc_name: continue if (claim_ref.get("namespace") or "").strip() != pvc_ns: continue claim_uid = (claim_ref.get("uid") or "").strip() if pvc_uid and claim_uid and pvc_uid == claim_uid: # Too risky: PV believes it is still claimed by this exact PVC. continue pv_modes = pv_spec.get("accessModes") or [] if pvc_modes: if not pv_modes: continue if not set(pvc_modes).issubset(set(pv_modes)): continue pv_mode = (pv_spec.get("volumeMode") or "").strip() if pvc_mode and pv_mode and pvc_mode != pv_mode: continue cap_storage = (((pv_spec.get("capacity") or {}).get("storage") or "").strip()) cap_bytes = self._parse_k8s_quantity_bytes(cap_storage) if req_bytes is not None and cap_bytes is not None: if req_bytes > cap_bytes: continue else: # If we can't compare sizes, be conservative. continue candidates.append((cap_bytes or 0, pv)) if not candidates: return None candidates.sort(key=lambda t: t[0]) return candidates[0][1] def _clear_pv_claim_ref(self, env: dict, pv_name: str) -> bool: pv_name = (pv_name or "").strip() if not pv_name: return False patch = '{"spec":{"claimRef":null}}' try: res = subprocess.run( [ "kubectl", "patch", "pv", pv_name, "--type", "merge", "-p", patch, ], env=env, capture_output=True, text=True, timeout=20, ) if res.returncode == 0: return True err = (res.stderr or "").strip() or (res.stdout or "").strip() if err: self.err(f"[WARN] Failed to patch pv/{pv_name} claimRef: {err}") return False except Exception as e: self.err(f"[WARN] Failed to patch pv/{pv_name} claimRef: {e}") return False def _repair_stale_released_pvs(self, env: dict, reset: bool = False) -> bool: """Repair stale `Released` PVs that retain an old `spec.claimRef`. Policy: - Normal repair is conservative. - When `reset=True`, we may proactively reclaim safe, strongly-matched PVs by clearing `spec.claimRef`. - Reset must NEVER delete PVs. """ service_ns = (env.get("SERVICE_NAMESPACE") or "").strip() or self._service_namespace() db_ns = ( (env.get("DB_NAMESPACE") or "").strip() or (self._get_input("init_password.db_namespace", "") or "").strip() or service_ns ) namespaces = sorted( { n for n in ( service_ns, db_ns, (env.get("NAMESPACE") or "").strip(), ) if (n or "").strip() } ) pending = self._list_pending_pvcs(env, namespaces=namespaces) if not pending: return False # Pull all Released PVs once; matching enforces storageClass/claimRef checks. released = self._list_released_pvs(env) if not released: return False changed = False for pvc in pending: meta = pvc.get("metadata") or {} spec = pvc.get("spec") or {} pvc_name = (meta.get("name") or "").strip() pvc_ns = (meta.get("namespace") or "").strip() pvc_sc = (spec.get("storageClassName") or "").strip() req = (((spec.get("resources") or {}).get("requests") or {}).get("storage") or "").strip() match = self._match_stale_released_pv(pvc, released) if not match: continue pv_name = ((match.get("metadata") or {}).get("name") or "").strip() pv_cap = (((match.get("spec") or {}).get("capacity") or {}).get("storage") or "").strip() claim_ref = (match.get("spec") or {}).get("claimRef") or {} claim_uid = (claim_ref.get("uid") or "").strip() prefix = "[RESET]" if reset else "[INFO]" self.log( f"{prefix} Detected pending PVC with matching Released PV claimRef; clearing claimRef " f"pvc/{pvc_ns}/{pvc_name} storageClass={pvc_sc or ''} request={req or ''} " f"-> pv/{pv_name} capacity={pv_cap or ''} claimRef.uid={claim_uid or ''}" ) if self._clear_pv_claim_ref(env, pv_name): self.log(f"[OK] Cleared claimRef on pv/{pv_name}") changed = True else: self.err(f"[WARN] Failed to clear claimRef on pv/{pv_name}") return changed def _reset_reclaim_stale_pvs(self, env: dict) -> bool: """Reset/reconcile hook to reclaim safe stale PVs. Reset may reclaim stale `Released` PVs by clearing `spec.claimRef`. Reset must NEVER delete PVs. """ return self._repair_stale_released_pvs(env, reset=True) def _run_cluster_repair_pipeline(self, env: dict, anomalies: list[str]) -> None: """Python-native cluster repair orchestration. This mirrors the intent of `etc/repair_pipeline.sh` while delegating actual component deployments to existing `etc/init_*.sh` scripts. """ mode = (env.get("PROLE_MODE") or env.get("DEPLOYMENT_MODE") or "").strip() service_ns = (env.get("SERVICE_NAMESPACE") or "").strip() or self._service_namespace() db_ns = ( (env.get("DB_NAMESPACE") or "").strip() or (self._get_input("init_password.db_namespace", "") or "").strip() or service_ns ) argocd_ns = (env.get("ARGOCD_NAMESPACE") or "").strip() or self._argocd_namespace() registry_ns = (env.get("REGISTRY_NAMESPACE") or "").strip() or self._registry_namespace() def _kubectl(args: list[str], timeout: int = 30) -> subprocess.CompletedProcess: return subprocess.run( ["kubectl"] + args, env=env, capture_output=True, text=True, timeout=timeout, ) self.log(f"[INFO] Cluster repair starting (mode={mode or ''})") # Basic reachability check (non-fatal; keeps behavior conservative). try: res = _kubectl(["cluster-info"], timeout=15) if res.returncode != 0: self.err( f"[WARN] kubectl cluster-info failed: {((res.stderr or '') or (res.stdout or '')).strip()}" ) except Exception as e: self.err(f"[WARN] kubectl cluster-info failed: {e}") # --- Dashboard repair (Helm-based) if "dashboard" in (anomalies or []): try: self.log("[INFO] Repairing kubernetes-dashboard...") # Try to create namespace (idempotent) _kubectl(["create", "ns", "kubernetes-dashboard"], timeout=20) # Helm install/upgrade as in the legacy pipeline subprocess.run( ["helm", "repo", "add", "kubernetes-dashboard", "https://kubernetes.github.io/dashboard/"], env=env, capture_output=True, text=True, timeout=60, ) subprocess.run( ["helm", "repo", "update"], env=env, capture_output=True, text=True, timeout=120, ) subprocess.run( [ "helm", "upgrade", "--install", "kubernetes-dashboard", "kubernetes-dashboard/kubernetes-dashboard", "--namespace", "kubernetes-dashboard", "--create-namespace", "--set", "kong.enabled=true", "--set", "nginx.enabled=false", ], env=env, capture_output=True, text=True, timeout=300, ) # Restart pods (idempotent, best-effort) _kubectl(["-n", "kubernetes-dashboard", "delete", "pods", "-l", "app.kubernetes.io/name=kubernetes-dashboard"], timeout=60) _kubectl(["-n", "kubernetes-dashboard", "delete", "pods", "-l", "app.kubernetes.io/name=kong"], timeout=60) except FileNotFoundError: self.err("[WARN] helm not found; skipping dashboard Helm repair") except Exception as e: self.err(f"[WARN] Dashboard repair failed: {e}") # --- k3s legacy default namespace dedupe try: if mode == "k3s" and service_ns != "default": res_legacy = _kubectl(["get", "ns", "default", "-o", "name"], timeout=10) if res_legacy.returncode == 0 and "default" in (res_legacy.stdout or ""): legacy_ns = "default" def _exists(kind: str, name: str, ns: str) -> bool: r = _kubectl(["-n", ns, "get", kind, name, "-o", "name"], timeout=10) return r.returncode == 0 def _delete_kind(kind: str, name: str, ns: str): _kubectl(["-n", ns, "delete", kind, name, "--ignore-not-found=true"], timeout=60) legacy_openbao = _exists("deploy", "openbao", legacy_ns) or _exists( "statefulset", "openbao", legacy_ns ) legacy_opentofu = _exists("deploy", "opentofu", legacy_ns) legacy_garage = _exists("statefulset", "garage", legacy_ns) legacy_registry = _exists("deploy", "registry", legacy_ns) legacy_kong = _exists("deploy", "prole-svc-kong", legacy_ns) or _exists( "svc", "prole-svc-kong", legacy_ns ) missing_openbao = not ( _exists("deploy", "openbao", service_ns) or _exists("statefulset", "openbao", service_ns) ) missing_opentofu = not _exists("deploy", "opentofu", service_ns) missing_garage = not _exists("statefulset", "garage", service_ns) missing_registry = not _exists("deploy", "registry", registry_ns) if legacy_registry and missing_registry: self.log( f"[INFO] Legacy registry in {legacy_ns}; redeploying to {registry_ns}" ) self._run_script( "init_k3s_registry.sh", args=["update", "--namespace", registry_ns], env=env, ) _delete_kind("deploy", "registry", legacy_ns) if legacy_openbao and missing_openbao: self.log( f"[INFO] Legacy OpenBao in {legacy_ns}; redeploying to {service_ns}" ) self._run_script( "init_openbao.sh", args=["update", "--namespace", service_ns], env=env, ) _delete_kind("deploy", "openbao", legacy_ns) _delete_kind("statefulset", "openbao", legacy_ns) _delete_kind("pvc", "data-openbao-0", legacy_ns) if legacy_opentofu and missing_opentofu: self.log( f"[INFO] Legacy OpenTofu in {legacy_ns}; redeploying to {service_ns}" ) self._run_script( "init_opentofu.sh", args=["update", "--namespace", service_ns], env=env, ) _delete_kind("deploy", "opentofu", legacy_ns) if legacy_garage and missing_garage: self.log( f"[INFO] Legacy Garage in {legacy_ns}; redeploying to {service_ns}" ) self._run_script( "init_garage_store.sh", args=["update", "--namespace", service_ns], env=env, ) _delete_kind("statefulset", "garage", legacy_ns) _delete_kind("pvc", "meta-garage-0", legacy_ns) _delete_kind("pvc", "data-garage-0", legacy_ns) if legacy_kong: missing_kong = not ( _exists("deploy", "prole-svc-kong", service_ns) or _exists("svc", "prole-svc-kong", service_ns) ) if missing_kong: self.log( f"[INFO] Legacy Kong (prole-svc-kong) in {legacy_ns}; redeploying to {service_ns}" ) self._run_script( "init_kong.sh", args=["-n", service_ns, "update"], env=env, ) # Always remove legacy Kong to avoid namespace drift. _kubectl( [ "-n", legacy_ns, "delete", "pod", "-l", "app=prole-svc-kong", "--ignore-not-found=true", ], timeout=60, ) _delete_kind("deploy", "prole-svc-kong", legacy_ns) _delete_kind("svc", "prole-svc-kong", legacy_ns) _delete_kind("configmap", "prole-svc-kong-config", legacy_ns) except Exception as e: self.err(f"[WARN] Legacy namespace dedupe failed: {e}") # --- ArgoCD repair (registry redeploy) try: argocd_apps = [ ("deployment", "argocd-repo-server"), ("deployment", "argocd-application-controller"), ("statefulset", "argocd-redis"), ] unhealthy = False for kind, name in argocd_apps: r = _kubectl( [ "-n", argocd_ns, "get", kind, name, "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode != 0: unhealthy = True break try: if int((r.stdout or "0").strip() or "0") < 1: unhealthy = True break except Exception: unhealthy = True break if unhealthy: self.log("[WARN] ArgoCD is not healthy; redeploying registry...") env2 = env.copy() env2["REGISTRY_NAMESPACE"] = registry_ns self._run_script( "init_registry.sh", args=["-n", registry_ns, "stop"], env=env2, ) self._run_script( "init_registry.sh", args=["-n", registry_ns, "update"], env=env2, ) except Exception as e: self.err(f"[WARN] ArgoCD repair step failed: {e}") # --- Service restarts in the service namespace try: # OpenBao r = _kubectl( [ "-n", service_ns, "get", "statefulset", "openbao", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode == 0: if int((r.stdout or "0").strip() or "0") < 1: self.log("[WARN] OpenBao is not ready; restarting...") self._run_script( "init_openbao.sh", args=["restart", "--namespace", service_ns], env=env, ) except Exception: pass try: # OpenTofu r = _kubectl( [ "-n", service_ns, "get", "deploy", "opentofu", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode == 0: if int((r.stdout or "0").strip() or "0") < 1: self.log("[WARN] OpenTofu is not ready; restarting...") self._run_script( "init_opentofu.sh", args=["restart", "--namespace", service_ns], env=env, ) except Exception: pass try: # Garage r = _kubectl( [ "-n", service_ns, "get", "statefulset", "garage", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode == 0: if int((r.stdout or "0").strip() or "0") < 1: self.log("[WARN] Garage is not ready; restarting...") self._run_script( "init_garage_store.sh", args=["restart", "--namespace", service_ns], env=env, ) except Exception: pass # --- Kerberos / KDC repair (gated) try: kerberos_enabled = self._get_input_bool( "kerberos_config.enabled", False ) or self._get_input_bool("init_cluster.kerberos_enabled", False) if kerberos_enabled: auth_missing = self._authority_context_missing() if not auth_missing: # Only attempt KDC init when authority context exists. env2 = env.copy() env2["DB_NAMESPACE"] = db_ns env2["SERVICE_NAMESPACE"] = service_ns self._run_script( "init_kdc.sh", args=["-n", service_ns], env=env2, ) except Exception as e: self.err(f"[WARN] KDC repair step failed: {e}") # --- Barman plugin repair (best-effort) try: r = _kubectl(["-n", "cnpg-system", "get", "deploy", "barman-cloud"], timeout=10) if r.returncode != 0: env2 = env.copy() env2["NAMESPACE"] = db_ns self._run_script( "init_cloudnative_pg.sh", args=["install-barman-plugin"], env=env2, ) else: r2 = _kubectl( [ "-n", "cnpg-system", "get", "deploy", "barman-cloud", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r2.returncode == 0: try: if int((r2.stdout or "0").strip() or "0") < 1: env2 = env.copy() env2["NAMESPACE"] = db_ns self._run_script( "init_cloudnative_pg.sh", args=["install-barman-plugin"], env=env2, ) _kubectl(["-n", "cnpg-system", "rollout", "restart", "deploy/barman-cloud"], timeout=60) except Exception: pass except Exception: pass # --- CNPG rollout (best-effort; defer to init script) try: # Only attempt if a Cluster exists in the DB namespace. r = _kubectl( ["-n", db_ns, "get", "clusters.postgresql.cnpg.io", "cluster", "-o", "name"], timeout=10, ) if r.returncode == 0 and (r.stdout or "").strip(): env2 = env.copy() env2["NAMESPACE"] = db_ns self._run_script( "init_cloudnative_pg.sh", args=["rollout"], env=env2, ) except Exception: pass # Final health/status report try: args = ["-n", service_ns] kerberos_enabled = self._get_input_bool( "kerberos_config.enabled", False ) or self._get_input_bool("init_cluster.kerberos_enabled", False) if kerberos_enabled: args.append("-k") self._run_script("status_common_services.sh", args=args, env=env) except Exception: pass self.log("[OK] Cluster repair pipeline complete") def _maybe_run_repair_pipeline(self, env: dict) -> None: # Always attempt stale Released PV claimRef repair first. try: self._repair_stale_released_pvs(env) except Exception as e: self.err(f"[WARN] Stale PV claimRef repair failed: {e}") # Best-effort namespace preflight before broader anomaly repairs. try: self._ensure_namespace_ready(env, self._service_namespace()) except Exception as e: self.err(f"[WARN] Namespace preflight failed: {e}") return if self._repair_ran: return anomalies = [] if self._dashboard_kong_missing(env): anomalies.append("dashboard") if self._authority_context_missing(): anomalies.append("authority") if not anomalies: return self._repair_ran = True service_ns = self._service_namespace() db_ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" self.log( f"[INFO] Detected anomalies ({', '.join(anomalies)}); running repair pipeline..." ) self._run_cluster_repair_pipeline(env=env, anomalies=anomalies) # ------------------------------------------ image helpers def _collect_dependent_images( self, include_supabase: bool, include_kerberos_proxy: bool ) -> list[str]: images: set[str] = set() for rel_dir in ("k8s/prole", "k8s/openbao"): base_dir = self.project_root / rel_dir if not base_dir.exists(): continue images.update(_collect_images_from_files(list(base_dir.glob("*.yaml")))) if include_supabase: supa_home = _resolve_supabase_home(self.project_root) if supa_home: docker_dir = supa_home / "docker" compose_files = [docker_dir / "docker-compose.yml"] if os.environ.get("SUPABASE_USE_DEV_COMPOSE") == "1": compose_files.append(docker_dir / "dev" / "docker-compose.dev.yml") images.update(_collect_images_from_files(compose_files)) if not include_supabase: images = {img for img in images if "supabase" not in img} if include_kerberos_proxy: krb_img = os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat") if krb_img: images.add(krb_img) return sorted(images) def _get_docker_import_dir(self) -> str: """Best-effort accessor for a docker-import directory. GUI uses a Tk variable with `.get()`, while console flows tend to store a plain string. This helper normalizes both. """ val = getattr(self, "docker_import_dir", None) if val is None: return "" try: getter = getattr(val, "get", None) if callable(getter): val = getter() except Exception: pass return (str(val) if val is not None else "").strip() def _prepull_images_to_registry( self, include_supabase: bool, include_kerberos_proxy: bool, log: Callable[[str], None] | None = None, ) -> bool: def _log(msg: str): if log: try: log(msg) except Exception: pass self.log(msg) cluster_env = "" try: cluster_env = self._get_input("init_cluster.cluster_env", "") except Exception: cluster_env = "" if cluster_env and not _local_registry_enabled(cluster_env): _log("[SKIP] Local registry disabled; skipping image pre-pull.") return True mode = _deployment_mode_from_env(cluster_env) if mode == "k3s": # Keep behavior consistent across installers: k3s relies on in-cluster # image pulls/import paths rather than a local docker registry. _log("[SKIP] Image pre-pull is not supported in k3s mode.") return True ensure_fn = getattr(self, "ensure_local_registry_available", None) if not callable(ensure_fn): ensure_fn = getattr(self, "_ensure_local_registry_available", None) info = ensure_fn() if callable(ensure_fn) else None if not info: _log("[SKIP] Local registry unavailable; skipping image pre-pull.") return False registry, _cluster_registry = info images = self._collect_dependent_images( include_supabase, include_kerberos_proxy ) if not images: _log("[SKIP] No dependent images found to pre-pull.") return True overall_ok = True import_dir = self._get_docker_import_dir() for image in images: local_tag = image if not image.startswith(f"{registry}/"): local_tag = f"{registry}/{image}" # 1. Check if already in local registry _log(f"[INFO] Checking if {image} exists in local registry...") check_reg = subprocess.run( ["docker", "pull", local_tag], capture_output=True, text=True ) if check_reg.returncode == 0: _log(f"[OK] {image} already exists in local registry as {local_tag}") continue # 2. Check if we already have it in local docker daemon check_local = subprocess.run( ["docker", "image", "inspect", image], capture_output=True, text=True, ) found = check_local.returncode == 0 if not found and import_dir and os.path.isdir(import_dir): # 3. Check import directory safe_name = image.replace("/", "_").replace(":", "_") tar_path = Path(import_dir) / f"{safe_name}.tar" if tar_path.exists(): _log(f"[INFO] Found {tar_path} in import directory, loading...") load = subprocess.run( ["docker", "load", "-i", str(tar_path)], capture_output=True, text=True, ) if load.returncode == 0: found = True else: overall_ok = False self.err(f"[WARN] Failed to load {tar_path}: {load.stderr}") if not found: # 4. Pull from upstream _log(f"[INFO] Pulling {image} from upstream...") pull = subprocess.run( ["docker", "pull", image], capture_output=True, text=True ) if pull.returncode != 0: overall_ok = False self.err(pull.stdout or "") self.err(pull.stderr or "") self.err(f"[ERROR] docker pull failed for {image}") continue found = True # If we have the image locally, tag and push to local registry if found: if local_tag != image: tag = subprocess.run( ["docker", "tag", image, local_tag], capture_output=True, text=True, ) if tag.returncode != 0: overall_ok = False self.err(tag.stdout or "") self.err(tag.stderr or "") self.err( f"[ERROR] docker tag failed for {image} to {local_tag}" ) continue _log(f"[INFO] Pushing {local_tag} to local registry...") if not _push_docker_image(local_tag, log_fn=self.log): overall_ok = False continue _log(f"[OK] Stored {image} in local registry as {local_tag}") return overall_ok class _TeeWriter: """Write to both the original stream and a log file simultaneously.""" def __init__(self, original, log_file): self._original = original self._log_file = log_file def write(self, data): self._original.write(data) self._original.flush() try: self._log_file.write(data) self._log_file.flush() except Exception: pass return len(data) def flush(self): self._original.flush() try: self._log_file.flush() except Exception: pass def fileno(self): return self._original.fileno() def isatty(self): return self._original.isatty() def reconfigure(self, **kwargs): if hasattr(self._original, "reconfigure"): self._original.reconfigure(**kwargs) def __getattr__(self, name): return getattr(self._original, name) class ProleConsoleInstaller(ProleInstaller): """Console-based unattended installer driven by prole.cfg inputs.""" silent = True def __init__( self, controller: ProleController, cfg_path: str | None = None, reset_cluster: bool = False, ): self.controller = controller self.project_root = controller.project_root self.dependencies = list(inst_config.DEPENDENCIES) self.cfg_path = self._normalize_cfg_path(cfg_path) # Ensure the controller also has the normalized config path self.controller.cfg_path = self.cfg_path self.reset_cluster = bool(reset_cluster) self.reset_requested = bool(reset_cluster) # Keep a persistent indicator for reset-oriented reconcile actions (e.g. # reclaiming safe stale Released PV claimRefs). `_perform_cluster_reset()` # consumes `reset_requested` and clears it, but we still want to run # reset reconcile steps later once the cluster is reachable. self._reset_reclaim_pvs_requested = bool(reset_cluster) self.inputs: dict = {} self._init_shared_state() self._db_built_success = False self._scripts_success = False self._cnpg_success = False self.docker_import_dir = None self._openbao_init_env = None self._openbao_init_password = None self._log_file = None def enable_log_file(self, path: str): """Enable tee-style logging: output goes to both stdout and the log file.""" self._log_file = open(path, "a") sys.stdout = _TeeWriter(sys.__stdout__, self._log_file) sys.stderr = _TeeWriter(sys.__stderr__, self._log_file) self.log(f"[LOG] Logging to {path}") # ---- Data access (from inputs dict) ---- def _get_input(self, key: str, default: str | None = None) -> str: if key in self.inputs: return _safe_str(self.inputs[key]) return default if default is not None else "" # ---------------- Config helpers ---------------- def _normalize_cfg_path(self, raw: str | None) -> Path: if raw: p = Path(_expand_path(raw)) if p.is_dir(): return p / "prole.cfg" return p conf_dir = prole_conf_mgr.resolve_prole_conf_dir(self.project_root) return prole_conf_mgr.entrypoint_path(conf_dir) def _load_inputs_from_cfg(self) -> dict: # Ensure env-dir layout exists and legacy entrypoint config is migrated before loading. try: if self.cfg_path.name == "prole.cfg": if self.cfg_path.parent.name in prole_conf_mgr.ENVIRONMENTS: prole_conf_mgr.ensure_entrypoint(self.cfg_path.parent.parent) else: prole_conf_mgr.ensure_entrypoint(self.cfg_path.parent) except Exception: pass if not self.cfg_path.exists(): raise FileNotFoundError(f"prole.cfg not found at {self.cfg_path}") cfg = prole_conf_mgr.load_layered_config(self.cfg_path) cfg_vars = _collect_cfg_vars(cfg) # Load all sections into prole_cfg_data to maintain idempotency for section in cfg.sections(): if section not in self.prole_cfg_data: self.prole_cfg_data[section] = {} for k, v in cfg.items(section): self.prole_cfg_data[section][k] = _expand_cfg_value(v, cfg_vars) if cfg.has_section("Global"): service_ns = _expand_cfg_value( cfg.get("Global", "SERVICE_NAMESPACE", fallback=""), cfg_vars ).strip() if service_ns: self.prole_cfg_data["Global"]["SERVICE_NAMESPACE"] = service_ns if cfg.has_section("Port Forwards"): self.prole_cfg_data["Port Forwards"] = { k: _expand_cfg_value(v, cfg_vars) for k, v in cfg.items("Port Forwards") } for (section, key), _spec in SECRET_KEY_SPECS.items(): if cfg.has_option(section, key): val = _expand_cfg_value( cfg.get(section, key, fallback="").strip(), cfg_vars ) if val: self._cfg_secret_cache[(section, key)] = val if _is_openbao_ref(val): self._secrets_finalized = True self._auto_input_keys = set() auto_kdc = None auto_enabled = None if cfg.has_section("Network"): sec = cfg["Network"] if "KDC_AUTO_DETECTED" in sec: auto_kdc = sec.get("KDC_AUTO_DETECTED", "") if "KERBEROS_AUTO_ENABLED" in sec: auto_enabled = sec.get("KERBEROS_AUTO_ENABLED", "") def mark_auto(inputs: dict): if auto_kdc: if ( inputs.get("kerberos_config.kdc", "").strip() == str(auto_kdc).strip() ): self._auto_input_keys.add("kerberos_config.kdc") if auto_enabled is not None: input_enabled = inputs.get("kerberos_config.enabled") input_bool = _parse_bool(input_enabled, None) auto_bool = _parse_bool(auto_enabled, None) if ( input_bool is not None and auto_bool is not None and input_bool == auto_bool ): self._auto_input_keys.add("kerberos_config.enabled") inputs: dict[str, str] = {} if cfg.has_section("Inputs"): inputs.update( {k: _expand_cfg_value(v, cfg_vars) for k, v in cfg.items("Inputs")} ) mark_auto(inputs) # Resolve encrypted/anchored secrets for runtime use for key in ( "init_password.db_password", "init_password.db_password_confirm", "kerberos_config.password", "init_cluster.k3s_token", ): if key in inputs and inputs[key]: inputs[key] = self._resolve_secret_value(inputs[key]) if inputs.get("init_password.db_password") and not inputs.get( "init_password.db_password_confirm" ): inputs["init_password.db_password_confirm"] = inputs[ "init_password.db_password" ] # Allow explicit runtime override of DB password in silent mode. env_db_pw = os.environ.get("KNOE_DB_PASSWORD") or os.environ.get( "DB_PASSWORD" ) if env_db_pw: raw_db_pw = inputs.get("init_password.db_password", "") if ( not raw_db_pw or _is_openbao_ref(raw_db_pw) or _is_prole_secret(raw_db_pw) ): inputs["init_password.db_password"] = env_db_pw inputs["init_password.db_password_confirm"] = env_db_pw # Fill Ollama inputs from config sections if missing if cfg.has_section("Ollama"): sec = cfg["Ollama"] if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_SERVER_HOST", "").strip() ): inputs["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if ( "ollama_config.server_port" not in inputs and sec.get("OLLAMA_SERVER_PORT", "").strip() ): inputs["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if ( "ollama_config.model" not in inputs and sec.get("OLLAMA_MODEL", "").strip() ): inputs["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_HOST", "").strip() ): host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: inputs["ollama_config.server_host"] = host if port and "ollama_config.server_port" not in inputs: inputs["ollama_config.server_port"] = port if cfg.has_section("Global"): sec = cfg["Global"] if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_SERVER_HOST", "").strip() ): inputs["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if ( "ollama_config.server_port" not in inputs and sec.get("OLLAMA_SERVER_PORT", "").strip() ): inputs["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if ( "ollama_config.model" not in inputs and sec.get("OLLAMA_MODEL", "").strip() ): inputs["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_HOST", "").strip() ): host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: inputs["ollama_config.server_host"] = host if port and "ollama_config.server_port" not in inputs: inputs["ollama_config.server_port"] = port return inputs # Legacy fallback mapping legacy = {} if cfg.has_section("System Environment"): sec = cfg["System Environment"] for k in ( "PROLE_HOME", "PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE", ): if k in sec: raw = (sec.get(k, "") or "").strip() # New policy: do not expand variables for filesystem paths. # Ignore unsafe variable-containing values. if raw and "$" not in raw: legacy[f"env_setup.{k}"] = raw if cfg.has_section("Global"): sec = cfg["Global"] if "PROLE_HOME" in sec: raw = (sec.get("PROLE_HOME", "") or "").strip() if raw and "$" not in raw: legacy["env_setup.PROLE_HOME"] = raw if "CLUSTER_ENV" in sec: legacy["init_cluster.cluster_env"] = _expand_cfg_value( sec.get("CLUSTER_ENV", ""), cfg_vars ) if "PROLE_K3S_SERVER" in sec: legacy["init_cluster.k3s_server_url"] = _expand_cfg_value( sec.get("PROLE_K3S_SERVER", ""), cfg_vars ) if "K3S_SERVER_URL" in sec and "init_cluster.k3s_server_url" not in legacy: legacy["init_cluster.k3s_server_url"] = _expand_cfg_value( sec.get("K3S_SERVER_URL", ""), cfg_vars ) if "PROLE_K3S_TOKEN" in sec: legacy["init_cluster.k3s_token"] = _expand_cfg_value( sec.get("PROLE_K3S_TOKEN", ""), cfg_vars ) if "K3S_TOKEN" in sec and "init_cluster.k3s_token" not in legacy: legacy["init_cluster.k3s_token"] = _expand_cfg_value( sec.get("K3S_TOKEN", ""), cfg_vars ) if "NAMESPACE" in sec: legacy["init_password.db_namespace"] = _expand_cfg_value( sec.get("NAMESPACE", ""), cfg_vars ) legacy["env_setup.NAMESPACE"] = _expand_cfg_value( sec.get("NAMESPACE", ""), cfg_vars ) if "KNOE_DB_USER" in sec: legacy["init_password.db_username"] = _expand_cfg_value( sec.get("KNOE_DB_USER", ""), cfg_vars ) if "DB_PASSWORD" in sec: legacy["init_password.db_password"] = _expand_cfg_value( sec.get("DB_PASSWORD", ""), cfg_vars ) legacy["init_password.db_password_confirm"] = _expand_cfg_value( sec.get("DB_PASSWORD", ""), cfg_vars ) if "DB_HOST_PORT" in sec: legacy["init_password.db_host_port"] = _expand_cfg_value( sec.get("DB_HOST_PORT", "5432"), cfg_vars ) if "DOCKER_IMPORT_DIR" in sec: self.docker_import_dir = _expand_cfg_value( sec.get("DOCKER_IMPORT_DIR", ""), cfg_vars ) self.prole_cfg_data["Global"][ "DOCKER_IMPORT_DIR" ] = self.docker_import_dir if "OLLAMA_SERVER_HOST" in sec: legacy["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if "OLLAMA_SERVER_PORT" in sec: legacy["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if "OLLAMA_MODEL" in sec: legacy["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if "OLLAMA_HOST" in sec and "ollama_config.server_host" not in legacy: host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: legacy["ollama_config.server_host"] = host if port: legacy.setdefault("ollama_config.server_port", port) if cfg.has_section("Network"): sec = cfg["Network"] if "KDC_AUTO_DETECTED" in sec: legacy["kerberos_config.kdc"] = _expand_cfg_value( sec.get("KDC_AUTO_DETECTED", ""), cfg_vars ) if "KERBEROS_AUTO_ENABLED" in sec: legacy["kerberos_config.enabled"] = _expand_cfg_value( sec.get("KERBEROS_AUTO_ENABLED", ""), cfg_vars ) if cfg.has_section("Kerberos Authentication"): sec = cfg["Kerberos Authentication"] for k, tgt in ( ("ENABLED", "kerberos_config.enabled"), ("REALM", "kerberos_config.realm"), ("KDC", "kerberos_config.kdc"), ("USER", "kerberos_config.user"), ("PASSWORD", "kerberos_config.password"), ): if k in sec: legacy[tgt] = _expand_cfg_value(sec.get(k, ""), cfg_vars) if cfg.has_section("Ollama"): sec = cfg["Ollama"] if "OLLAMA_SERVER_HOST" in sec: legacy["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if "OLLAMA_SERVER_PORT" in sec: legacy["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if "OLLAMA_MODEL" in sec: legacy["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if "OLLAMA_HOST" in sec and "ollama_config.server_host" not in legacy: host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: legacy["ollama_config.server_host"] = host if port: legacy.setdefault("ollama_config.server_port", port) if cfg.has_section("GitOps"): sec = cfg["GitOps"] if "NAMESPACE" in sec: legacy["gitops.namespace"] = _expand_cfg_value( sec.get("NAMESPACE", ""), cfg_vars ) if cfg.has_section("Optional Features"): sec = cfg["Optional Features"] if "SUPABASE_ENABLED" in sec: legacy["init_cluster.supabase_enabled"] = sec.get( "SUPABASE_ENABLED", "" ) if "GITOPS_ENABLED" in sec: legacy["init_cluster.gitops_enabled"] = sec.get( "GITOPS_ENABLED", "" ) if "KERBEROS_ENABLED" in sec: legacy["init_cluster.kerberos_enabled"] = sec.get( "KERBEROS_ENABLED", "" ) legacy["kerberos_config.enabled"] = sec.get("KERBEROS_ENABLED", "") if "AT_REST_ENCRYPTION_ENABLED" in sec: legacy["init_cluster.at_rest_encryption_enabled"] = sec.get( "AT_REST_ENCRYPTION_ENABLED", "" ) if cfg.has_section("Initialize Cluster"): sec = cfg["Initialize Cluster"] if "ENVIRONMENT" in sec: legacy["init_cluster.cluster_env"] = sec.get("ENVIRONMENT", "") if "K3S_SERVER_URL" in sec and "init_cluster.k3s_server_url" not in legacy: legacy["init_cluster.k3s_server_url"] = sec.get("K3S_SERVER_URL", "") if "K3S_TOKEN" in sec and "init_cluster.k3s_token" not in legacy: legacy["init_cluster.k3s_token"] = sec.get("K3S_TOKEN", "") if cfg.has_section("Database Creation"): sec = cfg["Database Creation"] if "DB_NAME" in sec: legacy["init_password.db_namespace"] = sec.get("DB_NAME", "") legacy["env_setup.NAMESPACE"] = sec.get("DB_NAME", "") if "NAMESPACE" in sec: legacy["init_password.db_namespace"] = sec.get("NAMESPACE", "") legacy["env_setup.NAMESPACE"] = sec.get("NAMESPACE", "") if "DB_USER" in sec: legacy["init_password.db_username"] = sec.get("DB_USER", "") mark_auto(legacy) for key in ( "init_password.db_password", "init_password.db_password_confirm", "kerberos_config.password", "init_cluster.k3s_token", ): if key in legacy and legacy[key]: legacy[key] = self._resolve_secret_value(legacy[key]) if legacy.get("init_password.db_password") and not legacy.get( "init_password.db_password_confirm" ): legacy["init_password.db_password_confirm"] = legacy[ "init_password.db_password" ] return legacy def _default_inputs(self) -> dict: namespace = self._initial_namespace() owner = self._get_local_owner() env_vals = self._env_defaults(namespace) inputs: dict[str, str] = {} # Dependencies inputs["dependencies.verify_all"] = _bool_str(False) inputs["dependencies.auto_install_missing"] = _bool_str( DEFAULT_ACTION_FLAGS.get("dependencies.auto_install_missing", True) ) for dep in self.dependencies: inputs[f"dependencies.{dep['id']}.install"] = _bool_str(True) # Network scan inputs["network_scan.run"] = _bool_str( DEFAULT_ACTION_FLAGS.get("network_scan.run", True) ) # Environment setup for k in ( "PROLE_HOME", "PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE", ): inputs[f"env_setup.{k}"] = env_vals.get(k, "") inputs["env_setup.NAMESPACE"] = namespace # Database creation inputs["init_password.db_namespace"] = namespace inputs["init_password.db_username"] = owner inputs["init_password.db_password"] = "" inputs["init_password.db_password_confirm"] = "" inputs["init_password.generate_ssh_key"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_password.generate_ssh_key", True) ) inputs["init_password.db_host_port"] = "5432" # Build DB image inputs["init_db_build.run_build"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True) ) # Cluster init + optional features inputs["init_cluster.cluster_env"] = "dev" inputs["init_cluster.k3s_server_url"] = "" inputs["init_cluster.k3s_token"] = "" inputs["init_cluster.supabase_enabled"] = _bool_str(False) inputs["init_cluster.gitops_enabled"] = _bool_str(False) inputs["init_cluster.kerberos_enabled"] = _bool_str(False) inputs["init_cluster.at_rest_encryption_enabled"] = _bool_str(True) inputs["init_cluster.start_cluster"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_cluster.start_cluster", True) ) # Kerberos config inputs["kerberos_config.enabled"] = _bool_str(False) inputs["kerberos_config.realm"] = "" inputs["kerberos_config.kdc"] = "" inputs["kerberos_config.user"] = "administrator" inputs["kerberos_config.password"] = "" inputs["kerberos_config.test_connection"] = _bool_str( DEFAULT_ACTION_FLAGS.get("kerberos_config.test_connection", False) ) inputs["kerberos_config.init_authority"] = _bool_str(False) # Ollama config inputs["ollama_config.server_host"] = "" inputs["ollama_config.server_port"] = DEFAULT_OLLAMA_PORT inputs["ollama_config.model"] = "" # Init scripts + deploy inputs["init_scripts.run_scripts"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_scripts.run_scripts", True) ) inputs["init_cnpg_deploy.run_deploy"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True) ) inputs["init_cnpg_deploy.force_rollout"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False) ) # GitOps inputs["gitops.namespace"] = "gitea" # Disk selection (installer packaging) inputs["disk_selection.disk_type"] = "local" inputs["disk_selection.removable_mount"] = "" inputs["disk_selection.local_path"] = str(Path.home()) # Build tools app inputs["build.deploy_env"] = "Dev" inputs["build.run_build"] = _bool_str( DEFAULT_ACTION_FLAGS.get("build.run_build", False) ) return inputs # ---------------- Process helpers ---------------- def _run_cmd( self, cmd, cwd=None, env=None, stdin_text=None, on_stdout=None, on_stderr=None ) -> int: stdin_handle = subprocess.PIPE if stdin_text else None if isinstance(cmd, str): proc = subprocess.Popen( ["bash", "-lc", cmd], cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=stdin_handle, text=True, bufsize=1, ) else: proc = subprocess.Popen( cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=stdin_handle, text=True, bufsize=1, ) if stdin_text and proc.stdin: try: proc.stdin.write(stdin_text) proc.stdin.close() except Exception: pass def _read_stream(stream, handler, is_err=False): if not stream: return for line in iter(stream.readline, ""): if handler: handler(line) else: if is_err: self.err(line.rstrip("\n")) else: self.log(line.rstrip("\n")) err_thread = threading.Thread( target=_read_stream, args=(proc.stderr, on_stderr, True), daemon=True ) err_thread.start() _read_stream(proc.stdout, on_stdout, False) rc = proc.wait() err_thread.join(timeout=2) return rc def _run_script( self, script_name: str, args=None, env=None, stdin_text=None, on_line=None ) -> int: def _stdout(line): self.log(line.rstrip("\n")) if on_line: on_line(line) def _stderr(line): self.err(line.rstrip("\n")) mode_args = [] if script_name.startswith("init_"): mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if mode: mode_args = ["--mode", mode] return self.controller.run_script( script_name, args=mode_args + (args or []), env=env, stdin_text=stdin_text, on_line=_stdout, on_stderr_line=_stderr, stderr_to_stdout=False, ) def _ensure_local_registry_available(self): """Ensure a local k3d registry is running and return (host_registry, cluster_registry).""" env_hint = self._get_input("init_cluster.cluster_env", "") mode = _deployment_mode_from_env(env_hint) if not _local_registry_enabled(env_hint): self.log("[SKIP] Local registry disabled; skipping registry setup.") return None if mode == "k3s": # In k3s mode we must not use localhost registries or k3d registry containers. server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if not host: self.err( "[WARN] k3s mode but K3S_SERVER_URL/PROLE_K3S_SERVER not set; cannot resolve registry." ) return None host_registry = f"{host}:5000" registry_ns = self._registry_namespace() cluster_registry = f"registry.{registry_ns}.svc.cluster.local:5000" self.local_registry_url = host_registry self.local_registry_internal = cluster_registry self.prole_cfg_data.setdefault("Docker Build", {})["LOCAL_REGISTRY"] = host_registry self.prole_cfg_data["Docker Build"]["LOCAL_REGISTRY_INTERNAL"] = cluster_registry return host_registry, cluster_registry reg_name = "prole-registry" registry_container = f"k3d-{reg_name}" host_registry = "localhost:5000" cluster_registry = f"{registry_container}.localhost:5000" if not self.controller.check_docker_running(): self.err("[WARN] Docker not running; cannot ensure local registry.") return None # 1. If already responsive, just use it if _http_ping_registry("localhost", 5000): self.log("[INFO] Local registry already responsive on localhost:5000.") lst = subprocess.run( ["k3d", "registry", "list"], capture_output=True, text=True ) if registry_container not in (lst.stdout or "") and reg_name not in ( lst.stdout or "" ): cluster_registry = "localhost:5000" self.local_registry_url = host_registry self.local_registry_internal = cluster_registry self.prole_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry self.prole_cfg_data["Docker Build"][ "LOCAL_REGISTRY_INTERNAL" ] = cluster_registry return host_registry, cluster_registry # 2. Require k3d to manage/create the local registry k3d_exists = ( subprocess.run(["which", "k3d"], capture_output=True).returncode == 0 ) if not k3d_exists: self.err("[WARN] k3d not found; cannot ensure local registry.") return None lst = subprocess.run( ["k3d", "registry", "list"], capture_output=True, text=True ) if registry_container not in (lst.stdout or "") and reg_name not in ( lst.stdout or "" ): self.log("[INFO] Creating local k3d registry...") subprocess.run( ["k3d", "registry", "create", reg_name, "--port", "0.0.0.0:5000"], check=True, ) running = subprocess.run( [ "docker", "ps", "--filter", f"name={registry_container}", "--format", "{{.Names}}", ], capture_output=True, text=True, ) if not (running.stdout or "").strip(): exists = subprocess.run( [ "docker", "ps", "-a", "--filter", f"name={registry_container}", "--format", "{{.Names}}", ], capture_output=True, text=True, ) if (exists.stdout or "").strip(): subprocess.run(["docker", "start", registry_container], check=True) self.local_registry_url = host_registry self.local_registry_internal = cluster_registry self.prole_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry self.prole_cfg_data["Docker Build"][ "LOCAL_REGISTRY_INTERNAL" ] = cluster_registry return host_registry, cluster_registry def _argocd_namespace(self) -> str: ns = ( (self.prole_cfg_data.get("Global", {}) or {}) .get("ARGOCD_NAMESPACE", "") .strip() ) if not ns: ns = (os.environ.get("ARGOCD_NAMESPACE") or "").strip() return ns or "argocd" def _registry_namespace(self) -> str: ns = ( (self.prole_cfg_data.get("Global", {}) or {}) .get("REGISTRY_NAMESPACE", "") .strip() ) if not ns: ns = (os.environ.get("REGISTRY_NAMESPACE") or "").strip() if ns: return ns # Registry is a common-core service; default to the configured service namespace. return self._service_namespace() def _ensure_registry_defaults(self, mode: str) -> None: mode = _deployment_mode_from_env(mode) or mode db = self.prole_cfg_data.get("Docker Build", {}) or {} if mode == "k3d" and _local_registry_enabled(mode): info = self._ensure_local_registry_available() if info: host_registry, cluster_registry = info db.setdefault("LOCAL_REGISTRY", host_registry) db.setdefault("LOCAL_REGISTRY_INTERNAL", cluster_registry) if mode == "k3s" and _local_registry_enabled(mode): cur = (db.get("LOCAL_REGISTRY") or "").strip() if not cur or cur == "localhost:5000": server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if host: db["LOCAL_REGISTRY"] = f"{host}:5000" cur_internal = (db.get("LOCAL_REGISTRY_INTERNAL") or "").strip() if not cur_internal or ".localhost" in cur_internal: registry_ns = self._registry_namespace() db["LOCAL_REGISTRY_INTERNAL"] = ( f"registry.{registry_ns}.svc.cluster.local:5000" ) self.prole_cfg_data["Docker Build"] = db def _fetch_k3s_kubeconfig(self) -> Path | None: """Fetch a fresh kubeconfig from k3s via the Ansible playbook.""" kubeconfig = self.project_root / "prole-k3s.kubeconfig" playbook = ( self.project_root / "infrastructure" / "playbooks" / "k3s_fetch_kubeconfig.yml" ) if not playbook.exists(): self.err(f"[WARN] Ansible playbook not found: {playbook}") return None vault_pass = self.project_root / ".vault_pass" cmd = ["ansible-playbook"] if vault_pass.exists(): cmd += ["--vault-password-file", str(vault_pass)] vault_file = (os.environ.get("ANSIBLE_VAULT_PASSWORD_FILE") or "").strip() if not vault_pass.exists() and vault_file: cmd += ["--vault-password-file", vault_file] cmd.append(str(playbook)) self.log("==> Fetching k3s kubeconfig via Ansible") try: subprocess.run( cmd, cwd=str(self.project_root), check=True, capture_output=True, text=True, timeout=30, ) except subprocess.CalledProcessError as e: self.err(f"[WARN] Failed to fetch kubeconfig: {e}") return None except FileNotFoundError: self.err("[WARN] 'ansible-playbook' command not found.") return None except subprocess.TimeoutExpired: self.err("[WARN] Kubeconfig fetch timed out.") return None if not kubeconfig.exists(): self.err("[WARN] Kubeconfig was not written after playbook run.") return None self.log(f"[OK] Kubeconfig fetched: {kubeconfig}") return kubeconfig # ---------------- Steps ---------------- def _apply_inputs(self): defaults = self._default_inputs() loaded = self._load_inputs_from_cfg() self.inputs = {**defaults, **loaded} try: self._apply_ansible_defaults(set(loaded.keys())) except Exception: pass mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if not mode: mode = _deployment_mode_from_env(self._get_input("init_cluster.mode", "")) if mode: os.environ["PROLE_MODE"] = mode for k in ( "env_setup.PROLE_HOME", "env_setup.PROLE_CONF", "env_setup.PROLE_DATA", "env_setup.PROLE_LOGS", "env_setup.PROLE_SERVICE", ): if k in self.inputs: self.inputs[k] = _expand_path(self.inputs[k]) def _apply_ansible_defaults(self, loaded_keys: set[str] | None = None): info = _detect_ansible_topology(self.project_root) if not info: return loaded_keys = loaded_keys or set() auto_keys = getattr(self, "_auto_input_keys", set()) ansible_kdc = info.get("kdc_ip") or "" try: net = self.prole_cfg_data.get("Network", {}) if info.get("topology_json"): net["ANSIBLE_TOPOLOGY"] = info["topology_json"] if info.get("inventory_path"): net["ANSIBLE_INVENTORY"] = info["inventory_path"] if info.get("infrastructure_path"): net["ANSIBLE_INFRASTRUCTURE"] = info["infrastructure_path"] if info.get("domain"): net["ANSIBLE_DOMAIN"] = info["domain"] if info.get("realm"): net["ANSIBLE_REALM"] = info["realm"] if info.get("ad_dc_host"): net["AD_DC_HOST"] = info["ad_dc_host"] if info.get("ad_dc_ip"): net["AD_DC_IP"] = info["ad_dc_ip"] if info.get("kdc_ip"): net["KDC_ANSIBLE_DETECTED"] = info["kdc_ip"] self.prole_cfg_data["Network"] = net except Exception: pass # Storage/node placement derived from Ansible host vars. # This is the authoritative source of truth for where volumes are mounted. try: inv_str = (info.get("inventory_path") or "").strip() inv_path = Path(inv_str) if inv_str else None except Exception: inv_path = None if inv_path is not None and inv_path.exists(): try: mounts = _detect_ansible_storage_mounts(inv_path) labels_by_host = _detect_ansible_node_labels(inv_path) storage_sec = self.prole_cfg_data.setdefault("Storage", {}) if mounts: storage_sec["ANSIBLE_ISCSI_MOUNTS"] = json.dumps( mounts, separators=(",", ":"), sort_keys=True ) if labels_by_host: storage_sec["ANSIBLE_NODE_LABELS"] = json.dumps( labels_by_host, separators=(",", ":"), sort_keys=True ) glob = self.prole_cfg_data.setdefault("Global", {}) cnpg_eligible_nodes = sorted( { str((details or {}).get("host") or "").strip() for details in mounts.values() if isinstance(details, dict) and str((details or {}).get("host") or "").strip() } ) if cnpg_eligible_nodes: glob["CNPG_ELIGIBLE_NODES"] = ",".join(cnpg_eligible_nodes) # CNPG protected storage defaults (k3s local PV) d001 = mounts.get("d001") or {} d001_base = (d001.get("path") or "").strip() d001_host = (d001.get("host") or "").strip() if d001_base and d001_host: glob["PROLE_PROTECTED_DATA_PATH"] = f"{d001_base}/data" glob["PROLE_PROTECTED_WAL_PATH"] = f"{d001_base}/wal" glob["CNPG_STAGE1_NODE"] = d001_host glob["CNPG_DB_NODE_SELECTOR"] = f"kubernetes.io/hostname={d001_host}" # Monitoring storage defaults (prefer d004; fall back to d002) mon_choice = None for vol_id in ("d004", "d002"): if mounts.get(vol_id): mon_choice = mounts[vol_id] break if mon_choice: mon_base = (mon_choice.get("path") or "").strip() mon_host = (mon_choice.get("host") or "").strip() if mon_base: glob["PROLE_MONITORING_DATA_DIR"] = mon_base if mon_host: self.prole_cfg_data.setdefault("Monitoring", {})[ "MONITORING_PRIMARY_NODE" ] = mon_host except Exception: pass if ansible_kdc and "kerberos_config.kdc" in auto_keys: self.inputs["kerberos_config.kdc"] = ansible_kdc if ansible_kdc and "kerberos_config.enabled" in auto_keys: self.inputs["kerberos_config.enabled"] = _bool_str(True) if "kerberos_config.kdc" not in loaded_keys: if not self._get_input("kerberos_config.kdc", "").strip() and info.get( "kdc_ip" ): self.inputs["kerberos_config.kdc"] = info["kdc_ip"] if "kerberos_config.realm" not in loaded_keys: if not self._get_input("kerberos_config.realm", "").strip() and info.get( "realm" ): self.inputs["kerberos_config.realm"] = info["realm"] if "kerberos_config.enabled" not in loaded_keys: if info.get("kdc_ip") and not self._get_input_bool( "kerberos_config.enabled", False ): self.inputs["kerberos_config.enabled"] = _bool_str(True) if "init_cluster.k3s_server_url" not in loaded_keys: if not self._get_input( "init_cluster.k3s_server_url", "" ).strip() and info.get("k3s_server_url"): self.inputs["init_cluster.k3s_server_url"] = info["k3s_server_url"] if "init_cluster.k3s_token" not in loaded_keys: if not self._get_input("init_cluster.k3s_token", "").strip() and info.get( "k3s_token" ): self.inputs["init_cluster.k3s_token"] = info["k3s_token"] def _write_cfg(self): inputs = dict(self.inputs) db_pw = self._get_input("init_password.db_password", "") db_pw_cfg = self._secret_cfg_value( "Inputs", "init_password.db_password", db_pw, "db", "password" ) if db_pw_cfg: inputs["init_password.db_password"] = db_pw_cfg inputs["init_password.db_password_confirm"] = db_pw_cfg elif db_pw: inputs["init_password.db_password"] = db_pw inputs["init_password.db_password_confirm"] = ( self._get_input("init_password.db_password_confirm", "") or db_pw ) krb_pw = self._get_input("kerberos_config.password", "") if krb_pw: inputs["kerberos_config.password"] = self._secret_cfg_value( "Inputs", "kerberos_config.password", krb_pw, "kerberos", "password" ) k3s_token = self._get_input("init_cluster.k3s_token", "") if k3s_token: inputs["init_cluster.k3s_token"] = self._secret_cfg_value( "Inputs", "init_cluster.k3s_token", k3s_token, "k3s", "token" ) mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) target_label = _deployment_target_label( self._get_input("init_cluster.cluster_env", "") ) globals_to_save = { "PROLE_HOME": self._get_input("env_setup.PROLE_HOME", ""), "KNOE_DB_USER": self._get_input("init_password.db_username", ""), "DB_PASSWORD": self._secret_cfg_value( "Global", "DB_PASSWORD", db_pw, "db", "password" ), "CLUSTER_ENV": self._get_input("init_cluster.cluster_env", ""), "DEPLOYMENT_MODE": mode, "DEPLOYMENT_TARGET": target_label, "NAMESPACE": ( self._get_input("init_password.db_namespace", "") or "" ).strip(), "DB_HOST_PORT": ( self._get_input("init_password.db_host_port", "5432") or "5432" ).strip(), "DOCKER_IMPORT_DIR": self.docker_import_dir or "", "PROLE_K3S_SERVER": ( self._get_input("init_cluster.k3s_server_url", "") or "" ).strip(), "PROLE_K3S_TOKEN": self._secret_cfg_value( "Global", "PROLE_K3S_TOKEN", self._get_input("init_cluster.k3s_token", ""), "k3s", "token", ), "PROLE_OPENTOFU_URL": _default_opentofu_pipeline_url(), "SERVICE_NAMESPACE": self._service_namespace(), } # Merge pre-existing Global values without overriding explicit inputs existing_global = dict(self.prole_cfg_data.get("Global", {})) for k, v in existing_global.items(): if k not in globals_to_save or not str(globals_to_save.get(k, "")).strip(): globals_to_save[k] = v globals_to_save.setdefault("DOCKER_PRELOAD", "false") # Re-assert critical/dynamic values to avoid accidental overwrite globals_to_save["DB_PASSWORD"] = self._secret_cfg_value( "Global", "DB_PASSWORD", db_pw, "db", "password" ) globals_to_save["DEPLOYMENT_MODE"] = mode globals_to_save["DEPLOYMENT_TARGET"] = target_label globals_to_save.setdefault("ARGOCD_NAMESPACE", self._argocd_namespace()) globals_to_save.setdefault("REGISTRY_NAMESPACE", self._registry_namespace()) # Monitoring # NOTE: Storage base paths for monitoring are currently pinned in the # monitoring deploy script/Helm values (not persisted in prole.cfg). mon_sec = self.prole_cfg_data.get("Monitoring", {}) if mon_sec is None: mon_sec = {} self.prole_cfg_data["Monitoring"] = mon_sec # Optional features (sourced from inputs) opt_sec = self.prole_cfg_data.setdefault("Optional Features", {}) opt_sec["SUPABASE_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_enabled", False) ) opt_sec["GITOPS_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.gitops_enabled", False) ) opt_sec["KERBEROS_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.kerberos_enabled", False) ) opt_sec["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) gitops_ns_val = ( self._get_input("gitops.namespace", "") or self.inputs.get("gitops.namespace", "") ) if gitops_ns_val: self.prole_cfg_data.setdefault("GitOps", {})[ "GITOPS_NAMESPACE" ] = gitops_ns_val ollama_host = (self._get_input("ollama_config.server_host", "") or "").strip() ollama_port = ( self._get_input("ollama_config.server_port", DEFAULT_OLLAMA_PORT) or "" ).strip() ollama_model = (self._get_input("ollama_config.model", "") or "").strip() ollama_section = dict(self.prole_cfg_data.get("Ollama", {})) if ollama_host: if not ollama_port: ollama_port = DEFAULT_OLLAMA_PORT ollama_section["OLLAMA_SERVER_HOST"] = ollama_host ollama_section["OLLAMA_SERVER_PORT"] = ollama_port ollama_section["OLLAMA_HOST"] = _format_ollama_host( ollama_host, ollama_port ) else: for key in ("OLLAMA_SERVER_HOST", "OLLAMA_SERVER_PORT", "OLLAMA_HOST"): ollama_section.pop(key, None) if ollama_model: ollama_section["OLLAMA_MODEL"] = ollama_model else: ollama_section.pop("OLLAMA_MODEL", None) if ollama_section: self.prole_cfg_data["Ollama"] = ollama_section self._sync_port_forward_mappings() sections = { k: self.prole_cfg_data.get(k, {}) for k in [ "Welcome", "Dependencies", "Network", "Port Forwards", "System Environment", "Monitoring", "Kerberos Authentication", "Ollama", "Optional Features", "GitOps", "Database Creation", "Initialize Cluster", "Docker Build", "Initialization Scripts", "Deployment", "Install", ] } deployment_section = dict(sections.get("Deployment", {})) if mode: deployment_section.setdefault("MODE", mode) if target_label: deployment_section.setdefault("TARGET", target_label) sections["Deployment"] = deployment_section sections["Dev Cluster (k3d)"] = { **self.prole_cfg_data.get("Dev Cluster (k3d)", {}), "MODE": "k3d", "CLUSTER_ENV": "k3d-knoe-dev-cluster", "DISPLAY_NAME": "knoe-dev-cluster", "KUBECTL_CONTEXT": self._get_input("init_cluster.cluster_env", ""), } sections["Service Cluster (k3s)"] = { **self.prole_cfg_data.get("Service Cluster (k3s)", {}), "MODE": "k3s", "CLUSTER_ENV": "prole-service-cluster", "DISPLAY_NAME": "prole-service-cluster", "K3S_SERVER_URL": ( self._get_input("init_cluster.k3s_server_url", "") or "" ).strip(), "K3S_TOKEN": self._secret_cfg_value( "Service Cluster (k3s)", "K3S_TOKEN", self._get_input("init_cluster.k3s_token", ""), "k3s", "token", ), "PIPELINE_URL": _default_opentofu_pipeline_url(), } sections["Prod Cluster (k8s)"] = { **self.prole_cfg_data.get("Prod Cluster (k8s)", {}), "MODE": "k8s", "CLUSTER_ENV": "prole-prod-cluster", "DISPLAY_NAME": "prole-prod-cluster", "ARTIFACTS_DIR": ( self._get_input("init_cluster.prod_artifacts_path", "") or "" ).strip(), "PIPELINE_URL": _default_opentofu_pipeline_url(), } sections = self._sanitize_sections_for_cfg(sections) # Check for existing config and timestamp to maintain idempotency existing_text = None existing_timestamp = None if self.cfg_path.exists(): existing_text = self.cfg_path.read_text() for line in existing_text.splitlines(): if line.startswith("; Generated by install.py on "): existing_timestamp = line[ len("; Generated by install.py on ") : ].strip() break cfg_text = _render_prole_cfg( inputs, globals_to_save, sections, generated_at=existing_timestamp ) should_write = True if existing_text: if existing_text == cfg_text: should_write = False if should_write: # If content changed but we reused existing_timestamp, it might be misleading. # But usually we want to know when it LAST changed. # If it's different, let's generate a new one. if existing_text and existing_text != cfg_text: cfg_text = _render_prole_cfg(inputs, globals_to_save, sections) self.cfg_path.parent.mkdir(parents=True, exist_ok=True) self.cfg_path.write_text(cfg_text) self.log(f"[CONFIG] Wrote {self.cfg_path}") def _step_dependencies(self) -> bool: self.log("==> Dependencies") missing = [] for dep in self.dependencies: ok, location, version = inst_config.get_dep_info(dep) if ok: self.log(f"[OK] {dep['name']} {version or ''}".strip()) continue missing.append(dep) self.log(f"[MISSING] {dep['name']}") if not missing: self.prole_cfg_data["Dependencies"]["STATUS"] = "All installed" return True auto_install = self._get_input_bool( "dependencies.auto_install_missing", DEFAULT_ACTION_FLAGS.get("dependencies.auto_install_missing", True), ) if not auto_install: self.err("[ERROR] Dependencies missing and auto-install disabled.") self.prole_cfg_data["Dependencies"]["STATUS"] = "Missing" return False for dep in missing: install_cmd = dep.get("install_cmd") if not install_cmd: self.err(f"[ERROR] No install command for {dep['name']}.") continue should_install = self._get_input_bool( f"dependencies.{dep['id']}.install", True ) if not should_install: self.err(f"[SKIP] {dep['name']} install disabled by config.") continue self.log(f"[INSTALL] {dep['name']} -> {install_cmd}") rc = self._run_cmd(install_cmd) if rc != 0: self.err(f"[ERROR] Install failed for {dep['name']} (code {rc})") still_missing = [] for dep in missing: ok, _, _ = inst_config.get_dep_info(dep) if not ok: still_missing.append(dep["name"]) if still_missing: self.err(f"[ERROR] Still missing: {', '.join(still_missing)}") self.prole_cfg_data["Dependencies"]["STATUS"] = "Missing" return False self.prole_cfg_data["Dependencies"]["STATUS"] = "All installed" return True def _step_network_scan(self) -> None: if not self._get_input_bool( "network_scan.run", DEFAULT_ACTION_FLAGS.get("network_scan.run", True) ): self.log("[SKIP] Network scan disabled.") return existing_kdc = self.prole_cfg_data.get("Network", {}).get("KDC_AUTO_DETECTED") if existing_kdc: self.log( f"[OK] Network scan already performed. KDC detected: {existing_kdc}" ) return self.log("==> Network scan (prole-agent)") ansible_kdc = "" try: ansible_kdc = (self.prole_cfg_data.get("Network", {}) or {}).get( "KDC_ANSIBLE_DETECTED", "" ) except Exception: ansible_kdc = "" scan_binary = get_resource_path("prole-net/prole-agent") if not scan_binary.exists(): self.err(f"[ERROR] Scan binary not found at {scan_binary}") return prole_home = resolve_prole_home(env={}) scan_dir = prole_home / "scan" scan_dir.mkdir(parents=True, exist_ok=True) kdc_found = None pending_line = "" def _handle_line(line: str): nonlocal kdc_found self.log(line.rstrip("\n")) if "KDC is:" in line: try: ip_part = line.split("KDC is:")[1].strip() ip = ip_part.split()[0].strip("[]():,") if ip: kdc_found = ip except Exception: pass elif "Active Directory" in line or "88" in line: for part in line.split(): try: socket.inet_aton(part.strip("[]():,")) kdc_found = part.strip("[]():,") break except Exception: continue def _handle_stdout(text: str): nonlocal pending_line pending_line += text while "\n" in pending_line: line, pending_line = pending_line.split("\n", 1) _handle_line(line + "\n") rc = run_streaming_cmd( [str(scan_binary)], cwd=str(scan_dir), on_stdout=_handle_stdout ) if pending_line: _handle_line(pending_line) if rc == 0: self.log("[OK] Scan complete.") else: self.err(f"[ERROR] Scan failed (code {rc})") if kdc_found and not ansible_kdc: self.inputs["kerberos_config.kdc"] = kdc_found self.inputs["kerberos_config.enabled"] = _bool_str(True) self.prole_cfg_data["Network"]["KDC_AUTO_DETECTED"] = kdc_found self.prole_cfg_data["Network"]["KERBEROS_AUTO_ENABLED"] = "True" elif kdc_found and ansible_kdc: self.log( f"[INFO] Ansible KDC detected ({ansible_kdc}); ignoring scan-detected KDC {kdc_found}." ) def _step_env_setup(self) -> None: self.log("==> Environment setup") global_sec = self.prole_cfg_data.get("Global", {}) or {} sys_sec = self.prole_cfg_data.get("System Environment", {}) or {} def _cfg_val(sec: dict, key: str) -> str: try: v = str((sec or {}).get(key, "") or "").strip() except Exception: v = "" if v.startswith("${") and v.endswith("}"): return "" return v ns_hint = ( self._get_input("env_setup.NAMESPACE", "") or self._get_input("init_password.db_namespace", "") or _cfg_val(global_sec, "NAMESPACE") ) vals = { "PROLE_HOME": ( self._get_input("env_setup.PROLE_HOME", "") or _cfg_val(sys_sec, "PROLE_HOME") or _cfg_val(global_sec, "PROLE_HOME") or str(resolve_prole_home(env={})) ), "PROLE_CONF": ( self._get_input("env_setup.PROLE_CONF", "") or _cfg_val(sys_sec, "PROLE_CONF") or _cfg_val(global_sec, "PROLE_CONF") ), "PROLE_DATA": ( self._get_input("env_setup.PROLE_DATA", "") or _cfg_val(sys_sec, "PROLE_DATA") or _cfg_val(global_sec, "PROLE_DATA") ), "PROLE_LOGS": ( self._get_input("env_setup.PROLE_LOGS", "") or _cfg_val(sys_sec, "PROLE_LOGS") or _cfg_val(global_sec, "PROLE_LOGS") ), "PROLE_SERVICE": ( self._get_input("env_setup.PROLE_SERVICE", "") or _cfg_val(sys_sec, "PROLE_SERVICE") or _cfg_val(global_sec, "PROLE_SERVICE") ), "PROLE_OPENTOFU_URL": (os.environ.get("PROLE_OPENTOFU_URL") or "").strip(), } defaults = self._env_defaults(ns_hint) for k in vals: if not vals[k]: vals[k] = defaults.get(k, "") if not vals.get("PROLE_OPENTOFU_URL"): vals["PROLE_OPENTOFU_URL"] = _default_opentofu_pipeline_url() vals["NAMESPACE"] = self._get_input("env_setup.NAMESPACE", "") or ns_hint if not vals.get("PROLE_HOME"): raise Exception("PROLE_HOME is required for env setup.") # Persist normalized inputs for k in ( "PROLE_HOME", "PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE", ): self.inputs[f"env_setup.{k}"] = vals[k] self.inputs["env_setup.NAMESPACE"] = vals.get("NAMESPACE", "") self._save_env_to_file(vals) self.reload_env_from_shell() for k in ( "PROLE_HOME", "PROLE_CONF", "PROLE_DATA", "PROLE_LOGS", "PROLE_SERVICE", "PROLE_OPENTOFU_URL", ): self.prole_cfg_data["System Environment"][k] = vals[k] def _step_init_password(self) -> None: self.log("==> Database creation") ns = _safe_str(self._get_input("init_password.db_namespace", "") or "").strip() if not ns: ns = self._get_input("env_setup.NAMESPACE", "") if not ns: ns = ( (self.prole_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "") or "" ).strip() if ns.startswith("${") and ns.endswith("}"): ns = "" if not ns: raise Exception("Database namespace cannot be empty.") if not self._is_valid_namespace(ns): raise Exception("Invalid database namespace.") user = (self._get_input("init_password.db_username", "") or "").strip() if not user: user = ( (self.prole_cfg_data.get("Global", {}) or {}).get("KNOE_DB_USER", "") or "" ).strip() if user.startswith("${") and user.endswith("}"): user = "" if not user: import getpass try: user = getpass.getuser() except Exception: user = "prole" self.log(f"[INFO] No database owner provided; using default: {user}") p1 = self._get_input("init_password.db_password", "") p2 = self._get_input("init_password.db_password_confirm", "") or p1 if not p1: p1 = self._generate_db_password() p2 = p1 self.log( "[INFO] No database password provided; generated default password." ) if p1 != p2: raise Exception("Database passwords do not match.") # If password is an OpenBao anchor, only skip when OpenBao is already initialized. if p1.startswith("${"): resolved = self._resolve_secret_value(p1) if resolved and not resolved.startswith("${"): p1 = resolved p2 = resolved else: service_dir = self._get_input("env_setup.PROLE_SERVICE", "").strip() if not service_dir: service_dir = self._env_defaults(ns).get("PROLE_SERVICE", "") token_path = Path(service_dir) / "secrets" / "openbao-root-token" local_token_path = ( resolve_prole_home(env={}) / "etc" / "secrets" / "openbao-root-token" ) has_token = ( bool(os.environ.get("OPENBAO_ROOT_TOKEN")) or token_path.exists() or local_token_path.exists() ) if self.reset_requested or not has_token: p1 = self._generate_db_password() p2 = p1 self.log( "[RESET] OpenBao anchor detected without token; generated new DB password for initialization." ) else: self.log( f"[OK] OpenBao secrets already anchored for namespace {ns}. Skipping initialization." ) return self.inputs["init_password.db_namespace"] = ns self.inputs["env_setup.NAMESPACE"] = ns self.inputs["init_password.db_password"] = p1 self.inputs["init_password.db_password_confirm"] = p2 self._update_env_namespace(ns) self.prole_cfg_data["Database Creation"]["DB_USER"] = user self.prole_cfg_data["Database Creation"]["DB_PASSWORD_SET"] = "true" self.prole_cfg_data["Database Creation"]["DB_NAME"] = ns self.prole_cfg_data["Database Creation"]["NAMESPACE"] = ns include_supabase = self._get_input_bool("init_cluster.supabase_enabled", False) include_kerberos = self._get_input_bool( "init_cluster.kerberos_enabled", False ) or self._get_input_bool("kerberos_config.enabled", False) if include_supabase: env_policy = self._script_env_for_namespace(ns) allowed, _count, reason = self._optional_workloads_policy(env_policy) if not allowed: self.log(f"[SKIP] Supabase image pre-pull disabled by policy: {reason}") include_supabase = False try: self._prepull_images_to_registry(include_supabase, include_kerberos) except Exception as e: self.err(f"[WARN] Image pre-pull error: {e}") if self._get_input_bool( "init_password.generate_ssh_key", DEFAULT_ACTION_FLAGS.get("init_password.generate_ssh_key", True), ): self._generate_ssh_key(user) # Initialize OpenBao and store keys/passwords for this namespace. # OpenBao is expected to run inside the cluster; defer initialization until cluster is ready. env = os.environ.copy() env["PROLE_HOME"] = str(self.project_root) env["PROLE_SERVICE"] = str(self.project_root) env["KNOE_DB_USER"] = user env["DB_PASSWORD"] = p1 env["GRAFANA_ADMIN_PASSWORD"] = p1 env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) realm = self._get_input("kerberos_config.realm", "").strip() kdc = self._get_input("kerberos_config.kdc", "").strip() krb_user = self._get_input("kerberos_config.user", "").strip() krb_pw = self._get_input("kerberos_config.password", "").strip() if realm: env["KRB5_REALM"] = realm env["REALM"] = realm env["DOMAIN"] = realm.lower() if kdc: env["KRB5_KDC"] = kdc env["KRB5_ADMIN"] = kdc if krb_user: env["KRB5_USER"] = krb_user if krb_pw: env["KRB5_PASSWORD"] = krb_pw self._openbao_init_env = env self._openbao_init_password = p1 self.log("[INFO] Deferring OpenBao initialization until cluster is ready.") def _generate_ssh_key(self, user: str): self.log("==> Environment Preparation (Secure Access)") key_path = Path.home() / ".ssh" / "id_prole_ed25519" key_path.parent.mkdir(parents=True, exist_ok=True) if key_path.exists(): self.log(f"[SKIP] Secure access keys already exist at {key_path}") return cmd = ["ssh-keygen", "-t", "ed25519", "-N", "", "-f", str(key_path), "-C", user] rc = self._run_cmd(cmd) if rc != 0: self.err(f"[WARN] Preparation failed (code {rc}), trying alternative.") cmd = [ "ssh-keygen", "-t", "rsa", "-b", "4096", "-N", "", "-f", str(key_path), "-C", user, ] rc = self._run_cmd(cmd) if rc != 0: self.err(f"[ERROR] Preparation failed (code {rc})") def _step_db_build(self) -> None: if not self._get_input_bool( "init_db_build.run_build", DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True), ): self.log("[SKIP] DB build disabled.") self.prole_cfg_data["Docker Build"]["STATUS"] = "Skipped" return tag = self.controller.get_knoe_db_version() image_name = f"knoe-db:{tag}" env_key = _normalize_cluster_env( self._get_input("init_cluster.cluster_env", "dev") ) cluster_name = "knoe-dev-cluster" # Check if image already exists rc_inspect = subprocess.run( ["docker", "inspect", image_name], capture_output=True ).returncode if rc_inspect == 0: self.log(f"[OK] {image_name} already exists. Skipping build.") if env_key == "dev": self.log(f"Importing image to {cluster_name}...") self._run_cmd( ["k3d", "image", "import", image_name, "-c", cluster_name] ) self._db_built_success = True elif env_key == "service": # In k3s mode, publish to the in-cluster registry instead of importing # tarballs into nodes. push_ok = True db_cfg = self.prole_cfg_data.get("Docker Build", {}) or {} registry = (db_cfg.get("LOCAL_REGISTRY") or "").strip() if not registry or registry == "localhost:5000": server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if host: registry = f"{host}:5000" db_cfg["LOCAL_REGISTRY"] = registry self.prole_cfg_data["Docker Build"] = db_cfg if not registry or registry == "localhost:5000": push_ok = False self.err( "[ERROR] k3s mode but registry host could not be resolved; cannot push knoe-db image." ) else: remote_tag = f"{registry}/{image_name}" self.log(f"Tagging {image_name} as {remote_tag}...") subprocess.run( ["docker", "tag", image_name, remote_tag], capture_output=True, ) if _registry_image_ref_exists(remote_tag): self.log( f"[SKIP] {remote_tag} already exists in registry; skipping push." ) else: self.log(f"Pushing {remote_tag} to registry...") if not _push_docker_image(remote_tag, log_fn=self.log): push_ok = False self.err(f"[ERROR] Failed to push image {remote_tag}") self._db_built_success = push_ok else: self._db_built_success = True self.prole_cfg_data["Docker Build"]["STATUS"] = ( "Built" if self._db_built_success else "Failed" ) return self.log("==> Build knoe-db image") prole_home = resolve_prole_home() mode_key = _deployment_mode_from_env(env_key) or "default" build_dir = prole_home / "build" / mode_key / "knoe-db" build_dir.mkdir(parents=True, exist_ok=True) source_dir = get_resource_path("knoe-db") if not source_dir.exists(): source_dir = get_resource_path("knoe-db") marker = build_dir / ".prole_build_context_ready" if not marker.exists(): copy_build_context_dir(source_dir, build_dir) pub_key_path = Path.home() / ".ssh" / "id_prole_ed25519.pub" pub_key = pub_key_path.read_text().strip() if pub_key_path.exists() else "" username = self._get_input("init_password.db_username", "") cmd = ["docker", "build"] cmd.extend(get_docker_build_platform_args(env_key)) cmd += [ "--build-arg", f"PROLE_USER={username}", "--build-arg", f"PROLE_SSH_PUB_KEY={pub_key}", "-t", image_name, ".", ] rc = self._run_cmd(cmd, cwd=str(build_dir)) if rc == 0: self._db_built_success = True self.log("[OK] Build successful.") push_ok = True if env_key == "dev": cluster_name = "knoe-dev-cluster" self.log(f"Importing image to {cluster_name}...") self._run_cmd( ["k3d", "image", "import", image_name, "-c", cluster_name] ) else: # For non-dev, tag and push to registry db_cfg = self.prole_cfg_data.get("Docker Build", {}) or {} registry = (db_cfg.get("LOCAL_REGISTRY") or "").strip() if env_key == "service" and ( not registry or registry == "localhost:5000" ): server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if host: registry = f"{host}:5000" db_cfg["LOCAL_REGISTRY"] = registry self.prole_cfg_data["Docker Build"] = db_cfg if env_key == "service" and ( not registry or registry == "localhost:5000" ): push_ok = False self.err( "[ERROR] k3s mode but registry host could not be resolved; cannot push knoe-db image." ) self._db_built_success = False else: if not registry: registry = "localhost:5000" remote_tag = f"{registry}/{image_name}" self.log(f"Tagging {image_name} as {remote_tag}...") subprocess.run(["docker", "tag", image_name, remote_tag], check=True) if _registry_image_ref_exists(remote_tag): self.log( f"[SKIP] {remote_tag} already exists in registry; skipping push." ) else: self.log(f"Pushing {remote_tag} to registry...") if not _push_docker_image(remote_tag, log_fn=self.log): push_ok = False self.err(f"[ERROR] Failed to push image {remote_tag}") self._db_built_success = False self.prole_cfg_data["Docker Build"]["STATUS"] = ( "Built" if self._db_built_success else "Failed" ) def _step_cluster(self) -> None: self.log("==> Cluster setup") cluster_env = self._get_input("init_cluster.cluster_env", "dev") env_key = _normalize_cluster_env(cluster_env) self.prole_cfg_data["Initialize Cluster"]["ENVIRONMENT"] = cluster_env self.prole_cfg_data["Initialize Cluster"]["K3S_SERVER_URL"] = self._get_input( "init_cluster.k3s_server_url", "" ) self.prole_cfg_data["Initialize Cluster"]["K3S_TOKEN"] = _encrypt_cfg_secret( self._get_input("init_cluster.k3s_token", "") or "" ) self.prole_cfg_data["Optional Features"]["SUPABASE_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_enabled", False) ) self.prole_cfg_data["Optional Features"]["KERBEROS_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.kerberos_enabled", False) ) self.prole_cfg_data["Optional Features"]["AT_REST_ENCRYPTION_ENABLED"] = ( _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) ) mode = _deployment_mode_from_env(cluster_env) if mode: self._ensure_registry_defaults(mode) if not self._get_input_bool( "init_cluster.start_cluster", DEFAULT_ACTION_FLAGS.get("init_cluster.start_cluster", True), ): self.log("[SKIP] Cluster start disabled.") return if env_key == "dev": if not self.controller.check_docker_running(): if platform.system() == "Darwin": self.log("Starting Docker...") self._run_cmd(["open", "-a", "Docker"]) for _ in range(30): time.sleep(2) if self.controller.check_docker_running(): break if not self.controller.check_docker_running(): raise Exception("Docker is not running.") cluster_name = "knoe-dev-cluster" res = subprocess.run( ["k3d", "cluster", "list", "--no-headers"], capture_output=True, text=True, ) res_stdout = res.stdout or "" if self.reset_cluster: self.log(f"[RESET] Deleting k3d cluster {cluster_name}...") self._run_cmd(["k3d", "cluster", "delete", cluster_name]) res_stdout = "" self.reset_cluster = False if cluster_name not in res_stdout: prole_data = self._resolve_env_dir("PROLE_DATA", "data") self._create_k3d_cluster(cluster_name, prole_data) else: # Check if it's already running if "running" not in res_stdout.lower(): self._run_cmd(["k3d", "cluster", "start", cluster_name]) else: self.log(f"[OK] Cluster {cluster_name} is already running.") self.log(f"[OK] Cluster ready: {cluster_name}") try: os.environ.pop("KUBECONFIG", None) subprocess.run( ["kubectl", "config", "use-context", f"k3d-{cluster_name}"], capture_output=True, ) except Exception: pass else: # Ensure we don't pin KUBECONFIG to a stale local file when another # kubeconfig (e.g. Ansible-fetched client-cert auth) is available. service_dir = self._get_input("env_setup.PROLE_SERVICE", "").strip() candidates: list[Path] = [] kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip() if kubeconfig_env: try: candidates.append(Path(kubeconfig_env).expanduser()) except Exception: pass # Prefer Ansible-fetched kubeconfig in the project root. candidates.append(self.project_root / "prole-k3s.kubeconfig") # Then standard user kubeconfig. candidates.append(Path.home() / ".kube" / "config") # Then prole-specific secret locations. if service_dir: candidates.append(Path(service_dir) / "secrets" / "k3s.kubeconfig") candidates.append(resolve_prole_home(env={}) / "secrets" / "k3s.kubeconfig") candidates.append(self.project_root / "etc" / "secrets" / "k3s.kubeconfig") selected = _select_existing_kubeconfig(candidates) if selected: os.environ["KUBECONFIG"] = str(selected) self.log(f"Using kubeconfig: {selected}") else: # No local kubeconfig found – try fetching from k3s via Ansible fetched = self._fetch_k3s_kubeconfig() if fetched: os.environ["KUBECONFIG"] = str(fetched) self.log(f"Using fetched kubeconfig: {fetched}") kubectl = subprocess.run(["which", "kubectl"], capture_output=True) if kubectl.returncode != 0: raise Exception( "kubectl not found. Please install kubectl and configure access to the target cluster." ) if env_key == "service": if self.reset_cluster: self.log("[RESET] Resetting k3s cluster (mode k3s)...") self._cleanup_local_k3s_artifacts() vault_args = [] v_file = os.environ.get("ANSIBLE_VAULT_PASSWORD_FILE") if v_file: vault_args = ["-v", v_file] delete_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_delete.yml", ] + vault_args self.log(f"Running: {' '.join(delete_cmd)}") rc = self._run_cmd(delete_cmd) if rc != 0: raise Exception(f"k3s delete failed with code {rc}") # 1) --reset should run the ansible k3s/tasks/cleanup on myrddin and then reinstall. # This is handled by k3s_reset.yml reset_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_reset.yml", "-l", "myrddin.prole.org", ] + vault_args self.log(f"Running: {' '.join(reset_cmd)}") rc = self._run_cmd(reset_cmd) if rc != 0: raise Exception(f"k3s reset failed with code {rc}") # 2) Capture new k3s token + TLS bundle from init server and update vault sync_token_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_sync.yml", "-l", "myrddin.prole.org", ] + vault_args self.log(f"Running: {' '.join(sync_token_cmd)}") rc = self._run_cmd(sync_token_cmd) if rc != 0: raise Exception(f"k3s sync (token) failed with code {rc}") # 3) Reinstall agent nodes with the refreshed token reset_agents_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_reset.yml", "-l", "pi.prole.org,retropie.prole.org", ] + vault_args self.log(f"Running: {' '.join(reset_agents_cmd)}") rc = self._run_cmd(reset_agents_cmd) if rc != 0: raise Exception(f"k3s reset (agents) failed with code {rc}") self.reset_cluster = False if not (os.environ.get("KUBECONFIG") or "").strip(): kubeconfig_path = _find_kubeconfig_file() if kubeconfig_path: os.environ["KUBECONFIG"] = kubeconfig_path self.log(f"Using local kubeconfig: {kubeconfig_path}") kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip() if kubeconfig_env and os.path.exists(kubeconfig_env): cmd = ["kubectl", "--kubeconfig", kubeconfig_env, "cluster-info"] else: server = ( self._get_input("init_cluster.k3s_server_url", "") or "" ).strip() token = ( self._get_input("init_cluster.k3s_token", "") or "" ).strip() if token: token = self._resolve_secret_value(token) if server and token: if not server.startswith("http"): server = f"https://{server}" cmd = [ "kubectl", "--server=" + server, "--token=" + token, "--insecure-skip-tls-verify=true", "cluster-info", ] else: raise Exception( "K3s server URL/token missing and no kubeconfig available for service cluster." ) else: cmd = ["kubectl", "cluster-info"] res = subprocess.run(cmd, capture_output=True, text=True) if res.returncode != 0: raise Exception("Cluster is not reachable.") self.log(f"[OK] Cluster ready: {env_key}") if env_key == "service": try: registry_ns = self._registry_namespace() env = self._script_env_for_namespace(self._service_namespace()) env["REGISTRY_NAMESPACE"] = registry_ns self.log("==> Registry preflight") self._run_script( "init_registry.sh", args=["-n", registry_ns, "update"], env=env, ) except Exception as e: self.err(f"[WARN] Registry preflight failed: {e}") # If this run was launched with a reset request, attempt to reclaim any # strongly-matched stale Released PVs early, before later deploy/init # steps that may depend on PVC scheduling. if getattr(self, "_reset_reclaim_pvs_requested", False): try: service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) self.log("==> Reset: reclaim stale Released PV claimRefs") self._reset_reclaim_stale_pvs(env) except Exception as e: self.err(f"[WARN] Reset stale-PV reclaim failed: {e}") self._reset_reclaim_pvs_requested = False if self._openbao_init_env: env = dict(self._openbao_init_env) mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if not mode: if env_key == "dev": mode = "k3d" elif env_key == "service": mode = "k3s" elif env_key == "prod": mode = "k8s" env.setdefault("PROLE_MODE", mode or env_key) self.log("==> OpenBao initialize (cluster)") db_ns = (self._get_input("init_password.db_namespace", "") or "").strip() if not db_ns: db_ns = self._service_namespace() rc_bao = self._run_script( "init_openbao.sh", args=[ "initialize", "--mode", mode, "--namespace", db_ns, "--config", str(self.controller.cfg_path), ], env=env, stdin_text=f"{self._openbao_init_password}\n", ) if rc_bao != 0: raise Exception(f"OpenBao initialization failed (code {rc_bao})") # Materialize DB secrets in Kubernetes now that we have a cluster and a user-provided password. # This prevents downstream init steps (e.g. CNPG) from failing due to missing secrets. self.ensure_db_k8s_secrets( db_ns, self._openbao_init_password or "", log_fn=self.log, ) self._openbao_init_env = None self._openbao_init_password = None # Run repair pipeline if anomalies are detected on a ready cluster try: service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) self._maybe_run_repair_pipeline(env) except Exception: pass if env_key == "service": self._deploy_common_services(env_key) else: self._deploy_opentofu(env_key) def _deploy_common_services(self, env_key: str) -> None: self.log("==> Common services deploy") service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) kerberos_enabled = self._get_input_bool( "kerberos_config.enabled", False ) or self._get_input_bool("init_cluster.kerberos_enabled", False) try: # Avoid deploying into a namespace that is currently being deleted. self._ensure_namespace_ready(env, service_ns) # Ensure node labels are present before init scripts that depend on them # (e.g. node-role selectors and pinned local PV workloads). try: self._apply_ansible_node_labels(env) except Exception: pass # Refuse to proceed when the cluster has hard prerequisites blocked # (e.g. PV binding / scheduling issues). This prevents blind retries # and surfaces actionable diagnostics. self._reconcile_blocked_cluster_state(env=env, service_ns=service_ns) # Check status before update status_args = ["-n", service_ns] if kerberos_enabled: status_args.append("-k") rc_status = self._run_script( "status_common_services.sh", args=status_args, env=env ) if rc_status == 0: self.log( f"[OK] Common services in {service_ns} are healthy. Skipping update." ) return self.log( f"[INFO] Common services unhealthy or missing in {service_ns}. Attempting repair/deploy..." ) svc_args = ["-n", service_ns] if kerberos_enabled: svc_args.append("-k") svc_args.append("update") rc = self._run_script("init_common_services.sh", args=svc_args, env=env) if rc != 0: self.err(f"[ERROR] Common services deploy failed (code {rc})") else: # Re-validate after fix rc_status = self._run_script( "status_common_services.sh", args=status_args, env=env ) if rc_status != 0: self.err( f"[ERROR] Common services still unhealthy after repair attempt." ) else: self.log(f"[OK] Common services in {service_ns} are now healthy.") finally: pass def _apply_ansible_node_labels(self, env: dict) -> None: """Apply Ansible-defined node labels to the current cluster. This makes node selection deterministic even when UI selections are stale or missing. """ inv = ( (self.prole_cfg_data.get("Network", {}) or {}).get("ANSIBLE_INVENTORY") or "" ).strip() if not inv: return inv_path = Path(inv) if not inv_path.exists(): return labels_by_host = _detect_ansible_node_labels(inv_path) if not labels_by_host: return self.log("==> Applying Ansible node labels") base_env = dict(os.environ) base_env.update(env or {}) for host, labels in sorted(labels_by_host.items()): if not labels: continue label_args = [f"{k}={v}" for k, v in sorted(labels.items())] def _label(node_name: str) -> bool: res = subprocess.run( ["kubectl", "label", "node", node_name, "--overwrite", *label_args], capture_output=True, text=True, env=base_env, ) if res.returncode == 0: return True return False # Prefer FQDN node names, but retry with short hostname when needed. if _label(host): continue if "." in host: short = host.split(".", 1)[0] if short and _label(short): continue self.err(f"[WARN] Failed to apply labels to node '{host}'.") def _deploy_opentofu(self, env_key: str) -> None: self.log("==> OpenTofu deploy") ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) try: rc = self._run_script("init_opentofu.sh", args=["start"], env=env) if rc != 0: self.err(f"[ERROR] OpenTofu deploy failed (code {rc})") finally: pass def _step_kerberos(self) -> None: enabled = self._get_input_bool("kerberos_config.enabled", False) if not enabled: self.log("[SKIP] Kerberos disabled.") return realm = self._get_input("kerberos_config.realm", "") kdc = self._get_input("kerberos_config.kdc", "") user = self._get_input("kerberos_config.user", "") password = self._get_input("kerberos_config.password", "") self.prole_cfg_data["Kerberos Authentication"]["ENABLED"] = _bool_str(enabled) self.prole_cfg_data["Kerberos Authentication"]["REALM"] = realm self.prole_cfg_data["Kerberos Authentication"]["KDC"] = kdc self.prole_cfg_data["Kerberos Authentication"]["SERVER"] = kdc self.prole_cfg_data["Kerberos Authentication"]["USER"] = user self.prole_cfg_data["Kerberos Authentication"]["PASSWORD"] = password self.prole_cfg_data["Kerberos Authentication"]["AD_PORT_FORWARD"] = ( os.environ.get("KRB5_AD_PORT_FORWARD", "1") ) self.prole_cfg_data["Kerberos Authentication"]["AD_TCP_PORTS"] = os.environ.get( "KRB5_AD_TCP_PORTS", "88 389 445 464 636" ) self.prole_cfg_data["Kerberos Authentication"]["AD_UDP_PORTS"] = os.environ.get( "KRB5_AD_UDP_PORTS", "88 464" ) self.prole_cfg_data["Kerberos Authentication"]["AD_PROXY_HOST_NETWORK"] = ( os.environ.get("KRB5_AD_PROXY_HOST_NETWORK", "1") ) self.prole_cfg_data["Kerberos Authentication"]["AD_PROXY_IMAGE"] = ( os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat") ) self.prole_cfg_data["Kerberos Authentication"]["AD_PROXY_SERVICE"] = ( os.environ.get("KRB5_AD_SERVICE_NAME", "prole-kerberos-ad-dc") ) ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) env["KERBEROS_ENABLED"] = _bool_str(enabled) env["ENABLED"] = env["KERBEROS_ENABLED"] env["KRB5_REALM"] = realm if env.get("PROLE_MODE") == "k3s": env.setdefault("KRB5_AD_PORT_FORWARD", "0") if not self._get_input_bool( "kerberos_config.test_connection", DEFAULT_ACTION_FLAGS.get("kerberos_config.test_connection", False), ): self.log("[SKIP] Kerberos test disabled.") return # Check if already tested successfully if ( self.prole_cfg_data.get("Kerberos Authentication", {}).get("TEST_STATUS") == "Success" ): self.log("[OK] Kerberos already tested successfully. Skipping.") return if not (realm and user and password and kdc): self.err("[WARN] Kerberos test skipped: missing realm/user/password/kdc.") return self.log("==> Kerberos test: init_kerberos.sh test") rc2 = self._run_script("init_kerberos.sh", args=["test"], env=env) if rc2 != 0: self.err(f"[ERROR] Kerberos test failed (code {rc2})") self.prole_cfg_data.setdefault("Kerberos Authentication", {})[ "TEST_STATUS" ] = "Failed" else: self.log("[OK] Kerberos test completed successfully.") self.prole_cfg_data.setdefault("Kerberos Authentication", {})[ "TEST_STATUS" ] = "Success" def _step_init_scripts(self) -> None: if not self._get_input_bool( "init_scripts.run_scripts", DEFAULT_ACTION_FLAGS.get("init_scripts.run_scripts", True), ): self.log("[SKIP] Init scripts disabled.") self.prole_cfg_data["Initialization Scripts"]["STATUS"] = "Skipped" return if ( self.prole_cfg_data.get("Initialization Scripts", {}).get("STATUS") == "Finished" ): self.log("[OK] Initialization scripts already finished.") return self.log("==> Initialization scripts") ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(env) password = self._get_input("init_password.db_password", "") kerberos_enabled = self._get_input_bool("kerberos_config.enabled", False) svc_args = ["update"] if kerberos_enabled: svc_args = ["-k", "update"] steps = [ ("init_common_services.sh", svc_args, False), ("init_certmgr.sh", ["initialize"], False), ("init_cloudnative_pg.sh", ["initialize"], False), ] if kerberos_enabled: steps.append(("init_kerberos.sh", ["initialize"], False)) steps.extend( [ ("init_cnpg_backup.sh", ["start"], False), ("init_kong.sh", ["start"], False), ] ) if opt_allowed: steps.append(("init_monitoring.sh", ["initialize"], False)) else: self.log(f"[SKIP] Monitoring disabled by policy: {opt_reason}") self.prole_cfg_data.setdefault("Monitoring", {})["STATUS"] = "Skipped" mode = self._deployment_mode() if mode != "k3d": steps.append(("init_nginx_ingress.sh", ["initialize"], False)) overall_success = True for script, args, needs_password in steps: self.log(f"--> {script} {' '.join(args)}") stdin_text = f"{password}\n" if needs_password else None # Ensure DB/CNPG secrets exist before CNPG initialization runs. if script == "init_cloudnative_pg.sh": try: self.ensure_db_k8s_secrets(ns, password, log_fn=self.log) except Exception as e: self.err(f"[ERROR] Failed to ensure DB secrets before CNPG init: {e}") overall_success = False break # Use custom line handler for monitoring to capture Grafana password and port mappings rc = self._run_script( script, args=args, env=env, stdin_text=stdin_text, on_line=self._process_script_output_line, ) if rc != 0: self.err(f"[ERROR] {script} failed (code {rc})") overall_success = False self._scripts_success = overall_success # Verify critical secrets self.log(f"==> Verifying critical secrets in namespace {ns}") critical_secrets = ["knoe-db-user", "knoe-db-superuser", "cnpg-admin-key"] missing_secrets = [] for secret in critical_secrets: rc_s = self._run_cmd(["kubectl", "get", "secret", secret, "-n", ns]) if rc_s != 0: missing_secrets.append(secret) if missing_secrets: self.err( f"[CRITICAL] Missing secrets in namespace '{ns}': {', '.join(missing_secrets)}" ) self.err("Database initialization will fail without these secrets.") self._scripts_success = False self.prole_cfg_data["Initialization Scripts"]["STATUS"] = ( "Completed" if self._scripts_success else "Attempted" ) def _step_cnpg_deploy(self) -> None: if not self._get_input_bool( "init_cnpg_deploy.run_deploy", DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True), ): self.log("[SKIP] CnPG deploy disabled.") self.prole_cfg_data["Deployment"]["STATUS"] = "Skipped" return self.log("==> Deploy CloudNative-PG") etc_dir = self.project_root / "etc" ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) rc = self._run_cmd( ["bash", str(etc_dir / "init_cloudnative_pg.sh"), "deploy", "latest"], env=env, ) if rc == 0: self._cnpg_success = True self.prole_cfg_data["Deployment"]["STATUS"] = "Deployed" else: self._cnpg_success = False self.prole_cfg_data["Deployment"]["STATUS"] = "Attempted" self.err(f"[ERROR] CnPG deploy failed (code {rc})") if self._get_input_bool( "init_cnpg_deploy.force_rollout", DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False), ): # Only attempt rollout if there are existing CNPG pods; otherwise this is a fresh install rc_check = subprocess.run( [ "kubectl", "get", "pods", "-n", ns, "-l", f"cnpg.io/cluster=knoe-db", "--no-headers", ], capture_output=True, text=True, env=env, ) has_pods = bool((rc_check.stdout or "").strip()) if has_pods: self.log("==> Force rollout") rc2 = self._run_cmd( ["bash", str(etc_dir / "init_cloudnative_pg.sh"), "rollout"], env=env, ) if rc2 != 0: self.err(f"[ERROR] Rollout failed (code {rc2})") else: self.log( "[SKIP] Force rollout skipped — no existing CNPG pods (new installation)." ) def _step_supabase(self) -> None: if not self._get_input_bool("init_cluster.supabase_enabled", False): self.log("[SKIP] Supabase deploy disabled.") self.prole_cfg_data["Supabase"] = {"STATUS": "Skipped"} return self.log("==> Deploy Supabase") ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(env) if not opt_allowed: self.log(f"[SKIP] Supabase deploy disabled by policy: {opt_reason}") self.prole_cfg_data["Supabase"] = {"STATUS": "Skipped"} self._supabase_success = False return script_path = self.project_root / "supabase" / "deploy.sh" if not script_path.exists(): self.err(f"[ERROR] Supabase deploy script not found: {script_path}") self._supabase_success = False self.prole_cfg_data["Supabase"] = {"STATUS": "Attempted"} return mode = ( os.environ.get("SUPABASE_DEPLOY_MODE") or os.environ.get("SUPABASE_MODE") or "" ).strip() if not mode: # Match cluster environment to supabase deploy mode cluster_env = _normalize_cluster_env( self._get_input("init_cluster.cluster_env", "dev") ) if cluster_env == "dev": mode = "k3d" elif cluster_env in ("service", "prod"): mode = "k8s" else: mode = "k3d" args = ["--mode", mode] cfg_path = None if self.cfg_path and self.cfg_path.exists(): cfg_path = self.cfg_path else: conf_dir = prole_conf_mgr.resolve_prole_conf_dir(self.project_root) candidate = prole_conf_mgr.entrypoint_path(conf_dir) if candidate.exists(): cfg_path = candidate if cfg_path: args.extend(["-c", str(cfg_path)]) if _parse_bool(os.environ.get("SUPABASE_USE_DEV_COMPOSE"), False): args.append("--with-dev-helpers") if _parse_bool(os.environ.get("SUPABASE_FOREGROUND"), False): args.append("--foreground") self.log(f"--> supabase/deploy.sh {' '.join(args)}") rc = self._run_cmd(["bash", str(script_path)] + args, env=env) if rc == 0: self._supabase_success = True self.prole_cfg_data["Supabase"] = {"STATUS": "Deployed"} else: self._supabase_success = False self.prole_cfg_data["Supabase"] = {"STATUS": "Attempted"} self.err(f"[ERROR] Supabase deploy failed (code {rc})") def _finalize_secrets(self) -> None: ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" if not ns or ns == "default": ns = self._get_input("env_setup.NAMESPACE", "") or "default" p1 = self._get_input("init_password.db_password", "") if p1.startswith("${"): self.log( f"[OK] Secrets already finalized for namespace {ns}. Skipping build-a-bao." ) return self.log("==> OpenBao: finalize secrets") env = self._script_env_for_namespace(ns) rc = self._run_script("build-a-bao.sh", env=env) if rc == 0: self._secrets_finalized = True self.log("[OK] Secrets saved to OpenBao and prole.cfg updated.") else: self.err(f"[ERROR] build-a-bao.sh failed (code {rc})") def _prepare_opentofu_pipeline(self) -> None: env_key = _normalize_cluster_env( self._get_input("init_cluster.cluster_env", "") ) mode = _deployment_mode_from_env(self._get_input("init_cluster.mode", "")) if not mode: mode = _deployment_mode_from_env(os.environ.get("PROLE_MODE", "")) # OpenTofu pipeline is k3s-scoped, but we still want it available for k3d # installs when the user provides k3s connection settings (e.g., staging # manifests for a service cluster while running a local dev cluster). if env_key != "service" and mode not in ("k3d", "k3s"): return namespace = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" k3s_server = (self._get_input("init_cluster.k3s_server_url", "") or "").strip() k3s_token = (self._get_input("init_cluster.k3s_token", "") or "").strip() if k3s_token: k3s_token = self._resolve_secret_value(k3s_token) if k3s_server and not k3s_server.startswith("http"): k3s_server = f"https://{k3s_server}" if not k3s_server: self.err("[WARN] OpenTofu pipeline skipped: missing k3s server URL.") return if not k3s_token: self.err("[WARN] OpenTofu pipeline missing k3s token; writing empty token.") pipeline_dir = _sync_opentofu_pipeline( self.project_root, namespace, k3s_server, k3s_token ) self.prole_cfg_data["Deployment"]["OPENTOFU_PIPELINE_DIR"] = str(pipeline_dir) self.log(f"[OK] OpenTofu pipeline prepared at {pipeline_dir}") def _create_k3d_cluster(self, cluster_name: str, prole_data: Path) -> None: volume_args = _k3d_prole_data_volume_args(str(prole_data)) reg_args = [] if _local_registry_enabled("k3d"): info = self._ensure_local_registry_available() if info: _host_registry, cluster_registry = info reg_use = cluster_registry or "" if reg_use.endswith(".localhost:5000"): reg_use = reg_use.replace(".localhost:5000", ":5000") elif reg_use.endswith(".localhost"): reg_use = reg_use[: -len(".localhost")] if reg_use: reg_args = ["--registry-use", reg_use] cmd = ( ["k3d", "cluster", "create", cluster_name, "-a", "2"] + volume_args + reg_args + ["--api-port", "0.0.0.0:6443"] ) rc = self._run_cmd(cmd) if rc == 0: return if platform.system() == "Darwin": fallback = resolve_prole_home(env={}) / "data" try: if prole_data.resolve() == fallback.resolve(): raise Exception("k3d cluster create failed.") except Exception: # If path resolution fails, still try the fallback pass self.err( f"[WARN] k3d create failed with {prole_data}; retrying with {fallback}..." ) rc = self._run_cmd( ["k3d", "cluster", "create", cluster_name, "-a", "2"] + _k3d_prole_data_volume_args(str(fallback)) + reg_args + ["--api-port", "0.0.0.0:6443"] ) if rc == 0: # Persist fallback for the rest of the run self.inputs["env_setup.PROLE_DATA"] = str(fallback) return raise Exception(f"Failed to create k3d cluster {cluster_name} (code {rc})") def _perform_cluster_reset(self) -> None: if not self.reset_requested: return cluster_env = self._get_input("init_cluster.cluster_env", "dev") env_key = _normalize_cluster_env(cluster_env) if env_key == "dev": self.log("==> Cluster reset (k3d)") if not self.controller.check_docker_running(): if platform.system() == "Darwin": self.log("Starting Docker...") self._run_cmd(["open", "-a", "Docker"]) for _ in range(30): time.sleep(2) if self.controller.check_docker_running(): break if not self.controller.check_docker_running(): raise Exception("Docker is not running.") cluster_name = "knoe-dev-cluster" rc = self._run_cmd(["k3d", "cluster", "delete", cluster_name]) if rc != 0: self.err( f"[WARN] Failed to delete k3d cluster {cluster_name} (code {rc}); continuing." ) prole_data = self._resolve_env_dir("PROLE_DATA", "data") self._create_k3d_cluster(cluster_name, prole_data) elif env_key in ("service", "prole-service-cluster", "k3s"): self.log("==> Cluster reset (k3s namespace cleanup)") ns = self._get_input("env_setup.NAMESPACE", "default") server = self._get_input("init_cluster.k3s_server_url", "") token = self._get_input("init_cluster.k3s_token", "") # For remote k3s clusters the config token is typically a *join token* # (e.g. `K10...::server:...`) which cannot be used to authenticate kubectl. # Ensure we have a cert-based kubeconfig (or a valid bearer token config) # before invoking any kubectl-based reset scripts. self._ensure_project_k3s_kubeconfig(server, token) _reset_k3s_namespace(self.project_root, ns, server, token) self._cleanup_local_k3s_artifacts() self.reset_requested = False def _cleanup_local_k3s_artifacts(self) -> None: paths = [ self.project_root / "etc" / "secrets" / "k3s.kubeconfig", resolve_prole_home(env={}) / "secrets" / "k3s.kubeconfig", ] for path in paths: try: if path.exists(): path.unlink() self.log(f"[RESET] Removed local kubeconfig {path}") except Exception as exc: self.err(f"[WARN] Failed to remove {path}: {exc}") def _ensure_project_k3s_kubeconfig(self, server: str, token: str) -> None: server = (server or "").strip() token = (token or "").strip() target = self.project_root / "prole-k3s.kubeconfig" backup = target.with_name(target.name + ".bak") def _looks_cert_based(path: Path) -> bool: try: txt = path.read_text(encoding="utf-8") except Exception: return False return "client-certificate-data" in txt def _looks_like_join_token_cfg(path: Path) -> bool: try: txt = path.read_text(encoding="utf-8") except Exception: return False # k3s join tokens include `::server:`; they are not Kubernetes bearer tokens. return "token:" in txt and "::server:" in txt and "client-certificate-data" not in txt # If we already have a kubeconfig, keep it unless it is clearly an invalid join-token config. if target.exists() and not _looks_like_join_token_cfg(target): return # Prefer restoring a known-good cert-based kubeconfig. if (not target.exists() or _looks_like_join_token_cfg(target)) and backup.exists() and _looks_cert_based(backup): try: target.write_text(backup.read_text(encoding="utf-8"), encoding="utf-8") os.chmod(target, 0o600) self.log(f"[OK] Restored kubeconfig from backup: {target}") return except Exception as exc: self.err(f"[WARN] Failed to restore kubeconfig from {backup}: {exc}") # Next best: fetch a fresh cert-based kubeconfig via Ansible. fetched = self._fetch_k3s_kubeconfig() if fetched and target.exists() and not _looks_like_join_token_cfg(target): return # Final fallback: only generate a token-based kubeconfig if the token actually looks like a # Kubernetes bearer token. (k3s join tokens are not usable for kubectl auth.) if server and token and _looks_like_k8s_bearer_token(token): try: generated = _write_k3s_kubeconfig(server, token) if generated.exists() and generated != target: target.write_text(generated.read_text(encoding="utf-8"), encoding="utf-8") os.chmod(target, 0o600) self.log(f"[OK] Wrote token-based kubeconfig: {target}") except Exception as exc: self.err(f"[WARN] Failed to generate token-based kubeconfig: {exc}") else: self.err( "[WARN] No usable kubeconfig available for remote k3s; please fetch/create a cert-based prole-k3s.kubeconfig." ) def _close_log_file(self): """Restore original stdout/stderr and close the log file.""" if self._log_file: sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ try: self._log_file.close() except Exception: pass self._log_file = None def _prompt_for_master_password(self) -> str: """Prompt user for master password in silent mode with a 10-minute timeout.""" import sys import time import select import termios import tty # Use sys.__stdout__ and sys.__stdin__ to ensure we bypass any redirection out = sys.__stdout__ inp = sys.__stdin__ # ANSI color codes GREEN = "\033[32m" RED = "\033[31m" RESET = "\033[0m" BOLD = "\033[1m" CHECK = "\u2713" out.write("\n" + "=" * 60 + "\n") out.write(f"{BOLD}DATABASE MASTER PASSWORD REQUIRED{RESET}\n") out.write("A master password is required to initialize the database cluster.\n") out.write("You are running in silent mode (-S) and no valid password was found.\n") out.write(f"Please provide it now. Action timer: {BOLD}10 minutes (600s){RESET}.\n") out.write("=" * 60 + "\n\n") out.flush() timeout = 600 start_time = time.time() def get_input_with_timeout(prompt, mask=True): out.write(prompt) out.flush() if not inp.isatty(): # Fallback for non-TTY environments if necessary, but requirement says "prompt" # In non-TTY, select might still work on stdin. pass fd = inp.fileno() old_settings = termios.tcgetattr(fd) try: tty.setcbreak(fd) res = "" while True: elapsed = time.time() - start_time if elapsed >= timeout: out.write( f"\n{RED}[ERROR] Timeout waiting for password input (600s). Cannot proceed.{RESET}\n" ) out.flush() sys.exit(1) rlist, _, _ = select.select([inp], [], [], 1.0) if rlist: char = inp.read(1) if char in ("\n", "\r"): out.write("\n") out.flush() return res elif char == "\x7f": # Backspace if res: res = res[:-1] if mask: out.write("\b \b") out.flush() elif char == "\x03": # Ctrl+C out.write(f"\n{RED}Installation aborted by user.{RESET}\n") out.flush() sys.exit(1) else: res += char if mask: out.write("*") out.flush() finally: termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) while True: p1 = get_input_with_timeout("Enter Master Password: ") p2 = get_input_with_timeout("Confirm Master Password: ") if p1 and p1 == p2: out.write(f"{GREEN}{CHECK} Password accepted.{RESET}\n\n") out.flush() return p1 elif not p1: out.write(f"{RED}Password cannot be empty.{RESET}\n") out.flush() else: out.write( f"{RED}Passwords do not match. Please try again.{RESET}\n" ) out.flush() def run(self) -> int: _configure_unbuffered_io() self.log(f"[CONFIG] Using {self.cfg_path}") try: self.inputs = {**self._default_inputs(), **self._load_inputs_from_cfg()} # Silent mode: ensure we have a master password source before proceeding. # If the value is an OpenBao reference or an encrypted `${PROLE_SECRET:...}` # placeholder, allow downstream secret-management steps to resolve it. # Only prompt/generate when the value is truly missing. db_pw = self._get_input("init_password.db_password", "") needs_prompt = not bool(db_pw) if needs_prompt: # In silent installs, never block on an interactive prompt in environments # that cannot provide input (CI/pytest/non-interactive runners). stdin_is_tty = getattr(sys.__stdin__, "isatty", lambda: False)() running_under_pytest = "PYTEST_CURRENT_TEST" in os.environ running_in_ci = bool(os.environ.get("CI")) allow_prompt = stdin_is_tty and not running_under_pytest and not running_in_ci if allow_prompt: new_pw = self._prompt_for_master_password() else: self.err( "[WARN] Silent install requires a database master password but prompting is not available; generating one automatically." ) new_pw = secrets.token_urlsafe(24) self.inputs["init_password.db_password"] = new_pw self.inputs["init_password.db_password_confirm"] = new_pw # Save immediately to prole.cfg so subsequent steps/scripts see it. self._write_cfg() # Synchronize controller state self.controller.state.inputs = self.inputs self.controller.state.config_data = self.prole_cfg_data except Exception as e: self.err(f"[FATAL] {e}") self._close_log_file() return 2 self._write_cfg() self._perform_cluster_reset() milestones = [ DependenciesMilestone(), NetworkScanMilestone(), EnvSetupMilestone(), SecretManagementMilestone(), DockerBuildMilestone(), DatabaseCreationMilestone(), ClusterLifecycleMilestone(), InitializationScriptsMilestone(), KerberosMilestone(), GitOpsMilestone(), SupabaseImagePreloadMilestone(), SupabaseMilestone(), DeploymentMilestone(), ] try: self.controller.run_milestones( milestones, progress_callback=lambda msg, p: self.log(f"[{p*100:0.0f}%] {msg}"), ) # Sync back results self.prole_cfg_data = self.controller.state.config_data self.prole_cfg_data["Install"]["STATUS"] = "Finished" self._write_cfg() self.log("[DONE] Silent install completed.") self._close_log_file() return 0 except Exception as e: self.err(f"[FATAL] {e}") try: self.prole_cfg_data["Install"]["STATUS"] = "Failed" self._write_cfg() except Exception: pass self._close_log_file() return 2 def _run_silent_install_test(project_root: Path, cfg_path: Path) -> int: script = project_root / "tests" / "silent_install_test.sh" if not script.exists(): print( f"[ERROR] Silent install test script not found: {script}", file=sys.stderr ) return 1 env = os.environ.copy() env.setdefault("SILENT_INSTALL_LOG", "true") res = subprocess.run(["bash", str(script), str(cfg_path)], env=env) return res.returncode def _attempt_k3s_repair( controller: ProleController, namespace: str, server: str, token: str, db_password: str, ) -> None: env = os.environ.copy() env["PROLE_HOME"] = str(controller.project_root) env["PROLE_SERVICE"] = str(controller.project_root) env["NAMESPACE"] = namespace env["PROLE_MODE"] = "k3s" if db_password: env["DB_PASSWORD"] = db_password env["OPENTOFU_ADMIN_PASSWORD"] = db_password if server and token and not env.get("KUBECONFIG"): repo_kubeconfig = controller.project_root / "prole-k3s.kubeconfig" if repo_kubeconfig.exists(): env["KUBECONFIG"] = str(repo_kubeconfig) elif _looks_like_k8s_bearer_token(token): kubeconfig_path = _write_k3s_kubeconfig(server, token) env["KUBECONFIG"] = str(kubeconfig_path) controller.run_script( "init_openbao.sh", args=[ "update", "--mode", "k3s", "--namespace", namespace, "--config", str(controller.cfg_path), ], env=env, ) controller.run_script( "init_opentofu.sh", args=[ "update", "--mode", "k3s", "--namespace", namespace, "--config", str(controller.cfg_path), ], env=env, ) def _reset_k3s_namespace( project_root: Path, namespace: str, server: str, token: str ) -> None: script = project_root / "scripts" / "reset-ns.sh" if not script.exists(): print(f"[WARN] Namespace reset script not found: {script}", file=sys.stderr) return env = os.environ.copy() if not env.get("KUBECONFIG"): repo_kubeconfig = project_root / "prole-k3s.kubeconfig" if repo_kubeconfig.exists(): env["KUBECONFIG"] = str(repo_kubeconfig) elif server and token and _looks_like_k8s_bearer_token(token): kubeconfig_path = _write_k3s_kubeconfig(server, token) env["KUBECONFIG"] = str(kubeconfig_path) subprocess.run(["bash", str(script), "-n", namespace], env=env) def _prepare_k3s_pipeline( controller: ProleController, log_fn=None, skip_validation: bool = False ) -> int: """Prepare OpenTofu k3s pipeline: stage manifests, tfvars, ArgoCD apps. When *skip_validation* is True the silent-install-test / repair loop is skipped entirely — the caller (e.g. a successful silent install) already validated the environment. The main prole.cfg is **not** overwritten in this case so it keeps whatever cluster_env the install used. """ def _log(msg): if log_fn: try: log_fn(msg) except: print(msg, end="", flush=True) else: print(msg, end="", flush=True) project_root = controller.project_root conf_dir = prole_conf_mgr.resolve_prole_conf_dir(project_root) cfg_path = prole_conf_mgr.entrypoint_path(conf_dir) _log("==> Preparing k3s pipeline\n") _log(f" Project root: {project_root}\n") _log(f" Config: {cfg_path}\n\n") _log("==> Detecting Ansible topology...\n") info = _detect_ansible_topology(project_root) k3s_server = (info.get("k3s_server_url") or "").strip() if info else "" k3s_token = (info.get("k3s_token") or "").strip() if info else "" if k3s_server and not k3s_server.startswith("http"): k3s_server = f"https://{k3s_server}" _log(f" k3s_server: {k3s_server or '(not detected)'}\n") _log(f" k3s_token: {'detected' if k3s_token else '(not detected)'}\n\n") # Load config to resolve namespace and secrets for pipeline tfvars _log("==> Loading knoe config...\n") knoe = ProleConsoleInstaller(controller, str(cfg_path)) try: existing_inputs = knoe._load_inputs_from_cfg() except Exception: existing_inputs = {} knoe.inputs = {**knoe._default_inputs(), **existing_inputs} if k3s_server and not knoe.inputs.get("init_cluster.k3s_server_url"): knoe.inputs["init_cluster.k3s_server_url"] = k3s_server if k3s_token and not knoe.inputs.get("init_cluster.k3s_token"): knoe.inputs["init_cluster.k3s_token"] = k3s_token namespace = (knoe._get_input("init_password.db_namespace", "") or "").strip() if not namespace: namespace = ( knoe._get_input("env_setup.NAMESPACE", "") or "" ).strip() or "default" _log(f" Target namespace: {namespace}\n") _log(" [OK] Config loaded\n\n") if not skip_validation: # Standalone mode: write a pipeline-specific cfg and validate knoe.inputs["init_cluster.cluster_env"] = "prole-service-cluster" knoe.inputs["init_cluster.supabase_enabled"] = _bool_str(False) knoe.inputs["init_cluster.kerberos_enabled"] = _bool_str(False) knoe.inputs["init_cluster.at_rest_encryption_enabled"] = _bool_str(True) knoe.inputs["kerberos_config.enabled"] = _bool_str(False) knoe.inputs["kerberos_config.test_connection"] = _bool_str(False) knoe.inputs["env_setup.PROLE_HOME"] = str(project_root) knoe.inputs["env_setup.PROLE_CONF"] = str(project_root / "conf") knoe.inputs["env_setup.PROLE_DATA"] = knoe._resolve_env_value( "PROLE_DATA", str(resolve_prole_home(env={}) / "data") ) or str(resolve_prole_home(env={}) / "data") knoe.inputs["env_setup.PROLE_LOGS"] = knoe._resolve_env_value( "PROLE_LOGS", str(resolve_prole_home(env={}) / "logs") ) or str(resolve_prole_home(env={}) / "logs") knoe.inputs["env_setup.PROLE_SERVICE"] = str(project_root / "etc") # Ensure required keys that the silent install test validates if not knoe.inputs.get("env_setup.NAMESPACE"): knoe.inputs["env_setup.NAMESPACE"] = ( knoe.inputs.get("init_password.db_namespace", "") or "default" ) if not knoe.inputs.get("init_password.db_namespace"): knoe.inputs["init_password.db_namespace"] = ( knoe.inputs.get("env_setup.NAMESPACE", "") or "default" ) if not knoe.inputs.get("kerberos_config.init_authority"): knoe.inputs["kerberos_config.init_authority"] = _bool_str(False) if not knoe.inputs.get("init_password.db_host_port"): knoe.inputs["init_password.db_host_port"] = "5432" _log(" [OK] Config overrides applied for k3s pipeline\n") _log("==> Writing prole.cfg...\n") knoe._write_cfg() _log(f" [OK] {cfg_path}\n\n") db_password = knoe._get_input("init_password.db_password", "").strip() _log(f"==> Target namespace: {namespace}\n\n") attempt = 1 total_attempts = 0 max_attempts_env = os.environ.get("PROLE_SILENT_TEST_MAX_ATTEMPTS", "").strip() max_attempts = int(max_attempts_env) if max_attempts_env.isdigit() else 0 _log("==> Running silent install validation...\n") while True: total_attempts += 1 _log(f" Attempt {total_attempts}...\n") rc = _run_silent_install_test(project_root, cfg_path) if rc == 0: _log(f" [OK] Silent install test passed\n\n") break _log(f" [WARN] Silent install test failed (code {rc})\n") cur_server = knoe.inputs.get("init_cluster.k3s_server_url", k3s_server) cur_token = knoe.inputs.get("init_cluster.k3s_token", k3s_token) if attempt == 1: _log(" Attempting k3s repair...\n") _attempt_k3s_repair( controller, namespace, cur_server, cur_token, db_password ) elif attempt == 2: _log(" Resetting k3s namespace...\n") _reset_k3s_namespace(project_root, namespace, cur_server, cur_token) else: attempt = 0 attempt += 1 if max_attempts and total_attempts >= max_attempts: _log(f" [ERROR] Max attempts ({max_attempts}) reached. Aborting.\n") return rc else: _log("==> Skipping validation (silent install already succeeded)\n\n") cur_server = knoe.inputs.get("init_cluster.k3s_server_url", k3s_server) cur_token = knoe.inputs.get("init_cluster.k3s_token", k3s_token) _sync_opentofu_pipeline(project_root, namespace, cur_server, cur_token, log_fn=_log) _log("\n==> k3s pipeline preparation complete.\n") return 0