""" Installer actions and unattended workflow helpers. """ from __future__ import annotations import shlex import socket import sys import threading import uuid import logging import secrets import string from knoe import knoe_conf as knoe_conf_mgr from knoe.config import ( _expand_path, _collect_cfg_vars, _expand_cfg_value, _parse_bool, _is_openbao_ref, _is_knoe_secret, _decrypt_knoe_secret, _encrypt_knoe_secret, _write_k3s_kubeconfig, _encrypt_cfg_secret, _merge_kubeconfig, ) from knoe.core.build_context import copy_build_context_dir from knoe.core.cnpg_placement import ( load_cnpg_placement_plan, plan_cnpg_placement, save_cnpg_placement_plan, ) from knoe.core.controller import KnoeController from knoe.core.env import * # noqa: F401,F403 from knoe.core.env import ( _parse_ollama_host, _bool_str, _deployment_mode_from_env, _openbao_placeholder, _looks_like_k8s_bearer_token, _find_kubeconfig_file, _normalize_cluster_env, _pf_upsert_mapping, _build_required_port_forwards, _local_registry_enabled, _collect_images_from_files, _http_ping_registry, _registry_image_ref_exists, _resolve_supabase_home, _push_docker_image, _detect_ansible_topology, _detect_ansible_storage_mounts, _detect_ansible_node_labels, _deployment_target_label, _default_opentofu_pipeline_url, _format_ollama_host, _host_from_url, _render_knoe_cfg, _k3d_knoe_data_volume_args, _sync_opentofu_pipeline, _read_k3s_cfg, _safe_str, _expand_cfg_value as _expand_env_cfg_value, _resolve_k3s_connection as _resolve_k3s_connection_fn, _kubectl_base_cmd_for_k3s as _kubectl_base_cmd_for_k3s_fn, _verify_k3s_services_status, ) from knoe.core.milestones import ( DependenciesMilestone, NetworkScanMilestone, EnvSetupMilestone, SecretManagementMilestone, DatabaseCreationMilestone, DockerBuildMilestone, ClusterLifecycleMilestone, InitializationScriptsMilestone, KerberosMilestone, DeploymentMilestone, SecurityMilestone, GitOpsMilestone, SupabaseImagePreloadMilestone, SupabaseMilestone, ) from knoe.core.ops import garage_store as garage_store_ops from knoe.core.ops import monitoring as monitoring_ops from knoe.core.ops import openbao as openbao_ops from knoe.core.ops import opentofu as opentofu_ops from knoe.core.ops import registry as registry_ops from knoe.core.ops.gke_clusters import ( GkeClusterSpec, build_kubectl_env_for_cluster, ensure_app_cluster, ensure_db_cluster, get_cluster_credentials, ) from knoe.core.ops.cloudnative_pg import ( initialize as cnpg_initialize, deploy as cnpg_deploy, install_barman_plugin as cnpg_install_barman_plugin, rollout as cnpg_rollout, ) from knoe.core.ops.storage import ( ClusterStorageSpec, StorageProvisioningError, provision_cluster_storage, ) from knoe.core.policy import ( POLICY_CFG_KEY, OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES, evaluate_optional_workloads_allowed, ) from knoe.core.stream_exec import run_streaming_cmd from typing import Callable, Sequence DEFAULT_APP_CLUSTER_NAME = "knoe-dev-0" DEFAULT_APP_CLUSTER_MODE = "standard" DEFAULT_APP_CLUSTER_MACHINE_TYPE = "e2-standard-2" DEFAULT_APP_CLUSTER_NODE_COUNT = 3 DEFAULT_DB_CLUSTER_NAME = "knoe-dev-cnpg-0" DEFAULT_DB_CLUSTER_MODE = "standard" DEFAULT_DB_CLUSTER_NODE_COUNT = 3 DEFAULT_DB_CLUSTER_MACHINE_TYPE = "e2-standard-2" DEFAULT_DB_BOOT_DISK_TYPE = "pd-standard" DEFAULT_DB_BOOT_DISK_SIZE_GB = 50 def _configure_unbuffered_io(): os.environ.setdefault("PYTHONUNBUFFERED", "1") for stream in (sys.stdout, sys.stderr): try: stream.reconfigure(line_buffering=True, write_through=True) except Exception: try: stream.flush() except Exception: pass def _kubeconfig_cluster_info_ok(kubeconfig: Path, timeout_s: int = 8) -> bool: try: res = subprocess.run( ["kubectl", "--kubeconfig", str(kubeconfig), "cluster-info"], capture_output=True, text=True, timeout=timeout_s, ) return res.returncode == 0 except Exception: return False def _select_existing_kubeconfig(candidates: Sequence[Path]) -> Path | None: """Return the first kubeconfig that exists and can reach/auth the cluster. If none of the existing candidates pass `kubectl cluster-info` (e.g. cluster not yet reachable), fall back to the first existing candidate to preserve previous behavior. """ first_existing: Path | None = None for kc in candidates: try: if not kc.exists(): continue except Exception: continue if first_existing is None: first_existing = kc if _kubeconfig_cluster_info_ok(kc): return kc return first_existing class KnoeInstaller: """Shared business logic for both silent and interactive installer modes. Subclasses must provide a :meth:`_get_input` implementation that bridges their respective data-access layers (``dict`` for silent, tk ``StringVar`` for the GUI). All common helpers – namespace, env, secrets, port-forwards, script-env, logging – live here so they are defined exactly once. """ # Subclasses should set this to ``True`` for headless operation. silent: bool = False # ------------------------------------------------------------------ init def _init_shared_state(self): """Initialise state common to every installer flavour.""" self.knoe_cfg_data = { "Global": {}, "Welcome": {}, "Dependencies": {}, "Network": {}, "Storage": {}, "Port Forwards": {}, "System Environment": {}, "Monitoring": {}, "Kerberos Authentication": {}, "Ollama": {}, "Optional Features": {}, "Database Options": {}, "Database Creation": {}, "Docker Build": {}, "Initialize Cluster": {}, "Initialization Scripts": {}, "GitOps": {}, "Deployment": {}, "Dev Cluster (k3d)": {}, "Service Cluster (k3s)": {}, "Prod Cluster (k8s)": {}, "Install": {}, } self._cfg_secret_cache: dict = {} self._secrets_finalized: bool = False self._managed_kubeconfig: str | None = None self._repair_ran: bool = False self._optional_workloads_policy_cache: tuple[bool, int, str] | None = None def _optional_workloads_policy(self, env: dict) -> tuple[bool, int, str]: """Evaluate and cache the optional-workloads policy. Central rule: `optional_workloads_allowed = ready_schedulable_nodes >= min_required` `min_required` is loaded from `knoe.cfg` Global key `OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES`. """ if self._optional_workloads_policy_cache is not None: return self._optional_workloads_policy_cache raw = ( (self.knoe_cfg_data.get("Global", {}) or {}).get(POLICY_CFG_KEY, "") ) try: min_nodes = int(str(raw).strip() or str(OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES)) except Exception: min_nodes = OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES allowed, ready_count, reason = evaluate_optional_workloads_allowed( env=env, min_nodes=min_nodes ) self._optional_workloads_policy_cache = (allowed, ready_count, reason) self.log(f"[POLICY] {reason}") return self._optional_workloads_policy_cache # --------------------------------------------------------- data access def _get_input(self, key: str, default: str | None = None) -> str: """Return user input for *key*. Subclasses **must** override.""" raise NotImplementedError def _get_input_bool(self, key: str, default: bool = False) -> bool: return _parse_bool(self._get_input(key, None), default=default) # --------------------------------------------------------- logging def log(self, msg: str): print(msg, flush=True) logging.info(msg) def err(self, msg: str): print(msg, file=sys.stderr, flush=True) logging.error(msg) # ------------------------------------------------ deployment helpers def _deployment_mode(self) -> str: return _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) def _secret_namespace(self) -> str: ns = (self._get_input("init_password.db_namespace", "") or "").strip() if not ns: ns = (self._get_input("env_setup.DATABASE_NAMESPACE", "") or "").strip() if not ns: ns = (self._get_input("env_setup.NAMESPACE", "") or "").strip() if not ns: ns = ( (self.knoe_cfg_data.get("Global", {}) or {}).get( "DATABASE_NAMESPACE", "" ) or "" ).strip() if not ns: ns = ( (self.knoe_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "") or "" ).strip() if ns.startswith("${") and ns.endswith("}"): ns = "" if ns: return ns try: if self._deployment_mode() == "k8s": return "knoe-db-0" except Exception: pass return "default" def _service_namespace(self) -> str: ns = ( (self.knoe_cfg_data.get("Global", {}) or {}) .get("SERVICE_NAMESPACE", "") .strip() ) if ns.startswith("${") and ns.endswith("}"): ns = "" if not ns: ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip() if ns: return ns # In k3s service clusters, core services live in knoe-system by default. try: if self._deployment_mode() == "k3s": return "knoe-system" except Exception: pass return "knoe-system" # ------------------------------------------ k3s connection helpers def _read_k3s_cfg_values( self, cfg_path: "Path | None" = None ) -> tuple[str, str, str]: """Return (cluster_env, server_url, token) from knoe.cfg. Delegates to the module-level ``_read_k3s_cfg`` so that every presentation layer (Tk, ncurses, silent) uses the same logic. """ if cfg_path is None: cfg_path = getattr(self, "cfg_path", None) return _read_k3s_cfg(cfg_path) def _resolve_k3s_connection(self) -> tuple[str, str]: """Resolve k3s server URL and token from all sources. Delegates to the module-level ``_resolve_k3s_connection`` so that every presentation layer uses identical resolution order. """ cfg_path = getattr(self, "cfg_path", None) root = getattr(self, "project_root", None) or PROJECT_ROOT topology = getattr(self, "ansible_topology", None) return _resolve_k3s_connection_fn( project_root=root, cfg_path=cfg_path, ansible_topology=topology, ) def _kubectl_base_cmd_for_k3s(self) -> list[str]: """Build a kubectl command prefix suitable for the k3s service cluster. Delegates to the module-level ``_kubectl_base_cmd_for_k3s``. """ managed = getattr(self, "_managed_kubeconfig", None) root = getattr(self, "project_root", None) or PROJECT_ROOT return _kubectl_base_cmd_for_k3s_fn( managed_kubeconfig=managed, project_root=root, ) def _kubectl_base_cmd(self, mode: str | None = None, role: str = "app") -> list[str]: """Build a consistent kubectl command based on the current cluster environment. Prefers environment-specific base commands (k3s/k8s) or KUBECONFIG, and ensures the selected context is used if provided. """ if not mode: mode = self._deployment_mode() # 1. Start with environment-specific base command if mode == "k3s": cmd = self._kubectl_base_cmd_for_k3s() elif mode == "k8s": kubeconfig = _find_kubeconfig_file() if kubeconfig: cmd = ["kubectl", "--kubeconfig", kubeconfig] else: cmd = ["kubectl"] elif mode == "k3d": # For k3d, check if we have a managed or merged config kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip() if kubeconfig_env: cmd = ["kubectl", "--kubeconfig", kubeconfig_env] else: cmd = ["kubectl"] else: cmd = ["kubectl"] # 2. Add context explicitly (and never rely on ambient current-context) if mode == "k8s": # In split-cluster GKE mode default kubectl access must target APP cluster # unless a step uses an explicit role-specific env override. ctx = self._cluster_kubecontext(role).strip() else: ctx = self._get_input("init_cluster.selected_kubectx", "").strip() if ctx and cmd[0] == "kubectl" and "--server" not in cmd and "--context" not in cmd: cmd.extend(["--context", ctx]) return cmd def ensure_db_k8s_secrets( self, namespace: str, db_password: str, *, log_fn=None, ) -> None: """Ensure DB/CNPG secrets exist in Kubernetes for *namespace*. `etc/init_cloudnative_pg.sh` expects the following secrets: - `knoe-db-user` with `username` and `password` - `knoe-db-superuser` with `username` and `password` - `cnpg-admin-key` with `admin.key` and `admin.pub` This uses the provided DB password (user-entered) and a locally generated ssh keypair (`~/.ssh/id_knoe_ed25519`) to bootstrap the admin key files in `KNOE_SERVICE/secrets/`. Notes: - `knoe-db-user` and `knoe-db-superuser` are always applied/updated. - `cnpg-admin-key` is created only if missing (no silent rotation). - Secret manifests are never logged (to avoid leaking credentials). """ ns = (namespace or "").strip() if not ns: raise Exception("Database namespace cannot be empty.") pw = (db_password or "").strip() if pw.startswith("${"): resolved = self._resolve_secret_value(pw) if resolved and not resolved.startswith("${"): pw = resolved if not pw or pw.startswith("${"): raise Exception("Database password is required to ensure Kubernetes secrets.") log = log_fn or getattr(self, "log", None) or (lambda *_: None) env = self._script_env_for_namespace(ns) # Determine the base kubectl command for the DB cluster. base_cmd = self._kubectl_base_cmd(role="db") # Extract the context name for logging if present. _db_ctx = "" if "--context" in base_cmd: idx = base_cmd.index("--context") if idx + 1 < len(base_cmd): _db_ctx = base_cmd[idx+1] if _db_ctx: log(f"Using DB cluster context: {_db_ctx}") def _run( cmd: list[str], *, input_text: str | None = None, log_output: bool = True, ) -> subprocess.CompletedProcess: # Replace 'kubectl' with the role-specific base command if applicable. final_cmd = list(cmd) if final_cmd and final_cmd[0] == "kubectl": final_cmd = base_cmd + final_cmd[1:] res = subprocess.run( final_cmd, input=input_text, text=True, capture_output=True, env=env, ) if log_output: out = (res.stdout or "").strip() err = (res.stderr or "").strip() if out: log(out) if err: log(err) return res def _ensure_namespace() -> None: chk = _run(["kubectl", "get", "namespace", ns], log_output=False) if chk.returncode == 0: return rc = _run(["kubectl", "create", "namespace", ns], log_output=True) if rc.returncode != 0: raise Exception(f"Failed to create namespace '{ns}'") def _apply_secret_from_create(create_cmd: list[str]) -> None: # 1) render YAML (do not log; contains secret material) rendered = _run(create_cmd, log_output=False) if rendered.returncode != 0 or not (rendered.stdout or "").strip(): raise Exception("Failed to render Kubernetes Secret manifest.") # 2) apply it (safe to log) applied = _run( ["kubectl", "apply", "-n", ns, "-f", "-"], input_text=rendered.stdout, log_output=True, ) if applied.returncode != 0: raise Exception("Failed to apply Kubernetes Secret manifest.") def _apply_literal_secret(name: str, data: dict[str, str]) -> None: args = ["kubectl", "create", "secret", "generic", name, "-n", ns] for k, v in data.items(): args.append(f"--from-literal={k}={v}") args.extend(["--dry-run=client", "-o", "yaml"]) _apply_secret_from_create(args) def _secret_exists(name: str) -> bool: res = _run(["kubectl", "-n", ns, "get", "secret", name], log_output=False) return res.returncode == 0 log(f"==> Ensuring DB secrets in namespace '{ns}'") _ensure_namespace() # Ensure expected admin key files exist on disk for CNPG tooling. service_dir = Path((env.get("KNOE_SERVICE") or "").strip()) secrets_dir = service_dir / "secrets" if service_dir else None if secrets_dir: secrets_dir.mkdir(parents=True, exist_ok=True) ssh_priv = Path.home() / ".ssh" / "id_knoe_ed25519" ssh_pub = Path.home() / ".ssh" / "id_knoe_ed25519.pub" if not (ssh_priv.exists() and ssh_pub.exists()): # Best-effort: generate if missing. try: self._generate_ssh_key(self._get_input("init_password.db_username", "knoe").strip() or "knoe") except Exception: pass if secrets_dir and ssh_priv.exists() and ssh_pub.exists(): # Write both generic and legacy filenames expected by scripts. try: admin_priv_generic = secrets_dir / "admin.key" admin_pub_generic = secrets_dir / "admin.pub" admin_priv_legacy = secrets_dir / "admin_ed25519" admin_pub_legacy = secrets_dir / "admin_ed25519.pub" if not admin_priv_generic.exists(): shutil.copyfile(ssh_priv, admin_priv_generic) os.chmod(admin_priv_generic, 0o600) if not admin_pub_generic.exists(): shutil.copyfile(ssh_pub, admin_pub_generic) if not admin_priv_legacy.exists(): shutil.copyfile(ssh_priv, admin_priv_legacy) os.chmod(admin_priv_legacy, 0o600) if not admin_pub_legacy.exists(): shutil.copyfile(ssh_pub, admin_pub_legacy) except Exception: pass # DB credentials secrets expected by CNPG init script. _apply_literal_secret("knoe-db-user", {"username": "knoe", "password": pw}) _apply_literal_secret( "knoe-db-superuser", {"username": "postgres", "password": pw} ) # Admin key secret only if missing (avoid rotation). if not _secret_exists("cnpg-admin-key"): if not secrets_dir: raise Exception("KNOE_SERVICE is not set; cannot locate secrets directory for CNPG admin key.") admin_priv = secrets_dir / "admin.key" admin_pub = secrets_dir / "admin.pub" if not (admin_priv.exists() and admin_pub.exists()): raise Exception( "CNPG admin key files are missing (admin.key/admin.pub); cannot create cnpg-admin-key secret." ) create = [ "kubectl", "create", "secret", "generic", "cnpg-admin-key", "-n", ns, f"--from-file=admin.key={admin_priv}", f"--from-file=admin.pub={admin_pub}", "--dry-run=client", "-o", "yaml", ] _apply_secret_from_create(create) def _verify_k3s_services_core(self) -> dict[str, str]: """Check registry, OpenBao and OpenTofu on a remote k3s cluster. Delegates to the module-level ``_verify_k3s_services_status``. """ managed = getattr(self, "_managed_kubeconfig", None) root = getattr(self, "project_root", None) or PROJECT_ROOT return _verify_k3s_services_status( project_root=root, managed_kubeconfig=managed, ) def _deploy_k3s_services_core(self, controller=None, log_fn=None): """Deploy registry, OpenBao and OpenTofu to a remote k3s cluster. *controller* must expose ``run_script(name, args, env, on_line)``. Falls back to ``self.controller`` when not provided. """ controller = controller or getattr(self, "controller", None) if controller is None: raise RuntimeError("No controller available for script execution") server, token = self._resolve_k3s_connection() if not server or not token: if log_fn: log_fn("Missing k3s server URL or token.\n") return ns = getattr(self, "db_namespace", None) or "default" if hasattr(ns, "get"): ns = ns.get() ns = (ns or "default").strip() db_pw = getattr(self, "db_password", "") or "" if hasattr(db_pw, "get"): db_pw = db_pw.get() env = os.environ.copy() root = getattr(self, "project_root", None) or PROJECT_ROOT env["KNOE_HOME"] = str(root) env["KNOE_SERVICE"] = str(root) env["DB_PASSWORD"] = db_pw env["OPENTOFU_ADMIN_PASSWORD"] = db_pw env["KNOE_MODE"] = "k3s" # Resolve KUBECONFIG through the same path the Tk UI and silent # installer use: prefer existing cert-based file, fall back to # token-generated kubeconfig. kubeconfig = _find_kubeconfig_file(env) if not kubeconfig: try: kubeconfig = str(_write_k3s_kubeconfig(server, token)) except Exception: pass if kubeconfig: env["KUBECONFIG"] = kubeconfig try: _merge_kubeconfig(kubeconfig) except Exception: pass service_ns = ( env.get("SERVICE_NAMESPACE") or (self.knoe_cfg_data.get("Global", {}) or {}).get("SERVICE_NAMESPACE", "") or "knoe-system" ) service_ns = str(service_ns).strip() or "knoe-system" argocd_ns = env.get("ARGOCD_NAMESPACE") or "argocd" registry_ns = env.get("REGISTRY_NAMESPACE") or service_ns if str(registry_ns).strip() == "default": registry_ns = service_ns env["ARGOCD_NAMESPACE"] = argocd_ns env["SERVICE_NAMESPACE"] = service_ns env["REGISTRY_NAMESPACE"] = registry_ns mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) try: registry_ops.update( namespace=registry_ns, env=env, project_root=root, mode=mode, log=log_fn, ) openbao_ops.update( namespace=ns, env=env, project_root=root, mode=mode, log=log_fn, ) opentofu_ops.update( namespace=ns, env=env, project_root=root, mode=mode, log=log_fn, ) except Exception as e: if log_fn: log_fn(f"Deploy error: {e}\n") # ------------------------------------------------- secret helpers def _secret_cfg_value( self, section: str, key: str, plaintext: str, leaf: str, bao_key: str ) -> str: if self._secrets_finalized: return _openbao_placeholder(self._secret_namespace(), leaf, bao_key) if not plaintext: return "" if _is_openbao_ref(plaintext) or _is_knoe_secret(plaintext): return plaintext existing = (self.knoe_cfg_data.get(section, {}) or {}).get(key) if existing and _is_knoe_secret(existing): try: if _decrypt_knoe_secret(existing) == plaintext: return existing except Exception: pass try: return _encrypt_knoe_secret(plaintext) except Exception: return plaintext def _resolve_openbao_ref(self, value: str) -> str: if not _is_openbao_ref(value): return value inner = value[len(OPENBAO_PREFIX) : -len(OPENBAO_SUFFIX)] path, key = (inner.split("#", 1) + [""])[:2] if not path or not key: return "" mount = "kv" secret_path = path if "/" in path: maybe_mount, rest = path.split("/", 1) if maybe_mount: mount = maybe_mount secret_path = rest token = os.environ.get("OPENBAO_ROOT_TOKEN", "") if not token: knoe_service = os.environ.get("KNOE_SERVICE") if knoe_service: token_path = Path(knoe_service) / "secrets" / "openbao-root-token" if token_path.exists(): token = token_path.read_text().strip() if not token: return value url = os.environ.get("PROLE_OPENBAO_URL") if not url: for p in ["8200", "18200"]: try: import urllib.request with urllib.request.urlopen( f"http://127.0.0.1:{p}/v1/sys/health", timeout=0.5 ) as r: if r.getcode() == 200: url = f"http://127.0.0.1:{p}" break except Exception: pass if not url: url = "http://127.0.0.1:8200" url = url.rstrip("/") try: req = urllib.request.Request(f"{url}/v1/{mount}/data/{secret_path}") req.add_header("X-Vault-Token", token) with urllib.request.urlopen(req, timeout=4) as resp: payload = json.loads(resp.read().decode("utf-8")) return payload.get("data", {}).get("data", {}).get(key, "") or "" except Exception: return value def _resolve_secretref_value(self, value: str) -> str: raw = str(value or "").strip() if not raw.startswith("secretref://"): return value ref = raw[len("secretref://") :].strip().strip("/") if not ref: return value env_candidates: list[str] = [] def _add_env(name: str) -> None: token = str(name or "").strip() if token and token not in env_candidates: env_candidates.append(token) _add_env(ref) normalized = ref.replace("-", "_").replace(".", "_").replace("/", "_") _add_env(normalized) _add_env(normalized.upper()) if ref == "google-oidc-client-id": for key in ( "GITLAB_OIDC_CLIENT_ID", "OIDC_CLIENT_ID", "GOOGLE_OIDC_CLIENT_ID", "GOOGLE_CLIENT_ID", ): _add_env(key) elif ref == "google-oidc-client-secret": for key in ( "GITLAB_OIDC_CLIENT_SECRET", "OIDC_CLIENT_SECRET", "GOOGLE_OIDC_CLIENT_SECRET", "GOOGLE_CLIENT_SECRET", ): _add_env(key) for env_key in env_candidates: candidate = str(os.environ.get(env_key, "") or "").strip() if candidate: return candidate file_candidates = [ Path(self.project_root) / "etc" / "secrets" / ref, Path(self.project_root) / "secrets" / ref, ] service_dir = str( self._get_input("env_setup.KNOE_SERVICE", "") or os.environ.get("KNOE_SERVICE", "") or "" ).strip() if service_dir: file_candidates.insert(0, Path(service_dir) / "secrets" / ref) for file_path in file_candidates: try: if file_path.exists() and file_path.is_file(): candidate = file_path.read_text(encoding="utf-8").strip() if candidate: return candidate except Exception: continue return value def _resolve_secret_value(self, value: str) -> str: if _is_knoe_secret(value): return _decrypt_knoe_secret(value) if _is_openbao_ref(value): return self._resolve_openbao_ref(value) if str(value or "").strip().startswith("secretref://"): return self._resolve_secretref_value(value) return value # --------------------------------------------- cfg sanitisation def _sanitize_sections_for_cfg(self, sections: dict) -> dict: sanitized = {k: dict(v) for k, v in sections.items()} # Persist fully expanded values in knoe.cfg (no $VAR / ${VAR} expansions). # Downstream tools treat these values as plain strings. env_map = dict(os.environ) try: env_map.update( {k: str(v or "") for k, v in (self._read_existing_env() or {}).items()} ) except Exception: pass for k in ( "KNOE_HOME", "KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE", "NAMESPACE", "SERVICE_NAMESPACE", ): val = (self._get_input(f"env_setup.{k}", "") or "").strip() if val: try: env_map[k] = self._expand_shell_path(val, env=env_map) except Exception: env_map[k] = val for section_name, section in sanitized.items(): for k, v in list(section.items()): if not isinstance(v, str) or not v.strip(): continue if _is_knoe_secret(v) or _is_openbao_ref(v): continue try: section[k] = self._cfgify_home_path( self._expand_shell_path(v, env=env_map) ) except Exception: section[k] = v if "Kerberos Authentication" in sanitized: val = sanitized["Kerberos Authentication"].get("PASSWORD", "") if val or ("Kerberos Authentication", "PASSWORD") in self._cfg_secret_cache: sanitized["Kerberos Authentication"]["PASSWORD"] = ( self._secret_cfg_value( "Kerberos Authentication", "PASSWORD", val, "kerberos", "password", ) ) if "Monitoring" in sanitized: val = sanitized["Monitoring"].get("GRAFANA_ADMIN_PASSWORD", "") if ( val or ("Monitoring", "GRAFANA_ADMIN_PASSWORD") in self._cfg_secret_cache ): sanitized["Monitoring"]["GRAFANA_ADMIN_PASSWORD"] = ( self._secret_cfg_value( "Monitoring", "GRAFANA_ADMIN_PASSWORD", val, "monitoring", "grafana_admin_password", ) ) return sanitized # ------------------------------------------- namespace helpers def _get_local_owner(self) -> str: try: return getpass.getuser() except Exception: try: return os.getlogin() except Exception: return "knoe" def _sanitize_namespace(self, name: str) -> str: cleaned = re.sub(r"[^a-z0-9-]+", "-", (name or "").lower()) cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-") if not cleaned: cleaned = "knoe-db" if len(cleaned) > 63: cleaned = cleaned[:63].rstrip("-") return cleaned def _generate_namespace_name(self) -> str: owner = self._sanitize_namespace(self._get_local_owner()) suffix = uuid.uuid4().hex[:6] base = f"knoe-db-{owner}-{suffix}" return self._sanitize_namespace(base) def _ensure_namespace_prefix(self, name: str) -> str: cleaned = (name or "").strip() if not cleaned: return "knoe-db" return cleaned def _read_existing_cfg_namespace(self) -> str | None: """Best-effort read of database namespace from the existing knoe.cfg. Silent retries should reuse the namespace stored in knoe.cfg unless the user explicitly overrides via env vars. """ cfg_path = getattr(self, "cfg_path", None) if not cfg_path: return None try: p = Path(str(cfg_path)).expanduser() if p.is_dir(): p = p / "knoe.cfg" if not p.exists(): return None cfg = configparser.ConfigParser(interpolation=None) cfg.optionxform = str cfg.read(p) # Current canonical location for section, key in ( ("User", "DATABASE_NAMESPACE"), ("User", "PROLE_NAMESPACE"), ("Global", "DATABASE_NAMESPACE"), # Legacy fallbacks ("User", "NAMESPACE"), ("Database Creation", "NAMESPACE"), ("Database Creation", "DATABASE_NAMESPACE"), ("Database Creation", "DB_NAME"), ): if cfg.has_option(section, key): ns = (cfg.get(section, key, fallback="") or "").strip() if ns: return ns except Exception: return None return None def _initial_namespace(self) -> str: # Explicit overrides must win. ns = ( os.environ.get("DATABASE_NAMESPACE") or os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE") ) if ns: return self._ensure_namespace_prefix(ns) try: existing = self._read_existing_env() ns = ( existing.get("DATABASE_NAMESPACE") or existing.get("NAMESPACE") or existing.get("PROLE_NAMESPACE") ) if ns: return self._ensure_namespace_prefix(ns) except Exception: pass # Prefer the namespace already stored in knoe.cfg to make retries stable. ns = self._read_existing_cfg_namespace() if ns: return self._ensure_namespace_prefix(ns) return self._ensure_namespace_prefix(self._generate_namespace_name()) def _initial_db_namespace(self) -> str: """Initial namespace suggestion for CNPG/Postgres. When a [CNPG Clusters] registry exists in knoe.cfg, the identity cluster's declared namespace is used unconditionally — no version suffix is appended. Version-aware namespace generation only applies as a last-resort fallback when no cluster registry is configured. """ # Registry-declared identity cluster wins unconditionally. identity = self._cnpg_identity_cluster() if identity and identity.get("namespace"): return self._sanitize_namespace( self._ensure_namespace_prefix(identity["namespace"]) ) # Explicit environment overrides. base = ( os.environ.get("DATABASE_NAMESPACE") or os.environ.get("KNOE_DB_NAMESPACE") or os.environ.get("DB_NAMESPACE") or os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE") or "" ).strip() if not base: try: existing = self._read_existing_env() base = ( (existing.get("DATABASE_NAMESPACE") or "") or (existing.get("NAMESPACE") or "") or (existing.get("PROLE_NAMESPACE") or "") ).strip() except Exception: base = "" if not base: base = (self._read_existing_cfg_namespace() or "").strip() or "knoe-db" return self._sanitize_namespace(self._ensure_namespace_prefix(base)) def _cnpg_cluster_registry(self) -> list[dict]: """Parse [CNPG Clusters] section from the active knoe.cfg. Returns a list of cluster dicts with keys: name, namespace, identity, instances, image. Validates that namespaces are unique and exactly one cluster is designated as the identity cluster. """ section = (getattr(self, "knoe_cfg_data", None) or {}).get("CNPG Clusters", {}) if not section: return [] clusters: dict[str, dict] = {} for raw_key, raw_val in section.items(): if "." not in raw_key: continue cluster_name, _, prop = raw_key.partition(".") cluster_name = cluster_name.strip() clusters.setdefault(cluster_name, {"name": cluster_name}) clusters[cluster_name][prop.strip()] = (raw_val or "").strip() result = list(clusters.values()) if not result: return [] # Validate: namespaces must be unique namespaces = [c.get("namespace", "") for c in result] if len(namespaces) != len(set(ns for ns in namespaces if ns)): raise ValueError("CNPG Clusters: duplicate namespace declared") # Validate: exactly one identity=true identity_count = sum( 1 for c in result if c.get("identity", "").lower() == "true" ) if identity_count != 1: raise ValueError( f"CNPG Clusters: exactly one cluster must have identity=true, found {identity_count}" ) return result def _cnpg_identity_cluster(self) -> dict | None: """Return the cluster entry with identity=true, or None if no registry.""" for c in self._cnpg_cluster_registry(): if c.get("identity", "").lower() == "true": return c return None def _is_valid_namespace(self, name: str) -> bool: if not name or len(name) > 63: return False if "db" not in name.lower(): return False return re.match(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", name) is not None def _generate_db_password(self, length: int = 24) -> str: alphabet = string.ascii_letters + string.digits return "".join(secrets.choice(alphabet) for _ in range(length)) # --------------------------------------------------- env helpers _FOREIGN_HOME_PREFIX_RE = re.compile(r"^/(Users|home)/([^/]+)(/.*)?$") def _rewrite_foreign_home_prefix(self, value: str) -> str: """Rewrite foreign home prefixes to the current user's home. Example: running on Linux with HOME=/home/, rewrite `/Users//dev/knoe` -> `/home//dev/knoe`. This helps prevent stale macOS paths (often stored in `~/.knoe/env.sh`) from being persisted into `knoe.cfg` on Linux. """ raw = (value or "").strip() if not raw or "$" in raw: return raw m = self._FOREIGN_HOME_PREFIX_RE.match(raw) if not m: return raw foreign_root = m.group(1) foreign_user = m.group(2) suffix = m.group(3) or "" try: current_home = Path.home() except Exception: return raw # Only rewrite when the path refers to the current username. if foreign_user != current_home.name: return raw system = platform.system() if system != "Darwin" and foreign_root == "Users": return str(current_home) + suffix if system == "Darwin" and foreign_root == "home": return str(current_home) + suffix return raw def _shellify_home_path(self, value: str) -> str: """Represent paths under the current home directory using `$HOME`. Keeps values unchanged if they already contain shell variables. """ raw = (value or "").strip() if not raw or "$" in raw: return raw try: p = Path(raw).expanduser() home = Path.home() rel = p.relative_to(home) except Exception: return raw rel_str = rel.as_posix() if not rel_str or rel_str == ".": return "$HOME" return f"$HOME/{rel_str}" def _expand_shell_path(self, value: str, env: dict[str, str] | None = None) -> str: raw = (value or "").strip() if not raw: return "" env_map = env or os.environ expanded = _expand_env_cfg_value(raw, env=env_map, max_depth=10) return os.path.expanduser(expanded) def _cfgify_home_path(self, value: str) -> str: """Represent paths under the current home directory using `$HOME`. Also rewrites stale foreign-home paths like `/Users//...` when the username matches the current user. """ raw = (value or "").strip() if not raw or "$" in raw: return raw raw = self._rewrite_foreign_home_prefix(raw) try: p = Path(raw).expanduser() home = Path.home() rel = p.relative_to(home) except Exception: return raw rel_str = rel.as_posix() if not rel_str or rel_str == ".": return "$HOME" return f"$HOME/{rel_str}" def _env_defaults(self, namespace: str | None = None) -> dict: # Default install layout: ~/.knoe # NOTE: UI may present this as $HOME/.knoe, but internally we keep # resolved literal paths for filesystem operations. default_home = resolve_knoe_home(env={}) resolved_home = self._resolve_env_value("KNOE_HOME", str(default_home)) or str( default_home ) env_map = dict(os.environ) env_map["KNOE_HOME"] = resolved_home expanded_home = self._expand_shell_path(resolved_home, env=env_map) or str(default_home) home = Path(expanded_home) env_map["KNOE_HOME"] = str(home) if namespace is None: namespace = ( self._get_input("init_password.db_namespace", "") or "" ).strip() if not namespace: namespace = os.environ.get("DATABASE_NAMESPACE", "") if not namespace: namespace = os.environ.get("NAMESPACE", "") return { "KNOE_HOME": str(home), "KNOE_CONF": self._expand_shell_path( self._resolve_env_value("KNOE_CONF", str(home / "conf")) or str(home / "conf"), env=env_map, ), "PROLE_DATA": self._expand_shell_path( self._resolve_env_value("PROLE_DATA", str(home / "data")) or str(home / "data"), env=env_map, ), "PROLE_LOGS": self._expand_shell_path( self._resolve_env_value("PROLE_LOGS", str(home / "logs")) or str(home / "logs"), env=env_map, ), "KNOE_SERVICE": self._expand_shell_path( self._resolve_env_value("KNOE_SERVICE", str(home / "etc")) or str(home / "etc"), env=env_map, ), "DATABASE_NAMESPACE": namespace or "", } def _resolve_env_value(self, key: str, fallback: str | None = None) -> str | None: val = os.environ.get(key) if val: return val try: env = self._read_existing_env() val = env.get(key) if val: return val except Exception: pass return fallback def _resolve_env_dir(self, key: str, default_suffix: str) -> Path: input_key = f"env_setup.{key}" val = (self._get_input(input_key, "") or "").strip() env_map = dict(os.environ) try: env_map.update({k: str(v or "") for k, v in (self._read_existing_env() or {}).items()}) except Exception: pass if val: try: return Path(self._expand_shell_path(val, env=env_map)) except Exception: pass val = self._resolve_env_value(key) if val: try: return Path(self._expand_shell_path(val, env=env_map)) except Exception: pass return Path.home() / "dev" / "knoe" / default_suffix def _read_existing_env(self) -> dict: env = {} knoe_home = os.environ.get("KNOE_HOME") candidates = [] if knoe_home: try: candidates.append(Path(self._expand_shell_path(knoe_home, env=os.environ)) / "env.sh") except Exception: candidates.append(Path(knoe_home) / "env.sh") candidates.append(resolve_knoe_home(env={}) / "env.sh") for p in candidates: try: if p.exists(): for line in p.read_text().splitlines(): line = line.strip() if not line or line.startswith("#"): continue if line.startswith("export "): line = line[len("export ") :] if "=" in line: k, v = line.split("=", 1) k = k.strip() v = v.strip().strip('"') if k in { "KNOE_HOME", "KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE", }: v = self._rewrite_foreign_home_prefix(v) env[k] = v break except Exception: pass return env def _save_env_to_file(self, values: dict): cluster_env = self._get_input("init_cluster.cluster_env", "dev") env_key = _normalize_cluster_env(cluster_env) env_map = dict(os.environ) try: env_map.update({str(k): str(v) for k, v in (values or {}).items() if v is not None}) except Exception: pass expanded_home = self._expand_shell_path(values.get("KNOE_HOME", ""), env=env_map) expanded_home = os.path.expanduser(expanded_home) if expanded_home else "" if expanded_home: env_map["KNOE_HOME"] = expanded_home home = Path(expanded_home) else: home = resolve_knoe_home(env={}) if env_key == "dev": home.mkdir(parents=True, exist_ok=True) for key in ("KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE"): try: raw = values.get(key, "") expanded = self._expand_shell_path(raw, env=env_map) expanded = os.path.expanduser(expanded) if expanded else "" if expanded: env_map[key] = expanded Path(expanded).mkdir(parents=True, exist_ok=True) except Exception: pass content = [] content.append("#!/usr/bin/env bash") content.append("# Knoe environment configuration") content.append( "# This file is generated by the installer. Source it in new shells, or execute as a wrapper:" ) content.append('# "$KNOE_HOME/env.sh" [args…]') content.append("# shellcheck shell=bash") for k in ( "KNOE_HOME", "KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE", ): content.append(f'export {k}="{self._shellify_home_path(values[k])}"') # NAMESPACE/PROLE_NAMESPACE must never be written to env.sh — only knoe.cfg is the source of truth content.append("") content.append("# Ensure PATH works for GUI-launched shells (Docker, etc.)") content.append( '_knoe_add_path() { case ":${PATH}:" in *":$1:"*) ;; *) PATH="$1:${PATH:-}" ;; esac; }' ) content.append('_knoe_add_path "$KNOE_HOME/bin"') content.append('_knoe_add_path "/opt/homebrew/bin"') content.append('_knoe_add_path "/usr/local/bin"') content.append('_knoe_add_path "/usr/bin"') content.append('_knoe_add_path "/bin"') content.append('_knoe_add_path "/usr/sbin"') content.append('_knoe_add_path "/sbin"') content.append("export PATH") content.append("") content.append("# Add custom paths below if needed (examples):") content.append("") content.append( "# If executed with arguments (and not sourced), run them under this environment" ) content.append('if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [ "$#" -gt 0 ]; then') content.append(' exec "$@"') content.append("fi") new_content = "\n".join(content) + "\n" out = home / "env.sh" if env_key != "dev": out = resolve_knoe_home(env={}) / "env.sh" out.parent.mkdir(parents=True, exist_ok=True) should_write = True if out.exists(): if out.read_text() == new_content: should_write = False self.log(f"[OK] {out} is up to date.") if should_write: tmp = out.parent / "env.sh.tmp" tmp.write_text(new_content) tmp.replace(out) try: os.chmod(out, 0o755) except Exception: pass self.log(f"[OK] Wrote {out}") self._deploy_env_resources(values) def _deploy_env_resources(self, values: dict): """Copy init scripts and etc directory into KNOE_HOME / KNOE_SERVICE.""" try: env_map = dict(os.environ) try: env_map.update({str(k): str(v) for k, v in (values or {}).items() if v is not None}) except Exception: pass expanded_home = self._expand_shell_path(values.get("KNOE_HOME", ""), env=env_map) expanded_home = os.path.expanduser(expanded_home) if expanded_home else "" if expanded_home: env_map["KNOE_HOME"] = expanded_home expanded_service = self._expand_shell_path(values.get("KNOE_SERVICE", ""), env=env_map) expanded_service = os.path.expanduser(expanded_service) if expanded_service else "" knoe_home = Path(expanded_home) if expanded_home else resolve_knoe_home(env={}) knoe_service = Path(expanded_service or (knoe_home / "etc")) init_pf_src_candidates = [ self.project_root / "src" / "knoe" / "etc" / "init-port-forward.sh", self.project_root / "etc" / "init-port-forward.sh", ] init_pf_src = next((p for p in init_pf_src_candidates if p.exists()), None) if init_pf_src is not None: init_pf_dst = knoe_home / "init-port-forward.sh" try: data = init_pf_src.read_bytes() init_pf_dst.write_bytes(data) os.chmod(init_pf_dst, 0o755) except Exception: pass etc_src_candidates = [ self.project_root / "src" / "knoe" / "etc", self.project_root / "etc", ] etc_src = next((p for p in etc_src_candidates if p.exists()), None) if etc_src is not None: try: knoe_service.mkdir(parents=True, exist_ok=True) if etc_src.resolve() != knoe_service.resolve(): if knoe_service.exists(): shutil.rmtree(knoe_service, ignore_errors=True) shutil.copytree(etc_src, knoe_service) except Exception: pass except Exception: pass def reload_env_from_shell(self) -> None: raw_home = self._get_input( "env_setup.KNOE_HOME", str(resolve_knoe_home(env={})) ) expanded_home = self._expand_shell_path(raw_home, env=os.environ) expanded_home = os.path.expanduser(expanded_home) if expanded_home else "" home = Path(expanded_home) if expanded_home else resolve_knoe_home(env={}) env_file = home / "env.sh" cmd = ( f"export KNOE_HOME={shlex.quote(str(home))}; " f"source {shlex.quote(str(env_file))}; " "env -0" ) try: out = subprocess.check_output(["bash", "-lc", cmd]) except Exception as e: raise Exception(f"Failed to reload environment: {e}") try: for raw in out.split(b"\x00"): if not raw: continue kv = raw.decode("utf-8", errors="ignore") if "=" not in kv: continue k, v = kv.split("=", 1) if k in ("PYTHONPATH", "PYTHONHOME"): continue os.environ[k] = v except Exception: pass def _update_env_namespace(self, namespace: str): # Namespace is persisted to knoe.cfg only — never to env.sh or os.environ. # This method is intentionally a no-op for environment mutation. pass # --------------------------------------------- port-forward helpers def _add_port_mapping(self, mapping_str): mode = self._deployment_mode() prefix = ( "PORT_FORWARD_K3S_MAPPING_" if mode == "k3s" else "PORT_FORWARD_K3D_MAPPING_" ) pf_section = self.knoe_cfg_data.get("Port Forwards", {}) if pf_section is None: pf_section = {} if not _pf_upsert_mapping(pf_section, prefix, mapping_str): return self.knoe_cfg_data["Port Forwards"] = pf_section if hasattr(self, "_save_knoe_cfg"): self._save_knoe_cfg() elif hasattr(self, "_write_cfg"): self._write_cfg() def _process_script_output_line(self, line): if "GRAFANA_ADMIN_PASSWORD=" in line: pwd = line.split("GRAFANA_ADMIN_PASSWORD=")[1].strip() if pwd: mon = self.knoe_cfg_data.get("Monitoring", {}) if mon is None: mon = {} mon["GRAFANA_ADMIN_PASSWORD"] = pwd self.knoe_cfg_data["Monitoring"] = mon if hasattr(self, "_save_knoe_cfg"): self._save_knoe_cfg() elif hasattr(self, "_write_cfg"): self._write_cfg() if "PORT_FORWARD_MAPPING:" in line: mapping = line.split("PORT_FORWARD_MAPPING:")[1].strip() if mapping: self._add_port_mapping(mapping) def _sync_port_forward_mappings(self): mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if not mode: mode = "k3d" prefix = ( "PORT_FORWARD_K3S_MAPPING_" if mode == "k3s" else "PORT_FORWARD_K3D_MAPPING_" ) pf_section = self.knoe_cfg_data.get("Port Forwards", {}) if pf_section is None: pf_section = {} _ids_to_remove_if_disabled = { "registry": True, "argocd": not self._get_input_bool("init_cluster.argocd_enabled", False), "dashboard": True, } for k, v in list(pf_section.items()): if not isinstance(v, str): continue for _id, _should_remove in _ids_to_remove_if_disabled.items(): if _should_remove and (f"id={_id};" in v or v.strip() == f"id={_id}"): del pf_section[k] break service_ns = (self._service_namespace() or "").strip() or "default" db_ns = (self._get_input("init_password.db_namespace", "") or "").strip() if not db_ns: db_ns = ( (self.knoe_cfg_data.get("Global", {}) or {}) .get("DATABASE_NAMESPACE", "") .strip() ) if not db_ns: db_ns = ( (self.knoe_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "").strip() ) if not db_ns: db_ns = ( os.environ.get("DATABASE_NAMESPACE") or os.environ.get("PROLE_NAMESPACE") or os.environ.get("NAMESPACE") or "" ).strip() if not db_ns: db_ns = "default" argocd_ns = ( (self.knoe_cfg_data.get("Global", {}) or {}) .get("ARGOCD_NAMESPACE", "") .strip() ) if not argocd_ns: argocd_ns = (os.environ.get("ARGOCD_NAMESPACE") or "").strip() if not argocd_ns: argocd_ns = "argocd" db_host_port = (self._get_input("init_password.db_host_port", "") or "").strip() if not db_host_port: db_host_port = ( (self.knoe_cfg_data.get("Global", {}) or {}) .get("DB_HOST_PORT", "") .strip() ) if not db_host_port: db_host_port = "5432" supabase_enabled = self._get_input_bool("init_cluster.supabase_enabled", False) supabase_ns = (os.environ.get("SUPABASE_NAMESPACE") or "").strip() if not supabase_ns: supabase_ns = ( (self.knoe_cfg_data.get("Supabase", {}) or {}) .get("NAMESPACE", "") .strip() ) if not supabase_ns: supabase_ns = "supabase" gitops_enabled = self._get_input_bool("init_cluster.gitops_enabled", False) gitops_ns = (os.environ.get("GITEA_NAMESPACE") or "").strip() if not gitops_ns: gitops_ns = ( (self.knoe_cfg_data.get("GitOps", {}) or {}) .get("GITOPS_NAMESPACE", "") .strip() ) if not gitops_ns: gitops_ns = (self._get_input("gitops.namespace", "") or "").strip() if not gitops_ns: gitops_ns = "gitea" argocd_enabled = self._get_input_bool("init_cluster.argocd_enabled", False) monitoring_release = ( (self.knoe_cfg_data.get("Monitoring", {}) or {}) .get("MONITORING_RELEASE", "") .strip() or os.environ.get("MONITORING_RELEASE", "").strip() or "prometheus" ) mappings = _build_required_port_forwards( mode=mode, service_ns=service_ns, argocd_ns=argocd_ns, db_ns=db_ns, db_host_port=db_host_port, supabase_enabled=supabase_enabled, supabase_namespace=supabase_ns, gitops_enabled=gitops_enabled, gitops_namespace=gitops_ns, argocd_enabled=argocd_enabled, dashboard_enabled=False, monitoring_release=monitoring_release, ) for mapping in mappings: _pf_upsert_mapping(pf_section, prefix, mapping) self.knoe_cfg_data["Port Forwards"] = pf_section # --------------------------------------------- script env def _script_env_for_namespace(self, namespace: str, cluster_role: str = "db") -> dict: namespace = _safe_str(namespace) cluster_role = "app" if str(cluster_role).strip().lower() == "app" else "db" env = os.environ.copy() root = getattr(self, "project_root", None) or PROJECT_ROOT cluster_name = ( self._app_cluster_name() if cluster_role == "app" else self._cnpg_cluster_name() ) env["KNOE_HOME"] = str(root) env["KNOE_SERVICE"] = str(root) env["DATABASE_NAMESPACE"] = namespace env["CLUSTER_NAME"] = cluster_name env["APP_CLUSTER_NAME"] = self._app_cluster_name() env["DB_CLUSTER_NAME"] = self._cnpg_cluster_name() env["APP_CLUSTER_MODE"] = self._app_cluster_mode() env["DB_CLUSTER_MODE"] = self._db_cluster_mode() # Transitional compatibility for scripts still reading NAMESPACE. env["NAMESPACE"] = namespace service_ns = self._service_namespace() env["KNOE_CONF"] = self._get_input( "env_setup.KNOE_CONF", str(root / "conf") ) env["PROLE_DATA"] = self._get_input( "env_setup.PROLE_DATA", str(root / "knoe-db" / "data") ) env["PROLE_LOGS"] = self._get_input( "env_setup.PROLE_LOGS", str(root / "logs") ) env["KNOE_SERVICE"] = self._get_input( "env_setup.KNOE_SERVICE", str(root / "etc") ) env["KNOE_DB_USER"] = self._get_input("init_password.db_username", "").strip() # Make the DB/root master password available to shell scripts. # OpenTofu bootstrap uses it to create the initial admin secret when missing. db_pw = (self._get_input("init_password.db_password", "") or "").strip() if db_pw: env.setdefault("DB_PASSWORD", db_pw) env.setdefault("OPENTOFU_ADMIN_PASSWORD", db_pw) mode = self._deployment_mode() if mode: env["KNOE_MODE"] = mode env["DEPLOYMENT_MODE"] = mode cluster_env = self._get_input("init_cluster.cluster_env", "dev") env["DEPLOYMENT_TARGET"] = _deployment_target_label(cluster_env) if mode == "k3s": # 1. Prefer an existing kubeconfig that already exists on disk. # Check knoe-specific paths before ~/.kube/config to avoid # picking up a stale k3d context from the default kubeconfig. root = getattr(self, "project_root", None) or PROJECT_ROOT _knoe_kc_candidates = [ str(Path(root) / "knoe-k3s.kubeconfig"), str(Path(root) / "etc" / "secrets" / "k3s.kubeconfig"), str(Path(root) / "secrets" / "k3s.kubeconfig"), ] knoe_service_dir = (env.get("KNOE_SERVICE") or "").strip() if knoe_service_dir: _knoe_kc_candidates.insert(1, str(Path(knoe_service_dir) / "secrets" / "k3s.kubeconfig")) kc = next( (p for p in _knoe_kc_candidates if p and Path(p).expanduser().exists()), "", ) or _find_kubeconfig_file(env) if kc: env["KUBECONFIG"] = kc else: # 2. Try fetching from k3s via Ansible or generate token-based fetched = self._fetch_k3s_kubeconfig() if fetched: env["KUBECONFIG"] = str(fetched) else: server, token = self._resolve_k3s_connection() if server and token and _looks_like_k8s_bearer_token(token): try: managed = _write_k3s_kubeconfig(server, token) self._managed_kubeconfig = str(managed) env["KUBECONFIG"] = self._managed_kubeconfig _merge_kubeconfig(self._managed_kubeconfig) except Exception: pass elif server and token: self.err( "[WARN] k3s token does not look like a Kubernetes bearer token; cannot generate kubeconfig. Ensure a cert-based knoe-k3s.kubeconfig exists (Ansible fetch)." ) raw_server, raw_token = self._resolve_k3s_connection() if raw_server: env["K3S_SERVER"] = raw_server if raw_token: env["K3S_TOKEN"] = raw_token elif mode == "k3d": # Ensure k3d dev clusters have a resolvable KUBECONFIG. cluster_name = self._get_input("init_cluster.deployment_target", "knoe-dev-cluster") if not cluster_name or cluster_name == "knoe-dev-cluster": # Fallback to check if we have a more specific one from UI if available cluster_name = getattr(self, "selected_k3d_cluster", None) if hasattr(cluster_name, "get"): cluster_name = cluster_name.get() cluster_name = cluster_name or "knoe-dev-cluster" try: subprocess.run( [ "k3d", "kubeconfig", "merge", cluster_name, "--kubeconfig-switch-context", ], capture_output=True, text=True, timeout=10, ) except Exception: pass default_kube = str(Path.home() / ".kube" / "config") if Path(default_kube).exists(): env["KUBECONFIG"] = default_kube elif mode == "k8s": # For GKE/prod split-cluster deploys we require explicit role context. gke_context = self._cluster_kubecontext(cluster_role).strip() default_kube = str(Path.home() / ".kube" / "config") if Path(default_kube).exists(): env["KUBECONFIG"] = default_kube if gke_context: env["KUBECONTEXT"] = gke_context env["KUBE_CONTEXT_NAME"] = gke_context env["KUBECTL_CONTEXT"] = gke_context else: env.pop("KUBECONFIG", None) env["SERVICE_NAMESPACE"] = service_ns env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) env["SUPABASE_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_enabled", False) ) env["SUPABASE_STUDIO_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_studio_enabled", False) ) default_supabase_studio_url = "db.0.knoe.dev" if mode == "k8s" else "db.knoe.org" env["SUPABASE_STUDIO_URL"] = ( self._get_input("init_cluster.supabase_studio_url", default_supabase_studio_url).strip() or default_supabase_studio_url ) env["SUPABASE_AUTH_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_auth_enabled", True) ) env["SUPABASE_REALTIME_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_realtime_enabled", True) ) env["SUPABASE_META_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_meta_enabled", True) ) env["SUPABASE_ANALYTICS_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_analytics_enabled", True) ) db_pw = self._get_input("init_password.db_password", "").strip() if db_pw: db_pw = self._resolve_secret_value(db_pw) env["DB_PASSWORD"] = db_pw env["OPENTOFU_ADMIN_PASSWORD"] = db_pw grafana_pw = ( self.knoe_cfg_data.get("Monitoring", {}) .get("GRAFANA_ADMIN_PASSWORD", "") .strip() ) resolved_grafana = self._resolve_secret_value(grafana_pw) if grafana_pw else "" if resolved_grafana and not ( _is_openbao_ref(resolved_grafana) or _is_knoe_secret(resolved_grafana) ): env["GRAFANA_ADMIN_PASSWORD"] = resolved_grafana elif db_pw: env["GRAFANA_ADMIN_PASSWORD"] = db_pw realm = self._get_input("kerberos_config.realm", "").strip() if realm: env["KRB5_REALM"] = realm env["REALM"] = realm env["DOMAIN"] = realm.lower() kdc = self._get_input("kerberos_config.kdc", "").strip() if kdc: env["KRB5_KDC"] = kdc env["KRB5_ADMIN"] = kdc env["KRB5_USER"] = self._get_input("kerberos_config.user", "") krb_pw = self._get_input("kerberos_config.password", "") if krb_pw: env["KRB5_PASSWORD"] = self._resolve_secret_value(krb_pw) else: env["KRB5_PASSWORD"] = "" # Registry settings reg_host = self.knoe_cfg_data.get("Docker Build", {}).get("LOCAL_REGISTRY") if reg_host: env["LOCAL_REGISTRY"] = reg_host reg_internal = self.knoe_cfg_data.get("Docker Build", {}).get( "LOCAL_REGISTRY_INTERNAL" ) if reg_internal: env["LOCAL_REGISTRY_INTERNAL"] = reg_internal # Artifact Registry is required by CNPG manifests in k8s mode. artifact_registry = str(env.get("ARTIFACT_REGISTRY") or "").strip().strip('"') if not artifact_registry: artifact_registry = ( (self.knoe_cfg_data.get("Global", {}) or {}) .get("ARTIFACT_REGISTRY", "") .strip() .strip('"') ) if not artifact_registry: gcp_cfg = self.knoe_cfg_data.get("GCP", {}) or {} project_id = ( (self._get_input("init_cluster.project_id", "") or "").strip() or str(gcp_cfg.get("project_id") or "").strip().strip('"') or str(gcp_cfg.get("PROJECT_ID") or "").strip().strip('"') or (os.environ.get("GCP_PROJECT") or "").strip() ) region = ( (self._get_input("init_cluster.db_cluster_region", "") or "").strip() or (self._get_input("init_cluster.app_cluster_region", "") or "").strip() or str(gcp_cfg.get("region") or "").strip().strip('"') or str(gcp_cfg.get("REGION") or "").strip().strip('"') or str(gcp_cfg.get("location") or "").strip().strip('"') or (os.environ.get("GCP_REGION") or "").strip() ) repo = (service_ns or "knoe-system").strip() or "knoe-system" if project_id and region: artifact_registry = f"{region}-docker.pkg.dev/{project_id}/{repo}" if artifact_registry: env["ARTIFACT_REGISTRY"] = artifact_registry.rstrip("/") try: placement_plan, placement_path = self._resolve_cnpg_placement_plan(namespace) eligible_nodes = placement_plan.get("eligible_nodes") or [] assignments = placement_plan.get("assignments") or {} assignment_pairs = [] for ordinal in sorted(assignments.keys(), key=lambda k: int(str(k))): assignment_pairs.append(f"{ordinal}:{assignments[ordinal]}") env["CNPG_PLACEMENT_PLAN_FILE"] = str(placement_path) env["CNPG_PLACEMENT_PLAN_ID"] = str(placement_plan.get("plan_id") or "") env["CNPG_PLACEMENT_PLAN_HASH"] = str( placement_plan.get("plan_hash") or "" ) env["CNPG_PLACEMENT_ELIGIBLE_NODES"] = ",".join(eligible_nodes) env["CNPG_PLACEMENT_ASSIGNMENTS"] = ",".join(assignment_pairs) env["CNPG_PLACEMENT_REUSED"] = _bool_str( bool((placement_plan.get("metadata") or {}).get("reused", False)) ) env.setdefault( "CNPG_CLUSTER_NAME", str(placement_plan.get("cluster_name") or "knoe-db"), ) env.setdefault( "CNPG_INSTANCES", str(placement_plan.get("desired_instances") or "3"), ) # Compatibility bridge for existing shell topology flow. if eligible_nodes: env.setdefault("CNPG_STAGE1_NODE", str(eligible_nodes[0])) if len(eligible_nodes) == 1: env.setdefault( "CNPG_DB_NODE_SELECTOR", f"kubernetes.io/hostname={eligible_nodes[0]}", ) except Exception as e: self.err(f"[WARN] CNPG placement planning failed; using fallback topology: {e}") # Only apply dual-cluster GKE targeting in k8s/prod mode. if mode == "k8s": return build_kubectl_env_for_cluster( base_env=env, kubecontext=self._cluster_kubecontext(cluster_role), cluster_name=cluster_name, cluster_role=cluster_role, ) return env def _cnpg_cluster_name(self) -> str: glob = self.knoe_cfg_data.get("Global", {}) or {} cluster_name = ( os.environ.get("CLUSTER_NAME") or os.environ.get("CNPG_CLUSTER_NAME") or self._get_input("init_password.db_cluster_name", "") or self._get_input("env_setup.DB_CLUSTER_NAME", "") or str(glob.get("CLUSTER_NAME") or "") or str(glob.get("CNPG_CLUSTER_NAME") or "") ).strip() return cluster_name or "knoe-db" def _app_cluster_name(self) -> str: glob = self.knoe_cfg_data.get("Global", {}) or {} name = ( self._get_input("init_password.app_cluster_name", "") or self._get_input("env_setup.APP_CLUSTER_NAME", "") or str(glob.get("APP_CLUSTER_NAME") or "") or DEFAULT_APP_CLUSTER_NAME ) return str(name).strip() or DEFAULT_APP_CLUSTER_NAME def _app_cluster_mode(self) -> str: glob = self.knoe_cfg_data.get("Global", {}) or {} mode = ( self._get_input("init_cluster.app_cluster_mode", "") or self._get_input("env_setup.APP_CLUSTER_MODE", "") or str(glob.get("APP_CLUSTER_MODE") or "") or DEFAULT_APP_CLUSTER_MODE ) return str(mode).strip() or DEFAULT_APP_CLUSTER_MODE def _db_cluster_mode(self) -> str: glob = self.knoe_cfg_data.get("Global", {}) or {} mode = ( self._get_input("init_cluster.db_cluster_mode", "") or self._get_input("env_setup.DB_CLUSTER_MODE", "") or str(glob.get("DB_CLUSTER_MODE") or "") or DEFAULT_DB_CLUSTER_MODE ) return str(mode).strip() or DEFAULT_DB_CLUSTER_MODE def _cluster_kubecontext(self, role: str) -> str: role = "app" if str(role).strip().lower() == "app" else "db" key = ( "init_cluster.app_cluster_kubecontext" if role == "app" else "init_cluster.db_cluster_kubecontext" ) fallback_key = ( "env_setup.APP_CLUSTER_KUBECONTEXT" if role == "app" else "env_setup.DB_CLUSTER_KUBECONTEXT" ) context = _safe_str(self._get_input(key, "") or self._get_input(fallback_key, "")) glob = self.knoe_cfg_data.get("Global", {}) or {} global_key = "APP_CLUSTER_KUBECONTEXT" if role == "app" else "DB_CLUSTER_KUBECONTEXT" if not context: context = _safe_str(glob.get(global_key, "")) opposite_key = ( "init_cluster.db_cluster_kubecontext" if role == "app" else "init_cluster.app_cluster_kubecontext" ) opposite_fallback_key = ( "env_setup.DB_CLUSTER_KUBECONTEXT" if role == "app" else "env_setup.APP_CLUSTER_KUBECONTEXT" ) opposite_global_key = "DB_CLUSTER_KUBECONTEXT" if role == "app" else "APP_CLUSTER_KUBECONTEXT" opposite_context = _safe_str( self._get_input(opposite_key, "") or self._get_input(opposite_fallback_key, "") or glob.get(opposite_global_key, "") ) if context and opposite_context and context == opposite_context: raise RuntimeError( f"Refusing {role.upper()} targeting: APP/DB contexts must differ, both resolve to '{context}'." ) if self._deployment_mode() == "k8s": if context: return context # Fall through to derive context from cluster name + region below. if context: return context selected_context = _safe_str( self._get_input("init_cluster.selected_kubectx", "") or glob.get("KUBECONTEXT", "") ) if selected_context and not selected_context.startswith("gke_"): return selected_context if role == "app": cluster_name = self._app_cluster_name() region = _safe_str( self._get_input("init_cluster.app_cluster_region", "") or self._get_input("env_setup.APP_CLUSTER_REGION", "") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("region") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("REGION") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("location") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("LOCATION") ) else: cluster_name = _safe_str( self._get_input("init_password.db_cluster_name", "") or self._get_input("env_setup.DB_CLUSTER_NAME", "") or glob.get("DB_CLUSTER_NAME", "") or glob.get("CNPG_CLUSTER_NAME", "") or DEFAULT_DB_CLUSTER_NAME ) or DEFAULT_DB_CLUSTER_NAME region = _safe_str( self._get_input("init_cluster.db_cluster_region", "") or self._get_input("env_setup.DB_CLUSTER_REGION", "") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("region") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("REGION") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("location") or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("LOCATION") ) gcp_cfg = self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {} project_id = _safe_str( self._get_input("init_cluster.project_id", "") or gcp_cfg.get("project_id") or gcp_cfg.get("PROJECT_ID") ) if not region: region = _safe_str( gcp_cfg.get("region") or gcp_cfg.get("REGION") or gcp_cfg.get("location") or gcp_cfg.get("LOCATION") ) if project_id and region: return f"gke_{project_id}_{region}_{cluster_name}" parts = selected_context.split("_", 3) if len(parts) == 4: return f"{parts[0]}_{parts[1]}_{parts[2]}_{cluster_name}" return selected_context def _cnpg_desired_instances(self) -> int: glob = self.knoe_cfg_data.get("Global", {}) or {} raw = (os.environ.get("CNPG_INSTANCES") or str(glob.get("CNPG_INSTANCES") or "")).strip() try: val = int(raw) if raw else 3 except Exception: val = 3 return max(1, val) def _cnpg_rebalance_requested(self) -> bool: glob = self.knoe_cfg_data.get("Global", {}) or {} raw = ( os.environ.get("CNPG_REBALANCE") or str(glob.get("CNPG_REBALANCE") or "") ) return _parse_bool(raw, default=False) def _cnpg_candidate_nodes(self) -> list[str]: glob = self.knoe_cfg_data.get("Global", {}) or {} nodes: set[str] = set() explicit = ( os.environ.get("CNPG_ELIGIBLE_NODES") or str(glob.get("CNPG_ELIGIBLE_NODES") or "") ) if explicit: for part in explicit.split(","): node = str(part or "").strip() if node: nodes.add(node) mounts_raw = str( (self.knoe_cfg_data.get("Storage", {}) or {}).get("ANSIBLE_ISCSI_MOUNTS", "") ).strip() if mounts_raw: try: mounts = json.loads(mounts_raw) except Exception: mounts = {} if isinstance(mounts, dict): for details in mounts.values(): if not isinstance(details, dict): continue node = str(details.get("host") or "").strip() if node: nodes.add(node) stage1 = str(glob.get("CNPG_STAGE1_NODE") or "").strip() if stage1: nodes.add(stage1) return sorted(nodes) def _cnpg_plan_path(self, namespace: str, cluster_name: str) -> Path: conf_root = self._get_input( "env_setup.KNOE_CONF", str((getattr(self, "project_root", None) or PROJECT_ROOT) / "conf"), ) conf_dir = Path(conf_root) def _safe_segment(raw: str) -> str: s = str(raw or "").strip() if not s: return "default" return "".join(ch if ch.isalnum() or ch in "-_." else "-" for ch in s) ns_seg = _safe_segment(namespace) cluster_seg = _safe_segment(cluster_name) return conf_dir / "cnpg-placement" / f"{ns_seg}-{cluster_seg}.json" def _resolve_cnpg_placement_plan(self, namespace: str) -> tuple[dict, Path]: cluster_name = self._cnpg_cluster_name() desired_instances = self._cnpg_desired_instances() candidate_nodes = self._cnpg_candidate_nodes() rebalance = self._cnpg_rebalance_requested() plan_path = self._cnpg_plan_path(namespace, cluster_name) prior_plan = load_cnpg_placement_plan(plan_path) placement_plan = plan_cnpg_placement( cluster_name=cluster_name, desired_instances=desired_instances, candidate_nodes=candidate_nodes, prior_plan=prior_plan, rebalance=rebalance, ) if not isinstance(prior_plan, dict) or prior_plan != placement_plan: save_cnpg_placement_plan(plan_path, placement_plan) glob = self.knoe_cfg_data.setdefault("Global", {}) glob["CNPG_PLACEMENT_PLAN_FILE"] = str(plan_path) glob["CNPG_PLACEMENT_PLAN_ID"] = str(placement_plan.get("plan_id") or "") glob["CNPG_PLACEMENT_PLAN_HASH"] = str(placement_plan.get("plan_hash") or "") return placement_plan, plan_path def _cnpg_storage_node(self, env: dict[str, str]) -> str: stage1 = str(env.get("CNPG_STAGE1_NODE") or "").strip() if stage1: return stage1 eligible_raw = str(env.get("CNPG_PLACEMENT_ELIGIBLE_NODES") or "").strip() if eligible_raw: for part in eligible_raw.split(","): node = str(part or "").strip() if node: return node candidates = self._cnpg_candidate_nodes() if candidates: return candidates[0] raise StorageProvisioningError( "Unable to determine CNPG node for local PV provisioning. " "Expected CNPG placement env (CNPG_STAGE1_NODE/CNPG_PLACEMENT_ELIGIBLE_NODES) " "or configured CNPG candidate nodes." ) def _ensure_cnpg_storage_provisioned(self, namespace: str, env: dict[str, str]) -> None: if self._deployment_mode() != "k3s": return cluster_name = str( env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") or self._cnpg_cluster_name() ).strip() if not cluster_name: cluster_name = "knoe-db" node_name = self._cnpg_storage_node(env) spec = ClusterStorageSpec( namespace=namespace, cluster_name=cluster_name, node_name=node_name, storage_class_name="synology-iscsi", command_env=env, ) provisioned = provision_cluster_storage(spec) env["CNPG_STORAGE_CLASS"] = spec.storage_class_name env["CNPG_DATA_SELECTOR_JSON"] = json.dumps( provisioned.data_selector, separators=(",", ":"), sort_keys=True, ) env["CNPG_WAL_SELECTOR_JSON"] = json.dumps( provisioned.wal_selector, separators=(",", ":"), sort_keys=True, ) env["CNPG_DATA_PV_NAME"] = provisioned.data_pv_name env["CNPG_WAL_PV_NAME"] = provisioned.wal_pv_name env["CNPG_DATA_HOST_PATH"] = provisioned.data_path env["CNPG_WAL_HOST_PATH"] = provisioned.wal_path env["SYNOLOGY_ROOTS"] = ",".join(spec.synology_roots) self.log( "[CNPG] Provisioned cluster storage " f"namespace={namespace} cluster={cluster_name} node={node_name} " f"data_pv={provisioned.data_pv_name} wal_pv={provisioned.wal_pv_name} " f"data_path={provisioned.data_path} wal_path={provisioned.wal_path}" ) def _cluster_storage_milestone_enabled(self) -> bool: raw = str( os.environ.get("PROLE_ENABLE_CLUSTER_STORAGE_MILESTONE", "") ).strip().lower() return raw in {"1", "true", "yes", "on"} # --------------------------------------------- authority / repair def _authority_context_missing(self) -> bool: enabled = self._get_input_bool( "kerberos_config.enabled", False ) or self._get_input_bool("init_cluster.kerberos_enabled", False) if not enabled: return False candidates = [] env_home = (os.environ.get("KNOE_HOME") or "").strip() if env_home: candidates.append(Path(env_home)) candidates.append(self.project_root) for base in candidates: try: if (base / "authority").is_dir(): return False if (base / "knoe" / "authority").is_dir(): return False except Exception: continue return True def _dashboard_kong_missing(self, env: dict) -> bool: ns = "kubernetes-dashboard" try: res_ns = subprocess.run( ["kubectl", "get", "ns", ns, "-o", "name"], env=env, capture_output=True, text=True, timeout=6, ) if res_ns.returncode != 0 or ns not in (res_ns.stdout or ""): return True res = subprocess.run( ["kubectl", "-n", ns, "get", "pods", "--no-headers"], env=env, capture_output=True, text=True, timeout=6, ) if res.returncode != 0: return True kong_lines = [ l for l in (res.stdout or "").splitlines() if "kong" in l.lower() ] if not kong_lines: return True for line in kong_lines: parts = line.split() if len(parts) < 3: return True if parts[2] != "Running": return True return False except Exception: return True def _parse_k8s_quantity_bytes(self, raw: str | None) -> int | None: """Parse a Kubernetes resource quantity (e.g. 10Gi, 500Mi) into bytes. Conservative: returns None when parsing fails. """ s = (raw or "").strip() if not s: return None # Common forms: 10Gi, 500Mi, 1Ti, 1000, 1.5Gi num = "" suf = "" for ch in s: if (ch.isdigit() or ch in (".", "+", "-")) and not suf: num += ch else: suf += ch try: value = float(num) except Exception: return None suf = suf.strip() if not suf: try: return int(value) except Exception: return None binary = { "Ki": 1024, "Mi": 1024**2, "Gi": 1024**3, "Ti": 1024**4, "Pi": 1024**5, "Ei": 1024**6, } decimal = { "K": 1000, "M": 1000**2, "G": 1000**3, "T": 1000**4, "P": 1000**5, "E": 1000**6, } if suf in binary: return int(value * binary[suf]) if suf in decimal: return int(value * decimal[suf]) # Be conservative: unknown suffix (e.g. m) => do not parse. return None def _kubectl_get_json(self, env: dict, args: list[str]) -> dict | None: try: res = subprocess.run( ["kubectl"] + list(args), env=env, capture_output=True, text=True, timeout=30, ) if res.returncode != 0: err = (res.stderr or "").strip() or (res.stdout or "").strip() if err: self.err(f"[WARN] kubectl {' '.join(args)} failed: {err}") return None out = (res.stdout or "").strip() if not out: return None return json.loads(out) except Exception as e: self.err(f"[WARN] kubectl {' '.join(args)} failed: {e}") return None def _kubectl_get_namespace_json(self, env: dict, name: str) -> dict | None: n = (name or "").strip() if not n: return None return self._kubectl_get_json(env, ["get", "namespace", n, "-o", "json"]) def _namespace_is_terminating(self, ns: dict | None) -> bool: if not ns: return False md = ns.get("metadata") or {} if (md.get("deletionTimestamp") or "").strip(): return True phase = ((ns.get("status") or {}).get("phase") or "").strip() return phase == "Terminating" # ------------------------------------------ blocked cluster reconciliation def _classify_blocked_pod(self, env: dict, pod: dict) -> dict: """Classify common pod blockers into stable, testable categories. Returns a dict with: - `type`: primary classification (storage-first) - `types`: list of all detected classification tags """ _ = env # reserved for future use meta = pod.get("metadata") or {} status = pod.get("status") or {} name = (meta.get("name") or "").strip() ns = (meta.get("namespace") or "").strip() phase = (status.get("phase") or "").strip() types: list[str] = [] # ContainerCreating / ImagePullBackOff etc. for cs in status.get("containerStatuses") or []: waiting = ((cs.get("state") or {}).get("waiting") or {}) reason = (waiting.get("reason") or "").strip() if reason == "ContainerCreating": types.append("container_creating") elif reason: types.append(f"container_waiting_{reason.lower()}") # Unschedulable reasons. for cond in status.get("conditions") or []: try: if (cond.get("type") or "").strip() != "PodScheduled": continue if (cond.get("status") or "").strip() != "False": continue if (cond.get("reason") or "").strip() != "Unschedulable": continue msg = ((cond.get("message") or "").strip() or "").lower() if "persistent volumes" in msg and "bind" in msg: types.append("unschedulable_due_to_pv_binding") if "didn't match pod's node affinity" in msg or "node affinity/selector" in msg: types.append("unschedulable_due_to_node_affinity") if "had untolerated taint" in msg: types.append("unschedulable_due_to_untolerated_taint") except Exception: continue # Primary type: prefer storage/root-cause categories. primary = "ok" for preferred in ( "unschedulable_due_to_pv_binding", "unschedulable_due_to_node_affinity", "unschedulable_due_to_untolerated_taint", "container_creating", ): if preferred in types: primary = preferred break if primary == "ok" and phase in ("Pending", "Unknown") and types: primary = types[0] # Dedupe while keeping order. seen: set[str] = set() deduped: list[str] = [] for t in types: if t not in seen: seen.add(t) deduped.append(t) return { "pod": f"{ns}/{name}" if ns and name else (name or ns or ""), "namespace": ns, "name": name, "phase": phase, "type": primary, "types": deduped, } def _reconcile_blocked_cluster_state(self, env: dict, service_ns: str) -> None: """Detect hard cluster blockers and raise before running deployment scripts. This is intentionally conservative: it attempts safe PV claimRef repair and then surfaces clear diagnostics for scheduling/storage blockers. """ svc_ns = (service_ns or "").strip() or self._service_namespace() # Always attempt safe stale Released PV claimRef repair first. try: self._repair_stale_released_pvs(env, reset=False) except Exception as e: self.err(f"[WARN] Stale PV claimRef repair failed: {e}") pods_data = self._kubectl_get_json(env, ["get", "pod", "-A", "-o", "json"]) or {} pods = pods_data.get("items") or [] classified = [self._classify_blocked_pod(env, p) for p in pods] blocked = [c for c in classified if (c.get("type") or "") != "ok"] # If we can't positively identify blocked pods, do not block deployment. # This keeps behavior conservative when `kubectl` is unavailable. if not blocked: return # Downstream missingness: surface but treat separately. missing: list[str] = [] for kind in ("svc", "deploy"): try: res = subprocess.run( ["kubectl", "-n", svc_ns, "get", kind, "knoe-svc-kong"], env=env, capture_output=True, text=True, timeout=10, ) if res.returncode != 0: missing.append(f"{kind}/knoe-svc-kong") except FileNotFoundError: # If kubectl isn't available, don't guess missing resources. pass except Exception: pass all_types: set[str] = set() for b in blocked: for t in b.get("types") or []: all_types.add(t) # Group summary for easier scanning. storage_types = [t for t in sorted(all_types) if "pv_binding" in t] sched_types = [ t for t in sorted(all_types) if "node_affinity" in t or "untolerated_taint" in t ] container_types = [t for t in sorted(all_types) if t.startswith("container_")] lines: list[str] = ["Cluster prerequisites are blocked"] if storage_types or any("pv_binding" in (b.get("type") or "") for b in blocked): lines.append("\nStorage blockers:") for t in storage_types or ["unschedulable_due_to_pv_binding"]: lines.append(f"- {t}") if sched_types: lines.append("\nScheduling blockers:") for t in sched_types: lines.append(f"- {t}") if container_types: lines.append("\nContainer blockers:") for t in container_types: lines.append(f"- {t}") if blocked: lines.append("\nBlocked pods:") for b in blocked: lines.append( f"- {b.get('pod')} type={b.get('type')} types={','.join(b.get('types') or [])}" ) if missing: lines.append("\nDownstream missing resources:") for m in sorted(set(missing)): lines.append(f"- {m}") raise Exception("\n".join(lines)) def _ensure_namespace_ready(self, env: dict, name: str, timeout_s: int = 90) -> None: """Ensure a namespace is usable for deployments. If the namespace is in `Terminating`, wait for deletion to complete. If it disappears, recreate it. If it remains terminating after the timeout, raise with an actionable message. """ n = (name or "").strip() if not n: return ns = self._kubectl_get_namespace_json(env, n) if self._namespace_is_terminating(ns): self.err( f"[WARN] Namespace '{n}' is terminating; waiting up to {timeout_s}s for deletion to complete..." ) deadline = time.time() + max(0, int(timeout_s)) while time.time() < deadline: time.sleep(3) ns = self._kubectl_get_namespace_json(env, n) if not ns: break if not self._namespace_is_terminating(ns): break ns = self._kubectl_get_namespace_json(env, n) if self._namespace_is_terminating(ns): raise Exception( f"Namespace '{n}' is still terminating. " f"Wait for it to finish deleting (kubectl get namespace {n}) or force-delete it, " f"then retry the installer." ) # Create namespace if missing. if not self._kubectl_get_namespace_json(env, n): self.log(f"[INFO] Creating namespace '{n}' ...") rc = self._run_cmd(["kubectl", "create", "namespace", n], env=env) if rc != 0: # Be idempotent: a concurrent creator may have raced us. self.err(f"[WARN] Failed to create namespace '{n}' (code {rc}); continuing") def _list_pending_pvcs( self, env: dict, namespaces: list[str] | None = None ) -> list[dict]: data = self._kubectl_get_json(env, ["get", "pvc", "-A", "-o", "json"]) items = (data or {}).get("items") or [] ns_filter = {n.strip() for n in (namespaces or []) if (n or "").strip()} out: list[dict] = [] for pvc in items: try: phase = ((pvc.get("status") or {}).get("phase") or "").strip() if phase != "Pending": continue ns = ((pvc.get("metadata") or {}).get("namespace") or "").strip() if ns_filter and ns not in ns_filter: continue out.append(pvc) except Exception: continue return out def _list_released_pvs(self, env: dict, storage_class: str | None = None) -> list[dict]: data = self._kubectl_get_json(env, ["get", "pv", "-o", "json"]) items = (data or {}).get("items") or [] sc = (storage_class or "").strip() out: list[dict] = [] for pv in items: try: phase = ((pv.get("status") or {}).get("phase") or "").strip() if phase != "Released": continue pv_sc = ((pv.get("spec") or {}).get("storageClassName") or "").strip() if sc and pv_sc != sc: continue out.append(pv) except Exception: continue return out def _match_stale_released_pv(self, pvc: dict, pvs: list[dict]) -> dict | None: pvc_meta = pvc.get("metadata") or {} pvc_spec = pvc.get("spec") or {} pvc_name = (pvc_meta.get("name") or "").strip() pvc_ns = (pvc_meta.get("namespace") or "").strip() pvc_uid = (pvc_meta.get("uid") or "").strip() pvc_sc = (pvc_spec.get("storageClassName") or "").strip() pvc_volume_name = (pvc_spec.get("volumeName") or "").strip() pvc_modes = pvc_spec.get("accessModes") or [] pvc_mode = (pvc_spec.get("volumeMode") or "").strip() req_storage = ( (((pvc_spec.get("resources") or {}).get("requests") or {}).get("storage") or "").strip() ) req_bytes = self._parse_k8s_quantity_bytes(req_storage) if not pvc_name or not pvc_ns: return None candidates: list[tuple[int, dict]] = [] for pv in pvs: pv_meta = pv.get("metadata") or {} pv_spec = pv.get("spec") or {} pv_status = pv.get("status") or {} pv_name = (pv_meta.get("name") or "").strip() if not pv_name: continue if (pv_status.get("phase") or "").strip() != "Released": continue if pvc_volume_name and pvc_volume_name != pv_name: continue pv_sc = (pv_spec.get("storageClassName") or "").strip() if pvc_sc and pv_sc and pvc_sc != pv_sc: continue if pvc_sc and not pv_sc: continue claim_ref = pv_spec.get("claimRef") or {} if not claim_ref: continue if (claim_ref.get("name") or "").strip() != pvc_name: continue if (claim_ref.get("namespace") or "").strip() != pvc_ns: continue claim_uid = (claim_ref.get("uid") or "").strip() if pvc_uid and claim_uid and pvc_uid == claim_uid: # Too risky: PV believes it is still claimed by this exact PVC. continue pv_modes = pv_spec.get("accessModes") or [] if pvc_modes: if not pv_modes: continue if not set(pvc_modes).issubset(set(pv_modes)): continue pv_mode = (pv_spec.get("volumeMode") or "").strip() if pvc_mode and pv_mode and pvc_mode != pv_mode: continue cap_storage = (((pv_spec.get("capacity") or {}).get("storage") or "").strip()) cap_bytes = self._parse_k8s_quantity_bytes(cap_storage) if req_bytes is not None and cap_bytes is not None: if req_bytes > cap_bytes: continue else: # If we can't compare sizes, be conservative. continue candidates.append((cap_bytes or 0, pv)) if not candidates: return None candidates.sort(key=lambda t: t[0]) return candidates[0][1] def _clear_pv_claim_ref(self, env: dict, pv_name: str) -> bool: pv_name = (pv_name or "").strip() if not pv_name: return False patch = '{"spec":{"claimRef":null}}' try: res = subprocess.run( [ "kubectl", "patch", "pv", pv_name, "--type", "merge", "-p", patch, ], env=env, capture_output=True, text=True, timeout=20, ) if res.returncode == 0: return True err = (res.stderr or "").strip() or (res.stdout or "").strip() if err: self.err(f"[WARN] Failed to patch pv/{pv_name} claimRef: {err}") return False except Exception as e: self.err(f"[WARN] Failed to patch pv/{pv_name} claimRef: {e}") return False def _repair_stale_released_pvs(self, env: dict, reset: bool = False) -> bool: """Repair stale `Released` PVs that retain an old `spec.claimRef`. Policy: - Normal repair is conservative. - When `reset=True`, we may proactively reclaim safe, strongly-matched PVs by clearing `spec.claimRef`. - Reset must NEVER delete PVs. """ service_ns = (env.get("SERVICE_NAMESPACE") or "").strip() or self._service_namespace() db_ns = ( (env.get("DB_NAMESPACE") or "").strip() or (self._get_input("init_password.db_namespace", "") or "").strip() or service_ns ) namespaces = sorted( { n for n in ( service_ns, db_ns, (env.get("NAMESPACE") or "").strip(), ) if (n or "").strip() } ) pending = self._list_pending_pvcs(env, namespaces=namespaces) if not pending: return False # Pull all Released PVs once; matching enforces storageClass/claimRef checks. released = self._list_released_pvs(env) if not released: return False changed = False for pvc in pending: meta = pvc.get("metadata") or {} spec = pvc.get("spec") or {} pvc_name = (meta.get("name") or "").strip() pvc_ns = (meta.get("namespace") or "").strip() pvc_sc = (spec.get("storageClassName") or "").strip() req = (((spec.get("resources") or {}).get("requests") or {}).get("storage") or "").strip() match = self._match_stale_released_pv(pvc, released) if not match: continue pv_name = ((match.get("metadata") or {}).get("name") or "").strip() pv_cap = (((match.get("spec") or {}).get("capacity") or {}).get("storage") or "").strip() claim_ref = (match.get("spec") or {}).get("claimRef") or {} claim_uid = (claim_ref.get("uid") or "").strip() prefix = "[RESET]" if reset else "[INFO]" self.log( f"{prefix} Detected pending PVC with matching Released PV claimRef; clearing claimRef " f"pvc/{pvc_ns}/{pvc_name} storageClass={pvc_sc or ''} request={req or ''} " f"-> pv/{pv_name} capacity={pv_cap or ''} claimRef.uid={claim_uid or ''}" ) if self._clear_pv_claim_ref(env, pv_name): self.log(f"[OK] Cleared claimRef on pv/{pv_name}") changed = True else: self.err(f"[WARN] Failed to clear claimRef on pv/{pv_name}") return changed def _reset_reclaim_stale_pvs(self, env: dict) -> bool: """Reset/reconcile hook to reclaim safe stale PVs. Reset may reclaim stale `Released` PVs by clearing `spec.claimRef`. Reset must NEVER delete PVs. """ return self._repair_stale_released_pvs(env, reset=True) def _run_cluster_repair_pipeline(self, env: dict, anomalies: list[str]) -> None: """Python-native cluster repair orchestration. This mirrors the intent of `etc/repair_pipeline.sh` while delegating actual component deployments to existing `etc/init_*.sh` scripts. """ mode = (env.get("KNOE_MODE") or env.get("DEPLOYMENT_MODE") or "").strip() service_ns = (env.get("SERVICE_NAMESPACE") or "").strip() or self._service_namespace() db_ns = ( (env.get("DB_NAMESPACE") or "").strip() or (self._get_input("init_password.db_namespace", "") or "").strip() or service_ns ) argocd_ns = (env.get("ARGOCD_NAMESPACE") or "").strip() or self._argocd_namespace() registry_ns = (env.get("REGISTRY_NAMESPACE") or "").strip() or self._registry_namespace() def _kubectl(args: list[str], timeout: int = 30) -> subprocess.CompletedProcess: return subprocess.run( ["kubectl"] + args, env=env, capture_output=True, text=True, timeout=timeout, ) self.log(f"[INFO] Cluster repair starting (mode={mode or ''})") # Basic reachability check (non-fatal; keeps behavior conservative). try: res = _kubectl(["cluster-info"], timeout=15) if res.returncode != 0: self.err( f"[WARN] kubectl cluster-info failed: {((res.stderr or '') or (res.stdout or '')).strip()}" ) except Exception as e: self.err(f"[WARN] kubectl cluster-info failed: {e}") # --- Dashboard repair (Helm-based) if "dashboard" in (anomalies or []): try: self.log("[INFO] Repairing kubernetes-dashboard...") # Try to create namespace (idempotent) _kubectl(["create", "ns", "kubernetes-dashboard"], timeout=20) # Helm install/upgrade as in the legacy pipeline subprocess.run( ["helm", "repo", "add", "kubernetes-dashboard", "https://kubernetes.github.io/dashboard/"], env=env, capture_output=True, text=True, timeout=60, ) subprocess.run( ["helm", "repo", "update"], env=env, capture_output=True, text=True, timeout=120, ) subprocess.run( [ "helm", "upgrade", "--install", "kubernetes-dashboard", "kubernetes-dashboard/kubernetes-dashboard", "--namespace", "kubernetes-dashboard", "--create-namespace", "--set", "kong.enabled=true", "--set", "nginx.enabled=false", ], env=env, capture_output=True, text=True, timeout=300, ) # Restart pods (idempotent, best-effort) _kubectl(["-n", "kubernetes-dashboard", "delete", "pods", "-l", "app.kubernetes.io/name=kubernetes-dashboard"], timeout=60) _kubectl(["-n", "kubernetes-dashboard", "delete", "pods", "-l", "app.kubernetes.io/name=kong"], timeout=60) except FileNotFoundError: self.err("[WARN] helm not found; skipping dashboard Helm repair") except Exception as e: self.err(f"[WARN] Dashboard repair failed: {e}") # --- k3s legacy default namespace dedupe try: if mode == "k3s" and service_ns != "default": res_legacy = _kubectl(["get", "ns", "default", "-o", "name"], timeout=10) if res_legacy.returncode == 0 and "default" in (res_legacy.stdout or ""): legacy_ns = "default" def _exists(kind: str, name: str, ns: str) -> bool: r = _kubectl(["-n", ns, "get", kind, name, "-o", "name"], timeout=10) return r.returncode == 0 def _delete_kind(kind: str, name: str, ns: str): _kubectl(["-n", ns, "delete", kind, name, "--ignore-not-found=true"], timeout=60) legacy_openbao = _exists("deploy", "openbao", legacy_ns) or _exists( "statefulset", "openbao", legacy_ns ) legacy_opentofu = _exists("deploy", "opentofu", legacy_ns) legacy_garage = _exists("statefulset", "garage", legacy_ns) legacy_registry = _exists("deploy", "registry", legacy_ns) legacy_kong = _exists("deploy", "knoe-svc-kong", legacy_ns) or _exists( "svc", "knoe-svc-kong", legacy_ns ) missing_openbao = not ( _exists("deploy", "openbao", service_ns) or _exists("statefulset", "openbao", service_ns) ) missing_opentofu = not _exists("deploy", "opentofu", service_ns) missing_garage = not _exists("statefulset", "garage", service_ns) missing_registry = not _exists("deploy", "registry", registry_ns) if legacy_registry and missing_registry: self.log( f"[INFO] Legacy registry in {legacy_ns}; redeploying to {registry_ns}" ) registry_ops.update(namespace=registry_ns, env=env, mode=mode, log=self.log) _delete_kind("deploy", "registry", legacy_ns) if legacy_openbao and missing_openbao: self.log( f"[INFO] Legacy OpenBao in {legacy_ns}; redeploying to {service_ns}" ) openbao_ops.update(namespace=service_ns, env=env, mode=mode, log=self.log) _delete_kind("deploy", "openbao", legacy_ns) _delete_kind("statefulset", "openbao", legacy_ns) _delete_kind("pvc", "data-openbao-0", legacy_ns) if legacy_opentofu and missing_opentofu: self.log( f"[INFO] Legacy OpenTofu in {legacy_ns}; redeploying to {service_ns}" ) opentofu_ops.update(namespace=service_ns, env=env, mode=mode, log=self.log) _delete_kind("deploy", "opentofu", legacy_ns) if legacy_garage and missing_garage: self.log( f"[INFO] Legacy Garage in {legacy_ns}; redeploying to {service_ns}" ) garage_store_ops.update( namespace=service_ns, env=env, mode=mode, log=self.log, ) _delete_kind("statefulset", "garage", legacy_ns) _delete_kind("pvc", "meta-garage-0", legacy_ns) _delete_kind("pvc", "data-garage-0", legacy_ns) if legacy_kong: missing_kong = not ( _exists("deploy", "knoe-svc-kong", service_ns) or _exists("svc", "knoe-svc-kong", service_ns) ) if missing_kong: self.log( f"[INFO] Legacy Kong (knoe-svc-kong) in {legacy_ns}; redeploying to {service_ns}" ) self._run_script( "init_kong.sh", args=["-n", service_ns, "update"], env=env, ) # Always remove legacy Kong to avoid namespace drift. _kubectl( [ "-n", legacy_ns, "delete", "pod", "-l", "app=knoe-svc-kong", "--ignore-not-found=true", ], timeout=60, ) _delete_kind("deploy", "knoe-svc-kong", legacy_ns) _delete_kind("svc", "knoe-svc-kong", legacy_ns) _delete_kind("configmap", "knoe-svc-kong-config", legacy_ns) # Clean up prole-era Kong (named "prole-svc-kong") from both # legacy_ns and service_ns — stale after the prole→knoe rebrand. legacy_prole_kong = ( _exists("deploy", "prole-svc-kong", legacy_ns) or _exists("svc", "prole-svc-kong", legacy_ns) or _exists("deploy", "prole-svc-kong", service_ns) or _exists("svc", "prole-svc-kong", service_ns) ) if legacy_prole_kong: self.log("[INFO] Cleaning up legacy prole-svc-kong resources") for _ns in (legacy_ns, service_ns): _kubectl( [ "-n", _ns, "delete", "pod", "-l", "app=prole-svc-kong", "--ignore-not-found=true", ], timeout=60, ) _delete_kind("deploy", "prole-svc-kong", _ns) _delete_kind("svc", "prole-svc-kong", _ns) _delete_kind("configmap", "prole-svc-kong-config", _ns) except Exception as e: self.err(f"[WARN] Legacy namespace dedupe failed: {e}") # --- ArgoCD repair (registry redeploy) try: argocd_apps = [ ("deployment", "argocd-repo-server"), ("deployment", "argocd-application-controller"), ("statefulset", "argocd-redis"), ] unhealthy = False for kind, name in argocd_apps: r = _kubectl( [ "-n", argocd_ns, "get", kind, name, "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode != 0: unhealthy = True break try: if int((r.stdout or "0").strip() or "0") < 1: unhealthy = True break except Exception: unhealthy = True break if unhealthy: self.log("[WARN] ArgoCD is not healthy; redeploying registry...") env2 = env.copy() env2["REGISTRY_NAMESPACE"] = registry_ns registry_ops.stop(namespace=registry_ns, env=env2, mode=mode, log=self.log) registry_ops.update(namespace=registry_ns, env=env2, mode=mode, log=self.log) except Exception as e: self.err(f"[WARN] ArgoCD repair step failed: {e}") # --- Service restarts in the service namespace try: # OpenBao r = _kubectl( [ "-n", service_ns, "get", "statefulset", "openbao", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode == 0: if int((r.stdout or "0").strip() or "0") < 1: self.log("[WARN] OpenBao is not ready; restarting...") openbao_ops.restart(namespace=service_ns, env=env, log=self.log) except Exception: pass try: # OpenTofu r = _kubectl( [ "-n", service_ns, "get", "deploy", "opentofu", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode == 0: if int((r.stdout or "0").strip() or "0") < 1: self.log("[WARN] OpenTofu is not ready; restarting...") opentofu_ops.restart(namespace=service_ns, env=env, log=self.log) except Exception: pass try: # Garage r = _kubectl( [ "-n", service_ns, "get", "statefulset", "garage", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r.returncode == 0: if int((r.stdout or "0").strip() or "0") < 1: self.log("[WARN] Garage is not ready; restarting...") garage_store_ops.restart(namespace=service_ns, env=env, log=self.log) except Exception: pass # --- Kerberos / KDC repair (gated) try: kerberos_enabled = self._get_input_bool( "kerberos_config.enabled", False ) or self._get_input_bool("init_cluster.kerberos_enabled", False) if kerberos_enabled: auth_missing = self._authority_context_missing() if not auth_missing: # Only attempt KDC init when authority context exists. env2 = env.copy() env2["DB_NAMESPACE"] = db_ns env2["SERVICE_NAMESPACE"] = service_ns self._run_script( "init_kdc.sh", args=["-n", service_ns], env=env2, ) except Exception as e: self.err(f"[WARN] KDC repair step failed: {e}") # --- Barman plugin repair (best-effort) try: r = _kubectl(["-n", "cnpg-system", "get", "deploy", "barman-cloud"], timeout=10) env2 = env.copy() env2["DATABASE_NAMESPACE"] = db_ns env2["CLUSTER_NAME"] = self._cnpg_cluster_name() if r.returncode != 0: cnpg_install_barman_plugin(env=env2, log=self.log) else: r2 = _kubectl( [ "-n", "cnpg-system", "get", "deploy", "barman-cloud", "-o", "jsonpath={.status.readyReplicas}", ], timeout=10, ) if r2.returncode == 0: try: if int((r2.stdout or "0").strip() or "0") < 1: cnpg_install_barman_plugin(env=env2, log=self.log) _kubectl(["-n", "cnpg-system", "rollout", "restart", "deploy/barman-cloud"], timeout=60) except Exception: pass except Exception: pass # --- CNPG rollout (best-effort) try: # Only attempt if a Cluster exists in the DB namespace. r = _kubectl( ["-n", db_ns, "get", "clusters.postgresql.cnpg.io", "cluster", "-o", "name"], timeout=10, ) if r.returncode == 0 and (r.stdout or "").strip(): env2 = env.copy() env2["DATABASE_NAMESPACE"] = db_ns env2["CLUSTER_NAME"] = self._cnpg_cluster_name() cnpg_rollout( namespace=db_ns, cluster_name=self._cnpg_cluster_name(), env=env2, log=self.log, ) except Exception: pass # Final health/status report try: status_ok = { "registry": registry_ops.status( namespace=self._registry_namespace(), env=env, mode=env.get("KNOE_MODE") or self._deployment_mode(), ), "openbao": openbao_ops.status( namespace=service_ns, env=env, mode=env.get("KNOE_MODE") or self._deployment_mode(), ), "opentofu": opentofu_ops.status(namespace=service_ns, env=env), "garage": garage_store_ops.status(namespace=service_ns, env=env), } for name, ok in status_ok.items(): self.log(f"[STATUS] {name}: {'ready' if ok else 'not-ready'}") except Exception: pass self.log("[OK] Cluster repair pipeline complete") def _maybe_run_repair_pipeline(self, env: dict) -> None: # Always attempt stale Released PV claimRef repair first. try: self._repair_stale_released_pvs(env) except Exception as e: self.err(f"[WARN] Stale PV claimRef repair failed: {e}") # Best-effort namespace preflight before broader anomaly repairs. try: self._ensure_namespace_ready(env, self._service_namespace()) except Exception as e: self.err(f"[WARN] Namespace preflight failed: {e}") return if self._repair_ran: return anomalies = [] if self._dashboard_kong_missing(env): anomalies.append("dashboard") if self._authority_context_missing(): anomalies.append("authority") if not anomalies: return self._repair_ran = True service_ns = self._service_namespace() db_ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" self.log( f"[INFO] Detected anomalies ({', '.join(anomalies)}); running repair pipeline..." ) self._run_cluster_repair_pipeline(env=env, anomalies=anomalies) # ------------------------------------------ image helpers def _collect_dependent_images( self, include_supabase: bool, include_kerberos_proxy: bool ) -> list[str]: images: set[str] = set() for rel_dir in ("k8s/knoe", "k8s/openbao"): base_dir = self.project_root / rel_dir if not base_dir.exists(): continue images.update(_collect_images_from_files(list(base_dir.glob("*.yaml")))) if include_supabase: supa_home = _resolve_supabase_home(self.project_root) if supa_home: docker_dir = supa_home / "docker" compose_files = [docker_dir / "docker-compose.yml"] if os.environ.get("SUPABASE_USE_DEV_COMPOSE") == "1": compose_files.append(docker_dir / "dev" / "docker-compose.dev.yml") images.update(_collect_images_from_files(compose_files)) if not include_supabase: images = {img for img in images if "supabase" not in img} if include_kerberos_proxy: krb_img = os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat") if krb_img: images.add(krb_img) return sorted(images) def _get_docker_import_dir(self) -> str: """Best-effort accessor for a docker-import directory. GUI uses a Tk variable with `.get()`, while console flows tend to store a plain string. This helper normalizes both. """ val = getattr(self, "docker_import_dir", None) if val is None: return "" try: getter = getattr(val, "get", None) if callable(getter): val = getter() except Exception: pass return (str(val) if val is not None else "").strip() def _prepull_images_to_registry( self, include_supabase: bool, include_kerberos_proxy: bool, log: Callable[[str], None] | None = None, ) -> bool: def _log(msg: str): if log: try: log(msg) except Exception: pass self.log(msg) cluster_env = "" try: cluster_env = self._get_input("init_cluster.cluster_env", "") except Exception: cluster_env = "" if cluster_env and not _local_registry_enabled(cluster_env): _log("[SKIP] Local registry disabled; skipping image pre-pull.") return True mode = _deployment_mode_from_env(cluster_env) if mode == "k3s": # Keep behavior consistent across installers: k3s relies on in-cluster # image pulls/import paths rather than a local docker registry. _log("[SKIP] Image pre-pull is not supported in k3s mode.") return True ensure_fn = getattr(self, "ensure_local_registry_available", None) if not callable(ensure_fn): ensure_fn = getattr(self, "_ensure_local_registry_available", None) info = ensure_fn() if callable(ensure_fn) else None if not info: _log("[SKIP] Local registry unavailable; skipping image pre-pull.") return False registry, _cluster_registry = info images = self._collect_dependent_images( include_supabase, include_kerberos_proxy ) if not images: _log("[SKIP] No dependent images found to pre-pull.") return True overall_ok = True import_dir = self._get_docker_import_dir() for image in images: local_tag = image if not image.startswith(f"{registry}/"): local_tag = f"{registry}/{image}" # 1. Check if already in local registry _log(f"[INFO] Checking if {image} exists in local registry...") check_reg = subprocess.run( ["docker", "pull", local_tag], capture_output=True, text=True ) if check_reg.returncode == 0: _log(f"[OK] {image} already exists in local registry as {local_tag}") continue # 2. Check if we already have it in local docker daemon check_local = subprocess.run( ["docker", "image", "inspect", image], capture_output=True, text=True, ) found = check_local.returncode == 0 if not found and import_dir and os.path.isdir(import_dir): # 3. Check import directory safe_name = image.replace("/", "_").replace(":", "_") tar_path = Path(import_dir) / f"{safe_name}.tar" if tar_path.exists(): _log(f"[INFO] Found {tar_path} in import directory, loading...") load = subprocess.run( ["docker", "load", "-i", str(tar_path)], capture_output=True, text=True, ) if load.returncode == 0: found = True else: overall_ok = False self.err(f"[WARN] Failed to load {tar_path}: {load.stderr}") if not found: # 4. Pull from upstream _log(f"[INFO] Pulling {image} from upstream...") pull = subprocess.run( ["docker", "pull", image], capture_output=True, text=True ) if pull.returncode != 0: overall_ok = False self.err(pull.stdout or "") self.err(pull.stderr or "") self.err(f"[ERROR] docker pull failed for {image}") continue found = True # If we have the image locally, tag and push to local registry if found: if local_tag != image: tag = subprocess.run( ["docker", "tag", image, local_tag], capture_output=True, text=True, ) if tag.returncode != 0: overall_ok = False self.err(tag.stdout or "") self.err(tag.stderr or "") self.err( f"[ERROR] docker tag failed for {image} to {local_tag}" ) continue _log(f"[INFO] Pushing {local_tag} to local registry...") if not _push_docker_image(local_tag, log_fn=self.log): overall_ok = False continue _log(f"[OK] Stored {image} in local registry as {local_tag}") return overall_ok class _TeeWriter: """Write to both the original stream and a log file simultaneously.""" def __init__(self, original, log_file): self._original = original self._log_file = log_file def write(self, data): self._original.write(data) self._original.flush() try: self._log_file.write(data) self._log_file.flush() except Exception: pass return len(data) def flush(self): self._original.flush() try: self._log_file.flush() except Exception: pass def fileno(self): return self._original.fileno() def isatty(self): return self._original.isatty() def reconfigure(self, **kwargs): if hasattr(self._original, "reconfigure"): self._original.reconfigure(**kwargs) def __getattr__(self, name): return getattr(self._original, name) class KnoeConsoleInstaller(KnoeInstaller): """Console-based unattended installer driven by knoe.cfg inputs.""" silent = True def __init__( self, controller: KnoeController, cfg_path: str | None = None, reset_cluster: bool = False, delete_db: bool = False, ): self.controller = controller self.project_root = controller.project_root self.dependencies = list(inst_config.DEPENDENCIES) self.cfg_path = self._normalize_cfg_path(cfg_path) # Ensure the controller also has the normalized config path self.controller.cfg_path = self.cfg_path self.reset_cluster = bool(reset_cluster) self.reset_requested = bool(reset_cluster) self.delete_db_requested = bool(delete_db) # Keep a persistent indicator for reset-oriented reconcile actions (e.g. # reclaiming safe stale Released PV claimRefs). `_perform_cluster_reset()` # consumes `reset_requested` and clears it, but we still want to run # reset reconcile steps later once the cluster is reachable. self._reset_reclaim_pvs_requested = bool(reset_cluster) self.inputs: dict = {} self._init_shared_state() self._db_built_success = False self._scripts_success = False self._cnpg_success = False self.docker_import_dir = None self._openbao_init_env = None self._openbao_init_password = None self._log_file = None def enable_log_file(self, path: str): """Enable tee-style logging: output goes to both stdout and the log file.""" self._log_file = open(path, "a") sys.stdout = _TeeWriter(sys.__stdout__, self._log_file) sys.stderr = _TeeWriter(sys.__stderr__, self._log_file) self.log(f"[LOG] Logging to {path}") # ---- Data access (from inputs dict) ---- def _get_input(self, key: str, default: str | None = None) -> str: if key in self.inputs: return _safe_str(self.inputs[key]) return default if default is not None else "" # ---------------- Config helpers ---------------- def _normalize_cfg_path(self, raw: str | None) -> Path: if raw: p = Path(_expand_path(raw)) if p.is_dir(): return p / "knoe.cfg" return p conf_dir = knoe_conf_mgr.resolve_knoe_conf_dir(self.project_root) return knoe_conf_mgr.entrypoint_path(conf_dir) def _load_inputs_from_cfg(self) -> dict: # Ensure env-dir layout exists and legacy entrypoint config is migrated before loading. try: if self.cfg_path.name == "knoe.cfg": if self.cfg_path.parent.name in knoe_conf_mgr.ENVIRONMENTS: knoe_conf_mgr.ensure_entrypoint(self.cfg_path.parent.parent) else: knoe_conf_mgr.ensure_entrypoint(self.cfg_path.parent) except Exception: pass if not self.cfg_path.exists(): raise FileNotFoundError(f"knoe.cfg not found at {self.cfg_path}") cfg = knoe_conf_mgr.load_layered_config(self.cfg_path) cfg_vars = _collect_cfg_vars(cfg) # Load all sections into knoe_cfg_data to maintain idempotency for section in cfg.sections(): if section not in self.knoe_cfg_data: self.knoe_cfg_data[section] = {} for k, v in cfg.items(section): self.knoe_cfg_data[section][k] = _expand_cfg_value(v, cfg_vars) if cfg.has_section("Global"): service_ns = _expand_cfg_value( cfg.get("Global", "SERVICE_NAMESPACE", fallback=""), cfg_vars ).strip() if service_ns: self.knoe_cfg_data["Global"]["SERVICE_NAMESPACE"] = service_ns if cfg.has_section("Port Forwards"): self.knoe_cfg_data["Port Forwards"] = { k: _expand_cfg_value(v, cfg_vars) for k, v in cfg.items("Port Forwards") } for (section, key), _spec in SECRET_KEY_SPECS.items(): if cfg.has_option(section, key): val = _expand_cfg_value( cfg.get(section, key, fallback="").strip(), cfg_vars ) if val: self._cfg_secret_cache[(section, key)] = val if _is_openbao_ref(val): self._secrets_finalized = True self._auto_input_keys = set() auto_kdc = None auto_enabled = None if cfg.has_section("Network"): sec = cfg["Network"] if "KDC_AUTO_DETECTED" in sec: auto_kdc = sec.get("KDC_AUTO_DETECTED", "") if "KERBEROS_AUTO_ENABLED" in sec: auto_enabled = sec.get("KERBEROS_AUTO_ENABLED", "") def mark_auto(inputs: dict): if auto_kdc: if ( inputs.get("kerberos_config.kdc", "").strip() == str(auto_kdc).strip() ): self._auto_input_keys.add("kerberos_config.kdc") if auto_enabled is not None: input_enabled = inputs.get("kerberos_config.enabled") input_bool = _parse_bool(input_enabled, None) auto_bool = _parse_bool(auto_enabled, None) if ( input_bool is not None and auto_bool is not None and input_bool == auto_bool ): self._auto_input_keys.add("kerberos_config.enabled") def _first_non_empty(*values: str) -> str: for value in values: candidate = str(value or "").strip() if candidate: return candidate return "" def _cfg_first_non_empty(section_name: str, *keys: str) -> str: if not cfg.has_section(section_name): return "" section = cfg[section_name] for key in keys: if key not in section: continue candidate = _expand_cfg_value(section.get(key, ""), cfg_vars).strip() if candidate: return candidate return "" def _hydrate_auth_oidc_inputs(target: dict[str, str]) -> None: if not str(target.get("auth.clientId", "") or "").strip(): client_id = _first_non_empty( target.get("oidc.clientId", ""), _cfg_first_non_empty( "Auth", "clientId", "CLIENT_ID", "OIDC_CLIENT_ID", "GOOGLE_OIDC_CLIENT_ID", "clientIdRef", "CLIENT_ID_REF", ), _cfg_first_non_empty( "Global", "GITLAB_OIDC_CLIENT_ID", "OIDC_CLIENT_ID", "GOOGLE_OIDC_CLIENT_ID", "OIDC_CLIENT_ID_REF", "GOOGLE_OIDC_CLIENT_ID_REF", ), os.environ.get("GITLAB_OIDC_CLIENT_ID", ""), os.environ.get("OIDC_CLIENT_ID", ""), os.environ.get("GOOGLE_OIDC_CLIENT_ID", ""), ) if client_id: target["auth.clientId"] = client_id if not str(target.get("auth.clientSecret", "") or "").strip(): client_secret = _first_non_empty( target.get("oidc.clientSecret", ""), _cfg_first_non_empty( "Auth", "clientSecret", "CLIENT_SECRET", "OIDC_CLIENT_SECRET", "GOOGLE_OIDC_CLIENT_SECRET", "clientSecretRef", "CLIENT_SECRET_REF", ), _cfg_first_non_empty( "Global", "GITLAB_OIDC_CLIENT_SECRET", "OIDC_CLIENT_SECRET", "GOOGLE_OIDC_CLIENT_SECRET", "OIDC_CLIENT_SECRET_REF", "GOOGLE_OIDC_CLIENT_SECRET_REF", ), os.environ.get("GITLAB_OIDC_CLIENT_SECRET", ""), os.environ.get("OIDC_CLIENT_SECRET", ""), os.environ.get("GOOGLE_OIDC_CLIENT_SECRET", ""), ) if client_secret: target["auth.clientSecret"] = client_secret inputs: dict[str, str] = {} if cfg.has_section("Inputs"): inputs.update( {k: _expand_cfg_value(v, cfg_vars) for k, v in cfg.items("Inputs")} ) mark_auto(inputs) # Resolve encrypted/anchored secrets for runtime use for key in ( "init_password.db_password", "init_password.db_password_confirm", "kerberos_config.password", "init_cluster.k3s_token", "auth.clientId", "auth.clientSecret", ): if key in inputs and inputs[key]: inputs[key] = self._resolve_secret_value(inputs[key]) if inputs.get("init_password.db_password") and not inputs.get( "init_password.db_password_confirm" ): inputs["init_password.db_password_confirm"] = inputs[ "init_password.db_password" ] # Allow explicit runtime override of DB password in silent mode. env_db_pw = os.environ.get("KNOE_DB_PASSWORD") or os.environ.get( "DB_PASSWORD" ) if env_db_pw: raw_db_pw = inputs.get("init_password.db_password", "") if ( not raw_db_pw or _is_openbao_ref(raw_db_pw) or _is_knoe_secret(raw_db_pw) ): inputs["init_password.db_password"] = env_db_pw inputs["init_password.db_password_confirm"] = env_db_pw # Fill Ollama inputs from config sections if missing if cfg.has_section("Ollama"): sec = cfg["Ollama"] if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_SERVER_HOST", "").strip() ): inputs["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if ( "ollama_config.server_port" not in inputs and sec.get("OLLAMA_SERVER_PORT", "").strip() ): inputs["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if ( "ollama_config.model" not in inputs and sec.get("OLLAMA_MODEL", "").strip() ): inputs["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_HOST", "").strip() ): host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: inputs["ollama_config.server_host"] = host if port and "ollama_config.server_port" not in inputs: inputs["ollama_config.server_port"] = port if cfg.has_section("Global"): sec = cfg["Global"] if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_SERVER_HOST", "").strip() ): inputs["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if ( "ollama_config.server_port" not in inputs and sec.get("OLLAMA_SERVER_PORT", "").strip() ): inputs["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if ( "ollama_config.model" not in inputs and sec.get("OLLAMA_MODEL", "").strip() ): inputs["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if ( "ollama_config.server_host" not in inputs and sec.get("OLLAMA_HOST", "").strip() ): host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: inputs["ollama_config.server_host"] = host if port and "ollama_config.server_port" not in inputs: inputs["ollama_config.server_port"] = port _hydrate_auth_oidc_inputs(inputs) return inputs # Legacy fallback mapping legacy = {} if cfg.has_section("System Environment"): sec = cfg["System Environment"] for k in ( "KNOE_HOME", "KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE", ): if k in sec: raw = (sec.get(k, "") or "").strip() # Keep shell-style variables (e.g. $HOME) literal in cfg. if raw: legacy[f"env_setup.{k}"] = raw if cfg.has_section("Global"): sec = cfg["Global"] if "KNOE_HOME" in sec: raw = (sec.get("KNOE_HOME", "") or "").strip() if raw: legacy["env_setup.KNOE_HOME"] = raw if "CLUSTER_ENV" in sec: legacy["init_cluster.cluster_env"] = _expand_cfg_value( sec.get("CLUSTER_ENV", ""), cfg_vars ) if "DATABASE_NAMESPACE" in sec: db_ns = _expand_cfg_value(sec.get("DATABASE_NAMESPACE", ""), cfg_vars) legacy["init_password.db_namespace"] = db_ns legacy["env_setup.DATABASE_NAMESPACE"] = db_ns if "K3S_SERVER" in sec: legacy["init_cluster.k3s_server_url"] = _expand_cfg_value( sec.get("K3S_SERVER", ""), cfg_vars ) if "K3S_SERVER_URL" in sec and "init_cluster.k3s_server_url" not in legacy: legacy["init_cluster.k3s_server_url"] = _expand_cfg_value( sec.get("K3S_SERVER_URL", ""), cfg_vars ) if "K3S_TOKEN" in sec: legacy["init_cluster.k3s_token"] = _expand_cfg_value( sec.get("K3S_TOKEN", ""), cfg_vars ) if "K3S_TOKEN" in sec and "init_cluster.k3s_token" not in legacy: legacy["init_cluster.k3s_token"] = _expand_cfg_value( sec.get("K3S_TOKEN", ""), cfg_vars ) if "NAMESPACE" in sec and "init_password.db_namespace" not in legacy: db_ns = _expand_cfg_value(sec.get("NAMESPACE", ""), cfg_vars) legacy["init_password.db_namespace"] = db_ns legacy["env_setup.DATABASE_NAMESPACE"] = db_ns if "CLUSTER_NAME" in sec: cluster_name = _expand_cfg_value(sec.get("CLUSTER_NAME", ""), cfg_vars) legacy["init_password.cluster_name"] = cluster_name legacy["env_setup.CLUSTER_NAME"] = cluster_name legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name) legacy.setdefault("init_password.db_cluster_name", cluster_name) if "CNPG_CLUSTER_NAME" in sec and "init_password.cluster_name" not in legacy: cluster_name = _expand_cfg_value( sec.get("CNPG_CLUSTER_NAME", ""), cfg_vars ) legacy["init_password.cluster_name"] = cluster_name legacy["env_setup.CLUSTER_NAME"] = cluster_name legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name) legacy.setdefault("init_password.db_cluster_name", cluster_name) if "APP_CLUSTER_NAME" in sec: app_cluster_name = _expand_cfg_value( sec.get("APP_CLUSTER_NAME", ""), cfg_vars ) if app_cluster_name: legacy["env_setup.APP_CLUSTER_NAME"] = app_cluster_name legacy["init_password.app_cluster_name"] = app_cluster_name if "APP_CLUSTER_MODE" in sec: legacy["env_setup.APP_CLUSTER_MODE"] = _expand_cfg_value( sec.get("APP_CLUSTER_MODE", ""), cfg_vars ) legacy["init_cluster.app_cluster_mode"] = legacy[ "env_setup.APP_CLUSTER_MODE" ] if "DB_CLUSTER_NAME" in sec: db_cluster_name = _expand_cfg_value( sec.get("DB_CLUSTER_NAME", ""), cfg_vars ) if db_cluster_name: legacy["env_setup.DB_CLUSTER_NAME"] = db_cluster_name legacy["init_password.db_cluster_name"] = db_cluster_name if "DB_CLUSTER_MODE" in sec: legacy["env_setup.DB_CLUSTER_MODE"] = _expand_cfg_value( sec.get("DB_CLUSTER_MODE", ""), cfg_vars ) legacy["init_cluster.db_cluster_mode"] = legacy["env_setup.DB_CLUSTER_MODE"] if "DB_CLUSTER_NODE_COUNT" in sec: legacy["init_cluster.db_cluster_node_count"] = _expand_cfg_value( sec.get("DB_CLUSTER_NODE_COUNT", ""), cfg_vars ) if "DB_CLUSTER_MACHINE_TYPE" in sec: legacy["init_cluster.db_cluster_machine_type"] = _expand_cfg_value( sec.get("DB_CLUSTER_MACHINE_TYPE", ""), cfg_vars ) if "DB_BOOT_DISK_TYPE" in sec: legacy["init_cluster.db_boot_disk_type"] = _expand_cfg_value( sec.get("DB_BOOT_DISK_TYPE", ""), cfg_vars ) if "DB_BOOT_DISK_SIZE_GB" in sec: legacy["init_cluster.db_boot_disk_size_gb"] = _expand_cfg_value( sec.get("DB_BOOT_DISK_SIZE_GB", ""), cfg_vars ) if "DB_CLUSTER_REGION" in sec: legacy["init_cluster.db_cluster_region"] = _expand_cfg_value( sec.get("DB_CLUSTER_REGION", ""), cfg_vars ) if "APP_CLUSTER_KUBECONTEXT" in sec: kubecontext = _expand_cfg_value( sec.get("APP_CLUSTER_KUBECONTEXT", ""), cfg_vars ) legacy["env_setup.APP_CLUSTER_KUBECONTEXT"] = kubecontext legacy["init_cluster.app_cluster_kubecontext"] = kubecontext if "DB_CLUSTER_KUBECONTEXT" in sec: kubecontext = _expand_cfg_value( sec.get("DB_CLUSTER_KUBECONTEXT", ""), cfg_vars ) legacy["env_setup.DB_CLUSTER_KUBECONTEXT"] = kubecontext legacy["init_cluster.db_cluster_kubecontext"] = kubecontext if "KNOE_DB_USER" in sec: legacy["init_password.db_username"] = _expand_cfg_value( sec.get("KNOE_DB_USER", ""), cfg_vars ) if "DB_PASSWORD" in sec: legacy["init_password.db_password"] = _expand_cfg_value( sec.get("DB_PASSWORD", ""), cfg_vars ) legacy["init_password.db_password_confirm"] = _expand_cfg_value( sec.get("DB_PASSWORD", ""), cfg_vars ) if "DB_HOST_PORT" in sec: legacy["init_password.db_host_port"] = _expand_cfg_value( sec.get("DB_HOST_PORT", "5432"), cfg_vars ) if "DOCKER_IMPORT_DIR" in sec: self.docker_import_dir = _expand_cfg_value( sec.get("DOCKER_IMPORT_DIR", ""), cfg_vars ) self.knoe_cfg_data["Global"][ "DOCKER_IMPORT_DIR" ] = self.docker_import_dir if "OLLAMA_SERVER_HOST" in sec: legacy["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if "OLLAMA_SERVER_PORT" in sec: legacy["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if "OLLAMA_MODEL" in sec: legacy["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if "OLLAMA_HOST" in sec and "ollama_config.server_host" not in legacy: host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: legacy["ollama_config.server_host"] = host if port: legacy.setdefault("ollama_config.server_port", port) if cfg.has_section("Network"): sec = cfg["Network"] if "KDC_AUTO_DETECTED" in sec: legacy["kerberos_config.kdc"] = _expand_cfg_value( sec.get("KDC_AUTO_DETECTED", ""), cfg_vars ) if "KERBEROS_AUTO_ENABLED" in sec: legacy["kerberos_config.enabled"] = _expand_cfg_value( sec.get("KERBEROS_AUTO_ENABLED", ""), cfg_vars ) if cfg.has_section("Kerberos Authentication"): sec = cfg["Kerberos Authentication"] for k, tgt in ( ("ENABLED", "kerberos_config.enabled"), ("REALM", "kerberos_config.realm"), ("KDC", "kerberos_config.kdc"), ("USER", "kerberos_config.user"), ("PASSWORD", "kerberos_config.password"), ): if k in sec: legacy[tgt] = _expand_cfg_value(sec.get(k, ""), cfg_vars) if cfg.has_section("Ollama"): sec = cfg["Ollama"] if "OLLAMA_SERVER_HOST" in sec: legacy["ollama_config.server_host"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars ) if "OLLAMA_SERVER_PORT" in sec: legacy["ollama_config.server_port"] = _expand_cfg_value( sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars ) if "OLLAMA_MODEL" in sec: legacy["ollama_config.model"] = _expand_cfg_value( sec.get("OLLAMA_MODEL", ""), cfg_vars ) if "OLLAMA_HOST" in sec and "ollama_config.server_host" not in legacy: host, port = _parse_ollama_host( _expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars) ) if host: legacy["ollama_config.server_host"] = host if port: legacy.setdefault("ollama_config.server_port", port) if cfg.has_section("GitOps"): sec = cfg["GitOps"] if "NAMESPACE" in sec: legacy["gitops.namespace"] = _expand_cfg_value( sec.get("NAMESPACE", ""), cfg_vars ) if cfg.has_section("Optional Features"): sec = cfg["Optional Features"] if "SUPABASE_ENABLED" in sec: legacy["init_cluster.supabase_enabled"] = sec.get( "SUPABASE_ENABLED", "" ) if "SUPABASE_STUDIO_ENABLED" in sec: legacy["init_cluster.supabase_studio_enabled"] = sec.get( "SUPABASE_STUDIO_ENABLED", "" ) if "SUPABASE_STUDIO_URL" in sec: legacy["init_cluster.supabase_studio_url"] = sec.get( "SUPABASE_STUDIO_URL", "" ) if "SUPABASE_AUTH_ENABLED" in sec: legacy["init_cluster.supabase_auth_enabled"] = sec.get( "SUPABASE_AUTH_ENABLED", "" ) if "SUPABASE_REALTIME_ENABLED" in sec: legacy["init_cluster.supabase_realtime_enabled"] = sec.get( "SUPABASE_REALTIME_ENABLED", "" ) if "SUPABASE_META_ENABLED" in sec: legacy["init_cluster.supabase_meta_enabled"] = sec.get( "SUPABASE_META_ENABLED", "" ) if "SUPABASE_ANALYTICS_ENABLED" in sec: legacy["init_cluster.supabase_analytics_enabled"] = sec.get( "SUPABASE_ANALYTICS_ENABLED", "" ) if "GITOPS_ENABLED" in sec: legacy["init_cluster.gitops_enabled"] = sec.get( "GITOPS_ENABLED", "" ) if "KERBEROS_ENABLED" in sec: legacy["init_cluster.kerberos_enabled"] = sec.get( "KERBEROS_ENABLED", "" ) legacy["kerberos_config.enabled"] = sec.get("KERBEROS_ENABLED", "") if "AT_REST_ENCRYPTION_ENABLED" in sec: legacy["init_cluster.at_rest_encryption_enabled"] = sec.get( "AT_REST_ENCRYPTION_ENABLED", "" ) if cfg.has_section("Initialize Cluster"): sec = cfg["Initialize Cluster"] if "ENVIRONMENT" in sec: legacy["init_cluster.cluster_env"] = sec.get("ENVIRONMENT", "") if "K3S_SERVER_URL" in sec and "init_cluster.k3s_server_url" not in legacy: legacy["init_cluster.k3s_server_url"] = sec.get("K3S_SERVER_URL", "") if "K3S_TOKEN" in sec and "init_cluster.k3s_token" not in legacy: legacy["init_cluster.k3s_token"] = sec.get("K3S_TOKEN", "") if cfg.has_section("Database Creation"): sec = cfg["Database Creation"] db_ns = "" for k in ("DB_NAME", "NAMESPACE", "DATABASE_NAMESPACE"): if k in sec: db_ns = sec.get(k, "") if db_ns: legacy["init_password.db_namespace"] = db_ns legacy["env_setup.DATABASE_NAMESPACE"] = db_ns if "CLUSTER_NAME" in sec: cluster_name = sec.get("CLUSTER_NAME", "") legacy["init_password.cluster_name"] = cluster_name legacy["env_setup.CLUSTER_NAME"] = cluster_name if "DB_USER" in sec: legacy["init_password.db_username"] = sec.get("DB_USER", "") _hydrate_auth_oidc_inputs(legacy) mark_auto(legacy) for key in ( "init_password.db_password", "init_password.db_password_confirm", "kerberos_config.password", "init_cluster.k3s_token", "auth.clientId", "auth.clientSecret", ): if key in legacy and legacy[key]: legacy[key] = self._resolve_secret_value(legacy[key]) if legacy.get("init_password.db_password") and not legacy.get( "init_password.db_password_confirm" ): legacy["init_password.db_password_confirm"] = legacy[ "init_password.db_password" ] return legacy def _default_inputs(self) -> dict: namespace = self._initial_db_namespace() cluster_name = self._cnpg_cluster_name() owner = self._get_local_owner() env_vals = self._env_defaults(namespace) inputs: dict[str, str] = {} # Dependencies inputs["dependencies.verify_all"] = _bool_str(False) inputs["dependencies.auto_install_missing"] = _bool_str( DEFAULT_ACTION_FLAGS.get("dependencies.auto_install_missing", True) ) for dep in self.dependencies: inputs[f"dependencies.{dep['id']}.install"] = _bool_str(True) # Network scan inputs["network_scan.run"] = _bool_str( DEFAULT_ACTION_FLAGS.get("network_scan.run", True) ) # Environment setup for k in ( "KNOE_HOME", "KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE", ): inputs[f"env_setup.{k}"] = env_vals.get(k, "") inputs["env_setup.DATABASE_NAMESPACE"] = namespace inputs["env_setup.CLUSTER_NAME"] = cluster_name inputs["env_setup.APP_CLUSTER_NAME"] = DEFAULT_APP_CLUSTER_NAME inputs["env_setup.APP_CLUSTER_MODE"] = DEFAULT_APP_CLUSTER_MODE inputs["env_setup.DB_CLUSTER_NAME"] = DEFAULT_DB_CLUSTER_NAME inputs["env_setup.DB_CLUSTER_MODE"] = DEFAULT_DB_CLUSTER_MODE inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = "" inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = "" # Database creation inputs["init_password.db_namespace"] = namespace inputs["init_password.cluster_name"] = cluster_name inputs["init_password.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME inputs["init_password.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME inputs["init_password.db_username"] = owner inputs["init_password.db_password"] = "" inputs["init_password.db_password_confirm"] = "" inputs["init_password.generate_ssh_key"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_password.generate_ssh_key", True) ) inputs["init_password.db_host_port"] = "5432" # Build DB image inputs["init_db_build.run_build"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True) ) # Cluster init + optional features inputs["init_cluster.cluster_env"] = "dev" inputs["init_cluster.k3s_server_url"] = "" inputs["init_cluster.k3s_token"] = "" inputs["init_cluster.supabase_enabled"] = _bool_str(False) inputs["init_cluster.supabase_studio_enabled"] = _bool_str(False) inputs["init_cluster.supabase_studio_url"] = "db.0.knoe.dev" inputs["init_cluster.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME inputs["init_cluster.app_cluster_mode"] = DEFAULT_APP_CLUSTER_MODE inputs["init_cluster.app_cluster_machine_type"] = DEFAULT_APP_CLUSTER_MACHINE_TYPE inputs["init_cluster.app_cluster_node_count"] = str(DEFAULT_APP_CLUSTER_NODE_COUNT) inputs["init_cluster.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME inputs["init_cluster.db_cluster_mode"] = DEFAULT_DB_CLUSTER_MODE inputs["init_cluster.db_cluster_node_count"] = str(DEFAULT_DB_CLUSTER_NODE_COUNT) inputs["init_cluster.db_cluster_machine_type"] = DEFAULT_DB_CLUSTER_MACHINE_TYPE inputs["init_cluster.db_boot_disk_type"] = DEFAULT_DB_BOOT_DISK_TYPE inputs["init_cluster.db_boot_disk_size_gb"] = str(DEFAULT_DB_BOOT_DISK_SIZE_GB) inputs["init_cluster.db_cluster_region"] = "" inputs["init_cluster.db_cluster_zones"] = "" inputs["init_cluster.app_cluster_kubecontext"] = "" inputs["init_cluster.db_cluster_kubecontext"] = "" inputs["init_cluster.supabase_auth_enabled"] = _bool_str(True) inputs["init_cluster.supabase_realtime_enabled"] = _bool_str(True) inputs["init_cluster.supabase_meta_enabled"] = _bool_str(True) inputs["init_cluster.supabase_analytics_enabled"] = _bool_str(True) inputs["init_cluster.gitops_enabled"] = _bool_str(False) inputs["init_cluster.kerberos_enabled"] = _bool_str(False) inputs["init_cluster.at_rest_encryption_enabled"] = _bool_str(True) inputs["init_cluster.start_cluster"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_cluster.start_cluster", True) ) # Kerberos config inputs["kerberos_config.enabled"] = _bool_str(False) inputs["kerberos_config.realm"] = "" inputs["kerberos_config.kdc"] = "" inputs["kerberos_config.user"] = "administrator" inputs["kerberos_config.password"] = "" inputs["kerberos_config.test_connection"] = _bool_str( DEFAULT_ACTION_FLAGS.get("kerberos_config.test_connection", False) ) inputs["kerberos_config.init_authority"] = _bool_str(False) # Ollama config inputs["ollama_config.server_host"] = "" inputs["ollama_config.server_port"] = DEFAULT_OLLAMA_PORT inputs["ollama_config.model"] = "" # Init scripts + deploy inputs["init_scripts.run_scripts"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_scripts.run_scripts", True) ) inputs["init_cnpg_deploy.run_deploy"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True) ) inputs["init_cnpg_deploy.force_rollout"] = _bool_str( DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False) ) # GitOps inputs["gitops.namespace"] = "gitea" inputs["auth.clientId"] = "secretref://google-oidc-client-id" inputs["auth.clientSecret"] = "secretref://google-oidc-client-secret" # Disk selection (installer packaging) inputs["disk_selection.disk_type"] = "local" inputs["disk_selection.removable_mount"] = "" inputs["disk_selection.local_path"] = str(Path.home()) # Build tools app inputs["build.deploy_env"] = "Dev" inputs["build.run_build"] = _bool_str( DEFAULT_ACTION_FLAGS.get("build.run_build", False) ) return inputs # ---------------- Process helpers ---------------- def _run_cmd( self, cmd, cwd=None, env=None, stdin_text=None, on_stdout=None, on_stderr=None ) -> int: if isinstance(cmd, (list, tuple)) and cmd: binary = Path(str(cmd[0])).name if binary in {"kubectl", "helm"}: target_role = "UNKNOWN" target_ctx = "" if isinstance(env, dict): target_role = str(env.get("KNOE_CLUSTER_ROLE") or "").strip().upper() or "UNKNOWN" target_ctx = str( env.get("KUBECTL_CONTEXT") or env.get("KUBE_CONTEXT_NAME") or env.get("KUBECONTEXT") or "" ).strip() self.log(f"[TARGET {target_role}] {binary} context={target_ctx or ''}") stdin_handle = subprocess.PIPE if stdin_text else None if isinstance(cmd, str): proc = subprocess.Popen( ["bash", "-lc", cmd], cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=stdin_handle, text=True, bufsize=1, ) else: proc = subprocess.Popen( cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=stdin_handle, text=True, bufsize=1, ) if stdin_text and proc.stdin: try: proc.stdin.write(stdin_text) proc.stdin.close() except Exception: pass def _read_stream(stream, handler, is_err=False): if not stream: return for line in iter(stream.readline, ""): if handler: handler(line) else: if is_err: self.err(line.rstrip("\n")) else: self.log(line.rstrip("\n")) err_thread = threading.Thread( target=_read_stream, args=(proc.stderr, on_stderr, True), daemon=True ) err_thread.start() _read_stream(proc.stdout, on_stdout, False) rc = proc.wait() err_thread.join(timeout=2) return rc def _run_script( self, script_name: str, args=None, env=None, stdin_text=None, on_line=None ) -> int: def _stdout(line): self.log(line.rstrip("\n")) if on_line: on_line(line) def _stderr(line): self.err(line.rstrip("\n")) mode_args = [] if script_name.startswith("init_"): mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if mode: mode_args = ["--mode", mode] target_role = "UNKNOWN" target_ctx = "" if isinstance(env, dict): target_role = str(env.get("KNOE_CLUSTER_ROLE") or "").strip().upper() or "UNKNOWN" target_ctx = str( env.get("KUBECTL_CONTEXT") or env.get("KUBE_CONTEXT_NAME") or env.get("KUBECONTEXT") or "" ).strip() if target_ctx or target_role != "UNKNOWN": self.log( f"[TARGET {target_role}] script={script_name} context={target_ctx or ''}" ) return self.controller.run_script( script_name, args=mode_args + (args or []), env=env, stdin_text=stdin_text, on_line=_stdout, on_stderr_line=_stderr, stderr_to_stdout=False, ) def _ensure_local_registry_available(self): """Ensure a local k3d registry is running and return (host_registry, cluster_registry).""" env_hint = self._get_input("init_cluster.cluster_env", "") mode = _deployment_mode_from_env(env_hint) if not _local_registry_enabled(env_hint): self.log("[SKIP] Local registry disabled; skipping registry setup.") return None if mode == "k3s": # In k3s mode we must not use localhost registries or k3d registry containers. server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if not host: self.err( "[WARN] k3s mode but K3S_SERVER_URL/K3S_SERVER not set; cannot resolve registry." ) return None host_registry = f"{host}:5000" registry_ns = self._registry_namespace() cluster_registry = f"registry.{registry_ns}.svc.cluster.local:5000" self.local_registry_url = host_registry self.local_registry_internal = cluster_registry self.knoe_cfg_data.setdefault("Docker Build", {})["LOCAL_REGISTRY"] = host_registry self.knoe_cfg_data["Docker Build"]["LOCAL_REGISTRY_INTERNAL"] = cluster_registry return host_registry, cluster_registry reg_name = "knoe-registry" registry_container = f"k3d-{reg_name}" host_registry = "localhost:5000" cluster_registry = f"{registry_container}.localhost:5000" if not self.controller.check_docker_running(): self.err("[WARN] Docker not running; cannot ensure local registry.") return None # 1. If already responsive, just use it if _http_ping_registry("localhost", 5000): self.log("[INFO] Local registry already responsive on localhost:5000.") lst = subprocess.run( ["k3d", "registry", "list"], capture_output=True, text=True ) if registry_container not in (lst.stdout or "") and reg_name not in ( lst.stdout or "" ): cluster_registry = "localhost:5000" self.local_registry_url = host_registry self.local_registry_internal = cluster_registry self.knoe_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry self.knoe_cfg_data["Docker Build"][ "LOCAL_REGISTRY_INTERNAL" ] = cluster_registry return host_registry, cluster_registry # 2. Require k3d to manage/create the local registry k3d_exists = ( subprocess.run(["which", "k3d"], capture_output=True).returncode == 0 ) if not k3d_exists: self.err("[WARN] k3d not found; cannot ensure local registry.") return None lst = subprocess.run( ["k3d", "registry", "list"], capture_output=True, text=True ) if registry_container not in (lst.stdout or "") and reg_name not in ( lst.stdout or "" ): self.log("[INFO] Creating local k3d registry...") subprocess.run( ["k3d", "registry", "create", reg_name, "--port", "0.0.0.0:5000"], check=True, ) running = subprocess.run( [ "docker", "ps", "--filter", f"name={registry_container}", "--format", "{{.Names}}", ], capture_output=True, text=True, ) if not (running.stdout or "").strip(): exists = subprocess.run( [ "docker", "ps", "-a", "--filter", f"name={registry_container}", "--format", "{{.Names}}", ], capture_output=True, text=True, ) if (exists.stdout or "").strip(): subprocess.run(["docker", "start", registry_container], check=True) self.local_registry_url = host_registry self.local_registry_internal = cluster_registry self.knoe_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry self.knoe_cfg_data["Docker Build"][ "LOCAL_REGISTRY_INTERNAL" ] = cluster_registry return host_registry, cluster_registry def _argocd_namespace(self) -> str: ns = ( (self.knoe_cfg_data.get("Global", {}) or {}) .get("ARGOCD_NAMESPACE", "") .strip() ) if not ns: ns = (os.environ.get("ARGOCD_NAMESPACE") or "").strip() return ns or "argocd" def _registry_namespace(self) -> str: ns = ( (self.knoe_cfg_data.get("Global", {}) or {}) .get("REGISTRY_NAMESPACE", "") .strip() ) if not ns: ns = (os.environ.get("REGISTRY_NAMESPACE") or "").strip() # In service-mode k3s, keep registry in the service namespace and # avoid falling back to default (which causes duplicate deployments). if ns and not (ns == "default" and self._deployment_mode() == "k3s"): return ns # Registry is a common-core service; default to the configured service namespace. return self._service_namespace() def _ensure_registry_defaults(self, mode: str) -> None: mode = _deployment_mode_from_env(mode) or mode db = self.knoe_cfg_data.get("Docker Build", {}) or {} if mode == "k3d" and _local_registry_enabled(mode): info = self._ensure_local_registry_available() if info: host_registry, cluster_registry = info db.setdefault("LOCAL_REGISTRY", host_registry) db.setdefault("LOCAL_REGISTRY_INTERNAL", cluster_registry) if mode == "k3s" and _local_registry_enabled(mode): cur = (db.get("LOCAL_REGISTRY") or "").strip() if not cur or cur == "localhost:5000": server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if host: db["LOCAL_REGISTRY"] = f"{host}:5000" cur_internal = (db.get("LOCAL_REGISTRY_INTERNAL") or "").strip() if not cur_internal or ".localhost" in cur_internal: registry_ns = self._registry_namespace() db["LOCAL_REGISTRY_INTERNAL"] = ( f"registry.{registry_ns}.svc.cluster.local:5000" ) self.knoe_cfg_data["Docker Build"] = db def _fetch_k3s_kubeconfig(self) -> Path | None: """Fetch a fresh kubeconfig from k3s via the Ansible playbook.""" kubeconfig = self.project_root / "knoe-k3s.kubeconfig" playbook = ( self.project_root / "infrastructure" / "playbooks" / "k3s_fetch_kubeconfig.yml" ) if not playbook.exists(): self.err(f"[WARN] Ansible playbook not found: {playbook}") return None cmd = ["ansible-playbook", str(playbook)] self.log("==> Fetching k3s kubeconfig via Ansible") try: subprocess.run( cmd, cwd=str(self.project_root), check=True, capture_output=True, text=True, timeout=30, ) except subprocess.CalledProcessError as e: self.err(f"[WARN] Failed to fetch kubeconfig: {e}") return None except FileNotFoundError: self.err("[WARN] 'ansible-playbook' command not found.") return None except subprocess.TimeoutExpired: self.err("[WARN] Kubeconfig fetch timed out.") return None if not kubeconfig.exists(): self.err("[WARN] Kubeconfig was not written after playbook run.") return None self.log(f"[OK] Kubeconfig fetched: {kubeconfig}") return kubeconfig # ---------------- Steps ---------------- def _apply_inputs(self): defaults = self._default_inputs() loaded = self._load_inputs_from_cfg() self.inputs = {**defaults, **loaded} try: self._apply_ansible_defaults(set(loaded.keys())) except Exception: pass mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if not mode: mode = _deployment_mode_from_env(self._get_input("init_cluster.mode", "")) if mode: os.environ["KNOE_MODE"] = mode def _apply_ansible_defaults(self, loaded_keys: set[str] | None = None): info = _detect_ansible_topology(self.project_root) if not info: return loaded_keys = loaded_keys or set() auto_keys = getattr(self, "_auto_input_keys", set()) ansible_kdc = info.get("kdc_ip") or "" try: net = self.knoe_cfg_data.get("Network", {}) if info.get("topology_json"): net["ANSIBLE_TOPOLOGY"] = info["topology_json"] if info.get("inventory_path"): net["ANSIBLE_INVENTORY"] = info["inventory_path"] if info.get("infrastructure_path"): net["ANSIBLE_INFRASTRUCTURE"] = info["infrastructure_path"] if info.get("domain"): net["ANSIBLE_DOMAIN"] = info["domain"] if info.get("realm"): net["ANSIBLE_REALM"] = info["realm"] if info.get("ad_dc_host"): net["AD_DC_HOST"] = info["ad_dc_host"] if info.get("ad_dc_ip"): net["AD_DC_IP"] = info["ad_dc_ip"] if info.get("kdc_ip"): net["KDC_ANSIBLE_DETECTED"] = info["kdc_ip"] self.knoe_cfg_data["Network"] = net except Exception: pass # Storage/node placement derived from Ansible host vars. # This is the authoritative source of truth for where volumes are mounted. try: inv_str = (info.get("inventory_path") or "").strip() inv_path = Path(inv_str) if inv_str else None except Exception: inv_path = None if inv_path is not None and inv_path.exists(): try: mounts = _detect_ansible_storage_mounts(inv_path) labels_by_host = _detect_ansible_node_labels(inv_path) storage_sec = self.knoe_cfg_data.setdefault("Storage", {}) if mounts: storage_sec["ANSIBLE_ISCSI_MOUNTS"] = json.dumps( mounts, separators=(",", ":"), sort_keys=True ) if labels_by_host: storage_sec["ANSIBLE_NODE_LABELS"] = json.dumps( labels_by_host, separators=(",", ":"), sort_keys=True ) glob = self.knoe_cfg_data.setdefault("Global", {}) cnpg_eligible_nodes = sorted( { str((details or {}).get("host") or "").strip() for details in mounts.values() if isinstance(details, dict) and str((details or {}).get("host") or "").strip() } ) if cnpg_eligible_nodes: glob["CNPG_ELIGIBLE_NODES"] = ",".join(cnpg_eligible_nodes) synology_roots = sorted( { str((details or {}).get("path") or "").strip() for details in mounts.values() if isinstance(details, dict) and str((details or {}).get("path") or "").strip() } ) if synology_roots: glob["SYNOLOGY_ROOTS"] = ",".join(synology_roots) # CNPG protected storage defaults (k3s local PV) d001 = mounts.get("d001") or {} d001_base = (d001.get("path") or "").strip() d001_host = (d001.get("host") or "").strip() if d001_base and d001_host: glob["CNPG_STAGE1_NODE"] = d001_host glob["CNPG_DB_NODE_SELECTOR"] = f"kubernetes.io/hostname={d001_host}" # Monitoring storage defaults (prefer d004; fall back to d002) mon_choice = None for vol_id in ("d004", "d002"): if mounts.get(vol_id): mon_choice = mounts[vol_id] break if mon_choice: mon_base = (mon_choice.get("path") or "").strip() mon_host = (mon_choice.get("host") or "").strip() if mon_base: glob["MONITORING_DATA_DIR"] = mon_base if mon_host: self.knoe_cfg_data.setdefault("Monitoring", {})[ "MONITORING_PRIMARY_NODE" ] = mon_host except Exception: pass if ansible_kdc and "kerberos_config.kdc" in auto_keys: self.inputs["kerberos_config.kdc"] = ansible_kdc if ansible_kdc and "kerberos_config.enabled" in auto_keys: self.inputs["kerberos_config.enabled"] = _bool_str(True) if "kerberos_config.kdc" not in loaded_keys: if not self._get_input("kerberos_config.kdc", "").strip() and info.get( "kdc_ip" ): self.inputs["kerberos_config.kdc"] = info["kdc_ip"] if "kerberos_config.realm" not in loaded_keys: if not self._get_input("kerberos_config.realm", "").strip() and info.get( "realm" ): self.inputs["kerberos_config.realm"] = info["realm"] if "kerberos_config.enabled" not in loaded_keys: if info.get("kdc_ip") and not self._get_input_bool( "kerberos_config.enabled", False ): self.inputs["kerberos_config.enabled"] = _bool_str(True) if "init_cluster.k3s_server_url" not in loaded_keys: if not self._get_input( "init_cluster.k3s_server_url", "" ).strip() and info.get("k3s_server_url"): self.inputs["init_cluster.k3s_server_url"] = info["k3s_server_url"] if "init_cluster.k3s_token" not in loaded_keys: if not self._get_input("init_cluster.k3s_token", "").strip() and info.get( "k3s_token" ): self.inputs["init_cluster.k3s_token"] = info["k3s_token"] def _write_cfg(self): inputs = dict(self.inputs) db_pw = self._get_input("init_password.db_password", "") db_pw_cfg = self._secret_cfg_value( "Inputs", "init_password.db_password", db_pw, "db", "password" ) if db_pw_cfg: inputs["init_password.db_password"] = db_pw_cfg inputs["init_password.db_password_confirm"] = db_pw_cfg elif db_pw: inputs["init_password.db_password"] = db_pw inputs["init_password.db_password_confirm"] = ( self._get_input("init_password.db_password_confirm", "") or db_pw ) krb_pw = self._get_input("kerberos_config.password", "") if krb_pw: inputs["kerberos_config.password"] = self._secret_cfg_value( "Inputs", "kerberos_config.password", krb_pw, "kerberos", "password" ) k3s_token = self._get_input("init_cluster.k3s_token", "") if k3s_token: inputs["init_cluster.k3s_token"] = self._secret_cfg_value( "Inputs", "init_cluster.k3s_token", k3s_token, "k3s", "token" ) mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) target_label = _deployment_target_label( self._get_input("init_cluster.cluster_env", "") ) globals_to_save = { "KNOE_HOME": self._get_input("env_setup.KNOE_HOME", ""), "KNOE_DB_USER": self._get_input("init_password.db_username", ""), "DB_PASSWORD": self._secret_cfg_value( "Global", "DB_PASSWORD", db_pw, "db", "password" ), "CLUSTER_ENV": self._get_input("init_cluster.cluster_env", ""), "DEPLOYMENT_MODE": mode, "DEPLOYMENT_TARGET": target_label, "DATABASE_NAMESPACE": ( self._get_input("init_password.db_namespace", "") or "" ).strip(), "CLUSTER_NAME": ( self._get_input("init_password.cluster_name", "") or self._get_input("env_setup.CLUSTER_NAME", "") or self._cnpg_cluster_name() or "" ).strip(), "DB_HOST_PORT": ( self._get_input("init_password.db_host_port", "5432") or "5432" ).strip(), "DOCKER_IMPORT_DIR": self.docker_import_dir or "", "K3S_SERVER": ( self._get_input("init_cluster.k3s_server_url", "") or "" ).strip(), "K3S_TOKEN": self._secret_cfg_value( "Global", "K3S_TOKEN", self._get_input("init_cluster.k3s_token", ""), "k3s", "token", ), "OPENTOFU_URL": _default_opentofu_pipeline_url(), "SERVICE_NAMESPACE": self._service_namespace(), } # Merge pre-existing Global values without overriding explicit inputs existing_global = dict(self.knoe_cfg_data.get("Global", {})) for k, v in existing_global.items(): if k not in globals_to_save or not str(globals_to_save.get(k, "")).strip(): globals_to_save[k] = v globals_to_save.setdefault("DOCKER_PRELOAD", "false") for _gk, _gv in list(globals_to_save.items()): if isinstance(_gv, str) and _gv.strip(): globals_to_save[_gk] = self._cfgify_home_path(_gv) # Re-assert critical/dynamic values to avoid accidental overwrite globals_to_save["DB_PASSWORD"] = self._secret_cfg_value( "Global", "DB_PASSWORD", db_pw, "db", "password" ) globals_to_save["DEPLOYMENT_MODE"] = mode globals_to_save["DEPLOYMENT_TARGET"] = target_label globals_to_save.pop("NAMESPACE", None) globals_to_save.pop("CNPG_CLUSTER_NAME", None) globals_to_save.pop("KUBECONTEXT", None) # machine-specific; always runtime-detected globals_to_save.setdefault("ARGOCD_NAMESPACE", self._argocd_namespace()) globals_to_save.setdefault("REGISTRY_NAMESPACE", self._registry_namespace()) # Preserve Supabase placement vars from cfg — never let pipeline state overwrite them for _supa_key in ( "SUPABASE_PRIMARY_NODE", "SUPABASE_PV_NODE", "SUPABASE_PV_BASE_DIR", "SUPABASE_STORAGE_CLASS", "SUPABASE_ADDITIONAL_REDIRECT_URLS", ): _supa_val = existing_global.get(_supa_key, "") if _supa_val: globals_to_save[_supa_key] = _supa_val # Monitoring # NOTE: Storage base paths for monitoring are currently pinned in the # monitoring deploy script/Helm values (not persisted in knoe.cfg). mon_sec = self.knoe_cfg_data.get("Monitoring", {}) if mon_sec is None: mon_sec = {} self.knoe_cfg_data["Monitoring"] = mon_sec # Optional features (sourced from inputs) opt_sec = self.knoe_cfg_data.setdefault("Optional Features", {}) opt_sec["SUPABASE_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_enabled", False) ) opt_sec["GITOPS_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.gitops_enabled", False) ) opt_sec["KERBEROS_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.kerberos_enabled", False) ) opt_sec["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) gitops_ns_val = ( self._get_input("gitops.namespace", "") or self.inputs.get("gitops.namespace", "") ) if gitops_ns_val: self.knoe_cfg_data.setdefault("GitOps", {})[ "GITOPS_NAMESPACE" ] = gitops_ns_val ollama_host = (self._get_input("ollama_config.server_host", "") or "").strip() ollama_port = ( self._get_input("ollama_config.server_port", DEFAULT_OLLAMA_PORT) or "" ).strip() ollama_model = (self._get_input("ollama_config.model", "") or "").strip() ollama_section = dict(self.knoe_cfg_data.get("Ollama", {})) if ollama_host: if not ollama_port: ollama_port = DEFAULT_OLLAMA_PORT ollama_section["OLLAMA_SERVER_HOST"] = ollama_host ollama_section["OLLAMA_SERVER_PORT"] = ollama_port ollama_section["OLLAMA_HOST"] = _format_ollama_host( ollama_host, ollama_port ) else: for key in ("OLLAMA_SERVER_HOST", "OLLAMA_SERVER_PORT", "OLLAMA_HOST"): ollama_section.pop(key, None) if ollama_model: ollama_section["OLLAMA_MODEL"] = ollama_model else: ollama_section.pop("OLLAMA_MODEL", None) if ollama_section: self.knoe_cfg_data["Ollama"] = ollama_section self._sync_port_forward_mappings() sections = { k: self.knoe_cfg_data.get(k, {}) for k in [ "Welcome", "Dependencies", "Network", "Port Forwards", "System Environment", "Monitoring", "Kerberos Authentication", "Ollama", "Optional Features", "GitOps", "CNPG Clusters", "Database Creation", "Initialize Cluster", "Docker Build", "Initialization Scripts", "Deployment", "Install", ] } deployment_section = dict(sections.get("Deployment", {})) if mode: deployment_section.setdefault("MODE", mode) if target_label: deployment_section.setdefault("TARGET", target_label) sections["Deployment"] = deployment_section sections["Dev Cluster (k3d)"] = { **self.knoe_cfg_data.get("Dev Cluster (k3d)", {}), "MODE": "k3d", "CLUSTER_ENV": "k3d-knoe-dev-cluster", "DISPLAY_NAME": "knoe-dev-cluster", "KUBECTL_CONTEXT": self._get_input("init_cluster.cluster_env", ""), } sections["Service Cluster (k3s)"] = { **self.knoe_cfg_data.get("Service Cluster (k3s)", {}), "MODE": "k3s", "CLUSTER_ENV": "knoe-service-cluster", "DISPLAY_NAME": "knoe-service-cluster", "K3S_SERVER_URL": ( self._get_input("init_cluster.k3s_server_url", "") or "" ).strip(), "K3S_TOKEN": self._secret_cfg_value( "Service Cluster (k3s)", "K3S_TOKEN", self._get_input("init_cluster.k3s_token", ""), "k3s", "token", ), "PIPELINE_URL": _default_opentofu_pipeline_url(), } sections["Prod Cluster (k8s)"] = { **self.knoe_cfg_data.get("Prod Cluster (k8s)", {}), "MODE": "k8s", "CLUSTER_ENV": "knoe-prod-cluster", "DISPLAY_NAME": "knoe-prod-cluster", "ARTIFACTS_DIR": ( self._get_input("init_cluster.prod_artifacts_path", "") or "" ).strip(), "PIPELINE_URL": _default_opentofu_pipeline_url(), } sections = self._sanitize_sections_for_cfg(sections) # Check for existing config and timestamp to maintain idempotency existing_text = None existing_timestamp = None if self.cfg_path.exists(): existing_text = self.cfg_path.read_text() for line in existing_text.splitlines(): if line.startswith("; Generated by install.py on "): existing_timestamp = line[ len("; Generated by install.py on ") : ].strip() break cfg_text = _render_knoe_cfg( inputs, globals_to_save, sections, generated_at=existing_timestamp ) should_write = True if existing_text: if existing_text == cfg_text: should_write = False if should_write: # If content changed but we reused existing_timestamp, it might be misleading. # But usually we want to know when it LAST changed. # If it's different, let's generate a new one. if existing_text and existing_text != cfg_text: cfg_text = _render_knoe_cfg(inputs, globals_to_save, sections) self.cfg_path.parent.mkdir(parents=True, exist_ok=True) self.cfg_path.write_text(cfg_text) self.log(f"[CONFIG] Wrote {self.cfg_path}") def _step_dependencies(self) -> bool: self.log("==> Dependencies") missing = [] for dep in self.dependencies: ok, location, version = inst_config.get_dep_info(dep) if ok: self.log(f"[OK] {dep['name']} {version or ''}".strip()) continue missing.append(dep) self.log(f"[MISSING] {dep['name']}") if not missing: if not self._ensure_gcloud_auth_for_k8s(): self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing" return False self.knoe_cfg_data["Dependencies"]["STATUS"] = "All installed" return True auto_install = self._get_input_bool( "dependencies.auto_install_missing", DEFAULT_ACTION_FLAGS.get("dependencies.auto_install_missing", True), ) if not auto_install: self.err("[ERROR] Dependencies missing and auto-install disabled.") self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing" return False for dep in missing: install_cmd = dep.get("install_cmd") if not install_cmd: self.err(f"[ERROR] No install command for {dep['name']}.") continue should_install = self._get_input_bool( f"dependencies.{dep['id']}.install", True ) if not should_install: self.err(f"[SKIP] {dep['name']} install disabled by config.") continue self.log(f"[INSTALL] {dep['name']} -> {install_cmd}") rc = self._run_cmd(install_cmd) if rc != 0: self.err(f"[ERROR] Install failed for {dep['name']} (code {rc})") still_missing = [] for dep in missing: ok, _, _ = inst_config.get_dep_info(dep) if not ok: still_missing.append(dep["name"]) if still_missing: self.err(f"[ERROR] Still missing: {', '.join(still_missing)}") self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing" return False if not self._ensure_gcloud_auth_for_k8s(): self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing" return False self.knoe_cfg_data["Dependencies"]["STATUS"] = "All installed" return True def _ensure_gcloud_auth_for_k8s(self) -> bool: if self._deployment_mode() != "k8s": return True gcloud_env = inst_config._augment_env_for_dependency_backend(os.environ.copy()) cmd_base = ["gcloud"] token_cmd = cmd_base + ["auth", "print-access-token", "--quiet"] try: token_res = subprocess.run( token_cmd, capture_output=True, text=True, env=gcloud_env, timeout=20, ) if token_res.returncode == 0 and (token_res.stdout or "").strip(): return True except Exception: pass interactive = bool(getattr(sys.stdin, "isatty", lambda: False)()) if interactive: login_cmd = cmd_base + ["auth", "login", "--no-launch-browser"] self.log( "[ACTION] No active gcloud session found. Starting login flow: " + " ".join(shlex.quote(part) for part in login_cmd) ) try: login_rc = self._run_cmd(login_cmd) if login_rc == 0: token_res = subprocess.run( token_cmd, capture_output=True, text=True, env=gcloud_env, timeout=20, ) if token_res.returncode == 0 and (token_res.stdout or "").strip(): return True except Exception: pass self.err( "[ERROR] gcloud is installed but no active auth session is available for GKE. " "Run `gcloud auth login --no-launch-browser`, complete the URL/code flow on a browser-capable machine, then retry deploy." ) return False def _step_network_scan(self) -> None: if not self._get_input_bool( "network_scan.run", DEFAULT_ACTION_FLAGS.get("network_scan.run", True) ): self.log("[SKIP] Network scan disabled.") return existing_kdc = self.knoe_cfg_data.get("Network", {}).get("KDC_AUTO_DETECTED") if existing_kdc: self.log( f"[OK] Network scan already performed. KDC detected: {existing_kdc}" ) return self.log("==> Network scan (network-agent)") ansible_kdc = "" try: ansible_kdc = (self.knoe_cfg_data.get("Network", {}) or {}).get( "KDC_ANSIBLE_DETECTED", "" ) except Exception: ansible_kdc = "" scan_binary = get_resource_path("scan/network-agent") if not scan_binary.exists(): self.err(f"[ERROR] Scan binary not found at {scan_binary}") return knoe_home = resolve_knoe_home(env={}) scan_dir = knoe_home / "scan" scan_dir.mkdir(parents=True, exist_ok=True) kdc_found = None pending_line = "" def _handle_line(line: str): nonlocal kdc_found self.log(line.rstrip("\n")) if "KDC is:" in line: try: ip_part = line.split("KDC is:")[1].strip() ip = ip_part.split()[0].strip("[]():,") if ip: kdc_found = ip except Exception: pass elif "Active Directory" in line or "88" in line: for part in line.split(): try: socket.inet_aton(part.strip("[]():,")) kdc_found = part.strip("[]():,") break except Exception: continue def _handle_stdout(text: str): nonlocal pending_line pending_line += text while "\n" in pending_line: line, pending_line = pending_line.split("\n", 1) _handle_line(line + "\n") rc = run_streaming_cmd( [str(scan_binary)], cwd=str(scan_dir), on_stdout=_handle_stdout ) if pending_line: _handle_line(pending_line) if rc == 0: self.log("[OK] Scan complete.") else: self.err(f"[ERROR] Scan failed (code {rc})") if kdc_found and not ansible_kdc: self.inputs["kerberos_config.kdc"] = kdc_found self.inputs["kerberos_config.enabled"] = _bool_str(True) self.knoe_cfg_data["Network"]["KDC_AUTO_DETECTED"] = kdc_found self.knoe_cfg_data["Network"]["KERBEROS_AUTO_ENABLED"] = "True" elif kdc_found and ansible_kdc: self.log( f"[INFO] Ansible KDC detected ({ansible_kdc}); ignoring scan-detected KDC {kdc_found}." ) def _step_env_setup(self) -> None: self.log("==> Environment setup") global_sec = self.knoe_cfg_data.get("Global", {}) or {} sys_sec = self.knoe_cfg_data.get("System Environment", {}) or {} def _cfg_val(sec: dict, key: str) -> str: try: v = str((sec or {}).get(key, "") or "").strip() except Exception: v = "" if v.startswith("${") and v.endswith("}"): return "" return v ns_hint = ( self._get_input("env_setup.DATABASE_NAMESPACE", "") or self._get_input("init_password.db_namespace", "") or _cfg_val(global_sec, "DATABASE_NAMESPACE") or self._get_input("env_setup.NAMESPACE", "") or self._get_input("init_password.db_namespace", "") or _cfg_val(global_sec, "NAMESPACE") ) cluster_hint = ( self._get_input("env_setup.CLUSTER_NAME", "") or self._get_input("init_password.cluster_name", "") or _cfg_val(global_sec, "CLUSTER_NAME") or _cfg_val(global_sec, "CNPG_CLUSTER_NAME") or "knoe-db" ) vals = { "KNOE_HOME": ( self._get_input("env_setup.KNOE_HOME", "") or _cfg_val(sys_sec, "KNOE_HOME") or _cfg_val(global_sec, "KNOE_HOME") or str(resolve_knoe_home(env={})) ), "KNOE_CONF": ( self._get_input("env_setup.KNOE_CONF", "") or _cfg_val(sys_sec, "KNOE_CONF") or _cfg_val(global_sec, "KNOE_CONF") ), "PROLE_DATA": ( self._get_input("env_setup.PROLE_DATA", "") or _cfg_val(sys_sec, "PROLE_DATA") or _cfg_val(global_sec, "PROLE_DATA") ), "PROLE_LOGS": ( self._get_input("env_setup.PROLE_LOGS", "") or _cfg_val(sys_sec, "PROLE_LOGS") or _cfg_val(global_sec, "PROLE_LOGS") ), "KNOE_SERVICE": ( self._get_input("env_setup.KNOE_SERVICE", "") or _cfg_val(sys_sec, "KNOE_SERVICE") or _cfg_val(global_sec, "KNOE_SERVICE") ), "OPENTOFU_URL": (os.environ.get("OPENTOFU_URL") or "").strip(), } defaults = self._env_defaults(ns_hint) for k in vals: if not vals[k]: vals[k] = defaults.get(k, "") if not vals.get("OPENTOFU_URL"): vals["OPENTOFU_URL"] = _default_opentofu_pipeline_url() vals["DATABASE_NAMESPACE"] = ( self._get_input("env_setup.DATABASE_NAMESPACE", "") or self._get_input("env_setup.NAMESPACE", "") or ns_hint ) vals["CLUSTER_NAME"] = ( self._get_input("env_setup.CLUSTER_NAME", "") or cluster_hint ) if not vals.get("KNOE_HOME"): raise Exception("KNOE_HOME is required for env setup.") # Persist normalized inputs for k in ( "KNOE_HOME", "KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE", ): self.inputs[f"env_setup.{k}"] = vals[k] self.inputs["env_setup.DATABASE_NAMESPACE"] = vals.get( "DATABASE_NAMESPACE", "" ) self.inputs["env_setup.CLUSTER_NAME"] = vals.get("CLUSTER_NAME", "") if not (self._get_input("init_password.db_namespace", "") or "").strip(): self.inputs["init_password.db_namespace"] = vals.get( "DATABASE_NAMESPACE", "" ) if not (self._get_input("init_password.cluster_name", "") or "").strip(): self.inputs["init_password.cluster_name"] = vals.get("CLUSTER_NAME", "") self._save_env_to_file(vals) self.reload_env_from_shell() for k in ( "KNOE_HOME", "KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE", "OPENTOFU_URL", ): self.knoe_cfg_data["System Environment"][k] = vals[k] def _step_init_password(self) -> None: self.log("==> Database creation") ns = _safe_str(self._get_input("init_password.db_namespace", "") or "").strip() if not ns: ns = self._get_input("env_setup.DATABASE_NAMESPACE", "") if not ns: ns = self._get_input("env_setup.NAMESPACE", "") if not ns: ns = ( (self.knoe_cfg_data.get("Global", {}) or {}).get( "DATABASE_NAMESPACE", "" ) or "" ).strip() if not ns: ns = ( (self.knoe_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "") or "" ).strip() if ns.startswith("${") and ns.endswith("}"): ns = "" if not ns: raise Exception("Database namespace cannot be empty.") if not self._is_valid_namespace(ns): raise Exception("Invalid database namespace.") cluster_name = ( _safe_str(self._get_input("init_password.cluster_name", "") or "").strip() ) if not cluster_name: cluster_name = ( _safe_str(self._get_input("env_setup.CLUSTER_NAME", "") or "").strip() ) if not cluster_name: cluster_name = ( (self.knoe_cfg_data.get("Global", {}) or {}).get("CLUSTER_NAME", "") or "" ).strip() if not cluster_name: cluster_name = ( (self.knoe_cfg_data.get("Global", {}) or {}).get( "CNPG_CLUSTER_NAME", "" ) or "" ).strip() if not cluster_name: cluster_name = "knoe-db" user = (self._get_input("init_password.db_username", "") or "").strip() if not user: user = ( (self.knoe_cfg_data.get("Global", {}) or {}).get("KNOE_DB_USER", "") or "" ).strip() if user.startswith("${") and user.endswith("}"): user = "" if not user: import getpass try: user = getpass.getuser() except Exception: user = "knoe" self.log(f"[INFO] No database owner provided; using default: {user}") p1 = self._get_input("init_password.db_password", "") p2 = self._get_input("init_password.db_password_confirm", "") or p1 if not p1: p1 = self._generate_db_password() p2 = p1 self.log( "[INFO] No database password provided; generated default password." ) if p1 != p2: raise Exception("Database passwords do not match.") # If password is an OpenBao anchor, only skip when OpenBao is already initialized. if p1.startswith("${"): resolved = self._resolve_secret_value(p1) if resolved and not resolved.startswith("${"): p1 = resolved p2 = resolved else: service_dir = self._get_input("env_setup.KNOE_SERVICE", "").strip() if not service_dir: service_dir = self._env_defaults(ns).get("KNOE_SERVICE", "") token_path = Path(service_dir) / "secrets" / "openbao-root-token" local_token_path = ( resolve_knoe_home(env={}) / "etc" / "secrets" / "openbao-root-token" ) has_token = ( bool(os.environ.get("OPENBAO_ROOT_TOKEN")) or token_path.exists() or local_token_path.exists() ) if self.reset_requested or not has_token: p1 = self._generate_db_password() p2 = p1 self.log( "[RESET] OpenBao anchor detected without token; generated new DB password for initialization." ) else: self.log( f"[OK] OpenBao secrets already anchored for namespace {ns}. Skipping initialization." ) return self.inputs["init_password.db_namespace"] = ns self.inputs["env_setup.DATABASE_NAMESPACE"] = ns self.inputs["init_password.cluster_name"] = cluster_name self.inputs["env_setup.CLUSTER_NAME"] = cluster_name self.inputs["init_password.db_password"] = p1 self.inputs["init_password.db_password_confirm"] = p2 self._update_env_namespace(ns) self.knoe_cfg_data["Database Creation"]["DB_USER"] = user self.knoe_cfg_data["Database Creation"]["DB_PASSWORD_SET"] = "true" self.knoe_cfg_data["Database Creation"]["DATABASE_NAMESPACE"] = ns self.knoe_cfg_data["Database Creation"]["CLUSTER_NAME"] = cluster_name self.knoe_cfg_data["Database Creation"].pop("DB_NAME", None) self.knoe_cfg_data["Database Creation"].pop("NAMESPACE", None) include_supabase = self._get_input_bool("init_cluster.supabase_enabled", False) include_kerberos = self._get_input_bool( "init_cluster.kerberos_enabled", False ) or self._get_input_bool("kerberos_config.enabled", False) if include_supabase: env_policy = self._script_env_for_namespace(ns) allowed, _count, reason = self._optional_workloads_policy(env_policy) if not allowed: self.log(f"[SKIP] Supabase image pre-pull disabled by policy: {reason}") include_supabase = False try: self._prepull_images_to_registry(include_supabase, include_kerberos) except Exception as e: self.err(f"[WARN] Image pre-pull error: {e}") if self._get_input_bool( "init_password.generate_ssh_key", DEFAULT_ACTION_FLAGS.get("init_password.generate_ssh_key", True), ): self._generate_ssh_key(user) # Initialize OpenBao and store keys/passwords for this namespace. # OpenBao is expected to run inside the cluster; defer initialization until cluster is ready. env = os.environ.copy() env["KNOE_HOME"] = str(self.project_root) env["KNOE_SERVICE"] = str(self.project_root) env["KNOE_DB_USER"] = user env["DB_PASSWORD"] = p1 env["GRAFANA_ADMIN_PASSWORD"] = p1 env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) realm = self._get_input("kerberos_config.realm", "").strip() kdc = self._get_input("kerberos_config.kdc", "").strip() krb_user = self._get_input("kerberos_config.user", "").strip() krb_pw = self._get_input("kerberos_config.password", "").strip() if realm: env["KRB5_REALM"] = realm env["REALM"] = realm env["DOMAIN"] = realm.lower() if kdc: env["KRB5_KDC"] = kdc env["KRB5_ADMIN"] = kdc if krb_user: env["KRB5_USER"] = krb_user if krb_pw: env["KRB5_PASSWORD"] = krb_pw self._openbao_init_env = env self._openbao_init_password = p1 self.log("[INFO] Deferring OpenBao initialization until cluster is ready.") def _generate_ssh_key(self, user: str): self.log("==> Environment Preparation (Secure Access)") key_path = Path.home() / ".ssh" / "id_knoe_ed25519" key_path.parent.mkdir(parents=True, exist_ok=True) if key_path.exists(): self.log(f"[SKIP] Secure access keys already exist at {key_path}") return cmd = ["ssh-keygen", "-t", "ed25519", "-N", "", "-f", str(key_path), "-C", user] rc = self._run_cmd(cmd) if rc != 0: self.err(f"[WARN] Preparation failed (code {rc}), trying alternative.") cmd = [ "ssh-keygen", "-t", "rsa", "-b", "4096", "-N", "", "-f", str(key_path), "-C", user, ] rc = self._run_cmd(cmd) if rc != 0: self.err(f"[ERROR] Preparation failed (code {rc})") def _step_db_build(self) -> None: if not self._get_input_bool( "init_db_build.run_build", DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True), ): self.log("[SKIP] DB build disabled.") self.knoe_cfg_data["Docker Build"]["STATUS"] = "Skipped" return tag = self.controller.get_knoe_db_version() image_name = f"knoe-db:{tag}" env_key = _normalize_cluster_env( self._get_input("init_cluster.cluster_env", "dev") ) cluster_name = "knoe-dev-cluster" # Check if image already exists rc_inspect = subprocess.run( ["docker", "inspect", image_name], capture_output=True ).returncode if rc_inspect == 0: self.log(f"[OK] {image_name} already exists. Skipping build.") if env_key == "dev": self.log(f"Importing image to {cluster_name}...") self._run_cmd( ["k3d", "image", "import", image_name, "-c", cluster_name] ) self._db_built_success = True elif env_key == "service": # In k3s mode, publish to the in-cluster registry instead of importing # tarballs into nodes. push_ok = True db_cfg = self.knoe_cfg_data.get("Docker Build", {}) or {} registry = (db_cfg.get("LOCAL_REGISTRY") or "").strip() # Prioritize KNOE_IMAGE_REGISTRY from [Global] if set global_registry = (self.knoe_cfg_data.get("Global", {}) or {}).get("KNOE_IMAGE_REGISTRY", "").strip() if global_registry: registry = global_registry if not registry or registry == "localhost:5000": server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if host: registry = f"{host}:5000" db_cfg["LOCAL_REGISTRY"] = registry self.knoe_cfg_data["Docker Build"] = db_cfg if not registry or registry == "localhost:5000": push_ok = False self.err( "[ERROR] k3s mode but registry host could not be resolved; cannot push knoe-db image." ) else: # Preflight check: verify registry is reachable reg_host = registry reg_port = 80 if ":" in registry: parts = registry.split(":") reg_host = parts[0] try: reg_port = int(parts[1]) except: pass self.log(f"Preflight: checking registry reachability at {reg_host}:{reg_port}...") if not _http_ping_registry(reg_host, reg_port): self.err(f"[ERROR] Registry {registry} is unreachable from this node.") self._db_built_success = False self.knoe_cfg_data["Docker Build"]["STATUS"] = "Failed" return remote_tag = f"{registry}/{image_name}" self.log(f"Tagging {image_name} as {remote_tag}...") subprocess.run( ["docker", "tag", image_name, remote_tag], capture_output=True, ) if _registry_image_ref_exists(remote_tag): self.log( f"[SKIP] {remote_tag} already exists in registry; skipping push." ) else: self.log(f"Pushing {remote_tag} to registry...") if not _push_docker_image(remote_tag, log_fn=self.log): push_ok = False self.err(f"[ERROR] Failed to push image {remote_tag}") self._db_built_success = push_ok else: self._db_built_success = True self.knoe_cfg_data["Docker Build"]["STATUS"] = ( "Built" if self._db_built_success else "Failed" ) return self.log("==> Build knoe-db image") knoe_home = resolve_knoe_home() mode_key = _deployment_mode_from_env(env_key) or "default" build_dir = knoe_home / "build" / mode_key / "knoe-db" build_dir.mkdir(parents=True, exist_ok=True) source_dir = get_resource_path("knoe-db") if not source_dir.exists(): source_dir = get_resource_path("knoe-db") marker = build_dir / ".knoe_build_context_ready" if not marker.exists(): copy_build_context_dir(source_dir, build_dir) pub_key_path = Path.home() / ".ssh" / "id_knoe_ed25519.pub" pub_key = pub_key_path.read_text().strip() if pub_key_path.exists() else "" username = self._get_input("init_password.db_username", "") cmd = ["docker", "build"] cmd.extend(get_docker_build_platform_args(env_key)) cmd += [ "--build-arg", f"PROLE_USER={username}", "--build-arg", f"PROLE_SSH_PUB_KEY={pub_key}", "-t", image_name, ".", ] rc = self._run_cmd(cmd, cwd=str(build_dir)) if rc == 0: self._db_built_success = True self.log("[OK] Build successful.") push_ok = True if env_key == "dev": cluster_name = "knoe-dev-cluster" self.log(f"Importing image to {cluster_name}...") self._run_cmd( ["k3d", "image", "import", image_name, "-c", cluster_name] ) else: # For non-dev, tag and push to registry db_cfg = self.knoe_cfg_data.get("Docker Build", {}) or {} registry = (db_cfg.get("LOCAL_REGISTRY") or "").strip() # Prioritize KNOE_IMAGE_REGISTRY from [Global] if set global_registry = (self.knoe_cfg_data.get("Global", {}) or {}).get("KNOE_IMAGE_REGISTRY", "").strip() if global_registry: registry = global_registry if env_key == "service" and ( not registry or registry == "localhost:5000" ): server, _token = _resolve_k3s_connection_fn( project_root=self.project_root, cfg_path=self.cfg_path, ) host = _host_from_url(server) if host: registry = f"{host}:5000" db_cfg["LOCAL_REGISTRY"] = registry self.knoe_cfg_data["Docker Build"] = db_cfg if env_key == "service" and ( not registry or registry == "localhost:5000" ): push_ok = False self.err( "[ERROR] k3s mode but registry host could not be resolved; cannot push knoe-db image." ) self._db_built_success = False else: if not registry: registry = "localhost:5000" remote_tag = f"{registry}/{image_name}" self.log(f"Tagging {image_name} as {remote_tag}...") subprocess.run(["docker", "tag", image_name, remote_tag], check=True) if _registry_image_ref_exists(remote_tag): self.log( f"[SKIP] {remote_tag} already exists in registry; skipping push." ) else: self.log(f"Pushing {remote_tag} to registry...") if not _push_docker_image(remote_tag, log_fn=self.log): push_ok = False self.err(f"[ERROR] Failed to push image {remote_tag}") self._db_built_success = False self.knoe_cfg_data["Docker Build"]["STATUS"] = ( "Built" if self._db_built_success else "Failed" ) def _step_cluster(self) -> None: self.log("==> Cluster setup") cluster_env = self._get_input("init_cluster.cluster_env", "dev") env_key = _normalize_cluster_env(cluster_env) self.knoe_cfg_data["Initialize Cluster"]["ENVIRONMENT"] = cluster_env self.knoe_cfg_data["Initialize Cluster"]["K3S_SERVER_URL"] = self._get_input( "init_cluster.k3s_server_url", "" ) self.knoe_cfg_data["Initialize Cluster"]["K3S_TOKEN"] = _encrypt_cfg_secret( self._get_input("init_cluster.k3s_token", "") or "" ) self.knoe_cfg_data["Optional Features"]["SUPABASE_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.supabase_enabled", False) ) self.knoe_cfg_data["Optional Features"]["KERBEROS_ENABLED"] = _bool_str( self._get_input_bool("init_cluster.kerberos_enabled", False) ) self.knoe_cfg_data["Optional Features"]["AT_REST_ENCRYPTION_ENABLED"] = ( _bool_str( self._get_input_bool("init_cluster.at_rest_encryption_enabled", False) ) ) mode = _deployment_mode_from_env(cluster_env) if mode: self._ensure_registry_defaults(mode) if not self._get_input_bool( "init_cluster.start_cluster", DEFAULT_ACTION_FLAGS.get("init_cluster.start_cluster", True), ): self.log("[SKIP] Cluster start disabled.") return if env_key == "dev": if not self.controller.check_docker_running(): if platform.system() == "Darwin": self.log("Starting Docker...") self._run_cmd(["open", "-a", "Docker"]) for _ in range(30): time.sleep(2) if self.controller.check_docker_running(): break if not self.controller.check_docker_running(): raise Exception("Docker is not running.") cluster_name = "knoe-dev-cluster" res = subprocess.run( ["k3d", "cluster", "list", "--no-headers"], capture_output=True, text=True, ) res_stdout = res.stdout or "" if self.reset_cluster: self.log(f"[RESET] Deleting k3d cluster {cluster_name}...") self._run_cmd(["k3d", "cluster", "delete", cluster_name]) res_stdout = "" self.reset_cluster = False if cluster_name not in res_stdout: knoe_data = self._resolve_env_dir("PROLE_DATA", "data") self._create_k3d_cluster(cluster_name, knoe_data) else: # Check if it's already running if "running" not in res_stdout.lower(): self._run_cmd(["k3d", "cluster", "start", cluster_name]) else: self.log(f"[OK] Cluster {cluster_name} is already running.") self.log(f"[OK] Cluster ready: {cluster_name}") try: os.environ.pop("KUBECONFIG", None) subprocess.run( ["kubectl", "config", "use-context", f"k3d-{cluster_name}"], capture_output=True, ) except Exception: pass self.inputs["init_cluster.app_cluster_kubecontext"] = f"k3d-{cluster_name}" self.inputs["init_cluster.db_cluster_kubecontext"] = f"k3d-{cluster_name}" self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}" self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}" else: # Ensure we don't pin KUBECONFIG to a stale local file when another # kubeconfig (e.g. Ansible-fetched client-cert auth) is available. service_dir = self._get_input("env_setup.KNOE_SERVICE", "").strip() candidates: list[Path] = [] kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip() if kubeconfig_env: try: candidates.append(Path(kubeconfig_env).expanduser()) except Exception: pass # Prefer Ansible-fetched kubeconfig in the project root. candidates.append(self.project_root / "knoe-k3s.kubeconfig") # Then standard user kubeconfig. candidates.append(Path.home() / ".kube" / "config") # Then knoe-specific secret locations. if service_dir: candidates.append(Path(service_dir) / "secrets" / "k3s.kubeconfig") candidates.append(resolve_knoe_home(env={}) / "secrets" / "k3s.kubeconfig") candidates.append(self.project_root / "etc" / "secrets" / "k3s.kubeconfig") selected = _select_existing_kubeconfig(candidates) if selected: os.environ["KUBECONFIG"] = str(selected) self.log(f"Using kubeconfig: {selected}") else: # No local kubeconfig found – try fetching from k3s via Ansible fetched = self._fetch_k3s_kubeconfig() if fetched: os.environ["KUBECONFIG"] = str(fetched) self.log(f"Using fetched kubeconfig: {fetched}") if mode == "k8s": gcp_cfg = self.gcp_cfg if isinstance(self.gcp_cfg, dict) else {} project_id = str( gcp_cfg.get("project_id") or gcp_cfg.get("PROJECT_ID") or self._get_input("init_cluster.project_id", "") or "" ).strip() if not project_id: raise Exception("GCP project_id is required for dual-cluster GKE setup.") app_cluster_name = self._app_cluster_name() app_cluster_mode = self._app_cluster_mode() app_machine_type = ( self._get_input("init_cluster.app_cluster_machine_type", "") or DEFAULT_APP_CLUSTER_MACHINE_TYPE ).strip() or DEFAULT_APP_CLUSTER_MACHINE_TYPE app_node_count = int( (self._get_input("init_cluster.app_cluster_node_count", "") or str(DEFAULT_APP_CLUSTER_NODE_COUNT)).strip() or str(DEFAULT_APP_CLUSTER_NODE_COUNT) ) db_cluster_name = self._cnpg_cluster_name() db_cluster_mode = self._db_cluster_mode() db_node_count = int( (self._get_input("init_cluster.db_cluster_node_count", "") or "3").strip() or "3" ) db_machine_type = ( self._get_input("init_cluster.db_cluster_machine_type", "") or DEFAULT_DB_CLUSTER_MACHINE_TYPE ).strip() or DEFAULT_DB_CLUSTER_MACHINE_TYPE db_boot_disk_type = ( self._get_input("init_cluster.db_boot_disk_type", "") or DEFAULT_DB_BOOT_DISK_TYPE ).strip() or DEFAULT_DB_BOOT_DISK_TYPE db_boot_disk_size_gb = int( ( self._get_input("init_cluster.db_boot_disk_size_gb", "") or str(DEFAULT_DB_BOOT_DISK_SIZE_GB) ).strip() or str(DEFAULT_DB_BOOT_DISK_SIZE_GB) ) db_location = ( self._get_input("init_cluster.db_cluster_region", "") or gcp_cfg.get("region") or gcp_cfg.get("REGION") or gcp_cfg.get("zone") or gcp_cfg.get("ZONE") or "us-central1" ) app_location = ( self._get_input("init_cluster.app_cluster_region", "") or gcp_cfg.get("region") or gcp_cfg.get("REGION") or db_location ) app_spec = GkeClusterSpec( name=app_cluster_name, mode=app_cluster_mode, location=str(app_location).strip(), machine_type=app_machine_type, node_count=max(1, app_node_count), boot_disk_type=DEFAULT_DB_BOOT_DISK_TYPE, boot_disk_size_gb=DEFAULT_DB_BOOT_DISK_SIZE_GB, ) db_spec = GkeClusterSpec( name=db_cluster_name, mode=db_cluster_mode, location=str(db_location).strip(), machine_type=db_machine_type, node_count=max(3, db_node_count), node_pool_name="cnpg-db-pool", boot_disk_type=db_boot_disk_type, boot_disk_size_gb=db_boot_disk_size_gb, ) self.log(f"[app-cluster {app_cluster_name}] validating cluster") ensure_app_cluster(project_id=project_id, spec=app_spec, log=self.log) self.log(f"[db-cluster {db_cluster_name}] ensuring Standard GKE cluster") ensure_db_cluster(project_id=project_id, spec=db_spec, log=self.log) app_ctx = get_cluster_credentials( project_id=project_id, cluster_name=app_cluster_name, location=app_spec.location, log=self.log, ) db_ctx = get_cluster_credentials( project_id=project_id, cluster_name=db_cluster_name, location=db_spec.location, log=self.log, ) self.inputs["init_cluster.app_cluster_kubecontext"] = app_ctx self.inputs["init_cluster.db_cluster_kubecontext"] = db_ctx self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = app_ctx self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = db_ctx self.knoe_cfg_data.setdefault("Global", {})["APP_CLUSTER_NAME"] = app_cluster_name self.knoe_cfg_data.setdefault("Global", {})["APP_CLUSTER_MODE"] = app_cluster_mode self.knoe_cfg_data.setdefault("Global", {})["APP_CLUSTER_KUBECONTEXT"] = app_ctx self.knoe_cfg_data.setdefault("Global", {})["DB_CLUSTER_NAME"] = db_cluster_name self.knoe_cfg_data.setdefault("Global", {})["DB_CLUSTER_MODE"] = db_cluster_mode self.knoe_cfg_data.setdefault("Global", {})["DB_CLUSTER_KUBECONTEXT"] = db_ctx # Back-compat default context points to app cluster for platform services. self.knoe_cfg_data.setdefault("Global", {})["KUBECONTEXT"] = app_ctx kubectl = subprocess.run(["which", "kubectl"], capture_output=True) if kubectl.returncode != 0: raise Exception( "kubectl not found. Please install kubectl and configure access to the target cluster." ) if env_key == "service": if self.reset_cluster: self.log("[RESET] Resetting k3s cluster (mode k3s)...") self._cleanup_local_k3s_artifacts() delete_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_delete.yml", ] self.log(f"Running: {' '.join(delete_cmd)}") rc = self._run_cmd(delete_cmd) if rc != 0: raise Exception(f"k3s delete failed with code {rc}") # 1) --reset should run the ansible k3s/tasks/cleanup on myrddin and then reinstall. # This is handled by k3s_reset.yml reset_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_reset.yml", "-l", "myrddin.knoe.org", ] + vault_args self.log(f"Running: {' '.join(reset_cmd)}") rc = self._run_cmd(reset_cmd) if rc != 0: raise Exception(f"k3s reset failed with code {rc}") # 2) Capture new k3s token + TLS bundle from init server and update vault sync_token_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_sync.yml", "-l", "myrddin.knoe.org", ] + vault_args self.log(f"Running: {' '.join(sync_token_cmd)}") rc = self._run_cmd(sync_token_cmd) if rc != 0: raise Exception(f"k3s sync (token) failed with code {rc}") # 3) Reinstall agent nodes with the refreshed token reset_agents_cmd = [ "./ansible.sh", "-p", "infrastructure/playbooks/k3s_reset.yml", "-l", "pi.knoe.org,retropie.knoe.org", ] + vault_args self.log(f"Running: {' '.join(reset_agents_cmd)}") rc = self._run_cmd(reset_agents_cmd) if rc != 0: raise Exception(f"k3s reset (agents) failed with code {rc}") self.reset_cluster = False if not (os.environ.get("KUBECONFIG") or "").strip(): kubeconfig_path = _find_kubeconfig_file() if kubeconfig_path: os.environ["KUBECONFIG"] = kubeconfig_path self.log(f"Using local kubeconfig: {kubeconfig_path}") kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip() if kubeconfig_env and os.path.exists(kubeconfig_env): cmd = ["kubectl", "--kubeconfig", kubeconfig_env, "cluster-info"] else: server = ( self._get_input("init_cluster.k3s_server_url", "") or "" ).strip() token = ( self._get_input("init_cluster.k3s_token", "") or "" ).strip() if token: token = self._resolve_secret_value(token) if server and token: if not server.startswith("http"): server = f"https://{server}" cmd = [ "kubectl", "--server=" + server, "--token=" + token, "--insecure-skip-tls-verify=true", "cluster-info", ] else: raise Exception( "K3s server URL/token missing and no kubeconfig available for service cluster." ) else: cmd = ["kubectl", "cluster-info"] res = subprocess.run(cmd, capture_output=True, text=True) if res.returncode != 0: raise Exception("Cluster is not reachable.") self.log(f"[OK] Cluster ready: {env_key}") if env_key == "service": try: registry_ns = self._registry_namespace() env = self._script_env_for_namespace(self._service_namespace()) env["REGISTRY_NAMESPACE"] = registry_ns self.log("==> Registry preflight") registry_ops.update( namespace=registry_ns, env=env, mode=env.get("KNOE_MODE"), project_root=self.project_root, log=self.log, ) except Exception as e: self.err(f"[WARN] Registry preflight failed: {e}") # If this run was launched with a reset request, attempt to reclaim any # strongly-matched stale Released PVs early, before later deploy/init # steps that may depend on PVC scheduling. if getattr(self, "_reset_reclaim_pvs_requested", False): try: service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) self.log("==> Reset: reclaim stale Released PV claimRefs") self._reset_reclaim_stale_pvs(env) except Exception as e: self.err(f"[WARN] Reset stale-PV reclaim failed: {e}") self._reset_reclaim_pvs_requested = False if self._openbao_init_env: env = dict(self._openbao_init_env) mode = _deployment_mode_from_env( self._get_input("init_cluster.cluster_env", "") ) if not mode: if env_key == "dev": mode = "k3d" elif env_key == "service": mode = "k3s" elif env_key == "prod": mode = "k8s" env.setdefault("KNOE_MODE", mode or env_key) self.log("==> OpenBao initialize (cluster)") db_ns = (self._get_input("init_password.db_namespace", "") or "").strip() if not db_ns: db_ns = self._service_namespace() openbao_ops.initialize( namespace=db_ns, env=env, project_root=self.project_root, mode=mode, log=self.log, ) # Materialize DB secrets in Kubernetes now that we have a cluster and a user-provided password. # This prevents downstream init steps (e.g. CNPG) from failing due to missing secrets. self.ensure_db_k8s_secrets( db_ns, self._openbao_init_password or "", log_fn=self.log, ) self._openbao_init_env = None self._openbao_init_password = None # Run repair pipeline if anomalies are detected on a ready cluster try: service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) self._maybe_run_repair_pipeline(env) except Exception: pass if env_key == "service": self._deploy_common_services(env_key) else: self._deploy_opentofu(env_key) def _deploy_common_services(self, env_key: str) -> None: self.log("==> Common services deploy") service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) mode = env.get("KNOE_MODE") or self._deployment_mode() manage_opentofu = str(mode).lower() != "k8s" registry_ns = self._registry_namespace() if not manage_opentofu: self.log("[INFO] k8s mode: skipping OpenTofu checks/deploy in common services.") try: # Avoid deploying into a namespace that is currently being deleted. self._ensure_namespace_ready(env, service_ns) # Ensure node labels are present before init scripts that depend on them # (e.g. node-role selectors and pinned local PV workloads). try: self._apply_ansible_node_labels(env) except Exception: pass # Refuse to proceed when the cluster has hard prerequisites blocked # (e.g. PV binding / scheduling issues). This prevents blind retries # and surfaces actionable diagnostics. self._reconcile_blocked_cluster_state(env=env, service_ns=service_ns) def _status_safe(check_fn) -> bool: try: return bool(check_fn()) except Exception: return False status_checks = [ _status_safe( lambda: registry_ops.status( namespace=registry_ns, env=env, mode=mode, ) ), _status_safe( lambda: openbao_ops.status( namespace=service_ns, env=env, mode=mode, ) ), _status_safe( lambda: garage_store_ops.status( namespace=service_ns, env=env, ) ), ] if manage_opentofu: status_checks.append( _status_safe( lambda: opentofu_ops.status( namespace=service_ns, env=env, ) ) ) is_healthy = all(status_checks) if is_healthy: self.log( f"[OK] Common services in {service_ns} are healthy. Skipping update." ) return self.log( f"[INFO] Common services unhealthy or missing in {service_ns}. Attempting repair/deploy..." ) try: registry_ops.update( namespace=registry_ns, env=env, mode=mode, project_root=self.project_root, log=self.log, ) openbao_ops.update( namespace=service_ns, env=env, mode=mode, project_root=self.project_root, log=self.log, ) garage_store_ops.update( namespace=service_ns, env=env, mode=mode, project_root=self.project_root, log=self.log, ) if manage_opentofu: opentofu_ops.update( namespace=service_ns, env=env, mode=mode, project_root=self.project_root, log=self.log, ) except Exception as e: self.err(f"[ERROR] Common services deploy failed: {e}") else: # Re-validate after fix. repair_checks = [ _status_safe( lambda: registry_ops.status( namespace=registry_ns, env=env, mode=mode, ) ), _status_safe( lambda: openbao_ops.status( namespace=service_ns, env=env, mode=mode, ) ), _status_safe( lambda: garage_store_ops.status( namespace=service_ns, env=env, ) ), ] if manage_opentofu: repair_checks.append( _status_safe( lambda: opentofu_ops.status( namespace=service_ns, env=env, ) ) ) repaired = all(repair_checks) if not repaired: self.err( f"[ERROR] Common services still unhealthy after repair attempt." ) else: self.log(f"[OK] Common services in {service_ns} are now healthy.") finally: pass def _apply_ansible_node_labels(self, env: dict) -> None: """Apply Ansible-defined node labels to the current cluster. This makes node selection deterministic even when UI selections are stale or missing. """ inv = ( (self.knoe_cfg_data.get("Network", {}) or {}).get("ANSIBLE_INVENTORY") or "" ).strip() if not inv: return inv_path = Path(inv) if not inv_path.exists(): return labels_by_host = _detect_ansible_node_labels(inv_path) if not labels_by_host: return self.log("==> Applying Ansible node labels") base_env = dict(os.environ) base_env.update(env or {}) for host, labels in sorted(labels_by_host.items()): if not labels: continue label_args = [f"{k}={v}" for k, v in sorted(labels.items())] def _label(node_name: str) -> bool: res = subprocess.run( ["kubectl", "label", "node", node_name, "--overwrite", *label_args], capture_output=True, text=True, env=base_env, ) if res.returncode == 0: return True return False # Prefer FQDN node names, but retry with short hostname when needed. if _label(host): continue if "." in host: short = host.split(".", 1)[0] if short and _label(short): continue self.err(f"[WARN] Failed to apply labels to node '{host}'.") def _deploy_opentofu(self, env_key: str) -> None: self.log("==> OpenTofu deploy") ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) try: opentofu_ops.start(namespace=ns, env=env, mode=env.get("KNOE_MODE"), log=self.log) finally: pass def _step_kerberos(self) -> None: enabled = self._get_input_bool("kerberos_config.enabled", False) if not enabled: self.log("[SKIP] Kerberos disabled.") return realm = self._get_input("kerberos_config.realm", "") kdc = self._get_input("kerberos_config.kdc", "") user = self._get_input("kerberos_config.user", "") password = self._get_input("kerberos_config.password", "") self.knoe_cfg_data["Kerberos Authentication"]["ENABLED"] = _bool_str(enabled) self.knoe_cfg_data["Kerberos Authentication"]["REALM"] = realm self.knoe_cfg_data["Kerberos Authentication"]["KDC"] = kdc self.knoe_cfg_data["Kerberos Authentication"]["SERVER"] = kdc self.knoe_cfg_data["Kerberos Authentication"]["USER"] = user self.knoe_cfg_data["Kerberos Authentication"]["PASSWORD"] = password self.knoe_cfg_data["Kerberos Authentication"]["AD_PORT_FORWARD"] = ( os.environ.get("KRB5_AD_PORT_FORWARD", "1") ) self.knoe_cfg_data["Kerberos Authentication"]["AD_TCP_PORTS"] = os.environ.get( "KRB5_AD_TCP_PORTS", "88 389 445 464 636" ) self.knoe_cfg_data["Kerberos Authentication"]["AD_UDP_PORTS"] = os.environ.get( "KRB5_AD_UDP_PORTS", "88 464" ) self.knoe_cfg_data["Kerberos Authentication"]["AD_PROXY_HOST_NETWORK"] = ( os.environ.get("KRB5_AD_PROXY_HOST_NETWORK", "1") ) self.knoe_cfg_data["Kerberos Authentication"]["AD_PROXY_IMAGE"] = ( os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat") ) self.knoe_cfg_data["Kerberos Authentication"]["AD_PROXY_SERVICE"] = ( os.environ.get("KRB5_AD_SERVICE_NAME", "knoe-kerberos-ad-dc") ) ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) env["KERBEROS_ENABLED"] = _bool_str(enabled) env["ENABLED"] = env["KERBEROS_ENABLED"] env["KRB5_REALM"] = realm if env.get("KNOE_MODE") == "k3s": env.setdefault("KRB5_AD_PORT_FORWARD", "0") if not self._get_input_bool( "kerberos_config.test_connection", DEFAULT_ACTION_FLAGS.get("kerberos_config.test_connection", False), ): self.log("[SKIP] Kerberos test disabled.") return # Check if already tested successfully if ( self.knoe_cfg_data.get("Kerberos Authentication", {}).get("TEST_STATUS") == "Success" ): self.log("[OK] Kerberos already tested successfully. Skipping.") return if not (realm and user and password and kdc): self.err("[WARN] Kerberos test skipped: missing realm/user/password/kdc.") return self.log("==> Kerberos test: init_kerberos.sh test") rc2 = self._run_script("init_kerberos.sh", args=["test"], env=env) if rc2 != 0: self.err(f"[ERROR] Kerberos test failed (code {rc2})") self.knoe_cfg_data.setdefault("Kerberos Authentication", {})[ "TEST_STATUS" ] = "Failed" else: self.log("[OK] Kerberos test completed successfully.") self.knoe_cfg_data.setdefault("Kerberos Authentication", {})[ "TEST_STATUS" ] = "Success" def _step_init_scripts(self) -> None: if not self._get_input_bool( "init_scripts.run_scripts", DEFAULT_ACTION_FLAGS.get("init_scripts.run_scripts", True), ): self.log("[SKIP] Init scripts disabled.") self.knoe_cfg_data["Initialization Scripts"]["STATUS"] = "Skipped" return if ( self.knoe_cfg_data.get("Initialization Scripts", {}).get("STATUS") == "Finished" ): self.log("[OK] Initialization scripts already finished.") return self.log("==> Initialization scripts") mode = self._deployment_mode() db_ns = self._secret_namespace() if mode == "k8s" and db_ns == "default": db_ns = "knoe-db-0" service_ns = self._service_namespace() db_env = self._script_env_for_namespace(db_ns, cluster_role="db") app_env = self._script_env_for_namespace(service_ns, cluster_role="app") opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(app_env) password = self._get_input("init_password.db_password", "") kerberos_enabled = self._get_input_bool("kerberos_config.enabled", False) pre_cnpg_steps = [ ("init_certmgr.sh", ["initialize"], False), ] if kerberos_enabled: pre_cnpg_steps.append(("init_kerberos.sh", ["initialize"], False)) post_cnpg_steps: list[tuple[str, list[str], bool, str]] = [ ("init_cnpg_backup.sh", ["start"], False, "db"), ("init_kong.sh", ["start"], False, "app"), ] if not opt_allowed: self.log(f"[SKIP] Monitoring disabled by policy: {opt_reason}") self.knoe_cfg_data.setdefault("Monitoring", {})["STATUS"] = "Skipped" if mode != "k3d": post_cnpg_steps.append(("init_nginx_ingress.sh", ["initialize"], False, "app")) overall_success = True # Common services (Python owners) — replaces init_common_services.sh self.log( f"[app-cluster {self._app_cluster_name()} context={app_env.get('KUBECONTEXT','')}] " f"common services namespace={service_ns}" ) try: registry_ns = str(app_env.get("REGISTRY_NAMESPACE") or service_ns).strip() or service_ns registry_ops.update(namespace=registry_ns, env=app_env, mode=mode, log=self.log) openbao_ops.update(namespace=service_ns, env=app_env, mode=mode, log=self.log) garage_store_ops.update(namespace=service_ns, env=app_env, mode=mode, log=self.log) opentofu_ops.update(namespace=service_ns, env=app_env, mode=mode, log=self.log) except Exception as e: self.err(f"[ERROR] common services (python owners) failed: {e}") overall_success = False # Run pre-CNPG shell scripts (cert-manager, optional kerberos) for script, args, needs_password in pre_cnpg_steps: self.log(f"--> {script} {' '.join(args)}") stdin_text = f"{password}\n" if needs_password else None rc = self._run_script( script, args=args, env=db_env, stdin_text=stdin_text, on_line=self._process_script_output_line, ) if rc != 0: self.err(f"[ERROR] {script} failed (code {rc})") overall_success = False # CNPG initialization (Python) — replaces init_cloudnative_pg.sh initialize if overall_success: self.log( f"[db-cluster {self._cnpg_cluster_name()} context={db_env.get('KUBECONTEXT','')}] " f"cnpg_initialize namespace={db_ns}" ) try: self.ensure_db_k8s_secrets(db_ns, password, log_fn=self.log) except Exception as e: self.err(f"[ERROR] Failed to ensure DB secrets before CNPG init: {e}") overall_success = False if overall_success: if self._cluster_storage_milestone_enabled(): try: self._ensure_cnpg_storage_provisioned(db_ns, db_env) except Exception as e: self.err(f"[WARN] Failed to provision CNPG storage before init: {e}") else: self.log( "[SKIP] Cluster storage milestone shelved; " "retaining utility workflows only." ) try: cnpg_initialize( namespace=db_ns, cluster_name=str(db_env.get("CLUSTER_NAME") or db_env.get("CNPG_CLUSTER_NAME") or "knoe-db").strip(), env=db_env, project_root=self.project_root, log=self.log, mode=mode, ) except Exception as e: self.err(f"[ERROR] CNPG initialization failed: {e}") overall_success = False # Run post-CNPG shell scripts (backup, kong, ingress) if overall_success: for script, args, needs_password, role in post_cnpg_steps: self.log(f"--> {script} {' '.join(args)}") stdin_text = f"{password}\n" if needs_password else None target_env = db_env if role == "db" else app_env rc = self._run_script( script, args=args, env=target_env, stdin_text=stdin_text, on_line=self._process_script_output_line, ) if rc != 0: self.err(f"[ERROR] {script} failed (code {rc})") overall_success = False if overall_success and opt_allowed: self.log(f"[app-cluster {self._app_cluster_name()}] monitoring (python owner) initialize") try: monitoring_ns = str(app_env.get("MONITORING_NAMESPACE") or "monitoring").strip() or "monitoring" monitoring_ops.initialize( namespace=monitoring_ns, env=app_env, mode=mode, log=self.log, ) except Exception as e: self.err(f"[ERROR] monitoring initialization failed: {e}") overall_success = False self._scripts_success = overall_success # Verify critical secrets self.log(f"==> Verifying critical secrets in namespace {db_ns}") critical_secrets = ["knoe-db-user", "knoe-db-superuser", "cnpg-admin-key"] missing_secrets = [] for secret in critical_secrets: rc_s = self._run_cmd( ["kubectl", "get", "secret", secret, "-n", db_ns], env=db_env, ) if rc_s != 0: missing_secrets.append(secret) if missing_secrets: self.err( f"[CRITICAL] Missing secrets in namespace '{db_ns}': {', '.join(missing_secrets)}" ) self.err("Database initialization will fail without these secrets.") self._scripts_success = False self.knoe_cfg_data["Initialization Scripts"]["STATUS"] = ( "Completed" if self._scripts_success else "Attempted" ) def _step_cnpg_deploy(self) -> None: if not self._get_input_bool( "init_cnpg_deploy.run_deploy", DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True), ): self.log("[SKIP] CnPG deploy disabled.") self.knoe_cfg_data["Deployment"]["STATUS"] = "Skipped" return db_cluster = self._cnpg_cluster_name() self.log(f"==> [db-cluster {db_cluster}] Deploy CloudNativePG") ns = self._secret_namespace() if self._deployment_mode() == "k8s" and ns == "default": ns = "knoe-db-0" env = self._script_env_for_namespace(ns, cluster_role="db") cluster_name = str( env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") or self._cnpg_cluster_name() ).strip() if not cluster_name: cluster_name = "knoe-db" mode = self._deployment_mode() if self._cluster_storage_milestone_enabled(): try: self._ensure_cnpg_storage_provisioned(ns, env) except Exception as exc: self._cnpg_success = False self.knoe_cfg_data["Deployment"]["STATUS"] = "Attempted" self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}") return else: self.log( "[SKIP] Cluster storage milestone shelved; " "retaining utility workflows only." ) try: self.log(f"[db-cluster {db_cluster}] applying knoe-db cluster") cnpg_deploy( namespace=ns, cluster_name=cluster_name, env=env, project_root=self.project_root, log=self.log, mode=mode, ) self._cnpg_success = True self.knoe_cfg_data["Deployment"]["STATUS"] = "Deployed" except Exception as exc: self._cnpg_success = False self.knoe_cfg_data["Deployment"]["STATUS"] = "Attempted" self.err(f"[ERROR] CnPG deploy failed: {exc}") if self._get_input_bool( "init_cnpg_deploy.force_rollout", DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False), ): # Only attempt rollout if there are existing CNPG pods; otherwise this is a fresh install rc_check = subprocess.run( [ "kubectl", "get", "pods", "-n", ns, "-l", f"cnpg.io/cluster={cluster_name}", "--no-headers", ], capture_output=True, text=True, env=env, ) has_pods = bool((rc_check.stdout or "").strip()) if has_pods: self.log(f"==> [db-cluster {db_cluster}] Force rollout") try: cnpg_rollout( namespace=ns, cluster_name=cluster_name, env=env, log=self.log, ) except Exception as exc2: self.err(f"[ERROR] Rollout failed: {exc2}") else: self.log( "[SKIP] Force rollout skipped — no existing CNPG pods (new installation)." ) def _step_supabase(self) -> None: if not self._get_input_bool("init_cluster.supabase_enabled", False): self.log("[SKIP] Supabase deploy disabled.") self.knoe_cfg_data["Supabase"] = {"STATUS": "Skipped"} return self.log("==> Deploy Supabase") ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" env = self._script_env_for_namespace(ns) opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(env) if not opt_allowed: self.log(f"[SKIP] Supabase deploy disabled by policy: {opt_reason}") self.knoe_cfg_data["Supabase"] = {"STATUS": "Skipped"} self._supabase_success = False return script_path = self.project_root / "supabase" / "deploy.sh" if not script_path.exists(): self.err(f"[ERROR] Supabase deploy script not found: {script_path}") self._supabase_success = False self.knoe_cfg_data["Supabase"] = {"STATUS": "Attempted"} return mode = ( os.environ.get("SUPABASE_DEPLOY_MODE") or os.environ.get("SUPABASE_MODE") or "" ).strip() if not mode: # Match cluster environment to supabase deploy mode cluster_env = _normalize_cluster_env( self._get_input("init_cluster.cluster_env", "dev") ) if cluster_env == "dev": mode = "k3d" elif cluster_env in ("service", "prod"): mode = "k8s" else: mode = "k3d" args = ["--mode", mode] cfg_path = None if self.cfg_path and self.cfg_path.exists(): cfg_path = self.cfg_path else: conf_dir = knoe_conf_mgr.resolve_knoe_conf_dir(self.project_root) candidate = knoe_conf_mgr.entrypoint_path(conf_dir) if candidate.exists(): cfg_path = candidate if cfg_path: args.extend(["-c", str(cfg_path)]) if _parse_bool(os.environ.get("SUPABASE_USE_DEV_COMPOSE"), False): args.append("--with-dev-helpers") if _parse_bool(os.environ.get("SUPABASE_FOREGROUND"), False): args.append("--foreground") self.log(f"--> supabase/deploy.sh {' '.join(args)}") rc = self._run_cmd(["bash", str(script_path)] + args, env=env) if rc == 0: self._supabase_success = True self.knoe_cfg_data["Supabase"] = {"STATUS": "Deployed"} else: self._supabase_success = False self.knoe_cfg_data["Supabase"] = {"STATUS": "Attempted"} self.err(f"[ERROR] Supabase deploy failed (code {rc})") def _finalize_secrets(self) -> None: ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" if not ns or ns == "default": ns = self._get_input("env_setup.DATABASE_NAMESPACE", "") or "default" if not ns or ns == "default": ns = self._get_input("env_setup.NAMESPACE", "") or "default" p1 = self._get_input("init_password.db_password", "") if p1.startswith("${"): self.log( f"[OK] Secrets already finalized for namespace {ns}. Skipping build-a-bao." ) return self.log("==> OpenBao: finalize secrets") env = self._script_env_for_namespace(ns) rc = self._run_script("build-a-bao.sh", env=env) if rc == 0: self._secrets_finalized = True self.log("[OK] Secrets saved to OpenBao and knoe.cfg updated.") else: self.err(f"[ERROR] build-a-bao.sh failed (code {rc})") def _prepare_opentofu_pipeline(self) -> None: env_key = _normalize_cluster_env( self._get_input("init_cluster.cluster_env", "") ) mode = _deployment_mode_from_env(self._get_input("init_cluster.mode", "")) if not mode: mode = _deployment_mode_from_env(os.environ.get("KNOE_MODE", "")) # OpenTofu pipeline is k3s-scoped, but we still want it available for k3d # installs when the user provides k3s connection settings (e.g., staging # manifests for a service cluster while running a local dev cluster). if env_key != "service" and mode not in ("k3d", "k3s"): return namespace = ( self._get_input("init_password.db_namespace", "") or "" ).strip() if not namespace: namespace = ( self._get_input("env_setup.DATABASE_NAMESPACE", "") or "" ).strip() if not namespace: namespace = ( self._get_input("env_setup.NAMESPACE", "") or "" ).strip() if not namespace: namespace = "default" k3s_server = (self._get_input("init_cluster.k3s_server_url", "") or "").strip() k3s_token = (self._get_input("init_cluster.k3s_token", "") or "").strip() if k3s_token: k3s_token = self._resolve_secret_value(k3s_token) if k3s_server and not k3s_server.startswith("http"): k3s_server = f"https://{k3s_server}" if not k3s_server: self.err("[WARN] OpenTofu pipeline skipped: missing k3s server URL.") return if not k3s_token: self.err("[WARN] OpenTofu pipeline missing k3s token; writing empty token.") pipeline_dir = _sync_opentofu_pipeline( self.project_root, namespace, k3s_server, k3s_token ) self.knoe_cfg_data["Deployment"]["OPENTOFU_PIPELINE_DIR"] = str(pipeline_dir) self.log(f"[OK] OpenTofu pipeline prepared at {pipeline_dir}") def _create_k3d_cluster(self, cluster_name: str, knoe_data: Path) -> None: volume_args = _k3d_knoe_data_volume_args(str(knoe_data)) reg_args = [] if _local_registry_enabled("k3d"): info = self._ensure_local_registry_available() if info: _host_registry, cluster_registry = info reg_use = cluster_registry or "" if reg_use.endswith(".localhost:5000"): reg_use = reg_use.replace(".localhost:5000", ":5000") elif reg_use.endswith(".localhost"): reg_use = reg_use[: -len(".localhost")] if reg_use: reg_args = ["--registry-use", reg_use] cmd = ( ["k3d", "cluster", "create", cluster_name, "-a", "2"] + volume_args + reg_args + ["--api-port", "0.0.0.0:6443"] ) rc = self._run_cmd(cmd) if rc == 0: return if platform.system() == "Darwin": fallback = resolve_knoe_home(env={}) / "data" try: if knoe_data.resolve() == fallback.resolve(): raise Exception("k3d cluster create failed.") except Exception: # If path resolution fails, still try the fallback pass self.err( f"[WARN] k3d create failed with {knoe_data}; retrying with {fallback}..." ) rc = self._run_cmd( ["k3d", "cluster", "create", cluster_name, "-a", "2"] + _k3d_knoe_data_volume_args(str(fallback)) + reg_args + ["--api-port", "0.0.0.0:6443"] ) if rc == 0: # Persist fallback for the rest of the run self.inputs["env_setup.PROLE_DATA"] = str(fallback) return raise Exception(f"Failed to create k3d cluster {cluster_name} (code {rc})") def _perform_cluster_reset(self) -> None: if not self.reset_requested: return cluster_env = self._get_input("init_cluster.cluster_env", "dev") env_key = _normalize_cluster_env(cluster_env) if env_key == "dev": self.log("==> Cluster reset (k3d)") if not self.controller.check_docker_running(): if platform.system() == "Darwin": self.log("Starting Docker...") self._run_cmd(["open", "-a", "Docker"]) for _ in range(30): time.sleep(2) if self.controller.check_docker_running(): break if not self.controller.check_docker_running(): raise Exception("Docker is not running.") cluster_name = "knoe-dev-cluster" rc = self._run_cmd(["k3d", "cluster", "delete", cluster_name]) if rc != 0: self.err( f"[WARN] Failed to delete k3d cluster {cluster_name} (code {rc}); continuing." ) knoe_data = self._resolve_env_dir("PROLE_DATA", "data") self._create_k3d_cluster(cluster_name, knoe_data) elif env_key in ("service", "knoe-service-cluster", "k3s"): self.log("==> Cluster reset (k3s namespace cleanup)") db_ns = self._get_input("env_setup.DATABASE_NAMESPACE", "") if not db_ns: db_ns = self._get_input("env_setup.NAMESPACE", "") if not db_ns: db_ns = "default" db_ns = db_ns.strip() or "default" service_ns = self._service_namespace().strip() or "knoe-system" monitoring_ns = ( ((self.knoe_cfg_data.get("Global", {}) or {}).get("MONITORING_NAMESPACE", "") or "").strip() or (os.environ.get("MONITORING_NAMESPACE") or "").strip() or "monitoring" ) cnpg_ns = ( ((self.knoe_cfg_data.get("Global", {}) or {}).get("CNPG_NAMESPACE", "") or "").strip() or (os.environ.get("CNPG_NAMESPACE") or "").strip() or "cnpg-system" ) argocd_ns = self._argocd_namespace().strip() or "argocd" registry_ns = service_ns reset_namespaces: list[str] = [] for candidate in [db_ns, service_ns, monitoring_ns, cnpg_ns, argocd_ns, "default"]: ns = (candidate or "").strip() if not ns or (ns.startswith("${") and ns.endswith("}")): continue if ns not in reset_namespaces: reset_namespaces.append(ns) server = self._get_input("init_cluster.k3s_server_url", "") token = self._get_input("init_cluster.k3s_token", "") # For remote k3s clusters the config token is typically a *join token* # (e.g. `K10...::server:...`) which cannot be used to authenticate kubectl. # Ensure we have a cert-based kubeconfig (or a valid bearer token config) # before invoking any kubectl-based reset scripts. self._ensure_project_k3s_kubeconfig(server, token) if self.delete_db_requested: self.log(f"[RESET] Namespaces to clean: {', '.join(reset_namespaces)}") self.log( f"[RESET] --delete-db enabled; database PVCs in namespace '{db_ns}' will be deleted." ) else: self.log( f"[RESET] Namespaces to clean (PVCs preserved): {', '.join(reset_namespaces)}" ) self.log( f"[RESET] Preserving registry workload in namespace: {registry_ns}" ) _reset_k3s_namespace( self.project_root, reset_namespaces, server, token, registry_namespace=registry_ns, clear_node_reservations=True, delete_pvcs_namespaces=[db_ns] if self.delete_db_requested else None, force_delete_bound_pvs=self.delete_db_requested, ) self._cleanup_local_k3s_artifacts() self.reset_requested = False self.delete_db_requested = False def _cleanup_local_k3s_artifacts(self) -> None: paths = [ self.project_root / "etc" / "secrets" / "k3s.kubeconfig", resolve_knoe_home(env={}) / "secrets" / "k3s.kubeconfig", ] for path in paths: try: if path.exists(): path.unlink() self.log(f"[RESET] Removed local kubeconfig {path}") except Exception as exc: self.err(f"[WARN] Failed to remove {path}: {exc}") def _ensure_project_k3s_kubeconfig(self, server: str, token: str) -> None: server = (server or "").strip() token = (token or "").strip() target = self.project_root / "knoe-k3s.kubeconfig" backup = target.with_name(target.name + ".bak") def _looks_cert_based(path: Path) -> bool: try: txt = path.read_text(encoding="utf-8") except Exception: return False return "client-certificate-data" in txt def _looks_like_join_token_cfg(path: Path) -> bool: try: txt = path.read_text(encoding="utf-8") except Exception: return False # k3s join tokens include `::server:`; they are not Kubernetes bearer tokens. return "token:" in txt and "::server:" in txt and "client-certificate-data" not in txt # If we already have a kubeconfig, keep it unless it is clearly an invalid join-token config. if target.exists() and not _looks_like_join_token_cfg(target): return # Prefer restoring a known-good cert-based kubeconfig. if (not target.exists() or _looks_like_join_token_cfg(target)) and backup.exists() and _looks_cert_based(backup): try: target.write_text(backup.read_text(encoding="utf-8"), encoding="utf-8") os.chmod(target, 0o600) self.log(f"[OK] Restored kubeconfig from backup: {target}") return except Exception as exc: self.err(f"[WARN] Failed to restore kubeconfig from {backup}: {exc}") # Next best: fetch a fresh cert-based kubeconfig via Ansible. fetched = self._fetch_k3s_kubeconfig() if fetched and target.exists() and not _looks_like_join_token_cfg(target): return # Final fallback: only generate a token-based kubeconfig if the token actually looks like a # Kubernetes bearer token. (k3s join tokens are not usable for kubectl auth.) if server and token and _looks_like_k8s_bearer_token(token): try: generated = _write_k3s_kubeconfig(server, token) if generated.exists() and generated != target: target.write_text(generated.read_text(encoding="utf-8"), encoding="utf-8") os.chmod(target, 0o600) self.log(f"[OK] Wrote token-based kubeconfig: {target}") except Exception as exc: self.err(f"[WARN] Failed to generate token-based kubeconfig: {exc}") else: self.err( "[WARN] No usable kubeconfig available for remote k3s; please fetch/create a cert-based knoe-k3s.kubeconfig." ) def _close_log_file(self): """Restore original stdout/stderr and close the log file.""" if self._log_file: sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ try: self._log_file.close() except Exception: pass self._log_file = None def _prompt_for_master_password(self) -> str: """Prompt user for master password in silent mode with a 10-minute timeout.""" import sys import time import select import termios import tty # Use sys.__stdout__ and sys.__stdin__ to ensure we bypass any redirection out = sys.__stdout__ inp = sys.__stdin__ # ANSI color codes GREEN = "\033[32m" RED = "\033[31m" RESET = "\033[0m" BOLD = "\033[1m" CHECK = "\u2713" out.write("\n" + "=" * 60 + "\n") out.write(f"{BOLD}DATABASE MASTER PASSWORD REQUIRED{RESET}\n") out.write("A master password is required to initialize the database cluster.\n") out.write("You are running in silent mode (-S) and no valid password was found.\n") out.write(f"Please provide it now. Action timer: {BOLD}10 minutes (600s){RESET}.\n") out.write("=" * 60 + "\n\n") out.flush() timeout = 600 start_time = time.time() def get_input_with_timeout(prompt, mask=True): out.write(prompt) out.flush() if not inp.isatty(): # Fallback for non-TTY environments if necessary, but requirement says "prompt" # In non-TTY, select might still work on stdin. pass fd = inp.fileno() old_settings = termios.tcgetattr(fd) try: tty.setcbreak(fd) res = "" while True: elapsed = time.time() - start_time if elapsed >= timeout: out.write( f"\n{RED}[ERROR] Timeout waiting for password input (600s). Cannot proceed.{RESET}\n" ) out.flush() sys.exit(1) rlist, _, _ = select.select([inp], [], [], 1.0) if rlist: char = inp.read(1) if char in ("\n", "\r"): out.write("\n") out.flush() return res elif char == "\x7f": # Backspace if res: res = res[:-1] if mask: out.write("\b \b") out.flush() elif char == "\x03": # Ctrl+C out.write(f"\n{RED}Installation aborted by user.{RESET}\n") out.flush() sys.exit(1) else: res += char if mask: out.write("*") out.flush() finally: termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) while True: p1 = get_input_with_timeout("Enter Master Password: ") p2 = get_input_with_timeout("Confirm Master Password: ") if p1 and p1 == p2: out.write(f"{GREEN}{CHECK} Password accepted.{RESET}\n\n") out.flush() return p1 elif not p1: out.write(f"{RED}Password cannot be empty.{RESET}\n") out.flush() else: out.write( f"{RED}Passwords do not match. Please try again.{RESET}\n" ) out.flush() def _load_db_password_from_1password(self, *, log_found: bool = True) -> str: try: from knoe.core.onepassword import get_secret, op_available if not op_available(): return "" password = get_secret("administrator", "password").strip() if password and log_found: self.log("[CONFIG] Loaded DB master password from 1Password knoey vault.") return password except Exception: return "" def _persist_db_password_to_1password(self, password: str) -> None: if not password: raise RuntimeError("Cannot persist an empty database master password.") try: from knoe.core.onepassword import set_secret, op_available if not op_available(): self.log("[WARN] 1Password CLI not available; skipping password persistence.") return set_secret("administrator", "password", password) self.log("[CONFIG] Persisted DB master password to 1Password knoey vault.") except Exception as e: raise RuntimeError(f"Failed to persist database master password to 1Password: {e}") from e def run(self) -> int: _configure_unbuffered_io() self.log(f"[CONFIG] Using {self.cfg_path}") try: # Merge caller-provided inputs with defaults and config-file values. # Caller inputs (self.inputs) take precedence to support programmatic overrides # in automated pipelines (like knoe.deploy_pipeline). self.inputs = { **self._default_inputs(), **self._load_inputs_from_cfg(), **self.inputs } # Silent mode: ensure we have a master password source before proceeding. # If the value is an OpenBao reference or an encrypted `${KNOE_SECRET:...}` # placeholder, allow downstream secret-management steps to resolve it. # Only prompt/generate when the value is truly missing. db_pw = (self._get_input("init_password.db_password", "") or "").strip() env_db_pw = ( os.environ.get("KNOE_DB_PASSWORD") or os.environ.get("DB_PASSWORD") or "" ).strip() # If runtime env provides a password, reconcile with 1Password for # consistency and bootstrap when missing. if db_pw and env_db_pw and db_pw == env_db_pw: vault_pw = self._load_db_password_from_1password(log_found=False) if vault_pw and vault_pw != db_pw: self.log( "[WARN] Runtime DB password differs from 1Password value; " "using 1Password value for consistency." ) self.inputs["init_password.db_password"] = vault_pw self.inputs["init_password.db_password_confirm"] = vault_pw db_pw = vault_pw elif not vault_pw: self._persist_db_password_to_1password(db_pw) self.log( "[CONFIG] Bootstrapped 1Password administrator from KNOE_DB_PASSWORD/DB_PASSWORD." ) if not db_pw: vault_pw = self._load_db_password_from_1password() if vault_pw: self.inputs["init_password.db_password"] = vault_pw self.inputs["init_password.db_password_confirm"] = vault_pw db_pw = vault_pw needs_prompt = not bool(db_pw) if needs_prompt: # In silent installs, never block on an interactive prompt in environments # that cannot provide input (CI/pytest/non-interactive runners). stdin_is_tty = getattr(sys.__stdin__, "isatty", lambda: False)() stdout_is_tty = getattr(sys.__stdout__, "isatty", lambda: False)() running_under_pytest = "PYTEST_CURRENT_TEST" in os.environ running_in_ci = bool(os.environ.get("CI")) allow_prompt = ( stdin_is_tty and stdout_is_tty and not running_under_pytest and not running_in_ci ) if allow_prompt: new_pw = self._prompt_for_master_password() self._persist_db_password_to_1password(new_pw) self.log( "[CONFIG] Master password captured interactively and saved to 1Password." ) else: raise RuntimeError( "Database master password is missing and prompting is unavailable. " "Run once in an interactive terminal to bootstrap the 1Password knoey vault, " "or provide KNOE_DB_PASSWORD/DB_PASSWORD." ) self.inputs["init_password.db_password"] = new_pw self.inputs["init_password.db_password_confirm"] = new_pw # Save immediately to knoe.cfg so subsequent steps/scripts see it. self._write_cfg() # Synchronize controller state self.controller.state.inputs = self.inputs self.controller.state.config_data = self.knoe_cfg_data # Delegate config writing to the installer (allows milestones to persist changes) self.controller.write_config = self._write_cfg except Exception as e: self.err(f"[FATAL] {e}") self._close_log_file() return 2 self._write_cfg() self._perform_cluster_reset() milestones = [ DependenciesMilestone(), NetworkScanMilestone(), EnvSetupMilestone(), SecretManagementMilestone(), DockerBuildMilestone(), DatabaseCreationMilestone(), ClusterLifecycleMilestone(), InitializationScriptsMilestone(), KerberosMilestone(), GitOpsMilestone(), SupabaseImagePreloadMilestone(), SupabaseMilestone(), DeploymentMilestone(), SecurityMilestone(), ] try: self.controller.run_milestones( milestones, progress_callback=lambda msg, p: self.log(f"[{p*100:0.0f}%] {msg}"), ) # Sync back results self.knoe_cfg_data = self.controller.state.config_data self.knoe_cfg_data["Install"]["STATUS"] = "Finished" self._write_cfg() self.log("[DONE] Silent install completed.") self._close_log_file() return 0 except Exception as e: self.err(f"[FATAL] {e}") try: self.knoe_cfg_data["Install"]["STATUS"] = "Failed" self._write_cfg() except Exception: pass self._close_log_file() return 2 # ------------------------------------------------------------------ # Repair / update paths # ------------------------------------------------------------------ def _validate_cluster_state(self, env: dict) -> dict[str, list[str]]: """Compare configured intent with live cluster resources. Returns a dict of anomaly category → list of description strings. Only kubectl is used so the check is non-destructive. """ anomalies: dict[str, list[str]] = {} try: supabase_ns = (os.environ.get("SUPABASE_NAMESPACE") or "").strip() or "supabase" kube_env = {k: v for k, v in env.items()} def _kubectl(*args: str) -> subprocess.CompletedProcess: return subprocess.run( ["kubectl", *args], capture_output=True, text=True, env=kube_env, timeout=30, ) # --- Supabase --- if self._get_input_bool("init_cluster.supabase_enabled", False): studio_enabled = self._get_input_bool("init_cluster.supabase_studio_enabled", False) r = _kubectl("get", "deployment", "supabase-studio", "-n", supabase_ns, "--ignore-not-found") studio_exists = bool(r.stdout.strip()) if studio_enabled and not studio_exists: anomalies.setdefault("supabase_studio", []).append("deployment missing") if studio_enabled and studio_exists: r2 = _kubectl( "get", "deployment", "supabase-studio", "-n", supabase_ns, "-o", "jsonpath={.status.readyReplicas}", ) ready = (r2.stdout.strip() or "0") if ready == "0": anomalies.setdefault("supabase_studio", []).append("deployment not ready") # Per-component checks for component, flag in ( ("supabase-auth", "supabase_auth_enabled"), ("supabase-realtime", "supabase_realtime_enabled"), ("supabase-meta", "supabase_meta_enabled"), ("supabase-analytics", "supabase_analytics_enabled"), ): if self._get_input_bool(f"init_cluster.{flag}", True): rc = _kubectl("get", "deployment", component, "-n", supabase_ns, "--ignore-not-found") if not rc.stdout.strip(): anomalies.setdefault("supabase_components", []).append(f"{component} deployment missing") # --- Core dashboard (Kong) --- if self._dashboard_kong_missing(env): anomalies.setdefault("dashboard", []).append("kong missing") # --- Authority context --- if self._authority_context_missing(): anomalies.setdefault("authority", []).append("context missing") except Exception as exc: self.err(f"[WARN] Cluster state validation error (non-fatal): {exc}") return anomalies def _reconcile_supabase_studio(self, env: dict) -> None: """Ensure Studio deployment/service/ingress/TLS exist and match config. Uses ``supabase_studio_url`` as the source of truth for ingress host. Operates only on Studio resources — does not restart other Supabase components. """ default_supabase_studio_url = ( "db.0.knoe.dev" if self._deployment_mode() == "k8s" else "db.knoe.org" ) studio_url = ( self._get_input("init_cluster.supabase_studio_url", default_supabase_studio_url).strip() or default_supabase_studio_url ) supabase_ns = (os.environ.get("SUPABASE_NAMESPACE") or "").strip() or "supabase" self.log(f"[REPAIR] Reconciling Supabase Studio → ingress host: {studio_url}") project_root_str = str(self.controller.project_root) cfg_path_str = str(self.cfg_path) if (self.cfg_path and self.cfg_path.exists()) else "" db_ns = ( self._get_input("init_password.db_namespace", "").strip() or (env.get("KNOE_DB_NAMESPACE") or "").strip() or "knoe-db" ) deploy_env = { **env, "SUPABASE_STUDIO_URL": studio_url, "SUPABASE_STUDIO_ENABLED": "true", "PROJECT_ROOT": project_root_str, "KNOE_HOME": project_root_str, "KNOE_DB_NAMESPACE": db_ns, "DATABASE_NAMESPACE": db_ns, **({"PROLE_CFG_PATH": cfg_path_str} if cfg_path_str else {}), } reconcile_script = self.controller.project_root / "etc" / "supabase_reconcile_studio.sh" if reconcile_script.exists(): self.controller.run_script( "supabase_reconcile_studio.sh", args=["--namespace", supabase_ns, "--studio-url", studio_url], env=deploy_env, ) else: # Dedicated reconcile script not present — fall back to the full # deploy.sh pipeline (render → helm upgrade --install) with Studio # env vars forced, mirroring _step_supabase. self.log("[REPAIR] supabase_reconcile_studio.sh not found; falling back to deploy.sh") deploy_sh = self.controller.project_root / "supabase" / "deploy.sh" if not deploy_sh.exists(): self.err(f"[ERROR] supabase/deploy.sh not found: {deploy_sh}") return cluster_env = _normalize_cluster_env( self._get_input("init_cluster.cluster_env", "dev") ) if cluster_env in ("service", "prod"): mode = "k8s" else: mode = "k3d" deploy_args = ["--mode", mode] if self.cfg_path and self.cfg_path.exists(): deploy_args.extend(["-c", str(self.cfg_path)]) self.log(f"[REPAIR] supabase/deploy.sh {' '.join(deploy_args)}") rc = self._run_cmd(["bash", str(deploy_sh)] + deploy_args, env=deploy_env) if rc != 0: self.err(f"[ERROR] Supabase deploy.sh failed (code {rc})") def _reconcile_supabase(self, env: dict) -> None: """Reconcile all enabled Supabase components toward configured intent. Honours per-component enable flags; avoids namespace-wide restarts. """ supabase_enabled = self._get_input_bool("init_cluster.supabase_enabled", False) if not supabase_enabled: self.log("[REPAIR] Supabase disabled in config — skipping Supabase reconcile.") return studio_enabled = self._get_input_bool("init_cluster.supabase_studio_enabled", False) if studio_enabled: self._reconcile_supabase_studio(env) else: self.log("[REPAIR] supabase_studio_enabled=false — skipping Studio reconcile.") component_flags = { "supabase_auth_enabled": "auth", "supabase_realtime_enabled": "realtime", "supabase_meta_enabled": "meta", "supabase_analytics_enabled": "analytics", } for flag, component in component_flags.items(): if self._get_input_bool(f"init_cluster.{flag}", True): self.log(f"[REPAIR] Verified Supabase component enabled in config: {component}") else: self.log(f"[REPAIR] {component} disabled in config — skipping.") def run_repair(self) -> int: """Validate current configuration against live cluster state and fix drift. Prefers patch/apply over delete/recreate. Does not restart healthy unrelated components. Safe to run repeatedly (idempotent). """ self.log("[REPAIR] Starting repair run…") try: self.inputs = { **self._default_inputs(), **self._load_inputs_from_cfg(), **self.inputs } self.controller.state.inputs = self.inputs self.controller.state.config_data = self.knoe_cfg_data # Delegate config writing to the installer (allows milestones to persist changes) self.controller.write_config = self._write_cfg except Exception as exc: self.err(f"[FATAL] Could not load config: {exc}") self._close_log_file() return 2 service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) env["SERVICE_NAMESPACE"] = service_ns self.log("[REPAIR] Validating cluster state…") anomalies = self._validate_cluster_state(env) if not anomalies: self.log("[REPAIR] No drift detected — cluster matches configured intent.") self._close_log_file() return 0 for category, details in anomalies.items(): for detail in details: self.log(f"[REPAIR] Anomaly [{category}]: {detail}") try: self._repair_stale_released_pvs(env) except Exception as exc: self.err(f"[WARN] PV repair error (non-fatal): {exc}") if "supabase_studio" in anomalies or "supabase_components" in anomalies: self._reconcile_supabase(env) core_anomalies = [k for k in anomalies if k not in ("supabase_studio", "supabase_components")] if core_anomalies: try: self._run_cluster_repair_pipeline(env=env, anomalies=core_anomalies) except Exception as exc: self.err(f"[WARN] Cluster repair pipeline error (non-fatal): {exc}") self.log("[REPAIR] Repair run complete.") self._close_log_file() return 0 def run_update(self) -> int: """Run repair-style reconciliation then apply any newly changed config intent. Only rolls components whose effective config changed since last write. Does not perform a full reinstall. """ self.log("[UPDATE] Starting update run (repair + config delta apply)…") rc = self.run_repair() if rc != 0: self.err("[UPDATE] Repair phase failed; aborting update.") return rc self.log("[UPDATE] Applying config changes…") try: self.inputs = { **self._default_inputs(), **self._load_inputs_from_cfg(), **self.inputs } self.controller.state.inputs = self.inputs self.controller.state.config_data = self.knoe_cfg_data # Delegate config writing to the installer (allows milestones to persist changes) self.controller.write_config = self._write_cfg except Exception as exc: self.err(f"[FATAL] Could not reload config for update: {exc}") self._close_log_file() return 2 service_ns = self._service_namespace() env = self._script_env_for_namespace(service_ns) if self._get_input_bool("init_db_build.run_build", DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True)): self.log("[UPDATE] Running DB image build…") self._step_db_build() if self._get_input_bool("init_cnpg_deploy.run_deploy", DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True)): configured_force_rollout = self._get_input_bool( "init_cnpg_deploy.force_rollout", DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False), ) self.log( "[UPDATE] Running CNPG deploy " f"(configured_force_rollout={configured_force_rollout}, effective_force_rollout=True)…" ) original_force_rollout = self.inputs.get("init_cnpg_deploy.force_rollout") self.inputs["init_cnpg_deploy.force_rollout"] = "true" try: self._step_cnpg_deploy() finally: if original_force_rollout is None: self.inputs.pop("init_cnpg_deploy.force_rollout", None) else: self.inputs["init_cnpg_deploy.force_rollout"] = original_force_rollout if self._get_input_bool("init_cluster.supabase_enabled", False): self._reconcile_supabase(env) self._write_cfg() self.log("[UPDATE] Update run complete.") self._close_log_file() return 0 def _run_silent_install_test(project_root: Path, cfg_path: Path) -> int: script = project_root / "tests" / "silent_install_test.sh" if not script.exists(): print( f"[ERROR] Silent install test script not found: {script}", file=sys.stderr ) return 1 env = os.environ.copy() env.setdefault("SILENT_INSTALL_LOG", "true") res = subprocess.run(["bash", str(script), str(cfg_path)], env=env) return res.returncode def _attempt_k3s_repair( controller: KnoeController, namespace: str, server: str, token: str, db_password: str, ) -> None: env = os.environ.copy() env["KNOE_HOME"] = str(controller.project_root) env["KNOE_SERVICE"] = str(controller.project_root) env["DATABASE_NAMESPACE"] = namespace env["CLUSTER_NAME"] = (os.environ.get("CLUSTER_NAME") or "knoe-db").strip() or "knoe-db" # Transitional compatibility for scripts still reading NAMESPACE. env["NAMESPACE"] = namespace env["KNOE_MODE"] = "k3s" if db_password: env["DB_PASSWORD"] = db_password env["OPENTOFU_ADMIN_PASSWORD"] = db_password if server and token and not env.get("KUBECONFIG"): repo_kubeconfig = controller.project_root / "knoe-k3s.kubeconfig" if repo_kubeconfig.exists(): env["KUBECONFIG"] = str(repo_kubeconfig) elif _looks_like_k8s_bearer_token(token): kubeconfig_path = _write_k3s_kubeconfig(server, token) env["KUBECONFIG"] = str(kubeconfig_path) openbao_ops.update( namespace=namespace, env=env, mode="k3s", project_root=controller.project_root, ) opentofu_ops.update( namespace=namespace, env=env, mode="k3s", project_root=controller.project_root, ) def _reset_k3s_namespace( project_root: Path, namespace: str | Sequence[str], server: str, token: str, *, registry_namespace: str | None = None, clear_node_reservations: bool = False, delete_pvcs_namespaces: Sequence[str] | None = None, force_delete_bound_pvs: bool = False, ) -> None: script = project_root / "scripts" / "reset-ns.sh" if not script.exists(): print(f"[WARN] Namespace reset script not found: {script}", file=sys.stderr) return if isinstance(namespace, str): raw_namespaces = [namespace] else: raw_namespaces = list(namespace or []) namespaces: list[str] = [] for ns in raw_namespaces: item = str(ns or "").strip() if not item: continue if item not in namespaces: namespaces.append(item) if not namespaces: return env = os.environ.copy() if not env.get("KUBECONFIG"): repo_kubeconfig = project_root / "knoe-k3s.kubeconfig" if repo_kubeconfig.exists(): env["KUBECONFIG"] = str(repo_kubeconfig) elif server and token and _looks_like_k8s_bearer_token(token): kubeconfig_path = _write_k3s_kubeconfig(server, token) env["KUBECONFIG"] = str(kubeconfig_path) reg_ns = (registry_namespace or "").strip() delete_pvcs_set = { str(ns or "").strip() for ns in (delete_pvcs_namespaces or []) if str(ns or "").strip() } def _collect_bound_pvs(target_ns: str) -> list[str]: res = subprocess.run( [ "kubectl", "-n", target_ns, "get", "pvc", "-o", "jsonpath={range .items[*]}{.spec.volumeName}{\"\\n\"}{end}", ], env=env, capture_output=True, text=True, ) if res.returncode != 0: print( f"[WARN] Could not list PVC-bound PVs in {target_ns} (code {res.returncode}).", file=sys.stderr, ) return [] pvs: list[str] = [] for line in (res.stdout or "").splitlines(): pv = (line or "").strip() if pv and pv not in pvs: pvs.append(pv) return pvs def _set_reclaim_policy_delete(pvs: Sequence[str]) -> None: for pv in pvs: patch = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' res = subprocess.run( ["kubectl", "patch", "pv", pv, "--type=merge", "-p", patch], env=env, capture_output=True, text=True, ) if res.returncode != 0: print( f"[WARN] Failed to set reclaimPolicy=Delete for pv/{pv} (code {res.returncode}).", file=sys.stderr, ) def _delete_pvs(pvs: Sequence[str]) -> None: for pv in pvs: res = subprocess.run( ["kubectl", "delete", "pv", pv, "--ignore-not-found", "--wait=false"], env=env, capture_output=True, text=True, ) if res.returncode != 0: print( f"[WARN] Failed to delete pv/{pv} (code {res.returncode}).", file=sys.stderr, ) for idx, ns in enumerate(namespaces): delete_pvcs = ns in delete_pvcs_set bound_pvs: list[str] = [] if delete_pvcs and force_delete_bound_pvs: bound_pvs = _collect_bound_pvs(ns) if bound_pvs: _set_reclaim_policy_delete(bound_pvs) cmd = ["bash", str(script), "-n", ns] if reg_ns and ns == reg_ns: cmd.append("--keep-registry") if delete_pvcs: cmd.append("--delete-pvcs") if clear_node_reservations and idx == (len(namespaces) - 1): cmd.append("--clear-node-reservations") res = subprocess.run(cmd, env=env) if res.returncode != 0: print( f"[WARN] Namespace reset failed for {ns} (code {res.returncode})", file=sys.stderr, ) continue if delete_pvcs and force_delete_bound_pvs and bound_pvs: _delete_pvs(bound_pvs) def _prepare_k3s_pipeline( controller: KnoeController, log_fn=None, skip_validation: bool = False ) -> int: """Prepare OpenTofu k3s pipeline: stage manifests, tfvars, ArgoCD apps. When *skip_validation* is True the silent-install-test / repair loop is skipped entirely — the caller (e.g. a successful silent install) already validated the environment. The main knoe.cfg is **not** overwritten in this case so it keeps whatever cluster_env the install used. """ def _log(msg): if log_fn: try: log_fn(msg) except: print(msg, end="", flush=True) else: print(msg, end="", flush=True) project_root = controller.project_root conf_dir = knoe_conf_mgr.resolve_knoe_conf_dir(project_root) cfg_path = knoe_conf_mgr.entrypoint_path(conf_dir) _log("==> Preparing k3s pipeline\n") _log(f" Project root: {project_root}\n") _log(f" Config: {cfg_path}\n\n") _log("==> Detecting Ansible topology...\n") info = _detect_ansible_topology(project_root) k3s_server = (info.get("k3s_server_url") or "").strip() if info else "" k3s_token = (info.get("k3s_token") or "").strip() if info else "" if k3s_server and not k3s_server.startswith("http"): k3s_server = f"https://{k3s_server}" _log(f" k3s_server: {k3s_server or '(not detected)'}\n") _log(f" k3s_token: {'detected' if k3s_token else '(not detected)'}\n\n") # Load config to resolve namespace and secrets for pipeline tfvars _log("==> Loading knoe config...\n") knoe = KnoeConsoleInstaller(controller, str(cfg_path)) try: existing_inputs = knoe._load_inputs_from_cfg() except Exception: existing_inputs = {} knoe.inputs = {**knoe._default_inputs(), **existing_inputs} if k3s_server and not knoe.inputs.get("init_cluster.k3s_server_url"): knoe.inputs["init_cluster.k3s_server_url"] = k3s_server if k3s_token and not knoe.inputs.get("init_cluster.k3s_token"): knoe.inputs["init_cluster.k3s_token"] = k3s_token namespace = (knoe._get_input("init_password.db_namespace", "") or "").strip() if not namespace: namespace = ( knoe._get_input("env_setup.DATABASE_NAMESPACE", "") or "" ).strip() if not namespace: namespace = ( knoe._get_input("env_setup.NAMESPACE", "") or "" ).strip() or "default" _log(f" Target namespace: {namespace}\n") _log(" [OK] Config loaded\n\n") if not skip_validation: # Standalone mode: write a pipeline-specific cfg and validate knoe.inputs["init_cluster.cluster_env"] = "knoe-service-cluster" # supabase_enabled is intentionally NOT overridden here — preserve the cfg value knoe.inputs["init_cluster.kerberos_enabled"] = _bool_str(False) knoe.inputs["init_cluster.at_rest_encryption_enabled"] = _bool_str(True) knoe.inputs["kerberos_config.enabled"] = _bool_str(False) knoe.inputs["kerberos_config.test_connection"] = _bool_str(False) knoe.inputs["env_setup.KNOE_HOME"] = str(project_root) knoe.inputs["env_setup.KNOE_CONF"] = str(project_root / "conf") knoe.inputs["env_setup.PROLE_DATA"] = knoe._resolve_env_value( "PROLE_DATA", str(resolve_knoe_home(env={}) / "data") ) or str(resolve_knoe_home(env={}) / "data") knoe.inputs["env_setup.PROLE_LOGS"] = knoe._resolve_env_value( "PROLE_LOGS", str(resolve_knoe_home(env={}) / "logs") ) or str(resolve_knoe_home(env={}) / "logs") knoe.inputs["env_setup.KNOE_SERVICE"] = str(project_root / "etc") # Ensure required keys that the silent install test validates if not knoe.inputs.get("env_setup.DATABASE_NAMESPACE"): knoe.inputs["env_setup.DATABASE_NAMESPACE"] = ( knoe.inputs.get("init_password.db_namespace", "") or "default" ) if not knoe.inputs.get("init_password.db_namespace"): knoe.inputs["init_password.db_namespace"] = ( knoe.inputs.get("env_setup.DATABASE_NAMESPACE", "") or knoe.inputs.get("env_setup.NAMESPACE", "") or "default" ) if not knoe.inputs.get("kerberos_config.init_authority"): knoe.inputs["kerberos_config.init_authority"] = _bool_str(False) if not knoe.inputs.get("init_password.db_host_port"): knoe.inputs["init_password.db_host_port"] = "5432" _log(" [OK] Config overrides applied for k3s pipeline\n") _log("==> Writing knoe.cfg...\n") knoe._write_cfg() _log(f" [OK] {cfg_path}\n\n") db_password = knoe._get_input("init_password.db_password", "").strip() _log(f"==> Target namespace: {namespace}\n\n") attempt = 1 total_attempts = 0 max_attempts_env = os.environ.get("SILENT_TEST_MAX_ATTEMPTS", "").strip() max_attempts = int(max_attempts_env) if max_attempts_env.isdigit() else 0 _log("==> Running silent install validation...\n") while True: total_attempts += 1 _log(f" Attempt {total_attempts}...\n") rc = _run_silent_install_test(project_root, cfg_path) if rc == 0: _log(f" [OK] Silent install test passed\n\n") break _log(f" [WARN] Silent install test failed (code {rc})\n") cur_server = knoe.inputs.get("init_cluster.k3s_server_url", k3s_server) cur_token = knoe.inputs.get("init_cluster.k3s_token", k3s_token) if attempt == 1: _log(" Attempting k3s repair...\n") _attempt_k3s_repair( controller, namespace, cur_server, cur_token, db_password ) elif attempt == 2: _log(" Resetting k3s namespace...\n") _reset_k3s_namespace(project_root, namespace, cur_server, cur_token) else: attempt = 0 attempt += 1 if max_attempts and total_attempts >= max_attempts: _log(f" [ERROR] Max attempts ({max_attempts}) reached. Aborting.\n") return rc else: _log("==> Skipping validation (silent install already succeeded)\n\n") cur_server = knoe.inputs.get("init_cluster.k3s_server_url", k3s_server) cur_token = knoe.inputs.get("init_cluster.k3s_token", k3s_token) _sync_opentofu_pipeline(project_root, namespace, cur_server, cur_token, log_fn=_log) _log("\n==> k3s pipeline preparation complete.\n") return 0