mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
2df7c91926
conf/dev/knoe.cfg had a stale SERVICE_NAMESPACE=default written by a prior TUI session. Also guard _service_namespace() so the value 'default' is never returned — it is never a valid service namespace and signals a stale generated config. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
7791 lines
314 KiB
Python
7791 lines
314 KiB
Python
"""
|
||
Installer actions and unattended workflow helpers.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import shlex
|
||
import socket
|
||
import sys
|
||
import threading
|
||
import uuid
|
||
|
||
import logging
|
||
import secrets
|
||
import string
|
||
from knoe import knoe_conf as knoe_conf_mgr
|
||
from knoe.config import (
|
||
_expand_path,
|
||
_collect_cfg_vars,
|
||
_expand_cfg_value,
|
||
_parse_bool,
|
||
_is_openbao_ref,
|
||
_is_knoe_secret,
|
||
_decrypt_knoe_secret,
|
||
_encrypt_knoe_secret,
|
||
_write_k3s_kubeconfig,
|
||
_encrypt_cfg_secret,
|
||
_merge_kubeconfig,
|
||
)
|
||
from knoe.core.build_context import copy_build_context_dir
|
||
from knoe.core.cnpg_placement import (
|
||
load_cnpg_placement_plan,
|
||
plan_cnpg_placement,
|
||
save_cnpg_placement_plan,
|
||
)
|
||
from knoe.core.controller import KnoeController
|
||
from knoe.core.env import * # noqa: F401,F403
|
||
from knoe.core.env import (
|
||
_parse_ollama_host,
|
||
_bool_str,
|
||
_deployment_mode_from_env,
|
||
_openbao_placeholder,
|
||
_looks_like_k8s_bearer_token,
|
||
_find_kubeconfig_file,
|
||
_normalize_cluster_env,
|
||
_pf_upsert_mapping,
|
||
_build_required_port_forwards,
|
||
_local_registry_enabled,
|
||
_collect_images_from_files,
|
||
_http_ping_registry,
|
||
_registry_image_ref_exists,
|
||
_resolve_supabase_home,
|
||
_push_docker_image,
|
||
_detect_ansible_topology,
|
||
_detect_ansible_storage_mounts,
|
||
_detect_ansible_node_labels,
|
||
_deployment_target_label,
|
||
_default_opentofu_pipeline_url,
|
||
_format_ollama_host,
|
||
_host_from_url,
|
||
_render_knoe_cfg,
|
||
_k3d_knoe_data_volume_args,
|
||
_sync_opentofu_pipeline,
|
||
_read_k3s_cfg,
|
||
_safe_str,
|
||
_expand_cfg_value as _expand_env_cfg_value,
|
||
_resolve_k3s_connection as _resolve_k3s_connection_fn,
|
||
_kubectl_base_cmd_for_k3s as _kubectl_base_cmd_for_k3s_fn,
|
||
_verify_k3s_services_status,
|
||
)
|
||
from knoe.core.milestones import (
|
||
DependenciesMilestone,
|
||
NetworkScanMilestone,
|
||
EnvSetupMilestone,
|
||
SecretManagementMilestone,
|
||
DatabaseCreationMilestone,
|
||
DockerBuildMilestone,
|
||
ClusterLifecycleMilestone,
|
||
InitializationScriptsMilestone,
|
||
KerberosMilestone,
|
||
DeploymentMilestone,
|
||
SecurityMilestone,
|
||
GitOpsMilestone,
|
||
SupabaseImagePreloadMilestone,
|
||
SupabaseMilestone,
|
||
)
|
||
from knoe.core.ops import garage_store as garage_store_ops
|
||
from knoe.core.ops import monitoring as monitoring_ops
|
||
from knoe.core.ops import openbao as openbao_ops
|
||
from knoe.core.ops import opentofu as opentofu_ops
|
||
from knoe.core.ops import registry as registry_ops
|
||
from knoe.core.ops.gke_clusters import (
|
||
GkeClusterSpec,
|
||
build_kubectl_env_for_cluster,
|
||
ensure_app_cluster,
|
||
ensure_db_cluster,
|
||
get_cluster_credentials,
|
||
)
|
||
from knoe.core.ops.cloudnative_pg import (
|
||
initialize as cnpg_initialize,
|
||
deploy as cnpg_deploy,
|
||
install_barman_plugin as cnpg_install_barman_plugin,
|
||
rollout as cnpg_rollout,
|
||
)
|
||
from knoe.core.ops.storage import (
|
||
ClusterStorageSpec,
|
||
StorageProvisioningError,
|
||
provision_cluster_storage,
|
||
)
|
||
from knoe.core.policy import (
|
||
POLICY_CFG_KEY,
|
||
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES,
|
||
evaluate_optional_workloads_allowed,
|
||
)
|
||
from knoe.core.stream_exec import run_streaming_cmd
|
||
from typing import Callable, Sequence
|
||
|
||
|
||
DEFAULT_APP_CLUSTER_NAME = "knoe-dev-0"
|
||
DEFAULT_APP_CLUSTER_MODE = "standard"
|
||
DEFAULT_APP_CLUSTER_MACHINE_TYPE = "e2-standard-2"
|
||
DEFAULT_APP_CLUSTER_NODE_COUNT = 3
|
||
DEFAULT_DB_CLUSTER_NAME = "knoe-dev-cnpg-0"
|
||
DEFAULT_DB_CLUSTER_MODE = "standard"
|
||
DEFAULT_DB_CLUSTER_NODE_COUNT = 3
|
||
DEFAULT_DB_CLUSTER_MACHINE_TYPE = "e2-standard-2"
|
||
DEFAULT_DB_BOOT_DISK_TYPE = "pd-standard"
|
||
DEFAULT_DB_BOOT_DISK_SIZE_GB = 50
|
||
|
||
|
||
def _configure_unbuffered_io():
|
||
os.environ.setdefault("PYTHONUNBUFFERED", "1")
|
||
for stream in (sys.stdout, sys.stderr):
|
||
try:
|
||
stream.reconfigure(line_buffering=True, write_through=True)
|
||
except Exception:
|
||
try:
|
||
stream.flush()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _kubeconfig_cluster_info_ok(kubeconfig: Path, timeout_s: int = 8) -> bool:
|
||
try:
|
||
res = subprocess.run(
|
||
["kubectl", "--kubeconfig", str(kubeconfig), "cluster-info"],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=timeout_s,
|
||
)
|
||
return res.returncode == 0
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _select_existing_kubeconfig(candidates: Sequence[Path]) -> Path | None:
|
||
"""Return the first kubeconfig that exists and can reach/auth the cluster.
|
||
|
||
If none of the existing candidates pass `kubectl cluster-info` (e.g. cluster
|
||
not yet reachable), fall back to the first existing candidate to preserve
|
||
previous behavior.
|
||
"""
|
||
|
||
first_existing: Path | None = None
|
||
for kc in candidates:
|
||
try:
|
||
if not kc.exists():
|
||
continue
|
||
except Exception:
|
||
continue
|
||
if first_existing is None:
|
||
first_existing = kc
|
||
if _kubeconfig_cluster_info_ok(kc):
|
||
return kc
|
||
return first_existing
|
||
|
||
|
||
class KnoeInstaller:
|
||
"""Shared business logic for both silent and interactive installer modes.
|
||
|
||
Subclasses must provide a :meth:`_get_input` implementation that bridges
|
||
their respective data-access layers (``dict`` for silent, tk ``StringVar``
|
||
for the GUI). All common helpers – namespace, env, secrets, port-forwards,
|
||
script-env, logging – live here so they are defined exactly once.
|
||
"""
|
||
|
||
# Subclasses should set this to ``True`` for headless operation.
|
||
silent: bool = False
|
||
|
||
# ------------------------------------------------------------------ init
|
||
def _init_shared_state(self):
|
||
"""Initialise state common to every installer flavour."""
|
||
self.knoe_cfg_data = {
|
||
"Global": {},
|
||
"Welcome": {},
|
||
"Dependencies": {},
|
||
"Network": {},
|
||
"Storage": {},
|
||
"Port Forwards": {},
|
||
"System Environment": {},
|
||
"Monitoring": {},
|
||
"Kerberos Authentication": {},
|
||
"Ollama": {},
|
||
"Optional Features": {},
|
||
"Database Options": {},
|
||
"Database Creation": {},
|
||
"Docker Build": {},
|
||
"Initialize Cluster": {},
|
||
"Initialization Scripts": {},
|
||
"GitOps": {},
|
||
"Deployment": {},
|
||
"Dev Cluster (k3d)": {},
|
||
"Service Cluster (k3s)": {},
|
||
"Prod Cluster (k8s)": {},
|
||
"Install": {},
|
||
}
|
||
self._cfg_secret_cache: dict = {}
|
||
self._secrets_finalized: bool = False
|
||
self._managed_kubeconfig: str | None = None
|
||
self._repair_ran: bool = False
|
||
self._optional_workloads_policy_cache: tuple[bool, int, str] | None = None
|
||
|
||
def _optional_workloads_policy(self, env: dict) -> tuple[bool, int, str]:
|
||
"""Evaluate and cache the optional-workloads policy.
|
||
|
||
Central rule:
|
||
`optional_workloads_allowed = ready_schedulable_nodes >= min_required`
|
||
|
||
`min_required` is loaded from `knoe.cfg` Global key
|
||
`OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES`.
|
||
"""
|
||
|
||
if self._optional_workloads_policy_cache is not None:
|
||
return self._optional_workloads_policy_cache
|
||
|
||
raw = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get(POLICY_CFG_KEY, "")
|
||
)
|
||
try:
|
||
min_nodes = int(str(raw).strip() or str(OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES))
|
||
except Exception:
|
||
min_nodes = OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES
|
||
|
||
allowed, ready_count, reason = evaluate_optional_workloads_allowed(
|
||
env=env, min_nodes=min_nodes
|
||
)
|
||
self._optional_workloads_policy_cache = (allowed, ready_count, reason)
|
||
self.log(f"[POLICY] {reason}")
|
||
return self._optional_workloads_policy_cache
|
||
|
||
# --------------------------------------------------------- data access
|
||
def _get_input(self, key: str, default: str | None = None) -> str:
|
||
"""Return user input for *key*. Subclasses **must** override."""
|
||
raise NotImplementedError
|
||
|
||
def _get_input_bool(self, key: str, default: bool = False) -> bool:
|
||
return _parse_bool(self._get_input(key, None), default=default)
|
||
|
||
# --------------------------------------------------------- logging
|
||
def log(self, msg: str):
|
||
print(msg, flush=True)
|
||
logging.info(msg)
|
||
|
||
def err(self, msg: str):
|
||
print(msg, file=sys.stderr, flush=True)
|
||
logging.error(msg)
|
||
|
||
# ------------------------------------------------ deployment helpers
|
||
def _deployment_mode(self) -> str:
|
||
return _deployment_mode_from_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
|
||
def _secret_namespace(self) -> str:
|
||
ns = (self._get_input("init_password.db_namespace", "") or "").strip()
|
||
if not ns:
|
||
ns = (self._get_input("env_setup.DATABASE_NAMESPACE", "") or "").strip()
|
||
if not ns:
|
||
ns = (self._get_input("env_setup.NAMESPACE", "") or "").strip()
|
||
if not ns:
|
||
ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get(
|
||
"DATABASE_NAMESPACE", ""
|
||
)
|
||
or ""
|
||
).strip()
|
||
if not ns:
|
||
ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "")
|
||
or ""
|
||
).strip()
|
||
if ns.startswith("${") and ns.endswith("}"):
|
||
ns = ""
|
||
if ns:
|
||
return ns
|
||
try:
|
||
if self._deployment_mode() == "k8s":
|
||
return "knoe-db-0"
|
||
except Exception:
|
||
pass
|
||
return "default"
|
||
|
||
def _service_namespace(self) -> str:
|
||
ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {})
|
||
.get("SERVICE_NAMESPACE", "")
|
||
.strip()
|
||
)
|
||
if ns.startswith("${") and ns.endswith("}"):
|
||
ns = ""
|
||
if ns == "default":
|
||
ns = ""
|
||
if not ns:
|
||
ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip()
|
||
if ns == "default":
|
||
ns = ""
|
||
|
||
if ns:
|
||
return ns
|
||
|
||
# In k3s service clusters, core services live in knoe-system by default.
|
||
try:
|
||
if self._deployment_mode() == "k3s":
|
||
return "knoe-system"
|
||
except Exception:
|
||
pass
|
||
|
||
return "knoe-system"
|
||
|
||
# ------------------------------------------ k3s connection helpers
|
||
def _read_k3s_cfg_values(
|
||
self, cfg_path: "Path | None" = None
|
||
) -> tuple[str, str, str]:
|
||
"""Return (cluster_env, server_url, token) from knoe.cfg.
|
||
|
||
Delegates to the module-level ``_read_k3s_cfg`` so that every
|
||
presentation layer (Tk, ncurses, silent) uses the same logic.
|
||
"""
|
||
if cfg_path is None:
|
||
cfg_path = getattr(self, "cfg_path", None)
|
||
return _read_k3s_cfg(cfg_path)
|
||
|
||
def _resolve_k3s_connection(self) -> tuple[str, str]:
|
||
"""Resolve k3s server URL and token from all sources.
|
||
|
||
Delegates to the module-level ``_resolve_k3s_connection`` so that
|
||
every presentation layer uses identical resolution order.
|
||
"""
|
||
cfg_path = getattr(self, "cfg_path", None)
|
||
root = getattr(self, "project_root", None) or PROJECT_ROOT
|
||
topology = getattr(self, "ansible_topology", None)
|
||
return _resolve_k3s_connection_fn(
|
||
project_root=root,
|
||
cfg_path=cfg_path,
|
||
ansible_topology=topology,
|
||
)
|
||
|
||
def _kubectl_base_cmd_for_k3s(self) -> list[str]:
|
||
"""Build a kubectl command prefix suitable for the k3s service cluster.
|
||
|
||
Delegates to the module-level ``_kubectl_base_cmd_for_k3s``.
|
||
"""
|
||
managed = getattr(self, "_managed_kubeconfig", None)
|
||
root = getattr(self, "project_root", None) or PROJECT_ROOT
|
||
return _kubectl_base_cmd_for_k3s_fn(
|
||
managed_kubeconfig=managed,
|
||
project_root=root,
|
||
)
|
||
|
||
def _kubectl_base_cmd(self, mode: str | None = None, role: str = "app") -> list[str]:
|
||
"""Build a consistent kubectl command based on the current cluster environment.
|
||
|
||
Prefers environment-specific base commands (k3s/k8s) or KUBECONFIG,
|
||
and ensures the selected context is used if provided.
|
||
"""
|
||
if not mode:
|
||
mode = self._deployment_mode()
|
||
|
||
# 1. Start with environment-specific base command
|
||
if mode == "k3s":
|
||
cmd = self._kubectl_base_cmd_for_k3s()
|
||
elif mode == "k8s":
|
||
kubeconfig = _find_kubeconfig_file()
|
||
if kubeconfig:
|
||
cmd = ["kubectl", "--kubeconfig", kubeconfig]
|
||
else:
|
||
cmd = ["kubectl"]
|
||
elif mode == "k3d":
|
||
# For k3d, check if we have a managed or merged config
|
||
kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip()
|
||
if kubeconfig_env:
|
||
cmd = ["kubectl", "--kubeconfig", kubeconfig_env]
|
||
else:
|
||
cmd = ["kubectl"]
|
||
else:
|
||
cmd = ["kubectl"]
|
||
|
||
# 2. Add context explicitly (and never rely on ambient current-context)
|
||
if mode == "k8s":
|
||
# In split-cluster GKE mode default kubectl access must target APP cluster
|
||
# unless a step uses an explicit role-specific env override.
|
||
ctx = self._cluster_kubecontext(role).strip()
|
||
else:
|
||
ctx = self._get_input("init_cluster.selected_kubectx", "").strip()
|
||
if ctx and cmd[0] == "kubectl" and "--server" not in cmd and "--context" not in cmd:
|
||
cmd.extend(["--context", ctx])
|
||
return cmd
|
||
|
||
def ensure_db_k8s_secrets(
|
||
self,
|
||
namespace: str,
|
||
db_password: str,
|
||
*,
|
||
log_fn=None,
|
||
) -> None:
|
||
"""Ensure DB/CNPG secrets exist in Kubernetes for *namespace*.
|
||
|
||
`etc/init_cloudnative_pg.sh` expects the following secrets:
|
||
- `knoe-db-user` with `username` and `password`
|
||
- `knoe-db-superuser` with `username` and `password`
|
||
- `cnpg-admin-key` with `admin.key` and `admin.pub`
|
||
|
||
This uses the provided DB password (user-entered) and a locally generated
|
||
ssh keypair (`~/.ssh/id_knoe_ed25519`) to bootstrap the admin key files
|
||
in `KNOE_SERVICE/secrets/`.
|
||
|
||
Notes:
|
||
- `knoe-db-user` and `knoe-db-superuser` are always applied/updated.
|
||
- `cnpg-admin-key` is created only if missing (no silent rotation).
|
||
- Secret manifests are never logged (to avoid leaking credentials).
|
||
"""
|
||
|
||
ns = (namespace or "").strip()
|
||
if not ns:
|
||
raise Exception("Database namespace cannot be empty.")
|
||
|
||
pw = (db_password or "").strip()
|
||
if pw.startswith("${"):
|
||
resolved = self._resolve_secret_value(pw)
|
||
if resolved and not resolved.startswith("${"):
|
||
pw = resolved
|
||
|
||
if not pw or pw.startswith("${"):
|
||
raise Exception("Database password is required to ensure Kubernetes secrets.")
|
||
|
||
log = log_fn or getattr(self, "log", None) or (lambda *_: None)
|
||
env = self._script_env_for_namespace(ns)
|
||
|
||
# Determine the base kubectl command for the DB cluster.
|
||
base_cmd = self._kubectl_base_cmd(role="db")
|
||
# Extract the context name for logging if present.
|
||
_db_ctx = ""
|
||
if "--context" in base_cmd:
|
||
idx = base_cmd.index("--context")
|
||
if idx + 1 < len(base_cmd):
|
||
_db_ctx = base_cmd[idx+1]
|
||
|
||
if _db_ctx:
|
||
log(f"Using DB cluster context: {_db_ctx}")
|
||
|
||
def _run(
|
||
cmd: list[str],
|
||
*,
|
||
input_text: str | None = None,
|
||
log_output: bool = True,
|
||
) -> subprocess.CompletedProcess:
|
||
# Replace 'kubectl' with the role-specific base command if applicable.
|
||
final_cmd = list(cmd)
|
||
if final_cmd and final_cmd[0] == "kubectl":
|
||
final_cmd = base_cmd + final_cmd[1:]
|
||
|
||
res = subprocess.run(
|
||
final_cmd,
|
||
input=input_text,
|
||
text=True,
|
||
capture_output=True,
|
||
env=env,
|
||
)
|
||
if log_output:
|
||
out = (res.stdout or "").strip()
|
||
err = (res.stderr or "").strip()
|
||
if out:
|
||
log(out)
|
||
if err:
|
||
log(err)
|
||
return res
|
||
|
||
def _ensure_namespace() -> None:
|
||
chk = _run(["kubectl", "get", "namespace", ns], log_output=False)
|
||
if chk.returncode == 0:
|
||
return
|
||
rc = _run(["kubectl", "create", "namespace", ns], log_output=True)
|
||
if rc.returncode != 0:
|
||
raise Exception(f"Failed to create namespace '{ns}'")
|
||
|
||
def _apply_secret_from_create(create_cmd: list[str]) -> None:
|
||
# 1) render YAML (do not log; contains secret material)
|
||
rendered = _run(create_cmd, log_output=False)
|
||
if rendered.returncode != 0 or not (rendered.stdout or "").strip():
|
||
raise Exception("Failed to render Kubernetes Secret manifest.")
|
||
# 2) apply it (safe to log)
|
||
applied = _run(
|
||
["kubectl", "apply", "-n", ns, "-f", "-"],
|
||
input_text=rendered.stdout,
|
||
log_output=True,
|
||
)
|
||
if applied.returncode != 0:
|
||
raise Exception("Failed to apply Kubernetes Secret manifest.")
|
||
|
||
def _apply_literal_secret(name: str, data: dict[str, str]) -> None:
|
||
args = ["kubectl", "create", "secret", "generic", name, "-n", ns]
|
||
for k, v in data.items():
|
||
args.append(f"--from-literal={k}={v}")
|
||
args.extend(["--dry-run=client", "-o", "yaml"])
|
||
_apply_secret_from_create(args)
|
||
|
||
def _secret_exists(name: str) -> bool:
|
||
res = _run(["kubectl", "-n", ns, "get", "secret", name], log_output=False)
|
||
return res.returncode == 0
|
||
|
||
log(f"==> Ensuring DB secrets in namespace '{ns}'")
|
||
_ensure_namespace()
|
||
|
||
# Ensure expected admin key files exist on disk for CNPG tooling.
|
||
service_dir = Path((env.get("KNOE_SERVICE") or "").strip())
|
||
secrets_dir = service_dir / "secrets" if service_dir else None
|
||
if secrets_dir:
|
||
secrets_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
ssh_priv = Path.home() / ".ssh" / "id_knoe_ed25519"
|
||
ssh_pub = Path.home() / ".ssh" / "id_knoe_ed25519.pub"
|
||
if not (ssh_priv.exists() and ssh_pub.exists()):
|
||
# Best-effort: generate if missing.
|
||
try:
|
||
self._generate_ssh_key(self._get_input("init_password.db_username", "knoe").strip() or "knoe")
|
||
except Exception:
|
||
pass
|
||
|
||
if secrets_dir and ssh_priv.exists() and ssh_pub.exists():
|
||
# Write both generic and legacy filenames expected by scripts.
|
||
try:
|
||
admin_priv_generic = secrets_dir / "admin.key"
|
||
admin_pub_generic = secrets_dir / "admin.pub"
|
||
admin_priv_legacy = secrets_dir / "admin_ed25519"
|
||
admin_pub_legacy = secrets_dir / "admin_ed25519.pub"
|
||
|
||
if not admin_priv_generic.exists():
|
||
shutil.copyfile(ssh_priv, admin_priv_generic)
|
||
os.chmod(admin_priv_generic, 0o600)
|
||
if not admin_pub_generic.exists():
|
||
shutil.copyfile(ssh_pub, admin_pub_generic)
|
||
if not admin_priv_legacy.exists():
|
||
shutil.copyfile(ssh_priv, admin_priv_legacy)
|
||
os.chmod(admin_priv_legacy, 0o600)
|
||
if not admin_pub_legacy.exists():
|
||
shutil.copyfile(ssh_pub, admin_pub_legacy)
|
||
except Exception:
|
||
pass
|
||
|
||
# DB credentials secrets expected by CNPG init script.
|
||
_apply_literal_secret("knoe-db-user", {"username": "knoe", "password": pw})
|
||
_apply_literal_secret(
|
||
"knoe-db-superuser", {"username": "postgres", "password": pw}
|
||
)
|
||
|
||
# Admin key secret only if missing (avoid rotation).
|
||
if not _secret_exists("cnpg-admin-key"):
|
||
if not secrets_dir:
|
||
raise Exception("KNOE_SERVICE is not set; cannot locate secrets directory for CNPG admin key.")
|
||
admin_priv = secrets_dir / "admin.key"
|
||
admin_pub = secrets_dir / "admin.pub"
|
||
if not (admin_priv.exists() and admin_pub.exists()):
|
||
raise Exception(
|
||
"CNPG admin key files are missing (admin.key/admin.pub); cannot create cnpg-admin-key secret."
|
||
)
|
||
create = [
|
||
"kubectl",
|
||
"create",
|
||
"secret",
|
||
"generic",
|
||
"cnpg-admin-key",
|
||
"-n",
|
||
ns,
|
||
f"--from-file=admin.key={admin_priv}",
|
||
f"--from-file=admin.pub={admin_pub}",
|
||
"--dry-run=client",
|
||
"-o",
|
||
"yaml",
|
||
]
|
||
_apply_secret_from_create(create)
|
||
|
||
def _verify_k3s_services_core(self) -> dict[str, str]:
|
||
"""Check registry, OpenBao and OpenTofu on a remote k3s cluster.
|
||
|
||
Delegates to the module-level ``_verify_k3s_services_status``.
|
||
"""
|
||
managed = getattr(self, "_managed_kubeconfig", None)
|
||
root = getattr(self, "project_root", None) or PROJECT_ROOT
|
||
return _verify_k3s_services_status(
|
||
project_root=root,
|
||
managed_kubeconfig=managed,
|
||
)
|
||
|
||
def _deploy_k3s_services_core(self, controller=None, log_fn=None):
|
||
"""Deploy registry, OpenBao and OpenTofu to a remote k3s cluster.
|
||
|
||
*controller* must expose ``run_script(name, args, env, on_line)``.
|
||
Falls back to ``self.controller`` when not provided.
|
||
"""
|
||
controller = controller or getattr(self, "controller", None)
|
||
if controller is None:
|
||
raise RuntimeError("No controller available for script execution")
|
||
|
||
server, token = self._resolve_k3s_connection()
|
||
if not server or not token:
|
||
if log_fn:
|
||
log_fn("Missing k3s server URL or token.\n")
|
||
return
|
||
|
||
ns = getattr(self, "db_namespace", None) or "default"
|
||
if hasattr(ns, "get"):
|
||
ns = ns.get()
|
||
ns = (ns or "default").strip()
|
||
db_pw = getattr(self, "db_password", "") or ""
|
||
if hasattr(db_pw, "get"):
|
||
db_pw = db_pw.get()
|
||
|
||
env = os.environ.copy()
|
||
root = getattr(self, "project_root", None) or PROJECT_ROOT
|
||
env["KNOE_HOME"] = str(root)
|
||
env["KNOE_SERVICE"] = str(root)
|
||
env["DB_PASSWORD"] = db_pw
|
||
env["OPENTOFU_ADMIN_PASSWORD"] = db_pw
|
||
env["KNOE_MODE"] = "k3s"
|
||
|
||
# Resolve KUBECONFIG through the same path the Tk UI and silent
|
||
# installer use: prefer existing cert-based file, fall back to
|
||
# token-generated kubeconfig.
|
||
kubeconfig = _find_kubeconfig_file(env)
|
||
if not kubeconfig:
|
||
try:
|
||
kubeconfig = str(_write_k3s_kubeconfig(server, token))
|
||
except Exception:
|
||
pass
|
||
if kubeconfig:
|
||
env["KUBECONFIG"] = kubeconfig
|
||
try:
|
||
_merge_kubeconfig(kubeconfig)
|
||
except Exception:
|
||
pass
|
||
|
||
service_ns = (
|
||
env.get("SERVICE_NAMESPACE")
|
||
or (self.knoe_cfg_data.get("Global", {}) or {}).get("SERVICE_NAMESPACE", "")
|
||
or "knoe-system"
|
||
)
|
||
service_ns = str(service_ns).strip() or "knoe-system"
|
||
argocd_ns = env.get("ARGOCD_NAMESPACE") or "argocd"
|
||
registry_ns = env.get("REGISTRY_NAMESPACE") or service_ns
|
||
if str(registry_ns).strip() == "default":
|
||
registry_ns = service_ns
|
||
env["ARGOCD_NAMESPACE"] = argocd_ns
|
||
env["SERVICE_NAMESPACE"] = service_ns
|
||
env["REGISTRY_NAMESPACE"] = registry_ns
|
||
|
||
mode = _deployment_mode_from_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
try:
|
||
registry_ops.update(
|
||
namespace=registry_ns,
|
||
env=env,
|
||
project_root=root,
|
||
mode=mode,
|
||
log=log_fn,
|
||
)
|
||
openbao_ops.update(
|
||
namespace=ns,
|
||
env=env,
|
||
project_root=root,
|
||
mode=mode,
|
||
log=log_fn,
|
||
)
|
||
opentofu_ops.update(
|
||
namespace=ns,
|
||
env=env,
|
||
project_root=root,
|
||
mode=mode,
|
||
log=log_fn,
|
||
)
|
||
except Exception as e:
|
||
if log_fn:
|
||
log_fn(f"Deploy error: {e}\n")
|
||
|
||
# ------------------------------------------------- secret helpers
|
||
def _secret_cfg_value(
|
||
self, section: str, key: str, plaintext: str, leaf: str, bao_key: str
|
||
) -> str:
|
||
if self._secrets_finalized:
|
||
return _openbao_placeholder(self._secret_namespace(), leaf, bao_key)
|
||
if not plaintext:
|
||
return ""
|
||
if _is_openbao_ref(plaintext) or _is_knoe_secret(plaintext):
|
||
return plaintext
|
||
existing = (self.knoe_cfg_data.get(section, {}) or {}).get(key)
|
||
if existing and _is_knoe_secret(existing):
|
||
try:
|
||
if _decrypt_knoe_secret(existing) == plaintext:
|
||
return existing
|
||
except Exception:
|
||
pass
|
||
try:
|
||
return _encrypt_knoe_secret(plaintext)
|
||
except Exception:
|
||
return plaintext
|
||
|
||
def _resolve_openbao_ref(self, value: str) -> str:
|
||
if not _is_openbao_ref(value):
|
||
return value
|
||
inner = value[len(OPENBAO_PREFIX) : -len(OPENBAO_SUFFIX)]
|
||
path, key = (inner.split("#", 1) + [""])[:2]
|
||
if not path or not key:
|
||
return ""
|
||
mount = "kv"
|
||
secret_path = path
|
||
if "/" in path:
|
||
maybe_mount, rest = path.split("/", 1)
|
||
if maybe_mount:
|
||
mount = maybe_mount
|
||
secret_path = rest
|
||
token = os.environ.get("OPENBAO_ROOT_TOKEN", "")
|
||
if not token:
|
||
knoe_service = os.environ.get("KNOE_SERVICE")
|
||
if knoe_service:
|
||
token_path = Path(knoe_service) / "secrets" / "openbao-root-token"
|
||
if token_path.exists():
|
||
token = token_path.read_text().strip()
|
||
if not token:
|
||
return value
|
||
url = os.environ.get("PROLE_OPENBAO_URL")
|
||
if not url:
|
||
for p in ["8200", "18200"]:
|
||
try:
|
||
import urllib.request
|
||
|
||
with urllib.request.urlopen(
|
||
f"http://127.0.0.1:{p}/v1/sys/health", timeout=0.5
|
||
) as r:
|
||
if r.getcode() == 200:
|
||
url = f"http://127.0.0.1:{p}"
|
||
break
|
||
except Exception:
|
||
pass
|
||
if not url:
|
||
url = "http://127.0.0.1:8200"
|
||
url = url.rstrip("/")
|
||
try:
|
||
req = urllib.request.Request(f"{url}/v1/{mount}/data/{secret_path}")
|
||
req.add_header("X-Vault-Token", token)
|
||
with urllib.request.urlopen(req, timeout=4) as resp:
|
||
payload = json.loads(resp.read().decode("utf-8"))
|
||
return payload.get("data", {}).get("data", {}).get(key, "") or ""
|
||
except Exception:
|
||
return value
|
||
|
||
def _resolve_secretref_value(self, value: str) -> str:
|
||
raw = str(value or "").strip()
|
||
if not raw.startswith("secretref://"):
|
||
return value
|
||
|
||
ref = raw[len("secretref://") :].strip().strip("/")
|
||
if not ref:
|
||
return value
|
||
|
||
env_candidates: list[str] = []
|
||
|
||
def _add_env(name: str) -> None:
|
||
token = str(name or "").strip()
|
||
if token and token not in env_candidates:
|
||
env_candidates.append(token)
|
||
|
||
_add_env(ref)
|
||
normalized = ref.replace("-", "_").replace(".", "_").replace("/", "_")
|
||
_add_env(normalized)
|
||
_add_env(normalized.upper())
|
||
|
||
if ref == "google-oidc-client-id":
|
||
for key in (
|
||
"GITLAB_OIDC_CLIENT_ID",
|
||
"OIDC_CLIENT_ID",
|
||
"GOOGLE_OIDC_CLIENT_ID",
|
||
"GOOGLE_CLIENT_ID",
|
||
):
|
||
_add_env(key)
|
||
elif ref == "google-oidc-client-secret":
|
||
for key in (
|
||
"GITLAB_OIDC_CLIENT_SECRET",
|
||
"OIDC_CLIENT_SECRET",
|
||
"GOOGLE_OIDC_CLIENT_SECRET",
|
||
"GOOGLE_CLIENT_SECRET",
|
||
):
|
||
_add_env(key)
|
||
|
||
for env_key in env_candidates:
|
||
candidate = str(os.environ.get(env_key, "") or "").strip()
|
||
if candidate:
|
||
return candidate
|
||
|
||
file_candidates = [
|
||
Path(self.project_root) / "etc" / "secrets" / ref,
|
||
Path(self.project_root) / "secrets" / ref,
|
||
]
|
||
service_dir = str(
|
||
self._get_input("env_setup.KNOE_SERVICE", "")
|
||
or os.environ.get("KNOE_SERVICE", "")
|
||
or ""
|
||
).strip()
|
||
if service_dir:
|
||
file_candidates.insert(0, Path(service_dir) / "secrets" / ref)
|
||
|
||
for file_path in file_candidates:
|
||
try:
|
||
if file_path.exists() and file_path.is_file():
|
||
candidate = file_path.read_text(encoding="utf-8").strip()
|
||
if candidate:
|
||
return candidate
|
||
except Exception:
|
||
continue
|
||
|
||
return value
|
||
|
||
def _resolve_secret_value(self, value: str) -> str:
|
||
if _is_knoe_secret(value):
|
||
return _decrypt_knoe_secret(value)
|
||
if _is_openbao_ref(value):
|
||
return self._resolve_openbao_ref(value)
|
||
if str(value or "").strip().startswith("secretref://"):
|
||
return self._resolve_secretref_value(value)
|
||
return value
|
||
|
||
# --------------------------------------------- cfg sanitisation
|
||
def _sanitize_sections_for_cfg(self, sections: dict) -> dict:
|
||
sanitized = {k: dict(v) for k, v in sections.items()}
|
||
|
||
# Persist fully expanded values in knoe.cfg (no $VAR / ${VAR} expansions).
|
||
# Downstream tools treat these values as plain strings.
|
||
env_map = dict(os.environ)
|
||
try:
|
||
env_map.update(
|
||
{k: str(v or "") for k, v in (self._read_existing_env() or {}).items()}
|
||
)
|
||
except Exception:
|
||
pass
|
||
for k in (
|
||
"KNOE_HOME",
|
||
"KNOE_CONF",
|
||
"PROLE_DATA",
|
||
"PROLE_LOGS",
|
||
"KNOE_SERVICE",
|
||
"NAMESPACE",
|
||
"SERVICE_NAMESPACE",
|
||
):
|
||
val = (self._get_input(f"env_setup.{k}", "") or "").strip()
|
||
if val:
|
||
try:
|
||
env_map[k] = self._expand_shell_path(val, env=env_map)
|
||
except Exception:
|
||
env_map[k] = val
|
||
|
||
for section_name, section in sanitized.items():
|
||
for k, v in list(section.items()):
|
||
if not isinstance(v, str) or not v.strip():
|
||
continue
|
||
if _is_knoe_secret(v) or _is_openbao_ref(v):
|
||
continue
|
||
try:
|
||
section[k] = self._cfgify_home_path(
|
||
self._expand_shell_path(v, env=env_map)
|
||
)
|
||
except Exception:
|
||
section[k] = v
|
||
if "Kerberos Authentication" in sanitized:
|
||
val = sanitized["Kerberos Authentication"].get("PASSWORD", "")
|
||
if val or ("Kerberos Authentication", "PASSWORD") in self._cfg_secret_cache:
|
||
sanitized["Kerberos Authentication"]["PASSWORD"] = (
|
||
self._secret_cfg_value(
|
||
"Kerberos Authentication",
|
||
"PASSWORD",
|
||
val,
|
||
"kerberos",
|
||
"password",
|
||
)
|
||
)
|
||
if "Monitoring" in sanitized:
|
||
val = sanitized["Monitoring"].get("GRAFANA_ADMIN_PASSWORD", "")
|
||
if (
|
||
val
|
||
or ("Monitoring", "GRAFANA_ADMIN_PASSWORD") in self._cfg_secret_cache
|
||
):
|
||
sanitized["Monitoring"]["GRAFANA_ADMIN_PASSWORD"] = (
|
||
self._secret_cfg_value(
|
||
"Monitoring",
|
||
"GRAFANA_ADMIN_PASSWORD",
|
||
val,
|
||
"monitoring",
|
||
"grafana_admin_password",
|
||
)
|
||
)
|
||
return sanitized
|
||
|
||
# ------------------------------------------- namespace helpers
|
||
def _get_local_owner(self) -> str:
|
||
try:
|
||
return getpass.getuser()
|
||
except Exception:
|
||
try:
|
||
return os.getlogin()
|
||
except Exception:
|
||
return "knoe"
|
||
|
||
def _sanitize_namespace(self, name: str) -> str:
|
||
cleaned = re.sub(r"[^a-z0-9-]+", "-", (name or "").lower())
|
||
cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-")
|
||
if not cleaned:
|
||
cleaned = "knoe-db"
|
||
if len(cleaned) > 63:
|
||
cleaned = cleaned[:63].rstrip("-")
|
||
return cleaned
|
||
|
||
def _generate_namespace_name(self) -> str:
|
||
owner = self._sanitize_namespace(self._get_local_owner())
|
||
suffix = uuid.uuid4().hex[:6]
|
||
base = f"knoe-db-{owner}-{suffix}"
|
||
return self._sanitize_namespace(base)
|
||
|
||
def _ensure_namespace_prefix(self, name: str) -> str:
|
||
cleaned = (name or "").strip()
|
||
if not cleaned:
|
||
return "knoe-db"
|
||
return cleaned
|
||
|
||
def _read_existing_cfg_namespace(self) -> str | None:
|
||
"""Best-effort read of database namespace from the existing knoe.cfg.
|
||
|
||
Silent retries should reuse the namespace stored in knoe.cfg unless the
|
||
user explicitly overrides via env vars.
|
||
"""
|
||
|
||
cfg_path = getattr(self, "cfg_path", None)
|
||
if not cfg_path:
|
||
return None
|
||
try:
|
||
p = Path(str(cfg_path)).expanduser()
|
||
if p.is_dir():
|
||
p = p / "knoe.cfg"
|
||
if not p.exists():
|
||
return None
|
||
|
||
cfg = configparser.ConfigParser(interpolation=None)
|
||
cfg.optionxform = str
|
||
cfg.read(p)
|
||
|
||
# Current canonical location
|
||
for section, key in (
|
||
("User", "DATABASE_NAMESPACE"),
|
||
("User", "PROLE_NAMESPACE"),
|
||
("Global", "DATABASE_NAMESPACE"),
|
||
# Legacy fallbacks
|
||
("User", "NAMESPACE"),
|
||
("Database Creation", "NAMESPACE"),
|
||
("Database Creation", "DATABASE_NAMESPACE"),
|
||
("Database Creation", "DB_NAME"),
|
||
):
|
||
if cfg.has_option(section, key):
|
||
ns = (cfg.get(section, key, fallback="") or "").strip()
|
||
if ns:
|
||
return ns
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
def _initial_namespace(self) -> str:
|
||
# Explicit overrides must win.
|
||
ns = (
|
||
os.environ.get("DATABASE_NAMESPACE")
|
||
or os.environ.get("PROLE_NAMESPACE")
|
||
or os.environ.get("NAMESPACE")
|
||
)
|
||
if ns:
|
||
return self._ensure_namespace_prefix(ns)
|
||
try:
|
||
existing = self._read_existing_env()
|
||
ns = (
|
||
existing.get("DATABASE_NAMESPACE")
|
||
or existing.get("NAMESPACE")
|
||
or existing.get("PROLE_NAMESPACE")
|
||
)
|
||
if ns:
|
||
return self._ensure_namespace_prefix(ns)
|
||
except Exception:
|
||
pass
|
||
|
||
# Prefer the namespace already stored in knoe.cfg to make retries stable.
|
||
ns = self._read_existing_cfg_namespace()
|
||
if ns:
|
||
return self._ensure_namespace_prefix(ns)
|
||
return self._ensure_namespace_prefix(self._generate_namespace_name())
|
||
|
||
def _initial_db_namespace(self) -> str:
|
||
"""Initial namespace suggestion for CNPG/Postgres.
|
||
|
||
When a [CNPG Clusters] registry exists in knoe.cfg, the identity
|
||
cluster's declared namespace is used unconditionally — no version suffix
|
||
is appended. Version-aware namespace generation only applies as a
|
||
last-resort fallback when no cluster registry is configured.
|
||
"""
|
||
|
||
# Registry-declared identity cluster wins unconditionally.
|
||
identity = self._cnpg_identity_cluster()
|
||
if identity and identity.get("namespace"):
|
||
return self._sanitize_namespace(
|
||
self._ensure_namespace_prefix(identity["namespace"])
|
||
)
|
||
|
||
# Explicit environment overrides.
|
||
base = (
|
||
os.environ.get("DATABASE_NAMESPACE")
|
||
or os.environ.get("KNOE_DB_NAMESPACE")
|
||
or os.environ.get("DB_NAMESPACE")
|
||
or os.environ.get("PROLE_NAMESPACE")
|
||
or os.environ.get("NAMESPACE")
|
||
or ""
|
||
).strip()
|
||
|
||
if not base:
|
||
try:
|
||
existing = self._read_existing_env()
|
||
base = (
|
||
(existing.get("DATABASE_NAMESPACE") or "")
|
||
or (existing.get("NAMESPACE") or "")
|
||
or (existing.get("PROLE_NAMESPACE") or "")
|
||
).strip()
|
||
except Exception:
|
||
base = ""
|
||
|
||
if not base:
|
||
base = (self._read_existing_cfg_namespace() or "").strip() or "knoe-db"
|
||
|
||
return self._sanitize_namespace(self._ensure_namespace_prefix(base))
|
||
|
||
def _cnpg_cluster_registry(self) -> list[dict]:
|
||
"""Parse [CNPG Clusters] section from the active knoe.cfg.
|
||
|
||
Returns a list of cluster dicts with keys: name, namespace, identity,
|
||
instances, image. Validates that namespaces are unique and exactly one
|
||
cluster is designated as the identity cluster.
|
||
"""
|
||
section = (getattr(self, "knoe_cfg_data", None) or {}).get("CNPG Clusters", {})
|
||
if not section:
|
||
return []
|
||
clusters: dict[str, dict] = {}
|
||
for raw_key, raw_val in section.items():
|
||
if "." not in raw_key:
|
||
continue
|
||
cluster_name, _, prop = raw_key.partition(".")
|
||
cluster_name = cluster_name.strip()
|
||
clusters.setdefault(cluster_name, {"name": cluster_name})
|
||
clusters[cluster_name][prop.strip()] = (raw_val or "").strip()
|
||
result = list(clusters.values())
|
||
if not result:
|
||
return []
|
||
# Validate: namespaces must be unique
|
||
namespaces = [c.get("namespace", "") for c in result]
|
||
if len(namespaces) != len(set(ns for ns in namespaces if ns)):
|
||
raise ValueError("CNPG Clusters: duplicate namespace declared")
|
||
# Validate: exactly one identity=true
|
||
identity_count = sum(
|
||
1 for c in result if c.get("identity", "").lower() == "true"
|
||
)
|
||
if identity_count != 1:
|
||
raise ValueError(
|
||
f"CNPG Clusters: exactly one cluster must have identity=true, found {identity_count}"
|
||
)
|
||
return result
|
||
|
||
def _cnpg_identity_cluster(self) -> dict | None:
|
||
"""Return the cluster entry with identity=true, or None if no registry."""
|
||
for c in self._cnpg_cluster_registry():
|
||
if c.get("identity", "").lower() == "true":
|
||
return c
|
||
return None
|
||
|
||
def _is_valid_namespace(self, name: str) -> bool:
|
||
if not name or len(name) > 63:
|
||
return False
|
||
if "db" not in name.lower():
|
||
return False
|
||
return re.match(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", name) is not None
|
||
|
||
def _generate_db_password(self, length: int = 24) -> str:
|
||
alphabet = string.ascii_letters + string.digits
|
||
return "".join(secrets.choice(alphabet) for _ in range(length))
|
||
|
||
# --------------------------------------------------- env helpers
|
||
|
||
_FOREIGN_HOME_PREFIX_RE = re.compile(r"^/(Users|home)/([^/]+)(/.*)?$")
|
||
|
||
def _rewrite_foreign_home_prefix(self, value: str) -> str:
|
||
"""Rewrite foreign home prefixes to the current user's home.
|
||
|
||
Example: running on Linux with HOME=/home/<user>, rewrite
|
||
`/Users/<user>/dev/knoe` -> `/home/<user>/dev/knoe`.
|
||
|
||
This helps prevent stale macOS paths (often stored in `~/.knoe/env.sh`) from
|
||
being persisted into `knoe.cfg` on Linux.
|
||
"""
|
||
|
||
raw = (value or "").strip()
|
||
if not raw or "$" in raw:
|
||
return raw
|
||
|
||
m = self._FOREIGN_HOME_PREFIX_RE.match(raw)
|
||
if not m:
|
||
return raw
|
||
|
||
foreign_root = m.group(1)
|
||
foreign_user = m.group(2)
|
||
suffix = m.group(3) or ""
|
||
|
||
try:
|
||
current_home = Path.home()
|
||
except Exception:
|
||
return raw
|
||
|
||
# Only rewrite when the path refers to the current username.
|
||
if foreign_user != current_home.name:
|
||
return raw
|
||
|
||
system = platform.system()
|
||
if system != "Darwin" and foreign_root == "Users":
|
||
return str(current_home) + suffix
|
||
if system == "Darwin" and foreign_root == "home":
|
||
return str(current_home) + suffix
|
||
|
||
return raw
|
||
|
||
def _shellify_home_path(self, value: str) -> str:
|
||
"""Represent paths under the current home directory using `$HOME`.
|
||
|
||
Keeps values unchanged if they already contain shell variables.
|
||
"""
|
||
|
||
raw = (value or "").strip()
|
||
if not raw or "$" in raw:
|
||
return raw
|
||
try:
|
||
p = Path(raw).expanduser()
|
||
home = Path.home()
|
||
rel = p.relative_to(home)
|
||
except Exception:
|
||
return raw
|
||
rel_str = rel.as_posix()
|
||
if not rel_str or rel_str == ".":
|
||
return "$HOME"
|
||
return f"$HOME/{rel_str}"
|
||
|
||
def _expand_shell_path(self, value: str, env: dict[str, str] | None = None) -> str:
|
||
raw = (value or "").strip()
|
||
if not raw:
|
||
return ""
|
||
env_map = env or os.environ
|
||
expanded = _expand_env_cfg_value(raw, env=env_map, max_depth=10)
|
||
return os.path.expanduser(expanded)
|
||
|
||
def _cfgify_home_path(self, value: str) -> str:
|
||
"""Represent paths under the current home directory using `$HOME`.
|
||
|
||
Also rewrites stale foreign-home paths like `/Users/<user>/...` when the
|
||
username matches the current user.
|
||
"""
|
||
|
||
raw = (value or "").strip()
|
||
if not raw or "$" in raw:
|
||
return raw
|
||
|
||
raw = self._rewrite_foreign_home_prefix(raw)
|
||
|
||
try:
|
||
p = Path(raw).expanduser()
|
||
home = Path.home()
|
||
rel = p.relative_to(home)
|
||
except Exception:
|
||
return raw
|
||
rel_str = rel.as_posix()
|
||
if not rel_str or rel_str == ".":
|
||
return "$HOME"
|
||
return f"$HOME/{rel_str}"
|
||
|
||
def _env_defaults(self, namespace: str | None = None) -> dict:
|
||
# Default install layout: ~/.knoe
|
||
# NOTE: UI may present this as $HOME/.knoe, but internally we keep
|
||
# resolved literal paths for filesystem operations.
|
||
default_home = resolve_knoe_home(env={})
|
||
resolved_home = self._resolve_env_value("KNOE_HOME", str(default_home)) or str(
|
||
default_home
|
||
)
|
||
env_map = dict(os.environ)
|
||
env_map["KNOE_HOME"] = resolved_home
|
||
expanded_home = self._expand_shell_path(resolved_home, env=env_map) or str(default_home)
|
||
home = Path(expanded_home)
|
||
env_map["KNOE_HOME"] = str(home)
|
||
if namespace is None:
|
||
namespace = (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip()
|
||
if not namespace:
|
||
namespace = os.environ.get("DATABASE_NAMESPACE", "")
|
||
if not namespace:
|
||
namespace = os.environ.get("NAMESPACE", "")
|
||
return {
|
||
"KNOE_HOME": str(home),
|
||
"KNOE_CONF": self._expand_shell_path(
|
||
self._resolve_env_value("KNOE_CONF", str(home / "conf")) or str(home / "conf"),
|
||
env=env_map,
|
||
),
|
||
"PROLE_DATA": self._expand_shell_path(
|
||
self._resolve_env_value("PROLE_DATA", str(home / "data")) or str(home / "data"),
|
||
env=env_map,
|
||
),
|
||
"PROLE_LOGS": self._expand_shell_path(
|
||
self._resolve_env_value("PROLE_LOGS", str(home / "logs")) or str(home / "logs"),
|
||
env=env_map,
|
||
),
|
||
"KNOE_SERVICE": self._expand_shell_path(
|
||
self._resolve_env_value("KNOE_SERVICE", str(home / "etc")) or str(home / "etc"),
|
||
env=env_map,
|
||
),
|
||
"DATABASE_NAMESPACE": namespace or "",
|
||
}
|
||
|
||
def _resolve_env_value(self, key: str, fallback: str | None = None) -> str | None:
|
||
val = os.environ.get(key)
|
||
if val:
|
||
return val
|
||
try:
|
||
env = self._read_existing_env()
|
||
val = env.get(key)
|
||
if val:
|
||
return val
|
||
except Exception:
|
||
pass
|
||
return fallback
|
||
|
||
def _resolve_env_dir(self, key: str, default_suffix: str) -> Path:
|
||
input_key = f"env_setup.{key}"
|
||
val = (self._get_input(input_key, "") or "").strip()
|
||
env_map = dict(os.environ)
|
||
try:
|
||
env_map.update({k: str(v or "") for k, v in (self._read_existing_env() or {}).items()})
|
||
except Exception:
|
||
pass
|
||
if val:
|
||
try:
|
||
return Path(self._expand_shell_path(val, env=env_map))
|
||
except Exception:
|
||
pass
|
||
val = self._resolve_env_value(key)
|
||
if val:
|
||
try:
|
||
return Path(self._expand_shell_path(val, env=env_map))
|
||
except Exception:
|
||
pass
|
||
return Path.home() / "dev" / "knoe" / default_suffix
|
||
|
||
def _read_existing_env(self) -> dict:
|
||
env = {}
|
||
knoe_home = os.environ.get("KNOE_HOME")
|
||
candidates = []
|
||
if knoe_home:
|
||
try:
|
||
candidates.append(Path(self._expand_shell_path(knoe_home, env=os.environ)) / "env.sh")
|
||
except Exception:
|
||
candidates.append(Path(knoe_home) / "env.sh")
|
||
candidates.append(resolve_knoe_home(env={}) / "env.sh")
|
||
for p in candidates:
|
||
try:
|
||
if p.exists():
|
||
for line in p.read_text().splitlines():
|
||
line = line.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
if line.startswith("export "):
|
||
line = line[len("export ") :]
|
||
if "=" in line:
|
||
k, v = line.split("=", 1)
|
||
k = k.strip()
|
||
v = v.strip().strip('"')
|
||
if k in {
|
||
"KNOE_HOME",
|
||
"KNOE_CONF",
|
||
"PROLE_DATA",
|
||
"PROLE_LOGS",
|
||
"KNOE_SERVICE",
|
||
}:
|
||
v = self._rewrite_foreign_home_prefix(v)
|
||
env[k] = v
|
||
break
|
||
except Exception:
|
||
pass
|
||
return env
|
||
|
||
def _save_env_to_file(self, values: dict):
|
||
cluster_env = self._get_input("init_cluster.cluster_env", "dev")
|
||
env_key = _normalize_cluster_env(cluster_env)
|
||
|
||
env_map = dict(os.environ)
|
||
try:
|
||
env_map.update({str(k): str(v) for k, v in (values or {}).items() if v is not None})
|
||
except Exception:
|
||
pass
|
||
|
||
expanded_home = self._expand_shell_path(values.get("KNOE_HOME", ""), env=env_map)
|
||
expanded_home = os.path.expanduser(expanded_home) if expanded_home else ""
|
||
if expanded_home:
|
||
env_map["KNOE_HOME"] = expanded_home
|
||
home = Path(expanded_home)
|
||
else:
|
||
home = resolve_knoe_home(env={})
|
||
if env_key == "dev":
|
||
home.mkdir(parents=True, exist_ok=True)
|
||
for key in ("KNOE_CONF", "PROLE_DATA", "PROLE_LOGS", "KNOE_SERVICE"):
|
||
try:
|
||
raw = values.get(key, "")
|
||
expanded = self._expand_shell_path(raw, env=env_map)
|
||
expanded = os.path.expanduser(expanded) if expanded else ""
|
||
if expanded:
|
||
env_map[key] = expanded
|
||
Path(expanded).mkdir(parents=True, exist_ok=True)
|
||
except Exception:
|
||
pass
|
||
|
||
content = []
|
||
content.append("#!/usr/bin/env bash")
|
||
content.append("# Knoe environment configuration")
|
||
content.append(
|
||
"# This file is generated by the installer. Source it in new shells, or execute as a wrapper:"
|
||
)
|
||
content.append('# "$KNOE_HOME/env.sh" <command> [args…]')
|
||
content.append("# shellcheck shell=bash")
|
||
for k in (
|
||
"KNOE_HOME",
|
||
"KNOE_CONF",
|
||
"PROLE_DATA",
|
||
"PROLE_LOGS",
|
||
"KNOE_SERVICE",
|
||
):
|
||
content.append(f'export {k}="{self._shellify_home_path(values[k])}"')
|
||
# NAMESPACE/PROLE_NAMESPACE must never be written to env.sh — only knoe.cfg is the source of truth
|
||
content.append("")
|
||
content.append("# Ensure PATH works for GUI-launched shells (Docker, etc.)")
|
||
content.append(
|
||
'_knoe_add_path() { case ":${PATH}:" in *":$1:"*) ;; *) PATH="$1:${PATH:-}" ;; esac; }'
|
||
)
|
||
content.append('_knoe_add_path "$KNOE_HOME/bin"')
|
||
content.append('_knoe_add_path "/opt/homebrew/bin"')
|
||
content.append('_knoe_add_path "/usr/local/bin"')
|
||
content.append('_knoe_add_path "/usr/bin"')
|
||
content.append('_knoe_add_path "/bin"')
|
||
content.append('_knoe_add_path "/usr/sbin"')
|
||
content.append('_knoe_add_path "/sbin"')
|
||
content.append("export PATH")
|
||
content.append("")
|
||
content.append("# Add custom paths below if needed (examples):")
|
||
content.append("")
|
||
content.append(
|
||
"# If executed with arguments (and not sourced), run them under this environment"
|
||
)
|
||
content.append('if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [ "$#" -gt 0 ]; then')
|
||
content.append(' exec "$@"')
|
||
content.append("fi")
|
||
|
||
new_content = "\n".join(content) + "\n"
|
||
out = home / "env.sh"
|
||
if env_key != "dev":
|
||
out = resolve_knoe_home(env={}) / "env.sh"
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
|
||
should_write = True
|
||
if out.exists():
|
||
if out.read_text() == new_content:
|
||
should_write = False
|
||
self.log(f"[OK] {out} is up to date.")
|
||
|
||
if should_write:
|
||
tmp = out.parent / "env.sh.tmp"
|
||
tmp.write_text(new_content)
|
||
tmp.replace(out)
|
||
try:
|
||
os.chmod(out, 0o755)
|
||
except Exception:
|
||
pass
|
||
self.log(f"[OK] Wrote {out}")
|
||
|
||
self._deploy_env_resources(values)
|
||
|
||
def _deploy_env_resources(self, values: dict):
|
||
"""Copy init scripts and etc directory into KNOE_HOME / KNOE_SERVICE."""
|
||
try:
|
||
env_map = dict(os.environ)
|
||
try:
|
||
env_map.update({str(k): str(v) for k, v in (values or {}).items() if v is not None})
|
||
except Exception:
|
||
pass
|
||
|
||
expanded_home = self._expand_shell_path(values.get("KNOE_HOME", ""), env=env_map)
|
||
expanded_home = os.path.expanduser(expanded_home) if expanded_home else ""
|
||
if expanded_home:
|
||
env_map["KNOE_HOME"] = expanded_home
|
||
expanded_service = self._expand_shell_path(values.get("KNOE_SERVICE", ""), env=env_map)
|
||
expanded_service = os.path.expanduser(expanded_service) if expanded_service else ""
|
||
|
||
knoe_home = Path(expanded_home) if expanded_home else resolve_knoe_home(env={})
|
||
knoe_service = Path(expanded_service or (knoe_home / "etc"))
|
||
|
||
init_pf_src_candidates = [
|
||
self.project_root / "src" / "knoe" / "etc" / "init-port-forward.sh",
|
||
self.project_root / "etc" / "init-port-forward.sh",
|
||
]
|
||
init_pf_src = next((p for p in init_pf_src_candidates if p.exists()), None)
|
||
if init_pf_src is not None:
|
||
init_pf_dst = knoe_home / "init-port-forward.sh"
|
||
try:
|
||
data = init_pf_src.read_bytes()
|
||
init_pf_dst.write_bytes(data)
|
||
os.chmod(init_pf_dst, 0o755)
|
||
except Exception:
|
||
pass
|
||
|
||
etc_src_candidates = [
|
||
self.project_root / "src" / "knoe" / "etc",
|
||
self.project_root / "etc",
|
||
]
|
||
etc_src = next((p for p in etc_src_candidates if p.exists()), None)
|
||
if etc_src is not None:
|
||
try:
|
||
knoe_service.mkdir(parents=True, exist_ok=True)
|
||
if etc_src.resolve() != knoe_service.resolve():
|
||
if knoe_service.exists():
|
||
shutil.rmtree(knoe_service, ignore_errors=True)
|
||
shutil.copytree(etc_src, knoe_service)
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
def reload_env_from_shell(self) -> None:
|
||
raw_home = self._get_input(
|
||
"env_setup.KNOE_HOME", str(resolve_knoe_home(env={}))
|
||
)
|
||
expanded_home = self._expand_shell_path(raw_home, env=os.environ)
|
||
expanded_home = os.path.expanduser(expanded_home) if expanded_home else ""
|
||
home = Path(expanded_home) if expanded_home else resolve_knoe_home(env={})
|
||
env_file = home / "env.sh"
|
||
cmd = (
|
||
f"export KNOE_HOME={shlex.quote(str(home))}; "
|
||
f"source {shlex.quote(str(env_file))}; "
|
||
"env -0"
|
||
)
|
||
try:
|
||
out = subprocess.check_output(["bash", "-lc", cmd])
|
||
except Exception as e:
|
||
raise Exception(f"Failed to reload environment: {e}")
|
||
try:
|
||
for raw in out.split(b"\x00"):
|
||
if not raw:
|
||
continue
|
||
kv = raw.decode("utf-8", errors="ignore")
|
||
if "=" not in kv:
|
||
continue
|
||
k, v = kv.split("=", 1)
|
||
if k in ("PYTHONPATH", "PYTHONHOME"):
|
||
continue
|
||
os.environ[k] = v
|
||
except Exception:
|
||
pass
|
||
|
||
def _update_env_namespace(self, namespace: str):
|
||
# Namespace is persisted to knoe.cfg only — never to env.sh or os.environ.
|
||
# This method is intentionally a no-op for environment mutation.
|
||
pass
|
||
|
||
# --------------------------------------------- port-forward helpers
|
||
def _add_port_mapping(self, mapping_str):
|
||
mode = self._deployment_mode()
|
||
prefix = (
|
||
"PORT_FORWARD_K3S_MAPPING_"
|
||
if mode == "k3s"
|
||
else "PORT_FORWARD_K3D_MAPPING_"
|
||
)
|
||
pf_section = self.knoe_cfg_data.get("Port Forwards", {})
|
||
if pf_section is None:
|
||
pf_section = {}
|
||
if not _pf_upsert_mapping(pf_section, prefix, mapping_str):
|
||
return
|
||
self.knoe_cfg_data["Port Forwards"] = pf_section
|
||
if hasattr(self, "_save_knoe_cfg"):
|
||
self._save_knoe_cfg()
|
||
elif hasattr(self, "_write_cfg"):
|
||
self._write_cfg()
|
||
|
||
def _process_script_output_line(self, line):
|
||
if "GRAFANA_ADMIN_PASSWORD=" in line:
|
||
pwd = line.split("GRAFANA_ADMIN_PASSWORD=")[1].strip()
|
||
if pwd:
|
||
mon = self.knoe_cfg_data.get("Monitoring", {})
|
||
if mon is None:
|
||
mon = {}
|
||
mon["GRAFANA_ADMIN_PASSWORD"] = pwd
|
||
self.knoe_cfg_data["Monitoring"] = mon
|
||
if hasattr(self, "_save_knoe_cfg"):
|
||
self._save_knoe_cfg()
|
||
elif hasattr(self, "_write_cfg"):
|
||
self._write_cfg()
|
||
if "PORT_FORWARD_MAPPING:" in line:
|
||
mapping = line.split("PORT_FORWARD_MAPPING:")[1].strip()
|
||
if mapping:
|
||
self._add_port_mapping(mapping)
|
||
|
||
def _sync_port_forward_mappings(self):
|
||
mode = _deployment_mode_from_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
if not mode:
|
||
mode = "k3d"
|
||
prefix = (
|
||
"PORT_FORWARD_K3S_MAPPING_"
|
||
if mode == "k3s"
|
||
else "PORT_FORWARD_K3D_MAPPING_"
|
||
)
|
||
pf_section = self.knoe_cfg_data.get("Port Forwards", {})
|
||
if pf_section is None:
|
||
pf_section = {}
|
||
_ids_to_remove_if_disabled = {
|
||
"registry": True,
|
||
"argocd": not self._get_input_bool("init_cluster.argocd_enabled", False),
|
||
"dashboard": True,
|
||
}
|
||
for k, v in list(pf_section.items()):
|
||
if not isinstance(v, str):
|
||
continue
|
||
for _id, _should_remove in _ids_to_remove_if_disabled.items():
|
||
if _should_remove and (f"id={_id};" in v or v.strip() == f"id={_id}"):
|
||
del pf_section[k]
|
||
break
|
||
service_ns = (self._service_namespace() or "").strip() or "default"
|
||
db_ns = (self._get_input("init_password.db_namespace", "") or "").strip()
|
||
if not db_ns:
|
||
db_ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {})
|
||
.get("DATABASE_NAMESPACE", "")
|
||
.strip()
|
||
)
|
||
if not db_ns:
|
||
db_ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "").strip()
|
||
)
|
||
if not db_ns:
|
||
db_ns = (
|
||
os.environ.get("DATABASE_NAMESPACE")
|
||
or os.environ.get("PROLE_NAMESPACE")
|
||
or os.environ.get("NAMESPACE")
|
||
or ""
|
||
).strip()
|
||
if not db_ns:
|
||
db_ns = "default"
|
||
argocd_ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {})
|
||
.get("ARGOCD_NAMESPACE", "")
|
||
.strip()
|
||
)
|
||
if not argocd_ns:
|
||
argocd_ns = (os.environ.get("ARGOCD_NAMESPACE") or "").strip()
|
||
if not argocd_ns:
|
||
argocd_ns = "argocd"
|
||
db_host_port = (self._get_input("init_password.db_host_port", "") or "").strip()
|
||
if not db_host_port:
|
||
db_host_port = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {})
|
||
.get("DB_HOST_PORT", "")
|
||
.strip()
|
||
)
|
||
if not db_host_port:
|
||
db_host_port = "5432"
|
||
supabase_enabled = self._get_input_bool("init_cluster.supabase_enabled", False)
|
||
supabase_ns = (os.environ.get("SUPABASE_NAMESPACE") or "").strip()
|
||
if not supabase_ns:
|
||
supabase_ns = (
|
||
(self.knoe_cfg_data.get("Supabase", {}) or {})
|
||
.get("NAMESPACE", "")
|
||
.strip()
|
||
)
|
||
if not supabase_ns:
|
||
supabase_ns = "supabase"
|
||
gitops_enabled = self._get_input_bool("init_cluster.gitops_enabled", False)
|
||
gitops_ns = (os.environ.get("GITEA_NAMESPACE") or "").strip()
|
||
if not gitops_ns:
|
||
gitops_ns = (
|
||
(self.knoe_cfg_data.get("GitOps", {}) or {})
|
||
.get("GITOPS_NAMESPACE", "")
|
||
.strip()
|
||
)
|
||
if not gitops_ns:
|
||
gitops_ns = (self._get_input("gitops.namespace", "") or "").strip()
|
||
if not gitops_ns:
|
||
gitops_ns = "gitea"
|
||
argocd_enabled = self._get_input_bool("init_cluster.argocd_enabled", False)
|
||
monitoring_release = (
|
||
(self.knoe_cfg_data.get("Monitoring", {}) or {})
|
||
.get("MONITORING_RELEASE", "")
|
||
.strip()
|
||
or os.environ.get("MONITORING_RELEASE", "").strip()
|
||
or "prometheus"
|
||
)
|
||
mappings = _build_required_port_forwards(
|
||
mode=mode,
|
||
service_ns=service_ns,
|
||
argocd_ns=argocd_ns,
|
||
db_ns=db_ns,
|
||
db_host_port=db_host_port,
|
||
supabase_enabled=supabase_enabled,
|
||
supabase_namespace=supabase_ns,
|
||
gitops_enabled=gitops_enabled,
|
||
gitops_namespace=gitops_ns,
|
||
argocd_enabled=argocd_enabled,
|
||
dashboard_enabled=False,
|
||
monitoring_release=monitoring_release,
|
||
)
|
||
for mapping in mappings:
|
||
_pf_upsert_mapping(pf_section, prefix, mapping)
|
||
self.knoe_cfg_data["Port Forwards"] = pf_section
|
||
|
||
# --------------------------------------------- script env
|
||
def _script_env_for_namespace(self, namespace: str, cluster_role: str = "db") -> dict:
|
||
namespace = _safe_str(namespace)
|
||
cluster_role = "app" if str(cluster_role).strip().lower() == "app" else "db"
|
||
env = os.environ.copy()
|
||
root = getattr(self, "project_root", None) or PROJECT_ROOT
|
||
cluster_name = (
|
||
self._app_cluster_name() if cluster_role == "app" else self._cnpg_cluster_name()
|
||
)
|
||
env["KNOE_HOME"] = str(root)
|
||
env["KNOE_SERVICE"] = str(root)
|
||
env["DATABASE_NAMESPACE"] = namespace
|
||
env["CLUSTER_NAME"] = cluster_name
|
||
env["APP_CLUSTER_NAME"] = self._app_cluster_name()
|
||
env["DB_CLUSTER_NAME"] = self._cnpg_cluster_name()
|
||
env["APP_CLUSTER_MODE"] = self._app_cluster_mode()
|
||
env["DB_CLUSTER_MODE"] = self._db_cluster_mode()
|
||
# Transitional compatibility for scripts still reading NAMESPACE.
|
||
env["NAMESPACE"] = namespace
|
||
service_ns = self._service_namespace()
|
||
|
||
env["KNOE_CONF"] = self._get_input(
|
||
"env_setup.KNOE_CONF", str(root / "conf")
|
||
)
|
||
env["PROLE_DATA"] = self._get_input(
|
||
"env_setup.PROLE_DATA", str(root / "knoe-db" / "data")
|
||
)
|
||
env["PROLE_LOGS"] = self._get_input(
|
||
"env_setup.PROLE_LOGS", str(root / "logs")
|
||
)
|
||
env["KNOE_SERVICE"] = self._get_input(
|
||
"env_setup.KNOE_SERVICE", str(root / "etc")
|
||
)
|
||
env["KNOE_DB_USER"] = self._get_input("init_password.db_username", "").strip()
|
||
|
||
# Make the DB/root master password available to shell scripts.
|
||
# OpenTofu bootstrap uses it to create the initial admin secret when missing.
|
||
db_pw = (self._get_input("init_password.db_password", "") or "").strip()
|
||
if db_pw:
|
||
env.setdefault("DB_PASSWORD", db_pw)
|
||
env.setdefault("OPENTOFU_ADMIN_PASSWORD", db_pw)
|
||
|
||
mode = self._deployment_mode()
|
||
if mode:
|
||
env["KNOE_MODE"] = mode
|
||
env["DEPLOYMENT_MODE"] = mode
|
||
cluster_env = self._get_input("init_cluster.cluster_env", "dev")
|
||
env["DEPLOYMENT_TARGET"] = _deployment_target_label(cluster_env)
|
||
|
||
if mode == "k3s":
|
||
# 1. Prefer an existing kubeconfig that already exists on disk.
|
||
# Check knoe-specific paths before ~/.kube/config to avoid
|
||
# picking up a stale k3d context from the default kubeconfig.
|
||
root = getattr(self, "project_root", None) or PROJECT_ROOT
|
||
_knoe_kc_candidates = [
|
||
str(Path(root) / "knoe-k3s.kubeconfig"),
|
||
str(Path(root) / "etc" / "secrets" / "k3s.kubeconfig"),
|
||
str(Path(root) / "secrets" / "k3s.kubeconfig"),
|
||
]
|
||
knoe_service_dir = (env.get("KNOE_SERVICE") or "").strip()
|
||
if knoe_service_dir:
|
||
_knoe_kc_candidates.insert(1, str(Path(knoe_service_dir) / "secrets" / "k3s.kubeconfig"))
|
||
kc = next(
|
||
(p for p in _knoe_kc_candidates if p and Path(p).expanduser().exists()),
|
||
"",
|
||
) or _find_kubeconfig_file(env)
|
||
if kc:
|
||
env["KUBECONFIG"] = kc
|
||
else:
|
||
# 2. Try fetching from k3s via Ansible or generate token-based
|
||
fetched = self._fetch_k3s_kubeconfig()
|
||
if fetched:
|
||
env["KUBECONFIG"] = str(fetched)
|
||
else:
|
||
server, token = self._resolve_k3s_connection()
|
||
if server and token and _looks_like_k8s_bearer_token(token):
|
||
try:
|
||
managed = _write_k3s_kubeconfig(server, token)
|
||
self._managed_kubeconfig = str(managed)
|
||
env["KUBECONFIG"] = self._managed_kubeconfig
|
||
_merge_kubeconfig(self._managed_kubeconfig)
|
||
except Exception:
|
||
pass
|
||
elif server and token:
|
||
self.err(
|
||
"[WARN] k3s token does not look like a Kubernetes bearer token; cannot generate kubeconfig. Ensure a cert-based knoe-k3s.kubeconfig exists (Ansible fetch)."
|
||
)
|
||
|
||
raw_server, raw_token = self._resolve_k3s_connection()
|
||
if raw_server:
|
||
env["K3S_SERVER"] = raw_server
|
||
if raw_token:
|
||
env["K3S_TOKEN"] = raw_token
|
||
|
||
elif mode == "k3d":
|
||
# Ensure k3d dev clusters have a resolvable KUBECONFIG.
|
||
cluster_name = self._get_input("init_cluster.deployment_target", "knoe-dev-cluster")
|
||
if not cluster_name or cluster_name == "knoe-dev-cluster":
|
||
# Fallback to check if we have a more specific one from UI if available
|
||
cluster_name = getattr(self, "selected_k3d_cluster", None)
|
||
if hasattr(cluster_name, "get"):
|
||
cluster_name = cluster_name.get()
|
||
cluster_name = cluster_name or "knoe-dev-cluster"
|
||
|
||
try:
|
||
subprocess.run(
|
||
[
|
||
"k3d",
|
||
"kubeconfig",
|
||
"merge",
|
||
cluster_name,
|
||
"--kubeconfig-switch-context",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
)
|
||
except Exception:
|
||
pass
|
||
default_kube = str(Path.home() / ".kube" / "config")
|
||
if Path(default_kube).exists():
|
||
env["KUBECONFIG"] = default_kube
|
||
elif mode == "k8s":
|
||
# For GKE/prod split-cluster deploys we require explicit role context.
|
||
gke_context = self._cluster_kubecontext(cluster_role).strip()
|
||
default_kube = str(Path.home() / ".kube" / "config")
|
||
if Path(default_kube).exists():
|
||
env["KUBECONFIG"] = default_kube
|
||
if gke_context:
|
||
env["KUBECONTEXT"] = gke_context
|
||
env["KUBE_CONTEXT_NAME"] = gke_context
|
||
env["KUBECTL_CONTEXT"] = gke_context
|
||
else:
|
||
env.pop("KUBECONFIG", None)
|
||
env["SERVICE_NAMESPACE"] = service_ns
|
||
env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.at_rest_encryption_enabled", False)
|
||
)
|
||
env["SUPABASE_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_enabled", False)
|
||
)
|
||
env["SUPABASE_STUDIO_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_studio_enabled", False)
|
||
)
|
||
default_supabase_studio_url = "db.0.knoe.dev" if mode == "k8s" else "db.knoe.org"
|
||
env["SUPABASE_STUDIO_URL"] = (
|
||
self._get_input("init_cluster.supabase_studio_url", default_supabase_studio_url).strip()
|
||
or default_supabase_studio_url
|
||
)
|
||
env["SUPABASE_AUTH_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_auth_enabled", True)
|
||
)
|
||
env["SUPABASE_REALTIME_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_realtime_enabled", True)
|
||
)
|
||
env["SUPABASE_META_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_meta_enabled", True)
|
||
)
|
||
env["SUPABASE_ANALYTICS_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_analytics_enabled", True)
|
||
)
|
||
db_pw = self._get_input("init_password.db_password", "").strip()
|
||
if db_pw:
|
||
db_pw = self._resolve_secret_value(db_pw)
|
||
env["DB_PASSWORD"] = db_pw
|
||
env["OPENTOFU_ADMIN_PASSWORD"] = db_pw
|
||
|
||
grafana_pw = (
|
||
self.knoe_cfg_data.get("Monitoring", {})
|
||
.get("GRAFANA_ADMIN_PASSWORD", "")
|
||
.strip()
|
||
)
|
||
resolved_grafana = self._resolve_secret_value(grafana_pw) if grafana_pw else ""
|
||
if resolved_grafana and not (
|
||
_is_openbao_ref(resolved_grafana) or _is_knoe_secret(resolved_grafana)
|
||
):
|
||
env["GRAFANA_ADMIN_PASSWORD"] = resolved_grafana
|
||
elif db_pw:
|
||
env["GRAFANA_ADMIN_PASSWORD"] = db_pw
|
||
|
||
realm = self._get_input("kerberos_config.realm", "").strip()
|
||
if realm:
|
||
env["KRB5_REALM"] = realm
|
||
env["REALM"] = realm
|
||
env["DOMAIN"] = realm.lower()
|
||
|
||
kdc = self._get_input("kerberos_config.kdc", "").strip()
|
||
if kdc:
|
||
env["KRB5_KDC"] = kdc
|
||
env["KRB5_ADMIN"] = kdc
|
||
|
||
env["KRB5_USER"] = self._get_input("kerberos_config.user", "")
|
||
krb_pw = self._get_input("kerberos_config.password", "")
|
||
if krb_pw:
|
||
env["KRB5_PASSWORD"] = self._resolve_secret_value(krb_pw)
|
||
else:
|
||
env["KRB5_PASSWORD"] = ""
|
||
|
||
# Registry settings
|
||
reg_host = self.knoe_cfg_data.get("Docker Build", {}).get("LOCAL_REGISTRY")
|
||
if reg_host:
|
||
env["LOCAL_REGISTRY"] = reg_host
|
||
reg_internal = self.knoe_cfg_data.get("Docker Build", {}).get(
|
||
"LOCAL_REGISTRY_INTERNAL"
|
||
)
|
||
if reg_internal:
|
||
env["LOCAL_REGISTRY_INTERNAL"] = reg_internal
|
||
|
||
# Artifact Registry is required by CNPG manifests in k8s mode.
|
||
artifact_registry = str(env.get("ARTIFACT_REGISTRY") or "").strip().strip('"')
|
||
if not artifact_registry:
|
||
artifact_registry = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {})
|
||
.get("ARTIFACT_REGISTRY", "")
|
||
.strip()
|
||
.strip('"')
|
||
)
|
||
if not artifact_registry:
|
||
gcp_cfg = self.knoe_cfg_data.get("GCP", {}) or {}
|
||
project_id = (
|
||
(self._get_input("init_cluster.project_id", "") or "").strip()
|
||
or str(gcp_cfg.get("project_id") or "").strip().strip('"')
|
||
or str(gcp_cfg.get("PROJECT_ID") or "").strip().strip('"')
|
||
or (os.environ.get("GCP_PROJECT") or "").strip()
|
||
)
|
||
region = (
|
||
(self._get_input("init_cluster.db_cluster_region", "") or "").strip()
|
||
or (self._get_input("init_cluster.app_cluster_region", "") or "").strip()
|
||
or str(gcp_cfg.get("region") or "").strip().strip('"')
|
||
or str(gcp_cfg.get("REGION") or "").strip().strip('"')
|
||
or str(gcp_cfg.get("location") or "").strip().strip('"')
|
||
or (os.environ.get("GCP_REGION") or "").strip()
|
||
)
|
||
repo = (service_ns or "knoe-system").strip() or "knoe-system"
|
||
if project_id and region:
|
||
artifact_registry = f"{region}-docker.pkg.dev/{project_id}/{repo}"
|
||
if artifact_registry:
|
||
env["ARTIFACT_REGISTRY"] = artifact_registry.rstrip("/")
|
||
|
||
try:
|
||
placement_plan, placement_path = self._resolve_cnpg_placement_plan(namespace)
|
||
eligible_nodes = placement_plan.get("eligible_nodes") or []
|
||
assignments = placement_plan.get("assignments") or {}
|
||
assignment_pairs = []
|
||
for ordinal in sorted(assignments.keys(), key=lambda k: int(str(k))):
|
||
assignment_pairs.append(f"{ordinal}:{assignments[ordinal]}")
|
||
|
||
env["CNPG_PLACEMENT_PLAN_FILE"] = str(placement_path)
|
||
env["CNPG_PLACEMENT_PLAN_ID"] = str(placement_plan.get("plan_id") or "")
|
||
env["CNPG_PLACEMENT_PLAN_HASH"] = str(
|
||
placement_plan.get("plan_hash") or ""
|
||
)
|
||
env["CNPG_PLACEMENT_ELIGIBLE_NODES"] = ",".join(eligible_nodes)
|
||
env["CNPG_PLACEMENT_ASSIGNMENTS"] = ",".join(assignment_pairs)
|
||
env["CNPG_PLACEMENT_REUSED"] = _bool_str(
|
||
bool((placement_plan.get("metadata") or {}).get("reused", False))
|
||
)
|
||
env.setdefault(
|
||
"CNPG_CLUSTER_NAME",
|
||
str(placement_plan.get("cluster_name") or "knoe-db"),
|
||
)
|
||
env.setdefault(
|
||
"CNPG_INSTANCES",
|
||
str(placement_plan.get("desired_instances") or "3"),
|
||
)
|
||
|
||
# Compatibility bridge for existing shell topology flow.
|
||
if eligible_nodes:
|
||
env.setdefault("CNPG_STAGE1_NODE", str(eligible_nodes[0]))
|
||
if len(eligible_nodes) == 1:
|
||
env.setdefault(
|
||
"CNPG_DB_NODE_SELECTOR",
|
||
f"kubernetes.io/hostname={eligible_nodes[0]}",
|
||
)
|
||
except Exception as e:
|
||
self.err(f"[WARN] CNPG placement planning failed; using fallback topology: {e}")
|
||
# Only apply dual-cluster GKE targeting in k8s/prod mode.
|
||
if mode == "k8s":
|
||
return build_kubectl_env_for_cluster(
|
||
base_env=env,
|
||
kubecontext=self._cluster_kubecontext(cluster_role),
|
||
cluster_name=cluster_name,
|
||
cluster_role=cluster_role,
|
||
)
|
||
return env
|
||
|
||
def _cnpg_cluster_name(self) -> str:
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
cluster_name = (
|
||
os.environ.get("CLUSTER_NAME")
|
||
or os.environ.get("CNPG_CLUSTER_NAME")
|
||
or self._get_input("init_password.db_cluster_name", "")
|
||
or self._get_input("env_setup.DB_CLUSTER_NAME", "")
|
||
or str(glob.get("CLUSTER_NAME") or "")
|
||
or str(glob.get("CNPG_CLUSTER_NAME") or "")
|
||
).strip()
|
||
return cluster_name or "knoe-db"
|
||
|
||
def _app_cluster_name(self) -> str:
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
name = (
|
||
self._get_input("init_password.app_cluster_name", "")
|
||
or self._get_input("env_setup.APP_CLUSTER_NAME", "")
|
||
or str(glob.get("APP_CLUSTER_NAME") or "")
|
||
or DEFAULT_APP_CLUSTER_NAME
|
||
)
|
||
return str(name).strip() or DEFAULT_APP_CLUSTER_NAME
|
||
|
||
def _app_cluster_mode(self) -> str:
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
mode = (
|
||
self._get_input("init_cluster.app_cluster_mode", "")
|
||
or self._get_input("env_setup.APP_CLUSTER_MODE", "")
|
||
or str(glob.get("APP_CLUSTER_MODE") or "")
|
||
or DEFAULT_APP_CLUSTER_MODE
|
||
)
|
||
return str(mode).strip() or DEFAULT_APP_CLUSTER_MODE
|
||
|
||
def _db_cluster_mode(self) -> str:
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
mode = (
|
||
self._get_input("init_cluster.db_cluster_mode", "")
|
||
or self._get_input("env_setup.DB_CLUSTER_MODE", "")
|
||
or str(glob.get("DB_CLUSTER_MODE") or "")
|
||
or DEFAULT_DB_CLUSTER_MODE
|
||
)
|
||
return str(mode).strip() or DEFAULT_DB_CLUSTER_MODE
|
||
|
||
def _cluster_kubecontext(self, role: str) -> str:
|
||
role = "app" if str(role).strip().lower() == "app" else "db"
|
||
key = (
|
||
"init_cluster.app_cluster_kubecontext"
|
||
if role == "app"
|
||
else "init_cluster.db_cluster_kubecontext"
|
||
)
|
||
fallback_key = (
|
||
"env_setup.APP_CLUSTER_KUBECONTEXT"
|
||
if role == "app"
|
||
else "env_setup.DB_CLUSTER_KUBECONTEXT"
|
||
)
|
||
context = _safe_str(self._get_input(key, "") or self._get_input(fallback_key, ""))
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
global_key = "APP_CLUSTER_KUBECONTEXT" if role == "app" else "DB_CLUSTER_KUBECONTEXT"
|
||
if not context:
|
||
context = _safe_str(glob.get(global_key, ""))
|
||
|
||
opposite_key = (
|
||
"init_cluster.db_cluster_kubecontext"
|
||
if role == "app"
|
||
else "init_cluster.app_cluster_kubecontext"
|
||
)
|
||
opposite_fallback_key = (
|
||
"env_setup.DB_CLUSTER_KUBECONTEXT"
|
||
if role == "app"
|
||
else "env_setup.APP_CLUSTER_KUBECONTEXT"
|
||
)
|
||
opposite_global_key = "DB_CLUSTER_KUBECONTEXT" if role == "app" else "APP_CLUSTER_KUBECONTEXT"
|
||
opposite_context = _safe_str(
|
||
self._get_input(opposite_key, "")
|
||
or self._get_input(opposite_fallback_key, "")
|
||
or glob.get(opposite_global_key, "")
|
||
)
|
||
if context and opposite_context and context == opposite_context:
|
||
raise RuntimeError(
|
||
f"Refusing {role.upper()} targeting: APP/DB contexts must differ, both resolve to '{context}'."
|
||
)
|
||
|
||
if self._deployment_mode() == "k8s":
|
||
if context:
|
||
return context
|
||
# Fall through to derive context from cluster name + region below.
|
||
|
||
if context:
|
||
return context
|
||
|
||
selected_context = _safe_str(
|
||
self._get_input("init_cluster.selected_kubectx", "") or glob.get("KUBECONTEXT", "")
|
||
)
|
||
if selected_context and not selected_context.startswith("gke_"):
|
||
return selected_context
|
||
|
||
if role == "app":
|
||
cluster_name = self._app_cluster_name()
|
||
region = _safe_str(
|
||
self._get_input("init_cluster.app_cluster_region", "")
|
||
or self._get_input("env_setup.APP_CLUSTER_REGION", "")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("region")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("REGION")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("location")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("LOCATION")
|
||
)
|
||
else:
|
||
cluster_name = _safe_str(
|
||
self._get_input("init_password.db_cluster_name", "")
|
||
or self._get_input("env_setup.DB_CLUSTER_NAME", "")
|
||
or glob.get("DB_CLUSTER_NAME", "")
|
||
or glob.get("CNPG_CLUSTER_NAME", "")
|
||
or DEFAULT_DB_CLUSTER_NAME
|
||
) or DEFAULT_DB_CLUSTER_NAME
|
||
region = _safe_str(
|
||
self._get_input("init_cluster.db_cluster_region", "")
|
||
or self._get_input("env_setup.DB_CLUSTER_REGION", "")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("region")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("REGION")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("location")
|
||
or (self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}).get("LOCATION")
|
||
)
|
||
|
||
gcp_cfg = self.gcp_cfg if isinstance(getattr(self, "gcp_cfg", None), dict) else {}
|
||
project_id = _safe_str(
|
||
self._get_input("init_cluster.project_id", "")
|
||
or gcp_cfg.get("project_id")
|
||
or gcp_cfg.get("PROJECT_ID")
|
||
)
|
||
if not region:
|
||
region = _safe_str(
|
||
gcp_cfg.get("region")
|
||
or gcp_cfg.get("REGION")
|
||
or gcp_cfg.get("location")
|
||
or gcp_cfg.get("LOCATION")
|
||
)
|
||
if project_id and region:
|
||
return f"gke_{project_id}_{region}_{cluster_name}"
|
||
|
||
parts = selected_context.split("_", 3)
|
||
if len(parts) == 4:
|
||
return f"{parts[0]}_{parts[1]}_{parts[2]}_{cluster_name}"
|
||
|
||
return selected_context
|
||
|
||
def _cnpg_desired_instances(self) -> int:
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
raw = (os.environ.get("CNPG_INSTANCES") or str(glob.get("CNPG_INSTANCES") or "")).strip()
|
||
try:
|
||
val = int(raw) if raw else 3
|
||
except Exception:
|
||
val = 3
|
||
return max(1, val)
|
||
|
||
def _cnpg_rebalance_requested(self) -> bool:
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
raw = (
|
||
os.environ.get("CNPG_REBALANCE")
|
||
or str(glob.get("CNPG_REBALANCE") or "")
|
||
)
|
||
return _parse_bool(raw, default=False)
|
||
|
||
def _cnpg_candidate_nodes(self) -> list[str]:
|
||
glob = self.knoe_cfg_data.get("Global", {}) or {}
|
||
nodes: set[str] = set()
|
||
|
||
explicit = (
|
||
os.environ.get("CNPG_ELIGIBLE_NODES")
|
||
or str(glob.get("CNPG_ELIGIBLE_NODES") or "")
|
||
)
|
||
if explicit:
|
||
for part in explicit.split(","):
|
||
node = str(part or "").strip()
|
||
if node:
|
||
nodes.add(node)
|
||
|
||
mounts_raw = str(
|
||
(self.knoe_cfg_data.get("Storage", {}) or {}).get("ANSIBLE_ISCSI_MOUNTS", "")
|
||
).strip()
|
||
if mounts_raw:
|
||
try:
|
||
mounts = json.loads(mounts_raw)
|
||
except Exception:
|
||
mounts = {}
|
||
if isinstance(mounts, dict):
|
||
for details in mounts.values():
|
||
if not isinstance(details, dict):
|
||
continue
|
||
node = str(details.get("host") or "").strip()
|
||
if node:
|
||
nodes.add(node)
|
||
|
||
stage1 = str(glob.get("CNPG_STAGE1_NODE") or "").strip()
|
||
if stage1:
|
||
nodes.add(stage1)
|
||
|
||
return sorted(nodes)
|
||
|
||
def _cnpg_plan_path(self, namespace: str, cluster_name: str) -> Path:
|
||
conf_root = self._get_input(
|
||
"env_setup.KNOE_CONF",
|
||
str((getattr(self, "project_root", None) or PROJECT_ROOT) / "conf"),
|
||
)
|
||
conf_dir = Path(conf_root)
|
||
|
||
def _safe_segment(raw: str) -> str:
|
||
s = str(raw or "").strip()
|
||
if not s:
|
||
return "default"
|
||
return "".join(ch if ch.isalnum() or ch in "-_." else "-" for ch in s)
|
||
|
||
ns_seg = _safe_segment(namespace)
|
||
cluster_seg = _safe_segment(cluster_name)
|
||
return conf_dir / "cnpg-placement" / f"{ns_seg}-{cluster_seg}.json"
|
||
|
||
def _resolve_cnpg_placement_plan(self, namespace: str) -> tuple[dict, Path]:
|
||
cluster_name = self._cnpg_cluster_name()
|
||
desired_instances = self._cnpg_desired_instances()
|
||
candidate_nodes = self._cnpg_candidate_nodes()
|
||
rebalance = self._cnpg_rebalance_requested()
|
||
plan_path = self._cnpg_plan_path(namespace, cluster_name)
|
||
prior_plan = load_cnpg_placement_plan(plan_path)
|
||
|
||
placement_plan = plan_cnpg_placement(
|
||
cluster_name=cluster_name,
|
||
desired_instances=desired_instances,
|
||
candidate_nodes=candidate_nodes,
|
||
prior_plan=prior_plan,
|
||
rebalance=rebalance,
|
||
)
|
||
|
||
if not isinstance(prior_plan, dict) or prior_plan != placement_plan:
|
||
save_cnpg_placement_plan(plan_path, placement_plan)
|
||
|
||
glob = self.knoe_cfg_data.setdefault("Global", {})
|
||
glob["CNPG_PLACEMENT_PLAN_FILE"] = str(plan_path)
|
||
glob["CNPG_PLACEMENT_PLAN_ID"] = str(placement_plan.get("plan_id") or "")
|
||
glob["CNPG_PLACEMENT_PLAN_HASH"] = str(placement_plan.get("plan_hash") or "")
|
||
|
||
return placement_plan, plan_path
|
||
|
||
def _cnpg_storage_node(self, env: dict[str, str]) -> str:
|
||
stage1 = str(env.get("CNPG_STAGE1_NODE") or "").strip()
|
||
if stage1:
|
||
return stage1
|
||
|
||
eligible_raw = str(env.get("CNPG_PLACEMENT_ELIGIBLE_NODES") or "").strip()
|
||
if eligible_raw:
|
||
for part in eligible_raw.split(","):
|
||
node = str(part or "").strip()
|
||
if node:
|
||
return node
|
||
|
||
candidates = self._cnpg_candidate_nodes()
|
||
if candidates:
|
||
return candidates[0]
|
||
|
||
raise StorageProvisioningError(
|
||
"Unable to determine CNPG node for local PV provisioning. "
|
||
"Expected CNPG placement env (CNPG_STAGE1_NODE/CNPG_PLACEMENT_ELIGIBLE_NODES) "
|
||
"or configured CNPG candidate nodes."
|
||
)
|
||
|
||
def _ensure_cnpg_storage_provisioned(self, namespace: str, env: dict[str, str]) -> None:
|
||
if self._deployment_mode() != "k3s":
|
||
return
|
||
|
||
cluster_name = str(
|
||
env.get("CLUSTER_NAME")
|
||
or env.get("CNPG_CLUSTER_NAME")
|
||
or self._cnpg_cluster_name()
|
||
).strip()
|
||
if not cluster_name:
|
||
cluster_name = "knoe-db"
|
||
node_name = self._cnpg_storage_node(env)
|
||
|
||
spec = ClusterStorageSpec(
|
||
namespace=namespace,
|
||
cluster_name=cluster_name,
|
||
node_name=node_name,
|
||
storage_class_name="synology-iscsi",
|
||
command_env=env,
|
||
)
|
||
provisioned = provision_cluster_storage(spec)
|
||
|
||
env["CNPG_STORAGE_CLASS"] = spec.storage_class_name
|
||
env["CNPG_DATA_SELECTOR_JSON"] = json.dumps(
|
||
provisioned.data_selector,
|
||
separators=(",", ":"),
|
||
sort_keys=True,
|
||
)
|
||
env["CNPG_WAL_SELECTOR_JSON"] = json.dumps(
|
||
provisioned.wal_selector,
|
||
separators=(",", ":"),
|
||
sort_keys=True,
|
||
)
|
||
env["CNPG_DATA_PV_NAME"] = provisioned.data_pv_name
|
||
env["CNPG_WAL_PV_NAME"] = provisioned.wal_pv_name
|
||
env["CNPG_DATA_HOST_PATH"] = provisioned.data_path
|
||
env["CNPG_WAL_HOST_PATH"] = provisioned.wal_path
|
||
env["SYNOLOGY_ROOTS"] = ",".join(spec.synology_roots)
|
||
|
||
self.log(
|
||
"[CNPG] Provisioned cluster storage "
|
||
f"namespace={namespace} cluster={cluster_name} node={node_name} "
|
||
f"data_pv={provisioned.data_pv_name} wal_pv={provisioned.wal_pv_name} "
|
||
f"data_path={provisioned.data_path} wal_path={provisioned.wal_path}"
|
||
)
|
||
|
||
def _cluster_storage_milestone_enabled(self) -> bool:
|
||
raw = str(
|
||
os.environ.get("PROLE_ENABLE_CLUSTER_STORAGE_MILESTONE", "")
|
||
).strip().lower()
|
||
return raw in {"1", "true", "yes", "on"}
|
||
|
||
# --------------------------------------------- authority / repair
|
||
def _authority_context_missing(self) -> bool:
|
||
enabled = self._get_input_bool(
|
||
"kerberos_config.enabled", False
|
||
) or self._get_input_bool("init_cluster.kerberos_enabled", False)
|
||
if not enabled:
|
||
return False
|
||
candidates = []
|
||
env_home = (os.environ.get("KNOE_HOME") or "").strip()
|
||
if env_home:
|
||
candidates.append(Path(env_home))
|
||
candidates.append(self.project_root)
|
||
for base in candidates:
|
||
try:
|
||
if (base / "authority").is_dir():
|
||
return False
|
||
if (base / "knoe" / "authority").is_dir():
|
||
return False
|
||
except Exception:
|
||
continue
|
||
return True
|
||
|
||
def _dashboard_kong_missing(self, env: dict) -> bool:
|
||
ns = "kubernetes-dashboard"
|
||
try:
|
||
res_ns = subprocess.run(
|
||
["kubectl", "get", "ns", ns, "-o", "name"],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=6,
|
||
)
|
||
if res_ns.returncode != 0 or ns not in (res_ns.stdout or ""):
|
||
return True
|
||
res = subprocess.run(
|
||
["kubectl", "-n", ns, "get", "pods", "--no-headers"],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=6,
|
||
)
|
||
if res.returncode != 0:
|
||
return True
|
||
kong_lines = [
|
||
l for l in (res.stdout or "").splitlines() if "kong" in l.lower()
|
||
]
|
||
if not kong_lines:
|
||
return True
|
||
for line in kong_lines:
|
||
parts = line.split()
|
||
if len(parts) < 3:
|
||
return True
|
||
if parts[2] != "Running":
|
||
return True
|
||
return False
|
||
except Exception:
|
||
return True
|
||
|
||
def _parse_k8s_quantity_bytes(self, raw: str | None) -> int | None:
|
||
"""Parse a Kubernetes resource quantity (e.g. 10Gi, 500Mi) into bytes.
|
||
|
||
Conservative: returns None when parsing fails.
|
||
"""
|
||
s = (raw or "").strip()
|
||
if not s:
|
||
return None
|
||
# Common forms: 10Gi, 500Mi, 1Ti, 1000, 1.5Gi
|
||
num = ""
|
||
suf = ""
|
||
for ch in s:
|
||
if (ch.isdigit() or ch in (".", "+", "-")) and not suf:
|
||
num += ch
|
||
else:
|
||
suf += ch
|
||
try:
|
||
value = float(num)
|
||
except Exception:
|
||
return None
|
||
|
||
suf = suf.strip()
|
||
if not suf:
|
||
try:
|
||
return int(value)
|
||
except Exception:
|
||
return None
|
||
|
||
binary = {
|
||
"Ki": 1024,
|
||
"Mi": 1024**2,
|
||
"Gi": 1024**3,
|
||
"Ti": 1024**4,
|
||
"Pi": 1024**5,
|
||
"Ei": 1024**6,
|
||
}
|
||
decimal = {
|
||
"K": 1000,
|
||
"M": 1000**2,
|
||
"G": 1000**3,
|
||
"T": 1000**4,
|
||
"P": 1000**5,
|
||
"E": 1000**6,
|
||
}
|
||
if suf in binary:
|
||
return int(value * binary[suf])
|
||
if suf in decimal:
|
||
return int(value * decimal[suf])
|
||
# Be conservative: unknown suffix (e.g. m) => do not parse.
|
||
return None
|
||
|
||
def _kubectl_get_json(self, env: dict, args: list[str]) -> dict | None:
|
||
try:
|
||
res = subprocess.run(
|
||
["kubectl"] + list(args),
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=30,
|
||
)
|
||
if res.returncode != 0:
|
||
err = (res.stderr or "").strip() or (res.stdout or "").strip()
|
||
if err:
|
||
self.err(f"[WARN] kubectl {' '.join(args)} failed: {err}")
|
||
return None
|
||
out = (res.stdout or "").strip()
|
||
if not out:
|
||
return None
|
||
return json.loads(out)
|
||
except Exception as e:
|
||
self.err(f"[WARN] kubectl {' '.join(args)} failed: {e}")
|
||
return None
|
||
|
||
def _kubectl_get_namespace_json(self, env: dict, name: str) -> dict | None:
|
||
n = (name or "").strip()
|
||
if not n:
|
||
return None
|
||
return self._kubectl_get_json(env, ["get", "namespace", n, "-o", "json"])
|
||
|
||
def _namespace_is_terminating(self, ns: dict | None) -> bool:
|
||
if not ns:
|
||
return False
|
||
md = ns.get("metadata") or {}
|
||
if (md.get("deletionTimestamp") or "").strip():
|
||
return True
|
||
phase = ((ns.get("status") or {}).get("phase") or "").strip()
|
||
return phase == "Terminating"
|
||
|
||
# ------------------------------------------ blocked cluster reconciliation
|
||
def _classify_blocked_pod(self, env: dict, pod: dict) -> dict:
|
||
"""Classify common pod blockers into stable, testable categories.
|
||
|
||
Returns a dict with:
|
||
- `type`: primary classification (storage-first)
|
||
- `types`: list of all detected classification tags
|
||
"""
|
||
|
||
_ = env # reserved for future use
|
||
meta = pod.get("metadata") or {}
|
||
status = pod.get("status") or {}
|
||
|
||
name = (meta.get("name") or "").strip()
|
||
ns = (meta.get("namespace") or "").strip()
|
||
phase = (status.get("phase") or "").strip()
|
||
|
||
types: list[str] = []
|
||
|
||
# ContainerCreating / ImagePullBackOff etc.
|
||
for cs in status.get("containerStatuses") or []:
|
||
waiting = ((cs.get("state") or {}).get("waiting") or {})
|
||
reason = (waiting.get("reason") or "").strip()
|
||
if reason == "ContainerCreating":
|
||
types.append("container_creating")
|
||
elif reason:
|
||
types.append(f"container_waiting_{reason.lower()}")
|
||
|
||
# Unschedulable reasons.
|
||
for cond in status.get("conditions") or []:
|
||
try:
|
||
if (cond.get("type") or "").strip() != "PodScheduled":
|
||
continue
|
||
if (cond.get("status") or "").strip() != "False":
|
||
continue
|
||
if (cond.get("reason") or "").strip() != "Unschedulable":
|
||
continue
|
||
|
||
msg = ((cond.get("message") or "").strip() or "").lower()
|
||
if "persistent volumes" in msg and "bind" in msg:
|
||
types.append("unschedulable_due_to_pv_binding")
|
||
if "didn't match pod's node affinity" in msg or "node affinity/selector" in msg:
|
||
types.append("unschedulable_due_to_node_affinity")
|
||
if "had untolerated taint" in msg:
|
||
types.append("unschedulable_due_to_untolerated_taint")
|
||
except Exception:
|
||
continue
|
||
|
||
# Primary type: prefer storage/root-cause categories.
|
||
primary = "ok"
|
||
for preferred in (
|
||
"unschedulable_due_to_pv_binding",
|
||
"unschedulable_due_to_node_affinity",
|
||
"unschedulable_due_to_untolerated_taint",
|
||
"container_creating",
|
||
):
|
||
if preferred in types:
|
||
primary = preferred
|
||
break
|
||
if primary == "ok" and phase in ("Pending", "Unknown") and types:
|
||
primary = types[0]
|
||
|
||
# Dedupe while keeping order.
|
||
seen: set[str] = set()
|
||
deduped: list[str] = []
|
||
for t in types:
|
||
if t not in seen:
|
||
seen.add(t)
|
||
deduped.append(t)
|
||
|
||
return {
|
||
"pod": f"{ns}/{name}" if ns and name else (name or ns or "<unknown>"),
|
||
"namespace": ns,
|
||
"name": name,
|
||
"phase": phase,
|
||
"type": primary,
|
||
"types": deduped,
|
||
}
|
||
|
||
def _reconcile_blocked_cluster_state(self, env: dict, service_ns: str) -> None:
|
||
"""Detect hard cluster blockers and raise before running deployment scripts.
|
||
|
||
This is intentionally conservative: it attempts safe PV claimRef repair
|
||
and then surfaces clear diagnostics for scheduling/storage blockers.
|
||
"""
|
||
|
||
svc_ns = (service_ns or "").strip() or self._service_namespace()
|
||
|
||
# Always attempt safe stale Released PV claimRef repair first.
|
||
try:
|
||
self._repair_stale_released_pvs(env, reset=False)
|
||
except Exception as e:
|
||
self.err(f"[WARN] Stale PV claimRef repair failed: {e}")
|
||
|
||
pods_data = self._kubectl_get_json(env, ["get", "pod", "-A", "-o", "json"]) or {}
|
||
pods = pods_data.get("items") or []
|
||
classified = [self._classify_blocked_pod(env, p) for p in pods]
|
||
blocked = [c for c in classified if (c.get("type") or "") != "ok"]
|
||
|
||
# If we can't positively identify blocked pods, do not block deployment.
|
||
# This keeps behavior conservative when `kubectl` is unavailable.
|
||
if not blocked:
|
||
return
|
||
|
||
# Downstream missingness: surface but treat separately.
|
||
missing: list[str] = []
|
||
for kind in ("svc", "deploy"):
|
||
try:
|
||
res = subprocess.run(
|
||
["kubectl", "-n", svc_ns, "get", kind, "knoe-svc-kong"],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
)
|
||
if res.returncode != 0:
|
||
missing.append(f"{kind}/knoe-svc-kong")
|
||
except FileNotFoundError:
|
||
# If kubectl isn't available, don't guess missing resources.
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
all_types: set[str] = set()
|
||
for b in blocked:
|
||
for t in b.get("types") or []:
|
||
all_types.add(t)
|
||
|
||
# Group summary for easier scanning.
|
||
storage_types = [t for t in sorted(all_types) if "pv_binding" in t]
|
||
sched_types = [
|
||
t
|
||
for t in sorted(all_types)
|
||
if "node_affinity" in t or "untolerated_taint" in t
|
||
]
|
||
container_types = [t for t in sorted(all_types) if t.startswith("container_")]
|
||
|
||
lines: list[str] = ["Cluster prerequisites are blocked"]
|
||
if storage_types or any("pv_binding" in (b.get("type") or "") for b in blocked):
|
||
lines.append("\nStorage blockers:")
|
||
for t in storage_types or ["unschedulable_due_to_pv_binding"]:
|
||
lines.append(f"- {t}")
|
||
if sched_types:
|
||
lines.append("\nScheduling blockers:")
|
||
for t in sched_types:
|
||
lines.append(f"- {t}")
|
||
if container_types:
|
||
lines.append("\nContainer blockers:")
|
||
for t in container_types:
|
||
lines.append(f"- {t}")
|
||
if blocked:
|
||
lines.append("\nBlocked pods:")
|
||
for b in blocked:
|
||
lines.append(
|
||
f"- {b.get('pod')} type={b.get('type')} types={','.join(b.get('types') or [])}"
|
||
)
|
||
if missing:
|
||
lines.append("\nDownstream missing resources:")
|
||
for m in sorted(set(missing)):
|
||
lines.append(f"- {m}")
|
||
|
||
raise Exception("\n".join(lines))
|
||
|
||
def _ensure_namespace_ready(self, env: dict, name: str, timeout_s: int = 90) -> None:
|
||
"""Ensure a namespace is usable for deployments.
|
||
|
||
If the namespace is in `Terminating`, wait for deletion to complete.
|
||
If it disappears, recreate it. If it remains terminating after the timeout,
|
||
raise with an actionable message.
|
||
"""
|
||
|
||
n = (name or "").strip()
|
||
if not n:
|
||
return
|
||
|
||
ns = self._kubectl_get_namespace_json(env, n)
|
||
if self._namespace_is_terminating(ns):
|
||
self.err(
|
||
f"[WARN] Namespace '{n}' is terminating; waiting up to {timeout_s}s for deletion to complete..."
|
||
)
|
||
deadline = time.time() + max(0, int(timeout_s))
|
||
while time.time() < deadline:
|
||
time.sleep(3)
|
||
ns = self._kubectl_get_namespace_json(env, n)
|
||
if not ns:
|
||
break
|
||
if not self._namespace_is_terminating(ns):
|
||
break
|
||
|
||
ns = self._kubectl_get_namespace_json(env, n)
|
||
if self._namespace_is_terminating(ns):
|
||
raise Exception(
|
||
f"Namespace '{n}' is still terminating. "
|
||
f"Wait for it to finish deleting (kubectl get namespace {n}) or force-delete it, "
|
||
f"then retry the installer."
|
||
)
|
||
|
||
# Create namespace if missing.
|
||
if not self._kubectl_get_namespace_json(env, n):
|
||
self.log(f"[INFO] Creating namespace '{n}' ...")
|
||
rc = self._run_cmd(["kubectl", "create", "namespace", n], env=env)
|
||
if rc != 0:
|
||
# Be idempotent: a concurrent creator may have raced us.
|
||
self.err(f"[WARN] Failed to create namespace '{n}' (code {rc}); continuing")
|
||
|
||
def _list_pending_pvcs(
|
||
self, env: dict, namespaces: list[str] | None = None
|
||
) -> list[dict]:
|
||
data = self._kubectl_get_json(env, ["get", "pvc", "-A", "-o", "json"])
|
||
items = (data or {}).get("items") or []
|
||
ns_filter = {n.strip() for n in (namespaces or []) if (n or "").strip()}
|
||
out: list[dict] = []
|
||
for pvc in items:
|
||
try:
|
||
phase = ((pvc.get("status") or {}).get("phase") or "").strip()
|
||
if phase != "Pending":
|
||
continue
|
||
ns = ((pvc.get("metadata") or {}).get("namespace") or "").strip()
|
||
if ns_filter and ns not in ns_filter:
|
||
continue
|
||
out.append(pvc)
|
||
except Exception:
|
||
continue
|
||
return out
|
||
|
||
def _list_released_pvs(self, env: dict, storage_class: str | None = None) -> list[dict]:
|
||
data = self._kubectl_get_json(env, ["get", "pv", "-o", "json"])
|
||
items = (data or {}).get("items") or []
|
||
sc = (storage_class or "").strip()
|
||
out: list[dict] = []
|
||
for pv in items:
|
||
try:
|
||
phase = ((pv.get("status") or {}).get("phase") or "").strip()
|
||
if phase != "Released":
|
||
continue
|
||
pv_sc = ((pv.get("spec") or {}).get("storageClassName") or "").strip()
|
||
if sc and pv_sc != sc:
|
||
continue
|
||
out.append(pv)
|
||
except Exception:
|
||
continue
|
||
return out
|
||
|
||
def _match_stale_released_pv(self, pvc: dict, pvs: list[dict]) -> dict | None:
|
||
pvc_meta = pvc.get("metadata") or {}
|
||
pvc_spec = pvc.get("spec") or {}
|
||
pvc_name = (pvc_meta.get("name") or "").strip()
|
||
pvc_ns = (pvc_meta.get("namespace") or "").strip()
|
||
pvc_uid = (pvc_meta.get("uid") or "").strip()
|
||
pvc_sc = (pvc_spec.get("storageClassName") or "").strip()
|
||
pvc_volume_name = (pvc_spec.get("volumeName") or "").strip()
|
||
pvc_modes = pvc_spec.get("accessModes") or []
|
||
pvc_mode = (pvc_spec.get("volumeMode") or "").strip()
|
||
req_storage = (
|
||
(((pvc_spec.get("resources") or {}).get("requests") or {}).get("storage") or "").strip()
|
||
)
|
||
req_bytes = self._parse_k8s_quantity_bytes(req_storage)
|
||
|
||
if not pvc_name or not pvc_ns:
|
||
return None
|
||
|
||
candidates: list[tuple[int, dict]] = []
|
||
for pv in pvs:
|
||
pv_meta = pv.get("metadata") or {}
|
||
pv_spec = pv.get("spec") or {}
|
||
pv_status = pv.get("status") or {}
|
||
pv_name = (pv_meta.get("name") or "").strip()
|
||
if not pv_name:
|
||
continue
|
||
if (pv_status.get("phase") or "").strip() != "Released":
|
||
continue
|
||
|
||
if pvc_volume_name and pvc_volume_name != pv_name:
|
||
continue
|
||
|
||
pv_sc = (pv_spec.get("storageClassName") or "").strip()
|
||
if pvc_sc and pv_sc and pvc_sc != pv_sc:
|
||
continue
|
||
if pvc_sc and not pv_sc:
|
||
continue
|
||
|
||
claim_ref = pv_spec.get("claimRef") or {}
|
||
if not claim_ref:
|
||
continue
|
||
if (claim_ref.get("name") or "").strip() != pvc_name:
|
||
continue
|
||
if (claim_ref.get("namespace") or "").strip() != pvc_ns:
|
||
continue
|
||
claim_uid = (claim_ref.get("uid") or "").strip()
|
||
if pvc_uid and claim_uid and pvc_uid == claim_uid:
|
||
# Too risky: PV believes it is still claimed by this exact PVC.
|
||
continue
|
||
|
||
pv_modes = pv_spec.get("accessModes") or []
|
||
if pvc_modes:
|
||
if not pv_modes:
|
||
continue
|
||
if not set(pvc_modes).issubset(set(pv_modes)):
|
||
continue
|
||
|
||
pv_mode = (pv_spec.get("volumeMode") or "").strip()
|
||
if pvc_mode and pv_mode and pvc_mode != pv_mode:
|
||
continue
|
||
|
||
cap_storage = (((pv_spec.get("capacity") or {}).get("storage") or "").strip())
|
||
cap_bytes = self._parse_k8s_quantity_bytes(cap_storage)
|
||
if req_bytes is not None and cap_bytes is not None:
|
||
if req_bytes > cap_bytes:
|
||
continue
|
||
else:
|
||
# If we can't compare sizes, be conservative.
|
||
continue
|
||
|
||
candidates.append((cap_bytes or 0, pv))
|
||
|
||
if not candidates:
|
||
return None
|
||
candidates.sort(key=lambda t: t[0])
|
||
return candidates[0][1]
|
||
|
||
def _clear_pv_claim_ref(self, env: dict, pv_name: str) -> bool:
|
||
pv_name = (pv_name or "").strip()
|
||
if not pv_name:
|
||
return False
|
||
patch = '{"spec":{"claimRef":null}}'
|
||
try:
|
||
res = subprocess.run(
|
||
[
|
||
"kubectl",
|
||
"patch",
|
||
"pv",
|
||
pv_name,
|
||
"--type",
|
||
"merge",
|
||
"-p",
|
||
patch,
|
||
],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=20,
|
||
)
|
||
if res.returncode == 0:
|
||
return True
|
||
err = (res.stderr or "").strip() or (res.stdout or "").strip()
|
||
if err:
|
||
self.err(f"[WARN] Failed to patch pv/{pv_name} claimRef: {err}")
|
||
return False
|
||
except Exception as e:
|
||
self.err(f"[WARN] Failed to patch pv/{pv_name} claimRef: {e}")
|
||
return False
|
||
|
||
def _repair_stale_released_pvs(self, env: dict, reset: bool = False) -> bool:
|
||
"""Repair stale `Released` PVs that retain an old `spec.claimRef`.
|
||
|
||
Policy:
|
||
- Normal repair is conservative.
|
||
- When `reset=True`, we may proactively reclaim safe, strongly-matched PVs by
|
||
clearing `spec.claimRef`.
|
||
- Reset must NEVER delete PVs.
|
||
"""
|
||
|
||
service_ns = (env.get("SERVICE_NAMESPACE") or "").strip() or self._service_namespace()
|
||
db_ns = (
|
||
(env.get("DB_NAMESPACE") or "").strip()
|
||
or (self._get_input("init_password.db_namespace", "") or "").strip()
|
||
or service_ns
|
||
)
|
||
namespaces = sorted(
|
||
{
|
||
n
|
||
for n in (
|
||
service_ns,
|
||
db_ns,
|
||
(env.get("NAMESPACE") or "").strip(),
|
||
)
|
||
if (n or "").strip()
|
||
}
|
||
)
|
||
|
||
pending = self._list_pending_pvcs(env, namespaces=namespaces)
|
||
if not pending:
|
||
return False
|
||
|
||
# Pull all Released PVs once; matching enforces storageClass/claimRef checks.
|
||
released = self._list_released_pvs(env)
|
||
if not released:
|
||
return False
|
||
|
||
changed = False
|
||
for pvc in pending:
|
||
meta = pvc.get("metadata") or {}
|
||
spec = pvc.get("spec") or {}
|
||
pvc_name = (meta.get("name") or "").strip()
|
||
pvc_ns = (meta.get("namespace") or "").strip()
|
||
pvc_sc = (spec.get("storageClassName") or "").strip()
|
||
req = (((spec.get("resources") or {}).get("requests") or {}).get("storage") or "").strip()
|
||
|
||
match = self._match_stale_released_pv(pvc, released)
|
||
if not match:
|
||
continue
|
||
|
||
pv_name = ((match.get("metadata") or {}).get("name") or "").strip()
|
||
pv_cap = (((match.get("spec") or {}).get("capacity") or {}).get("storage") or "").strip()
|
||
claim_ref = (match.get("spec") or {}).get("claimRef") or {}
|
||
claim_uid = (claim_ref.get("uid") or "").strip()
|
||
|
||
prefix = "[RESET]" if reset else "[INFO]"
|
||
self.log(
|
||
f"{prefix} Detected pending PVC with matching Released PV claimRef; clearing claimRef "
|
||
f"pvc/{pvc_ns}/{pvc_name} storageClass={pvc_sc or '<unset>'} request={req or '<unset>'} "
|
||
f"-> pv/{pv_name} capacity={pv_cap or '<unset>'} claimRef.uid={claim_uid or '<unset>'}"
|
||
)
|
||
if self._clear_pv_claim_ref(env, pv_name):
|
||
self.log(f"[OK] Cleared claimRef on pv/{pv_name}")
|
||
changed = True
|
||
else:
|
||
self.err(f"[WARN] Failed to clear claimRef on pv/{pv_name}")
|
||
|
||
return changed
|
||
|
||
def _reset_reclaim_stale_pvs(self, env: dict) -> bool:
|
||
"""Reset/reconcile hook to reclaim safe stale PVs.
|
||
|
||
Reset may reclaim stale `Released` PVs by clearing `spec.claimRef`.
|
||
Reset must NEVER delete PVs.
|
||
"""
|
||
|
||
return self._repair_stale_released_pvs(env, reset=True)
|
||
|
||
def _run_cluster_repair_pipeline(self, env: dict, anomalies: list[str]) -> None:
|
||
"""Python-native cluster repair orchestration.
|
||
|
||
This mirrors the intent of `etc/repair_pipeline.sh` while delegating
|
||
actual component deployments to existing `etc/init_*.sh` scripts.
|
||
"""
|
||
mode = (env.get("KNOE_MODE") or env.get("DEPLOYMENT_MODE") or "").strip()
|
||
service_ns = (env.get("SERVICE_NAMESPACE") or "").strip() or self._service_namespace()
|
||
db_ns = (
|
||
(env.get("DB_NAMESPACE") or "").strip()
|
||
or (self._get_input("init_password.db_namespace", "") or "").strip()
|
||
or service_ns
|
||
)
|
||
argocd_ns = (env.get("ARGOCD_NAMESPACE") or "").strip() or self._argocd_namespace()
|
||
registry_ns = (env.get("REGISTRY_NAMESPACE") or "").strip() or self._registry_namespace()
|
||
|
||
def _kubectl(args: list[str], timeout: int = 30) -> subprocess.CompletedProcess:
|
||
return subprocess.run(
|
||
["kubectl"] + args,
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=timeout,
|
||
)
|
||
|
||
self.log(f"[INFO] Cluster repair starting (mode={mode or '<unset>'})")
|
||
|
||
# Basic reachability check (non-fatal; keeps behavior conservative).
|
||
try:
|
||
res = _kubectl(["cluster-info"], timeout=15)
|
||
if res.returncode != 0:
|
||
self.err(
|
||
f"[WARN] kubectl cluster-info failed: {((res.stderr or '') or (res.stdout or '')).strip()}"
|
||
)
|
||
except Exception as e:
|
||
self.err(f"[WARN] kubectl cluster-info failed: {e}")
|
||
|
||
# --- Dashboard repair (Helm-based)
|
||
if "dashboard" in (anomalies or []):
|
||
try:
|
||
self.log("[INFO] Repairing kubernetes-dashboard...")
|
||
# Try to create namespace (idempotent)
|
||
_kubectl(["create", "ns", "kubernetes-dashboard"], timeout=20)
|
||
# Helm install/upgrade as in the legacy pipeline
|
||
subprocess.run(
|
||
["helm", "repo", "add", "kubernetes-dashboard", "https://kubernetes.github.io/dashboard/"],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=60,
|
||
)
|
||
subprocess.run(
|
||
["helm", "repo", "update"],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=120,
|
||
)
|
||
subprocess.run(
|
||
[
|
||
"helm",
|
||
"upgrade",
|
||
"--install",
|
||
"kubernetes-dashboard",
|
||
"kubernetes-dashboard/kubernetes-dashboard",
|
||
"--namespace",
|
||
"kubernetes-dashboard",
|
||
"--create-namespace",
|
||
"--set",
|
||
"kong.enabled=true",
|
||
"--set",
|
||
"nginx.enabled=false",
|
||
],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=300,
|
||
)
|
||
# Restart pods (idempotent, best-effort)
|
||
_kubectl(["-n", "kubernetes-dashboard", "delete", "pods", "-l", "app.kubernetes.io/name=kubernetes-dashboard"], timeout=60)
|
||
_kubectl(["-n", "kubernetes-dashboard", "delete", "pods", "-l", "app.kubernetes.io/name=kong"], timeout=60)
|
||
except FileNotFoundError:
|
||
self.err("[WARN] helm not found; skipping dashboard Helm repair")
|
||
except Exception as e:
|
||
self.err(f"[WARN] Dashboard repair failed: {e}")
|
||
|
||
# --- k3s legacy default namespace dedupe
|
||
try:
|
||
if mode == "k3s" and service_ns != "default":
|
||
res_legacy = _kubectl(["get", "ns", "default", "-o", "name"], timeout=10)
|
||
if res_legacy.returncode == 0 and "default" in (res_legacy.stdout or ""):
|
||
legacy_ns = "default"
|
||
|
||
def _exists(kind: str, name: str, ns: str) -> bool:
|
||
r = _kubectl(["-n", ns, "get", kind, name, "-o", "name"], timeout=10)
|
||
return r.returncode == 0
|
||
|
||
def _delete_kind(kind: str, name: str, ns: str):
|
||
_kubectl(["-n", ns, "delete", kind, name, "--ignore-not-found=true"], timeout=60)
|
||
|
||
legacy_openbao = _exists("deploy", "openbao", legacy_ns) or _exists(
|
||
"statefulset", "openbao", legacy_ns
|
||
)
|
||
legacy_opentofu = _exists("deploy", "opentofu", legacy_ns)
|
||
legacy_garage = _exists("statefulset", "garage", legacy_ns)
|
||
legacy_registry = _exists("deploy", "registry", legacy_ns)
|
||
legacy_kong = _exists("deploy", "knoe-svc-kong", legacy_ns) or _exists(
|
||
"svc", "knoe-svc-kong", legacy_ns
|
||
)
|
||
|
||
missing_openbao = not (
|
||
_exists("deploy", "openbao", service_ns)
|
||
or _exists("statefulset", "openbao", service_ns)
|
||
)
|
||
missing_opentofu = not _exists("deploy", "opentofu", service_ns)
|
||
missing_garage = not _exists("statefulset", "garage", service_ns)
|
||
missing_registry = not _exists("deploy", "registry", registry_ns)
|
||
|
||
if legacy_registry and missing_registry:
|
||
self.log(
|
||
f"[INFO] Legacy registry in {legacy_ns}; redeploying to {registry_ns}"
|
||
)
|
||
registry_ops.update(namespace=registry_ns, env=env, mode=mode, log=self.log)
|
||
_delete_kind("deploy", "registry", legacy_ns)
|
||
|
||
if legacy_openbao and missing_openbao:
|
||
self.log(
|
||
f"[INFO] Legacy OpenBao in {legacy_ns}; redeploying to {service_ns}"
|
||
)
|
||
openbao_ops.update(namespace=service_ns, env=env, mode=mode, log=self.log)
|
||
_delete_kind("deploy", "openbao", legacy_ns)
|
||
_delete_kind("statefulset", "openbao", legacy_ns)
|
||
_delete_kind("pvc", "data-openbao-0", legacy_ns)
|
||
|
||
if legacy_opentofu and missing_opentofu:
|
||
self.log(
|
||
f"[INFO] Legacy OpenTofu in {legacy_ns}; redeploying to {service_ns}"
|
||
)
|
||
opentofu_ops.update(namespace=service_ns, env=env, mode=mode, log=self.log)
|
||
_delete_kind("deploy", "opentofu", legacy_ns)
|
||
|
||
if legacy_garage and missing_garage:
|
||
self.log(
|
||
f"[INFO] Legacy Garage in {legacy_ns}; redeploying to {service_ns}"
|
||
)
|
||
garage_store_ops.update(
|
||
namespace=service_ns,
|
||
env=env,
|
||
mode=mode,
|
||
log=self.log,
|
||
)
|
||
_delete_kind("statefulset", "garage", legacy_ns)
|
||
_delete_kind("pvc", "meta-garage-0", legacy_ns)
|
||
_delete_kind("pvc", "data-garage-0", legacy_ns)
|
||
|
||
if legacy_kong:
|
||
missing_kong = not (
|
||
_exists("deploy", "knoe-svc-kong", service_ns)
|
||
or _exists("svc", "knoe-svc-kong", service_ns)
|
||
)
|
||
if missing_kong:
|
||
self.log(
|
||
f"[INFO] Legacy Kong (knoe-svc-kong) in {legacy_ns}; redeploying to {service_ns}"
|
||
)
|
||
self._run_script(
|
||
"init_kong.sh",
|
||
args=["-n", service_ns, "update"],
|
||
env=env,
|
||
)
|
||
|
||
# Always remove legacy Kong to avoid namespace drift.
|
||
_kubectl(
|
||
[
|
||
"-n",
|
||
legacy_ns,
|
||
"delete",
|
||
"pod",
|
||
"-l",
|
||
"app=knoe-svc-kong",
|
||
"--ignore-not-found=true",
|
||
],
|
||
timeout=60,
|
||
)
|
||
_delete_kind("deploy", "knoe-svc-kong", legacy_ns)
|
||
_delete_kind("svc", "knoe-svc-kong", legacy_ns)
|
||
_delete_kind("configmap", "knoe-svc-kong-config", legacy_ns)
|
||
|
||
# Clean up prole-era Kong (named "prole-svc-kong") from both
|
||
# legacy_ns and service_ns — stale after the prole→knoe rebrand.
|
||
legacy_prole_kong = (
|
||
_exists("deploy", "prole-svc-kong", legacy_ns)
|
||
or _exists("svc", "prole-svc-kong", legacy_ns)
|
||
or _exists("deploy", "prole-svc-kong", service_ns)
|
||
or _exists("svc", "prole-svc-kong", service_ns)
|
||
)
|
||
if legacy_prole_kong:
|
||
self.log("[INFO] Cleaning up legacy prole-svc-kong resources")
|
||
for _ns in (legacy_ns, service_ns):
|
||
_kubectl(
|
||
[
|
||
"-n",
|
||
_ns,
|
||
"delete",
|
||
"pod",
|
||
"-l",
|
||
"app=prole-svc-kong",
|
||
"--ignore-not-found=true",
|
||
],
|
||
timeout=60,
|
||
)
|
||
_delete_kind("deploy", "prole-svc-kong", _ns)
|
||
_delete_kind("svc", "prole-svc-kong", _ns)
|
||
_delete_kind("configmap", "prole-svc-kong-config", _ns)
|
||
except Exception as e:
|
||
self.err(f"[WARN] Legacy namespace dedupe failed: {e}")
|
||
|
||
# --- ArgoCD repair (registry redeploy)
|
||
try:
|
||
argocd_apps = [
|
||
("deployment", "argocd-repo-server"),
|
||
("deployment", "argocd-application-controller"),
|
||
("statefulset", "argocd-redis"),
|
||
]
|
||
unhealthy = False
|
||
for kind, name in argocd_apps:
|
||
r = _kubectl(
|
||
[
|
||
"-n",
|
||
argocd_ns,
|
||
"get",
|
||
kind,
|
||
name,
|
||
"-o",
|
||
"jsonpath={.status.readyReplicas}",
|
||
],
|
||
timeout=10,
|
||
)
|
||
if r.returncode != 0:
|
||
unhealthy = True
|
||
break
|
||
try:
|
||
if int((r.stdout or "0").strip() or "0") < 1:
|
||
unhealthy = True
|
||
break
|
||
except Exception:
|
||
unhealthy = True
|
||
break
|
||
|
||
if unhealthy:
|
||
self.log("[WARN] ArgoCD is not healthy; redeploying registry...")
|
||
env2 = env.copy()
|
||
env2["REGISTRY_NAMESPACE"] = registry_ns
|
||
registry_ops.stop(namespace=registry_ns, env=env2, mode=mode, log=self.log)
|
||
registry_ops.update(namespace=registry_ns, env=env2, mode=mode, log=self.log)
|
||
except Exception as e:
|
||
self.err(f"[WARN] ArgoCD repair step failed: {e}")
|
||
|
||
# --- Service restarts in the service namespace
|
||
try:
|
||
# OpenBao
|
||
r = _kubectl(
|
||
[
|
||
"-n",
|
||
service_ns,
|
||
"get",
|
||
"statefulset",
|
||
"openbao",
|
||
"-o",
|
||
"jsonpath={.status.readyReplicas}",
|
||
],
|
||
timeout=10,
|
||
)
|
||
if r.returncode == 0:
|
||
if int((r.stdout or "0").strip() or "0") < 1:
|
||
self.log("[WARN] OpenBao is not ready; restarting...")
|
||
openbao_ops.restart(namespace=service_ns, env=env, log=self.log)
|
||
except Exception:
|
||
pass
|
||
|
||
try:
|
||
# OpenTofu
|
||
r = _kubectl(
|
||
[
|
||
"-n",
|
||
service_ns,
|
||
"get",
|
||
"deploy",
|
||
"opentofu",
|
||
"-o",
|
||
"jsonpath={.status.readyReplicas}",
|
||
],
|
||
timeout=10,
|
||
)
|
||
if r.returncode == 0:
|
||
if int((r.stdout or "0").strip() or "0") < 1:
|
||
self.log("[WARN] OpenTofu is not ready; restarting...")
|
||
opentofu_ops.restart(namespace=service_ns, env=env, log=self.log)
|
||
except Exception:
|
||
pass
|
||
|
||
try:
|
||
# Garage
|
||
r = _kubectl(
|
||
[
|
||
"-n",
|
||
service_ns,
|
||
"get",
|
||
"statefulset",
|
||
"garage",
|
||
"-o",
|
||
"jsonpath={.status.readyReplicas}",
|
||
],
|
||
timeout=10,
|
||
)
|
||
if r.returncode == 0:
|
||
if int((r.stdout or "0").strip() or "0") < 1:
|
||
self.log("[WARN] Garage is not ready; restarting...")
|
||
garage_store_ops.restart(namespace=service_ns, env=env, log=self.log)
|
||
except Exception:
|
||
pass
|
||
|
||
# --- Kerberos / KDC repair (gated)
|
||
try:
|
||
kerberos_enabled = self._get_input_bool(
|
||
"kerberos_config.enabled", False
|
||
) or self._get_input_bool("init_cluster.kerberos_enabled", False)
|
||
if kerberos_enabled:
|
||
auth_missing = self._authority_context_missing()
|
||
if not auth_missing:
|
||
# Only attempt KDC init when authority context exists.
|
||
env2 = env.copy()
|
||
env2["DB_NAMESPACE"] = db_ns
|
||
env2["SERVICE_NAMESPACE"] = service_ns
|
||
self._run_script(
|
||
"init_kdc.sh",
|
||
args=["-n", service_ns],
|
||
env=env2,
|
||
)
|
||
except Exception as e:
|
||
self.err(f"[WARN] KDC repair step failed: {e}")
|
||
|
||
# --- Barman plugin repair (best-effort)
|
||
try:
|
||
r = _kubectl(["-n", "cnpg-system", "get", "deploy", "barman-cloud"], timeout=10)
|
||
env2 = env.copy()
|
||
env2["DATABASE_NAMESPACE"] = db_ns
|
||
env2["CLUSTER_NAME"] = self._cnpg_cluster_name()
|
||
if r.returncode != 0:
|
||
cnpg_install_barman_plugin(env=env2, log=self.log)
|
||
else:
|
||
r2 = _kubectl(
|
||
[
|
||
"-n",
|
||
"cnpg-system",
|
||
"get",
|
||
"deploy",
|
||
"barman-cloud",
|
||
"-o",
|
||
"jsonpath={.status.readyReplicas}",
|
||
],
|
||
timeout=10,
|
||
)
|
||
if r2.returncode == 0:
|
||
try:
|
||
if int((r2.stdout or "0").strip() or "0") < 1:
|
||
cnpg_install_barman_plugin(env=env2, log=self.log)
|
||
_kubectl(["-n", "cnpg-system", "rollout", "restart", "deploy/barman-cloud"], timeout=60)
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
# --- CNPG rollout (best-effort)
|
||
try:
|
||
# Only attempt if a Cluster exists in the DB namespace.
|
||
r = _kubectl(
|
||
["-n", db_ns, "get", "clusters.postgresql.cnpg.io", "cluster", "-o", "name"],
|
||
timeout=10,
|
||
)
|
||
if r.returncode == 0 and (r.stdout or "").strip():
|
||
env2 = env.copy()
|
||
env2["DATABASE_NAMESPACE"] = db_ns
|
||
env2["CLUSTER_NAME"] = self._cnpg_cluster_name()
|
||
cnpg_rollout(
|
||
namespace=db_ns,
|
||
cluster_name=self._cnpg_cluster_name(),
|
||
env=env2,
|
||
log=self.log,
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
# Final health/status report
|
||
try:
|
||
status_ok = {
|
||
"registry": registry_ops.status(
|
||
namespace=self._registry_namespace(),
|
||
env=env,
|
||
mode=env.get("KNOE_MODE") or self._deployment_mode(),
|
||
),
|
||
"openbao": openbao_ops.status(
|
||
namespace=service_ns,
|
||
env=env,
|
||
mode=env.get("KNOE_MODE") or self._deployment_mode(),
|
||
),
|
||
"opentofu": opentofu_ops.status(namespace=service_ns, env=env),
|
||
"garage": garage_store_ops.status(namespace=service_ns, env=env),
|
||
}
|
||
for name, ok in status_ok.items():
|
||
self.log(f"[STATUS] {name}: {'ready' if ok else 'not-ready'}")
|
||
except Exception:
|
||
pass
|
||
|
||
self.log("[OK] Cluster repair pipeline complete")
|
||
|
||
def _maybe_run_repair_pipeline(self, env: dict) -> None:
|
||
# Always attempt stale Released PV claimRef repair first.
|
||
try:
|
||
self._repair_stale_released_pvs(env)
|
||
except Exception as e:
|
||
self.err(f"[WARN] Stale PV claimRef repair failed: {e}")
|
||
|
||
# Best-effort namespace preflight before broader anomaly repairs.
|
||
try:
|
||
self._ensure_namespace_ready(env, self._service_namespace())
|
||
except Exception as e:
|
||
self.err(f"[WARN] Namespace preflight failed: {e}")
|
||
return
|
||
|
||
if self._repair_ran:
|
||
return
|
||
anomalies = []
|
||
if self._dashboard_kong_missing(env):
|
||
anomalies.append("dashboard")
|
||
if self._authority_context_missing():
|
||
anomalies.append("authority")
|
||
if not anomalies:
|
||
return
|
||
self._repair_ran = True
|
||
service_ns = self._service_namespace()
|
||
db_ns = (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip() or "default"
|
||
self.log(
|
||
f"[INFO] Detected anomalies ({', '.join(anomalies)}); running repair pipeline..."
|
||
)
|
||
self._run_cluster_repair_pipeline(env=env, anomalies=anomalies)
|
||
|
||
# ------------------------------------------ image helpers
|
||
def _collect_dependent_images(
|
||
self, include_supabase: bool, include_kerberos_proxy: bool
|
||
) -> list[str]:
|
||
images: set[str] = set()
|
||
|
||
for rel_dir in ("k8s/knoe", "k8s/openbao"):
|
||
base_dir = self.project_root / rel_dir
|
||
if not base_dir.exists():
|
||
continue
|
||
images.update(_collect_images_from_files(list(base_dir.glob("*.yaml"))))
|
||
|
||
if include_supabase:
|
||
supa_home = _resolve_supabase_home(self.project_root)
|
||
if supa_home:
|
||
docker_dir = supa_home / "docker"
|
||
compose_files = [docker_dir / "docker-compose.yml"]
|
||
if os.environ.get("SUPABASE_USE_DEV_COMPOSE") == "1":
|
||
compose_files.append(docker_dir / "dev" / "docker-compose.dev.yml")
|
||
images.update(_collect_images_from_files(compose_files))
|
||
|
||
if not include_supabase:
|
||
images = {img for img in images if "supabase" not in img}
|
||
|
||
if include_kerberos_proxy:
|
||
krb_img = os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat")
|
||
if krb_img:
|
||
images.add(krb_img)
|
||
|
||
return sorted(images)
|
||
|
||
def _get_docker_import_dir(self) -> str:
|
||
"""Best-effort accessor for a docker-import directory.
|
||
|
||
GUI uses a Tk variable with `.get()`, while console flows tend to store a
|
||
plain string. This helper normalizes both.
|
||
"""
|
||
|
||
val = getattr(self, "docker_import_dir", None)
|
||
if val is None:
|
||
return ""
|
||
try:
|
||
getter = getattr(val, "get", None)
|
||
if callable(getter):
|
||
val = getter()
|
||
except Exception:
|
||
pass
|
||
return (str(val) if val is not None else "").strip()
|
||
|
||
def _prepull_images_to_registry(
|
||
self,
|
||
include_supabase: bool,
|
||
include_kerberos_proxy: bool,
|
||
log: Callable[[str], None] | None = None,
|
||
) -> bool:
|
||
def _log(msg: str):
|
||
if log:
|
||
try:
|
||
log(msg)
|
||
except Exception:
|
||
pass
|
||
self.log(msg)
|
||
|
||
cluster_env = ""
|
||
try:
|
||
cluster_env = self._get_input("init_cluster.cluster_env", "")
|
||
except Exception:
|
||
cluster_env = ""
|
||
|
||
if cluster_env and not _local_registry_enabled(cluster_env):
|
||
_log("[SKIP] Local registry disabled; skipping image pre-pull.")
|
||
return True
|
||
|
||
mode = _deployment_mode_from_env(cluster_env)
|
||
if mode == "k3s":
|
||
# Keep behavior consistent across installers: k3s relies on in-cluster
|
||
# image pulls/import paths rather than a local docker registry.
|
||
_log("[SKIP] Image pre-pull is not supported in k3s mode.")
|
||
return True
|
||
|
||
ensure_fn = getattr(self, "ensure_local_registry_available", None)
|
||
if not callable(ensure_fn):
|
||
ensure_fn = getattr(self, "_ensure_local_registry_available", None)
|
||
info = ensure_fn() if callable(ensure_fn) else None
|
||
if not info:
|
||
_log("[SKIP] Local registry unavailable; skipping image pre-pull.")
|
||
return False
|
||
|
||
registry, _cluster_registry = info
|
||
|
||
images = self._collect_dependent_images(
|
||
include_supabase, include_kerberos_proxy
|
||
)
|
||
if not images:
|
||
_log("[SKIP] No dependent images found to pre-pull.")
|
||
return True
|
||
|
||
overall_ok = True
|
||
import_dir = self._get_docker_import_dir()
|
||
|
||
for image in images:
|
||
local_tag = image
|
||
if not image.startswith(f"{registry}/"):
|
||
local_tag = f"{registry}/{image}"
|
||
|
||
# 1. Check if already in local registry
|
||
_log(f"[INFO] Checking if {image} exists in local registry...")
|
||
check_reg = subprocess.run(
|
||
["docker", "pull", local_tag], capture_output=True, text=True
|
||
)
|
||
if check_reg.returncode == 0:
|
||
_log(f"[OK] {image} already exists in local registry as {local_tag}")
|
||
continue
|
||
|
||
# 2. Check if we already have it in local docker daemon
|
||
check_local = subprocess.run(
|
||
["docker", "image", "inspect", image],
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
found = check_local.returncode == 0
|
||
|
||
if not found and import_dir and os.path.isdir(import_dir):
|
||
# 3. Check import directory
|
||
safe_name = image.replace("/", "_").replace(":", "_")
|
||
tar_path = Path(import_dir) / f"{safe_name}.tar"
|
||
if tar_path.exists():
|
||
_log(f"[INFO] Found {tar_path} in import directory, loading...")
|
||
load = subprocess.run(
|
||
["docker", "load", "-i", str(tar_path)],
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
if load.returncode == 0:
|
||
found = True
|
||
else:
|
||
overall_ok = False
|
||
self.err(f"[WARN] Failed to load {tar_path}: {load.stderr}")
|
||
|
||
if not found:
|
||
# 4. Pull from upstream
|
||
_log(f"[INFO] Pulling {image} from upstream...")
|
||
pull = subprocess.run(
|
||
["docker", "pull", image], capture_output=True, text=True
|
||
)
|
||
if pull.returncode != 0:
|
||
overall_ok = False
|
||
self.err(pull.stdout or "")
|
||
self.err(pull.stderr or "")
|
||
self.err(f"[ERROR] docker pull failed for {image}")
|
||
continue
|
||
found = True
|
||
|
||
# If we have the image locally, tag and push to local registry
|
||
if found:
|
||
if local_tag != image:
|
||
tag = subprocess.run(
|
||
["docker", "tag", image, local_tag],
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
if tag.returncode != 0:
|
||
overall_ok = False
|
||
self.err(tag.stdout or "")
|
||
self.err(tag.stderr or "")
|
||
self.err(
|
||
f"[ERROR] docker tag failed for {image} to {local_tag}"
|
||
)
|
||
continue
|
||
|
||
_log(f"[INFO] Pushing {local_tag} to local registry...")
|
||
if not _push_docker_image(local_tag, log_fn=self.log):
|
||
overall_ok = False
|
||
continue
|
||
_log(f"[OK] Stored {image} in local registry as {local_tag}")
|
||
|
||
return overall_ok
|
||
|
||
|
||
class _TeeWriter:
|
||
"""Write to both the original stream and a log file simultaneously."""
|
||
|
||
def __init__(self, original, log_file):
|
||
self._original = original
|
||
self._log_file = log_file
|
||
|
||
def write(self, data):
|
||
self._original.write(data)
|
||
self._original.flush()
|
||
try:
|
||
self._log_file.write(data)
|
||
self._log_file.flush()
|
||
except Exception:
|
||
pass
|
||
return len(data)
|
||
|
||
def flush(self):
|
||
self._original.flush()
|
||
try:
|
||
self._log_file.flush()
|
||
except Exception:
|
||
pass
|
||
|
||
def fileno(self):
|
||
return self._original.fileno()
|
||
|
||
def isatty(self):
|
||
return self._original.isatty()
|
||
|
||
def reconfigure(self, **kwargs):
|
||
if hasattr(self._original, "reconfigure"):
|
||
self._original.reconfigure(**kwargs)
|
||
|
||
def __getattr__(self, name):
|
||
return getattr(self._original, name)
|
||
|
||
|
||
class KnoeConsoleInstaller(KnoeInstaller):
|
||
"""Console-based unattended installer driven by knoe.cfg inputs."""
|
||
|
||
silent = True
|
||
|
||
def __init__(
|
||
self,
|
||
controller: KnoeController,
|
||
cfg_path: str | None = None,
|
||
reset_cluster: bool = False,
|
||
delete_db: bool = False,
|
||
):
|
||
self.controller = controller
|
||
self.project_root = controller.project_root
|
||
self.dependencies = list(inst_config.DEPENDENCIES)
|
||
self.cfg_path = self._normalize_cfg_path(cfg_path)
|
||
# Ensure the controller also has the normalized config path
|
||
self.controller.cfg_path = self.cfg_path
|
||
self.reset_cluster = bool(reset_cluster)
|
||
self.reset_requested = bool(reset_cluster)
|
||
self.delete_db_requested = bool(delete_db)
|
||
# Keep a persistent indicator for reset-oriented reconcile actions (e.g.
|
||
# reclaiming safe stale Released PV claimRefs). `_perform_cluster_reset()`
|
||
# consumes `reset_requested` and clears it, but we still want to run
|
||
# reset reconcile steps later once the cluster is reachable.
|
||
self._reset_reclaim_pvs_requested = bool(reset_cluster)
|
||
self.inputs: dict = {}
|
||
self._init_shared_state()
|
||
self._db_built_success = False
|
||
self._scripts_success = False
|
||
self._cnpg_success = False
|
||
self.docker_import_dir = None
|
||
self._openbao_init_env = None
|
||
self._openbao_init_password = None
|
||
self._log_file = None
|
||
|
||
def enable_log_file(self, path: str):
|
||
"""Enable tee-style logging: output goes to both stdout and the log file."""
|
||
self._log_file = open(path, "a")
|
||
sys.stdout = _TeeWriter(sys.__stdout__, self._log_file)
|
||
sys.stderr = _TeeWriter(sys.__stderr__, self._log_file)
|
||
self.log(f"[LOG] Logging to {path}")
|
||
|
||
# ---- Data access (from inputs dict) ----
|
||
def _get_input(self, key: str, default: str | None = None) -> str:
|
||
if key in self.inputs:
|
||
return _safe_str(self.inputs[key])
|
||
return default if default is not None else ""
|
||
|
||
# ---------------- Config helpers ----------------
|
||
def _normalize_cfg_path(self, raw: str | None) -> Path:
|
||
if raw:
|
||
p = Path(_expand_path(raw))
|
||
if p.is_dir():
|
||
return p / "knoe.cfg"
|
||
return p
|
||
conf_dir = knoe_conf_mgr.resolve_knoe_conf_dir(self.project_root)
|
||
return knoe_conf_mgr.entrypoint_path(conf_dir)
|
||
|
||
def _load_inputs_from_cfg(self) -> dict:
|
||
# Ensure env-dir layout exists and legacy entrypoint config is migrated before loading.
|
||
try:
|
||
if self.cfg_path.name == "knoe.cfg":
|
||
if self.cfg_path.parent.name in knoe_conf_mgr.ENVIRONMENTS:
|
||
knoe_conf_mgr.ensure_entrypoint(self.cfg_path.parent.parent)
|
||
else:
|
||
knoe_conf_mgr.ensure_entrypoint(self.cfg_path.parent)
|
||
except Exception:
|
||
pass
|
||
|
||
if not self.cfg_path.exists():
|
||
raise FileNotFoundError(f"knoe.cfg not found at {self.cfg_path}")
|
||
cfg = knoe_conf_mgr.load_layered_config(self.cfg_path)
|
||
cfg_vars = _collect_cfg_vars(cfg)
|
||
|
||
# Load all sections into knoe_cfg_data to maintain idempotency
|
||
for section in cfg.sections():
|
||
if section not in self.knoe_cfg_data:
|
||
self.knoe_cfg_data[section] = {}
|
||
for k, v in cfg.items(section):
|
||
self.knoe_cfg_data[section][k] = _expand_cfg_value(v, cfg_vars)
|
||
|
||
if cfg.has_section("Global"):
|
||
service_ns = _expand_cfg_value(
|
||
cfg.get("Global", "SERVICE_NAMESPACE", fallback=""), cfg_vars
|
||
).strip()
|
||
if service_ns:
|
||
self.knoe_cfg_data["Global"]["SERVICE_NAMESPACE"] = service_ns
|
||
if cfg.has_section("Port Forwards"):
|
||
self.knoe_cfg_data["Port Forwards"] = {
|
||
k: _expand_cfg_value(v, cfg_vars) for k, v in cfg.items("Port Forwards")
|
||
}
|
||
for (section, key), _spec in SECRET_KEY_SPECS.items():
|
||
if cfg.has_option(section, key):
|
||
val = _expand_cfg_value(
|
||
cfg.get(section, key, fallback="").strip(), cfg_vars
|
||
)
|
||
if val:
|
||
self._cfg_secret_cache[(section, key)] = val
|
||
if _is_openbao_ref(val):
|
||
self._secrets_finalized = True
|
||
self._auto_input_keys = set()
|
||
auto_kdc = None
|
||
auto_enabled = None
|
||
if cfg.has_section("Network"):
|
||
sec = cfg["Network"]
|
||
if "KDC_AUTO_DETECTED" in sec:
|
||
auto_kdc = sec.get("KDC_AUTO_DETECTED", "")
|
||
if "KERBEROS_AUTO_ENABLED" in sec:
|
||
auto_enabled = sec.get("KERBEROS_AUTO_ENABLED", "")
|
||
|
||
def mark_auto(inputs: dict):
|
||
if auto_kdc:
|
||
if (
|
||
inputs.get("kerberos_config.kdc", "").strip()
|
||
== str(auto_kdc).strip()
|
||
):
|
||
self._auto_input_keys.add("kerberos_config.kdc")
|
||
if auto_enabled is not None:
|
||
input_enabled = inputs.get("kerberos_config.enabled")
|
||
input_bool = _parse_bool(input_enabled, None)
|
||
auto_bool = _parse_bool(auto_enabled, None)
|
||
if (
|
||
input_bool is not None
|
||
and auto_bool is not None
|
||
and input_bool == auto_bool
|
||
):
|
||
self._auto_input_keys.add("kerberos_config.enabled")
|
||
|
||
def _first_non_empty(*values: str) -> str:
|
||
for value in values:
|
||
candidate = str(value or "").strip()
|
||
if candidate:
|
||
return candidate
|
||
return ""
|
||
|
||
def _cfg_first_non_empty(section_name: str, *keys: str) -> str:
|
||
if not cfg.has_section(section_name):
|
||
return ""
|
||
section = cfg[section_name]
|
||
for key in keys:
|
||
if key not in section:
|
||
continue
|
||
candidate = _expand_cfg_value(section.get(key, ""), cfg_vars).strip()
|
||
if candidate:
|
||
return candidate
|
||
return ""
|
||
|
||
def _hydrate_auth_oidc_inputs(target: dict[str, str]) -> None:
|
||
if not str(target.get("auth.clientId", "") or "").strip():
|
||
client_id = _first_non_empty(
|
||
target.get("oidc.clientId", ""),
|
||
_cfg_first_non_empty(
|
||
"Auth",
|
||
"clientId",
|
||
"CLIENT_ID",
|
||
"OIDC_CLIENT_ID",
|
||
"GOOGLE_OIDC_CLIENT_ID",
|
||
"clientIdRef",
|
||
"CLIENT_ID_REF",
|
||
),
|
||
_cfg_first_non_empty(
|
||
"Global",
|
||
"GITLAB_OIDC_CLIENT_ID",
|
||
"OIDC_CLIENT_ID",
|
||
"GOOGLE_OIDC_CLIENT_ID",
|
||
"OIDC_CLIENT_ID_REF",
|
||
"GOOGLE_OIDC_CLIENT_ID_REF",
|
||
),
|
||
os.environ.get("GITLAB_OIDC_CLIENT_ID", ""),
|
||
os.environ.get("OIDC_CLIENT_ID", ""),
|
||
os.environ.get("GOOGLE_OIDC_CLIENT_ID", ""),
|
||
)
|
||
if client_id:
|
||
target["auth.clientId"] = client_id
|
||
|
||
if not str(target.get("auth.clientSecret", "") or "").strip():
|
||
client_secret = _first_non_empty(
|
||
target.get("oidc.clientSecret", ""),
|
||
_cfg_first_non_empty(
|
||
"Auth",
|
||
"clientSecret",
|
||
"CLIENT_SECRET",
|
||
"OIDC_CLIENT_SECRET",
|
||
"GOOGLE_OIDC_CLIENT_SECRET",
|
||
"clientSecretRef",
|
||
"CLIENT_SECRET_REF",
|
||
),
|
||
_cfg_first_non_empty(
|
||
"Global",
|
||
"GITLAB_OIDC_CLIENT_SECRET",
|
||
"OIDC_CLIENT_SECRET",
|
||
"GOOGLE_OIDC_CLIENT_SECRET",
|
||
"OIDC_CLIENT_SECRET_REF",
|
||
"GOOGLE_OIDC_CLIENT_SECRET_REF",
|
||
),
|
||
os.environ.get("GITLAB_OIDC_CLIENT_SECRET", ""),
|
||
os.environ.get("OIDC_CLIENT_SECRET", ""),
|
||
os.environ.get("GOOGLE_OIDC_CLIENT_SECRET", ""),
|
||
)
|
||
if client_secret:
|
||
target["auth.clientSecret"] = client_secret
|
||
|
||
inputs: dict[str, str] = {}
|
||
if cfg.has_section("Inputs"):
|
||
inputs.update(
|
||
{k: _expand_cfg_value(v, cfg_vars) for k, v in cfg.items("Inputs")}
|
||
)
|
||
mark_auto(inputs)
|
||
# Resolve encrypted/anchored secrets for runtime use
|
||
for key in (
|
||
"init_password.db_password",
|
||
"init_password.db_password_confirm",
|
||
"kerberos_config.password",
|
||
"init_cluster.k3s_token",
|
||
"auth.clientId",
|
||
"auth.clientSecret",
|
||
):
|
||
if key in inputs and inputs[key]:
|
||
inputs[key] = self._resolve_secret_value(inputs[key])
|
||
if inputs.get("init_password.db_password") and not inputs.get(
|
||
"init_password.db_password_confirm"
|
||
):
|
||
inputs["init_password.db_password_confirm"] = inputs[
|
||
"init_password.db_password"
|
||
]
|
||
# Allow explicit runtime override of DB password in silent mode.
|
||
env_db_pw = os.environ.get("KNOE_DB_PASSWORD") or os.environ.get(
|
||
"DB_PASSWORD"
|
||
)
|
||
if env_db_pw:
|
||
raw_db_pw = inputs.get("init_password.db_password", "")
|
||
if (
|
||
not raw_db_pw
|
||
or _is_openbao_ref(raw_db_pw)
|
||
or _is_knoe_secret(raw_db_pw)
|
||
):
|
||
inputs["init_password.db_password"] = env_db_pw
|
||
inputs["init_password.db_password_confirm"] = env_db_pw
|
||
# Fill Ollama inputs from config sections if missing
|
||
if cfg.has_section("Ollama"):
|
||
sec = cfg["Ollama"]
|
||
if (
|
||
"ollama_config.server_host" not in inputs
|
||
and sec.get("OLLAMA_SERVER_HOST", "").strip()
|
||
):
|
||
inputs["ollama_config.server_host"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars
|
||
)
|
||
if (
|
||
"ollama_config.server_port" not in inputs
|
||
and sec.get("OLLAMA_SERVER_PORT", "").strip()
|
||
):
|
||
inputs["ollama_config.server_port"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars
|
||
)
|
||
if (
|
||
"ollama_config.model" not in inputs
|
||
and sec.get("OLLAMA_MODEL", "").strip()
|
||
):
|
||
inputs["ollama_config.model"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_MODEL", ""), cfg_vars
|
||
)
|
||
if (
|
||
"ollama_config.server_host" not in inputs
|
||
and sec.get("OLLAMA_HOST", "").strip()
|
||
):
|
||
host, port = _parse_ollama_host(
|
||
_expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars)
|
||
)
|
||
if host:
|
||
inputs["ollama_config.server_host"] = host
|
||
if port and "ollama_config.server_port" not in inputs:
|
||
inputs["ollama_config.server_port"] = port
|
||
if cfg.has_section("Global"):
|
||
sec = cfg["Global"]
|
||
if (
|
||
"ollama_config.server_host" not in inputs
|
||
and sec.get("OLLAMA_SERVER_HOST", "").strip()
|
||
):
|
||
inputs["ollama_config.server_host"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars
|
||
)
|
||
if (
|
||
"ollama_config.server_port" not in inputs
|
||
and sec.get("OLLAMA_SERVER_PORT", "").strip()
|
||
):
|
||
inputs["ollama_config.server_port"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars
|
||
)
|
||
if (
|
||
"ollama_config.model" not in inputs
|
||
and sec.get("OLLAMA_MODEL", "").strip()
|
||
):
|
||
inputs["ollama_config.model"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_MODEL", ""), cfg_vars
|
||
)
|
||
if (
|
||
"ollama_config.server_host" not in inputs
|
||
and sec.get("OLLAMA_HOST", "").strip()
|
||
):
|
||
host, port = _parse_ollama_host(
|
||
_expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars)
|
||
)
|
||
if host:
|
||
inputs["ollama_config.server_host"] = host
|
||
if port and "ollama_config.server_port" not in inputs:
|
||
inputs["ollama_config.server_port"] = port
|
||
_hydrate_auth_oidc_inputs(inputs)
|
||
return inputs
|
||
# Legacy fallback mapping
|
||
legacy = {}
|
||
if cfg.has_section("System Environment"):
|
||
sec = cfg["System Environment"]
|
||
for k in (
|
||
"KNOE_HOME",
|
||
"KNOE_CONF",
|
||
"PROLE_DATA",
|
||
"PROLE_LOGS",
|
||
"KNOE_SERVICE",
|
||
):
|
||
if k in sec:
|
||
raw = (sec.get(k, "") or "").strip()
|
||
# Keep shell-style variables (e.g. $HOME) literal in cfg.
|
||
if raw:
|
||
legacy[f"env_setup.{k}"] = raw
|
||
if cfg.has_section("Global"):
|
||
sec = cfg["Global"]
|
||
if "KNOE_HOME" in sec:
|
||
raw = (sec.get("KNOE_HOME", "") or "").strip()
|
||
if raw:
|
||
legacy["env_setup.KNOE_HOME"] = raw
|
||
if "CLUSTER_ENV" in sec:
|
||
legacy["init_cluster.cluster_env"] = _expand_cfg_value(
|
||
sec.get("CLUSTER_ENV", ""), cfg_vars
|
||
)
|
||
if "DATABASE_NAMESPACE" in sec:
|
||
db_ns = _expand_cfg_value(sec.get("DATABASE_NAMESPACE", ""), cfg_vars)
|
||
legacy["init_password.db_namespace"] = db_ns
|
||
legacy["env_setup.DATABASE_NAMESPACE"] = db_ns
|
||
if "K3S_SERVER" in sec:
|
||
legacy["init_cluster.k3s_server_url"] = _expand_cfg_value(
|
||
sec.get("K3S_SERVER", ""), cfg_vars
|
||
)
|
||
if "K3S_SERVER_URL" in sec and "init_cluster.k3s_server_url" not in legacy:
|
||
legacy["init_cluster.k3s_server_url"] = _expand_cfg_value(
|
||
sec.get("K3S_SERVER_URL", ""), cfg_vars
|
||
)
|
||
if "K3S_TOKEN" in sec:
|
||
legacy["init_cluster.k3s_token"] = _expand_cfg_value(
|
||
sec.get("K3S_TOKEN", ""), cfg_vars
|
||
)
|
||
if "K3S_TOKEN" in sec and "init_cluster.k3s_token" not in legacy:
|
||
legacy["init_cluster.k3s_token"] = _expand_cfg_value(
|
||
sec.get("K3S_TOKEN", ""), cfg_vars
|
||
)
|
||
if "NAMESPACE" in sec and "init_password.db_namespace" not in legacy:
|
||
db_ns = _expand_cfg_value(sec.get("NAMESPACE", ""), cfg_vars)
|
||
legacy["init_password.db_namespace"] = db_ns
|
||
legacy["env_setup.DATABASE_NAMESPACE"] = db_ns
|
||
if "CLUSTER_NAME" in sec:
|
||
cluster_name = _expand_cfg_value(sec.get("CLUSTER_NAME", ""), cfg_vars)
|
||
legacy["init_password.cluster_name"] = cluster_name
|
||
legacy["env_setup.CLUSTER_NAME"] = cluster_name
|
||
legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name)
|
||
legacy.setdefault("init_password.db_cluster_name", cluster_name)
|
||
if "CNPG_CLUSTER_NAME" in sec and "init_password.cluster_name" not in legacy:
|
||
cluster_name = _expand_cfg_value(
|
||
sec.get("CNPG_CLUSTER_NAME", ""), cfg_vars
|
||
)
|
||
legacy["init_password.cluster_name"] = cluster_name
|
||
legacy["env_setup.CLUSTER_NAME"] = cluster_name
|
||
legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name)
|
||
legacy.setdefault("init_password.db_cluster_name", cluster_name)
|
||
if "APP_CLUSTER_NAME" in sec:
|
||
app_cluster_name = _expand_cfg_value(
|
||
sec.get("APP_CLUSTER_NAME", ""), cfg_vars
|
||
)
|
||
if app_cluster_name:
|
||
legacy["env_setup.APP_CLUSTER_NAME"] = app_cluster_name
|
||
legacy["init_password.app_cluster_name"] = app_cluster_name
|
||
if "APP_CLUSTER_MODE" in sec:
|
||
legacy["env_setup.APP_CLUSTER_MODE"] = _expand_cfg_value(
|
||
sec.get("APP_CLUSTER_MODE", ""), cfg_vars
|
||
)
|
||
legacy["init_cluster.app_cluster_mode"] = legacy[
|
||
"env_setup.APP_CLUSTER_MODE"
|
||
]
|
||
if "DB_CLUSTER_NAME" in sec:
|
||
db_cluster_name = _expand_cfg_value(
|
||
sec.get("DB_CLUSTER_NAME", ""), cfg_vars
|
||
)
|
||
if db_cluster_name:
|
||
legacy["env_setup.DB_CLUSTER_NAME"] = db_cluster_name
|
||
legacy["init_password.db_cluster_name"] = db_cluster_name
|
||
if "DB_CLUSTER_MODE" in sec:
|
||
legacy["env_setup.DB_CLUSTER_MODE"] = _expand_cfg_value(
|
||
sec.get("DB_CLUSTER_MODE", ""), cfg_vars
|
||
)
|
||
legacy["init_cluster.db_cluster_mode"] = legacy["env_setup.DB_CLUSTER_MODE"]
|
||
if "DB_CLUSTER_NODE_COUNT" in sec:
|
||
legacy["init_cluster.db_cluster_node_count"] = _expand_cfg_value(
|
||
sec.get("DB_CLUSTER_NODE_COUNT", ""), cfg_vars
|
||
)
|
||
if "DB_CLUSTER_MACHINE_TYPE" in sec:
|
||
legacy["init_cluster.db_cluster_machine_type"] = _expand_cfg_value(
|
||
sec.get("DB_CLUSTER_MACHINE_TYPE", ""), cfg_vars
|
||
)
|
||
if "DB_BOOT_DISK_TYPE" in sec:
|
||
legacy["init_cluster.db_boot_disk_type"] = _expand_cfg_value(
|
||
sec.get("DB_BOOT_DISK_TYPE", ""), cfg_vars
|
||
)
|
||
if "DB_BOOT_DISK_SIZE_GB" in sec:
|
||
legacy["init_cluster.db_boot_disk_size_gb"] = _expand_cfg_value(
|
||
sec.get("DB_BOOT_DISK_SIZE_GB", ""), cfg_vars
|
||
)
|
||
if "DB_CLUSTER_REGION" in sec:
|
||
legacy["init_cluster.db_cluster_region"] = _expand_cfg_value(
|
||
sec.get("DB_CLUSTER_REGION", ""), cfg_vars
|
||
)
|
||
if "APP_CLUSTER_KUBECONTEXT" in sec:
|
||
kubecontext = _expand_cfg_value(
|
||
sec.get("APP_CLUSTER_KUBECONTEXT", ""), cfg_vars
|
||
)
|
||
legacy["env_setup.APP_CLUSTER_KUBECONTEXT"] = kubecontext
|
||
legacy["init_cluster.app_cluster_kubecontext"] = kubecontext
|
||
if "DB_CLUSTER_KUBECONTEXT" in sec:
|
||
kubecontext = _expand_cfg_value(
|
||
sec.get("DB_CLUSTER_KUBECONTEXT", ""), cfg_vars
|
||
)
|
||
legacy["env_setup.DB_CLUSTER_KUBECONTEXT"] = kubecontext
|
||
legacy["init_cluster.db_cluster_kubecontext"] = kubecontext
|
||
if "KNOE_DB_USER" in sec:
|
||
legacy["init_password.db_username"] = _expand_cfg_value(
|
||
sec.get("KNOE_DB_USER", ""), cfg_vars
|
||
)
|
||
if "DB_PASSWORD" in sec:
|
||
legacy["init_password.db_password"] = _expand_cfg_value(
|
||
sec.get("DB_PASSWORD", ""), cfg_vars
|
||
)
|
||
legacy["init_password.db_password_confirm"] = _expand_cfg_value(
|
||
sec.get("DB_PASSWORD", ""), cfg_vars
|
||
)
|
||
if "DB_HOST_PORT" in sec:
|
||
legacy["init_password.db_host_port"] = _expand_cfg_value(
|
||
sec.get("DB_HOST_PORT", "5432"), cfg_vars
|
||
)
|
||
if "DOCKER_IMPORT_DIR" in sec:
|
||
self.docker_import_dir = _expand_cfg_value(
|
||
sec.get("DOCKER_IMPORT_DIR", ""), cfg_vars
|
||
)
|
||
self.knoe_cfg_data["Global"][
|
||
"DOCKER_IMPORT_DIR"
|
||
] = self.docker_import_dir
|
||
if "OLLAMA_SERVER_HOST" in sec:
|
||
legacy["ollama_config.server_host"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars
|
||
)
|
||
if "OLLAMA_SERVER_PORT" in sec:
|
||
legacy["ollama_config.server_port"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars
|
||
)
|
||
if "OLLAMA_MODEL" in sec:
|
||
legacy["ollama_config.model"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_MODEL", ""), cfg_vars
|
||
)
|
||
if "OLLAMA_HOST" in sec and "ollama_config.server_host" not in legacy:
|
||
host, port = _parse_ollama_host(
|
||
_expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars)
|
||
)
|
||
if host:
|
||
legacy["ollama_config.server_host"] = host
|
||
if port:
|
||
legacy.setdefault("ollama_config.server_port", port)
|
||
if cfg.has_section("Network"):
|
||
sec = cfg["Network"]
|
||
if "KDC_AUTO_DETECTED" in sec:
|
||
legacy["kerberos_config.kdc"] = _expand_cfg_value(
|
||
sec.get("KDC_AUTO_DETECTED", ""), cfg_vars
|
||
)
|
||
if "KERBEROS_AUTO_ENABLED" in sec:
|
||
legacy["kerberos_config.enabled"] = _expand_cfg_value(
|
||
sec.get("KERBEROS_AUTO_ENABLED", ""), cfg_vars
|
||
)
|
||
if cfg.has_section("Kerberos Authentication"):
|
||
sec = cfg["Kerberos Authentication"]
|
||
for k, tgt in (
|
||
("ENABLED", "kerberos_config.enabled"),
|
||
("REALM", "kerberos_config.realm"),
|
||
("KDC", "kerberos_config.kdc"),
|
||
("USER", "kerberos_config.user"),
|
||
("PASSWORD", "kerberos_config.password"),
|
||
):
|
||
if k in sec:
|
||
legacy[tgt] = _expand_cfg_value(sec.get(k, ""), cfg_vars)
|
||
if cfg.has_section("Ollama"):
|
||
sec = cfg["Ollama"]
|
||
if "OLLAMA_SERVER_HOST" in sec:
|
||
legacy["ollama_config.server_host"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_HOST", ""), cfg_vars
|
||
)
|
||
if "OLLAMA_SERVER_PORT" in sec:
|
||
legacy["ollama_config.server_port"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_SERVER_PORT", ""), cfg_vars
|
||
)
|
||
if "OLLAMA_MODEL" in sec:
|
||
legacy["ollama_config.model"] = _expand_cfg_value(
|
||
sec.get("OLLAMA_MODEL", ""), cfg_vars
|
||
)
|
||
if "OLLAMA_HOST" in sec and "ollama_config.server_host" not in legacy:
|
||
host, port = _parse_ollama_host(
|
||
_expand_cfg_value(sec.get("OLLAMA_HOST", ""), cfg_vars)
|
||
)
|
||
if host:
|
||
legacy["ollama_config.server_host"] = host
|
||
if port:
|
||
legacy.setdefault("ollama_config.server_port", port)
|
||
if cfg.has_section("GitOps"):
|
||
sec = cfg["GitOps"]
|
||
if "NAMESPACE" in sec:
|
||
legacy["gitops.namespace"] = _expand_cfg_value(
|
||
sec.get("NAMESPACE", ""), cfg_vars
|
||
)
|
||
if cfg.has_section("Optional Features"):
|
||
sec = cfg["Optional Features"]
|
||
if "SUPABASE_ENABLED" in sec:
|
||
legacy["init_cluster.supabase_enabled"] = sec.get(
|
||
"SUPABASE_ENABLED", ""
|
||
)
|
||
if "SUPABASE_STUDIO_ENABLED" in sec:
|
||
legacy["init_cluster.supabase_studio_enabled"] = sec.get(
|
||
"SUPABASE_STUDIO_ENABLED", ""
|
||
)
|
||
if "SUPABASE_STUDIO_URL" in sec:
|
||
legacy["init_cluster.supabase_studio_url"] = sec.get(
|
||
"SUPABASE_STUDIO_URL", ""
|
||
)
|
||
if "SUPABASE_AUTH_ENABLED" in sec:
|
||
legacy["init_cluster.supabase_auth_enabled"] = sec.get(
|
||
"SUPABASE_AUTH_ENABLED", ""
|
||
)
|
||
if "SUPABASE_REALTIME_ENABLED" in sec:
|
||
legacy["init_cluster.supabase_realtime_enabled"] = sec.get(
|
||
"SUPABASE_REALTIME_ENABLED", ""
|
||
)
|
||
if "SUPABASE_META_ENABLED" in sec:
|
||
legacy["init_cluster.supabase_meta_enabled"] = sec.get(
|
||
"SUPABASE_META_ENABLED", ""
|
||
)
|
||
if "SUPABASE_ANALYTICS_ENABLED" in sec:
|
||
legacy["init_cluster.supabase_analytics_enabled"] = sec.get(
|
||
"SUPABASE_ANALYTICS_ENABLED", ""
|
||
)
|
||
if "GITOPS_ENABLED" in sec:
|
||
legacy["init_cluster.gitops_enabled"] = sec.get(
|
||
"GITOPS_ENABLED", ""
|
||
)
|
||
if "KERBEROS_ENABLED" in sec:
|
||
legacy["init_cluster.kerberos_enabled"] = sec.get(
|
||
"KERBEROS_ENABLED", ""
|
||
)
|
||
legacy["kerberos_config.enabled"] = sec.get("KERBEROS_ENABLED", "")
|
||
if "AT_REST_ENCRYPTION_ENABLED" in sec:
|
||
legacy["init_cluster.at_rest_encryption_enabled"] = sec.get(
|
||
"AT_REST_ENCRYPTION_ENABLED", ""
|
||
)
|
||
if cfg.has_section("Initialize Cluster"):
|
||
sec = cfg["Initialize Cluster"]
|
||
if "ENVIRONMENT" in sec:
|
||
legacy["init_cluster.cluster_env"] = sec.get("ENVIRONMENT", "")
|
||
if "K3S_SERVER_URL" in sec and "init_cluster.k3s_server_url" not in legacy:
|
||
legacy["init_cluster.k3s_server_url"] = sec.get("K3S_SERVER_URL", "")
|
||
if "K3S_TOKEN" in sec and "init_cluster.k3s_token" not in legacy:
|
||
legacy["init_cluster.k3s_token"] = sec.get("K3S_TOKEN", "")
|
||
if cfg.has_section("Database Creation"):
|
||
sec = cfg["Database Creation"]
|
||
db_ns = ""
|
||
for k in ("DB_NAME", "NAMESPACE", "DATABASE_NAMESPACE"):
|
||
if k in sec:
|
||
db_ns = sec.get(k, "")
|
||
if db_ns:
|
||
legacy["init_password.db_namespace"] = db_ns
|
||
legacy["env_setup.DATABASE_NAMESPACE"] = db_ns
|
||
if "CLUSTER_NAME" in sec:
|
||
cluster_name = sec.get("CLUSTER_NAME", "")
|
||
legacy["init_password.cluster_name"] = cluster_name
|
||
legacy["env_setup.CLUSTER_NAME"] = cluster_name
|
||
if "DB_USER" in sec:
|
||
legacy["init_password.db_username"] = sec.get("DB_USER", "")
|
||
_hydrate_auth_oidc_inputs(legacy)
|
||
mark_auto(legacy)
|
||
for key in (
|
||
"init_password.db_password",
|
||
"init_password.db_password_confirm",
|
||
"kerberos_config.password",
|
||
"init_cluster.k3s_token",
|
||
"auth.clientId",
|
||
"auth.clientSecret",
|
||
):
|
||
if key in legacy and legacy[key]:
|
||
legacy[key] = self._resolve_secret_value(legacy[key])
|
||
if legacy.get("init_password.db_password") and not legacy.get(
|
||
"init_password.db_password_confirm"
|
||
):
|
||
legacy["init_password.db_password_confirm"] = legacy[
|
||
"init_password.db_password"
|
||
]
|
||
return legacy
|
||
|
||
def _default_inputs(self) -> dict:
|
||
namespace = self._initial_db_namespace()
|
||
cluster_name = self._cnpg_cluster_name()
|
||
owner = self._get_local_owner()
|
||
env_vals = self._env_defaults(namespace)
|
||
inputs: dict[str, str] = {}
|
||
|
||
# Dependencies
|
||
inputs["dependencies.verify_all"] = _bool_str(False)
|
||
inputs["dependencies.auto_install_missing"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("dependencies.auto_install_missing", True)
|
||
)
|
||
for dep in self.dependencies:
|
||
inputs[f"dependencies.{dep['id']}.install"] = _bool_str(True)
|
||
|
||
# Network scan
|
||
inputs["network_scan.run"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("network_scan.run", True)
|
||
)
|
||
|
||
# Environment setup
|
||
for k in (
|
||
"KNOE_HOME",
|
||
"KNOE_CONF",
|
||
"PROLE_DATA",
|
||
"PROLE_LOGS",
|
||
"KNOE_SERVICE",
|
||
):
|
||
inputs[f"env_setup.{k}"] = env_vals.get(k, "")
|
||
inputs["env_setup.DATABASE_NAMESPACE"] = namespace
|
||
inputs["env_setup.CLUSTER_NAME"] = cluster_name
|
||
inputs["env_setup.APP_CLUSTER_NAME"] = DEFAULT_APP_CLUSTER_NAME
|
||
inputs["env_setup.APP_CLUSTER_MODE"] = DEFAULT_APP_CLUSTER_MODE
|
||
inputs["env_setup.DB_CLUSTER_NAME"] = DEFAULT_DB_CLUSTER_NAME
|
||
inputs["env_setup.DB_CLUSTER_MODE"] = DEFAULT_DB_CLUSTER_MODE
|
||
inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = ""
|
||
inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = ""
|
||
|
||
# Database creation
|
||
inputs["init_password.db_namespace"] = namespace
|
||
inputs["init_password.cluster_name"] = cluster_name
|
||
inputs["init_password.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME
|
||
inputs["init_password.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME
|
||
inputs["init_password.db_username"] = owner
|
||
inputs["init_password.db_password"] = ""
|
||
inputs["init_password.db_password_confirm"] = ""
|
||
inputs["init_password.generate_ssh_key"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("init_password.generate_ssh_key", True)
|
||
)
|
||
inputs["init_password.db_host_port"] = "5432"
|
||
|
||
# Build DB image
|
||
inputs["init_db_build.run_build"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True)
|
||
)
|
||
|
||
# Cluster init + optional features
|
||
inputs["init_cluster.cluster_env"] = "dev"
|
||
inputs["init_cluster.k3s_server_url"] = ""
|
||
inputs["init_cluster.k3s_token"] = ""
|
||
inputs["init_cluster.supabase_enabled"] = _bool_str(False)
|
||
inputs["init_cluster.supabase_studio_enabled"] = _bool_str(False)
|
||
inputs["init_cluster.supabase_studio_url"] = "db.0.knoe.dev"
|
||
inputs["init_cluster.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME
|
||
inputs["init_cluster.app_cluster_mode"] = DEFAULT_APP_CLUSTER_MODE
|
||
inputs["init_cluster.app_cluster_machine_type"] = DEFAULT_APP_CLUSTER_MACHINE_TYPE
|
||
inputs["init_cluster.app_cluster_node_count"] = str(DEFAULT_APP_CLUSTER_NODE_COUNT)
|
||
inputs["init_cluster.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME
|
||
inputs["init_cluster.db_cluster_mode"] = DEFAULT_DB_CLUSTER_MODE
|
||
inputs["init_cluster.db_cluster_node_count"] = str(DEFAULT_DB_CLUSTER_NODE_COUNT)
|
||
inputs["init_cluster.db_cluster_machine_type"] = DEFAULT_DB_CLUSTER_MACHINE_TYPE
|
||
inputs["init_cluster.db_boot_disk_type"] = DEFAULT_DB_BOOT_DISK_TYPE
|
||
inputs["init_cluster.db_boot_disk_size_gb"] = str(DEFAULT_DB_BOOT_DISK_SIZE_GB)
|
||
inputs["init_cluster.db_cluster_region"] = ""
|
||
inputs["init_cluster.db_cluster_zones"] = ""
|
||
inputs["init_cluster.app_cluster_kubecontext"] = ""
|
||
inputs["init_cluster.db_cluster_kubecontext"] = ""
|
||
inputs["init_cluster.supabase_auth_enabled"] = _bool_str(True)
|
||
inputs["init_cluster.supabase_realtime_enabled"] = _bool_str(True)
|
||
inputs["init_cluster.supabase_meta_enabled"] = _bool_str(True)
|
||
inputs["init_cluster.supabase_analytics_enabled"] = _bool_str(True)
|
||
inputs["init_cluster.gitops_enabled"] = _bool_str(False)
|
||
inputs["init_cluster.kerberos_enabled"] = _bool_str(False)
|
||
inputs["init_cluster.at_rest_encryption_enabled"] = _bool_str(True)
|
||
inputs["init_cluster.start_cluster"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("init_cluster.start_cluster", True)
|
||
)
|
||
|
||
# Kerberos config
|
||
inputs["kerberos_config.enabled"] = _bool_str(False)
|
||
inputs["kerberos_config.realm"] = ""
|
||
inputs["kerberos_config.kdc"] = ""
|
||
inputs["kerberos_config.user"] = "administrator"
|
||
inputs["kerberos_config.password"] = ""
|
||
inputs["kerberos_config.test_connection"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("kerberos_config.test_connection", False)
|
||
)
|
||
inputs["kerberos_config.init_authority"] = _bool_str(False)
|
||
|
||
# Ollama config
|
||
inputs["ollama_config.server_host"] = ""
|
||
inputs["ollama_config.server_port"] = DEFAULT_OLLAMA_PORT
|
||
inputs["ollama_config.model"] = ""
|
||
|
||
# Init scripts + deploy
|
||
inputs["init_scripts.run_scripts"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("init_scripts.run_scripts", True)
|
||
)
|
||
inputs["init_cnpg_deploy.run_deploy"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True)
|
||
)
|
||
inputs["init_cnpg_deploy.force_rollout"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False)
|
||
)
|
||
|
||
# GitOps
|
||
inputs["gitops.namespace"] = "gitea"
|
||
inputs["auth.clientId"] = "secretref://google-oidc-client-id"
|
||
inputs["auth.clientSecret"] = "secretref://google-oidc-client-secret"
|
||
|
||
# Disk selection (installer packaging)
|
||
inputs["disk_selection.disk_type"] = "local"
|
||
inputs["disk_selection.removable_mount"] = ""
|
||
inputs["disk_selection.local_path"] = str(Path.home())
|
||
|
||
# Build tools app
|
||
inputs["build.deploy_env"] = "Dev"
|
||
inputs["build.run_build"] = _bool_str(
|
||
DEFAULT_ACTION_FLAGS.get("build.run_build", False)
|
||
)
|
||
|
||
return inputs
|
||
|
||
# ---------------- Process helpers ----------------
|
||
def _run_cmd(
|
||
self, cmd, cwd=None, env=None, stdin_text=None, on_stdout=None, on_stderr=None
|
||
) -> int:
|
||
if isinstance(cmd, (list, tuple)) and cmd:
|
||
binary = Path(str(cmd[0])).name
|
||
if binary in {"kubectl", "helm"}:
|
||
target_role = "UNKNOWN"
|
||
target_ctx = ""
|
||
if isinstance(env, dict):
|
||
target_role = str(env.get("KNOE_CLUSTER_ROLE") or "").strip().upper() or "UNKNOWN"
|
||
target_ctx = str(
|
||
env.get("KUBECTL_CONTEXT")
|
||
or env.get("KUBE_CONTEXT_NAME")
|
||
or env.get("KUBECONTEXT")
|
||
or ""
|
||
).strip()
|
||
self.log(f"[TARGET {target_role}] {binary} context={target_ctx or '<unset>'}")
|
||
|
||
stdin_handle = subprocess.PIPE if stdin_text else None
|
||
if isinstance(cmd, str):
|
||
proc = subprocess.Popen(
|
||
["bash", "-lc", cmd],
|
||
cwd=cwd,
|
||
env=env,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
stdin=stdin_handle,
|
||
text=True,
|
||
bufsize=1,
|
||
)
|
||
else:
|
||
proc = subprocess.Popen(
|
||
cmd,
|
||
cwd=cwd,
|
||
env=env,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
stdin=stdin_handle,
|
||
text=True,
|
||
bufsize=1,
|
||
)
|
||
if stdin_text and proc.stdin:
|
||
try:
|
||
proc.stdin.write(stdin_text)
|
||
proc.stdin.close()
|
||
except Exception:
|
||
pass
|
||
|
||
def _read_stream(stream, handler, is_err=False):
|
||
if not stream:
|
||
return
|
||
for line in iter(stream.readline, ""):
|
||
if handler:
|
||
handler(line)
|
||
else:
|
||
if is_err:
|
||
self.err(line.rstrip("\n"))
|
||
else:
|
||
self.log(line.rstrip("\n"))
|
||
|
||
err_thread = threading.Thread(
|
||
target=_read_stream, args=(proc.stderr, on_stderr, True), daemon=True
|
||
)
|
||
err_thread.start()
|
||
_read_stream(proc.stdout, on_stdout, False)
|
||
rc = proc.wait()
|
||
err_thread.join(timeout=2)
|
||
return rc
|
||
|
||
def _run_script(
|
||
self, script_name: str, args=None, env=None, stdin_text=None, on_line=None
|
||
) -> int:
|
||
def _stdout(line):
|
||
self.log(line.rstrip("\n"))
|
||
if on_line:
|
||
on_line(line)
|
||
|
||
def _stderr(line):
|
||
self.err(line.rstrip("\n"))
|
||
|
||
mode_args = []
|
||
if script_name.startswith("init_"):
|
||
mode = _deployment_mode_from_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
if mode:
|
||
mode_args = ["--mode", mode]
|
||
|
||
target_role = "UNKNOWN"
|
||
target_ctx = ""
|
||
if isinstance(env, dict):
|
||
target_role = str(env.get("KNOE_CLUSTER_ROLE") or "").strip().upper() or "UNKNOWN"
|
||
target_ctx = str(
|
||
env.get("KUBECTL_CONTEXT")
|
||
or env.get("KUBE_CONTEXT_NAME")
|
||
or env.get("KUBECONTEXT")
|
||
or ""
|
||
).strip()
|
||
if target_ctx or target_role != "UNKNOWN":
|
||
self.log(
|
||
f"[TARGET {target_role}] script={script_name} context={target_ctx or '<unset>'}"
|
||
)
|
||
|
||
return self.controller.run_script(
|
||
script_name,
|
||
args=mode_args + (args or []),
|
||
env=env,
|
||
stdin_text=stdin_text,
|
||
on_line=_stdout,
|
||
on_stderr_line=_stderr,
|
||
stderr_to_stdout=False,
|
||
)
|
||
|
||
def _ensure_local_registry_available(self):
|
||
"""Ensure a local k3d registry is running and return (host_registry, cluster_registry)."""
|
||
env_hint = self._get_input("init_cluster.cluster_env", "")
|
||
mode = _deployment_mode_from_env(env_hint)
|
||
if not _local_registry_enabled(env_hint):
|
||
self.log("[SKIP] Local registry disabled; skipping registry setup.")
|
||
return None
|
||
|
||
if mode == "k3s":
|
||
# In k3s mode we must not use localhost registries or k3d registry containers.
|
||
server, _token = _resolve_k3s_connection_fn(
|
||
project_root=self.project_root,
|
||
cfg_path=self.cfg_path,
|
||
)
|
||
host = _host_from_url(server)
|
||
if not host:
|
||
self.err(
|
||
"[WARN] k3s mode but K3S_SERVER_URL/K3S_SERVER not set; cannot resolve registry."
|
||
)
|
||
return None
|
||
host_registry = f"{host}:5000"
|
||
registry_ns = self._registry_namespace()
|
||
cluster_registry = f"registry.{registry_ns}.svc.cluster.local:5000"
|
||
self.local_registry_url = host_registry
|
||
self.local_registry_internal = cluster_registry
|
||
self.knoe_cfg_data.setdefault("Docker Build", {})["LOCAL_REGISTRY"] = host_registry
|
||
self.knoe_cfg_data["Docker Build"]["LOCAL_REGISTRY_INTERNAL"] = cluster_registry
|
||
return host_registry, cluster_registry
|
||
|
||
reg_name = "knoe-registry"
|
||
registry_container = f"k3d-{reg_name}"
|
||
host_registry = "localhost:5000"
|
||
cluster_registry = f"{registry_container}.localhost:5000"
|
||
|
||
if not self.controller.check_docker_running():
|
||
self.err("[WARN] Docker not running; cannot ensure local registry.")
|
||
return None
|
||
|
||
# 1. If already responsive, just use it
|
||
if _http_ping_registry("localhost", 5000):
|
||
self.log("[INFO] Local registry already responsive on localhost:5000.")
|
||
lst = subprocess.run(
|
||
["k3d", "registry", "list"], capture_output=True, text=True
|
||
)
|
||
if registry_container not in (lst.stdout or "") and reg_name not in (
|
||
lst.stdout or ""
|
||
):
|
||
cluster_registry = "localhost:5000"
|
||
|
||
self.local_registry_url = host_registry
|
||
self.local_registry_internal = cluster_registry
|
||
self.knoe_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry
|
||
self.knoe_cfg_data["Docker Build"][
|
||
"LOCAL_REGISTRY_INTERNAL"
|
||
] = cluster_registry
|
||
return host_registry, cluster_registry
|
||
|
||
# 2. Require k3d to manage/create the local registry
|
||
k3d_exists = (
|
||
subprocess.run(["which", "k3d"], capture_output=True).returncode == 0
|
||
)
|
||
if not k3d_exists:
|
||
self.err("[WARN] k3d not found; cannot ensure local registry.")
|
||
return None
|
||
|
||
lst = subprocess.run(
|
||
["k3d", "registry", "list"], capture_output=True, text=True
|
||
)
|
||
if registry_container not in (lst.stdout or "") and reg_name not in (
|
||
lst.stdout or ""
|
||
):
|
||
self.log("[INFO] Creating local k3d registry...")
|
||
subprocess.run(
|
||
["k3d", "registry", "create", reg_name, "--port", "0.0.0.0:5000"],
|
||
check=True,
|
||
)
|
||
|
||
running = subprocess.run(
|
||
[
|
||
"docker",
|
||
"ps",
|
||
"--filter",
|
||
f"name={registry_container}",
|
||
"--format",
|
||
"{{.Names}}",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
if not (running.stdout or "").strip():
|
||
exists = subprocess.run(
|
||
[
|
||
"docker",
|
||
"ps",
|
||
"-a",
|
||
"--filter",
|
||
f"name={registry_container}",
|
||
"--format",
|
||
"{{.Names}}",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
if (exists.stdout or "").strip():
|
||
subprocess.run(["docker", "start", registry_container], check=True)
|
||
|
||
self.local_registry_url = host_registry
|
||
self.local_registry_internal = cluster_registry
|
||
self.knoe_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry
|
||
self.knoe_cfg_data["Docker Build"][
|
||
"LOCAL_REGISTRY_INTERNAL"
|
||
] = cluster_registry
|
||
return host_registry, cluster_registry
|
||
|
||
def _argocd_namespace(self) -> str:
|
||
ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {})
|
||
.get("ARGOCD_NAMESPACE", "")
|
||
.strip()
|
||
)
|
||
if not ns:
|
||
ns = (os.environ.get("ARGOCD_NAMESPACE") or "").strip()
|
||
return ns or "argocd"
|
||
|
||
def _registry_namespace(self) -> str:
|
||
ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {})
|
||
.get("REGISTRY_NAMESPACE", "")
|
||
.strip()
|
||
)
|
||
if not ns:
|
||
ns = (os.environ.get("REGISTRY_NAMESPACE") or "").strip()
|
||
# In service-mode k3s, keep registry in the service namespace and
|
||
# avoid falling back to default (which causes duplicate deployments).
|
||
if ns and not (ns == "default" and self._deployment_mode() == "k3s"):
|
||
return ns
|
||
|
||
# Registry is a common-core service; default to the configured service namespace.
|
||
return self._service_namespace()
|
||
|
||
def _ensure_registry_defaults(self, mode: str) -> None:
|
||
mode = _deployment_mode_from_env(mode) or mode
|
||
db = self.knoe_cfg_data.get("Docker Build", {}) or {}
|
||
|
||
if mode == "k3d" and _local_registry_enabled(mode):
|
||
info = self._ensure_local_registry_available()
|
||
if info:
|
||
host_registry, cluster_registry = info
|
||
db.setdefault("LOCAL_REGISTRY", host_registry)
|
||
db.setdefault("LOCAL_REGISTRY_INTERNAL", cluster_registry)
|
||
|
||
if mode == "k3s" and _local_registry_enabled(mode):
|
||
cur = (db.get("LOCAL_REGISTRY") or "").strip()
|
||
if not cur or cur == "localhost:5000":
|
||
server, _token = _resolve_k3s_connection_fn(
|
||
project_root=self.project_root,
|
||
cfg_path=self.cfg_path,
|
||
)
|
||
host = _host_from_url(server)
|
||
if host:
|
||
db["LOCAL_REGISTRY"] = f"{host}:5000"
|
||
|
||
cur_internal = (db.get("LOCAL_REGISTRY_INTERNAL") or "").strip()
|
||
if not cur_internal or ".localhost" in cur_internal:
|
||
registry_ns = self._registry_namespace()
|
||
db["LOCAL_REGISTRY_INTERNAL"] = (
|
||
f"registry.{registry_ns}.svc.cluster.local:5000"
|
||
)
|
||
|
||
self.knoe_cfg_data["Docker Build"] = db
|
||
|
||
def _fetch_k3s_kubeconfig(self) -> Path | None:
|
||
"""Fetch a fresh kubeconfig from k3s via the Ansible playbook."""
|
||
kubeconfig = self.project_root / "knoe-k3s.kubeconfig"
|
||
playbook = (
|
||
self.project_root
|
||
/ "infrastructure"
|
||
/ "playbooks"
|
||
/ "k3s_fetch_kubeconfig.yml"
|
||
)
|
||
if not playbook.exists():
|
||
self.err(f"[WARN] Ansible playbook not found: {playbook}")
|
||
return None
|
||
|
||
cmd = ["ansible-playbook", str(playbook)]
|
||
|
||
self.log("==> Fetching k3s kubeconfig via Ansible")
|
||
try:
|
||
subprocess.run(
|
||
cmd,
|
||
cwd=str(self.project_root),
|
||
check=True,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=30,
|
||
)
|
||
except subprocess.CalledProcessError as e:
|
||
self.err(f"[WARN] Failed to fetch kubeconfig: {e}")
|
||
return None
|
||
except FileNotFoundError:
|
||
self.err("[WARN] 'ansible-playbook' command not found.")
|
||
return None
|
||
except subprocess.TimeoutExpired:
|
||
self.err("[WARN] Kubeconfig fetch timed out.")
|
||
return None
|
||
|
||
if not kubeconfig.exists():
|
||
self.err("[WARN] Kubeconfig was not written after playbook run.")
|
||
return None
|
||
|
||
self.log(f"[OK] Kubeconfig fetched: {kubeconfig}")
|
||
return kubeconfig
|
||
|
||
# ---------------- Steps ----------------
|
||
def _apply_inputs(self):
|
||
defaults = self._default_inputs()
|
||
loaded = self._load_inputs_from_cfg()
|
||
self.inputs = {**defaults, **loaded}
|
||
try:
|
||
self._apply_ansible_defaults(set(loaded.keys()))
|
||
except Exception:
|
||
pass
|
||
|
||
mode = _deployment_mode_from_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
if not mode:
|
||
mode = _deployment_mode_from_env(self._get_input("init_cluster.mode", ""))
|
||
if mode:
|
||
os.environ["KNOE_MODE"] = mode
|
||
|
||
def _apply_ansible_defaults(self, loaded_keys: set[str] | None = None):
|
||
info = _detect_ansible_topology(self.project_root)
|
||
if not info:
|
||
return
|
||
loaded_keys = loaded_keys or set()
|
||
auto_keys = getattr(self, "_auto_input_keys", set())
|
||
ansible_kdc = info.get("kdc_ip") or ""
|
||
try:
|
||
net = self.knoe_cfg_data.get("Network", {})
|
||
if info.get("topology_json"):
|
||
net["ANSIBLE_TOPOLOGY"] = info["topology_json"]
|
||
if info.get("inventory_path"):
|
||
net["ANSIBLE_INVENTORY"] = info["inventory_path"]
|
||
if info.get("infrastructure_path"):
|
||
net["ANSIBLE_INFRASTRUCTURE"] = info["infrastructure_path"]
|
||
if info.get("domain"):
|
||
net["ANSIBLE_DOMAIN"] = info["domain"]
|
||
if info.get("realm"):
|
||
net["ANSIBLE_REALM"] = info["realm"]
|
||
if info.get("ad_dc_host"):
|
||
net["AD_DC_HOST"] = info["ad_dc_host"]
|
||
if info.get("ad_dc_ip"):
|
||
net["AD_DC_IP"] = info["ad_dc_ip"]
|
||
if info.get("kdc_ip"):
|
||
net["KDC_ANSIBLE_DETECTED"] = info["kdc_ip"]
|
||
self.knoe_cfg_data["Network"] = net
|
||
except Exception:
|
||
pass
|
||
|
||
# Storage/node placement derived from Ansible host vars.
|
||
# This is the authoritative source of truth for where volumes are mounted.
|
||
try:
|
||
inv_str = (info.get("inventory_path") or "").strip()
|
||
inv_path = Path(inv_str) if inv_str else None
|
||
except Exception:
|
||
inv_path = None
|
||
|
||
if inv_path is not None and inv_path.exists():
|
||
try:
|
||
mounts = _detect_ansible_storage_mounts(inv_path)
|
||
labels_by_host = _detect_ansible_node_labels(inv_path)
|
||
|
||
storage_sec = self.knoe_cfg_data.setdefault("Storage", {})
|
||
if mounts:
|
||
storage_sec["ANSIBLE_ISCSI_MOUNTS"] = json.dumps(
|
||
mounts, separators=(",", ":"), sort_keys=True
|
||
)
|
||
if labels_by_host:
|
||
storage_sec["ANSIBLE_NODE_LABELS"] = json.dumps(
|
||
labels_by_host, separators=(",", ":"), sort_keys=True
|
||
)
|
||
|
||
glob = self.knoe_cfg_data.setdefault("Global", {})
|
||
|
||
cnpg_eligible_nodes = sorted(
|
||
{
|
||
str((details or {}).get("host") or "").strip()
|
||
for details in mounts.values()
|
||
if isinstance(details, dict)
|
||
and str((details or {}).get("host") or "").strip()
|
||
}
|
||
)
|
||
if cnpg_eligible_nodes:
|
||
glob["CNPG_ELIGIBLE_NODES"] = ",".join(cnpg_eligible_nodes)
|
||
|
||
synology_roots = sorted(
|
||
{
|
||
str((details or {}).get("path") or "").strip()
|
||
for details in mounts.values()
|
||
if isinstance(details, dict)
|
||
and str((details or {}).get("path") or "").strip()
|
||
}
|
||
)
|
||
if synology_roots:
|
||
glob["SYNOLOGY_ROOTS"] = ",".join(synology_roots)
|
||
|
||
# CNPG protected storage defaults (k3s local PV)
|
||
d001 = mounts.get("d001") or {}
|
||
d001_base = (d001.get("path") or "").strip()
|
||
d001_host = (d001.get("host") or "").strip()
|
||
if d001_base and d001_host:
|
||
glob["CNPG_STAGE1_NODE"] = d001_host
|
||
glob["CNPG_DB_NODE_SELECTOR"] = f"kubernetes.io/hostname={d001_host}"
|
||
|
||
# Monitoring storage defaults (prefer d004; fall back to d002)
|
||
mon_choice = None
|
||
for vol_id in ("d004", "d002"):
|
||
if mounts.get(vol_id):
|
||
mon_choice = mounts[vol_id]
|
||
break
|
||
if mon_choice:
|
||
mon_base = (mon_choice.get("path") or "").strip()
|
||
mon_host = (mon_choice.get("host") or "").strip()
|
||
if mon_base:
|
||
glob["MONITORING_DATA_DIR"] = mon_base
|
||
if mon_host:
|
||
self.knoe_cfg_data.setdefault("Monitoring", {})[
|
||
"MONITORING_PRIMARY_NODE"
|
||
] = mon_host
|
||
except Exception:
|
||
pass
|
||
|
||
if ansible_kdc and "kerberos_config.kdc" in auto_keys:
|
||
self.inputs["kerberos_config.kdc"] = ansible_kdc
|
||
if ansible_kdc and "kerberos_config.enabled" in auto_keys:
|
||
self.inputs["kerberos_config.enabled"] = _bool_str(True)
|
||
|
||
if "kerberos_config.kdc" not in loaded_keys:
|
||
if not self._get_input("kerberos_config.kdc", "").strip() and info.get(
|
||
"kdc_ip"
|
||
):
|
||
self.inputs["kerberos_config.kdc"] = info["kdc_ip"]
|
||
if "kerberos_config.realm" not in loaded_keys:
|
||
if not self._get_input("kerberos_config.realm", "").strip() and info.get(
|
||
"realm"
|
||
):
|
||
self.inputs["kerberos_config.realm"] = info["realm"]
|
||
if "kerberos_config.enabled" not in loaded_keys:
|
||
if info.get("kdc_ip") and not self._get_input_bool(
|
||
"kerberos_config.enabled", False
|
||
):
|
||
self.inputs["kerberos_config.enabled"] = _bool_str(True)
|
||
|
||
if "init_cluster.k3s_server_url" not in loaded_keys:
|
||
if not self._get_input(
|
||
"init_cluster.k3s_server_url", ""
|
||
).strip() and info.get("k3s_server_url"):
|
||
self.inputs["init_cluster.k3s_server_url"] = info["k3s_server_url"]
|
||
if "init_cluster.k3s_token" not in loaded_keys:
|
||
if not self._get_input("init_cluster.k3s_token", "").strip() and info.get(
|
||
"k3s_token"
|
||
):
|
||
self.inputs["init_cluster.k3s_token"] = info["k3s_token"]
|
||
|
||
def _write_cfg(self):
|
||
inputs = dict(self.inputs)
|
||
db_pw = self._get_input("init_password.db_password", "")
|
||
db_pw_cfg = self._secret_cfg_value(
|
||
"Inputs", "init_password.db_password", db_pw, "db", "password"
|
||
)
|
||
if db_pw_cfg:
|
||
inputs["init_password.db_password"] = db_pw_cfg
|
||
inputs["init_password.db_password_confirm"] = db_pw_cfg
|
||
elif db_pw:
|
||
inputs["init_password.db_password"] = db_pw
|
||
inputs["init_password.db_password_confirm"] = (
|
||
self._get_input("init_password.db_password_confirm", "") or db_pw
|
||
)
|
||
krb_pw = self._get_input("kerberos_config.password", "")
|
||
if krb_pw:
|
||
inputs["kerberos_config.password"] = self._secret_cfg_value(
|
||
"Inputs", "kerberos_config.password", krb_pw, "kerberos", "password"
|
||
)
|
||
k3s_token = self._get_input("init_cluster.k3s_token", "")
|
||
if k3s_token:
|
||
inputs["init_cluster.k3s_token"] = self._secret_cfg_value(
|
||
"Inputs", "init_cluster.k3s_token", k3s_token, "k3s", "token"
|
||
)
|
||
|
||
mode = _deployment_mode_from_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
target_label = _deployment_target_label(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
globals_to_save = {
|
||
"KNOE_HOME": self._get_input("env_setup.KNOE_HOME", ""),
|
||
"KNOE_DB_USER": self._get_input("init_password.db_username", ""),
|
||
"DB_PASSWORD": self._secret_cfg_value(
|
||
"Global", "DB_PASSWORD", db_pw, "db", "password"
|
||
),
|
||
"CLUSTER_ENV": self._get_input("init_cluster.cluster_env", ""),
|
||
"DEPLOYMENT_MODE": mode,
|
||
"DEPLOYMENT_TARGET": target_label,
|
||
"DATABASE_NAMESPACE": (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip(),
|
||
"CLUSTER_NAME": (
|
||
self._get_input("init_password.cluster_name", "")
|
||
or self._get_input("env_setup.CLUSTER_NAME", "")
|
||
or self._cnpg_cluster_name()
|
||
or ""
|
||
).strip(),
|
||
"DB_HOST_PORT": (
|
||
self._get_input("init_password.db_host_port", "5432") or "5432"
|
||
).strip(),
|
||
"DOCKER_IMPORT_DIR": self.docker_import_dir or "",
|
||
"K3S_SERVER": (
|
||
self._get_input("init_cluster.k3s_server_url", "") or ""
|
||
).strip(),
|
||
"K3S_TOKEN": self._secret_cfg_value(
|
||
"Global",
|
||
"K3S_TOKEN",
|
||
self._get_input("init_cluster.k3s_token", ""),
|
||
"k3s",
|
||
"token",
|
||
),
|
||
"OPENTOFU_URL": _default_opentofu_pipeline_url(),
|
||
"SERVICE_NAMESPACE": self._service_namespace(),
|
||
}
|
||
# Merge pre-existing Global values without overriding explicit inputs
|
||
existing_global = dict(self.knoe_cfg_data.get("Global", {}))
|
||
for k, v in existing_global.items():
|
||
if k not in globals_to_save or not str(globals_to_save.get(k, "")).strip():
|
||
globals_to_save[k] = v
|
||
globals_to_save.setdefault("DOCKER_PRELOAD", "false")
|
||
for _gk, _gv in list(globals_to_save.items()):
|
||
if isinstance(_gv, str) and _gv.strip():
|
||
globals_to_save[_gk] = self._cfgify_home_path(_gv)
|
||
# Re-assert critical/dynamic values to avoid accidental overwrite
|
||
globals_to_save["DB_PASSWORD"] = self._secret_cfg_value(
|
||
"Global", "DB_PASSWORD", db_pw, "db", "password"
|
||
)
|
||
globals_to_save["DEPLOYMENT_MODE"] = mode
|
||
globals_to_save["DEPLOYMENT_TARGET"] = target_label
|
||
globals_to_save.pop("NAMESPACE", None)
|
||
globals_to_save.pop("CNPG_CLUSTER_NAME", None)
|
||
globals_to_save.pop("KUBECONTEXT", None) # machine-specific; always runtime-detected
|
||
globals_to_save.setdefault("ARGOCD_NAMESPACE", self._argocd_namespace())
|
||
globals_to_save.setdefault("REGISTRY_NAMESPACE", self._registry_namespace())
|
||
# Preserve Supabase placement vars from cfg — never let pipeline state overwrite them
|
||
for _supa_key in (
|
||
"SUPABASE_PRIMARY_NODE", "SUPABASE_PV_NODE",
|
||
"SUPABASE_PV_BASE_DIR", "SUPABASE_STORAGE_CLASS",
|
||
"SUPABASE_ADDITIONAL_REDIRECT_URLS",
|
||
):
|
||
_supa_val = existing_global.get(_supa_key, "")
|
||
if _supa_val:
|
||
globals_to_save[_supa_key] = _supa_val
|
||
|
||
# Monitoring
|
||
# NOTE: Storage base paths for monitoring are currently pinned in the
|
||
# monitoring deploy script/Helm values (not persisted in knoe.cfg).
|
||
mon_sec = self.knoe_cfg_data.get("Monitoring", {})
|
||
if mon_sec is None:
|
||
mon_sec = {}
|
||
self.knoe_cfg_data["Monitoring"] = mon_sec
|
||
|
||
# Optional features (sourced from inputs)
|
||
opt_sec = self.knoe_cfg_data.setdefault("Optional Features", {})
|
||
opt_sec["SUPABASE_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_enabled", False)
|
||
)
|
||
opt_sec["GITOPS_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.gitops_enabled", False)
|
||
)
|
||
opt_sec["KERBEROS_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.kerberos_enabled", False)
|
||
)
|
||
opt_sec["AT_REST_ENCRYPTION_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.at_rest_encryption_enabled", False)
|
||
)
|
||
|
||
gitops_ns_val = (
|
||
self._get_input("gitops.namespace", "")
|
||
or self.inputs.get("gitops.namespace", "")
|
||
)
|
||
if gitops_ns_val:
|
||
self.knoe_cfg_data.setdefault("GitOps", {})[
|
||
"GITOPS_NAMESPACE"
|
||
] = gitops_ns_val
|
||
|
||
ollama_host = (self._get_input("ollama_config.server_host", "") or "").strip()
|
||
ollama_port = (
|
||
self._get_input("ollama_config.server_port", DEFAULT_OLLAMA_PORT) or ""
|
||
).strip()
|
||
ollama_model = (self._get_input("ollama_config.model", "") or "").strip()
|
||
ollama_section = dict(self.knoe_cfg_data.get("Ollama", {}))
|
||
if ollama_host:
|
||
if not ollama_port:
|
||
ollama_port = DEFAULT_OLLAMA_PORT
|
||
ollama_section["OLLAMA_SERVER_HOST"] = ollama_host
|
||
ollama_section["OLLAMA_SERVER_PORT"] = ollama_port
|
||
ollama_section["OLLAMA_HOST"] = _format_ollama_host(
|
||
ollama_host, ollama_port
|
||
)
|
||
else:
|
||
for key in ("OLLAMA_SERVER_HOST", "OLLAMA_SERVER_PORT", "OLLAMA_HOST"):
|
||
ollama_section.pop(key, None)
|
||
if ollama_model:
|
||
ollama_section["OLLAMA_MODEL"] = ollama_model
|
||
else:
|
||
ollama_section.pop("OLLAMA_MODEL", None)
|
||
if ollama_section:
|
||
self.knoe_cfg_data["Ollama"] = ollama_section
|
||
|
||
self._sync_port_forward_mappings()
|
||
|
||
sections = {
|
||
k: self.knoe_cfg_data.get(k, {})
|
||
for k in [
|
||
"Welcome",
|
||
"Dependencies",
|
||
"Network",
|
||
"Port Forwards",
|
||
"System Environment",
|
||
"Monitoring",
|
||
"Kerberos Authentication",
|
||
"Ollama",
|
||
"Optional Features",
|
||
"GitOps",
|
||
"CNPG Clusters",
|
||
"Database Creation",
|
||
"Initialize Cluster",
|
||
"Docker Build",
|
||
"Initialization Scripts",
|
||
"Deployment",
|
||
"Install",
|
||
]
|
||
}
|
||
deployment_section = dict(sections.get("Deployment", {}))
|
||
if mode:
|
||
deployment_section.setdefault("MODE", mode)
|
||
if target_label:
|
||
deployment_section.setdefault("TARGET", target_label)
|
||
sections["Deployment"] = deployment_section
|
||
|
||
sections["Dev Cluster (k3d)"] = {
|
||
**self.knoe_cfg_data.get("Dev Cluster (k3d)", {}),
|
||
"MODE": "k3d",
|
||
"CLUSTER_ENV": "k3d-knoe-dev-cluster",
|
||
"DISPLAY_NAME": "knoe-dev-cluster",
|
||
"KUBECTL_CONTEXT": self._get_input("init_cluster.cluster_env", ""),
|
||
}
|
||
sections["Service Cluster (k3s)"] = {
|
||
**self.knoe_cfg_data.get("Service Cluster (k3s)", {}),
|
||
"MODE": "k3s",
|
||
"CLUSTER_ENV": "knoe-service-cluster",
|
||
"DISPLAY_NAME": "knoe-service-cluster",
|
||
"K3S_SERVER_URL": (
|
||
self._get_input("init_cluster.k3s_server_url", "") or ""
|
||
).strip(),
|
||
"K3S_TOKEN": self._secret_cfg_value(
|
||
"Service Cluster (k3s)",
|
||
"K3S_TOKEN",
|
||
self._get_input("init_cluster.k3s_token", ""),
|
||
"k3s",
|
||
"token",
|
||
),
|
||
"PIPELINE_URL": _default_opentofu_pipeline_url(),
|
||
}
|
||
sections["Prod Cluster (k8s)"] = {
|
||
**self.knoe_cfg_data.get("Prod Cluster (k8s)", {}),
|
||
"MODE": "k8s",
|
||
"CLUSTER_ENV": "knoe-prod-cluster",
|
||
"DISPLAY_NAME": "knoe-prod-cluster",
|
||
"ARTIFACTS_DIR": (
|
||
self._get_input("init_cluster.prod_artifacts_path", "") or ""
|
||
).strip(),
|
||
"PIPELINE_URL": _default_opentofu_pipeline_url(),
|
||
}
|
||
sections = self._sanitize_sections_for_cfg(sections)
|
||
|
||
# Check for existing config and timestamp to maintain idempotency
|
||
existing_text = None
|
||
existing_timestamp = None
|
||
if self.cfg_path.exists():
|
||
existing_text = self.cfg_path.read_text()
|
||
for line in existing_text.splitlines():
|
||
if line.startswith("; Generated by install.py on "):
|
||
existing_timestamp = line[
|
||
len("; Generated by install.py on ") :
|
||
].strip()
|
||
break
|
||
|
||
cfg_text = _render_knoe_cfg(
|
||
inputs, globals_to_save, sections, generated_at=existing_timestamp
|
||
)
|
||
|
||
should_write = True
|
||
if existing_text:
|
||
if existing_text == cfg_text:
|
||
should_write = False
|
||
|
||
if should_write:
|
||
# If content changed but we reused existing_timestamp, it might be misleading.
|
||
# But usually we want to know when it LAST changed.
|
||
# If it's different, let's generate a new one.
|
||
if existing_text and existing_text != cfg_text:
|
||
cfg_text = _render_knoe_cfg(inputs, globals_to_save, sections)
|
||
|
||
self.cfg_path.parent.mkdir(parents=True, exist_ok=True)
|
||
self.cfg_path.write_text(cfg_text)
|
||
self.log(f"[CONFIG] Wrote {self.cfg_path}")
|
||
|
||
def _step_dependencies(self) -> bool:
|
||
self.log("==> Dependencies")
|
||
missing = []
|
||
for dep in self.dependencies:
|
||
ok, location, version = inst_config.get_dep_info(dep)
|
||
if ok:
|
||
self.log(f"[OK] {dep['name']} {version or ''}".strip())
|
||
continue
|
||
missing.append(dep)
|
||
self.log(f"[MISSING] {dep['name']}")
|
||
|
||
if not missing:
|
||
if not self._ensure_gcloud_auth_for_k8s():
|
||
self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing"
|
||
return False
|
||
self.knoe_cfg_data["Dependencies"]["STATUS"] = "All installed"
|
||
return True
|
||
|
||
auto_install = self._get_input_bool(
|
||
"dependencies.auto_install_missing",
|
||
DEFAULT_ACTION_FLAGS.get("dependencies.auto_install_missing", True),
|
||
)
|
||
if not auto_install:
|
||
self.err("[ERROR] Dependencies missing and auto-install disabled.")
|
||
self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing"
|
||
return False
|
||
|
||
for dep in missing:
|
||
install_cmd = dep.get("install_cmd")
|
||
if not install_cmd:
|
||
self.err(f"[ERROR] No install command for {dep['name']}.")
|
||
continue
|
||
should_install = self._get_input_bool(
|
||
f"dependencies.{dep['id']}.install", True
|
||
)
|
||
if not should_install:
|
||
self.err(f"[SKIP] {dep['name']} install disabled by config.")
|
||
continue
|
||
self.log(f"[INSTALL] {dep['name']} -> {install_cmd}")
|
||
rc = self._run_cmd(install_cmd)
|
||
if rc != 0:
|
||
self.err(f"[ERROR] Install failed for {dep['name']} (code {rc})")
|
||
|
||
still_missing = []
|
||
for dep in missing:
|
||
ok, _, _ = inst_config.get_dep_info(dep)
|
||
if not ok:
|
||
still_missing.append(dep["name"])
|
||
if still_missing:
|
||
self.err(f"[ERROR] Still missing: {', '.join(still_missing)}")
|
||
self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing"
|
||
return False
|
||
if not self._ensure_gcloud_auth_for_k8s():
|
||
self.knoe_cfg_data["Dependencies"]["STATUS"] = "Missing"
|
||
return False
|
||
self.knoe_cfg_data["Dependencies"]["STATUS"] = "All installed"
|
||
return True
|
||
|
||
def _ensure_gcloud_auth_for_k8s(self) -> bool:
|
||
if self._deployment_mode() != "k8s":
|
||
return True
|
||
|
||
gcloud_env = inst_config._augment_env_for_dependency_backend(os.environ.copy())
|
||
cmd_base = ["gcloud"]
|
||
token_cmd = cmd_base + ["auth", "print-access-token", "--quiet"]
|
||
try:
|
||
token_res = subprocess.run(
|
||
token_cmd,
|
||
capture_output=True,
|
||
text=True,
|
||
env=gcloud_env,
|
||
timeout=20,
|
||
)
|
||
if token_res.returncode == 0 and (token_res.stdout or "").strip():
|
||
return True
|
||
except Exception:
|
||
pass
|
||
|
||
interactive = bool(getattr(sys.stdin, "isatty", lambda: False)())
|
||
if interactive:
|
||
login_cmd = cmd_base + ["auth", "login", "--no-launch-browser"]
|
||
self.log(
|
||
"[ACTION] No active gcloud session found. Starting login flow: "
|
||
+ " ".join(shlex.quote(part) for part in login_cmd)
|
||
)
|
||
try:
|
||
login_rc = self._run_cmd(login_cmd)
|
||
if login_rc == 0:
|
||
token_res = subprocess.run(
|
||
token_cmd,
|
||
capture_output=True,
|
||
text=True,
|
||
env=gcloud_env,
|
||
timeout=20,
|
||
)
|
||
if token_res.returncode == 0 and (token_res.stdout or "").strip():
|
||
return True
|
||
except Exception:
|
||
pass
|
||
|
||
self.err(
|
||
"[ERROR] gcloud is installed but no active auth session is available for GKE. "
|
||
"Run `gcloud auth login --no-launch-browser`, complete the URL/code flow on a browser-capable machine, then retry deploy."
|
||
)
|
||
return False
|
||
|
||
def _step_network_scan(self) -> None:
|
||
if not self._get_input_bool(
|
||
"network_scan.run", DEFAULT_ACTION_FLAGS.get("network_scan.run", True)
|
||
):
|
||
self.log("[SKIP] Network scan disabled.")
|
||
return
|
||
|
||
existing_kdc = self.knoe_cfg_data.get("Network", {}).get("KDC_AUTO_DETECTED")
|
||
if existing_kdc:
|
||
self.log(
|
||
f"[OK] Network scan already performed. KDC detected: {existing_kdc}"
|
||
)
|
||
return
|
||
|
||
self.log("==> Network scan (network-agent)")
|
||
ansible_kdc = ""
|
||
try:
|
||
ansible_kdc = (self.knoe_cfg_data.get("Network", {}) or {}).get(
|
||
"KDC_ANSIBLE_DETECTED", ""
|
||
)
|
||
except Exception:
|
||
ansible_kdc = ""
|
||
scan_binary = get_resource_path("scan/network-agent")
|
||
if not scan_binary.exists():
|
||
self.err(f"[ERROR] Scan binary not found at {scan_binary}")
|
||
return
|
||
|
||
knoe_home = resolve_knoe_home(env={})
|
||
scan_dir = knoe_home / "scan"
|
||
scan_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
kdc_found = None
|
||
pending_line = ""
|
||
|
||
def _handle_line(line: str):
|
||
nonlocal kdc_found
|
||
self.log(line.rstrip("\n"))
|
||
if "KDC is:" in line:
|
||
try:
|
||
ip_part = line.split("KDC is:")[1].strip()
|
||
ip = ip_part.split()[0].strip("[]():,")
|
||
if ip:
|
||
kdc_found = ip
|
||
except Exception:
|
||
pass
|
||
elif "Active Directory" in line or "88" in line:
|
||
for part in line.split():
|
||
try:
|
||
socket.inet_aton(part.strip("[]():,"))
|
||
kdc_found = part.strip("[]():,")
|
||
break
|
||
except Exception:
|
||
continue
|
||
|
||
def _handle_stdout(text: str):
|
||
nonlocal pending_line
|
||
pending_line += text
|
||
while "\n" in pending_line:
|
||
line, pending_line = pending_line.split("\n", 1)
|
||
_handle_line(line + "\n")
|
||
|
||
rc = run_streaming_cmd(
|
||
[str(scan_binary)], cwd=str(scan_dir), on_stdout=_handle_stdout
|
||
)
|
||
if pending_line:
|
||
_handle_line(pending_line)
|
||
if rc == 0:
|
||
self.log("[OK] Scan complete.")
|
||
else:
|
||
self.err(f"[ERROR] Scan failed (code {rc})")
|
||
if kdc_found and not ansible_kdc:
|
||
self.inputs["kerberos_config.kdc"] = kdc_found
|
||
self.inputs["kerberos_config.enabled"] = _bool_str(True)
|
||
self.knoe_cfg_data["Network"]["KDC_AUTO_DETECTED"] = kdc_found
|
||
self.knoe_cfg_data["Network"]["KERBEROS_AUTO_ENABLED"] = "True"
|
||
elif kdc_found and ansible_kdc:
|
||
self.log(
|
||
f"[INFO] Ansible KDC detected ({ansible_kdc}); ignoring scan-detected KDC {kdc_found}."
|
||
)
|
||
|
||
def _step_env_setup(self) -> None:
|
||
self.log("==> Environment setup")
|
||
global_sec = self.knoe_cfg_data.get("Global", {}) or {}
|
||
sys_sec = self.knoe_cfg_data.get("System Environment", {}) or {}
|
||
|
||
def _cfg_val(sec: dict, key: str) -> str:
|
||
try:
|
||
v = str((sec or {}).get(key, "") or "").strip()
|
||
except Exception:
|
||
v = ""
|
||
if v.startswith("${") and v.endswith("}"):
|
||
return ""
|
||
return v
|
||
|
||
ns_hint = (
|
||
self._get_input("env_setup.DATABASE_NAMESPACE", "")
|
||
or self._get_input("init_password.db_namespace", "")
|
||
or _cfg_val(global_sec, "DATABASE_NAMESPACE")
|
||
or self._get_input("env_setup.NAMESPACE", "")
|
||
or self._get_input("init_password.db_namespace", "")
|
||
or _cfg_val(global_sec, "NAMESPACE")
|
||
)
|
||
cluster_hint = (
|
||
self._get_input("env_setup.CLUSTER_NAME", "")
|
||
or self._get_input("init_password.cluster_name", "")
|
||
or _cfg_val(global_sec, "CLUSTER_NAME")
|
||
or _cfg_val(global_sec, "CNPG_CLUSTER_NAME")
|
||
or "knoe-db"
|
||
)
|
||
vals = {
|
||
"KNOE_HOME": (
|
||
self._get_input("env_setup.KNOE_HOME", "")
|
||
or _cfg_val(sys_sec, "KNOE_HOME")
|
||
or _cfg_val(global_sec, "KNOE_HOME")
|
||
or str(resolve_knoe_home(env={}))
|
||
),
|
||
"KNOE_CONF": (
|
||
self._get_input("env_setup.KNOE_CONF", "")
|
||
or _cfg_val(sys_sec, "KNOE_CONF")
|
||
or _cfg_val(global_sec, "KNOE_CONF")
|
||
),
|
||
"PROLE_DATA": (
|
||
self._get_input("env_setup.PROLE_DATA", "")
|
||
or _cfg_val(sys_sec, "PROLE_DATA")
|
||
or _cfg_val(global_sec, "PROLE_DATA")
|
||
),
|
||
"PROLE_LOGS": (
|
||
self._get_input("env_setup.PROLE_LOGS", "")
|
||
or _cfg_val(sys_sec, "PROLE_LOGS")
|
||
or _cfg_val(global_sec, "PROLE_LOGS")
|
||
),
|
||
"KNOE_SERVICE": (
|
||
self._get_input("env_setup.KNOE_SERVICE", "")
|
||
or _cfg_val(sys_sec, "KNOE_SERVICE")
|
||
or _cfg_val(global_sec, "KNOE_SERVICE")
|
||
),
|
||
"OPENTOFU_URL": (os.environ.get("OPENTOFU_URL") or "").strip(),
|
||
}
|
||
defaults = self._env_defaults(ns_hint)
|
||
for k in vals:
|
||
if not vals[k]:
|
||
vals[k] = defaults.get(k, "")
|
||
if not vals.get("OPENTOFU_URL"):
|
||
vals["OPENTOFU_URL"] = _default_opentofu_pipeline_url()
|
||
vals["DATABASE_NAMESPACE"] = (
|
||
self._get_input("env_setup.DATABASE_NAMESPACE", "")
|
||
or self._get_input("env_setup.NAMESPACE", "")
|
||
or ns_hint
|
||
)
|
||
vals["CLUSTER_NAME"] = (
|
||
self._get_input("env_setup.CLUSTER_NAME", "") or cluster_hint
|
||
)
|
||
|
||
if not vals.get("KNOE_HOME"):
|
||
raise Exception("KNOE_HOME is required for env setup.")
|
||
# Persist normalized inputs
|
||
for k in (
|
||
"KNOE_HOME",
|
||
"KNOE_CONF",
|
||
"PROLE_DATA",
|
||
"PROLE_LOGS",
|
||
"KNOE_SERVICE",
|
||
):
|
||
self.inputs[f"env_setup.{k}"] = vals[k]
|
||
self.inputs["env_setup.DATABASE_NAMESPACE"] = vals.get(
|
||
"DATABASE_NAMESPACE", ""
|
||
)
|
||
self.inputs["env_setup.CLUSTER_NAME"] = vals.get("CLUSTER_NAME", "")
|
||
if not (self._get_input("init_password.db_namespace", "") or "").strip():
|
||
self.inputs["init_password.db_namespace"] = vals.get(
|
||
"DATABASE_NAMESPACE", ""
|
||
)
|
||
if not (self._get_input("init_password.cluster_name", "") or "").strip():
|
||
self.inputs["init_password.cluster_name"] = vals.get("CLUSTER_NAME", "")
|
||
self._save_env_to_file(vals)
|
||
self.reload_env_from_shell()
|
||
|
||
for k in (
|
||
"KNOE_HOME",
|
||
"KNOE_CONF",
|
||
"PROLE_DATA",
|
||
"PROLE_LOGS",
|
||
"KNOE_SERVICE",
|
||
"OPENTOFU_URL",
|
||
):
|
||
self.knoe_cfg_data["System Environment"][k] = vals[k]
|
||
|
||
def _step_init_password(self) -> None:
|
||
self.log("==> Database creation")
|
||
ns = _safe_str(self._get_input("init_password.db_namespace", "") or "").strip()
|
||
if not ns:
|
||
ns = self._get_input("env_setup.DATABASE_NAMESPACE", "")
|
||
if not ns:
|
||
ns = self._get_input("env_setup.NAMESPACE", "")
|
||
if not ns:
|
||
ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get(
|
||
"DATABASE_NAMESPACE", ""
|
||
)
|
||
or ""
|
||
).strip()
|
||
if not ns:
|
||
ns = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get("NAMESPACE", "") or ""
|
||
).strip()
|
||
if ns.startswith("${") and ns.endswith("}"):
|
||
ns = ""
|
||
if not ns:
|
||
raise Exception("Database namespace cannot be empty.")
|
||
if not self._is_valid_namespace(ns):
|
||
raise Exception("Invalid database namespace.")
|
||
|
||
cluster_name = (
|
||
_safe_str(self._get_input("init_password.cluster_name", "") or "").strip()
|
||
)
|
||
if not cluster_name:
|
||
cluster_name = (
|
||
_safe_str(self._get_input("env_setup.CLUSTER_NAME", "") or "").strip()
|
||
)
|
||
if not cluster_name:
|
||
cluster_name = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get("CLUSTER_NAME", "")
|
||
or ""
|
||
).strip()
|
||
if not cluster_name:
|
||
cluster_name = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get(
|
||
"CNPG_CLUSTER_NAME", ""
|
||
)
|
||
or ""
|
||
).strip()
|
||
if not cluster_name:
|
||
cluster_name = "knoe-db"
|
||
user = (self._get_input("init_password.db_username", "") or "").strip()
|
||
if not user:
|
||
user = (
|
||
(self.knoe_cfg_data.get("Global", {}) or {}).get("KNOE_DB_USER", "")
|
||
or ""
|
||
).strip()
|
||
if user.startswith("${") and user.endswith("}"):
|
||
user = ""
|
||
if not user:
|
||
import getpass
|
||
|
||
try:
|
||
user = getpass.getuser()
|
||
except Exception:
|
||
user = "knoe"
|
||
self.log(f"[INFO] No database owner provided; using default: {user}")
|
||
p1 = self._get_input("init_password.db_password", "")
|
||
p2 = self._get_input("init_password.db_password_confirm", "") or p1
|
||
if not p1:
|
||
p1 = self._generate_db_password()
|
||
p2 = p1
|
||
self.log(
|
||
"[INFO] No database password provided; generated default password."
|
||
)
|
||
if p1 != p2:
|
||
raise Exception("Database passwords do not match.")
|
||
|
||
# If password is an OpenBao anchor, only skip when OpenBao is already initialized.
|
||
if p1.startswith("${"):
|
||
resolved = self._resolve_secret_value(p1)
|
||
if resolved and not resolved.startswith("${"):
|
||
p1 = resolved
|
||
p2 = resolved
|
||
else:
|
||
service_dir = self._get_input("env_setup.KNOE_SERVICE", "").strip()
|
||
if not service_dir:
|
||
service_dir = self._env_defaults(ns).get("KNOE_SERVICE", "")
|
||
token_path = Path(service_dir) / "secrets" / "openbao-root-token"
|
||
local_token_path = (
|
||
resolve_knoe_home(env={})
|
||
/ "etc"
|
||
/ "secrets"
|
||
/ "openbao-root-token"
|
||
)
|
||
has_token = (
|
||
bool(os.environ.get("OPENBAO_ROOT_TOKEN"))
|
||
or token_path.exists()
|
||
or local_token_path.exists()
|
||
)
|
||
|
||
if self.reset_requested or not has_token:
|
||
p1 = self._generate_db_password()
|
||
p2 = p1
|
||
self.log(
|
||
"[RESET] OpenBao anchor detected without token; generated new DB password for initialization."
|
||
)
|
||
else:
|
||
self.log(
|
||
f"[OK] OpenBao secrets already anchored for namespace {ns}. Skipping initialization."
|
||
)
|
||
return
|
||
|
||
self.inputs["init_password.db_namespace"] = ns
|
||
self.inputs["env_setup.DATABASE_NAMESPACE"] = ns
|
||
self.inputs["init_password.cluster_name"] = cluster_name
|
||
self.inputs["env_setup.CLUSTER_NAME"] = cluster_name
|
||
self.inputs["init_password.db_password"] = p1
|
||
self.inputs["init_password.db_password_confirm"] = p2
|
||
self._update_env_namespace(ns)
|
||
|
||
self.knoe_cfg_data["Database Creation"]["DB_USER"] = user
|
||
self.knoe_cfg_data["Database Creation"]["DB_PASSWORD_SET"] = "true"
|
||
self.knoe_cfg_data["Database Creation"]["DATABASE_NAMESPACE"] = ns
|
||
self.knoe_cfg_data["Database Creation"]["CLUSTER_NAME"] = cluster_name
|
||
self.knoe_cfg_data["Database Creation"].pop("DB_NAME", None)
|
||
self.knoe_cfg_data["Database Creation"].pop("NAMESPACE", None)
|
||
|
||
include_supabase = self._get_input_bool("init_cluster.supabase_enabled", False)
|
||
include_kerberos = self._get_input_bool(
|
||
"init_cluster.kerberos_enabled", False
|
||
) or self._get_input_bool("kerberos_config.enabled", False)
|
||
|
||
if include_supabase:
|
||
env_policy = self._script_env_for_namespace(ns)
|
||
allowed, _count, reason = self._optional_workloads_policy(env_policy)
|
||
if not allowed:
|
||
self.log(f"[SKIP] Supabase image pre-pull disabled by policy: {reason}")
|
||
include_supabase = False
|
||
try:
|
||
self._prepull_images_to_registry(include_supabase, include_kerberos)
|
||
except Exception as e:
|
||
self.err(f"[WARN] Image pre-pull error: {e}")
|
||
|
||
if self._get_input_bool(
|
||
"init_password.generate_ssh_key",
|
||
DEFAULT_ACTION_FLAGS.get("init_password.generate_ssh_key", True),
|
||
):
|
||
self._generate_ssh_key(user)
|
||
|
||
# Initialize OpenBao and store keys/passwords for this namespace.
|
||
# OpenBao is expected to run inside the cluster; defer initialization until cluster is ready.
|
||
env = os.environ.copy()
|
||
env["KNOE_HOME"] = str(self.project_root)
|
||
env["KNOE_SERVICE"] = str(self.project_root)
|
||
env["KNOE_DB_USER"] = user
|
||
env["DB_PASSWORD"] = p1
|
||
env["GRAFANA_ADMIN_PASSWORD"] = p1
|
||
env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.at_rest_encryption_enabled", False)
|
||
)
|
||
realm = self._get_input("kerberos_config.realm", "").strip()
|
||
kdc = self._get_input("kerberos_config.kdc", "").strip()
|
||
krb_user = self._get_input("kerberos_config.user", "").strip()
|
||
krb_pw = self._get_input("kerberos_config.password", "").strip()
|
||
if realm:
|
||
env["KRB5_REALM"] = realm
|
||
env["REALM"] = realm
|
||
env["DOMAIN"] = realm.lower()
|
||
if kdc:
|
||
env["KRB5_KDC"] = kdc
|
||
env["KRB5_ADMIN"] = kdc
|
||
if krb_user:
|
||
env["KRB5_USER"] = krb_user
|
||
if krb_pw:
|
||
env["KRB5_PASSWORD"] = krb_pw
|
||
self._openbao_init_env = env
|
||
self._openbao_init_password = p1
|
||
self.log("[INFO] Deferring OpenBao initialization until cluster is ready.")
|
||
|
||
def _generate_ssh_key(self, user: str):
|
||
self.log("==> Environment Preparation (Secure Access)")
|
||
key_path = Path.home() / ".ssh" / "id_knoe_ed25519"
|
||
key_path.parent.mkdir(parents=True, exist_ok=True)
|
||
if key_path.exists():
|
||
self.log(f"[SKIP] Secure access keys already exist at {key_path}")
|
||
return
|
||
cmd = ["ssh-keygen", "-t", "ed25519", "-N", "", "-f", str(key_path), "-C", user]
|
||
rc = self._run_cmd(cmd)
|
||
if rc != 0:
|
||
self.err(f"[WARN] Preparation failed (code {rc}), trying alternative.")
|
||
cmd = [
|
||
"ssh-keygen",
|
||
"-t",
|
||
"rsa",
|
||
"-b",
|
||
"4096",
|
||
"-N",
|
||
"",
|
||
"-f",
|
||
str(key_path),
|
||
"-C",
|
||
user,
|
||
]
|
||
rc = self._run_cmd(cmd)
|
||
if rc != 0:
|
||
self.err(f"[ERROR] Preparation failed (code {rc})")
|
||
|
||
def _step_db_build(self) -> None:
|
||
if not self._get_input_bool(
|
||
"init_db_build.run_build",
|
||
DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True),
|
||
):
|
||
self.log("[SKIP] DB build disabled.")
|
||
self.knoe_cfg_data["Docker Build"]["STATUS"] = "Skipped"
|
||
return
|
||
|
||
tag = self.controller.get_knoe_db_version()
|
||
image_name = f"knoe-db:{tag}"
|
||
env_key = _normalize_cluster_env(
|
||
self._get_input("init_cluster.cluster_env", "dev")
|
||
)
|
||
cluster_name = "knoe-dev-cluster"
|
||
|
||
# Check if image already exists
|
||
rc_inspect = subprocess.run(
|
||
["docker", "inspect", image_name], capture_output=True
|
||
).returncode
|
||
if rc_inspect == 0:
|
||
self.log(f"[OK] {image_name} already exists. Skipping build.")
|
||
if env_key == "dev":
|
||
self.log(f"Importing image to {cluster_name}...")
|
||
self._run_cmd(
|
||
["k3d", "image", "import", image_name, "-c", cluster_name]
|
||
)
|
||
self._db_built_success = True
|
||
elif env_key == "service":
|
||
# In k3s mode, publish to the in-cluster registry instead of importing
|
||
# tarballs into nodes.
|
||
push_ok = True
|
||
db_cfg = self.knoe_cfg_data.get("Docker Build", {}) or {}
|
||
registry = (db_cfg.get("LOCAL_REGISTRY") or "").strip()
|
||
|
||
# Prioritize KNOE_IMAGE_REGISTRY from [Global] if set
|
||
global_registry = (self.knoe_cfg_data.get("Global", {}) or {}).get("KNOE_IMAGE_REGISTRY", "").strip()
|
||
if global_registry:
|
||
registry = global_registry
|
||
|
||
if not registry or registry == "localhost:5000":
|
||
server, _token = _resolve_k3s_connection_fn(
|
||
project_root=self.project_root,
|
||
cfg_path=self.cfg_path,
|
||
)
|
||
host = _host_from_url(server)
|
||
if host:
|
||
registry = f"{host}:5000"
|
||
db_cfg["LOCAL_REGISTRY"] = registry
|
||
self.knoe_cfg_data["Docker Build"] = db_cfg
|
||
|
||
if not registry or registry == "localhost:5000":
|
||
push_ok = False
|
||
self.err(
|
||
"[ERROR] k3s mode but registry host could not be resolved; cannot push knoe-db image."
|
||
)
|
||
else:
|
||
# Preflight check: verify registry is reachable
|
||
reg_host = registry
|
||
reg_port = 80
|
||
if ":" in registry:
|
||
parts = registry.split(":")
|
||
reg_host = parts[0]
|
||
try:
|
||
reg_port = int(parts[1])
|
||
except:
|
||
pass
|
||
|
||
self.log(f"Preflight: checking registry reachability at {reg_host}:{reg_port}...")
|
||
if not _http_ping_registry(reg_host, reg_port):
|
||
self.err(f"[ERROR] Registry {registry} is unreachable from this node.")
|
||
self._db_built_success = False
|
||
self.knoe_cfg_data["Docker Build"]["STATUS"] = "Failed"
|
||
return
|
||
|
||
remote_tag = f"{registry}/{image_name}"
|
||
self.log(f"Tagging {image_name} as {remote_tag}...")
|
||
subprocess.run(
|
||
["docker", "tag", image_name, remote_tag],
|
||
capture_output=True,
|
||
)
|
||
|
||
if _registry_image_ref_exists(remote_tag):
|
||
self.log(
|
||
f"[SKIP] {remote_tag} already exists in registry; skipping push."
|
||
)
|
||
else:
|
||
self.log(f"Pushing {remote_tag} to registry...")
|
||
if not _push_docker_image(remote_tag, log_fn=self.log):
|
||
push_ok = False
|
||
self.err(f"[ERROR] Failed to push image {remote_tag}")
|
||
self._db_built_success = push_ok
|
||
else:
|
||
self._db_built_success = True
|
||
|
||
self.knoe_cfg_data["Docker Build"]["STATUS"] = (
|
||
"Built" if self._db_built_success else "Failed"
|
||
)
|
||
return
|
||
|
||
self.log("==> Build knoe-db image")
|
||
knoe_home = resolve_knoe_home()
|
||
mode_key = _deployment_mode_from_env(env_key) or "default"
|
||
build_dir = knoe_home / "build" / mode_key / "knoe-db"
|
||
build_dir.mkdir(parents=True, exist_ok=True)
|
||
source_dir = get_resource_path("knoe-db")
|
||
if not source_dir.exists():
|
||
source_dir = get_resource_path("knoe-db")
|
||
marker = build_dir / ".knoe_build_context_ready"
|
||
if not marker.exists():
|
||
copy_build_context_dir(source_dir, build_dir)
|
||
|
||
pub_key_path = Path.home() / ".ssh" / "id_knoe_ed25519.pub"
|
||
pub_key = pub_key_path.read_text().strip() if pub_key_path.exists() else ""
|
||
username = self._get_input("init_password.db_username", "")
|
||
|
||
cmd = ["docker", "build"]
|
||
cmd.extend(get_docker_build_platform_args(env_key))
|
||
cmd += [
|
||
"--build-arg",
|
||
f"PROLE_USER={username}",
|
||
"--build-arg",
|
||
f"PROLE_SSH_PUB_KEY={pub_key}",
|
||
"-t",
|
||
image_name,
|
||
".",
|
||
]
|
||
rc = self._run_cmd(cmd, cwd=str(build_dir))
|
||
if rc == 0:
|
||
self._db_built_success = True
|
||
self.log("[OK] Build successful.")
|
||
push_ok = True
|
||
if env_key == "dev":
|
||
cluster_name = "knoe-dev-cluster"
|
||
self.log(f"Importing image to {cluster_name}...")
|
||
self._run_cmd(
|
||
["k3d", "image", "import", image_name, "-c", cluster_name]
|
||
)
|
||
else:
|
||
# For non-dev, tag and push to registry
|
||
db_cfg = self.knoe_cfg_data.get("Docker Build", {}) or {}
|
||
registry = (db_cfg.get("LOCAL_REGISTRY") or "").strip()
|
||
|
||
# Prioritize KNOE_IMAGE_REGISTRY from [Global] if set
|
||
global_registry = (self.knoe_cfg_data.get("Global", {}) or {}).get("KNOE_IMAGE_REGISTRY", "").strip()
|
||
if global_registry:
|
||
registry = global_registry
|
||
|
||
if env_key == "service" and (
|
||
not registry or registry == "localhost:5000"
|
||
):
|
||
server, _token = _resolve_k3s_connection_fn(
|
||
project_root=self.project_root,
|
||
cfg_path=self.cfg_path,
|
||
)
|
||
host = _host_from_url(server)
|
||
if host:
|
||
registry = f"{host}:5000"
|
||
db_cfg["LOCAL_REGISTRY"] = registry
|
||
self.knoe_cfg_data["Docker Build"] = db_cfg
|
||
|
||
if env_key == "service" and (
|
||
not registry or registry == "localhost:5000"
|
||
):
|
||
push_ok = False
|
||
self.err(
|
||
"[ERROR] k3s mode but registry host could not be resolved; cannot push knoe-db image."
|
||
)
|
||
self._db_built_success = False
|
||
else:
|
||
if not registry:
|
||
registry = "localhost:5000"
|
||
remote_tag = f"{registry}/{image_name}"
|
||
self.log(f"Tagging {image_name} as {remote_tag}...")
|
||
subprocess.run(["docker", "tag", image_name, remote_tag], check=True)
|
||
|
||
if _registry_image_ref_exists(remote_tag):
|
||
self.log(
|
||
f"[SKIP] {remote_tag} already exists in registry; skipping push."
|
||
)
|
||
else:
|
||
self.log(f"Pushing {remote_tag} to registry...")
|
||
if not _push_docker_image(remote_tag, log_fn=self.log):
|
||
push_ok = False
|
||
self.err(f"[ERROR] Failed to push image {remote_tag}")
|
||
self._db_built_success = False
|
||
self.knoe_cfg_data["Docker Build"]["STATUS"] = (
|
||
"Built" if self._db_built_success else "Failed"
|
||
)
|
||
|
||
def _step_cluster(self) -> None:
|
||
self.log("==> Cluster setup")
|
||
cluster_env = self._get_input("init_cluster.cluster_env", "dev")
|
||
env_key = _normalize_cluster_env(cluster_env)
|
||
self.knoe_cfg_data["Initialize Cluster"]["ENVIRONMENT"] = cluster_env
|
||
self.knoe_cfg_data["Initialize Cluster"]["K3S_SERVER_URL"] = self._get_input(
|
||
"init_cluster.k3s_server_url", ""
|
||
)
|
||
self.knoe_cfg_data["Initialize Cluster"]["K3S_TOKEN"] = _encrypt_cfg_secret(
|
||
self._get_input("init_cluster.k3s_token", "") or ""
|
||
)
|
||
self.knoe_cfg_data["Optional Features"]["SUPABASE_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.supabase_enabled", False)
|
||
)
|
||
self.knoe_cfg_data["Optional Features"]["KERBEROS_ENABLED"] = _bool_str(
|
||
self._get_input_bool("init_cluster.kerberos_enabled", False)
|
||
)
|
||
self.knoe_cfg_data["Optional Features"]["AT_REST_ENCRYPTION_ENABLED"] = (
|
||
_bool_str(
|
||
self._get_input_bool("init_cluster.at_rest_encryption_enabled", False)
|
||
)
|
||
)
|
||
|
||
mode = _deployment_mode_from_env(cluster_env)
|
||
if mode:
|
||
self._ensure_registry_defaults(mode)
|
||
|
||
if not self._get_input_bool(
|
||
"init_cluster.start_cluster",
|
||
DEFAULT_ACTION_FLAGS.get("init_cluster.start_cluster", True),
|
||
):
|
||
self.log("[SKIP] Cluster start disabled.")
|
||
return
|
||
|
||
if env_key == "dev":
|
||
if not self.controller.check_docker_running():
|
||
if platform.system() == "Darwin":
|
||
self.log("Starting Docker...")
|
||
self._run_cmd(["open", "-a", "Docker"])
|
||
for _ in range(30):
|
||
time.sleep(2)
|
||
if self.controller.check_docker_running():
|
||
break
|
||
if not self.controller.check_docker_running():
|
||
raise Exception("Docker is not running.")
|
||
|
||
cluster_name = "knoe-dev-cluster"
|
||
res = subprocess.run(
|
||
["k3d", "cluster", "list", "--no-headers"],
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
res_stdout = res.stdout or ""
|
||
if self.reset_cluster:
|
||
self.log(f"[RESET] Deleting k3d cluster {cluster_name}...")
|
||
self._run_cmd(["k3d", "cluster", "delete", cluster_name])
|
||
res_stdout = ""
|
||
self.reset_cluster = False
|
||
|
||
if cluster_name not in res_stdout:
|
||
knoe_data = self._resolve_env_dir("PROLE_DATA", "data")
|
||
self._create_k3d_cluster(cluster_name, knoe_data)
|
||
else:
|
||
# Check if it's already running
|
||
if "running" not in res_stdout.lower():
|
||
self._run_cmd(["k3d", "cluster", "start", cluster_name])
|
||
else:
|
||
self.log(f"[OK] Cluster {cluster_name} is already running.")
|
||
self.log(f"[OK] Cluster ready: {cluster_name}")
|
||
try:
|
||
os.environ.pop("KUBECONFIG", None)
|
||
subprocess.run(
|
||
["kubectl", "config", "use-context", f"k3d-{cluster_name}"],
|
||
capture_output=True,
|
||
)
|
||
except Exception:
|
||
pass
|
||
self.inputs["init_cluster.app_cluster_kubecontext"] = f"k3d-{cluster_name}"
|
||
self.inputs["init_cluster.db_cluster_kubecontext"] = f"k3d-{cluster_name}"
|
||
self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}"
|
||
self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}"
|
||
else:
|
||
# Ensure we don't pin KUBECONFIG to a stale local file when another
|
||
# kubeconfig (e.g. Ansible-fetched client-cert auth) is available.
|
||
service_dir = self._get_input("env_setup.KNOE_SERVICE", "").strip()
|
||
candidates: list[Path] = []
|
||
|
||
kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip()
|
||
if kubeconfig_env:
|
||
try:
|
||
candidates.append(Path(kubeconfig_env).expanduser())
|
||
except Exception:
|
||
pass
|
||
|
||
# Prefer Ansible-fetched kubeconfig in the project root.
|
||
candidates.append(self.project_root / "knoe-k3s.kubeconfig")
|
||
# Then standard user kubeconfig.
|
||
candidates.append(Path.home() / ".kube" / "config")
|
||
|
||
# Then knoe-specific secret locations.
|
||
if service_dir:
|
||
candidates.append(Path(service_dir) / "secrets" / "k3s.kubeconfig")
|
||
candidates.append(resolve_knoe_home(env={}) / "secrets" / "k3s.kubeconfig")
|
||
candidates.append(self.project_root / "etc" / "secrets" / "k3s.kubeconfig")
|
||
|
||
selected = _select_existing_kubeconfig(candidates)
|
||
if selected:
|
||
os.environ["KUBECONFIG"] = str(selected)
|
||
self.log(f"Using kubeconfig: {selected}")
|
||
else:
|
||
# No local kubeconfig found – try fetching from k3s via Ansible
|
||
fetched = self._fetch_k3s_kubeconfig()
|
||
if fetched:
|
||
os.environ["KUBECONFIG"] = str(fetched)
|
||
self.log(f"Using fetched kubeconfig: {fetched}")
|
||
|
||
if mode == "k8s":
|
||
gcp_cfg = self.gcp_cfg if isinstance(self.gcp_cfg, dict) else {}
|
||
project_id = str(
|
||
gcp_cfg.get("project_id")
|
||
or gcp_cfg.get("PROJECT_ID")
|
||
or self._get_input("init_cluster.project_id", "")
|
||
or ""
|
||
).strip()
|
||
if not project_id:
|
||
raise Exception("GCP project_id is required for dual-cluster GKE setup.")
|
||
|
||
app_cluster_name = self._app_cluster_name()
|
||
app_cluster_mode = self._app_cluster_mode()
|
||
app_machine_type = (
|
||
self._get_input("init_cluster.app_cluster_machine_type", "")
|
||
or DEFAULT_APP_CLUSTER_MACHINE_TYPE
|
||
).strip() or DEFAULT_APP_CLUSTER_MACHINE_TYPE
|
||
app_node_count = int(
|
||
(self._get_input("init_cluster.app_cluster_node_count", "") or str(DEFAULT_APP_CLUSTER_NODE_COUNT)).strip()
|
||
or str(DEFAULT_APP_CLUSTER_NODE_COUNT)
|
||
)
|
||
db_cluster_name = self._cnpg_cluster_name()
|
||
db_cluster_mode = self._db_cluster_mode()
|
||
db_node_count = int(
|
||
(self._get_input("init_cluster.db_cluster_node_count", "") or "3").strip() or "3"
|
||
)
|
||
db_machine_type = (
|
||
self._get_input("init_cluster.db_cluster_machine_type", "")
|
||
or DEFAULT_DB_CLUSTER_MACHINE_TYPE
|
||
).strip() or DEFAULT_DB_CLUSTER_MACHINE_TYPE
|
||
db_boot_disk_type = (
|
||
self._get_input("init_cluster.db_boot_disk_type", "")
|
||
or DEFAULT_DB_BOOT_DISK_TYPE
|
||
).strip() or DEFAULT_DB_BOOT_DISK_TYPE
|
||
db_boot_disk_size_gb = int(
|
||
(
|
||
self._get_input("init_cluster.db_boot_disk_size_gb", "")
|
||
or str(DEFAULT_DB_BOOT_DISK_SIZE_GB)
|
||
).strip()
|
||
or str(DEFAULT_DB_BOOT_DISK_SIZE_GB)
|
||
)
|
||
db_location = (
|
||
self._get_input("init_cluster.db_cluster_region", "")
|
||
or gcp_cfg.get("region")
|
||
or gcp_cfg.get("REGION")
|
||
or gcp_cfg.get("zone")
|
||
or gcp_cfg.get("ZONE")
|
||
or "us-central1"
|
||
)
|
||
app_location = (
|
||
self._get_input("init_cluster.app_cluster_region", "")
|
||
or gcp_cfg.get("region")
|
||
or gcp_cfg.get("REGION")
|
||
or db_location
|
||
)
|
||
|
||
app_spec = GkeClusterSpec(
|
||
name=app_cluster_name,
|
||
mode=app_cluster_mode,
|
||
location=str(app_location).strip(),
|
||
machine_type=app_machine_type,
|
||
node_count=max(1, app_node_count),
|
||
boot_disk_type=DEFAULT_DB_BOOT_DISK_TYPE,
|
||
boot_disk_size_gb=DEFAULT_DB_BOOT_DISK_SIZE_GB,
|
||
)
|
||
db_spec = GkeClusterSpec(
|
||
name=db_cluster_name,
|
||
mode=db_cluster_mode,
|
||
location=str(db_location).strip(),
|
||
machine_type=db_machine_type,
|
||
node_count=max(3, db_node_count),
|
||
node_pool_name="cnpg-db-pool",
|
||
boot_disk_type=db_boot_disk_type,
|
||
boot_disk_size_gb=db_boot_disk_size_gb,
|
||
)
|
||
|
||
self.log(f"[app-cluster {app_cluster_name}] validating cluster")
|
||
ensure_app_cluster(project_id=project_id, spec=app_spec, log=self.log)
|
||
self.log(f"[db-cluster {db_cluster_name}] ensuring Standard GKE cluster")
|
||
ensure_db_cluster(project_id=project_id, spec=db_spec, log=self.log)
|
||
|
||
app_ctx = get_cluster_credentials(
|
||
project_id=project_id,
|
||
cluster_name=app_cluster_name,
|
||
location=app_spec.location,
|
||
log=self.log,
|
||
)
|
||
db_ctx = get_cluster_credentials(
|
||
project_id=project_id,
|
||
cluster_name=db_cluster_name,
|
||
location=db_spec.location,
|
||
log=self.log,
|
||
)
|
||
self.inputs["init_cluster.app_cluster_kubecontext"] = app_ctx
|
||
self.inputs["init_cluster.db_cluster_kubecontext"] = db_ctx
|
||
self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = app_ctx
|
||
self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = db_ctx
|
||
self.knoe_cfg_data.setdefault("Global", {})["APP_CLUSTER_NAME"] = app_cluster_name
|
||
self.knoe_cfg_data.setdefault("Global", {})["APP_CLUSTER_MODE"] = app_cluster_mode
|
||
self.knoe_cfg_data.setdefault("Global", {})["APP_CLUSTER_KUBECONTEXT"] = app_ctx
|
||
self.knoe_cfg_data.setdefault("Global", {})["DB_CLUSTER_NAME"] = db_cluster_name
|
||
self.knoe_cfg_data.setdefault("Global", {})["DB_CLUSTER_MODE"] = db_cluster_mode
|
||
self.knoe_cfg_data.setdefault("Global", {})["DB_CLUSTER_KUBECONTEXT"] = db_ctx
|
||
# Back-compat default context points to app cluster for platform services.
|
||
self.knoe_cfg_data.setdefault("Global", {})["KUBECONTEXT"] = app_ctx
|
||
|
||
kubectl = subprocess.run(["which", "kubectl"], capture_output=True)
|
||
if kubectl.returncode != 0:
|
||
raise Exception(
|
||
"kubectl not found. Please install kubectl and configure access to the target cluster."
|
||
)
|
||
|
||
if env_key == "service":
|
||
if self.reset_cluster:
|
||
self.log("[RESET] Resetting k3s cluster (mode k3s)...")
|
||
self._cleanup_local_k3s_artifacts()
|
||
|
||
delete_cmd = [
|
||
"./ansible.sh",
|
||
"-p",
|
||
"infrastructure/playbooks/k3s_delete.yml",
|
||
]
|
||
self.log(f"Running: {' '.join(delete_cmd)}")
|
||
rc = self._run_cmd(delete_cmd)
|
||
if rc != 0:
|
||
raise Exception(f"k3s delete failed with code {rc}")
|
||
|
||
# 1) --reset should run the ansible k3s/tasks/cleanup on myrddin and then reinstall.
|
||
# This is handled by k3s_reset.yml
|
||
reset_cmd = [
|
||
"./ansible.sh",
|
||
"-p",
|
||
"infrastructure/playbooks/k3s_reset.yml",
|
||
"-l",
|
||
"myrddin.knoe.org",
|
||
] + vault_args
|
||
self.log(f"Running: {' '.join(reset_cmd)}")
|
||
rc = self._run_cmd(reset_cmd)
|
||
if rc != 0:
|
||
raise Exception(f"k3s reset failed with code {rc}")
|
||
|
||
# 2) Capture new k3s token + TLS bundle from init server and update vault
|
||
sync_token_cmd = [
|
||
"./ansible.sh",
|
||
"-p",
|
||
"infrastructure/playbooks/k3s_sync.yml",
|
||
"-l",
|
||
"myrddin.knoe.org",
|
||
] + vault_args
|
||
self.log(f"Running: {' '.join(sync_token_cmd)}")
|
||
rc = self._run_cmd(sync_token_cmd)
|
||
if rc != 0:
|
||
raise Exception(f"k3s sync (token) failed with code {rc}")
|
||
|
||
# 3) Reinstall agent nodes with the refreshed token
|
||
reset_agents_cmd = [
|
||
"./ansible.sh",
|
||
"-p",
|
||
"infrastructure/playbooks/k3s_reset.yml",
|
||
"-l",
|
||
"pi.knoe.org,retropie.knoe.org",
|
||
] + vault_args
|
||
self.log(f"Running: {' '.join(reset_agents_cmd)}")
|
||
rc = self._run_cmd(reset_agents_cmd)
|
||
if rc != 0:
|
||
raise Exception(f"k3s reset (agents) failed with code {rc}")
|
||
|
||
self.reset_cluster = False
|
||
|
||
if not (os.environ.get("KUBECONFIG") or "").strip():
|
||
kubeconfig_path = _find_kubeconfig_file()
|
||
if kubeconfig_path:
|
||
os.environ["KUBECONFIG"] = kubeconfig_path
|
||
self.log(f"Using local kubeconfig: {kubeconfig_path}")
|
||
|
||
kubeconfig_env = (os.environ.get("KUBECONFIG") or "").strip()
|
||
if kubeconfig_env and os.path.exists(kubeconfig_env):
|
||
cmd = ["kubectl", "--kubeconfig", kubeconfig_env, "cluster-info"]
|
||
else:
|
||
server = (
|
||
self._get_input("init_cluster.k3s_server_url", "") or ""
|
||
).strip()
|
||
token = (
|
||
self._get_input("init_cluster.k3s_token", "") or ""
|
||
).strip()
|
||
if token:
|
||
token = self._resolve_secret_value(token)
|
||
if server and token:
|
||
if not server.startswith("http"):
|
||
server = f"https://{server}"
|
||
cmd = [
|
||
"kubectl",
|
||
"--server=" + server,
|
||
"--token=" + token,
|
||
"--insecure-skip-tls-verify=true",
|
||
"cluster-info",
|
||
]
|
||
else:
|
||
raise Exception(
|
||
"K3s server URL/token missing and no kubeconfig available for service cluster."
|
||
)
|
||
else:
|
||
cmd = ["kubectl", "cluster-info"]
|
||
res = subprocess.run(cmd, capture_output=True, text=True)
|
||
if res.returncode != 0:
|
||
raise Exception("Cluster is not reachable.")
|
||
self.log(f"[OK] Cluster ready: {env_key}")
|
||
|
||
if env_key == "service":
|
||
try:
|
||
registry_ns = self._registry_namespace()
|
||
env = self._script_env_for_namespace(self._service_namespace())
|
||
env["REGISTRY_NAMESPACE"] = registry_ns
|
||
self.log("==> Registry preflight")
|
||
registry_ops.update(
|
||
namespace=registry_ns,
|
||
env=env,
|
||
mode=env.get("KNOE_MODE"),
|
||
project_root=self.project_root,
|
||
log=self.log,
|
||
)
|
||
except Exception as e:
|
||
self.err(f"[WARN] Registry preflight failed: {e}")
|
||
|
||
# If this run was launched with a reset request, attempt to reclaim any
|
||
# strongly-matched stale Released PVs early, before later deploy/init
|
||
# steps that may depend on PVC scheduling.
|
||
if getattr(self, "_reset_reclaim_pvs_requested", False):
|
||
try:
|
||
service_ns = self._service_namespace()
|
||
env = self._script_env_for_namespace(service_ns)
|
||
self.log("==> Reset: reclaim stale Released PV claimRefs")
|
||
self._reset_reclaim_stale_pvs(env)
|
||
except Exception as e:
|
||
self.err(f"[WARN] Reset stale-PV reclaim failed: {e}")
|
||
self._reset_reclaim_pvs_requested = False
|
||
|
||
if self._openbao_init_env:
|
||
env = dict(self._openbao_init_env)
|
||
mode = _deployment_mode_from_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
if not mode:
|
||
if env_key == "dev":
|
||
mode = "k3d"
|
||
elif env_key == "service":
|
||
mode = "k3s"
|
||
elif env_key == "prod":
|
||
mode = "k8s"
|
||
env.setdefault("KNOE_MODE", mode or env_key)
|
||
self.log("==> OpenBao initialize (cluster)")
|
||
db_ns = (self._get_input("init_password.db_namespace", "") or "").strip()
|
||
if not db_ns:
|
||
db_ns = self._service_namespace()
|
||
openbao_ops.initialize(
|
||
namespace=db_ns,
|
||
env=env,
|
||
project_root=self.project_root,
|
||
mode=mode,
|
||
log=self.log,
|
||
)
|
||
|
||
# Materialize DB secrets in Kubernetes now that we have a cluster and a user-provided password.
|
||
# This prevents downstream init steps (e.g. CNPG) from failing due to missing secrets.
|
||
self.ensure_db_k8s_secrets(
|
||
db_ns,
|
||
self._openbao_init_password or "",
|
||
log_fn=self.log,
|
||
)
|
||
self._openbao_init_env = None
|
||
self._openbao_init_password = None
|
||
|
||
# Run repair pipeline if anomalies are detected on a ready cluster
|
||
try:
|
||
service_ns = self._service_namespace()
|
||
env = self._script_env_for_namespace(service_ns)
|
||
self._maybe_run_repair_pipeline(env)
|
||
except Exception:
|
||
pass
|
||
|
||
if env_key == "service":
|
||
self._deploy_common_services(env_key)
|
||
else:
|
||
self._deploy_opentofu(env_key)
|
||
|
||
def _deploy_common_services(self, env_key: str) -> None:
|
||
self.log("==> Common services deploy")
|
||
service_ns = self._service_namespace()
|
||
env = self._script_env_for_namespace(service_ns)
|
||
mode = env.get("KNOE_MODE") or self._deployment_mode()
|
||
manage_opentofu = str(mode).lower() != "k8s"
|
||
registry_ns = self._registry_namespace()
|
||
if not manage_opentofu:
|
||
self.log("[INFO] k8s mode: skipping OpenTofu checks/deploy in common services.")
|
||
|
||
try:
|
||
# Avoid deploying into a namespace that is currently being deleted.
|
||
self._ensure_namespace_ready(env, service_ns)
|
||
|
||
# Ensure node labels are present before init scripts that depend on them
|
||
# (e.g. node-role selectors and pinned local PV workloads).
|
||
try:
|
||
self._apply_ansible_node_labels(env)
|
||
except Exception:
|
||
pass
|
||
|
||
# Refuse to proceed when the cluster has hard prerequisites blocked
|
||
# (e.g. PV binding / scheduling issues). This prevents blind retries
|
||
# and surfaces actionable diagnostics.
|
||
self._reconcile_blocked_cluster_state(env=env, service_ns=service_ns)
|
||
|
||
def _status_safe(check_fn) -> bool:
|
||
try:
|
||
return bool(check_fn())
|
||
except Exception:
|
||
return False
|
||
|
||
status_checks = [
|
||
_status_safe(
|
||
lambda: registry_ops.status(
|
||
namespace=registry_ns,
|
||
env=env,
|
||
mode=mode,
|
||
)
|
||
),
|
||
_status_safe(
|
||
lambda: openbao_ops.status(
|
||
namespace=service_ns,
|
||
env=env,
|
||
mode=mode,
|
||
)
|
||
),
|
||
_status_safe(
|
||
lambda: garage_store_ops.status(
|
||
namespace=service_ns,
|
||
env=env,
|
||
)
|
||
),
|
||
]
|
||
if manage_opentofu:
|
||
status_checks.append(
|
||
_status_safe(
|
||
lambda: opentofu_ops.status(
|
||
namespace=service_ns,
|
||
env=env,
|
||
)
|
||
)
|
||
)
|
||
is_healthy = all(status_checks)
|
||
if is_healthy:
|
||
self.log(
|
||
f"[OK] Common services in {service_ns} are healthy. Skipping update."
|
||
)
|
||
return
|
||
|
||
self.log(
|
||
f"[INFO] Common services unhealthy or missing in {service_ns}. Attempting repair/deploy..."
|
||
)
|
||
try:
|
||
registry_ops.update(
|
||
namespace=registry_ns,
|
||
env=env,
|
||
mode=mode,
|
||
project_root=self.project_root,
|
||
log=self.log,
|
||
)
|
||
openbao_ops.update(
|
||
namespace=service_ns,
|
||
env=env,
|
||
mode=mode,
|
||
project_root=self.project_root,
|
||
log=self.log,
|
||
)
|
||
garage_store_ops.update(
|
||
namespace=service_ns,
|
||
env=env,
|
||
mode=mode,
|
||
project_root=self.project_root,
|
||
log=self.log,
|
||
)
|
||
if manage_opentofu:
|
||
opentofu_ops.update(
|
||
namespace=service_ns,
|
||
env=env,
|
||
mode=mode,
|
||
project_root=self.project_root,
|
||
log=self.log,
|
||
)
|
||
except Exception as e:
|
||
self.err(f"[ERROR] Common services deploy failed: {e}")
|
||
else:
|
||
# Re-validate after fix.
|
||
repair_checks = [
|
||
_status_safe(
|
||
lambda: registry_ops.status(
|
||
namespace=registry_ns,
|
||
env=env,
|
||
mode=mode,
|
||
)
|
||
),
|
||
_status_safe(
|
||
lambda: openbao_ops.status(
|
||
namespace=service_ns,
|
||
env=env,
|
||
mode=mode,
|
||
)
|
||
),
|
||
_status_safe(
|
||
lambda: garage_store_ops.status(
|
||
namespace=service_ns,
|
||
env=env,
|
||
)
|
||
),
|
||
]
|
||
if manage_opentofu:
|
||
repair_checks.append(
|
||
_status_safe(
|
||
lambda: opentofu_ops.status(
|
||
namespace=service_ns,
|
||
env=env,
|
||
)
|
||
)
|
||
)
|
||
repaired = all(repair_checks)
|
||
if not repaired:
|
||
self.err(
|
||
f"[ERROR] Common services still unhealthy after repair attempt."
|
||
)
|
||
else:
|
||
self.log(f"[OK] Common services in {service_ns} are now healthy.")
|
||
finally:
|
||
pass
|
||
|
||
def _apply_ansible_node_labels(self, env: dict) -> None:
|
||
"""Apply Ansible-defined node labels to the current cluster.
|
||
|
||
This makes node selection deterministic even when UI selections are stale
|
||
or missing.
|
||
"""
|
||
|
||
inv = (
|
||
(self.knoe_cfg_data.get("Network", {}) or {}).get("ANSIBLE_INVENTORY")
|
||
or ""
|
||
).strip()
|
||
if not inv:
|
||
return
|
||
|
||
inv_path = Path(inv)
|
||
if not inv_path.exists():
|
||
return
|
||
|
||
labels_by_host = _detect_ansible_node_labels(inv_path)
|
||
if not labels_by_host:
|
||
return
|
||
|
||
self.log("==> Applying Ansible node labels")
|
||
base_env = dict(os.environ)
|
||
base_env.update(env or {})
|
||
|
||
for host, labels in sorted(labels_by_host.items()):
|
||
if not labels:
|
||
continue
|
||
label_args = [f"{k}={v}" for k, v in sorted(labels.items())]
|
||
|
||
def _label(node_name: str) -> bool:
|
||
res = subprocess.run(
|
||
["kubectl", "label", "node", node_name, "--overwrite", *label_args],
|
||
capture_output=True,
|
||
text=True,
|
||
env=base_env,
|
||
)
|
||
if res.returncode == 0:
|
||
return True
|
||
return False
|
||
|
||
# Prefer FQDN node names, but retry with short hostname when needed.
|
||
if _label(host):
|
||
continue
|
||
if "." in host:
|
||
short = host.split(".", 1)[0]
|
||
if short and _label(short):
|
||
continue
|
||
|
||
self.err(f"[WARN] Failed to apply labels to node '{host}'.")
|
||
|
||
def _deploy_opentofu(self, env_key: str) -> None:
|
||
self.log("==> OpenTofu deploy")
|
||
ns = (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip() or "default"
|
||
env = self._script_env_for_namespace(ns)
|
||
|
||
try:
|
||
opentofu_ops.start(namespace=ns, env=env, mode=env.get("KNOE_MODE"), log=self.log)
|
||
finally:
|
||
pass
|
||
|
||
def _step_kerberos(self) -> None:
|
||
enabled = self._get_input_bool("kerberos_config.enabled", False)
|
||
if not enabled:
|
||
self.log("[SKIP] Kerberos disabled.")
|
||
return
|
||
realm = self._get_input("kerberos_config.realm", "")
|
||
kdc = self._get_input("kerberos_config.kdc", "")
|
||
user = self._get_input("kerberos_config.user", "")
|
||
password = self._get_input("kerberos_config.password", "")
|
||
self.knoe_cfg_data["Kerberos Authentication"]["ENABLED"] = _bool_str(enabled)
|
||
self.knoe_cfg_data["Kerberos Authentication"]["REALM"] = realm
|
||
self.knoe_cfg_data["Kerberos Authentication"]["KDC"] = kdc
|
||
self.knoe_cfg_data["Kerberos Authentication"]["SERVER"] = kdc
|
||
self.knoe_cfg_data["Kerberos Authentication"]["USER"] = user
|
||
self.knoe_cfg_data["Kerberos Authentication"]["PASSWORD"] = password
|
||
self.knoe_cfg_data["Kerberos Authentication"]["AD_PORT_FORWARD"] = (
|
||
os.environ.get("KRB5_AD_PORT_FORWARD", "1")
|
||
)
|
||
self.knoe_cfg_data["Kerberos Authentication"]["AD_TCP_PORTS"] = os.environ.get(
|
||
"KRB5_AD_TCP_PORTS", "88 389 445 464 636"
|
||
)
|
||
self.knoe_cfg_data["Kerberos Authentication"]["AD_UDP_PORTS"] = os.environ.get(
|
||
"KRB5_AD_UDP_PORTS", "88 464"
|
||
)
|
||
self.knoe_cfg_data["Kerberos Authentication"]["AD_PROXY_HOST_NETWORK"] = (
|
||
os.environ.get("KRB5_AD_PROXY_HOST_NETWORK", "1")
|
||
)
|
||
self.knoe_cfg_data["Kerberos Authentication"]["AD_PROXY_IMAGE"] = (
|
||
os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat")
|
||
)
|
||
self.knoe_cfg_data["Kerberos Authentication"]["AD_PROXY_SERVICE"] = (
|
||
os.environ.get("KRB5_AD_SERVICE_NAME", "knoe-kerberos-ad-dc")
|
||
)
|
||
|
||
ns = (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip() or "default"
|
||
env = self._script_env_for_namespace(ns)
|
||
env["KERBEROS_ENABLED"] = _bool_str(enabled)
|
||
env["ENABLED"] = env["KERBEROS_ENABLED"]
|
||
env["KRB5_REALM"] = realm
|
||
if env.get("KNOE_MODE") == "k3s":
|
||
env.setdefault("KRB5_AD_PORT_FORWARD", "0")
|
||
|
||
if not self._get_input_bool(
|
||
"kerberos_config.test_connection",
|
||
DEFAULT_ACTION_FLAGS.get("kerberos_config.test_connection", False),
|
||
):
|
||
self.log("[SKIP] Kerberos test disabled.")
|
||
return
|
||
|
||
# Check if already tested successfully
|
||
if (
|
||
self.knoe_cfg_data.get("Kerberos Authentication", {}).get("TEST_STATUS")
|
||
== "Success"
|
||
):
|
||
self.log("[OK] Kerberos already tested successfully. Skipping.")
|
||
return
|
||
if not (realm and user and password and kdc):
|
||
self.err("[WARN] Kerberos test skipped: missing realm/user/password/kdc.")
|
||
return
|
||
|
||
self.log("==> Kerberos test: init_kerberos.sh test")
|
||
rc2 = self._run_script("init_kerberos.sh", args=["test"], env=env)
|
||
if rc2 != 0:
|
||
self.err(f"[ERROR] Kerberos test failed (code {rc2})")
|
||
self.knoe_cfg_data.setdefault("Kerberos Authentication", {})[
|
||
"TEST_STATUS"
|
||
] = "Failed"
|
||
else:
|
||
self.log("[OK] Kerberos test completed successfully.")
|
||
self.knoe_cfg_data.setdefault("Kerberos Authentication", {})[
|
||
"TEST_STATUS"
|
||
] = "Success"
|
||
|
||
def _step_init_scripts(self) -> None:
|
||
if not self._get_input_bool(
|
||
"init_scripts.run_scripts",
|
||
DEFAULT_ACTION_FLAGS.get("init_scripts.run_scripts", True),
|
||
):
|
||
self.log("[SKIP] Init scripts disabled.")
|
||
self.knoe_cfg_data["Initialization Scripts"]["STATUS"] = "Skipped"
|
||
return
|
||
|
||
if (
|
||
self.knoe_cfg_data.get("Initialization Scripts", {}).get("STATUS")
|
||
== "Finished"
|
||
):
|
||
self.log("[OK] Initialization scripts already finished.")
|
||
return
|
||
|
||
self.log("==> Initialization scripts")
|
||
mode = self._deployment_mode()
|
||
db_ns = self._secret_namespace()
|
||
if mode == "k8s" and db_ns == "default":
|
||
db_ns = "knoe-db-0"
|
||
service_ns = self._service_namespace()
|
||
db_env = self._script_env_for_namespace(db_ns, cluster_role="db")
|
||
app_env = self._script_env_for_namespace(service_ns, cluster_role="app")
|
||
|
||
opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(app_env)
|
||
|
||
password = self._get_input("init_password.db_password", "")
|
||
kerberos_enabled = self._get_input_bool("kerberos_config.enabled", False)
|
||
pre_cnpg_steps = [
|
||
("init_certmgr.sh", ["initialize"], False),
|
||
]
|
||
if kerberos_enabled:
|
||
pre_cnpg_steps.append(("init_kerberos.sh", ["initialize"], False))
|
||
post_cnpg_steps: list[tuple[str, list[str], bool, str]] = [
|
||
("init_cnpg_backup.sh", ["start"], False, "db"),
|
||
("init_kong.sh", ["start"], False, "app"),
|
||
]
|
||
if not opt_allowed:
|
||
self.log(f"[SKIP] Monitoring disabled by policy: {opt_reason}")
|
||
self.knoe_cfg_data.setdefault("Monitoring", {})["STATUS"] = "Skipped"
|
||
if mode != "k3d":
|
||
post_cnpg_steps.append(("init_nginx_ingress.sh", ["initialize"], False, "app"))
|
||
|
||
overall_success = True
|
||
|
||
# Common services (Python owners) — replaces init_common_services.sh
|
||
self.log(
|
||
f"[app-cluster {self._app_cluster_name()} context={app_env.get('KUBECONTEXT','')}] "
|
||
f"common services namespace={service_ns}"
|
||
)
|
||
try:
|
||
registry_ns = str(app_env.get("REGISTRY_NAMESPACE") or service_ns).strip() or service_ns
|
||
registry_ops.update(namespace=registry_ns, env=app_env, mode=mode, log=self.log)
|
||
openbao_ops.update(namespace=service_ns, env=app_env, mode=mode, log=self.log)
|
||
garage_store_ops.update(namespace=service_ns, env=app_env, mode=mode, log=self.log)
|
||
opentofu_ops.update(namespace=service_ns, env=app_env, mode=mode, log=self.log)
|
||
except Exception as e:
|
||
self.err(f"[ERROR] common services (python owners) failed: {e}")
|
||
overall_success = False
|
||
|
||
# Run pre-CNPG shell scripts (cert-manager, optional kerberos)
|
||
for script, args, needs_password in pre_cnpg_steps:
|
||
self.log(f"--> {script} {' '.join(args)}")
|
||
stdin_text = f"{password}\n" if needs_password else None
|
||
rc = self._run_script(
|
||
script,
|
||
args=args,
|
||
env=db_env,
|
||
stdin_text=stdin_text,
|
||
on_line=self._process_script_output_line,
|
||
)
|
||
if rc != 0:
|
||
self.err(f"[ERROR] {script} failed (code {rc})")
|
||
overall_success = False
|
||
|
||
# CNPG initialization (Python) — replaces init_cloudnative_pg.sh initialize
|
||
if overall_success:
|
||
self.log(
|
||
f"[db-cluster {self._cnpg_cluster_name()} context={db_env.get('KUBECONTEXT','')}] "
|
||
f"cnpg_initialize namespace={db_ns}"
|
||
)
|
||
try:
|
||
self.ensure_db_k8s_secrets(db_ns, password, log_fn=self.log)
|
||
except Exception as e:
|
||
self.err(f"[ERROR] Failed to ensure DB secrets before CNPG init: {e}")
|
||
overall_success = False
|
||
if overall_success:
|
||
if self._cluster_storage_milestone_enabled():
|
||
try:
|
||
self._ensure_cnpg_storage_provisioned(db_ns, db_env)
|
||
except Exception as e:
|
||
self.err(f"[WARN] Failed to provision CNPG storage before init: {e}")
|
||
else:
|
||
self.log(
|
||
"[SKIP] Cluster storage milestone shelved; "
|
||
"retaining utility workflows only."
|
||
)
|
||
try:
|
||
cnpg_initialize(
|
||
namespace=db_ns,
|
||
cluster_name=str(db_env.get("CLUSTER_NAME") or db_env.get("CNPG_CLUSTER_NAME") or "knoe-db").strip(),
|
||
env=db_env,
|
||
project_root=self.project_root,
|
||
log=self.log,
|
||
mode=mode,
|
||
)
|
||
except Exception as e:
|
||
self.err(f"[ERROR] CNPG initialization failed: {e}")
|
||
overall_success = False
|
||
|
||
# Run post-CNPG shell scripts (backup, kong, ingress)
|
||
if overall_success:
|
||
for script, args, needs_password, role in post_cnpg_steps:
|
||
self.log(f"--> {script} {' '.join(args)}")
|
||
stdin_text = f"{password}\n" if needs_password else None
|
||
target_env = db_env if role == "db" else app_env
|
||
rc = self._run_script(
|
||
script,
|
||
args=args,
|
||
env=target_env,
|
||
stdin_text=stdin_text,
|
||
on_line=self._process_script_output_line,
|
||
)
|
||
if rc != 0:
|
||
self.err(f"[ERROR] {script} failed (code {rc})")
|
||
overall_success = False
|
||
|
||
if overall_success and opt_allowed:
|
||
self.log(f"[app-cluster {self._app_cluster_name()}] monitoring (python owner) initialize")
|
||
try:
|
||
monitoring_ns = str(app_env.get("MONITORING_NAMESPACE") or "monitoring").strip() or "monitoring"
|
||
monitoring_ops.initialize(
|
||
namespace=monitoring_ns,
|
||
env=app_env,
|
||
mode=mode,
|
||
log=self.log,
|
||
)
|
||
except Exception as e:
|
||
self.err(f"[ERROR] monitoring initialization failed: {e}")
|
||
overall_success = False
|
||
|
||
self._scripts_success = overall_success
|
||
|
||
# Verify critical secrets
|
||
self.log(f"==> Verifying critical secrets in namespace {db_ns}")
|
||
critical_secrets = ["knoe-db-user", "knoe-db-superuser", "cnpg-admin-key"]
|
||
missing_secrets = []
|
||
for secret in critical_secrets:
|
||
rc_s = self._run_cmd(
|
||
["kubectl", "get", "secret", secret, "-n", db_ns],
|
||
env=db_env,
|
||
)
|
||
if rc_s != 0:
|
||
missing_secrets.append(secret)
|
||
|
||
if missing_secrets:
|
||
self.err(
|
||
f"[CRITICAL] Missing secrets in namespace '{db_ns}': {', '.join(missing_secrets)}"
|
||
)
|
||
self.err("Database initialization will fail without these secrets.")
|
||
self._scripts_success = False
|
||
|
||
self.knoe_cfg_data["Initialization Scripts"]["STATUS"] = (
|
||
"Completed" if self._scripts_success else "Attempted"
|
||
)
|
||
|
||
def _step_cnpg_deploy(self) -> None:
|
||
if not self._get_input_bool(
|
||
"init_cnpg_deploy.run_deploy",
|
||
DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True),
|
||
):
|
||
self.log("[SKIP] CnPG deploy disabled.")
|
||
self.knoe_cfg_data["Deployment"]["STATUS"] = "Skipped"
|
||
return
|
||
db_cluster = self._cnpg_cluster_name()
|
||
self.log(f"==> [db-cluster {db_cluster}] Deploy CloudNativePG")
|
||
ns = self._secret_namespace()
|
||
if self._deployment_mode() == "k8s" and ns == "default":
|
||
ns = "knoe-db-0"
|
||
env = self._script_env_for_namespace(ns, cluster_role="db")
|
||
cluster_name = str(
|
||
env.get("CLUSTER_NAME")
|
||
or env.get("CNPG_CLUSTER_NAME")
|
||
or self._cnpg_cluster_name()
|
||
).strip()
|
||
if not cluster_name:
|
||
cluster_name = "knoe-db"
|
||
mode = self._deployment_mode()
|
||
|
||
if self._cluster_storage_milestone_enabled():
|
||
try:
|
||
self._ensure_cnpg_storage_provisioned(ns, env)
|
||
except Exception as exc:
|
||
self._cnpg_success = False
|
||
self.knoe_cfg_data["Deployment"]["STATUS"] = "Attempted"
|
||
self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}")
|
||
return
|
||
else:
|
||
self.log(
|
||
"[SKIP] Cluster storage milestone shelved; "
|
||
"retaining utility workflows only."
|
||
)
|
||
|
||
try:
|
||
self.log(f"[db-cluster {db_cluster}] applying knoe-db cluster")
|
||
cnpg_deploy(
|
||
namespace=ns,
|
||
cluster_name=cluster_name,
|
||
env=env,
|
||
project_root=self.project_root,
|
||
log=self.log,
|
||
mode=mode,
|
||
)
|
||
self._cnpg_success = True
|
||
self.knoe_cfg_data["Deployment"]["STATUS"] = "Deployed"
|
||
except Exception as exc:
|
||
self._cnpg_success = False
|
||
self.knoe_cfg_data["Deployment"]["STATUS"] = "Attempted"
|
||
self.err(f"[ERROR] CnPG deploy failed: {exc}")
|
||
|
||
if self._get_input_bool(
|
||
"init_cnpg_deploy.force_rollout",
|
||
DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False),
|
||
):
|
||
# Only attempt rollout if there are existing CNPG pods; otherwise this is a fresh install
|
||
rc_check = subprocess.run(
|
||
[
|
||
"kubectl", "get", "pods",
|
||
"-n", ns, "-l", f"cnpg.io/cluster={cluster_name}",
|
||
"--no-headers",
|
||
],
|
||
capture_output=True, text=True, env=env,
|
||
)
|
||
has_pods = bool((rc_check.stdout or "").strip())
|
||
if has_pods:
|
||
self.log(f"==> [db-cluster {db_cluster}] Force rollout")
|
||
try:
|
||
cnpg_rollout(
|
||
namespace=ns,
|
||
cluster_name=cluster_name,
|
||
env=env,
|
||
log=self.log,
|
||
)
|
||
except Exception as exc2:
|
||
self.err(f"[ERROR] Rollout failed: {exc2}")
|
||
else:
|
||
self.log(
|
||
"[SKIP] Force rollout skipped — no existing CNPG pods (new installation)."
|
||
)
|
||
|
||
def _step_supabase(self) -> None:
|
||
if not self._get_input_bool("init_cluster.supabase_enabled", False):
|
||
self.log("[SKIP] Supabase deploy disabled.")
|
||
self.knoe_cfg_data["Supabase"] = {"STATUS": "Skipped"}
|
||
return
|
||
self.log("==> Deploy Supabase")
|
||
ns = (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip() or "default"
|
||
env = self._script_env_for_namespace(ns)
|
||
|
||
opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(env)
|
||
if not opt_allowed:
|
||
self.log(f"[SKIP] Supabase deploy disabled by policy: {opt_reason}")
|
||
self.knoe_cfg_data["Supabase"] = {"STATUS": "Skipped"}
|
||
self._supabase_success = False
|
||
return
|
||
|
||
script_path = self.project_root / "supabase" / "deploy.sh"
|
||
if not script_path.exists():
|
||
self.err(f"[ERROR] Supabase deploy script not found: {script_path}")
|
||
self._supabase_success = False
|
||
self.knoe_cfg_data["Supabase"] = {"STATUS": "Attempted"}
|
||
return
|
||
mode = (
|
||
os.environ.get("SUPABASE_DEPLOY_MODE")
|
||
or os.environ.get("SUPABASE_MODE")
|
||
or ""
|
||
).strip()
|
||
if not mode:
|
||
# Match cluster environment to supabase deploy mode
|
||
cluster_env = _normalize_cluster_env(
|
||
self._get_input("init_cluster.cluster_env", "dev")
|
||
)
|
||
if cluster_env == "dev":
|
||
mode = "k3d"
|
||
elif cluster_env in ("service", "prod"):
|
||
mode = "k8s"
|
||
else:
|
||
mode = "k3d"
|
||
args = ["--mode", mode]
|
||
cfg_path = None
|
||
if self.cfg_path and self.cfg_path.exists():
|
||
cfg_path = self.cfg_path
|
||
else:
|
||
conf_dir = knoe_conf_mgr.resolve_knoe_conf_dir(self.project_root)
|
||
candidate = knoe_conf_mgr.entrypoint_path(conf_dir)
|
||
if candidate.exists():
|
||
cfg_path = candidate
|
||
if cfg_path:
|
||
args.extend(["-c", str(cfg_path)])
|
||
if _parse_bool(os.environ.get("SUPABASE_USE_DEV_COMPOSE"), False):
|
||
args.append("--with-dev-helpers")
|
||
if _parse_bool(os.environ.get("SUPABASE_FOREGROUND"), False):
|
||
args.append("--foreground")
|
||
self.log(f"--> supabase/deploy.sh {' '.join(args)}")
|
||
rc = self._run_cmd(["bash", str(script_path)] + args, env=env)
|
||
if rc == 0:
|
||
self._supabase_success = True
|
||
self.knoe_cfg_data["Supabase"] = {"STATUS": "Deployed"}
|
||
else:
|
||
self._supabase_success = False
|
||
self.knoe_cfg_data["Supabase"] = {"STATUS": "Attempted"}
|
||
self.err(f"[ERROR] Supabase deploy failed (code {rc})")
|
||
|
||
def _finalize_secrets(self) -> None:
|
||
ns = (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip() or "default"
|
||
if not ns or ns == "default":
|
||
ns = self._get_input("env_setup.DATABASE_NAMESPACE", "") or "default"
|
||
if not ns or ns == "default":
|
||
ns = self._get_input("env_setup.NAMESPACE", "") or "default"
|
||
|
||
p1 = self._get_input("init_password.db_password", "")
|
||
if p1.startswith("${"):
|
||
self.log(
|
||
f"[OK] Secrets already finalized for namespace {ns}. Skipping build-a-bao."
|
||
)
|
||
return
|
||
|
||
self.log("==> OpenBao: finalize secrets")
|
||
env = self._script_env_for_namespace(ns)
|
||
rc = self._run_script("build-a-bao.sh", env=env)
|
||
if rc == 0:
|
||
self._secrets_finalized = True
|
||
self.log("[OK] Secrets saved to OpenBao and knoe.cfg updated.")
|
||
else:
|
||
self.err(f"[ERROR] build-a-bao.sh failed (code {rc})")
|
||
|
||
def _prepare_opentofu_pipeline(self) -> None:
|
||
env_key = _normalize_cluster_env(
|
||
self._get_input("init_cluster.cluster_env", "")
|
||
)
|
||
mode = _deployment_mode_from_env(self._get_input("init_cluster.mode", ""))
|
||
if not mode:
|
||
mode = _deployment_mode_from_env(os.environ.get("KNOE_MODE", ""))
|
||
|
||
# OpenTofu pipeline is k3s-scoped, but we still want it available for k3d
|
||
# installs when the user provides k3s connection settings (e.g., staging
|
||
# manifests for a service cluster while running a local dev cluster).
|
||
if env_key != "service" and mode not in ("k3d", "k3s"):
|
||
return
|
||
namespace = (
|
||
self._get_input("init_password.db_namespace", "") or ""
|
||
).strip()
|
||
if not namespace:
|
||
namespace = (
|
||
self._get_input("env_setup.DATABASE_NAMESPACE", "") or ""
|
||
).strip()
|
||
if not namespace:
|
||
namespace = (
|
||
self._get_input("env_setup.NAMESPACE", "") or ""
|
||
).strip()
|
||
if not namespace:
|
||
namespace = "default"
|
||
k3s_server = (self._get_input("init_cluster.k3s_server_url", "") or "").strip()
|
||
k3s_token = (self._get_input("init_cluster.k3s_token", "") or "").strip()
|
||
if k3s_token:
|
||
k3s_token = self._resolve_secret_value(k3s_token)
|
||
if k3s_server and not k3s_server.startswith("http"):
|
||
k3s_server = f"https://{k3s_server}"
|
||
if not k3s_server:
|
||
self.err("[WARN] OpenTofu pipeline skipped: missing k3s server URL.")
|
||
return
|
||
if not k3s_token:
|
||
self.err("[WARN] OpenTofu pipeline missing k3s token; writing empty token.")
|
||
pipeline_dir = _sync_opentofu_pipeline(
|
||
self.project_root, namespace, k3s_server, k3s_token
|
||
)
|
||
self.knoe_cfg_data["Deployment"]["OPENTOFU_PIPELINE_DIR"] = str(pipeline_dir)
|
||
self.log(f"[OK] OpenTofu pipeline prepared at {pipeline_dir}")
|
||
|
||
def _create_k3d_cluster(self, cluster_name: str, knoe_data: Path) -> None:
|
||
volume_args = _k3d_knoe_data_volume_args(str(knoe_data))
|
||
reg_args = []
|
||
if _local_registry_enabled("k3d"):
|
||
info = self._ensure_local_registry_available()
|
||
if info:
|
||
_host_registry, cluster_registry = info
|
||
reg_use = cluster_registry or ""
|
||
if reg_use.endswith(".localhost:5000"):
|
||
reg_use = reg_use.replace(".localhost:5000", ":5000")
|
||
elif reg_use.endswith(".localhost"):
|
||
reg_use = reg_use[: -len(".localhost")]
|
||
if reg_use:
|
||
reg_args = ["--registry-use", reg_use]
|
||
cmd = (
|
||
["k3d", "cluster", "create", cluster_name, "-a", "2"]
|
||
+ volume_args
|
||
+ reg_args
|
||
+ ["--api-port", "0.0.0.0:6443"]
|
||
)
|
||
rc = self._run_cmd(cmd)
|
||
if rc == 0:
|
||
return
|
||
if platform.system() == "Darwin":
|
||
fallback = resolve_knoe_home(env={}) / "data"
|
||
try:
|
||
if knoe_data.resolve() == fallback.resolve():
|
||
raise Exception("k3d cluster create failed.")
|
||
except Exception:
|
||
# If path resolution fails, still try the fallback
|
||
pass
|
||
self.err(
|
||
f"[WARN] k3d create failed with {knoe_data}; retrying with {fallback}..."
|
||
)
|
||
rc = self._run_cmd(
|
||
["k3d", "cluster", "create", cluster_name, "-a", "2"]
|
||
+ _k3d_knoe_data_volume_args(str(fallback))
|
||
+ reg_args
|
||
+ ["--api-port", "0.0.0.0:6443"]
|
||
)
|
||
if rc == 0:
|
||
# Persist fallback for the rest of the run
|
||
self.inputs["env_setup.PROLE_DATA"] = str(fallback)
|
||
return
|
||
raise Exception(f"Failed to create k3d cluster {cluster_name} (code {rc})")
|
||
|
||
def _perform_cluster_reset(self) -> None:
|
||
if not self.reset_requested:
|
||
return
|
||
|
||
cluster_env = self._get_input("init_cluster.cluster_env", "dev")
|
||
env_key = _normalize_cluster_env(cluster_env)
|
||
|
||
if env_key == "dev":
|
||
self.log("==> Cluster reset (k3d)")
|
||
if not self.controller.check_docker_running():
|
||
if platform.system() == "Darwin":
|
||
self.log("Starting Docker...")
|
||
self._run_cmd(["open", "-a", "Docker"])
|
||
for _ in range(30):
|
||
time.sleep(2)
|
||
if self.controller.check_docker_running():
|
||
break
|
||
if not self.controller.check_docker_running():
|
||
raise Exception("Docker is not running.")
|
||
|
||
cluster_name = "knoe-dev-cluster"
|
||
rc = self._run_cmd(["k3d", "cluster", "delete", cluster_name])
|
||
if rc != 0:
|
||
self.err(
|
||
f"[WARN] Failed to delete k3d cluster {cluster_name} (code {rc}); continuing."
|
||
)
|
||
|
||
knoe_data = self._resolve_env_dir("PROLE_DATA", "data")
|
||
self._create_k3d_cluster(cluster_name, knoe_data)
|
||
elif env_key in ("service", "knoe-service-cluster", "k3s"):
|
||
self.log("==> Cluster reset (k3s namespace cleanup)")
|
||
db_ns = self._get_input("env_setup.DATABASE_NAMESPACE", "")
|
||
if not db_ns:
|
||
db_ns = self._get_input("env_setup.NAMESPACE", "")
|
||
if not db_ns:
|
||
db_ns = "default"
|
||
db_ns = db_ns.strip() or "default"
|
||
|
||
service_ns = self._service_namespace().strip() or "knoe-system"
|
||
monitoring_ns = (
|
||
((self.knoe_cfg_data.get("Global", {}) or {}).get("MONITORING_NAMESPACE", "") or "").strip()
|
||
or (os.environ.get("MONITORING_NAMESPACE") or "").strip()
|
||
or "monitoring"
|
||
)
|
||
cnpg_ns = (
|
||
((self.knoe_cfg_data.get("Global", {}) or {}).get("CNPG_NAMESPACE", "") or "").strip()
|
||
or (os.environ.get("CNPG_NAMESPACE") or "").strip()
|
||
or "cnpg-system"
|
||
)
|
||
argocd_ns = self._argocd_namespace().strip() or "argocd"
|
||
registry_ns = service_ns
|
||
|
||
reset_namespaces: list[str] = []
|
||
for candidate in [db_ns, service_ns, monitoring_ns, cnpg_ns, argocd_ns, "default"]:
|
||
ns = (candidate or "").strip()
|
||
if not ns or (ns.startswith("${") and ns.endswith("}")):
|
||
continue
|
||
if ns not in reset_namespaces:
|
||
reset_namespaces.append(ns)
|
||
|
||
server = self._get_input("init_cluster.k3s_server_url", "")
|
||
token = self._get_input("init_cluster.k3s_token", "")
|
||
|
||
# For remote k3s clusters the config token is typically a *join token*
|
||
# (e.g. `K10...::server:...`) which cannot be used to authenticate kubectl.
|
||
# Ensure we have a cert-based kubeconfig (or a valid bearer token config)
|
||
# before invoking any kubectl-based reset scripts.
|
||
self._ensure_project_k3s_kubeconfig(server, token)
|
||
if self.delete_db_requested:
|
||
self.log(f"[RESET] Namespaces to clean: {', '.join(reset_namespaces)}")
|
||
self.log(
|
||
f"[RESET] --delete-db enabled; database PVCs in namespace '{db_ns}' will be deleted."
|
||
)
|
||
else:
|
||
self.log(
|
||
f"[RESET] Namespaces to clean (PVCs preserved): {', '.join(reset_namespaces)}"
|
||
)
|
||
self.log(
|
||
f"[RESET] Preserving registry workload in namespace: {registry_ns}"
|
||
)
|
||
_reset_k3s_namespace(
|
||
self.project_root,
|
||
reset_namespaces,
|
||
server,
|
||
token,
|
||
registry_namespace=registry_ns,
|
||
clear_node_reservations=True,
|
||
delete_pvcs_namespaces=[db_ns] if self.delete_db_requested else None,
|
||
force_delete_bound_pvs=self.delete_db_requested,
|
||
)
|
||
self._cleanup_local_k3s_artifacts()
|
||
|
||
self.reset_requested = False
|
||
self.delete_db_requested = False
|
||
|
||
def _cleanup_local_k3s_artifacts(self) -> None:
|
||
paths = [
|
||
self.project_root / "etc" / "secrets" / "k3s.kubeconfig",
|
||
resolve_knoe_home(env={}) / "secrets" / "k3s.kubeconfig",
|
||
]
|
||
for path in paths:
|
||
try:
|
||
if path.exists():
|
||
path.unlink()
|
||
self.log(f"[RESET] Removed local kubeconfig {path}")
|
||
except Exception as exc:
|
||
self.err(f"[WARN] Failed to remove {path}: {exc}")
|
||
|
||
def _ensure_project_k3s_kubeconfig(self, server: str, token: str) -> None:
|
||
server = (server or "").strip()
|
||
token = (token or "").strip()
|
||
target = self.project_root / "knoe-k3s.kubeconfig"
|
||
backup = target.with_name(target.name + ".bak")
|
||
|
||
def _looks_cert_based(path: Path) -> bool:
|
||
try:
|
||
txt = path.read_text(encoding="utf-8")
|
||
except Exception:
|
||
return False
|
||
return "client-certificate-data" in txt
|
||
|
||
def _looks_like_join_token_cfg(path: Path) -> bool:
|
||
try:
|
||
txt = path.read_text(encoding="utf-8")
|
||
except Exception:
|
||
return False
|
||
# k3s join tokens include `::server:`; they are not Kubernetes bearer tokens.
|
||
return "token:" in txt and "::server:" in txt and "client-certificate-data" not in txt
|
||
|
||
# If we already have a kubeconfig, keep it unless it is clearly an invalid join-token config.
|
||
if target.exists() and not _looks_like_join_token_cfg(target):
|
||
return
|
||
|
||
# Prefer restoring a known-good cert-based kubeconfig.
|
||
if (not target.exists() or _looks_like_join_token_cfg(target)) and backup.exists() and _looks_cert_based(backup):
|
||
try:
|
||
target.write_text(backup.read_text(encoding="utf-8"), encoding="utf-8")
|
||
os.chmod(target, 0o600)
|
||
self.log(f"[OK] Restored kubeconfig from backup: {target}")
|
||
return
|
||
except Exception as exc:
|
||
self.err(f"[WARN] Failed to restore kubeconfig from {backup}: {exc}")
|
||
|
||
# Next best: fetch a fresh cert-based kubeconfig via Ansible.
|
||
fetched = self._fetch_k3s_kubeconfig()
|
||
if fetched and target.exists() and not _looks_like_join_token_cfg(target):
|
||
return
|
||
|
||
# Final fallback: only generate a token-based kubeconfig if the token actually looks like a
|
||
# Kubernetes bearer token. (k3s join tokens are not usable for kubectl auth.)
|
||
if server and token and _looks_like_k8s_bearer_token(token):
|
||
try:
|
||
generated = _write_k3s_kubeconfig(server, token)
|
||
if generated.exists() and generated != target:
|
||
target.write_text(generated.read_text(encoding="utf-8"), encoding="utf-8")
|
||
os.chmod(target, 0o600)
|
||
self.log(f"[OK] Wrote token-based kubeconfig: {target}")
|
||
except Exception as exc:
|
||
self.err(f"[WARN] Failed to generate token-based kubeconfig: {exc}")
|
||
else:
|
||
self.err(
|
||
"[WARN] No usable kubeconfig available for remote k3s; please fetch/create a cert-based knoe-k3s.kubeconfig."
|
||
)
|
||
|
||
def _close_log_file(self):
|
||
"""Restore original stdout/stderr and close the log file."""
|
||
if self._log_file:
|
||
sys.stdout = sys.__stdout__
|
||
sys.stderr = sys.__stderr__
|
||
try:
|
||
self._log_file.close()
|
||
except Exception:
|
||
pass
|
||
self._log_file = None
|
||
|
||
def _prompt_for_master_password(self) -> str:
|
||
"""Prompt user for master password in silent mode with a 10-minute timeout."""
|
||
import sys
|
||
import time
|
||
import select
|
||
import termios
|
||
import tty
|
||
|
||
# Use sys.__stdout__ and sys.__stdin__ to ensure we bypass any redirection
|
||
out = sys.__stdout__
|
||
inp = sys.__stdin__
|
||
|
||
# ANSI color codes
|
||
GREEN = "\033[32m"
|
||
RED = "\033[31m"
|
||
RESET = "\033[0m"
|
||
BOLD = "\033[1m"
|
||
CHECK = "\u2713"
|
||
|
||
out.write("\n" + "=" * 60 + "\n")
|
||
out.write(f"{BOLD}DATABASE MASTER PASSWORD REQUIRED{RESET}\n")
|
||
out.write("A master password is required to initialize the database cluster.\n")
|
||
out.write("You are running in silent mode (-S) and no valid password was found.\n")
|
||
out.write(f"Please provide it now. Action timer: {BOLD}10 minutes (600s){RESET}.\n")
|
||
out.write("=" * 60 + "\n\n")
|
||
out.flush()
|
||
|
||
timeout = 600
|
||
start_time = time.time()
|
||
|
||
def get_input_with_timeout(prompt, mask=True):
|
||
out.write(prompt)
|
||
out.flush()
|
||
|
||
if not inp.isatty():
|
||
# Fallback for non-TTY environments if necessary, but requirement says "prompt"
|
||
# In non-TTY, select might still work on stdin.
|
||
pass
|
||
|
||
fd = inp.fileno()
|
||
old_settings = termios.tcgetattr(fd)
|
||
try:
|
||
tty.setcbreak(fd)
|
||
res = ""
|
||
while True:
|
||
elapsed = time.time() - start_time
|
||
if elapsed >= timeout:
|
||
out.write(
|
||
f"\n{RED}[ERROR] Timeout waiting for password input (600s). Cannot proceed.{RESET}\n"
|
||
)
|
||
out.flush()
|
||
sys.exit(1)
|
||
|
||
rlist, _, _ = select.select([inp], [], [], 1.0)
|
||
if rlist:
|
||
char = inp.read(1)
|
||
if char in ("\n", "\r"):
|
||
out.write("\n")
|
||
out.flush()
|
||
return res
|
||
elif char == "\x7f": # Backspace
|
||
if res:
|
||
res = res[:-1]
|
||
if mask:
|
||
out.write("\b \b")
|
||
out.flush()
|
||
elif char == "\x03": # Ctrl+C
|
||
out.write(f"\n{RED}Installation aborted by user.{RESET}\n")
|
||
out.flush()
|
||
sys.exit(1)
|
||
else:
|
||
res += char
|
||
if mask:
|
||
out.write("*")
|
||
out.flush()
|
||
finally:
|
||
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
|
||
|
||
while True:
|
||
p1 = get_input_with_timeout("Enter Master Password: ")
|
||
p2 = get_input_with_timeout("Confirm Master Password: ")
|
||
|
||
if p1 and p1 == p2:
|
||
out.write(f"{GREEN}{CHECK} Password accepted.{RESET}\n\n")
|
||
out.flush()
|
||
return p1
|
||
elif not p1:
|
||
out.write(f"{RED}Password cannot be empty.{RESET}\n")
|
||
out.flush()
|
||
else:
|
||
out.write(
|
||
f"{RED}Passwords do not match. Please try again.{RESET}\n"
|
||
)
|
||
out.flush()
|
||
|
||
def _load_db_password_from_1password(self, *, log_found: bool = True) -> str:
|
||
try:
|
||
from knoe.core.onepassword import get_secret, op_available
|
||
if not op_available():
|
||
return ""
|
||
password = get_secret("administrator", "password").strip()
|
||
if password and log_found:
|
||
self.log("[CONFIG] Loaded DB master password from 1Password knoey vault.")
|
||
return password
|
||
except Exception:
|
||
return ""
|
||
|
||
def _persist_db_password_to_1password(self, password: str) -> None:
|
||
if not password:
|
||
raise RuntimeError("Cannot persist an empty database master password.")
|
||
try:
|
||
from knoe.core.onepassword import set_secret, op_available
|
||
if not op_available():
|
||
self.log("[WARN] 1Password CLI not available; skipping password persistence.")
|
||
return
|
||
set_secret("administrator", "password", password)
|
||
self.log("[CONFIG] Persisted DB master password to 1Password knoey vault.")
|
||
except Exception as e:
|
||
raise RuntimeError(f"Failed to persist database master password to 1Password: {e}") from e
|
||
|
||
def run(self) -> int:
|
||
_configure_unbuffered_io()
|
||
self.log(f"[CONFIG] Using {self.cfg_path}")
|
||
try:
|
||
# Merge caller-provided inputs with defaults and config-file values.
|
||
# Caller inputs (self.inputs) take precedence to support programmatic overrides
|
||
# in automated pipelines (like knoe.deploy_pipeline).
|
||
self.inputs = {
|
||
**self._default_inputs(),
|
||
**self._load_inputs_from_cfg(),
|
||
**self.inputs
|
||
}
|
||
|
||
# Silent mode: ensure we have a master password source before proceeding.
|
||
# If the value is an OpenBao reference or an encrypted `${KNOE_SECRET:...}`
|
||
# placeholder, allow downstream secret-management steps to resolve it.
|
||
# Only prompt/generate when the value is truly missing.
|
||
db_pw = (self._get_input("init_password.db_password", "") or "").strip()
|
||
|
||
env_db_pw = (
|
||
os.environ.get("KNOE_DB_PASSWORD")
|
||
or os.environ.get("DB_PASSWORD")
|
||
or ""
|
||
).strip()
|
||
|
||
# If runtime env provides a password, reconcile with 1Password for
|
||
# consistency and bootstrap when missing.
|
||
if db_pw and env_db_pw and db_pw == env_db_pw:
|
||
vault_pw = self._load_db_password_from_1password(log_found=False)
|
||
if vault_pw and vault_pw != db_pw:
|
||
self.log(
|
||
"[WARN] Runtime DB password differs from 1Password value; "
|
||
"using 1Password value for consistency."
|
||
)
|
||
self.inputs["init_password.db_password"] = vault_pw
|
||
self.inputs["init_password.db_password_confirm"] = vault_pw
|
||
db_pw = vault_pw
|
||
elif not vault_pw:
|
||
self._persist_db_password_to_1password(db_pw)
|
||
self.log(
|
||
"[CONFIG] Bootstrapped 1Password administrator from KNOE_DB_PASSWORD/DB_PASSWORD."
|
||
)
|
||
|
||
if not db_pw:
|
||
vault_pw = self._load_db_password_from_1password()
|
||
if vault_pw:
|
||
self.inputs["init_password.db_password"] = vault_pw
|
||
self.inputs["init_password.db_password_confirm"] = vault_pw
|
||
db_pw = vault_pw
|
||
needs_prompt = not bool(db_pw)
|
||
|
||
if needs_prompt:
|
||
# In silent installs, never block on an interactive prompt in environments
|
||
# that cannot provide input (CI/pytest/non-interactive runners).
|
||
stdin_is_tty = getattr(sys.__stdin__, "isatty", lambda: False)()
|
||
stdout_is_tty = getattr(sys.__stdout__, "isatty", lambda: False)()
|
||
running_under_pytest = "PYTEST_CURRENT_TEST" in os.environ
|
||
running_in_ci = bool(os.environ.get("CI"))
|
||
allow_prompt = (
|
||
stdin_is_tty
|
||
and stdout_is_tty
|
||
and not running_under_pytest
|
||
and not running_in_ci
|
||
)
|
||
if allow_prompt:
|
||
new_pw = self._prompt_for_master_password()
|
||
self._persist_db_password_to_1password(new_pw)
|
||
self.log(
|
||
"[CONFIG] Master password captured interactively and saved to 1Password."
|
||
)
|
||
else:
|
||
raise RuntimeError(
|
||
"Database master password is missing and prompting is unavailable. "
|
||
"Run once in an interactive terminal to bootstrap the 1Password knoey vault, "
|
||
"or provide KNOE_DB_PASSWORD/DB_PASSWORD."
|
||
)
|
||
self.inputs["init_password.db_password"] = new_pw
|
||
self.inputs["init_password.db_password_confirm"] = new_pw
|
||
# Save immediately to knoe.cfg so subsequent steps/scripts see it.
|
||
self._write_cfg()
|
||
# Synchronize controller state
|
||
self.controller.state.inputs = self.inputs
|
||
self.controller.state.config_data = self.knoe_cfg_data
|
||
# Delegate config writing to the installer (allows milestones to persist changes)
|
||
self.controller.write_config = self._write_cfg
|
||
except Exception as e:
|
||
self.err(f"[FATAL] {e}")
|
||
self._close_log_file()
|
||
return 2
|
||
|
||
self._write_cfg()
|
||
|
||
self._perform_cluster_reset()
|
||
|
||
milestones = [
|
||
DependenciesMilestone(),
|
||
NetworkScanMilestone(),
|
||
EnvSetupMilestone(),
|
||
SecretManagementMilestone(),
|
||
DockerBuildMilestone(),
|
||
DatabaseCreationMilestone(),
|
||
ClusterLifecycleMilestone(),
|
||
InitializationScriptsMilestone(),
|
||
KerberosMilestone(),
|
||
GitOpsMilestone(),
|
||
SupabaseImagePreloadMilestone(),
|
||
SupabaseMilestone(),
|
||
DeploymentMilestone(),
|
||
SecurityMilestone(),
|
||
]
|
||
|
||
try:
|
||
self.controller.run_milestones(
|
||
milestones,
|
||
progress_callback=lambda msg, p: self.log(f"[{p*100:0.0f}%] {msg}"),
|
||
)
|
||
|
||
# Sync back results
|
||
self.knoe_cfg_data = self.controller.state.config_data
|
||
self.knoe_cfg_data["Install"]["STATUS"] = "Finished"
|
||
self._write_cfg()
|
||
|
||
self.log("[DONE] Silent install completed.")
|
||
self._close_log_file()
|
||
return 0
|
||
except Exception as e:
|
||
self.err(f"[FATAL] {e}")
|
||
try:
|
||
self.knoe_cfg_data["Install"]["STATUS"] = "Failed"
|
||
self._write_cfg()
|
||
except Exception:
|
||
pass
|
||
self._close_log_file()
|
||
return 2
|
||
|
||
# ------------------------------------------------------------------
|
||
# Repair / update paths
|
||
# ------------------------------------------------------------------
|
||
|
||
def _validate_cluster_state(self, env: dict) -> dict[str, list[str]]:
|
||
"""Compare configured intent with live cluster resources.
|
||
|
||
Returns a dict of anomaly category → list of description strings.
|
||
Only kubectl is used so the check is non-destructive.
|
||
"""
|
||
anomalies: dict[str, list[str]] = {}
|
||
try:
|
||
supabase_ns = (os.environ.get("SUPABASE_NAMESPACE") or "").strip() or "supabase"
|
||
kube_env = {k: v for k, v in env.items()}
|
||
|
||
def _kubectl(*args: str) -> subprocess.CompletedProcess:
|
||
return subprocess.run(
|
||
["kubectl", *args],
|
||
capture_output=True, text=True, env=kube_env, timeout=30,
|
||
)
|
||
|
||
# --- Supabase ---
|
||
if self._get_input_bool("init_cluster.supabase_enabled", False):
|
||
studio_enabled = self._get_input_bool("init_cluster.supabase_studio_enabled", False)
|
||
r = _kubectl("get", "deployment", "supabase-studio", "-n", supabase_ns, "--ignore-not-found")
|
||
studio_exists = bool(r.stdout.strip())
|
||
if studio_enabled and not studio_exists:
|
||
anomalies.setdefault("supabase_studio", []).append("deployment missing")
|
||
if studio_enabled and studio_exists:
|
||
r2 = _kubectl(
|
||
"get", "deployment", "supabase-studio", "-n", supabase_ns,
|
||
"-o", "jsonpath={.status.readyReplicas}",
|
||
)
|
||
ready = (r2.stdout.strip() or "0")
|
||
if ready == "0":
|
||
anomalies.setdefault("supabase_studio", []).append("deployment not ready")
|
||
# Per-component checks
|
||
for component, flag in (
|
||
("supabase-auth", "supabase_auth_enabled"),
|
||
("supabase-realtime", "supabase_realtime_enabled"),
|
||
("supabase-meta", "supabase_meta_enabled"),
|
||
("supabase-analytics", "supabase_analytics_enabled"),
|
||
):
|
||
if self._get_input_bool(f"init_cluster.{flag}", True):
|
||
rc = _kubectl("get", "deployment", component, "-n", supabase_ns, "--ignore-not-found")
|
||
if not rc.stdout.strip():
|
||
anomalies.setdefault("supabase_components", []).append(f"{component} deployment missing")
|
||
|
||
# --- Core dashboard (Kong) ---
|
||
if self._dashboard_kong_missing(env):
|
||
anomalies.setdefault("dashboard", []).append("kong missing")
|
||
|
||
# --- Authority context ---
|
||
if self._authority_context_missing():
|
||
anomalies.setdefault("authority", []).append("context missing")
|
||
|
||
except Exception as exc:
|
||
self.err(f"[WARN] Cluster state validation error (non-fatal): {exc}")
|
||
return anomalies
|
||
|
||
def _reconcile_supabase_studio(self, env: dict) -> None:
|
||
"""Ensure Studio deployment/service/ingress/TLS exist and match config.
|
||
|
||
Uses ``supabase_studio_url`` as the source of truth for ingress host.
|
||
Operates only on Studio resources — does not restart other Supabase components.
|
||
"""
|
||
default_supabase_studio_url = (
|
||
"db.0.knoe.dev" if self._deployment_mode() == "k8s" else "db.knoe.org"
|
||
)
|
||
studio_url = (
|
||
self._get_input("init_cluster.supabase_studio_url", default_supabase_studio_url).strip()
|
||
or default_supabase_studio_url
|
||
)
|
||
supabase_ns = (os.environ.get("SUPABASE_NAMESPACE") or "").strip() or "supabase"
|
||
self.log(f"[REPAIR] Reconciling Supabase Studio → ingress host: {studio_url}")
|
||
project_root_str = str(self.controller.project_root)
|
||
cfg_path_str = str(self.cfg_path) if (self.cfg_path and self.cfg_path.exists()) else ""
|
||
db_ns = (
|
||
self._get_input("init_password.db_namespace", "").strip()
|
||
or (env.get("KNOE_DB_NAMESPACE") or "").strip()
|
||
or "knoe-db"
|
||
)
|
||
deploy_env = {
|
||
**env,
|
||
"SUPABASE_STUDIO_URL": studio_url,
|
||
"SUPABASE_STUDIO_ENABLED": "true",
|
||
"PROJECT_ROOT": project_root_str,
|
||
"KNOE_HOME": project_root_str,
|
||
"KNOE_DB_NAMESPACE": db_ns,
|
||
"DATABASE_NAMESPACE": db_ns,
|
||
**({"PROLE_CFG_PATH": cfg_path_str} if cfg_path_str else {}),
|
||
}
|
||
reconcile_script = self.controller.project_root / "etc" / "supabase_reconcile_studio.sh"
|
||
if reconcile_script.exists():
|
||
self.controller.run_script(
|
||
"supabase_reconcile_studio.sh",
|
||
args=["--namespace", supabase_ns, "--studio-url", studio_url],
|
||
env=deploy_env,
|
||
)
|
||
else:
|
||
# Dedicated reconcile script not present — fall back to the full
|
||
# deploy.sh pipeline (render → helm upgrade --install) with Studio
|
||
# env vars forced, mirroring _step_supabase.
|
||
self.log("[REPAIR] supabase_reconcile_studio.sh not found; falling back to deploy.sh")
|
||
deploy_sh = self.controller.project_root / "supabase" / "deploy.sh"
|
||
if not deploy_sh.exists():
|
||
self.err(f"[ERROR] supabase/deploy.sh not found: {deploy_sh}")
|
||
return
|
||
cluster_env = _normalize_cluster_env(
|
||
self._get_input("init_cluster.cluster_env", "dev")
|
||
)
|
||
if cluster_env in ("service", "prod"):
|
||
mode = "k8s"
|
||
else:
|
||
mode = "k3d"
|
||
deploy_args = ["--mode", mode]
|
||
if self.cfg_path and self.cfg_path.exists():
|
||
deploy_args.extend(["-c", str(self.cfg_path)])
|
||
self.log(f"[REPAIR] supabase/deploy.sh {' '.join(deploy_args)}")
|
||
rc = self._run_cmd(["bash", str(deploy_sh)] + deploy_args, env=deploy_env)
|
||
if rc != 0:
|
||
self.err(f"[ERROR] Supabase deploy.sh failed (code {rc})")
|
||
|
||
def _reconcile_supabase(self, env: dict) -> None:
|
||
"""Reconcile all enabled Supabase components toward configured intent.
|
||
|
||
Honours per-component enable flags; avoids namespace-wide restarts.
|
||
"""
|
||
supabase_enabled = self._get_input_bool("init_cluster.supabase_enabled", False)
|
||
if not supabase_enabled:
|
||
self.log("[REPAIR] Supabase disabled in config — skipping Supabase reconcile.")
|
||
return
|
||
|
||
studio_enabled = self._get_input_bool("init_cluster.supabase_studio_enabled", False)
|
||
if studio_enabled:
|
||
self._reconcile_supabase_studio(env)
|
||
else:
|
||
self.log("[REPAIR] supabase_studio_enabled=false — skipping Studio reconcile.")
|
||
|
||
component_flags = {
|
||
"supabase_auth_enabled": "auth",
|
||
"supabase_realtime_enabled": "realtime",
|
||
"supabase_meta_enabled": "meta",
|
||
"supabase_analytics_enabled": "analytics",
|
||
}
|
||
for flag, component in component_flags.items():
|
||
if self._get_input_bool(f"init_cluster.{flag}", True):
|
||
self.log(f"[REPAIR] Verified Supabase component enabled in config: {component}")
|
||
else:
|
||
self.log(f"[REPAIR] {component} disabled in config — skipping.")
|
||
|
||
def run_repair(self) -> int:
|
||
"""Validate current configuration against live cluster state and fix drift.
|
||
|
||
Prefers patch/apply over delete/recreate. Does not restart healthy
|
||
unrelated components. Safe to run repeatedly (idempotent).
|
||
"""
|
||
self.log("[REPAIR] Starting repair run…")
|
||
try:
|
||
self.inputs = {
|
||
**self._default_inputs(),
|
||
**self._load_inputs_from_cfg(),
|
||
**self.inputs
|
||
}
|
||
self.controller.state.inputs = self.inputs
|
||
self.controller.state.config_data = self.knoe_cfg_data
|
||
# Delegate config writing to the installer (allows milestones to persist changes)
|
||
self.controller.write_config = self._write_cfg
|
||
except Exception as exc:
|
||
self.err(f"[FATAL] Could not load config: {exc}")
|
||
self._close_log_file()
|
||
return 2
|
||
|
||
service_ns = self._service_namespace()
|
||
env = self._script_env_for_namespace(service_ns)
|
||
env["SERVICE_NAMESPACE"] = service_ns
|
||
|
||
self.log("[REPAIR] Validating cluster state…")
|
||
anomalies = self._validate_cluster_state(env)
|
||
if not anomalies:
|
||
self.log("[REPAIR] No drift detected — cluster matches configured intent.")
|
||
self._close_log_file()
|
||
return 0
|
||
|
||
for category, details in anomalies.items():
|
||
for detail in details:
|
||
self.log(f"[REPAIR] Anomaly [{category}]: {detail}")
|
||
|
||
try:
|
||
self._repair_stale_released_pvs(env)
|
||
except Exception as exc:
|
||
self.err(f"[WARN] PV repair error (non-fatal): {exc}")
|
||
|
||
if "supabase_studio" in anomalies or "supabase_components" in anomalies:
|
||
self._reconcile_supabase(env)
|
||
|
||
core_anomalies = [k for k in anomalies if k not in ("supabase_studio", "supabase_components")]
|
||
if core_anomalies:
|
||
try:
|
||
self._run_cluster_repair_pipeline(env=env, anomalies=core_anomalies)
|
||
except Exception as exc:
|
||
self.err(f"[WARN] Cluster repair pipeline error (non-fatal): {exc}")
|
||
|
||
self.log("[REPAIR] Repair run complete.")
|
||
self._close_log_file()
|
||
return 0
|
||
|
||
def run_update(self) -> int:
|
||
"""Run repair-style reconciliation then apply any newly changed config intent.
|
||
|
||
Only rolls components whose effective config changed since last write.
|
||
Does not perform a full reinstall.
|
||
"""
|
||
self.log("[UPDATE] Starting update run (repair + config delta apply)…")
|
||
|
||
rc = self.run_repair()
|
||
if rc != 0:
|
||
self.err("[UPDATE] Repair phase failed; aborting update.")
|
||
return rc
|
||
|
||
self.log("[UPDATE] Applying config changes…")
|
||
try:
|
||
self.inputs = {
|
||
**self._default_inputs(),
|
||
**self._load_inputs_from_cfg(),
|
||
**self.inputs
|
||
}
|
||
self.controller.state.inputs = self.inputs
|
||
self.controller.state.config_data = self.knoe_cfg_data
|
||
# Delegate config writing to the installer (allows milestones to persist changes)
|
||
self.controller.write_config = self._write_cfg
|
||
except Exception as exc:
|
||
self.err(f"[FATAL] Could not reload config for update: {exc}")
|
||
self._close_log_file()
|
||
return 2
|
||
|
||
service_ns = self._service_namespace()
|
||
env = self._script_env_for_namespace(service_ns)
|
||
|
||
if self._get_input_bool("init_db_build.run_build", DEFAULT_ACTION_FLAGS.get("init_db_build.run_build", True)):
|
||
self.log("[UPDATE] Running DB image build…")
|
||
self._step_db_build()
|
||
|
||
if self._get_input_bool("init_cnpg_deploy.run_deploy", DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.run_deploy", True)):
|
||
configured_force_rollout = self._get_input_bool(
|
||
"init_cnpg_deploy.force_rollout",
|
||
DEFAULT_ACTION_FLAGS.get("init_cnpg_deploy.force_rollout", False),
|
||
)
|
||
self.log(
|
||
"[UPDATE] Running CNPG deploy "
|
||
f"(configured_force_rollout={configured_force_rollout}, effective_force_rollout=True)…"
|
||
)
|
||
original_force_rollout = self.inputs.get("init_cnpg_deploy.force_rollout")
|
||
self.inputs["init_cnpg_deploy.force_rollout"] = "true"
|
||
try:
|
||
self._step_cnpg_deploy()
|
||
finally:
|
||
if original_force_rollout is None:
|
||
self.inputs.pop("init_cnpg_deploy.force_rollout", None)
|
||
else:
|
||
self.inputs["init_cnpg_deploy.force_rollout"] = original_force_rollout
|
||
|
||
if self._get_input_bool("init_cluster.supabase_enabled", False):
|
||
self._reconcile_supabase(env)
|
||
|
||
self._write_cfg()
|
||
self.log("[UPDATE] Update run complete.")
|
||
self._close_log_file()
|
||
return 0
|
||
|
||
|
||
def _run_silent_install_test(project_root: Path, cfg_path: Path) -> int:
|
||
script = project_root / "tests" / "silent_install_test.sh"
|
||
if not script.exists():
|
||
print(
|
||
f"[ERROR] Silent install test script not found: {script}", file=sys.stderr
|
||
)
|
||
return 1
|
||
env = os.environ.copy()
|
||
env.setdefault("SILENT_INSTALL_LOG", "true")
|
||
res = subprocess.run(["bash", str(script), str(cfg_path)], env=env)
|
||
return res.returncode
|
||
|
||
|
||
def _attempt_k3s_repair(
|
||
controller: KnoeController,
|
||
namespace: str,
|
||
server: str,
|
||
token: str,
|
||
db_password: str,
|
||
) -> None:
|
||
env = os.environ.copy()
|
||
env["KNOE_HOME"] = str(controller.project_root)
|
||
env["KNOE_SERVICE"] = str(controller.project_root)
|
||
env["DATABASE_NAMESPACE"] = namespace
|
||
env["CLUSTER_NAME"] = (os.environ.get("CLUSTER_NAME") or "knoe-db").strip() or "knoe-db"
|
||
# Transitional compatibility for scripts still reading NAMESPACE.
|
||
env["NAMESPACE"] = namespace
|
||
env["KNOE_MODE"] = "k3s"
|
||
if db_password:
|
||
env["DB_PASSWORD"] = db_password
|
||
env["OPENTOFU_ADMIN_PASSWORD"] = db_password
|
||
if server and token and not env.get("KUBECONFIG"):
|
||
repo_kubeconfig = controller.project_root / "knoe-k3s.kubeconfig"
|
||
if repo_kubeconfig.exists():
|
||
env["KUBECONFIG"] = str(repo_kubeconfig)
|
||
elif _looks_like_k8s_bearer_token(token):
|
||
kubeconfig_path = _write_k3s_kubeconfig(server, token)
|
||
env["KUBECONFIG"] = str(kubeconfig_path)
|
||
openbao_ops.update(
|
||
namespace=namespace,
|
||
env=env,
|
||
mode="k3s",
|
||
project_root=controller.project_root,
|
||
)
|
||
opentofu_ops.update(
|
||
namespace=namespace,
|
||
env=env,
|
||
mode="k3s",
|
||
project_root=controller.project_root,
|
||
)
|
||
|
||
|
||
def _reset_k3s_namespace(
|
||
project_root: Path,
|
||
namespace: str | Sequence[str],
|
||
server: str,
|
||
token: str,
|
||
*,
|
||
registry_namespace: str | None = None,
|
||
clear_node_reservations: bool = False,
|
||
delete_pvcs_namespaces: Sequence[str] | None = None,
|
||
force_delete_bound_pvs: bool = False,
|
||
) -> None:
|
||
script = project_root / "scripts" / "reset-ns.sh"
|
||
if not script.exists():
|
||
print(f"[WARN] Namespace reset script not found: {script}", file=sys.stderr)
|
||
return
|
||
|
||
if isinstance(namespace, str):
|
||
raw_namespaces = [namespace]
|
||
else:
|
||
raw_namespaces = list(namespace or [])
|
||
|
||
namespaces: list[str] = []
|
||
for ns in raw_namespaces:
|
||
item = str(ns or "").strip()
|
||
if not item:
|
||
continue
|
||
if item not in namespaces:
|
||
namespaces.append(item)
|
||
|
||
if not namespaces:
|
||
return
|
||
|
||
env = os.environ.copy()
|
||
if not env.get("KUBECONFIG"):
|
||
repo_kubeconfig = project_root / "knoe-k3s.kubeconfig"
|
||
if repo_kubeconfig.exists():
|
||
env["KUBECONFIG"] = str(repo_kubeconfig)
|
||
elif server and token and _looks_like_k8s_bearer_token(token):
|
||
kubeconfig_path = _write_k3s_kubeconfig(server, token)
|
||
env["KUBECONFIG"] = str(kubeconfig_path)
|
||
|
||
reg_ns = (registry_namespace or "").strip()
|
||
delete_pvcs_set = {
|
||
str(ns or "").strip()
|
||
for ns in (delete_pvcs_namespaces or [])
|
||
if str(ns or "").strip()
|
||
}
|
||
|
||
def _collect_bound_pvs(target_ns: str) -> list[str]:
|
||
res = subprocess.run(
|
||
[
|
||
"kubectl",
|
||
"-n",
|
||
target_ns,
|
||
"get",
|
||
"pvc",
|
||
"-o",
|
||
"jsonpath={range .items[*]}{.spec.volumeName}{\"\\n\"}{end}",
|
||
],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
if res.returncode != 0:
|
||
print(
|
||
f"[WARN] Could not list PVC-bound PVs in {target_ns} (code {res.returncode}).",
|
||
file=sys.stderr,
|
||
)
|
||
return []
|
||
pvs: list[str] = []
|
||
for line in (res.stdout or "").splitlines():
|
||
pv = (line or "").strip()
|
||
if pv and pv not in pvs:
|
||
pvs.append(pv)
|
||
return pvs
|
||
|
||
def _set_reclaim_policy_delete(pvs: Sequence[str]) -> None:
|
||
for pv in pvs:
|
||
patch = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
|
||
res = subprocess.run(
|
||
["kubectl", "patch", "pv", pv, "--type=merge", "-p", patch],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
if res.returncode != 0:
|
||
print(
|
||
f"[WARN] Failed to set reclaimPolicy=Delete for pv/{pv} (code {res.returncode}).",
|
||
file=sys.stderr,
|
||
)
|
||
|
||
def _delete_pvs(pvs: Sequence[str]) -> None:
|
||
for pv in pvs:
|
||
res = subprocess.run(
|
||
["kubectl", "delete", "pv", pv, "--ignore-not-found", "--wait=false"],
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
)
|
||
if res.returncode != 0:
|
||
print(
|
||
f"[WARN] Failed to delete pv/{pv} (code {res.returncode}).",
|
||
file=sys.stderr,
|
||
)
|
||
|
||
for idx, ns in enumerate(namespaces):
|
||
delete_pvcs = ns in delete_pvcs_set
|
||
bound_pvs: list[str] = []
|
||
if delete_pvcs and force_delete_bound_pvs:
|
||
bound_pvs = _collect_bound_pvs(ns)
|
||
if bound_pvs:
|
||
_set_reclaim_policy_delete(bound_pvs)
|
||
|
||
cmd = ["bash", str(script), "-n", ns]
|
||
if reg_ns and ns == reg_ns:
|
||
cmd.append("--keep-registry")
|
||
if delete_pvcs:
|
||
cmd.append("--delete-pvcs")
|
||
if clear_node_reservations and idx == (len(namespaces) - 1):
|
||
cmd.append("--clear-node-reservations")
|
||
res = subprocess.run(cmd, env=env)
|
||
if res.returncode != 0:
|
||
print(
|
||
f"[WARN] Namespace reset failed for {ns} (code {res.returncode})",
|
||
file=sys.stderr,
|
||
)
|
||
continue
|
||
if delete_pvcs and force_delete_bound_pvs and bound_pvs:
|
||
_delete_pvs(bound_pvs)
|
||
|
||
|
||
def _prepare_k3s_pipeline(
|
||
controller: KnoeController, log_fn=None, skip_validation: bool = False
|
||
) -> int:
|
||
"""Prepare OpenTofu k3s pipeline: stage manifests, tfvars, ArgoCD apps.
|
||
|
||
When *skip_validation* is True the silent-install-test / repair loop is
|
||
skipped entirely — the caller (e.g. a successful silent install) already
|
||
validated the environment. The main knoe.cfg is **not** overwritten in
|
||
this case so it keeps whatever cluster_env the install used.
|
||
"""
|
||
|
||
def _log(msg):
|
||
if log_fn:
|
||
try:
|
||
log_fn(msg)
|
||
except:
|
||
print(msg, end="", flush=True)
|
||
else:
|
||
print(msg, end="", flush=True)
|
||
|
||
project_root = controller.project_root
|
||
conf_dir = knoe_conf_mgr.resolve_knoe_conf_dir(project_root)
|
||
cfg_path = knoe_conf_mgr.entrypoint_path(conf_dir)
|
||
|
||
_log("==> Preparing k3s pipeline\n")
|
||
_log(f" Project root: {project_root}\n")
|
||
_log(f" Config: {cfg_path}\n\n")
|
||
|
||
_log("==> Detecting Ansible topology...\n")
|
||
info = _detect_ansible_topology(project_root)
|
||
k3s_server = (info.get("k3s_server_url") or "").strip() if info else ""
|
||
k3s_token = (info.get("k3s_token") or "").strip() if info else ""
|
||
if k3s_server and not k3s_server.startswith("http"):
|
||
k3s_server = f"https://{k3s_server}"
|
||
_log(f" k3s_server: {k3s_server or '(not detected)'}\n")
|
||
_log(f" k3s_token: {'detected' if k3s_token else '(not detected)'}\n\n")
|
||
|
||
# Load config to resolve namespace and secrets for pipeline tfvars
|
||
_log("==> Loading knoe config...\n")
|
||
knoe = KnoeConsoleInstaller(controller, str(cfg_path))
|
||
try:
|
||
existing_inputs = knoe._load_inputs_from_cfg()
|
||
except Exception:
|
||
existing_inputs = {}
|
||
knoe.inputs = {**knoe._default_inputs(), **existing_inputs}
|
||
|
||
if k3s_server and not knoe.inputs.get("init_cluster.k3s_server_url"):
|
||
knoe.inputs["init_cluster.k3s_server_url"] = k3s_server
|
||
if k3s_token and not knoe.inputs.get("init_cluster.k3s_token"):
|
||
knoe.inputs["init_cluster.k3s_token"] = k3s_token
|
||
|
||
namespace = (knoe._get_input("init_password.db_namespace", "") or "").strip()
|
||
if not namespace:
|
||
namespace = (
|
||
knoe._get_input("env_setup.DATABASE_NAMESPACE", "") or ""
|
||
).strip()
|
||
if not namespace:
|
||
namespace = (
|
||
knoe._get_input("env_setup.NAMESPACE", "") or ""
|
||
).strip() or "default"
|
||
_log(f" Target namespace: {namespace}\n")
|
||
_log(" [OK] Config loaded\n\n")
|
||
|
||
if not skip_validation:
|
||
# Standalone mode: write a pipeline-specific cfg and validate
|
||
knoe.inputs["init_cluster.cluster_env"] = "knoe-service-cluster"
|
||
# supabase_enabled is intentionally NOT overridden here — preserve the cfg value
|
||
knoe.inputs["init_cluster.kerberos_enabled"] = _bool_str(False)
|
||
knoe.inputs["init_cluster.at_rest_encryption_enabled"] = _bool_str(True)
|
||
knoe.inputs["kerberos_config.enabled"] = _bool_str(False)
|
||
knoe.inputs["kerberos_config.test_connection"] = _bool_str(False)
|
||
|
||
knoe.inputs["env_setup.KNOE_HOME"] = str(project_root)
|
||
knoe.inputs["env_setup.KNOE_CONF"] = str(project_root / "conf")
|
||
knoe.inputs["env_setup.PROLE_DATA"] = knoe._resolve_env_value(
|
||
"PROLE_DATA", str(resolve_knoe_home(env={}) / "data")
|
||
) or str(resolve_knoe_home(env={}) / "data")
|
||
knoe.inputs["env_setup.PROLE_LOGS"] = knoe._resolve_env_value(
|
||
"PROLE_LOGS", str(resolve_knoe_home(env={}) / "logs")
|
||
) or str(resolve_knoe_home(env={}) / "logs")
|
||
knoe.inputs["env_setup.KNOE_SERVICE"] = str(project_root / "etc")
|
||
|
||
# Ensure required keys that the silent install test validates
|
||
if not knoe.inputs.get("env_setup.DATABASE_NAMESPACE"):
|
||
knoe.inputs["env_setup.DATABASE_NAMESPACE"] = (
|
||
knoe.inputs.get("init_password.db_namespace", "") or "default"
|
||
)
|
||
if not knoe.inputs.get("init_password.db_namespace"):
|
||
knoe.inputs["init_password.db_namespace"] = (
|
||
knoe.inputs.get("env_setup.DATABASE_NAMESPACE", "")
|
||
or knoe.inputs.get("env_setup.NAMESPACE", "")
|
||
or "default"
|
||
)
|
||
if not knoe.inputs.get("kerberos_config.init_authority"):
|
||
knoe.inputs["kerberos_config.init_authority"] = _bool_str(False)
|
||
if not knoe.inputs.get("init_password.db_host_port"):
|
||
knoe.inputs["init_password.db_host_port"] = "5432"
|
||
_log(" [OK] Config overrides applied for k3s pipeline\n")
|
||
|
||
_log("==> Writing knoe.cfg...\n")
|
||
knoe._write_cfg()
|
||
_log(f" [OK] {cfg_path}\n\n")
|
||
|
||
db_password = knoe._get_input("init_password.db_password", "").strip()
|
||
_log(f"==> Target namespace: {namespace}\n\n")
|
||
|
||
attempt = 1
|
||
total_attempts = 0
|
||
max_attempts_env = os.environ.get("SILENT_TEST_MAX_ATTEMPTS", "").strip()
|
||
max_attempts = int(max_attempts_env) if max_attempts_env.isdigit() else 0
|
||
|
||
_log("==> Running silent install validation...\n")
|
||
while True:
|
||
total_attempts += 1
|
||
_log(f" Attempt {total_attempts}...\n")
|
||
rc = _run_silent_install_test(project_root, cfg_path)
|
||
if rc == 0:
|
||
_log(f" [OK] Silent install test passed\n\n")
|
||
break
|
||
|
||
_log(f" [WARN] Silent install test failed (code {rc})\n")
|
||
cur_server = knoe.inputs.get("init_cluster.k3s_server_url", k3s_server)
|
||
cur_token = knoe.inputs.get("init_cluster.k3s_token", k3s_token)
|
||
|
||
if attempt == 1:
|
||
_log(" Attempting k3s repair...\n")
|
||
_attempt_k3s_repair(
|
||
controller, namespace, cur_server, cur_token, db_password
|
||
)
|
||
elif attempt == 2:
|
||
_log(" Resetting k3s namespace...\n")
|
||
_reset_k3s_namespace(project_root, namespace, cur_server, cur_token)
|
||
else:
|
||
attempt = 0
|
||
|
||
attempt += 1
|
||
if max_attempts and total_attempts >= max_attempts:
|
||
_log(f" [ERROR] Max attempts ({max_attempts}) reached. Aborting.\n")
|
||
return rc
|
||
else:
|
||
_log("==> Skipping validation (silent install already succeeded)\n\n")
|
||
|
||
cur_server = knoe.inputs.get("init_cluster.k3s_server_url", k3s_server)
|
||
cur_token = knoe.inputs.get("init_cluster.k3s_token", k3s_token)
|
||
_sync_opentofu_pipeline(project_root, namespace, cur_server, cur_token, log_fn=_log)
|
||
|
||
_log("\n==> k3s pipeline preparation complete.\n")
|
||
return 0
|