"""Initialization scripts, CNPG deployment and service-layer overlays.""" import os import platform import secrets import shlex import subprocess import threading import time import webbrowser from pathlib import Path from datetime import datetime import tkinter as tk from tkinter import ttk, messagebox, filedialog from knoe.screen import TerminalConsole from knoe import screen as ui from knoe.core.env import ( PROJECT_ROOT, _bool_str, _deployment_mode_from_env, _deployment_target_label, _normalize_cluster_env, ) from knoe.core.policy import ( POLICY_CFG_KEY, OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES, evaluate_optional_workloads_allowed, ) from knoe.core.ops.cloudnative_pg import ( initialize as cnpg_initialize, deploy as cnpg_deploy, rollout as cnpg_rollout, ) from knoe.core.ops import openbao as openbao_ops from knoe.core.ops import registry as registry_ops from knoe.core.ops import garage_store as garage_store_ops from knoe.core.ops import opentofu as opentofu_ops from knoe.core.ops import monitoring as monitoring_ops class ServicesScreenMixin: """Initialization scripts, CNPG deployment and service-layer overlays.""" def _render_init_scripts_page(self): # Letterhead at top right content_width = self.bg_canvas.winfo_width() or 975 right_margin = content_width - 48 ui.canvas_text( self, right_margin, 40, "Prole", fill="#6e6e73", font=("SF Pro Text", 32, "bold"), anchor="ne", ) ui.canvas_text( self, right_margin, 85, "Infrastructure Automated.", fill="#6e6e73", font=("SF Pro Text", 18), anchor="ne", ) self._render_title("Initialization Scripts", y=150) self._render_paragraph( "Running initialization scripts to set up Garage, CloudNative-PG, and backups.", y=200, ) y = 260 y = self._render_init_scripts_section(48, y, 900, 450, title="Execution Output") # Use tk.Button self._init_scripts_button = tk.Button( self.bg_canvas, text="Run Scripts", command=self.run_init_scripts, bg="#F5F5DC", fg="black", activebackground="#E5E5D5", highlightbackground="#F5F5DC", highlightthickness=0, relief="flat", font=("SF Pro Text", 11), padx=16, pady=8, ) btn_window = self.bg_canvas.create_window( 48, y + 20, window=self._init_scripts_button, anchor="nw", width=180 ) self._canvas_items.append(btn_window) self._overlay_widgets.append(self._init_scripts_button) # Status Label self._init_scripts_status_label = ui.canvas_text( self, 240, y + 32, "", fill="black", font=("SF Pro Text", 12) ) self._canvas_items.append(self._init_scripts_status_label) def _render_init_scripts_section( self, x: int, y: int, width: int, height: int, title: str | None = "Execution Output", ) -> int: if title: title_item = ui.canvas_text( self, x, y, title, fill="#1d1d1f", font=("SF Pro Text", 12, "bold") ) self._canvas_items.append(title_item) y += 30 # Use a background frame for the notebook to hide potential system borders notebook_bg = tk.Frame(self.bg_canvas, bg="white", highlightthickness=0, bd=0) self.script_tabs = ttk.Notebook(notebook_bg, style="TNotebook") self.script_tabs.pack(fill="both", expand=True, padx=1, pady=1) tab_window = self.bg_canvas.create_window( x, y, window=notebook_bg, anchor="nw", width=width, height=height ) self._canvas_items.append(tab_window) self._overlay_widgets.append(notebook_bg) self._overlay_widgets.append(self.script_tabs) self.script_consoles = {} scripts = self._init_scripts_list() self._script_tab_index = {} for idx, (title, fname) in enumerate(scripts): # Use a background frame to ensure NO borders are visible around the console console_bg = tk.Frame( self.script_tabs, bg="white", highlightthickness=0, bd=0 ) self.script_tabs.add(console_bg, text=title) # Use TerminalConsole for consistent styling if fname == self.KUBECTL_STATUS_TAB: mono = ( ("Menlo", 9) if platform.system() == "Darwin" else ("Consolas", 9) ) console = ui.TerminalConsole( console_bg, highlightthickness=0, bd=0, font=mono, wrap="none", show_horizontal=True, ) else: console = ui.TerminalConsole(console_bg, highlightthickness=0, bd=0) console.pack(fill="both", expand=True, padx=1, pady=1) self.script_consoles[fname] = console self._script_tab_index[fname] = idx self._overlay_widgets.append(console_bg) self._overlay_widgets.append(console) self.script_tabs.bind( "<>", self._on_init_scripts_tab_changed ) self.safe_after(self._refresh_kubectl_status_tab, delay=50) return y + height def _init_scripts_list(self): scripts = [ ("Kubectl Status", self.KUBECTL_STATUS_TAB), ("Common Services", "init_common_services.sh"), ("Common Services Log", "common_services_log"), ("CloudNative-PG", "init_cloudnative_pg.sh"), ] scripts.append(("Prole DB Backup", "init_cnpg_backup.sh")) scripts.append(("Kong API Gateway", "init_kong.sh")) scripts.append(("Monitoring", "init_monitoring.sh")) mode = self._deployment_mode() if mode != "k3d": scripts.append(("Nginx Ingress", "init_nginx_ingress.sh")) if mode != "k3s": scripts.append(("Port Forwards", "init_port_forwards.sh")) return scripts def run_init_scripts(self): self._action_flags["init_scripts.run_scripts"] = True def worker(): def _select_tab(script_name): idx = getattr(self, "_script_tab_index", {}).get(script_name) if idx is None: return self.safe_after( lambda: ( self.script_tabs.select(idx) if self.script_tabs.winfo_exists() else None ) ) self.safe_after( lambda: ( self._init_scripts_button.configure(state="disabled") if self._init_scripts_button.winfo_exists() else None ) ) self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._init_scripts_status_label, text="Running scripts...", fill="blue", ) if self.bg_canvas.winfo_exists() else None ) ) password = self.db_password.get() # Prepare environment for scripts env = os.environ.copy() env["PROLE_HOME"] = str(PROJECT_ROOT) env["PROLE_SERVICE"] = str(PROJECT_ROOT) env["KNOE_DB_USER"] = self.db_username.get() env["DB_PASSWORD"] = password env["GRAFANA_ADMIN_PASSWORD"] = password db_namespace = (self.db_namespace.get() or "").strip() if not db_namespace: db_namespace = ( ((getattr(self, "prole_cfg_data", {}) or {}).get("Global", {}) or {}) .get("DATABASE_NAMESPACE", "") .strip() ) if not db_namespace: db_namespace = "default" env["DATABASE_NAMESPACE"] = db_namespace env["NAMESPACE"] = db_namespace env["PROLE_NAMESPACE"] = db_namespace env["SERVICE_NAMESPACE"] = self._get_service_namespace() if self.kerberos_realm.get().strip(): env["KRB5_REALM"] = self.kerberos_realm.get().strip() env["REALM"] = self.kerberos_realm.get().strip() env["DOMAIN"] = self.kerberos_realm.get().strip().lower() if self.kerberos_kdc.get().strip(): env["KRB5_KDC"] = self.kerberos_kdc.get().strip() env["KRB5_ADMIN"] = self.kerberos_kdc.get().strip() if self.kerberos_user.get().strip(): env["KRB5_USER"] = self.kerberos_user.get().strip() if self.kerberos_password.get().strip(): env["KRB5_PASSWORD"] = self.kerberos_password.get().strip() env["KERBEROS_ENABLED"] = _bool_str(self.kerberos_enabled.get()) env["ENABLED"] = env["KERBEROS_ENABLED"] mode = self._deployment_mode() if mode: env["PROLE_MODE"] = mode env["DEPLOYMENT_MODE"] = mode env["DEPLOYMENT_TARGET"] = _deployment_target_label( self.cluster_env.get() ) mode_args = ["--mode", mode] if mode else [] raw_min = ( (getattr(self, "prole_cfg_data", {}) or {}) .get("Global", {}) .get(POLICY_CFG_KEY, "") ) try: min_nodes = int( str(raw_min).strip() or str(OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES) ) except Exception: min_nodes = OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES opt_allowed, _count, opt_reason = evaluate_optional_workloads_allowed( env=env, min_nodes=min_nodes ) logs_dir = self._resolve_prole_logs_dir() try: logs_dir.mkdir(parents=True, exist_ok=True) except Exception: pass env.setdefault("PROLE_LOGS", str(logs_dir)) def _log_path_for(script_name: str) -> Path: base = Path(script_name).stem if base in ("common_services_log", "init_common_services"): return logs_dir / "init_common_services.log" return logs_dir / f"{base}.log" # Ensure port-forward mappings and port-mapping.cfg are populated # before init_nginx_ingress.sh runs (it reads port-mapping.cfg). try: self._sync_port_forward_mappings() self._update_legacy_port_mapping() except Exception: pass # 1. init_common_services.sh start overall_success = True if overall_success: script = "init_common_services.sh" _select_tab(script) self.script_consoles[script].clear() self.script_consoles[script].write(f"Running {script} start...\n") svc_log_path = _log_path_for(script) self._last_common_services_log_path = svc_log_path env["COMMON_SERVICES_INIT_LOG"] = str(svc_log_path) self._record_install_log(svc_log_path) self.script_consoles["common_services_log"].clear() self.script_consoles["common_services_log"].write( f"Log file: {svc_log_path}\n\n" ) self.script_consoles[script].write(f"Log file: {svc_log_path}\n\n") try: svc_fp = svc_log_path.open("a", encoding="utf-8") except Exception: svc_fp = None def _svc_line(line): self.script_consoles[script].write(line) self.script_consoles["common_services_log"].write(line) self._process_script_output_line(line) if svc_fp: try: svc_fp.write(line) svc_fp.flush() except Exception: pass rc_svc = 0 svc_namespace = ( env.get("SERVICE_NAMESPACE") or env.get("NAMESPACE") or "default" ) registry_ns = env.get("REGISTRY_NAMESPACE") or svc_namespace project_root = getattr(self.controller, "project_root", PROJECT_ROOT) def _svc_log(msg: str) -> None: _svc_line(msg if msg.endswith("\n") else msg + "\n") try: _svc_log(f"[INFO] Registry update namespace={registry_ns}") registry_ops.update( namespace=registry_ns, env=env, project_root=project_root, mode=mode, log=_svc_log, ) _svc_log(f"[INFO] OpenBao update namespace={svc_namespace}") openbao_ops.update( namespace=svc_namespace, env=env, project_root=project_root, mode=mode, log=_svc_log, ) _svc_log(f"[INFO] Garage update namespace={svc_namespace}") garage_store_ops.update( namespace=svc_namespace, env=env, project_root=project_root, mode=mode, log=_svc_log, ) _svc_log(f"[INFO] OpenTofu update namespace={svc_namespace}") opentofu_ops.update( namespace=svc_namespace, env=env, project_root=project_root, mode=mode, log=_svc_log, ) except Exception as _svc_exc: _svc_log(f"[ERROR] Common services Python execution failed: {_svc_exc}") rc_svc = 1 if svc_fp: try: svc_fp.close() except Exception: pass if rc_svc != 0: self.script_consoles[script].write( f"\nERROR: {script} start failed with code {rc_svc}\n" ) overall_success = False else: self.script_consoles[script].write( f"\n{script} completed successfully.\n" ) else: self.script_consoles["init_common_services.sh"].write( "Skipping common services initialization because previous steps failed.\n" ) # 2. CloudNative-PG initialization (Python) cnpg_tab = "init_cloudnative_pg.sh" if overall_success: _select_tab(cnpg_tab) self.script_consoles[cnpg_tab].clear() self.script_consoles[cnpg_tab].write("Running CloudNative-PG initialization...\n") log_path = _log_path_for(cnpg_tab) self._record_install_log(log_path) try: log_fp = log_path.open("a", encoding="utf-8") except Exception: log_fp = None def _cnpg_log(msg): line = msg if msg.endswith("\n") else msg + "\n" self.script_consoles[cnpg_tab].write(line) self._process_script_output_line(line) if log_fp: try: log_fp.write(line) log_fp.flush() except Exception: pass cnpg_ns = env.get("DATABASE_NAMESPACE") or env.get("NAMESPACE", "default") cnpg_cluster = env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") or "knoe-db" cnpg_project_root = getattr(self.controller, "project_root", PROJECT_ROOT) try: cnpg_initialize( namespace=cnpg_ns, cluster_name=cnpg_cluster, env=env, project_root=cnpg_project_root, log=_cnpg_log, mode=mode, ) self.script_consoles[cnpg_tab].write( "\nCloudNative-PG initialization completed successfully.\n" ) except Exception as _cnpg_exc: self.script_consoles[cnpg_tab].write( f"\nERROR: CloudNative-PG initialization failed: {_cnpg_exc}\n" ) overall_success = False finally: if log_fp: try: log_fp.close() except Exception: pass else: if cnpg_tab in self.script_consoles: self.script_consoles[cnpg_tab].write( "Skipping CloudNative-PG initialization because previous steps failed.\n" ) # 4. init_cnpg_backup.sh start if overall_success: script = "init_cnpg_backup.sh" _select_tab(script) self.script_consoles[script].clear() self.script_consoles[script].write(f"Running {script} start...\n") log_path = _log_path_for(script) self._record_install_log(log_path) try: log_fp = log_path.open("a", encoding="utf-8") except Exception: log_fp = None def _backup_line(line): self.script_consoles[script].write(line) self._process_script_output_line(line) if log_fp: try: log_fp.write(line) log_fp.flush() except Exception: pass rc_backup = self.controller.run_script( script, args=mode_args + ["start"], env=env, on_line=_backup_line ) if log_fp: try: log_fp.close() except Exception: pass if rc_backup != 0: self.script_consoles[script].write( f"\nERROR: {script} start failed with code {rc_backup}\n" ) overall_success = False else: self.script_consoles[script].write( f"\n{script} completed successfully.\n" ) else: self.script_consoles["init_cnpg_backup.sh"].write( "Skipping Prole DB backup because previous steps failed.\n" ) # 5. init_kong.sh start if overall_success: script = "init_kong.sh" if script in self.script_consoles: _select_tab(script) self.script_consoles[script].clear() self.script_consoles[script].write(f"Running {script} start...\n") log_path = _log_path_for(script) self._record_install_log(log_path) try: log_fp = log_path.open("a", encoding="utf-8") except Exception: log_fp = None def _kong_line(line): self.script_consoles[script].write(line) self._process_script_output_line(line) if log_fp: try: log_fp.write(line) log_fp.flush() except Exception: pass kong_ns = env.get("NAMESPACE") or "knoe-db" rc_kong = self.controller.run_script( script, args=mode_args + ["-n", kong_ns, "start"], env=env, on_line=_kong_line ) if log_fp: try: log_fp.close() except Exception: pass if rc_kong != 0: self.script_consoles[script].write( f"\nERROR: {script} start failed with code {rc_kong}\n" ) overall_success = False else: self.script_consoles[script].write( f"\n{script} completed successfully.\n" ) else: if "init_kong.sh" in self.script_consoles: self.script_consoles["init_kong.sh"].write( "Skipping Kong because previous steps failed.\n" ) # 6. init_monitoring.sh initialize if overall_success and opt_allowed: script = "init_monitoring.sh" _select_tab(script) self.script_consoles[script].clear() self.script_consoles[script].write(f"Running {script} initialize...\n") log_path = _log_path_for(script) self._record_install_log(log_path) try: log_fp = log_path.open("a", encoding="utf-8") except Exception: log_fp = None def _monitoring_line(line): self.script_consoles[script].write(line) self._process_script_output_line(line) if log_fp: try: log_fp.write(line) log_fp.flush() except Exception: pass # Extract Grafana password if present if "GRAFANA_ADMIN_PASSWORD=" in line: pwd = line.split("GRAFANA_ADMIN_PASSWORD=")[1].strip() if pwd: mon = self.prole_cfg_data.get("Monitoring", {}) if mon is None: mon = {} mon["GRAFANA_ADMIN_PASSWORD"] = pwd self.prole_cfg_data["Monitoring"] = mon self._save_prole_cfg() rc_mon = 0 monitoring_ns = env.get("MONITORING_NAMESPACE") or "monitoring" def _monitoring_log(msg: str) -> None: _monitoring_line(msg if msg.endswith("\n") else msg + "\n") try: monitoring_ops.initialize( namespace=monitoring_ns, env=env, mode=mode, log=_monitoring_log, ) except Exception as _mon_exc: _monitoring_log(f"[ERROR] Monitoring Python execution failed: {_mon_exc}") rc_mon = 1 if log_fp: try: log_fp.close() except Exception: pass if rc_mon != 0: self.script_consoles[script].write( f"\nERROR: {script} initialize failed with code {rc_mon}\n" ) overall_success = False else: mon = self.prole_cfg_data.get("Monitoring", {}) or {} if not str(mon.get("GRAFANA_ADMIN_PASSWORD") or "").strip(): mon["GRAFANA_ADMIN_PASSWORD"] = env.get( "GRAFANA_ADMIN_PASSWORD", "" ) self.prole_cfg_data["Monitoring"] = mon try: self._save_prole_cfg() except Exception: pass self.script_consoles[script].write( f"\n{script} completed successfully.\n" ) elif overall_success and not opt_allowed: script = "init_monitoring.sh" _select_tab(script) self.script_consoles[script].clear() msg = f"[SKIP] Monitoring disabled by policy: {opt_reason}\n" self.script_consoles[script].write(msg) mon = (getattr(self, "prole_cfg_data", {}) or {}).get("Monitoring") if mon is None: mon = {} mon["STATUS"] = "Skipped" self.prole_cfg_data["Monitoring"] = mon try: self._save_prole_cfg() except Exception: pass else: if "init_monitoring.sh" in self.script_consoles: self.script_consoles["init_monitoring.sh"].write( "Skipping Monitoring initialization because previous steps failed.\n" ) # 7. init_nginx_ingress.sh initialize (k3s only — not used in k3d) if overall_success and mode != "k3d": script = "init_nginx_ingress.sh" _select_tab(script) self.script_consoles[script].clear() self.script_consoles[script].write(f"Running {script} initialize...\n") log_path = _log_path_for(script) self._record_install_log(log_path) try: log_fp = log_path.open("a", encoding="utf-8") except Exception: log_fp = None def _ingress_line(line): self.script_consoles[script].write(line) self._process_script_output_line(line) if log_fp: try: log_fp.write(line) log_fp.flush() except Exception: pass rc_ing = self.controller.run_script( script, args=mode_args + ["initialize"], env=env, on_line=_ingress_line, ) if log_fp: try: log_fp.close() except Exception: pass if rc_ing != 0: self.script_consoles[script].write( f"\nERROR: {script} initialize failed with code {rc_ing}\n" ) overall_success = False else: self.script_consoles[script].write( f"\n{script} completed successfully.\n" ) elif mode != "k3d": self.script_consoles["init_nginx_ingress.sh"].write( "Skipping Nginx Ingress because previous steps failed.\n" ) # 8. init_port_forwards.sh start (k3d only — not used in k3s) if overall_success and mode != "k3s": script = "init_port_forwards.sh" if script in self.script_consoles: _select_tab(script) self.script_consoles[script].clear() self.script_consoles[script].write(f"Running {script} start...\n") def _pf_line(line): self.script_consoles[script].write(line) self._process_script_output_line(line) env_pf = dict(env) env_pf.setdefault("PORT_FORWARD_SKIP_VALIDATE", "1") env_pf.setdefault("PORT_FORWARD_SKIP_WAIT", "1") env_pf.setdefault("PORT_FORWARD_WAIT_TIMEOUT", "20") env_pf.setdefault("PORT_FORWARD_WAIT_INTERVAL", "2") rc_pf = self.controller.run_script( script, args=["--force", "--mode", mode, "start"], env=env_pf, on_line=_pf_line, ) if rc_pf != 0: self.script_consoles[script].write( f"\nERROR: {script} start failed with code {rc_pf}\n" ) overall_success = False else: running = False for _ in range(10): status_lines = [] def _status_line(line): status_lines.append(line) self.script_consoles[script].write(line) rc_status = self.controller.run_script( script, args=["--mode", mode, "status", "grafana"], env=env_pf, on_line=_status_line, ) running = any( l.lstrip().startswith("RUNNING") and "grafana" in l for l in status_lines ) if rc_status == 0 and running: break time.sleep(1) if not running: self.script_consoles[script].write( "\nERROR: Grafana port-forward not running after start.\n" ) overall_success = False else: self.script_consoles[script].write( "\nPort-forwards running (grafana ok).\n" ) # Verify critical secrets ns = ( str(env.get("DATABASE_NAMESPACE") or env.get("NAMESPACE") or "default").strip() or "default" ) kubectl_base_cmd = ["kubectl"] try: resolved_base_cmd = list(self._kubectl_base_cmd(mode)) if resolved_base_cmd: kubectl_base_cmd = resolved_base_cmd except Exception: pass self.script_consoles["init_cloudnative_pg.sh"].write( f"\nVerifying critical secrets in namespace {ns}...\n" ) critical_secrets = ["knoe-db-user", "knoe-db-superuser", "cnpg-admin-key"] missing_secrets = [] for secret in critical_secrets: rc_s, _ = self._run_cmd_capture( kubectl_base_cmd + ["get", "secret", secret, "-n", ns] ) if rc_s != 0: missing_secrets.append(secret) has_secret_check_warnings = False if missing_secrets: has_secret_check_warnings = True self.script_consoles["init_cloudnative_pg.sh"].write( f"WARNING: Missing secrets in namespace '{ns}': {', '.join(missing_secrets)}.\n" ) self.script_consoles["init_cloudnative_pg.sh"].write( "Continuing because initialization scripts succeeded; verify CNPG secrets before production use.\n" ) if overall_success: status_text = "Initialization complete!" status_fill = "#34c759" if has_secret_check_warnings: status_text = "Initialization complete (warnings)." status_fill = "#ff9f0a" self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._init_scripts_status_label, text=status_text, fill=status_fill, ) if self.bg_canvas.winfo_exists() else None ) ) self._scripts_success = True else: self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._init_scripts_status_label, text="Initialization failed.", fill="#ff3b30", ) if self.bg_canvas.winfo_exists() else None ) ) self._scripts_success = False try: self.prole_cfg_data["Initialization Scripts"]["STATUS"] = ( "Completed" if self._scripts_success else "Attempted" ) self.safe_after(self._save_prole_cfg) except Exception: pass self.safe_after(lambda: self.update_footer()) self.safe_after( lambda: ( self._init_scripts_button.configure(state="normal") if self._init_scripts_button.winfo_exists() else None ) ) self.safe_after(self.check_services_status_async) threading.Thread(target=worker, daemon=True).start() def _on_init_scripts_tab_changed(self, _event=None): try: current_idx = self.script_tabs.index("current") except Exception: return if current_idx == self._script_tab_index.get(self.KUBECTL_STATUS_TAB): self._refresh_kubectl_status_tab() def _refresh_kubectl_status_tab(self): console = self.script_consoles.get(self.KUBECTL_STATUS_TAB) if not console: return namespace = ( (self.db_namespace.get() or "").strip() or os.environ.get("NAMESPACE") or "default" ) base_cmd = self._kubectl_base_cmd() def _display_cmd(args: list[str]) -> str: redacted = [] for item in base_cmd + args: if item.startswith("--token="): redacted.append("--token=***") else: redacted.append(item) return " ".join(shlex.quote(x) for x in redacted) def _run_kubectl(args: list[str]): env = os.environ.copy() env.setdefault("PROLE_HOME", str(PROJECT_ROOT)) try: res = subprocess.run( base_cmd + args, capture_output=True, text=True, env=env ) out = res.stdout or "" err = res.stderr or "" return res.returncode, out, err except Exception as exc: return 1, "", f"{exc}" def worker(): cmd_args = ["get", "pods", "-A", "-o", "wide"] code, out, err = _run_kubectl(cmd_args) now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") header = f"# {_display_cmd(cmd_args)} ({now}) [highlight: namespace={namespace}]" lines = [ln.rstrip("\n") for ln in out.splitlines() if ln.strip() != ""] if code != 0 and not lines: lines = ["(no output)"] pod_rows = [] unhappy = [] if lines: for idx, line in enumerate(lines): if idx == 0: pod_rows.append({"raw": line, "ns": "", "unhappy": False}) continue parts = line.split() if len(parts) < 4: pod_rows.append({"raw": line, "ns": "", "unhappy": False}) continue ns, name, ready, status = parts[0], parts[1], parts[2], parts[3] status_lc = status.lower() happy_status = status_lc in ("running", "completed", "succeeded") ready_ok = True if status_lc == "running" and "/" in ready: try: a, b = ready.split("/", 1) ready_ok = a == b except Exception: ready_ok = False is_happy = happy_status and ( ready_ok or status_lc in ("completed", "succeeded") ) if not is_happy: unhappy.append( {"ns": ns, "name": name, "status": status, "ready": ready} ) pod_rows.append({"raw": line, "ns": ns, "unhappy": not is_happy}) details = "" if unhappy: blocks = ["", "# Unhealthy pods detected; collecting events + logs"] for pod in unhappy: ns = pod["ns"] name = pod["name"] blocks.append(f"\n## Events for {ns}/{name}") ec, eout, eerr = _run_kubectl( [ "get", "events", "-n", ns, "--field-selector", f"involvedObject.kind=Pod,involvedObject.name={name}", "--sort-by=.lastTimestamp", ] ) blocks.append(eout.strip() or eerr.strip() or "(no events output)") blocks.append(f"\n## Logs for {ns}/{name}") lc, lout, lerr = _run_kubectl( ["logs", "-n", ns, name, "--all-containers", "--tail=200"] ) blocks.append(lout.strip() or lerr.strip() or "(no logs output)") details = "\n".join(blocks).rstrip() + "\n" elif err.strip(): details = "\n# kubectl stderr\n" + err.strip() + "\n" def render(): if not console.text.winfo_exists(): return text = console.text text.configure(state="normal") text.delete("1.0", "end") text.tag_configure("header", foreground="#6e6e73") text.tag_configure("ns", background="#DDF4FF") text.tag_configure("bad", foreground="#B00020") text.insert("end", header + "\n\n", ("header",)) for row in pod_rows: tags = [] if row["ns"] == namespace: tags.append("ns") if row["unhappy"]: tags.append("bad") if row["raw"].strip(): text.insert("end", row["raw"] + "\n", tuple(tags)) if details: text.insert("end", "\n" + details) text.see("1.0") text.configure(state="disabled") self.safe_after(render) threading.Thread(target=worker, daemon=True).start() def _render_init_cnpg_deploy_page(self): # Letterhead at top right content_width = self.bg_canvas.winfo_width() or 975 right_margin = content_width - 48 ui.canvas_text( self, right_margin, 40, "Prole", fill="#6e6e73", font=("SF Pro Text", 32, "bold"), anchor="ne", ) ui.canvas_text( self, right_margin, 85, "Infrastructure Automated.", fill="#6e6e73", font=("SF Pro Text", 18), anchor="ne", ) self._render_title("Deployment", y=150) self._render_paragraph( "Deploy the CloudNative-PG operator and cluster manifests to Kubernetes.", y=200, ) # Output Console self._cnpg_deploy_console = self._create_console_output( y=260, title="Deployment Output", width=900, height=520 ) # Use tk.Button self._cnpg_deploy_button = tk.Button( self.bg_canvas, text="Run Deployment", command=self.run_cnpg_deploy, bg="#F5F5DC", fg="black", activebackground="#E5E5D5", highlightbackground="#F5F5DC", highlightthickness=0, relief="flat", font=("SF Pro Text", 11), padx=16, pady=8, ) self._cnpg_deploy_button.update_idletasks() deploy_btn_height = self._cnpg_deploy_button.winfo_reqheight() or 32 # Deployment mode selector mode_label = ui.canvas_text( self, 48, 782, "Mode", fill="#6e6e73", font=("SF Pro Text", 10, "bold") ) self._canvas_items.append(mode_label) mode_values = [ "knoe-dev-cluster", "prole-service-cluster", "prole-prod-cluster", ] mode_frame = tk.Frame( self.bg_canvas, bg="#F5F5DC", width=180, height=deploy_btn_height ) mode_frame.pack_propagate(False) mode_combo = ttk.Combobox( mode_frame, textvariable=self.deploy_target, values=mode_values, state="readonly", width=24, ) mode_combo.pack(fill="both", expand=True) mode_combo_window = self.bg_canvas.create_window( 48, 800, window=mode_frame, anchor="nw", width=180, height=deploy_btn_height ) self._canvas_items.append(mode_combo_window) self._overlay_widgets.append(mode_frame) btn_window = self.bg_canvas.create_window( 240, 800, window=self._cnpg_deploy_button, anchor="nw", width=180 ) self._canvas_items.append(btn_window) self._overlay_widgets.append(self._cnpg_deploy_button) # Force Rollout Button self._cnpg_rollout_button = tk.Button( self.bg_canvas, text="Force Rollout", command=self.run_cnpg_rollout, bg="#F5F5DC", fg="black", activebackground="#E5E5D5", highlightbackground="#F5F5DC", highlightthickness=0, relief="flat", font=("SF Pro Text", 11), padx=16, pady=8, ) rollout_btn_window = self.bg_canvas.create_window( 420, 800, window=self._cnpg_rollout_button, anchor="nw", width=160 ) self._canvas_items.append(rollout_btn_window) self._overlay_widgets.append(self._cnpg_rollout_button) # Save Deployment Button (moved from Post Install) self._save_deployment_button = tk.Button( self.bg_canvas, text="Save Deployment", command=self.run_final_deployment, bg="#F5F5DC", fg="black", activebackground="#E5E5D5", highlightbackground="#F5F5DC", highlightthickness=0, relief="flat", font=("SF Pro Text", 11), padx=16, pady=8, ) save_btn_window = self.bg_canvas.create_window( 600, 800, window=self._save_deployment_button, anchor="nw", width=180 ) self._canvas_items.append(save_btn_window) self._overlay_widgets.append(self._save_deployment_button) # Status Labels (below the buttons row to avoid overlap) self._cnpg_deploy_status_label = ui.canvas_text( self, 48, 840, "", fill="black", font=("SF Pro Text", 12) ) self._canvas_items.append(self._cnpg_deploy_status_label) self._save_deployment_status_label = ui.canvas_text( self, 600, 840, "", fill="black", font=("SF Pro Text", 12) ) self._canvas_items.append(self._save_deployment_status_label) def run_cnpg_deploy(self): self._action_flags["init_cnpg_deploy.run_deploy"] = True def worker(): self.safe_after( lambda: ( self._cnpg_deploy_button.configure(state="disabled") if self._cnpg_deploy_button.winfo_exists() else None ) ) self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._cnpg_deploy_status_label, text="Deploying...", fill="blue" ) if self.bg_canvas.winfo_exists() else None ) ) target_value = "" try: target_value = (self.deploy_target.get() or "").strip() except Exception: target_value = "" if not target_value: try: target_value = (self.cluster_env.get() or "").strip() except Exception: target_value = "" target_key = _normalize_cluster_env(target_value) mode = _deployment_mode_from_env(target_value or self.cluster_env.get()) if target_key in ("service", "prod"): url = self._deployment_pipeline_url(target_key) self._cnpg_deploy_console.clear() self._cnpg_deploy_console.write( f"Opening deployment pipeline for {target_value or target_key}...\n" ) self._cnpg_deploy_console.write(f"{url}\n") try: webbrowser.open(url) except Exception: pass self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._cnpg_deploy_status_label, text="Deployment pipeline opened.", fill="#34c759", ) if self.bg_canvas.winfo_exists() else None ) ) self._cnpg_success = False self.safe_after( lambda: ( self._cnpg_deploy_button.configure(state="normal") if self._cnpg_deploy_button.winfo_exists() else None ) ) return etc_dir = PROJECT_ROOT / "etc" # Prepare environment for scripts env = os.environ.copy() env["PROLE_HOME"] = str(PROJECT_ROOT) env["PROLE_SERVICE"] = str(PROJECT_ROOT) env["NAMESPACE"] = (self.db_namespace.get() or "").strip() env["NAMESPACE"] = (self.db_namespace.get() or "").strip() if mode: env["PROLE_MODE"] = mode env["DEPLOYMENT_MODE"] = mode env["DEPLOYMENT_TARGET"] = _deployment_target_label( self.cluster_env.get() ) ns = env.get("NAMESPACE", "default") cluster_name = env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") or "knoe-db" cnpg_project_root = getattr(self.controller, "project_root", PROJECT_ROOT) self._cnpg_deploy_console.clear() self._cnpg_deploy_console.write("Starting CloudNative-PG deployment (Python)...\n") def _deploy_log(msg): line = msg if msg.endswith("\n") else msg + "\n" self._cnpg_deploy_console.write(line) try: cnpg_deploy( namespace=ns, cluster_name=cluster_name, env=env, project_root=cnpg_project_root, log=_deploy_log, mode=mode, ) self._cnpg_deploy_console.write("\nDeployment successful!\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._cnpg_deploy_status_label, text="Deployment successful!", fill="#34c759", ) if self.bg_canvas.winfo_exists() else None ) ) self._cnpg_success = True except Exception as _deploy_exc: self._cnpg_deploy_console.write(f"\nDeployment failed: {_deploy_exc}\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._cnpg_deploy_status_label, text=f"Deployment failed: {_deploy_exc}", fill="#ff3b30", ) if self.bg_canvas.winfo_exists() else None ) ) self._cnpg_success = False self.safe_after( lambda: ( self._cnpg_deploy_button.configure(state="normal") if self._cnpg_deploy_button.winfo_exists() else None ) ) threading.Thread(target=worker, daemon=True).start() def run_cnpg_rollout(self): self._action_flags["init_cnpg_deploy.force_rollout"] = True def worker(): self.safe_after( lambda: ( self._cnpg_deploy_button.configure(state="disabled") if self._cnpg_deploy_button.winfo_exists() else None ) ) self.safe_after( lambda: ( self._cnpg_rollout_button.configure(state="disabled") if self._cnpg_rollout_button.winfo_exists() else None ) ) self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._cnpg_deploy_status_label, text="Rolling out...", fill="blue", ) if self.bg_canvas.winfo_exists() else None ) ) # Prepare environment env = os.environ.copy() env["PROLE_HOME"] = str(PROJECT_ROOT) env["PROLE_SERVICE"] = str(PROJECT_ROOT) mode = self._deployment_mode() if mode: env["PROLE_MODE"] = mode env["DEPLOYMENT_MODE"] = mode env["DEPLOYMENT_TARGET"] = _deployment_target_label( self.cluster_env.get() ) ns = (self.db_namespace.get() or "").strip() or env.get("NAMESPACE", "default") env["NAMESPACE"] = ns cluster_name = env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") or "knoe-db" cnpg_project_root = getattr(self.controller, "project_root", PROJECT_ROOT) self._cnpg_deploy_console.clear() self._cnpg_deploy_console.write( "Starting manual recreate rollout for knoe-db cluster (Python)...\n" ) def _rollout_log(msg): line = msg if msg.endswith("\n") else msg + "\n" self._cnpg_deploy_console.write(line) try: cnpg_rollout( namespace=ns, cluster_name=cluster_name, env=env, log=_rollout_log, ) self._cnpg_deploy_console.write("\nRollout successful!\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._cnpg_deploy_status_label, text="Rollout successful!", fill="#34c759", ) if self.bg_canvas.winfo_exists() else None ) ) except Exception as _rollout_exc: self._cnpg_deploy_console.write(f"\nRollout failed: {_rollout_exc}\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._cnpg_deploy_status_label, text=f"Rollout failed: {_rollout_exc}", fill="#ff3b30", ) if self.bg_canvas.winfo_exists() else None ) ) self.safe_after( lambda: ( self._cnpg_deploy_button.configure(state="normal") if self._cnpg_deploy_button.winfo_exists() else None ) ) self.safe_after( lambda: ( self._cnpg_rollout_button.configure(state="normal") if self._cnpg_rollout_button.winfo_exists() else None ) ) def _run_service_layer_inline(self, next_page: str = "init_password"): """Run service-layer migration in the background without showing the overlay screen. Used when all common services are already green so the user can skip directly to Database Creation.""" def _inline_worker(): try: namespace = (self.db_namespace.get() or "").strip() or "default" env = self._script_env_for_namespace(namespace) env["SERVICE_NAMESPACE"] = self._get_service_namespace() rc = self.controller.run_script( "init_service_layer.sh", args=["migrate"], env=env, ) if rc == 0: self._common_services_success = True except Exception: pass # Navigate regardless — the service layer migration is best-effort # when all traffic lights are already green. self._common_services_success = True self.safe_after(lambda: self.show_page(next_page)) threading.Thread(target=_inline_worker, daemon=True).start() def _run_service_layer_overlay(self, next_page: str = "init_password"): """Ensure service-layer resources are ready before database creation.""" self._showing_service_overlay = True self._common_services_success = False self.update_footer() self._clear_canvas_page() # Letterhead at top right content_width = self.bg_canvas.winfo_width() or 975 right_margin = content_width - 48 ui.canvas_text( self, right_margin, 40, "Prole", fill="#6e6e73", font=("SF Pro Text", 32, "bold"), anchor="ne", ) ui.canvas_text( self, right_margin, 85, "Infrastructure Automated.", fill="#6e6e73", font=("SF Pro Text", 18), anchor="ne", ) self._render_title("Preparing Service Layer", y=150) self._render_paragraph( "Ensuring service-layer resources are ready before database creation.", y=200, ) console = self._create_console_output( y=260, title="Service Layer Output", width=900, height=520 ) status_label = ui.canvas_text( self, 240, 812, "Ready to deploy common services", fill="black", font=("SF Pro Text", 12), ) self._canvas_items.append(status_label) log_path = self._common_services_log_path() try: console.write(f"Log file: {log_path}\n\n") except Exception: pass def worker(): try: self.safe_after( lambda: ( self._deploy_services_button.configure(state="disabled") if self._deploy_services_button.winfo_exists() else None ) ) namespace = (self.db_namespace.get() or "").strip() or "default" env = self._script_env_for_namespace(namespace) env["SERVICE_NAMESPACE"] = self._get_service_namespace() try: log_fp = log_path.open("a", encoding="utf-8") log_fp.write( f"\n# Service layer deploy @ {time.strftime('%Y-%m-%d %H:%M:%S')}\n" ) log_fp.flush() except Exception: log_fp = None def _line(line: str): console.write(line) if log_fp: try: log_fp.write(line) log_fp.flush() except Exception: pass self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Deploying service layer...", fill="blue" ) if self.bg_canvas.winfo_exists() else None ) ) rc = self.controller.run_script( "init_service_layer.sh", args=["migrate"], env=env, on_line=_line ) if log_fp: try: log_fp.write(f"\nExit status: {rc}\n") log_fp.close() except Exception: pass if rc != 0: console.write(f"\nService layer setup failed (code {rc}).\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text=f"Service layer failed (code {rc})", fill="#ff3b30", ) if self.bg_canvas.winfo_exists() else None ) ) self.safe_after( lambda: ( self._deploy_services_button.configure(state="normal") if self._deploy_services_button.winfo_exists() else None ) ) return self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Service layer ready. You may proceed.", fill="#34c759", ) if self.bg_canvas.winfo_exists() else None ) ) self._common_services_success = True self.safe_after(self.update_footer) except Exception as e: console.write(f"\nService layer error: {e}\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Service layer error", fill="#ff3b30" ) if self.bg_canvas.winfo_exists() else None ) ) self.safe_after( lambda: ( self._deploy_services_button.configure(state="normal") if self._deploy_services_button.winfo_exists() else None ) ) # Use tk.Button explicitly for control over button appearance and behavior self._deploy_services_button = tk.Button( self.bg_canvas, text="Deploy Services", command=lambda: threading.Thread(target=worker, daemon=True).start(), bg="#F5F5DC", fg="black", activebackground="#E5E5D5", highlightbackground="#F5F5DC", highlightthickness=0, relief="flat", font=("SF Pro Text", 11), padx=16, pady=8, ) btn_window = self.bg_canvas.create_window( 48, 800, window=self._deploy_services_button, anchor="nw", width=180 ) self._canvas_items.append(btn_window) self._overlay_widgets.append(self._deploy_services_button) def _run_preparation_overlay(self): """Prepare environment for database creation.""" self._action_flags["init_password.generate_ssh_key"] = True self._clear_canvas_page() # Letterhead at top right content_width = self.bg_canvas.winfo_width() or 975 right_margin = content_width - 48 ui.canvas_text( self, right_margin, 40, "Prole", fill="#6e6e73", font=("SF Pro Text", 32, "bold"), anchor="ne", ) ui.canvas_text( self, right_margin, 85, "Infrastructure Automated.", fill="#6e6e73", font=("SF Pro Text", 18), anchor="ne", ) self._render_title("Preparing", y=150) self._render_paragraph( "Preparing your environment for database creation.", y=200 ) # Output Console - standardized to match Docker Build screen console = self._create_console_output(y=260, title="", width=900, height=520) # Status Label status_label = ui.canvas_text( self, 48, 812, "Initializing...", fill="black", font=("SF Pro Text", 12) ) self._canvas_items.append(status_label) def _show_return_button(): def _place(): try: if getattr(self, "_prep_return_btn", None): self.bg_canvas.delete(getattr(self, "_prep_return_btn_window", None)) except Exception: pass btn = tk.Button( self.bg_canvas, text="Back to Database", command=lambda: self.show_page("init_password"), bg="#F5F5DC", fg="black", activebackground="#E5E5D5", highlightbackground="#F5F5DC", highlightthickness=0, relief="flat", font=("SF Pro Text", 11), padx=14, pady=6, ) btn_window = self.bg_canvas.create_window( 48, 840, window=btn, anchor="nw", width=180 ) self._prep_return_btn = btn self._prep_return_btn_window = btn_window self._overlay_widgets.append(btn) self._canvas_items.append(btn_window) self.safe_after(_place) def worker(): key_path = Path.home() / ".ssh" / "id_prole_ed25519" key_path.parent.mkdir(parents=True, exist_ok=True) password = self.db_password.get() namespace = (self.db_namespace.get() or "").strip() mode = _deployment_mode_from_env(self.cluster_env.get()) # Pre-pull dependent images into local registry before initialization if mode == "k3s": console.write( "[INFO] Skipping image pre-pull in k3s mode (images are handled by the in-cluster registry/import).\n" ) else: try: self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Pre-pulling images...", fill="blue" ) if self.bg_canvas.winfo_exists() else None ) ) pull_ok = self._prepull_images_to_registry( include_supabase=self.supabase_enabled.get(), include_kerberos_proxy=self.kerberos_enabled.get(), log=console.write, ) if not pull_ok: console.write( "\n[WARN] Image pre-pull failed or incomplete. Continuing...\n" ) except Exception as e: console.write(f"\n[WARN] Image pre-pull error: {e}\n") if key_path.exists(): console.write(f"Key already exists at {key_path}. Proceeding...\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Keys present. Proceeding...", fill="#34c759", ) if self.bg_canvas.winfo_exists() else None ) ) time.sleep(0.5) else: self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Preparing keys...", fill="blue" ) if self.bg_canvas.winfo_exists() else None ) ) cmd = [ "ssh-keygen", "-t", "ed25519", "-N", "", "-f", str(key_path), "-C", self.db_username.get(), ] console.write(f"Initializing secure access: {' '.join(cmd)}\n\n") proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) while True: line = proc.stdout.readline() if not line and proc.poll() is not None: break if line: console.write(line) if proc.returncode != 0: console.write( f"\nPreparation failed, trying alternative (code {proc.returncode})\n" ) cmd = [ "ssh-keygen", "-t", "rsa", "-b", "4096", "-N", "", "-f", str(key_path), "-C", self.db_username.get(), ] console.write(f"Initializing secure access: {' '.join(cmd)}\n\n") proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) while True: line = proc.stdout.readline() if not line and proc.poll() is not None: break if line: console.write(line) if proc.returncode == 0: console.write("\nPreparation completed successfully.\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Preparation completed successfully.", fill="#34c759", ) if self.bg_canvas.winfo_exists() else None ) ) else: console.write( f"\nError during preparation (code {proc.returncode})\n" ) self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text=f"Error during preparation (code {proc.returncode})", fill="#ff3b30", ) if self.bg_canvas.winfo_exists() else None ) ) _show_return_button() return # Initialize OpenBao and store keys/passwords self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Initializing OpenBao...", fill="blue" ) if self.bg_canvas.winfo_exists() else None ) ) env = self._script_env_for_namespace(namespace) env["KNOE_DB_USER"] = self.db_username.get().strip() env["DB_PASSWORD"] = password env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( self.at_rest_encryption_enabled.get() ) if self.at_rest_encryption_enabled.get(): console.write( "At-rest encryption enabled: generating/storing TDE keys in OpenBao...\n" ) rc = 0 try: openbao_ops.initialize( namespace=namespace, env=env, project_root=getattr(self.controller, "project_root", PROJECT_ROOT), mode=mode, log=lambda l: console.write(l if l.endswith("\n") else l + "\n"), ) except Exception as _bao_exc: rc = 1 console.write(f"\nOpenBao initialization failed: {_bao_exc}\n") if rc != 0: self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text=f"OpenBao init failed (code {rc})", fill="#ff3b30", ) if self.bg_canvas.winfo_exists() else None ) ) _show_return_button() return # Materialize DB/CNPG secrets in Kubernetes using the user-entered password. # This prevents later CNPG initialization from failing due to missing secrets. self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Ensuring database secrets...", fill="blue", ) if self.bg_canvas.winfo_exists() else None ) ) try: self.ensure_db_k8s_secrets( namespace, password, log_fn=lambda m: console.write(f"{m}\n") if m else None, ) except Exception as e: console.write(f"\nFailed to ensure database secrets: {e}\n") self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="Database secrets error", fill="#ff3b30", ) if self.bg_canvas.winfo_exists() else None ) ) _show_return_button() return # Move to next page self.safe_after( lambda: ( self.bg_canvas.itemconfig( status_label, text="OpenBao ready. Proceeding...", fill="#34c759", ) if self.bg_canvas.winfo_exists() else None ) ) self.root.after(1000, lambda: self.show_page("init_scripts")) threading.Thread(target=worker, daemon=True).start()