prole/knoe/core/ops/opentofu.py
chrisfu 15890187a3 Stabilize CNPG reset/update flow and finalize 3-node recovery
- add idempotent CNPG operator reconciliation to prevent duplicate active controller ReplicaSets
- restore and validate CNPG 3-instance convergence with PV claimRef recovery and storage path readiness
- wire canonical launcher aliases/reset behavior and improve namespace cleanup semantics
- harden backup/objectstore readiness handling and retry behavior
- enforce service namespace usage for common services and remove default-namespace drift
- enable Kerberos milestone auto-activation when realm/kdc are configured and apply CNPG GSS pg_hba rules
- keep final runtime healthy: knoe-db 3/3, operator stable, perfsnap captured

Co-authored-by: Junie <junie@jetbrains.com>
2026-03-27 21:54:53 -07:00

265 lines
7.4 KiB
Python

from __future__ import annotations
from pathlib import Path
import json
import secrets
import subprocess
from ._services_common import (
_LogFn,
_detect_mode,
_ensure_namespace,
_exists,
_kubectl,
_log,
_manifest_path,
_namespace,
_prune_named_workload_other_namespaces,
_reconcile_deployment_replicasets,
_to_bool,
_wait_rollout,
)
def _opentofu_namespace(namespace: str | None, env: dict | None) -> str:
if env:
explicit = str(env.get("OPENTOFU_NAMESPACE") or "").strip()
if explicit:
return explicit
return _namespace(namespace, env, default="default")
def _manifest(project_root: str | Path) -> Path:
return _manifest_path(project_root, "k8s", "opentofu", "deployment.yaml")
def _build_basic_auth_entry(password: str) -> str:
pwd = (password or "").strip() or "changeme"
try:
res = subprocess.run(
["openssl", "passwd", "-apr1", pwd],
capture_output=True,
text=True,
timeout=10,
)
if res.returncode == 0:
hashed = (res.stdout or "").strip()
if hashed:
return f"admin:{hashed}"
except Exception:
pass
return f"admin:{{PLAIN}}{pwd}"
def _ensure_admin_secret(
*,
namespace: str,
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
force = _to_bool((env or {}).get("PROLE_OPENTOFU_FORCE_SECRET"), default=False)
if not force and _exists("secret", "opentofu-admin", namespace, env=env):
return
admin_password = (
str((env or {}).get("OPENTOFU_ADMIN_PASSWORD") or "").strip()
or str((env or {}).get("DB_PASSWORD") or "").strip()
or secrets.token_urlsafe(18)
)
auth_entry = _build_basic_auth_entry(admin_password)
secret_yaml = (
"apiVersion: v1\n"
"kind: Secret\n"
"metadata:\n"
" name: opentofu-admin\n"
"type: Opaque\n"
"stringData:\n"
f" admin_password: {json.dumps(admin_password)}\n"
f" auth: {json.dumps(auth_entry)}\n"
)
_kubectl(
["-n", namespace, "apply", "-f", "-"],
env=env,
input_text=secret_yaml,
timeout=90,
check=True,
)
_log(log, f"[OPENTOFU] Ensured secret/opentofu-admin in namespace {namespace}")
def _deployment_is_available(
*,
deployment: str,
namespace: str,
env: dict | None = None,
) -> bool:
res = _kubectl(
["-n", namespace, "get", "deployment", deployment, "-o", "json"],
env=env,
timeout=20,
)
if res.returncode != 0 or not res.stdout.strip():
return False
try:
payload = json.loads(res.stdout)
except Exception:
return False
spec = payload.get("spec") or {}
status = payload.get("status") or {}
desired = int(spec.get("replicas") or 1)
available = int(status.get("availableReplicas") or 0)
return desired > 0 and available >= desired
def initialize(
*,
namespace: str | None = None,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str | None = None,
) -> None:
update(namespace=namespace, env=env, project_root=project_root, log=log, mode=mode)
def start(
*,
namespace: str | None = None,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str | None = None,
) -> None:
update(namespace=namespace, env=env, project_root=project_root, log=log, mode=mode)
def update(
*,
namespace: str | None = None,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str | None = None,
) -> None:
_detect_mode(mode, env) # mode kept for API parity
target_ns = _opentofu_namespace(namespace, env)
_ensure_namespace(target_ns, env)
_ensure_admin_secret(namespace=target_ns, env=env, log=log)
_prune_named_workload_other_namespaces(
kind="deployment",
name="opentofu",
target_namespace=target_ns,
label_selector="app=opentofu",
env=env,
log=log,
)
_prune_named_workload_other_namespaces(
kind="service",
name="opentofu",
target_namespace=target_ns,
label_selector="app=opentofu",
env=env,
log=log,
)
force_apply = _to_bool((env or {}).get("PROLE_OPENTOFU_FORCE_APPLY"), default=False)
if not force_apply and _exists("deployment", "opentofu", target_ns, env=env) and _exists(
"service", "opentofu", target_ns, env=env
):
_reconcile_deployment_replicasets(
deployment="opentofu",
namespace=target_ns,
label_selector="app=opentofu",
env=env,
log=log,
)
if _deployment_is_available(deployment="opentofu", namespace=target_ns, env=env):
_log(
log,
f"[OPENTOFU] Healthy deployment already present in namespace {target_ns}; skipping re-apply",
)
return
_log(
log,
f"[OPENTOFU] Existing deployment in namespace {target_ns} is not ready; applying manifest for recovery",
)
if force_apply:
_log(
log,
f"[OPENTOFU] Force apply enabled; reconciling manifest in namespace {target_ns}",
)
manifest = _manifest(project_root)
if not manifest.exists():
raise RuntimeError(f"OpenTofu manifest not found: {manifest}")
_log(log, f"[OPENTOFU] Applying {manifest}")
_kubectl(
["-n", target_ns, "apply", "-f", str(manifest)],
env=env,
timeout=240,
check=True,
)
_reconcile_deployment_replicasets(
deployment="opentofu",
namespace=target_ns,
label_selector="app=opentofu",
env=env,
log=log,
)
_wait_rollout("deployment", "opentofu", target_ns, env=env)
def stop(
*,
namespace: str | None = None,
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
target_ns = _opentofu_namespace(namespace, env)
if _exists("deployment", "opentofu", target_ns, env=env):
_log(log, f"[OPENTOFU] Scaling deployment/opentofu to 0 in namespace {target_ns}")
_kubectl(
["-n", target_ns, "scale", "deployment/opentofu", "--replicas=0"],
env=env,
timeout=90,
check=True,
)
def restart(
*,
namespace: str | None = None,
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
target_ns = _opentofu_namespace(namespace, env)
if _exists("deployment", "opentofu", target_ns, env=env):
_log(log, f"[OPENTOFU] Restarting deployment/opentofu in namespace {target_ns}")
_kubectl(
["-n", target_ns, "rollout", "restart", "deployment/opentofu"],
env=env,
timeout=120,
check=True,
)
_wait_rollout("deployment", "opentofu", target_ns, env=env)
def status(
*,
namespace: str | None = None,
env: dict | None = None,
) -> bool:
target_ns = _opentofu_namespace(namespace, env)
dep = _kubectl(
["-n", target_ns, "get", "deployment", "opentofu", "-o", "jsonpath={.status.readyReplicas}"],
env=env,
timeout=20,
)
svc = _kubectl(["-n", target_ns, "get", "svc", "opentofu"], env=env, timeout=20)
return _to_bool(
dep.returncode == 0 and (dep.stdout or "0").strip() not in {"", "0"} and svc.returncode == 0
)