chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-27 13:24:30 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
ad36f4ee54
prole/knoe/core/ops/cloudnative_pg.py
chrisfu ad36f4ee54 Stage 2: complete CNPG shell-to-Python cutover, remove init_cloudnative_pg.sh 
All active runtime paths that previously shelled out to etc/init_cloudnative_pg.sh
now dispatch through Python. The shell script has been removed from git.

New files:
- knoe/core/ops/cloudnative_pg.py: canonical Python owner for CNPG lifecycle with
  public API (initialize, deploy, rollout, ensure_operator, pin_controller,
  install_barman_plugin) and internal helpers (_apply_manifest, _wait_cnpg_pods,
  _reconcile_instances, etc.)
- prole/tools/run_cnpg_coverage.py: coverage entry points for k3d/k3s modes,
  report sub-command, and check-shell scanner to confirm no live Python dispatch
  to the removed shell script

Modified files:
- knoe/core/actions.py: replace shell dispatch in _step_init_scripts,
  _step_cnpg_deploy, and repair pipeline with Python calls
- knoe/core/milestones.py: replace shell dispatch in InitializationScriptsMilestone
  and DeploymentMilestone
- knoe/ui/screens/services.py: replace shell dispatch for init-scripts step,
  deploy button, and rollout button
- prole/deployment.py: replace shell dispatch in _run_post_apply_scripts
- status.py: remove init_cloudnative_pg.sh from _STATUS_SCRIPTS list
- tests/installer/test_actions_helpers.py: mock Python functions, assert shell
  script is never dispatched
- tests/installer/test_milestones.py: same
- tests/installer/test_services_init_scripts.py: same

Removed:
- etc/init_cloudnative_pg.sh (git rm)

Verification:
- prole.tools.run_cnpg_coverage check-shell reports clean
- All 61 tests in the affected test files pass

Co-authored-by: Junie <junie@jetbrains.com>
2026-03-23 19:28:04 -07:00

600 lines
22 KiB
Python
Raw Blame History

"""
knoe/core/ops/cloudnative_pg.py
Canonical Python owner for CloudNativePG lifecycle behavior.
Replaces etc/init_cloudnative_pg.sh runtime dispatch for the four active
call paths:
initialize -- install operator, apply cluster manifests, wait for readiness
deploy -- image-aware manifest apply + instance reconcile
rollout -- rolling restart via ordered pod deletion / promotion
install_barman_plugin -- install the Barman Cloud operator plugin
Public API
----------
ensure_operator(env, log, mode) -> None
pin_controller(env, log, mode) -> None
install_barman_plugin(env, log) -> None
initialize(namespace, cluster_name, env, project_root, log, mode) -> None
deploy(namespace, cluster_name, env, project_root, log, mode) -> None
rollout(namespace, cluster_name, env, log) -> None
"""
from __future__ import annotations
import json
import logging
import subprocess
import time
from pathlib import Path
from typing import Callable
_LOG = logging.getLogger(__name__)
_LogFn = Callable[[str], None]
_CNPG_OPERATOR_FALLBACK_VERSION = "1.28.1"
_BARMAN_PLUGIN_FALLBACK_VERSION = "0.6.0"
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
def _log(log: _LogFn | None, msg: str) -> None:
if log is not None:
log(msg)
else:
_LOG.info(msg)
def _kubectl(
args: list[str],
env: dict | None = None,
timeout: int = 30,
check: bool = False,
) -> subprocess.CompletedProcess:
return subprocess.run(
["kubectl"] + args,
env=env,
capture_output=True,
text=True,
timeout=timeout,
check=check,
)
def _kubectl_ok(args: list[str], env: dict | None = None, timeout: int = 15) -> bool:
return _kubectl(args, env=env, timeout=timeout).returncode == 0
def _resolve_cnpg_manifest(project_root: str | Path, env: dict | None) -> Path:
"""Resolve the CNPG cluster manifest path, mirroring the shell K8S_PROLE_DIR logic."""
if env:
override = (env.get("CNPG_MANIFEST_OVERRIDE") or "").strip()
if override:
p = Path(override)
if p.exists():
return p
root = Path(project_root)
candidates = [
root / "deploy" / "opentofu" / "k3s" / "manifests" / "prole" / "knoe-db.yaml",
root / "k8s" / "prole" / "knoe-db.yaml",
]
if env:
prole_home = (env.get("PROLE_HOME") or "").strip()
if prole_home:
candidates.insert(0, Path(prole_home) / "k8s" / "prole" / "knoe-db.yaml")
for candidate in candidates:
if candidate.exists():
return candidate
return root / "k8s" / "prole" / "knoe-db.yaml"
def _apply_manifest(
namespace: str,
manifest: Path,
env: dict | None,
log: _LogFn | None,
attempts: int = 8,
) -> None:
"""Apply a CNPG cluster manifest with retry on transient API errors."""
if not manifest.exists():
raise FileNotFoundError(f"CNPG manifest not found: {manifest}")
_log(log, f"Applying CNPG manifest {manifest} in namespace {namespace}...")
last_err = ""
for i in range(1, attempts + 1):
r = _kubectl(["-n", namespace, "apply", "-f", str(manifest)], env=env, timeout=60)
if r.returncode == 0:
if r.stdout.strip():
_log(log, r.stdout.strip())
return
last_err = (r.stderr or r.stdout or "").strip()
transient = any(
kw in last_err.lower()
for kw in ("etcdserver", "connection refused", "timeout", "tls handshake", "webhook")
)
if transient and i < attempts:
_log(log, f"WARN: manifest apply failed (attempt {i}/{attempts}); retrying in 5s...")
time.sleep(5)
continue
break
raise RuntimeError(f"CNPG manifest apply failed after {attempts} attempts: {last_err}")
def _wait_pod_ready(
namespace: str, pod: str, env: dict | None, timeout: int = 300, log: _LogFn | None = None
) -> None:
r = _kubectl(
["-n", namespace, "wait", "--for=condition=Ready", f"pod/{pod}", f"--timeout={timeout}s"],
env=env,
timeout=timeout + 10,
)
if r.returncode != 0:
_log(log, f"WARN: pod {pod} did not become Ready within {timeout}s")
def _wait_cnpg_pods(
namespace: str,
cluster_name: str,
env: dict | None,
timeout: int = 900,
log: _LogFn | None = None,
) -> None:
"""Wait for all CNPG cluster pods to reach Ready state."""
# Determine expected pod count from cluster spec
r = _kubectl(
["-n", namespace, "get", "cluster", cluster_name, "-o", "jsonpath={.spec.instances}"],
env=env,
timeout=15,
)
target = 1
if r.returncode == 0:
try:
target = max(1, int((r.stdout or "1").strip() or "1"))
except ValueError:
target = 1
_log(log, f"Waiting for {target} CNPG pod(s) for cluster '{cluster_name}' in '{namespace}' (timeout={timeout}s)...")
deadline = time.monotonic() + timeout
last_feedback = 0.0
while True:
now = time.monotonic()
if now >= deadline:
raise RuntimeError(
f"Timed out ({timeout}s) waiting for {target} CNPG pods to be Ready "
f"in namespace '{namespace}'"
)
r2 = _kubectl(
[
"-n", namespace, "get", "pods",
"-l", f"cnpg.io/cluster={cluster_name}",
"-o",
"jsonpath={range .items[*]}{.metadata.name}={.status.conditions[?(@.type==\"Ready\")].status}\\n{end}",
],
env=env,
timeout=15,
)
if r2.returncode == 0:
lines = [l for l in (r2.stdout or "").splitlines() if "=" in l]
ready = sum(1 for l in lines if l.split("=", 1)[1].strip() == "True")
if ready >= target:
_log(log, f"All {ready}/{target} CNPG pods are Ready.")
return
elapsed = timeout - (deadline - now)
if elapsed - last_feedback >= 30:
last_feedback = elapsed
_log(log, f" [{int(elapsed)}s/{timeout}s] {ready}/{target} CNPG pods ready...")
time.sleep(5)
def _reconcile_instances(
namespace: str,
cluster_name: str,
env: dict | None,
log: _LogFn | None,
) -> None:
"""
Cap CNPG cluster instances to the number of eligible schedulable nodes,
then patch the cluster spec if the count has changed.
Mirrors reconcile_cnpg_instances() from the shell script.
"""
# Current spec instances
r = _kubectl(
["-n", namespace, "get", "cluster", cluster_name, "-o", "jsonpath={.spec.instances}"],
env=env, timeout=15,
)
if r.returncode != 0 or not (r.stdout or "").strip():
return # cluster may not exist yet
try:
current = max(0, int((r.stdout or "0").strip()))
except ValueError:
return
desired_raw = int((env or {}).get("CNPG_INSTANCES", "3") or "3")
desired = max(1, desired_raw)
# Count eligible nodes (nodes with cnpg.io/node-role=db or all schedulable nodes)
node_selector = (env or {}).get("CNPG_NODE_SELECTOR", "")
if node_selector:
r2 = _kubectl(["get", "nodes", "-l", node_selector, "--no-headers"], env=env, timeout=10)
else:
r2 = _kubectl(["get", "nodes", "--no-headers"], env=env, timeout=10)
eligible = len([l for l in (r2.stdout or "").splitlines() if l.strip()]) if r2.returncode == 0 else 1
eligible = max(1, eligible)
if desired > eligible:
desired = eligible
if current == desired:
return
# Clamp maxSyncReplicas so the spec stays valid (maxSyncReplicas < instances)
base_max_sync = int((env or {}).get("CNPG_MAX_SYNC_REPLICAS", "1") or "1")
if desired <= 1:
max_sync = 0
else:
max_sync = min(base_max_sync, desired - 1)
_log(log, f"Reconciling CNPG instances: {current} → {desired} (eligible_nodes={eligible})")
patch = json.dumps({"spec": {"instances": desired, "maxSyncReplicas": max_sync}})
for attempt in range(1, 9):
r3 = _kubectl(
["-n", namespace, "patch", "cluster", cluster_name, "--type", "merge", "-p", patch],
env=env, timeout=30,
)
if r3.returncode == 0:
_log(log, f"Cluster instances patched to {desired}.")
return
err = (r3.stderr or r3.stdout or "").strip().lower()
transient = any(kw in err for kw in ("etcd", "connection refused", "timeout", "webhook"))
if transient and attempt < 8:
_log(log, f"WARN: patch failed (attempt {attempt}/8); retrying in 5s...")
time.sleep(5)
else:
_log(log, f"WARN: could not patch CNPG instances: {err}")
return
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def ensure_operator(
env: dict | None = None,
log: _LogFn | None = None,
mode: str = "",
) -> None:
"""Install the CloudNativePG operator if not already present, then wait for readiness."""
if _kubectl_ok(["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager"], env=env):
_log(log, "CloudNative-PG operator already installed; waiting for readiness...")
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=180s"], env=env, timeout=190)
if mode == "k3s":
_tune_operator_for_k3s(env, log)
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=300s"], env=env, timeout=310)
return
version = ""
if env:
version = (env.get("CNPG_OPERATOR_VERSION") or env.get("CNPG_VERSION") or "").strip()
if not version or version == "latest":
version = _CNPG_OPERATOR_FALLBACK_VERSION
minor = ".".join(version.split(".")[:2])
url = (
f"https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg"
f"/release-{minor}/releases/cnpg-{version}.yaml"
)
_log(log, f"Installing CloudNative-PG operator version {version}...")
r = _kubectl(["apply", "--server-side", "-f", url], env=env, timeout=120)
if r.returncode != 0:
raise RuntimeError(f"Failed to install CNPG operator: {(r.stderr or r.stdout or '').strip()}")
if _kubectl_ok(["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager"], env=env, timeout=20):
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=180s"], env=env, timeout=190)
if mode == "k3s":
_tune_operator_for_k3s(env, log)
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=300s"], env=env, timeout=310)
def _tune_operator_for_k3s(env: dict | None, log: _LogFn | None) -> None:
"""Relax CNPG controller probes/resources for single-node k3s environments."""
_log(log, "Tuning CNPG operator deployment probes/resources for k3s...")
patch = json.dumps({
"spec": {
"template": {
"spec": {
"containers": [
{
"name": "manager",
"resources": {
"requests": {"cpu": "250m", "memory": "512Mi"},
"limits": {"cpu": "500m", "memory": "1Gi"},
},
"livenessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
"readinessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
"startupProbe": {"timeoutSeconds": 5, "failureThreshold": 60},
}
]
}
}
}
})
_kubectl(
["-n", "cnpg-system", "patch", "deploy", "cnpg-controller-manager", "--type", "merge", "-p", patch],
env=env, timeout=30,
)
def pin_controller(
env: dict | None = None,
log: _LogFn | None = None,
mode: str = "",
) -> None:
"""Pin the CNPG controller manager to storage-designated nodes (k3s) or as configured."""
selector = (env or {}).get("CNPG_CONTROLLER_NODE_SELECTOR", "").strip()
if not selector and mode == "k3s":
r = _kubectl(["get", "nodes", "-l", "storage=primary", "--no-headers"], env=env, timeout=10)
if r.returncode == 0 and (r.stdout or "").strip():
selector = "storage=primary"
else:
r2 = _kubectl(
["get", "nodes", "-l", "node-role.kubernetes.io/control-plane",
"-o", "jsonpath={.items[0].metadata.name}"],
env=env, timeout=10,
)
if r2.returncode == 0 and (r2.stdout or "").strip():
selector = f"kubernetes.io/hostname={r2.stdout.strip()}"
if not selector:
return
if "=" not in selector:
_log(log, f"WARN: CNPG_CONTROLLER_NODE_SELECTOR must be key=value (got '{selector}'). Skipping pin.")
return
key, value = selector.split("=", 1)
if not _kubectl_ok(["get", "nodes", "-l", f"{key}={value}", "--no-headers"], env=env):
_log(log, f"WARN: No nodes match CNPG_CONTROLLER_NODE_SELECTOR={selector}. Skipping pin.")
return
_log(log, f"Pinning cnpg-controller-manager to nodes with {selector}...")
patch = json.dumps({"spec": {"template": {"spec": {"nodeSelector": {key: value}}}}})
_kubectl(
["-n", "cnpg-system", "patch", "deployment", "cnpg-controller-manager",
"--type", "merge", "-p", patch],
env=env, timeout=30,
)
def install_barman_plugin(
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
"""Install the Barman Cloud operator plugin, retrying on cert-manager webhook errors."""
url = ((env or {}).get("BARMAN_PLUGIN_MANIFEST_URL") or "").strip()
if not url:
tag = _resolve_barman_plugin_tag(env, log)
url = f"https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/{tag}/manifest.yaml"
_log(log, f"Installing Barman Cloud plugin from {url}...")
r = subprocess.run(
["kubectl", "apply", "-f", url],
env=env, capture_output=True, text=True, timeout=120,
)
if r.returncode != 0:
combined = (r.stdout or "") + (r.stderr or "")
if "webhook.cert-manager.io" in combined.lower() or "cert-manager" in combined.lower():
_log(log, "WARN: cert-manager webhook error; restarting cert-manager and retrying...")
for deploy in ["cert-manager", "cert-manager-webhook", "cert-manager-cainjector"]:
_kubectl(["-n", "cert-manager", "rollout", "restart", f"deploy/{deploy}"], env=env, timeout=30)
_kubectl(["-n", "cert-manager", "rollout", "status", "deploy/cert-manager-webhook", "--timeout=180s"], env=env, timeout=190)
_kubectl(["-n", "cert-manager", "rollout", "status", "deploy/cert-manager", "--timeout=180s"], env=env, timeout=190)
r2 = subprocess.run(
["kubectl", "apply", "-f", url],
env=env, capture_output=True, text=True, timeout=120,
)
if r2.returncode != 0:
raise RuntimeError(
f"Failed to apply Barman Cloud plugin after cert-manager restart: "
f"{(r2.stderr or r2.stdout or '').strip()}"
)
else:
raise RuntimeError(
f"Failed to apply Barman Cloud plugin manifest: {(r.stderr or r.stdout or '').strip()}"
)
elif r.stdout.strip():
_log(log, r.stdout.strip())
# Wait for barman-cloud deployment if it was created
r3 = _kubectl(["-n", "cnpg-system", "get", "deploy", "barman-cloud"], env=env, timeout=15)
if r3.returncode == 0:
timeout_s = int((env or {}).get("BARMAN_DEPLOY_TIMEOUT", "300") or "300")
_kubectl(
["-n", "cnpg-system", "rollout", "status", "deploy/barman-cloud", f"--timeout={timeout_s}s"],
env=env, timeout=timeout_s + 10,
)
def _resolve_barman_plugin_tag(env: dict | None, log: _LogFn | None) -> str:
"""Fetch the latest Barman Cloud plugin release tag from GitHub, with fallback."""
fallback = ((env or {}).get("BARMAN_PLUGIN_FALLBACK_VERSION") or _BARMAN_PLUGIN_FALLBACK_VERSION).strip()
try:
import urllib.request as _req
import json as _json
with _req.urlopen(
"https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest",
timeout=10,
) as resp:
data = _json.loads(resp.read())
tag = (data.get("tag_name") or "").strip()
if tag and tag != "null":
return tag
except Exception as exc:
_log(log, f"WARN: could not fetch latest Barman plugin version: {exc}; using v{fallback}")
return f"v{fallback}"
def initialize(
namespace: str,
cluster_name: str,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str = "",
) -> None:
"""
Full CNPG initialization pipeline.
Steps (matching init_cloudnative_pg.sh initialize):
1. Ensure the CNPG operator is installed and ready
2. Pin the controller to storage nodes (k3s)
3. Install the Barman Cloud plugin
4. Preflight: verify required secrets exist
5. Apply the CNPG cluster manifest
6. Wait for cluster pods to become Ready
"""
_log(log, f"Initializing CNPG cluster '{cluster_name}' in namespace '{namespace}'...")
ensure_operator(env=env, log=log, mode=mode)
pin_controller(env=env, log=log, mode=mode)
install_barman_plugin(env=env, log=log)
# Preflight: required secrets must exist before applying the cluster
missing = []
for secret in ["knoe-db-user", f"{cluster_name}-tls", f"{cluster_name}-ca"]:
if not _kubectl_ok(["-n", namespace, "get", "secret", secret], env=env):
missing.append(secret)
if missing:
raise RuntimeError(
f"Required secrets missing in namespace '{namespace}': {', '.join(missing)}. "
"The CNPG cluster cannot start without these secrets."
)
_log(log, "Pre-flight check passed: all required secrets present.")
manifest = _resolve_cnpg_manifest(project_root, env)
_apply_manifest(namespace, manifest, env, log)
wait_timeout = int((env or {}).get("CNPG_WAIT_TIMEOUT", "900") or "900")
_wait_cnpg_pods(namespace, cluster_name, env, timeout=wait_timeout, log=log)
_log(log, f"Initialization complete for CNPG cluster '{cluster_name}'.")
def deploy(
namespace: str,
cluster_name: str,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str = "",
) -> None:
"""
Image-aware CNPG deploy pipeline.
Steps (matching init_cloudnative_pg.sh deploy_cluster):
1. Ensure the CNPG operator is installed and ready
2. Pin the controller to storage nodes (k3s)
3. Install the Barman Cloud plugin
4. Apply the CNPG cluster manifest
5. Reconcile instance count to match eligible node count
"""
_log(log, f"Deploying CNPG cluster '{cluster_name}' in namespace '{namespace}'...")
ensure_operator(env=env, log=log, mode=mode)
pin_controller(env=env, log=log, mode=mode)
install_barman_plugin(env=env, log=log)
manifest = _resolve_cnpg_manifest(project_root, env)
_apply_manifest(namespace, manifest, env, log)
_reconcile_instances(namespace, cluster_name, env, log)
_log(log, f"Deploy complete for CNPG cluster '{cluster_name}'.")
def rollout(
namespace: str,
cluster_name: str,
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
"""
Rolling restart of a CNPG cluster via ordered pod deletion.
Steps (matching init_cloudnative_pg.sh rollout_cluster):
1. Identify the current primary pod
2. Delete replica pods one-by-one, waiting for readiness
3. Promote a replica to primary (if kubectl-cnpg plugin is available)
4. Delete and recreate the old primary pod
"""
_log(log, f"Starting rollout for CNPG cluster '{cluster_name}' in namespace '{namespace}'...")
# Identify current primary
r = _kubectl(
["-n", namespace, "get", "cluster", cluster_name, "-o", "jsonpath={.status.currentPrimary}"],
env=env, timeout=15,
)
primary = (r.stdout or "").strip()
if not primary:
raise RuntimeError(f"Could not identify primary instance for cluster '{cluster_name}'")
_log(log, f"Primary instance: {primary}")
# List all cluster pods
r2 = _kubectl(
["-n", namespace, "get", "pods", "-l", f"cnpg.io/cluster={cluster_name}",
"-o", "jsonpath={.items[*].metadata.name}"],
env=env, timeout=15,
)
pods = (r2.stdout or "").strip().split()
if not pods:
raise RuntimeError(f"No pods found for cluster '{cluster_name}'")
has_plugin = _kubectl_ok(["cnpg", "version"], env=env, timeout=10)
# Delete non-primary pods first
new_primary: str | None = None
for pod in pods:
if pod == primary:
continue
_log(log, f"Recreating non-primary pod: {pod}...")
_kubectl(["-n", namespace, "delete", "pod", pod], env=env, timeout=60)
_wait_pod_ready(namespace, pod, env, timeout=300, log=log)
if new_primary is None:
new_primary = pod
# Promote a replica if the cnpg plugin is available
if new_primary and has_plugin:
_log(log, f"Promoting {new_primary}...")
_kubectl(["cnpg", "promote", cluster_name, new_primary, "-n", namespace], env=env, timeout=30)
deadline = time.monotonic() + 300
while time.monotonic() < deadline:
r3 = _kubectl(
["-n", namespace, "get", "cluster", cluster_name,
"-o", "jsonpath={.status.currentPrimary}"],
env=env, timeout=10,
)
if (r3.stdout or "").strip() == new_primary:
_log(log, f"{new_primary} is now the primary.")
break
time.sleep(5)
elif new_primary and not has_plugin:
_log(log, "WARN: kubectl cnpg plugin not available; skipping explicit promotion.")
# Delete the old primary last
_log(log, f"Recreating old primary pod: {primary}...")
_kubectl(["-n", namespace, "delete", "pod", "-n", namespace, primary], env=env, timeout=60)
_wait_pod_ready(namespace, primary, env, timeout=300, log=log)
if has_plugin:
_log(log, "Rollout complete. Final cluster status:")
_kubectl(["cnpg", "status", cluster_name, "-n", namespace], env=env, timeout=30)
else:
_log(log, "Rollout complete.")
Reference in New Issue View Git Blame Copy Permalink