#!/usr/bin/env bash set -euo pipefail # init_cloudnative_pg.sh # Purpose: # - Distribute administrator ed25519 key pair to CloudNative‑PG as a Kubernetes Secret for cert auth # - Patch CNPG cluster to enable TLS where possible # # Usage: # ./init_cloudnative_pg.sh start|stop|status|restart # ./init_cloudnative_pg.sh initialize # install CNPG, create cluster, configure secrets # ./init_cloudnative_pg.sh update|reload # re-apply/patch # ./init_cloudnative_pg.sh deploy [version] # apply CNPG manifest and update image # ./init_cloudnative_pg.sh rollout # rolling restart of CNPG pods # # Requirements: # - init_openbao.sh has been run (OpenBao running as a local container) # - $PROLE_HOME/env.sh or $HOME/.prole/env.sh defining PROLE_SERVICE # - Optional: CNPG_MANIFEST_OVERRIDE to apply a recovery manifest instead of kustomize # Initialize SCRIPT_DIR SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Load environment and config via prole_cfg.sh # shellcheck disable=SC1090 source "$SCRIPT_DIR/prole_cfg.sh" if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then prole_set_mode "${2:-}" shift 2 elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then prole_set_mode "${1#*=}" shift fi if [[ -z "${PROLE_SERVICE:-}" ]]; then echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2 exit 1 fi ACTION=${1:-} CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-prole-db} VERSION=${2:-latest} OPENBAO_NAME=${OPENBAO_NAME:-openbao} REALM=${REALM:-PROLE.ORG} DOMAIN=${DOMAIN:-prole.org} CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-} PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup} BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800} RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl" BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-} BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0} CERT_MANAGER_MANIFEST_URL=${CERT_MANAGER_MANIFEST_URL:-} CERT_MANAGER_FALLBACK_VERSION=${CERT_MANAGER_FALLBACK_VERSION:-1.19.3} if [[ "$ACTION" != "deploy" && "$ACTION" != "rollout" && "$ACTION" != "force-rollout" ]]; then if [[ -n "${2:-}" ]]; then CNPG_CLUSTER_NAME="${2}" fi VERSION=${3:-latest} fi # Support both PROLE_HOME/k8s and sibling k8s directory if [[ -d "$SCRIPT_DIR/../k8s/prole" ]]; then K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole" elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/prole" ]]; then K8S_PROLE_DIR="$PROLE_HOME/k8s/prole" else K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole" fi CNPG_MANIFEST="$K8S_PROLE_DIR/prole-db.yaml" BARMAN_OBJECTSTORE_MANIFEST="$K8S_PROLE_DIR/prole-db-barman-objectstore.yaml" SECRETS_DIR="$PROLE_SERVICE/secrets" # Resolving CNPG admin keys. # We prefer names without algorithm suffixes to be more generic, matching install.py fallback strategy. ADMIN_PRIV_ED25519="$SECRETS_DIR/admin_ed25519.key" ADMIN_PUB_ED25519="$SECRETS_DIR/admin_ed25519.pub" ADMIN_PRIV_GENERIC="$SECRETS_DIR/admin.key" ADMIN_PUB_GENERIC="$SECRETS_DIR/admin.pub" OPENBAO_TOKEN_FILE="$SECRETS_DIR/openbao-root-token" BAO_NAMESPACE="${NAMESPACE:-default}" OPENBAO_NAMESPACE="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}" BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}" BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin" BAO_PATH_DB="${BAO_PATH_PREFIX}/db" BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring" ensure_tools() { for t in kubectl curl openssl base64 jq; do command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; } done } ensure_namespace() { if [[ -z "${NAMESPACE:-}" ]]; then echo "ERROR: NAMESPACE is empty. Check env.sh or conf/prole.cfg." >&2 exit 1 fi if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then echo "Creating namespace '$NAMESPACE' ..." kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true fi } get_latest_image() { local pg_version_file release_file pg_version release if [[ -f "$SCRIPT_DIR/../conf/postgresql/.version" ]]; then pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/conf/postgresql/.version" ]]; then pg_version_file="$PROLE_HOME/conf/postgresql/.version" else pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" fi if [[ -f "$SCRIPT_DIR/../prole-db/.version" ]]; then release_file="$SCRIPT_DIR/../prole-db/.version" elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/prole-db/.version" ]]; then release_file="$PROLE_HOME/prole-db/.version" else release_file="$SCRIPT_DIR/../prole-db/.version" fi if [[ -f "$pg_version_file" ]]; then pg_version=$(tr -d '[:space:]' < "$pg_version_file") else pg_version="17.7" fi if [[ -f "$release_file" ]]; then release=$(tr -d '[:space:]' < "$release_file") else release="43" fi if [[ "$release" =~ ^[0-9]+$ ]]; then release=$(printf "%03d" "$release") fi echo "prole-db:${pg_version}-${release}" } resolve_cnpg_image() { local image="$1" if _prole_local_registry_enabled; then local registry="${LOCAL_REGISTRY:-}" if [[ -z "$registry" ]]; then registry="${LOCAL_REGISTRY_INTERNAL:-}" fi if [[ -n "$registry" ]]; then local first="${image%%/*}" if [[ "$image" != */* ]]; then image="${registry}/${image}" elif [[ "$first" != *"."* && "$first" != *":"* && "$first" != "localhost" ]]; then image="${registry}/${image}" fi fi fi printf '%s' "$image" } sync_manifest_image() { local image="$1" local files=() if [[ -n "$CNPG_MANIFEST" ]]; then files+=("$CNPG_MANIFEST") fi if [[ -f "$RECOVERY_TEMPLATE" ]]; then files+=("$RECOVERY_TEMPLATE") fi local f tmp for f in "${files[@]}"; do if [[ -f "$f" ]] && grep -qE '^[[:space:]]*imageName:' "$f"; then tmp=$(mktemp) sed -E "s|^([[:space:]]*imageName:).*|\\1 ${image}|" "$f" > "$tmp" mv "$tmp" "$f" fi done } openbao_url() { if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then echo "$PROLE_OPENBAO_URL" return 0 fi if prole_is_in_cluster; then echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200" return 0 fi if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then echo "http://127.0.0.1:8200" return 0 elif curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then echo "http://127.0.0.1:18200" return 0 else echo "" return 0 fi } openbao_token() { if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then cat "$OPENBAO_TOKEN_FILE" else echo "${OPENBAO_ROOT_TOKEN:-}" fi } fetch_openbao_secret() { local path="$1" local key="$2" local token url token=$(openbao_token) url=$(openbao_url) if [[ -z "$token" || -z "$url" ]]; then echo "" return 0 fi curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo "" } resolve_db_password() { local db_pw="${DB_PASSWORD:-}" if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then local fetched_db fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password") if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then db_pw="$fetched_db" fi fi printf '%s' "$db_pw" } ensure_grafana_admin_secret() { local db_pw db_pw=$(resolve_db_password) if [[ -z "$db_pw" ]]; then echo "ERROR: DB_PASSWORD unavailable; cannot create Grafana admin secret." >&2 return 1 fi kubectl create secret generic grafana-admin \ -n "$NAMESPACE" \ --from-literal=admin-password="$db_pw" \ --dry-run=client -o yaml | kubectl apply -f - >/dev/null local token url token=$(openbao_token) url=$(openbao_url) if [[ -n "$token" && -n "$url" ]]; then curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \ -X POST "$url/v1/kv/data/$BAO_PATH_MONITORING" \ -d "{\"data\":{\"grafana_admin_password\":\"$db_pw\"}}" >/dev/null || true fi } ensure_cnpg_operator() { if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true wait_for_cnpg_webhook 180 || true return 0 fi local latest_version minor_version yaml_url latest_version=$(get_latest_cnpg_version) minor_version=$(echo "$latest_version" | cut -d. -f1,2) yaml_url="https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${minor_version}/releases/cnpg-${latest_version}.yaml" echo "Installing CloudNative-PG operator version ${latest_version} ..." kubectl apply --server-side -f "$yaml_url" if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true wait_for_cnpg_webhook 180 || true fi } get_latest_barman_plugin_version() { local version tag tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "") if [[ -z "$tag" || "$tag" == "null" ]]; then echo "v${BARMAN_PLUGIN_FALLBACK_VERSION}" return 0 fi echo "$tag" } resolve_barman_plugin_manifest_url() { if [[ -n "$BARMAN_PLUGIN_MANIFEST_URL" ]]; then echo "$BARMAN_PLUGIN_MANIFEST_URL" return 0 fi local tag tag=$(get_latest_barman_plugin_version) echo "https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/${tag}/manifest.yaml" } cert_manager_ready() { if ! kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then return 1 fi if ! kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then return 1 fi return 0 } resolve_control_plane_selector() { local node="" node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "$node" ]]; then printf 'kubernetes.io/hostname=%s' "$node" fi } pin_cert_manager() { local selector="${CERT_MANAGER_NODE_SELECTOR:-}" if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then selector="$(resolve_control_plane_selector)" fi if [[ -z "$selector" ]]; then return 0 fi local key value key="${selector%%=*}" value="${selector#*=}" if [[ -z "$key" || -z "$value" ]]; then echo "WARN: CERT_MANAGER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 return 0 fi if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then echo "WARN: No nodes match CERT_MANAGER_NODE_SELECTOR=${selector}. Skipping pin." >&2 return 0 fi echo "Pinning cert-manager deployments to nodes with ${selector} ..." for dep in cert-manager cert-manager-webhook cert-manager-cainjector; do kubectl -n cert-manager patch deployment "$dep" --type merge \ -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true done kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true } pin_barman_cloud() { local selector="${BARMAN_NODE_SELECTOR:-}" if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then selector="$(resolve_control_plane_selector)" fi if [[ -z "$selector" ]]; then return 0 fi local key value key="${selector%%=*}" value="${selector#*=}" if [[ -z "$key" || -z "$value" ]]; then echo "WARN: BARMAN_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 return 0 fi if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then echo "WARN: No nodes match BARMAN_NODE_SELECTOR=${selector}. Skipping pin." >&2 return 0 fi echo "Pinning barman-cloud deployment to nodes with ${selector} ..." kubectl -n cnpg-system patch deployment barman-cloud --type merge \ -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true kubectl -n cnpg-system rollout restart deploy/barman-cloud >/dev/null 2>&1 || true } get_latest_cert_manager_version() { local tag tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cert-manager/cert-manager/releases/latest" | jq -r '.tag_name' || echo "") if [[ -z "$tag" || "$tag" == "null" ]]; then echo "v${CERT_MANAGER_FALLBACK_VERSION}" return 0 fi echo "$tag" } resolve_cert_manager_manifest_url() { if [[ -n "$CERT_MANAGER_MANIFEST_URL" ]]; then echo "$CERT_MANAGER_MANIFEST_URL" return 0 fi local tag tag=$(get_latest_cert_manager_version) echo "https://github.com/cert-manager/cert-manager/releases/download/${tag}/cert-manager.yaml" } ensure_cert_manager() { if cert_manager_ready; then pin_cert_manager if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true fi return 0 fi local cm_url cm_url=$(resolve_cert_manager_manifest_url) echo "Installing cert-manager from ${cm_url} ..." kubectl apply -f "$cm_url" pin_cert_manager if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true fi } wait_for_barman_crd() { local timeout=${1:-120} local start_time start_time=$(date +%s) while true; do if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then return 0 fi if (( $(date +%s) - start_time > timeout )); then return 1 fi sleep 5 done } wait_for_barman_tls_secrets() { local timeout=${1:-180} local start_time start_time=$(date +%s) while true; do if kubectl -n cnpg-system get secret barman-cloud-client-tls >/dev/null 2>&1 \ && kubectl -n cnpg-system get secret barman-cloud-server-tls >/dev/null 2>&1; then return 0 fi if (( $(date +%s) - start_time > timeout )); then return 1 fi sleep 5 done } ensure_barman_plugin() { ensure_cert_manager local plugin_url plugin_url=$(resolve_barman_plugin_manifest_url) echo "Installing Barman Cloud plugin from ${plugin_url} ..." local apply_out="" if ! apply_out=$(kubectl apply -f "$plugin_url" 2>&1); then echo "$apply_out" >&2 if echo "$apply_out" | grep -qi "webhook.cert-manager.io"; then echo "WARN: cert-manager webhook error detected; restarting cert-manager components and retrying..." >&2 kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s >/dev/null 2>&1 || true kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s >/dev/null 2>&1 || true kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s >/dev/null 2>&1 || true kubectl apply -f "$plugin_url" || true fi else printf '%s\n' "$apply_out" fi if ! wait_for_barman_crd 120; then echo "WARN: Barman Cloud ObjectStore CRD not ready after install." >&2 fi if ! wait_for_barman_tls_secrets 180; then echo "WARN: Barman Cloud TLS secrets not ready after install." >&2 fi pin_barman_cloud if kubectl -n cnpg-system get deploy barman-cloud >/dev/null 2>&1; then kubectl -n cnpg-system rollout status deploy/barman-cloud --timeout=180s || true fi } pin_cnpg_controller() { local selector="${CNPG_CONTROLLER_NODE_SELECTOR:-}" if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then if kubectl get nodes -l "storage=primary" >/dev/null 2>&1; then selector="storage=primary" else local cp_node="" cp_node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "$cp_node" ]]; then selector="kubernetes.io/hostname=${cp_node}" fi fi fi if [[ -z "$selector" ]]; then return 0 fi local key value key="${selector%%=*}" value="${selector#*=}" if [[ -z "$key" || -z "$value" ]]; then echo "WARN: CNPG_CONTROLLER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 return 0 fi if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then echo "WARN: No nodes match CNPG_CONTROLLER_NODE_SELECTOR=${selector}. Skipping pin." >&2 return 0 fi echo "Pinning cnpg-controller-manager to nodes with ${selector} ..." kubectl -n cnpg-system patch deployment cnpg-controller-manager --type merge \ -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true } wait_for_cnpg_webhook() { local timeout=${1:-120} local start_time start_time=$(date +%s) echo "Waiting for CNPG webhook service endpoints to be ready..." while true; do local endpoints endpoints=$(kubectl -n cnpg-system get endpoints cnpg-webhook-service -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true) if [[ -n "$endpoints" ]]; then echo "CNPG webhook service has endpoints." return 0 fi if (( $(date +%s) - start_time > timeout )); then echo "WARN: CNPG webhook endpoints not ready after ${timeout}s." return 1 fi sleep 5 done } apply_barman_objectstore_if_present() { if [[ ! -f "$BARMAN_OBJECTSTORE_MANIFEST" ]]; then return 0 fi if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then local endpoint endpoint="http://garage.${SERVICE_NAMESPACE}.svc.cluster.local:3900" prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" \ | sed -E "s|^[[:space:]]*endpointURL:.*| endpointURL: ${endpoint}|" \ | kubectl apply -n "$NAMESPACE" -f - else prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" | kubectl apply -n "$NAMESPACE" -f - fi else echo "WARN: Barman Cloud ObjectStore CRD not found; skipping $BARMAN_OBJECTSTORE_MANIFEST." fi } apply_prole_manifest_file() { local file="$1" local output="" if output=$(prole_render_manifest "$file" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then printf '%s\n' "$output" return 0 fi if [[ "${PROLE_MODE:-}" == "k3d" && "$(basename "$file")" == "garage-statefulset.yaml" ]] \ && echo "$output" | grep -q "updates to statefulset spec"; then echo "WARN: Garage StatefulSet immutable in k3d; skipping apply." return 0 fi echo "$output" >&2 return 1 } ensure_prole_stack_resources() { echo "Applying CloudNative-PG cluster and related resources ..." ensure_grafana_admin_secret local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}" local image="" if [[ -n "$image_override" ]]; then image="$image_override" else if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then image=$(get_latest_image) else image="prole-db:$VERSION" fi fi image=$(resolve_cnpg_image "$image") sync_manifest_image "$image" if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2 return 1 fi local dir file dir="$K8S_PROLE_DIR" for file in "$dir"/*.yaml; do case "$(basename "$file")" in prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml) continue ;; openbao-statefulset.yaml|openbao-service.yaml) if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then continue fi ;; garage-*.yaml|grafana-*.yaml|prometheus-*.yaml) if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then continue fi ;; esac apply_prole_manifest_file "$file" done apply_barman_objectstore_if_present apply_cnpg_cluster_manifest "$CNPG_MANIFEST_OVERRIDE" else local dir file dir="$K8S_PROLE_DIR" for file in "$dir"/*.yaml; do case "$(basename "$file")" in prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml) continue ;; openbao-statefulset.yaml|openbao-service.yaml) if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then continue fi ;; garage-*.yaml|grafana-*.yaml|prometheus-*.yaml) if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then continue fi ;; esac apply_prole_manifest_file "$file" done apply_barman_objectstore_if_present apply_cnpg_cluster_manifest "$CNPG_MANIFEST" fi # Ensure prole-index-html exists for prole deployment readiness probe if ! kubectl get configmap prole-index-html -n "$NAMESPACE" >/dev/null 2>&1; then echo "Creating prole-index-html configmap..." printf "

Prole

" > /tmp/index.html kubectl create configmap prole-index-html --from-file=/tmp/index.html -n "$NAMESPACE" rm /tmp/index.html fi # Ensure prole-nginx-tls exists (self-signed for dev) if ! kubectl get secret prole-nginx-tls -n "$NAMESPACE" >/dev/null 2>&1; then echo "Generating self-signed prole-nginx-tls for development..." openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout /tmp/nginx-tls.key -out /tmp/nginx-tls.crt \ -subj "/CN=prole.org" >/dev/null 2>&1 kubectl create secret tls prole-nginx-tls --key /tmp/nginx-tls.key --cert /tmp/nginx-tls.crt -n "$NAMESPACE" rm /tmp/nginx-tls.key /tmp/nginx-tls.crt fi } apply_cnpg_cluster_manifest() { local manifest="$1" local attempts=${CNPG_APPLY_RETRIES:-6} local i out for ((i=1; i<=attempts; i++)); do if out=$(prole_render_manifest "$manifest" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then printf '%s\n' "$out" return 0 fi if echo "$out" | grep -q "cnpg-webhook-service"; then echo "CNPG webhook not ready yet (attempt $i/$attempts). Retrying..." sleep 5 continue fi echo "$out" >&2 return 1 done echo "ERROR: Failed to apply CNPG manifest after $attempts attempts." >&2 echo "$out" >&2 return 1 } wait_for_cnpg_pods() { local timeout=${1:-${CNPG_WAIT_TIMEOUT:-900}} local start_time start_time=$(date +%s) local target_pods="${CNPG_TARGET_PODS:-}" if [[ -z "$target_pods" ]]; then target_pods=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.instances}' 2>/dev/null || true) fi if [[ -z "$target_pods" ]]; then target_pods=3 fi echo "Waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' to be Running..." while true; do local pods pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null || true) if [[ -n "$pods" ]]; then # Check for Error or CrashLoopBackOff if echo "$pods" | grep -E "Error|CrashLoopBackOff" >/dev/null; then echo "ERROR: Some CNPG pods are in Error or CrashLoopBackOff state:" >&2 echo "$pods" | grep -E "Error|CrashLoopBackOff" >&2 return 1 fi # Count Running pods by name (exclude initdb) local running_pods running_pods=$(kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null \ | grep "^${CNPG_CLUSTER_NAME}-" \ | grep -v "initdb" \ | grep "Running" \ | wc -l | xargs) if [[ "$running_pods" -ge "$target_pods" ]]; then echo "All $running_pods/$target_pods pods are Running." return 0 fi fi if (( $(date +%s) - start_time > timeout )); then echo "ERROR: Timed out waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE'." >&2 return 1 fi sleep 5 done } cluster_exists() { kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1 } cluster_has_pods() { kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null | grep -q . } latest_backup_name() { kubectl -n "$NAMESPACE" get backup \ --sort-by=.metadata.creationTimestamp \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 } latest_completed_backup_name() { kubectl -n "$NAMESPACE" get backup \ --sort-by=.metadata.creationTimestamp \ -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null | \ awk -F'|' '{p=tolower($2); if (p=="completed" || p=="succeeded") {name=$1}} END {print name}' } wait_for_backup() { local backup_name="$1" local start_time now phase phase_lc start_time=$(date +%s) while true; do phase=$(kubectl -n "$NAMESPACE" get backup "$backup_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) phase_lc=$(printf '%s' "$phase" | tr '[:upper:]' '[:lower:]') case "$phase_lc" in completed|succeeded) echo "Backup $backup_name completed." return 0 ;; failed|error) echo "Backup $backup_name failed (phase=$phase)." >&2 return 1 ;; esac now=$(date +%s) if (( now - start_time > BACKUP_WAIT_TIMEOUT )); then echo "Timed out waiting for backup $backup_name." >&2 return 1 fi echo "Waiting for backup $backup_name to complete (phase=${phase:-unknown}) ..." sleep 10 done } run_garage_backup() { if [[ ! -x "$SCRIPT_DIR/init_prole-db-backup.sh" ]]; then echo "WARN: init_prole-db-backup.sh not found; skipping Garage backup." >&2 return 1 fi echo "Running Garage backup via init_prole-db-backup.sh ..." if ! "$SCRIPT_DIR/init_prole-db-backup.sh" start; then echo "Garage backup script failed." >&2 return 1 fi sleep 2 local backup_name backup_name=$(latest_backup_name) if [[ -z "$backup_name" ]]; then echo "No backup resource detected after triggering backup." >&2 return 1 fi wait_for_backup "$backup_name" } get_primary_pod() { local primary primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) if [[ -z "$primary" ]]; then primary=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) fi printf '%s' "$primary" } decode_b64() { local data="$1" if [[ -z "$data" ]]; then return 1 fi printf '%s' "$data" | base64 -d 2>/dev/null } resolve_db_credentials() { local user_b64 pass_b64 user pass user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.username}' 2>/dev/null || true) pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.password}' 2>/dev/null || true) user=$(decode_b64 "$user_b64" || true) pass=$(decode_b64 "$pass_b64" || true) if [[ -z "$user" || -z "$pass" ]]; then user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.username}' 2>/dev/null || true) pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.password}' 2>/dev/null || true) user=$(decode_b64 "$user_b64" || true) pass=$(decode_b64 "$pass_b64" || true) fi if [[ -z "$user" || -z "$pass" ]]; then return 1 fi printf '%s\n%s' "$user" "$pass" } pgdump_local() { local pod user pass db_name dump_dir dump_file timestamp pod=$(get_primary_pod) if [[ -z "$pod" ]]; then echo "ERROR: No CNPG pod available for pg_dump." >&2 return 1 fi local creds if ! creds=$(resolve_db_credentials); then echo "ERROR: Unable to resolve database credentials for pg_dump." >&2 return 1 fi user=$(printf '%s' "$creds" | sed -n '1p') pass=$(printf '%s' "$creds" | sed -n '2p') db_name=${PROLE_DB_NAME:-prole-db} dump_dir="$BACKUP_DIR" mkdir -p "$dump_dir" timestamp=$(date +%Y%m%d%H%M%S) dump_file="$dump_dir/${CNPG_CLUSTER_NAME}-pgdump-${timestamp}.dump" echo "Running pg_dump against pod $pod (db=$db_name) ..." if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- env PGPASSWORD="$pass" \ pg_dump -U "$user" -d "$db_name" -Fc > "$dump_file"; then echo "pg_dump saved to $dump_file" return 0 fi echo "pg_dump failed; removing partial file." >&2 rm -f "$dump_file" return 1 } attempt_backup_if_active() { if ! cluster_exists; then return 0 fi if ! cluster_has_pods; then echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no pods detected; skipping backup." >&2 return 0 fi echo "CNPG cluster '$CNPG_CLUSTER_NAME' detected; attempting Garage backup ..." if run_garage_backup; then return 0 fi echo "Garage backup failed; attempting local pg_dump ..." >&2 if pgdump_local; then return 0 fi echo "WARN: Both Garage backup and local pg_dump failed." >&2 return 0 } wait_for_pod_ready() { local pod="$1" if ! kubectl -n "$NAMESPACE" wait --for=condition=Ready pod "$pod" --timeout=300s >/dev/null 2>&1; then echo "WARN: pod $pod did not become Ready within timeout." >&2 return 1 fi return 0 } rollout_cluster() { ensure_tools ensure_namespace echo "Starting manual recreate rollout for $CNPG_CLUSTER_NAME in $NAMESPACE..." local has_cnpg_plugin="0" if kubectl cnpg version >/dev/null 2>&1; then has_cnpg_plugin="1" kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true fi local primary primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) if [[ -z "$primary" ]]; then echo "ERROR: Could not identify primary instance." >&2 return 1 fi echo "Primary instance: $primary" local instances instances=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true) if [[ -z "$instances" ]]; then echo "ERROR: No pods found for cluster $CNPG_CLUSTER_NAME." >&2 return 1 fi local pod for pod in $instances; do if [[ "$pod" != "$primary" ]]; then echo "Recreating non-primary pod: $pod..." kubectl delete pod -n "$NAMESPACE" "$pod" wait_for_pod_ready "$pod" || true if [[ "$has_cnpg_plugin" == "1" ]]; then kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true fi fi done local new_primary="" for pod in $instances; do if [[ "$pod" != "$primary" ]]; then new_primary="$pod" break fi done if [[ -n "$new_primary" && "$has_cnpg_plugin" == "1" ]]; then echo "Promoting $new_primary..." kubectl cnpg promote "$CNPG_CLUSTER_NAME" "$new_primary" -n "$NAMESPACE" || true echo "Waiting for $new_primary to become primary..." for _ in {1..60}; do local current_primary current_primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) if [[ "$current_primary" == "$new_primary" ]]; then echo "$new_primary is now the primary." break fi sleep 5 done else if [[ -n "$new_primary" ]]; then echo "WARN: kubectl cnpg plugin not available; skipping explicit promotion." fi fi echo "Recreating the old primary pod: $primary..." kubectl delete pod -n "$NAMESPACE" "$primary" wait_for_pod_ready "$primary" || true if [[ "$has_cnpg_plugin" == "1" ]]; then echo "Rollout complete. Final cluster status:" kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true fi } force_rollout() { echo "Attempting force rollout of CNPG pods ..." rollout_cluster } reset_and_reinit() { local backup_name tmp_manifest backup_name=$(latest_completed_backup_name) echo "Resetting CNPG cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' ..." kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then tmp_manifest=$(mktemp) sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest" echo "Re-initializing from backup $backup_name ..." CNPG_MANIFEST_OVERRIDE="$tmp_manifest" ensure_prole_stack_resources rm -f "$tmp_manifest" else if [[ -n "$backup_name" && ! -f "$RECOVERY_TEMPLATE" ]]; then echo "WARN: Recovery template not found: $RECOVERY_TEMPLATE" >&2 fi if [[ -z "$backup_name" ]]; then echo "No completed backups found; starting fresh initialization." >&2 fi ensure_prole_stack_resources fi wait_for_cnpg_pods 300 } # Resolve OpenBao URL: prefer explicit env, then in-cluster service, then localhost port-forward bao_service_url() { if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then echo "$PROLE_OPENBAO_URL" return 0 fi if prole_is_in_cluster; then echo "http://$OPENBAO_NAME.${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}.svc.cluster.local:8200" return 0 fi if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then echo "http://127.0.0.1:8200" return 0 fi # Prefer standard local port-forward managed by etc/init_port_forwards.sh if curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then echo "http://127.0.0.1:18200" return 0 fi echo "" } fetch_admin_keys_and_db_pass_from_bao_or_local() { local token url if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then token=$(cat "$OPENBAO_TOKEN_FILE") else token="" fi url=$(bao_service_url) if [[ -n "$token" && -n "$url" ]]; then echo "Attempting to read admin key pair from OpenBao kv/$BAO_PATH_ADMIN ..." if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -e '.data.data' >/dev/null 2>&1; then local priv_b64 pub_b64 priv_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_private_key_b64') pub_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_public_key_b64') # Use a temporary file to determine where to save based on existing legacy or generic preference local target_priv="$ADMIN_PRIV_GENERIC" local target_pub="$ADMIN_PUB_GENERIC" # If legacy keys exist, we might want to overwrite them too for compatibility printf "%s" "$priv_b64" | base64 -d >"$target_priv" printf "%s" "$pub_b64" | base64 -d >"$target_pub" chmod 0600 "$target_priv" # Mirror to legacy path if it was expected by other scripts cp "$target_priv" "$ADMIN_PRIV_ED25519" 2>/dev/null || true cp "$target_pub" "$ADMIN_PUB_ED25519" 2>/dev/null || true fi echo "Attempting to read database password from OpenBao kv/$BAO_PATH_DB ..." if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -e '.data.data' >/dev/null 2>&1; then local db_pass db_pass=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -r '.data.data.password') if [[ -n "$db_pass" ]]; then echo "Updating database user secret 'prole-db-user' from OpenBao ..." kubectl create secret generic prole-db-user -n "$NAMESPACE" \ --from-literal=username=prole \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - echo "Updating database superuser secret 'prole-db-superuser' from OpenBao ..." kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \ --from-literal=username=postgres \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - fi fi fi # Fallback: use DB_PASSWORD from env if OpenBao is unreachable or empty if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then local db_pass="${DB_PASSWORD:-}" if [[ -n "$db_pass" && "$db_pass" != '${OPENBAO:'* && "$db_pass" != '${PROLE_SECRET:'* ]]; then echo "Updating database user secret 'prole-db-user' from DB_PASSWORD env ..." kubectl create secret generic prole-db-user -n "$NAMESPACE" \ --from-literal=username=prole \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - echo "Updating database superuser secret 'prole-db-superuser' from DB_PASSWORD env ..." kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \ --from-literal=username=postgres \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - fi fi if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then echo "Using local admin key pair at $SECRETS_DIR" return 0 fi if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then echo "Using local legacy admin key pair at $SECRETS_DIR" # Link or copy to generic for consistent use below cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC" cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC" return 0 fi echo "ERROR: Could not obtain admin key pair from OpenBao and no local files found." >&2 exit 1 } apply_cnpg_admin_secret() { echo "Creating/updating Secret cnpg-admin-key ..." kubectl create secret generic cnpg-admin-key -n "$NAMESPACE" \ --from-file=admin.key="$ADMIN_PRIV_GENERIC" \ --from-file=admin.pub="$ADMIN_PUB_GENERIC" \ --dry-run=client -o yaml | kubectl apply -f - } generate_tls_if_missing() { local ca_secret_name="${CNPG_CLUSTER_NAME}-ca" if kubectl -n "$NAMESPACE" get secret "$ca_secret_name" >/dev/null 2>&1; then echo "CA secret $ca_secret_name already exists; skipping generation." return 0 fi echo "Generating self-signed CA (RSA 4096) for CNPG ..." local TMPD TMPD=$(mktemp -d) openssl genrsa -out "$TMPD/ca.key" 4096 openssl req -x509 -new -key "$TMPD/ca.key" -out "$TMPD/ca.crt" -days 3650 -subj "/CN=Prole CNPG CA" kubectl -n "$NAMESPACE" create secret generic "$ca_secret_name" \ --from-file=ca.crt="$TMPD/ca.crt" \ --from-file=ca.key="$TMPD/ca.key" \ --dry-run=client -o yaml | kubectl apply -f - rm -rf "$TMPD" } # Resolve latest CNPG version from GitHub if possible, fallback to a sensible default. get_latest_cnpg_version() { local version version=$(curl -s "https://api.github.com/repos/cloudnative-pg/cloudnative-pg/releases/latest" | jq -r '.tag_name' | sed 's/^v//' || echo "") if [[ -z "$version" || "$version" == "null" ]]; then echo "1.27.0" else echo "$version" fi } initialize() { ensure_tools ensure_namespace attempt_backup_if_active ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin echo "Using namespace: $NAMESPACE" fetch_admin_keys_and_db_pass_from_bao_or_local apply_cnpg_admin_secret ensure_prole_stack_resources if ! wait_for_cnpg_pods 300; then echo "WARN: CNPG pods did not become ready after init; attempting force rollout ..." >&2 if force_rollout; then if ! wait_for_cnpg_pods 300; then echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2 if ! reset_and_reinit; then return 1 fi fi else echo "WARN: Force rollout failed; attempting reset and re-init ..." >&2 if ! reset_and_reinit; then return 1 fi fi fi echo "Initialization complete for CNPG + cert artifacts." prole_register_port_forward "postgres" "${NAMESPACE:-default}" "svc/${CNPG_CLUSTER_NAME}-rw" "5432" "5432" "0.0.0.0" "TCP" "PostgreSQL" } update_reload() { initialize } deploy_cluster() { ensure_tools ensure_namespace ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin local image if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then image=$(get_latest_image) else image="prole-db:$VERSION" fi image=$(resolve_cnpg_image "$image") sync_manifest_image "$image" echo "Deploying $image to cluster $CNPG_CLUSTER_NAME in namespace $NAMESPACE..." apply_cnpg_cluster_manifest "$CNPG_MANIFEST" local current_image current_image=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "") if [[ -n "$current_image" && "$current_image" != "$image" ]]; then echo "Patching cluster to use image '$image' (was '$current_image')..." kubectl -n "$NAMESPACE" patch cluster "$CNPG_CLUSTER_NAME" --type merge -p "{\"spec\": {\"imageName\": \"$image\"}}" if kubectl cnpg version >/dev/null 2>&1; then kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true fi fi } case "$ACTION" in recreate) ensure_tools "$0" delete "$CNPG_CLUSTER_NAME" "$0" create "$CNPG_CLUSTER_NAME" ;; create) ensure_tools ensure_namespace initialize ;; delete) ensure_tools echo "Deleting all resources for '$CNPG_CLUSTER_NAME' ..." kubectl delete -n "$NAMESPACE" -k "$SCRIPT_DIR/../k8s/prole" --ignore-not-found ;; start) ensure_tools ensure_namespace ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin if [[ ! -f "$CNPG_MANIFEST" ]]; then echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2 exit 1 fi echo "Starting CloudNative-PG cluster from $CNPG_MANIFEST in namespace $NAMESPACE..." prole_render_manifest "$CNPG_MANIFEST" | kubectl apply -n "$NAMESPACE" -f - ;; stop) ensure_tools if [[ ! -f "$CNPG_MANIFEST" ]]; then echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2 exit 1 fi echo "Stopping CloudNative-PG cluster using $CNPG_MANIFEST ..." kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found ;; status) ensure_tools echo "--- CloudNative-PG Cluster Status ($CNPG_CLUSTER_NAME) ---" if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" echo "" echo "CNPG Plugin Status:" if kubectl cnpg version >/dev/null 2>&1; then kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" else echo "Note: 'kubectl cnpg' plugin not found; skipping detailed status." fi else echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'." fi ;; restart) ensure_tools "$0" stop "$0" start ;; initialize) if ! initialize; then exit 1 fi ;; update|reload) if ! update_reload; then exit 1 fi ;; deploy) deploy_cluster ;; rollout|force-rollout) rollout_cluster ;; install-barman-plugin) ensure_tools ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin ;; *) echo "Usage: $0 {create|delete|recreate|start|stop|status|restart|initialize|update|reload|deploy|rollout|install-barman-plugin} [cluster] [version]" >&2 exit 2 ;; esac