#!/usr/bin/env bash

set -euo pipefail

# init_cloudnative_pg.sh
# Purpose:
# - Distribute administrator ed25519 key pair to CloudNative‑PG as a Kubernetes Secret for cert auth
# - Patch CNPG cluster to enable TLS where possible
#
# Usage:
#   ./init_cloudnative_pg.sh start|stop|status|restart
#   ./init_cloudnative_pg.sh initialize          # install CNPG, create cluster, configure secrets
#   ./init_cloudnative_pg.sh update|reload       # re-apply/patch
#   ./init_cloudnative_pg.sh deploy [version]    # apply CNPG manifest and update image
#   ./init_cloudnative_pg.sh rollout             # rolling restart of CNPG pods
#
# Requirements:
# - init_openbao.sh has been run (OpenBao running as a local container)
# - $PROLE_HOME/env.sh or $HOME/.prole/env.sh defining PROLE_SERVICE
# - Optional: CNPG_MANIFEST_OVERRIDE to apply a recovery manifest instead of kustomize

# Initialize SCRIPT_DIR
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

# Load environment and config via prole_cfg.sh
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
  prole_set_mode "${2:-}"
  shift 2
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
  prole_set_mode "${1#*=}"
  shift
fi

if [[ -z "${PROLE_SERVICE:-}" ]]; then
  echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2
  exit 1
fi

ACTION=${1:-}
CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-prole-db}
VERSION=${2:-latest}
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
REALM=${REALM:-PROLE.ORG}
DOMAIN=${DOMAIN:-prole.org}
CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-}
PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}
BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup}
BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800}
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
  CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-900}
else
  CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-300}
fi
RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl"
BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-}
BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0}
CERT_MANAGER_MANIFEST_URL=${CERT_MANAGER_MANIFEST_URL:-}
CERT_MANAGER_FALLBACK_VERSION=${CERT_MANAGER_FALLBACK_VERSION:-1.19.3}

if [[ "$ACTION" != "deploy" && "$ACTION" != "rollout" && "$ACTION" != "force-rollout" ]]; then
  if [[ -n "${2:-}" ]]; then
    CNPG_CLUSTER_NAME="${2}"
  fi
  VERSION=${3:-latest}
fi

# Support both PROLE_HOME/k8s and sibling k8s directory
if [[ -d "$SCRIPT_DIR/../k8s/prole" ]]; then
  K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole"
elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/prole" ]]; then
  K8S_PROLE_DIR="$PROLE_HOME/k8s/prole"
else
  K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole"
fi
CNPG_MANIFEST="$K8S_PROLE_DIR/prole-db.yaml"
BARMAN_OBJECTSTORE_MANIFEST="$K8S_PROLE_DIR/prole-db-barman-objectstore.yaml"

SECRETS_DIR="$PROLE_SERVICE/secrets"
# Resolving CNPG admin keys. 
# We prefer names without algorithm suffixes to be more generic, matching install.py fallback strategy.
ADMIN_PRIV_ED25519="$SECRETS_DIR/admin_ed25519.key"
ADMIN_PUB_ED25519="$SECRETS_DIR/admin_ed25519.pub"
ADMIN_PRIV_GENERIC="$SECRETS_DIR/admin.key"
ADMIN_PUB_GENERIC="$SECRETS_DIR/admin.pub"
OPENBAO_TOKEN_FILE="$SECRETS_DIR/openbao-root-token"
# NAMESPACE is always derived from prole.cfg via PROLE_NAMESPACE — never from the environment
NAMESPACE="${PROLE_NAMESPACE}"
BAO_NAMESPACE="${PROLE_NAMESPACE}"
OPENBAO_NAMESPACE="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${PROLE_NAMESPACE}}}"
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
BAO_PATH_DB="${BAO_PATH_PREFIX}/db"
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"

ensure_tools() {
  for t in kubectl curl openssl base64 jq; do
    command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
  done
}

ensure_namespace() {
  if [[ -z "${PROLE_NAMESPACE:-}" ]]; then
    echo "ERROR: PROLE_NAMESPACE is empty. Set NAMESPACE in conf/prole.cfg." >&2
    exit 1
  fi
  if ! kubectl get namespace "$PROLE_NAMESPACE" >/dev/null 2>&1; then
    echo "Creating namespace '$PROLE_NAMESPACE' ..."
    kubectl create namespace "$PROLE_NAMESPACE" >/dev/null 2>&1 || true
  fi
}

get_latest_image() {
  local pg_version_file release_file pg_version release
  if [[ -f "$SCRIPT_DIR/../conf/postgresql/.version" ]]; then
    pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version"
  elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/conf/postgresql/.version" ]]; then
    pg_version_file="$PROLE_HOME/conf/postgresql/.version"
  else
    pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version"
  fi

  if [[ -f "$SCRIPT_DIR/../prole-db/.version" ]]; then
    release_file="$SCRIPT_DIR/../prole-db/.version"
  elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/prole-db/.version" ]]; then
    release_file="$PROLE_HOME/prole-db/.version"
  else
    release_file="$SCRIPT_DIR/../prole-db/.version"
  fi

  if [[ -f "$pg_version_file" ]]; then
    pg_version=$(tr -d '[:space:]' < "$pg_version_file")
  else
    pg_version="17.7"
  fi

  if [[ -f "$release_file" ]]; then
    release=$(tr -d '[:space:]' < "$release_file")
  else
    release="43"
  fi

  if [[ "$release" =~ ^[0-9]+$ ]]; then
    release=$(printf "%03d" "$release")
  fi

  echo "prole-db:${pg_version}-${release}"
}

resolve_cnpg_image() {
  local image="$1"
  if _prole_local_registry_enabled; then
    # Prefer the in-cluster/internal registry for k3d/k3s nodes pulling images.
    # Using LOCAL_REGISTRY (e.g., localhost:5000) breaks inside cluster and may
    # also prefer IPv6 ::1, leading to connection refused. Avoid it.
    local registry=""

    _cnpg_registry_looks_like_k3d() {
      local r="${1:-}"
      [[ -n "$r" ]] || return 1
      case "$r" in
        k3d-*|*"/k3d-"*) return 0 ;;
      esac
      return 1
    }

    _cnpg_registry_is_localhostish() {
      local r="${1:-}"
      [[ -n "$r" ]] || return 1
      case "$r" in
        localhost:5000|127.0.0.1:5000|*.localhost|*.localhost:5000) return 0 ;;
      esac
      return 1
    }

    _cnpg_k3s_internal_registry() {
      # For k3s, never use k3d registry names or localhost-ish endpoints.
      local r="${LOCAL_REGISTRY_INTERNAL:-}"
      if _cnpg_registry_looks_like_k3d "$r" || _cnpg_registry_is_localhostish "$r"; then
        r=""
      fi
      if [[ -n "$r" ]]; then
        printf '%s' "$r"
        return 0
      fi

      local host=""
      if command -v _prole_host_from_url >/dev/null 2>&1; then
        host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
      fi
      if [[ -n "${host:-}" ]]; then
        printf '%s' "${host}:5000"
        return 0
      fi

      local ns="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}"
      printf '%s' "registry.${ns}.svc.cluster.local:5000"
    }

    local _mode
    _mode=$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}")
    case "${_mode}" in
      k3s)
        registry=$(_cnpg_k3s_internal_registry)
        ;;
      *)
        if [[ -n "${LOCAL_REGISTRY_INTERNAL:-}" ]]; then
          registry="${LOCAL_REGISTRY_INTERNAL}"
        fi
        ;;
    esac

    if [[ -z "$registry" && -n "${LOCAL_REGISTRY:-}" ]]; then
      # Only fall back to LOCAL_REGISTRY when INTERNAL is not available
      # and it's not pointing at localhost (which is invalid for cluster pulls).
      if [[ "${LOCAL_REGISTRY}" != "localhost:5000" && "${LOCAL_REGISTRY}" != "127.0.0.1:5000" ]]; then
        registry="${LOCAL_REGISTRY}"
      fi
    fi

    if [[ -n "$registry" ]]; then
      local first="${image%%/*}"
      # If the image is unqualified (no registry), prefix it with the chosen registry
      if [[ "$image" != */* ]]; then
        image="${registry}/${image}"
      # If the first path segment has no dot/colon, it's still unqualified (e.g., prole-db:TAG)
      elif [[ "$first" != *"."* && "$first" != *":"* ]]; then
        image="${registry}/${image}"
      fi
    fi
  fi
  printf '%s' "$image"
}

sync_manifest_image() {
  local image="$1"
  local files=()
  if [[ -n "$CNPG_MANIFEST" ]]; then
    files+=("$CNPG_MANIFEST")
  fi
  if [[ -f "$RECOVERY_TEMPLATE" ]]; then
    files+=("$RECOVERY_TEMPLATE")
  fi
  local f tmp
  for f in "${files[@]}"; do
    if [[ -f "$f" ]] && grep -qE '^[[:space:]]*imageName:' "$f"; then
      tmp=$(mktemp)
      sed -E "s|^([[:space:]]*imageName:).*|\\1 ${image}|" "$f" > "$tmp"
      mv "$tmp" "$f"
    fi
  done
}

openbao_url() {
  if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
    echo "$PROLE_OPENBAO_URL"
    return 0
  fi
  if prole_is_in_cluster; then
    echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200"
    return 0
  fi

  # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
  # (never via localhost or kubectl port-forward).
  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
    if command -v _prole_host_from_url >/dev/null 2>&1; then
      local host
      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
      if [[ -n "${host:-}" ]]; then
        echo "http://${host}:8200"
        return 0
      fi
    fi
    echo ""
    return 0
  fi

  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    echo "http://127.0.0.1:8200"
    return 0
  fi
  echo ""
  return 0
}

openbao_token() {
  if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
    cat "$OPENBAO_TOKEN_FILE"
  else
    echo "${OPENBAO_ROOT_TOKEN:-}"
  fi
}

fetch_openbao_secret() {
  local path="$1"
  local key="$2"
  local token url
  token=$(openbao_token)
  url=$(openbao_url)
  if [[ -z "$token" || -z "$url" ]]; then
    echo ""
    return 0
  fi
  curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
}

resolve_db_password() {
  local db_pw="${DB_PASSWORD:-}"
  if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
    local fetched_db
    fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password")
    if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
      db_pw="$fetched_db"
    fi
  fi
  printf '%s' "$db_pw"
}


ensure_cnpg_operator() {
  if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
    kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
    wait_for_cnpg_webhook 180 || true
    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
      tune_cnpg_operator_for_k3s || true
      kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true
      wait_for_cnpg_webhook 300 || true
    fi
    return 0
  fi

  local target_version minor_version yaml_url
  target_version="${CNPG_OPERATOR_VERSION:-${CNPG_VERSION:-${CNPG_OPERATOR_FALLBACK_VERSION:-1.28.1}}}"
  if [[ -z "$target_version" || "$target_version" == "latest" ]]; then
    target_version=$(get_latest_cnpg_version)
  fi
  minor_version=$(echo "$target_version" | cut -d. -f1,2)
  yaml_url="https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${minor_version}/releases/cnpg-${target_version}.yaml"

  echo "Installing CloudNative-PG operator version ${target_version} ..."
  kubectl apply --server-side -f "$yaml_url"

  if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
    kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
    wait_for_cnpg_webhook 180 || true
    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
      tune_cnpg_operator_for_k3s || true
      kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true
      wait_for_cnpg_webhook 300 || true
    fi
  fi
}

tune_cnpg_operator_for_k3s() {
  # k3s on a single node can experience short API/server or scheduling stalls under load.
  # CNPG's default probes are very aggressive (timeoutSeconds=1), which can cause flapping
  # readiness and webhook endpoints disappearing mid-apply.
  echo "Tuning CNPG operator deployment probes/resources for k3s ..."
  kubectl -n cnpg-system patch deploy cnpg-controller-manager --type merge -p '
  {
    "spec": {
      "template": {
        "spec": {
          "containers": [
            {
              "name": "manager",
              "resources": {
                "requests": {"cpu": "250m", "memory": "512Mi"},
                "limits": {"cpu": "500m", "memory": "1Gi"}
              },
              "livenessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
              "readinessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
              "startupProbe": {"timeoutSeconds": 5, "failureThreshold": 60}
            }
          ]
        }
      }
    }
  }' >/dev/null 2>&1 || return 1
}

get_latest_barman_plugin_version() {
  local version tag
  tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "")
  if [[ -z "$tag" || "$tag" == "null" ]]; then
    echo "v${BARMAN_PLUGIN_FALLBACK_VERSION}"
    return 0
  fi
  echo "$tag"
}

resolve_barman_plugin_manifest_url() {
  if [[ -n "$BARMAN_PLUGIN_MANIFEST_URL" ]]; then
    echo "$BARMAN_PLUGIN_MANIFEST_URL"
    return 0
  fi
  local tag
  tag=$(get_latest_barman_plugin_version)
  echo "https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/${tag}/manifest.yaml"
}

cert_manager_ready() {
  if ! kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
    return 1
  fi
  if ! kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
    return 1
  fi
  return 0
}

resolve_control_plane_selector() {
  local node=""
  node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -n "$node" ]]; then
    printf 'kubernetes.io/hostname=%s' "$node"
  fi
}

pin_cert_manager() {
  local selector="${CERT_MANAGER_NODE_SELECTOR:-}"
  if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then
    selector="$(resolve_control_plane_selector)"
  fi
  if [[ -z "$selector" ]]; then
    return 0
  fi
  local key value
  key="${selector%%=*}"
  value="${selector#*=}"
  if [[ -z "$key" || -z "$value" ]]; then
    echo "WARN: CERT_MANAGER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2
    return 0
  fi
  if [[ -z "$(kubectl get nodes -l "${key}=${value}" --no-headers 2>/dev/null)" ]]; then
    echo "WARN: No nodes match CERT_MANAGER_NODE_SELECTOR=${selector}. Skipping pin." >&2
    return 0
  fi
  echo "Pinning cert-manager deployments to nodes with ${selector} ..."
  for dep in cert-manager cert-manager-webhook cert-manager-cainjector; do
    kubectl -n cert-manager patch deployment "$dep" --type merge \
      -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true
  done
  kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true
}

pin_barman_cloud() {
  local selector="${BARMAN_NODE_SELECTOR:-}"
  if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then
    selector="$(resolve_control_plane_selector)"
  fi
  if [[ -z "$selector" ]]; then
    return 0
  fi
  local key value
  key="${selector%%=*}"
  value="${selector#*=}"
  if [[ -z "$key" || -z "$value" ]]; then
    echo "WARN: BARMAN_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2
    return 0
  fi
  if [[ -z "$(kubectl get nodes -l "${key}=${value}" --no-headers 2>/dev/null)" ]]; then
    echo "WARN: No nodes match BARMAN_NODE_SELECTOR=${selector}. Skipping pin." >&2
    return 0
  fi
  echo "Pinning barman-cloud deployment to nodes with ${selector} ..."
  kubectl -n cnpg-system patch deployment barman-cloud --type merge \
    -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true
  kubectl -n cnpg-system rollout restart deploy/barman-cloud >/dev/null 2>&1 || true
}

get_latest_cert_manager_version() {
  local tag
  tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cert-manager/cert-manager/releases/latest" | jq -r '.tag_name' || echo "")
  if [[ -z "$tag" || "$tag" == "null" ]]; then
    echo "v${CERT_MANAGER_FALLBACK_VERSION}"
    return 0
  fi
  echo "$tag"
}

resolve_cert_manager_manifest_url() {
  if [[ -n "$CERT_MANAGER_MANIFEST_URL" ]]; then
    echo "$CERT_MANAGER_MANIFEST_URL"
    return 0
  fi
  local tag
  tag=$(get_latest_cert_manager_version)
  echo "https://github.com/cert-manager/cert-manager/releases/download/${tag}/cert-manager.yaml"
}

ensure_cert_manager() {
  if cert_manager_ready; then
    pin_cert_manager
    if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
      kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true
      kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true
      kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true
    fi
    return 0
  fi

  local cm_url
  cm_url=$(resolve_cert_manager_manifest_url)
  echo "Installing cert-manager from ${cm_url} ..."
  kubectl apply -f "$cm_url"
  pin_cert_manager

  if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
    kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true
    kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true
    kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true
  fi
}

wait_for_barman_crd() {
  local timeout=${1:-120}
  local start_time
  start_time=$(date +%s)
  echo "Waiting for Barman Cloud CRD (timeout: ${timeout}s)..."
  while true; do
    if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then
      echo "Barman Cloud CRD is available."
      return 0
    fi
    local elapsed=$(( $(date +%s) - start_time ))
    if (( elapsed > timeout )); then
      echo "Barman Cloud CRD not ready after ${elapsed}s." >&2
      return 1
    fi
    if (( elapsed % 30 < 6 && elapsed > 5 )); then
      echo "  [${elapsed}s/${timeout}s] Still waiting for Barman Cloud CRD..."
    fi
    sleep 5
  done
}

wait_for_barman_tls_secrets() {
  local timeout=${1:-180}
  local start_time
  start_time=$(date +%s)
  echo "Waiting for Barman Cloud TLS secrets (timeout: ${timeout}s)..."
  while true; do
    if kubectl -n cnpg-system get secret barman-cloud-client-tls >/dev/null 2>&1 \
      && kubectl -n cnpg-system get secret barman-cloud-server-tls >/dev/null 2>&1; then
      echo "Barman Cloud TLS secrets are available."
      return 0
    fi
    local elapsed=$(( $(date +%s) - start_time ))
    if (( elapsed > timeout )); then
      echo "Barman Cloud TLS secrets not ready after ${elapsed}s." >&2
      return 1
    fi
    if (( elapsed % 30 < 6 && elapsed > 5 )); then
      echo "  [${elapsed}s/${timeout}s] Still waiting for Barman Cloud TLS secrets..."
    fi
    sleep 5
  done
}

ensure_barman_plugin() {
  ensure_cert_manager

  local plugin_url
  plugin_url=$(resolve_barman_plugin_manifest_url)
  echo "Installing Barman Cloud plugin from ${plugin_url} ..."
  local apply_out=""
  if ! apply_out=$(kubectl apply -f "$plugin_url" 2>&1); then
    echo "$apply_out" >&2
    if echo "$apply_out" | grep -qi "webhook.cert-manager.io"; then
      echo "WARN: cert-manager webhook error detected; restarting cert-manager components and retrying..." >&2
      kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true
      kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s >/dev/null 2>&1 || true
      kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s >/dev/null 2>&1 || true
      kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s >/dev/null 2>&1 || true
      kubectl apply -f "$plugin_url" || true
    fi
  else
    printf '%s\n' "$apply_out"
  fi

  if ! wait_for_barman_crd 120; then
    echo "WARN: Barman Cloud ObjectStore CRD not ready after install." >&2
  fi

  if ! wait_for_barman_tls_secrets 180; then
    echo "WARN: Barman Cloud TLS secrets not ready after install." >&2
  fi

  pin_barman_cloud

  if kubectl -n cnpg-system get deploy barman-cloud >/dev/null 2>&1; then
    kubectl -n cnpg-system rollout status deploy/barman-cloud --timeout=180s || true
  fi
}

pin_cnpg_controller() {
  local selector="${CNPG_CONTROLLER_NODE_SELECTOR:-}"
  if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then
    if [[ -n "$(kubectl get nodes -l "storage=primary" --no-headers 2>/dev/null)" ]]; then
      selector="storage=primary"
    else
      local cp_node=""
      cp_node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
      if [[ -n "$cp_node" ]]; then
        selector="kubernetes.io/hostname=${cp_node}"
      fi
    fi
  fi
  if [[ -z "$selector" ]]; then
    return 0
  fi
  local key value
  key="${selector%%=*}"
  value="${selector#*=}"
  if [[ -z "$key" || -z "$value" ]]; then
    echo "WARN: CNPG_CONTROLLER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2
    return 0
  fi
  if [[ -z "$(kubectl get nodes -l "${key}=${value}" --no-headers 2>/dev/null)" ]]; then
    echo "WARN: No nodes match CNPG_CONTROLLER_NODE_SELECTOR=${selector}. Skipping pin." >&2
    return 0
  fi
  echo "Pinning cnpg-controller-manager to nodes with ${selector} ..."
  kubectl -n cnpg-system patch deployment cnpg-controller-manager --type merge \
    -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true
}

wait_for_cnpg_webhook() {
  local timeout=${1:-120}
  local start_time
  start_time=$(date +%s)

  echo "Waiting for CNPG webhook service endpoints to be ready (timeout: ${timeout}s)..."
  while true; do
    local endpoints
    endpoints=$(kubectl -n cnpg-system get endpoints cnpg-webhook-service -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
    if [[ -n "$endpoints" ]]; then
      echo "CNPG webhook service has endpoints."
      return 0
    fi
    local elapsed=$(( $(date +%s) - start_time ))
    if (( elapsed > timeout )); then
      echo "WARN: CNPG webhook endpoints not ready after ${elapsed}s." >&2
      kubectl -n cnpg-system get pods 2>/dev/null >&2 || true
      return 1
    fi
    if (( elapsed % 30 < 6 && elapsed > 5 )); then
      echo "  [${elapsed}s/${timeout}s] Waiting for CNPG webhook endpoints..."
      kubectl -n cnpg-system get pods --no-headers 2>/dev/null | sed 's/^/    /' || true
    fi
    sleep 5
  done
}

apply_barman_objectstore_if_present() {
  if [[ ! -f "$BARMAN_OBJECTSTORE_MANIFEST" ]]; then
    return 0
  fi
  if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then
    if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
      local endpoint
      endpoint="http://garage.${SERVICE_NAMESPACE}.svc.cluster.local:3900"
      prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" \
        | sed -E "s|^[[:space:]]*endpointURL:.*|    endpointURL: ${endpoint}|" \
        | kubectl apply -n "$NAMESPACE" -f -
    else
      prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" | kubectl apply -n "$NAMESPACE" -f -
    fi
  else
    echo "WARN: Barman Cloud ObjectStore CRD not found; skipping $BARMAN_OBJECTSTORE_MANIFEST."
  fi
}

apply_prole_manifest_file() {
  local file="$1"
  local output=""
  if output=$(prole_render_manifest "$file" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then
    printf '%s\n' "$output"
    return 0
  fi
  if [[ "${PROLE_MODE:-}" == "k3d" && "$(basename "$file")" == "garage-statefulset.yaml" ]] \
    && echo "$output" | grep -q "updates to statefulset spec"; then
    echo "WARN: Garage StatefulSet immutable in k3d; skipping apply."
    return 0
  fi
  echo "$output" >&2
  return 1
}

_push_to_k3s_registry() {
  local image="$1"
  local push_host="${LOCAL_REGISTRY:-${LOCAL_REGISTRY_INTERNAL:-myrddin.prole.org:5000}}"
  local plain_image="${image##*/}"

  # Normalize push_host (strip scheme if provided)
  push_host="${push_host#http://}"
  push_host="${push_host#https://}"

  if command -v skopeo >/dev/null 2>&1; then
    # Avoid long partial uploads when the host registry endpoint is unreachable.
    if command -v curl >/dev/null 2>&1; then
      if ! curl -fsS -m 2 "http://${push_host}/v2/" >/dev/null 2>&1; then
        echo "  WARN: Registry endpoint 'http://${push_host}/v2/' is not reachable; will try port-forward fallback." >&2
      else
        echo "  Pushing to k3s registry at '${push_host}' using skopeo ..."
        if skopeo copy --dest-tls-verify=false docker-daemon:"$image" docker://"${push_host}/${plain_image}"; then
          echo "  ✓ Image pushed to registry at '${push_host}'."
          return 0
        fi
      fi
    else
    echo "  Pushing to k3s registry at '${push_host}' using skopeo ..."
    if skopeo copy --dest-tls-verify=false docker-daemon:"$image" docker://"${push_host}/${plain_image}"; then
      echo "  ✓ Image pushed to registry at '${push_host}'."
      return 0
    fi
    fi
  fi

  # Fallback: port-forward the in-cluster registry service and push via localhost.
  # This avoids relying on hostPort / firewall rules for ${push_host}.
  if command -v kubectl >/dev/null 2>&1 && command -v skopeo >/dev/null 2>&1; then
    local reg_ns="${REGISTRY_NAMESPACE:-${COMMON_SERVICES_NAMESPACE:-common-services}}"
    local pf_port="${PROLE_REGISTRY_PORT_FORWARD_LOCAL:-55000}"
    local pf_log
    pf_log="$(mktemp -t prole-registry-pf.XXXXXX)"

    echo "  Trying registry push via kubectl port-forward (namespace='${reg_ns}', local=127.0.0.1:${pf_port} -> svc/registry:5000) ..."
    kubectl -n "$reg_ns" port-forward --address 127.0.0.1 svc/registry "${pf_port}:5000" >"$pf_log" 2>&1 &
    local pf_pid=$!

    local ready=0
    if command -v curl >/dev/null 2>&1; then
      for _i in {1..40}; do
        if curl -fsS -m 1 "http://127.0.0.1:${pf_port}/v2/" >/dev/null 2>&1; then
          ready=1
          break
        fi
        if ! kill -0 "$pf_pid" >/dev/null 2>&1; then
          break
        fi
        sleep 0.5
      done
    else
      # Without curl, best-effort short delay before attempting push.
      sleep 2
      ready=1
    fi

    local rc=1
    if [[ "$ready" == "1" ]]; then
      if skopeo copy --dest-tls-verify=false docker-daemon:"$image" docker://"127.0.0.1:${pf_port}/${plain_image}"; then
        echo "  ✓ Image pushed to registry via port-forward."
        rc=0
      fi
    else
      echo "  WARN: registry port-forward did not become ready (see $pf_log)." >&2
    fi

    kill "$pf_pid" >/dev/null 2>&1 || true
    wait "$pf_pid" >/dev/null 2>&1 || true
    rm -f "$pf_log" >/dev/null 2>&1 || true
    if [[ $rc -eq 0 ]]; then
      return 0
    fi

    if _import_image_to_k3s_nodes "$image"; then
      return 0
    fi
  fi

  # Fallback to docker push if skopeo is missing or fails (might fail if daemon not configured)
  local push_ref="${push_host}/${plain_image}"
  docker tag "$image" "$push_ref" 2>/dev/null || true
  if docker push "$push_ref" 2>/dev/null; then
    echo "  ✓ Image pushed to registry at '${push_host}' via docker push."
    return 0
  fi

  echo "ERROR: Failed to push image '$image' to k3s registry at '${push_host}'." >&2
  return 1
}

# Push image to local registry (via LOCAL_REGISTRY host address) with k3d import fallback.
_push_to_k3d_registry() {
  local image="$1"
  local cluster_name="$2"
  local push_host="${LOCAL_REGISTRY:-localhost:5000}"
  local plain_image="${image##*/}"  # strip registry prefix, e.g. prole-db:18-088

  if [[ -n "$push_host" ]]; then
    local push_ref="${push_host}/${plain_image}"
    docker tag "$image" "$push_ref" 2>/dev/null || true
    if docker push "$push_ref" 2>/dev/null; then
      echo "  ✓ Image pushed to registry at '${push_host}'."
      return 0
    fi
    echo "  WARN: push to '${push_host}' failed; falling back to k3d image import ..." >&2
  fi

  if k3d image import "$image" -c "$cluster_name" 2>/dev/null; then
    echo "  ✓ Image '$image' imported directly into k3d cluster '$cluster_name'."
    return 0
  fi
  echo "ERROR: Failed to push or import image '$image'." >&2
  return 1
}

# Pre-flight: ensure the prole-db image is available in the k3d cluster before
# the CNPG operator ever tries to pull it, avoiding ErrImagePull backoff loops.
# Steps: containerd cache → Docker daemon (registry tag) → Docker daemon (plain tag)
#        → tar import → docker build + push/import.
_ensure_prole_db_image() {
  if [[ "${PROLE_MODE:-}" != "k3d" && "${PROLE_MODE:-}" != "k3s" ]]; then
    return 0
  fi

  local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
  local image=""
  if [[ -n "$image_override" ]]; then
    image="$image_override"
  else
    if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then
      image=$(get_latest_image)
    else
      image="prole-db:$VERSION"
    fi
  fi
  image=$(resolve_cnpg_image "$image")
  local prole_db_dir="${PROLE_HOME:-$SCRIPT_DIR/..}/prole-db"
  local plain_image="${image##*/}"  # e.g. prole-db:18-088

  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
    echo "Pre-flight: ensuring image '$image' is available in k3s registry/import path ..."
  else
    # Auto-detect active k3d cluster name
    local cluster_name="${K3D_CLUSTER_NAME:-}"
    if [[ -z "$cluster_name" ]]; then
      cluster_name=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | head -1)
    fi
    cluster_name="${cluster_name:-knoe-dev-cluster}"

    echo "Pre-flight: verifying image '$image' is available in k3d cluster '$cluster_name' ..."

    # Step 1: check if already present in k3d containerd with matching digest
    local containerd_digest local_digest containerd_sha
    containerd_digest=$(docker exec "k3d-${cluster_name}-server-0" \
      ctr images ls -q 2>/dev/null | grep -F "$image" | head -1 || true)
    if [[ -n "$containerd_digest" ]]; then
      local_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$image" 2>/dev/null \
        | awk -F@ '{print $2}' || true)
      containerd_sha=$(docker exec "k3d-${cluster_name}-server-0" \
        ctr images ls 2>/dev/null | grep -F "$image" | awk '{print $3}' | head -1 || true)
      if [[ -z "$local_digest" || "$containerd_sha" == "$local_digest" ]]; then
        echo "  ✓ Image '$image' already in k3d containerd (digest match); no import needed."
        return 0
      fi
      echo "  Image '$image' in containerd but digest mismatch (local: ${local_digest:-unknown}, containerd: ${containerd_sha:-unknown}); re-importing ..."
      docker exec "k3d-${cluster_name}-server-0" ctr images rm "$image" 2>/dev/null || true
    fi
  fi

  # Step 2a: registry-tagged image in local Docker daemon → push + import
  if docker image inspect "$image" >/dev/null 2>&1; then
    echo "  Image '$image' found in Docker daemon; pushing to registry ..."
    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
      _push_to_k3s_registry "$image"
    else
      _push_to_k3d_registry "$image" "$cluster_name"
    fi
    return $?
  fi

  # Step 2b: plain-tagged image in local Docker daemon → tag + push + import
  if docker image inspect "$plain_image" >/dev/null 2>&1; then
    echo "  Plain image '$plain_image' found in Docker daemon; tagging as '$image' and pushing ..."
    docker tag "$plain_image" "$image"
    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
      _push_to_k3s_registry "$image"
    else
      _push_to_k3d_registry "$image" "$cluster_name"
    fi
    return $?
  fi

  # Step 3: look for a matching tar in the docker-import directory
  local prole_data="${PROLE_DATA:-$HOME/.prole/data}"
  local docker_import_dir="${DOCKER_IMPORT_DIR:-${prole_data}/docker-import}"
  local name_part="${plain_image%%:*}"  # e.g. prole-db
  local tag_part="${plain_image##*:}"   # e.g. 18-088
  local found_tar=""
  if [[ -d "$docker_import_dir" ]]; then
    for _t in "$docker_import_dir"/*.tar; do
      [[ -f "$_t" ]] || continue
      local _bn
      _bn=$(basename "$_t" .tar)
      if [[ "$_bn" == *"$name_part"* && "$_bn" == *"$tag_part"* ]]; then
        found_tar="$_t"
        break
      fi
    done
  fi
  if [[ -n "$found_tar" ]]; then
    echo "  Loading tar '$(basename "$found_tar")' into Docker daemon ..."
    docker load -i "$found_tar"
    docker tag "$plain_image" "$image" 2>/dev/null || true
    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
      _push_to_k3s_registry "$image"
    else
      _push_to_k3d_registry "$image" "$cluster_name"
    fi
    return $?
  fi

  # Step 4: image not found anywhere — build from source then push + import
  if [[ ! -f "$prole_db_dir/Dockerfile" ]]; then
    echo "ERROR: Dockerfile not found in '$prole_db_dir'; cannot build prole-db image." >&2
    return 1
  fi
  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
    echo "  Image '$image' not found in Docker daemon or docker-import dir."
  else
    echo "  Image '$image' not found in k3d, Docker daemon, or docker-import dir."
  fi
  echo "  Building prole-db image from '$prole_db_dir' ..."
  if ! docker build -t "$plain_image" "$prole_db_dir"; then
    echo "ERROR: docker build failed for image '$plain_image'." >&2
    return 1
  fi
  docker tag "$plain_image" "$image"
  echo "  Build complete. Pushing '$image' to registry ..."
  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
    _push_to_k3s_registry "$image"
  else
    _push_to_k3d_registry "$image" "$cluster_name"
  fi
  return $?
}

cleanup_unintended_cnpg_services() {
  # Historical manifest bug: we used to create a CNPG-managed Service named `prole-db-001`
  # as `type: LoadBalancer`. In k3s this spawns `svclb-prole-db-001` pods, and it can
  # interfere with CNPG startup/reconciliation. Ensure it is removed if it exists.
  local svc_name="prole-db-001"
  if kubectl -n "$NAMESPACE" get svc "$svc_name" >/dev/null 2>&1; then
    echo "Removing unintended Service '$svc_name' from namespace '$NAMESPACE' ..."
    kubectl -n "$NAMESPACE" delete svc "$svc_name" --ignore-not-found >/dev/null 2>&1 || true
  fi
}

ensure_prole_stack_resources() {
  echo "Applying CloudNative-PG cluster and related resources ..."
  cleanup_unintended_cnpg_services || true
  local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
  local image=""
  if [[ -n "$image_override" ]]; then
    image="$image_override"
  else
    if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then
      image=$(get_latest_image)
    else
      image="prole-db:$VERSION"
    fi
  fi
  image=$(resolve_cnpg_image "$image")
  sync_manifest_image "$image"
  if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then
    if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then
      echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2
      return 1
    fi
    local dir file
    dir="$K8S_PROLE_DIR"
    for file in "$dir"/*.yaml; do
      case "$(basename "$file")" in
        prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml|ingress.yaml)
          continue
          ;;
        openbao-statefulset.yaml|openbao-service.yaml)
          if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
            continue
          fi
          ;;
        garage-*.yaml|grafana-*.yaml|prometheus-*.yaml)
          if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
            continue
          fi
          ;;
      esac
      apply_prole_manifest_file "$file"
    done
    # Apply ingress.yaml without -n flag so each document targets its own namespace
    if [[ -f "$dir/ingress.yaml" ]]; then
      # Ensure referenced namespaces exist before applying multi-namespace ingress
      for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do
        if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then
          echo "Creating namespace '$_ing_ns' for ingress resource ..."
          kubectl create namespace "$_ing_ns" 2>/dev/null || true
        fi
      done
      prole_render_manifest "$dir/ingress.yaml" | kubectl apply -f - 2>&1 || true
    fi
    apply_barman_objectstore_if_present
    apply_cnpg_cluster_manifest "$CNPG_MANIFEST_OVERRIDE"
  else
    local dir file
    dir="$K8S_PROLE_DIR"
    for file in "$dir"/*.yaml; do
      case "$(basename "$file")" in
        prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml|ingress.yaml)
          continue
          ;;
        openbao-statefulset.yaml|openbao-service.yaml)
          if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
            continue
          fi
          ;;
        garage-*.yaml|grafana-*.yaml|prometheus-*.yaml)
          if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
            continue
          fi
          ;;
      esac
      apply_prole_manifest_file "$file"
    done
    # Apply ingress.yaml without -n flag so each document targets its own namespace
    if [[ -f "$dir/ingress.yaml" ]]; then
      # Ensure referenced namespaces exist before applying multi-namespace ingress
      for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do
        if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then
          echo "Creating namespace '$_ing_ns' for ingress resource ..."
          kubectl create namespace "$_ing_ns" 2>/dev/null || true
        fi
      done
      prole_render_manifest "$dir/ingress.yaml" | kubectl apply -f - 2>&1 || true
    fi
    apply_barman_objectstore_if_present
    apply_cnpg_cluster_manifest "$CNPG_MANIFEST"
  fi

  # Ensure prole-index-html exists for prole deployment readiness probe
  if ! kubectl get configmap prole-index-html -n "$NAMESPACE" >/dev/null 2>&1; then
    echo "Creating prole-index-html configmap..."
    printf "<html><body><h1>Prole</h1></body></html>" > /tmp/index.html
    kubectl create configmap prole-index-html --from-file=/tmp/index.html -n "$NAMESPACE"
    rm /tmp/index.html
  fi

  # Ensure prole-nginx-tls exists (self-signed for dev)
  if ! kubectl get secret prole-nginx-tls -n "$NAMESPACE" >/dev/null 2>&1; then
    echo "Generating self-signed prole-nginx-tls for development..."
    openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
      -keyout /tmp/nginx-tls.key -out /tmp/nginx-tls.crt \
      -subj "/CN=prole.org" >/dev/null 2>&1
    kubectl create secret tls prole-nginx-tls --key /tmp/nginx-tls.key --cert /tmp/nginx-tls.crt -n "$NAMESPACE"
    rm /tmp/nginx-tls.key /tmp/nginx-tls.crt
  fi
}

apply_cnpg_cluster_manifest() {
  local manifest="$1"
  local attempts=${CNPG_APPLY_RETRIES:-6}
  local i out

  for ((i=1; i<=attempts; i++)); do
    if out=$(prole_render_manifest "$manifest" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then
      printf '%s\n' "$out"
      return 0
    fi
    if echo "$out" | grep -q "cnpg-webhook-service"; then
      echo "CNPG webhook not ready yet (attempt $i/$attempts). Retrying..."
      sleep 5
      continue
    fi
    echo "$out" >&2
    return 1
  done

  echo "ERROR: Failed to apply CNPG manifest after $attempts attempts." >&2
  echo "$out" >&2
  return 1
}

wait_for_cnpg_pods() {
  local timeout=${1:-${CNPG_WAIT_TIMEOUT:-900}}
  local start_time
  start_time=$(date +%s)

  local target_pods="${CNPG_TARGET_PODS:-}"
  if [[ -z "$target_pods" ]]; then
    target_pods=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.instances}' 2>/dev/null || true)
  fi
  if [[ -z "$target_pods" ]]; then
    target_pods=3
  fi

  echo "Waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' to be Ready..."
  echo "  Timeout: ${timeout}s"
  local last_feedback=0
  local feedback_interval=30  # print detailed status every 30s
  local _image_pull_remediated=""  # set after first k3d import attempt to avoid loops
  while true; do
    local now elapsed
    now=$(date +%s)
    elapsed=$(( now - start_time ))

    local pods
    pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null || true)

    if [[ -n "$pods" ]]; then
      # Check for Error or CrashLoopBackOff (terminal failures)
      if echo "$pods" | grep -E "Error|CrashLoopBackOff" >/dev/null; then
        echo "ERROR: Some CNPG pods are in Error or CrashLoopBackOff state:" >&2
        echo "$pods" | grep -E "Error|CrashLoopBackOff" >&2
        return 1
      fi

      # Check for image pull failures — attempt remediation in k3d/k3s mode
      if echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >/dev/null; then
        if [[ "${PROLE_MODE:-}" == "k3d" || "${PROLE_MODE:-}" == "k3s" ]]; then
          if [[ -z "$_image_pull_remediated" ]]; then
            _image_pull_remediated=1
            local fail_pod fail_image
            fail_pod=$(echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" | awk '{print $1}' | head -1)
            # Try containers then initContainers
            fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \
              -o jsonpath='{.spec.containers[0].image}' 2>/dev/null || true)
            if [[ -z "$fail_image" ]]; then
              fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \
                -o jsonpath='{.spec.initContainers[0].image}' 2>/dev/null || true)
            fi
            echo "WARN: Pod '$fail_pod' cannot pull image '${fail_image:-unknown}' (ErrImagePull/ImagePullBackOff)." >&2
            import_dir="${DOCKER_IMPORT_DIR:-${PROLE_DATA:+${PROLE_DATA}/docker-import}}"
            # Re-run full image ensure logic to push/import the failing image
            echo "  Attempting image remediation via _ensure_prole_db_image ..." >&2
            _ensure_prole_db_image >&2 || true
            echo "  Remediation complete; resuming wait ..." >&2
          fi
          # Continue the wait loop — do not return 1
        else
          echo "ERROR: Image pull failure in non-k3d mode — cannot auto-recover:" >&2
          echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >&2
          return 1
        fi
      fi

      # Count Ready pods by name (exclude initdb)
      local ready_pods
      ready_pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep "True" | wc -l | xargs)

      if [[ "$ready_pods" -ge "$target_pods" ]]; then
        echo "All $ready_pods/$target_pods pods are Ready."
        # Verify DB connectivity if kubectl-cnpg is available
        if kubectl cnpg version >/dev/null 2>&1; then
           echo "Verifying database connectivity via 'kubectl cnpg psql'..."
           if kubectl cnpg -n "$NAMESPACE" psql "$CNPG_CLUSTER_NAME" -- -c "SELECT 1" >/dev/null 2>&1; then
             echo "Database connection verified."
             return 0
           else
             echo "  [${elapsed}s/${timeout}s] Pods are Ready but database connection failed — waiting for database to accept connections..."
           fi
        else
           # Fallback if no cnpg plugin: just return 0 if pods are ready
           return 0
        fi
      fi

      # Periodic detailed feedback
      if (( now - last_feedback >= feedback_interval )); then
        last_feedback=$now
        echo ""
        echo "  [${elapsed}s/${timeout}s] Waiting — ${ready_pods:-0}/${target_pods} pods Ready"
        echo "  Pod status:"
        echo "$pods" | sed 's/^/    /'

        # Show cluster status if available
        local cluster_phase
        cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)
        if [[ -n "$cluster_phase" ]]; then
          echo "  Cluster phase: ${cluster_phase}"
        fi

        # Show recent events (last 3)
        local events
        events=$(kubectl -n "$NAMESPACE" get events \
          --sort-by='.lastTimestamp' \
          --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \
          -o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \
          --no-headers 2>/dev/null | tail -3 || true)
        if [[ -n "$events" ]]; then
          echo "  Recent cluster events:"
          echo "$events" | sed 's/^/    /'
        fi
      fi
    else
      # No pods yet — give feedback
      if (( now - last_feedback >= feedback_interval )); then
        last_feedback=$now
        echo "  [${elapsed}s/${timeout}s] No CNPG pods found yet for cluster '$CNPG_CLUSTER_NAME' — waiting for operator to create them..."
        local cluster_phase
        cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)
        if [[ -n "$cluster_phase" ]]; then
          echo "  Cluster phase: ${cluster_phase}"
        fi
        # If stuck in "Unable to create required cluster objects", show conditions and events for diagnosis
        if [[ "$cluster_phase" == *"Unable to create"* ]]; then
          echo "  Cluster conditions:"
          kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \
            -o jsonpath='{range .status.conditions[*]}    {.type}: {.status} — {.message}{"\n"}{end}' 2>/dev/null || true
          echo "  Recent events:"
          kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \
            --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \
            -o custom-columns='TYPE:.type,REASON:.reason,MESSAGE:.message' \
            --no-headers 2>/dev/null | tail -5 | sed 's/^/    /' || true
        fi
      fi
    fi

    if (( elapsed > timeout )); then
      echo ""
      echo "ERROR: Timed out after ${elapsed}s waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE'." >&2
      echo "Final pod status:" >&2
      kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" 2>/dev/null >&2 || true
      echo "Recent events:" >&2
      kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \
        --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \
        -o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \
        --no-headers 2>/dev/null | tail -5 >&2 || true
      return 1
    fi
    sleep 5
  done
}

cluster_exists() {
  kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1
}

cluster_has_pods() {
  kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null | grep -q .
}

cluster_has_ready_pods() {
  kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" \
    -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Ready")].status=="True")].metadata.name}' 2>/dev/null | grep -q .
}

latest_backup_name() {
  kubectl -n "$NAMESPACE" get backup \
    --sort-by=.metadata.creationTimestamp \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1
}

latest_completed_backup_name() {
  kubectl -n "$NAMESPACE" get backup \
    --sort-by=.metadata.creationTimestamp \
    -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null | \
    awk -F'|' '{p=tolower($2); if (p=="completed" || p=="succeeded") {name=$1}} END {print name}'
}

wait_for_backup() {
  local backup_name="$1"
  local start_time now phase phase_lc
  start_time=$(date +%s)

  while true; do
    phase=$(kubectl -n "$NAMESPACE" get backup "$backup_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
    phase_lc=$(printf '%s' "$phase" | tr '[:upper:]' '[:lower:]')

    case "$phase_lc" in
      completed|succeeded)
        echo "Backup $backup_name completed."
        return 0
        ;;
      failed|error)
        echo "Backup $backup_name failed (phase=$phase)." >&2
        return 1
        ;;
    esac

    now=$(date +%s)
    if (( now - start_time > BACKUP_WAIT_TIMEOUT )); then
      echo "Timed out waiting for backup $backup_name." >&2
      return 1
    fi

    echo "Waiting for backup $backup_name to complete (phase=${phase:-unknown}) ..."
    sleep 10
  done
}

run_garage_backup() {
  if [[ ! -x "$SCRIPT_DIR/init_cnpg_backup.sh" ]]; then
    echo "WARN: init_cnpg_backup.sh not found; skipping Garage backup." >&2
    return 1
  fi

  echo "Running Garage backup via init_cnpg_backup.sh ..."
  if ! "$SCRIPT_DIR/init_cnpg_backup.sh" start; then
    echo "Garage backup script failed." >&2
    return 1
  fi

  sleep 2
  local backup_name
  backup_name=$(latest_backup_name)
  if [[ -z "$backup_name" ]]; then
    echo "No backup resource detected after triggering backup." >&2
    return 1
  fi

  wait_for_backup "$backup_name"
}

get_primary_pod() {
  local primary
  primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
  if [[ -z "$primary" ]]; then
    primary=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  fi
  printf '%s' "$primary"
}

decode_b64() {
  local data="$1"
  if [[ -z "$data" ]]; then
    return 1
  fi
  printf '%s' "$data" | base64 -d 2>/dev/null
}

resolve_db_credentials() {
  local user_b64 pass_b64 user pass

  user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.username}' 2>/dev/null || true)
  pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.password}' 2>/dev/null || true)
  user=$(decode_b64 "$user_b64" || true)
  pass=$(decode_b64 "$pass_b64" || true)

  if [[ -z "$user" || -z "$pass" ]]; then
    user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.username}' 2>/dev/null || true)
    pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.password}' 2>/dev/null || true)
    user=$(decode_b64 "$user_b64" || true)
    pass=$(decode_b64 "$pass_b64" || true)
  fi

  if [[ -z "$user" || -z "$pass" ]]; then
    return 1
  fi

  printf '%s\n%s' "$user" "$pass"
}

pgdump_local() {
  local pod user pass db_name dump_dir dump_file timestamp
  pod=$(get_primary_pod)
  if [[ -z "$pod" ]]; then
    echo "ERROR: No CNPG pod available for pg_dump." >&2
    return 1
  fi

  local creds
  if ! creds=$(resolve_db_credentials); then
    echo "ERROR: Unable to resolve database credentials for pg_dump." >&2
    return 1
  fi
  user=$(printf '%s' "$creds" | sed -n '1p')
  pass=$(printf '%s' "$creds" | sed -n '2p')

  db_name=${PROLE_DB_NAME:-prole-db}
  dump_dir="$BACKUP_DIR"
  mkdir -p "$dump_dir"

  timestamp=$(date +%Y%m%d%H%M%S)
  dump_file="$dump_dir/${CNPG_CLUSTER_NAME}-pgdump-${timestamp}.dump"

  echo "Running pg_dump against pod $pod (db=$db_name) ..."
  if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- env PGPASSWORD="$pass" \
      pg_dump -U "$user" -d "$db_name" -Fc > "$dump_file"; then
    echo "pg_dump saved to $dump_file"
    return 0
  fi

  echo "pg_dump failed; removing partial file." >&2
  rm -f "$dump_file"
  return 1
}

attempt_backup_if_active() {
  if ! cluster_exists; then
    return 0
  fi

  if ! cluster_has_pods; then
    echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no pods detected; skipping backup." >&2
    return 0
  fi

  if ! cluster_has_ready_pods; then
    echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no ready pods detected; skipping backup." >&2
    return 0
  fi

  echo "CNPG cluster '$CNPG_CLUSTER_NAME' detected; attempting Garage backup ..."
  if run_garage_backup; then
    return 0
  fi

  echo "Garage backup failed; attempting local pg_dump ..." >&2
  if pgdump_local; then
    return 0
  fi

  echo "WARN: Both Garage backup and local pg_dump failed." >&2
  return 0
}

wait_for_pod_ready() {
  local pod="$1"
  if ! kubectl -n "$NAMESPACE" wait --for=condition=Ready pod "$pod" --timeout=300s >/dev/null 2>&1; then
    echo "WARN: pod $pod did not become Ready within timeout." >&2
    return 1
  fi
  return 0
}

rollout_cluster() {
  ensure_tools
  ensure_namespace
  echo "Starting manual recreate rollout for $CNPG_CLUSTER_NAME in $NAMESPACE..."

  local has_cnpg_plugin="0"
  if kubectl cnpg version >/dev/null 2>&1; then
    has_cnpg_plugin="1"
    kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
  fi

  local primary
  primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
  if [[ -z "$primary" ]]; then
    echo "ERROR: Could not identify primary instance." >&2
    return 1
  fi
  echo "Primary instance: $primary"

  local instances
  instances=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)
  if [[ -z "$instances" ]]; then
    echo "ERROR: No pods found for cluster $CNPG_CLUSTER_NAME." >&2
    return 1
  fi

  local pod
  for pod in $instances; do
    if [[ "$pod" != "$primary" ]]; then
      echo "Recreating non-primary pod: $pod..."
      kubectl delete pod -n "$NAMESPACE" "$pod"
      wait_for_pod_ready "$pod" || true
      if [[ "$has_cnpg_plugin" == "1" ]]; then
        kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
      fi
    fi
  done

  local new_primary=""
  for pod in $instances; do
    if [[ "$pod" != "$primary" ]]; then
      new_primary="$pod"
      break
    fi
  done

  if [[ -n "$new_primary" && "$has_cnpg_plugin" == "1" ]]; then
    echo "Promoting $new_primary..."
    kubectl cnpg promote "$CNPG_CLUSTER_NAME" "$new_primary" -n "$NAMESPACE" || true
    echo "Waiting for $new_primary to become primary..."
    for _ in {1..60}; do
      local current_primary
      current_primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
      if [[ "$current_primary" == "$new_primary" ]]; then
        echo "$new_primary is now the primary."
        break
      fi
      sleep 5
    done
  else
    if [[ -n "$new_primary" ]]; then
      echo "WARN: kubectl cnpg plugin not available; skipping explicit promotion."
    fi
  fi

  echo "Recreating the old primary pod: $primary..."
  kubectl delete pod -n "$NAMESPACE" "$primary"
  wait_for_pod_ready "$primary" || true

  if [[ "$has_cnpg_plugin" == "1" ]]; then
    echo "Rollout complete. Final cluster status:"
    kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
  fi
}

force_rollout() {
  echo "Attempting force rollout of CNPG pods ..."
  rollout_cluster
}

reset_and_reinit() {
  local backup_name tmp_manifest
  backup_name=$(latest_completed_backup_name)

  echo "Resetting CNPG cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' ..."
  kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
  kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true

  recycle_released_prole_iscsi_pvs || true

  if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then
    tmp_manifest=$(mktemp)
    sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest"
    echo "Re-initializing from backup $backup_name ..."
    CNPG_MANIFEST_OVERRIDE="$tmp_manifest" ensure_prole_stack_resources
    rm -f "$tmp_manifest"
  else
    if [[ -n "$backup_name" && ! -f "$RECOVERY_TEMPLATE" ]]; then
      echo "WARN: Recovery template not found: $RECOVERY_TEMPLATE" >&2
    fi
    if [[ -z "$backup_name" ]]; then
      echo "No completed backups found; starting fresh initialization." >&2
    fi
    ensure_prole_stack_resources
  fi

  wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"
}

recycle_released_prole_iscsi_pvs() {
  # StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. After a CNPG reset, PVs can remain
  # in `Released` with a stale `claimRef`, which prevents new PVCs (same names) from binding.
  # In that case, CNPG init jobs remain `Pending` with "didn't find available persistent volumes".
  echo "Recycling Released prole-iscsi PVs (clearing stale claimRefs) for namespace '$NAMESPACE' ..."
  local pvs
  pvs=$(kubectl get pv -o json \
    | jq -r --arg ns "$NAMESPACE" '.items[]
      | select(.spec.storageClassName == "prole-iscsi")
      | select(.status.phase == "Released")
      | select((.spec.claimRef.namespace // "") == $ns)
      | .metadata.name' 2>/dev/null || true)

  if [[ -z "${pvs:-}" ]]; then
    return 0
  fi

  local pv
  for pv in $pvs; do
    echo "  - Clearing claimRef on PV: $pv"
    kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
  done
}

# Resolve OpenBao URL: prefer explicit env, then in-cluster service.
# In k3s mode, out-of-cluster runs must use the k3s host (no localhost/port-forward).
bao_service_url() {
  if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
    echo "$PROLE_OPENBAO_URL"
    return 0
  fi
  if prole_is_in_cluster; then
    local ns="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}"
    echo "http://$OPENBAO_NAME.$ns.svc.cluster.local:8200"
    return 0
  fi

  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
    if command -v _prole_host_from_url >/dev/null 2>&1; then
      local host
      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
      if [[ -n "${host:-}" ]]; then
        echo "http://${host}:8200"
        return 0
      fi
    fi
    echo ""
    return 0
  fi

  if curl -sS --connect-timeout 2 "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    echo "http://127.0.0.1:8200"
    return 0
  fi
  echo ""
}

fetch_admin_keys_and_db_pass_from_bao_or_local() {
  local token url
  if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
    token=$(cat "$OPENBAO_TOKEN_FILE")
  else
    token=""
  fi
  url=$(bao_service_url)
  if [[ -n "$token" && -n "$url" ]]; then
    echo "Attempting to read admin key pair from OpenBao kv/$BAO_PATH_ADMIN ..."
    if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -e '.data.data' >/dev/null 2>&1; then
      local priv_b64 pub_b64
      priv_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_private_key_b64')
      pub_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_public_key_b64')
      # Use a temporary file to determine where to save based on existing legacy or generic preference
      local target_priv="$ADMIN_PRIV_GENERIC"
      local target_pub="$ADMIN_PUB_GENERIC"
      
      # If legacy keys exist, we might want to overwrite them too for compatibility
      printf "%s" "$priv_b64" | base64 -d >"$target_priv"
      printf "%s" "$pub_b64" | base64 -d >"$target_pub"
      chmod 0600 "$target_priv"
      
      # Mirror to legacy path if it was expected by other scripts
      cp "$target_priv" "$ADMIN_PRIV_ED25519" 2>/dev/null || true
      cp "$target_pub" "$ADMIN_PUB_ED25519" 2>/dev/null || true
    fi

    echo "Attempting to read database password from OpenBao kv/$BAO_PATH_DB ..."
    if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -e '.data.data' >/dev/null 2>&1; then
      local db_pass
      db_pass=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -r '.data.data.password')
      if [[ -n "$db_pass" ]]; then
        echo "Updating database user secret 'prole-db-user' from OpenBao ..."
        kubectl create secret generic prole-db-user -n "$NAMESPACE" \
          --from-literal=username=prole \
          --from-literal=password="$db_pass" \
          --dry-run=client -o yaml | kubectl apply -f -

        echo "Updating database superuser secret 'prole-db-superuser' from OpenBao ..."
        kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \
          --from-literal=username=postgres \
          --from-literal=password="$db_pass" \
          --dry-run=client -o yaml | kubectl apply -f -
      fi
    fi
  fi

  # No env fallback: if OpenBao is unreachable and secrets are missing, fail clearly

  if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
    echo "ERROR: 'prole-db-user' secret is missing in namespace '$NAMESPACE' and could not be resolved from OpenBao." >&2
    return 1
  fi

  if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then
    echo "Using local admin key pair at $SECRETS_DIR"
    return 0
  fi
  if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then
    echo "Using local legacy admin key pair at $SECRETS_DIR"
    # Link or copy to generic for consistent use below
    cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC"
    cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC"
    return 0
  fi
  echo "ERROR: Could not obtain admin key pair from OpenBao and no local files found." >&2
  exit 1
}

apply_cnpg_admin_secret() {
  echo "Creating/updating Secret cnpg-admin-key ..."
  kubectl create secret generic cnpg-admin-key -n "$NAMESPACE" \
    --from-file=admin.key="$ADMIN_PRIV_GENERIC" \
    --from-file=admin.pub="$ADMIN_PUB_GENERIC" \
    --dry-run=client -o yaml | kubectl apply -f -
}

detect_and_reprovision_unencrypted_cluster() {
  local encryption_enabled="${AT_REST_ENCRYPTION_ENABLED:-false}"
  if [[ "$encryption_enabled" != "true" && "$encryption_enabled" != "True" && "$encryption_enabled" != "1" ]]; then
    return 0
  fi

  if ! kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
    return 0
  fi

  local server_tls
  server_tls=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.certificates.serverTLSSecret}' 2>/dev/null || echo "")
  if [[ -n "$server_tls" ]]; then
    echo "Existing cluster '$CNPG_CLUSTER_NAME' already has TLS configured ($server_tls); no conversion needed."
    return 0
  fi

  echo "WARN: Existing cluster '$CNPG_CLUSTER_NAME' does not have TLS/at-rest-encryption configured."
  echo "At-rest encryption is now enabled; the cluster must be reprovisioned."
  echo "NOTE: External full backup and restore from barman backup is not yet available."
  echo "      Removing existing cluster to allow re-creation with encryption enabled ..."

  attempt_backup_if_active

  kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
  kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true
  echo "Existing unencrypted cluster removed. Will be re-created with TLS and encryption."
}

validate_ca_secret() {
  local secret_name="$1"
  local ns="$2"
  if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then
    return 1
  fi
  local crt key
  crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true)
  key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.key}' 2>/dev/null || true)
  if [[ -z "$crt" || -z "$key" ]]; then
    echo "WARN: CA secret '$secret_name' exists but is missing ca.crt or ca.key data." >&2
    return 1
  fi
  if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then
    echo "WARN: CA secret '$secret_name' has invalid certificate data." >&2
    return 1
  fi
  # CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8
  local key_header
  key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1)
  if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then
    echo "WARN: CA secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2
    return 1
  fi
  return 0
}

validate_tls_secret() {
  local secret_name="$1"
  local ns="$2"
  if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then
    return 1
  fi
  local crt key
  crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true)
  key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.key}' 2>/dev/null || true)
  if [[ -z "$crt" || -z "$key" ]]; then
    echo "WARN: TLS secret '$secret_name' exists but is missing tls.crt or tls.key data." >&2
    return 1
  fi
  if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then
    echo "WARN: TLS secret '$secret_name' has invalid certificate data." >&2
    return 1
  fi
  # CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8
  local key_header
  key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1)
  if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then
    echo "WARN: TLS secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2
    return 1
  fi
  return 0
}

generate_tls_if_missing() {
  local ca_secret_name="${CNPG_CLUSTER_NAME}-ca"
  local tls_secret_name="${CNPG_CLUSTER_NAME}-tls"
  local need_ca=0
  local need_tls=0

  if validate_ca_secret "$ca_secret_name" "$NAMESPACE"; then
    echo "CA secret $ca_secret_name already exists and is valid."
  else
    if kubectl -n "$NAMESPACE" get secret "$ca_secret_name" >/dev/null 2>&1; then
      echo "Removing invalid CA secret '$ca_secret_name' ..."
      kubectl -n "$NAMESPACE" delete secret "$ca_secret_name" --ignore-not-found
    fi
    need_ca=1
  fi

  if validate_tls_secret "$tls_secret_name" "$NAMESPACE"; then
    echo "TLS secret $tls_secret_name already exists and is valid."
  else
    if kubectl -n "$NAMESPACE" get secret "$tls_secret_name" >/dev/null 2>&1; then
      echo "Removing invalid TLS secret '$tls_secret_name' ..."
      kubectl -n "$NAMESPACE" delete secret "$tls_secret_name" --ignore-not-found
    fi
    need_tls=1
  fi

  if [[ "$need_ca" -eq 0 && "$need_tls" -eq 0 ]]; then
    echo "TLS secrets already present and valid; skipping generation."
    return 0
  fi

  local TMPD
  TMPD=$(mktemp -d)

  if [[ "$need_ca" -eq 1 ]]; then
    echo "Generating self-signed CA (EC P-256) for CNPG ..."
    openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/ca.key"
    openssl req -x509 -new -key "$TMPD/ca.key" -out "$TMPD/ca.crt" -days 3650 -subj "/CN=Prole CNPG CA"
    kubectl -n "$NAMESPACE" create secret generic "$ca_secret_name" \
      --from-file=ca.crt="$TMPD/ca.crt" \
      --from-file=ca.key="$TMPD/ca.key" \
      --dry-run=client -o yaml | kubectl apply -f -
  else
    # Extract existing CA for signing the server certificate
    kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.crt}' | base64 -d > "$TMPD/ca.crt"
    kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.key}' | base64 -d > "$TMPD/ca.key"
  fi

  if [[ "$need_tls" -eq 1 ]]; then
    echo "Generating server TLS certificate for CNPG ($tls_secret_name) ..."
    openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/tls.key"
    openssl req -new -key "$TMPD/tls.key" -out "$TMPD/tls.csr" \
      -subj "/CN=${CNPG_CLUSTER_NAME}.${NAMESPACE}.svc"
    openssl x509 -req -in "$TMPD/tls.csr" -CA "$TMPD/ca.crt" -CAkey "$TMPD/ca.key" \
      -CAcreateserial -out "$TMPD/tls.crt" -days 365 \
      -extfile <(printf "subjectAltName=DNS:%s,DNS:%s-rw,DNS:%s-rw.%s.svc,DNS:%s-r,DNS:%s-ro" \
        "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$NAMESPACE" \
        "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME")
    kubectl -n "$NAMESPACE" create secret tls "$tls_secret_name" \
      --cert="$TMPD/tls.crt" \
      --key="$TMPD/tls.key" \
      --dry-run=client -o yaml | kubectl apply -f -
  fi

  rm -rf "$TMPD"
}

# Resolve latest CNPG version from GitHub if possible, fallback to a sensible default.
get_latest_cnpg_version() {
  local version
  version=$(curl -s "https://api.github.com/repos/cloudnative-pg/cloudnative-pg/releases/latest" | jq -r '.tag_name' | sed 's/^v//' || echo "")
  if [[ -z "$version" || "$version" == "null" ]]; then
    echo "1.27.0"
  else
    echo "$version"
  fi
}

initialize() {
  ensure_tools
  ensure_namespace
  attempt_backup_if_active
  ensure_cnpg_operator
  pin_cnpg_controller
  ensure_barman_plugin
  echo "Using namespace: $NAMESPACE"
  
  if ! fetch_admin_keys_and_db_pass_from_bao_or_local; then
    echo "ERROR: Failed to fetch/ensure database secrets and admin keys." >&2
    return 1
  fi
  apply_cnpg_admin_secret
  detect_and_reprovision_unencrypted_cluster
  generate_tls_if_missing

  # Pre-flight: verify required secrets exist before deploying cluster
  local _missing_secrets=""
  if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
    _missing_secrets="${_missing_secrets} prole-db-user"
  fi
  if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-tls" >/dev/null 2>&1; then
    _missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-tls"
  fi
  if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-ca" >/dev/null 2>&1; then
    _missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-ca"
  fi
  if [[ -n "$_missing_secrets" ]]; then
    echo "ERROR: Required secrets missing in namespace '$NAMESPACE':${_missing_secrets}" >&2
    echo "The CNPG cluster cannot start without these secrets." >&2
    return 1
  fi
  echo "Pre-flight check passed: all required secrets present."

  if ! _ensure_prole_db_image; then
    echo "ERROR: Pre-flight image check failed; aborting cluster initialization." >&2
    return 1
  fi

  ensure_prole_stack_resources

  if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then
    if cluster_has_pods; then
      echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2
      if force_rollout; then
        if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then
          echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2
          if ! reset_and_reinit; then
            return 1
          fi
        fi
      else
        echo "WARN: Force rollout failed; attempting reset and re-init ..." >&2
        if ! reset_and_reinit; then
          return 1
        fi
      fi
    else
      echo "WARN: No CNPG pods found — this is a new installation." >&2
      echo "The CNPG operator was unable to create cluster pods." >&2
      echo "Check operator logs:  kubectl -n cnpg-system logs -l app.kubernetes.io/name=cloudnative-pg" >&2
      echo "Check cluster status: kubectl -n $NAMESPACE describe cluster $CNPG_CLUSTER_NAME" >&2
      return 1
    fi
  fi

  echo "Initialization complete for CNPG + cert artifacts."
  prole_register_port_forward "postgres" "${NAMESPACE:-default}" "svc/${CNPG_CLUSTER_NAME}-rw" "5432" "5432" "0.0.0.0" "TCP" "PostgreSQL"
}

update_reload() {
  initialize
}

deploy_cluster() {
  ensure_tools
  ensure_namespace
  ensure_cnpg_operator
  pin_cnpg_controller
  ensure_barman_plugin

  local image
  if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then
    image=$(get_latest_image)
  else
    image="prole-db:$VERSION"
  fi
  image=$(resolve_cnpg_image "$image")
  sync_manifest_image "$image"

  generate_tls_if_missing

  echo "Deploying $image to cluster $CNPG_CLUSTER_NAME in namespace $NAMESPACE..."
  _ensure_prole_db_image || true
  apply_cnpg_cluster_manifest "$CNPG_MANIFEST"

  local current_image
  current_image=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "")
  if [[ -n "$current_image" && "$current_image" != "$image" ]]; then
    echo "Patching cluster to use image '$image' (was '$current_image')..."
    kubectl -n "$NAMESPACE" patch cluster "$CNPG_CLUSTER_NAME" --type merge -p "{\"spec\": {\"imageName\": \"$image\"}}"
    if kubectl cnpg version >/dev/null 2>&1; then
      kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
    fi
  fi
}

preflight_image() {
  ensure_tools
  ensure_namespace
  _ensure_prole_db_image
}

case "$ACTION" in
  preflight-image|image-preflight|preflight_image)
    preflight_image
    ;;
  recreate)
    ensure_tools
    "$0" delete "$CNPG_CLUSTER_NAME"
    # Clean up PVCs left behind by the CNPG operator (not in static manifests)
    echo "Cleaning up PVCs in namespace '$NAMESPACE' ..."
    kubectl -n "$NAMESPACE" delete pvc --all --ignore-not-found 2>/dev/null || true
    # Wait briefly for pods to fully terminate before re-creating
    echo "Waiting for pods to terminate in namespace '$NAMESPACE' ..."
    _wait_term=0
    while kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null | grep -qv '^No resources'; do
      sleep 3
      _wait_term=$(( _wait_term + 3 ))
      if (( _wait_term >= 60 )); then
        echo "WARN: Pods still present after 60s; proceeding anyway." >&2
        break
      fi
    done
    "$0" create "$CNPG_CLUSTER_NAME"
    ;;
  create)
    ensure_tools
    ensure_namespace
    initialize
    ;;
  delete)
    ensure_tools
    echo "Deleting all resources for '$CNPG_CLUSTER_NAME' ..."
    # Delete each manifest individually, mirroring the apply pattern:
    # ingress.yaml contains multi-namespace resources and must be deleted without -n.
    for _del_f in "$SCRIPT_DIR/../k8s/prole"/*.yaml; do
      _del_base=$(basename "$_del_f")
      case "$_del_base" in
        kustomization.yaml|ingress.yaml) continue ;;
        *) kubectl delete -n "$NAMESPACE" -f "$_del_f" --ignore-not-found 2>&1 \
             | grep -v "^Error from server (NotFound)" || true ;;
      esac
    done
    if [[ -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" ]]; then
      kubectl delete -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" --ignore-not-found 2>&1 \
        | grep -v "^Error from server (NotFound)" || true
    fi
    ;;
  start)
    ensure_tools
    ensure_namespace
    ensure_cnpg_operator
    pin_cnpg_controller
    ensure_barman_plugin
    if [[ ! -f "$CNPG_MANIFEST" ]]; then
      echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
      exit 1
    fi
    echo "Starting CloudNative-PG cluster from $CNPG_MANIFEST in namespace $NAMESPACE..."
    prole_render_manifest "$CNPG_MANIFEST" | kubectl apply -n "$NAMESPACE" -f -
    ;;
  stop)
    ensure_tools
    if [[ ! -f "$CNPG_MANIFEST" ]]; then
      echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
      exit 1
    fi
    echo "Stopping CloudNative-PG cluster using $CNPG_MANIFEST ..."
    kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found
    ;;
  status)
    ensure_tools
    echo "--- CloudNative-PG Cluster Status ($CNPG_CLUSTER_NAME) ---"
    if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
      kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME"
      echo ""
      echo "CNPG Plugin Status:"
      if kubectl cnpg version >/dev/null 2>&1; then
        kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE"
      else
        echo "Note: 'kubectl cnpg' plugin not found; skipping detailed status."
      fi
    else
      echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'."
    fi
    ;;
  restart)
    ensure_tools
    "$0" stop
    "$0" start
    ;;
  initialize)
    if ! initialize; then
      exit 1
    fi
    ;;
  update|reload)
    if ! update_reload; then
      exit 1
    fi
    ;;
  deploy)
    deploy_cluster
    ;;
  rollout|force-rollout)
    rollout_cluster
    ;;
  install-barman-plugin)
    ensure_tools
    ensure_cnpg_operator
    pin_cnpg_controller
    ensure_barman_plugin
    ;;
  *)
    echo "Usage: $0 {create|delete|recreate|start|stop|status|restart|initialize|update|reload|deploy|rollout|install-barman-plugin} [cluster] [version]" >&2
    exit 2
    ;;
esac