local token url if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then token=$(cat "$OPENBAO_TOKEN_FILE") else token="" fi url=$(bao_service_url) if [[ -n "$token" && -n "$url" ]]; then echo "Attempting to read admin key pair from OpenBao kv/$BAO_PATH_ADMIN ..." if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -e '.data.data' >/dev/null 2>&1; then local priv_b64 pub_b64 priv_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_private_key_b64') pub_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_public_key_b64') # Use a temporary file to determine where to save based on existing legacy or generic preference local target_priv="$ADMIN_PRIV_GENERIC" local target_pub="$ADMIN_PUB_GENERIC" # If legacy keys exist, we might want to overwrite them too for compatibility printf "%s" "$priv_b64" | base64 -d >"$target_priv" printf "%s" "$pub_b64" | base64 -d >"$target_pub" chmod 0600 "$target_priv" # Mirror to legacy path if it was expected by other scripts cp "$target_priv" "$ADMIN_PRIV_ED25519" 2>/dev/null || true cp "$target_pub" "$ADMIN_PUB_ED25519" 2>/dev/null || true fi echo "Attempting to read database password from OpenBao kv/$BAO_PATH_DB ..." if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -e '.data.data' >/dev/null 2>&1; then local db_pass db_pass=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -r '.data.data.password') if [[ -n "$db_pass" ]]; then echo "Updating database user secret 'prole-db-user' from OpenBao ..." kubectl create secret generic prole-db-user -n "$NAMESPACE" \ --from-literal=username=prole \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - echo "Updating database superuser secret 'prole-db-superuser' from OpenBao ..." kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \ --from-literal=username=postgres \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - fi fi fi # Fallback: use DB_PASSWORD from env if OpenBao is unreachable or empty if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then local db_pass="${DB_PASSWORD:-}" if [[ -n "$db_pass" && "$db_pass" != '${OPENBAO:'* && "$db_pass" != '${PROLE_SECRET:'* ]]; then echo "Updating database user secret 'prole-db-user' from DB_PASSWORD env ..." kubectl create secret generic prole-db-user -n "$NAMESPACE" \ --from-literal=username=prole \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - echo "Updating database superuser secret 'prole-db-superuser' from DB_PASSWORD env ..." kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \ --from-literal=username=postgres \ --from-literal=password="$db_pass" \ --dry-run=client -o yaml | kubectl apply -f - fi fi if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then echo "ERROR: 'prole-db-user' secret is missing in namespace '$NAMESPACE' and could not be resolved from OpenBao or DB_PASSWORD env." >&2 return 1 fi ensure_local_admin_keypair if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then echo "Using local admin key pair at $SECRETS_DIR" return 0 fi echo "ERROR: Could not obtain admin key pair from OpenBao and could not generate local keys." >&2 return 1 } apply_cnpg_admin_secret() { echo "Creating/updating Secret cnpg-admin-key ..." kubectl create secret generic cnpg-admin-key -n "$NAMESPACE" \ --from-file=admin.key="$ADMIN_PRIV_GENERIC" \ --from-file=admin.pub="$ADMIN_PUB_GENERIC" \ --dry-run=client -o yaml | kubectl apply -f - } detect_and_reprovision_unencrypted_cluster() { local encryption_enabled="${AT_REST_ENCRYPTION_ENABLED:-false}" if [[ "$encryption_enabled" != "true" && "$encryption_enabled" != "True" && "$encryption_enabled" != "1" ]]; then return 0 fi if ! kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then return 0 fi local server_tls server_tls=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.certificates.serverTLSSecret}' 2>/dev/null || echo "") if [[ -n "$server_tls" ]]; then echo "Existing cluster '$CNPG_CLUSTER_NAME' already has TLS configured ($server_tls); no conversion needed." return 0 fi echo "WARN: Existing cluster '$CNPG_CLUSTER_NAME' does not have TLS/at-rest-encryption configured." echo "At-rest encryption is now enabled; the cluster must be reprovisioned." echo "NOTE: External full backup and restore from barman backup is not yet available." echo " Removing existing cluster to allow re-creation with encryption enabled ..." attempt_backup_if_active kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true echo "Existing unencrypted cluster removed. Will be re-created with TLS and encryption." } validate_ca_secret() { local secret_name="$1" local ns="$2" if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then return 1 fi local crt key crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true) key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.key}' 2>/dev/null || true) if [[ -z "$crt" || -z "$key" ]]; then echo "WARN: CA secret '$secret_name' exists but is missing ca.crt or ca.key data." >&2 return 1 fi if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then echo "WARN: CA secret '$secret_name' has invalid certificate data." >&2 return 1 fi # CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8 local key_header key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1) if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then echo "WARN: CA secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2 return 1 fi return 0 } validate_tls_secret() { local secret_name="$1" local ns="$2" if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then return 1 fi local crt key crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true) key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.key}' 2>/dev/null || true) if [[ -z "$crt" || -z "$key" ]]; then echo "WARN: TLS secret '$secret_name' exists but is missing tls.crt or tls.key data." >&2 return 1 fi if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then echo "WARN: TLS secret '$secret_name' has invalid certificate data." >&2 return 1 fi # CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8 local key_header key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1) if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then echo "WARN: TLS secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2 return 1 fi return 0 } generate_tls_if_missing() { local ca_secret_name="${CNPG_CLUSTER_NAME}-ca" local tls_secret_name="${CNPG_CLUSTER_NAME}-tls" local need_ca=0 local need_tls=0 if validate_ca_secret "$ca_secret_name" "$NAMESPACE"; then echo "CA secret $ca_secret_name already exists and is valid." else if kubectl -n "$NAMESPACE" get secret "$ca_secret_name" >/dev/null 2>&1; then echo "Removing invalid CA secret '$ca_secret_name' ..." kubectl -n "$NAMESPACE" delete secret "$ca_secret_name" --ignore-not-found fi need_ca=1 fi if validate_tls_secret "$tls_secret_name" "$NAMESPACE"; then echo "TLS secret $tls_secret_name already exists and is valid." else if kubectl -n "$NAMESPACE" get secret "$tls_secret_name" >/dev/null 2>&1; then echo "Removing invalid TLS secret '$tls_secret_name' ..." kubectl -n "$NAMESPACE" delete secret "$tls_secret_name" --ignore-not-found fi need_tls=1 fi if [[ "$need_ca" -eq 0 && "$need_tls" -eq 0 ]]; then echo "TLS secrets already present and valid; skipping generation." return 0 fi local TMPD TMPD=$(mktemp -d) if [[ "$need_ca" -eq 1 ]]; then echo "Generating self-signed CA (EC P-256) for CNPG ..." openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/ca.key" openssl req -x509 -new -key "$TMPD/ca.key" -out "$TMPD/ca.crt" -days 3650 -subj "/CN=Prole CNPG CA" kubectl -n "$NAMESPACE" create secret generic "$ca_secret_name" \ --from-file=ca.crt="$TMPD/ca.crt" \ --from-file=ca.key="$TMPD/ca.key" \ --dry-run=client -o yaml | kubectl apply -f - else # Extract existing CA for signing the server certificate kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.crt}' | base64 -d > "$TMPD/ca.crt" kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.key}' | base64 -d > "$TMPD/ca.key" fi if [[ "$need_tls" -eq 1 ]]; then echo "Generating server TLS certificate for CNPG ($tls_secret_name) ..." openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/tls.key" openssl req -new -key "$TMPD/tls.key" -out "$TMPD/tls.csr" \ -subj "/CN=${CNPG_CLUSTER_NAME}.${NAMESPACE}.svc" openssl x509 -req -in "$TMPD/tls.csr" -CA "$TMPD/ca.crt" -CAkey "$TMPD/ca.key" \ -CAcreateserial -out "$TMPD/tls.crt" -days 365 \ -extfile <(printf "subjectAltName=DNS:%s,DNS:%s-rw,DNS:%s-rw.%s.svc,DNS:%s-r,DNS:%s-ro" \ "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$NAMESPACE" \ "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME") kubectl -n "$NAMESPACE" create secret tls "$tls_secret_name" \ --cert="$TMPD/tls.crt" \ --key="$TMPD/tls.key" \ --dry-run=client -o yaml | kubectl apply -f - fi rm -rf "$TMPD" } # Resolve latest CNPG version from GitHub if possible, fallback to a sensible default. get_latest_cnpg_version() { local version version=$(curl -s "https://api.github.com/repos/cloudnative-pg/cloudnative-pg/releases/latest" | jq -r '.tag_name' | sed 's/^v//' || echo "") if [[ -z "$version" || "$version" == "null" ]]; then echo "1.27.0" else echo "$version" fi } initialize() { ensure_tools ensure_namespace attempt_backup_if_active ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin echo "Using namespace: $NAMESPACE" if ! fetch_admin_keys_and_db_pass_from_bao_or_local; then echo "ERROR: Failed to fetch/ensure database secrets and admin keys." >&2 return 1 fi apply_cnpg_admin_secret detect_and_reprovision_unencrypted_cluster generate_tls_if_missing # Pre-flight: verify required secrets exist before deploying cluster local _missing_secrets="" if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then _missing_secrets="${_missing_secrets} prole-db-user" fi if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-tls" >/dev/null 2>&1; then _missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-tls" fi if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-ca" >/dev/null 2>&1; then _missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-ca" fi if [[ -n "$_missing_secrets" ]]; then echo "ERROR: Required secrets missing in namespace '$NAMESPACE':${_missing_secrets}" >&2 echo "The CNPG cluster cannot start without these secrets." >&2 return 1 fi echo "Pre-flight check passed: all required secrets present." if ! _ensure_prole_db_image; then echo "ERROR: Pre-flight image check failed; aborting cluster initialization." >&2 return 1 fi ensure_prole_stack_resources if ! wait_for_cnpg_pods 300; then if cluster_has_pods; then echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2 if force_rollout; then if ! wait_for_cnpg_pods 300; then echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2 if ! reset_and_reinit; then return 1 fi fi else echo "WARN: Force rollout failed; attempting reset and re-init ..." >&2 if ! reset_and_reinit; then return 1 fi fi else echo "WARN: No CNPG pods found — this is a new installation." >&2 echo "The CNPG operator was unable to create cluster pods." >&2 echo "Check operator logs: kubectl -n cnpg-system logs -l app.kubernetes.io/name=cloudnative-pg" >&2 echo "Check cluster status: kubectl -n $NAMESPACE describe cluster $CNPG_CLUSTER_NAME" >&2 return 1 fi fi echo "Initialization complete for CNPG + cert artifacts." prole_register_port_forward "postgres" "${NAMESPACE:-default}" "svc/${CNPG_CLUSTER_NAME}-rw" "5432" "5432" "0.0.0.0" "TCP" "PostgreSQL" } update_reload() { initialize } deploy_cluster() { ensure_tools ensure_namespace ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin local image if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then image=$(get_latest_image) else image="prole-db:$VERSION" fi image=$(resolve_cnpg_image "$image") sync_manifest_image "$image" generate_tls_if_missing echo "Deploying $image to cluster $CNPG_CLUSTER_NAME in namespace $NAMESPACE..." _ensure_prole_db_image || true apply_cnpg_cluster_manifest "$CNPG_MANIFEST" local current_image current_image=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "") if [[ -n "$current_image" && "$current_image" != "$image" ]]; then echo "Patching cluster to use image '$image' (was '$current_image')..." kubectl -n "$NAMESPACE" patch cluster "$CNPG_CLUSTER_NAME" --type merge -p "{\"spec\": {\"imageName\": \"$image\"}}" if kubectl cnpg version >/dev/null 2>&1; then kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true fi fi } case "$ACTION" in recreate) ensure_tools "$0" delete "$CNPG_CLUSTER_NAME" # Clean up PVCs left behind by the CNPG operator (not in static manifests) echo "Cleaning up PVCs in namespace '$NAMESPACE' ..." kubectl -n "$NAMESPACE" delete pvc --all --ignore-not-found 2>/dev/null || true # Wait briefly for pods to fully terminate before re-creating echo "Waiting for pods to terminate in namespace '$NAMESPACE' ..." _wait_term=0 while kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null | grep -qv '^No resources'; do sleep 3 _wait_term=$(( _wait_term + 3 )) if (( _wait_term >= 60 )); then echo "WARN: Pods still present after 60s; proceeding anyway." >&2 break fi done "$0" create "$CNPG_CLUSTER_NAME" ;; create) ensure_tools ensure_namespace initialize ;; delete) ensure_tools echo "Deleting all resources for '$CNPG_CLUSTER_NAME' ..." # Delete each manifest individually, mirroring the apply pattern: # ingress.yaml contains multi-namespace resources and must be deleted without -n. for _del_f in "$SCRIPT_DIR/../k8s/prole"/*.yaml; do _del_base=$(basename "$_del_f") case "$_del_base" in kustomization.yaml|ingress.yaml) continue ;; *) kubectl delete -n "$NAMESPACE" -f "$_del_f" --ignore-not-found 2>&1 \ | grep -v "^Error from server (NotFound)" || true ;; esac done if [[ -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" ]]; then kubectl delete -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" --ignore-not-found 2>&1 \ | grep -v "^Error from server (NotFound)" || true fi ;; start) ensure_tools ensure_namespace ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin if [[ ! -f "$CNPG_MANIFEST" ]]; then echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2 exit 1 fi echo "Starting CloudNative-PG cluster from $CNPG_MANIFEST in namespace $NAMESPACE..." prole_render_manifest "$CNPG_MANIFEST" | kubectl apply -n "$NAMESPACE" -f - ;; stop) ensure_tools if [[ ! -f "$CNPG_MANIFEST" ]]; then echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2 exit 1 fi echo "Stopping CloudNative-PG cluster using $CNPG_MANIFEST ..." kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found ;; status) ensure_tools echo "--- CloudNative-PG Cluster Status ($CNPG_CLUSTER_NAME) ---" if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" echo "" echo "CNPG Plugin Status:" if kubectl cnpg version >/dev/null 2>&1; then kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" else echo "Note: 'kubectl cnpg' plugin not found; skipping detailed status." fi else echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'." fi ;; restart) ensure_tools "$0" stop "$0" start ;; initialize) if ! initialize; then exit 1 fi ;; update|reload) if ! update_reload; then exit 1 fi ;; deploy) deploy_cluster ;; rollout|force-rollout) rollout_cluster ;; install-barman-plugin) ensure_tools ensure_cnpg_operator pin_cnpg_controller ensure_barman_plugin ;; *) echo "Usage: $0 {create|delete|recreate|start|stop|status|restart|initialize|update|reload|deploy|rollout|install-barman-plugin} [cluster] [version]" >&2 exit 2 ;; esac