#!/usr/bin/env bash
set -euo pipefail
# init_cloudnative_pg.sh
# Purpose:
# - Distribute administrator ed25519 key pair to CloudNative‑PG as a Kubernetes Secret for cert auth
# - Configure Kerberos (GSSAPI) using external Kerberos KDC
# - Patch CNPG cluster to enable TLS and GSSAPI where possible
#
# Usage:
# ./init_cloudnative_pg.sh start|stop|status|restart
# ./init_cloudnative_pg.sh initialize # install CNPG, create cluster, configure secrets + Kerberos
# ./init_cloudnative_pg.sh update|reload # re-apply/patch
#
# Requirements:
# - init_openbao.sh has been run (OpenBao running as a local container)
# - $PROLE_HOME/env.sh or $HOME/.prole/env.sh defining PROLE_SERVICE
# - Optional: CNPG_MANIFEST_OVERRIDE to apply a recovery manifest instead of kustomize
# Initialize SCRIPT_DIR
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Load environment and config via prole_cfg.sh
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"
if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
prole_set_mode "${2:-}"
shift 2
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
prole_set_mode "${1#*=}"
shift
fi
if [[ -z "${PROLE_SERVICE:-}" ]]; then
echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2
exit 1
fi
ACTION=${1:-}
CNPG_CLUSTER_NAME=${2:-${CNPG_CLUSTER_NAME:-prole-db}}
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
REALM=${REALM:-PROLE.ORG}
DOMAIN=${DOMAIN:-prole.org}
KRB5_KDC=${KRB5_KDC:-}
KRB5_ADMIN=${KRB5_ADMIN:-}
CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-}
PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}
BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup}
BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800}
RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl"
CNPG_FORCE_ROLLOUT_SCRIPT="$SCRIPT_DIR/init_prole-db.sh"
BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-}
BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0}
CERT_MANAGER_MANIFEST_URL=${CERT_MANAGER_MANIFEST_URL:-}
CERT_MANAGER_FALLBACK_VERSION=${CERT_MANAGER_FALLBACK_VERSION:-1.19.3}
# Support both PROLE_HOME/k8s and sibling k8s directory
if [[ -d "$SCRIPT_DIR/../k8s/prole" ]]; then
K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole"
elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/prole" ]]; then
K8S_PROLE_DIR="$PROLE_HOME/k8s/prole"
else
K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole"
fi
CNPG_MANIFEST="$K8S_PROLE_DIR/prole-db.yaml"
BARMAN_OBJECTSTORE_MANIFEST="$K8S_PROLE_DIR/prole-db-barman-objectstore.yaml"
SECRETS_DIR="$PROLE_SERVICE/secrets"
# Resolving CNPG admin keys.
# We prefer names without algorithm suffixes to be more generic, matching install.py fallback strategy.
ADMIN_PRIV_ED25519="$SECRETS_DIR/admin_ed25519.key"
ADMIN_PUB_ED25519="$SECRETS_DIR/admin_ed25519.pub"
ADMIN_PRIV_GENERIC="$SECRETS_DIR/admin.key"
ADMIN_PUB_GENERIC="$SECRETS_DIR/admin.pub"
OPENBAO_TOKEN_FILE="$SECRETS_DIR/openbao-root-token"
BAO_NAMESPACE="${NAMESPACE:-default}"
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
BAO_PATH_DB="${BAO_PATH_PREFIX}/db"
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"
ensure_tools() {
for t in kubectl curl openssl base64 jq; do
command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
done
}
ensure_namespace() {
if [[ -z "${NAMESPACE:-}" ]]; then
echo "ERROR: NAMESPACE is empty. Check env.sh or conf/prole.cfg." >&2
exit 1
fi
if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
echo "Creating namespace '$NAMESPACE' ..."
kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true
fi
}
openbao_url() {
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
echo "$PROLE_OPENBAO_URL"
elif curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:18200"
else
echo "http://openbao.${NAMESPACE}.svc.cluster.local:8200"
fi
}
openbao_token() {
if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
cat "$OPENBAO_TOKEN_FILE"
else
echo "${OPENBAO_ROOT_TOKEN:-}"
fi
}
fetch_openbao_secret() {
local path="$1"
local key="$2"
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -z "$token" ]]; then
echo ""
return 0
fi
curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
}
resolve_db_password() {
local db_pw="${DB_PASSWORD:-}"
if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
local fetched_db
fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password")
if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
db_pw="$fetched_db"
fi
fi
printf '%s' "$db_pw"
}
ensure_grafana_admin_secret() {
local db_pw
db_pw=$(resolve_db_password)
if [[ -z "$db_pw" ]]; then
echo "ERROR: DB_PASSWORD unavailable; cannot create Grafana admin secret." >&2
return 1
fi
kubectl create secret generic grafana-admin \
-n "$NAMESPACE" \
--from-literal=admin-password="$db_pw" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -n "$token" ]]; then
curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
-X POST "$url/v1/kv/data/$BAO_PATH_MONITORING" \
-d "{\"data\":{\"grafana_admin_password\":\"$db_pw\"}}" >/dev/null || true
fi
}
ensure_cnpg_operator() {
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
wait_for_cnpg_webhook 180 || true
return 0
fi
local latest_version minor_version yaml_url
latest_version=$(get_latest_cnpg_version)
minor_version=$(echo "$latest_version" | cut -d. -f1,2)
yaml_url="https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${minor_version}/releases/cnpg-${latest_version}.yaml"
echo "Installing CloudNative-PG operator version ${latest_version} ..."
kubectl apply --server-side -f "$yaml_url"
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
wait_for_cnpg_webhook 180 || true
fi
}
get_latest_barman_plugin_version() {
local version tag
tag=$(curl -s "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "")
if [[ -z "$tag" || "$tag" == "null" ]]; then
echo "v${BARMAN_PLUGIN_FALLBACK_VERSION}"
return 0
fi
echo "$tag"
}
resolve_barman_plugin_manifest_url() {
if [[ -n "$BARMAN_PLUGIN_MANIFEST_URL" ]]; then
echo "$BARMAN_PLUGIN_MANIFEST_URL"
return 0
fi
local tag
tag=$(get_latest_barman_plugin_version)
echo "https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/${tag}/manifest.yaml"
}
cert_manager_ready() {
kubectl get crd certificates.cert-manager.io >/dev/null 2>&1
}
get_latest_cert_manager_version() {
local tag
tag=$(curl -s "https://api.github.com/repos/cert-manager/cert-manager/releases/latest" | jq -r '.tag_name' || echo "")
if [[ -z "$tag" || "$tag" == "null" ]]; then
echo "v${CERT_MANAGER_FALLBACK_VERSION}"
return 0
fi
echo "$tag"
}
resolve_cert_manager_manifest_url() {
if [[ -n "$CERT_MANAGER_MANIFEST_URL" ]]; then
echo "$CERT_MANAGER_MANIFEST_URL"
return 0
fi
local tag
tag=$(get_latest_cert_manager_version)
echo "https://github.com/cert-manager/cert-manager/releases/download/${tag}/cert-manager.yaml"
}
ensure_cert_manager() {
if cert_manager_ready; then
if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true
kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true
kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true
fi
return 0
fi
local cm_url
cm_url=$(resolve_cert_manager_manifest_url)
echo "Installing cert-manager from ${cm_url} ..."
kubectl apply -f "$cm_url"
if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true
kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true
kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true
fi
}
wait_for_barman_crd() {
local timeout=${1:-120}
local start_time
start_time=$(date +%s)
while true; do
if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then
return 0
fi
if (( $(date +%s) - start_time > timeout )); then
return 1
fi
sleep 5
done
}
ensure_barman_plugin() {
local crd_present deploy_present
crd_present=false
deploy_present=false
if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then
crd_present=true
fi
if kubectl -n cnpg-system get deploy barman-cloud >/dev/null 2>&1; then
deploy_present=true
fi
if [[ "$crd_present" == "true" && "$deploy_present" == "true" ]]; then
kubectl -n cnpg-system rollout status deploy/barman-cloud --timeout=180s || true
return 0
fi
ensure_cert_manager
local plugin_url
plugin_url=$(resolve_barman_plugin_manifest_url)
echo "Installing Barman Cloud plugin from ${plugin_url} ..."
kubectl apply -f "$plugin_url"
if ! wait_for_barman_crd 120; then
echo "WARN: Barman Cloud ObjectStore CRD not ready after install." >&2
fi
if kubectl -n cnpg-system get deploy barman-cloud >/dev/null 2>&1; then
kubectl -n cnpg-system rollout status deploy/barman-cloud --timeout=180s || true
fi
}
wait_for_cnpg_webhook() {
local timeout=${1:-120}
local start_time
start_time=$(date +%s)
echo "Waiting for CNPG webhook service endpoints to be ready..."
while true; do
local endpoints
endpoints=$(kubectl -n cnpg-system get endpoints cnpg-webhook-service -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
if [[ -n "$endpoints" ]]; then
echo "CNPG webhook service has endpoints."
return 0
fi
if (( $(date +%s) - start_time > timeout )); then
echo "WARN: CNPG webhook endpoints not ready after ${timeout}s."
return 1
fi
sleep 5
done
}
apply_barman_objectstore_if_present() {
if [[ ! -f "$BARMAN_OBJECTSTORE_MANIFEST" ]]; then
return 0
fi
if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then
kubectl apply -n "$NAMESPACE" -f "$BARMAN_OBJECTSTORE_MANIFEST"
else
echo "WARN: Barman Cloud ObjectStore CRD not found; skipping $BARMAN_OBJECTSTORE_MANIFEST."
fi
}
ensure_prole_stack_resources() {
echo "Applying CloudNative-PG cluster and related resources ..."
ensure_grafana_admin_secret
if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then
if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then
echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2
return 1
fi
local dir file
dir="$K8S_PROLE_DIR"
for file in "$dir"/*.yaml; do
case "$(basename "$file")" in
prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml)
continue
;;
esac
kubectl apply -n "$NAMESPACE" -f "$file"
done
apply_barman_objectstore_if_present
apply_cnpg_cluster_manifest "$CNPG_MANIFEST_OVERRIDE"
else
local dir file
dir="$K8S_PROLE_DIR"
for file in "$dir"/*.yaml; do
case "$(basename "$file")" in
prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml)
continue
;;
esac
kubectl apply -n "$NAMESPACE" -f "$file"
done
apply_barman_objectstore_if_present
apply_cnpg_cluster_manifest "$CNPG_MANIFEST"
fi
# Ensure prole-index-html exists for prole deployment readiness probe
if ! kubectl get configmap prole-index-html -n "$NAMESPACE" >/dev/null 2>&1; then
echo "Creating prole-index-html configmap..."
printf "
Prole
" > /tmp/index.html
kubectl create configmap prole-index-html --from-file=/tmp/index.html -n "$NAMESPACE"
rm /tmp/index.html
fi
# Ensure prole-nginx-tls exists (self-signed for dev)
if ! kubectl get secret prole-nginx-tls -n "$NAMESPACE" >/dev/null 2>&1; then
echo "Generating self-signed prole-nginx-tls for development..."
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
-keyout /tmp/nginx-tls.key -out /tmp/nginx-tls.crt \
-subj "/CN=prole.org" >/dev/null 2>&1
kubectl create secret tls prole-nginx-tls --key /tmp/nginx-tls.key --cert /tmp/nginx-tls.crt -n "$NAMESPACE"
rm /tmp/nginx-tls.key /tmp/nginx-tls.crt
fi
}
apply_cnpg_cluster_manifest() {
local manifest="$1"
local attempts=${CNPG_APPLY_RETRIES:-6}
local i out
for ((i=1; i<=attempts; i++)); do
if out=$(kubectl apply -n "$NAMESPACE" -f "$manifest" 2>&1); then
printf '%s\n' "$out"
return 0
fi
if echo "$out" | grep -q "cnpg-webhook-service"; then
echo "CNPG webhook not ready yet (attempt $i/$attempts). Retrying..."
sleep 5
continue
fi
echo "$out" >&2
return 1
done
echo "ERROR: Failed to apply CNPG manifest after $attempts attempts." >&2
echo "$out" >&2
return 1
}
wait_for_cnpg_pods() {
local timeout=${1:-300}
local start_time
start_time=$(date +%s)
local target_pods="${CNPG_TARGET_PODS:-}"
if [[ -z "$target_pods" ]]; then
target_pods=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.instances}' 2>/dev/null || true)
fi
if [[ -z "$target_pods" ]]; then
target_pods=3
fi
echo "Waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' to be Running..."
while true; do
local pods
pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null || true)
if [[ -n "$pods" ]]; then
# Check for Error or CrashLoopBackOff
if echo "$pods" | grep -E "Error|CrashLoopBackOff" >/dev/null; then
echo "ERROR: Some CNPG pods are in Error or CrashLoopBackOff state:" >&2
echo "$pods" | grep -E "Error|CrashLoopBackOff" >&2
return 1
fi
# Count Running pods by name (exclude initdb)
local running_pods
running_pods=$(kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null \
| grep "^${CNPG_CLUSTER_NAME}-" \
| grep -v "initdb" \
| grep "Running" \
| wc -l | xargs)
if [[ "$running_pods" -ge "$target_pods" ]]; then
echo "All $running_pods/$target_pods pods are Running."
return 0
fi
fi
if (( $(date +%s) - start_time > timeout )); then
echo "ERROR: Timed out waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE'." >&2
return 1
fi
sleep 5
done
}
cluster_exists() {
kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1
}
cluster_has_pods() {
kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null | grep -q .
}
latest_backup_name() {
kubectl -n "$NAMESPACE" get backup \
--sort-by=.metadata.creationTimestamp \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1
}
latest_completed_backup_name() {
kubectl -n "$NAMESPACE" get backup \
--sort-by=.metadata.creationTimestamp \
-o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null | \
awk -F'|' '{p=tolower($2); if (p=="completed" || p=="succeeded") {name=$1}} END {print name}'
}
wait_for_backup() {
local backup_name="$1"
local start_time now phase phase_lc
start_time=$(date +%s)
while true; do
phase=$(kubectl -n "$NAMESPACE" get backup "$backup_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
phase_lc=$(printf '%s' "$phase" | tr '[:upper:]' '[:lower:]')
case "$phase_lc" in
completed|succeeded)
echo "Backup $backup_name completed."
return 0
;;
failed|error)
echo "Backup $backup_name failed (phase=$phase)." >&2
return 1
;;
esac
now=$(date +%s)
if (( now - start_time > BACKUP_WAIT_TIMEOUT )); then
echo "Timed out waiting for backup $backup_name." >&2
return 1
fi
echo "Waiting for backup $backup_name to complete (phase=${phase:-unknown}) ..."
sleep 10
done
}
run_garage_backup() {
if [[ ! -x "$SCRIPT_DIR/init_prole-db-backup.sh" ]]; then
echo "WARN: init_prole-db-backup.sh not found; skipping Garage backup." >&2
return 1
fi
echo "Running Garage backup via init_prole-db-backup.sh ..."
if ! "$SCRIPT_DIR/init_prole-db-backup.sh" start; then
echo "Garage backup script failed." >&2
return 1
fi
sleep 2
local backup_name
backup_name=$(latest_backup_name)
if [[ -z "$backup_name" ]]; then
echo "No backup resource detected after triggering backup." >&2
return 1
fi
wait_for_backup "$backup_name"
}
get_primary_pod() {
local primary
primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
if [[ -z "$primary" ]]; then
primary=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
fi
printf '%s' "$primary"
}
decode_b64() {
local data="$1"
if [[ -z "$data" ]]; then
return 1
fi
printf '%s' "$data" | base64 -d 2>/dev/null
}
resolve_db_credentials() {
local user_b64 pass_b64 user pass
user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.username}' 2>/dev/null || true)
pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.password}' 2>/dev/null || true)
user=$(decode_b64 "$user_b64" || true)
pass=$(decode_b64 "$pass_b64" || true)
if [[ -z "$user" || -z "$pass" ]]; then
user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.username}' 2>/dev/null || true)
pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.password}' 2>/dev/null || true)
user=$(decode_b64 "$user_b64" || true)
pass=$(decode_b64 "$pass_b64" || true)
fi
if [[ -z "$user" || -z "$pass" ]]; then
return 1
fi
printf '%s\n%s' "$user" "$pass"
}
pgdump_local() {
local pod user pass db_name dump_dir dump_file timestamp
pod=$(get_primary_pod)
if [[ -z "$pod" ]]; then
echo "ERROR: No CNPG pod available for pg_dump." >&2
return 1
fi
local creds
if ! creds=$(resolve_db_credentials); then
echo "ERROR: Unable to resolve database credentials for pg_dump." >&2
return 1
fi
user=$(printf '%s' "$creds" | sed -n '1p')
pass=$(printf '%s' "$creds" | sed -n '2p')
db_name=${PROLE_DB_NAME:-prole-db}
dump_dir="$BACKUP_DIR"
mkdir -p "$dump_dir"
timestamp=$(date +%Y%m%d%H%M%S)
dump_file="$dump_dir/${CNPG_CLUSTER_NAME}-pgdump-${timestamp}.dump"
echo "Running pg_dump against pod $pod (db=$db_name) ..."
if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- env PGPASSWORD="$pass" \
pg_dump -U "$user" -d "$db_name" -Fc > "$dump_file"; then
echo "pg_dump saved to $dump_file"
return 0
fi
echo "pg_dump failed; removing partial file." >&2
rm -f "$dump_file"
return 1
}
attempt_backup_if_active() {
if ! cluster_exists; then
return 0
fi
if ! cluster_has_pods; then
echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no pods detected; skipping backup." >&2
return 0
fi
echo "CNPG cluster '$CNPG_CLUSTER_NAME' detected; attempting Garage backup ..."
if run_garage_backup; then
return 0
fi
echo "Garage backup failed; attempting local pg_dump ..." >&2
if pgdump_local; then
return 0
fi
echo "WARN: Both Garage backup and local pg_dump failed." >&2
return 0
}
force_rollout() {
if [[ ! -x "$CNPG_FORCE_ROLLOUT_SCRIPT" ]]; then
echo "WARN: Force rollout script not found at $CNPG_FORCE_ROLLOUT_SCRIPT" >&2
return 1
fi
echo "Attempting force rollout of CNPG pods ..."
"$CNPG_FORCE_ROLLOUT_SCRIPT" rollout
}
reset_and_reinit() {
local backup_name tmp_manifest
backup_name=$(latest_completed_backup_name)
echo "Resetting CNPG cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' ..."
kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true
if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then
tmp_manifest=$(mktemp)
sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest"
echo "Re-initializing from backup $backup_name ..."
CNPG_MANIFEST_OVERRIDE="$tmp_manifest" ensure_prole_stack_resources
rm -f "$tmp_manifest"
else
if [[ -n "$backup_name" && ! -f "$RECOVERY_TEMPLATE" ]]; then
echo "WARN: Recovery template not found: $RECOVERY_TEMPLATE" >&2
fi
if [[ -z "$backup_name" ]]; then
echo "No completed backups found; starting fresh initialization." >&2
fi
ensure_prole_stack_resources
fi
wait_for_cnpg_pods 300
}
# Resolve OpenBao URL: prefer explicit env, then localhost port-forward, then cluster DNS
bao_service_url() {
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
echo "$PROLE_OPENBAO_URL"
return 0
fi
# Prefer standard local port-forward managed by etc/init_port_forwards.sh
if curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:18200"
return 0
fi
echo "http://$OPENBAO_NAME.$NAMESPACE.svc.cluster.local:8200"
}
fetch_admin_keys_and_db_pass_from_bao_or_local() {
local token url
if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
token=$(cat "$OPENBAO_TOKEN_FILE")
else
token=""
fi
url=$(bao_service_url)
if [[ -n "$token" ]]; then
echo "Attempting to read admin key pair from OpenBao kv/$BAO_PATH_ADMIN ..."
if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -e '.data.data' >/dev/null 2>&1; then
local priv_b64 pub_b64
priv_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_private_key_b64')
pub_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_public_key_b64')
# Use a temporary file to determine where to save based on existing legacy or generic preference
local target_priv="$ADMIN_PRIV_GENERIC"
local target_pub="$ADMIN_PUB_GENERIC"
# If legacy keys exist, we might want to overwrite them too for compatibility
printf "%s" "$priv_b64" | base64 -d >"$target_priv"
printf "%s" "$pub_b64" | base64 -d >"$target_pub"
chmod 0600 "$target_priv"
# Mirror to legacy path if it was expected by other scripts
cp "$target_priv" "$ADMIN_PRIV_ED25519" 2>/dev/null || true
cp "$target_pub" "$ADMIN_PUB_ED25519" 2>/dev/null || true
fi
echo "Attempting to read database password from OpenBao kv/$BAO_PATH_DB ..."
if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -e '.data.data' >/dev/null 2>&1; then
local db_pass
db_pass=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -r '.data.data.password')
if [[ -n "$db_pass" ]]; then
echo "Updating database user secret 'prole-db-user' from OpenBao ..."
kubectl create secret generic prole-db-user -n "$NAMESPACE" \
--from-literal=username=prole \
--from-literal=password="$db_pass" \
--dry-run=client -o yaml | kubectl apply -f -
echo "Updating database superuser secret 'prole-db-superuser' from OpenBao ..."
kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \
--from-literal=username=postgres \
--from-literal=password="$db_pass" \
--dry-run=client -o yaml | kubectl apply -f -
fi
fi
fi
if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then
echo "Using local admin key pair at $SECRETS_DIR"
return 0
fi
if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then
echo "Using local legacy admin key pair at $SECRETS_DIR"
# Link or copy to generic for consistent use below
cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC"
cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC"
return 0
fi
echo "ERROR: Could not obtain admin key pair from OpenBao and no local files found." >&2
exit 1
}
apply_cnpg_admin_secret() {
echo "Creating/updating Secret cnpg-admin-key ..."
kubectl create secret generic cnpg-admin-key -n "$NAMESPACE" \
--from-file=admin.key="$ADMIN_PRIV_GENERIC" \
--from-file=admin.pub="$ADMIN_PUB_GENERIC" \
--dry-run=client -o yaml | kubectl apply -f -
}
generate_tls_if_missing() {
local ca_secret_name="${CNPG_CLUSTER_NAME}-ca"
if kubectl -n "$NAMESPACE" get secret "$ca_secret_name" >/dev/null 2>&1; then
echo "CA secret $ca_secret_name already exists; skipping generation."
return 0
fi
echo "Generating self-signed CA (RSA 4096) for CNPG ..."
local TMPD
TMPD=$(mktemp -d)
openssl genrsa -out "$TMPD/ca.key" 4096
openssl req -x509 -new -key "$TMPD/ca.key" -out "$TMPD/ca.crt" -days 3650 -subj "/CN=Prole CNPG CA"
kubectl -n "$NAMESPACE" create secret generic "$ca_secret_name" \
--from-file=ca.crt="$TMPD/ca.crt" \
--from-file=ca.key="$TMPD/ca.key" \
--dry-run=client -o yaml | kubectl apply -f -
rm -rf "$TMPD"
}
# Resolve latest CNPG version from GitHub if possible, fallback to a sensible default.
get_latest_cnpg_version() {
local version
version=$(curl -s "https://api.github.com/repos/cloudnative-pg/cloudnative-pg/releases/latest" | jq -r '.tag_name' | sed 's/^v//' || echo "")
if [[ -z "$version" || "$version" == "null" ]]; then
echo "1.27.0"
else
echo "$version"
fi
}
initialize() {
ensure_tools
ensure_namespace
attempt_backup_if_active
ensure_cnpg_operator
ensure_barman_plugin
echo "Using namespace: $NAMESPACE"
fetch_admin_keys_and_db_pass_from_bao_or_local
apply_cnpg_admin_secret
ensure_prole_stack_resources
if ! wait_for_cnpg_pods 300; then
echo "WARN: CNPG pods did not become ready after init; attempting force rollout ..." >&2
if force_rollout; then
if ! wait_for_cnpg_pods 300; then
echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2
if ! reset_and_reinit; then
return 1
fi
fi
else
echo "WARN: Force rollout failed; attempting reset and re-init ..." >&2
if ! reset_and_reinit; then
return 1
fi
fi
fi
# Support both names for the toggle from prole.cfg
local kerberos_enabled="${KERBEROS_ENABLED:-${ENABLED:-false}}"
if [[ "$kerberos_enabled" == "true" || "$kerberos_enabled" == "True" || "$kerberos_enabled" == "1" ]]; then
if [[ -x "$SCRIPT_DIR/init_kerberos.sh" ]]; then
echo "Configuring Kerberos for CNPG pods ..."
if ! "$SCRIPT_DIR/init_kerberos.sh" initialize; then
echo "WARN: Kerberos initialization did not complete successfully."
fi
else
echo "WARN: init_kerberos.sh not found; skipping Kerberos configuration."
fi
else
echo "Kerberos is disabled; skipping Kerberos configuration."
fi
# Ensure port-forward is running for Postgres (local access)
echo "Ensuring port-forward for Postgres is active ..."
"$SCRIPT_DIR/init_port_forwards.sh" stop postgres || true
"$SCRIPT_DIR/init_port_forwards.sh" start postgres &
sleep 2
echo "Initialization complete for CNPG + Kerberos + cert artifacts."
}
update_reload() {
initialize
}
case "$ACTION" in
recreate)
ensure_tools
"$0" delete "$CNPG_CLUSTER_NAME"
"$0" create "$CNPG_CLUSTER_NAME"
;;
create)
ensure_tools
ensure_namespace
initialize
;;
delete)
ensure_tools
echo "Deleting all resources for '$CNPG_CLUSTER_NAME' ..."
kubectl delete -n "$NAMESPACE" -k "$SCRIPT_DIR/../k8s/prole" --ignore-not-found
;;
start)
ensure_tools
ensure_namespace
ensure_cnpg_operator
ensure_barman_plugin
if [[ ! -f "$CNPG_MANIFEST" ]]; then
echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
exit 1
fi
echo "Starting CloudNative-PG cluster from $CNPG_MANIFEST in namespace $NAMESPACE..."
kubectl apply -n "$NAMESPACE" -f "$CNPG_MANIFEST"
;;
stop)
ensure_tools
if [[ ! -f "$CNPG_MANIFEST" ]]; then
echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
exit 1
fi
echo "Stopping CloudNative-PG cluster using $CNPG_MANIFEST ..."
kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found
;;
status)
ensure_tools
echo "--- CloudNative-PG Cluster Status ($CNPG_CLUSTER_NAME) ---"
if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME"
echo ""
echo "CNPG Plugin Status:"
if kubectl cnpg version >/dev/null 2>&1; then
kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE"
else
echo "Note: 'kubectl cnpg' plugin not found; skipping detailed status."
fi
else
echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'."
fi
;;
restart)
ensure_tools
"$0" stop
"$0" start
;;
initialize)
if ! initialize; then
exit 1
fi
;;
update|reload)
if ! update_reload; then
exit 1
fi
;;
install-barman-plugin)
ensure_tools
ensure_cnpg_operator
ensure_barman_plugin
;;
*)
echo "Usage: $0 {create|delete|recreate|start|stop|status|restart|initialize|update|reload|install-barman-plugin} [dbname]" >&2
exit 2
;;
esac