#!/usr/bin/env bash set -euo pipefail # init_prole-db.sh # Purpose: # - Manage prole-db CloudNative-PG cluster operations (deploy, start, stop, etc.) # Initialize SCRIPT_DIR SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Load environment and config via prole_cfg.sh # shellcheck disable=SC1090 source "$SCRIPT_DIR/prole_cfg.sh" ACTION=${1:-} VERSION=${2:-latest} CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-prole-db} NAMESPACE=${NAMESPACE:-default} # Support both PROLE_HOME/k8s and sibling k8s directory if [[ -d "$SCRIPT_DIR/../k8s/prole" ]]; then CNPG_MANIFEST="$SCRIPT_DIR/../k8s/prole/prole-db.yaml" elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/prole" ]]; then CNPG_MANIFEST="$PROLE_HOME/k8s/prole/prole-db.yaml" else CNPG_MANIFEST="$SCRIPT_DIR/../k8s/prole/prole-db.yaml" fi ensure_tools() { for t in kubectl jq curl; do command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; } done } ensure_namespace() { if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then echo "Creating namespace '$NAMESPACE' ..." kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true fi } get_latest_image() { local pg_version_file release_file pg_version release if [[ -f "$SCRIPT_DIR/../conf/postgresql/.version" ]]; then pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/conf/postgresql/.version" ]]; then pg_version_file="$PROLE_HOME/conf/postgresql/.version" else pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" fi if [[ -f "$SCRIPT_DIR/../prole-db/.version" ]]; then release_file="$SCRIPT_DIR/../prole-db/.version" elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/prole-db/.version" ]]; then release_file="$PROLE_HOME/prole-db/.version" else release_file="$SCRIPT_DIR/../prole-db/.version" fi if [[ -f "$pg_version_file" ]]; then pg_version=$(cat "$pg_version_file" | tr -d '[:space:]') else pg_version="17.7" fi if [[ -f "$release_file" ]]; then release=$(cat "$release_file" | tr -d '[:space:]') else release="43" fi if [[ "$release" =~ ^[0-9]+$ ]]; then release=$(printf "%03d" "$release") fi echo "prole-db:${pg_version}-${release}" } sync_manifest_image() { local image="$1" local manifest_dir manifest_dir=$(dirname "$CNPG_MANIFEST") local files=("$CNPG_MANIFEST") if [[ -f "$manifest_dir/prole-db-recovery.yaml.tpl" ]]; then files+=("$manifest_dir/prole-db-recovery.yaml.tpl") fi local f tmp for f in "${files[@]}"; do if [[ -f "$f" ]] && grep -qE '^[[:space:]]*imageName:' "$f"; then tmp=$(mktemp) sed -E "s|^([[:space:]]*imageName:).*|\\1 ${image}|" "$f" > "$tmp" mv "$tmp" "$f" fi done } start() { ensure_tools ensure_namespace echo "Checking dependencies..." # 1. k3d is running if ! command -v k3d >/dev/null || ! k3d cluster list >/dev/null 2>&1; then echo "ERROR: k3d is not running or not installed." >&2 exit 1 fi # 2. cnpg operator is loaded if ! kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then echo "ERROR: CloudNative-PG operator is not loaded." >&2 exit 1 fi # 3. openbao is configured (local container) local openbao_url="${PROLE_OPENBAO_URL:-http://127.0.0.1:18200}" if ! curl -sS "$openbao_url/v1/sys/health" >/dev/null 2>&1; then echo "ERROR: OpenBao is not reachable at $openbao_url." >&2 exit 1 fi # 4. garage is configured if ! kubectl get statefulset garage -n "$NAMESPACE" >/dev/null 2>&1 && ! kubectl get deployment garage -n "$NAMESPACE" >/dev/null 2>&1; then echo "ERROR: Garage is not deployed." >&2 exit 1 fi # Ensure cluster exists before attempting to patch image if ! kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'. Applying manifest..." kubectl apply -n "$NAMESPACE" -f "$CNPG_MANIFEST" fi # Compare latest image with deployed local latest_image latest_image=$(get_latest_image) sync_manifest_image "$latest_image" local current_image current_image=$(kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "") if kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then if [[ "$current_image" != "$latest_image" ]]; then echo "Updating cluster image from '$current_image' to '$latest_image'..." kubectl patch cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --type merge -p "{\"spec\": {\"imageName\": \"$latest_image\"}}" # Force a rollout to ensure the new image is pulled even if it was just a tag update (though we use unique tags) kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true else echo "Cluster is already using the latest image: $latest_image" fi else echo "WARN: Cluster '$CNPG_CLUSTER_NAME' still not found after applying manifest." fi echo "Starting prole-db cluster (ensuring manifest is applied)..." kubectl apply -n "$NAMESPACE" -f "$CNPG_MANIFEST" } stop() { ensure_tools echo "Stopping prole-db cluster $CNPG_CLUSTER_NAME..." # Identify instances local instances instances=$(kubectl get pods -n "$NAMESPACE" -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[*].metadata.name}') if [[ -z "$instances" ]]; then echo "No instances found for cluster $CNPG_CLUSTER_NAME." return fi # Determine replicas and primary local primary primary=$(kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" -o jsonpath='{.status.currentPrimary}') echo "Primary: $primary" # Shutdown replicas first for pod in $instances; do if [[ "$pod" != "$primary" ]]; then echo "Shutting down replica $pod..." kubectl exec -n "$NAMESPACE" "$pod" -c postgres -- psql -U postgres -c "CHECKPOINT;" || true fi done # Shutdown primary last if [[ -n "$primary" ]]; then echo "Shutting down primary $primary..." kubectl exec -n "$NAMESPACE" "$primary" -c postgres -- psql -U postgres -c "CHECKPOINT;" || true fi echo "Deleting cluster resource..." kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found } restart() { ensure_tools ensure_namespace echo "Restarting prole-db cluster..." kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" } deploy() { ensure_tools ensure_namespace local image if [[ "$VERSION" == "latest" ]]; then image=$(get_latest_image) else image="prole-db:$VERSION" fi sync_manifest_image "$image" echo "Deploying $image to cluster $CNPG_CLUSTER_NAME..." # If cluster doesn't exist, use init_cloudnative_pg.sh create first if ! kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then echo "Cluster not found. Running init_cloudnative_pg.sh create..." bash "$SCRIPT_DIR/init_cloudnative_pg.sh" create fi # Ensure image is updated if manifest has an older version # First, apply the manifest to ensure the cluster exists/is updated echo "Applying manifest $CNPG_MANIFEST..." kubectl apply -n "$NAMESPACE" -f "$CNPG_MANIFEST" # Then force the specific image version via patch if different local current_image current_image=$(kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "") if [[ "$current_image" != "$image" ]]; then echo "Patching cluster to use image '$image' (was '$current_image')..." kubectl patch cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --type merge -p "{\"spec\": {\"imageName\": \"$image\"}}" # Force a rollout kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true fi } rollback() { ensure_tools echo "Rollback: Updating to previous image (manually specified version or fallback)..." if [[ "$VERSION" == "latest" ]]; then echo "Please specify a version to rollback to. Usage: $0 rollback " exit 1 fi deploy } rollout() { ensure_tools ensure_namespace echo "Starting manual recreate rollout for $CNPG_CLUSTER_NAME in $NAMESPACE..." # 1. Monitor status echo "Current status:" kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" # 2. Identify the primary instance local primary primary=$(kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" -o jsonpath='{.status.currentPrimary}') if [[ -z "$primary" ]]; then echo "ERROR: Could not identify primary instance." >&2 exit 1 fi echo "Primary instance: $primary" # 3. Identify all pod instances local instances instances=$(kubectl get pods -n "$NAMESPACE" -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[*].metadata.name}') # 4. Recreate non-primary instances one at a time for pod in $instances; do if [[ "$pod" != "$primary" ]]; then echo "Recreating non-primary pod: $pod..." kubectl delete pod -n "$NAMESPACE" "$pod" echo "Waiting for $pod to be recreated and healthy..." while true; do if kubectl get pod -n "$NAMESPACE" "$pod" >/dev/null 2>&1; then local status status=$(kubectl get pod -n "$NAMESPACE" "$pod" -o jsonpath='{.status.phase}') local ready ready=$(kubectl get pod -n "$NAMESPACE" "$pod" -o jsonpath='{.status.containerStatuses[0].ready}') if [[ "$status" == "Running" && "$ready" == "true" ]]; then echo "Pod $pod is active and healthy." break fi fi echo -n "." sleep 5 done echo "" # Additional wait for CNPG to recognize it as joined and healthy echo "Waiting for CNPG cluster to stabilize..." sleep 10 kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" fi done # 5. When the last primary instance needs to be recreated, promote a new primary echo "Promoting a new primary to recreate the old primary $primary..." # We promote one of the newly recreated instances. # kubectl cnpg promote local new_primary="" for pod in $instances; do if [[ "$pod" != "$primary" ]]; then new_primary="$pod" break fi done if [[ -z "$new_primary" ]]; then echo "ERROR: No candidate for new primary found." >&2 exit 1 fi echo "Promoting $new_primary..." kubectl cnpg promote "$CNPG_CLUSTER_NAME" "$new_primary" -n "$NAMESPACE" || { echo "Promotion failed, but continuing rollout (CNPG might have already promoted it)..." } echo "Waiting for $new_primary to become primary..." while true; do local current_primary current_primary=$(kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" -o jsonpath='{.status.currentPrimary}') if [[ "$current_primary" == "$new_primary" ]]; then echo "$new_primary is now the primary." break fi echo -n "." sleep 5 done echo "" # 6. Rebuild the last pod (the old primary) to make the cluster consistent echo "Recreating the old primary pod: $primary..." kubectl delete pod -n "$NAMESPACE" "$primary" echo "Waiting for $primary to be recreated and healthy..." while true; do if kubectl get pod -n "$NAMESPACE" "$primary" >/dev/null 2>&1; then local status status=$(kubectl get pod -n "$NAMESPACE" "$primary" -o jsonpath='{.status.phase}') local ready ready=$(kubectl get pod -n "$NAMESPACE" "$primary" -o jsonpath='{.status.containerStatuses[0].ready}') if [[ "$status" == "Running" && "$ready" == "true" ]]; then echo "Pod $primary is active and healthy." break fi fi echo -n "." sleep 5 done echo "" echo "Rollout complete. Final cluster status:" kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" } backup() { ensure_tools bash "$SCRIPT_DIR/init_prole-db-backup.sh" start } reset() { ensure_tools echo "Resetting prole-db cluster..." echo "Removing cnpg instances and pods..." kubectl delete cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --ignore-not-found # Wait for deletion kubectl wait --for=delete cluster/"$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --timeout=60s || true echo "Reinstantiating..." deploy } case "$ACTION" in start) start ;; stop) stop ;; restart) restart ;; deploy) deploy ;; rollback) rollback ;; rollout) rollout ;; backup) backup ;; reset) reset ;; *) echo "Usage: $0 {start|stop|restart|deploy|rollback|backup|reset} [version]" >&2 exit 2 ;; esac