#!/usr/bin/env bash # patch_garage_cross_cluster.sh # # One-shot patch for the dual-cluster garage/barman situation: # # Problem: install.sh incorrectly deployed Garage into knoe-cnpg-0 (DB cluster) # because app_cluster_kubecontext was not persisted in conf/gke.cfg. # CNPG backups should use GCS with Workload Identity, not Garage. # # Fix applied by this script: # 1. Remove Garage from knoe-cnpg-0 (statefulset, service, configmap, PVC) # 2. Apply cnpg-backup-sa ServiceAccount with Workload Identity annotation # 3. Apply the GCS barman ObjectStore in knoe-db-0 # 4. Refresh CNPG_ELIGIBLE_NODES in conf/gke.cfg from live knoe-cnpg-0 nodes # # Note: GCS buckets and GCP SA must already exist (created by init_cnpg_gke.sh or # the GCP console). If not, run: # ./etc/init_cnpg_gke.sh --project plenary-truck-485623-p7 --region us-west3 \ # --cluster knoe-dev-cnpg-0 # # Usage: # CONFIRM=true ./scripts/patch_garage_cross_cluster.sh # # Overrides: # GCP_PROJECT (default: plenary-truck-485623-p7) # GCP_REGION (default: us-west3) # DB_CLUSTER (default: knoe-dev-cnpg-0) # APP_CLUSTER (default: knoe-dev-0) # SERVICE_NS (default: knoe-system) # DB_NS (default: knoe-db-0) # CONFIRM REQUIRED: must be "true" set -euo pipefail export PATH="/opt/homebrew/share/google-cloud-sdk/bin:$PATH" GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}" GCP_REGION="${GCP_REGION:-us-west3}" DB_CLUSTER="${DB_CLUSTER:-knoe-dev-cnpg-0}" APP_CLUSTER="${APP_CLUSTER:-knoe-dev-0}" SERVICE_NS="${SERVICE_NS:-knoe-system}" DB_NS="${DB_NS:-knoe-db-0}" CONFIRM="${CONFIRM:-false}" DB_CTX="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}" APP_CTX="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } die() { log "ERROR: $*" >&2; exit 1; } # --------------------------------------------------------------------------- # Pre-flight # --------------------------------------------------------------------------- if [[ "${CONFIRM}" != "true" ]]; then echo "" echo " This script will:" echo " 1. Remove Garage from ${DB_CLUSTER}/${SERVICE_NS}" echo " 2. Apply cnpg-backup-sa (Workload Identity) in ${DB_CLUSTER}/${DB_NS}" echo " 3. Apply the GCS barman ObjectStore in ${DB_CLUSTER}/${DB_NS}" echo " 4. Refresh CNPG_ELIGIBLE_NODES in conf/gke.cfg" echo "" echo " Set CONFIRM=true to proceed:" echo " CONFIRM=true ./scripts/patch_garage_cross_cluster.sh" echo "" exit 1 fi for tool in gcloud kubectl; do command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool" done command -v gke-gcloud-auth-plugin >/dev/null 2>&1 \ || die "gke-gcloud-auth-plugin not found — install with: gcloud components install gke-gcloud-auth-plugin" log "==> patch_garage_cross_cluster (GCS mode)" log " DB cluster: ${DB_CTX}" log " App cluster: ${APP_CTX}" # --------------------------------------------------------------------------- # Phase 1: Remove Garage from the DB cluster (knoe-cnpg-0) # --------------------------------------------------------------------------- log "[phase 1] Removing Garage from ${DB_CLUSTER}/${SERVICE_NS} ..." for resource in "statefulset/garage" "service/garage" "service/garage-s3-ilb" "configmap/garage-config"; do if kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" get "${resource}" \ >/dev/null 2>&1; then log " Deleting ${resource} from ${DB_CLUSTER}/${SERVICE_NS}" kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" delete "${resource}" \ --ignore-not-found=true --wait=false else log " ${resource} not found in ${DB_CLUSTER}/${SERVICE_NS} — skipping" fi done if kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" get pvc data-garage-0 \ >/dev/null 2>&1; then log " Deleting PVC data-garage-0 from ${DB_CLUSTER}/${SERVICE_NS}" kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" delete pvc data-garage-0 \ --ignore-not-found=true fi if kubectl --context="${DB_CTX}" get storageclass garage-hdd >/dev/null 2>&1; then log " Deleting StorageClass garage-hdd from ${DB_CLUSTER}" kubectl --context="${DB_CTX}" delete storageclass garage-hdd --ignore-not-found=true fi log " Garage removed from ${DB_CLUSTER}." # --------------------------------------------------------------------------- # Phase 2: Apply cnpg-backup-sa ServiceAccount with Workload Identity # --------------------------------------------------------------------------- log "[phase 2] Applying cnpg-backup-sa (Workload Identity) in ${DB_CLUSTER}/${DB_NS} ..." GCS_MANIFEST="${REPO_ROOT}/deploy/gcp/gke/knoe-db-backup-gcs.yaml" [[ -f "${GCS_MANIFEST}" ]] || die "GCS manifest not found: ${GCS_MANIFEST}" GCP_PROJECT_ID="${GCP_PROJECT}" envsubst '${GCP_PROJECT_ID}' < "${GCS_MANIFEST}" \ | kubectl --context="${DB_CTX}" apply -f - log " cnpg-backup-sa applied." # --------------------------------------------------------------------------- # Phase 3: Apply GCS barman ObjectStore # --------------------------------------------------------------------------- log "[phase 3] Applying GCS barman ObjectStore in ${DB_CLUSTER}/${DB_NS} ..." GCS_OBJ_MANIFEST="${REPO_ROOT}/k8s/knoe/knoe-db-barman-objectstore-gcs.yaml" [[ -f "${GCS_OBJ_MANIFEST}" ]] || die "GCS ObjectStore manifest not found: ${GCS_OBJ_MANIFEST}" kubectl --context="${DB_CTX}" -n "${DB_NS}" apply -f "${GCS_OBJ_MANIFEST}" log " GCS ObjectStore applied." # --------------------------------------------------------------------------- # Phase 4: Refresh CNPG_ELIGIBLE_NODES in conf/gke.cfg # --------------------------------------------------------------------------- log "[phase 4] Refreshing CNPG_ELIGIBLE_NODES from ${DB_CLUSTER} ..." CNPG_NODES=$(kubectl --context="${DB_CTX}" get nodes \ -o jsonpath='{range .items[*]}{.metadata.name}{","}{end}' 2>/dev/null \ | sed 's/,$//' || true) KNOE_CFG="${REPO_ROOT}/conf/gke.cfg" if [[ -n "${CNPG_NODES}" && -f "${KNOE_CFG}" ]]; then STAGE1_NODE=$(echo "${CNPG_NODES}" | cut -d',' -f1) log " Updating CNPG_ELIGIBLE_NODES=${CNPG_NODES}" log " Updating CNPG_STAGE1_NODE=${STAGE1_NODE}" # Replace CNPG_ELIGIBLE_NODES line sed -i.bak \ "s|^CNPG_ELIGIBLE_NODES = .*|CNPG_ELIGIBLE_NODES = ${CNPG_NODES}|" \ "${KNOE_CFG}" # Replace CNPG_STAGE1_NODE line sed -i.bak \ "s|^CNPG_STAGE1_NODE = .*|CNPG_STAGE1_NODE = ${STAGE1_NODE}|" \ "${KNOE_CFG}" rm -f "${KNOE_CFG}.bak" log " knoe.cfg updated." else log " WARNING: Could not read nodes from ${DB_CLUSTER} — CNPG_ELIGIBLE_NODES not updated." fi # --------------------------------------------------------------------------- # Done # --------------------------------------------------------------------------- echo "" log "==> patch_garage_cross_cluster complete." echo "" echo "────────────────────────────────────────────────────────────────" echo " Verify:" echo "" echo " # Garage gone from DB cluster" echo " kubectl --context=${DB_CTX} -n ${SERVICE_NS} get statefulset garage" echo "" echo " # WI service account" echo " kubectl --context=${DB_CTX} -n ${DB_NS} get sa cnpg-backup-sa -o yaml" echo "" echo " # GCS ObjectStore" echo " kubectl --context=${DB_CTX} -n ${DB_NS} get objectstore knoe-db-barman-objectstore -o yaml" echo "" echo " If barman plugin isn't configured yet:" echo " KUBECONTEXT=${DB_CTX} ./etc/init_cnpg_backup.sh start" echo "────────────────────────────────────────────────────────────────"