mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
5d17325c10
Defects A, B, C from docs/plans/junie/06-patch-garage-script-fixes.md:
A. DB_CLUSTER default was knoe-cnpg-0 (stale rebrand artifact);
corrected to knoe-dev-cnpg-0 to match conf/gke.cfg and the
cluster table in CLAUDE.md. Header comment + overrides block
also updated.
B. Phase 1 deletion loop missed service/garage-s3-ilb (the
LoadBalancer the ObjectStore endpoint historically pointed at).
Now included alongside service/garage.
C. Phase 2 was applying knoe-db-backup-gcs.yaml whole, including a
legacy ScheduledBackup using method:barmanObjectStore (being
removed in CNPG v1.30). Manifest split handled in prior commit
(34a25dd); script's Phase 2 is now safe to re-run.
bash -n clean; CONFIRM=false dry-run prints knoe-dev-cnpg-0 correctly.
docs/TODO.md: queue items #2 and #6 + drift rows R5 and R9 archived to Done.
Closes queue item #6 in docs/TODO.md (drift R9).
Co-authored-by: Junie <junie@jetbrains.com>
186 lines
7.7 KiB
Bash
Executable File
186 lines
7.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# patch_garage_cross_cluster.sh
|
|
#
|
|
# One-shot patch for the dual-cluster garage/barman situation:
|
|
#
|
|
# Problem: install.sh incorrectly deployed Garage into knoe-cnpg-0 (DB cluster)
|
|
# because app_cluster_kubecontext was not persisted in conf/gke.cfg.
|
|
# CNPG backups should use GCS with Workload Identity, not Garage.
|
|
#
|
|
# Fix applied by this script:
|
|
# 1. Remove Garage from knoe-cnpg-0 (statefulset, service, configmap, PVC)
|
|
# 2. Apply cnpg-backup-sa ServiceAccount with Workload Identity annotation
|
|
# 3. Apply the GCS barman ObjectStore in knoe-db-0
|
|
# 4. Refresh CNPG_ELIGIBLE_NODES in conf/gke.cfg from live knoe-cnpg-0 nodes
|
|
#
|
|
# Note: GCS buckets and GCP SA must already exist (created by init_cnpg_gke.sh or
|
|
# the GCP console). If not, run:
|
|
# ./etc/init_cnpg_gke.sh --project plenary-truck-485623-p7 --region us-west3 \
|
|
# --cluster knoe-dev-cnpg-0
|
|
#
|
|
# Usage:
|
|
# CONFIRM=true ./scripts/patch_garage_cross_cluster.sh
|
|
#
|
|
# Overrides:
|
|
# GCP_PROJECT (default: plenary-truck-485623-p7)
|
|
# GCP_REGION (default: us-west3)
|
|
# DB_CLUSTER (default: knoe-dev-cnpg-0)
|
|
# APP_CLUSTER (default: knoe-dev-0)
|
|
# SERVICE_NS (default: knoe-system)
|
|
# DB_NS (default: knoe-db-0)
|
|
# CONFIRM REQUIRED: must be "true"
|
|
|
|
set -euo pipefail
|
|
|
|
export PATH="/opt/homebrew/share/google-cloud-sdk/bin:$PATH"
|
|
|
|
GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
|
|
GCP_REGION="${GCP_REGION:-us-west3}"
|
|
DB_CLUSTER="${DB_CLUSTER:-knoe-dev-cnpg-0}"
|
|
APP_CLUSTER="${APP_CLUSTER:-knoe-dev-0}"
|
|
SERVICE_NS="${SERVICE_NS:-knoe-system}"
|
|
DB_NS="${DB_NS:-knoe-db-0}"
|
|
CONFIRM="${CONFIRM:-false}"
|
|
|
|
DB_CTX="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
|
|
APP_CTX="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
|
|
die() { log "ERROR: $*" >&2; exit 1; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pre-flight
|
|
# ---------------------------------------------------------------------------
|
|
if [[ "${CONFIRM}" != "true" ]]; then
|
|
echo ""
|
|
echo " This script will:"
|
|
echo " 1. Remove Garage from ${DB_CLUSTER}/${SERVICE_NS}"
|
|
echo " 2. Apply cnpg-backup-sa (Workload Identity) in ${DB_CLUSTER}/${DB_NS}"
|
|
echo " 3. Apply the GCS barman ObjectStore in ${DB_CLUSTER}/${DB_NS}"
|
|
echo " 4. Refresh CNPG_ELIGIBLE_NODES in conf/gke.cfg"
|
|
echo ""
|
|
echo " Set CONFIRM=true to proceed:"
|
|
echo " CONFIRM=true ./scripts/patch_garage_cross_cluster.sh"
|
|
echo ""
|
|
exit 1
|
|
fi
|
|
|
|
for tool in gcloud kubectl; do
|
|
command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool"
|
|
done
|
|
command -v gke-gcloud-auth-plugin >/dev/null 2>&1 \
|
|
|| die "gke-gcloud-auth-plugin not found — install with: gcloud components install gke-gcloud-auth-plugin"
|
|
|
|
log "==> patch_garage_cross_cluster (GCS mode)"
|
|
log " DB cluster: ${DB_CTX}"
|
|
log " App cluster: ${APP_CTX}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: Remove Garage from the DB cluster (knoe-cnpg-0)
|
|
# ---------------------------------------------------------------------------
|
|
log "[phase 1] Removing Garage from ${DB_CLUSTER}/${SERVICE_NS} ..."
|
|
|
|
for resource in "statefulset/garage" "service/garage" "service/garage-s3-ilb" "configmap/garage-config"; do
|
|
if kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" get "${resource}" \
|
|
>/dev/null 2>&1; then
|
|
log " Deleting ${resource} from ${DB_CLUSTER}/${SERVICE_NS}"
|
|
kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" delete "${resource}" \
|
|
--ignore-not-found=true --wait=false
|
|
else
|
|
log " ${resource} not found in ${DB_CLUSTER}/${SERVICE_NS} — skipping"
|
|
fi
|
|
done
|
|
|
|
if kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" get pvc data-garage-0 \
|
|
>/dev/null 2>&1; then
|
|
log " Deleting PVC data-garage-0 from ${DB_CLUSTER}/${SERVICE_NS}"
|
|
kubectl --context="${DB_CTX}" -n "${SERVICE_NS}" delete pvc data-garage-0 \
|
|
--ignore-not-found=true
|
|
fi
|
|
|
|
if kubectl --context="${DB_CTX}" get storageclass garage-hdd >/dev/null 2>&1; then
|
|
log " Deleting StorageClass garage-hdd from ${DB_CLUSTER}"
|
|
kubectl --context="${DB_CTX}" delete storageclass garage-hdd --ignore-not-found=true
|
|
fi
|
|
|
|
log " Garage removed from ${DB_CLUSTER}."
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: Apply cnpg-backup-sa ServiceAccount with Workload Identity
|
|
# ---------------------------------------------------------------------------
|
|
log "[phase 2] Applying cnpg-backup-sa (Workload Identity) in ${DB_CLUSTER}/${DB_NS} ..."
|
|
|
|
GCS_MANIFEST="${REPO_ROOT}/deploy/gcp/gke/knoe-db-backup-gcs.yaml"
|
|
[[ -f "${GCS_MANIFEST}" ]] || die "GCS manifest not found: ${GCS_MANIFEST}"
|
|
|
|
GCP_PROJECT_ID="${GCP_PROJECT}" envsubst '${GCP_PROJECT_ID}' < "${GCS_MANIFEST}" \
|
|
| kubectl --context="${DB_CTX}" apply -f -
|
|
|
|
log " cnpg-backup-sa applied."
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: Apply GCS barman ObjectStore
|
|
# ---------------------------------------------------------------------------
|
|
log "[phase 3] Applying GCS barman ObjectStore in ${DB_CLUSTER}/${DB_NS} ..."
|
|
|
|
GCS_OBJ_MANIFEST="${REPO_ROOT}/k8s/knoe/knoe-db-barman-objectstore-gcs.yaml"
|
|
[[ -f "${GCS_OBJ_MANIFEST}" ]] || die "GCS ObjectStore manifest not found: ${GCS_OBJ_MANIFEST}"
|
|
|
|
kubectl --context="${DB_CTX}" -n "${DB_NS}" apply -f "${GCS_OBJ_MANIFEST}"
|
|
|
|
log " GCS ObjectStore applied."
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: Refresh CNPG_ELIGIBLE_NODES in conf/gke.cfg
|
|
# ---------------------------------------------------------------------------
|
|
log "[phase 4] Refreshing CNPG_ELIGIBLE_NODES from ${DB_CLUSTER} ..."
|
|
|
|
CNPG_NODES=$(kubectl --context="${DB_CTX}" get nodes \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{","}{end}' 2>/dev/null \
|
|
| sed 's/,$//' || true)
|
|
|
|
KNOE_CFG="${REPO_ROOT}/conf/gke.cfg"
|
|
|
|
if [[ -n "${CNPG_NODES}" && -f "${KNOE_CFG}" ]]; then
|
|
STAGE1_NODE=$(echo "${CNPG_NODES}" | cut -d',' -f1)
|
|
log " Updating CNPG_ELIGIBLE_NODES=${CNPG_NODES}"
|
|
log " Updating CNPG_STAGE1_NODE=${STAGE1_NODE}"
|
|
# Replace CNPG_ELIGIBLE_NODES line
|
|
sed -i.bak \
|
|
"s|^CNPG_ELIGIBLE_NODES = .*|CNPG_ELIGIBLE_NODES = ${CNPG_NODES}|" \
|
|
"${KNOE_CFG}"
|
|
# Replace CNPG_STAGE1_NODE line
|
|
sed -i.bak \
|
|
"s|^CNPG_STAGE1_NODE = .*|CNPG_STAGE1_NODE = ${STAGE1_NODE}|" \
|
|
"${KNOE_CFG}"
|
|
rm -f "${KNOE_CFG}.bak"
|
|
log " knoe.cfg updated."
|
|
else
|
|
log " WARNING: Could not read nodes from ${DB_CLUSTER} — CNPG_ELIGIBLE_NODES not updated."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Done
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
log "==> patch_garage_cross_cluster complete."
|
|
echo ""
|
|
echo "────────────────────────────────────────────────────────────────"
|
|
echo " Verify:"
|
|
echo ""
|
|
echo " # Garage gone from DB cluster"
|
|
echo " kubectl --context=${DB_CTX} -n ${SERVICE_NS} get statefulset garage"
|
|
echo ""
|
|
echo " # WI service account"
|
|
echo " kubectl --context=${DB_CTX} -n ${DB_NS} get sa cnpg-backup-sa -o yaml"
|
|
echo ""
|
|
echo " # GCS ObjectStore"
|
|
echo " kubectl --context=${DB_CTX} -n ${DB_NS} get objectstore knoe-db-barman-objectstore -o yaml"
|
|
echo ""
|
|
echo " If barman plugin isn't configured yet:"
|
|
echo " KUBECONTEXT=${DB_CTX} ./etc/init_cnpg_backup.sh start"
|
|
echo "────────────────────────────────────────────────────────────────"
|