#!/usr/bin/env bash # init_cnpg_gke.sh # Provision CloudNativePG on GKE with GCS backup via Workload Identity. # # Usage: # ./etc/init_cnpg_gke.sh [--project PROJECT_ID] [--region REGION] [--cluster CLUSTER_NAME] # # Env vars (override flags): # GCP_PROJECT_ID, GCP_REGION, GKE_CLUSTER, GCS_BACKUP_BUCKET, # CNPG_NAMESPACE, ARTIFACT_REGISTRY, DB_PASSWORD, CLOUDSDK_AUTH_ACCESS_TOKEN set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" GKE_MANIFEST_DIR="$REPO_ROOT/deploy/gcp/gke" # ── Defaults ──────────────────────────────────────────────────────────────── GCP_PROJECT_ID="${GCP_PROJECT_ID:-}" GCP_REGION="${GCP_REGION:-us-central1}" GKE_CLUSTER="${GKE_CLUSTER:-knoe-dev-0}" GCS_BACKUP_BUCKET="${GCS_BACKUP_BUCKET:-knoe-0-backups}" GCS_WAL_BUCKET="${GCS_WAL_BUCKET:-knoe-0-wal}" CNPG_NAMESPACE="${CNPG_NAMESPACE:-knoe-db-0}" CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-knoe-db}" CNPG_BACKUP_SA="${CNPG_BACKUP_SA:-cnpg-backup}" ARTIFACT_REGISTRY="${ARTIFACT_REGISTRY:-}" KNOE_DB_IMAGE_TAG="${KNOE_DB_IMAGE_TAG:-}" DB_PASSWORD="${DB_PASSWORD:-}" CNPG_DB_OWNER="${CNPG_DB_OWNER:-knoe}" CNPG_STORAGE_CLASS_ORDER="${CNPG_STORAGE_CLASS_ORDER:-premium-rwo,ssd,premium,standard-rwo,garage-hdd,standard,dynamic-rwo}" CNPG_STORAGE_WAIT_TIMEOUT="${CNPG_STORAGE_WAIT_TIMEOUT:-240}" # v1.29.0+ is required: spec.serviceAccountName lets the cluster pods run as # cnpg-backup-sa (annotated with iam.gke.io/gcp-service-account for WI), which # is how the GCS-backed barman ObjectStore authenticates without a static key. # Older operators (we used 1.24.0 historically) silently drop the field. CNPG_OPERATOR_VERSION="${CNPG_OPERATOR_VERSION:-1.29.0}" # ── Argument parsing ───────────────────────────────────────────────────────── while [[ $# -gt 0 ]]; do case "$1" in --project) GCP_PROJECT_ID="$2"; shift 2 ;; --region) GCP_REGION="$2"; shift 2 ;; --cluster) GKE_CLUSTER="$2"; shift 2 ;; --bucket) GCS_BACKUP_BUCKET="$2"; shift 2 ;; --namespace) CNPG_NAMESPACE="$2"; shift 2 ;; --registry) ARTIFACT_REGISTRY="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 1 ;; esac done [[ -z "$GCP_PROJECT_ID" ]] && { # Try to read from gcp.cfg GCP_CFG="$REPO_ROOT/conf/prod/gcp.cfg" if [[ -f "$GCP_CFG" ]]; then GCP_PROJECT_ID=$(grep '^project_id' "$GCP_CFG" | sed 's/.*=\s*"\?\([^"]*\)"\?.*/\1/' | tr -d '[:space:]') fi } [[ -z "$GCP_PROJECT_ID" ]] && { echo "ERROR: GCP_PROJECT_ID not set and conf/prod/gcp.cfg not found." >&2; exit 1; } GCP_SA_EMAIL="${CNPG_BACKUP_SA}@${GCP_PROJECT_ID}.iam.gserviceaccount.com" log() { echo "[init_cnpg_gke] $*"; } resolve_knoe_db_image_tag() { [[ -n "$KNOE_DB_IMAGE_TAG" ]] && return local mode_key="k8s" local knoe_home knoe_home="${KNOE_HOME:-$REPO_ROOT}" local pg_version_file="$knoe_home/modes/$mode_key/conf/postgresql/.version" local release_file="$knoe_home/modes/$mode_key/knoe-db/.version" [[ -f "$pg_version_file" ]] || pg_version_file="$knoe_home/conf/postgresql/.version" [[ -f "$release_file" ]] || release_file="$knoe_home/knoe-db/.version" local pg_version release if [[ -f "$pg_version_file" ]]; then pg_version="$(tr -d '[:space:]' < "$pg_version_file")" else pg_version="17.7" fi if [[ -f "$release_file" ]]; then release="$(tr -d '[:space:]' < "$release_file")" else release="43" fi if [[ "$release" =~ ^[0-9]+$ ]]; then release="$(printf '%03d' "$release")" fi KNOE_DB_IMAGE_TAG="${pg_version}-${release}" } storage_class_available() { local candidate="$1" [[ -n "$candidate" ]] || return 1 kubectl get storageclass "$candidate" >/dev/null 2>&1 } append_storage_candidate() { local candidate="$1" [[ -n "$candidate" ]] || return case ",${CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED}," in *",${candidate},"*) return ;; esac if [[ -n "$CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED" ]]; then CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED+="${IFS_COMMA}${candidate}" else CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED="$candidate" fi } storage_class_is_compatible() { local candidate="$1" case "$candidate" in *rwx*|*RWX*|gcsfuse*|parallelstore-*|enterprise-rwx|enterprise-multishare-rwx|regional-rwx|premium-rwx|standard-rwx|synology-*|*iscsi*) return 1 ;; *) return 0 ;; esac } append_by_name_pattern() { local pattern="$1" local sc while IFS= read -r sc; do [[ -n "$sc" ]] || continue storage_class_is_compatible "$sc" || continue case "$sc" in *"$pattern"*) append_storage_candidate "$sc" ;; esac done <<< "$AVAILABLE_STORAGE_CLASSES" } resolve_storage_class_candidates() { AVAILABLE_STORAGE_CLASSES="$(kubectl get storageclass -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)" [[ -n "$AVAILABLE_STORAGE_CLASSES" ]] || { echo "ERROR: unable to enumerate Kubernetes storage classes." >&2 exit 1 } CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED="" IFS=',' read -r -a requested_classes <<< "$CNPG_STORAGE_CLASS_ORDER" local requested for requested in "${requested_classes[@]}"; do requested="${requested//[[:space:]]/}" [[ -n "$requested" ]] || continue case "$requested" in ssd) append_by_name_pattern "ssd" ;; premium) append_by_name_pattern "premium" ;; *) if storage_class_available "$requested" && storage_class_is_compatible "$requested"; then append_storage_candidate "$requested" fi ;; esac done local sc while IFS= read -r sc; do [[ -n "$sc" ]] || continue storage_class_is_compatible "$sc" || continue append_storage_candidate "$sc" done <<< "$AVAILABLE_STORAGE_CLASSES" [[ -n "$CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED" ]] || { echo "ERROR: no compatible storage classes available for CNPG provisioning." >&2 exit 1 } } render_cnpg_manifest() { local storage_class="$1" local rendered_manifest="$2" ARTIFACT_REGISTRY="${ARTIFACT_REGISTRY}" KNOE_DB_IMAGE_TAG="${KNOE_DB_IMAGE_TAG}" \ envsubst '${ARTIFACT_REGISTRY} ${KNOE_DB_IMAGE_TAG}' < "$GKE_MANIFEST_DIR/knoe-db.yaml" > "$rendered_manifest" sed -E "s#^([[:space:]]*storageClassName:).*#\\1 ${storage_class}#" "$rendered_manifest" > "${rendered_manifest}.tmp" mv "${rendered_manifest}.tmp" "$rendered_manifest" } cnpg_attempt_ready() { local expected_storage_class="$1" local pfx current_storage_class pfx="${CNPG_CLUSTER_NAME}-" current_storage_class="$(kubectl -n "$CNPG_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.storage.pvcTemplate.storageClassName}' 2>/dev/null || true)" [[ "$current_storage_class" == "$expected_storage_class" ]] || return 1 kubectl -n "$CNPG_NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.storageClassName}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null \ | awk -F'|' -v expected="$expected_storage_class" -v prefix="$pfx" '$1 ~ ("^" prefix) && $2 == expected && $3 == "Bound" { found=1 } END { exit(found ? 0 : 1) }' } cnpg_attempt_has_pvc_for_class() { local expected_storage_class="$1" local pfx pfx="${CNPG_CLUSTER_NAME}-" kubectl -n "$CNPG_NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.storageClassName}{"\n"}{end}' 2>/dev/null \ | awk -F'|' -v expected="$expected_storage_class" -v prefix="$pfx" '$1 ~ ("^" prefix) && $2 == expected { found=1 } END { exit(found ? 0 : 1) }' } cnpg_recent_events() { kubectl -n "$CNPG_NAMESPACE" get events --sort-by=.lastTimestamp 2>/dev/null | tail -n 80 || true } wait_for_storage_outcome() { local expected_storage_class="$1" local started_at current_ts events started_at="$(date +%s)" while true; do if cnpg_attempt_ready "$expected_storage_class"; then return 0 fi if cnpg_attempt_has_pvc_for_class "$expected_storage_class"; then events="$(cnpg_recent_events)" if printf '%s\n' "$events" | grep -q "SSD_TOTAL_GB"; then return 10 fi if printf '%s\n' "$events" | grep -Eqi "storageclass.*not found|failed to provision volume"; then return 11 fi fi current_ts="$(date +%s)" if (( current_ts - started_at >= CNPG_STORAGE_WAIT_TIMEOUT )); then return 1 fi sleep 10 done } reset_cnpg_cluster_attempt() { kubectl -n "$CNPG_NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found=true --wait=true --timeout=180s >/dev/null 2>&1 || true kubectl -n "$CNPG_NAMESPACE" delete job -l cnpg.io/cluster="$CNPG_CLUSTER_NAME" --ignore-not-found=true >/dev/null 2>&1 || true kubectl -n "$CNPG_NAMESPACE" delete pvc -l cnpg.io/cluster="$CNPG_CLUSTER_NAME" --ignore-not-found=true >/dev/null 2>&1 || true kubectl -n "$CNPG_NAMESPACE" delete pod -l cnpg.io/cluster="$CNPG_CLUSTER_NAME" --ignore-not-found=true >/dev/null 2>&1 || true } ensure_cnpg_tls_secrets() { local cluster_name="knoe-db" if kubectl -n "$CNPG_NAMESPACE" get secret "${cluster_name}-ca" >/dev/null 2>&1 \ && kubectl -n "$CNPG_NAMESPACE" get secret "${cluster_name}-tls" >/dev/null 2>&1; then local ca_key_b64 local ca_key_pem ca_key_b64="$(kubectl -n "$CNPG_NAMESPACE" get secret "${cluster_name}-ca" -o jsonpath='{.data.ca\.key}' 2>/dev/null || true)" ca_key_pem="$(printf '%s' "$ca_key_b64" | openssl base64 -d -A 2>/dev/null || true)" if [[ -n "$ca_key_b64" && "$ca_key_pem" == *"BEGIN EC PRIVATE KEY"* ]]; then log "CNPG TLS secrets already present in ${CNPG_NAMESPACE}." return fi log "CNPG CA secret is missing/invalid for CNPG in ${CNPG_NAMESPACE}; regenerating CNPG TLS secrets ..." fi log "Bootstrapping CNPG TLS secrets in ${CNPG_NAMESPACE} ..." local tmp_dir tmp_dir="$(mktemp -d)" local fqdn cn fqdn="${cluster_name}-rw.${CNPG_NAMESPACE}.svc.cluster.local" cn="$fqdn" if [[ ${#cn} -gt 64 ]]; then cn="${cluster_name}-rw" fi openssl ecparam -name prime256v1 -genkey -noout -out "$tmp_dir/ca.key" >/dev/null 2>&1 openssl req -x509 -new -nodes -key "$tmp_dir/ca.key" -sha256 -days 3650 \ -subj "/CN=${cluster_name}-ca" -out "$tmp_dir/ca.crt" >/dev/null 2>&1 openssl ecparam -name prime256v1 -genkey -noout -out "$tmp_dir/tls.key" >/dev/null 2>&1 openssl req -new -key "$tmp_dir/tls.key" -subj "/CN=${cn}" -out "$tmp_dir/tls.csr" >/dev/null 2>&1 cat > "$tmp_dir/ext.cnf" </dev/null 2>&1 kubectl -n "$CNPG_NAMESPACE" create secret generic "${cluster_name}-ca" \ --from-file=ca.crt="$tmp_dir/ca.crt" \ --from-file=ca.key="$tmp_dir/ca.key" \ --dry-run=client -o yaml | kubectl apply -f - kubectl -n "$CNPG_NAMESPACE" create secret tls "${cluster_name}-tls" \ --cert="$tmp_dir/tls.crt" \ --key="$tmp_dir/tls.key" \ --dry-run=client -o yaml | kubectl apply -f - rm -rf "$tmp_dir" } ensure_db_user_secret() { if kubectl -n "$CNPG_NAMESPACE" get secret knoe-db-user >/dev/null 2>&1; then log "knoe-db-user secret already present in ${CNPG_NAMESPACE}." else [[ -n "$DB_PASSWORD" && "$DB_PASSWORD" != \$\{* ]] || { echo "ERROR: secret 'knoe-db-user' missing in '${CNPG_NAMESPACE}' and DB_PASSWORD is not set." >&2 exit 1 } log "Creating missing knoe-db-user secret in ${CNPG_NAMESPACE} ..." kubectl -n "$CNPG_NAMESPACE" create secret generic knoe-db-user \ --from-literal=username="$CNPG_DB_OWNER" \ --from-literal=password="$DB_PASSWORD" \ --dry-run=client -o yaml | kubectl apply -f - fi if kubectl -n "$CNPG_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then return fi [[ -n "$DB_PASSWORD" && "$DB_PASSWORD" != \$\{* ]] || return log "Creating missing knoe-db-superuser secret in ${CNPG_NAMESPACE} ..." kubectl -n "$CNPG_NAMESPACE" create secret generic knoe-db-superuser \ --from-literal=username=postgres \ --from-literal=password="$DB_PASSWORD" \ --dry-run=client -o yaml | kubectl apply -f - } # ── Tool checks ────────────────────────────────────────────────────────────── for tool in gcloud kubectl gsutil openssl; do command -v "$tool" >/dev/null || { echo "ERROR: $tool not found in PATH." >&2; exit 1; } done # ── 1. Acquire kubeconfig ──────────────────────────────────────────────────── log "Fetching kubeconfig for $GKE_CLUSTER in $GCP_REGION ..." gcloud container clusters get-credentials "$GKE_CLUSTER" \ --project "$GCP_PROJECT_ID" \ --region "$GCP_REGION" # ── 2. Create GCS buckets ──────────────────────────────────────────────────── for bucket in "$GCS_BACKUP_BUCKET" "$GCS_WAL_BUCKET"; do if gsutil ls -b "gs://$bucket" >/dev/null 2>&1; then log "Bucket gs://$bucket already exists." else log "Creating bucket gs://$bucket ..." gsutil mb -p "$GCP_PROJECT_ID" -l "$GCP_REGION" "gs://$bucket" gsutil versioning set on "gs://$bucket" gsutil lifecycle set /dev/stdin "gs://$bucket" </dev/null 2>&1; then log "Service account $GCP_SA_EMAIL already exists." else log "Creating service account $GCP_SA_EMAIL ..." gcloud iam service-accounts create "$CNPG_BACKUP_SA" \ --display-name="CNPG GCS Backup" \ --project="$GCP_PROJECT_ID" fi # IAM propagation is eventually consistent right after service-account creation. # Wait briefly so downstream gsutil IAM grants don't fail with transient 400. for i in $(seq 1 12); do if gcloud iam service-accounts describe "$GCP_SA_EMAIL" --project "$GCP_PROJECT_ID" >/dev/null 2>&1; then break fi log "Waiting for service account propagation ($i/12) ..." sleep 5 done # ── 4. Grant storage access ─────────────────────────────────────────────────── log "Granting objectAdmin on backup buckets to $GCP_SA_EMAIL ..." for bucket in "$GCS_BACKUP_BUCKET" "$GCS_WAL_BUCKET"; do ok=0 for i in $(seq 1 6); do if gsutil iam ch \ "serviceAccount:${GCP_SA_EMAIL}:objectAdmin" \ "gs://$bucket"; then ok=1 break fi log "WARN: IAM grant failed for gs://$bucket (attempt $i/6); retrying in 5s ..." sleep 5 done [[ $ok -eq 1 ]] || { echo "ERROR: failed to grant objectAdmin on gs://$bucket to $GCP_SA_EMAIL" >&2 exit 1 } done # ── 5. Bind Workload Identity ───────────────────────────────────────────────── log "Binding Workload Identity ..." gcloud iam service-accounts add-iam-policy-binding "$GCP_SA_EMAIL" \ --role=roles/iam.workloadIdentityUser \ --member="serviceAccount:${GCP_PROJECT_ID}.svc.id.goog[${CNPG_NAMESPACE}/cnpg-backup-sa]" \ --project="$GCP_PROJECT_ID" # ── 6. Apply namespace ──────────────────────────────────────────────────────── log "Applying namespace $CNPG_NAMESPACE ..." kubectl apply -f "$GKE_MANIFEST_DIR/namespace.yaml" # ── 7. Install CNPG operator (if not present) ───────────────────────────────── if ! kubectl get crd clusters.postgresql.cnpg.io >/dev/null 2>&1; then log "Installing CloudNativePG operator v${CNPG_OPERATOR_VERSION} ..." kubectl apply --server-side -f \ "https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-${CNPG_OPERATOR_VERSION}.yaml" log "Waiting for CNPG operator to be ready ..." kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=120s else log "CNPG operator already installed." fi # ── 8. Apply ServiceAccount + annotate with WI ─────────────────────────────── log "Applying CNPG backup ServiceAccount with Workload Identity annotation ..." GCP_PROJECT_ID="$GCP_PROJECT_ID" envsubst '${GCP_PROJECT_ID}' < "$GKE_MANIFEST_DIR/knoe-db-backup-gcs.yaml" \ | kubectl apply -f - kubectl annotate serviceaccount cnpg-backup-sa \ -n "$CNPG_NAMESPACE" \ "iam.gke.io/gcp-service-account=${GCP_SA_EMAIL}" \ --overwrite # ── 9. Bootstrap CNPG secrets ─────────────────────────────────────────────── ensure_cnpg_tls_secrets ensure_db_user_secret # ── 10. Apply CNPG cluster ─────────────────────────────────────────────────── IFS_COMMA="," resolve_knoe_db_image_tag log "Using knoe-db image tag: ${KNOE_DB_IMAGE_TAG}" resolve_storage_class_candidates IFS=',' read -r -a storage_classes <<< "$CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED" selected_storage_class="" attempt_total="${#storage_classes[@]}" attempt_index=0 if kubectl -n "$CNPG_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then existing_phase="$(kubectl -n "$CNPG_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)" if [[ "$existing_phase" != "Cluster in healthy state" ]]; then log "Existing cluster phase ${existing_phase:-unknown} is not healthy; resetting stale resources before storage-class evaluation." reset_cnpg_cluster_attempt fi fi for storage_class in "${storage_classes[@]}"; do storage_class="${storage_class//[[:space:]]/}" [[ -n "$storage_class" ]] || continue attempt_index=$((attempt_index + 1)) log "Applying knoe-db CNPG cluster with storageClass=${storage_class} (${attempt_index}/${attempt_total}) ..." tmp_manifest="$(mktemp)" render_cnpg_manifest "$storage_class" "$tmp_manifest" kubectl apply -f "$tmp_manifest" rm -f "$tmp_manifest" if wait_for_storage_outcome "$storage_class"; then selected_storage_class="$storage_class" break fi outcome_rc=$? case "$outcome_rc" in 10) log "WARN: storageClass=${storage_class} failed due to SSD quota pressure; stepping down." ;; 11) log "WARN: storageClass=${storage_class} failed provisioning; stepping down." ;; *) log "WARN: storageClass=${storage_class} did not become ready within ${CNPG_STORAGE_WAIT_TIMEOUT}s; stepping down." ;; esac reset_cnpg_cluster_attempt done [[ -n "$selected_storage_class" ]] || { echo "ERROR: CNPG failed to provision across storage class candidates: ${CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED}" >&2 exit 1 } log "CNPG provisioning selected storageClass=${selected_storage_class}." # ── 11. Ensure RoleBindings reference cnpg-backup-sa ───────────────────────── # CNPG auto-creates knoe-db and knoe-db-barman-cloud RoleBindings at cluster # creation. With spec.serviceAccountName set from the start the operator may # use cnpg-backup-sa as the sole subject, but if it still uses the default SA # name (cluster name) we patch both bindings to add cnpg-backup-sa — matching # the live state after the 2026-04-29 stabilization session. log "Ensuring CNPG-managed RoleBindings include cnpg-backup-sa ..." for rb in "${CNPG_CLUSTER_NAME}" "${CNPG_CLUSTER_NAME}-barman-cloud"; do for _ in 1 2 3 4 5; do kubectl -n "${CNPG_NAMESPACE}" get rolebinding "${rb}" >/dev/null 2>&1 && break sleep 2 done if kubectl -n "${CNPG_NAMESPACE}" get rolebinding "${rb}" >/dev/null 2>&1; then if ! kubectl -n "${CNPG_NAMESPACE}" get rolebinding "${rb}" \ -o jsonpath='{.subjects[*].name}' | grep -qw cnpg-backup-sa; then patch_file="$(mktemp)" cat > "${patch_file}" <