mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
c94c62e1bb
- persist and load DB master password via Ansible Vault bootstrap flow - enforce knoe-system service namespace and explicit app/db kubecontext targeting - improve OpenBao/CNPG deploy reliability and logging; add retries/readiness diagnostics - tighten reset/delete cluster behavior and expand installer/deploy pipeline test coverage Co-authored-by: Junie <junie@jetbrains.com>
471 lines
16 KiB
Bash
Executable File
471 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# reset_clusters.sh — Delete and recreate both GKE clusters with correct configuration.
|
||
#
|
||
# Problem: Both clusters were created via GCloud Console as Autopilot, which:
|
||
# - Prevents manual node pool management (patch_clusters.sh cannot work)
|
||
# - Uses pd-balanced boot disks (counts against SSD_TOTAL_GB quota, 300 GB at limit)
|
||
#
|
||
# Solution: Delete both, recreate with pd-standard boot disks:
|
||
# knoe-dev-0 → Standard (app workloads: GitLab, platform), e2-small × 3, VPA, pd-standard boot
|
||
# knoe-cnpg-0 → Standard (CNPG only, 3× e2-standard-2), pd-standard boot
|
||
#
|
||
# SSD quota budget after reset:
|
||
# Boot disks: pd-standard — does NOT count against SSD_TOTAL_GB
|
||
# PGDATA PVCs (premium-rwo): 3 × 50 Gi = 150 Gi \ Apply knoe-db.yaml AFTER
|
||
# WAL PVCs (premium-rwo): 3 × 50 Gi = 150 Gi / quota increase to 2 TB
|
||
#
|
||
# Usage:
|
||
# CONFIRM=true ./scripts/reset_clusters.sh
|
||
# CONFIRM=true DRY_RUN=true ./scripts/reset_clusters.sh # inspect only
|
||
#
|
||
# Environment overrides:
|
||
# GCP_PROJECT (default: plenary-truck-485623-p7)
|
||
# GCP_REGION (default: us-west3)
|
||
# APP_CLUSTER (default: knoe-dev-0)
|
||
# DB_CLUSTER (default: knoe-cnpg-0)
|
||
# DB_MACHINE_TYPE (default: e2-standard-2)
|
||
# DB_DISK_TYPE (default: pd-standard)
|
||
# DB_DISK_SIZE_GB (default: 50)
|
||
# DB_NODES_PER_ZONE (default: 1 → 3 nodes across 3 zones)
|
||
# APP_MACHINE_TYPE (default: e2-small)
|
||
# APP_DISK_TYPE (default: pd-standard)
|
||
# APP_DISK_SIZE_GB (default: 50)
|
||
# APP_NODES_PER_ZONE (default: 1 → 3 nodes across 3 zones)
|
||
# CONFIRM REQUIRED: must be "true" to allow destructive operations
|
||
# DRY_RUN (default: false)
|
||
|
||
set -euo pipefail
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Configuration
|
||
# ---------------------------------------------------------------------------
|
||
|
||
GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
|
||
GCP_REGION="${GCP_REGION:-us-west3}"
|
||
APP_CLUSTER="${APP_CLUSTER:-knoe-dev-0}"
|
||
DB_CLUSTER="${DB_CLUSTER:-knoe-cnpg-0}"
|
||
DB_MACHINE_TYPE="${DB_MACHINE_TYPE:-e2-standard-2}"
|
||
DB_DISK_TYPE="${DB_DISK_TYPE:-pd-standard}"
|
||
DB_DISK_SIZE_GB="${DB_DISK_SIZE_GB:-50}"
|
||
DB_NODES_PER_ZONE="${DB_NODES_PER_ZONE:-1}" # regional cluster = 3 zones = 3 nodes total
|
||
APP_MACHINE_TYPE="${APP_MACHINE_TYPE:-e2-small}"
|
||
APP_DISK_TYPE="${APP_DISK_TYPE:-pd-standard}"
|
||
APP_DISK_SIZE_GB="${APP_DISK_SIZE_GB:-50}"
|
||
APP_NODES_PER_ZONE="${APP_NODES_PER_ZONE:-1}" # regional cluster = 3 zones = 3 nodes total
|
||
CONFIRM="${CONFIRM:-false}"
|
||
DRY_RUN="${DRY_RUN:-false}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
|
||
|
||
die() { log "ERROR: $*" >&2; exit 1; }
|
||
|
||
require_tool() {
|
||
command -v "$1" >/dev/null 2>&1 || die "required tool not found: $1"
|
||
}
|
||
|
||
run_cmd() {
|
||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||
echo "[DRY-RUN] $*"
|
||
else
|
||
"$@"
|
||
fi
|
||
}
|
||
|
||
cluster_exists() {
|
||
local name="$1"
|
||
gcloud container clusters describe "${name}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--format="value(name)" \
|
||
--quiet 2>/dev/null | grep -q "${name}"
|
||
}
|
||
|
||
wait_for_cluster_absent() {
|
||
local name="$1"
|
||
local timeout_s="${2:-900}"
|
||
local poll_s="${3:-15}"
|
||
local elapsed=0
|
||
|
||
while cluster_exists "${name}"; do
|
||
if [[ ${elapsed} -ge ${timeout_s} ]]; then
|
||
return 1
|
||
fi
|
||
log " ${name} still exists; waiting ${poll_s}s (${elapsed}s/${timeout_s}s) ..."
|
||
sleep "${poll_s}"
|
||
elapsed=$(( elapsed + poll_s ))
|
||
done
|
||
|
||
return 0
|
||
}
|
||
|
||
delete_cluster_until_absent() {
|
||
local name="$1"
|
||
local max_attempts="${2:-6}"
|
||
local retry_sleep_s="${3:-20}"
|
||
local attempt=1
|
||
local output=""
|
||
local status=0
|
||
|
||
if ! cluster_exists "${name}"; then
|
||
log " ${name} not found — skipping."
|
||
return 0
|
||
fi
|
||
|
||
while [[ ${attempt} -le ${max_attempts} ]]; do
|
||
log " Deleting ${name} (attempt ${attempt}/${max_attempts}) ..."
|
||
|
||
set +e
|
||
output=$(gcloud container clusters delete "${name}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--quiet 2>&1)
|
||
status=$?
|
||
set -e
|
||
|
||
if [[ ${status} -ne 0 ]]; then
|
||
if grep -qi "incompatible operation" <<<"${output}"; then
|
||
log " ${name} has an incompatible operation in progress; retrying after ${retry_sleep_s}s."
|
||
elif grep -qi "not found" <<<"${output}"; then
|
||
log " ${name} already absent."
|
||
return 0
|
||
else
|
||
printf "%s\n" "${output}" >&2
|
||
die "cluster delete failed for ${name}"
|
||
fi
|
||
fi
|
||
|
||
if wait_for_cluster_absent "${name}" 900 15; then
|
||
log " ${name} deletion confirmed."
|
||
return 0
|
||
fi
|
||
|
||
log " ${name} still present after delete attempt ${attempt}; retrying."
|
||
sleep "${retry_sleep_s}"
|
||
attempt=$(( attempt + 1 ))
|
||
done
|
||
|
||
die "timed out deleting ${name} after ${max_attempts} attempts"
|
||
}
|
||
|
||
_ssd_quota_yaml() {
|
||
# Emit the 3-line YAML block for SSD_TOTAL_GB quota entry, e.g.:
|
||
# - limit: 300.0
|
||
# metric: SSD_TOTAL_GB
|
||
# usage: 300.0
|
||
gcloud compute regions describe "${GCP_REGION}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--format=yaml \
|
||
--quiet 2>/dev/null | grep -B 1 -A 1 "metric: SSD_TOTAL_GB" || true
|
||
}
|
||
|
||
ssd_usage_gb() {
|
||
local block
|
||
block=$(_ssd_quota_yaml)
|
||
echo "${block}" | awk '/usage:/{print $2}' | head -1 || echo "unknown"
|
||
}
|
||
|
||
ssd_limit_gb() {
|
||
local block
|
||
block=$(_ssd_quota_yaml)
|
||
echo "${block}" | awk '/limit:/{print $2}' | head -1 || echo "unknown"
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Pre-flight
|
||
# ---------------------------------------------------------------------------
|
||
|
||
require_tool gcloud
|
||
require_tool kubectl
|
||
|
||
if [[ "${CONFIRM}" != "true" ]]; then
|
||
echo ""
|
||
echo " This script will DELETE and RECREATE both GKE clusters:"
|
||
echo " ${APP_CLUSTER} (Autopilot) and ${DB_CLUSTER} (Standard)"
|
||
echo ""
|
||
echo " Set CONFIRM=true to proceed:"
|
||
echo " CONFIRM=true ./scripts/reset_clusters.sh"
|
||
echo ""
|
||
exit 1
|
||
fi
|
||
|
||
log "==> Cluster reset: ${APP_CLUSTER} (Standard) + ${DB_CLUSTER} (Standard)"
|
||
log " Project : ${GCP_PROJECT}"
|
||
log " Region : ${GCP_REGION}"
|
||
log " DRY_RUN : ${DRY_RUN}"
|
||
echo ""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 1: Show current SSD quota
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 1] Current SSD quota (SSD_TOTAL_GB) in ${GCP_REGION} ..."
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
usage=$(ssd_usage_gb)
|
||
limit=$(ssd_limit_gb)
|
||
log " SSD usage: ${usage} GB / ${limit} GB limit"
|
||
if [[ "${usage}" == "unknown" || "${limit}" == "unknown" ]]; then
|
||
log " WARNING: Could not read SSD quota — proceeding anyway."
|
||
fi
|
||
else
|
||
log " [DRY-RUN] Would read SSD quota from ${GCP_REGION}"
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 2: Delete existing clusters
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 2] Deleting existing clusters ..."
|
||
|
||
for cluster in "${DB_CLUSTER}" "${APP_CLUSTER}"; do
|
||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||
echo "[DRY-RUN] gcloud container clusters delete ${cluster} --project=${GCP_PROJECT} --region=${GCP_REGION} --quiet"
|
||
else
|
||
delete_cluster_until_absent "${cluster}"
|
||
fi
|
||
done
|
||
|
||
# Confirm all target clusters are absent before proceeding
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
log " Waiting for cluster deletions to complete ..."
|
||
for cluster in "${DB_CLUSTER}" "${APP_CLUSTER}"; do
|
||
wait_for_cluster_absent "${cluster}" 900 15 \
|
||
|| die "cluster ${cluster} still exists after deletion phase"
|
||
done
|
||
log " All deletions complete."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 2b: Delete orphaned pd-ssd disks left behind by cluster deletion
|
||
# ---------------------------------------------------------------------------
|
||
# GKE cluster deletion does not remove PersistentVolume-backed GCP disks.
|
||
# Orphaned pd-ssd disks hold SSD quota and prevent reset_clusters.sh from
|
||
# proceeding (Phase 3 waits for SSD usage < 50 GB).
|
||
|
||
log "[phase 2b] Deleting orphaned pd-ssd disks in ${GCP_REGION} ..."
|
||
|
||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||
log " [DRY-RUN] Would list and delete orphaned pd-ssd disks in ${GCP_REGION}"
|
||
else
|
||
# List all disks in the project, filter to pd-ssd in this region in bash
|
||
mapfile -t ssd_disks < <(
|
||
gcloud compute disks list \
|
||
--project="${GCP_PROJECT}" \
|
||
--format="csv[no-heading](name,zone,type)" \
|
||
--quiet 2>/dev/null \
|
||
| awk -F',' "/${GCP_REGION}/ && /pd-ssd/ {print \$1 \",\" \$2}" \
|
||
|| true
|
||
)
|
||
|
||
if [[ ${#ssd_disks[@]} -eq 0 ]]; then
|
||
log " No orphaned pd-ssd disks found."
|
||
else
|
||
for entry in "${ssd_disks[@]}"; do
|
||
disk_name="${entry%%,*}"
|
||
disk_zone="${entry##*,}"
|
||
# Extract just the zone name (may be a full URL)
|
||
disk_zone="${disk_zone##*/}"
|
||
log " Deleting pd-ssd disk: ${disk_name} (zone: ${disk_zone})"
|
||
gcloud compute disks delete "${disk_name}" \
|
||
--zone="${disk_zone}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--quiet 2>/dev/null \
|
||
&& log " Deleted ${disk_name}." \
|
||
|| log " WARNING: Could not delete ${disk_name} — may already be gone."
|
||
done
|
||
log " Orphaned pd-ssd disk cleanup complete."
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 3: Wait for SSD quota to be released
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 3] Waiting for SSD quota to be released ..."
|
||
|
||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||
log " [DRY-RUN] Would poll SSD_TOTAL_GB until usage < 50 GB"
|
||
else
|
||
max_wait_s=1200 # 20 min max
|
||
poll_s=30
|
||
elapsed=0
|
||
while true; do
|
||
usage=$(ssd_usage_gb)
|
||
log " SSD usage: ${usage} GB (${elapsed}s elapsed)"
|
||
if [[ "${usage}" == "unknown" ]]; then
|
||
log " WARNING: Could not read quota — treating as released."
|
||
break
|
||
fi
|
||
# Cast to int for comparison
|
||
usage_int=${usage%.*}
|
||
if [[ "${usage_int}" -lt 50 ]]; then
|
||
log " SSD quota released (${usage} GB remaining usage)."
|
||
break
|
||
fi
|
||
if [[ ${elapsed} -ge ${max_wait_s} ]]; then
|
||
log " WARNING: SSD quota did not fully release within ${max_wait_s}s."
|
||
log " Current usage: ${usage} GB — proceeding with pd-standard (no SSD impact)."
|
||
break
|
||
fi
|
||
sleep "${poll_s}"
|
||
elapsed=$(( elapsed + poll_s ))
|
||
done
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 4: Create knoe-cnpg-0 as Standard GKE cluster
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 4] Creating ${DB_CLUSTER} (Standard, ${DB_MACHINE_TYPE}, ${DB_DISK_TYPE}, ${DB_DISK_SIZE_GB}GB) ..."
|
||
|
||
run_cmd gcloud container clusters create "${DB_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--cluster-version=latest \
|
||
--machine-type="${DB_MACHINE_TYPE}" \
|
||
--node-labels="workload=db" \
|
||
--disk-type="${DB_DISK_TYPE}" \
|
||
--disk-size="${DB_DISK_SIZE_GB}" \
|
||
--num-nodes="${DB_NODES_PER_ZONE}" \
|
||
--enable-ip-alias \
|
||
--workload-pool="${GCP_PROJECT}.svc.id.goog" \
|
||
--quiet
|
||
|
||
log " ${DB_CLUSTER} created."
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 5: Create knoe-dev-0 as Standard cluster
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 5] Creating ${APP_CLUSTER} (Standard, ${APP_MACHINE_TYPE}, ${APP_DISK_TYPE}, ${APP_DISK_SIZE_GB}GB, VPA) ..."
|
||
|
||
run_cmd gcloud container clusters create "${APP_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--cluster-version=latest \
|
||
--machine-type="${APP_MACHINE_TYPE}" \
|
||
--disk-type="${APP_DISK_TYPE}" \
|
||
--disk-size="${APP_DISK_SIZE_GB}" \
|
||
--num-nodes="${APP_NODES_PER_ZONE}" \
|
||
--enable-vertical-pod-autoscaling \
|
||
--enable-ip-alias \
|
||
--workload-pool="${GCP_PROJECT}.svc.id.goog" \
|
||
--quiet
|
||
# Standard mode: pd-standard boot disks consume ZERO SSD quota (vs Autopilot's
|
||
# pd-balanced 100GB/node which exhausted the full 300GB SSD_TOTAL_GB quota).
|
||
# VPA handles dynamic resource adjustment for burstable e2-medium nodes.
|
||
|
||
log " ${APP_CLUSTER} created."
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 6: Fetch credentials and verify
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 6] Fetching kubeconfig credentials ..."
|
||
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
run_cmd gcloud container clusters get-credentials "${DB_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--quiet
|
||
db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
|
||
log " DB cluster context: ${db_ctx}"
|
||
|
||
run_cmd gcloud container clusters get-credentials "${APP_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--quiet
|
||
app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
|
||
log " App cluster context: ${app_ctx}"
|
||
|
||
log " Verifying cluster connectivity ..."
|
||
kubectl --context="${db_ctx}" cluster-info --request-timeout=15s \
|
||
&& log " ${DB_CLUSTER}: OK" \
|
||
|| log " WARNING: ${DB_CLUSTER} not yet reachable — may need a moment."
|
||
kubectl --context="${app_ctx}" cluster-info --request-timeout=15s \
|
||
&& log " ${APP_CLUSTER}: OK" \
|
||
|| log " WARNING: ${APP_CLUSTER} not yet reachable — may need a moment."
|
||
else
|
||
db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
|
||
app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
|
||
log " [DRY-RUN] Would fetch credentials for both clusters."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 7: Register clusters in knoe-0 fleet + enable service mesh
|
||
# ---------------------------------------------------------------------------
|
||
# Cloud Service Mesh (CSM) is enabled per-cluster via fleet membership.
|
||
# This automates what was previously done via the GCP web console.
|
||
|
||
log "[phase 7] Registering clusters in fleet and enabling service mesh ..."
|
||
|
||
_register_and_mesh() {
|
||
local cluster_name="$1"
|
||
log " Registering ${cluster_name} in fleet ..."
|
||
if ! run_cmd gcloud container fleet memberships register "${cluster_name}" \
|
||
--gke-cluster="${GCP_REGION}/${cluster_name}" \
|
||
--enable-workload-identity \
|
||
--project="${GCP_PROJECT}" \
|
||
--quiet 2>&1; then
|
||
log " WARNING: Fleet registration for ${cluster_name} failed (may already be registered — continuing)."
|
||
fi
|
||
|
||
log " Enabling automatic service mesh management for ${cluster_name} ..."
|
||
if ! run_cmd gcloud container fleet mesh update \
|
||
--management=automatic \
|
||
--memberships="${cluster_name}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--quiet 2>&1; then
|
||
log " WARNING: Service mesh update for ${cluster_name} failed — enable manually via GCP console."
|
||
fi
|
||
}
|
||
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
_register_and_mesh "${DB_CLUSTER}"
|
||
_register_and_mesh "${APP_CLUSTER}"
|
||
log " Fleet + mesh registration submitted. Mesh provisioning is async (~10 min)."
|
||
log " Check status: gcloud container fleet mesh describe --project=${GCP_PROJECT}"
|
||
else
|
||
log " [DRY-RUN] Would register ${DB_CLUSTER} and ${APP_CLUSTER} in fleet + enable service mesh."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 8: Show final SSD quota
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 8] Final SSD quota ..."
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
usage=$(ssd_usage_gb)
|
||
limit=$(ssd_limit_gb)
|
||
log " SSD usage: ${usage} GB / ${limit} GB (pd-standard boot disks use 0 SSD quota)"
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Done
|
||
# ---------------------------------------------------------------------------
|
||
|
||
echo ""
|
||
log "==> Reset complete."
|
||
log ""
|
||
log "Contexts:"
|
||
log " App (Standard) : ${app_ctx}"
|
||
log " DB (Standard) : ${db_ctx}"
|
||
log ""
|
||
log "Next steps:"
|
||
log " 1. Run the installer to configure workloads:"
|
||
log " ./install.sh"
|
||
log ""
|
||
log " 2. Once the SSD quota increase (300 GB → 2 TB) is approved, apply CNPG storage:"
|
||
log " kubectl --context=${db_ctx} apply -f deploy/gcp/gke/knoe-db.yaml"
|
||
log " # This provisions 3×50Gi PGDATA + 3×50Gi WAL = 300 Gi pd-ssd"
|
||
log ""
|
||
log " 3. Check your quota increase request:"
|
||
log " gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} \\"
|
||
log " --format='table(quotas.metric,quotas.usage,quotas.limit)' | grep SSD"
|
||
log ""
|
||
log " 4. Check service mesh provisioning status (~10 min after cluster creation):"
|
||
log " gcloud container fleet mesh describe --project=${GCP_PROJECT}"
|