mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
64e1cef16c
- make cluster-storage milestone opt-in and remove installer cluster-storage step from UI navigation\n- add cluster storage browser and GKE cluster ops helpers with CLI coverage\n- update k8s/CNPG config and install flow files for corrected cluster setup\n- add/refresh tests for storage browser, GKE ops, prod config, and service-layer navigation Co-authored-by: Junie <junie@jetbrains.com>
318 lines
11 KiB
Bash
Executable File
318 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# reset_clusters.sh — Delete and recreate both GKE clusters with correct configuration.
|
||
#
|
||
# Problem: Both clusters were created via GCloud Console as Autopilot, which:
|
||
# - Prevents manual node pool management (patch_clusters.sh cannot work)
|
||
# - Uses pd-balanced boot disks (counts against SSD_TOTAL_GB quota, 300 GB at limit)
|
||
#
|
||
# Solution: Delete both, recreate with pd-standard boot disks:
|
||
# knoe-dev-0 → Autopilot (app workloads: GitLab, platform), pd-standard boot
|
||
# knoe-cnpg-0 → Standard (CNPG only, 3× e2-standard-2), pd-standard boot
|
||
#
|
||
# SSD quota budget after reset:
|
||
# Boot disks: pd-standard — does NOT count against SSD_TOTAL_GB
|
||
# PGDATA PVCs (premium-rwo): 3 × 50 Gi = 150 Gi \ Apply knoe-db.yaml AFTER
|
||
# WAL PVCs (premium-rwo): 3 × 50 Gi = 150 Gi / quota increase to 2 TB
|
||
#
|
||
# Usage:
|
||
# CONFIRM=true ./scripts/reset_clusters.sh
|
||
# CONFIRM=true DRY_RUN=true ./scripts/reset_clusters.sh # inspect only
|
||
#
|
||
# Environment overrides:
|
||
# GCP_PROJECT (default: plenary-truck-485623-p7)
|
||
# GCP_REGION (default: us-west3)
|
||
# APP_CLUSTER (default: knoe-dev-0)
|
||
# DB_CLUSTER (default: knoe-cnpg-0)
|
||
# DB_MACHINE_TYPE (default: e2-standard-2)
|
||
# DB_DISK_TYPE (default: pd-standard)
|
||
# DB_DISK_SIZE_GB (default: 50)
|
||
# DB_NODES_PER_ZONE (default: 1 → 3 nodes across 3 zones)
|
||
# APP_DISK_TYPE (default: pd-standard)
|
||
# APP_DISK_SIZE_GB (default: 50)
|
||
# CONFIRM REQUIRED: must be "true" to allow destructive operations
|
||
# DRY_RUN (default: false)
|
||
|
||
set -euo pipefail
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Configuration
|
||
# ---------------------------------------------------------------------------
|
||
|
||
GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
|
||
GCP_REGION="${GCP_REGION:-us-west3}"
|
||
APP_CLUSTER="${APP_CLUSTER:-knoe-dev-0}"
|
||
DB_CLUSTER="${DB_CLUSTER:-knoe-cnpg-0}"
|
||
DB_MACHINE_TYPE="${DB_MACHINE_TYPE:-e2-standard-2}"
|
||
DB_DISK_TYPE="${DB_DISK_TYPE:-pd-standard}"
|
||
DB_DISK_SIZE_GB="${DB_DISK_SIZE_GB:-50}"
|
||
DB_NODES_PER_ZONE="${DB_NODES_PER_ZONE:-1}" # regional cluster = 3 zones = 3 nodes total
|
||
APP_DISK_TYPE="${APP_DISK_TYPE:-pd-standard}"
|
||
APP_DISK_SIZE_GB="${APP_DISK_SIZE_GB:-50}"
|
||
CONFIRM="${CONFIRM:-false}"
|
||
DRY_RUN="${DRY_RUN:-false}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
|
||
|
||
die() { log "ERROR: $*" >&2; exit 1; }
|
||
|
||
require_tool() {
|
||
command -v "$1" >/dev/null 2>&1 || die "required tool not found: $1"
|
||
}
|
||
|
||
run_cmd() {
|
||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||
echo "[DRY-RUN] $*"
|
||
else
|
||
"$@"
|
||
fi
|
||
}
|
||
|
||
cluster_exists() {
|
||
local name="$1"
|
||
gcloud container clusters describe "${name}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--format="value(name)" \
|
||
--quiet 2>/dev/null | grep -q "${name}"
|
||
}
|
||
|
||
_ssd_quota_yaml() {
|
||
# Emit the 3-line YAML block for SSD_TOTAL_GB quota entry, e.g.:
|
||
# - limit: 300.0
|
||
# metric: SSD_TOTAL_GB
|
||
# usage: 300.0
|
||
gcloud compute regions describe "${GCP_REGION}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--format=yaml \
|
||
--quiet 2>/dev/null | grep -B 1 -A 1 "metric: SSD_TOTAL_GB" || true
|
||
}
|
||
|
||
ssd_usage_gb() {
|
||
local block
|
||
block=$(_ssd_quota_yaml)
|
||
echo "${block}" | awk '/usage:/{print $2}' | head -1 || echo "unknown"
|
||
}
|
||
|
||
ssd_limit_gb() {
|
||
local block
|
||
block=$(_ssd_quota_yaml)
|
||
echo "${block}" | awk '/limit:/{print $2}' | head -1 || echo "unknown"
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Pre-flight
|
||
# ---------------------------------------------------------------------------
|
||
|
||
require_tool gcloud
|
||
require_tool kubectl
|
||
|
||
if [[ "${CONFIRM}" != "true" ]]; then
|
||
echo ""
|
||
echo " This script will DELETE and RECREATE both GKE clusters:"
|
||
echo " ${APP_CLUSTER} (Autopilot) and ${DB_CLUSTER} (Standard)"
|
||
echo ""
|
||
echo " Set CONFIRM=true to proceed:"
|
||
echo " CONFIRM=true ./scripts/reset_clusters.sh"
|
||
echo ""
|
||
exit 1
|
||
fi
|
||
|
||
log "==> Cluster reset: ${APP_CLUSTER} (Autopilot) + ${DB_CLUSTER} (Standard)"
|
||
log " Project : ${GCP_PROJECT}"
|
||
log " Region : ${GCP_REGION}"
|
||
log " DRY_RUN : ${DRY_RUN}"
|
||
echo ""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 1: Show current SSD quota
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 1] Current SSD quota (SSD_TOTAL_GB) in ${GCP_REGION} ..."
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
usage=$(ssd_usage_gb)
|
||
limit=$(ssd_limit_gb)
|
||
log " SSD usage: ${usage} GB / ${limit} GB limit"
|
||
if [[ "${usage}" == "unknown" || "${limit}" == "unknown" ]]; then
|
||
log " WARNING: Could not read SSD quota — proceeding anyway."
|
||
fi
|
||
else
|
||
log " [DRY-RUN] Would read SSD quota from ${GCP_REGION}"
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 2: Delete existing clusters
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 2] Deleting existing clusters ..."
|
||
|
||
for cluster in "${DB_CLUSTER}" "${APP_CLUSTER}"; do
|
||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||
echo "[DRY-RUN] gcloud container clusters delete ${cluster} --project=${GCP_PROJECT} --region=${GCP_REGION} --quiet"
|
||
else
|
||
if cluster_exists "${cluster}"; then
|
||
log " Deleting ${cluster} (this takes ~5-10 min) ..."
|
||
gcloud container clusters delete "${cluster}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--quiet &
|
||
log " ${cluster} deletion running in background (PID $!)."
|
||
else
|
||
log " ${cluster} not found — skipping."
|
||
fi
|
||
fi
|
||
done
|
||
|
||
# Wait for all background deletions to finish
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
log " Waiting for cluster deletions to complete ..."
|
||
wait
|
||
log " All deletions complete."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 3: Wait for SSD quota to be released
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 3] Waiting for SSD quota to be released ..."
|
||
|
||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||
log " [DRY-RUN] Would poll SSD_TOTAL_GB until usage < 50 GB"
|
||
else
|
||
max_wait_s=1200 # 20 min max
|
||
poll_s=30
|
||
elapsed=0
|
||
while true; do
|
||
usage=$(ssd_usage_gb)
|
||
log " SSD usage: ${usage} GB (${elapsed}s elapsed)"
|
||
if [[ "${usage}" == "unknown" ]]; then
|
||
log " WARNING: Could not read quota — treating as released."
|
||
break
|
||
fi
|
||
# Cast to int for comparison
|
||
usage_int=${usage%.*}
|
||
if [[ "${usage_int}" -lt 50 ]]; then
|
||
log " SSD quota released (${usage} GB remaining usage)."
|
||
break
|
||
fi
|
||
if [[ ${elapsed} -ge ${max_wait_s} ]]; then
|
||
log " WARNING: SSD quota did not fully release within ${max_wait_s}s."
|
||
log " Current usage: ${usage} GB — proceeding with pd-standard (no SSD impact)."
|
||
break
|
||
fi
|
||
sleep "${poll_s}"
|
||
elapsed=$(( elapsed + poll_s ))
|
||
done
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 4: Create knoe-cnpg-0 as Standard GKE cluster
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 4] Creating ${DB_CLUSTER} (Standard, ${DB_MACHINE_TYPE}, ${DB_DISK_TYPE}, ${DB_DISK_SIZE_GB}GB) ..."
|
||
|
||
run_cmd gcloud container clusters create "${DB_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--cluster-version=latest \
|
||
--machine-type="${DB_MACHINE_TYPE}" \
|
||
--disk-type="${DB_DISK_TYPE}" \
|
||
--disk-size="${DB_DISK_SIZE_GB}" \
|
||
--num-nodes="${DB_NODES_PER_ZONE}" \
|
||
--enable-ip-alias \
|
||
--workload-pool="${GCP_PROJECT}.svc.id.goog" \
|
||
--quiet
|
||
|
||
log " ${DB_CLUSTER} created."
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 5: Create knoe-dev-0 as Autopilot cluster
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 5] Creating ${APP_CLUSTER} (Autopilot, ${APP_DISK_TYPE}, ${APP_DISK_SIZE_GB}GB) ..."
|
||
|
||
run_cmd gcloud container clusters create-auto "${APP_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--cluster-version=latest \
|
||
--workload-policies=allow-net-admin \
|
||
--quiet
|
||
# NOTE: Autopilot manages node infrastructure (disk type, size) automatically.
|
||
# Boot disk type cannot be set at Autopilot cluster creation time.
|
||
# Autopilot only provisions nodes when pods are scheduled — no quota impact
|
||
# while the cluster is idle. Deploy workloads after the SSD quota increase
|
||
# (300 GB → 2 TB) is approved to ensure headroom for GitLab node boot disks.
|
||
|
||
log " ${APP_CLUSTER} created."
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 6: Fetch credentials and verify
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 6] Fetching kubeconfig credentials ..."
|
||
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
run_cmd gcloud container clusters get-credentials "${DB_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--quiet
|
||
db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
|
||
log " DB cluster context: ${db_ctx}"
|
||
|
||
run_cmd gcloud container clusters get-credentials "${APP_CLUSTER}" \
|
||
--project="${GCP_PROJECT}" \
|
||
--region="${GCP_REGION}" \
|
||
--quiet
|
||
app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
|
||
log " App cluster context: ${app_ctx}"
|
||
|
||
log " Verifying cluster connectivity ..."
|
||
kubectl --context="${db_ctx}" cluster-info --request-timeout=15s \
|
||
&& log " ${DB_CLUSTER}: OK" \
|
||
|| log " WARNING: ${DB_CLUSTER} not yet reachable — may need a moment."
|
||
kubectl --context="${app_ctx}" cluster-info --request-timeout=15s \
|
||
&& log " ${APP_CLUSTER}: OK" \
|
||
|| log " WARNING: ${APP_CLUSTER} not yet reachable — may need a moment."
|
||
else
|
||
db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
|
||
app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
|
||
log " [DRY-RUN] Would fetch credentials for both clusters."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 7: Show final SSD quota
|
||
# ---------------------------------------------------------------------------
|
||
|
||
log "[phase 7] Final SSD quota ..."
|
||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||
usage=$(ssd_usage_gb)
|
||
limit=$(ssd_limit_gb)
|
||
log " SSD usage: ${usage} GB / ${limit} GB (pd-standard boot disks use 0 SSD quota)"
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Done
|
||
# ---------------------------------------------------------------------------
|
||
|
||
echo ""
|
||
log "==> Reset complete."
|
||
log ""
|
||
log "Contexts:"
|
||
log " App (Autopilot): ${app_ctx}"
|
||
log " DB (Standard) : ${db_ctx}"
|
||
log ""
|
||
log "Next steps:"
|
||
log " 1. Run the installer to configure workloads:"
|
||
log " ./install.sh"
|
||
log ""
|
||
log " 2. Once the SSD quota increase (300 GB → 2 TB) is approved, apply CNPG storage:"
|
||
log " kubectl --context=${db_ctx} apply -f deploy/gcp/gke/knoe-db.yaml"
|
||
log " # This provisions 3×50Gi PGDATA + 3×50Gi WAL = 300 Gi pd-ssd"
|
||
log ""
|
||
log " 3. Check your quota increase request:"
|
||
log " gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} \\"
|
||
log " --format='table(quotas.metric,quotas.usage,quotas.limit)' | grep SSD"
|