mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
64e1cef16c
- make cluster-storage milestone opt-in and remove installer cluster-storage step from UI navigation\n- add cluster storage browser and GKE cluster ops helpers with CLI coverage\n- update k8s/CNPG config and install flow files for corrected cluster setup\n- add/refresh tests for storage browser, GKE ops, prod config, and service-layer navigation Co-authored-by: Junie <junie@jetbrains.com>
252 lines
9.0 KiB
Bash
Executable File
252 lines
9.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# patch_clusters.sh — Migrate knoe-cnpg-0 boot disks from pd-ssd to pd-standard.
|
|
#
|
|
# The GCloud Console creates node pools with pd-ssd boot disks by default, which
|
|
# consumes the entire 300 GB pd-ssd quota before any PGDATA/WAL PVCs can be
|
|
# provisioned. This script replaces the existing default-pool with a new
|
|
# cnpg-db-pool that uses pd-standard boot disks, freeing the quota for CNPG PVCs.
|
|
#
|
|
# What this does:
|
|
# 1. Detects the current node pool disk configuration.
|
|
# 2. Creates cnpg-db-pool (pd-standard boot, e2-standard-2, 3 nodes).
|
|
# 3. Cordons and drains all nodes in the old pool.
|
|
# 4. Deletes the old pool.
|
|
# 5. Verifies 3 CNPG pods return to Running.
|
|
#
|
|
# Usage:
|
|
# ./scripts/patch_clusters.sh
|
|
#
|
|
# Environment overrides:
|
|
# GCP_PROJECT — GCP project ID (default: plenary-truck-485623-p7)
|
|
# GCP_REGION — GCP region (default: us-west3)
|
|
# CLUSTER_NAME — GKE cluster (default: knoe-cnpg-0)
|
|
# OLD_POOL — Pool to remove (default: default-pool)
|
|
# NEW_POOL — Pool to create (default: cnpg-db-pool)
|
|
# MACHINE_TYPE — Node machine type (default: e2-standard-2)
|
|
# DISK_TYPE — Boot disk type (default: pd-standard)
|
|
# DISK_SIZE_GB — Boot disk size GB (default: 50)
|
|
# NODE_COUNT — Nodes per zone (default: 1, regional = 3 zones)
|
|
# CNPG_NAMESPACE — Namespace for CNPG pods (default: knoe-db-0)
|
|
# DRAIN_TIMEOUT — kubectl drain timeout (default: 300s)
|
|
# DRY_RUN — Set to "true" to print commands without running (default: false)
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
|
|
GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
|
|
GCP_REGION="${GCP_REGION:-us-west3}"
|
|
CLUSTER_NAME="${CLUSTER_NAME:-knoe-cnpg-0}"
|
|
OLD_POOL="${OLD_POOL:-default-pool}"
|
|
NEW_POOL="${NEW_POOL:-cnpg-db-pool}"
|
|
MACHINE_TYPE="${MACHINE_TYPE:-e2-standard-2}"
|
|
DISK_TYPE="${DISK_TYPE:-pd-standard}"
|
|
DISK_SIZE_GB="${DISK_SIZE_GB:-50}"
|
|
NODE_COUNT="${NODE_COUNT:-1}" # 1 per zone; regional cluster spans 3 zones = 3 nodes total
|
|
CNPG_NAMESPACE="${CNPG_NAMESPACE:-knoe-db-0}"
|
|
DRAIN_TIMEOUT="${DRAIN_TIMEOUT:-300s}"
|
|
DRY_RUN="${DRY_RUN:-false}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log() {
|
|
printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"
|
|
}
|
|
|
|
require_tool() {
|
|
command -v "$1" >/dev/null 2>&1 || { log "Error: required tool not found: $1" >&2; exit 1; }
|
|
}
|
|
|
|
run_cmd() {
|
|
if [[ "${DRY_RUN}" == "true" ]]; then
|
|
echo "[DRY-RUN] $*"
|
|
else
|
|
"$@"
|
|
fi
|
|
}
|
|
|
|
pool_exists() {
|
|
local pool="$1"
|
|
gcloud container node-pools describe "${pool}" \
|
|
--cluster="${CLUSTER_NAME}" \
|
|
--project="${GCP_PROJECT}" \
|
|
--region="${GCP_REGION}" \
|
|
--format="value(name)" \
|
|
--quiet 2>/dev/null | grep -q "${pool}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pre-flight
|
|
# ---------------------------------------------------------------------------
|
|
|
|
require_tool gcloud
|
|
require_tool kubectl
|
|
|
|
log "==> Patch: migrate ${CLUSTER_NAME}/${OLD_POOL} → ${NEW_POOL} (${DISK_TYPE}, ${DISK_SIZE_GB}GB)"
|
|
log " Project : ${GCP_PROJECT}"
|
|
log " Region : ${GCP_REGION}"
|
|
log " DRY_RUN : ${DRY_RUN}"
|
|
echo ""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: Detect current configuration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log "[phase 1] Detecting current node pool configuration ..."
|
|
|
|
if ! gcloud container clusters describe "${CLUSTER_NAME}" \
|
|
--project="${GCP_PROJECT}" \
|
|
--region="${GCP_REGION}" \
|
|
--format="value(name)" \
|
|
--quiet 2>/dev/null | grep -q "${CLUSTER_NAME}"; then
|
|
log "Error: cluster '${CLUSTER_NAME}' not found in project '${GCP_PROJECT}' region '${GCP_REGION}'." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if pool_exists "${OLD_POOL}"; then
|
|
current_disk=$(gcloud container node-pools describe "${OLD_POOL}" \
|
|
--cluster="${CLUSTER_NAME}" \
|
|
--project="${GCP_PROJECT}" \
|
|
--region="${GCP_REGION}" \
|
|
--format="value(config.diskType)" \
|
|
--quiet 2>/dev/null || echo "unknown")
|
|
current_size=$(gcloud container node-pools describe "${OLD_POOL}" \
|
|
--cluster="${CLUSTER_NAME}" \
|
|
--project="${GCP_PROJECT}" \
|
|
--region="${GCP_REGION}" \
|
|
--format="value(config.diskSizeGb)" \
|
|
--quiet 2>/dev/null || echo "unknown")
|
|
log " ${OLD_POOL}: disk_type=${current_disk}, disk_size_gb=${current_size}"
|
|
if [[ "${current_disk}" == "pd-standard" ]]; then
|
|
log " NOTE: ${OLD_POOL} already uses pd-standard. Continuing to ensure pool is renamed to '${NEW_POOL}'."
|
|
fi
|
|
else
|
|
log " ${OLD_POOL} not found — may have been removed already."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: Create new node pool
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log "[phase 2] Ensuring node pool '${NEW_POOL}' exists ..."
|
|
|
|
if pool_exists "${NEW_POOL}"; then
|
|
log " ${NEW_POOL} already exists — skipping creation."
|
|
else
|
|
log " Creating ${NEW_POOL} (machine=${MACHINE_TYPE}, disk=${DISK_TYPE}, size=${DISK_SIZE_GB}GB, num-nodes=${NODE_COUNT}/zone) ..."
|
|
run_cmd gcloud container node-pools create "${NEW_POOL}" \
|
|
--cluster="${CLUSTER_NAME}" \
|
|
--project="${GCP_PROJECT}" \
|
|
--region="${GCP_REGION}" \
|
|
--machine-type="${MACHINE_TYPE}" \
|
|
--disk-type="${DISK_TYPE}" \
|
|
--disk-size="${DISK_SIZE_GB}" \
|
|
--num-nodes="${NODE_COUNT}" \
|
|
--quiet
|
|
log " ${NEW_POOL} created."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: Cordon and drain old pool
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log "[phase 3] Cordoning and draining '${OLD_POOL}' ..."
|
|
|
|
if ! pool_exists "${OLD_POOL}"; then
|
|
log " ${OLD_POOL} not found — nothing to drain."
|
|
else
|
|
old_nodes=$(kubectl get nodes \
|
|
-l "cloud.google.com/gke-nodepool=${OLD_POOL}" \
|
|
--no-headers \
|
|
-o custom-columns=":metadata.name" 2>/dev/null || true)
|
|
|
|
if [[ -z "${old_nodes}" ]]; then
|
|
log " No nodes found labelled with nodepool=${OLD_POOL} — already drained or pool empty."
|
|
else
|
|
for node in ${old_nodes}; do
|
|
log " Cordoning ${node} ..."
|
|
run_cmd kubectl cordon "${node}"
|
|
done
|
|
|
|
for node in ${old_nodes}; do
|
|
log " Draining ${node} (timeout ${DRAIN_TIMEOUT}) ..."
|
|
run_cmd kubectl drain "${node}" \
|
|
--ignore-daemonsets \
|
|
--delete-emptydir-data \
|
|
--timeout="${DRAIN_TIMEOUT}" \
|
|
--force
|
|
done
|
|
log " All nodes in ${OLD_POOL} drained."
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: Delete old pool
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log "[phase 4] Deleting old pool '${OLD_POOL}' ..."
|
|
|
|
if ! pool_exists "${OLD_POOL}"; then
|
|
log " ${OLD_POOL} does not exist — already removed."
|
|
else
|
|
run_cmd gcloud container node-pools delete "${OLD_POOL}" \
|
|
--cluster="${CLUSTER_NAME}" \
|
|
--project="${GCP_PROJECT}" \
|
|
--region="${GCP_REGION}" \
|
|
--quiet
|
|
log " ${OLD_POOL} deleted."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 5: Verify CNPG pods
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log "[phase 5] Verifying CNPG pods in namespace '${CNPG_NAMESPACE}' ..."
|
|
|
|
if [[ "${DRY_RUN}" == "true" ]]; then
|
|
log " [DRY-RUN] Would check: kubectl -n ${CNPG_NAMESPACE} get pods -l cnpg.io/cluster=knoe-db"
|
|
else
|
|
max_attempts=20
|
|
attempt=0
|
|
while [[ ${attempt} -lt ${max_attempts} ]]; do
|
|
running=$(kubectl -n "${CNPG_NAMESPACE}" get pods \
|
|
-l "cnpg.io/cluster=knoe-db" \
|
|
--no-headers 2>/dev/null \
|
|
| grep -c "Running" || true)
|
|
log " Running CNPG pods: ${running} / 3 expected (attempt $((attempt + 1))/${max_attempts})"
|
|
if [[ "${running}" -ge 3 ]]; then
|
|
log " All CNPG pods Running on ${NEW_POOL}."
|
|
kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db"
|
|
break
|
|
fi
|
|
attempt=$(( attempt + 1 ))
|
|
if [[ ${attempt} -lt ${max_attempts} ]]; then
|
|
sleep 30
|
|
fi
|
|
done
|
|
|
|
if [[ "${running:-0}" -lt 3 ]]; then
|
|
log "WARNING: Not all CNPG pods reached Running state within timeout." >&2
|
|
kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db" || true
|
|
log "Check pod events: kubectl -n ${CNPG_NAMESPACE} describe pods -l cnpg.io/cluster=knoe-db"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Done
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo ""
|
|
log "==> Patch complete."
|
|
log ""
|
|
log "Next steps:"
|
|
log " 1. Verify pd-ssd quota freed:"
|
|
log " gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} --format='value(quotas[disksSSD].usage,quotas[disksSSD].limit)'"
|
|
log " 2. Apply updated CNPG manifest (premium-rwo PVCs, 50Gi PGDATA + WAL):"
|
|
log " kubectl --context=<cnpg-context> apply -f deploy/gcp/gke/knoe-db.yaml"
|
|
log " 3. Watch PVC expansion and pod restart:"
|
|
log " kubectl -n ${CNPG_NAMESPACE} get pvc -w"
|