#!/usr/bin/env bash # ensure_default_storage_class.sh # # Preflight / remediator that asserts the cluster-wide default StorageClass is # pd-standard (HDD), not pd-balanced / pd-ssd. GKE ships `standard-rwo` as the # default, which provisions `pd-balanced` under the hood and draws from the # SSD_TOTAL_GB quota. That has repeatedly wedged provisioning on low-SSD-quota # projects (see init_cnpg_gke.sh step-down logic, which compensates on the # consumer side). # # This script fixes the root cause cluster-side: it un-defaults any built-in # SSD-backed class and promotes `standard-hdd` (pd-standard) as the default. # Workloads that actually need SSD latency (CNPG) opt in explicitly via # `storageClassName: premium-rwo` in their PVC template — they are unaffected # by the default change. # # Usage # ----- # ./etc/ensure_default_storage_class.sh [--check | --apply] [--kube-context CTX] # # --check (default) Report current state; exit 0 if compliant, 1 if drift # detected. No cluster mutations. # --apply Remediate: create standard-hdd if absent, clear the # default annotation from any SSD-backed class, set standard-hdd # as the default. Idempotent. # --kube-context CTX Use the named kube-context (default: current). # # Exit codes # ---------- # 0 Compliant (or successfully remediated) # 1 Drift detected in --check mode # 2 Remediation failed (in --apply mode) # 3 Cluster not a GKE cluster (no pd.csi.storage.gke.io provisioner seen) # 4 Invalid arguments / missing tooling set -euo pipefail SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")" DEFAULT_SC_NAME="${PROLE_DEFAULT_SC_NAME:-standard-hdd}" DEFAULT_SC_DISK_TYPE="pd-standard" GKE_CSI_PROVISIONER="pd.csi.storage.gke.io" MODE="check" KUBE_CONTEXT="" log() { printf '[%s] %s\n' "$SCRIPT_NAME" "$*"; } warn() { printf '[%s] WARN: %s\n' "$SCRIPT_NAME" "$*" >&2; } die() { printf '[%s] ERROR: %s\n' "$SCRIPT_NAME" "$*" >&2; exit "${2:-4}"; } while [[ $# -gt 0 ]]; do case "$1" in --check) MODE="check"; shift ;; --apply) MODE="apply"; shift ;; --kube-context) [[ -n "${2:-}" ]] || die "--kube-context requires an argument" 4 KUBE_CONTEXT="$2"; shift 2 ;; -h|--help) sed -n '1,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' exit 0 ;; *) die "unknown argument: $1" 4 ;; esac done command -v kubectl >/dev/null || die "kubectl not found in PATH" 4 command -v python3 >/dev/null || die "python3 not found in PATH" 4 kctl() { if [[ -n "$KUBE_CONTEXT" ]]; then kubectl --context "$KUBE_CONTEXT" "$@" else kubectl "$@" fi } # ── 1. Confirm we're talking to a GKE cluster ──────────────────────────────── sc_json="$(kctl get storageclass -o json 2>/dev/null || true)" if [[ -z "$sc_json" ]] || ! printf '%s' "$sc_json" | python3 -c ' import json, sys data = json.load(sys.stdin) items = data.get("items") or [] sys.exit(0 if any(it.get("provisioner") == "pd.csi.storage.gke.io" for it in items) else 1) '; then die "no ${GKE_CSI_PROVISIONER} StorageClasses found; this does not look like a GKE cluster" 3 fi # ── 2. Inspect current state ───────────────────────────────────────────────── # Emits: ||| for every StorageClass. sc_summary="$(printf '%s' "$sc_json" | python3 - <<'PY' import json, sys data = json.load(sys.stdin) for it in data.get("items") or []: meta = it.get("metadata") or {} name = meta.get("name", "") ann = (meta.get("annotations") or {}) is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "") or "").lower() == "true" prov = it.get("provisioner", "") or "" disk_type = ((it.get("parameters") or {}).get("type", "")) or "" print(f"{name}|{prov}|{disk_type}|{'true' if is_default else 'false'}") PY )" log "Current StorageClasses:" while IFS='|' read -r name prov disk_type is_default; do [[ -n "$name" ]] || continue if [[ "$is_default" == "true" ]]; then log " * ${name} prov=${prov} type=${disk_type} (default)" else log " ${name} prov=${prov} type=${disk_type}" fi done <<< "$sc_summary" # Identify the current default + its disk type. current_default="" current_default_type="" ssd_backed_defaults=() while IFS='|' read -r name prov disk_type is_default; do [[ -n "$name" ]] || continue if [[ "$is_default" == "true" ]]; then current_default="$name" current_default_type="$disk_type" if [[ "$disk_type" == "pd-balanced" || "$disk_type" == "pd-ssd" ]]; then ssd_backed_defaults+=("$name") fi fi done <<< "$sc_summary" # Does standard-hdd (or PROLE_DEFAULT_SC_NAME override) exist with the right shape? target_exists="false" target_disk_type="" target_is_default="false" while IFS='|' read -r name prov disk_type is_default; do if [[ "$name" == "$DEFAULT_SC_NAME" ]]; then target_exists="true" target_disk_type="$disk_type" target_is_default="$is_default" fi done <<< "$sc_summary" compliant="true" issues=() if [[ "$target_exists" != "true" ]]; then compliant="false" issues+=("target StorageClass '${DEFAULT_SC_NAME}' does not exist") elif [[ "$target_disk_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then compliant="false" issues+=("'${DEFAULT_SC_NAME}' has type='${target_disk_type}', want '${DEFAULT_SC_DISK_TYPE}'") fi if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then compliant="false" issues+=("SSD-backed class(es) still marked default: ${ssd_backed_defaults[*]}") fi if [[ "$target_exists" == "true" && "$target_is_default" != "true" ]]; then compliant="false" issues+=("'${DEFAULT_SC_NAME}' is not annotated is-default-class=true") fi if [[ "$compliant" == "true" ]]; then log "OK — default StorageClass is '${current_default}' (type=${current_default_type}). No action needed." exit 0 fi log "Drift detected:" for issue in "${issues[@]}"; do log " - ${issue}" done if [[ "$MODE" == "check" ]]; then log "Run with --apply to remediate." exit 1 fi # ── 3. Remediate ───────────────────────────────────────────────────────────── log "Applying remediation ..." # 3a. Create/update standard-hdd — but NOT yet marked default so we never have # a window with two defaults racing. cat </dev/null || true)" if [[ "$live_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then warn "live '${DEFAULT_SC_NAME}' has type='${live_type}' (want '${DEFAULT_SC_DISK_TYPE}'). Recreating ..." kctl delete storageclass "$DEFAULT_SC_NAME" --wait=true >/dev/null 2>&1 || die "failed to delete '${DEFAULT_SC_NAME}' for recreation" 2 cat </dev/null || true)" [[ "$live_type" == "$DEFAULT_SC_DISK_TYPE" ]] || die "after recreate, '${DEFAULT_SC_NAME}' still has type='${live_type}'" 2 fi # 3b. Clear the default annotation from any SSD-backed class that currently # holds it. Patch to "false" (not remove) so we leave a clear audit trail. if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then for sc in "${ssd_backed_defaults[@]}"; do log "Un-defaulting SSD-backed class: ${sc}" kctl annotate storageclass "$sc" \ "storageclass.kubernetes.io/is-default-class=false" --overwrite >/dev/null \ || die "failed to un-default '${sc}'" 2 done fi # 3c. Promote standard-hdd to default — atomic with respect to other defaults # because we cleared them in 3b first. kctl annotate storageclass "$DEFAULT_SC_NAME" \ "storageclass.kubernetes.io/is-default-class=true" --overwrite >/dev/null \ || die "failed to mark '${DEFAULT_SC_NAME}' as default" 2 # ── 4. Re-verify ───────────────────────────────────────────────────────────── final_default="$(kctl get storageclass \ -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \ 2>/dev/null | head -n1)" if [[ "$final_default" != "$DEFAULT_SC_NAME" ]]; then die "post-remediation: default is '${final_default:-}', expected '${DEFAULT_SC_NAME}'" 2 fi # Count defaults — having two is a worse state than having zero. default_count="$(kctl get storageclass \ -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \ 2>/dev/null | grep -cv '^$' || true)" if [[ "$default_count" != "1" ]]; then die "post-remediation: ${default_count} default StorageClasses exist (want exactly 1)" 2 fi log "Remediation OK. Default StorageClass is now '${DEFAULT_SC_NAME}' (${DEFAULT_SC_DISK_TYPE})." log "Re-run 'kubectl get storageclass' to confirm." exit 0