#!/usr/bin/env bash # diag_gke_storage.sh — inspect StorageClass + SSD quota state on a GKE cluster. # # Safe, read-only. Prints: # 1. Every StorageClass, whether it's default, and its underlying pd type # 2. All PVs grouped by storage class, with capacity + bound claim # 3. Totals: how many GiB are pinned to each disk type (pd-ssd / pd-balanced # count against SSD_TOTAL_GB; pd-standard is the separate HDD quota) # 4. Any PVC stuck in Pending with a QUOTA_EXCEEDED / ProvisioningFailed event # # Usage: ./etc/diag_gke_storage.sh [--kube-context CTX] set -euo pipefail KUBE_CONTEXT="" while [[ $# -gt 0 ]]; do case "$1" in --kube-context) KUBE_CONTEXT="$2"; shift 2 ;; -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "unknown arg: $1" >&2; exit 2 ;; esac done kctl() { if [[ -n "$KUBE_CONTEXT" ]]; then kubectl --context "$KUBE_CONTEXT" "$@" else kubectl "$@" fi } TMPDIR="$(mktemp -d)" trap 'rm -rf "$TMPDIR"' EXIT SC_JSON="$TMPDIR/sc.json" PV_JSON="$TMPDIR/pv.json" PVC_JSON="$TMPDIR/pvc.json" PY_SCRIPT="$TMPDIR/diag.py" kctl get storageclass -o json > "$SC_JSON" kctl get pv -o json > "$PV_JSON" kctl get pvc -A -o json > "$PVC_JSON" # Single python script does all four sections — reads the three JSON files by path. cat > "$PY_SCRIPT" <<'PY' import json, re, sys from collections import defaultdict sc_path, pv_path, pvc_path = sys.argv[1], sys.argv[2], sys.argv[3] with open(sc_path) as f: scs_raw = json.load(f).get("items") or [] with open(pv_path) as f: pvs = json.load(f).get("items") or [] with open(pvc_path) as f: pvcs = json.load(f).get("items") or [] # --- Section 1: StorageClasses ------------------------------------------------ print("=== StorageClasses ===") print(f"{'NAME':<24} {'PROVISIONER':<30} {'DISK_TYPE':<14} {'DEFAULT':<7} {'RECLAIM'}") sc_by_name = {} for sc in scs_raw: m = sc.get("metadata") or {} name = m.get("name", "") ann = m.get("annotations") or {} is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "")).lower() == "true" prov = sc.get("provisioner", "") or "" disk = ((sc.get("parameters") or {}).get("type") or "") reclaim = sc.get("reclaimPolicy", "") or "" sc_by_name[name] = {"disk": disk, "prov": prov, "default": is_default} print(f"{name:<24} {prov:<30} {disk:<14} {'YES' if is_default else '-':<7} {reclaim}") # --- Section 2: PVs grouped by SC -------------------------------------------- print() print("=== PVs by StorageClass ===") groups = defaultdict(list) for pv in pvs: spec = pv.get("spec", {}) or {} sc = spec.get("storageClassName", "") or "" cap = (spec.get("capacity") or {}).get("storage", "") phase = (pv.get("status") or {}).get("phase", "") cr = spec.get("claimRef") or {} claim = f"{cr.get('namespace','')}/{cr.get('name','')}" if cr else "" groups[sc].append((pv.get("metadata", {}).get("name", ""), cap, phase, claim)) if not groups: print(" (no PVs)") else: for sc, rows in sorted(groups.items()): print(f"-- {sc} --") for name, cap, phase, claim in rows: print(f" {name:<50} {cap:<8} {phase:<10} {claim}") # --- Section 3: GiB per disk type -------------------------------------------- def to_gib(cap): if not cap: return 0.0 m = re.match(r"^(\d+(?:\.\d+)?)\s*([KMGTP]i?)?$", cap) if not m: return 0.0 n = float(m.group(1)) u = m.group(2) or "" unit = { "": 1/1024**3, "Ki": 1/1024**2, "Mi": 1/1024, "Gi": 1, "Ti": 1024, "Pi": 1024**2, "K": 1e3/1024**3, "M": 1e6/1024**3, "G": 1e9/1024**3, "T": 1e12/1024**3, }.get(u, 0) return n * unit print() print("=== GiB per disk type (what counts against which quota) ===") totals = defaultdict(float) for pv in pvs: spec = pv.get("spec", {}) or {} sc = spec.get("storageClassName", "") or "" disk = sc_by_name.get(sc, {}).get("disk", "") cap = (spec.get("capacity") or {}).get("storage", "") totals[disk] += to_gib(cap) ssd_quota = 0.0 hdd_quota = 0.0 if not totals: print(" (no PVs)") else: for disk, gib in sorted(totals.items()): if disk in ("pd-balanced", "pd-ssd", "pd-extreme"): bucket = "SSD_TOTAL_GB" ssd_quota += gib elif disk == "pd-standard": bucket = "HDD (pd-standard)" hdd_quota += gib else: bucket = "other" print(f" {disk:<16} {gib:>8.1f} GiB ({bucket})") print() print(f" TOTAL counted against SSD_TOTAL_GB: {ssd_quota:.1f} GiB") print(f" TOTAL on HDD (pd-standard): {hdd_quota:.1f} GiB") # --- Section 4: Pending PVCs -------------------------------------------------- print() print("=== Pending PVCs ===") pending = [] for pvc in pvcs: phase = (pvc.get("status") or {}).get("phase", "") if phase != "Pending": continue m = pvc.get("metadata") or {} sc = (pvc.get("spec") or {}).get("storageClassName") or "" pending.append((m.get("namespace", ""), m.get("name", ""), sc)) if not pending: print(" (none)") else: for ns, name, sc in pending: print(f" {ns}/{name} (storageClass={sc})") # Print the namespace/name list on stderr so the shell can loop over it. import sys as _sys for ns, name, _sc in pending: print(f"{ns}/{name}", file=_sys.stderr) PY # Run the python analysis; stderr carries the pending-PVC list for the shell # loop below. PENDING_LIST="$(python3 "$PY_SCRIPT" "$SC_JSON" "$PV_JSON" "$PVC_JSON" 2>"$TMPDIR/pending.txt")" printf '%s\n' "$PENDING_LIST" echo echo "=== Provisioning events for any Pending PVCs (tail) ===" if [[ ! -s "$TMPDIR/pending.txt" ]]; then echo " (no Pending PVCs)" else while IFS='/' read -r ns pvc; do [[ -n "$ns" && -n "$pvc" ]] || continue echo "-- ${ns}/${pvc} --" kctl -n "$ns" describe pvc "$pvc" \ | awk '/^Events:/{flag=1} flag' \ | head -n 20 done < "$TMPDIR/pending.txt" fi