prole/etc/diag_gke_storage.sh

#!/usr/bin/env bash
# diag_gke_storage.sh — inspect StorageClass + SSD quota state on a GKE cluster.
#
# Safe, read-only. Prints:
#   1. Every StorageClass, whether it's default, and its underlying pd type
#   2. All PVs grouped by storage class, with capacity + bound claim
#   3. Totals: how many GiB are pinned to each disk type (pd-ssd / pd-balanced
#      count against SSD_TOTAL_GB; pd-standard is the separate HDD quota)
#   4. Any PVC stuck in Pending with a QUOTA_EXCEEDED / ProvisioningFailed event
#
# Usage: ./etc/diag_gke_storage.sh [--kube-context CTX]

set -euo pipefail

KUBE_CONTEXT=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
    *) echo "unknown arg: $1" >&2; exit 2 ;;
  esac
done

kctl() {
  if [[ -n "$KUBE_CONTEXT" ]]; then
    kubectl --context "$KUBE_CONTEXT" "$@"
  else
    kubectl "$@"
  fi
}

TMPDIR="$(mktemp -d)"
trap 'rm -rf "$TMPDIR"' EXIT

SC_JSON="$TMPDIR/sc.json"
PV_JSON="$TMPDIR/pv.json"
PVC_JSON="$TMPDIR/pvc.json"
PY_SCRIPT="$TMPDIR/diag.py"

kctl get storageclass -o json > "$SC_JSON"
kctl get pv -o json > "$PV_JSON"
kctl get pvc -A -o json > "$PVC_JSON"

# Single python script does all four sections — reads the three JSON files by path.
cat > "$PY_SCRIPT" <<'PY'
import json, re, sys
from collections import defaultdict

sc_path, pv_path, pvc_path = sys.argv[1], sys.argv[2], sys.argv[3]

with open(sc_path) as f:
    scs_raw = json.load(f).get("items") or []
with open(pv_path) as f:
    pvs = json.load(f).get("items") or []
with open(pvc_path) as f:
    pvcs = json.load(f).get("items") or []

# --- Section 1: StorageClasses ------------------------------------------------
print("=== StorageClasses ===")
print(f"{'NAME':<24} {'PROVISIONER':<30} {'DISK_TYPE':<14} {'DEFAULT':<7} {'RECLAIM'}")
sc_by_name = {}
for sc in scs_raw:
    m = sc.get("metadata") or {}
    name = m.get("name", "")
    ann = m.get("annotations") or {}
    is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "")).lower() == "true"
    prov = sc.get("provisioner", "") or ""
    disk = ((sc.get("parameters") or {}).get("type") or "")
    reclaim = sc.get("reclaimPolicy", "") or ""
    sc_by_name[name] = {"disk": disk, "prov": prov, "default": is_default}
    print(f"{name:<24} {prov:<30} {disk:<14} {'YES' if is_default else '-':<7} {reclaim}")

# --- Section 2: PVs grouped by SC --------------------------------------------
print()
print("=== PVs by StorageClass ===")
groups = defaultdict(list)
for pv in pvs:
    spec = pv.get("spec", {}) or {}
    sc = spec.get("storageClassName", "") or "<none>"
    cap = (spec.get("capacity") or {}).get("storage", "")
    phase = (pv.get("status") or {}).get("phase", "")
    cr = spec.get("claimRef") or {}
    claim = f"{cr.get('namespace','')}/{cr.get('name','')}" if cr else ""
    groups[sc].append((pv.get("metadata", {}).get("name", ""), cap, phase, claim))

if not groups:
    print("  (no PVs)")
else:
    for sc, rows in sorted(groups.items()):
        print(f"-- {sc} --")
        for name, cap, phase, claim in rows:
            print(f"  {name:<50} {cap:<8} {phase:<10} {claim}")

# --- Section 3: GiB per disk type --------------------------------------------
def to_gib(cap):
    if not cap:
        return 0.0
    m = re.match(r"^(\d+(?:\.\d+)?)\s*([KMGTP]i?)?$", cap)
    if not m:
        return 0.0
    n = float(m.group(1))
    u = m.group(2) or ""
    unit = {
        "": 1/1024**3, "Ki": 1/1024**2, "Mi": 1/1024, "Gi": 1, "Ti": 1024, "Pi": 1024**2,
        "K": 1e3/1024**3, "M": 1e6/1024**3, "G": 1e9/1024**3, "T": 1e12/1024**3,
    }.get(u, 0)
    return n * unit

print()
print("=== GiB per disk type (what counts against which quota) ===")
totals = defaultdict(float)
for pv in pvs:
    spec = pv.get("spec", {}) or {}
    sc = spec.get("storageClassName", "") or "<none>"
    disk = sc_by_name.get(sc, {}).get("disk", "<unknown-sc>")
    cap = (spec.get("capacity") or {}).get("storage", "")
    totals[disk] += to_gib(cap)

ssd_quota = 0.0
hdd_quota = 0.0
if not totals:
    print("  (no PVs)")
else:
    for disk, gib in sorted(totals.items()):
        if disk in ("pd-balanced", "pd-ssd", "pd-extreme"):
            bucket = "SSD_TOTAL_GB"
            ssd_quota += gib
        elif disk == "pd-standard":
            bucket = "HDD (pd-standard)"
            hdd_quota += gib
        else:
            bucket = "other"
        print(f"  {disk:<16}  {gib:>8.1f} GiB  ({bucket})")
    print()
    print(f"  TOTAL counted against SSD_TOTAL_GB: {ssd_quota:.1f} GiB")
    print(f"  TOTAL on HDD (pd-standard):         {hdd_quota:.1f} GiB")

# --- Section 4: Pending PVCs --------------------------------------------------
print()
print("=== Pending PVCs ===")
pending = []
for pvc in pvcs:
    phase = (pvc.get("status") or {}).get("phase", "")
    if phase != "Pending":
        continue
    m = pvc.get("metadata") or {}
    sc = (pvc.get("spec") or {}).get("storageClassName") or "<none>"
    pending.append((m.get("namespace", ""), m.get("name", ""), sc))

if not pending:
    print("  (none)")
else:
    for ns, name, sc in pending:
        print(f"  {ns}/{name}  (storageClass={sc})")

# Print the namespace/name list on stderr so the shell can loop over it.
import sys as _sys
for ns, name, _sc in pending:
    print(f"{ns}/{name}", file=_sys.stderr)
PY

# Run the python analysis; stderr carries the pending-PVC list for the shell
# loop below.
PENDING_LIST="$(python3 "$PY_SCRIPT" "$SC_JSON" "$PV_JSON" "$PVC_JSON" 2>"$TMPDIR/pending.txt")"
printf '%s\n' "$PENDING_LIST"

echo
echo "=== Provisioning events for any Pending PVCs (tail) ==="
if [[ ! -s "$TMPDIR/pending.txt" ]]; then
  echo "  (no Pending PVCs)"
else
  while IFS='/' read -r ns pvc; do
    [[ -n "$ns" && -n "$pvc" ]] || continue
    echo "-- ${ns}/${pvc} --"
    kctl -n "$ns" describe pvc "$pvc" \
      | awk '/^Events:/{flag=1} flag' \
      | head -n 20
  done < "$TMPDIR/pending.txt"
fi