2026-09-23 10:13:58 +00:00 · 2026-04-20 18:51:41 -07:00 · 2026-04-20 18:51:41 -07:00 · e44ff8e476
commit e44ff8e476
parent 84105e1b0b
9 changed files with 864 additions and 24 deletions
--- a/conf/gke.cfg
+++ b/conf/gke.cfg
@ -1,5 +1,5 @@
 ; Prole Master Configuration File
-; Generated by install.py on 2026-04-18 20:45:32
+; Generated by install.py on 2026-04-20 14:42:17
 ; This file is used as input for Ansible deployment and k8s cluster creation.

 [User]
@ -8,8 +8,8 @@

 [Inputs]
 ; Screen-scoped inputs used for unattended replays (-S)
-auth.clientId = secretref://google-oidc-client-id
-auth.clientSecret = secretref://google-oidc-client-secret
+auth.clientId = 200699021716-pbaekj5q8i3fplhof0u7gccrvkvvacug.apps.googleusercontent.com
+auth.clientSecret = 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
 build.deploy_env = Prod
 build.run_build = false
 dependencies.ansible.install = true
@ -79,8 +79,8 @@ init_password.cluster_name = ${CLUSTER_NAME}
 init_password.db_cluster_name = knoe-dev-cnpg-0
 init_password.db_host_port = 5432
 init_password.db_namespace = ${DATABASE_NAMESPACE}
-init_password.db_password = ${PROLE_SECRET:v1:5RlyU1vwz3CDoq6u:UfdWQw93uyPDIY_TWxRzdIbyZQCGv4sBpjNFmQMDRdmrypdVxcwz0w==}
-init_password.db_password_confirm = ${PROLE_SECRET:v1:5RlyU1vwz3CDoq6u:UfdWQw93uyPDIY_TWxRzdIbyZQCGv4sBpjNFmQMDRdmrypdVxcwz0w==}
+init_password.db_password = ${PROLE_SECRET:v1:t152p6lozmV1olHq:cJCCJAqk8iusivgp9akabPgYgi5RZYqVI9grBY4pRzaWcuQviqizZg==}
+init_password.db_password_confirm = ${PROLE_SECRET:v1:t152p6lozmV1olHq:cJCCJAqk8iusivgp9akabPgYgi5RZYqVI9grBY4pRzaWcuQviqizZg==}
 init_password.db_username = chrisfu
 init_password.generate_ssh_key = true
 init_scripts.run_scripts = true
@ -114,7 +114,7 @@ CNPG_SIZE_PROFILE = small
 DATABASE_NAMESPACE = knoe-db-0
 DB_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-cnpg-0
 DB_HOST_PORT = 5432
-DB_PASSWORD = ${PROLE_SECRET:v1:v42vq25mZFVFIwuZ:YK1Bz8WwwhkEQk712JAa2aCWqokdAyVG_Z1bP_Rj5RxVCoU6_lDYYA==}
+DB_PASSWORD = ${PROLE_SECRET:v1:AX1bdMcT41sPTZbS:wvvf8AVN-hvKOcJWg1EIBtly78nrTL7rBrjfLXboqqMUD_qw-lSoSQ==}
 DEPLOYMENT_MODE = k8s
 DEPLOYMENT_TARGET = prole-prod-cluster
 DOCKER_IMPORT_DIR = 
@ -123,22 +123,16 @@ GARAGE_PRIVATE_S3_ENDPOINT = http://10.180.15.239:3900
 GITLAB_DOMAIN = git.knoe.dev
 GITLAB_GITALY_STORAGE_CLASS = standard
 GITLAB_INGRESS_CLASS = gce
-; OIDC client credentials are indirected through secretref://; the loader
-; (knoe/core/actions.py:_resolve_secretref_value) reads them from env or from
-; etc/secrets/<ref> (the etc/secrets/ directory is gitignored). This keeps
-; plaintext client_secret out of the repo so partial public forks stay clean.
 GITLAB_OIDC_CLIENT_ID = secretref://google-oidc-client-id
 GITLAB_OIDC_CLIENT_SECRET = secretref://google-oidc-client-secret
-; Path A: Google Workspace as OIDC OP directly (bypassing knoe-auth, which
-; doesn't yet implement an OIDC provider surface). See docs/oidc-setup.md.
 GITLAB_OIDC_ISSUER = https://accounts.google.com
 GITLAB_PUBLIC_HOSTS = git.knoe.dev
 GITLAB_REPAIR_BLOCKED_AUTOCLEAN = 1
-GITLAB_WEBSERVICE_LIMITS_MEMORY = 1800M
+GITLAB_WEBSERVICE_LIMITS_MEMORY = 3Gi
 GITLAB_WEBSERVICE_PUMA_THREADS_MAX = 2
 GITLAB_WEBSERVICE_PUMA_THREADS_MIN = 2
-GITLAB_WEBSERVICE_REQUESTS_CPU = 200m
-GITLAB_WEBSERVICE_REQUESTS_MEMORY = 900M
+GITLAB_WEBSERVICE_REQUESTS_CPU = 500m
+GITLAB_WEBSERVICE_REQUESTS_MEMORY = 2Gi
 GITLAB_WEBSERVICE_WORKER_PROCESSES = 1
 GRAFANA_HOSTNAME = svc.knoe.dev
 K3S_SERVER = 
@ -159,11 +153,11 @@ REDIS_HOST = redis-master.knoe-system.svc.cluster.local
 REGISTRY_NAMESPACE = knoe-system
 SERVICE_NAMESPACE = knoe-system
 SUPABASE_API_HOSTNAME = api.0.knoe.dev
+SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS = true
+SUPABASE_AUTO_CLEAN_RETAINED_PVS = true
 SUPABASE_HOSTNAME = db.0.knoe.dev
 SUPABASE_INGRESS_CLASS = gce
 SUPABASE_STUDIO_HOSTNAME = db.0.knoe.dev
-SUPABASE_AUTO_CLEAN_RETAINED_PVS = true
-SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS = true

 [Welcome]
 ; No configuration values captured yet for this section.
--- a/conf/port-mapping.cfg
+++ b/conf/port-mapping.cfg
@ -2,12 +2,17 @@
 # Format: key: local=... remote=... ns=... svc=... address=...

 argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0
+supabase-studio: local=18080 remote=3000 ns=supabase svc=studio address=0.0.0.0
+supabase-auth: local=9999 remote=9999 ns=supabase svc=auth address=127.0.0.1
+supabase-rest: local=3001 remote=3000 ns=supabase svc=rest address=0.0.0.0
+supabase-realtime: local=4000 remote=4000 ns=supabase svc=realtime address=0.0.0.0
+gitea-http: local=13000 remote=3000 ns=gitea svc=gitea-http address=0.0.0.0
 gitea-ssh: local=22 remote=22 ns=gitea svc=gitea-ssh address=0.0.0.0
 garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0
 openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0
 opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0
 dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1
-postgres: local=5432 remote=5432 ns=knoe-db svc=knoe-db-rw address=0.0.0.0
+postgres: local=5432 remote=5432 ns=knoe-db-0 svc=knoe-db-rw address=0.0.0.0
 prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1
 grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0
-gitea-http: local=13000 remote=3000 ns=gitea svc=gitea-http address=0.0.0.0
+supabase-kong: local=8000 remote=8000 ns=supabase svc=kong address=0.0.0.0
--- a/deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml
+++ b/deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml
@ -20,6 +20,12 @@ stringData:
        - profile
        - email
      response_type: code
+      # discovery: true triggers a one-time fetch of
+      # ${issuer}/.well-known/openid-configuration at webservice boot, which
+      # populates the OpenIDConnect client's host/port/scheme/endpoints.
+      # Without it omniauth-openid_connect fails every callback with
+      # "Could not authenticate you from OpenIDConnect because 'No host info'".
+      discovery: true
      issuer: "${GITLAB_OIDC_ISSUER}"
      client_auth_method: query
      uid_field: email
--- a/etc/diag_gitlab_boot.sh
+++ b/etc/diag_gitlab_boot.sh
@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# diag_gitlab_boot.sh — read-only snapshot of GitLab backend health.
+#
+# Investigates why the frontdoor returns "Waiting for GitLab to boot" even
+# after DNS + ingress are green. Focuses on:
+#   - webservice pod states + last-terminated reasons (OOM, probe failures)
+#   - webservice + workhorse log tail on the likely-serving pod
+#   - Gitaly readiness
+#   - recent Warning events
+#
+# Usage: ./etc/diag_gitlab_boot.sh [--namespace NS] [--kube-context CTX]
+#   --namespace defaults to 'gitlab'
+
+set -euo pipefail
+
+NS="gitlab"
+KUBE_CONTEXT=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --namespace) NS="$2"; shift 2 ;;
+    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@"
+  else
+    kubectl -n "$NS" "$@"
+  fi
+}
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+WEB_JSON="$TMPDIR/web.json"
+kctl get pods -l app=webservice -o json > "$WEB_JSON"
+
+echo "=== Namespace: $NS ==="
+echo
+echo "=== Webservice pods (restarts + last termination) ==="
+
+PY_SCRIPT="$TMPDIR/analyze.py"
+cat > "$PY_SCRIPT" <<'PY'
+import json, sys
+
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+
+pods = data.get("items") or []
+if not pods:
+    print("  (no webservice pods found)")
+
+for pod in pods:
+    name = (pod.get("metadata") or {}).get("name", "")
+    phase = (pod.get("status") or {}).get("phase", "")
+    print(f"\n-- {name}  phase={phase} --")
+    for cs in (pod.get("status") or {}).get("containerStatuses") or []:
+        cname = cs.get("name", "")
+        ready = cs.get("ready")
+        restart = cs.get("restartCount", 0)
+        state = cs.get("state") or {}
+        last = cs.get("lastState") or {}
+        state_key = next(iter(state.keys()), "unknown")
+        state_detail = state.get(state_key) or {}
+        last_term = (last.get("terminated") or {}) if last else {}
+        print(f"  container={cname:<22} ready={ready} restarts={restart} state={state_key}")
+        if state_detail.get("reason"):
+            print(f"    state.reason={state_detail.get('reason')} msg={(state_detail.get('message') or '')[:180]}")
+        if last_term:
+            print(f"    last terminated: exit={last_term.get('exitCode')} reason={last_term.get('reason')} "
+                  f"signal={last_term.get('signal')}")
+            print(f"    last start={last_term.get('startedAt')} finish={last_term.get('finishedAt')}")
+            if last_term.get("message"):
+                print(f"    last msg: {(last_term.get('message') or '')[:240]}")
+
+# Emit the best-candidate pod name to a sidecar file so the shell can pick it up
+def score(p):
+    cs = (p.get("status") or {}).get("containerStatuses") or []
+    ready = all(c.get("ready") for c in cs) if cs else False
+    restarts = max((c.get("restartCount", 0) for c in cs), default=0)
+    return (1 if ready else 0, -restarts)
+
+pods.sort(key=score, reverse=True)
+target = pods[0]["metadata"]["name"] if pods else ""
+with open(sys.argv[2], "w") as f:
+    f.write(target)
+PY
+
+TARGET_FILE="$TMPDIR/target.txt"
+python3 "$PY_SCRIPT" "$WEB_JSON" "$TARGET_FILE"
+POD="$(cat "$TARGET_FILE")"
+
+echo
+echo "=== Most-likely serving webservice pod: tail 60 lines (webservice container) ==="
+if [[ -n "$POD" ]]; then
+  echo "Pod: $POD"
+  kctl logs "$POD" -c webservice --tail=60 2>&1 | sed 's/^/  /' || true
+  echo
+  echo "=== ... gitlab-workhorse container (tail 30) ==="
+  kctl logs "$POD" -c gitlab-workhorse --tail=30 2>&1 | sed 's/^/  /' || true
+else
+  echo "  (no webservice pod found)"
+fi
+
+echo
+echo "=== Gitaly pods ==="
+kctl get pods -l app=gitaly -o wide || true
+
+echo
+echo "=== Recent Warning events (last 25) ==="
+kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 25 || true
+
+echo
+echo "=== GitLab CR status (if present) ==="
+GL_JSON="$TMPDIR/gl.json"
+if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then
+  python3 - "$GL_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for it in data.get("items") or []:
+    m = it.get("metadata") or {}
+    s = it.get("status") or {}
+    print(f"  {m.get('name')}: phase={s.get('phase','?')}")
+    for c in (s.get("conditions") or [])[:10]:
+        print(f"    {c.get('type')}={c.get('status')} reason={c.get('reason')} "
+              f"msg={(c.get('message') or '')[:140]}")
+PY
+else
+  echo "  (no GitLab CR or CRD not installed)"
+fi
--- a/etc/diag_gitlab_webservice_oom.sh
+++ b/etc/diag_gitlab_webservice_oom.sh
@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# diag_gitlab_webservice_oom.sh — memory ceilings + Puma tuning for webservice.
+#
+# Read-only. Prints:
+#   1. webservice container resources (requests/limits)
+#   2. Puma worker/thread env vars from pod spec
+#   3. node allocatable memory so we know upper bound
+#   4. prometheus-style /metrics snapshot from the running pod if available
+#
+# Usage: ./etc/diag_gitlab_webservice_oom.sh [--namespace NS] [--kube-context CTX]
+
+set -euo pipefail
+
+NS="gitlab"
+KUBE_CONTEXT=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --namespace) NS="$2"; shift 2 ;;
+    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@"
+  else
+    kubectl -n "$NS" "$@"
+  fi
+}
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+DEPLOY_JSON="$TMPDIR/deploy.json"
+POD_JSON="$TMPDIR/pod.json"
+NODE_JSON="$TMPDIR/node.json"
+
+kctl get deployment -l app=webservice -o json > "$DEPLOY_JSON"
+kctl get pods -l app=webservice -o json > "$POD_JSON"
+
+echo "=== webservice container resources (from deployment) ==="
+python3 - "$DEPLOY_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for d in data.get("items") or []:
+    name = (d.get("metadata") or {}).get("name", "")
+    print(f"\n-- deployment/{name} --")
+    spec = ((d.get("spec") or {}).get("template") or {}).get("spec") or {}
+    for c in spec.get("containers") or []:
+        cname = c.get("name", "")
+        res = c.get("resources") or {}
+        req = res.get("requests") or {}
+        lim = res.get("limits") or {}
+        print(f"  container={cname}")
+        print(f"    requests: cpu={req.get('cpu','?')}  mem={req.get('memory','?')}")
+        print(f"    limits:   cpu={lim.get('cpu','?')}  mem={lim.get('memory','?')}")
+        # Pick out Puma-relevant env vars
+        env = c.get("env") or []
+        interesting = {
+            "PUMA_WORKERS", "PUMA_THREADS_MIN", "PUMA_THREADS_MAX",
+            "WORKER_PROCESSES", "SIDEKIQ_CONCURRENCY",
+            "GITLAB_MEMORY_WATCHDOG_ENABLED", "GITLAB_MEMORY_WATCHDOG_PUMA_ENABLED",
+            "GITLAB_MEMORY_WATCHDOG_MAX_HEAP_FRAG_THRESHOLD_MB",
+            "GITLAB_MEMORY_WATCHDOG_MAX_STRIKES",
+            "MALLOC_ARENA_MAX",
+        }
+        seen = [e for e in env if e.get("name") in interesting]
+        if seen:
+            print("    puma/watchdog env:")
+            for e in seen:
+                val = e.get("value", "<valueFrom>") if "value" in e else "<valueFrom>"
+                print(f"      {e.get('name')}={val}")
+PY
+
+echo
+echo "=== Node memory allocatable ==="
+kctl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory,MEM_CAPACITY:.status.capacity.memory 2>/dev/null \
+  || kubectl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory
+
+echo
+echo "=== Running webservice pod memory usage (kubectl top if metrics-server present) ==="
+kctl top pod -l app=webservice --containers 2>&1 || echo "  (metrics-server not installed or not returning data)"
+
+echo
+echo "=== Pod QoS class + restart history ==="
+python3 - "$POD_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for p in data.get("items") or []:
+    m = p.get("metadata") or {}
+    s = p.get("status") or {}
+    cs = s.get("containerStatuses") or []
+    ws = next((c for c in cs if c.get("name") == "webservice"), None)
+    print(f"  {m.get('name'):<48}  qos={s.get('qosClass','?'):<10} "
+          f"phase={s.get('phase','?'):<10} restarts={ws.get('restartCount', '?') if ws else '?'}")
+PY
+
+echo
+echo "=== Last 40 Warning events (whole namespace, for noise diagnosis) ==="
+kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 40 || true
+
+echo
+echo "=== All PVCs in namespace (what the operator might be trying to reconcile) ==="
+kctl get pvc -o wide || true
+
+echo
+echo "=== GitLab operator CR spec.gitaly (replicas + storage) ==="
+GL_JSON="$TMPDIR/gl.json"
+if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then
+  python3 - "$GL_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for g in data.get("items") or []:
+    m = g.get("metadata") or {}
+    name = m.get("name", "")
+    chart = ((g.get("spec") or {}).get("chart") or {}).get("values") or {}
+    gitaly = ((chart.get("global") or {}).get("gitaly") or {})
+    wsvc = (chart.get("gitlab") or {}).get("webservice") or {}
+    print(f"  gitlab/{name}")
+    print(f"    global.gitaly: {json.dumps(gitaly, indent=6)[:800]}")
+    print(f"    gitlab.webservice.replicas: {wsvc.get('replicaCount', wsvc.get('minReplicas','?'))}")
+    print(f"    gitlab.webservice.resources: {json.dumps(wsvc.get('resources', {}), indent=6)}")
+    print(f"    gitlab.webservice.workerProcesses: {wsvc.get('workerProcesses','?')}")
+PY
+else
+  echo "  (no GitLab CR or not readable)"
+fi
--- a/etc/diag_gke_storage.sh
+++ b/etc/diag_gke_storage.sh
@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+# diag_gke_storage.sh — inspect StorageClass + SSD quota state on a GKE cluster.
+#
+# Safe, read-only. Prints:
+#   1. Every StorageClass, whether it's default, and its underlying pd type
+#   2. All PVs grouped by storage class, with capacity + bound claim
+#   3. Totals: how many GiB are pinned to each disk type (pd-ssd / pd-balanced
+#      count against SSD_TOTAL_GB; pd-standard is the separate HDD quota)
+#   4. Any PVC stuck in Pending with a QUOTA_EXCEEDED / ProvisioningFailed event
+#
+# Usage: ./etc/diag_gke_storage.sh [--kube-context CTX]
+
+set -euo pipefail
+
+KUBE_CONTEXT=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" "$@"
+  else
+    kubectl "$@"
+  fi
+}
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+SC_JSON="$TMPDIR/sc.json"
+PV_JSON="$TMPDIR/pv.json"
+PVC_JSON="$TMPDIR/pvc.json"
+PY_SCRIPT="$TMPDIR/diag.py"
+
+kctl get storageclass -o json > "$SC_JSON"
+kctl get pv -o json > "$PV_JSON"
+kctl get pvc -A -o json > "$PVC_JSON"
+
+# Single python script does all four sections — reads the three JSON files by path.
+cat > "$PY_SCRIPT" <<'PY'
+import json, re, sys
+from collections import defaultdict
+
+sc_path, pv_path, pvc_path = sys.argv[1], sys.argv[2], sys.argv[3]
+
+with open(sc_path) as f:
+    scs_raw = json.load(f).get("items") or []
+with open(pv_path) as f:
+    pvs = json.load(f).get("items") or []
+with open(pvc_path) as f:
+    pvcs = json.load(f).get("items") or []
+
+# --- Section 1: StorageClasses ------------------------------------------------
+print("=== StorageClasses ===")
+print(f"{'NAME':<24} {'PROVISIONER':<30} {'DISK_TYPE':<14} {'DEFAULT':<7} {'RECLAIM'}")
+sc_by_name = {}
+for sc in scs_raw:
+    m = sc.get("metadata") or {}
+    name = m.get("name", "")
+    ann = m.get("annotations") or {}
+    is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "")).lower() == "true"
+    prov = sc.get("provisioner", "") or ""
+    disk = ((sc.get("parameters") or {}).get("type") or "")
+    reclaim = sc.get("reclaimPolicy", "") or ""
+    sc_by_name[name] = {"disk": disk, "prov": prov, "default": is_default}
+    print(f"{name:<24} {prov:<30} {disk:<14} {'YES' if is_default else '-':<7} {reclaim}")
+
+# --- Section 2: PVs grouped by SC --------------------------------------------
+print()
+print("=== PVs by StorageClass ===")
+groups = defaultdict(list)
+for pv in pvs:
+    spec = pv.get("spec", {}) or {}
+    sc = spec.get("storageClassName", "") or "<none>"
+    cap = (spec.get("capacity") or {}).get("storage", "")
+    phase = (pv.get("status") or {}).get("phase", "")
+    cr = spec.get("claimRef") or {}
+    claim = f"{cr.get('namespace','')}/{cr.get('name','')}" if cr else ""
+    groups[sc].append((pv.get("metadata", {}).get("name", ""), cap, phase, claim))
+
+if not groups:
+    print("  (no PVs)")
+else:
+    for sc, rows in sorted(groups.items()):
+        print(f"-- {sc} --")
+        for name, cap, phase, claim in rows:
+            print(f"  {name:<50} {cap:<8} {phase:<10} {claim}")
+
+# --- Section 3: GiB per disk type --------------------------------------------
+def to_gib(cap):
+    if not cap:
+        return 0.0
+    m = re.match(r"^(\d+(?:\.\d+)?)\s*([KMGTP]i?)?$", cap)
+    if not m:
+        return 0.0
+    n = float(m.group(1))
+    u = m.group(2) or ""
+    unit = {
+        "": 1/1024**3, "Ki": 1/1024**2, "Mi": 1/1024, "Gi": 1, "Ti": 1024, "Pi": 1024**2,
+        "K": 1e3/1024**3, "M": 1e6/1024**3, "G": 1e9/1024**3, "T": 1e12/1024**3,
+    }.get(u, 0)
+    return n * unit
+
+print()
+print("=== GiB per disk type (what counts against which quota) ===")
+totals = defaultdict(float)
+for pv in pvs:
+    spec = pv.get("spec", {}) or {}
+    sc = spec.get("storageClassName", "") or "<none>"
+    disk = sc_by_name.get(sc, {}).get("disk", "<unknown-sc>")
+    cap = (spec.get("capacity") or {}).get("storage", "")
+    totals[disk] += to_gib(cap)
+
+ssd_quota = 0.0
+hdd_quota = 0.0
+if not totals:
+    print("  (no PVs)")
+else:
+    for disk, gib in sorted(totals.items()):
+        if disk in ("pd-balanced", "pd-ssd", "pd-extreme"):
+            bucket = "SSD_TOTAL_GB"
+            ssd_quota += gib
+        elif disk == "pd-standard":
+            bucket = "HDD (pd-standard)"
+            hdd_quota += gib
+        else:
+            bucket = "other"
+        print(f"  {disk:<16}  {gib:>8.1f} GiB  ({bucket})")
+    print()
+    print(f"  TOTAL counted against SSD_TOTAL_GB: {ssd_quota:.1f} GiB")
+    print(f"  TOTAL on HDD (pd-standard):         {hdd_quota:.1f} GiB")
+
+# --- Section 4: Pending PVCs --------------------------------------------------
+print()
+print("=== Pending PVCs ===")
+pending = []
+for pvc in pvcs:
+    phase = (pvc.get("status") or {}).get("phase", "")
+    if phase != "Pending":
+        continue
+    m = pvc.get("metadata") or {}
+    sc = (pvc.get("spec") or {}).get("storageClassName") or "<none>"
+    pending.append((m.get("namespace", ""), m.get("name", ""), sc))
+
+if not pending:
+    print("  (none)")
+else:
+    for ns, name, sc in pending:
+        print(f"  {ns}/{name}  (storageClass={sc})")
+
+# Print the namespace/name list on stderr so the shell can loop over it.
+import sys as _sys
+for ns, name, _sc in pending:
+    print(f"{ns}/{name}", file=_sys.stderr)
+PY
+
+# Run the python analysis; stderr carries the pending-PVC list for the shell
+# loop below.
+PENDING_LIST="$(python3 "$PY_SCRIPT" "$SC_JSON" "$PV_JSON" "$PVC_JSON" 2>"$TMPDIR/pending.txt")"
+printf '%s\n' "$PENDING_LIST"
+
+echo
+echo "=== Provisioning events for any Pending PVCs (tail) ==="
+if [[ ! -s "$TMPDIR/pending.txt" ]]; then
+  echo "  (no Pending PVCs)"
+else
+  while IFS='/' read -r ns pvc; do
+    [[ -n "$ns" && -n "$pvc" ]] || continue
+    echo "-- ${ns}/${pvc} --"
+    kctl -n "$ns" describe pvc "$pvc" \
+      | awk '/^Events:/{flag=1} flag' \
+      | head -n 20
+  done < "$TMPDIR/pending.txt"
+fi
--- a/etc/ensure_default_storage_class.sh
+++ b/etc/ensure_default_storage_class.sh
@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+# ensure_default_storage_class.sh
+#
+# Preflight / remediator that asserts the cluster-wide default StorageClass is
+# pd-standard (HDD), not pd-balanced / pd-ssd. GKE ships `standard-rwo` as the
+# default, which provisions `pd-balanced` under the hood and draws from the
+# SSD_TOTAL_GB quota. That has repeatedly wedged provisioning on low-SSD-quota
+# projects (see init_cnpg_gke.sh step-down logic, which compensates on the
+# consumer side).
+#
+# This script fixes the root cause cluster-side: it un-defaults any built-in
+# SSD-backed class and promotes `standard-hdd` (pd-standard) as the default.
+# Workloads that actually need SSD latency (CNPG) opt in explicitly via
+# `storageClassName: premium-rwo` in their PVC template — they are unaffected
+# by the default change.
+#
+# Usage
+# -----
+#   ./etc/ensure_default_storage_class.sh [--check | --apply] [--kube-context CTX]
+#
+#   --check   (default) Report current state; exit 0 if compliant, 1 if drift
+#             detected. No cluster mutations.
+#   --apply             Remediate: create standard-hdd if absent, clear the
+#             default annotation from any SSD-backed class, set standard-hdd
+#             as the default. Idempotent.
+#   --kube-context CTX  Use the named kube-context (default: current).
+#
+# Exit codes
+# ----------
+#   0  Compliant (or successfully remediated)
+#   1  Drift detected in --check mode
+#   2  Remediation failed (in --apply mode)
+#   3  Cluster not a GKE cluster (no pd.csi.storage.gke.io provisioner seen)
+#   4  Invalid arguments / missing tooling
+
+set -euo pipefail
+
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+DEFAULT_SC_NAME="${PROLE_DEFAULT_SC_NAME:-standard-hdd}"
+DEFAULT_SC_DISK_TYPE="pd-standard"
+GKE_CSI_PROVISIONER="pd.csi.storage.gke.io"
+MODE="check"
+KUBE_CONTEXT=""
+
+log() { printf '[%s] %s\n' "$SCRIPT_NAME" "$*"; }
+warn() { printf '[%s] WARN: %s\n' "$SCRIPT_NAME" "$*" >&2; }
+die() { printf '[%s] ERROR: %s\n' "$SCRIPT_NAME" "$*" >&2; exit "${2:-4}"; }
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check)   MODE="check"; shift ;;
+    --apply)   MODE="apply"; shift ;;
+    --kube-context)
+      [[ -n "${2:-}" ]] || die "--kube-context requires an argument" 4
+      KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help)
+      sed -n '1,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
+      exit 0 ;;
+    *) die "unknown argument: $1" 4 ;;
+  esac
+done
+
+command -v kubectl >/dev/null || die "kubectl not found in PATH" 4
+command -v python3 >/dev/null || die "python3 not found in PATH" 4
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" "$@"
+  else
+    kubectl "$@"
+  fi
+}
+
+# ── 1. Confirm we're talking to a GKE cluster ────────────────────────────────
+sc_json="$(kctl get storageclass -o json 2>/dev/null || true)"
+if [[ -z "$sc_json" ]] || ! printf '%s' "$sc_json" | python3 -c '
+import json, sys
+data = json.load(sys.stdin)
+items = data.get("items") or []
+sys.exit(0 if any(it.get("provisioner") == "pd.csi.storage.gke.io" for it in items) else 1)
+'; then
+  die "no ${GKE_CSI_PROVISIONER} StorageClasses found; this does not look like a GKE cluster" 3
+fi
+
+# ── 2. Inspect current state ─────────────────────────────────────────────────
+# Emits: <name>|<provisioner>|<type>|<is_default>  for every StorageClass.
+sc_summary="$(printf '%s' "$sc_json" | python3 - <<'PY'
+import json, sys
+data = json.load(sys.stdin)
+for it in data.get("items") or []:
+    meta = it.get("metadata") or {}
+    name = meta.get("name", "")
+    ann = (meta.get("annotations") or {})
+    is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "") or "").lower() == "true"
+    prov = it.get("provisioner", "") or ""
+    disk_type = ((it.get("parameters") or {}).get("type", "")) or ""
+    print(f"{name}|{prov}|{disk_type}|{'true' if is_default else 'false'}")
+PY
+)"
+
+log "Current StorageClasses:"
+while IFS='|' read -r name prov disk_type is_default; do
+  [[ -n "$name" ]] || continue
+  if [[ "$is_default" == "true" ]]; then
+    log "  * ${name}  prov=${prov}  type=${disk_type}  (default)"
+  else
+    log "    ${name}  prov=${prov}  type=${disk_type}"
+  fi
+done <<< "$sc_summary"
+
+# Identify the current default + its disk type.
+current_default=""
+current_default_type=""
+ssd_backed_defaults=()
+while IFS='|' read -r name prov disk_type is_default; do
+  [[ -n "$name" ]] || continue
+  if [[ "$is_default" == "true" ]]; then
+    current_default="$name"
+    current_default_type="$disk_type"
+    if [[ "$disk_type" == "pd-balanced" || "$disk_type" == "pd-ssd" ]]; then
+      ssd_backed_defaults+=("$name")
+    fi
+  fi
+done <<< "$sc_summary"
+
+# Does standard-hdd (or PROLE_DEFAULT_SC_NAME override) exist with the right shape?
+target_exists="false"
+target_disk_type=""
+target_is_default="false"
+while IFS='|' read -r name prov disk_type is_default; do
+  if [[ "$name" == "$DEFAULT_SC_NAME" ]]; then
+    target_exists="true"
+    target_disk_type="$disk_type"
+    target_is_default="$is_default"
+  fi
+done <<< "$sc_summary"
+
+compliant="true"
+issues=()
+
+if [[ "$target_exists" != "true" ]]; then
+  compliant="false"
+  issues+=("target StorageClass '${DEFAULT_SC_NAME}' does not exist")
+elif [[ "$target_disk_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then
+  compliant="false"
+  issues+=("'${DEFAULT_SC_NAME}' has type='${target_disk_type}', want '${DEFAULT_SC_DISK_TYPE}'")
+fi
+
+if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then
+  compliant="false"
+  issues+=("SSD-backed class(es) still marked default: ${ssd_backed_defaults[*]}")
+fi
+
+if [[ "$target_exists" == "true" && "$target_is_default" != "true" ]]; then
+  compliant="false"
+  issues+=("'${DEFAULT_SC_NAME}' is not annotated is-default-class=true")
+fi
+
+if [[ "$compliant" == "true" ]]; then
+  log "OK — default StorageClass is '${current_default}' (type=${current_default_type}). No action needed."
+  exit 0
+fi
+
+log "Drift detected:"
+for issue in "${issues[@]}"; do
+  log "  - ${issue}"
+done
+
+if [[ "$MODE" == "check" ]]; then
+  log "Run with --apply to remediate."
+  exit 1
+fi
+
+# ── 3. Remediate ─────────────────────────────────────────────────────────────
+log "Applying remediation ..."
+
+# 3a. Create/update standard-hdd — but NOT yet marked default so we never have
+# a window with two defaults racing.
+cat <<EOF | kctl apply -f -
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${DEFAULT_SC_NAME}
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: ${GKE_CSI_PROVISIONER}
+parameters:
+  type: ${DEFAULT_SC_DISK_TYPE}
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+EOF
+
+# Verify the live class has the right disk type. If `parameters` drifted on an
+# existing class, the apply above would silently no-op (parameters are
+# immutable). Catch that and hard-fail with a clear message.
+live_type="$(kctl get storageclass "$DEFAULT_SC_NAME" -o jsonpath='{.parameters.type}' 2>/dev/null || true)"
+if [[ "$live_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then
+  warn "live '${DEFAULT_SC_NAME}' has type='${live_type}' (want '${DEFAULT_SC_DISK_TYPE}'). Recreating ..."
+  kctl delete storageclass "$DEFAULT_SC_NAME" --wait=true >/dev/null 2>&1 || die "failed to delete '${DEFAULT_SC_NAME}' for recreation" 2
+  cat <<EOF | kctl apply -f -
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${DEFAULT_SC_NAME}
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: ${GKE_CSI_PROVISIONER}
+parameters:
+  type: ${DEFAULT_SC_DISK_TYPE}
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+EOF
+  live_type="$(kctl get storageclass "$DEFAULT_SC_NAME" -o jsonpath='{.parameters.type}' 2>/dev/null || true)"
+  [[ "$live_type" == "$DEFAULT_SC_DISK_TYPE" ]] || die "after recreate, '${DEFAULT_SC_NAME}' still has type='${live_type}'" 2
+fi
+
+# 3b. Clear the default annotation from any SSD-backed class that currently
+# holds it. Patch to "false" (not remove) so we leave a clear audit trail.
+if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then
+  for sc in "${ssd_backed_defaults[@]}"; do
+    log "Un-defaulting SSD-backed class: ${sc}"
+    kctl annotate storageclass "$sc" \
+      "storageclass.kubernetes.io/is-default-class=false" --overwrite >/dev/null \
+      || die "failed to un-default '${sc}'" 2
+  done
+fi
+
+# 3c. Promote standard-hdd to default — atomic with respect to other defaults
+# because we cleared them in 3b first.
+kctl annotate storageclass "$DEFAULT_SC_NAME" \
+  "storageclass.kubernetes.io/is-default-class=true" --overwrite >/dev/null \
+  || die "failed to mark '${DEFAULT_SC_NAME}' as default" 2
+
+# ── 4. Re-verify ─────────────────────────────────────────────────────────────
+final_default="$(kctl get storageclass \
+  -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \
+  2>/dev/null | head -n1)"
+
+if [[ "$final_default" != "$DEFAULT_SC_NAME" ]]; then
+  die "post-remediation: default is '${final_default:-<none>}', expected '${DEFAULT_SC_NAME}'" 2
+fi
+
+# Count defaults — having two is a worse state than having zero.
+default_count="$(kctl get storageclass \
+  -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \
+  2>/dev/null | grep -cv '^$' || true)"
+if [[ "$default_count" != "1" ]]; then
+  die "post-remediation: ${default_count} default StorageClasses exist (want exactly 1)" 2
+fi
+
+log "Remediation OK. Default StorageClass is now '${DEFAULT_SC_NAME}' (${DEFAULT_SC_DISK_TYPE})."
+log "Re-run 'kubectl get storageclass' to confirm."
+exit 0
--- a/etc/init_gitlab.sh
+++ b/etc/init_gitlab.sh
@ -4,6 +4,37 @@ set -euo pipefail
 PROG="init_gitlab"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

+# Standalone-invocation fix: pre-scan $@ for --config and export PROLE_DEPLOY_CFG
+# BEFORE sourcing prole_cfg.sh. deploy.sh injects PROLE_DEPLOY_CFG in its env,
+# but direct invocations (./etc/init_gitlab.sh --config conf/gke.cfg deploy) do
+# not — prole_cfg.sh's auto-selection then picks conf/k3d.cfg first, which
+# exports KUBECONTEXT=dev and downstream kubectl calls fail with
+# `error: context "dev" does not exist`.
+if [[ -z "${PROLE_DEPLOY_CFG:-}" ]]; then
+  _gitlab_prescan_cfg=""
+  _gitlab_prescan_want_next=0
+  for _gitlab_prescan_tok in "$@"; do
+    if [[ "$_gitlab_prescan_want_next" == "1" ]]; then
+      _gitlab_prescan_cfg="$_gitlab_prescan_tok"
+      _gitlab_prescan_want_next=0
+      break
+    fi
+    case "$_gitlab_prescan_tok" in
+      --config=*)  _gitlab_prescan_cfg="${_gitlab_prescan_tok#*=}"; break ;;
+      -c|--config) _gitlab_prescan_want_next=1 ;;
+    esac
+  done
+  if [[ -n "$_gitlab_prescan_cfg" ]]; then
+    if [[ "$_gitlab_prescan_cfg" != /* ]]; then
+      _gitlab_prescan_cfg="$(cd "$(dirname "$_gitlab_prescan_cfg")" 2>/dev/null && pwd)/$(basename "$_gitlab_prescan_cfg")"
+    fi
+    if [[ -f "$_gitlab_prescan_cfg" ]]; then
+      export PROLE_DEPLOY_CFG="$_gitlab_prescan_cfg"
+    fi
+  fi
+  unset _gitlab_prescan_cfg _gitlab_prescan_tok _gitlab_prescan_want_next
+fi
+
 # shellcheck disable=SC1090
 source "$SCRIPT_DIR/prole_cfg.sh"

@ -2483,6 +2514,72 @@ GITLAB_OIDC_REDIRECT_URI="${GITLAB_OIDC_REDIRECT_URI:-${_default_gitlab_oidc_red
 GITLAB_OIDC_CLIENT_ID="${GITLAB_OIDC_CLIENT_ID:-${OIDC_CLIENT_ID:-${GOOGLE_OIDC_CLIENT_ID:-${GOOGLE_CLIENT_ID:-}}}}"
 GITLAB_OIDC_CLIENT_SECRET="${GITLAB_OIDC_CLIENT_SECRET:-${OIDC_CLIENT_SECRET:-${GOOGLE_OIDC_CLIENT_SECRET:-${GOOGLE_CLIENT_SECRET:-}}}}"

+# Resolve `secretref://<ref>` values for standalone invocations. deploy.sh
+# resolves these in Python (knoe/core/actions.py:_resolve_secretref_value)
+# before exec; direct invocations need the same lookup inline. Searches env
+# aliases, then PROLE_SERVICE/secrets, etc/secrets, and secrets directories
+# for a file named <ref>. Returns the raw input unchanged on miss.
+_init_gitlab_resolve_secretref() {
+  local raw="${1:-}"
+  local aliases="${2:-}"
+  case "$raw" in
+    secretref://*) ;;
+    *) printf '%s' "$raw"; return 0 ;;
+  esac
+
+  local ref="${raw#secretref://}"
+  ref="${ref#/}"; ref="${ref%/}"
+  [[ -n "$ref" ]] || { printf '%s' "$raw"; return 0; }
+
+  local normalized="${ref//-/_}"
+  normalized="${normalized//\//_}"
+  normalized="${normalized//./_}"
+  local upper_norm="${normalized^^}"
+
+  local -a candidates=("$ref" "$normalized" "$upper_norm")
+  local alias
+  for alias in $aliases; do
+    candidates+=("$alias")
+  done
+
+  local key val
+  for key in "${candidates[@]}"; do
+    [[ -z "$key" ]] && continue
+    [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
+    val="${!key-}"
+    if [[ -n "$val" && "$val" != secretref://* ]]; then
+      printf '%s' "$val"
+      return 0
+    fi
+  done
+
+  local root="${PROLE_HOME:-${SCRIPT_DIR}/..}"
+  local svc="${PROLE_SERVICE:-}"
+  local -a paths=()
+  [[ -n "$svc" ]] && paths+=("$svc/secrets/$ref")
+  paths+=("$root/etc/secrets/$ref" "$root/secrets/$ref")
+
+  local path
+  for path in "${paths[@]}"; do
+    if [[ -f "$path" && -r "$path" ]]; then
+      val="$(< "$path")"
+      val="${val#"${val%%[![:space:]]*}"}"
+      val="${val%"${val##*[![:space:]]}"}"
+      if [[ -n "$val" ]]; then
+        printf '%s' "$val"
+        return 0
+      fi
+    fi
+  done
+
+  printf '%s' "$raw"
+}
+
+GITLAB_OIDC_CLIENT_ID="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_ID" \
+  "GITLAB_OIDC_CLIENT_ID OIDC_CLIENT_ID GOOGLE_OIDC_CLIENT_ID GOOGLE_CLIENT_ID")"
+GITLAB_OIDC_CLIENT_SECRET="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_SECRET" \
+  "GITLAB_OIDC_CLIENT_SECRET OIDC_CLIENT_SECRET GOOGLE_OIDC_CLIENT_SECRET GOOGLE_CLIENT_SECRET")"
+
 is_unresolved_secret_ref() {
  local raw="${1:-}"
  case "$raw" in
@ -3445,13 +3542,21 @@ setup_garage_for_gitlab() {

  # ---- Create buckets and grant permissions ----
  # --key takes the accessKeyId (not the name) in garage v1.x
+  # Bucket names must match GitLab Helm chart defaults because the GitLab CR's
+  # global.appConfig.object_store block only specifies the connection secret —
+  # it does NOT override per-object bucket names. Rails therefore writes to
+  # gitlab-uploads / gitlab-artifacts / etc. (no -storage suffix). A prior
+  # naming with a -storage suffix caused Excon::Error::NotFound / NoSuchBucket
+  # on first login when CarrierWave tried to upload the duo-bot avatar to
+  # /gitlab-uploads/user/avatar/8/duo-bot.png. Keep registry / terraform-state /
+  # ci-secure-files verbatim — those are the names both sides already agreed on.
  local buckets=(
    registry
-    gitlab-artifacts-storage
-    gitlab-lfs-storage
-    gitlab-uploads-storage
-    gitlab-packages-storage
-    gitlab-dependency-proxy-storage
+    gitlab-artifacts
+    gitlab-lfs
+    gitlab-uploads
+    gitlab-packages
+    gitlab-dependency-proxy
    gitlab-terraform-state
    gitlab-ci-secure-files
  )
--- a/k8s/prole/storageclass-gcp-standard-hdd.yaml
+++ b/k8s/prole/storageclass-gcp-standard-hdd.yaml
@ -0,0 +1,30 @@
+# Generic HDD (pd-standard) StorageClass for GKE — the cluster-wide default.
+#
+# Why this exists
+# ---------------
+# GKE ships with `standard-rwo` as the default StorageClass, which under the
+# hood provisions `pd-balanced` (SSD-backed hybrid). That draws from the
+# SSD_TOTAL_GB quota and has repeatedly caused provisioning to wedge on small
+# projects. For workloads that don't need SSD latency (OpenBao, registry
+# storage, generic scratch PVCs, Garage) we want `pd-standard` (HDD) — which
+# uses the separate HDD quota and is cheaper per GiB.
+#
+# This class is the canonical cluster-wide default. Workloads that DO need
+# SSD latency (CNPG) opt in explicitly via storageClassName=premium-rwo in
+# their PVC template.
+#
+# To make this the default, apply this manifest and then run
+# etc/ensure_default_storage_class.sh --apply (which clears the default
+# annotation off standard-rwo and sets it here).
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard-hdd
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: pd.csi.storage.gke.io
+parameters:
+  type: pd-standard
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true