chore: refactor Supabase PV cleanup logic and improve artifact handling

- Added `supabase_collect_stale_retained_pv_info` for clearer separation of logic and improved reusability.
- Enhanced zone detection for retained PVs using regex-based fallback.
- Updated cleanup logic to auto-enable `SUPABASE_AUTO_CLEAN_RETAINED_PVS` and streamline handling of GCE disk deletion.
- Introduced verification and retry mechanism to ensure stale PV artifacts are fully removed.
- Improved error handling for remaining GCE disk cleanup with detailed remediation instructions.

deploy.sh

@@ -69,6 +69,7 @@ import subprocess
 import json
 import configparser
 import time
+import re
 
 def get_config(path):
     c = configparser.ConfigParser()
@@ -197,18 +198,24 @@ def check_supabase_pvcs(ctx):
             csi = spec.get("csi") or {}
             gce = spec.get("gcePersistentDisk") or {}
             labels = (pv.get("metadata") or {}).get("labels") or {}
+            disk_handle = csi.get("volumeHandle") or gce.get("pdName") or "<unknown>"
+            zone = (
+                labels.get("topology.kubernetes.io/zone")
+                or labels.get("failure-domain.beta.kubernetes.io/zone")
+                or (csi.get("volumeAttributes") or {}).get("topology.gke.io/zone")
+                or ""
+            )
+            if not zone and isinstance(disk_handle, str):
+                m = re.search(r"/zones/([^/]+)/disks/[^/]+$", disk_handle)
+                if m:
+                    zone = m.group(1)
             stale_retained.append({
                 "pv": (pv.get("metadata") or {}).get("name", "?"),
                 "pvc": claim_name or "?",
                 "phase": phase or "Unknown",
                 "storageClass": spec.get("storageClassName") or "<unset>",
-                "diskHandle": csi.get("volumeHandle") or gce.get("pdName") or "<unknown>",
+                "diskHandle": disk_handle,
-                "zone": (
+                "zone": zone or "<unknown>",
-                    labels.get("topology.kubernetes.io/zone")
-                    or labels.get("failure-domain.beta.kubernetes.io/zone")
-                    or (csi.get("volumeAttributes") or {}).get("topology.gke.io/zone")
-                    or "<unknown>"
-                ),
             })
 
         if blocking_pvcs or stale_retained:
@@ -392,7 +399,7 @@ while time.time() - start < timeout:
                 print(f"    PVC: {item['pvc']}")
                 print(f"    Phase/Reclaim: {item['phase']}/{item.get('storageClass','<unset>')} Retain")
                 print(f"    Disk: {item['diskHandle']} (zone: {item['zone']})")
-            print("\nAction: delete stale Supabase Retain PVs and backing GCE disks, or rerun with SUPABASE_AUTO_CLEAN_RETAINED_PVS=true (and SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true for disk deletion).")
+            print("\nAction: stale Supabase Retain PVs should be auto-cleaned during deploy; delete the remaining backing GCE disks listed above, or enable SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true with gcloud/project env configured.")
             sys.exit(1)
 
         blockers = pvc_state.get("blockingPVCs") or []

supabase/deploy.sh

@@ -91,7 +91,7 @@ check_supabase_minio_blocked() {
   fi
 }
 
-check_supabase_retained_pv_blocked() {
+supabase_collect_stale_retained_pv_info() {
   local ns="$1"
   local kube_context="${2:-}"
   local ctx_args=()
@@ -99,9 +99,9 @@ check_supabase_retained_pv_blocked() {
     ctx_args+=(--context "$kube_context")
   fi
 
-  local stale_info
+  kubectl "${ctx_args[@]}" get pv -o json 2>/dev/null | python3 - "$ns" <<'PY'
-  stale_info=$(kubectl "${ctx_args[@]}" get pv -o json 2>/dev/null | python3 - "$ns" <<'PY'
 import json
+import re
 import sys
 
 ns = sys.argv[1]
@@ -142,6 +142,10 @@ for item in data.get("items", []) or []:
         or (csi.get("volumeAttributes") or {}).get("topology.gke.io/zone")
         or ""
     ).strip()
+    if not zone and handle:
+        m = re.search(r"/zones/([^/]+)/disks/[^/]+$", handle)
+        if m:
+            zone = m.group(1)
     sc_name = (spec.get("storageClassName") or "<unset>").strip()
     pv_name = ((item.get("metadata") or {}).get("name") or "").strip()
 
@@ -155,36 +159,39 @@ for item in data.get("items", []) or []:
         zone,
     ]))
 PY
-) || true
+}
+
+check_supabase_retained_pv_blocked() {
+  local ns="$1"
+  local kube_context="${2:-}"
+  local ctx_args=()
+  if [[ -n "$kube_context" ]]; then
+    ctx_args+=(--context "$kube_context")
+  fi
+
+  local stale_info
+  stale_info="$(supabase_collect_stale_retained_pv_info "$ns" "$kube_context")" || true
 
   if [[ -z "$stale_info" ]]; then
     return 0
   fi
 
-  local auto_clean_raw auto_clean_pvs auto_clean_disks_raw auto_clean_disks
+  local auto_clean_raw auto_clean_disks_raw auto_clean_disks
   auto_clean_raw="$(printf '%s' "${SUPABASE_AUTO_CLEAN_RETAINED_PVS:-false}" | tr '[:upper:]' '[:lower:]')"
   auto_clean_disks_raw="$(printf '%s' "${SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS:-false}" | tr '[:upper:]' '[:lower:]')"
-  auto_clean_pvs="false"
   auto_clean_disks="false"
-  if [[ "$auto_clean_raw" =~ ^(1|true|yes|on)$ ]]; then
-    auto_clean_pvs="true"
-  fi
   if [[ "$auto_clean_disks_raw" =~ ^(1|true|yes|on)$ ]]; then
     auto_clean_disks="true"
   fi
-
+  if [[ "$auto_clean_raw" =~ ^(1|true|yes|on)$ ]]; then
-  local summary_lines=()
+    export SUPABASE_AUTO_CLEAN_RETAINED_PVS="true"
-  local pv_name pvc_name phase sc_name reclaim disk_handle zone
+  else
-  while IFS=$'\t' read -r pv_name pvc_name phase sc_name reclaim disk_handle zone; do
+    warn "Detected stale Supabase Retain PV artifacts; auto-enabling SUPABASE_AUTO_CLEAN_RETAINED_PVS=true for this run."
-    [[ -n "$pv_name" ]] || continue
+    export SUPABASE_AUTO_CLEAN_RETAINED_PVS="true"
-    summary_lines+=("pv=${pv_name}, pvc=${pvc_name:-<unknown>}, phase=${phase:-?}, storageClass=${sc_name:-<unset>}, reclaim=${reclaim:-?}, disk=${disk_handle:-<unknown>}, zone=${zone:-<unknown>}")
-  done <<< "$stale_info"
-
-  if [[ "$auto_clean_pvs" != "true" ]]; then
-    repair_blocked "Supabase retained PVs from previous failed/rerun attempts are still present and can cause GCE quota exhaustion." \
-      "Delete stale Supabase Retain PV/disk artifacts before rerun. Blocking artifacts: ${summary_lines[*]}. Optional controlled cleanup: set SUPABASE_AUTO_CLEAN_RETAINED_PVS=true (and SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true with gcloud + project env to delete backing disks)."
   fi
 
+  local remaining_disk_lines=()
+  local pv_name pvc_name phase sc_name reclaim disk_handle zone
   warn "SUPABASE_AUTO_CLEAN_RETAINED_PVS=true; deleting stale Supabase Retain PV objects before Helm reconcile."
   local project_id
   project_id="${GCP_PROJECT_ID:-${PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}}"
@@ -194,7 +201,14 @@ PY
     warn "Deleting stale PV '${pv_name}' (pvc=${pvc_name:-<unknown>}, disk=${disk_handle:-<unknown>}, zone=${zone:-<unknown>})."
     kubectl "${ctx_args[@]}" delete pv "$pv_name" >/dev/null 2>&1 || true
 
-    if [[ "$auto_clean_disks" != "true" || -z "$disk_handle" ]]; then
+    if [[ "$auto_clean_disks" != "true" ]]; then
+      if [[ -n "$disk_handle" ]]; then
+        remaining_disk_lines+=("${disk_handle} (zone=${zone:-<unknown>})")
+      fi
+      continue
+    fi
+
+    if [[ -z "$disk_handle" ]]; then
       continue
     fi
 
@@ -207,21 +221,63 @@ PY
 
     if ! command -v gcloud >/dev/null 2>&1; then
       warn "Skipping disk cleanup for '${disk_handle}': gcloud is not installed."
+      remaining_disk_lines+=("${disk_handle} (zone=${disk_zone:-<unknown>})")
       continue
     fi
     if [[ -z "$project_id" ]]; then
       warn "Skipping disk cleanup for '${disk_handle}': GCP project is not set (expected GCP_PROJECT_ID/PROJECT_ID/GOOGLE_CLOUD_PROJECT)."
+      remaining_disk_lines+=("${disk_handle} (zone=${disk_zone:-<unknown>})")
       continue
     fi
     if [[ -z "$disk_zone" ]]; then
       warn "Skipping disk cleanup for '${disk_handle}': could not determine disk zone."
+      remaining_disk_lines+=("${disk_handle} (zone=<unknown>)")
       continue
     fi
 
     warn "Deleting retained GCE disk '${disk_name}' (zone=${disk_zone}, project=${project_id})."
-    gcloud compute disks delete "$disk_name" --project "$project_id" --zone "$disk_zone" --quiet >/dev/null 2>&1 || \
+    if ! gcloud compute disks delete "$disk_name" --project "$project_id" --zone "$disk_zone" --quiet >/dev/null 2>&1; then
       warn "Failed to delete retained GCE disk '${disk_name}' in zone '${disk_zone}'."
+      remaining_disk_lines+=("${disk_handle} (zone=${disk_zone})")
+    fi
   done <<< "$stale_info"
+
+  local verify_stale_info=""
+  local verify_attempt
+  for verify_attempt in 1 2 3 4; do
+    verify_stale_info="$(supabase_collect_stale_retained_pv_info "$ns" "$kube_context")" || true
+    if [[ -z "$verify_stale_info" ]]; then
+      break
+    fi
+    warn "Supabase retained-PV cleanup verification poll ${verify_attempt}/4: stale PV objects still visible; waiting 2s..."
+    sleep 2
+  done
+
+  if [[ -n "$verify_stale_info" ]]; then
+    local verify_lines=()
+    while IFS=$'\t' read -r pv_name pvc_name phase sc_name reclaim disk_handle zone; do
+      [[ -n "$pv_name" ]] || continue
+      if [[ -z "$zone" && "$disk_handle" =~ /zones/([^/]+)/disks/([^/]+)$ ]]; then
+        zone="${BASH_REMATCH[1]}"
+      fi
+      verify_lines+=("pv=${pv_name}, pvc=${pvc_name:-<unknown>}, phase=${phase:-?}, storageClass=${sc_name:-<unset>}, reclaim=${reclaim:-?}, disk=${disk_handle:-<unknown>}, zone=${zone:-<unknown>}")
+    done <<< "$verify_stale_info"
+    repair_blocked "Supabase retained PV cleanup did not converge; stale Supabase Retain PV artifacts are still present." \
+      "Remaining stale artifacts after cleanup: ${verify_lines[*]}"
+  fi
+
+  if (( ${#remaining_disk_lines[@]} > 0 )); then
+    local remaining_disks
+    remaining_disks=$(IFS='; '; printf '%s' "${remaining_disk_lines[*]}")
+    if [[ "$auto_clean_disks" != "true" ]]; then
+      repair_blocked "Stale Supabase Retain PVs were cleaned automatically, but retained backing GCE disks remain and can still consume quota." \
+        "Delete the remaining retained disk handles and rerun deploy. Remaining handles: ${remaining_disks}. To auto-delete in-script, set SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true (with gcloud and project env configured)."
+    fi
+    repair_blocked "Automatic retained GCE disk cleanup did not fully succeed for Supabase stale artifacts." \
+      "Delete the remaining retained disk handles and rerun deploy. Remaining handles: ${remaining_disks}."
+  fi
+
+  warn "Supabase retained-PV preflight cleanup completed; stale PV blockers are cleared for this run."
 }
 
 check_supabase_pvc_quota_blocked() {