mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
chore: refactor Supabase PV cleanup logic and improve artifact handling
- Added `supabase_collect_stale_retained_pv_info` for clearer separation of logic and improved reusability. - Enhanced zone detection for retained PVs using regex-based fallback. - Updated cleanup logic to auto-enable `SUPABASE_AUTO_CLEAN_RETAINED_PVS` and streamline handling of GCE disk deletion. - Introduced verification and retry mechanism to ensure stale PV artifacts are fully removed. - Improved error handling for remaining GCE disk cleanup with detailed remediation instructions.
This commit is contained in:
chrisfu
parent
6ec631c6d3
commit
be6ff2f1a8
23
deploy.sh
23
deploy.sh
@ -69,6 +69,7 @@ import subprocess
|
||||
import json
|
||||
import configparser
|
||||
import time
|
||||
import re
|
||||
|
||||
def get_config(path):
|
||||
c = configparser.ConfigParser()
|
||||
@ -197,18 +198,24 @@ def check_supabase_pvcs(ctx):
|
||||
csi = spec.get("csi") or {}
|
||||
gce = spec.get("gcePersistentDisk") or {}
|
||||
labels = (pv.get("metadata") or {}).get("labels") or {}
|
||||
disk_handle = csi.get("volumeHandle") or gce.get("pdName") or "<unknown>"
|
||||
zone = (
|
||||
labels.get("topology.kubernetes.io/zone")
|
||||
or labels.get("failure-domain.beta.kubernetes.io/zone")
|
||||
or (csi.get("volumeAttributes") or {}).get("topology.gke.io/zone")
|
||||
or ""
|
||||
)
|
||||
if not zone and isinstance(disk_handle, str):
|
||||
m = re.search(r"/zones/([^/]+)/disks/[^/]+$", disk_handle)
|
||||
if m:
|
||||
zone = m.group(1)
|
||||
stale_retained.append({
|
||||
"pv": (pv.get("metadata") or {}).get("name", "?"),
|
||||
"pvc": claim_name or "?",
|
||||
"phase": phase or "Unknown",
|
||||
"storageClass": spec.get("storageClassName") or "<unset>",
|
||||
"diskHandle": csi.get("volumeHandle") or gce.get("pdName") or "<unknown>",
|
||||
"zone": (
|
||||
labels.get("topology.kubernetes.io/zone")
|
||||
or labels.get("failure-domain.beta.kubernetes.io/zone")
|
||||
or (csi.get("volumeAttributes") or {}).get("topology.gke.io/zone")
|
||||
or "<unknown>"
|
||||
),
|
||||
"diskHandle": disk_handle,
|
||||
"zone": zone or "<unknown>",
|
||||
})
|
||||
|
||||
if blocking_pvcs or stale_retained:
|
||||
@ -392,7 +399,7 @@ while time.time() - start < timeout:
|
||||
print(f" PVC: {item['pvc']}")
|
||||
print(f" Phase/Reclaim: {item['phase']}/{item.get('storageClass','<unset>')} Retain")
|
||||
print(f" Disk: {item['diskHandle']} (zone: {item['zone']})")
|
||||
print("\nAction: delete stale Supabase Retain PVs and backing GCE disks, or rerun with SUPABASE_AUTO_CLEAN_RETAINED_PVS=true (and SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true for disk deletion).")
|
||||
print("\nAction: stale Supabase Retain PVs should be auto-cleaned during deploy; delete the remaining backing GCE disks listed above, or enable SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true with gcloud/project env configured.")
|
||||
sys.exit(1)
|
||||
|
||||
blockers = pvc_state.get("blockingPVCs") or []
|
||||
|
||||
@ -91,7 +91,7 @@ check_supabase_minio_blocked() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_supabase_retained_pv_blocked() {
|
||||
supabase_collect_stale_retained_pv_info() {
|
||||
local ns="$1"
|
||||
local kube_context="${2:-}"
|
||||
local ctx_args=()
|
||||
@ -99,9 +99,9 @@ check_supabase_retained_pv_blocked() {
|
||||
ctx_args+=(--context "$kube_context")
|
||||
fi
|
||||
|
||||
local stale_info
|
||||
stale_info=$(kubectl "${ctx_args[@]}" get pv -o json 2>/dev/null | python3 - "$ns" <<'PY'
|
||||
kubectl "${ctx_args[@]}" get pv -o json 2>/dev/null | python3 - "$ns" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
ns = sys.argv[1]
|
||||
@ -142,6 +142,10 @@ for item in data.get("items", []) or []:
|
||||
or (csi.get("volumeAttributes") or {}).get("topology.gke.io/zone")
|
||||
or ""
|
||||
).strip()
|
||||
if not zone and handle:
|
||||
m = re.search(r"/zones/([^/]+)/disks/[^/]+$", handle)
|
||||
if m:
|
||||
zone = m.group(1)
|
||||
sc_name = (spec.get("storageClassName") or "<unset>").strip()
|
||||
pv_name = ((item.get("metadata") or {}).get("name") or "").strip()
|
||||
|
||||
@ -155,36 +159,39 @@ for item in data.get("items", []) or []:
|
||||
zone,
|
||||
]))
|
||||
PY
|
||||
) || true
|
||||
}
|
||||
|
||||
check_supabase_retained_pv_blocked() {
|
||||
local ns="$1"
|
||||
local kube_context="${2:-}"
|
||||
local ctx_args=()
|
||||
if [[ -n "$kube_context" ]]; then
|
||||
ctx_args+=(--context "$kube_context")
|
||||
fi
|
||||
|
||||
local stale_info
|
||||
stale_info="$(supabase_collect_stale_retained_pv_info "$ns" "$kube_context")" || true
|
||||
|
||||
if [[ -z "$stale_info" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local auto_clean_raw auto_clean_pvs auto_clean_disks_raw auto_clean_disks
|
||||
local auto_clean_raw auto_clean_disks_raw auto_clean_disks
|
||||
auto_clean_raw="$(printf '%s' "${SUPABASE_AUTO_CLEAN_RETAINED_PVS:-false}" | tr '[:upper:]' '[:lower:]')"
|
||||
auto_clean_disks_raw="$(printf '%s' "${SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS:-false}" | tr '[:upper:]' '[:lower:]')"
|
||||
auto_clean_pvs="false"
|
||||
auto_clean_disks="false"
|
||||
if [[ "$auto_clean_raw" =~ ^(1|true|yes|on)$ ]]; then
|
||||
auto_clean_pvs="true"
|
||||
fi
|
||||
if [[ "$auto_clean_disks_raw" =~ ^(1|true|yes|on)$ ]]; then
|
||||
auto_clean_disks="true"
|
||||
fi
|
||||
|
||||
local summary_lines=()
|
||||
local pv_name pvc_name phase sc_name reclaim disk_handle zone
|
||||
while IFS=$'\t' read -r pv_name pvc_name phase sc_name reclaim disk_handle zone; do
|
||||
[[ -n "$pv_name" ]] || continue
|
||||
summary_lines+=("pv=${pv_name}, pvc=${pvc_name:-<unknown>}, phase=${phase:-?}, storageClass=${sc_name:-<unset>}, reclaim=${reclaim:-?}, disk=${disk_handle:-<unknown>}, zone=${zone:-<unknown>}")
|
||||
done <<< "$stale_info"
|
||||
|
||||
if [[ "$auto_clean_pvs" != "true" ]]; then
|
||||
repair_blocked "Supabase retained PVs from previous failed/rerun attempts are still present and can cause GCE quota exhaustion." \
|
||||
"Delete stale Supabase Retain PV/disk artifacts before rerun. Blocking artifacts: ${summary_lines[*]}. Optional controlled cleanup: set SUPABASE_AUTO_CLEAN_RETAINED_PVS=true (and SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true with gcloud + project env to delete backing disks)."
|
||||
if [[ "$auto_clean_raw" =~ ^(1|true|yes|on)$ ]]; then
|
||||
export SUPABASE_AUTO_CLEAN_RETAINED_PVS="true"
|
||||
else
|
||||
warn "Detected stale Supabase Retain PV artifacts; auto-enabling SUPABASE_AUTO_CLEAN_RETAINED_PVS=true for this run."
|
||||
export SUPABASE_AUTO_CLEAN_RETAINED_PVS="true"
|
||||
fi
|
||||
|
||||
local remaining_disk_lines=()
|
||||
local pv_name pvc_name phase sc_name reclaim disk_handle zone
|
||||
warn "SUPABASE_AUTO_CLEAN_RETAINED_PVS=true; deleting stale Supabase Retain PV objects before Helm reconcile."
|
||||
local project_id
|
||||
project_id="${GCP_PROJECT_ID:-${PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}}"
|
||||
@ -194,7 +201,14 @@ PY
|
||||
warn "Deleting stale PV '${pv_name}' (pvc=${pvc_name:-<unknown>}, disk=${disk_handle:-<unknown>}, zone=${zone:-<unknown>})."
|
||||
kubectl "${ctx_args[@]}" delete pv "$pv_name" >/dev/null 2>&1 || true
|
||||
|
||||
if [[ "$auto_clean_disks" != "true" || -z "$disk_handle" ]]; then
|
||||
if [[ "$auto_clean_disks" != "true" ]]; then
|
||||
if [[ -n "$disk_handle" ]]; then
|
||||
remaining_disk_lines+=("${disk_handle} (zone=${zone:-<unknown>})")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -z "$disk_handle" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
@ -207,21 +221,63 @@ PY
|
||||
|
||||
if ! command -v gcloud >/dev/null 2>&1; then
|
||||
warn "Skipping disk cleanup for '${disk_handle}': gcloud is not installed."
|
||||
remaining_disk_lines+=("${disk_handle} (zone=${disk_zone:-<unknown>})")
|
||||
continue
|
||||
fi
|
||||
if [[ -z "$project_id" ]]; then
|
||||
warn "Skipping disk cleanup for '${disk_handle}': GCP project is not set (expected GCP_PROJECT_ID/PROJECT_ID/GOOGLE_CLOUD_PROJECT)."
|
||||
remaining_disk_lines+=("${disk_handle} (zone=${disk_zone:-<unknown>})")
|
||||
continue
|
||||
fi
|
||||
if [[ -z "$disk_zone" ]]; then
|
||||
warn "Skipping disk cleanup for '${disk_handle}': could not determine disk zone."
|
||||
remaining_disk_lines+=("${disk_handle} (zone=<unknown>)")
|
||||
continue
|
||||
fi
|
||||
|
||||
warn "Deleting retained GCE disk '${disk_name}' (zone=${disk_zone}, project=${project_id})."
|
||||
gcloud compute disks delete "$disk_name" --project "$project_id" --zone "$disk_zone" --quiet >/dev/null 2>&1 || \
|
||||
if ! gcloud compute disks delete "$disk_name" --project "$project_id" --zone "$disk_zone" --quiet >/dev/null 2>&1; then
|
||||
warn "Failed to delete retained GCE disk '${disk_name}' in zone '${disk_zone}'."
|
||||
remaining_disk_lines+=("${disk_handle} (zone=${disk_zone})")
|
||||
fi
|
||||
done <<< "$stale_info"
|
||||
|
||||
local verify_stale_info=""
|
||||
local verify_attempt
|
||||
for verify_attempt in 1 2 3 4; do
|
||||
verify_stale_info="$(supabase_collect_stale_retained_pv_info "$ns" "$kube_context")" || true
|
||||
if [[ -z "$verify_stale_info" ]]; then
|
||||
break
|
||||
fi
|
||||
warn "Supabase retained-PV cleanup verification poll ${verify_attempt}/4: stale PV objects still visible; waiting 2s..."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [[ -n "$verify_stale_info" ]]; then
|
||||
local verify_lines=()
|
||||
while IFS=$'\t' read -r pv_name pvc_name phase sc_name reclaim disk_handle zone; do
|
||||
[[ -n "$pv_name" ]] || continue
|
||||
if [[ -z "$zone" && "$disk_handle" =~ /zones/([^/]+)/disks/([^/]+)$ ]]; then
|
||||
zone="${BASH_REMATCH[1]}"
|
||||
fi
|
||||
verify_lines+=("pv=${pv_name}, pvc=${pvc_name:-<unknown>}, phase=${phase:-?}, storageClass=${sc_name:-<unset>}, reclaim=${reclaim:-?}, disk=${disk_handle:-<unknown>}, zone=${zone:-<unknown>}")
|
||||
done <<< "$verify_stale_info"
|
||||
repair_blocked "Supabase retained PV cleanup did not converge; stale Supabase Retain PV artifacts are still present." \
|
||||
"Remaining stale artifacts after cleanup: ${verify_lines[*]}"
|
||||
fi
|
||||
|
||||
if (( ${#remaining_disk_lines[@]} > 0 )); then
|
||||
local remaining_disks
|
||||
remaining_disks=$(IFS='; '; printf '%s' "${remaining_disk_lines[*]}")
|
||||
if [[ "$auto_clean_disks" != "true" ]]; then
|
||||
repair_blocked "Stale Supabase Retain PVs were cleaned automatically, but retained backing GCE disks remain and can still consume quota." \
|
||||
"Delete the remaining retained disk handles and rerun deploy. Remaining handles: ${remaining_disks}. To auto-delete in-script, set SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS=true (with gcloud and project env configured)."
|
||||
fi
|
||||
repair_blocked "Automatic retained GCE disk cleanup did not fully succeed for Supabase stale artifacts." \
|
||||
"Delete the remaining retained disk handles and rerun deploy. Remaining handles: ${remaining_disks}."
|
||||
fi
|
||||
|
||||
warn "Supabase retained-PV preflight cleanup completed; stale PV blockers are cleared for this run."
|
||||
}
|
||||
|
||||
check_supabase_pvc_quota_blocked() {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user