chore: add Supabase PVC quota diagnostics and improve ingress handling

- Added PVC quota error detection with detailed logging in `deploy.sh`.
- Enhanced ingress readiness checks with additional Supabase ingress resources.
- Updated Helm templates to standardize and always enable ingress for API and Studio.
This commit is contained in:
chrisfu 2026-04-15 22:51:57 -07:00
parent 7c46b52e7e
commit bdeb0b62f3
3 changed files with 71 additions and 6 deletions

View File

@ -55,6 +55,26 @@ def get_ip(host, ctx):
# api.knoe.dev, git.knoe.dev, svc.knoe.dev, api.0.knoe.dev, db.0.knoe.dev
return "pending"
def check_supabase_pvcs(ctx):
if not ctx: return None
try:
out = subprocess.check_output(["kubectl", "--context", ctx, "get", "events", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
events = json.loads(out).get("items", [])
blocking_pvcs = {}
for event in events:
obj = event.get("involvedObject", {})
if obj.get("kind") == "PersistentVolumeClaim":
msg = event.get("message", "")
reason = event.get("reason", "")
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
if "exceeded" in msg.lower() or "quota" in msg.lower():
pvc_name = obj.get("name")
blocking_pvcs[pvc_name] = msg
if blocking_pvcs:
return blocking_pvcs
except: pass
return None
app_ctx, db_ctx = get_config(sys.argv[1])
# Targets we want to wait for if they are pending
@ -63,6 +83,14 @@ timeout = 300 # 5 minutes
start = time.time()
while time.time() - start < timeout:
pvc_errors = check_supabase_pvcs(app_ctx)
if pvc_errors:
print("\nERROR: Supabase PVC provisioning failed (Storage Quota Exceeded).")
for pvc, msg in pvc_errors.items():
print(f" - PVC: {pvc}")
print(f" GKE Error: {msg}")
sys.exit(1)
pending = False
for h in wait_hosts:
if get_ip(h, app_ctx) == "pending":
@ -78,7 +106,8 @@ if time.time() - start >= timeout:
try:
subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False)
print("\nIngress Resource Status:")
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-public", "-n", "supabase"], check=False)
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-kong", "-n", "supabase"], check=False)
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-studio", "-n", "supabase"], check=False)
except:
pass
sys.exit(1)

View File

@ -91,6 +91,41 @@ check_supabase_minio_blocked() {
fi
}
check_supabase_pvc_quota_blocked() {
local ns="$1"
local kube_context="${2:-}"
local ctx_args=()
if [[ -n "$kube_context" ]]; then
ctx_args+=(--context "$kube_context")
fi
local error_info
error_info=$(kubectl "${ctx_args[@]}" get events -n "$ns" -o json 2>/dev/null | python3 - <<'PY'
import json
import sys
try:
data = json.load(sys.stdin)
except:
sys.exit(0)
for event in data.get("items", []):
obj = event.get("involvedObject", {})
if obj.get("kind") == "PersistentVolumeClaim":
msg = event.get("message", "")
reason = event.get("reason", "")
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
if "exceeded" in msg.lower() or "quota" in msg.lower():
print(f"{obj.get('name')}|{msg}")
sys.exit(0)
PY
)
if [[ -n "$error_info" ]]; then
local pvc_name="${error_info%%|*}"
local gke_msg="${error_info#*|}"
repair_blocked "Supabase PVC provisioning failed (Storage Quota Exceeded)" \
"PVC '${pvc_name}' is blocked. GKE Error: ${gke_msg}"
fi
}
warn() {
echo "Warning: $*" >&2
}
@ -483,6 +518,7 @@ wait_for_supabase_ready() {
# Targeted blocked-state checks
check_supabase_minio_blocked "$ns" "$kube_context"
check_supabase_pvc_quota_blocked "$ns" "$kube_context"
local pending_pvcs not_ready_pods terminating_pods pvc_blockers pod_blockers status_metrics
status_metrics=$(STATUS_JSON="$status_json" python3 - <<'PY'

View File

@ -823,7 +823,7 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
"nameOverride": "supabase",
"fullnameOverride": "supabase",
"publicIngress": {
"enabled": mode == "k8s",
"enabled": False,
"rules": rules_public,
"annotations": {
"kubernetes.io/ingress.class": api_ingress_class,
@ -894,16 +894,16 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
"s3": {"keyId": garage_s3_key_id, "accessKey": garage_s3_access_key},
},
"ingress": {
"enabled": mode != "k8s",
"className": api_ingress_class if mode != "k8s" else "",
"enabled": True,
"className": "" if mode == "k8s" else api_ingress_class,
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in api_host_entries],
"annotations": {
**({"kubernetes.io/ingress.class": api_ingress_class} if mode == "k8s" else {}),
}
},
"studioIngress": {
"enabled": mode != "k8s",
"className": studio_ingress_class if mode != "k8s" else "",
"enabled": True,
"className": "" if mode == "k8s" else studio_ingress_class,
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in studio_host_entries],
"annotations": {
**({"kubernetes.io/ingress.class": studio_ingress_class} if mode == "k8s" else {}),