mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
chore: add Supabase PVC quota diagnostics and improve ingress handling
- Added PVC quota error detection with detailed logging in `deploy.sh`. - Enhanced ingress readiness checks with additional Supabase ingress resources. - Updated Helm templates to standardize and always enable ingress for API and Studio.
This commit is contained in:
parent
7c46b52e7e
commit
bdeb0b62f3
31
deploy.sh
31
deploy.sh
@ -55,6 +55,26 @@ def get_ip(host, ctx):
|
||||
# api.knoe.dev, git.knoe.dev, svc.knoe.dev, api.0.knoe.dev, db.0.knoe.dev
|
||||
return "pending"
|
||||
|
||||
def check_supabase_pvcs(ctx):
|
||||
if not ctx: return None
|
||||
try:
|
||||
out = subprocess.check_output(["kubectl", "--context", ctx, "get", "events", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
|
||||
events = json.loads(out).get("items", [])
|
||||
blocking_pvcs = {}
|
||||
for event in events:
|
||||
obj = event.get("involvedObject", {})
|
||||
if obj.get("kind") == "PersistentVolumeClaim":
|
||||
msg = event.get("message", "")
|
||||
reason = event.get("reason", "")
|
||||
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
|
||||
if "exceeded" in msg.lower() or "quota" in msg.lower():
|
||||
pvc_name = obj.get("name")
|
||||
blocking_pvcs[pvc_name] = msg
|
||||
if blocking_pvcs:
|
||||
return blocking_pvcs
|
||||
except: pass
|
||||
return None
|
||||
|
||||
app_ctx, db_ctx = get_config(sys.argv[1])
|
||||
|
||||
# Targets we want to wait for if they are pending
|
||||
@ -63,6 +83,14 @@ timeout = 300 # 5 minutes
|
||||
start = time.time()
|
||||
|
||||
while time.time() - start < timeout:
|
||||
pvc_errors = check_supabase_pvcs(app_ctx)
|
||||
if pvc_errors:
|
||||
print("\nERROR: Supabase PVC provisioning failed (Storage Quota Exceeded).")
|
||||
for pvc, msg in pvc_errors.items():
|
||||
print(f" - PVC: {pvc}")
|
||||
print(f" GKE Error: {msg}")
|
||||
sys.exit(1)
|
||||
|
||||
pending = False
|
||||
for h in wait_hosts:
|
||||
if get_ip(h, app_ctx) == "pending":
|
||||
@ -78,7 +106,8 @@ if time.time() - start >= timeout:
|
||||
try:
|
||||
subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False)
|
||||
print("\nIngress Resource Status:")
|
||||
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-public", "-n", "supabase"], check=False)
|
||||
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-kong", "-n", "supabase"], check=False)
|
||||
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-studio", "-n", "supabase"], check=False)
|
||||
except:
|
||||
pass
|
||||
sys.exit(1)
|
||||
|
||||
@ -91,6 +91,41 @@ check_supabase_minio_blocked() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_supabase_pvc_quota_blocked() {
|
||||
local ns="$1"
|
||||
local kube_context="${2:-}"
|
||||
local ctx_args=()
|
||||
if [[ -n "$kube_context" ]]; then
|
||||
ctx_args+=(--context "$kube_context")
|
||||
fi
|
||||
|
||||
local error_info
|
||||
error_info=$(kubectl "${ctx_args[@]}" get events -n "$ns" -o json 2>/dev/null | python3 - <<'PY'
|
||||
import json
|
||||
import sys
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
except:
|
||||
sys.exit(0)
|
||||
for event in data.get("items", []):
|
||||
obj = event.get("involvedObject", {})
|
||||
if obj.get("kind") == "PersistentVolumeClaim":
|
||||
msg = event.get("message", "")
|
||||
reason = event.get("reason", "")
|
||||
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
|
||||
if "exceeded" in msg.lower() or "quota" in msg.lower():
|
||||
print(f"{obj.get('name')}|{msg}")
|
||||
sys.exit(0)
|
||||
PY
|
||||
)
|
||||
if [[ -n "$error_info" ]]; then
|
||||
local pvc_name="${error_info%%|*}"
|
||||
local gke_msg="${error_info#*|}"
|
||||
repair_blocked "Supabase PVC provisioning failed (Storage Quota Exceeded)" \
|
||||
"PVC '${pvc_name}' is blocked. GKE Error: ${gke_msg}"
|
||||
fi
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo "Warning: $*" >&2
|
||||
}
|
||||
@ -483,6 +518,7 @@ wait_for_supabase_ready() {
|
||||
|
||||
# Targeted blocked-state checks
|
||||
check_supabase_minio_blocked "$ns" "$kube_context"
|
||||
check_supabase_pvc_quota_blocked "$ns" "$kube_context"
|
||||
|
||||
local pending_pvcs not_ready_pods terminating_pods pvc_blockers pod_blockers status_metrics
|
||||
status_metrics=$(STATUS_JSON="$status_json" python3 - <<'PY'
|
||||
|
||||
@ -823,7 +823,7 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
||||
"nameOverride": "supabase",
|
||||
"fullnameOverride": "supabase",
|
||||
"publicIngress": {
|
||||
"enabled": mode == "k8s",
|
||||
"enabled": False,
|
||||
"rules": rules_public,
|
||||
"annotations": {
|
||||
"kubernetes.io/ingress.class": api_ingress_class,
|
||||
@ -894,16 +894,16 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
||||
"s3": {"keyId": garage_s3_key_id, "accessKey": garage_s3_access_key},
|
||||
},
|
||||
"ingress": {
|
||||
"enabled": mode != "k8s",
|
||||
"className": api_ingress_class if mode != "k8s" else "",
|
||||
"enabled": True,
|
||||
"className": "" if mode == "k8s" else api_ingress_class,
|
||||
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in api_host_entries],
|
||||
"annotations": {
|
||||
**({"kubernetes.io/ingress.class": api_ingress_class} if mode == "k8s" else {}),
|
||||
}
|
||||
},
|
||||
"studioIngress": {
|
||||
"enabled": mode != "k8s",
|
||||
"className": studio_ingress_class if mode != "k8s" else "",
|
||||
"enabled": True,
|
||||
"className": "" if mode == "k8s" else studio_ingress_class,
|
||||
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in studio_host_entries],
|
||||
"annotations": {
|
||||
**({"kubernetes.io/ingress.class": studio_ingress_class} if mode == "k8s" else {}),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user