mirror of
https://github.com/dredx/prole.git
synced 2026-09-24 15:44:33 +00:00
chore: add Supabase PVC quota diagnostics and improve ingress handling
- Added PVC quota error detection with detailed logging in `deploy.sh`. - Enhanced ingress readiness checks with additional Supabase ingress resources. - Updated Helm templates to standardize and always enable ingress for API and Studio.
This commit is contained in:
parent
7c46b52e7e
commit
bdeb0b62f3
31
deploy.sh
31
deploy.sh
@ -55,6 +55,26 @@ def get_ip(host, ctx):
|
|||||||
# api.knoe.dev, git.knoe.dev, svc.knoe.dev, api.0.knoe.dev, db.0.knoe.dev
|
# api.knoe.dev, git.knoe.dev, svc.knoe.dev, api.0.knoe.dev, db.0.knoe.dev
|
||||||
return "pending"
|
return "pending"
|
||||||
|
|
||||||
|
def check_supabase_pvcs(ctx):
|
||||||
|
if not ctx: return None
|
||||||
|
try:
|
||||||
|
out = subprocess.check_output(["kubectl", "--context", ctx, "get", "events", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
|
||||||
|
events = json.loads(out).get("items", [])
|
||||||
|
blocking_pvcs = {}
|
||||||
|
for event in events:
|
||||||
|
obj = event.get("involvedObject", {})
|
||||||
|
if obj.get("kind") == "PersistentVolumeClaim":
|
||||||
|
msg = event.get("message", "")
|
||||||
|
reason = event.get("reason", "")
|
||||||
|
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
|
||||||
|
if "exceeded" in msg.lower() or "quota" in msg.lower():
|
||||||
|
pvc_name = obj.get("name")
|
||||||
|
blocking_pvcs[pvc_name] = msg
|
||||||
|
if blocking_pvcs:
|
||||||
|
return blocking_pvcs
|
||||||
|
except: pass
|
||||||
|
return None
|
||||||
|
|
||||||
app_ctx, db_ctx = get_config(sys.argv[1])
|
app_ctx, db_ctx = get_config(sys.argv[1])
|
||||||
|
|
||||||
# Targets we want to wait for if they are pending
|
# Targets we want to wait for if they are pending
|
||||||
@ -63,6 +83,14 @@ timeout = 300 # 5 minutes
|
|||||||
start = time.time()
|
start = time.time()
|
||||||
|
|
||||||
while time.time() - start < timeout:
|
while time.time() - start < timeout:
|
||||||
|
pvc_errors = check_supabase_pvcs(app_ctx)
|
||||||
|
if pvc_errors:
|
||||||
|
print("\nERROR: Supabase PVC provisioning failed (Storage Quota Exceeded).")
|
||||||
|
for pvc, msg in pvc_errors.items():
|
||||||
|
print(f" - PVC: {pvc}")
|
||||||
|
print(f" GKE Error: {msg}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
pending = False
|
pending = False
|
||||||
for h in wait_hosts:
|
for h in wait_hosts:
|
||||||
if get_ip(h, app_ctx) == "pending":
|
if get_ip(h, app_ctx) == "pending":
|
||||||
@ -78,7 +106,8 @@ if time.time() - start >= timeout:
|
|||||||
try:
|
try:
|
||||||
subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False)
|
subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False)
|
||||||
print("\nIngress Resource Status:")
|
print("\nIngress Resource Status:")
|
||||||
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-public", "-n", "supabase"], check=False)
|
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-kong", "-n", "supabase"], check=False)
|
||||||
|
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-studio", "-n", "supabase"], check=False)
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|||||||
@ -91,6 +91,41 @@ check_supabase_minio_blocked() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
check_supabase_pvc_quota_blocked() {
|
||||||
|
local ns="$1"
|
||||||
|
local kube_context="${2:-}"
|
||||||
|
local ctx_args=()
|
||||||
|
if [[ -n "$kube_context" ]]; then
|
||||||
|
ctx_args+=(--context "$kube_context")
|
||||||
|
fi
|
||||||
|
|
||||||
|
local error_info
|
||||||
|
error_info=$(kubectl "${ctx_args[@]}" get events -n "$ns" -o json 2>/dev/null | python3 - <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
try:
|
||||||
|
data = json.load(sys.stdin)
|
||||||
|
except:
|
||||||
|
sys.exit(0)
|
||||||
|
for event in data.get("items", []):
|
||||||
|
obj = event.get("involvedObject", {})
|
||||||
|
if obj.get("kind") == "PersistentVolumeClaim":
|
||||||
|
msg = event.get("message", "")
|
||||||
|
reason = event.get("reason", "")
|
||||||
|
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
|
||||||
|
if "exceeded" in msg.lower() or "quota" in msg.lower():
|
||||||
|
print(f"{obj.get('name')}|{msg}")
|
||||||
|
sys.exit(0)
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
if [[ -n "$error_info" ]]; then
|
||||||
|
local pvc_name="${error_info%%|*}"
|
||||||
|
local gke_msg="${error_info#*|}"
|
||||||
|
repair_blocked "Supabase PVC provisioning failed (Storage Quota Exceeded)" \
|
||||||
|
"PVC '${pvc_name}' is blocked. GKE Error: ${gke_msg}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
warn() {
|
warn() {
|
||||||
echo "Warning: $*" >&2
|
echo "Warning: $*" >&2
|
||||||
}
|
}
|
||||||
@ -483,6 +518,7 @@ wait_for_supabase_ready() {
|
|||||||
|
|
||||||
# Targeted blocked-state checks
|
# Targeted blocked-state checks
|
||||||
check_supabase_minio_blocked "$ns" "$kube_context"
|
check_supabase_minio_blocked "$ns" "$kube_context"
|
||||||
|
check_supabase_pvc_quota_blocked "$ns" "$kube_context"
|
||||||
|
|
||||||
local pending_pvcs not_ready_pods terminating_pods pvc_blockers pod_blockers status_metrics
|
local pending_pvcs not_ready_pods terminating_pods pvc_blockers pod_blockers status_metrics
|
||||||
status_metrics=$(STATUS_JSON="$status_json" python3 - <<'PY'
|
status_metrics=$(STATUS_JSON="$status_json" python3 - <<'PY'
|
||||||
|
|||||||
@ -823,7 +823,7 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
|||||||
"nameOverride": "supabase",
|
"nameOverride": "supabase",
|
||||||
"fullnameOverride": "supabase",
|
"fullnameOverride": "supabase",
|
||||||
"publicIngress": {
|
"publicIngress": {
|
||||||
"enabled": mode == "k8s",
|
"enabled": False,
|
||||||
"rules": rules_public,
|
"rules": rules_public,
|
||||||
"annotations": {
|
"annotations": {
|
||||||
"kubernetes.io/ingress.class": api_ingress_class,
|
"kubernetes.io/ingress.class": api_ingress_class,
|
||||||
@ -894,16 +894,16 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
|||||||
"s3": {"keyId": garage_s3_key_id, "accessKey": garage_s3_access_key},
|
"s3": {"keyId": garage_s3_key_id, "accessKey": garage_s3_access_key},
|
||||||
},
|
},
|
||||||
"ingress": {
|
"ingress": {
|
||||||
"enabled": mode != "k8s",
|
"enabled": True,
|
||||||
"className": api_ingress_class if mode != "k8s" else "",
|
"className": "" if mode == "k8s" else api_ingress_class,
|
||||||
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in api_host_entries],
|
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in api_host_entries],
|
||||||
"annotations": {
|
"annotations": {
|
||||||
**({"kubernetes.io/ingress.class": api_ingress_class} if mode == "k8s" else {}),
|
**({"kubernetes.io/ingress.class": api_ingress_class} if mode == "k8s" else {}),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"studioIngress": {
|
"studioIngress": {
|
||||||
"enabled": mode != "k8s",
|
"enabled": True,
|
||||||
"className": studio_ingress_class if mode != "k8s" else "",
|
"className": "" if mode == "k8s" else studio_ingress_class,
|
||||||
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in studio_host_entries],
|
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in studio_host_entries],
|
||||||
"annotations": {
|
"annotations": {
|
||||||
**({"kubernetes.io/ingress.class": studio_ingress_class} if mode == "k8s" else {}),
|
**({"kubernetes.io/ingress.class": studio_ingress_class} if mode == "k8s" else {}),
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user