mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
d0fc4af23c
- Added PVC quota error detection with detailed logging in `deploy.sh`. - Enhanced ingress readiness checks with additional Supabase ingress resources. - Updated Helm templates to standardize and always enable ingress for API and Studio.
180 lines
7.0 KiB
Bash
Executable File
180 lines
7.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CONFIG_PATH="${PROLE_DEPLOY_CFG:-${ROOT_DIR}/conf/gke.cfg}"
|
|
|
|
if [[ -x "${ROOT_DIR}/.venv/bin/python3" ]]; then
|
|
PYTHON_BIN="${ROOT_DIR}/.venv/bin/python3"
|
|
elif [[ -x "${ROOT_DIR}/bin/python3" ]]; then
|
|
PYTHON_BIN="${ROOT_DIR}/bin/python3"
|
|
else
|
|
PYTHON_BIN="${PYTHON:-python3}"
|
|
fi
|
|
|
|
# Validate StorageClass on GKE before deployment
|
|
if [[ "$CONFIG_PATH" == *"gke.cfg"* ]]; then
|
|
echo "Validating Supabase StorageClass configuration..."
|
|
"${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
|
|
import sys
|
|
import subprocess
|
|
import json
|
|
import configparser
|
|
|
|
def die(msg):
|
|
print(f"ERROR: {msg}")
|
|
sys.exit(1)
|
|
|
|
c = configparser.ConfigParser()
|
|
c.read(sys.argv[1])
|
|
g = c["Global"] if "Global" in c else {}
|
|
e = c["env_setup"] if "env_setup" in c else {}
|
|
i = c["init_cluster"] if "init_cluster" in c else {}
|
|
|
|
ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "")
|
|
if not ctx:
|
|
sys.exit(0)
|
|
|
|
# Resolve StorageClass name
|
|
sc_name = g.get("SUPABASE_STORAGE_CLASS") or c.get("Supabase", "STORAGE_CLASS", fallback="supabase-standard")
|
|
|
|
try:
|
|
out = subprocess.check_output(["kubectl", "--context", ctx, "get", "storageclass", sc_name, "-o", "json"], stderr=subprocess.DEVNULL, text=True)
|
|
sc = json.loads(out)
|
|
provisioner = sc.get("provisioner", "")
|
|
params = sc.get("parameters", {})
|
|
disk_type = params.get("type", "")
|
|
|
|
if provisioner == "pd.csi.storage.gke.io":
|
|
if disk_type in ["pd-ssd", "pd-balanced"]:
|
|
die(f"storage-class configuration error: StorageClass '{sc_name}' is SSD-backed (type: {disk_type}). "
|
|
"Supabase must use HDD-backed storage (pd-standard) on GKE to avoid quota issues.")
|
|
elif disk_type != "pd-standard":
|
|
die(f"storage-class configuration error: StorageClass '{sc_name}' has ambiguous disk type '{disk_type}'.")
|
|
print(f"Verified Supabase StorageClass '{sc_name}' (type: {disk_type or 'unknown'}).")
|
|
except subprocess.CalledProcessError:
|
|
# If it's the very first deploy, the StorageClass might be missing because it's in the Helm chart we're about to apply.
|
|
# However, if it's already missing and we're NOT on the first deploy, it might be an issue.
|
|
# But usually, if it's in the chart, we shouldn't fail if it's missing *before* the first deploy.
|
|
# But the requirement says "If the required HDD-backed StorageClass is missing, fail clearly."
|
|
# Let's assume for now that if it's missing, we allow it to be created by the deploy,
|
|
# UNLESS it's already there and wrong.
|
|
print(f"Note: StorageClass '{sc_name}' not found; will be created by deployment.")
|
|
PY
|
|
fi
|
|
|
|
"${PYTHON_BIN}" -m prole.deploy_pipeline --config "${CONFIG_PATH}" "$@"
|
|
|
|
# Post-deploy summary
|
|
echo ""
|
|
echo "Public Endpoints:"
|
|
"${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
|
|
import sys
|
|
import subprocess
|
|
import json
|
|
import configparser
|
|
import time
|
|
|
|
def get_config(path):
|
|
c = configparser.ConfigParser()
|
|
c.read(path)
|
|
# Check [Global] then [env_setup] then [init_cluster] for contexts
|
|
g = c["Global"] if "Global" in c else {}
|
|
e = c["env_setup"] if "env_setup" in c else {}
|
|
i = c["init_cluster"] if "init_cluster" in c else {}
|
|
|
|
app_ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "")
|
|
db_ctx = g.get("DB_CLUSTER_KUBECONTEXT") or e.get("DB_CLUSTER_KUBECONTEXT") or i.get("db_cluster_kubecontext", "")
|
|
return app_ctx, db_ctx
|
|
|
|
def get_ip(host, ctx):
|
|
if not ctx: return "pending"
|
|
# Try Ingress first
|
|
try:
|
|
out = subprocess.check_output(["kubectl", "--context", ctx, "get", "ingress", "-A", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
|
|
items = json.loads(out).get("items", [])
|
|
for item in items:
|
|
for rule in item.get("spec", {}).get("rules", []):
|
|
if rule.get("host") == host:
|
|
ing = item.get("status", {}).get("loadBalancer", {}).get("ingress", [])
|
|
if ing:
|
|
val = ing[0].get("ip") or ing[0].get("hostname")
|
|
if val: return val
|
|
except: pass
|
|
|
|
# Public APP endpoints must be sourced from Ingress status
|
|
# api.knoe.dev, git.knoe.dev, svc.knoe.dev, api.0.knoe.dev, db.0.knoe.dev
|
|
return "pending"
|
|
|
|
def check_supabase_pvcs(ctx):
|
|
if not ctx: return None
|
|
try:
|
|
out = subprocess.check_output(["kubectl", "--context", ctx, "get", "events", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
|
|
events = json.loads(out).get("items", [])
|
|
blocking_pvcs = {}
|
|
for event in events:
|
|
obj = event.get("involvedObject", {})
|
|
if obj.get("kind") == "PersistentVolumeClaim":
|
|
msg = event.get("message", "")
|
|
reason = event.get("reason", "")
|
|
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
|
|
if "exceeded" in msg.lower() or "quota" in msg.lower():
|
|
pvc_name = obj.get("name")
|
|
blocking_pvcs[pvc_name] = msg
|
|
if blocking_pvcs:
|
|
return blocking_pvcs
|
|
except: pass
|
|
return None
|
|
|
|
app_ctx, db_ctx = get_config(sys.argv[1])
|
|
|
|
# Targets we want to wait for if they are pending
|
|
wait_hosts = ["api.0.knoe.dev", "db.0.knoe.dev"]
|
|
timeout = 300 # 5 minutes
|
|
start = time.time()
|
|
|
|
while time.time() - start < timeout:
|
|
pvc_errors = check_supabase_pvcs(app_ctx)
|
|
if pvc_errors:
|
|
print("\nERROR: Supabase PVC provisioning failed (Storage Quota Exceeded).")
|
|
for pvc, msg in pvc_errors.items():
|
|
print(f" - PVC: {pvc}")
|
|
print(f" GKE Error: {msg}")
|
|
sys.exit(1)
|
|
|
|
pending = False
|
|
for h in wait_hosts:
|
|
if get_ip(h, app_ctx) == "pending":
|
|
pending = True
|
|
break
|
|
if not pending:
|
|
break
|
|
time.sleep(10)
|
|
|
|
if time.time() - start >= timeout:
|
|
print("\nERROR: Timeout waiting for Supabase ingress reconciliation.")
|
|
print("Ingress Controller Diagnosis (kubectl get events -n supabase):")
|
|
try:
|
|
subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False)
|
|
print("\nIngress Resource Status:")
|
|
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-kong", "-n", "supabase"], check=False)
|
|
subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-studio", "-n", "supabase"], check=False)
|
|
except:
|
|
pass
|
|
sys.exit(1)
|
|
|
|
endpoints = [
|
|
("api.knoe.dev", "knoe-svc-kong", "APP", app_ctx),
|
|
("git.knoe.dev", "GitLab", "APP", app_ctx),
|
|
("db.0.knoe.dev", "Supabase Studio", "APP", app_ctx),
|
|
("api.0.knoe.dev", "Supabase Kong", "APP", app_ctx),
|
|
("svc.knoe.dev", "Grafana", "APP", app_ctx),
|
|
]
|
|
|
|
print(f"{'hostname':<20} {'service':<20} {'cluster':<10} {'external IP':<15}")
|
|
print("-" * 70)
|
|
for host, svc, clus, ctx in endpoints:
|
|
ip = get_ip(host, ctx)
|
|
print(f"{host:<20} {svc:<20} {clus:<10} {ip:<15}")
|
|
PY
|