#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CONFIG_PATH="${PROLE_DEPLOY_CFG:-${ROOT_DIR}/conf/gke.cfg}" if [[ -x "${ROOT_DIR}/.venv/bin/python3" ]]; then PYTHON_BIN="${ROOT_DIR}/.venv/bin/python3" elif [[ -x "${ROOT_DIR}/bin/python3" ]]; then PYTHON_BIN="${ROOT_DIR}/bin/python3" else PYTHON_BIN="${PYTHON:-python3}" fi # Validate StorageClass on GKE before deployment if [[ "$CONFIG_PATH" == *"gke.cfg"* ]]; then echo "Validating Supabase StorageClass configuration..." "${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY' import sys import subprocess import json import configparser def die(msg): print(f"ERROR: {msg}", file=sys.stderr) sys.exit(1) c = configparser.ConfigParser() c.read(sys.argv[1]) g = c["Global"] if "Global" in c else {} e = c["env_setup"] if "env_setup" in c else {} i = c["init_cluster"] if "init_cluster" in c else {} ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "") if not ctx: sys.exit(0) # Resolve StorageClass name sc_name = g.get("SUPABASE_STORAGE_CLASS") or c.get("Supabase", "STORAGE_CLASS", fallback="supabase-standard") try: out = subprocess.check_output(["kubectl", "--context", ctx, "get", "storageclass", sc_name, "-o", "json"], stderr=subprocess.DEVNULL, text=True) sc = json.loads(out) provisioner = sc.get("provisioner", "") params = sc.get("parameters", {}) disk_type = params.get("type", "") if provisioner != "pd.csi.storage.gke.io": die(f"storage-class configuration error: StorageClass '{sc_name}' must use provisioner 'pd.csi.storage.gke.io', but uses '{provisioner}'.") if disk_type != "pd-standard": die(f"storage-class configuration error: StorageClass '{sc_name}' must use parameters.type 'pd-standard', but uses '{disk_type}'.") print(f"Verified live Supabase StorageClass '{sc_name}' (type: {disk_type}).", file=sys.stderr) except subprocess.CalledProcessError: die(f"storage-class configuration error: Required StorageClass '{sc_name}' does not exist on cluster '{ctx}'.") PY fi "${PYTHON_BIN}" -m prole.deploy_pipeline --config "${CONFIG_PATH}" "$@" # Post-deploy summary echo "" echo "Public Endpoints:" "${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY' import sys import subprocess import json import configparser import time def get_config(path): c = configparser.ConfigParser() c.read(path) # Check [Global] then [env_setup] then [init_cluster] for contexts g = c["Global"] if "Global" in c else {} e = c["env_setup"] if "env_setup" in c else {} i = c["init_cluster"] if "init_cluster" in c else {} app_ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "") db_ctx = g.get("DB_CLUSTER_KUBECONTEXT") or e.get("DB_CLUSTER_KUBECONTEXT") or i.get("db_cluster_kubecontext", "") # GitLab namespace usually defaults to 'gitlab' gitlab_ns = g.get("GITLAB_NAMESPACE") or e.get("GITLAB_NAMESPACE") or i.get("gitlab_namespace", "gitlab") return app_ctx, db_ctx, gitlab_ns def get_ip(host, ctx): if not ctx: return "pending" # Try Ingress first try: out = subprocess.check_output(["kubectl", "--context", ctx, "get", "ingress", "-A", "-o", "json"], stderr=subprocess.DEVNULL, text=True) items = json.loads(out).get("items", []) for item in items: for rule in item.get("spec", {}).get("rules", []): if rule.get("host") == host: ing = item.get("status", {}).get("loadBalancer", {}).get("ingress", []) if ing: val = ing[0].get("ip") or ing[0].get("hostname") if val: return val except: pass return "pending" def check_supabase_pvcs(ctx): if not ctx: return None try: from datetime import datetime, timezone out = subprocess.check_output(["kubectl", "--context", ctx, "get", "events", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True) events = json.loads(out).get("items", []) # Fetch current PVC states to ignore Bound ones out_pvc = subprocess.check_output(["kubectl", "--context", ctx, "get", "pvc", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True) pvcs = json.loads(out_pvc).get("items", []) pvc_status = {p.get("metadata", {}).get("name"): p.get("status", {}).get("phase") for p in pvcs} blocking_pvcs = {} for event in events: obj = event.get("involvedObject", {}) if obj.get("kind") == "PersistentVolumeClaim": pvc_name = obj.get("name") # Skip Bound PVCs if pvc_status.get(pvc_name) == "Bound": continue # Skip old events (> 10 mins) last_ts = event.get("lastTimestamp") or event.get("eventTime") if last_ts: try: ts = datetime.fromisoformat(last_ts.replace('Z', '+00:00')) if (datetime.now(timezone.utc) - ts).total_seconds() > 600: continue except: pass msg = event.get("message", "") reason = event.get("reason", "") if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason: if "exceeded" in msg.lower() or "quota" in msg.lower(): blocking_pvcs[pvc_name] = msg if blocking_pvcs: return blocking_pvcs except: pass return None def reconcile_gitlab(ctx, ns): if not ctx: return print(f"\nReconciling GitLab workloads in namespace '{ns}'...") try: # Get all deployments in one go out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployments", "-o", "json"], text=True) deploys = json.loads(out).get("items", []) # Get all replicasets in one go out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "replicasets", "-o", "json"], text=True) all_rss = json.loads(out).get("items", []) except Exception as e: print(f" Warning: Could not fetch GitLab workloads for reconciliation: {e}") return # Targets specifically mentioned in requirements or operator managed gitlab_workloads = [ "gitlab-sidekiq-all-in-1-v2", "gitlab-gitlab-shell", "gitlab-kas", "gitlab-registry", "gitlab-webservice-default" ] summary = [] over_deployed_detected = False for dep in deploys: name = dep['metadata']['name'] # Filter for operator-managed GitLab workloads only (Requirement 7) is_gitlab = False if name in gitlab_workloads: is_gitlab = True elif name.startswith("gitlab-") and not any(x in name for x in ["-migration", "-secrets", "-minio-create-buckets", "-db-config"]): is_gitlab = True if not is_gitlab: continue spec = dep.get('spec', {}) status = dep.get('status', {}) # Target should be 1 for these specific GitLab workloads in this environment (Requirement 11) # Reading from spec.replicas can drift from the intended target. if name in gitlab_workloads: desired = 1 else: desired = spec.get('replicas', 0) live = status.get('replicas', 0) updated = status.get('updatedReplicas', 0) # Identify RS owned by this deployment (Requirement 4) dep_uid = dep['metadata']['uid'] my_rss = [rs for rs in all_rss if any(o.get('uid') == dep_uid for o in rs['metadata'].get('ownerReferences', []))] current_revision = dep['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision') stale_rss_with_pods = [] for rs in my_rss: rev = rs['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision') rs_replicas = rs.get('status', {}).get('replicas', 0) if rev != current_revision and rs_replicas > 0: stale_rss_with_pods.append(rs) # Forced reconciliation if over-deployed (Requirement 3 & 5) if (live > desired or stale_rss_with_pods) and stale_rss_with_pods: print(f" [FIX] Deployment '{name}' is over-deployed (live={live}, desired={desired}). Scaling stale ReplicaSets to 0...") for rs in stale_rss_with_pods: rs_name = rs['metadata']['name'] print(f" - Scaling stale RS '{rs_name}' down from {rs.get('status', {}).get('replicas')} to 0") try: subprocess.run(["kubectl", "--context", ctx, "-n", ns, "scale", "rs", rs_name, "--replicas=0"], check=True) except: print(f" FAILED to scale RS {rs_name}") over_deployed_detected = True summary.append({ "name": name, "desired": desired, "live": live, "updated": updated, "stale_rs": [rs['metadata']['name'] for rs in stale_rss_with_pods] }) # Print summary (Requirement 8) print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8} {'Stale RS'}") print("-" * 85) for s in summary: stale_str = ",".join(s['stale_rs']) if s['stale_rs'] else "none" print(f"{s['name']:<35} {s['desired']:<8} {s['live']:<8} {s['updated']:<8} {stale_str}") # Final Convergence Check (Requirement 6) still_over_deployed = False for s in summary: # If we scaled them down, they SHOULD be 0 now, but let's re-verify from current state # For simplicity, if we detect over-deployment in the initial check, we already know it was bad. # Requirement says "Fail clearly if GitLab IS still over-deployed" after the reconciliation pass. if over_deployed_detected: # Re-fetch live status for the final check try: out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployment", s['name'], "-o", "json"], text=True) d = json.loads(out) st = d.get('status', {}) if st.get('replicas', 0) > d.get('spec', {}).get('replicas', 0) or st.get('updatedReplicas', 0) != d.get('spec', {}).get('replicas', 0): print(f"ERROR: GitLab deployment '{s['name']}' failed to converge.") still_over_deployed = True except: pass elif s['live'] > s['desired'] or s['updated'] != s['desired']: print(f"ERROR: GitLab deployment '{s['name']}' is not converged.") still_over_deployed = True if still_over_deployed: sys.exit(1) app_ctx, db_ctx, gitlab_ns = get_config(sys.argv[1]) # Run GitLab reconciliation first reconcile_gitlab(app_ctx, gitlab_ns) # Targets we want to wait for if they are pending wait_hosts = ["api.0.knoe.dev", "db.0.knoe.dev"] timeout = 300 # 5 minutes start = time.time() while time.time() - start < timeout: pvc_errors = check_supabase_pvcs(app_ctx) if pvc_errors: print("\nERROR: Supabase PVC provisioning failed (Storage Quota Exceeded).") for pvc, msg in pvc_errors.items(): print(f" - PVC: {pvc}") print(f" GKE Error: {msg}") sys.exit(1) pending = False for h in wait_hosts: if get_ip(h, app_ctx) == "pending": pending = True break if not pending: break time.sleep(10) if time.time() - start >= timeout: print("\nERROR: Timeout waiting for Supabase ingress reconciliation.") print("Ingress Controller Diagnosis (kubectl get events -n supabase):") try: subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False) print("\nIngress Resource Status:") subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-kong", "-n", "supabase"], check=False) subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-studio", "-n", "supabase"], check=False) except: pass sys.exit(1) endpoints = [ ("api.knoe.dev", "knoe-svc-kong", "APP", app_ctx), ("git.knoe.dev", "GitLab", "APP", app_ctx), ("db.0.knoe.dev", "Supabase Studio", "APP", app_ctx), ("api.0.knoe.dev", "Supabase Kong", "APP", app_ctx), ("svc.knoe.dev", "Grafana", "APP", app_ctx), ] print(f"\n{'hostname':<20} {'service':<20} {'cluster':<10} {'external IP':<15}") print("-" * 70) for host, svc, clus, ctx in endpoints: ip = get_ip(host, ctx) print(f"{host:<20} {svc:<20} {clus:<10} {ip:<15}") PY