prole/deploy.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_PATH="${PROLE_DEPLOY_CFG:-${ROOT_DIR}/conf/gke.cfg}"

if [[ -x "${ROOT_DIR}/.venv/bin/python3" ]]; then
  PYTHON_BIN="${ROOT_DIR}/.venv/bin/python3"
elif [[ -x "${ROOT_DIR}/bin/python3" ]]; then
  PYTHON_BIN="${ROOT_DIR}/bin/python3"
else
  PYTHON_BIN="${PYTHON:-python3}"
fi

# Validate StorageClass on GKE before deployment
if [[ "$CONFIG_PATH" == *"gke.cfg"* ]]; then
  echo "Validating Supabase StorageClass configuration..."
  "${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
import sys
import subprocess
import json
import configparser

def die(msg):
    print(f"ERROR: {msg}", file=sys.stderr)
    sys.exit(1)

c = configparser.ConfigParser()
c.read(sys.argv[1])
g = c["Global"] if "Global" in c else {}
e = c["env_setup"] if "env_setup" in c else {}
i = c["init_cluster"] if "init_cluster" in c else {}

ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "")
if not ctx:
    sys.exit(0)

# Resolve StorageClass name
sc_name = g.get("SUPABASE_STORAGE_CLASS") or c.get("Supabase", "STORAGE_CLASS", fallback="supabase-standard")

try:
    out = subprocess.check_output(["kubectl", "--context", ctx, "get", "storageclass", sc_name, "-o", "json"], stderr=subprocess.DEVNULL, text=True)
    sc = json.loads(out)
    provisioner = sc.get("provisioner", "")
    params = sc.get("parameters", {})
    disk_type = params.get("type", "")
    
    if provisioner != "pd.csi.storage.gke.io":
        die(f"storage-class configuration error: StorageClass '{sc_name}' must use provisioner 'pd.csi.storage.gke.io', but uses '{provisioner}'.")
    
    if disk_type != "pd-standard":
        die(f"storage-class configuration error: StorageClass '{sc_name}' must use parameters.type 'pd-standard', but uses '{disk_type}'.")
    
    print(f"Verified live Supabase StorageClass '{sc_name}' (type: {disk_type}).", file=sys.stderr)
except subprocess.CalledProcessError:
    die(f"storage-class configuration error: Required StorageClass '{sc_name}' does not exist on cluster '{ctx}'.")
PY
fi

"${PYTHON_BIN}" -m prole.deploy_pipeline --config "${CONFIG_PATH}" "$@"

# Post-deploy summary
echo ""
echo "Public Endpoints:"
"${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
import sys
import subprocess
import json
import configparser
import time

def get_config(path):
    c = configparser.ConfigParser()
    c.read(path)
    # Check [Global] then [env_setup] then [init_cluster] for contexts
    g = c["Global"] if "Global" in c else {}
    e = c["env_setup"] if "env_setup" in c else {}
    i = c["init_cluster"] if "init_cluster" in c else {}
    
    app_ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "")
    db_ctx = g.get("DB_CLUSTER_KUBECONTEXT") or e.get("DB_CLUSTER_KUBECONTEXT") or i.get("db_cluster_kubecontext", "")
    # GitLab namespace usually defaults to 'gitlab'
    gitlab_ns = g.get("GITLAB_NAMESPACE") or e.get("GITLAB_NAMESPACE") or i.get("gitlab_namespace", "gitlab")
    return app_ctx, db_ctx, gitlab_ns

def get_ip(host, ctx):
    if not ctx: return "pending"
    # Try Ingress first
    try:
        out = subprocess.check_output(["kubectl", "--context", ctx, "get", "ingress", "-A", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
        items = json.loads(out).get("items", [])
        for item in items:
            for rule in item.get("spec", {}).get("rules", []):
                if rule.get("host") == host:
                    ing = item.get("status", {}).get("loadBalancer", {}).get("ingress", [])
                    if ing:
                        val = ing[0].get("ip") or ing[0].get("hostname")
                        if val: return val
    except: pass
    
    return "pending"

def check_supabase_pvcs(ctx):
    if not ctx: return None
    try:
        from datetime import datetime, timezone
        out = subprocess.check_output(["kubectl", "--context", ctx, "get", "events", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
        events = json.loads(out).get("items", [])
        
        # Fetch current PVC states to ignore Bound ones
        out_pvc = subprocess.check_output(["kubectl", "--context", ctx, "get", "pvc", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
        pvcs = json.loads(out_pvc).get("items", [])
        pvc_status = {p.get("metadata", {}).get("name"): p.get("status", {}).get("phase") for p in pvcs}

        blocking_pvcs = {}
        for event in events:
            obj = event.get("involvedObject", {})
            if obj.get("kind") == "PersistentVolumeClaim":
                pvc_name = obj.get("name")
                
                # Skip Bound PVCs
                if pvc_status.get(pvc_name) == "Bound":
                    continue

                # Skip old events (> 10 mins)
                last_ts = event.get("lastTimestamp") or event.get("eventTime")
                if last_ts:
                    try:
                        ts = datetime.fromisoformat(last_ts.replace('Z', '+00:00'))
                        if (datetime.now(timezone.utc) - ts).total_seconds() > 600:
                            continue
                    except: pass

                msg = event.get("message", "")
                reason = event.get("reason", "")
                if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
                    if "exceeded" in msg.lower() or "quota" in msg.lower():
                        blocking_pvcs[pvc_name] = msg
        if blocking_pvcs:
            return blocking_pvcs
    except: pass
    return None

def reconcile_gitlab(ctx, ns):
    if not ctx: return
    print(f"\nReconciling GitLab workloads in namespace '{ns}'...")
    
    try:
        # Get all deployments in one go
        out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployments", "-o", "json"], text=True)
        deploys = json.loads(out).get("items", [])
        
        # Get all replicasets in one go
        out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "replicasets", "-o", "json"], text=True)
        all_rss = json.loads(out).get("items", [])
    except Exception as e:
        print(f"  Warning: Could not fetch GitLab workloads for reconciliation: {e}")
        return

    # Targets specifically mentioned in requirements or operator managed
    gitlab_workloads = [
        "gitlab-sidekiq-all-in-1-v2",
        "gitlab-gitlab-shell",
        "gitlab-kas",
        "gitlab-registry",
        "gitlab-webservice-default"
    ]
    
    summary = []
    over_deployed_detected = False
    
    for dep in deploys:
        name = dep['metadata']['name']
        # Filter for operator-managed GitLab workloads only (Requirement 7)
        is_gitlab = False
        if name in gitlab_workloads:
            is_gitlab = True
        elif name.startswith("gitlab-") and not any(x in name for x in ["-migration", "-secrets", "-minio-create-buckets", "-db-config"]):
            is_gitlab = True
        
        if not is_gitlab:
            continue
            
        spec = dep.get('spec', {})
        status = dep.get('status', {})
        
        # Target should be 1 for these specific GitLab workloads in this environment (Requirement 11)
        # Reading from spec.replicas can drift from the intended target.
        if name in gitlab_workloads:
            desired = 1
        else:
            desired = spec.get('replicas', 0)
            
        live = status.get('replicas', 0)
        updated = status.get('updatedReplicas', 0)
        
        # Identify RS owned by this deployment (Requirement 4)
        dep_uid = dep['metadata']['uid']
        my_rss = [rs for rs in all_rss if any(o.get('uid') == dep_uid for o in rs['metadata'].get('ownerReferences', []))]
        
        current_revision = dep['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision')
        
        stale_rss_with_pods = []
        for rs in my_rss:
            rev = rs['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision')
            rs_replicas = rs.get('status', {}).get('replicas', 0)
            if rev != current_revision and rs_replicas > 0:
                stale_rss_with_pods.append(rs)

        # Forced reconciliation if over-deployed (Requirement 3 & 5)
        if (live > desired or stale_rss_with_pods) and stale_rss_with_pods:
            print(f"  [FIX] Deployment '{name}' is over-deployed (live={live}, desired={desired}). Scaling stale ReplicaSets to 0...")
            for rs in stale_rss_with_pods:
                rs_name = rs['metadata']['name']
                print(f"    - Scaling stale RS '{rs_name}' down from {rs.get('status', {}).get('replicas')} to 0")
                try:
                    subprocess.run(["kubectl", "--context", ctx, "-n", ns, "scale", "rs", rs_name, "--replicas=0"], check=True)
                except:
                    print(f"      FAILED to scale RS {rs_name}")
            over_deployed_detected = True

        summary.append({
            "name": name,
            "desired": desired,
            "live": live,
            "updated": updated,
            "stale_rs": [rs['metadata']['name'] for rs in stale_rss_with_pods]
        })

    # Print summary (Requirement 8)
    print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8} {'Stale RS'}")
    print("-" * 85)
    for s in summary:
        stale_str = ",".join(s['stale_rs']) if s['stale_rs'] else "none"
        print(f"{s['name']:<35} {s['desired']:<8} {s['live']:<8} {s['updated']:<8} {stale_str}")

    # Final Convergence Check (Requirement 6)
    still_over_deployed = False
    for s in summary:
        # If we scaled them down, they SHOULD be 0 now, but let's re-verify from current state
        # For simplicity, if we detect over-deployment in the initial check, we already know it was bad.
        # Requirement says "Fail clearly if GitLab IS still over-deployed" after the reconciliation pass.
        if over_deployed_detected:
            # Re-fetch live status for the final check
            try:
                out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployment", s['name'], "-o", "json"], text=True)
                d = json.loads(out)
                st = d.get('status', {})
                if st.get('replicas', 0) > d.get('spec', {}).get('replicas', 0) or st.get('updatedReplicas', 0) != d.get('spec', {}).get('replicas', 0):
                    print(f"ERROR: GitLab deployment '{s['name']}' failed to converge.")
                    still_over_deployed = True
            except:
                pass
        elif s['live'] > s['desired'] or s['updated'] != s['desired']:
             print(f"ERROR: GitLab deployment '{s['name']}' is not converged.")
             still_over_deployed = True

    if still_over_deployed:
        sys.exit(1)

app_ctx, db_ctx, gitlab_ns = get_config(sys.argv[1])

# Run GitLab reconciliation first
reconcile_gitlab(app_ctx, gitlab_ns)

# Targets we want to wait for if they are pending
wait_hosts = ["api.0.knoe.dev", "db.0.knoe.dev"]
timeout = 300 # 5 minutes
start = time.time()

while time.time() - start < timeout:
    pvc_errors = check_supabase_pvcs(app_ctx)
    if pvc_errors:
        print("\nERROR: Supabase PVC provisioning failed (Storage Quota Exceeded).")
        for pvc, msg in pvc_errors.items():
            print(f"  - PVC: {pvc}")
            print(f"    GKE Error: {msg}")
        sys.exit(1)

    pending = False
    for h in wait_hosts:
        if get_ip(h, app_ctx) == "pending":
            pending = True
            break
    if not pending:
        break
    time.sleep(10)

if time.time() - start >= timeout:
    print("\nERROR: Timeout waiting for Supabase ingress reconciliation.")
    print("Ingress Controller Diagnosis (kubectl get events -n supabase):")
    try:
        subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False)
        print("\nIngress Resource Status:")
        subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-kong", "-n", "supabase"], check=False)
        subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-studio", "-n", "supabase"], check=False)
    except:
        pass
    sys.exit(1)

endpoints = [
    ("api.knoe.dev",   "knoe-svc-kong",   "APP", app_ctx),
    ("git.knoe.dev",   "GitLab",          "APP", app_ctx),
    ("db.0.knoe.dev",  "Supabase Studio", "APP", app_ctx),
    ("api.0.knoe.dev", "Supabase Kong",   "APP", app_ctx),
    ("svc.knoe.dev",   "Grafana",         "APP", app_ctx),
]

print(f"\n{'hostname':<20} {'service':<20} {'cluster':<10} {'external IP':<15}")
print("-" * 70)
for host, svc, clus, ctx in endpoints:
    ip = get_ip(host, ctx)
    print(f"{host:<20} {svc:<20} {clus:<10} {ip:<15}")
PY