prole/deploy.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_PATH="${PROLE_DEPLOY_CFG:-${ROOT_DIR}/conf/gke.cfg}"

if [[ -x "${ROOT_DIR}/.venv/bin/python3" ]]; then
  PYTHON_BIN="${ROOT_DIR}/.venv/bin/python3"
elif [[ -x "${ROOT_DIR}/bin/python3" ]]; then
  PYTHON_BIN="${ROOT_DIR}/bin/python3"
else
  PYTHON_BIN="${PYTHON:-python3}"
fi

# Validate StorageClass on GKE before deployment
if [[ "$CONFIG_PATH" == *"gke.cfg"* ]]; then
  echo "Validating Supabase StorageClass configuration..."
  "${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
import sys
import subprocess
import json
import configparser

def die(msg):
    print(f"ERROR: {msg}", file=sys.stderr)
    sys.exit(1)

c = configparser.ConfigParser()
c.read(sys.argv[1])
g = c["Global"] if "Global" in c else {}
e = c["env_setup"] if "env_setup" in c else {}
i = c["init_cluster"] if "init_cluster" in c else {}

ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "")
if not ctx:
    sys.exit(0)

# Resolve StorageClass name
sc_name = g.get("SUPABASE_STORAGE_CLASS") or c.get("Supabase", "STORAGE_CLASS", fallback="supabase-standard")

try:
    out = subprocess.check_output(["kubectl", "--context", ctx, "get", "storageclass", sc_name, "-o", "json"], stderr=subprocess.DEVNULL, text=True)
    sc = json.loads(out)
    provisioner = sc.get("provisioner", "")
    params = sc.get("parameters", {})
    disk_type = params.get("type", "")
    
    if provisioner != "pd.csi.storage.gke.io":
        die(f"storage-class configuration error: StorageClass '{sc_name}' must use provisioner 'pd.csi.storage.gke.io', but uses '{provisioner}'.")
    
    if disk_type != "pd-standard":
        die(f"storage-class configuration error: StorageClass '{sc_name}' must use parameters.type 'pd-standard', but uses '{disk_type}'.")
    
    print(f"Verified live Supabase StorageClass '{sc_name}' (type: {disk_type}).", file=sys.stderr)
except subprocess.CalledProcessError:
    die(f"storage-class configuration error: Required StorageClass '{sc_name}' does not exist on cluster '{ctx}'.")
PY
fi

"${PYTHON_BIN}" -m prole.deploy_pipeline --config "${CONFIG_PATH}" "$@"

# Post-deploy summary
echo ""
echo "Public Endpoints:"
ROOT_DIR="${ROOT_DIR}" "${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
import sys
import os
import subprocess
import json
import configparser
import time

def get_config(path):
    c = configparser.ConfigParser()
    c.read(path)
    # Check [Global] then [env_setup] then [init_cluster] for contexts
    g = c["Global"] if "Global" in c else {}
    e = c["env_setup"] if "env_setup" in c else {}
    i = c["init_cluster"] if "init_cluster" in c else {}
    
    app_ctx = g.get("APP_CLUSTER_KUBECONTEXT") or e.get("APP_CLUSTER_KUBECONTEXT") or i.get("app_cluster_kubecontext", "")
    db_ctx = g.get("DB_CLUSTER_KUBECONTEXT") or e.get("DB_CLUSTER_KUBECONTEXT") or i.get("db_cluster_kubecontext", "")
    # GitLab namespace usually defaults to 'gitlab'
    gitlab_ns = g.get("GITLAB_NAMESPACE") or e.get("GITLAB_NAMESPACE") or i.get("gitlab_namespace", "gitlab")
    return app_ctx, db_ctx, gitlab_ns

def get_ip(host, ctx):
    if not ctx: return "pending"
    # Try Ingress first
    try:
        out = subprocess.check_output(["kubectl", "--context", ctx, "get", "ingress", "-A", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
        items = json.loads(out).get("items", [])
        for item in items:
            for rule in item.get("spec", {}).get("rules", []):
                if rule.get("host") == host:
                    ing = item.get("status", {}).get("loadBalancer", {}).get("ingress", [])
                    if ing:
                        val = ing[0].get("ip") or ing[0].get("hostname")
                        if val: return val
    except: pass
    
    return "pending"

def check_supabase_pvcs(ctx):
    if not ctx: return None
    try:
        from datetime import datetime, timezone
        out = subprocess.check_output(["kubectl", "--context", ctx, "get", "events", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
        events = json.loads(out).get("items", [])
        
        # Fetch current PVC states to ignore Bound ones
        out_pvc = subprocess.check_output(["kubectl", "--context", ctx, "get", "pvc", "-n", "supabase", "-o", "json"], stderr=subprocess.DEVNULL, text=True)
        pvcs = json.loads(out_pvc).get("items", [])
        pvc_status = {p.get("metadata", {}).get("name"): p.get("status", {}).get("phase") for p in pvcs}

        blocking_pvcs = {}
        for event in events:
            obj = event.get("involvedObject", {})
            if obj.get("kind") == "PersistentVolumeClaim":
                pvc_name = obj.get("name")
                
                # Skip Bound PVCs
                if pvc_status.get(pvc_name) == "Bound":
                    continue

                # Skip old events (> 10 mins)
                last_ts = event.get("lastTimestamp") or event.get("eventTime")
                if last_ts:
                    try:
                        ts = datetime.fromisoformat(last_ts.replace('Z', '+00:00'))
                        if (datetime.now(timezone.utc) - ts).total_seconds() > 600:
                            continue
                    except: pass

                msg = event.get("message", "")
                reason = event.get("reason", "")
                if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
                    if "exceeded" in msg.lower() or "quota" in msg.lower():
                        blocking_pvcs[pvc_name] = msg
        if blocking_pvcs:
            return blocking_pvcs
    except: pass
    return None

def reconcile_gitlab(ctx, ns):
    if not ctx: return
    print(f"\nReconciling GitLab workloads in namespace '{ns}'...")
    
    try:
        # Get all deployments in one go
        out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployments", "-o", "json"], text=True)
        deploys = json.loads(out).get("items", [])
        
        # Get all replicasets in one go
        out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "replicasets", "-o", "json"], text=True)
        all_rss = json.loads(out).get("items", [])
    except Exception as e:
        print(f"  Warning: Could not fetch GitLab workloads for reconciliation: {e}")
        return

    # Targets specifically mentioned in requirements or operator managed
    gitlab_workloads = [
        "gitlab-sidekiq-all-in-1-v2",
        "gitlab-gitlab-shell",
        "gitlab-kas",
        "gitlab-registry",
        "gitlab-webservice-default"
    ]
    
    summary = []
    over_deployed_detected = False
    
    for dep in deploys:
        name = dep['metadata']['name']
        # Filter for operator-managed GitLab workloads only
        is_gitlab = False
        if name in gitlab_workloads:
            is_gitlab = True
        elif name.startswith("gitlab-") and not any(x in name for x in ["-migration", "-secrets", "-minio-create-buckets", "-db-config"]):
            is_gitlab = True
        
        if not is_gitlab:
            continue
            
        spec = dep.get('spec', {})
        status = dep.get('status', {})
        
        # Target should be 1 for these specific GitLab workloads (Requirement 11)
        if name in gitlab_workloads:
            desired = 1
        else:
            desired = spec.get('replicas', 0)
            
        live = status.get('replicas', 0)
        updated = status.get('updatedReplicas', 0)
        
        # Enforce desired replica count (Requirement 3 & 4)
        # Reconciliation compares:
        # - desired replicas (from Helm values)
        # - actual replicas (from Deployment.spec.replicas)
        actual_spec_replicas = spec.get('replicas', 0)
        if name in gitlab_workloads and actual_spec_replicas != desired:
            print(f"  [FIX] Deployment '{name}' spec.replicas ({actual_spec_replicas}) differs from desired ({desired}).")
            over_deployed_detected = True

        summary.append({
            "name": name,
            "desired": desired,
            "live": live,
            "updated": updated
        })

    # Check for Gitaly storage class drift on GKE (Requirement 3)
    is_gke = "gke" in sys.argv[1].lower()
    if is_gke:
        try:
            out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "gitlab", "-o", "json"], text=True)
            crs = json.loads(out).get("items", [])
            for cr in crs:
                spec = cr.get("spec", {})
                live_sc = spec.get("chart", {}).get("values", {}).get("global", {}).get("persistence", {}).get("storageClass")
                if not live_sc:
                    live_sc = spec.get("chart", {}).get("values", {}).get("gitlab", {}).get("gitaly", {}).get("persistence", {}).get("storageClass")
                
                # Desired is 'standard' in this environment by default
                desired_sc = "standard"
                
                if live_sc:
                    # Normalize comparison: standard and standard-rwo are equivalent on GKE
                    gke_standard_variants = ["standard", "standard-rwo"]
                    if live_sc in gke_standard_variants and desired_sc in gke_standard_variants:
                        continue # Equivalent
                    
                    if live_sc != desired_sc:
                        print(f"  [FIX] GitLab CR '{cr['metadata']['name']}' storageClass '{live_sc}' differs from desired '{desired_sc}'.")
                        over_deployed_detected = True
        except:
            pass

    # Print summary (Requirement 8)
    print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8}")
    print("-" * 70)
    for s in summary:
        print(f"{s['name']:<35} {s['desired']:<8} {s['live']:<8} {s['updated']:<8}")

    if over_deployed_detected:
        print("\nEnforcing desired configuration by re-applying GitLab Helm release/CR...")
        # Re-apply the Helm release (Requirement 2 & 4)
        try:
            root_dir = os.environ.get("ROOT_DIR", ".")
            subprocess.run(["/usr/bin/env", "bash", f"{root_dir}/etc/init_gitlab.sh", "deploy"], check=True)
        except Exception as e:
            print(f"  ERROR: Failed to re-apply GitLab configuration: {e}")
            sys.exit(1)

    # Final Convergence Check (Requirement 6)
    still_over_deployed = False
    for s in summary:
        # If we detected drift, we re-applied the configuration.
        # Now we check if it converged.
        if over_deployed_detected:
            # Re-fetch live status for the final check
            try:
                out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployment", s['name'], "-o", "json"], text=True)
                d = json.loads(out)
                st = d.get('status', {})
                # Compare against the intended 'desired' count (1), not necessarily the (potentially stale) spec.replicas
                if st.get('replicas', 0) > s['desired'] or st.get('updatedReplicas', 0) != s['desired']:
                    print(f"ERROR: GitLab deployment '{s['name']}' failed to converge (live={st.get('replicas')}, desired={s['desired']}).")
                    still_over_deployed = True
            except:
                pass
        elif s['live'] > s['desired'] or s['updated'] != s['desired']:
             print(f"ERROR: GitLab deployment '{s['name']}' is not converged.")
             still_over_deployed = True

    if still_over_deployed:
        sys.exit(1)

app_ctx, db_ctx, gitlab_ns = get_config(sys.argv[1])

# Run GitLab reconciliation first
reconcile_gitlab(app_ctx, gitlab_ns)

# Targets we want to wait for if they are pending
wait_hosts = ["api.0.knoe.dev", "db.0.knoe.dev"]
timeout = 300 # 5 minutes
start = time.time()

while time.time() - start < timeout:
    pvc_errors = check_supabase_pvcs(app_ctx)
    if pvc_errors:
        print("\nERROR: Supabase PVC provisioning failed (Storage Quota Exceeded).")
        for pvc, msg in pvc_errors.items():
            print(f"  - PVC: {pvc}")
            print(f"    GKE Error: {msg}")
        sys.exit(1)

    pending = False
    for h in wait_hosts:
        if get_ip(h, app_ctx) == "pending":
            pending = True
            break
    if not pending:
        break
    time.sleep(10)

if time.time() - start >= timeout:
    print("\nERROR: Timeout waiting for Supabase ingress reconciliation.")
    print("Ingress Controller Diagnosis (kubectl get events -n supabase):")
    try:
        subprocess.run(["kubectl", "--context", app_ctx, "get", "events", "-n", "supabase", "--sort-by=.lastTimestamp"], check=False)
        print("\nIngress Resource Status:")
        subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-kong", "-n", "supabase"], check=False)
        subprocess.run(["kubectl", "--context", app_ctx, "describe", "ingress", "supabase-studio", "-n", "supabase"], check=False)
    except:
        pass
    sys.exit(1)

endpoints = [
    ("api.knoe.dev",   "knoe-svc-kong",   "APP", app_ctx),
    ("git.knoe.dev",   "GitLab",          "APP", app_ctx),
    ("db.0.knoe.dev",  "Supabase Studio", "APP", app_ctx),
    ("api.0.knoe.dev", "Supabase Kong",   "APP", app_ctx),
    ("svc.knoe.dev",   "Grafana",         "APP", app_ctx),
]

print(f"\n{'hostname':<20} {'service':<20} {'cluster':<10} {'external IP':<15}")
print("-" * 70)
for host, svc, clus, ctx in endpoints:
    ip = get_ip(host, ctx)
    print(f"{host:<20} {svc:<20} {clus:<10} {ip:<15}")
PY