From 391c4f5fc968494161ab725aa6b8d80d88f9e066 Mon Sep 17 00:00:00 2001 From: chrisfu Date: Tue, 21 Apr 2026 10:31:06 -0700 Subject: [PATCH] fix(net): BackendConfig healthchecks + externally-managed supabase-kong ingress GCE L7 Ingresses for knoe-svc-kong (svc.knoe.dev / api.knoe.dev), supabase-kong (api.0.knoe.dev) and supabase-studio (db.0.knoe.dev) were all stuck UNHEALTHY: the default GCE healthCheck is HTTP GET `/` on the backend port, but Kong returns 404 on any unrouted path and Studio returns a 301 redirect -- neither passes the default probe, so the LB serves "Server Error" instead of reaching the pod. Replicate the pattern already working for gitlab-webservice-default: emit a BackendConfig CRD with a TCP healthCheck on the service port and annotate the Service with cloud.google.com/backend-config so GCE picks it up. TCP is sufficient for LB-level liveness -- the backend is "alive" as long as the process is accepting connections. - etc/init_kong.sh: new SVC_KNOE_BACKEND_CONFIG_NAME; apply BackendConfig inside k8s/GCE branch; annotate Service post-apply. - knoe-supabase chart: new kong/backendconfig.yaml + studio/backendconfig.yaml (TCP on 8000 / 3000), gated on service.{kong,studio}.backendConfigName. - knoe-supabase chart: kong/service.yaml + studio/service.yaml pick up cloud.google.com/backend-config when backendConfigName is set. - render_supabase.py: sets service.{kong,studio}.backendConfigName in k8s mode so the above wires up automatically. Separately, the chart-managed supabase-kong Ingress was being reaped from the cluster seconds after helm install (manifest present in the release, gone via `kubectl get`). Root cause TBD -- suspected meta.helm.sh/* annotation ownership colliding with a GKE/Anthos audit controller. Workaround: render_supabase.py now emits a standalone public-ingress-kong.yaml (no helm metadata) that supabase/deploy.sh applies alongside public-ingress-tls.yaml, and the chart template gains an `ingress.externallyManaged` guard so it no-ops in k8s mode. Default `false` keeps k3d/k3s behavior unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- etc/init_kong.sh | 46 ++++++++++ supabase/deploy.sh | 32 +++++++ .../templates/kong/backendconfig.yaml | 34 +++++++ .../knoe-supabase/templates/kong/ingress.yaml | 12 +++ .../knoe-supabase/templates/kong/service.yaml | 11 +++ .../templates/studio/backendconfig.yaml | 28 ++++++ .../templates/studio/service.yaml | 11 +++ supabase/helm/render_supabase.py | 88 +++++++++++++++++++ 8 files changed, 262 insertions(+) create mode 100644 supabase/helm/knoe-supabase/templates/kong/backendconfig.yaml create mode 100644 supabase/helm/knoe-supabase/templates/studio/backendconfig.yaml diff --git a/etc/init_kong.sh b/etc/init_kong.sh index a6feeeb..bac2b82 100755 --- a/etc/init_kong.sh +++ b/etc/init_kong.sh @@ -153,6 +153,15 @@ SERVICE_TLS_CLUSTER_ISSUER="${SERVICE_TLS_CLUSTER_ISSUER:-letsencrypt-prod}" # ingress delete/recreate. Leave blank to let GCE assign ephemerally. SVC_KNOE_GLOBAL_STATIC_IP_NAME="${SVC_KNOE_GLOBAL_STATIC_IP_NAME:-}" +# BackendConfig name for the knoe-svc-kong Service. GCE's default L7 health +# check hits HTTP `/` on the backend port, which Kong responds to with 404 +# (no route) -- marking the backend UNHEALTHY and causing the LB to return +# "Server Error" instead of reaching Kong. We instead point the GCE LB at a +# TCP health check (port-level liveness) so the backend passes as long as +# Kong is accepting connections, which is sufficient for our traffic shape. +# Mirrors the pattern in etc/init_gitlab.sh (gitlab-webservice-backendconfig). +SVC_KNOE_BACKEND_CONFIG_NAME="${SVC_KNOE_BACKEND_CONFIG_NAME:-knoe-svc-kong-backendconfig}" + # Legacy: svc-check used to own svc.prole.org. We now route the service hostname # to Grafana, so remove any leftover svc-check resources to avoid conflicts. SVC_CHECK_NAMESPACE="${SVC_CHECK_NAMESPACE:-svc-check}" @@ -596,6 +605,31 @@ EOF return 1 fi echo "Confirmed: ManagedCertificate/${service_managed_cert_name} and FrontendConfig/${service_frontend_config_name} exist in ns=${NAMESPACE}." + + # BackendConfig: GCE default healthCheck is HTTP GET / on the backend port + # and Kong returns 404 on an unrouted path, so the backend never goes + # HEALTHY. A TCP healthCheck on the proxy port is sufficient for our case + # (backend is alive as long as Kong is accepting connections). Service is + # annotated below with cloud.google.com/backend-config so GCE picks this up. + echo "Reconciling BackendConfig (${SVC_KNOE_BACKEND_CONFIG_NAME}) for ${KONG_NAME} in ns=${NAMESPACE} ..." + kubectl apply -f - <&1) echo "$svc_out" + # In k8s/GCE mode, annotate the Service so GCE LB picks up the BackendConfig + # with the TCP health check. Matches etc/init_gitlab.sh's pattern for + # gitlab-webservice-default. Additive annotation; survives manifest + # re-applies (the YAML in deploy/opentofu/ doesn't set it). + if [[ "${PROLE_MODE:-}" == "k8s" ]]; then + echo "Annotating Service ${KONG_NAME} with cloud.google.com/backend-config=${SVC_KNOE_BACKEND_CONFIG_NAME}..." + kubectl -n "$NAMESPACE" annotate svc "$KONG_NAME" \ + "cloud.google.com/backend-config={\"default\":\"${SVC_KNOE_BACKEND_CONFIG_NAME}\"}" \ + --overwrite >/dev/null || \ + echo "WARN: Failed to annotate ${KONG_NAME} with backend-config; GCE LB will fall back to default healthcheck (likely UNHEALTHY)." >&2 + fi + echo "Waiting for $KONG_NAME rollout ..." kubectl rollout status deployment/"$KONG_NAME" -n "$NAMESPACE" --timeout=120s diff --git a/supabase/deploy.sh b/supabase/deploy.sh index 2244090..067473d 100755 --- a/supabase/deploy.sh +++ b/supabase/deploy.sh @@ -817,6 +817,33 @@ apply_supabase_public_ingress_tls_resources() { fi } +# Apply the standalone supabase-kong Ingress (api.0.knoe.dev) in k8s/GKE mode. +# render_supabase.py emits this manifest outside the helm chart because the +# chart-rendered ingress kept getting reaped from the cluster minutes after +# helm install. Gated by the chart's `ingress.externallyManaged` flag which +# render_supabase.py sets to true in k8s mode, so the chart no-ops +# kong/ingress.yaml and this standalone manifest is the single source of +# truth for the api.0 ingress. +apply_supabase_public_ingress_kong_resource() { + local kube_context="$1" + local namespace="$2" + local manifest_path="$3" + + if [[ -z "$manifest_path" || ! -f "$manifest_path" ]]; then + return 0 + fi + + local ctx_args=() + if [[ -n "$kube_context" ]]; then + ctx_args+=(--context "$kube_context") + fi + + log "Applying standalone supabase-kong Ingress from '$manifest_path'..." + if ! kubectl "${ctx_args[@]}" -n "$namespace" apply -f "$manifest_path"; then + die "Failed applying standalone supabase-kong Ingress from '$manifest_path'." + fi +} + apply_defaults() { DEV_HOME="${DEV_HOME:-$HOME/dev}" DEV_HOME="${DEV_HOME/#\~/$HOME}" @@ -2949,13 +2976,16 @@ print(val)" 2>/dev/null || true) local values="$PROJECT_ROOT/supabase/helm/generated/values.generated.json" local db_frontdoor_manifest_path="" local public_ingress_tls_manifest_path="" + local public_ingress_kong_manifest_path="" local ns="supabase" ns="$(load_manifest_summary_path supabase_namespace)" ns="${ns:-supabase}" db_frontdoor_manifest_path="$(load_manifest_summary_path manifests_frontdoor_db)" public_ingress_tls_manifest_path="$(load_manifest_summary_path manifests_public_ingress_tls)" + public_ingress_kong_manifest_path="$(load_manifest_summary_path manifests_public_ingress_kong)" apply_supabase_public_ingress_tls_resources "" "$ns" "$public_ingress_tls_manifest_path" + apply_supabase_public_ingress_kong_resource "" "$ns" "$public_ingress_kong_manifest_path" # Pre-apply storage preflight: rendered PVC-bearing components must all use # the resolved storage class source-of-truth. @@ -3093,6 +3123,8 @@ print((data.get("info") or {}).get("status") or "unknown")' || true)" ensure_k8s_supabase_static_pvs "$storage_class" public_ingress_tls_manifest_path="$(load_manifest_summary_path manifests_public_ingress_tls)" apply_supabase_public_ingress_tls_resources "" "$ns" "$public_ingress_tls_manifest_path" + public_ingress_kong_manifest_path="$(load_manifest_summary_path manifests_public_ingress_kong)" + apply_supabase_public_ingress_kong_resource "" "$ns" "$public_ingress_kong_manifest_path" fi check_supabase_retained_pv_blocked "$ns" diff --git a/supabase/helm/knoe-supabase/templates/kong/backendconfig.yaml b/supabase/helm/knoe-supabase/templates/kong/backendconfig.yaml new file mode 100644 index 0000000..553d2a7 --- /dev/null +++ b/supabase/helm/knoe-supabase/templates/kong/backendconfig.yaml @@ -0,0 +1,34 @@ +{{- /* +GCE L7 BackendConfig for supabase-kong. + +GCE's default Ingress healthCheck is HTTP GET `/` on the backend port; Kong +returns 404 on any unrouted path, so the backend never goes HEALTHY and the +LB serves "Server Error" instead of reaching the proxy. A TCP healthCheck on +the Kong proxy port is sufficient for our deployment shape -- the backend is +"alive" as long as Kong is accepting connections, and GCE only probes for +reachability; per-route health is not required here. + +The companion Service template (kong/service.yaml) annotates the Service +with cloud.google.com/backend-config so GCE picks this up. + +Only rendered when .Values.service.kong.backendConfigName is set +(render_supabase.py populates it in k8s/GKE mode). +*/ -}} +{{- if and .Values.deployment.kong.enabled (.Values.service.kong.backendConfigName | default "") -}} +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: {{ .Values.service.kong.backendConfigName | quote }} + labels: + {{- include "supabase.labels" . | nindent 4 }} +spec: + healthCheck: + type: TCP + port: {{ .Values.service.kong.port | default 8000 }} + checkIntervalSec: 15 + timeoutSec: 5 + healthyThreshold: 1 + unhealthyThreshold: 3 + connectionDraining: + drainingTimeoutSec: 30 +{{- end }} diff --git a/supabase/helm/knoe-supabase/templates/kong/ingress.yaml b/supabase/helm/knoe-supabase/templates/kong/ingress.yaml index 8384088..f074db0 100644 --- a/supabase/helm/knoe-supabase/templates/kong/ingress.yaml +++ b/supabase/helm/knoe-supabase/templates/kong/ingress.yaml @@ -1,5 +1,16 @@ +{{- /* +In k8s/GKE mode the supabase-kong Ingress is owned by render_supabase.py as +a standalone manifest (supabase/k8s/public-ingress-kong.yaml) applied by +deploy.sh outside the helm chart. The chart-rendered ingress was getting +silently reaped from the cluster within seconds of install; root cause is +suspected to involve meta.helm.sh/* annotation ownership colliding with a +GKE/Anthos audit controller, but diagnosis is deferred. externallyManaged +defaults to false so `helm install` in local k3d/k3s modes (where +render_supabase.py is not in the path) keeps the historical behavior. +*/ -}} {{- if .Values.deployment.kong.enabled -}} {{- if .Values.ingress.enabled -}} +{{- if not (.Values.ingress.externallyManaged | default false) -}} {{- $fullName := include "supabase.kong.fullname" . -}} {{- $svcPort := .Values.service.kong.port -}} {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} @@ -55,3 +66,4 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/supabase/helm/knoe-supabase/templates/kong/service.yaml b/supabase/helm/knoe-supabase/templates/kong/service.yaml index ec1120b..6fd2d4e 100644 --- a/supabase/helm/knoe-supabase/templates/kong/service.yaml +++ b/supabase/helm/knoe-supabase/templates/kong/service.yaml @@ -5,6 +5,17 @@ metadata: name: {{ include "supabase.kong.fullname" . }} labels: {{- include "supabase.labels" . | nindent 4 }} + {{- $backendConfigName := .Values.service.kong.backendConfigName | default "" }} + {{- $extraAnnotations := .Values.service.kong.annotations | default dict }} + {{- if or $backendConfigName $extraAnnotations }} + annotations: + {{- if $backendConfigName }} + cloud.google.com/backend-config: '{"default":"{{ $backendConfigName }}"}' + {{- end }} + {{- with $extraAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} spec: type: {{ .Values.service.kong.type }} ports: diff --git a/supabase/helm/knoe-supabase/templates/studio/backendconfig.yaml b/supabase/helm/knoe-supabase/templates/studio/backendconfig.yaml new file mode 100644 index 0000000..d25aa5a --- /dev/null +++ b/supabase/helm/knoe-supabase/templates/studio/backendconfig.yaml @@ -0,0 +1,28 @@ +{{- /* +GCE L7 BackendConfig for supabase-studio. + +Mirror of kong/backendconfig.yaml. Studio is a Next.js app on port 3000 +that returns 301 redirects on `/` (not 200), which trips GCE's default +HTTP healthCheck and marks the backend UNHEALTHY. A TCP healthCheck is +sufficient for LB-level liveness. + +Rendered only when .Values.service.studio.backendConfigName is set. +*/ -}} +{{- if and .Values.deployment.studio.enabled (.Values.service.studio.backendConfigName | default "") -}} +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: {{ .Values.service.studio.backendConfigName | quote }} + labels: + {{- include "supabase.labels" . | nindent 4 }} +spec: + healthCheck: + type: TCP + port: {{ .Values.service.studio.port | default 3000 }} + checkIntervalSec: 15 + timeoutSec: 5 + healthyThreshold: 1 + unhealthyThreshold: 3 + connectionDraining: + drainingTimeoutSec: 30 +{{- end }} diff --git a/supabase/helm/knoe-supabase/templates/studio/service.yaml b/supabase/helm/knoe-supabase/templates/studio/service.yaml index c5354e8..898f46a 100644 --- a/supabase/helm/knoe-supabase/templates/studio/service.yaml +++ b/supabase/helm/knoe-supabase/templates/studio/service.yaml @@ -5,6 +5,17 @@ metadata: name: {{ include "supabase.studio.fullname" . }} labels: {{- include "supabase.labels" . | nindent 4 }} + {{- $backendConfigName := .Values.service.studio.backendConfigName | default "" }} + {{- $extraAnnotations := .Values.service.studio.annotations | default dict }} + {{- if or $backendConfigName $extraAnnotations }} + annotations: + {{- if $backendConfigName }} + cloud.google.com/backend-config: '{"default":"{{ $backendConfigName }}"}' + {{- end }} + {{- with $extraAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} spec: type: {{ .Values.service.studio.type }} ports: diff --git a/supabase/helm/render_supabase.py b/supabase/helm/render_supabase.py index 2cfaa70..dc39afa 100755 --- a/supabase/helm/render_supabase.py +++ b/supabase/helm/render_supabase.py @@ -991,6 +991,13 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) -> }, "ingress": { "enabled": True, + # In k8s/GKE mode the chart-rendered supabase-kong Ingress kept + # getting deleted from the cluster minutes after install (helm + # thought it owned it, something on the cluster kept reaping it). + # We sidestep by emitting a standalone public-ingress-kong.yaml + # below and telling the chart NOT to render its own ingress: + # kong/ingress.yaml has a top-level `if not externallyManaged` guard. + "externallyManaged": mode == "k8s", "className": "" if mode == "k8s" else api_ingress_class, "hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in api_host_entries], "annotations": { @@ -1024,6 +1031,20 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) -> } if frontdoor_auth_enabled and mode != "k8s" else {}), } }, + # BackendConfig wiring for GCE LB -- Kong's default "/" returns 404 + # and Studio returns 301, both fail GCE's default HTTP healthCheck + # and mark the backends UNHEALTHY. Our chart templates emit a + # BackendConfig with a TCP healthCheck when backendConfigName is set, + # and the Service gets a cloud.google.com/backend-config annotation + # referencing that name. + "service": { + "kong": { + "backendConfigName": "supabase-kong-backendconfig" if mode == "k8s" else "", + }, + "studio": { + "backendConfigName": "supabase-studio-backendconfig" if mode == "k8s" else "", + }, + }, } if supabase_storage_class: @@ -1188,6 +1209,72 @@ def render(args: argparse.Namespace) -> None: elif ingress_tls_manifest_path.exists(): ingress_tls_manifest_path.unlink() + # Standalone supabase-kong Ingress for GCE L7 (api.0.knoe.dev). + # The chart's templates/kong/ingress.yaml had the ingress vanish out from + # under us even though helm's manifest still showed it (root cause TBD -- + # suspected meta.helm.sh annotation mismatch interacting with a GKE audit + # reaper). We sidestep by: + # 1. Setting overlay.ingress.externallyManaged=true (k8s only), which + # makes the chart no-op its kong/ingress.yaml template. + # 2. Emitting the ingress here as a standalone manifest that deploy.sh + # applies alongside public-ingress-tls.yaml. No helm ownership labels + # or meta annotations -- a plain Kubernetes Ingress the chart won't + # try to adopt. + ingress_kong_manifest_path = k8s_dir / "public-ingress-kong.yaml" + if meta.get("mode") == "k8s": + api_ingress_cfg = overlay.get("ingress") or {} + hosts = api_ingress_cfg.get("hosts") or [] + api_annotations = dict(api_ingress_cfg.get("annotations") or {}) + # Drop chart-internal keys before emitting + api_annotations.pop("externallyManaged", None) + kong_ingress_rules: list[dict[str, Any]] = [] + for host_entry in hosts: + if not isinstance(host_entry, dict): + continue + host = str(host_entry.get("host") or "").strip() + if not host: + continue + kong_ingress_rules.append( + { + "host": host, + "http": { + "paths": [ + { + "path": p.get("path", "/"), + "pathType": p.get("pathType", "Prefix"), + "backend": { + "service": { + "name": "supabase-kong", + "port": {"number": 8000}, + } + }, + } + for p in (host_entry.get("paths") or [{"path": "/", "pathType": "Prefix"}]) + ] + }, + } + ) + if kong_ingress_rules and api_annotations: + kong_ingress_doc = { + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": { + "name": "supabase-kong", + "namespace": meta["supabase_namespace"], + "annotations": api_annotations, + }, + "spec": {"rules": kong_ingress_rules}, + } + ingress_kong_manifest_path.write_text( + yaml.safe_dump(kong_ingress_doc, sort_keys=False) + ) + print(f"\nGenerated Supabase kong public Ingress manifest: {ingress_kong_manifest_path}") + print(yaml.safe_dump(kong_ingress_doc, sort_keys=False).rstrip()) + elif ingress_kong_manifest_path.exists(): + ingress_kong_manifest_path.unlink() + elif ingress_kong_manifest_path.exists(): + ingress_kong_manifest_path.unlink() + # Render chart to a single manifest file (Helm accepts JSON values) rendered_path = k8s_dir / "supabase-helm.yaml" cmd = [ @@ -1246,6 +1333,7 @@ def render(args: argparse.Namespace) -> None: "manifests_app": str(app_rendered_path), "manifests_frontdoor_db": str(frontdoor_rendered_path), "manifests_public_ingress_tls": str(ingress_tls_manifest_path), + "manifests_public_ingress_kong": str(ingress_kong_manifest_path), "db_host": meta["db_host"], "supabase_namespace": meta["supabase_namespace"], }