From 82a2c2ed2ed6df34e93e26becc51c72a81e8ebc9 Mon Sep 17 00:00:00 2001 From: chrisfu Date: Mon, 20 Apr 2026 22:09:51 -0700 Subject: [PATCH] net: pin public ingresses to reserved static IPs + wire gitlab-shell SSH LB Reserved 5 named external static IPs and threaded the kubernetes.io/ingress.global-static-ip-name annotation through every public GCE L7 ingress the installer renders, so ingress delete/recreate stops churning DNS. Also turned on a dedicated regional L4 LoadBalancer for gitlab-shell (port 22) so SSH git workflows work. New config surface in conf/gke.cfg: GITLAB_GLOBAL_STATIC_IP_NAME = git-knoe-dev (34.102.141.87) SVC_KNOE_GLOBAL_STATIC_IP_NAME = svc-knoe (34.111.197.33) SUPABASE_API_GLOBAL_STATIC_IP_NAME = supabase-api (34.120.221.5) SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME = supabase-studio (136.110.189.6) GITLAB_SSH_HOST = git-ssh.knoe.dev GITLAB_SHELL_LOADBALANCER_IP = 34.106.243.154 (regional us-west3) GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY = Local (default) Why SSH lives on a separate hostname+IP: Google-managed certs require a GCE global L7 IP; port 22 needs a regional Network LB. Those can not share an IP on GCP, so git.knoe.dev stays on HTTPS and git-ssh.knoe.dev takes SSH. Wiring per surface: - etc/init_gitlab.sh * Added GITLAB_SSH_HOST (default git-ssh. in k8s mode), GITLAB_SHELL_LOADBALANCER_IP, GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY, GITLAB_GLOBAL_STATIC_IP_NAME config keys. * CR global.hosts.ssh now reads ${GITLAB_SSH_HOST}. * gitlab-shell block conditionally renders service: {type: LoadBalancer, loadBalancerIP, externalTrafficPolicy} when the LB IP is set. * GITLAB_GCE_TLS_ANNOTATIONS_YAML gains kubernetes.io/ingress.global-static-ip-name: "" when GITLAB_GLOBAL_STATIC_IP_NAME is set. - etc/init_kong.sh * Added SVC_KNOE_GLOBAL_STATIC_IP_NAME var near other SERVICE_TLS_* defaults. * gce_tls_annotations heredoc gets the static-IP line appended on the same condition. - supabase/helm/render_supabase.py * Two new _first(env, cfg, default) extractions for SUPABASE_API_GLOBAL_STATIC_IP_NAME and SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME near the existing managed-cert / frontend-config vars. * Two new dict-spread blocks in the Kong + Studio ingress annotations that emit the static-IP annotation only when mode=k8s and the value is non-empty. All three ingress surfaces already use kubectl apply (merge-friendly); re-running init_gitlab.sh / init_kong.sh / deploy.sh (supabase step) is enough to pick up the new annotation. The GKE LB controller will swap each ingress's forwarding rule from the auto-generated k8s2-fr-* reservation to the named reservation, then release the old ephemeral. Ingress IPs change; DNS records need updating (TTL <=300s while iterating). Google-managed cert for gitlab-managed-cert will briefly Provision again during the swap; the other three were already Provisioning. Co-Authored-By: Claude Opus 4.7 (1M context) --- conf/gke.cfg | 15 ++++++++++++ etc/init_gitlab.sh | 42 +++++++++++++++++++++++++++++++- etc/init_kong.sh | 11 +++++++++ supabase/helm/render_supabase.py | 25 +++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/conf/gke.cfg b/conf/gke.cfg index 34a7e90..0bf953f 100644 --- a/conf/gke.cfg +++ b/conf/gke.cfg @@ -127,6 +127,21 @@ GITLAB_OIDC_CLIENT_ID = secretref://google-oidc-client-id GITLAB_OIDC_CLIENT_SECRET = secretref://google-oidc-client-secret GITLAB_OIDC_ISSUER = https://accounts.google.com GITLAB_PUBLIC_HOSTS = git.knoe.dev +; Pin the GCE L7 HTTPS ingresses to reserved GLOBAL static IPs (gcloud compute +; addresses create --global). Prevents IP churn on ingress delete/recreate. +GITLAB_GLOBAL_STATIC_IP_NAME = git-knoe-dev +; Kong (knoe-system/svc-knoe-ingress, serves svc.knoe.dev + api.knoe.dev) +SVC_KNOE_GLOBAL_STATIC_IP_NAME = svc-knoe +; Supabase Kong API ingress (supabase/supabase-kong, serves api.0.knoe.dev) +SUPABASE_API_GLOBAL_STATIC_IP_NAME = supabase-api +; Supabase Studio ingress (supabase/supabase-studio, serves db.0.knoe.dev) +SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME = supabase-studio +; SSH ingress — dedicated REGIONAL external static IP (us-west3) bound to +; gitlab-shell Service of type LoadBalancer. Must be created via +; gcloud compute addresses create git-knoe-ssh --region=us-west3 +; then fill in the .address value below. +GITLAB_SSH_HOST = git-ssh.knoe.dev +GITLAB_SHELL_LOADBALANCER_IP = 34.106.243.154 GITLAB_REPAIR_BLOCKED_AUTOCLEAN = 1 GITLAB_WEBSERVICE_LIMITS_MEMORY = 3Gi GITLAB_WEBSERVICE_PUMA_THREADS_MAX = 2 diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh index 2bbc687..c1d0a4a 100755 --- a/etc/init_gitlab.sh +++ b/etc/init_gitlab.sh @@ -2428,6 +2428,31 @@ if [[ "$MODE" == "k8s" ]]; then fi GITLAB_INGRESS_CLASS="${GITLAB_INGRESS_CLASS:-$_default_gitlab_ingress_class}" +# --------------------------------------------------------------------------- +# Git SSH hostname + LoadBalancer IP +# --------------------------------------------------------------------------- +# On GKE with Google-managed certs, HTTPS has to live on a GCE global L7 IP +# (ManagedCertificate CRD only binds there). Port 22 requires a regional +# Network LB which can't share an IP with a global L7. So in k8s mode we +# default to a dedicated SSH hostname (git-ssh.) bound to a +# user-reserved regional external static IP via GITLAB_SHELL_LOADBALANCER_IP. +# In k3d/k3s modes the gitlab-shell Service stays ClusterIP (typically +# port-forwarded) and global.hosts.ssh tracks the main gitlab hostname. +_default_gitlab_ssh_host="$GITLAB_DOMAIN" +if [[ "$MODE" == "k8s" ]]; then + _default_gitlab_ssh_host="git-ssh.${PRIMARY_GITLAB_DOMAIN_ROOT}" +fi +GITLAB_SSH_HOST="${GITLAB_SSH_HOST:-$_default_gitlab_ssh_host}" + +# Pre-reserved regional external static IP for the gitlab-shell LoadBalancer +# Service. If blank, no service override is emitted (ClusterIP default). +GITLAB_SHELL_LOADBALANCER_IP="${GITLAB_SHELL_LOADBALANCER_IP:-}" + +# externalTrafficPolicy preserves client source IPs in SSH auth logs — useful +# for abuse triage and rate-limiting. Requires at least one gitlab-shell pod +# per node in the LB backend; our replicaCount=1 is fine. +GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY="${GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY:-Local}" + # Single GitLab front-door owner model. # - operator: GitLab chart/operator-managed ingress owns ${GITLAB_PUBLIC_HOSTS} # - fallback: custom fallback ingress owns ${GITLAB_PUBLIC_HOSTS} @@ -3831,7 +3856,7 @@ fi) domain: ${PRIMARY_GITLAB_DOMAIN_ROOT} gitlab: name: ${GITLAB_DOMAIN} - ssh: ${GITLAB_DOMAIN} + ssh: ${GITLAB_SSH_HOST} ingress: class: ${GITLAB_INGRESS_CLASS} configureCertmanager: false @@ -4008,6 +4033,14 @@ ${WORKLOAD_JEMALLOC_VALUES_YAML} replicaCount: 1 minReplicas: 1 maxReplicas: 1 +$(if [[ -n "${GITLAB_SHELL_LOADBALANCER_IP:-}" ]]; then +cat <&2 fi diff --git a/supabase/helm/render_supabase.py b/supabase/helm/render_supabase.py index 6c3bb00..2cfaa70 100755 --- a/supabase/helm/render_supabase.py +++ b/supabase/helm/render_supabase.py @@ -630,6 +630,25 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) -> "ManagedCertificate + FrontendConfig is the single TLS source of truth.\n" ) + # Optional: pin each GCE L7 ingress to a pre-reserved GLOBAL external + # static IP (gcloud compute addresses create --global). Prevents IP churn + # on ingress delete/recreate. Blank = GCE assigns ephemerally. + supabase_api_global_static_ip_name = _first( + os.environ.get("SUPABASE_API_GLOBAL_STATIC_IP_NAME", ""), + _cfg_get(cfg, "Global", "SUPABASE_API_GLOBAL_STATIC_IP_NAME"), + default="", + ).strip() + if _is_placeholder(supabase_api_global_static_ip_name): + supabase_api_global_static_ip_name = "" + + supabase_studio_global_static_ip_name = _first( + os.environ.get("SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME", ""), + _cfg_get(cfg, "Global", "SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME"), + default="", + ).strip() + if _is_placeholder(supabase_studio_global_static_ip_name): + supabase_studio_global_static_ip_name = "" + allow_traefik_public_ingress = _as_bool( _first( os.environ.get("SUPABASE_ALLOW_TRAEFIK_INGRESS", ""), @@ -980,6 +999,9 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) -> "networking.gke.io/managed-certificates": supabase_api_managed_cert_name, "networking.gke.io/v1beta1.FrontendConfig": supabase_api_frontend_config_name, } if mode == "k8s" else {}), + **({ + "kubernetes.io/ingress.global-static-ip-name": supabase_api_global_static_ip_name, + } if mode == "k8s" and supabase_api_global_static_ip_name else {}), } }, "studioIngress": { @@ -992,6 +1014,9 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) -> "networking.gke.io/managed-certificates": supabase_studio_managed_cert_name, "networking.gke.io/v1beta1.FrontendConfig": supabase_studio_frontend_config_name, } if mode == "k8s" else {}), + **({ + "kubernetes.io/ingress.global-static-ip-name": supabase_studio_global_static_ip_name, + } if mode == "k8s" and supabase_studio_global_static_ip_name else {}), **({ "nginx.ingress.kubernetes.io/auth-url": auth_verify_url, "nginx.ingress.kubernetes.io/auth-signin": auth_signin_url,