mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
net: pin public ingresses to reserved static IPs + wire gitlab-shell SSH LB
Reserved 5 named external static IPs and threaded the
kubernetes.io/ingress.global-static-ip-name annotation through every
public GCE L7 ingress the installer renders, so ingress delete/recreate
stops churning DNS. Also turned on a dedicated regional L4
LoadBalancer for gitlab-shell (port 22) so SSH git workflows work.
New config surface in conf/gke.cfg:
GITLAB_GLOBAL_STATIC_IP_NAME = git-knoe-dev (34.102.141.87)
SVC_KNOE_GLOBAL_STATIC_IP_NAME = svc-knoe (34.111.197.33)
SUPABASE_API_GLOBAL_STATIC_IP_NAME = supabase-api (34.120.221.5)
SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME = supabase-studio (136.110.189.6)
GITLAB_SSH_HOST = git-ssh.knoe.dev
GITLAB_SHELL_LOADBALANCER_IP = 34.106.243.154 (regional us-west3)
GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY = Local (default)
Why SSH lives on a separate hostname+IP: Google-managed certs require
a GCE global L7 IP; port 22 needs a regional Network LB. Those can not
share an IP on GCP, so git.knoe.dev stays on HTTPS and git-ssh.knoe.dev
takes SSH.
Wiring per surface:
- etc/init_gitlab.sh
* Added GITLAB_SSH_HOST (default git-ssh.<root> in k8s mode),
GITLAB_SHELL_LOADBALANCER_IP, GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY,
GITLAB_GLOBAL_STATIC_IP_NAME config keys.
* CR global.hosts.ssh now reads ${GITLAB_SSH_HOST}.
* gitlab-shell block conditionally renders service: {type:
LoadBalancer, loadBalancerIP, externalTrafficPolicy} when the LB IP
is set.
* GITLAB_GCE_TLS_ANNOTATIONS_YAML gains
kubernetes.io/ingress.global-static-ip-name: "<name>" when
GITLAB_GLOBAL_STATIC_IP_NAME is set.
- etc/init_kong.sh
* Added SVC_KNOE_GLOBAL_STATIC_IP_NAME var near other SERVICE_TLS_*
defaults.
* gce_tls_annotations heredoc gets the static-IP line appended on the
same condition.
- supabase/helm/render_supabase.py
* Two new _first(env, cfg, default) extractions for
SUPABASE_API_GLOBAL_STATIC_IP_NAME and
SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME near the existing managed-cert
/ frontend-config vars.
* Two new dict-spread blocks in the Kong + Studio ingress annotations
that emit the static-IP annotation only when mode=k8s and the value
is non-empty.
All three ingress surfaces already use kubectl apply (merge-friendly);
re-running init_gitlab.sh / init_kong.sh / deploy.sh (supabase step) is
enough to pick up the new annotation. The GKE LB controller will swap
each ingress's forwarding rule from the auto-generated k8s2-fr-*
reservation to the named reservation, then release the old ephemeral.
Ingress IPs change; DNS records need updating (TTL <=300s while
iterating). Google-managed cert for gitlab-managed-cert will briefly
Provision again during the swap; the other three were already
Provisioning.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
chrisfu
parent
15df3ecc08
commit
82a2c2ed2e
15
conf/gke.cfg
15
conf/gke.cfg
@ -127,6 +127,21 @@ GITLAB_OIDC_CLIENT_ID = secretref://google-oidc-client-id
|
||||
GITLAB_OIDC_CLIENT_SECRET = secretref://google-oidc-client-secret
|
||||
GITLAB_OIDC_ISSUER = https://accounts.google.com
|
||||
GITLAB_PUBLIC_HOSTS = git.knoe.dev
|
||||
; Pin the GCE L7 HTTPS ingresses to reserved GLOBAL static IPs (gcloud compute
|
||||
; addresses create --global). Prevents IP churn on ingress delete/recreate.
|
||||
GITLAB_GLOBAL_STATIC_IP_NAME = git-knoe-dev
|
||||
; Kong (knoe-system/svc-knoe-ingress, serves svc.knoe.dev + api.knoe.dev)
|
||||
SVC_KNOE_GLOBAL_STATIC_IP_NAME = svc-knoe
|
||||
; Supabase Kong API ingress (supabase/supabase-kong, serves api.0.knoe.dev)
|
||||
SUPABASE_API_GLOBAL_STATIC_IP_NAME = supabase-api
|
||||
; Supabase Studio ingress (supabase/supabase-studio, serves db.0.knoe.dev)
|
||||
SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME = supabase-studio
|
||||
; SSH ingress — dedicated REGIONAL external static IP (us-west3) bound to
|
||||
; gitlab-shell Service of type LoadBalancer. Must be created via
|
||||
; gcloud compute addresses create git-knoe-ssh --region=us-west3
|
||||
; then fill in the .address value below.
|
||||
GITLAB_SSH_HOST = git-ssh.knoe.dev
|
||||
GITLAB_SHELL_LOADBALANCER_IP = 34.106.243.154
|
||||
GITLAB_REPAIR_BLOCKED_AUTOCLEAN = 1
|
||||
GITLAB_WEBSERVICE_LIMITS_MEMORY = 3Gi
|
||||
GITLAB_WEBSERVICE_PUMA_THREADS_MAX = 2
|
||||
|
||||
@ -2428,6 +2428,31 @@ if [[ "$MODE" == "k8s" ]]; then
|
||||
fi
|
||||
GITLAB_INGRESS_CLASS="${GITLAB_INGRESS_CLASS:-$_default_gitlab_ingress_class}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Git SSH hostname + LoadBalancer IP
|
||||
# ---------------------------------------------------------------------------
|
||||
# On GKE with Google-managed certs, HTTPS has to live on a GCE global L7 IP
|
||||
# (ManagedCertificate CRD only binds there). Port 22 requires a regional
|
||||
# Network LB which can't share an IP with a global L7. So in k8s mode we
|
||||
# default to a dedicated SSH hostname (git-ssh.<root domain>) bound to a
|
||||
# user-reserved regional external static IP via GITLAB_SHELL_LOADBALANCER_IP.
|
||||
# In k3d/k3s modes the gitlab-shell Service stays ClusterIP (typically
|
||||
# port-forwarded) and global.hosts.ssh tracks the main gitlab hostname.
|
||||
_default_gitlab_ssh_host="$GITLAB_DOMAIN"
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
_default_gitlab_ssh_host="git-ssh.${PRIMARY_GITLAB_DOMAIN_ROOT}"
|
||||
fi
|
||||
GITLAB_SSH_HOST="${GITLAB_SSH_HOST:-$_default_gitlab_ssh_host}"
|
||||
|
||||
# Pre-reserved regional external static IP for the gitlab-shell LoadBalancer
|
||||
# Service. If blank, no service override is emitted (ClusterIP default).
|
||||
GITLAB_SHELL_LOADBALANCER_IP="${GITLAB_SHELL_LOADBALANCER_IP:-}"
|
||||
|
||||
# externalTrafficPolicy preserves client source IPs in SSH auth logs — useful
|
||||
# for abuse triage and rate-limiting. Requires at least one gitlab-shell pod
|
||||
# per node in the LB backend; our replicaCount=1 is fine.
|
||||
GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY="${GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY:-Local}"
|
||||
|
||||
# Single GitLab front-door owner model.
|
||||
# - operator: GitLab chart/operator-managed ingress owns ${GITLAB_PUBLIC_HOSTS}
|
||||
# - fallback: custom fallback ingress owns ${GITLAB_PUBLIC_HOSTS}
|
||||
@ -3831,7 +3856,7 @@ fi)
|
||||
domain: ${PRIMARY_GITLAB_DOMAIN_ROOT}
|
||||
gitlab:
|
||||
name: ${GITLAB_DOMAIN}
|
||||
ssh: ${GITLAB_DOMAIN}
|
||||
ssh: ${GITLAB_SSH_HOST}
|
||||
ingress:
|
||||
class: ${GITLAB_INGRESS_CLASS}
|
||||
configureCertmanager: false
|
||||
@ -4008,6 +4033,14 @@ ${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
replicaCount: 1
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
$(if [[ -n "${GITLAB_SHELL_LOADBALANCER_IP:-}" ]]; then
|
||||
cat <<SHELLSVC
|
||||
service:
|
||||
type: LoadBalancer
|
||||
loadBalancerIP: ${GITLAB_SHELL_LOADBALANCER_IP}
|
||||
externalTrafficPolicy: ${GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY}
|
||||
SHELLSVC
|
||||
fi)
|
||||
kas:
|
||||
replicaCount: 1
|
||||
minReplicas: 1
|
||||
@ -4384,12 +4417,19 @@ GITLAB_GCE_TLS_ANNOTATIONS_YAML=""
|
||||
GITLAB_MANAGED_CERT_NAME="gitlab-managed-cert"
|
||||
GITLAB_FRONTEND_CONFIG_NAME="gitlab-frontend-config"
|
||||
GITLAB_BACKEND_CONFIG_NAME="gitlab-webservice-backendconfig"
|
||||
# Optional: pin the GCE L7 ingress to a pre-reserved global external static IP
|
||||
# (gcloud compute addresses create --global). Prevents IP churn on ingress
|
||||
# delete/recreate. Leave blank to let GCE assign ephemerally.
|
||||
GITLAB_GLOBAL_STATIC_IP_NAME="${GITLAB_GLOBAL_STATIC_IP_NAME:-}"
|
||||
if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
|
||||
GITLAB_GCE_TLS_ANNOTATIONS_YAML=$(cat <<EOTLS
|
||||
networking.gke.io/managed-certificates: "${GITLAB_MANAGED_CERT_NAME}"
|
||||
networking.gke.io/v1beta1.FrontendConfig: "${GITLAB_FRONTEND_CONFIG_NAME}"
|
||||
EOTLS
|
||||
)
|
||||
if [[ -n "$GITLAB_GLOBAL_STATIC_IP_NAME" ]]; then
|
||||
GITLAB_GCE_TLS_ANNOTATIONS_YAML+=$'\n'" kubernetes.io/ingress.global-static-ip-name: \"${GITLAB_GLOBAL_STATIC_IP_NAME}\""
|
||||
fi
|
||||
# Build the ManagedCertificate domains list from GITLAB_PUBLIC_HOSTS.
|
||||
_gitlab_mc_domains_yaml=""
|
||||
for _gitlab_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
|
||||
|
||||
@ -148,6 +148,11 @@ KONG_GITEA_SSH_PORT="${KONG_GITEA_SSH_PORT:-3022}"
|
||||
SERVICE_TLS_SECRET_NAME="${SERVICE_TLS_SECRET_NAME:-${SERVICE_HOSTNAME//./-}-tls}"
|
||||
SERVICE_TLS_CLUSTER_ISSUER="${SERVICE_TLS_CLUSTER_ISSUER:-letsencrypt-prod}"
|
||||
|
||||
# Optional: pin the GCE L7 svc-knoe-ingress to a pre-reserved GLOBAL external
|
||||
# static IP (gcloud compute addresses create --global). Prevents IP churn on
|
||||
# ingress delete/recreate. Leave blank to let GCE assign ephemerally.
|
||||
SVC_KNOE_GLOBAL_STATIC_IP_NAME="${SVC_KNOE_GLOBAL_STATIC_IP_NAME:-}"
|
||||
|
||||
# Legacy: svc-check used to own svc.prole.org. We now route the service hostname
|
||||
# to Grafana, so remove any leftover svc-check resources to avoid conflicts.
|
||||
SVC_CHECK_NAMESPACE="${SVC_CHECK_NAMESPACE:-svc-check}"
|
||||
@ -596,6 +601,12 @@ EOF
|
||||
networking.gke.io/v1beta1.FrontendConfig: ${service_frontend_config_name}
|
||||
EOF
|
||||
)
|
||||
# Pin to a reserved global external static IP when configured. Prevents
|
||||
# IP churn on ingress delete/recreate (paired with
|
||||
# conf/gke.cfg:SVC_KNOE_GLOBAL_STATIC_IP_NAME).
|
||||
if [[ -n "${SVC_KNOE_GLOBAL_STATIC_IP_NAME:-}" ]]; then
|
||||
gce_tls_annotations+=$'\n'" kubernetes.io/ingress.global-static-ip-name: \"${SVC_KNOE_GLOBAL_STATIC_IP_NAME}\""
|
||||
fi
|
||||
if [[ -n "$service_pre_shared_cert" ]]; then
|
||||
echo "WARN: SERVICE_PRE_SHARED_CERT is ignored for ${NAMESPACE}/${ingress_name} in k8s/gce mode; using ManagedCertificate + FrontendConfig only." >&2
|
||||
fi
|
||||
|
||||
@ -630,6 +630,25 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
||||
"ManagedCertificate + FrontendConfig is the single TLS source of truth.\n"
|
||||
)
|
||||
|
||||
# Optional: pin each GCE L7 ingress to a pre-reserved GLOBAL external
|
||||
# static IP (gcloud compute addresses create --global). Prevents IP churn
|
||||
# on ingress delete/recreate. Blank = GCE assigns ephemerally.
|
||||
supabase_api_global_static_ip_name = _first(
|
||||
os.environ.get("SUPABASE_API_GLOBAL_STATIC_IP_NAME", ""),
|
||||
_cfg_get(cfg, "Global", "SUPABASE_API_GLOBAL_STATIC_IP_NAME"),
|
||||
default="",
|
||||
).strip()
|
||||
if _is_placeholder(supabase_api_global_static_ip_name):
|
||||
supabase_api_global_static_ip_name = ""
|
||||
|
||||
supabase_studio_global_static_ip_name = _first(
|
||||
os.environ.get("SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME", ""),
|
||||
_cfg_get(cfg, "Global", "SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME"),
|
||||
default="",
|
||||
).strip()
|
||||
if _is_placeholder(supabase_studio_global_static_ip_name):
|
||||
supabase_studio_global_static_ip_name = ""
|
||||
|
||||
allow_traefik_public_ingress = _as_bool(
|
||||
_first(
|
||||
os.environ.get("SUPABASE_ALLOW_TRAEFIK_INGRESS", ""),
|
||||
@ -980,6 +999,9 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
||||
"networking.gke.io/managed-certificates": supabase_api_managed_cert_name,
|
||||
"networking.gke.io/v1beta1.FrontendConfig": supabase_api_frontend_config_name,
|
||||
} if mode == "k8s" else {}),
|
||||
**({
|
||||
"kubernetes.io/ingress.global-static-ip-name": supabase_api_global_static_ip_name,
|
||||
} if mode == "k8s" and supabase_api_global_static_ip_name else {}),
|
||||
}
|
||||
},
|
||||
"studioIngress": {
|
||||
@ -992,6 +1014,9 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
||||
"networking.gke.io/managed-certificates": supabase_studio_managed_cert_name,
|
||||
"networking.gke.io/v1beta1.FrontendConfig": supabase_studio_frontend_config_name,
|
||||
} if mode == "k8s" else {}),
|
||||
**({
|
||||
"kubernetes.io/ingress.global-static-ip-name": supabase_studio_global_static_ip_name,
|
||||
} if mode == "k8s" and supabase_studio_global_static_ip_name else {}),
|
||||
**({
|
||||
"nginx.ingress.kubernetes.io/auth-url": auth_verify_url,
|
||||
"nginx.ingress.kubernetes.io/auth-signin": auth_signin_url,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user