chore: add GKE TLS management with ManagedCertificate and FrontendConfig annotations

- Updated `deploy.sh` and `init_kong.sh` to support GKE ManagedCertificate and FrontendConfig reconciliation in k8s mode with GCE ingress class.
- Enhanced TLS path diagnostics to distinguish between missing, attached-but-not-serving, and actively serving configurations.
- Added HTTPS probing for GKE-managed public ingress paths and validation for managed certificate statuses.
- Updated `render_supabase.py` to generate and attach TLS annotations for Supabase API and Studio ingresses in k8s mode.
- Added tests to validate TLS path handling, probing, and manifest generation for GKE deployments.
This commit is contained in:
chrisfu 2026-04-20 08:19:54 -07:00
parent 8eff5dba0d
commit 49c54ae6ee
4 changed files with 350 additions and 7 deletions

114
deploy.sh
View File

@ -250,9 +250,11 @@ def _ingress_tls_diagnostics(item, host):
pre_shared_cert = str(annotations.get("ingress.gcp.kubernetes.io/pre-shared-cert") or "").strip()
has_tls_path = bool(tls_host_match or managed_cert or pre_shared_cert)
tls_path_state = "TLS_PATH_ATTACHED" if has_tls_path else "MISSING_TLS_PATH"
return {
"has_tls_path": has_tls_path,
"tls_path_state": tls_path_state,
"tls_hosts": ",".join(sorted(set(tls_hosts))) if tls_hosts else "-",
"tls_secrets": ",".join(sorted(set(tls_secret_names))) if tls_secret_names else "-",
"managed_cert": managed_cert or "-",
@ -289,6 +291,70 @@ def get_ingresses_for_host(inventory, host):
})
return result
def _pick_authoritative_owner(owners, expected):
if not owners:
return None
if expected:
for owner in owners:
if owner.get("namespace") == expected.get("namespace") and owner.get("name") == expected.get("name"):
return owner
return owners[0]
def _managed_cert_status(ctx, namespace, cert_name):
if not ctx or not namespace or not cert_name:
return "Unknown"
try:
out = subprocess.check_output([
"kubectl", "--context", ctx, "get", "managedcertificate", cert_name,
"-n", namespace, "-o", "json"
], stderr=subprocess.DEVNULL, text=True)
item = json.loads(out)
status = (item.get("status") or {}) if isinstance(item, dict) else {}
cert_status = str(status.get("certificateStatus") or status.get("status") or "").strip()
return cert_status or "Unknown"
except Exception:
return "Unknown"
def _managed_cert_statuses(ctx, namespace, managed_cert_annotation):
cert_names = [token.strip() for token in str(managed_cert_annotation or "").split(",") if token.strip()]
statuses = {}
for cert_name in cert_names:
statuses[cert_name] = _managed_cert_status(ctx, namespace, cert_name)
return statuses
def _probe_https_endpoint(host, timeout_seconds=15):
url = f"https://{host}/"
try:
completed = subprocess.run([
"curl",
"--silent",
"--show-error",
"--location",
"--output", "/dev/null",
"--write-out", "%{http_code}",
"--connect-timeout", "5",
"--max-time", str(timeout_seconds),
url,
], check=False, capture_output=True, text=True)
http_code = str(completed.stdout or "").strip()
if completed.returncode == 0 and re.fullmatch(r"\d{3}", http_code):
return True, f"http={http_code}"
detail = str(completed.stderr or "").strip()
if not detail:
detail = str(completed.stdout or "").strip()
if detail:
detail = detail.splitlines()[-1]
if not detail:
detail = f"curl_exit={completed.returncode}"
return False, detail[:220]
except Exception as exc:
return False, str(exc)
def get_ingress_status(ctx, namespace, ingress_name):
status = {
"name": ingress_name,
@ -771,30 +837,54 @@ for host in public_hosts:
host_diagnostics[host] = owners
if not owners:
broken_hosts.append(f"{host}: no ingress claims this host")
print(f" - host={host} ingress=(none) class=- address=pending tls=- managedCert=- backend=-")
print(
f" - host={host} ingress=(none) class=- address=pending tlsState=MISSING_TLS_PATH tlsHosts=- tlsSecrets=- managedCert=- managedCertStatus=- preSharedCert=- httpsProbe=not-run backend=-"
)
continue
expected = expected_owner.get(host)
owner = _pick_authoritative_owner(owners, expected)
if not owner:
broken_hosts.append(f"{host}: unable to determine authoritative ingress owner")
continue
if len(owners) > 1:
owner_desc = ", ".join(f"{o['namespace']}/{o['name']}[class={o['ingressClass']}]" for o in owners)
broken_hosts.append(f"{host}: multiple ingress owners found ({owner_desc})")
owner = owners[0]
managed_cert_statuses = _managed_cert_statuses(app_ctx, owner["namespace"], owner["managed_cert"])
managed_cert_status_summary = (
",".join(f"{name}:{status}" for name, status in managed_cert_statuses.items()) if managed_cert_statuses else "-"
)
owner["managed_cert_status"] = managed_cert_status_summary
probe_ok = False
probe_result = "not-run"
if owner["has_tls_path"]:
probe_ok, probe_result = _probe_https_endpoint(host)
owner["tls_path_state"] = "TLS_PATH_SERVING" if probe_ok else "TLS_PATH_ATTACHED_BUT_NOT_SERVING"
else:
owner["tls_path_state"] = "MISSING_TLS_PATH"
owner["https_probe"] = probe_result
print(
" - host={host} ingress={ns}/{name} class={cls} address={addr} tlsHosts={tls_hosts} tlsSecrets={tls_secrets} managedCert={managed} preSharedCert={pre_shared} backend={backend}".format(
" - host={host} ingress={ns}/{name} class={cls} address={addr} tlsState={tls_state} tlsHosts={tls_hosts} tlsSecrets={tls_secrets} managedCert={managed} managedCertStatus={managed_status} preSharedCert={pre_shared} httpsProbe={https_probe} backend={backend}".format(
host=host,
ns=owner["namespace"],
name=owner["name"],
cls=owner["ingressClass"],
addr=owner["address"],
tls_state=owner["tls_path_state"],
tls_hosts=owner["tls_hosts"],
tls_secrets=owner["tls_secrets"],
managed=owner["managed_cert"],
managed_status=owner.get("managed_cert_status", "-"),
pre_shared=owner["pre_shared_cert"],
https_probe=owner.get("https_probe", "not-run"),
backend=owner["backend"],
)
)
expected = expected_owner.get(host)
if expected and (owner["namespace"] != expected["namespace"] or owner["name"] != expected["name"]):
broken_hosts.append(
f"{host}: unexpected ingress owner {owner['namespace']}/{owner['name']} (expected {expected['namespace']}/{expected['name']})"
@ -803,11 +893,23 @@ for host in public_hosts:
if owner["address"] == "pending":
broken_hosts.append(f"{host}: ingress {owner['namespace']}/{owner['name']} has no load-balancer address yet")
if not owner["has_tls_path"]:
if owner["tls_path_state"] == "MISSING_TLS_PATH":
broken_hosts.append(
f"{host}: ingress {owner['namespace']}/{owner['name']} has no configured TLS path (tlsHosts={owner['tls_hosts']}, managedCert={owner['managed_cert']}, preSharedCert={owner['pre_shared_cert']})"
f"{host}: MISSING_TLS_PATH on ingress {owner['namespace']}/{owner['name']} (tlsHosts={owner['tls_hosts']}, managedCert={owner['managed_cert']}, preSharedCert={owner['pre_shared_cert']})"
)
if owner["tls_path_state"] == "TLS_PATH_ATTACHED_BUT_NOT_SERVING":
broken_hosts.append(
f"{host}: TLS_PATH_ATTACHED_BUT_NOT_SERVING on ingress {owner['namespace']}/{owner['name']} (httpsProbe={owner.get('https_probe', '-')})"
)
if host == runtime["gitlab_host"] and managed_cert_statuses:
non_active = [f"{name}:{status}" for name, status in managed_cert_statuses.items() if status != "Active"]
if non_active:
broken_hosts.append(
f"{host}: managed certificate status is not Active ({', '.join(non_active)})"
)
if broken_hosts:
print("\nERROR: Public HTTPS front-door validation failed.")
for issue in broken_hosts:

View File

@ -519,6 +519,7 @@ EOF
local extra_annotations=""
local tls_enabled=0
local tls_annotations=""
local gce_tls_annotations=""
local tls_block=""
if is_truthy "${SERVICE_INGRESS_TLS_ENABLED:-0}"; then
tls_enabled=1
@ -548,6 +549,46 @@ EOF
ingress_host_csv=$(IFS=, ; echo "${ingress_hosts[*]}")
assert_unique_ingress_host_claims "$ingress_name" "$NAMESPACE" "$ingress_host_csv"
if [[ "${PROLE_MODE:-}" == "k8s" && "$ingress_class" == "gce" ]]; then
local service_managed_cert_name="${SERVICE_MANAGED_CERT_NAME:-svc-knoe-managed-cert}"
local service_frontend_config_name="${SERVICE_FRONTEND_CONFIG_NAME:-svc-knoe-frontend-config}"
local service_pre_shared_cert="${SERVICE_PRE_SHARED_CERT:-}"
local managed_domains_yaml=""
local ingress_tls_host
for ingress_tls_host in "${ingress_hosts[@]}"; do
managed_domains_yaml+=$'\n'" - ${ingress_tls_host}"
done
echo "Reconciling GKE ManagedCertificate (${service_managed_cert_name}) + FrontendConfig (${service_frontend_config_name}) for ${ingress_name} ..."
kubectl apply -f - <<EOF
apiVersion: networking.gke.io/v1
kind: ManagedCertificate
metadata:
name: ${service_managed_cert_name}
namespace: ${NAMESPACE}
spec:
domains:${managed_domains_yaml}
---
apiVersion: networking.gke.io/v1beta1
kind: FrontendConfig
metadata:
name: ${service_frontend_config_name}
namespace: ${NAMESPACE}
spec:
redirectToHttps:
enabled: true
responseCodeName: MOVED_PERMANENTLY_DEFAULT
EOF
gce_tls_annotations=$(cat <<EOF
networking.gke.io/managed-certificates: ${service_managed_cert_name}
networking.gke.io/v1beta1.FrontendConfig: ${service_frontend_config_name}
EOF
)
if [[ -n "$service_pre_shared_cert" ]]; then
gce_tls_annotations+=$'\n'" ingress.gcp.kubernetes.io/pre-shared-cert: ${service_pre_shared_cert}"
fi
unset service_managed_cert_name service_frontend_config_name service_pre_shared_cert managed_domains_yaml ingress_tls_host
fi
if (( tls_enabled == 1 )); then
tls_annotations=$(cat <<EOF
cert-manager.io/cluster-issuer: ${SERVICE_TLS_CLUSTER_ISSUER}
@ -577,7 +618,7 @@ metadata:
namespace: ${NAMESPACE}
annotations:
kubernetes.io/ingress.class: ${ingress_class}
${extra_annotations}${tls_annotations}
${extra_annotations}${tls_annotations}${gce_tls_annotations}
spec:
${tls_block}
rules:

View File

@ -568,6 +568,63 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
if not studio_ingress_class or _is_placeholder(studio_ingress_class):
studio_ingress_class = api_ingress_class
supabase_api_managed_cert_name = _first(
os.environ.get("SUPABASE_API_MANAGED_CERT_NAME", ""),
os.environ.get("SUPABASE_MANAGED_CERT_NAME", ""),
_cfg_get(cfg, "Global", "SUPABASE_API_MANAGED_CERT_NAME"),
_cfg_get(cfg, "Global", "SUPABASE_MANAGED_CERT_NAME"),
default="supabase-api-managed-cert",
).strip()
if not supabase_api_managed_cert_name or _is_placeholder(supabase_api_managed_cert_name):
supabase_api_managed_cert_name = "supabase-api-managed-cert"
supabase_studio_managed_cert_name = _first(
os.environ.get("SUPABASE_STUDIO_MANAGED_CERT_NAME", ""),
_cfg_get(cfg, "Global", "SUPABASE_STUDIO_MANAGED_CERT_NAME"),
_cfg_get(cfg, "Global", "SUPABASE_MANAGED_CERT_NAME"),
default="supabase-studio-managed-cert",
).strip()
if not supabase_studio_managed_cert_name or _is_placeholder(supabase_studio_managed_cert_name):
supabase_studio_managed_cert_name = "supabase-studio-managed-cert"
supabase_api_frontend_config_name = _first(
os.environ.get("SUPABASE_API_FRONTEND_CONFIG_NAME", ""),
os.environ.get("SUPABASE_FRONTEND_CONFIG_NAME", ""),
_cfg_get(cfg, "Global", "SUPABASE_API_FRONTEND_CONFIG_NAME"),
_cfg_get(cfg, "Global", "SUPABASE_FRONTEND_CONFIG_NAME"),
default="supabase-api-frontend-config",
).strip()
if not supabase_api_frontend_config_name or _is_placeholder(supabase_api_frontend_config_name):
supabase_api_frontend_config_name = "supabase-api-frontend-config"
supabase_studio_frontend_config_name = _first(
os.environ.get("SUPABASE_STUDIO_FRONTEND_CONFIG_NAME", ""),
_cfg_get(cfg, "Global", "SUPABASE_STUDIO_FRONTEND_CONFIG_NAME"),
_cfg_get(cfg, "Global", "SUPABASE_FRONTEND_CONFIG_NAME"),
default="supabase-studio-frontend-config",
).strip()
if not supabase_studio_frontend_config_name or _is_placeholder(supabase_studio_frontend_config_name):
supabase_studio_frontend_config_name = "supabase-studio-frontend-config"
supabase_api_pre_shared_cert = _first(
os.environ.get("SUPABASE_API_PRE_SHARED_CERT", ""),
os.environ.get("SUPABASE_PRE_SHARED_CERT", ""),
_cfg_get(cfg, "Global", "SUPABASE_API_PRE_SHARED_CERT"),
_cfg_get(cfg, "Global", "SUPABASE_PRE_SHARED_CERT"),
default="",
).strip()
if _is_placeholder(supabase_api_pre_shared_cert):
supabase_api_pre_shared_cert = ""
supabase_studio_pre_shared_cert = _first(
os.environ.get("SUPABASE_STUDIO_PRE_SHARED_CERT", ""),
_cfg_get(cfg, "Global", "SUPABASE_STUDIO_PRE_SHARED_CERT"),
_cfg_get(cfg, "Global", "SUPABASE_PRE_SHARED_CERT"),
default="",
).strip()
if _is_placeholder(supabase_studio_pre_shared_cert):
supabase_studio_pre_shared_cert = ""
allow_traefik_public_ingress = _as_bool(
_first(
os.environ.get("SUPABASE_ALLOW_TRAEFIK_INGRESS", ""),
@ -914,6 +971,12 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in api_host_entries],
"annotations": {
**({"kubernetes.io/ingress.class": api_ingress_class} if mode == "k8s" else {}),
**({
"networking.gke.io/managed-certificates": supabase_api_managed_cert_name,
"networking.gke.io/v1beta1.FrontendConfig": supabase_api_frontend_config_name,
} if mode == "k8s" else {}),
**({"ingress.gcp.kubernetes.io/pre-shared-cert": supabase_api_pre_shared_cert}
if mode == "k8s" and supabase_api_pre_shared_cert else {}),
}
},
"studioIngress": {
@ -922,6 +985,12 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
"hosts": [{"host": host, "paths": [{"path": "/", "pathType": "Prefix"}]} for host in studio_host_entries],
"annotations": {
**({"kubernetes.io/ingress.class": studio_ingress_class} if mode == "k8s" else {}),
**({
"networking.gke.io/managed-certificates": supabase_studio_managed_cert_name,
"networking.gke.io/v1beta1.FrontendConfig": supabase_studio_frontend_config_name,
} if mode == "k8s" else {}),
**({"ingress.gcp.kubernetes.io/pre-shared-cert": supabase_studio_pre_shared_cert}
if mode == "k8s" and supabase_studio_pre_shared_cert else {}),
**({
"nginx.ingress.kubernetes.io/auth-url": auth_verify_url,
"nginx.ingress.kubernetes.io/auth-signin": auth_signin_url,
@ -1007,6 +1076,61 @@ def render(args: argparse.Namespace) -> None:
}
(k8s_dir / "namespace.yaml").write_text(json.dumps(namespace_manifest))
gke_tls_resources: list[dict[str, Any]] = []
if meta.get("mode") == "k8s":
for ingress_cfg in (overlay.get("ingress") or {}, overlay.get("studioIngress") or {}):
if not isinstance(ingress_cfg, dict):
continue
annotations = ingress_cfg.get("annotations") or {}
if not isinstance(annotations, dict):
annotations = {}
managed_cert_name = str(annotations.get("networking.gke.io/managed-certificates") or "").strip()
frontend_config_name = str(annotations.get("networking.gke.io/v1beta1.FrontendConfig") or "").strip()
host_entries = ingress_cfg.get("hosts") or []
hosts = []
for host_entry in host_entries:
if not isinstance(host_entry, dict):
continue
host = str(host_entry.get("host") or "").strip()
if host and host not in hosts:
hosts.append(host)
if managed_cert_name and hosts:
gke_tls_resources.append(
{
"apiVersion": "networking.gke.io/v1",
"kind": "ManagedCertificate",
"metadata": {
"name": managed_cert_name,
"namespace": meta["supabase_namespace"],
},
"spec": {"domains": hosts},
}
)
if frontend_config_name:
gke_tls_resources.append(
{
"apiVersion": "networking.gke.io/v1beta1",
"kind": "FrontendConfig",
"metadata": {
"name": frontend_config_name,
"namespace": meta["supabase_namespace"],
},
"spec": {
"redirectToHttps": {
"enabled": True,
"responseCodeName": "MOVED_PERMANENTLY_DEFAULT",
}
},
}
)
ingress_tls_manifest_path = k8s_dir / "public-ingress-tls.yaml"
if gke_tls_resources:
ingress_tls_manifest_path.write_text(yaml.safe_dump_all(gke_tls_resources, sort_keys=False))
elif ingress_tls_manifest_path.exists():
ingress_tls_manifest_path.unlink()
# Render chart to a single manifest file (Helm accepts JSON values)
rendered_path = k8s_dir / "supabase-helm.yaml"
cmd = [

View File

@ -431,6 +431,68 @@ def test_render_supabase_studio_disabled_sets_enabled_false(tmp_path, monkeypatc
assert overlay["deployment"]["studio"]["enabled"] is False
def test_render_supabase_k8s_adds_gce_tls_annotations_for_api_and_studio(tmp_path, monkeypatch):
"""k8s/GKE overlay must attach explicit ManagedCertificate + FrontendConfig annotations."""
monkeypatch.setenv("PROLE_MODE", "k8s")
monkeypatch.setenv("SUPABASE_API_HOSTNAME", "api.0.knoe.dev")
monkeypatch.setenv("SUPABASE_STUDIO_HOSTNAME", "db.0.knoe.dev")
cfg = _minimal_cfg(tmp_path)
args = Namespace(output_dir=str(tmp_path / "gen"), manifests_dir=str(tmp_path / "k8s"))
overlay, _ = _build_overlay(cfg, args)
api_annotations = overlay.get("ingress", {}).get("annotations", {})
studio_annotations = overlay.get("studioIngress", {}).get("annotations", {})
assert api_annotations.get("kubernetes.io/ingress.class") == "gce"
assert studio_annotations.get("kubernetes.io/ingress.class") == "gce"
assert api_annotations.get("networking.gke.io/managed-certificates") == "supabase-api-managed-cert"
assert studio_annotations.get("networking.gke.io/managed-certificates") == "supabase-studio-managed-cert"
assert api_annotations.get("networking.gke.io/v1beta1.FrontendConfig") == "supabase-api-frontend-config"
assert studio_annotations.get("networking.gke.io/v1beta1.FrontendConfig") == "supabase-studio-frontend-config"
def test_render_supabase_render_writes_gke_public_ingress_tls_manifest(tmp_path, monkeypatch):
"""render() must emit ManagedCertificate/FrontendConfig manifests for Supabase public ingresses in k8s mode."""
import subprocess as sp
monkeypatch.setenv("PROLE_MODE", "k8s")
monkeypatch.setenv("SUPABASE_API_HOSTNAME", "api.0.knoe.dev")
monkeypatch.setenv("SUPABASE_STUDIO_HOSTNAME", "db.0.knoe.dev")
cfg_path = tmp_path / "prole.cfg"
_write_cfg(
cfg_path,
"""
[Inputs]
init_password.db_password = test-password
init_password.db_host_port = 5432
[Global]
NAMESPACE = supabase
STORAGE_BACKEND = local
DEPLOYMENT_MODE = k8s
""",
)
def _fake_run(cmd, capture_output=True, text=True):
return sp.CompletedProcess(cmd, 0, stdout="apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: supabase-placeholder\n", stderr="")
monkeypatch.setattr(_render_mod.subprocess, "run", _fake_run)
args = Namespace(config=str(cfg_path), output_dir=str(tmp_path / "gen"), manifests_dir=str(tmp_path / "k8s"))
_render_mod.render(args)
tls_manifest = (tmp_path / "k8s" / "public-ingress-tls.yaml")
assert tls_manifest.exists(), "Expected public-ingress-tls.yaml to be generated in k8s mode"
rendered = tls_manifest.read_text(encoding="utf-8")
assert "kind: ManagedCertificate" in rendered
assert "kind: FrontendConfig" in rendered
assert "supabase-api-managed-cert" in rendered
assert "supabase-studio-managed-cert" in rendered
# ===========================================================================
# 4. render_supabase.py — per-component flags respected
# ===========================================================================
@ -822,6 +884,20 @@ def test_init_kong_allows_disabling_svc_ingress_tls_for_k8s_path():
assert 'SERVICE_INGRESS_TLS_ENABLED="${SERVICE_INGRESS_TLS_ENABLED:-1}"' in script
assert 'if is_truthy "${SERVICE_INGRESS_TLS_ENABLED:-0}"; then' in script
assert "Rendering svc ingress without TLS" in script
assert 'SERVICE_MANAGED_CERT_NAME:-svc-knoe-managed-cert' in script
assert 'SERVICE_FRONTEND_CONFIG_NAME:-svc-knoe-frontend-config' in script
assert 'networking.gke.io/managed-certificates: ${service_managed_cert_name}' in script
assert 'networking.gke.io/v1beta1.FrontendConfig: ${service_frontend_config_name}' in script
def test_deploy_frontdoor_validation_tracks_tls_path_states_and_https_probe():
"""deploy.sh diagnostics must distinguish missing TLS path from attached-but-not-serving and require HTTPS probing."""
script = (REPO_ROOT / "deploy.sh").read_text(encoding="utf-8")
assert "MISSING_TLS_PATH" in script
assert "TLS_PATH_ATTACHED_BUT_NOT_SERVING" in script
assert "def _probe_https_endpoint" in script
assert "managed certificate status is not Active" in script
def test_reset_k3s_namespace_delete_db_deletes_pvcs_and_bound_pvs(tmp_path, monkeypatch):