prole/etc/init_kong.sh

#!/usr/bin/env bash

set -euo pipefail

# init_kong.sh
# Purpose:
# - Deploy Kong API Gateway (DB-less) into the service namespace
# - Replaces the prole nginx deployment as the API endpoint
# - Routes /backup/* to knoe-db-manager
# - Creates the kong declarative config as a ConfigMap
# - Applies the kong deployment and service manifests
# - Provides start/stop/status/restart actions

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"

# Inject default config if not provided
_has_config=0
for _arg in "$@"; do
  [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
done
if [[ $_has_config -eq 0 ]]; then
  _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)"
  if [[ -n "$_default_cfg" ]]; then
    set -- "-c" "$_default_cfg" "$@"
  fi
fi
unset _has_config _arg _default_cfg

common_core_preparse_config "$@"

# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

set -- "${COMMON_CORE_ARGS[@]}"
common_core_parse_args "$@"

if [[ -z "${PROLE_MODE:-}" ]]; then
  export PROLE_MODE="k3s"
fi

resolve_explicit_kube_context() {
  local ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
  if [[ -n "$ctx" ]]; then
    printf '%s' "$ctx"
    return 0
  fi
  return 1
}

enforce_app_cluster_targeting() {
  if [[ "${PROLE_MODE:-}" != "k8s" ]]; then
    return 0
  fi
  local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
  local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
  local target_ctx
  target_ctx="$(resolve_explicit_kube_context || true)"

  if [[ -z "$app_ctx" ]]; then
    echo "ERROR: APP_CLUSTER_KUBECONTEXT is required for k8s Kong deployment." >&2
    exit 2
  fi
  if [[ -z "$target_ctx" ]]; then
    echo "ERROR: explicit kubectl context is required for k8s Kong deployment." >&2
    exit 2
  fi
  if [[ -n "$db_ctx" && "$target_ctx" == "$db_ctx" ]]; then
    echo "ERROR: refusing Kong APP step against DB context '$target_ctx'." >&2
    exit 2
  fi
  if [[ "$target_ctx" != "$app_ctx" ]]; then
    echo "ERROR: Kong APP step must target APP_CLUSTER_KUBECONTEXT='$app_ctx' (got '$target_ctx')." >&2
    exit 2
  fi

  export KUBECTL_CONTEXT="$app_ctx"
  export KUBE_CONTEXT_NAME="$app_ctx"
  export KUBECONTEXT="$app_ctx"
}

kubectl() {
  local target_ctx
  target_ctx="$(resolve_explicit_kube_context || true)"
  if [[ "${PROLE_MODE:-}" == "k8s" && -z "$target_ctx" ]]; then
    echo "ERROR: explicit kubectl context is required in k8s mode." >&2
    return 2
  fi

  local arg has_context=0
  for arg in "$@"; do
    case "$arg" in
      --context|--context=*|--server|--server=*)
        has_context=1
        break
        ;;
    esac
  done

  if [[ -n "$target_ctx" && $has_context -eq 0 ]]; then
    command kubectl --context "$target_ctx" "$@"
  else
    command kubectl "$@"
  fi
}

enforce_app_cluster_targeting

if [[ "${COMMON_CORE_HELP:-0}" == 1 ]]; then
  common_core_usage "$0"
  exit 0
fi

if [[ -n "${COMMON_CORE_PARSE_ERROR:-}" ]]; then
  echo "ERROR: ${COMMON_CORE_PARSE_ERROR}" >&2
  common_core_usage "$0"
  exit 2
fi

ACTION="$COMMON_CORE_ACTION"
NAMESPACE="$(common_core_resolve_namespace "default")"
common_core_apply_namespace "$NAMESPACE"

PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}
KONG_IMAGE="${KONG_IMAGE:-kong:3.9}"
# k8s/GKE prod mode uses knoe.dev domain and knoe-svc-kong; all other modes use prole.org
if [[ "${PROLE_MODE:-}" == "k8s" ]]; then
  KONG_NAME="${KONG_NAME:-knoe-svc-kong}"
  KONG_CONFIG_NAME="${KONG_CONFIG_NAME:-knoe-svc-kong-config}"
  SERVICE_HOSTNAME="${SERVICE_HOSTNAME:-svc.knoe.dev}"
  AUTH_HOSTNAME="${AUTH_HOSTNAME:-api.knoe.dev}"
  GITEA_HOSTNAME="${GITEA_HOSTNAME:-${GITEA_DOMAIN:-git.knoe.dev}}"
  SERVICE_INGRESS_TLS_ENABLED="${SERVICE_INGRESS_TLS_ENABLED:-0}"
else
  KONG_NAME="${KONG_NAME:-prole-svc-kong}"
  KONG_CONFIG_NAME="${KONG_CONFIG_NAME:-prole-svc-kong-config}"
  SERVICE_HOSTNAME="${SERVICE_HOSTNAME:-svc.prole.org}"
  AUTH_HOSTNAME="${AUTH_HOSTNAME:-api.prole.org}"
  GITEA_HOSTNAME="${GITEA_HOSTNAME:-${GITEA_DOMAIN:-git.prole.org}}"
  SERVICE_INGRESS_TLS_ENABLED="${SERVICE_INGRESS_TLS_ENABLED:-1}"
fi
KONG_PROXY_PORT="${KONG_PROXY_PORT:-8000}"
KONG_ADMIN_PORT="${KONG_ADMIN_PORT:-8001}"
KONG_GITEA_SSH_PORT="${KONG_GITEA_SSH_PORT:-3022}"
SERVICE_TLS_SECRET_NAME="${SERVICE_TLS_SECRET_NAME:-${SERVICE_HOSTNAME//./-}-tls}"
SERVICE_TLS_CLUSTER_ISSUER="${SERVICE_TLS_CLUSTER_ISSUER:-letsencrypt-prod}"

# Optional: pin the GCE L7 svc-knoe-ingress to a pre-reserved GLOBAL external
# static IP (gcloud compute addresses create --global). Prevents IP churn on
# ingress delete/recreate. Leave blank to let GCE assign ephemerally.
SVC_KNOE_GLOBAL_STATIC_IP_NAME="${SVC_KNOE_GLOBAL_STATIC_IP_NAME:-}"

# BackendConfig name for the knoe-svc-kong Service. GCE's default L7 health
# check hits HTTP `/` on the backend port, which Kong responds to with 404
# (no route) -- marking the backend UNHEALTHY and causing the LB to return
# "Server Error" instead of reaching Kong. We instead point the GCE LB at a
# TCP health check (port-level liveness) so the backend passes as long as
# Kong is accepting connections, which is sufficient for our traffic shape.
# Mirrors the pattern in etc/init_gitlab.sh (gitlab-webservice-backendconfig).
SVC_KNOE_BACKEND_CONFIG_NAME="${SVC_KNOE_BACKEND_CONFIG_NAME:-knoe-svc-kong-backendconfig}"

# Legacy: svc-check used to own svc.prole.org. We now route the service hostname
# to Grafana, so remove any leftover svc-check resources to avoid conflicts.
SVC_CHECK_NAMESPACE="${SVC_CHECK_NAMESPACE:-svc-check}"

# kubectl robustness knobs (timeouts/retries for transient apiserver slowness)
KUBECTL_REQUEST_TIMEOUT="${KUBECTL_REQUEST_TIMEOUT:-30s}"
KUBECTL_APPLY_RETRIES="${KUBECTL_APPLY_RETRIES:-5}"
KUBECTL_APPLY_RETRY_DELAY="${KUBECTL_APPLY_RETRY_DELAY:-2}"

# Upstream service defaults
DB_MANAGER_SERVICE="${DB_MANAGER_SERVICE:-knoe-db-manager}"
DB_MANAGER_PORT="${DB_MANAGER_PORT:-80}"
DB_MANAGER_NAMESPACE="${DB_MANAGER_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
PROLE_SERVICE_UPSTREAM_URL="${PROLE_SERVICE_UPSTREAM_URL:-http://prole-svc.${NAMESPACE}.svc.cluster.local:8080}"
GRAFANA_UPSTREAM_URL="${GRAFANA_UPSTREAM_URL:-http://kps-grafana.monitoring.svc.cluster.local:80}"
GITEA_HTTP_UPSTREAM_URL="${GITEA_HTTP_UPSTREAM_URL:-http://gitea-http.gitea.svc.cluster.local:3000}"
GITEA_SSH_UPSTREAM_HOST="${GITEA_SSH_UPSTREAM_HOST:-gitea-ssh.gitea.svc.cluster.local}"
GITEA_SSH_UPSTREAM_PORT="${GITEA_SSH_UPSTREAM_PORT:-22}"

# SSO wiring knobs
PROLE_GRAFANA_SSO_ENABLED="${PROLE_GRAFANA_SSO_ENABLED:-0}"
GRAFANA_PROXY_UPSTREAM_URL="${GRAFANA_PROXY_UPSTREAM_URL:-http://prole-grafana-proxy.${NAMESPACE}.svc.cluster.local:80}"
KNOE_AUTH_UPSTREAM_URL="${KNOE_AUTH_UPSTREAM_URL:-http://knoe-auth.${SERVICE_NAMESPACE:-${NAMESPACE}}.svc.cluster.local:8080}"

usage() {
  cat <<USAGE
Usage: $0 [--mode MODE] [-n NAMESPACE] [${COMMON_CORE_ACTIONS//|/|}] [-c conf/{k3d|k3s|gke}.cfg]

Actions:
  start     Create ConfigMap and deploy Kong
  stop      Remove Kong deployment, service, and ConfigMap
  status    Show Kong pod/service status
  restart   Restart Kong pods
  update    Create or update Kong resources
USAGE
  exit 1
}

ensure_tools() {
  for t in kubectl; do
    command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
  done
}

is_truthy() {
  case "${1:-}" in
    1|true|TRUE|True|yes|YES|on|ON|y|Y)
      return 0
      ;;
    *)
      return 1
      ;;
  esac
}

assert_public_ingress_targeting() {
  local ingress_class="${1:-}"
  shift || true
  local hosts=("$@")

  if [[ "${PROLE_MODE:-}" != "k8s" ]]; then
    return 0
  fi

  local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
  local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
  local active_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"

  if [[ -z "$app_ctx" || -z "$active_ctx" ]]; then
    echo "ERROR: explicit APP cluster context is required for public Kong ingress in k8s mode." >&2
    exit 1
  fi

  local host_count=0
  local h
  for h in "${hosts[@]}"; do
    [[ -n "${h:-}" ]] && host_count=$((host_count + 1))
  done

  if [[ "$host_count" -gt 0 && -n "$db_ctx" && "$active_ctx" == "$db_ctx" ]]; then
    echo "ERROR: refusing to render/apply public Kong ingress in DB cluster context '${active_ctx}' (hosts: ${hosts[*]})." >&2
    exit 1
  fi

  if [[ "$host_count" -gt 0 && -n "$app_ctx" && -n "$active_ctx" && "$active_ctx" != "$app_ctx" ]]; then
    echo "ERROR: public Kong ingress must target APP cluster context '${app_ctx}', active context is '${active_ctx}'." >&2
    exit 1
  fi

  if [[ -n "$ingress_class" ]]; then
    local normalized_class="${ingress_class,,}"
    if [[ "$normalized_class" == traefik* ]] && ! is_truthy "${ALLOW_TRAEFIK_PUBLIC_INGRESS:-${KONG_ALLOW_TRAEFIK_INGRESS:-0}}"; then
      echo "ERROR: ingress class '${ingress_class}' is incompatible with k8s mode unless Traefik public ingress is explicitly enabled." >&2
      exit 1
    fi
  fi
}

assert_unique_ingress_host_claims() {
  local ingress_name="${1:-}"
  local ingress_namespace="${2:-}"
  local host_csv="${3:-}"
  [[ -n "$host_csv" ]] || return 0

  local target_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
  if [[ "${PROLE_MODE:-}" == "k8s" && -z "$target_ctx" ]]; then
    echo "ERROR: explicit kubectl context is required for ingress ownership checks in k8s mode." >&2
    return 1
  fi

  if ! python3 - "$host_csv" "$ingress_namespace" "$ingress_name" "$target_ctx" <<'PY'
import json
import subprocess
import sys

requested_hosts = {h.strip().lower() for h in (sys.argv[1] or "").split(",") if h.strip()}
target_ns = sys.argv[2]
target_name = sys.argv[3]
target_ctx = (sys.argv[4] or "").strip()

cmd = ["kubectl"]
if target_ctx:
    cmd.extend(["--context", target_ctx])
cmd.extend(["get", "ingress", "-A", "-o", "json"])

try:
    raw = subprocess.check_output(cmd, text=True)
except Exception:
    raise SystemExit(0)

if not raw.strip():
    raise SystemExit(0)

payload = json.loads(raw)
conflicts: list[str] = []
for item in payload.get("items", []) or []:
    md = item.get("metadata", {}) or {}
    ns = (md.get("namespace") or "").strip()
    name = (md.get("name") or "").strip()
    if ns == target_ns and name == target_name:
        continue

    spec = item.get("spec", {}) or {}
    rules = spec.get("rules", []) or []
    for rule in rules:
        host = (rule.get("host") or "").strip().lower()
        if not host or host not in requested_hosts:
            continue
        http = rule.get("http", {}) or {}
        paths = http.get("paths", []) or [{"path": "/"}]
        for path_item in paths:
            path = (path_item.get("path") or "/").strip() or "/"
            if path in {"/", ""}:
                conflicts.append(f"{host}{path} already owned by {ns}/{name}")

if conflicts:
    raise SystemExit("; ".join(conflicts))
PY
  then
    echo "ERROR: duplicate ingress host/path claim detected for Kong ingress '${ingress_namespace}/${ingress_name}'." >&2
    return 1
  fi
}

ensure_namespace() {
  if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
    echo "Creating namespace '$NAMESPACE' ..."
    kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true
  fi
}

kubectl_rt() {
  kubectl --request-timeout="$KUBECTL_REQUEST_TIMEOUT" "$@"
}

kubectl_apply_retry() {
  local attempt=1
  local delay="$KUBECTL_APPLY_RETRY_DELAY"
  while true; do
    if kubectl_rt apply "$@"; then
      return 0
    fi
    local rc=$?
    if [[ "$attempt" -ge "$KUBECTL_APPLY_RETRIES" ]]; then
      return "$rc"
    fi
    echo "WARN: kubectl apply failed (attempt ${attempt}/${KUBECTL_APPLY_RETRIES}); retrying in ${delay}s ..." >&2
    sleep "$delay"
    attempt=$((attempt + 1))
    delay=$((delay * 2))
  done
}

# Sets KONG_CONFIG_CHANGED=1 in the caller's scope when the ConfigMap was
# created or updated; leaves it 0 when kubectl reported "unchanged".
# A temp file is used to communicate the result out of the subshell.
KONG_CONFIG_CHANGED=0
_KONG_CONFIG_CHANGED_FILE=""

create_kong_config() {
  echo "Creating/updating Kong declarative config '$KONG_CONFIG_NAME' in namespace '$NAMESPACE' ..."

  local grafana_url
  grafana_url="$GRAFANA_UPSTREAM_URL"
  case "${PROLE_GRAFANA_SSO_ENABLED:-0}" in
    1|true|TRUE|True|yes|YES|on|ON)
      grafana_url="$GRAFANA_PROXY_UPSTREAM_URL"
      ;;
  esac

  local kong_yml
  kong_yml=$(cat <<KONGEOF
_format_version: "3.0"
_transform: true

services:
  # Dedicated health endpoint for the GCE LB BackendConfig. request-termination
  # returns 200 synchronously with no upstream call, so the probe passes as long
  # as the Kong proxy itself is running. GCE rejects \`type: TCP\` in BackendConfig
  # (only HTTP/HTTPS/HTTP2 accepted), so we use this route for HTTP liveness.
  - name: healthz
    url: http://127.0.0.1:${KONG_PROXY_PORT}/
    routes:
      - name: healthz
        paths:
          - /healthz
        strip_path: true
    plugins:
      - name: request-termination
        config:
          status_code: 200
          message: ok

  - name: prole-service
    url: ${PROLE_SERVICE_UPSTREAM_URL}
    routes:
      - name: prole-k3s-kubeconfig
        hosts:
          - ${SERVICE_HOSTNAME}
        paths:
          - /k3s/kube_config.sh
        strip_path: false

  - name: db-manager
    url: http://${DB_MANAGER_SERVICE}.${DB_MANAGER_NAMESPACE}.svc.cluster.local:${DB_MANAGER_PORT}
    routes:
      - name: backup-route
        hosts:
          - ${SERVICE_HOSTNAME}
        paths:
          - /backup
        strip_path: false

  - name: grafana
    url: ${grafana_url}
    routes:
      - name: grafana-root
        hosts:
          - ${SERVICE_HOSTNAME}
        paths:
          - /
        strip_path: false

  - name: knoe-auth
    url: ${KNOE_AUTH_UPSTREAM_URL}
    routes:
      - name: knoe-auth-root
        hosts:
          - ${AUTH_HOSTNAME}
        paths:
          - /
        strip_path: false

  - name: gitea-http
    url: ${GITEA_HTTP_UPSTREAM_URL}
    routes:
      - name: gitea-root
        hosts:
          - ${GITEA_HOSTNAME}
        paths:
          - /
        strip_path: false

  - name: gitea-ssh
    host: ${GITEA_SSH_UPSTREAM_HOST}
    port: ${GITEA_SSH_UPSTREAM_PORT}
    protocol: tcp
    routes:
      - name: gitea-ssh-tcp
        protocols:
          - tcp
        destinations:
          - port: ${KONG_GITEA_SSH_PORT}
KONGEOF
)

  # Use a subshell to ensure the cleanup trap doesn't leak globally (and trip `set -u` later).
  # A sentinel file is used to communicate a config change back to the parent shell.
  _KONG_CONFIG_CHANGED_FILE="$(mktemp)"
  (
    tmp="$(mktemp)"
    trap 'rm -f "${tmp:-}"' EXIT

    kubectl create configmap "$KONG_CONFIG_NAME" \
      --namespace="$NAMESPACE" \
      --from-literal=kong.yml="$kong_yml" \
      --dry-run=client -o yaml >"$tmp"

    local apply_out
    apply_out=$(kubectl_apply_retry -f "$tmp" 2>&1)
    echo "$apply_out"
    if ! echo "$apply_out" | grep -q 'unchanged'; then
      echo "1" >"$_KONG_CONFIG_CHANGED_FILE"
    fi
  )
  if [[ -f "$_KONG_CONFIG_CHANGED_FILE" ]] && [[ "$(cat "$_KONG_CONFIG_CHANGED_FILE")" == "1" ]]; then
    KONG_CONFIG_CHANGED=1
  fi
  rm -f "$_KONG_CONFIG_CHANGED_FILE"
  _KONG_CONFIG_CHANGED_FILE=""

  echo "ConfigMap '$KONG_CONFIG_NAME' ready."
}

cleanup_legacy_svc_check() {
  # Best-effort cleanup: older installs applied a static check page (svc-check)
  # that owned the service hostname via its own Ingress and injected routes into
  # the shared Kong declarative config ConfigMap.
  kubectl -n "$NAMESPACE" delete ingress svc-check-ingress --ignore-not-found >/dev/null 2>&1 || true
  kubectl delete namespace "$SVC_CHECK_NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
}

apply_service_ingress() {
  local host="${SERVICE_HOSTNAME:-}"
  if [[ -z "$host" ]]; then
    echo "WARN: SERVICE_HOSTNAME is empty; skipping service Ingress." >&2
    return 0
  fi

  local auth_host="${AUTH_HOSTNAME:-}"
  local gitea_host="${GITEA_HOSTNAME:-}"
  local include_aux_hosts=1
  local include_gitea_host=1
  if [[ "${PROLE_MODE:-}" == "k8s" ]]; then
    # In k8s mode, GitLab handles its own ingress.
    # But api.knoe.dev must be public via knoe-svc-kong.
    include_gitea_host=0
  fi
  local tls_hosts_extra=""
  local rules_extra=""
  if [[ "$include_aux_hosts" -eq 1 && -n "$auth_host" && "$auth_host" != "$host" ]]; then
    tls_hosts_extra=$'\n        - '"${auth_host}"
    rules_extra=$(cat <<EOF
    - host: ${auth_host}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: ${KONG_NAME}
                port:
                  number: ${KONG_PROXY_PORT}
EOF
)
  fi

  if [[ "$include_gitea_host" -eq 1 && -n "$gitea_host" && "$gitea_host" != "$host" && "$gitea_host" != "$auth_host" ]]; then
    tls_hosts_extra+=$'\n        - '"${gitea_host}"
    rules_extra+=$'\n'$(cat <<EOF
    - host: ${gitea_host}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: ${KONG_NAME}
                port:
                  number: ${KONG_PROXY_PORT}
EOF
)
  fi

  local ingress_name="svc-knoe-ingress"
  local extra_annotations=""
  local tls_enabled=0
  local tls_annotations=""
  local gce_tls_annotations=""
  local tls_block=""
  local service_managed_cert_name=""
  local service_frontend_config_name=""
  local service_pre_shared_cert="${SERVICE_PRE_SHARED_CERT:-}"
  if is_truthy "${SERVICE_INGRESS_TLS_ENABLED:-0}"; then
    tls_enabled=1
  fi

  if [[ "${PROLE_MODE:-}" != "k8s" ]]; then
    extra_annotations=$'    traefik.ingress.kubernetes.io/router.priority: "10"\n'
  fi
  local ingress_class="${KONG_INGRESS_CLASS:-}"
  if [[ -z "$ingress_class" ]]; then
    if [[ "${PROLE_MODE:-}" == "k8s" ]]; then
      ingress_class="gce"
    else
      ingress_class="traefik"
    fi
  fi

  local ingress_hosts=("$host")
  if [[ "$include_aux_hosts" -eq 1 && -n "$auth_host" && "$auth_host" != "$host" ]]; then
    ingress_hosts+=("$auth_host")
  fi
  if [[ "$include_gitea_host" -eq 1 && -n "$gitea_host" && "$gitea_host" != "$host" && "$gitea_host" != "$auth_host" ]]; then
    ingress_hosts+=("$gitea_host")
  fi
  assert_public_ingress_targeting "$ingress_class" "${ingress_hosts[@]}"
  local ingress_host_csv
  ingress_host_csv=$(IFS=, ; echo "${ingress_hosts[*]}")
  assert_unique_ingress_host_claims "$ingress_name" "$NAMESPACE" "$ingress_host_csv"

  if [[ "${PROLE_MODE:-}" == "k8s" && "$ingress_class" == "gce" ]]; then
    service_managed_cert_name="${SERVICE_MANAGED_CERT_NAME:-svc-knoe-managed-cert}"
    service_frontend_config_name="${SERVICE_FRONTEND_CONFIG_NAME:-svc-knoe-frontend-config}"
    local managed_domains_yaml=""
    local ingress_tls_host
    for ingress_tls_host in "${ingress_hosts[@]}"; do
      managed_domains_yaml+=$'\n'"    - ${ingress_tls_host}"
    done
    echo "Reconciling GKE ManagedCertificate (${service_managed_cert_name}) + FrontendConfig (${service_frontend_config_name}) for ${ingress_name} ..."
    # Requirement 1: apply ManagedCertificate and FrontendConfig and confirm they exist BEFORE the ingress is applied.
    kubectl apply -f - <<EOF
apiVersion: networking.gke.io/v1
kind: ManagedCertificate
metadata:
  name: ${service_managed_cert_name}
  namespace: ${NAMESPACE}
spec:
  domains:${managed_domains_yaml}
---
apiVersion: networking.gke.io/v1beta1
kind: FrontendConfig
metadata:
  name: ${service_frontend_config_name}
  namespace: ${NAMESPACE}
spec:
  redirectToHttps:
    enabled: true
    responseCodeName: MOVED_PERMANENTLY_DEFAULT
EOF
    # Confirm ManagedCertificate and FrontendConfig exist before proceeding to ingress apply.
    if ! kubectl -n "$NAMESPACE" get managedcertificate "$service_managed_cert_name" >/dev/null 2>&1; then
      echo "ERROR: ManagedCertificate ${NAMESPACE}/${service_managed_cert_name} was not found after apply — cannot safely apply GCE ingress." >&2
      return 1
    fi
    if ! kubectl -n "$NAMESPACE" get frontendconfig "$service_frontend_config_name" >/dev/null 2>&1; then
      echo "ERROR: FrontendConfig ${NAMESPACE}/${service_frontend_config_name} was not found after apply — cannot safely apply GCE ingress." >&2
      return 1
    fi
    echo "Confirmed: ManagedCertificate/${service_managed_cert_name} and FrontendConfig/${service_frontend_config_name} exist in ns=${NAMESPACE}."

    # BackendConfig: GCE default healthCheck is HTTP GET / on the backend port
    # and Kong returns 404 on an unrouted path, so the backend never goes
    # HEALTHY. We wanted TCP (Kong is alive as long as it accepts connections),
    # but GCE's BackendConfig CRD rejects `type: TCP` with
    #   `Protocol "TCP" is not valid, must be one of [HTTP,HTTPS,HTTP2]`
    # so we fall back to HTTP against the `/healthz` route we add to the
    # knoe-svc-kong declarative config above (request-termination plugin
    # returns 200 synchronously, no upstream dependency -- equivalent liveness
    # semantics to a TCP check but over a protocol GCE accepts).
    echo "Reconciling BackendConfig (${SVC_KNOE_BACKEND_CONFIG_NAME}) for ${KONG_NAME} in ns=${NAMESPACE} ..."
    kubectl apply -f - <<EOF
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
  name: ${SVC_KNOE_BACKEND_CONFIG_NAME}
  namespace: ${NAMESPACE}
spec:
  healthCheck:
    type: HTTP
    requestPath: /healthz
    port: ${KONG_PROXY_PORT}
    checkIntervalSec: 15
    timeoutSec: 5
    healthyThreshold: 1
    unhealthyThreshold: 3
  connectionDraining:
    drainingTimeoutSec: 30
EOF

    gce_tls_annotations=$(cat <<EOF
    networking.gke.io/managed-certificates: ${service_managed_cert_name}
    networking.gke.io/v1beta1.FrontendConfig: ${service_frontend_config_name}
EOF
)
    # Pin to a reserved global external static IP when configured. Prevents
    # IP churn on ingress delete/recreate (paired with
    # conf/gke.cfg:SVC_KNOE_GLOBAL_STATIC_IP_NAME).
    if [[ -n "${SVC_KNOE_GLOBAL_STATIC_IP_NAME:-}" ]]; then
      gce_tls_annotations+=$'\n'"    kubernetes.io/ingress.global-static-ip-name: \"${SVC_KNOE_GLOBAL_STATIC_IP_NAME}\""
    fi
    if [[ -n "$service_pre_shared_cert" ]]; then
      echo "WARN: SERVICE_PRE_SHARED_CERT is ignored for ${NAMESPACE}/${ingress_name} in k8s/gce mode; using ManagedCertificate + FrontendConfig only." >&2
    fi
    unset managed_domains_yaml ingress_tls_host
  fi

  if (( tls_enabled == 1 )); then
    tls_annotations=$(cat <<EOF
    cert-manager.io/cluster-issuer: ${SERVICE_TLS_CLUSTER_ISSUER}
EOF
)
    tls_block=$(cat <<EOF
  tls:
    - hosts:
        - ${host}
${tls_hosts_extra}
      secretName: ${SERVICE_TLS_SECRET_NAME}
EOF
)
  else
    echo "INFO: Rendering svc ingress without TLS (SERVICE_INGRESS_TLS_ENABLED=${SERVICE_INGRESS_TLS_ENABLED:-0})."
  fi

  # Diagnostics: show current ingress, managedcertificate, and frontendconfig state before apply.
  if [[ "${PROLE_MODE:-}" == "k8s" && "$ingress_class" == "gce" ]]; then
    echo "--- GCE svc ingress diagnostics (pre-apply) ---"
    kubectl get ingress -A --no-headers 2>/dev/null || true
    echo "ManagedCertificates:"
    kubectl get managedcertificate -A --no-headers 2>/dev/null || true
    echo "FrontendConfigs:"
    kubectl get frontendconfig -A --no-headers 2>/dev/null || true
    if kubectl -n "$NAMESPACE" get ingress "$ingress_name" >/dev/null 2>&1; then
      echo "Current annotations for ${NAMESPACE}/${ingress_name} (pre-apply):"
      kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true
      echo
    fi
    echo "---"
  fi

  echo "Rendered svc ingress (pre-apply): ns=${NAMESPACE} ingress=${ingress_name} class=${ingress_class} managedCert=${service_managed_cert_name:--} frontendConfig=${service_frontend_config_name:--} preSharedCert=${service_pre_shared_cert:--} tlsEnabled=${tls_enabled} tlsSecret=${SERVICE_TLS_SECRET_NAME:--} hosts=${ingress_host_csv} backend=${KONG_NAME}:${KONG_PROXY_PORT}"

  if kubectl -n "$NAMESPACE" get ingress "$ingress_name" >/dev/null 2>&1; then
    local live_spec_class=""
    local live_ann_class=""
    local live_class=""
    local live_managed_cert=""
    local live_frontend_config=""
    local live_pre_shared_cert=""
    live_spec_class="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.spec.ingressClassName}' 2>/dev/null || true)"
    live_ann_class="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.kubernetes\.io/ingress\.class}' 2>/dev/null || true)"
    live_class="${live_spec_class:-$live_ann_class}"
    live_managed_cert="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.networking\.gke\.io/managed-certificates}' 2>/dev/null || true)"
    live_frontend_config="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.networking\.gke\.io/v1beta1\.FrontendConfig}' 2>/dev/null || true)"
    live_pre_shared_cert="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.ingress\.gcp\.kubernetes\.io/pre-shared-cert}' 2>/dev/null || true)"

    local replace_reason=""
    local patch_only=0
    if [[ -n "$live_class" && "$live_class" != "$ingress_class" ]]; then
      # ingressClass change requires recreation (immutable field).
      replace_reason="ingressClass drift (live=${live_class}, desired=${ingress_class})"
    elif [[ "${PROLE_MODE:-}" == "k8s" && "$ingress_class" == "gce" ]]; then
      # Requirement 2 & 3: never delete/recreate to clear stale cert annotations.
      # Patch annotations in place instead.
      if [[ -n "$live_pre_shared_cert" ]]; then
        echo "Patching stale pre-shared-cert annotation from ${NAMESPACE}/${ingress_name} in place (was: ${live_pre_shared_cert})."
        kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \
          ingress.gcp.kubernetes.io/pre-shared-cert- \
          "networking.gke.io/managed-certificates=${service_managed_cert_name}" \
          "networking.gke.io/v1beta1.FrontendConfig=${service_frontend_config_name}" \
          --overwrite >/dev/null 2>&1 || echo "WARN: Failed to patch pre-shared-cert annotation from ${ingress_name}." >&2
        patch_only=1
      elif [[ -n "$live_managed_cert" && "$live_managed_cert" != "$service_managed_cert_name" ]]; then
        echo "Patching managed certificate annotation on ${NAMESPACE}/${ingress_name} in place (live=${live_managed_cert}, desired=${service_managed_cert_name})."
        kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \
          "networking.gke.io/managed-certificates=${service_managed_cert_name}" \
          --overwrite >/dev/null 2>&1 || echo "WARN: Failed to patch managed-certificates annotation on ${ingress_name}." >&2
        patch_only=1
      elif [[ -n "$live_frontend_config" && "$live_frontend_config" != "$service_frontend_config_name" ]]; then
        echo "Patching frontend config annotation on ${NAMESPACE}/${ingress_name} in place (live=${live_frontend_config}, desired=${service_frontend_config_name})."
        kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \
          "networking.gke.io/v1beta1.FrontendConfig=${service_frontend_config_name}" \
          --overwrite >/dev/null 2>&1 || echo "WARN: Failed to patch FrontendConfig annotation on ${ingress_name}." >&2
        patch_only=1
      fi
    fi

    if [[ -n "$replace_reason" && "$patch_only" -eq 0 ]]; then
      echo "Ingress shape change detected for ${NAMESPACE}/${ingress_name}: ${replace_reason}. Replacing ingress (host/rule shape change)."
      kubectl -n "$NAMESPACE" delete ingress "$ingress_name" --ignore-not-found >/dev/null || true
    fi
    unset patch_only
  fi

  echo "Applying service Ingress for host '${host}' -> ${KONG_NAME}:${KONG_PROXY_PORT} (namespace=${NAMESPACE}) ..."
  (
    tmp="$(mktemp)"
    trap 'rm -f "${tmp:-}"' EXIT
    cat >"$tmp" <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ${ingress_name}
  namespace: ${NAMESPACE}
  annotations:
    kubernetes.io/ingress.class: ${ingress_class}
${extra_annotations}${tls_annotations}${gce_tls_annotations}
spec:
${tls_block}
  rules:
    - host: ${host}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: ${KONG_NAME}
                port:
                  number: ${KONG_PROXY_PORT}
${rules_extra}
EOF
    kubectl_apply_retry -f "$tmp"
  )

  if [[ "${PROLE_MODE:-}" == "k8s" && "$ingress_class" == "gce" ]]; then
    kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \
      ingress.gcp.kubernetes.io/pre-shared-cert- \
      --overwrite >/dev/null 2>&1 || true
    # Diagnostics: show annotations after apply.
    echo "Annotations for ${NAMESPACE}/${ingress_name} (post-apply):"
    kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true
    echo
    kubectl -n "$NAMESPACE" describe ingress "$ingress_name" 2>/dev/null || true
  else
    kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \
      networking.gke.io/managed-certificates- \
      networking.gke.io/v1beta1.FrontendConfig- \
      ingress.gcp.kubernetes.io/pre-shared-cert- \
      --overwrite >/dev/null 2>&1 || true
  fi
}

deploy() {
  echo "Deploying $KONG_NAME to namespace '$NAMESPACE' ..."

  local manifests_dir
  if [[ "${PROLE_MODE:-}" == "k8s" ]]; then
    manifests_dir="$PROLE_HOME/deploy/opentofu/k8s/manifests/prole"
  else
    manifests_dir="$PROLE_HOME/deploy/opentofu/k3s/manifests/prole"
  fi

  # Determine whether the Deployment already exists before applying manifests.
  local deployment_existed=0
  kubectl_rt get deployment "$KONG_NAME" -n "$NAMESPACE" >/dev/null 2>&1 && deployment_existed=1

  local deploy_out
  deploy_out=$(kubectl_apply_retry -f "$manifests_dir/kong-deployment.yaml" -n "$NAMESPACE" 2>&1)
  echo "$deploy_out"
  local svc_out
  svc_out=$(kubectl_apply_retry -f "$manifests_dir/kong-service.yaml" -n "$NAMESPACE" 2>&1)
  echo "$svc_out"

  # In k8s/GCE mode, annotate the Service so GCE LB picks up the BackendConfig
  # with the TCP health check. Matches etc/init_gitlab.sh's pattern for
  # gitlab-webservice-default. Additive annotation; survives manifest
  # re-applies (the YAML in deploy/opentofu/ doesn't set it).
  if [[ "${PROLE_MODE:-}" == "k8s" ]]; then
    echo "Annotating Service ${KONG_NAME} with cloud.google.com/backend-config=${SVC_KNOE_BACKEND_CONFIG_NAME}..."
    kubectl -n "$NAMESPACE" annotate svc "$KONG_NAME" \
      "cloud.google.com/backend-config={\"default\":\"${SVC_KNOE_BACKEND_CONFIG_NAME}\"}" \
      --overwrite >/dev/null || \
      echo "WARN: Failed to annotate ${KONG_NAME} with backend-config; GCE LB will fall back to default healthcheck (likely UNHEALTHY)." >&2
  fi

  echo "Waiting for $KONG_NAME rollout ..."
  kubectl rollout status deployment/"$KONG_NAME" -n "$NAMESPACE" --timeout=120s

  # ConfigMaps do not trigger a Deployment rollout by default.  Only restart
  # Kong when something actually changed: either this is a fresh deployment or
  # the declarative config ConfigMap was modified.  Skipping the restart when
  # nothing changed prevents a new ReplicaSet from being created every run.
  local need_restart=0
  [[ "$deployment_existed" -eq 0 ]] && need_restart=1
  [[ "${KONG_CONFIG_CHANGED:-0}" -eq 1 ]] && need_restart=1

  if [[ "$need_restart" -eq 1 ]]; then
    echo "Restarting $KONG_NAME to reload declarative config ..."
    kubectl rollout restart deployment/"$KONG_NAME" -n "$NAMESPACE" >/dev/null 2>&1 || true
    kubectl rollout status deployment/"$KONG_NAME" -n "$NAMESPACE" --timeout=120s >/dev/null 2>&1 || true
  else
    echo "$KONG_NAME config unchanged; skipping rollout restart."
  fi

  echo "$KONG_NAME deployed successfully."
}

stop() {
  echo "Removing $KONG_NAME from namespace '$NAMESPACE' ..."
  kubectl delete deployment "$KONG_NAME" -n "$NAMESPACE" --ignore-not-found=true
  kubectl delete service "$KONG_NAME" -n "$NAMESPACE" --ignore-not-found=true
  kubectl delete configmap "$KONG_CONFIG_NAME" -n "$NAMESPACE" --ignore-not-found=true
  echo "$KONG_NAME removed."
}

status() {
  echo "=== $KONG_NAME pods ==="
  kubectl get pods -n "$NAMESPACE" -l app="$KONG_NAME" 2>/dev/null || echo "No pods found"
  echo ""
  echo "=== $KONG_NAME service ==="
  kubectl get svc "$KONG_NAME" -n "$NAMESPACE" 2>/dev/null || echo "No service found"
}

restart() {
  echo "Restarting $KONG_NAME ..."
  kubectl rollout restart deployment/"$KONG_NAME" -n "$NAMESPACE"
  kubectl rollout status deployment/"$KONG_NAME" -n "$NAMESPACE" --timeout=120s
  echo "$KONG_NAME restarted."
}

action_update() {
  ensure_tools
  ensure_namespace
  cleanup_legacy_svc_check
  create_kong_config
  apply_service_ingress
  deploy
}

case "$ACTION" in
  start|initialize|update|reload)
    action_update
    ;;
  stop)
    ensure_tools
    stop
    ;;
  status)
    ensure_tools
    status
    ;;
  restart)
    ensure_tools
    restart
    ;;
  *)
    usage
    ;;
esac