#!/usr/bin/env bash
set -euo pipefail

PROG="init_gitlab"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# Standalone-invocation fix: pre-scan $@ for --config and export PROLE_DEPLOY_CFG
# BEFORE sourcing knoe_cfg.sh. deploy.sh injects PROLE_DEPLOY_CFG in its env,
# but direct invocations (./etc/init_gitlab.sh --config conf/gke.cfg deploy) do
# not — knoe_cfg.sh's auto-selection then picks conf/k3d.cfg first, which
# exports KUBECONTEXT=dev and downstream kubectl calls fail with
# `error: context "dev" does not exist`.
if [[ -z "${PROLE_DEPLOY_CFG:-}" ]]; then
  _gitlab_prescan_cfg=""
  _gitlab_prescan_want_next=0
  for _gitlab_prescan_tok in "$@"; do
    if [[ "$_gitlab_prescan_want_next" == "1" ]]; then
      _gitlab_prescan_cfg="$_gitlab_prescan_tok"
      _gitlab_prescan_want_next=0
      break
    fi
    case "$_gitlab_prescan_tok" in
      --config=*)  _gitlab_prescan_cfg="${_gitlab_prescan_tok#*=}"; break ;;
      -c|--config) _gitlab_prescan_want_next=1 ;;
    esac
  done
  if [[ -n "$_gitlab_prescan_cfg" ]]; then
    if [[ "$_gitlab_prescan_cfg" != /* ]]; then
      _gitlab_prescan_cfg="$(cd "$(dirname "$_gitlab_prescan_cfg")" 2>/dev/null && pwd)/$(basename "$_gitlab_prescan_cfg")"
    fi
    if [[ -f "$_gitlab_prescan_cfg" ]]; then
      export PROLE_DEPLOY_CFG="$_gitlab_prescan_cfg"
    fi
  fi
  unset _gitlab_prescan_cfg _gitlab_prescan_tok _gitlab_prescan_want_next
fi

# shellcheck disable=SC1090
source "$SCRIPT_DIR/knoe_cfg.sh"

MODE="$(knoe_normalize_mode "${KNOE_MODE:-${DEPLOYMENT_MODE:-k3d}}")"
# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden.
# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps).
NAMESPACE="${GITLAB_NAMESPACE:-gitlab}"
CFG_PATH=""
FORCE=0
# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE).
# Setting this would pin ALL global components to one node, exhausting RAM.
NODE_SELECTOR=""

usage() {
  cat <<EOF
Usage:
  $PROG [options] [deploy]

Options:
  --mode <k3d|k3s|k8s|local>   Deployment mode (default: ${MODE:-k3d})
  -n, --namespace <name>        Target namespace (default: gitlab)
  -c, --config <config.cfg>     Path to config file (defaults to detected)
  --node-selector <node>        Node to pin all GitLab workloads (optional)
  --force                       Remove existing GitLab and Gitea releases before deploy
  --help                        Show this help

Behavior:
  - Removes any pre-existing GitLab-domain Gitea configurations (helm release,
    namespace, Kong routes) when --force is specified or when gitea is detected.
  - Installs the GitLab Operator via Helm into the gitlab namespace.
  - Provisions a GitLab CR that uses the knoe-db CloudNativePG cluster as its
    external PostgreSQL data store.
  - Configures the GitLab public hostname as Kubernetes ingress pointing to GitLab.
  - Legacy/local modes pin gitaly storage to GITLAB_STORAGE_NODE (required).
EOF
}

die()  { echo "[ERROR] $*" >&2; exit 2; }
log()  { echo "[INFO]  $*" >&2; }
warn() { echo "[WARN]  $*" >&2; }

resolve_explicit_kube_context() {
  local ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
  if [[ -n "$ctx" ]]; then
    printf '%s' "$ctx"
    return 0
  fi
  return 1
}

enforce_app_cluster_targeting() {
  if [[ "$MODE" != "k8s" ]]; then
    return 0
  fi
  local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
  local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
  local target_ctx
  target_ctx="$(resolve_explicit_kube_context || true)"

  log "Reconciling GitLab with APP context: ${app_ctx:-"(unset)"}"
  [[ -n "$db_ctx" ]] && log "Reconciling GitLab with DB context:  ${db_ctx}"

  [[ -n "$app_ctx" ]] || die "APP_CLUSTER_KUBECONTEXT is required for k8s GitLab deployment."
  [[ -n "$target_ctx" ]] || die "Explicit kubectl context is required for k8s GitLab deployment."

  if [[ -n "$db_ctx" && "$target_ctx" == "$db_ctx" ]]; then
    die "Refusing GitLab APP step against DB context '$target_ctx'."
  fi
  if [[ "$target_ctx" != "$app_ctx" ]]; then
    die "GitLab APP step must target APP_CLUSTER_KUBECONTEXT='${app_ctx}' (got '${target_ctx}')."
  fi

  export KUBECTL_CONTEXT="$app_ctx"
  export KUBE_CONTEXT_NAME="$app_ctx"
  export KUBECONTEXT="$app_ctx"
}

gitlab_split_cluster_ownership_diagnostics() {
  if [[ "$MODE" != "k8s" ]]; then
    return 0
  fi

  local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
  local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
  local gitlab_instance="${GITLAB_RELEASE:-gitlab}"

  if [[ -z "$db_ctx" || "$db_ctx" == "$app_ctx" ]]; then
    log "GitLab ownership policy: single-cluster mode (APP=${app_ctx:-unknown}); operator and app workloads are expected in this context."
    return 0
  fi

  log "GitLab split-cluster ownership policy: APP context '${app_ctx}' is authoritative for GitLab operator + app workloads in namespace '${NAMESPACE}'."
  log "GitLab split-cluster ownership policy: DB context '${db_ctx}' must not host GitLab app workloads (gitaly/webservice/sidekiq/kas/registry/toolbox)."

  local app_gitlab_objects db_gitlab_app_objects db_gitlab_operator_objects
  app_gitlab_objects=$(kubectl -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
    -l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
  db_gitlab_app_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
    -l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
  db_gitlab_operator_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deployment -o name 2>/dev/null \
    | grep -E 'deployment.apps/(gitlab-controller-manager|gitlab-operator|.*gitlab.*controller-manager)' || true)

  if [[ -n "$app_gitlab_objects" ]]; then
    log "GitLab APP-context resource snapshot (${app_ctx}):"
    while IFS= read -r _obj; do
      [[ -n "$_obj" ]] || continue
      log "  - ${_obj}"
    done <<< "$app_gitlab_objects"
  else
    warn "GitLab APP-context snapshot has no resources with app.kubernetes.io/instance=${gitlab_instance} yet (this can be transient during first reconcile)."
  fi

  if [[ -n "$db_gitlab_operator_objects" ]]; then
    warn "Detected GitLab operator control-plane resources in DB context '${db_ctx}' (stale/legacy install likely):"
    while IFS= read -r _obj; do
      [[ -n "$_obj" ]] || continue
      warn "  - ${_obj}"
    done <<< "$db_gitlab_operator_objects"
    warn "DB-context GitLab operator resources are not authoritative for APP GitLab workloads in split-cluster mode."
  fi

  if [[ -n "$db_gitlab_app_objects" ]]; then
    local db_app_report=""
    while IFS= read -r _obj; do
      [[ -n "$_obj" ]] || continue
      db_app_report+="- ${_obj}"$'\n'
    done <<< "$db_gitlab_app_objects"
    repair_blocked "GitLab split-cluster ownership violation: DB cluster contains GitLab app workloads" \
      "APP context: ${app_ctx}. DB context: ${db_ctx}. GitLab app resources detected in DB context:\n${db_app_report}Expected policy: GitLab app workloads run only in APP context. Remove stale DB GitLab app workloads and rerun deploy."
  fi
}

kubectl() {
  local target_ctx
  target_ctx="$(resolve_explicit_kube_context || true)"
  if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then
    die "Explicit kubectl context is required in k8s mode."
  fi

  local arg has_context=0
  for arg in "$@"; do
    case "$arg" in
      --context|--context=*|--server|--server=*)
        has_context=1
        break
        ;;
    esac
  done

  if [[ -n "$target_ctx" && $has_context -eq 0 ]]; then
    command kubectl --context "$target_ctx" "$@"
  else
    command kubectl "$@"
  fi
}

resolve_db_cluster_context() {
  if [[ "$MODE" == "k8s" ]]; then
    local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
    [[ -n "$db_ctx" ]] || die "DB_CLUSTER_KUBECONTEXT is required for GitLab DB setup in k8s mode."
    log "Using DB cluster context: ${db_ctx}"
    printf '%s' "$db_ctx"
    return 0
  fi
  return 1
}

repair_blocked() {
  local reason="$1"
  local remediation="${2:-}"
  echo "" >&2
  echo "[REPAIR_BLOCKED] ${reason}" >&2
  if [[ -n "$remediation" ]]; then
    echo "Remediation: ${remediation}" >&2
  fi
  echo "" >&2
  exit 1
}

gitlab_selector_for_deployment() {
  local deployment_name="$1"
  local selector_lines selector=""

  selector_lines=$(kubectl -n "$NAMESPACE" get deployment "$deployment_name" \
    -o go-template='{{range $k, $v := .spec.selector.matchLabels}}{{printf "%s=%s\n" $k $v}}{{end}}' 2>/dev/null || true)
  [[ -n "$selector_lines" ]] || return 1

  local selector_line
  while IFS= read -r selector_line; do
    [[ -n "$selector_line" ]] || continue
    if [[ -n "$selector" ]]; then
      selector+=",${selector_line}"
    else
      selector="$selector_line"
    fi
  done <<< "$selector_lines"

  [[ -n "$selector" ]] || return 1
  printf '%s' "$selector"
}

gitlab_non_terminal_pod_count_for_app() {
  local app_name="$1"
  local pod_selector="$app_name"
  if [[ -z "$pod_selector" ]]; then
    printf '0'
    return 0
  fi
  if [[ "$pod_selector" != *"="* ]]; then
    pod_selector="app=${app_name}"
  fi
  local count
  count=$(kubectl -n "$NAMESPACE" get pods -l "$pod_selector" \
    --field-selector=status.phase!=Succeeded,status.phase!=Failed \
    --no-headers 2>/dev/null | wc -l | xargs || echo "0")
  if [[ -z "$count" || ! "$count" =~ ^[0-9]+$ ]]; then
    count=0
  fi
  printf '%s' "$count"
}

gitlab_non_terminal_pod_names_for_app() {
  local app_name="$1"
  local pod_selector="$app_name"
  if [[ -z "$pod_selector" ]]; then
    return 0
  fi
  if [[ "$pod_selector" != *"="* ]]; then
    pod_selector="app=${app_name}"
  fi
  local names
  names=$(kubectl -n "$NAMESPACE" get pods -l "$pod_selector" \
    --field-selector=status.phase!=Succeeded,status.phase!=Failed \
    -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}' 2>/dev/null || true)
  names="${names% }"
  if [[ -n "$names" ]]; then
    printf '%s' "$names"
  fi
  return 0
}

gitlab_old_replicaset_live_summary() {
  local deployment_name="$1"
  local current_revision rs_rows summary=""

  current_revision=$(kubectl -n "$NAMESPACE" get deployment "$deployment_name" \
    -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)
  [[ -n "$current_revision" ]] || return 0

  rs_rows=$(kubectl -n "$NAMESPACE" get rs -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.ownerReferences[0].kind}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.metadata.annotations.deployment\.kubernetes\.io/revision}{"|"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true)
  [[ -n "$rs_rows" ]] || return 0

  local rs_name owner_kind owner_name rs_revision rs_replicas
  while IFS='|' read -r rs_name owner_kind owner_name rs_revision rs_replicas; do
    [[ -n "$rs_name" ]] || continue
    [[ "$owner_kind" == "Deployment" && "$owner_name" == "$deployment_name" ]] || continue
    [[ "$rs_revision" != "$current_revision" ]] || continue

    if [[ -z "$rs_replicas" || ! "$rs_replicas" =~ ^[0-9]+$ ]]; then
      rs_replicas=0
    fi
    if (( rs_replicas > 0 )); then
      summary+="${rs_name}:${rs_replicas},"
    fi
  done <<< "$rs_rows"

  summary="${summary%,}"
  if [[ -n "$summary" ]]; then
    printf '%s' "$summary"
  fi
  return 0
}

gitlab_replica_source_of_truth_report() {
  local target_replicas="$1"
  local report=""
  local dep_suffix dep_name live_replicas

  for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2"; do
    dep_name="${GITLAB_RELEASE}-${dep_suffix}"
    live_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)

    if [[ -z "$live_replicas" ]]; then
      report+="- ${dep_name}: spec=<missing>, desired=${target_replicas}"$'\n'
      continue
    fi

    if [[ ! "$live_replicas" =~ ^[0-9]+$ ]]; then
      report+="- ${dep_name}: spec=${live_replicas}, desired=${target_replicas}"$'\n'
      continue
    fi

    if [[ "$live_replicas" != "$target_replicas" ]]; then
      report+="- ${dep_name}: spec=${live_replicas}, desired=${target_replicas}"$'\n'
    fi
  done

  printf '%s' "$report"
}

gitlab_rendered_replica_source_fields_from_cr() {
  local desired_cr="$1"
  [[ -n "$desired_cr" ]] || return 0

  echo "$desired_cr" | awk '
BEGIN {
  in_values = 0
  component = ""
}
{
  line = $0
  if (line ~ /^    values:[[:space:]]*$/) {
    in_values = 1
    next
  }
  if (!in_values) {
    next
  }

  if (line ~ /^        webservice:[[:space:]]*$/) {
    component = "gitlab.webservice"
    next
  }
  if (line ~ /^        sidekiq:[[:space:]]*$/) {
    component = "gitlab.sidekiq"
    next
  }
  if (line ~ /^        gitlab-shell:[[:space:]]*$/) {
    component = "gitlab.gitlab-shell"
    next
  }
  if (line ~ /^        kas:[[:space:]]*$/) {
    component = "gitlab.kas"
    next
  }
  if (line ~ /^      registry:[[:space:]]*$/) {
    component = "registry"
    next
  }

  if (line ~ /^      gitaly:[[:space:]]*$/ || line ~ /^      postgresql:[[:space:]]*$/ || line ~ /^      redis:[[:space:]]*$/) {
    component = ""
  }
  if (line ~ /^        gitaly:[[:space:]]*$/ || line ~ /^        toolbox:[[:space:]]*$/) {
    component = ""
  }

  if (component != "" && line ~ /^[[:space:]]*(replicaCount|minReplicas|maxReplicas|hpa):[[:space:]]*/) {
    sub(/^[[:space:]]+/, "", line)
    print "- " component "." line
  }
}'
}

gitlab_verify_replica_source_of_truth() {
  local target_replicas="$1"
  local verify_timeout_s="${GITLAB_SOURCE_REPLICA_VERIFY_TIMEOUT:-180}"
  local verify_poll_interval_s="${GITLAB_SOURCE_REPLICA_VERIFY_POLL_INTERVAL:-10}"
  local rendered_replica_fields

  rendered_replica_fields="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")"
  if [[ -z "$rendered_replica_fields" ]]; then
    rendered_replica_fields="- (no replica-related fields detected in rendered CR values)"
  fi

  if [[ -z "$verify_timeout_s" || ! "$verify_timeout_s" =~ ^[0-9]+$ || "$verify_timeout_s" == "0" ]]; then
    verify_timeout_s=180
  fi
  if [[ -z "$verify_poll_interval_s" || ! "$verify_poll_interval_s" =~ ^[0-9]+$ || "$verify_poll_interval_s" == "0" ]]; then
    verify_poll_interval_s=10
  fi

  local verify_start_ts verify_now_ts mismatch_report
  verify_start_ts=$(date +%s)
  while true; do
    mismatch_report=$(gitlab_replica_source_of_truth_report "$target_replicas")
    if [[ -z "$mismatch_report" ]]; then
      log "GitLab operator desired replicas are source-of-truth converged (gitlab-shell/kas/registry/sidekiq spec=${target_replicas})."
      return 0
    fi

    verify_now_ts=$(date +%s)
    if (( verify_now_ts - verify_start_ts >= verify_timeout_s )); then
      repair_blocked "GitLab operator desired replica source-of-truth mismatch" \
        "Operator-managed Deployment specs did not converge to desired=${target_replicas} after GitLab CR apply:\n${mismatch_report}Rendered GitLab CR replica source fields:\n${rendered_replica_fields}\nThis is a source-of-truth issue (CR values still resolve to replicas>1), not a rollout lag issue."
    fi

    log "Waiting for GitLab operator to apply source-of-truth replicas (desired=${target_replicas}) before settle verification..."
    while IFS= read -r mismatch_line; do
      [[ -n "$mismatch_line" ]] || continue
      log "  ${mismatch_line}"
    done <<< "$mismatch_report"
    sleep "$verify_poll_interval_s"
  done
}

gitlab_webservice_blocked_reasons() {
  local webservice_app="$1"
  local webservice_selector="$webservice_app"
  if [[ -z "$webservice_selector" ]]; then
    return 0
  fi
  if [[ "$webservice_selector" != *"="* ]]; then
    webservice_selector="app=${webservice_app}"
  fi
  local not_ready_block_s="${GITLAB_WEBSERVICE_NOT_READY_BLOCK_SECONDS:-180}"
  if [[ -z "$not_ready_block_s" || ! "$not_ready_block_s" =~ ^[0-9]+$ ]]; then
    not_ready_block_s=180
  fi
  local pod_names
  pod_names=$(kubectl -n "$NAMESPACE" get pods -l "$webservice_selector" \
    --field-selector=status.phase!=Succeeded,status.phase!=Failed \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
  [[ -n "$pod_names" ]] || return 0

  local pod_name
  while IFS= read -r pod_name; do
    [[ -n "$pod_name" ]] || continue

    local pod_phase pod_reason
    pod_phase=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
    pod_reason=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.reason}' 2>/dev/null || true)

    if [[ "$pod_reason" == "ContainerStatusUnknown" || "$pod_phase" == "Unknown" ]]; then
      printf '%s\n' "pod ${pod_name} status is ${pod_reason:-$pod_phase}"
    fi

    local pod_ready_status pod_ready_transition
    pod_ready_status=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
    pod_ready_transition=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].lastTransitionTime}' 2>/dev/null || true)

    local container_rows
    container_rows=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"|"}{.state.waiting.reason}{"|"}{.state.terminated.reason}{"\n"}{end}' 2>/dev/null || true)
    [[ -n "$container_rows" ]] || continue

    local container_name waiting_reason terminated_reason
    while IFS='|' read -r container_name waiting_reason terminated_reason; do
      [[ -n "$container_name" ]] || continue

      case "$waiting_reason" in
        CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerError|CreateContainerConfigError|RunContainerError|Error)
          printf '%s\n' "pod ${pod_name}/${container_name} waiting reason=${waiting_reason}"
          ;;
      esac
      case "$terminated_reason" in
        Error|ContainerCannotRun)
          printf '%s\n' "pod ${pod_name}/${container_name} terminated reason=${terminated_reason}"
          ;;
      esac
    done <<< "$container_rows"

    if [[ "$pod_ready_status" != "True" ]]; then
      local ready_false_age_s=0
      if [[ -n "$pod_ready_transition" ]]; then
        ready_false_age_s=$(python3 - "$pod_ready_transition" <<'PY'
import datetime
import sys

raw = (sys.argv[1] if len(sys.argv) > 1 else "").strip()
if not raw:
    print("0")
    raise SystemExit(0)

try:
    ts = datetime.datetime.fromisoformat(raw.replace("Z", "+00:00"))
    now = datetime.datetime.now(datetime.timezone.utc)
    age = int((now - ts).total_seconds())
    print(str(max(age, 0)))
except Exception:
    print("0")
PY
)
      fi
      if [[ -z "$ready_false_age_s" || ! "$ready_false_age_s" =~ ^[0-9]+$ ]]; then
        ready_false_age_s=0
      fi
      if (( ready_false_age_s >= not_ready_block_s )); then
        printf '%s\n' "pod ${pod_name} Ready=False for ${ready_false_age_s}s (threshold=${not_ready_block_s}s)"
      fi
    fi
  done <<< "$pod_names"
}

wait_for_gitlab_workload_convergence() {
  local timeout_s="${GITLAB_WORKLOAD_CONVERGENCE_TIMEOUT:-420}"
  local poll_interval_s="${GITLAB_WORKLOAD_CONVERGENCE_POLL_INTERVAL:-10}"

  if [[ -z "$timeout_s" || ! "$timeout_s" =~ ^[0-9]+$ ]]; then
    timeout_s=420
  fi
  if [[ -z "$poll_interval_s" || ! "$poll_interval_s" =~ ^[0-9]+$ || "$poll_interval_s" == "0" ]]; then
    poll_interval_s=10
  fi

  local start_ts
  start_ts=$(date +%s)
  local last_report=""

  while true; do
    local -a blockers=()
    local dep_suffix dep_name dep_selector desired_replicas live_non_terminal old_rs_summary
    local ready_replicas available_replicas
    for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
      dep_name="${GITLAB_RELEASE}-${dep_suffix}"

      desired_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
      if [[ -z "$desired_replicas" ]]; then
        blockers+=("${dep_name}: deployment not found")
        continue
      fi
      if [[ ! "$desired_replicas" =~ ^[0-9]+$ ]]; then
        desired_replicas=1
      fi

      dep_selector=$(gitlab_selector_for_deployment "$dep_name" 2>/dev/null || true)
      if [[ -z "$dep_selector" ]]; then
        blockers+=("${dep_name}: deployment selector is empty")
        continue
      fi

      live_non_terminal=$(gitlab_non_terminal_pod_count_for_app "$dep_selector")
      if (( live_non_terminal > desired_replicas )); then
        blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} > desired=${desired_replicas} (selector=${dep_selector})")
      elif (( live_non_terminal < desired_replicas )); then
        blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} < desired=${desired_replicas} (selector=${dep_selector})")
      fi

      ready_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
      available_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
      if [[ -z "$ready_replicas" || ! "$ready_replicas" =~ ^[0-9]+$ ]]; then
        ready_replicas=0
      fi
      if [[ -z "$available_replicas" || ! "$available_replicas" =~ ^[0-9]+$ ]]; then
        available_replicas=0
      fi
      if (( ready_replicas < desired_replicas )); then
        blockers+=("${dep_name}: readyReplicas=${ready_replicas} < desired=${desired_replicas}")
      fi
      if (( available_replicas < desired_replicas )); then
        blockers+=("${dep_name}: availableReplicas=${available_replicas} < desired=${desired_replicas}")
      fi

      old_rs_summary=$(gitlab_old_replicaset_live_summary "$dep_name")
      if [[ -n "$old_rs_summary" ]]; then
        blockers+=("${dep_name}: old ReplicaSet pods still running (${old_rs_summary})")
      fi

      if [[ "$dep_suffix" == "webservice-default" ]]; then
        local webservice_reasons
        webservice_reasons=$(gitlab_webservice_blocked_reasons "$dep_selector" 2>/dev/null || true)
        if [[ -n "$webservice_reasons" ]]; then
          local webservice_reason
          while IFS= read -r webservice_reason; do
            [[ -n "$webservice_reason" ]] || continue
            blockers+=("${dep_name}: ${webservice_reason}")
          done <<< "$webservice_reasons"
        fi
      fi
    done

    if (( ${#blockers[@]} == 0 )); then
      log "GitLab workloads converged: pod counts, ReplicaSets, and webservice health are clean."
      return 0
    fi

    local report=""
    local blocker
    for blocker in "${blockers[@]}"; do
      report+="- ${blocker}"$'\n'
    done

    local now_ts
    now_ts=$(date +%s)
    if (( now_ts - start_ts >= timeout_s )); then
      repair_blocked "GitLab workloads are not converged after reconcile" \
        "Namespace: ${NAMESPACE}. Remaining blockers:
${report}Check: kubectl -n ${NAMESPACE} get deploy,rs,pods -o wide"
    fi

    if [[ "$report" != "$last_report" ]]; then
      warn "Waiting for GitLab workload convergence..."
      local report_line
      while IFS= read -r report_line; do
        [[ -n "$report_line" ]] || continue
        warn "  ${report_line}"
      done <<< "$report"
      last_report="$report"
    fi
    sleep "$poll_interval_s"
  done
}

get_gitlab_migrations_diagnostics() {
  local job_name="$1"
  [[ -n "$job_name" ]] || return 0

  local _pod_name=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$job_name" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
  if [[ -z "$_pod_name" ]]; then
    echo "No pods found for migration job $job_name."
    return 0
  fi

  local _diag=""
  _diag+="--- Migrations Diagnostics for Pod: $_pod_name ---"$'\n'
  _diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o wide 2>/dev/null || true)"$'\n'

  _diag+=$'\n'"[Container Statuses]"$'\n'
  _diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}: state={.state.waiting.reason}{.state.terminated.reason}{.state.running.startedAt}, exitCode={.state.terminated.exitCode}, restarts={.restartCount}{"\n"}{end}' 2>/dev/null || true)"$'\n'

  _diag+=$'\n'"[Migrations Logs (tail=100)]"$'\n'
  _diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --tail=100 2>/dev/null || echo "(no logs available)")"$'\n'

  _diag+=$'\n'"[Previous Migrations Logs (if any)]"$'\n'
  _diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --previous --tail=100 2>/dev/null || echo "(no previous logs)")"$'\n'

  _diag+=$'\n'"[Pod Events]"$'\n'
  _diag+="$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$_pod_name" --sort-by='.lastTimestamp' 2>/dev/null | tail -n 10 || true)"

  echo "$_diag"
}

check_gitlab_migrations_blocked() {
  local _latest_job
  _latest_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
  [[ -n "$_latest_job" ]] || return 0

  local _job_failed
  _job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
  
  # 1. Active Failure Detection
  local _latest_pod
  _latest_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
  if [[ -n "$_latest_pod" ]]; then
     local _restarts
     _restarts=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
     local _waiting_reason
     _waiting_reason=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
     local _exit_code
     _exit_code=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.terminated.exitCode}' 2>/dev/null || echo "")

     if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_exit_code" && "$_exit_code" != "0" ) ]]; then
        log "GitLab migrations job is actively failing: ${_latest_job} (pod: ${_latest_pod}, restarts: ${_restarts}, exitCode: ${_exit_code:-unknown})"
        
        # Enhanced DB connectivity diagnostics for split-cluster visibility
        local _db_info="DB_HOST=${DB_HOST}, DB_PORT=${DB_PORT}"
        local _split_cluster="No"
        local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
        local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
        if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then _split_cluster="Yes (APP: ${app_ctx}, DB: ${db_ctx})"; fi

        local _diag
        _diag=$(get_gitlab_migrations_diagnostics "$_latest_job")
        
        local _connectivity_hint=""
        if echo "$_diag" | grep -qi "connecting with your hostname"; then
           _connectivity_hint=" (Likely DNS/resolution failure)"
        elif echo "$_diag" | grep -qiE "connection refused|timeout"; then
           _connectivity_hint=" (Likely TCP connectivity/firewall failure)"
        fi

        repair_blocked "GitLab migrations job is actively failing" \
          "Job: ${_latest_job}. Pod: ${_latest_pod}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_exit_code:-unknown}. Restarts: ${_restarts}.
DB Config: ${_db_info}${_connectivity_hint}
Split-cluster: ${_split_cluster}
Diagnostics:
${_diag}"
     fi
  fi

  # 2. Stale Failed Job Repair
  local _job_creation_ts
  _job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
  local _job_pods_running
  _job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
  local _job_pods_pending
  _job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
  
  local _now_ts
  _now_ts=$(date +%s)
  local _creation_ts
  _creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0")
  local _age
  _age=$(( _now_ts - _creation_ts ))

  if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then
    log "GitLab operator is stalled on stale failed migrations job: ${_latest_job} (age: ${_age}s, no active pods)"
    log "Deleting stale failed migrations job to trigger repair..."
    kubectl -n "$NAMESPACE" delete job "$_latest_job" --wait=true 2>/dev/null || true
    kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_job" --force --grace-period=0 2>/dev/null || true
    log "Stale migrations job deleted. Operator should recreate it shortly."
  fi
}

check_host_resolves() {
  local target_host="$1"
  # Strip port if present
  local host_only="${target_host%:*}"
  local target_ctx
  target_ctx="$(resolve_explicit_kube_context || true)"

  local is_k8s_svc=0
  if [[ "$host_only" == *".svc.cluster.local" ]]; then
    is_k8s_svc=1
  elif [[ "$host_only" =~ ^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+$ ]]; then
    # Simple <service>.<namespace> format (exactly one dot, no numbers to avoid IPs)
    is_k8s_svc=1
  fi

  if [[ $is_k8s_svc -eq 1 ]]; then
    # Parse service and namespace (first and second labels)
    local svc_name svc_ns
    svc_name=$(echo "$host_only" | cut -d. -f1)
    svc_ns=$(echo "$host_only" | cut -d. -f2)

    log "Redis validation mode: kubernetes-service (cluster: ${target_ctx:-default})"
    log "Parsed REDIS_HOST ${target_host} -> service=${svc_name} namespace=${svc_ns}"

    if kubectl -n "$svc_ns" get service "$svc_name" >/dev/null 2>&1; then
      log "Service ${svc_name} found in namespace ${svc_ns}"
      # Preferably verify it has endpoints / ready backing pods
      local endpoints_found
      endpoints_found=$(kubectl -n "$svc_ns" get endpoints "$svc_name" -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
      if [[ -n "$endpoints_found" ]]; then
        log "Redis endpoints present for ${svc_name}"
      else
        warn "Service ${svc_name} found in namespace ${svc_ns} but has NO ready endpoints (yet)."
      fi
      return 0
    else
      warn "Service ${svc_name} NOT found in namespace ${svc_ns} (cluster: ${target_ctx:-default})"
      return 1
    fi
  else
    log "Redis validation mode: external-host"
    if ! host "$host_only" >/dev/null 2>&1; then
      # If it's an IP, host might fail. Check if it's an IP.
      if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
         log "Host ${host_only} is an IP address, skipping DNS check."
         return 0
      fi
      return 1
    fi
  fi
  return 0
}

check_db_connectivity() {
  local target_host="$1"
  local target_port="${2:-5432}"
  local host_only="${target_host%:*}"

  log "Validating GitLab database connectivity to ${target_host}:${target_port}..."

  # DNS sanity check
  if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
    log "Host ${host_only} is an IP address, skipping DNS check."
  else
    # In k8s mode, internal DNS resolution might fail from the script runner
    # but work from within the cluster. We do a basic 'host' check as a hint.
    if ! host "$host_only" >/dev/null 2>&1; then
      warn "DNS resolution (host) failed for database host: ${host_only}. This may be expected if only resolvable in-cluster."
    else
      log "DNS resolution successful for ${host_only}"
    fi
  fi

  # TCP check via temporary pod in the APP cluster namespace
  log "Performing in-cluster TCP connectivity probe to ${host_only}:${target_port} (namespace: ${NAMESPACE})..."
  if kubectl -n "$NAMESPACE" run db-probe \
    --image=alpine:latest --restart=Never --rm --attach --timeout=30s \
    --command -- sh -c "nc -zv -w 5 ${host_only} ${target_port}" >/dev/null 2>&1; then
    log "TCP connectivity to ${host_only}:${target_port} SUCCESSFUL."
    return 0
  else
    warn "TCP connectivity probe to ${host_only}:${target_port} FAILED."
    return 1
  fi
}

check_gitlab_pre_apply_blocked() {
  # --- Registry Endpoint Placeholder Check ---
  if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
    # If the configured value itself is still a placeholder, we MUST block.
    # The operator hasn't provided a real value yet.
    repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
      "GARAGE_PRIVATE_S3_ENDPOINT is set to the literal placeholder. Set it to the real DB cluster Garage endpoint."
  fi

  # --- Redis Host Resolution Check ---
  if ! check_host_resolves "$REDIS_HOST"; then
     # This is potentially repairable if we can infer a better host,
     # but for now we block if it's clearly invalid.
     repair_blocked "Redis host does not resolve" \
        "Resource: REDIS_HOST. Value: ${REDIS_HOST} does not resolve. Fix: Ensure Redis is deployed and REDIS_NAMESPACE is correct."
  fi

  # --- Database Connectivity Check ---
  if [[ "$MODE" == "k8s" ]]; then
     if ! check_db_connectivity "$DB_HOST" "$DB_PORT"; then
        local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
        local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
        repair_blocked "GitLab database host is not reachable from APP cluster" \
          "Host: ${DB_HOST}. Port: ${DB_PORT}. App Context: ${app_ctx}. DB Context: ${db_ctx}. Fix: Ensure cross-cluster networking (ILB) is functional."
     fi
  fi
}

get_gitlab_blocker_context() {
  local ctx=""
  local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
  local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
  if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then
     ctx+="Split-cluster (APP: ${app_ctx}, DB: ${db_ctx}). "
  fi
  ctx+="DB_HOST: ${DB_HOST}. "

  local _last_mig=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
  if [[ -n "$_last_mig" ]]; then
    local _m_ts=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
    local _m_failed=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
    local _m_pods=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --no-headers 2>/dev/null | wc -l | xargs || echo "0")
    
    local _m_pod_ctx=""
    local _m_last_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
    if [[ -n "$_m_last_pod" ]]; then
       local _m_pod_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.phase}' 2>/dev/null || true)
       local _m_restarts=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
       local _m_waiting=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
       _m_pod_ctx="Pod: ${_m_last_pod} (${_m_pod_status}, restarts: ${_m_restarts}, waiting: ${_m_waiting:-None}). "
    fi
    ctx+="Migration Job: ${_last_mig} (Created: ${_m_ts}, Failed: ${_m_failed:-False}, Pods: ${_m_pods}). ${_m_pod_ctx}"
  fi

  local _sts_name="${GITLAB_RELEASE}-gitaly"
  local _sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$_sts_name" -o yaml 2>/dev/null || true)
  if [[ -n "$_sts_yaml" ]]; then
    local _sts_ns=$(echo "$_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
    local _sts_sc=$(echo "$_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
    ctx+="Gitaly STS: nodeSelector=[${_sts_ns}], storageClass=[${_sts_sc}]. "
  fi

  local _pvc_name="repo-data-gitlab-gitaly-0"
  local _pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$_pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
  if [[ -n "$_pvc_phase" ]]; then
    ctx+="Gitaly PVC: ${_pvc_phase}. "
  fi

  local _op_pod=$(kubectl -n "$NAMESPACE" get pods -l "control-plane=controller-manager" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -n "$_op_pod" ]]; then
    local _op_log=$(kubectl -n "$NAMESPACE" logs "$_op_pod" -c manager --tail=1 2>/dev/null || true)
    ctx+="Operator Log: ${_op_log}. "
  fi
  echo "$ctx"
}

gitlab_storage_class_matches_expected() {
  local live_sc="$1"
  local expected_sc="$2"
  [[ -z "$expected_sc" ]] && return 0
  [[ "$live_sc" == "$expected_sc" ]]
}

gitlab_rendered_storage_fields_from_cr() {
  local desired_cr="$1"
  local rendered_global_sc=""
  local rendered_gitlab_gitaly_sc=""
  local rendered_chart_gitaly_sc=""

  if [[ -n "$desired_cr" ]]; then
    rendered_global_sc=$(echo "$desired_cr" | sed -n '/^      global:/,/^      postgresql:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true)
    rendered_gitlab_gitaly_sc=$(echo "$desired_cr" | sed -n '/^        gitaly:/,/^        toolbox:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true)
    rendered_chart_gitaly_sc=$(echo "$desired_cr" | sed -n '/^      gitaly:/,/^      registry:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true)
  fi

  printf '%s|%s|%s\n' "$rendered_global_sc" "$rendered_gitlab_gitaly_sc" "$rendered_chart_gitaly_sc"
}

gitlab_chart_major_version() {
  local chart_version="${GITLAB_CHART_VERSION:-}"
  chart_version="${chart_version#v}"
  chart_version="${chart_version%%-*}"
  chart_version="${chart_version%%.*}"
  printf '%s' "$chart_version"
}

gitlab_chart_is_v9_or_newer() {
  local chart_major
  chart_major="$(gitlab_chart_major_version)"
  [[ "$chart_major" =~ ^[0-9]+$ ]] || return 1
  (( chart_major >= 9 ))
}

gitlab_chart9_find_deprecated_top_level_replica_keys() {
  local desired_cr="$1"
  [[ -n "$desired_cr" ]] || return 0

  echo "$desired_cr" | awk '
function leading_spaces(str,   i, c, n) {
  n = 0
  for (i = 1; i <= length(str); i++) {
    c = substr(str, i, 1)
    if (c == " ") {
      n++
    } else {
      break
    }
  }
  return n
}
BEGIN {
  in_values = 0
  in_registry = 0
  registry_indent = -1
  in_hpa = 0
  hpa_indent = -1
}
{
  line = $0
  if (line ~ /^    values:[[:space:]]*$/) {
    in_values = 1
    next
  }
  if (!in_values) {
    next
  }

  indent = leading_spaces(line)
  if (in_hpa && indent <= hpa_indent) {
    in_hpa = 0
    hpa_indent = -1
  }
  if (in_registry && indent <= registry_indent && line !~ /^      registry:[[:space:]]*$/) {
    in_registry = 0
    registry_indent = -1
  }

  if (line ~ /^      registry:[[:space:]]*$/) {
    in_registry = 1
    registry_indent = indent
    next
  }
  if (!in_registry) {
    next
  }

  if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) {
    in_hpa = 1
    hpa_indent = indent
    next
  }

  if (!in_hpa && line ~ /^[[:space:]]*(minReplicas|maxReplicas):[[:space:]]*/) {
    print NR ":" line
  }
}'
}

gitlab_chart9_strip_deprecated_top_level_replica_keys() {
  local desired_cr="$1"
  [[ -n "$desired_cr" ]] || {
    printf '%s' "$desired_cr"
    return 0
  }

  echo "$desired_cr" | awk '
function leading_spaces(str,   i, c, n) {
  n = 0
  for (i = 1; i <= length(str); i++) {
    c = substr(str, i, 1)
    if (c == " ") {
      n++
    } else {
      break
    }
  }
  return n
}
BEGIN {
  in_values = 0
  in_registry = 0
  registry_indent = -1
  in_hpa = 0
  hpa_indent = -1
}
{
  line = $0
  if (line ~ /^    values:[[:space:]]*$/) {
    in_values = 1
    print line
    next
  }
  if (!in_values) {
    print line
    next
  }

  indent = leading_spaces(line)
  if (in_hpa && indent <= hpa_indent) {
    in_hpa = 0
    hpa_indent = -1
  }
  if (in_registry && indent <= registry_indent && line !~ /^      registry:[[:space:]]*$/) {
    in_registry = 0
    registry_indent = -1
  }

  if (line ~ /^      registry:[[:space:]]*$/) {
    in_registry = 1
    registry_indent = indent
    print line
    next
  }
  if (!in_registry) {
    print line
    next
  }

  if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) {
    in_hpa = 1
    hpa_indent = indent
    print line
    next
  }

  if (!in_hpa && line ~ /^[[:space:]]*(minReplicas|maxReplicas):[[:space:]]*/) {
    next
  }

  print line
}'
}

sanitize_gitlab_cr_rendered_values_for_chart_version() {
  local desired_cr="$1"
  [[ -n "$desired_cr" ]] || {
    printf '%s' "$desired_cr"
    return 0
  }

  if ! gitlab_chart_is_v9_or_newer; then
    printf '%s' "$desired_cr"
    return 0
  fi

  local deprecated_keys_before
  deprecated_keys_before="$(gitlab_chart9_find_deprecated_top_level_replica_keys "$desired_cr")"
  if [[ -n "$deprecated_keys_before" ]]; then
    log "GitLab chart ${GITLAB_CHART_VERSION}: stripping deprecated top-level replica keys before CR apply:"
    while IFS= read -r deprecated_key; do
      [[ -n "$deprecated_key" ]] || continue
      log "  - ${deprecated_key}"
    done <<< "$deprecated_keys_before"
    desired_cr="$(gitlab_chart9_strip_deprecated_top_level_replica_keys "$desired_cr")"
  fi

  printf '%s' "$desired_cr"
}

preflight_validate_gitlab_cr_rendered_values() {
  local desired_cr="$1"
  [[ -n "$desired_cr" ]] || return 0

  if ! gitlab_chart_is_v9_or_newer; then
    return 0
  fi

  local deprecated_keys_remaining
  deprecated_keys_remaining="$(gitlab_chart9_find_deprecated_top_level_replica_keys "$desired_cr")"
  if [[ -n "$deprecated_keys_remaining" ]]; then
    repair_blocked "Rendered GitLab CR values contain removed chart v9+ keys" \
      "Local preflight rejected GitLab CR apply. Deprecated top-level replica keys remain (for example registry.minReplicas/registry.maxReplicas). Remaining keys: ${deprecated_keys_remaining//$'\n'/; }."
  fi
}

gitlab_live_gitaly_repo_data_template_storage_class() {
  local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
  local live_template_sc
  live_template_sc=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o jsonpath='{range .spec.volumeClaimTemplates[?(@.metadata.name=="repo-data")]}{.spec.storageClassName}{end}' 2>/dev/null || true)
  if [[ -z "$live_template_sc" ]]; then
    live_template_sc=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o jsonpath='{.spec.volumeClaimTemplates[0].spec.storageClassName}' 2>/dev/null || true)
  fi
  printf '%s' "$live_template_sc"
}

cleanup_gitlab_wrong_gitaly_template_storage() {
  local expected_sc="$1"
  local live_template_sc="$2"
  local pvc_mismatch_records="$3"
  local desired_cr="$4"
  local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"

  export GITALY_AUTOCLEAN_PERFORMED=1
  log "AUTOCLEAN: repairing wrong-class Gitaly StatefulSet template storageClass (expected='${expected_sc}', live='${live_template_sc:-<unset>}')."
  log "Rendered GitLab CR storage fields prior to repair: $(gitlab_rendered_storage_fields_from_cr "$desired_cr")"

  kubectl -n "$NAMESPACE" scale statefulset "$gitaly_sts_name" --replicas=0 --timeout=30s 2>/dev/null || true
  if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then
    log "Deleting StatefulSet ${gitaly_sts_name} to force operator recreation of repo-data claim template..."
    kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true
  fi

  if [[ -n "$pvc_mismatch_records" ]]; then
    cleanup_gitlab_wrong_storage_class_records "$pvc_mismatch_records"
  fi

  # Also clear retained stale GKE dynamic-provisioning leftovers for repo-data claim.
  repair_stale_gke_gitaly_dynamic_storage "$expected_sc"
}

is_gke_stale_gitaly_storage_class_for_standard_target() {
  local desired_sc="$1"
  local candidate_sc="$2"
  [[ "$desired_sc" == "standard" ]] || return 1
  [[ -n "$candidate_sc" && "$candidate_sc" != "$desired_sc" ]] && return 0
  return 1
}

is_gke_blocking_disk_type_for_standard_target() {
  local disk_type="$1"
  case "$disk_type" in
    pd-balanced|pd-ssd)
      return 0
      ;;
  esac
  return 1
}

gke_fetch_disk_type_from_volume_handle() {
  local volume_handle="$1"
  [[ -n "$volume_handle" ]] || return 0
  command -v gcloud >/dev/null 2>&1 || return 0

  local parsed_ref
  parsed_ref=$(gke_parse_disk_ref_from_volume_handle "$volume_handle" || true)
  [[ -n "$parsed_ref" ]] || return 0

  local disk_scope disk_location disk_name
  IFS='|' read -r disk_scope disk_location disk_name <<< "$parsed_ref"
  [[ -n "$disk_name" ]] || return 0

  local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
  if [[ -z "$project_id" ]]; then
    project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
  fi

  local -a common_args=()
  if [[ -n "$project_id" ]]; then
    common_args+=(--project "$project_id")
  fi

  local disk_type=""
  if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then
    disk_type=$(gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" --format='value(type.basename())' 2>/dev/null || true)
  elif [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then
    disk_type=$(gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" --format='value(type.basename())' 2>/dev/null || true)
  else
    local matched
    matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](type.basename())' 2>/dev/null | head -n1 || true)
    disk_type="$matched"
  fi

  printf '%s' "$disk_type"
}

collect_gitlab_storage_mismatch_records() {
  local expected_sc="$1"
  [[ -n "$expected_sc" ]] || return 0

  local pvc_rows
  pvc_rows=$(kubectl -n "$NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.storageClassName}{"|"}{.status.phase}{"|"}{.spec.volumeName}{"|"}{.metadata.labels.app\.kubernetes\.io/instance}{"\n"}{end}' 2>/dev/null || true)
  [[ -n "$pvc_rows" ]] || return 0

  while IFS='|' read -r pvc_name pvc_sc pvc_phase pvc_pv pvc_instance; do
    [[ -n "$pvc_name" ]] || continue

    local is_gitlab_pvc=0
    if [[ "$pvc_instance" == "$GITLAB_RELEASE" || "$pvc_name" == "repo-data-${GITLAB_RELEASE}-"* || "$pvc_name" == "${GITLAB_RELEASE}-"* ]]; then
      is_gitlab_pvc=1
    fi
    [[ "$is_gitlab_pvc" == "1" ]] || continue

    local pv_sc="" disk_handle="" disk_type="" disk_scope="" disk_location="" disk_name=""
    if [[ -n "$pvc_pv" ]]; then
      local pv_details pv_disk_name
      pv_details=$(kubectl get pv "$pvc_pv" -o jsonpath='{.spec.storageClassName}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"|"}{.spec.csi.volumeAttributes.type}' 2>/dev/null || true)
      if [[ -n "$pv_details" ]]; then
        IFS='|' read -r pv_sc disk_handle pv_disk_name disk_type <<< "$pv_details"
      fi

      if [[ -z "$disk_handle" && -n "$pv_disk_name" ]]; then
        disk_handle="$pv_disk_name"
      fi

      if [[ -n "$disk_handle" ]]; then
        local parsed_ref
        parsed_ref=$(gke_parse_disk_ref_from_volume_handle "$disk_handle" || true)
        if [[ -n "$parsed_ref" ]]; then
          IFS='|' read -r disk_scope disk_location disk_name <<< "$parsed_ref"
        fi
      fi

      if [[ -z "$disk_type" && -n "$disk_handle" ]]; then
        disk_type=$(gke_fetch_disk_type_from_volume_handle "$disk_handle" || true)
      fi
    fi

    local effective_live_sc="$pvc_sc"
    if [[ -z "$effective_live_sc" && -n "$pv_sc" ]]; then
      effective_live_sc="$pv_sc"
    fi

    if ! gitlab_storage_class_matches_expected "$effective_live_sc" "$expected_sc"; then
      printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
        "$pvc_name" "$pvc_sc" "$pvc_phase" "$pvc_pv" "$pv_sc" "$disk_handle" "$disk_scope" "$disk_location" "$disk_name" "$disk_type"
    fi
  done <<< "$pvc_rows"
}

collect_gitlab_statefulset_template_mismatch_records() {
  local expected_sc="$1"
  [[ -n "$expected_sc" ]] || return 0

  local sts_rows
  sts_rows=$(kubectl -n "$NAMESPACE" get statefulset -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.labels.app\.kubernetes\.io/instance}{"|"}{range .spec.volumeClaimTemplates[*]}{.metadata.name}{":"}{.spec.storageClassName}{","}{end}{"\n"}{end}' 2>/dev/null || true)
  [[ -n "$sts_rows" ]] || return 0

  while IFS='|' read -r sts_name sts_instance claims; do
    [[ -n "$sts_name" ]] || continue

    local is_gitlab_sts=0
    if [[ "$sts_instance" == "$GITLAB_RELEASE" || "$sts_name" == "${GITLAB_RELEASE}-"* ]]; then
      is_gitlab_sts=1
    fi
    [[ "$is_gitlab_sts" == "1" ]] || continue
    [[ -n "$claims" ]] || continue

    IFS=',' read -ra claim_pairs <<< "$claims"
    local pair
    for pair in "${claim_pairs[@]}"; do
      [[ -n "$pair" ]] || continue
      local claim_name claim_sc
      claim_name="${pair%%:*}"
      claim_sc="${pair#*:}"
      if ! gitlab_storage_class_matches_expected "$claim_sc" "$expected_sc"; then
        printf '%s|%s|%s\n' "$sts_name" "$claim_name" "$claim_sc"
      fi
    done
  done <<< "$sts_rows"
}

cleanup_gitlab_wrong_storage_class_records() {
  local mismatch_records="$1"
  [[ -n "$mismatch_records" ]] || return 0

  export GITALY_AUTOCLEAN_PERFORMED=1
  log "AUTOCLEAN: removing wrong-class GitLab PVC/PV artifacts to enforce storageClass='${GITALY_STORAGE_CLASS:-}'..."

  local processed_disks="|"
  local _disk_key
  while IFS='|' read -r pvc_name _pvc_sc _pvc_phase pvc_pv _pv_sc disk_handle disk_scope disk_location disk_name _disk_type; do
    [[ -n "$pvc_name" ]] || continue

    if [[ "$pvc_name" == "repo-data-${GITLAB_RELEASE}-gitaly-0" ]]; then
      kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
    fi

    if kubectl -n "$NAMESPACE" get pvc "$pvc_name" >/dev/null 2>&1; then
      log "Deleting wrong-class PVC ${pvc_name}..."
      kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true
    fi

    if [[ -n "$pvc_pv" ]] && kubectl get pv "$pvc_pv" >/dev/null 2>&1; then
      log "Deleting wrong-class PV ${pvc_pv} (PVC=${pvc_name})..."
      kubectl delete pv "$pvc_pv" --wait=false 2>/dev/null || true
    fi

    local disk_ref=""
    if [[ -n "$disk_handle" ]]; then
      disk_ref=$(gke_parse_disk_ref_from_volume_handle "$disk_handle" || true)
    fi
    if [[ -n "$disk_ref" ]]; then
      IFS='|' read -r disk_scope disk_location disk_name <<< "$disk_ref"
    fi
    if [[ -z "$disk_name" && -n "$disk_handle" ]]; then
      disk_scope="name"
      disk_location=""
      disk_name="$disk_handle"
    fi

    if [[ -n "$disk_name" ]]; then
      _disk_key="${disk_scope}|${disk_location}|${disk_name}"
      if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
        processed_disks+="${_disk_key}|"
        gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true
      fi
    fi
  done <<< "$mismatch_records"

  sleep 2
}

enforce_gitlab_storage_class_target() {
  local expected_sc="$1"
  local desired_cr="$2"

  [[ "$MODE" == "k8s" ]] || return 0
  [[ -n "$expected_sc" ]] || return 0

  local rendered_global_sc=""
  local rendered_gitlab_gitaly_sc=""
  local rendered_chart_gitaly_sc=""
  IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")"

  log "GitLab storage target: configured='${expected_sc}', rendered.global.persistence.storageClass='${rendered_global_sc:-<unset>}', rendered.gitlab.gitaly.persistence.storageClass='${rendered_gitlab_gitaly_sc:-<unset>}', rendered.gitaly.persistence.storageClass='${rendered_chart_gitaly_sc:-<unset>}'"

  if [[ -n "$rendered_global_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_global_sc" "$expected_sc"; then
    repair_blocked "GitLab CR global persistence storageClass mismatch" \
      "Configured target=[${expected_sc}] but rendered global.persistence.storageClass=[${rendered_global_sc}]. This indicates GitLab CR rendering is wrong."
  fi
  if [[ -n "$rendered_gitlab_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_gitlab_gitaly_sc" "$expected_sc"; then
    repair_blocked "GitLab CR gitaly persistence storageClass mismatch" \
      "Configured target=[${expected_sc}] but rendered gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc}]. This indicates GitLab CR rendering is wrong."
  fi
  if [[ -n "$rendered_chart_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_chart_gitaly_sc" "$expected_sc"; then
    repair_blocked "GitLab CR chart gitaly persistence storageClass mismatch" \
      "Configured target=[${expected_sc}] but rendered gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc}]. This indicates GitLab CR rendering is wrong."
  fi

  local template_mismatches
  template_mismatches=$(collect_gitlab_statefulset_template_mismatch_records "$expected_sc")
  local pvc_mismatches
  pvc_mismatches=$(collect_gitlab_storage_mismatch_records "$expected_sc")

  local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
  local gitaly_sts_exists=0
  if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then
    gitaly_sts_exists=1
  fi

  local live_gitaly_template_sc
  live_gitaly_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class)
  local live_gitaly_pvc_sc
  live_gitaly_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "repo-data-${GITLAB_RELEASE}-gitaly-0" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
  log "GitLab Gitaly live storage snapshot: statefulset=${gitaly_sts_name} claimTemplate.repo-data.storageClassName='${live_gitaly_template_sc:-<unset>}' pvc.repo-data-${GITLAB_RELEASE}-gitaly-0.storageClassName='${live_gitaly_pvc_sc:-<unset>}'"
  if [[ "$gitaly_sts_exists" == "1" ]] && ! gitlab_storage_class_matches_expected "$live_gitaly_template_sc" "$expected_sc"; then
    if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
      repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass mismatch" \
        "Configured target=[${expected_sc}] rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}] live.statefulset=${GITLAB_RELEASE}-gitaly live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_template_sc:-<unset>}]. Delete statefulset/${GITLAB_RELEASE}-gitaly, pvc/repo-data-${GITLAB_RELEASE}-gitaly-0, and any wrong-class bound/released PV+disk artifacts, or enable GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 for automatic destructive repair."
    fi

    cleanup_gitlab_wrong_gitaly_template_storage "$expected_sc" "$live_gitaly_template_sc" "$pvc_mismatches" "$desired_cr"

    # Re-check after cleanup so subsequent gates reflect live post-repair state.
    template_mismatches=$(collect_gitlab_statefulset_template_mismatch_records "$expected_sc")
    pvc_mismatches=$(collect_gitlab_storage_mismatch_records "$expected_sc")
    gitaly_sts_exists=0
    if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then
      gitaly_sts_exists=1
    fi
    live_gitaly_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class)
    if [[ "$gitaly_sts_exists" == "1" ]] && ! gitlab_storage_class_matches_expected "$live_gitaly_template_sc" "$expected_sc"; then
      repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass remains wrong after repair" \
        "Configured target=[${expected_sc}] rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}] live.statefulset=${GITLAB_RELEASE}-gitaly live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_template_sc:-<unset>}]."
    fi
  fi

  if [[ -n "$template_mismatches" ]]; then
    log "GitLab StatefulSet volumeClaimTemplate storageClass diagnostics (expected='${expected_sc}')"
    while IFS='|' read -r sts_name claim_name claim_sc; do
      [[ -n "$sts_name" ]] || continue
      log "  - workload=statefulset/${sts_name} claimTemplate=${claim_name} storageClass=${claim_sc}"
    done <<< "$template_mismatches"
  fi

  if [[ -n "$pvc_mismatches" ]]; then
    log "GitLab PVC/PV storageClass diagnostics (expected='${expected_sc}')"
    while IFS='|' read -r pvc_name pvc_sc pvc_phase pvc_pv pv_sc disk_handle disk_scope disk_location disk_name disk_type; do
      [[ -n "$pvc_name" ]] || continue
      log "  - pvc=${pvc_name} phase=${pvc_phase:-unknown} pvc.storageClass=${pvc_sc:-<unset>} pv=${pvc_pv:-<none>} pv.storageClass=${pv_sc:-<unset>} diskHandle=${disk_handle:-<none>} diskRef=${disk_scope:-<none>}:${disk_location:-<none>}/${disk_name:-<none>} diskType=${disk_type:-<unknown>}"
    done <<< "$pvc_mismatches"
  fi

  if [[ -z "$template_mismatches" && -z "$pvc_mismatches" ]]; then
    return 0
  fi

  if [[ -n "$template_mismatches" ]]; then
    repair_blocked "GitLab StatefulSet claim-template storageClass mismatch" \
      "Configured target=[${expected_sc}] but one or more live GitLab StatefulSet volumeClaimTemplates use a different storageClass. For Gitaly this is a hard mismatch and must be repaired with authoritative CR fields + destructive cleanup gate."
  fi

  if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
    repair_blocked "GitLab storageClass mismatch detected" \
      "Configured target=[${expected_sc}] but one or more GitLab StatefulSet/PVC/PV resources are on a different storageClass. Enable GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to delete wrong-class GitLab PVC/PV/disk artifacts for reprovision."
  fi

  cleanup_gitlab_wrong_storage_class_records "$pvc_mismatches"
}

is_gke_cluster_detected() {
  local provider_ids gke_pool_labels gke_topology_labels
  provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true)
  gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true)
  gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true)
  if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then
    return 0
  fi
  return 1
}

gke_parse_disk_ref_from_volume_handle() {
  local handle="$1"
  [[ -n "$handle" ]] || return 1

  if [[ "$handle" =~ /zones/([^/]+)/disks/([^/]+)$ ]]; then
    printf 'zone|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
    return 0
  fi
  if [[ "$handle" =~ /regions/([^/]+)/disks/([^/]+)$ ]]; then
    printf 'region|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
    return 0
  fi
  if [[ "$handle" =~ ^[^/]+$ ]]; then
    printf 'name||%s' "$handle"
    return 0
  fi
  return 1
}

gke_delete_disk_ref_if_present() {
  local disk_scope="$1"
  local disk_location="$2"
  local disk_name="$3"
  [[ -n "$disk_name" ]] || return 0

  if ! command -v gcloud >/dev/null 2>&1; then
    warn "gcloud not found; cannot auto-delete stale disk '${disk_name}'."
    return 1
  fi

  local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
  if [[ -z "$project_id" ]]; then
    project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
  fi

  local -a common_args=(--quiet)
  if [[ -n "$project_id" ]]; then
    common_args+=(--project "$project_id")
  fi

  if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then
    if gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
      log "Deleting stale GKE disk ${disk_name} (zone=${disk_location})..."
      gcloud compute disks delete "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${disk_location})."
    else
      log "Stale disk ${disk_name} already absent in zone ${disk_location}."
    fi
    return 0
  fi

  if [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then
    if gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
      log "Deleting stale GKE regional disk ${disk_name} (region=${disk_location})..."
      gcloud compute disks delete "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${disk_location})."
    else
      log "Stale regional disk ${disk_name} already absent in region ${disk_location}."
    fi
    return 0
  fi

  local matched
  matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](zone.basename(),region.basename())' 2>/dev/null || true)
  if [[ -z "$matched" ]]; then
    log "Stale disk ${disk_name} already absent."
    return 0
  fi

  while IFS='|' read -r zone_name region_name; do
    [[ -n "$zone_name" || -n "$region_name" ]] || continue
    if [[ -n "$zone_name" ]]; then
      log "Deleting stale GKE disk ${disk_name} (zone=${zone_name})..."
      gcloud compute disks delete "$disk_name" --zone "$zone_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${zone_name})."
    elif [[ -n "$region_name" ]]; then
      log "Deleting stale GKE regional disk ${disk_name} (region=${region_name})..."
      gcloud compute disks delete "$disk_name" --region "$region_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${region_name})."
    fi
  done <<< "$matched"
}

collect_stale_gke_gitaly_pv_records() {
  local desired_sc="$1"
  [[ "$desired_sc" == "standard" ]] || return 0

  local pvc_name="repo-data-gitlab-gitaly-0"
  local pvc_phase
  pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)

  local pv_rows
  pv_rows=$(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"|"}{.spec.storageClassName}{"|"}{.spec.claimRef.namespace}{"|"}{.spec.claimRef.name}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"\n"}{end}' 2>/dev/null || true)
  [[ -n "$pv_rows" ]] || return 0

  while IFS='|' read -r pv_name pv_phase pv_sc pv_claim_ns pv_claim_name pv_handle pv_gce_pd; do
    [[ -n "$pv_name" ]] || continue
    [[ "$pv_claim_ns" == "$NAMESPACE" && "$pv_claim_name" == "$pvc_name" ]] || continue
    is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$pv_sc" || continue

    local stale_reason=""
    if [[ "$pv_phase" == "Released" || "$pv_phase" == "Failed" ]]; then
      stale_reason="pv-phase-${pv_phase}"
    elif [[ -z "$pvc_phase" ]]; then
      stale_reason="missing-live-pvc"
    fi
    [[ -n "$stale_reason" ]] || continue

    printf '%s|%s|%s|%s|%s|%s\n' "$pv_name" "$pv_phase" "$pv_sc" "$pv_handle" "$pv_gce_pd" "$stale_reason"
  done <<< "$pv_rows"
}

collect_stale_gke_gitaly_disk_records_without_pv() {
  local desired_sc="$1"
  [[ "$desired_sc" == "standard" ]] || return 0
  command -v gcloud >/dev/null 2>&1 || return 0

  local pvc_name="repo-data-gitlab-gitaly-0"
  local pvc_phase
  pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
  [[ -z "$pvc_phase" ]] || return 0

  local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
  if [[ -z "$project_id" ]]; then
    project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
  fi
  local -a common_args=(--quiet)
  if [[ -n "$project_id" ]]; then
    common_args+=(--project "$project_id")
  fi

  local rows
  rows=$(gcloud compute disks list "${common_args[@]}" \
    --filter="labels.kubernetes-io-created-for-pvc-name=${pvc_name} AND labels.kubernetes-io-created-for-pvc-namespace=${NAMESPACE}" \
    --format='csv[no-heading,separator="|"](name,zone.basename(),region.basename(),type.basename())' 2>/dev/null || true)
  [[ -n "$rows" ]] || return 0

  while IFS='|' read -r disk_name disk_zone disk_region disk_type; do
    [[ -n "$disk_name" ]] || continue
    is_gke_blocking_disk_type_for_standard_target "$disk_type" || continue
    if [[ -n "$disk_zone" ]]; then
      printf 'zone|%s|%s|%s|%s\n' "$disk_zone" "$disk_name" "$disk_type" "label-scan"
    elif [[ -n "$disk_region" ]]; then
      printf 'region|%s|%s|%s|%s\n' "$disk_region" "$disk_name" "$disk_type" "label-scan"
    else
      printf 'name||%s|%s|%s\n' "$disk_name" "$disk_type" "label-scan"
    fi
  done <<< "$rows"
}

repair_stale_gke_gitaly_dynamic_storage() {
  local desired_sc="$1"
  [[ "$MODE" == "k8s" ]] || return 0
  [[ "$desired_sc" == "standard" ]] || return 0
  is_gke_cluster_detected || return 0

  local pvc_name="repo-data-gitlab-gitaly-0"
  local stale_pv_records
  stale_pv_records=$(collect_stale_gke_gitaly_pv_records "$desired_sc")
  local stale_disk_records
  stale_disk_records=$(collect_stale_gke_gitaly_disk_records_without_pv "$desired_sc")

  if [[ -z "$stale_pv_records" && -z "$stale_disk_records" ]]; then
    return 0
  fi

  local summary=""
  if [[ -n "$stale_pv_records" ]]; then
    while IFS='|' read -r pv_name pv_phase pv_sc _pv_handle _pv_gce_pd stale_reason; do
      [[ -n "$pv_name" ]] || continue
      summary+="PV ${pv_name} (phase=${pv_phase:-unknown}, sc=${pv_sc:-unknown}, reason=${stale_reason}). "
    done <<< "$stale_pv_records"
  fi
  if [[ -n "$stale_disk_records" ]]; then
    while IFS='|' read -r disk_scope disk_location disk_name disk_type disk_source; do
      [[ -n "$disk_name" ]] || continue
      summary+="Disk ${disk_name} (${disk_scope}:${disk_location:-n/a}, type=${disk_type:-unknown}, source=${disk_source}). "
    done <<< "$stale_disk_records"
  fi

  if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
    repair_blocked "Detected stale GKE Gitaly dynamic storage artifacts blocking class '${desired_sc}'" \
      "Artifacts are scoped to PVC ${pvc_name} only: ${summary}Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to auto-clean stale PV/PD leftovers from old standard-rwo/pd-balanced attempts."
  fi

  export GITALY_AUTOCLEAN_PERFORMED=1
  log "AUTOCLEAN: repairing stale GKE Gitaly dynamic storage artifacts for PVC ${pvc_name}..."
  kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true

  local live_pvc_sc live_pvc_phase
  live_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
  live_pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
  if [[ -n "$live_pvc_phase" ]] && is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$live_pvc_sc" && [[ "$live_pvc_phase" != "Bound" ]]; then
    log "Deleting stale live PVC ${pvc_name} (phase=${live_pvc_phase}, sc=${live_pvc_sc}) before retry..."
    kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true
  fi

  local processed_disks="|"
  local _disk_key
  local _disk_ref _disk_scope _disk_location _disk_name
  if [[ -n "$stale_pv_records" ]]; then
    while IFS='|' read -r pv_name _pv_phase _pv_sc pv_handle pv_gce_pd _stale_reason; do
      [[ -n "$pv_name" ]] || continue
      log "Deleting stale Gitaly PV ${pv_name}..."
      kubectl delete pv "$pv_name" --wait=false 2>/dev/null || true

      _disk_ref=""
      if [[ -n "$pv_handle" ]]; then
        _disk_ref=$(gke_parse_disk_ref_from_volume_handle "$pv_handle" || true)
      fi
      if [[ -z "$_disk_ref" && -n "$pv_gce_pd" ]]; then
        _disk_ref="name||${pv_gce_pd}"
      fi
      if [[ -n "$_disk_ref" ]]; then
        IFS='|' read -r _disk_scope _disk_location _disk_name <<< "$_disk_ref"
        _disk_key="${_disk_scope}|${_disk_location}|${_disk_name}"
        if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
          processed_disks+="${_disk_key}|"
          gke_delete_disk_ref_if_present "$_disk_scope" "$_disk_location" "$_disk_name" || true
        fi
      fi
    done <<< "$stale_pv_records"
  fi

  if [[ -n "$stale_disk_records" ]]; then
    while IFS='|' read -r disk_scope disk_location disk_name _disk_type _disk_source; do
      [[ -n "$disk_name" ]] || continue
      _disk_key="${disk_scope}|${disk_location}|${disk_name}"
      if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
        processed_disks+="${_disk_key}|"
        gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true
      fi
    done <<< "$stale_disk_records"
  fi

  sleep 2
}

check_gitlab_post_apply_blocked() {
  # --- Migrations Check ---
  check_gitlab_migrations_blocked

  # --- Gitaly Check ---
  if [[ "$MODE" == "k8s" ]]; then
    local desired_cr="${GITLAB_CR_RENDERED:-}"
    if [[ -n "$desired_cr" ]]; then
      local rendered_global_sc=""
      local rendered_gitlab_gitaly_sc=""
      local rendered_chart_gitaly_sc=""
      IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")"

      local desired_gitaly_node_selector
      desired_gitaly_node_selector=$(echo "$desired_cr" | sed -n '/gitaly:/,/toolbox:/p' | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
      local desired_gitaly_storage_class="$rendered_gitlab_gitaly_sc"

      log "Desired Gitaly state (from CR): nodeSelector=[${desired_gitaly_node_selector}], rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}], rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}], rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}]"
      log "Internal script GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]"

      if [[ "$desired_cr" == *"gandalf.prole.org"* ]]; then
         repair_blocked "GitLab CR contains legacy nodeSelector" \
           "Desired GitLab CR still contains gandalf.prole.org. This is a configuration bug."
      fi
      if [[ "$desired_cr" == *"gitlab-gitaly-static"* ]]; then
         repair_blocked "GitLab CR contains legacy storageClass" \
           "Desired GitLab CR still contains gitlab-gitaly-static. This is a configuration bug."
      fi
      if [[ -n "$desired_gitaly_storage_class" ]] && ! gitlab_storage_class_matches_expected "$desired_gitaly_storage_class" "${GITALY_STORAGE_CLASS:-}"; then
         repair_blocked "GitLab CR storageClass mismatch" \
           "Desired GitLab CR storageClass=[${desired_gitaly_storage_class}] does not match GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]. Configuration bug."
      fi
      if [[ -n "$rendered_chart_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_chart_gitaly_sc" "${GITALY_STORAGE_CLASS:-}"; then
         repair_blocked "GitLab CR chart-level Gitaly storageClass mismatch" \
           "Rendered gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc}] does not match GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]. Configuration bug."
      fi

      enforce_gitlab_storage_class_target "${GITALY_STORAGE_CLASS:-}" "$desired_cr"
    fi

    local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"

    # If autoclean was performed, wait for convergence before checking live state.
    # This prevents false positives when the operator hasn't yet updated the stale StatefulSet.
    if [[ "${GITALY_AUTOCLEAN_PERFORMED:-0}" == "1" ]]; then
       log "AUTOCLEAN was performed. Waiting for Gitaly StatefulSet to converge (removing legacy storage)..."
       local grace_start=$(date +%s)
       local grace_timeout=60
       local converged=0
       while true; do
         local live_sts_yaml
         live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
         if [[ -z "$live_sts_yaml" ]]; then
            log "Gitaly StatefulSet not found (awaiting operator action)..."
         elif [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
            log "Gitaly StatefulSet converged to corrected state (clean nodeSelector/storageClass)."
            local _live_sc
            _live_sc=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
            if [[ -n "$_live_sc" ]] && ! gitlab_storage_class_matches_expected "$_live_sc" "${GITALY_STORAGE_CLASS:-}"; then
               log "WARNING: Converged StatefulSet uses storageClass=[${_live_sc}], expected [${GITALY_STORAGE_CLASS:-}]."
            fi
            converged=1
            break
         else
            local live_gitaly_node_selector
            live_gitaly_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
            local live_gitaly_storage_class
            live_gitaly_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
            log "Still waiting for Gitaly convergence (live nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}])..."
         fi

         if (( $(date +%s) - grace_start > grace_timeout )); then
            log "Grace period (${grace_timeout}s) expired."
            break
         fi
         sleep 15
       done

       if [[ "$converged" == "0" ]]; then
          local live_sts_yaml
          live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
          if [[ -n "$live_sts_yaml" && ( "$live_sts_yaml" == *"gandalf.prole.org"* || "$live_sts_yaml" == *"gitlab-gitaly-static"* ) ]]; then
             log "Live StatefulSet still legacy after grace period. Performing explicit replacement..."
             # We already confirmed desired CR is corrected at the start of this function.
             kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true
             log "Legacy StatefulSet deleted. Waiting for operator recreation..."

             local recreate_start=$(date +%s)
             local recreate_timeout=300
             while true; do
               live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
               if [[ -n "$live_sts_yaml" ]]; then
                  local recreated_node_selector
                  recreated_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
                  local recreated_storage_class
                  recreated_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
                  log "StatefulSet recreated. nodeSelector=[${recreated_node_selector}], storageClass=[${recreated_storage_class}]"

                  if [[ "$live_sts_yaml" != *"gandalf.knoe.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
                     if [[ -n "$recreated_storage_class" ]] && ! gitlab_storage_class_matches_expected "$recreated_storage_class" "${GITALY_STORAGE_CLASS:-}"; then
                        repair_blocked "Recreated Gitaly StatefulSet uses wrong storageClass" \
                          "Resource: statefulset/${gitaly_sts_name}. Value: storageClassName=[${recreated_storage_class}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check operator reconciliation."
                     fi
                     log "Gitaly StatefulSet recreated in corrected state."
                     converged=1
                     break
                  else
                     log "Recreated StatefulSet STILL contains legacy fields. Waiting for operator to correct it..."
                  fi
               fi

               if (( $(date +%s) - recreate_start > recreate_timeout )); then
                  repair_blocked "Gitaly failed to recreate clean StatefulSet after ${recreate_timeout}s" \
                    "Resource: statefulset/${gitaly_sts_name}. Fix: Check operator logs and desired GitLab CR."
               fi
               sleep 15
             done
          else
             log "No legacy StatefulSet found after grace period (may have been deleted or converged)."
             converged=1
          fi
       fi

       if [[ "$converged" == "1" ]]; then
          # Wait for PVC provisioning if autoclean was performed
          log "Waiting for PVC repo-data-gitlab-gitaly-0 to be provisioned and Bound..."
          local pvc_name="repo-data-gitlab-gitaly-0"
          local pvc_start=$(date +%s)
          local pvc_timeout=600
          local pvc_uid=""
          while true; do
            local pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
            local pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)

            if [[ -n "$pvc_phase" ]]; then
               pvc_uid=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
               local _sc_info=""
               if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-balanced / wrong-class for strict standard target)"; fi
               if [[ "$pvc_sc" == "premium-rwo" ]]; then _sc_info=" (pd-ssd)"; fi
               if [[ "$pvc_sc" == "standard" ]]; then _sc_info=" (pd-standard)"; fi

               log "PVC ${pvc_name}: storageClass=[${pvc_sc}${_sc_info}], phase=[${pvc_phase}]"
               if [[ -n "$pvc_sc" ]] && ! gitlab_storage_class_matches_expected "$pvc_sc" "${GITALY_STORAGE_CLASS:-}"; then
                  repair_blocked "Gitaly PVC uses wrong storageClass" \
                    "PVC: ${pvc_name}. Value: storageClass=[${pvc_sc}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check StatefulSet volumeClaimTemplates and operator reconciliation."
               fi

               if [[ "$pvc_phase" == "Bound" ]]; then
                  if [[ "$pvc_sc" == "${GITALY_STORAGE_CLASS:-}" ]]; then
                     log "PVC ${pvc_name} successfully provisioned on '${pvc_sc}' storage."
                     break
                  fi
               fi

               # Check provisioning failures only for the *current* PVC object.
               local provisioning_fail=""
               if [[ -n "$pvc_uid" ]]; then
                  provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
               fi
               if [[ -n "$provisioning_fail" ]]; then
                  if [[ "$provisioning_fail" == *"quota"* || "$provisioning_fail" == *"QUOTA"* ]]; then
                     repair_blocked "Gitaly PVC provisioning failed (Quota Exceeded)" \
                       "PVC: ${pvc_name}. Error: ${provisioning_fail}. Fix: Check GKE storage quotas and ensure Gitaly uses storageClass='${GITALY_STORAGE_CLASS:-standard}'."
                   else
                      log "PVC ${pvc_name} provisioning event: ${provisioning_fail}"
                   fi
               fi
            else
               # Check migrations first to fail fast
               check_gitlab_migrations_blocked
               log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..."
            fi

            if (( $(date +%s) - pvc_start > pvc_timeout )); then
               local last_msg=""
               if [[ -n "$pvc_uid" ]]; then
                  last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
               else
                  last_msg="No live PVC UID observed yet; ignoring stale historical PVC events from previous claims."
               fi
               repair_blocked "Gitaly PVC failed to bind after ${pvc_timeout}s" \
                 "PVC: ${pvc_name}. Status: ${pvc_phase:-NotFound}. Last event: ${last_msg}. Fix: Check storage provider and quota."
            fi
            sleep 15
          done

          export GITALY_AUTOCLEAN_PERFORMED=0
       fi

       # Also wait for PV deletion to settle if it exists
       if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
          log "Waiting for legacy PV gitlab-gitaly-synology to be removed..."
          local pv_wait_start=$(date +%s)
          while kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; do
             if (( $(date +%s) - pv_wait_start > 120 )); then
                log "Legacy PV still exists after 120s; continuing (it may be stuck in Terminating)."
                break
             fi
             sleep 5
          done
       fi
    fi

    # Final live state checks
    local gitaly_sts_yaml
    gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
    if [[ -n "$gitaly_sts_yaml" ]]; then
      local rendered_global_sc=""
      local rendered_gitlab_gitaly_sc=""
      local rendered_chart_gitaly_sc=""
      IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")"

      local live_gitaly_node_selector
      live_gitaly_node_selector=$(echo "$gitaly_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
      local live_gitaly_storage_class
      live_gitaly_storage_class=$(echo "$gitaly_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
      local live_gitaly_repo_data_template_sc
      live_gitaly_repo_data_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class)

      log "Live Gitaly state: nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}]"
      log "Live Gitaly StatefulSet repo-data claim template storageClassName=[${live_gitaly_repo_data_template_sc:-<unset>}]"

      if ! gitlab_storage_class_matches_expected "$live_gitaly_repo_data_template_sc" "${GITALY_STORAGE_CLASS:-}"; then
        if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
          repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass mismatch" \
            "Configured target=[${GITALY_STORAGE_CLASS:-}] rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}] live.statefulset=${gitaly_sts_name} live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_repo_data_template_sc:-<unset>}]. Delete statefulset/${gitaly_sts_name}, pvc/repo-data-${GITLAB_RELEASE}-gitaly-0, and wrong-class PV+disk artifacts or set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 for automatic destructive repair."
        fi

        local gitaly_pvc_mismatches
        gitaly_pvc_mismatches=$(collect_gitlab_storage_mismatch_records "${GITALY_STORAGE_CLASS:-}")
        cleanup_gitlab_wrong_gitaly_template_storage "${GITALY_STORAGE_CLASS:-}" "$live_gitaly_repo_data_template_sc" "$gitaly_pvc_mismatches" "$desired_cr"
      fi

      if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* ]]; then
        repair_blocked "Gitaly using legacy Synology storage" \
          "Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Ensure GITLAB_STORAGE_NODE is not set in k8s mode."
      fi
      if [[ "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
        repair_blocked "Gitaly using legacy Synology storage" \
          "Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden in k8s mode."
      fi
    fi

    if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
       repair_blocked "Gitaly using legacy Synology storage" \
          "Resource: pv/gitlab-gitaly-synology. Value: exists. Fix: Delete legacy PV or use GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1."
    fi

    local unschedulable_gitaly
    unschedulable_gitaly=$(kubectl -n "$NAMESPACE" get pods -l "app=gitaly" -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="PodScheduled")].status=="False")]}{.metadata.name}:{.status.conditions[?(@.type=="PodScheduled")].reason}{"\n"}{end}' 2>/dev/null | grep "Unschedulable" || true)
    if [[ -n "$unschedulable_gitaly" ]]; then
       repair_blocked "Gitaly pod(s) unschedulable" \
         "Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storageClass='${GITALY_STORAGE_CLASS:-standard}' and no hostname nodeSelector. Ensure wrong-class PV/PVC artifacts were repaired."
    fi

    # --- Sidekiq Config Check ---
    log "Desired Sidekiq state: concurrency=${GITLAB_SIDEKIQ_CONCURRENCY}, requests=[cpu=${GITLAB_SIDEKIQ_REQUESTS_CPU}, mem=${GITLAB_SIDEKIQ_REQUESTS_MEMORY}], limits=[cpu=${GITLAB_SIDEKIQ_LIMITS_CPU}, mem=${GITLAB_SIDEKIQ_LIMITS_MEMORY}]"
    if [[ "${gitlab_apply_changed:-0}" == "1" || "${gitlab_spec_changed:-0}" == "1" ]]; then
       log "GitLab CR was updated to fix configuration drift (Sidekiq or other fields)."
    fi

    local sidekiq_deploy_name="${GITLAB_RELEASE}-sidekiq-all-in-1-v2"
    local live_sidekiq_cpu_req
    live_sidekiq_cpu_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}' 2>/dev/null || true)
    local live_sidekiq_mem_req
    live_sidekiq_mem_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.memory}' 2>/dev/null || true)
    local live_sidekiq_cpu_lim
    live_sidekiq_cpu_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.cpu}' 2>/dev/null || true)
    local live_sidekiq_mem_lim
    live_sidekiq_mem_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.memory}' 2>/dev/null || true)
    local live_sidekiq_concurrency
    live_sidekiq_concurrency=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="SIDEKIQ_CONCURRENCY")].value}' 2>/dev/null || true)

    if [[ -n "$live_sidekiq_cpu_req" ]]; then
       log "Live Sidekiq state: concurrency=${live_sidekiq_concurrency}, requests=[cpu=${live_sidekiq_cpu_req}, mem=${live_sidekiq_mem_req}], limits=[cpu=${live_sidekiq_cpu_lim}, mem=${live_sidekiq_mem_lim}]"
    fi
  fi

  # --- Registry Config Check ---
  local registry_secret_config
  registry_secret_config=$(kubectl -n "$NAMESPACE" get secret "${GITLAB_RELEASE}-registry-storage" -o jsonpath='{.data.config}' 2>/dev/null | base64 -d 2>/dev/null || true)
  if [[ "$registry_secret_config" == *"<private-db-cluster-garage-endpoint>"* ]]; then
    if [[ "$GARAGE_S3_ENDPOINT" != *"<private-db-cluster-garage-endpoint>"* ]]; then
       log "Registry secret still contains placeholder but config is updated. Re-applying secret..."
       # Re-triggering the secret creation (this is safer than just blocking)
       setup_garage_for_gitlab
    else
       repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder in live secret)" \
         "Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint."
    fi
  fi

  # --- Registry Logs Check ---
  local registry_pod
  registry_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=registry" -o name 2>/dev/null | head -n1 || true)
  if [[ -n "$registry_pod" ]]; then
    local logs
    logs=$(kubectl -n "$NAMESPACE" logs "$registry_pod" --tail=100 2>&1 || true)
    if [[ "$logs" == *"DNS failure"* || "$logs" == *"AccessDenied"* || "$logs" == *"No such key"* ]]; then
       # If it's a "No such key" or "AccessDenied", and we have a custom endpoint, it might be the WRONG cluster Garage.
       repair_blocked "Registry endpoint invalid or wrong Garage (S3 error detected)" \
          "Logs: ${logs}. Fix: Verify GARAGE_S3_ENDPOINT points to the DB cluster Garage (not the APP cluster one)."
    fi
  fi

  # --- KAS Logs Check ---
  local kas_pod
  kas_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=kas" -o name 2>/dev/null | head -n1 || true)
  if [[ -n "$kas_pod" ]]; then
    local logs
    logs=$(kubectl -n "$NAMESPACE" logs "$kas_pod" --tail=100 2>&1 || true)
    if [[ "$logs" == *"no such host"* && "$logs" == *"redis"* ]]; then
       repair_blocked "Redis host does not resolve (detected in KAS logs)" \
          "Logs: ${logs}. Fix: Verify REDIS_HOST (${REDIS_HOST}) and ensure Redis service is healthy."
    fi
  fi
}

kubectl_db() {
  local db_ctx=""
  db_ctx="$(resolve_db_cluster_context || true)"
  if [[ -n "$db_ctx" ]]; then
    command kubectl --context "$db_ctx" "$@"
  else
    kubectl "$@"
  fi
}

resolve_garage_admin_context() {
  if [[ "$MODE" == "k8s" ]]; then
    local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
    local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
    local explicit_ctx=""
    explicit_ctx="$(resolve_explicit_kube_context || true)"

    if [[ -n "$db_ctx" && -n "$app_ctx" && "$db_ctx" != "$app_ctx" ]]; then
      printf '%s' "$db_ctx"
      return 0
    fi

    if [[ -n "$db_ctx" ]]; then
      printf '%s' "$db_ctx"
      return 0
    fi
    if [[ -n "$explicit_ctx" ]]; then
      printf '%s' "$explicit_ctx"
      return 0
    fi
    if [[ -n "$app_ctx" ]]; then
      printf '%s' "$app_ctx"
      return 0
    fi

    die "Explicit Garage admin context is required in k8s mode."
  fi
  return 1
}

kubectl_garage_admin() {
  local garage_ctx=""
  garage_ctx="$(resolve_garage_admin_context || true)"
  if [[ -n "$garage_ctx" ]]; then
    command kubectl --context "$garage_ctx" "$@"
  else
    kubectl "$@"
  fi
}

db_kubectl() {
  local db_ctx
  db_ctx="$(resolve_db_cluster_context || true)"
  if [[ -n "$db_ctx" ]]; then
    command kubectl --context "$db_ctx" "$@"
  else
    kubectl "$@"
  fi
}

ensure_cross_cluster_db_host() {
  if [[ "$MODE" != "k8s" ]]; then
    return 0
  fi

  local app_ctx db_ctx
  app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
  if [[ -z "$app_ctx" ]]; then
    app_ctx="$(kubectl config current-context 2>/dev/null || true)"
  fi
  db_ctx="$(resolve_db_cluster_context || true)"

  if [[ -z "$db_ctx" || -z "$app_ctx" || "$db_ctx" == "$app_ctx" ]]; then
    return 0
  fi

  local db_cluster_name="${CNPG_CLUSTER_NAME:-knoe-db}"
  local db_ns="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
  local ilb_service="${GITLAB_DB_ILB_SERVICE:-knoe-db-rw-ilb}"

  log "Split-cluster mode detected (APP: ${app_ctx}, DB: ${db_ctx})"
  log "Ensuring cross-cluster Postgres ILB service '${ilb_service}' in namespace '${db_ns}'..."
  
  db_kubectl -n "$db_ns" apply -f - <<EOF >/dev/null
apiVersion: v1
kind: Service
metadata:
  name: ${ilb_service}
  annotations:
    networking.gke.io/load-balancer-type: "Internal"
spec:
  type: LoadBalancer
  selector:
    cnpg.io/cluster: ${db_cluster_name}
    cnpg.io/instanceRole: primary
  ports:
    - name: postgres
      port: 5432
      targetPort: 5432
      protocol: TCP
EOF

  local db_host=""
  local attempts=0
  log "Waiting for cross-cluster DB host (ILB IP/hostname)..."
  while (( attempts < 60 )); do
    attempts=$((attempts + 1))
    db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)"
    if [[ -z "$db_host" ]]; then
      db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)"
    fi
    if [[ -n "$db_host" ]]; then
      break
    fi
    sleep 5
  done

  if [[ -z "$db_host" ]]; then
    warn "Cross-cluster Postgres ILB '${ilb_service}' has no ingress address yet after 5 minutes."
    return 0
  fi

  log "Successfully resolved cross-cluster DB host: ${db_host}"
  GITLAB_CROSS_CLUSTER_DB_HOST="$db_host"
  export GITLAB_CROSS_CLUSTER_DB_HOST
}

enforce_app_cluster_targeting

is_truthy() {
  case "${1:-}" in
    1|true|TRUE|True|yes|YES|on|ON|y|Y)
      return 0
      ;;
    *)
      return 1
      ;;
  esac
}

assert_public_ingress_targeting() {
  local ingress_class="${1:-}"
  shift || true
  local hosts=("$@")

  if [[ "$MODE" != "k8s" ]]; then
    return 0
  fi

  local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
  local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
  local active_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"

  if [[ -z "$app_ctx" || -z "$active_ctx" ]]; then
    die "Explicit APP cluster context is required for GitLab ingress operations in k8s mode."
  fi

  local host_count=0
  local h
  for h in "${hosts[@]}"; do
    [[ -n "${h:-}" ]] && host_count=$((host_count + 1))
  done

  if [[ "$host_count" -gt 0 && -n "$db_ctx" && "$active_ctx" == "$db_ctx" ]]; then
    die "Refusing to render/apply GitLab public ingress in DB cluster context '${active_ctx}' (hosts: ${hosts[*]})."
  fi

  if [[ "$host_count" -gt 0 && -n "$app_ctx" && -n "$active_ctx" && "$active_ctx" != "$app_ctx" ]]; then
    die "GitLab public ingress must target APP cluster context '${app_ctx}', active context is '${active_ctx}'."
  fi

  local normalized_class="${ingress_class,,}"
  if [[ "$normalized_class" == traefik* ]] && ! is_truthy "${ALLOW_TRAEFIK_PUBLIC_INGRESS:-${GITLAB_ALLOW_TRAEFIK_INGRESS:-0}}"; then
    die "Ingress class '${ingress_class}' is incompatible with k8s mode unless Traefik public ingress is explicitly enabled."
  fi
}

assert_unique_ingress_host_claims() {
  local ingress_name="${1:-}"
  local ingress_namespace="${2:-}"
  local host_csv="${3:-}"
  local gitlab_release="${4:-}"
  local expected_backend_service="${5:-}"
  [[ -n "$host_csv" ]] || return 0

  local target_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
  if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then
    die "Explicit kubectl context is required for ingress ownership checks in k8s mode."
  fi

  INGRESS_HOST_CLAIM_DETAILS=""

  local claim_result claim_status
  claim_result=$(python3 - "$host_csv" "$ingress_namespace" "$ingress_name" "$target_ctx" "$gitlab_release" "$expected_backend_service" <<'PY'
import json
import subprocess
import sys

requested_hosts = {h.strip().lower() for h in (sys.argv[1] or "").split(",") if h.strip()}
target_ns = sys.argv[2]
target_name = sys.argv[3]
target_ctx = (sys.argv[4] or "").strip()
target_release = (sys.argv[5] or "").strip()
expected_backend_service = (sys.argv[6] or "").strip()

cmd = ["kubectl"]
if target_ctx:
    cmd.extend(["--context", target_ctx])
cmd.extend(["get", "ingress", "-A", "-o", "json"])

try:
    raw = subprocess.check_output(cmd, text=True)
except Exception:
    print("OK|")
    raise SystemExit(0)

if not raw.strip():
    print("OK|")
    raise SystemExit(0)

try:
    payload = json.loads(raw)
except Exception:
    print("OK|")
    raise SystemExit(0)

same_owner_conflicts: list[str] = []
foreign_conflicts: list[str] = []
for item in payload.get("items", []) or []:
    md = item.get("metadata", {}) or {}
    ns = (md.get("namespace") or "").strip()
    name = (md.get("name") or "").strip()
    if ns == target_ns and name == target_name:
        continue

    labels = md.get("labels", {}) or {}
    spec = item.get("spec", {}) or {}
    rules = spec.get("rules", []) or []
    for rule in rules:
        host = (rule.get("host") or "").strip().lower()
        if not host or host not in requested_hosts:
            continue
        http = rule.get("http", {}) or {}
        paths = http.get("paths", []) or [{"path": "/"}]
        for path_item in paths:
            path = (path_item.get("path") or "/").strip() or "/"
            if path not in {"/", ""}:
                continue

            backend = path_item.get("backend", {}) or {}
            backend_svc = backend.get("service", {}) or {}
            backend_name = (backend_svc.get("name") or "").strip()

            same_owner = False
            if ns == target_ns:
                if expected_backend_service and backend_name == expected_backend_service:
                    same_owner = True
                if target_release and backend_name.startswith(f"{target_release}-webservice"):
                    same_owner = True

                instance_label = (labels.get("app.kubernetes.io/instance") or "").strip()
                part_of_label = (labels.get("app.kubernetes.io/part-of") or "").strip().lower()
                if target_release and (
                    name == f"{target_release}-webservice-default"
                    or name.startswith(f"{target_release}-")
                    or instance_label == target_release
                    or part_of_label == "gitlab"
                ):
                    same_owner = True

            owner_desc = f"{host}{path} already owned by {ns}/{name}"
            if backend_name:
                owner_desc += f" (backend={backend_name})"

            if same_owner:
                same_owner_conflicts.append(owner_desc)
            else:
                foreign_conflicts.append(owner_desc)

if foreign_conflicts:
    print("CONFLICT|" + "; ".join(sorted(set(foreign_conflicts))))
elif same_owner_conflicts:
    print("OWNED_BY_GITLAB|" + "; ".join(sorted(set(same_owner_conflicts))))
else:
    print("OK|")
PY
)
  claim_status="${claim_result%%|*}"
  INGRESS_HOST_CLAIM_DETAILS="${claim_result#*|}"

  case "$claim_status" in
    OK|"")
      return 0
      ;;
    OWNED_BY_GITLAB)
      return 10
      ;;
    CONFLICT)
      return 11
      ;;
    *)
      INGRESS_HOST_CLAIM_DETAILS="$claim_result"
      return 11
      ;;
  esac
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --mode)          MODE="$(knoe_normalize_mode "${2:-}")"; shift 2 ;;
    --mode=*)        MODE="$(knoe_normalize_mode "${1#*=}")"; shift 1 ;;
    -n|--namespace)  NAMESPACE="${2:-}"; shift 2 ;;
    --namespace=*)   NAMESPACE="${1#*=}"; shift 1 ;;
    -c|--config)     CFG_PATH="${2:-}"; shift 2 ;;
    --config=*)      CFG_PATH="${1#*=}"; shift 1 ;;
    --node-selector) NODE_SELECTOR="${2:-}"; shift 2 ;;
    --node-selector=*) NODE_SELECTOR="${1#*=}"; shift 1 ;;
    --force)         FORCE=1; shift 1 ;;
    -h|--help)       usage; exit 0 ;;
    --trace-namespace)
      # Resolve namespace then exit — used by diagnostic scripts only
      _TRACE_NS=1; shift 1 ;;
    *) break ;;
  esac
done

# ---------------------------------------------------------------------------
# Resolve config path
# ---------------------------------------------------------------------------
if [[ -z "$CFG_PATH" && -n "${KNOE_CONF:-}" && -f "${KNOE_CONF}/knoe.cfg" ]]; then
  CFG_PATH="$(_knoe_cfg_select_cfg_file "${KNOE_CONF}")"
fi
if [[ -z "$CFG_PATH" ]]; then
  CFG_PATH="$(_knoe_cfg_select_cfg_file "$SCRIPT_DIR/../conf")"
fi

# cfg-based namespace override (only GITLAB_NAMESPACE; GITOPS_NAMESPACE intentionally excluded)
if [[ -z "$NAMESPACE" || "$NAMESPACE" == "gitlab" ]] && [[ -n "$CFG_PATH" ]]; then
  maybe_ns="$(_knoe_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")"
  [[ -n "$maybe_ns" ]] && NAMESPACE="$maybe_ns"
fi
# Final safety net — always default to 'gitlab'
NAMESPACE="${NAMESPACE:-gitlab}"
export GITLAB_NAMESPACE="$NAMESPACE"

# Diagnostic exit — used by tmp/sim_installer_ns.sh
if [[ "${_TRACE_NS:-0}" == "1" ]]; then
  echo "TRACE: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" >&2
  echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-<unset>}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-<unset>}'" >&2
  exit 0
fi

case "$MODE" in
  k3d|k3s|k8s|local) ;;
  *) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;;
esac
export KNOE_MODE="$MODE"

command -v kubectl >/dev/null || die "kubectl not found"
command -v helm    >/dev/null || die "helm not found (required for GitLab Operator install)"

# ---------------------------------------------------------------------------
# knoe-db (CNPG) settings — reuse the same cluster gitea uses
# ---------------------------------------------------------------------------
DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-${CLUSTER_NAME:-knoe-db}}"

ensure_cross_cluster_db_host

GITLAB_DB_NAME="${GITLAB_DB_NAME:-gitlabhq_production}"
GITLAB_DB_USER="${GITLAB_DB_USER:-gitlab}"
GITLAB_DB_PASSWORD="${GITLAB_DB_PASSWORD:-}"
DB_HOST="${GITLAB_DB_HOST:-${GITLAB_CROSS_CLUSTER_DB_HOST:-${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local}}"
DB_PORT="${GITLAB_DB_PORT:-5432}"

# ---------------------------------------------------------------------------
# Redis — shared common service (deployed by init_redis.sh in knoe-system)
# ---------------------------------------------------------------------------
REDIS_NAMESPACE="${REDIS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}"
REDIS_HOST="${GITLAB_REDIS_HOST:-redis-master.${REDIS_NAMESPACE}.svc.cluster.local}"
REDIS_PORT="${GITLAB_REDIS_PORT:-6379}"

# GitLab public domain (configurable; default follows deployment mode)
_default_gitlab_domain="git.prole.org"
if [[ "$MODE" == "k8s" ]]; then
  _default_gitlab_domain="git.knoe.dev"
fi
GITLAB_DOMAIN="${GITLAB_DOMAIN:-${GITLAB_HOSTNAME:-$_default_gitlab_domain}}"
GITLAB_PUBLIC_HOSTS_RAW="${GITLAB_PUBLIC_HOSTS:-$GITLAB_DOMAIN}"

trim_csv_token() {
  local token="$1"
  token="${token#${token%%[![:space:]]*}}"
  token="${token%${token##*[![:space:]]}}"
  printf '%s' "$token"
}

_gitlab_hosts_csv=""
GITLAB_PUBLIC_HOSTS=()
IFS=',' read -r -a _gitlab_host_candidates <<< "$GITLAB_PUBLIC_HOSTS_RAW"
for _gitlab_host in "${_gitlab_host_candidates[@]}"; do
  _gitlab_host="$(trim_csv_token "$_gitlab_host")"
  [[ -n "$_gitlab_host" ]] || continue
  case ",${_gitlab_hosts_csv}," in
    *,"${_gitlab_host}",*) ;;
    *)
      GITLAB_PUBLIC_HOSTS+=("$_gitlab_host")
      _gitlab_hosts_csv="${_gitlab_hosts_csv:+${_gitlab_hosts_csv},}${_gitlab_host}"
      ;;
  esac
done
unset _gitlab_host_candidates _gitlab_host

if [[ ${#GITLAB_PUBLIC_HOSTS[@]} -eq 0 ]]; then
  GITLAB_PUBLIC_HOSTS=("$GITLAB_DOMAIN")
fi

PRIMARY_GITLAB_HOST="${GITLAB_PUBLIC_HOSTS[0]}"
GITLAB_DOMAIN="$PRIMARY_GITLAB_HOST"
PRIMARY_GITLAB_DOMAIN_ROOT="${PRIMARY_GITLAB_HOST#*.}"
if [[ "$PRIMARY_GITLAB_DOMAIN_ROOT" == "$PRIMARY_GITLAB_HOST" || -z "$PRIMARY_GITLAB_DOMAIN_ROOT" ]]; then
  PRIMARY_GITLAB_DOMAIN_ROOT="prole.org"
fi

# Public ingress class (k8s/GKE defaults to gce; local clusters keep kong)
_default_gitlab_ingress_class="kong"
if [[ "$MODE" == "k8s" ]]; then
  _default_gitlab_ingress_class="gce"
fi
GITLAB_INGRESS_CLASS="${GITLAB_INGRESS_CLASS:-$_default_gitlab_ingress_class}"

# ---------------------------------------------------------------------------
# Git SSH hostname + LoadBalancer IP
# ---------------------------------------------------------------------------
# On GKE with Google-managed certs, HTTPS has to live on a GCE global L7 IP
# (ManagedCertificate CRD only binds there). Port 22 requires a regional
# Network LB which can't share an IP with a global L7. So in k8s mode we
# default to a dedicated SSH hostname (git-ssh.<root domain>) bound to a
# user-reserved regional external static IP via GITLAB_SHELL_LOADBALANCER_IP.
# In k3d/k3s modes the gitlab-shell Service stays ClusterIP (typically
# port-forwarded) and global.hosts.ssh tracks the main gitlab hostname.
_default_gitlab_ssh_host="$GITLAB_DOMAIN"
if [[ "$MODE" == "k8s" ]]; then
  _default_gitlab_ssh_host="git-ssh.${PRIMARY_GITLAB_DOMAIN_ROOT}"
fi
GITLAB_SSH_HOST="${GITLAB_SSH_HOST:-$_default_gitlab_ssh_host}"

# Pre-reserved regional external static IP for the gitlab-shell LoadBalancer
# Service. If blank, no service override is emitted (ClusterIP default).
GITLAB_SHELL_LOADBALANCER_IP="${GITLAB_SHELL_LOADBALANCER_IP:-}"

# externalTrafficPolicy preserves client source IPs in SSH auth logs — useful
# for abuse triage and rate-limiting. Requires at least one gitlab-shell pod
# per node in the LB backend; our replicaCount=1 is fine.
GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY="${GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY:-Local}"

# Single GitLab front-door owner model.
# - operator: GitLab chart/operator-managed ingress owns ${GITLAB_PUBLIC_HOSTS}
# - fallback: custom fallback ingress owns ${GITLAB_PUBLIC_HOSTS}
# In k8s/GKE mode, default to fallback ownership so GitLab frontdoor remains
# anchored on the explicit GCE ingress path.
_default_gitlab_frontdoor_owner="fallback"
GITLAB_FRONTDOOR_OWNER="${GITLAB_FRONTDOOR_OWNER:-$_default_gitlab_frontdoor_owner}"
GITLAB_FRONTDOOR_OWNER="${GITLAB_FRONTDOOR_OWNER,,}"
GITLAB_FALLBACK_INGRESS_NAME="${GITLAB_FALLBACK_INGRESS_NAME:-gitlab-frontdoor-ingress}"
GITLAB_LEGACY_FALLBACK_INGRESS_NAME="gitlab-kong-ingress"
case "$GITLAB_FRONTDOOR_OWNER" in
  operator|fallback)
    ;;
  *)
    die "Unsupported GITLAB_FRONTDOOR_OWNER='${GITLAB_FRONTDOOR_OWNER}'. Supported values: operator, fallback."
    ;;
esac

# Trusted proxies used by GitLab Rails/Workhorse to accept forwarded host/proto
# from ingress/load-balancer hops.
_default_gitlab_trusted_proxies="127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10"
if [[ "$MODE" == "k8s" ]]; then
  _default_gitlab_trusted_proxies+=" ,130.211.0.0/22,35.191.0.0/16"
fi
_default_gitlab_trusted_proxies="${_default_gitlab_trusted_proxies// /}"
GITLAB_TRUSTED_PROXIES_RAW="${GITLAB_TRUSTED_PROXIES:-${_default_gitlab_trusted_proxies}}"
_gitlab_trusted_proxies_csv=""
GITLAB_TRUSTED_PROXIES=()
IFS=',' read -r -a _gitlab_trusted_proxy_candidates <<< "$GITLAB_TRUSTED_PROXIES_RAW"
for _gitlab_trusted_proxy in "${_gitlab_trusted_proxy_candidates[@]}"; do
  _gitlab_trusted_proxy="$(trim_csv_token "$_gitlab_trusted_proxy")"
  [[ -n "$_gitlab_trusted_proxy" ]] || continue
  case ",${_gitlab_trusted_proxies_csv}," in
    *,"${_gitlab_trusted_proxy}",*) ;;
    *)
      GITLAB_TRUSTED_PROXIES+=("$_gitlab_trusted_proxy")
      _gitlab_trusted_proxies_csv="${_gitlab_trusted_proxies_csv:+${_gitlab_trusted_proxies_csv},}${_gitlab_trusted_proxy}"
      ;;
  esac
done
unset _gitlab_trusted_proxy_candidates _gitlab_trusted_proxy
if [[ ${#GITLAB_TRUSTED_PROXIES[@]} -eq 0 ]]; then
  GITLAB_TRUSTED_PROXIES=("127.0.0.1/32")
  _gitlab_trusted_proxies_csv="127.0.0.1/32"
fi

GITLAB_WEBSERVICE_INGRESS_ENABLED="${GITLAB_WEBSERVICE_INGRESS_ENABLED:-true}"
if [[ "$MODE" == "k8s" ]]; then
  if [[ "$GITLAB_FRONTDOOR_OWNER" == "operator" ]]; then
    GITLAB_WEBSERVICE_INGRESS_ENABLED=true
  elif [[ "$GITLAB_FRONTDOOR_OWNER" == "fallback" ]]; then
    GITLAB_WEBSERVICE_INGRESS_ENABLED=false
  fi
fi

# Google Workspace OIDC — FRONTDOOR_HOST gates OmniAuth configuration.
# On k3s: api.knoe.org is the knoe-auth SSO gateway (knoe-auth service).
# Requires k8s secret 'gitlab-google-oidc' in GITLAB_NAMESPACE with Google
# OAuth2 client credentials (client_id, client_secret, redirect_uri).
# To disable OIDC: unset FRONTDOOR_HOST before running.
_default_auth_hostname="api.prole.org"
if [[ "$MODE" == "k8s" ]]; then
  _default_auth_hostname="api.knoe.dev"
fi
AUTH_HOSTNAME="${AUTH_HOSTNAME:-${FRONTDOOR_HOST:-${_default_auth_hostname}}}"
FRONTDOOR_HOST="${FRONTDOOR_HOST:-$AUTH_HOSTNAME}"
FRONTDOOR_AUTH_ENABLED="${FRONTDOOR_AUTH_ENABLED:-${AUTHORITY_ENABLED:-1}}"
AUTH_VERIFY_PATH="${AUTH_VERIFY_PATH:-/auth/verify}"
AUTH_LOGIN_PATH="${AUTH_LOGIN_PATH:-/auth/login}"
AUTH_RESPONSE_HEADERS="${AUTH_RESPONSE_HEADERS:-X-Knoe-User,X-Knoe-Email,X-Knoe-Groups}"
AUTH_VERIFY_URL="${AUTH_VERIFY_URL:-https://${AUTH_HOSTNAME}${AUTH_VERIFY_PATH}}"
AUTH_SIGNIN_URL="${AUTH_SIGNIN_URL:-https://${AUTH_HOSTNAME}${AUTH_LOGIN_PATH}?next=\$scheme://\$host\$escaped_request_uri}"
_default_gitlab_oidc_issuer="https://${AUTH_HOSTNAME}/auth"
if [[ "$MODE" == "k8s" ]]; then
  _default_gitlab_oidc_issuer="https://api.knoe.dev/auth"
fi
_default_gitlab_oidc_redirect_uri="https://${GITLAB_DOMAIN}/users/auth/openid_connect/callback"
if [[ "$MODE" == "k8s" ]]; then
  _default_gitlab_oidc_redirect_uri="https://git.knoe.dev/users/auth/openid_connect/callback"
fi
GITLAB_OIDC_PROVIDER_NAME="${GITLAB_OIDC_PROVIDER_NAME:-openid_connect}"
GITLAB_OIDC_ISSUER="${GITLAB_OIDC_ISSUER:-${OIDC_ISSUER:-${_default_gitlab_oidc_issuer}}}"
GITLAB_OIDC_REDIRECT_URI="${GITLAB_OIDC_REDIRECT_URI:-${_default_gitlab_oidc_redirect_uri}}"
GITLAB_OIDC_CLIENT_ID="${GITLAB_OIDC_CLIENT_ID:-${OIDC_CLIENT_ID:-${GOOGLE_OIDC_CLIENT_ID:-${GOOGLE_CLIENT_ID:-}}}}"
GITLAB_OIDC_CLIENT_SECRET="${GITLAB_OIDC_CLIENT_SECRET:-${OIDC_CLIENT_SECRET:-${GOOGLE_OIDC_CLIENT_SECRET:-${GOOGLE_CLIENT_SECRET:-}}}}"

# Resolve `secretref://<ref>` values for standalone invocations. deploy.sh
# resolves these in Python (knoe/core/actions.py:_resolve_secretref_value)
# before exec; direct invocations need the same lookup inline. Searches env
# aliases, then KNOE_SERVICE/secrets, etc/secrets, and secrets directories
# for a file named <ref>. Returns the raw input unchanged on miss.
_init_gitlab_resolve_secretref() {
  local raw="${1:-}"
  local aliases="${2:-}"
  case "$raw" in
    secretref://*) ;;
    *) printf '%s' "$raw"; return 0 ;;
  esac

  local ref="${raw#secretref://}"
  ref="${ref#/}"; ref="${ref%/}"
  [[ -n "$ref" ]] || { printf '%s' "$raw"; return 0; }

  local normalized="${ref//-/_}"
  normalized="${normalized//\//_}"
  normalized="${normalized//./_}"
  local upper_norm="${normalized^^}"

  local -a candidates=("$ref" "$normalized" "$upper_norm")
  local alias
  for alias in $aliases; do
    candidates+=("$alias")
  done

  local key val
  for key in "${candidates[@]}"; do
    [[ -z "$key" ]] && continue
    [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
    val="${!key-}"
    if [[ -n "$val" && "$val" != secretref://* ]]; then
      printf '%s' "$val"
      return 0
    fi
  done

  local root="${KNOE_HOME:-${SCRIPT_DIR}/..}"
  local svc="${KNOE_SERVICE:-}"
  local -a paths=()
  [[ -n "$svc" ]] && paths+=("$svc/secrets/$ref")
  paths+=("$root/etc/secrets/$ref" "$root/secrets/$ref")

  local path
  for path in "${paths[@]}"; do
    if [[ -f "$path" && -r "$path" ]]; then
      val="$(< "$path")"
      val="${val#"${val%%[![:space:]]*}"}"
      val="${val%"${val##*[![:space:]]}"}"
      if [[ -n "$val" ]]; then
        printf '%s' "$val"
        return 0
      fi
    fi
  done

  printf '%s' "$raw"
}

GITLAB_OIDC_CLIENT_ID="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_ID" \
  "GITLAB_OIDC_CLIENT_ID OIDC_CLIENT_ID GOOGLE_OIDC_CLIENT_ID GOOGLE_CLIENT_ID")"
GITLAB_OIDC_CLIENT_SECRET="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_SECRET" \
  "GITLAB_OIDC_CLIENT_SECRET OIDC_CLIENT_SECRET GOOGLE_OIDC_CLIENT_SECRET GOOGLE_CLIENT_SECRET")"

is_unresolved_secret_ref() {
  local raw="${1:-}"
  case "$raw" in
    ""|secretref://*|'${OPENBAO:'*|'${KNOE_SECRET:'*)
      return 0
      ;;
  esac
  return 1
}
GITLAB_RELEASE="gitlab"

gitlab_split_cluster_ownership_diagnostics

assert_public_ingress_targeting "$GITLAB_INGRESS_CLASS" "${GITLAB_PUBLIC_HOSTS[@]}"

# ---------------------------------------------------------------------------
# Garage S3 (knoe-system) — replaces embedded minio for object storage
# ---------------------------------------------------------------------------
GARAGE_NAMESPACE="${GARAGE_NAMESPACE:-knoe-system}"
GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-}"
GARAGE_S3_ENDPOINT="${GARAGE_S3_ENDPOINT:-${GARAGE_PRIVATE_S3_ENDPOINT:-${GARAGE_PRIVATE_ENDPOINT:-}}}"
if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
  repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
    "Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint in your config."
fi
if [[ -z "$GARAGE_S3_ENDPOINT" ]]; then
  if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]]; then
    die "GARAGE_S3_ENDPOINT (or GARAGE_PRIVATE_S3_ENDPOINT) must be set to an explicit private cross-cluster endpoint when APP and DB clusters differ."
  fi
  GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-garage.${GARAGE_NAMESPACE}.svc.cluster.local}"
  GARAGE_S3_ENDPOINT="http://${GARAGE_SVC_HOST}:3900"
fi
if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]] && [[ "$GARAGE_S3_ENDPOINT" == *".svc.cluster.local"* ]]; then
  die "GARAGE_S3_ENDPOINT must use a private cross-cluster endpoint and cannot use cluster-local service DNS (${GARAGE_S3_ENDPOINT})."
fi
if [[ -z "$GARAGE_SVC_HOST" ]]; then
  _garage_endpoint_hostport="${GARAGE_S3_ENDPOINT#http://}"
  _garage_endpoint_hostport="${_garage_endpoint_hostport#https://}"
  _garage_endpoint_hostport="${_garage_endpoint_hostport%%/*}"
  GARAGE_SVC_HOST="${_garage_endpoint_hostport%%:*}"
  unset _garage_endpoint_hostport
fi
GARAGE_S3_KEY_NAME="${GARAGE_S3_KEY_NAME:-gitlab-s3}"
GITLAB_OBJECT_STORAGE_REQUIRED="${GITLAB_OBJECT_STORAGE_REQUIRED:-1}"

# ---------------------------------------------------------------------------
# Operator chart coordinates
# ---------------------------------------------------------------------------
OPERATOR_REPO_NAME="gitlab-operator"
OPERATOR_REPO_URL="https://gitlab.com/api/v4/projects/18899486/packages/helm/stable"
OPERATOR_CHART="gitlab-operator/gitlab-operator"
OPERATOR_RELEASE="gitlab-operator"
OPERATOR_NAMESPACE="${GITLAB_OPERATOR_NAMESPACE:-${NAMESPACE}}"
GITLAB_OPERATOR_CHART_VERSION="${GITLAB_OPERATOR_CHART_VERSION:-}"
# GitLab Helm chart version required in the CR; auto-detected if not set.
GITLAB_CHART_VERSION="${GITLAB_CHART_VERSION:-}"
GITLAB_CHART_REPO_NAME="gitlab"
GITLAB_CHART_REPO_URL="https://charts.gitlab.io/"
gitlab_db_secret_changed=0
gitlab_object_storage_secret_changed=0
gitlab_registry_storage_secret_changed=0

resolve_helm_release_chart_version() {
  local release_namespace="$1"
  local release_name="$2"
  local chart_version
  chart_version=$({ helm -n "$release_namespace" list -f "^${release_name}$" -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys

try:
    rows = json.load(sys.stdin)
except Exception:
    print("")
    raise SystemExit(0)

if not rows:
    print("")
    raise SystemExit(0)

chart = str(rows[0].get("chart", ""))
if not chart:
    print("")
    raise SystemExit(0)

if "-" in chart:
    print(chart.rsplit("-", 1)[-1])
else:
    print("")
PY
)
  if [[ -n "$chart_version" ]]; then
    printf '%s' "$chart_version"
    return 0
  fi

  { helm -n "$release_namespace" status "$release_name" -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys

try:
    payload = json.load(sys.stdin)
except Exception:
    print("")
    raise SystemExit(0)

chart = str(payload.get("chart", ""))
if not chart:
    print("")
    raise SystemExit(0)

if "-" in chart:
    print(chart.rsplit("-", 1)[-1])
else:
    print("")
PY
}

resolve_operator_watch_namespace() {
  local watch_namespace
  watch_namespace=$({ helm -n "$OPERATOR_NAMESPACE" get values "$OPERATOR_RELEASE" --all -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys

try:
    values = json.load(sys.stdin)
except Exception:
    print("")
    raise SystemExit(0)

watch_namespace = values.get("watchNamespace", "")
if isinstance(watch_namespace, str):
    print(watch_namespace.strip())
else:
    print("")
PY
)
  if [[ -n "$watch_namespace" ]]; then
    printf '%s' "$watch_namespace"
    return 0
  fi

  kubectl -n "$OPERATOR_NAMESPACE" get deploy "$OPERATOR_RELEASE" \
    -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="WATCH_NAMESPACE")].value}' 2>/dev/null || true
}

kubectl_apply_reports_changed() {
  local apply_output="${1:-}"
  if [[ -z "${apply_output//[[:space:]]/}" ]]; then
    # Be conservative: unknown apply output should force full reconcile checks.
    return 0
  fi

  case "$apply_output" in
    *" created"*|*" configured"*|*" patched"*) return 0 ;;
  esac

  case "$apply_output" in
    *" unchanged"*) return 1 ;;
  esac

  # Unknown token from kubectl apply output -> treat as changed.
  return 0
}

# ---------------------------------------------------------------------------
# Helper: resolve knoe-db primary pod (same pattern as init_gitea.sh)
# ---------------------------------------------------------------------------
resolve_knoe_db_primary_pod() {
  local primary
  primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \
    -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME},cnpg.io/instanceRole=primary" \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -z "$primary" ]]; then
    primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \
      -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" \
      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  fi
  printf '%s' "$primary"
}

sql_escape_literal() {
  printf '%s' "${1:-}" | sed "s/'/''/g"
}

# ---------------------------------------------------------------------------
# Resolve GitLab DB password (pull from knoe-db-superuser secret as fallback)
# ---------------------------------------------------------------------------
resolve_gitlab_db_password() {
  if [[ -n "${GITLAB_DB_PASSWORD:-}" ]]; then
    return 0
  fi
  local resolved=""
  if kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then
    resolved=$(kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser \
      -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true)
  fi
  if [[ -n "$resolved" ]]; then
    GITLAB_DB_PASSWORD="$resolved"
  else
    # Generate a random password when none is available
    GITLAB_DB_PASSWORD="$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom 2>/dev/null | head -c 32 || true)"
    warn "Generated random GitLab DB password; store it in GITLAB_DB_PASSWORD for future runs."
  fi
}

# ---------------------------------------------------------------------------
# Provision GitLab role + database in knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab() {
  local primary db_ctx
  db_ctx="$(resolve_db_cluster_context || true)"

  primary="$(resolve_knoe_db_primary_pod)"
  if [[ -z "$primary" ]]; then
    if [[ "$MODE" == "k8s" ]]; then
      die "No knoe-db pod found in namespace '${DB_NAMESPACE}' on DB_CLUSTER_KUBECONTEXT='${db_ctx}'."
    fi
    warn "No knoe-db pod found in namespace '${DB_NAMESPACE}'; skipping GitLab DB setup."
    return 0
  fi

  resolve_gitlab_db_password

  local admin_user=""
  local candidate
  for candidate in postgres root; do
    if kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
      psql -U "$candidate" -d postgres -tAc "SELECT 1" >/dev/null 2>&1; then
      admin_user="$candidate"
      break
    fi
  done

  if [[ -z "$admin_user" ]]; then
    if [[ "$MODE" == "k8s" ]]; then
      die "Unable to connect to knoe-db as admin on DB_CLUSTER_KUBECONTEXT='${db_ctx}'."
    fi
    warn "Unable to connect to knoe-db as admin; skipping GitLab DB setup."
    return 0
  fi

  local esc_pw
  esc_pw="$(sql_escape_literal "$GITLAB_DB_PASSWORD")"

  if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
    psql -U "$admin_user" -d postgres -c "
      DO \$\$ BEGIN
        IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${GITLAB_DB_USER}') THEN
          CREATE ROLE ${GITLAB_DB_USER} LOGIN PASSWORD '${esc_pw}';
        ELSE
          ALTER ROLE ${GITLAB_DB_USER} WITH PASSWORD '${esc_pw}';
        END IF;
      END \$\$;
    " >/dev/null 2>&1; then
    if [[ "$MODE" == "k8s" ]]; then
      die "Could not create/update role '${GITLAB_DB_USER}' on DB cluster context '${db_ctx}'."
    fi
    warn "Could not create/update role '${GITLAB_DB_USER}'."
  fi

  local db_exists
  db_exists=$(kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
    psql -U "$admin_user" -d postgres -tAc \
    "SELECT 1 FROM pg_database WHERE datname='${GITLAB_DB_NAME}';" 2>/dev/null || true)

  if [[ "$db_exists" != "1" ]]; then
    if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
      psql -U "$admin_user" -d postgres -c \
      "CREATE DATABASE ${GITLAB_DB_NAME} OWNER ${GITLAB_DB_USER};" \
      >/dev/null 2>&1; then
      if [[ "$MODE" == "k8s" ]]; then
        die "Could not create database '${GITLAB_DB_NAME}' on DB cluster context '${db_ctx}'."
      fi
      warn "Could not create database '${GITLAB_DB_NAME}'."
    fi
  fi

  if [[ -n "$db_ctx" ]]; then
    log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE}, ctx=${db_ctx})."
  else
    log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE})."
  fi
}

# ---------------------------------------------------------------------------
# Clean up any pre-existing Gitea / git.prole.org configurations
# ---------------------------------------------------------------------------
cleanup_gitea() {
  log "Checking for pre-existing Gitea deployment to remove before GitLab install..."

  local gitea_ns="${GITEA_NAMESPACE:-gitea}"

  # Remove Gitea Helm release
  if helm -n "$gitea_ns" status gitea >/dev/null 2>&1; then
    log "Uninstalling Gitea Helm release from namespace '$gitea_ns'..."
    helm -n "$gitea_ns" uninstall gitea >/dev/null 2>&1 || true
  fi

  # Remove any leftover raw Gitea resources
  kubectl -n "$gitea_ns" delete deploy/gitea svc/gitea-http svc/gitea-ssh \
    >/dev/null 2>&1 || true

  # Remove Kong routes/services registered for git.prole.org (best-effort)
  local kong_ns="${KONG_NAMESPACE:-${NAMESPACE:-kong}}"
  for res_type in kongplugins kongingresses; do
    kubectl -n "$gitea_ns" delete "$res_type" --all >/dev/null 2>&1 || true
  done

  # Remove gitea namespace Ingress objects that route git.prole.org
  kubectl -n "$gitea_ns" delete ingress \
    -l "app.kubernetes.io/name=gitea" >/dev/null 2>&1 || true
  kubectl -n "$gitea_ns" delete ingress \
    --field-selector="metadata.name=gitea" >/dev/null 2>&1 || true

  log "Gitea cleanup complete."
}

# ---------------------------------------------------------------------------
# Ensure gitlab namespace exists
# ---------------------------------------------------------------------------
kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null

# ---------------------------------------------------------------------------
# --force: remove existing GitLab operator and release first
# ---------------------------------------------------------------------------
if [[ "$FORCE" -eq 1 ]]; then
  warn "--force: removing existing GitLab resources in namespace '$NAMESPACE'..."
  if kubectl get crd gitlabs.apps.gitlab.com >/dev/null 2>&1; then
    kubectl -n "$NAMESPACE" delete gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
    # Wait briefly for operator to clean up managed resources
    sleep 10
  fi
  helm -n "$NAMESPACE" uninstall "$GITLAB_RELEASE"        >/dev/null 2>&1 || true
  helm -n "$NAMESPACE" uninstall "$OPERATOR_RELEASE"      >/dev/null 2>&1 || true
  cleanup_gitea
fi

# ---------------------------------------------------------------------------
# Always remove gitea if its helm release exists (non-destructive path)
# gitea and gitlab both claim git.prole.org; they cannot coexist.
# ---------------------------------------------------------------------------
if helm -n "${GITEA_NAMESPACE:-gitea}" status gitea >/dev/null 2>&1; then
  warn "Gitea release detected — removing to free git.knoe.org for GitLab..."
  cleanup_gitea
fi

# ---------------------------------------------------------------------------
# Prepare knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab

# Persist the DB password as a k8s Secret the operator CR can reference
resolve_gitlab_db_password
gitlab_db_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-db-password \
  --from-literal=password="$GITLAB_DB_PASSWORD" \
  --dry-run=client -o yaml | kubectl apply -f - 2>&1)"
if kubectl_apply_reports_changed "$gitlab_db_secret_apply_output"; then
  gitlab_db_secret_changed=1
fi
log "gitlab-db-password secret applied in namespace '$NAMESPACE' (${gitlab_db_secret_apply_output})."

# ---------------------------------------------------------------------------
# Install / upgrade the GitLab Operator
# ---------------------------------------------------------------------------
operator_changed=0
operator_release_exists=0
operator_requires_upgrade=1
installed_operator_chart_version=""
installed_operator_watch_namespace=""
operator_watch_namespace_matches=1

if helm -n "$OPERATOR_NAMESPACE" status "$OPERATOR_RELEASE" >/dev/null 2>&1; then
  operator_release_exists=1
  installed_operator_chart_version="$(resolve_helm_release_chart_version "$OPERATOR_NAMESPACE" "$OPERATOR_RELEASE")"
  installed_operator_watch_namespace="$(resolve_operator_watch_namespace)"
fi

if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" && -n "$installed_operator_chart_version" ]]; then
  GITLAB_OPERATOR_CHART_VERSION="$installed_operator_chart_version"
  log "Reusing installed GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
fi

if [[ -n "$installed_operator_watch_namespace" && "$installed_operator_watch_namespace" != "$NAMESPACE" ]]; then
  operator_watch_namespace_matches=0
fi

if [[ "$operator_release_exists" == "1" && -n "$installed_operator_chart_version" \
      && "$installed_operator_chart_version" == "$GITLAB_OPERATOR_CHART_VERSION" \
      && "$operator_watch_namespace_matches" == "1" \
      && "${GITLAB_OPERATOR_FORCE_UPGRADE:-0}" != "1" ]]; then
  operator_requires_upgrade=0
fi

if [[ "$operator_requires_upgrade" == "1" ]]; then
  if [[ "$operator_watch_namespace_matches" == "0" ]]; then
    warn "Installed GitLab Operator watchNamespace (${installed_operator_watch_namespace}) differs from target (${NAMESPACE}); forcing operator upgrade."
  fi

  log "Adding/updating GitLab Operator Helm repo..."
  helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true
  helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..."

  if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then
    GITLAB_OPERATOR_CHART_VERSION=$(helm search repo "$OPERATOR_CHART" --output table 2>/dev/null \
      | awk 'NR==2{print $2}' || true)
    if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then
      die "Cannot detect GitLab Operator chart version. Set GITLAB_OPERATOR_CHART_VERSION explicitly and re-run."
    fi
    log "Auto-detected GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
  else
    log "Using GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
  fi

  log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..."

  # The operator needs cluster-scoped RBAC; it watches all namespaces by default.
  helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \
    -n "$OPERATOR_NAMESPACE" \
    --create-namespace \
    --version "$GITLAB_OPERATOR_CHART_VERSION" \
    --timeout 10m \
    --wait \
    --set watchNamespace="$NAMESPACE"

  operator_changed=1
  log "GitLab Operator ready."
else
  log "GitLab Operator already matches desired chart/watchNamespace; skipping operator upgrade."
fi

# ---------------------------------------------------------------------------
# Resolve GitLab chart version (required by the Operator CR since >= v0.28)
# ---------------------------------------------------------------------------
installed_gitlab_chart_version=""
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
  installed_gitlab_chart_version="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.spec.chart.version}' 2>/dev/null || true)"
  if [[ -n "$installed_gitlab_chart_version" ]]; then
    GITLAB_CHART_VERSION="$installed_gitlab_chart_version"
    log "Reusing installed GitLab chart version from existing GitLab CR: ${GITLAB_CHART_VERSION}"
  else
    log "Auto-detecting latest GitLab chart version..."
    helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true
    helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..."
    GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \
      | awk 'NR==2{print $2}' || true)
    if [[ -z "$GITLAB_CHART_VERSION" ]]; then
      die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run."
    fi
    log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}"
  fi
fi
CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\""
gitlab_chart_version_changed=0
if [[ -z "$installed_gitlab_chart_version" || "$installed_gitlab_chart_version" != "$GITLAB_CHART_VERSION" ]]; then
  gitlab_chart_version_changed=1
fi

# ---------------------------------------------------------------------------
# Determine node selector block for the GitLab CR
# ---------------------------------------------------------------------------
# Storage node is only used for legacy/local static local-PV mode.
if [[ "$MODE" == "k8s" ]]; then
  # In k8s/GKE mode, we do NOT use static storage nodes or nodeSelectors.
  STORAGE_NODE=""
  if [[ "$NODE_SELECTOR" == *"gandalf.prole.org"* ]]; then
    warn "Stripping legacy nodeSelector '${NODE_SELECTOR}' in k8s mode."
    NODE_SELECTOR=""
  fi
else
  STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
fi
[[ "$MODE" == "k8s" || -n "$STORAGE_NODE" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set in ${MODE} mode."
# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned
NODE_SELECTOR="${NODE_SELECTOR:-}"
NODE_SELECTOR_KEY="${GITLAB_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"
STORAGE_NODE_SELECTOR_KEY="${GITLAB_STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY}}"
NODE_SELECTOR_YAML=""
if [[ -n "$NODE_SELECTOR" ]]; then
  log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}"
  NODE_SELECTOR_YAML="${NODE_SELECTOR_KEY}: ${NODE_SELECTOR}"
fi
STORAGE_NODE_SELECTOR_YAML=""
if [[ -n "$STORAGE_NODE" ]]; then
  STORAGE_NODE_SELECTOR_YAML="${STORAGE_NODE_SELECTOR_KEY}: ${STORAGE_NODE}"
fi

GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}"
if [[ "$MODE" == "k8s" ]]; then
  # Strip legacy values that are meaningless in k8s mode — fall through to defaults.
  if [[ "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
    warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode."
    GITALY_STORAGE_CLASS=""
  fi

  # Default to strict 'standard' (pd-standard / HDD) in k8s mode.
  # Never auto-substitute to 'standard-rwo' for GitLab storage.
  if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
    GITALY_STORAGE_CLASS="standard"
    log "Defaulting Gitaly StorageClass to strict target 'standard' (pd-standard / HDD)."
  else
    log "Honoring explicit GITLAB_GITALY_STORAGE_CLASS=${GITALY_STORAGE_CLASS} (no auto-normalization)."
  fi

  if [[ "$GITALY_STORAGE_CLASS" == "standard" ]] && ! kubectl get storageclass standard >/dev/null 2>&1; then
    repair_blocked "Required GitLab StorageClass is missing" \
      "Configured target storageClass is 'standard' but StorageClass/standard does not exist on the cluster. Do not substitute to standard-rwo; create/restore StorageClass 'standard' or set an explicit non-standard class."
  fi
else
  if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
    GITALY_STORAGE_CLASS="gitlab-gitaly-static"
  fi
fi

# Sidekiq resources and concurrency
GITLAB_SIDEKIQ_REQUESTS_CPU="${GITLAB_SIDEKIQ_REQUESTS_CPU:-250m}"
GITLAB_SIDEKIQ_REQUESTS_MEMORY="${GITLAB_SIDEKIQ_REQUESTS_MEMORY:-1500Mi}"
GITLAB_SIDEKIQ_LIMITS_CPU="${GITLAB_SIDEKIQ_LIMITS_CPU:-1}"
GITLAB_SIDEKIQ_LIMITS_MEMORY="${GITLAB_SIDEKIQ_LIMITS_MEMORY:-3Gi}"
GITLAB_SIDEKIQ_CONCURRENCY="${GITLAB_SIDEKIQ_CONCURRENCY:-5}"

# Webservice (Puma) resources and concurrency
# Defaults calibrated for e2-standard-2 nodes (~7.1 GB allocatable).
# - workerProcesses=1 keeps steady-state RSS under ~1.1 GB.
# - Memory *limit* must exceed the boot spike (Rails preload + worker fork),
#   which peaks around 1.4–1.6 GB.  1800M gives headroom without exceeding
#   the node's 1/4-node per-pod budget.
# - puma.threads.{min,max} are the correct chart paths; PUMA_THREADS_{MIN,MAX}
#   env vars do NOT propagate through the chart's ERB-rendered puma config.
GITLAB_WEBSERVICE_REQUESTS_CPU="${GITLAB_WEBSERVICE_REQUESTS_CPU:-200m}"
GITLAB_WEBSERVICE_REQUESTS_MEMORY="${GITLAB_WEBSERVICE_REQUESTS_MEMORY:-900M}"
GITLAB_WEBSERVICE_LIMITS_MEMORY="${GITLAB_WEBSERVICE_LIMITS_MEMORY:-1800M}"
GITLAB_WEBSERVICE_WORKER_PROCESSES="${GITLAB_WEBSERVICE_WORKER_PROCESSES:-1}"
GITLAB_WEBSERVICE_PUMA_THREADS_MIN="${GITLAB_WEBSERVICE_PUMA_THREADS_MIN:-2}"
GITLAB_WEBSERVICE_PUMA_THREADS_MAX="${GITLAB_WEBSERVICE_PUMA_THREADS_MAX:-2}"

JEMALLOC_HOSTPATH_DIR="/opt/gitlab-jemalloc"
JEMALLOC_HOSTPATH_LIB="${JEMALLOC_HOSTPATH_DIR}/libjemalloc.so.2"
GITLAB_JEMALLOC_MODE="${GITLAB_JEMALLOC_MODE:-auto}"
if is_truthy "${GITLAB_JEMALLOC_REQUIRED:-0}"; then
  GITLAB_JEMALLOC_MODE="force"
fi

JEMALLOC_HOSTPATH_SUPPORTED=1
JEMALLOC_HOSTPATH_REASON=""
JEMALLOC_HOSTPATH_WANTED=0
JEMALLOC_HOSTPATH_ACTIVE=0

detect_jemalloc_hostpath_support() {
  local provider_ids gke_pool_labels gke_topology_labels os_images

  provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true)
  gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true)
  gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true)
  os_images=$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.nodeInfo.osImage}{"\n"}{end}' 2>/dev/null || true)

  if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then
    JEMALLOC_HOSTPATH_SUPPORTED=0
    JEMALLOC_HOSTPATH_REASON="Detected GKE/GCE node metadata (providerID/labels)."
    return 0
  fi

  if [[ "${os_images,,}" == *"container-optimized os"* ]]; then
    JEMALLOC_HOSTPATH_SUPPORTED=0
    JEMALLOC_HOSTPATH_REASON="Detected Container-Optimized OS node image."
  fi
}

configure_jemalloc_hostpath_mode() {
  local mode_normalized="${GITLAB_JEMALLOC_MODE,,}"

  detect_jemalloc_hostpath_support

  case "$mode_normalized" in
    auto|"")
      if (( JEMALLOC_HOSTPATH_SUPPORTED == 1 )); then
        JEMALLOC_HOSTPATH_WANTED=1
      else
        JEMALLOC_HOSTPATH_WANTED=0
        warn "Skipping hostPath jemalloc optimization: ${JEMALLOC_HOSTPATH_REASON}"
      fi
      ;;
    off|false|0|disabled)
      JEMALLOC_HOSTPATH_WANTED=0
      ;;
    force|required|on|true|1)
      JEMALLOC_HOSTPATH_WANTED=1
      if (( JEMALLOC_HOSTPATH_SUPPORTED == 0 )); then
        die "HostPath jemalloc mode is forced but unsupported in this cluster: ${JEMALLOC_HOSTPATH_REASON} Disable jemalloc (GITLAB_JEMALLOC_MODE=off) or use a non-hostPath approach."
      fi
      ;;
    *)
      die "Unsupported GITLAB_JEMALLOC_MODE='${GITLAB_JEMALLOC_MODE}' (expected: auto, off, force)."
      ;;
  esac
}

# ---------------------------------------------------------------------------
# Create the GitLab CR  (operator reconciles this into the full deployment)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Apply gitlab-google-oidc secret (OIDC provider config for OmniAuth)
# ---------------------------------------------------------------------------
if [[ -n "$FRONTDOOR_HOST" ]]; then
  oidc_secret_tmpl="${SCRIPT_DIR}/../deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
  if is_unresolved_secret_ref "$GITLAB_OIDC_CLIENT_ID" || is_unresolved_secret_ref "$GITLAB_OIDC_CLIENT_SECRET"; then
    repair_blocked "GitLab OIDC client credentials are missing or unresolved" \
      "Set non-empty OIDC client values in installer inputs/config (auth.clientId/auth.clientSecret or GITLAB_OIDC_CLIENT_ID/GITLAB_OIDC_CLIENT_SECRET) and rerun deploy."
  fi
  if [[ -f "$oidc_secret_tmpl" ]]; then
    log "Applying gitlab-google-oidc secret (issuer=${GITLAB_OIDC_ISSUER}, redirect=${GITLAB_OIDC_REDIRECT_URI}) ..."
    GITLAB_OIDC_PROVIDER_NAME="$GITLAB_OIDC_PROVIDER_NAME" \
      GITLAB_OIDC_ISSUER="$GITLAB_OIDC_ISSUER" \
      GITLAB_OIDC_CLIENT_ID="$GITLAB_OIDC_CLIENT_ID" \
      GITLAB_OIDC_CLIENT_SECRET="$GITLAB_OIDC_CLIENT_SECRET" \
      GITLAB_OIDC_REDIRECT_URI="$GITLAB_OIDC_REDIRECT_URI" \
      envsubst < "$oidc_secret_tmpl" | kubectl apply -n "$NAMESPACE" -f - >/dev/null || \
      repair_blocked "Could not apply gitlab-google-oidc secret" "GitLab OIDC provider secret apply failed for namespace ${NAMESPACE}."
  else
    repair_blocked "gitlab-google-oidc-secret.example.yaml not found" "Missing ${oidc_secret_tmpl}; cannot configure GitLab OIDC automatically."
  fi
fi

# ---------------------------------------------------------------------------
# ARM64 / 16KB-page jemalloc fix — build glibc jemalloc on every node once
# All RPi nodes use 16KB kernel pages; GitLab's bundled jemalloc is 4KB-only.
# We compile a compatible jemalloc-5.3.0 with --with-lg-page=14 via DaemonSet.
# ---------------------------------------------------------------------------
setup_jemalloc_on_nodes() {
  local jlib="${JEMALLOC_HOSTPATH_LIB}"
  local ds_name="jemalloc-builder"

  if (( JEMALLOC_HOSTPATH_WANTED == 0 )); then
    log "HostPath jemalloc optimization disabled for this environment."
    if kubectl -n "$NAMESPACE" get ds "$ds_name" >/dev/null 2>&1; then
      log "Deleting existing ${ds_name} DaemonSet because hostPath jemalloc is disabled."
      kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \
        warn "Could not delete ${ds_name} DaemonSet; continuing."
    fi
    JEMALLOC_HOSTPATH_ACTIVE=0
    return 0
  fi

  # Check if already present on all schedulable nodes
  local nodes
  nodes=$(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | tr '\n' ' ')
  local all_ready=true
  for node in $nodes; do
    result=$(kubectl -n "$NAMESPACE" run "jcheck-${node//\./-}" \
      --image=ubuntu:22.04 --restart=Never --rm --attach --quiet \
      --overrides="{\"spec\":{\"nodeName\":\"${node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"jlib\",\"hostPath\":{\"path\":\"${JEMALLOC_HOSTPATH_DIR}\",\"type\":\"DirectoryOrCreate\"}}],\"containers\":[{\"name\":\"c\",\"image\":\"ubuntu:22.04\",\"command\":[\"bash\",\"-c\",\"test -f /jlib/libjemalloc.so.2 && echo OK || echo MISSING\"],\"volumeMounts\":[{\"name\":\"jlib\",\"mountPath\":\"/jlib\"}]}]}}" 2>/dev/null || echo "MISSING")
    if [[ "$result" != *"OK"* ]]; then
      all_ready=false
      break
    fi
  done

  if $all_ready; then
    log "jemalloc-16k already present on all nodes — skipping build."
    JEMALLOC_HOSTPATH_ACTIVE=1
    return 0
  fi

  log "Deploying jemalloc-builder DaemonSet (compiles jemalloc-5.3.0 with --with-lg-page=14 on each node)..."
  local _tmpds
  _tmpds=$(mktemp /tmp/jemalloc-ds-XXXXXX.yaml)
  cat > "$_tmpds" <<'JEDS'
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: jemalloc-builder
  namespace: NAMESPACE_PLACEHOLDER
  labels:
    app: jemalloc-builder
spec:
  selector:
    matchLabels:
      app: jemalloc-builder
  template:
    metadata:
      labels:
        app: jemalloc-builder
    spec:
      tolerations:
      - operator: Exists
      initContainers:
      - name: build
        image: ubuntu:22.04
        command:
        - bash
        - -c
        - |
          set -e
          TARGET=/hostlib/libjemalloc.so.2
          if [ -f "$TARGET" ]; then echo "Already built"; exit 0; fi
          apt-get update -qq && apt-get install -y -q gcc make wget bzip2 2>&1
          cd /tmp
          wget -q -O jemalloc.tar.bz2 \
            https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
          tar xjf jemalloc.tar.bz2 && cd jemalloc-5.3.0
          ./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1
          make -j2 lib/libjemalloc.so.2 2>&1
          cp lib/libjemalloc.so.2 "$TARGET" && chmod 755 "$TARGET"
          echo "Done: $(ls -lh $TARGET)"
        volumeMounts:
        - name: hostlib
          mountPath: /hostlib
      containers:
      - name: keepalive
        image: ubuntu:22.04
        command: [bash, -c, "sleep infinity"]
      volumes:
      - name: hostlib
        hostPath:
          path: JEMALLOC_HOSTPATH_DIR_PLACEHOLDER
          type: DirectoryOrCreate
JEDS
  sed -i.bak "s/NAMESPACE_PLACEHOLDER/${NAMESPACE}/g" "$_tmpds"
  sed -i.bak2 "s#JEMALLOC_HOSTPATH_DIR_PLACEHOLDER#${JEMALLOC_HOSTPATH_DIR}#g" "$_tmpds"
  kubectl apply -f "$_tmpds"
  rm -f "$_tmpds" "${_tmpds}.bak" "${_tmpds}.bak2"

  log "Waiting up to 20 minutes for jemalloc to build on all nodes..."
  local deadline=$((SECONDS + 1200))
  while (( SECONDS < deadline )); do
    ready=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
      -o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0)
    desired=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
      -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo 1)
    [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]] && break
    log "  jemalloc build progress: ${ready}/${desired} nodes ready..."
    sleep 30
  done

  if [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]]; then
    JEMALLOC_HOSTPATH_ACTIVE=1
    log "jemalloc-16k ready on all nodes."
    return 0
  fi

  JEMALLOC_HOSTPATH_ACTIVE=0
  if [[ "${GITLAB_JEMALLOC_MODE,,}" == "force" || "${GITLAB_JEMALLOC_MODE,,}" == "required" || "${GITLAB_JEMALLOC_MODE,,}" == "1" || "${GITLAB_JEMALLOC_MODE,,}" == "true" || "${GITLAB_JEMALLOC_MODE,,}" == "on" ]]; then
    die "jemalloc-builder did not become ready (${ready}/${desired}) and jemalloc is required."
  fi
  kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \
    warn "Could not delete ${ds_name} DaemonSet after timeout; continuing without jemalloc."
  warn "jemalloc-builder did not become ready (${ready}/${desired}); continuing without hostPath jemalloc optimization."
}

# ---------------------------------------------------------------------------
# Garage S3 key + bucket provisioning for GitLab object storage
# Creates a dedicated "gitlab-s3" key in Garage, provisions required buckets,
# and writes the k8s secrets consumed by the GitLab CR.
# ---------------------------------------------------------------------------
setup_garage_for_gitlab() {
  local garage_admin_context garage_admin_context_label
  garage_admin_context="$(resolve_garage_admin_context || true)"
  if [[ -n "$garage_admin_context" ]]; then
    garage_admin_context_label="$garage_admin_context"
  else
    garage_admin_context_label="(active/default)"
  fi

  log "Configuring Garage S3 (${GARAGE_NAMESPACE}) as GitLab object storage..."
  log "Garage admin context: ${garage_admin_context_label}"
  log "Garage namespace: ${GARAGE_NAMESPACE}"
  log "GitLab object storage S3 endpoint: ${GARAGE_S3_ENDPOINT}"

  garage_setup_failure() {
    local reason="$1"
    if is_truthy "${GITLAB_OBJECT_STORAGE_REQUIRED:-1}"; then
      die "${reason} (object storage is required for GitLab deployment)."
    fi
    warn "${reason} (continuing because GITLAB_OBJECT_STORAGE_REQUIRED=${GITLAB_OBJECT_STORAGE_REQUIRED})."
    return 0
  }

  local garage_pod
  garage_pod=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get pods -l app=garage \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -z "$garage_pod" ]]; then
    garage_setup_failure "Garage pod not found in namespace '${GARAGE_NAMESPACE}' on context '${garage_admin_context_label}'"
    return 0
  fi

  local garage_admin_token_b64 garage_admin_token
  garage_admin_token_b64=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get secret garage-secrets \
    -o jsonpath='{.data.admin_token}' 2>/dev/null || true)
  if [[ -z "$garage_admin_token_b64" ]]; then
    garage_setup_failure "Garage admin token missing in secret '${GARAGE_NAMESPACE}/garage-secrets' on context '${garage_admin_context_label}'"
    return 0
  fi

  garage_admin_token=$(printf '%s' "$garage_admin_token_b64" | base64 -d 2>/dev/null || true)
  garage_admin_token="${garage_admin_token//$'\r'/}"
  garage_admin_token="${garage_admin_token//$'\n'/}"
  if [[ -z "$garage_admin_token" ]]; then
    garage_setup_failure "Garage admin token in secret '${GARAGE_NAMESPACE}/garage-secrets' is malformed or empty on context '${garage_admin_context_label}'"
    return 0
  fi

  local garage_cmd_last_reason=""
  local garage_cmd_output=""
  garage_exec() {
    local cmd_desc="$1"
    shift

    local out_file err_file stderr_text lower_stderr
    out_file=$(mktemp)
    err_file=$(mktemp)

    if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
      /garage --admin-token "$garage_admin_token" "$@" >"$out_file" 2>"$err_file"; then
      garage_cmd_output=$(cat "$out_file")
      rm -f "$out_file" "$err_file"
      garage_cmd_last_reason=""
      return 0
    fi

    stderr_text=$(cat "$err_file" 2>/dev/null || true)
    if [[ "$stderr_text" == *"--admin-token"* ]] && [[ "$stderr_text" == *"unknown"* || "$stderr_text" == *"unexpected"* || "$stderr_text" == *"wasn't expected"* ]]; then
      if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
        /garage "$@" >"$out_file" 2>"$err_file"; then
        garage_cmd_output=$(cat "$out_file")
        rm -f "$out_file" "$err_file"
        garage_cmd_last_reason=""
        return 0
      fi
      stderr_text=$(cat "$err_file" 2>/dev/null || true)
    fi

    lower_stderr="${stderr_text,,}"
    if [[ "$lower_stderr" == *"unauth"* || "$lower_stderr" == *"forbidden"* || "$lower_stderr" == *"invalid token"* || "$lower_stderr" == *"permission denied"* || "$lower_stderr" == *" 401"* || "$lower_stderr" == *" 403"* ]]; then
      garage_cmd_last_reason="Unauthenticated admin command while running '${cmd_desc}'"
    else
      garage_cmd_last_reason="Garage admin command failed while running '${cmd_desc}'"
    fi
    if [[ -n "$stderr_text" ]]; then
      garage_cmd_last_reason+=" (${stderr_text})"
    fi
    garage_cmd_output=""

    rm -f "$out_file" "$err_file"
    return 1
  }

  # ---- Resolve or create the gitlab-s3 access key ----
  # garage v1.x outputs plaintext (not JSON) for key list / key info / key create.
  # key list format:   "  GKxxxxxxxxx  <name>"
  # key create/info format includes lines:
  #   "Key ID: GKxxxxxxxxx"
  #   "Secret key: <secret>"
  local access_key secret_key existing_id key_output rotated_key_name key_list_output create_failure_reason
  if ! garage_exec "garage key list" key list; then
    garage_setup_failure "${garage_cmd_last_reason}"
    return 0
  fi
  key_list_output="$garage_cmd_output"
  existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true)

  if [[ -n "$existing_id" ]]; then
    log "Garage key '${GARAGE_S3_KEY_NAME}' already exists (id=${existing_id}); fetching info..."
    if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then
      garage_setup_failure "${garage_cmd_last_reason}"
      return 0
    fi
    key_output="$garage_cmd_output"
  else
    log "Creating Garage key '${GARAGE_S3_KEY_NAME}'..."
    create_failure_reason=""
    if ! garage_exec "garage key create ${GARAGE_S3_KEY_NAME}" key create "$GARAGE_S3_KEY_NAME"; then
      create_failure_reason="$garage_cmd_last_reason"
    fi

    if ! garage_exec "garage key list" key list; then
      if [[ -n "$create_failure_reason" ]]; then
        garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}"
      else
        garage_setup_failure "${garage_cmd_last_reason}"
      fi
      return 0
    fi
    key_list_output="$garage_cmd_output"
    existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true)

    if [[ -z "$existing_id" ]]; then
      if [[ -n "$create_failure_reason" ]]; then
        garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}' (${create_failure_reason})"
      else
        garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'"
      fi
      return 0
    fi

    if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then
      if [[ -n "$create_failure_reason" ]]; then
        garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}"
      else
        garage_setup_failure "${garage_cmd_last_reason}"
      fi
      return 0
    fi
    key_output="$garage_cmd_output"
  fi

  if [[ -z "$key_output" ]]; then
    garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'"
    return 0
  fi

  # Parse plaintext output — match "Key ID: GKxxx" and "Secret key: xxx" lines.
  # Also try JSON in case a future garage version changes format.
  access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1)
  secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1)
  # JSON fallback
  if [[ -z "$access_key" ]]; then
    access_key=$(printf '%s' "$key_output" | python3 -c \
      "import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true)
  fi
  if [[ -z "$secret_key" ]]; then
    secret_key=$(printf '%s' "$key_output" | python3 -c \
      "import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true)
  fi

  if [[ "$secret_key" == "(redacted)" ]]; then
    warn "Garage key '${GARAGE_S3_KEY_NAME}' secret is redacted; creating a deterministic fallback key for GitLab object storage."
    rotated_key_name="${GARAGE_S3_KEY_NAME}-gitlab"
    if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then
      if ! garage_exec "garage key create ${rotated_key_name}" key create "$rotated_key_name"; then
        garage_setup_failure "${garage_cmd_last_reason}"
        return 0
      fi
      if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then
        garage_setup_failure "${garage_cmd_last_reason}"
        return 0
      fi
    fi
    key_output="$garage_cmd_output"
    access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1)
    secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1)
    if [[ -z "$access_key" ]]; then
      access_key=$(printf '%s' "$key_output" | python3 -c \
        "import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true)
    fi
    if [[ -z "$secret_key" ]]; then
      secret_key=$(printf '%s' "$key_output" | python3 -c \
        "import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true)
    fi
    if [[ -n "$access_key" && -n "$secret_key" && "$secret_key" != "(redacted)" ]]; then
      GARAGE_S3_KEY_NAME="$rotated_key_name"
      log "Using rotated Garage key '${GARAGE_S3_KEY_NAME}'."
    fi
  fi

  if [[ -z "$access_key" || -z "$secret_key" || "$secret_key" == "(redacted)" ]]; then
    garage_setup_failure "Malformed Garage key output for '${GARAGE_S3_KEY_NAME}' (missing access key or secret key)"
    warn "Raw output: ${key_output}"
    return 0
  fi

  # ---- Create buckets and grant permissions ----
  # --key takes the accessKeyId (not the name) in garage v1.x
  # Bucket names must match GitLab Helm chart defaults because the GitLab CR's
  # global.appConfig.object_store block only specifies the connection secret —
  # it does NOT override per-object bucket names. Rails therefore writes to
  # gitlab-uploads / gitlab-artifacts / etc. (no -storage suffix). A prior
  # naming with a -storage suffix caused Excon::Error::NotFound / NoSuchBucket
  # on first login when CarrierWave tried to upload the duo-bot avatar to
  # /gitlab-uploads/user/avatar/8/duo-bot.png. Keep registry / terraform-state /
  # ci-secure-files verbatim — those are the names both sides already agreed on.
  local buckets=(
    registry
    gitlab-artifacts
    gitlab-lfs
    gitlab-uploads
    gitlab-packages
    gitlab-dependency-proxy
    gitlab-terraform-state
    gitlab-ci-secure-files
  )
  for bucket in "${buckets[@]}"; do
    if ! garage_exec "garage bucket create ${bucket}" bucket create "$bucket"; then
      local bucket_create_failure_reason lower_bucket_create_failure_reason
      bucket_create_failure_reason="${garage_cmd_last_reason}"
      lower_bucket_create_failure_reason="${bucket_create_failure_reason,,}"
      if [[ "$lower_bucket_create_failure_reason" == *"already exists"* ]]; then
        log "Garage bucket '${bucket}' already exists; reusing existing bucket."
      else
        garage_setup_failure "${bucket_create_failure_reason}"
        return 0
      fi
    fi
    # bucket name is positional; --read/--write/--owner are boolean flags
    if ! garage_exec "garage bucket allow ${bucket}" bucket allow "$bucket" --key "$access_key" \
      --read --write --owner; then
      garage_setup_failure "${garage_cmd_last_reason}"
      return 0
    fi
  done
  log "Garage buckets provisioned for GitLab."

  # ---- Write k8s secrets consumed by the GitLab CR ----
  # Rails object storage connection (artifacts, LFS, uploads, packages, etc.)
  local object_storage_secret_apply_output registry_storage_secret_apply_output
  object_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-object-storage \
    --from-literal=connection="provider: AWS
region: garage
aws_access_key_id: ${access_key}
aws_secret_access_key: ${secret_key}
endpoint: '${GARAGE_S3_ENDPOINT}'
path_style: true" \
    --dry-run=client -o yaml | kubectl apply -f - 2>&1)"
  if kubectl_apply_reports_changed "$object_storage_secret_apply_output"; then
    gitlab_object_storage_secret_changed=1
  fi
  log "gitlab-object-storage secret applied (${object_storage_secret_apply_output})."

  # Docker registry S3 storage driver config.
  # Use s3_v2 (AWS SDK v2) — s3_v1 has signature computation issues with
  # non-standard ports (Host: endpoint:3900) against Garage.  SDK v2 always
  # uses SigV4; v4auth is not a recognised key and must be omitted.
  registry_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-registry-storage \
    --from-literal=config="s3_v2:
  accesskey: ${access_key}
  secretkey: ${secret_key}
  bucket: registry
  regionendpoint: ${GARAGE_S3_ENDPOINT}
  region: garage
  pathstyle: true" \
    --dry-run=client -o yaml | kubectl apply -f - 2>&1)"
  if kubectl_apply_reports_changed "$registry_storage_secret_apply_output"; then
    gitlab_registry_storage_secret_changed=1
  fi
  log "gitlab-registry-storage secret applied (${registry_storage_secret_apply_output})."
}

# ---------------------------------------------------------------------------
# Legacy/local static PV + synology directory setup for gitaly
# minio PV removed — object storage is provided by knoe-system/garage
# ---------------------------------------------------------------------------
setup_gitlab_legacy_storage() {
  local synology_node="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
  local storage_selector_key="${STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"
  local base_path="${GITLAB_STORAGE_BASE:-/synology/d005/gitlab}"
  local gitaly_size="${GITLAB_GITALY_PV_SIZE:-50Gi}"

  [[ -n "$synology_node" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set for legacy GitLab storage setup."

  log "Creating synology gitaly directory on ${synology_node}..."
  kubectl -n "$NAMESPACE" run gitlab-dirprep \
    --image=alpine:latest --restart=Never --rm --attach \
    --overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"s\",\"hostPath\":{\"path\":\"$(dirname ${base_path})\",\"type\":\"Directory\"}}],\"containers\":[{\"name\":\"dirprep\",\"image\":\"alpine:latest\",\"command\":[\"sh\",\"-c\",\"mkdir -p ${base_path}/gitaly && chmod 777 ${base_path}/gitaly && echo done\"],\"volumeMounts\":[{\"name\":\"s\",\"mountPath\":\"$(dirname ${base_path})\"}]}]}}" \
    2>&1 || warn "Could not create synology dirs (may already exist)"

  # A named no-provisioner StorageClass is required so the helm chart renders
  # storageClassName in the PVC template (an empty string is treated as falsy
  # by the chart and omitted, causing the PVC to use the cluster default).
  log "Ensuring gitlab-gitaly-static StorageClass (no-provisioner)..."
  kubectl apply -f - <<SCYAML
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gitlab-gitaly-static
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: Immediate
reclaimPolicy: Retain
SCYAML

  log "Creating static PV for gitaly (${gitaly_size})..."
  kubectl apply -f - <<PVYAML
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-gitaly-synology
spec:
  capacity:
    storage: ${gitaly_size}
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: gitlab-gitaly-static
  local:
    path: ${base_path}/gitaly
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: ${storage_selector_key}
          operator: In
          values: [${synology_node}]
PVYAML
  log "GitLab gitaly storage PV ready."

}

setup_gitlab_storage() {
  if [[ "$MODE" == "k8s" ]]; then
    log "Skipping legacy Synology/local-PV gitaly storage prep in k8s mode; using dynamic StorageClass '${GITALY_STORAGE_CLASS}'."

    # Check for ACTUAL legacy resources on GKE/k8s.
    #
    # Historical bug: the bare check `kubectl get pvc repo-data-gitlab-gitaly-0`
    # flagged has_legacy=1 on every run once the PVC existed — which is always
    # true after the first deploy. Combined with GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1
    # that triggered a destructive PVC delete on every init_gitlab.sh run,
    # silently wiping the gitaly git-repo PV. The chart's normal PVC is not
    # legacy; the real legacy markers are synology PV, gandalf.knoe.org
    # nodeSelector, or gitlab-gitaly-static storageClass.
    local has_legacy=0
    if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then has_legacy=1; fi

    local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
    local gitaly_sts_yaml
    gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
    if [[ -n "$gitaly_sts_yaml" ]]; then
      if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* || "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
         has_legacy=1
      fi
    fi

    # Also treat PVC as legacy only when its actual storageClassName is a
    # known legacy class — not merely because the object exists.
    local pvc_sc
    pvc_sc=$(kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 \
      -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
    if [[ "$pvc_sc" == "gitlab-gitaly-static" || "$pvc_sc" == "synology-iscsi" ]]; then
      has_legacy=1
    fi

    if [[ "$has_legacy" == "1" ]]; then
       if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then
          export GITALY_AUTOCLEAN_PERFORMED=1
          log "AUTOCLEAN: repairing legacy Gitaly storage (scaling down and deleting legacy PV/PVC)..."
          kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
          kubectl -n "$NAMESPACE" delete pvc repo-data-gitlab-gitaly-0 --wait=false 2>/dev/null || true
          kubectl delete pv gitlab-gitaly-synology --wait=false 2>/dev/null || true
          # Give it a moment to process deletions
          sleep 2
       else
          repair_blocked "Legacy Gitaly PV/PVC still bound to gitlab-gitaly-static / gandalf.prole.org" \
            "Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to automatically repair by deleting legacy PVC/PV, or run:
kubectl -n $NAMESPACE scale sts ${GITLAB_RELEASE}-gitaly --replicas=0
kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0
kubectl delete pv gitlab-gitaly-synology"
       fi
    fi

    repair_stale_gke_gitaly_dynamic_storage "${GITALY_STORAGE_CLASS:-}"
  else
    setup_gitlab_legacy_storage
  fi

  # Object storage (registry, artifacts, LFS, uploads, etc.) is provided by
  # knoe-system/garage — credentials and buckets are set up here.
  setup_garage_for_gitlab
}

configure_jemalloc_hostpath_mode
setup_jemalloc_on_nodes
check_gitlab_pre_apply_blocked
setup_gitlab_storage

log "Pre-creating gitlab-app-nonroot ServiceAccount (required by chart v9+)..."
kubectl -n "${NAMESPACE}" apply -f - <<SAYAML
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitlab-app-nonroot
  namespace: ${NAMESPACE}
SAYAML

GITLAB_ALLOWED_HOSTS_YAML=""
_gitlab_allowed_hosts_csv=""
GITLAB_ALLOWED_HOSTS=()
append_gitlab_allowed_host() {
  local candidate="$1"
  [[ -n "$candidate" ]] || return 0
  case ",${_gitlab_allowed_hosts_csv}," in
    *,"${candidate}",*) return 0 ;;
  esac
  GITLAB_ALLOWED_HOSTS+=("$candidate")
  _gitlab_allowed_hosts_csv="${_gitlab_allowed_hosts_csv:+${_gitlab_allowed_hosts_csv},}${candidate}"
  return 0
}

for _gitlab_allowed_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
  append_gitlab_allowed_host "$_gitlab_allowed_host"
  _gitlab_host_root="${_gitlab_allowed_host#*.}"
  if [[ -n "$_gitlab_host_root" && "$_gitlab_host_root" != "$_gitlab_allowed_host" ]]; then
    append_gitlab_allowed_host "$_gitlab_host_root"
  fi
done
unset _gitlab_allowed_host _gitlab_host_root

for _gitlab_allowed_host in "${GITLAB_ALLOWED_HOSTS[@]}"; do
  GITLAB_ALLOWED_HOSTS_YAML+=$'\n'"          - ${_gitlab_allowed_host}"
done
unset _gitlab_allowed_host

GITLAB_TRUSTED_PROXIES_YAML=""
for _gitlab_trusted_proxy in "${GITLAB_TRUSTED_PROXIES[@]}"; do
  GITLAB_TRUSTED_PROXIES_YAML+=$'\n'"            - ${_gitlab_trusted_proxy}"
done
unset _gitlab_trusted_proxy
log "GitLab trusted_proxies configured: ${_gitlab_trusted_proxies_csv}"

GLOBAL_JEMALLOC_VALUES_YAML=""
WORKLOAD_JEMALLOC_VALUES_YAML=""
if (( JEMALLOC_HOSTPATH_ACTIVE == 1 )); then
  GLOBAL_JEMALLOC_VALUES_YAML=$(cat <<JEMGLOBAL
        # ARM64 nodes: inject glibc jemalloc compiled with --with-lg-page=14
        # The DaemonSet above places it at ${JEMALLOC_HOSTPATH_LIB}
        extraEnv:
          LD_PRELOAD: ${JEMALLOC_HOSTPATH_LIB}
JEMGLOBAL
)

  WORKLOAD_JEMALLOC_VALUES_YAML=$(cat <<JEMWORKLOAD
          extraEnv:
            LD_PRELOAD: ${JEMALLOC_HOSTPATH_LIB}
          extraVolumes: |
            - name: jemalloc-16k
              hostPath:
                path: ${JEMALLOC_HOSTPATH_DIR}
                type: Directory
          extraVolumeMounts: |
            - name: jemalloc-16k
              mountPath: ${JEMALLOC_HOSTPATH_DIR}
              readOnly: true
JEMWORKLOAD
)
else
  log "Rendering GitLab values without hostPath jemalloc mounts/LD_PRELOAD."
fi

gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_exists_before=0
if [[ -n "$gitlab_generation_before" ]]; then
  gitlab_exists_before=1
fi

GITLAB_CR_RENDERED="$(cat <<EOF
apiVersion: apps.gitlab.com/v1beta1
kind: GitLab
metadata:
  name: ${GITLAB_RELEASE}
  namespace: ${NAMESPACE}
spec:
  chart:
    ${CHART_VERSION_YAML}
    values:
      global:
        persistence:
          storageClass: ${GITALY_STORAGE_CLASS}
${GLOBAL_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<GNODE
        nodeSelector:
          ${NODE_SELECTOR_YAML}
GNODE
fi)
        # Explicit allowedHosts as flat strings — chart 9.x renders list-of-maps
        # by default which breaks URI::InvalidComponentError in 7_gitlab_http.rb
        allowedHosts:${GITLAB_ALLOWED_HOSTS_YAML}
        hosts:
          domain: ${PRIMARY_GITLAB_DOMAIN_ROOT}
          gitlab:
            name: ${GITLAB_DOMAIN}
          ssh: ${GITLAB_SSH_HOST}
        ingress:
          class: ${GITLAB_INGRESS_CLASS}
          configureCertmanager: false
          tls:
            enabled: false
        psql:
          host: ${DB_HOST}
          port: ${DB_PORT}
          username: ${GITLAB_DB_USER}
          database: ${GITLAB_DB_NAME}
          password:
            secret: gitlab-db-password
            key: password
        redis:
          host: ${REDIS_HOST}
          port: ${REDIS_PORT}
          auth:
            enabled: false
        # chart v9+ requires the SA to be pre-created when a custom name is used
        serviceAccount:
          create: false
        # Disable bundled minio globally — object storage is provided by garage
        minio:
          enabled: false
        # Object storage via knoe-system/garage (S3-compatible)
        appConfig:
          object_store:
            enabled: true
            proxy_download: true
            connection:
              secret: gitlab-object-storage
              key: connection
          trusted_proxies:${GITLAB_TRUSTED_PROXIES_YAML}
$(if [[ -n "$FRONTDOOR_HOST" ]]; then
cat <<OMNIAUTH
          omniauth:
            enabled: true
            autoSignInWithProvider: openid_connect
            allowSingleSignOn:
              - openid_connect
            blockAutoCreatedUsers: false
            providers:
              - secret: gitlab-google-oidc
                key: provider
OMNIAUTH
fi)
        registry:
          bucket: registry
      # Disable bundled sub-charts — we provide our own data stores
      postgresql:
        install: false
      redis:
        install: false
      certmanager:
        install: false
      prometheus:
        install: false
      gitlab-runner:
        install: false
      # minio sub-chart is disabled via global.minio.enabled: false (set above under global:).
      # Do NOT set minio.enabled here — chart v9 removed the sub-chart level property
      # and treats it as a hard validation error (NOTES.txt:234).
      gitlab:
        webservice:
          # Shrunk Puma for modest-sized GKE nodes (currently e2-standard-2,
          # ~7.1 GB allocatable — four pods share it).
          # Default workerProcesses=2 × 4 threads → ~2.2 GB steady state
          # (and ~2.8 GB boot spike) → OOMKilled under a 2 GB limit.
          # workerProcesses=1 + threads=2 keeps steady RSS ~0.9–1.1 GB and
          # boot spike ~1.4–1.6 GB, which fits within ${GITLAB_WEBSERVICE_LIMITS_MEMORY}.
          replicaCount: 1
          workerProcesses: ${GITLAB_WEBSERVICE_WORKER_PROCESSES}
          puma:
            # Canonical chart path — env-var overrides are ignored because the
            # chart renders puma.rb from these values via ERB.
            threads:
              min: ${GITLAB_WEBSERVICE_PUMA_THREADS_MIN}
              max: ${GITLAB_WEBSERVICE_PUMA_THREADS_MAX}
          resources:
            requests:
              cpu: ${GITLAB_WEBSERVICE_REQUESTS_CPU}
              memory: ${GITLAB_WEBSERVICE_REQUESTS_MEMORY}
            limits:
              memory: ${GITLAB_WEBSERVICE_LIMITS_MEMORY}
          # Rails preloading is slow on shared nodes — protect against
          # probe kills during the boot window, and against transient Gitaly
          # unavailability (readiness depends on /-/readiness reaching Gitaly).
          livenessProbe:
            initialDelaySeconds: 3600
            periodSeconds: 60
            timeoutSeconds: 30
            failureThreshold: 5
          readinessProbe:
            initialDelaySeconds: 120
            periodSeconds: 30
            timeoutSeconds: 30
            failureThreshold: 60
          ingress:
            enabled: ${GITLAB_WEBSERVICE_INGRESS_ENABLED}
            annotations:
              kubernetes.io/ingress.class: ${GITLAB_INGRESS_CLASS}
          minReplicas: 1
          maxReplicas: 1
${WORKLOAD_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
          nodeSelector:
            ${NODE_SELECTOR_YAML}
NODE
fi)
        sidekiq:
          replicaCount: 1
          minReplicas: 1
          maxReplicas: 1
          concurrency: ${GITLAB_SIDEKIQ_CONCURRENCY}
          resources:
            requests:
              cpu: ${GITLAB_SIDEKIQ_REQUESTS_CPU}
              memory: ${GITLAB_SIDEKIQ_REQUESTS_MEMORY}
            limits:
              cpu: ${GITLAB_SIDEKIQ_LIMITS_CPU}
              memory: ${GITLAB_SIDEKIQ_LIMITS_MEMORY}
          # Rails loading takes 25-40min on RPi; protect against liveness kills
          livenessProbe:
            initialDelaySeconds: 3600
            periodSeconds: 60
            timeoutSeconds: 30
            failureThreshold: 5
          readinessProbe:
            initialDelaySeconds: 120
            periodSeconds: 30
            timeoutSeconds: 30
            failureThreshold: 60
${WORKLOAD_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
          nodeSelector:
            ${NODE_SELECTOR_YAML}
NODE
fi)
        gitaly:
          # Storage class is mode-specific:
          # - k8s: dynamic PVCs on cluster storage class
          # - legacy/local: static local-PV storage class created by setup_gitlab_storage()
          persistence:
            storageClass: ${GITALY_STORAGE_CLASS}
            size: ${GITLAB_GITALY_PV_SIZE:-50Gi}
${WORKLOAD_JEMALLOC_VALUES_YAML}
          # Give gitaly gRPC health check time to initialise before probes start
          livenessProbe:
            initialDelaySeconds: 30
          readinessProbe:
            initialDelaySeconds: 15
$(if [[ -n "$STORAGE_NODE_SELECTOR_YAML" ]]; then
cat <<NODE
          # Legacy/local mode pins gitaly to storage node for local PV.
          nodeSelector:
            ${STORAGE_NODE_SELECTOR_YAML}
NODE
fi)
        toolbox:
          backups:
            objectStorage:
              backend: s3
              config:
                secret: gitlab-object-storage
                key: connection
${WORKLOAD_JEMALLOC_VALUES_YAML}
        migrations:
${WORKLOAD_JEMALLOC_VALUES_YAML}
        gitlab-exporter:
${WORKLOAD_JEMALLOC_VALUES_YAML}
        gitlab-shell:
          replicaCount: 1
          minReplicas: 1
          maxReplicas: 1
$(if [[ -n "${GITLAB_SHELL_LOADBALANCER_IP:-}" ]]; then
cat <<SHELLSVC
          service:
            type: LoadBalancer
            loadBalancerIP: ${GITLAB_SHELL_LOADBALANCER_IP}
            externalTrafficPolicy: ${GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY}
SHELLSVC
fi)
        kas:
          replicaCount: 1
          minReplicas: 1
          maxReplicas: 1
      gitaly:
        persistence:
          storageClass: ${GITALY_STORAGE_CLASS}
          size: ${GITLAB_GITALY_PV_SIZE:-50Gi}
      registry:
        replicaCount: 1
        hpa:
          minReplicas: 1
          maxReplicas: 1
        storage:
          secret: gitlab-registry-storage
          key: config
        ingress:
          enabled: false
EOF
)"

GITLAB_CR_RENDERED="$(sanitize_gitlab_cr_rendered_values_for_chart_version "$GITLAB_CR_RENDERED")"
preflight_validate_gitlab_cr_rendered_values "$GITLAB_CR_RENDERED"

log "Applying GitLab CR (hosts=${_gitlab_hosts_csv}, db=${GITLAB_DB_NAME}@${DB_HOST})..."
gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_apply_output="$(echo "$GITLAB_CR_RENDERED" | kubectl apply -f -)"
gitlab_generation_after="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"

gitlab_spec_changed=1
if [[ -n "$gitlab_generation_before" && -n "$gitlab_generation_after" && "$gitlab_generation_before" == "$gitlab_generation_after" ]]; then
  gitlab_spec_changed=0
fi
gitlab_apply_changed=0
if kubectl_apply_reports_changed "$gitlab_apply_output"; then
  if [[ "$MODE" == "k8s" && "$gitlab_spec_changed" == "0" ]]; then
    # Some apply operations may report "configured" without changing generation.
    # In that case, avoid triggering a full reconcile solely from apply output text.
    log "GitLab CR apply reported change, but generation is unchanged. Skipping drift trigger."
  else
    gitlab_apply_changed=1
  fi
fi

gitlab_config_changed=0
if [[ "$gitlab_db_secret_changed" == "1" || "$gitlab_object_storage_secret_changed" == "1" || "$gitlab_registry_storage_secret_changed" == "1" ]]; then
  gitlab_config_changed=1
fi

gitlab_reconcile_required=0
# Check actual replicas to ensure they match desired count (1) - Requirement 3
gitlab_replica_drift=0
for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
  _dep_name="${GITLAB_RELEASE}-${_dep_suffix}"
  _actual=$(kubectl -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")
  if [[ "$_actual" != "1" ]]; then
    log "Detected replica drift for ${_dep_name}: actual=${_actual}, desired=1"
    gitlab_replica_drift=1
    break
  fi
done

if [[ "$gitlab_exists_before" == "0" || "$operator_changed" == "1" || "$gitlab_chart_version_changed" == "1" || "$gitlab_spec_changed" == "1" || "$gitlab_apply_changed" == "1" || "$gitlab_config_changed" == "1" || "$gitlab_replica_drift" == "1" ]]; then
  gitlab_reconcile_required=1
fi

check_gitlab_post_apply_blocked

# minio is disabled (global.minio.enabled: false) — no ARM64 credential fix needed.

# ---------------------------------------------------------------------------
# ARM64 configure check: GitLab v17+ (chart v8+) ships multi-arch CNG images
# that include ARM64 variants. The old workaround of replacing the configure
# init container with alpine:latest is no longer needed and breaks things
# because /templates/configure is baked into the gitlab-base image, not
# mounted from a ConfigMap.
# This function is kept as a no-op to avoid breaking callers; remove entirely
# once the chart version floor is confirmed stable at v9+.
# ---------------------------------------------------------------------------
fix_registry_arm64_configure() {
  log "Skipping ARM64 configure patch — GitLab v18 CNG images are multi-arch natively."
}

fix_registry_arm64_configure

log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)."
log "Monitor progress:  kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w"
log "Watch pods:        kubectl -n ${NAMESPACE} get pods -w"

# ---------------------------------------------------------------------------
# Optionally wait for GitLab to become available
# ---------------------------------------------------------------------------
WAIT_TIMEOUT="${GITLAB_WAIT_TIMEOUT:-1200}"   # default 20 minutes
if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
  if [[ "$gitlab_reconcile_required" == "0" ]]; then
    log "Fast-path: no GitLab reconcile triggers detected; skipping long wait."
    gitlab_available_status="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{range .status.conditions[?(@.type=="Available")]}{.status}{end}' 2>/dev/null || true)"
    if [[ "$gitlab_available_status" == "True" ]]; then
      log "Short health check OK: GitLab CR condition Available=True."
      check_gitlab_post_apply_blocked
    else
      warn "Short health check: GitLab CR Available condition is '${gitlab_available_status:-unknown}'."
      kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
    fi
  else
    log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status (checking for blocked states every 30s)..."
    _wait_start=$(date +%s)
    while true; do
      if kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \
           --for=condition=Available \
           --timeout=30s >/dev/null 2>&1; then
        log "GitLab CR condition Available=True."
        break
      fi

      check_gitlab_post_apply_blocked

      _now=$(date +%s)
      if (( _now - _wait_start >= WAIT_TIMEOUT )); then
        _ctx=$(get_gitlab_blocker_context)
        repair_blocked "Timed out waiting for GitLab CR condition=Available" \
          "The deployment is taking too long. Context: ${_ctx}Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}"
      fi
    done
  fi
fi

# ---------------------------------------------------------------------------
# Requirement: Explicitly scale down over-replicated GitLab deployments
# Some components may remain at 2 replicas even after CR is Available.
# ---------------------------------------------------------------------------
log "Verifying GitLab deployment replica counts (desired=1)..."
_gitlab_replica_target=1

if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
  gitlab_verify_replica_source_of_truth "$_gitlab_replica_target"
fi

gitlab_post_remediation_success=0
_replica_drift_found=0
_corrected_deployments=()
_drifted_deployments=()
# Explicit list of components to verify and scale if needed (Requirement 2)
for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2"; do
  _dep_name="${GITLAB_RELEASE}-${_dep_suffix}"
  
  # Read live spec.replicas (Requirement 2 & 6)
  _live_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
  if [[ -z "$_live_replicas" || ! "$_live_replicas" =~ ^[0-9]+$ ]]; then
    _live_replicas=0
  fi
  
  if [[ "$_live_replicas" != "$_gitlab_replica_target" ]]; then
    log "Detected replica drift for ${_dep_name}: spec=${_live_replicas}, desired=${_gitlab_replica_target}"
    _replica_drift_found=1
    _drifted_deployments+=("$_dep_name")
  else
    log "Deployment ${_dep_name} replica count is correct: spec=${_live_replicas}, desired=${_gitlab_replica_target}"
  fi
done

if [[ "$_replica_drift_found" == "1" ]]; then
  log "Replica drift detected. Re-applying GitLab CR source-of-truth (gitlab-shell/kas/registry/sidekiq desired=${_gitlab_replica_target})."
  if ! echo "$GITLAB_CR_RENDERED" | kubectl --context "$KUBECTL_CONTEXT" apply -f - >/dev/null; then
    repair_blocked "Failed to re-apply GitLab CR for replica drift remediation" \
      "Could not reconcile source-of-truth replica values."
  fi

  gitlab_verify_replica_source_of_truth "$_gitlab_replica_target"

  _source_of_truth_mismatch_report=""

  for _dep_name in "${_drifted_deployments[@]}"; do
    _post_reconcile_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
    if [[ -z "$_post_reconcile_replicas" || ! "$_post_reconcile_replicas" =~ ^[0-9]+$ ]]; then
      _post_reconcile_replicas=0
    fi

    _corrected_deployments+=("$_dep_name")
    if [[ "$_post_reconcile_replicas" != "$_gitlab_replica_target" ]]; then
      _source_of_truth_mismatch_report+="- ${_dep_name}: spec=${_post_reconcile_replicas}, desired=${_gitlab_replica_target}"$'\n'
    fi
    log "Deployment ${_dep_name} after CR reconcile: spec=${_post_reconcile_replicas}, desired=${_gitlab_replica_target}."
  done

  if [[ -n "$_source_of_truth_mismatch_report" ]]; then
    _rendered_replica_fields_report="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")"
    if [[ -z "$_rendered_replica_fields_report" ]]; then
      _rendered_replica_fields_report="- (no replica-related fields detected in rendered CR values)"
    fi
    repair_blocked "GitLab operator desired replica source-of-truth mismatch after CR re-apply" \
      "Deployment specs still do not match desired=${_gitlab_replica_target}:\n${_source_of_truth_mismatch_report}Rendered GitLab CR replica source fields:\n${_rendered_replica_fields_report}\nThis indicates CR source-of-truth still resolves to replicas>1."
  fi

  log "Waiting for rollout of reconciled deployments..."
  _remediation_rollout_failed=0
  for _dep_name in "${_corrected_deployments[@]}"; do
    if ! kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" rollout status deployment/"$_dep_name" --timeout=300s; then
      warn "Rollout status check did not report success for ${_dep_name}; final replica state will decide pass/fail."
      _remediation_rollout_failed=1
    fi
  done
  
  # Final verification with settle window to tolerate transient old pod linger
  _settle_timeout_s="${GITLAB_POST_REMEDIATION_SETTLE_TIMEOUT:-90}"
  _settle_poll_interval_s="${GITLAB_POST_REMEDIATION_SETTLE_POLL_INTERVAL:-5}"
  if [[ -z "$_settle_timeout_s" || ! "$_settle_timeout_s" =~ ^[0-9]+$ || "$_settle_timeout_s" == "0" ]]; then
    _settle_timeout_s=90
  fi
  if [[ -z "$_settle_poll_interval_s" || ! "$_settle_poll_interval_s" =~ ^[0-9]+$ || "$_settle_poll_interval_s" == "0" ]]; then
    _settle_poll_interval_s=5
  fi

  log "Post-remediation settle verification for corrected deployments (timeout=${_settle_timeout_s}s, poll=${_settle_poll_interval_s}s):"
  _remediation_final_failed=0
  _remediation_failure_class=""
  _remediation_failure_report=""
  if (( ${#_corrected_deployments[@]} > 0 )); then
    _settle_start_ts=$(date +%s)
    _settle_had_errexit=0
    if [[ $- == *e* ]]; then
      _settle_had_errexit=1
      set +e
    fi
    _settle_iteration=0
    while true; do
      _settle_iteration=$((_settle_iteration + 1))
      _remediation_failure_report=""
      _iteration_blocked=0
      _source_of_truth_reversion=0
      for _dep_name in "${_corrected_deployments[@]}"; do
        _target_desired_replicas="${_gitlab_replica_target}"
        _desired_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
        if [[ -z "$_desired_replicas" || ! "$_desired_replicas" =~ ^[0-9]+$ ]]; then
          _desired_replicas=0
        fi

        _status_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.replicas}' 2>/dev/null || true)
        _status_ready=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
        _status_available=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
        _status_updated=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.updatedReplicas}' 2>/dev/null || true)
        _status_replicas=${_status_replicas:-0}
        _status_ready=${_status_ready:-0}
        _status_available=${_status_available:-0}
        _status_updated=${_status_updated:-0}

        _dep_selector=$(gitlab_selector_for_deployment "$_dep_name" 2>/dev/null || true)
        if [[ -n "$_dep_selector" ]]; then
          _final_live=$(gitlab_non_terminal_pod_count_for_app "$_dep_selector" 2>/dev/null || true)
          if [[ -z "$_final_live" || ! "$_final_live" =~ ^[0-9]+$ ]]; then
            _final_live=0
          fi
          _matching_pods=$(gitlab_non_terminal_pod_names_for_app "$_dep_selector" 2>/dev/null || true)
        else
          _final_live=0
          _matching_pods=""
        fi
        if [[ -z "$_matching_pods" ]]; then
          _matching_pods="(none)"
        fi

        log "Settle poll #${_settle_iteration}: deployment ${_dep_name} selector=${_dep_selector:-<unresolved>} spec=${_desired_replicas}, desired=${_target_desired_replicas}, live=${_final_live}, status(replicas/ready/available/updated)=${_status_replicas}/${_status_ready}/${_status_available}/${_status_updated}, pods=[${_matching_pods}]"

        if [[ -z "$_final_live" || ! "$_final_live" =~ ^[0-9]+$ || "$_desired_replicas" != "$_target_desired_replicas" || "$_final_live" != "$_target_desired_replicas" ]]; then
          _iteration_blocked=1
          if [[ "$_desired_replicas" != "$_target_desired_replicas" ]]; then
            _source_of_truth_reversion=1
          fi
          _remediation_failure_report+="- ${_dep_name}: spec=${_desired_replicas}, final_live=${_final_live:-unknown}, desired=${_target_desired_replicas}, selector=${_dep_selector:-<unresolved>}, status=${_status_replicas}/${_status_ready}/${_status_available}/${_status_updated}, pods=[${_matching_pods}]"$'\n'
        fi
      done

      if [[ "$_source_of_truth_reversion" == "1" ]]; then
        _remediation_final_failed=1
        _remediation_failure_class="source_of_truth"
        break
      fi

      if [[ "$_iteration_blocked" == "0" ]]; then
        break
      fi

      _settle_now_ts=$(date +%s)
      if (( _settle_now_ts - _settle_start_ts >= _settle_timeout_s )); then
        _remediation_final_failed=1
        _remediation_failure_class="convergence"
        break
      fi

      sleep "$_settle_poll_interval_s"
    done
    if [[ "$_settle_had_errexit" == "1" ]]; then
      set -e
    fi
  fi

  if [[ "$_remediation_final_failed" == "1" ]]; then
    if [[ "$_remediation_failure_class" == "source_of_truth" ]]; then
      repair_blocked "GitLab operator reverted deployment spec.replicas after source-of-truth reconcile" \
        "Operator-managed desired state diverged during settle verification:\n${_remediation_failure_report}This is a source-of-truth failure (CR values path), not a rollout lag issue."
    else
      repair_blocked "GitLab post-remediation replica verification failed" \
        "Source-of-truth reconcile ran, but final convergence did not meet required conditions (spec.replicas==desired and live selector pod count==desired):\n${_remediation_failure_report}Check: kubectl -n ${NAMESPACE} get deploy -l app.kubernetes.io/instance=${GITLAB_RELEASE}"
    fi
  fi

  gitlab_replica_drift=0
  gitlab_reconcile_required=0
  gitlab_post_remediation_success=1
  if [[ "$_remediation_rollout_failed" == "1" ]]; then
    log "GitLab corrective action converged by final replica state (spec and live selector counts match desired=${_gitlab_replica_target})."
  else
    log "GitLab corrective action succeeded: rollout completed and all spec/live selector counts match desired=${_gitlab_replica_target}."
  fi
fi

if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
  if [[ "${gitlab_post_remediation_success:-0}" == "1" ]]; then
    log "GitLab corrective action success is authoritative; skipping additional post-remediation convergence gate."
  else
    wait_for_gitlab_workload_convergence
  fi
else
  warn "GITLAB_NO_WAIT=1, skipping strict GitLab workload convergence checks."
fi

# ---------------------------------------------------------------------------
# Public ingress for configured GitLab domain -> gitlab-webservice
# The GitLab Operator creates the Ingress; this block ensures a stable public
# class/host mapping if the CR-managed ingress is absent or not usable.
# ---------------------------------------------------------------------------
WEBSERVICE_SVC="${GITLAB_RELEASE}-webservice-default"

GITLAB_INGRESS_RULES_YAML=""
for _gitlab_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
  GITLAB_INGRESS_RULES_YAML+=$'\n'"    - host: ${_gitlab_host}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: ${WEBSERVICE_SVC}
                port:
                  number: 8181"
done
unset _gitlab_host

GITLAB_AUTH_ANNOTATIONS_YAML=""
case "${FRONTDOOR_AUTH_ENABLED}" in
  1|true|TRUE|True|yes|YES|Yes|on|ON|On)
    GITLAB_AUTH_ANNOTATIONS_YAML=$(cat <<EOAUTH
    nginx.ingress.kubernetes.io/auth-url: "${AUTH_VERIFY_URL}"
    nginx.ingress.kubernetes.io/auth-signin: "${AUTH_SIGNIN_URL}"
    nginx.ingress.kubernetes.io/auth-response-headers: "${AUTH_RESPONSE_HEADERS}"
EOAUTH
)
    ;;
esac

# -- GKE-managed TLS (only when ingressClassName=gce) ------------------------
# With GCE/GCLB ingress, the idiomatic TLS story is ManagedCertificate +
# FrontendConfig (HTTP->HTTPS redirect).  We create both alongside the
# Ingress and wire them up via annotations.  No-op for nginx/kong classes.
#
# We also create a BackendConfig so GCLB's health check hits
# gitlab-workhorse's `/-/readiness` endpoint instead of `/`.
# The default `/` check returns 302 (workhorse redirects to /users/sign_in),
# which GCLB interprets as an unhealthy backend — the backend flaps between
# UNHEALTHY/HEALTHY and the ingress surfaces as HTTP 502.
# The Service is annotated with cloud.google.com/backend-config below so
# GCLB picks the BackendConfig up for the workhorse port (8181).
GITLAB_GCE_TLS_ANNOTATIONS_YAML=""
GITLAB_MANAGED_CERT_NAME="gitlab-managed-cert"
GITLAB_FRONTEND_CONFIG_NAME="gitlab-frontend-config"
GITLAB_BACKEND_CONFIG_NAME="gitlab-webservice-backendconfig"
# Optional: pin the GCE L7 ingress to a pre-reserved global external static IP
# (gcloud compute addresses create --global). Prevents IP churn on ingress
# delete/recreate. Leave blank to let GCE assign ephemerally.
GITLAB_GLOBAL_STATIC_IP_NAME="${GITLAB_GLOBAL_STATIC_IP_NAME:-}"
if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
  GITLAB_GCE_TLS_ANNOTATIONS_YAML=$(cat <<EOTLS
    networking.gke.io/managed-certificates: "${GITLAB_MANAGED_CERT_NAME}"
    networking.gke.io/v1beta1.FrontendConfig: "${GITLAB_FRONTEND_CONFIG_NAME}"
EOTLS
)
  if [[ -n "$GITLAB_GLOBAL_STATIC_IP_NAME" ]]; then
    GITLAB_GCE_TLS_ANNOTATIONS_YAML+=$'\n'"    kubernetes.io/ingress.global-static-ip-name: \"${GITLAB_GLOBAL_STATIC_IP_NAME}\""
  fi
  # Build the ManagedCertificate domains list from GITLAB_PUBLIC_HOSTS.
  _gitlab_mc_domains_yaml=""
  for _gitlab_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
    _gitlab_mc_domains_yaml+=$'\n'"    - ${_gitlab_host}"
  done
  unset _gitlab_host

  log "Applying GKE ManagedCertificate (${GITLAB_MANAGED_CERT_NAME}) + FrontendConfig (${GITLAB_FRONTEND_CONFIG_NAME}) + BackendConfig (${GITLAB_BACKEND_CONFIG_NAME}) in ns=${NAMESPACE}..."
  # Requirement 1: apply ManagedCertificate and FrontendConfig and confirm they exist BEFORE the ingress is applied.
  kubectl apply -f - <<EOF
apiVersion: networking.gke.io/v1
kind: ManagedCertificate
metadata:
  name: ${GITLAB_MANAGED_CERT_NAME}
  namespace: ${NAMESPACE}
spec:
  domains:${_gitlab_mc_domains_yaml}
---
apiVersion: networking.gke.io/v1beta1
kind: FrontendConfig
metadata:
  name: ${GITLAB_FRONTEND_CONFIG_NAME}
  namespace: ${NAMESPACE}
spec:
  redirectToHttps:
    enabled: true
    responseCodeName: MOVED_PERMANENTLY_DEFAULT
---
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
  name: ${GITLAB_BACKEND_CONFIG_NAME}
  namespace: ${NAMESPACE}
spec:
  healthCheck:
    type: HTTP
    requestPath: /-/readiness
    port: 8181
    # Puma + Rails boot is slow; be generous before marking backends unhealthy.
    checkIntervalSec: 30
    timeoutSec: 10
    healthyThreshold: 1
    unhealthyThreshold: 3
  # Drain a little more generously — GitLab requests can be long-running.
  connectionDraining:
    drainingTimeoutSec: 60
EOF
  unset _gitlab_mc_domains_yaml

  # Confirm ManagedCertificate and FrontendConfig exist before proceeding to ingress apply.
  if ! kubectl -n "$NAMESPACE" get managedcertificate "$GITLAB_MANAGED_CERT_NAME" >/dev/null 2>&1; then
    die "ManagedCertificate ${NAMESPACE}/${GITLAB_MANAGED_CERT_NAME} was not found after apply — cannot safely apply GCE ingress."
  fi
  if ! kubectl -n "$NAMESPACE" get frontendconfig "$GITLAB_FRONTEND_CONFIG_NAME" >/dev/null 2>&1; then
    die "FrontendConfig ${NAMESPACE}/${GITLAB_FRONTEND_CONFIG_NAME} was not found after apply — cannot safely apply GCE ingress."
  fi
  log "Confirmed: ManagedCertificate/${GITLAB_MANAGED_CERT_NAME} and FrontendConfig/${GITLAB_FRONTEND_CONFIG_NAME} exist in ns=${NAMESPACE}."

  # Annotate the webservice Service so GCLB picks up the BackendConfig.
  # The GitLab Operator owns this Service, so we apply the annotation
  # out-of-band.  It is additive (not owned by the chart) and survives
  # operator reconciles.
  _webservice_svc="${GITLAB_RELEASE}-webservice-default"
  if kubectl -n "$NAMESPACE" get svc "$_webservice_svc" >/dev/null 2>&1; then
    log "Annotating Service ${_webservice_svc} with cloud.google.com/backend-config=${GITLAB_BACKEND_CONFIG_NAME}..."
    kubectl -n "$NAMESPACE" annotate svc "$_webservice_svc" \
      "cloud.google.com/backend-config={\"default\":\"${GITLAB_BACKEND_CONFIG_NAME}\"}" \
      --overwrite >/dev/null || warn "Failed to annotate ${_webservice_svc}; will retry after CR reconcile."
  else
    log "Service ${_webservice_svc} not present yet; BackendConfig annotation will be (re-)applied after the CR reconcile."
  fi
  unset _webservice_svc
fi

# Diagnostics: show current ingress, managedcertificate, and frontendconfig state before reconcile.
if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
  log "--- GCE ingress diagnostics (pre-apply) ---"
  kubectl get ingress -A --no-headers 2>/dev/null || true
  log "ManagedCertificates:"
  kubectl get managedcertificate -A --no-headers 2>/dev/null || true
  log "FrontendConfigs:"
  kubectl get frontendconfig -A --no-headers 2>/dev/null || true
  # Show current annotations for each expected ingress before patch.
  for _diag_ing in "${GITLAB_FALLBACK_INGRESS_NAME}" "${GITLAB_RELEASE}-webservice-default"; do
    if kubectl -n "$NAMESPACE" get ingress "$_diag_ing" >/dev/null 2>&1; then
      log "Current annotations for ${NAMESPACE}/${_diag_ing} (pre-apply):"
      kubectl -n "$NAMESPACE" get ingress "$_diag_ing" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true
      echo
    fi
  done
  unset _diag_ing
  log "---"
fi

log "Reconciling GitLab front-door owner='${GITLAB_FRONTDOOR_OWNER}' (ingressClass=${GITLAB_INGRESS_CLASS}) for hosts=${_gitlab_hosts_csv} -> ${WEBSERVICE_SVC}:8181 ..."
log "Rendered GitLab ingress (pre-apply): ns=${NAMESPACE} owner=${GITLAB_FRONTDOOR_OWNER} class=${GITLAB_INGRESS_CLASS} managedCert=${GITLAB_MANAGED_CERT_NAME} frontendConfig=${GITLAB_FRONTEND_CONFIG_NAME} preSharedCert=- hosts=${_gitlab_hosts_csv} backend=${WEBSERVICE_SVC}:8181"

if [[ "$GITLAB_FRONTDOOR_OWNER" == "operator" ]]; then
  if assert_unique_ingress_host_claims "${GITLAB_RELEASE}-webservice-default" "$NAMESPACE" "$_gitlab_hosts_csv" "$GITLAB_RELEASE" "$WEBSERVICE_SVC"; then
    :
  else
    _ingress_claim_rc=$?
    case "$_ingress_claim_rc" in
      10)
        log "GitLab operator ingress host ownership confirmed in namespace '${NAMESPACE}': ${INGRESS_HOST_CLAIM_DETAILS}"
        ;;
      *)
        die "Duplicate ingress host/path claim detected for GitLab operator ingress '${NAMESPACE}/${GITLAB_RELEASE}-webservice-default': ${INGRESS_HOST_CLAIM_DETAILS}"
        ;;
    esac
  fi

  if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
    _operator_ingress_name="${GITLAB_RELEASE}-webservice-default"
    if kubectl -n "$NAMESPACE" get ingress "$_operator_ingress_name" >/dev/null 2>&1; then
      log "Annotating ingress ${_operator_ingress_name} with ManagedCertificate=${GITLAB_MANAGED_CERT_NAME} and FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}..."
      kubectl -n "$NAMESPACE" annotate ingress "$_operator_ingress_name" \
        "networking.gke.io/managed-certificates=${GITLAB_MANAGED_CERT_NAME}" \
        "networking.gke.io/v1beta1.FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}" \
        ingress.gcp.kubernetes.io/pre-shared-cert- \
        --overwrite >/dev/null || warn "Failed to annotate ${_operator_ingress_name} with GCE ingress TLS annotations."
    else
      warn "Operator ingress ${_operator_ingress_name} not found yet; cannot apply GCE ingress annotations in this pass."
    fi
    unset _operator_ingress_name
  else
    _operator_ingress_name="${GITLAB_RELEASE}-webservice-default"
    if kubectl -n "$NAMESPACE" get ingress "$_operator_ingress_name" >/dev/null 2>&1; then
      log "Removing stale GCE TLS annotations from operator ingress ${_operator_ingress_name} (class=${GITLAB_INGRESS_CLASS})..."
      kubectl -n "$NAMESPACE" annotate ingress "$_operator_ingress_name" \
        networking.gke.io/managed-certificates- \
        networking.gke.io/v1beta1.FrontendConfig- \
        ingress.gcp.kubernetes.io/pre-shared-cert- \
        --overwrite >/dev/null 2>&1 || true
    fi
    unset _operator_ingress_name
  fi

  if kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then
    log "Deleting stale fallback ingress ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} to enforce single front-door ownership (${GITLAB_FRONTDOOR_OWNER})."
    kubectl -n "$NAMESPACE" delete ingress "$GITLAB_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || \
      warn "Failed to delete stale fallback ingress ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME}."
  fi
  if [[ "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" != "$GITLAB_FALLBACK_INGRESS_NAME" ]] && kubectl -n "$NAMESPACE" get ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then
    log "Deleting legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME} to avoid owner/controller ambiguity."
    kubectl -n "$NAMESPACE" delete ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || \
      warn "Failed to delete legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME}."
  fi
else
  if assert_unique_ingress_host_claims "$GITLAB_FALLBACK_INGRESS_NAME" "$NAMESPACE" "$_gitlab_hosts_csv" "$GITLAB_RELEASE" "$WEBSERVICE_SVC"; then
    :
  else
    _ingress_claim_rc=$?
    case "$_ingress_claim_rc" in
      10)
        log "GitLab-managed ingress already owns host/path in namespace '${NAMESPACE}': ${INGRESS_HOST_CLAIM_DETAILS}"
        _operator_ingress_name="${GITLAB_RELEASE}-webservice-default"
        if kubectl -n "$NAMESPACE" get ingress "$_operator_ingress_name" >/dev/null 2>&1; then
          log "Fallback front-door owner selected; deleting stale operator ingress ${NAMESPACE}/${_operator_ingress_name}."
          kubectl -n "$NAMESPACE" delete ingress "$_operator_ingress_name" --ignore-not-found >/dev/null || \
            warn "Failed to delete stale operator ingress ${NAMESPACE}/${_operator_ingress_name}."
        fi
        unset _operator_ingress_name
        ;;
      *)
        die "Duplicate ingress host/path claim detected for GitLab fallback ingress '${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME}': ${INGRESS_HOST_CLAIM_DETAILS}"
        ;;
    esac
  fi

  if [[ "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" != "$GITLAB_FALLBACK_INGRESS_NAME" ]] && kubectl -n "$NAMESPACE" get ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then
    log "Deleting legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME} before applying ${GITLAB_FALLBACK_INGRESS_NAME}."
    kubectl -n "$NAMESPACE" delete ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || \
      warn "Failed to delete legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME}."
  fi

  if kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then
    _live_spec_class="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.spec.ingressClassName}' 2>/dev/null || true)"
    _live_ann_class="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.kubernetes\.io/ingress\.class}' 2>/dev/null || true)"
    _live_class="${_live_spec_class:-$_live_ann_class}"
    _live_managed="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.networking\.gke\.io/managed-certificates}' 2>/dev/null || true)"
    _live_frontend="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.networking\.gke\.io/v1beta1\.FrontendConfig}' 2>/dev/null || true)"
    _live_pre_shared="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.ingress\.gcp\.kubernetes\.io/pre-shared-cert}' 2>/dev/null || true)"
    _replace_reason=""
    _patch_only=0
    if [[ -n "$_live_class" && "$_live_class" != "$GITLAB_INGRESS_CLASS" ]]; then
      # ingressClass change requires recreation (immutable field).
      _replace_reason="ingressClass drift (live=${_live_class}, desired=${GITLAB_INGRESS_CLASS})"
    elif [[ "$GITLAB_INGRESS_CLASS" == "gce" ]]; then
      # Requirement 2 & 3: never delete/recreate to clear stale cert annotations.
      # Patch annotations in place instead.
      if [[ -n "$_live_pre_shared" ]]; then
        log "Patching stale pre-shared-cert annotation from ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} in place (was: ${_live_pre_shared})."
        kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \
          ingress.gcp.kubernetes.io/pre-shared-cert- \
          "networking.gke.io/managed-certificates=${GITLAB_MANAGED_CERT_NAME}" \
          "networking.gke.io/v1beta1.FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}" \
          --overwrite >/dev/null 2>&1 || warn "Failed to patch pre-shared-cert annotation from ${GITLAB_FALLBACK_INGRESS_NAME}."
        _patch_only=1
      elif [[ -n "$_live_managed" && "$_live_managed" != "$GITLAB_MANAGED_CERT_NAME" ]]; then
        log "Patching managed certificate annotation on ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} in place (live=${_live_managed}, desired=${GITLAB_MANAGED_CERT_NAME})."
        kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \
          "networking.gke.io/managed-certificates=${GITLAB_MANAGED_CERT_NAME}" \
          --overwrite >/dev/null 2>&1 || warn "Failed to patch managed-certificates annotation on ${GITLAB_FALLBACK_INGRESS_NAME}."
        _patch_only=1
      elif [[ -n "$_live_frontend" && "$_live_frontend" != "$GITLAB_FRONTEND_CONFIG_NAME" ]]; then
        log "Patching frontend config annotation on ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} in place (live=${_live_frontend}, desired=${GITLAB_FRONTEND_CONFIG_NAME})."
        kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \
          "networking.gke.io/v1beta1.FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}" \
          --overwrite >/dev/null 2>&1 || warn "Failed to patch FrontendConfig annotation on ${GITLAB_FALLBACK_INGRESS_NAME}."
        _patch_only=1
      fi
    fi
    if [[ -n "$_replace_reason" && "$_patch_only" -eq 0 ]]; then
      log "Ingress shape change detected for ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME}: ${_replace_reason}. Replacing ingress (host/rule shape change)."
      kubectl -n "$NAMESPACE" delete ingress "$GITLAB_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || true
    fi
    unset _live_spec_class _live_ann_class _live_class _live_managed _live_frontend _live_pre_shared _replace_reason _patch_only
  fi

  kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ${GITLAB_FALLBACK_INGRESS_NAME}
  namespace: ${NAMESPACE}
  annotations:
    kubernetes.io/ingress.class: ${GITLAB_INGRESS_CLASS}
    konghq.com/strip-path: "false"
${GITLAB_GCE_TLS_ANNOTATIONS_YAML}
${GITLAB_AUTH_ANNOTATIONS_YAML}
spec:
  ingressClassName: ${GITLAB_INGRESS_CLASS}
  rules:${GITLAB_INGRESS_RULES_YAML}
EOF

  if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
    kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \
      ingress.gcp.kubernetes.io/pre-shared-cert- \
      --overwrite >/dev/null 2>&1 || true
  else
    kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \
      networking.gke.io/managed-certificates- \
      networking.gke.io/v1beta1.FrontendConfig- \
      ingress.gcp.kubernetes.io/pre-shared-cert- \
      --overwrite >/dev/null 2>&1 || true
  fi

  # Diagnostics: show annotations after apply.
  if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
    log "Annotations for ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} (post-apply):"
    kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true
    echo
    kubectl -n "$NAMESPACE" describe ingress "$GITLAB_FALLBACK_INGRESS_NAME" 2>/dev/null || true
  fi
fi

# ---------------------------------------------------------------------------
# Registry migration: registry:2 (knoe-system) → gitlab-registry
# Runs unattended after GitLab is Ready.  Delegates to init_registry.sh
# migrate which uses skopeo when available, otherwise prints commands.
# Skipped if the old registry:2 has no images or is already gone.
# Set SKIP_REGISTRY_MIGRATE=1 to suppress.
# ---------------------------------------------------------------------------
if [[ "${SKIP_REGISTRY_MIGRATE:-0}" != "1" ]]; then
  _init_registry_sh="${SCRIPT_DIR}/init_registry.sh"
  if [[ -x "$_init_registry_sh" ]]; then
    log "--- Registry migration: registry:2 → gitlab-registry ---"
    # Pass the gitlab namespace so the migrate action knows where to send images.
    GITLAB_NAMESPACE="$NAMESPACE" \
    REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}" \
      "$_init_registry_sh" migrate || \
      warn "Registry migration encountered errors — check output above."
  else
    warn "init_registry.sh not found at ${_init_registry_sh}; skipping registry migration."
    warn "Run manually:  ./etc/init_registry.sh migrate"
  fi
  unset _init_registry_sh
fi

log "Done."
log "GitLab will be reachable at hosts=${_gitlab_hosts_csv} once pods are Running."
log "Initial root password:  kubectl -n ${NAMESPACE} get secret ${GITLAB_RELEASE}-gitlab-initial-root-password -o jsonpath='{.data.password}' | base64 -d"