#!/usr/bin/env bash
set -euo pipefail

PROG="init_gitlab"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")"
# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden.
# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps).
NAMESPACE="${GITLAB_NAMESPACE:-gitlab}"
CFG_PATH=""
FORCE=0
# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE).
# Setting this would pin ALL global components to one node, exhausting RAM.
NODE_SELECTOR=""

usage() {
  cat <<EOF
Usage:
  $PROG [options] [deploy]

Options:
  --mode <k3d|k3s|k8s|local>   Deployment mode (default: ${MODE:-k3d})
  -n, --namespace <name>        Target namespace (default: gitlab)
  -c, --config <prole.cfg>      Path to prole.cfg (defaults to detected)
  --node-selector <node>        Node to pin GitLab workloads (default: gandalf.prole.org)
  --force                       Remove existing GitLab and Gitea releases before deploy
  --help                        Show this help

Behavior:
  - Removes any pre-existing git.prole.org Gitea configurations (helm release,
    namespace, Kong routes) when --force is specified or when gitea is detected.
  - Installs the GitLab Operator via Helm into the gitlab namespace.
  - Provisions a GitLab CR that uses the knoe-db CloudNativePG cluster as its
    external PostgreSQL data store.
  - Configures git.prole.org as a Kong-managed ingress pointing to GitLab.
  - Pins all GitLab workloads to the selected node (default: gandalf.prole.org).
EOF
}

die()  { echo "[ERROR] $*" >&2; exit 2; }
log()  { echo "[INFO]  $*"; }
warn() { echo "[WARN]  $*" >&2; }

while [[ $# -gt 0 ]]; do
  case "$1" in
    --mode)          MODE="$(prole_normalize_mode "${2:-}")"; shift 2 ;;
    --mode=*)        MODE="$(prole_normalize_mode "${1#*=}")"; shift 1 ;;
    -n|--namespace)  NAMESPACE="${2:-}"; shift 2 ;;
    --namespace=*)   NAMESPACE="${1#*=}"; shift 1 ;;
    -c|--config)     CFG_PATH="${2:-}"; shift 2 ;;
    --config=*)      CFG_PATH="${1#*=}"; shift 1 ;;
    --node-selector) NODE_SELECTOR="${2:-}"; shift 2 ;;
    --node-selector=*) NODE_SELECTOR="${1#*=}"; shift 1 ;;
    --force)         FORCE=1; shift 1 ;;
    -h|--help)       usage; exit 0 ;;
    --trace-namespace)
      # Resolve namespace then exit — used by diagnostic scripts only
      _TRACE_NS=1; shift 1 ;;
    *) break ;;
  esac
done

# ---------------------------------------------------------------------------
# Resolve config path
# ---------------------------------------------------------------------------
if [[ -z "$CFG_PATH" && -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then
  CFG_PATH="${PROLE_CONF}/prole.cfg"
elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then
  CFG_PATH="$SCRIPT_DIR/../conf/prole.cfg"
fi

# cfg-based namespace override (only GITLAB_NAMESPACE; GITOPS_NAMESPACE intentionally excluded)
if [[ -z "$NAMESPACE" || "$NAMESPACE" == "gitlab" ]] && [[ -n "$CFG_PATH" ]]; then
  maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")"
  [[ -n "$maybe_ns" ]] && NAMESPACE="$maybe_ns"
fi
# Final safety net — always default to 'gitlab'
NAMESPACE="${NAMESPACE:-gitlab}"
export GITLAB_NAMESPACE="$NAMESPACE"

# Diagnostic exit — used by tmp/sim_installer_ns.sh
if [[ "${_TRACE_NS:-0}" == "1" ]]; then
  echo "TRACE: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" >&2
  echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-<unset>}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-<unset>}'" >&2
  exit 0
fi

case "$MODE" in
  k3d|k3s|k8s|local) ;;
  *) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;;
esac
export PROLE_MODE="$MODE"

command -v kubectl >/dev/null || die "kubectl not found"
command -v helm    >/dev/null || die "helm not found (required for GitLab Operator install)"

# ---------------------------------------------------------------------------
# knoe-db (CNPG) settings — reuse the same cluster gitea uses
# ---------------------------------------------------------------------------
DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-${CLUSTER_NAME:-knoe-db}}"
GITLAB_DB_NAME="${GITLAB_DB_NAME:-gitlabhq_production}"
GITLAB_DB_USER="${GITLAB_DB_USER:-gitlab}"
GITLAB_DB_PASSWORD="${GITLAB_DB_PASSWORD:-}"
DB_HOST="${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local"
DB_PORT="${GITLAB_DB_PORT:-5432}"

# ---------------------------------------------------------------------------
# Redis — shared common service (deployed by init_redis.sh in knoe-system)
# ---------------------------------------------------------------------------
REDIS_NAMESPACE="${REDIS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}"
REDIS_HOST="${GITLAB_REDIS_HOST:-redis-master.${REDIS_NAMESPACE}.svc.cluster.local}"
REDIS_PORT="${GITLAB_REDIS_PORT:-6379}"

# GitLab domain (reuses the git.prole.org shared hostname)
GITLAB_DOMAIN="${GITLAB_DOMAIN:-git.prole.org}"

# Google Workspace OIDC via knoe-auth — set FRONTDOOR_HOST to enable
FRONTDOOR_HOST="${FRONTDOOR_HOST:-}"
GITLAB_RELEASE="gitlab"

# ---------------------------------------------------------------------------
# Garage S3 (knoe-system) — replaces embedded minio for object storage
# ---------------------------------------------------------------------------
GARAGE_NAMESPACE="${GARAGE_NAMESPACE:-knoe-system}"
GARAGE_SVC_HOST="garage.${GARAGE_NAMESPACE}.svc.cluster.local"
GARAGE_S3_KEY_NAME="gitlab-s3"

# ---------------------------------------------------------------------------
# Operator chart coordinates
# ---------------------------------------------------------------------------
OPERATOR_REPO_NAME="gitlab-operator"
OPERATOR_REPO_URL="https://gitlab.com/api/v4/projects/18899486/packages/helm/stable"
OPERATOR_CHART="gitlab-operator/gitlab-operator"
OPERATOR_RELEASE="gitlab-operator"
OPERATOR_NAMESPACE="${GITLAB_OPERATOR_NAMESPACE:-${NAMESPACE}}"
# GitLab Helm chart version required in the CR; auto-detected if not set.
GITLAB_CHART_VERSION="${GITLAB_CHART_VERSION:-}"
GITLAB_CHART_REPO_NAME="gitlab"
GITLAB_CHART_REPO_URL="https://charts.gitlab.io/"

# ---------------------------------------------------------------------------
# Helper: resolve knoe-db primary pod (same pattern as init_gitea.sh)
# ---------------------------------------------------------------------------
resolve_knoe_db_primary_pod() {
  local primary
  primary=$(kubectl -n "$DB_NAMESPACE" get pods \
    -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME},cnpg.io/instanceRole=primary" \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -z "$primary" ]]; then
    primary=$(kubectl -n "$DB_NAMESPACE" get pods \
      -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" \
      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  fi
  printf '%s' "$primary"
}

sql_escape_literal() {
  printf '%s' "${1:-}" | sed "s/'/''/g"
}

# ---------------------------------------------------------------------------
# Resolve GitLab DB password (pull from knoe-db-superuser secret as fallback)
# ---------------------------------------------------------------------------
resolve_gitlab_db_password() {
  if [[ -n "${GITLAB_DB_PASSWORD:-}" ]]; then
    return 0
  fi
  local resolved=""
  if kubectl -n "$DB_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then
    resolved=$(kubectl -n "$DB_NAMESPACE" get secret knoe-db-superuser \
      -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true)
  fi
  if [[ -n "$resolved" ]]; then
    GITLAB_DB_PASSWORD="$resolved"
  else
    # Generate a random password when none is available
    GITLAB_DB_PASSWORD="$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom 2>/dev/null | head -c 32 || true)"
    warn "Generated random GitLab DB password; store it in GITLAB_DB_PASSWORD for future runs."
  fi
}

# ---------------------------------------------------------------------------
# Provision GitLab role + database in knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab() {
  local primary
  primary="$(resolve_knoe_db_primary_pod)"
  if [[ -z "$primary" ]]; then
    warn "No knoe-db pod found in namespace '${DB_NAMESPACE}'; skipping GitLab DB setup."
    return 0
  fi

  resolve_gitlab_db_password

  local admin_user=""
  local candidate
  for candidate in postgres root; do
    if kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
      psql -U "$candidate" -d postgres -tAc "SELECT 1" >/dev/null 2>&1; then
      admin_user="$candidate"
      break
    fi
  done

  if [[ -z "$admin_user" ]]; then
    warn "Unable to connect to knoe-db as admin; skipping GitLab DB setup."
    return 0
  fi

  local esc_pw
  esc_pw="$(sql_escape_literal "$GITLAB_DB_PASSWORD")"

  kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
    psql -U "$admin_user" -d postgres -c "
      DO \$\$ BEGIN
        IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${GITLAB_DB_USER}') THEN
          CREATE ROLE ${GITLAB_DB_USER} LOGIN PASSWORD '${esc_pw}';
        ELSE
          ALTER ROLE ${GITLAB_DB_USER} WITH PASSWORD '${esc_pw}';
        END IF;
      END \$\$;
    " >/dev/null 2>&1 || warn "Could not create/update role '${GITLAB_DB_USER}'."

  local db_exists
  db_exists=$(kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
    psql -U "$admin_user" -d postgres -tAc \
    "SELECT 1 FROM pg_database WHERE datname='${GITLAB_DB_NAME}';" 2>/dev/null || true)

  if [[ "$db_exists" != "1" ]]; then
    kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
      psql -U "$admin_user" -d postgres -c \
      "CREATE DATABASE ${GITLAB_DB_NAME} OWNER ${GITLAB_DB_USER};" \
      >/dev/null 2>&1 || warn "Could not create database '${GITLAB_DB_NAME}'."
  fi

  log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE})."
}

# ---------------------------------------------------------------------------
# Clean up any pre-existing Gitea / git.prole.org configurations
# ---------------------------------------------------------------------------
cleanup_gitea() {
  log "Checking for pre-existing Gitea deployment to remove before GitLab install..."

  local gitea_ns="${GITEA_NAMESPACE:-gitea}"

  # Remove Gitea Helm release
  if helm -n "$gitea_ns" status gitea >/dev/null 2>&1; then
    log "Uninstalling Gitea Helm release from namespace '$gitea_ns'..."
    helm -n "$gitea_ns" uninstall gitea >/dev/null 2>&1 || true
  fi

  # Remove any leftover raw Gitea resources
  kubectl -n "$gitea_ns" delete deploy/gitea svc/gitea-http svc/gitea-ssh \
    >/dev/null 2>&1 || true

  # Remove Kong routes/services registered for git.prole.org (best-effort)
  local kong_ns="${KONG_NAMESPACE:-${NAMESPACE:-kong}}"
  for res_type in kongplugins kongingresses; do
    kubectl -n "$gitea_ns" delete "$res_type" --all >/dev/null 2>&1 || true
  done

  # Remove gitea namespace Ingress objects that route git.prole.org
  kubectl -n "$gitea_ns" delete ingress \
    -l "app.kubernetes.io/name=gitea" >/dev/null 2>&1 || true
  kubectl -n "$gitea_ns" delete ingress \
    --field-selector="metadata.name=gitea" >/dev/null 2>&1 || true

  log "Gitea cleanup complete."
}

# ---------------------------------------------------------------------------
# Ensure gitlab namespace exists
# ---------------------------------------------------------------------------
kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null

# ---------------------------------------------------------------------------
# --force: remove existing GitLab operator and release first
# ---------------------------------------------------------------------------
if [[ "$FORCE" -eq 1 ]]; then
  warn "--force: removing existing GitLab resources in namespace '$NAMESPACE'..."
  if kubectl get crd gitlabs.apps.gitlab.com >/dev/null 2>&1; then
    kubectl -n "$NAMESPACE" delete gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
    # Wait briefly for operator to clean up managed resources
    sleep 10
  fi
  helm -n "$NAMESPACE" uninstall "$GITLAB_RELEASE"        >/dev/null 2>&1 || true
  helm -n "$NAMESPACE" uninstall "$OPERATOR_RELEASE"      >/dev/null 2>&1 || true
  cleanup_gitea
fi

# ---------------------------------------------------------------------------
# Always remove gitea if its helm release exists (non-destructive path)
# gitea and gitlab both claim git.prole.org; they cannot coexist.
# ---------------------------------------------------------------------------
if helm -n "${GITEA_NAMESPACE:-gitea}" status gitea >/dev/null 2>&1; then
  warn "Gitea release detected — removing to free git.prole.org for GitLab..."
  cleanup_gitea
fi

# ---------------------------------------------------------------------------
# Prepare knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab

# Persist the DB password as a k8s Secret the operator CR can reference
resolve_gitlab_db_password
kubectl -n "$NAMESPACE" create secret generic gitlab-db-password \
  --from-literal=password="$GITLAB_DB_PASSWORD" \
  --dry-run=client -o yaml | kubectl apply -f - >/dev/null
log "gitlab-db-password secret applied in namespace '$NAMESPACE'."

# ---------------------------------------------------------------------------
# Install / upgrade the GitLab Operator
# ---------------------------------------------------------------------------
log "Adding/updating GitLab Operator Helm repo..."
helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true
helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..."

log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..."

# The operator needs cluster-scoped RBAC; it watches all namespaces by default.
helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \
  -n "$OPERATOR_NAMESPACE" \
  --create-namespace \
  --timeout 10m \
  --wait \
  --set watchNamespace="$NAMESPACE"

log "GitLab Operator ready."

# ---------------------------------------------------------------------------
# Resolve GitLab chart version (required by the Operator CR since >= v0.28)
# ---------------------------------------------------------------------------
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
  log "Auto-detecting latest GitLab chart version..."
  helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true
  helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..."
  GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \
    | awk 'NR==2{print $2}' || true)
  if [[ -z "$GITLAB_CHART_VERSION" ]]; then
    die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run."
  fi
  log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}"
fi
CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\""

# ---------------------------------------------------------------------------
# Determine node selector block for the GitLab CR
# ---------------------------------------------------------------------------
# Storage node for gitaly + minio only (local PVs require co-location)
# Other components spread across the cluster via default scheduler.
STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-gandalf.prole.org}}"
# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned
NODE_SELECTOR="${NODE_SELECTOR:-}"
NODE_SELECTOR_YAML=""
if [[ -n "$NODE_SELECTOR" ]]; then
  log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}"
  NODE_SELECTOR_YAML="kubernetes.io/hostname: ${NODE_SELECTOR}"
fi
STORAGE_NODE_SELECTOR_YAML="kubernetes.io/hostname: ${STORAGE_NODE}"

# ---------------------------------------------------------------------------
# Create the GitLab CR  (operator reconciles this into the full deployment)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Apply gitlab-google-oidc secret (OIDC provider config for OmniAuth)
# ---------------------------------------------------------------------------
if [[ -n "$FRONTDOOR_HOST" ]]; then
  PLATFORM_DOMAIN="${PLATFORM_DOMAIN:-}"
  oidc_secret_tmpl="${SCRIPT_DIR}/../deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
  if [[ -f "$oidc_secret_tmpl" ]]; then
    log "Applying gitlab-google-oidc secret (issuer=https://${FRONTDOOR_HOST}/auth) ..."
    FRONTDOOR_HOST="$FRONTDOOR_HOST" PLATFORM_DOMAIN="$PLATFORM_DOMAIN" \
      envsubst < "$oidc_secret_tmpl" | kubectl apply -n "$NAMESPACE" -f - >/dev/null || \
      warn "Could not apply gitlab-google-oidc secret; create it manually from deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
  else
    warn "gitlab-google-oidc-secret.example.yaml not found; skipping OIDC secret (create it manually)."
  fi
fi

# ---------------------------------------------------------------------------
# ARM64 / 16KB-page jemalloc fix — build glibc jemalloc on every node once
# All RPi nodes use 16KB kernel pages; GitLab's bundled jemalloc is 4KB-only.
# We compile a compatible jemalloc-5.3.0 with --with-lg-page=14 via DaemonSet.
# ---------------------------------------------------------------------------
setup_jemalloc_on_nodes() {
  local jlib="/opt/gitlab-jemalloc/libjemalloc.so.2"
  local ds_name="jemalloc-builder"

  # Check if already present on all schedulable nodes
  local nodes
  nodes=$(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | tr '\n' ' ')
  local all_ready=true
  for node in $nodes; do
    result=$(kubectl -n "$NAMESPACE" run "jcheck-${node//\./-}" \
      --image=ubuntu:22.04 --restart=Never --rm --attach --quiet \
      --overrides="{\"spec\":{\"nodeName\":\"${node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"jlib\",\"hostPath\":{\"path\":\"/opt/gitlab-jemalloc\",\"type\":\"DirectoryOrCreate\"}}],\"containers\":[{\"name\":\"c\",\"image\":\"ubuntu:22.04\",\"command\":[\"bash\",\"-c\",\"test -f /jlib/libjemalloc.so.2 && echo OK || echo MISSING\"],\"volumeMounts\":[{\"name\":\"jlib\",\"mountPath\":\"/jlib\"}]}]}}" 2>/dev/null || echo "MISSING")
    if [[ "$result" != *"OK"* ]]; then
      all_ready=false
      break
    fi
  done

  if $all_ready; then
    log "jemalloc-16k already present on all nodes — skipping build."
    return 0
  fi

  log "Deploying jemalloc-builder DaemonSet (compiles jemalloc-5.3.0 with --with-lg-page=14 on each node)..."
  local _tmpds
  _tmpds=$(mktemp /tmp/jemalloc-ds-XXXXXX.yaml)
  cat > "$_tmpds" <<'JEDS'
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: jemalloc-builder
  namespace: NAMESPACE_PLACEHOLDER
  labels:
    app: jemalloc-builder
spec:
  selector:
    matchLabels:
      app: jemalloc-builder
  template:
    metadata:
      labels:
        app: jemalloc-builder
    spec:
      tolerations:
      - operator: Exists
      initContainers:
      - name: build
        image: ubuntu:22.04
        command:
        - bash
        - -c
        - |
          set -e
          TARGET=/hostlib/libjemalloc.so.2
          if [ -f "$TARGET" ]; then echo "Already built"; exit 0; fi
          apt-get update -qq && apt-get install -y -q gcc make wget bzip2 2>&1
          cd /tmp
          wget -q -O jemalloc.tar.bz2 \
            https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
          tar xjf jemalloc.tar.bz2 && cd jemalloc-5.3.0
          ./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1
          make -j2 lib/libjemalloc.so.2 2>&1
          cp lib/libjemalloc.so.2 "$TARGET" && chmod 755 "$TARGET"
          echo "Done: $(ls -lh $TARGET)"
        volumeMounts:
        - name: hostlib
          mountPath: /hostlib
      containers:
      - name: keepalive
        image: ubuntu:22.04
        command: [bash, -c, "sleep infinity"]
      volumes:
      - name: hostlib
        hostPath:
          path: /opt/gitlab-jemalloc
          type: DirectoryOrCreate
JEDS
  sed -i.bak "s/NAMESPACE_PLACEHOLDER/${NAMESPACE}/g" "$_tmpds"
  kubectl apply -f "$_tmpds"
  rm -f "$_tmpds" "${_tmpds}.bak"

  log "Waiting up to 20 minutes for jemalloc to build on all nodes..."
  local deadline=$((SECONDS + 1200))
  while (( SECONDS < deadline )); do
    ready=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
      -o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0)
    desired=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
      -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo 1)
    [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]] && break
    log "  jemalloc build progress: ${ready}/${desired} nodes ready..."
    sleep 30
  done
  log "jemalloc-16k ready on all nodes."
}

# ---------------------------------------------------------------------------
# Garage S3 key + bucket provisioning for GitLab object storage
# Creates a dedicated "gitlab-s3" key in Garage, provisions required buckets,
# and writes the k8s secrets consumed by the GitLab CR.
# ---------------------------------------------------------------------------
setup_garage_for_gitlab() {
  log "Configuring Garage S3 (${GARAGE_NAMESPACE}) as GitLab object storage..."

  local garage_pod
  garage_pod=$(kubectl -n "$GARAGE_NAMESPACE" get pods -l app=garage \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -z "$garage_pod" ]]; then
    warn "Garage pod not found in namespace '${GARAGE_NAMESPACE}'; skipping garage setup."
    return 0
  fi

  # ---- Resolve or create the gitlab-s3 access key ----
  # garage v1.x outputs plaintext (not JSON) for key list / key info / key create.
  # key list format:   "  GKxxxxxxxxx  <name>"
  # key create/info format includes lines:
  #   "Key ID: GKxxxxxxxxx"
  #   "Secret key: <secret>"
  local access_key secret_key existing_id key_output
  existing_id=$(kubectl -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
    /garage key list 2>/dev/null \
    | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true)

  if [[ -n "$existing_id" ]]; then
    log "Garage key '${GARAGE_S3_KEY_NAME}' already exists (id=${existing_id}); fetching info..."
    key_output=$(kubectl -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
      /garage key info "$existing_id" 2>/dev/null || true)
  else
    log "Creating Garage key '${GARAGE_S3_KEY_NAME}'..."
    key_output=$(kubectl -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
      /garage key create "$GARAGE_S3_KEY_NAME" 2>/dev/null || true)
  fi

  if [[ -z "$key_output" ]]; then
    warn "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'; skipping garage setup."
    return 0
  fi

  # Parse plaintext output — match "Key ID: GKxxx" and "Secret key: xxx" lines.
  # Also try JSON in case a future garage version changes format.
  access_key=$(printf '%s' "$key_output" | awk '/^Key ID:/{print $NF; exit}' || true)
  secret_key=$(printf '%s' "$key_output" | awk '/^Secret key:/{print $NF; exit}' || true)
  # JSON fallback
  if [[ -z "$access_key" ]]; then
    access_key=$(printf '%s' "$key_output" | python3 -c \
      "import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true)
  fi
  if [[ -z "$secret_key" ]]; then
    secret_key=$(printf '%s' "$key_output" | python3 -c \
      "import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true)
  fi

  if [[ -z "$access_key" || -z "$secret_key" ]]; then
    warn "Could not extract credentials from Garage key output; skipping garage setup."
    warn "Raw output: ${key_output}"
    return 0
  fi

  # ---- Create buckets and grant permissions ----
  # --key takes the accessKeyId (not the name) in garage v1.x
  local buckets=(
    registry
    gitlab-artifacts-storage
    gitlab-lfs-storage
    gitlab-uploads-storage
    gitlab-packages-storage
    gitlab-dependency-proxy-storage
    gitlab-terraform-state
    gitlab-ci-secure-files
  )
  for bucket in "${buckets[@]}"; do
    kubectl -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
      /garage bucket create "$bucket" 2>/dev/null || true
    # bucket name is positional; --read/--write/--owner are boolean flags
    kubectl -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
      /garage bucket allow "$bucket" --key "$access_key" \
      --read --write --owner 2>/dev/null || true
  done
  log "Garage buckets provisioned for GitLab."

  # ---- Write k8s secrets consumed by the GitLab CR ----
  # Rails object storage connection (artifacts, LFS, uploads, packages, etc.)
  kubectl -n "$NAMESPACE" create secret generic gitlab-object-storage \
    --from-literal=connection="provider: AWS
region: garage
aws_access_key_id: ${access_key}
aws_secret_access_key: ${secret_key}
endpoint: 'http://${GARAGE_SVC_HOST}:3900'
path_style: true" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null
  log "gitlab-object-storage secret applied."

  # Docker registry S3 storage driver config.
  # Use s3_v2 (AWS SDK v2) — s3_v1 has signature computation issues with
  # non-standard ports (Host: endpoint:3900) against Garage.  SDK v2 always
  # uses SigV4; v4auth is not a recognised key and must be omitted.
  kubectl -n "$NAMESPACE" create secret generic gitlab-registry-storage \
    --from-literal=config="s3_v2:
  accesskey: ${access_key}
  secretkey: ${secret_key}
  bucket: registry
  regionendpoint: http://${GARAGE_SVC_HOST}:3900
  region: garage
  pathstyle: true" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null
  log "gitlab-registry-storage secret applied."
}

# ---------------------------------------------------------------------------
# Static PV + synology directory setup for gitaly on gandalf
# (no dynamic provisioner is available in the service cluster)
# minio PV removed — object storage is provided by knoe-system/garage
# ---------------------------------------------------------------------------
setup_gitlab_storage() {
  local synology_node="${GITLAB_STORAGE_NODE:-gandalf.prole.org}"
  local base_path="${GITLAB_STORAGE_BASE:-/synology/d005/gitlab}"
  local gitaly_size="${GITLAB_GITALY_PV_SIZE:-50Gi}"

  log "Creating synology gitaly directory on ${synology_node}..."
  kubectl -n "$NAMESPACE" run gitlab-dirprep \
    --image=alpine:latest --restart=Never --rm --attach \
    --overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"s\",\"hostPath\":{\"path\":\"$(dirname ${base_path})\",\"type\":\"Directory\"}}],\"containers\":[{\"name\":\"dirprep\",\"image\":\"alpine:latest\",\"command\":[\"sh\",\"-c\",\"mkdir -p ${base_path}/gitaly && chmod 777 ${base_path}/gitaly && echo done\"],\"volumeMounts\":[{\"name\":\"s\",\"mountPath\":\"$(dirname ${base_path})\"}]}]}}" \
    2>&1 || warn "Could not create synology dirs (may already exist)"

  # A named no-provisioner StorageClass is required so the helm chart renders
  # storageClassName in the PVC template (an empty string is treated as falsy
  # by the chart and omitted, causing the PVC to use the cluster default).
  log "Ensuring gitlab-gitaly-static StorageClass (no-provisioner)..."
  kubectl apply -f - <<SCYAML
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gitlab-gitaly-static
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: Immediate
reclaimPolicy: Retain
SCYAML

  log "Creating static PV for gitaly (${gitaly_size})..."
  kubectl apply -f - <<PVYAML
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-gitaly-synology
spec:
  capacity:
    storage: ${gitaly_size}
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: gitlab-gitaly-static
  local:
    path: ${base_path}/gitaly
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values: [${synology_node}]
PVYAML
  log "GitLab gitaly storage PV ready."

  # Object storage (registry, artifacts, LFS, uploads, etc.) is provided by
  # knoe-system/garage — credentials and buckets are set up here.
  setup_garage_for_gitlab
}

setup_jemalloc_on_nodes
setup_gitlab_storage

log "Pre-creating gitlab-app-nonroot ServiceAccount (required by chart v9+)..."
kubectl -n "${NAMESPACE}" apply -f - <<SAYAML
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitlab-app-nonroot
  namespace: ${NAMESPACE}
SAYAML

log "Applying GitLab CR (domain=${GITLAB_DOMAIN}, db=${GITLAB_DB_NAME}@${DB_HOST})..."

kubectl apply -f - <<EOF
apiVersion: apps.gitlab.com/v1beta1
kind: GitLab
metadata:
  name: ${GITLAB_RELEASE}
  namespace: ${NAMESPACE}
spec:
  chart:
    ${CHART_VERSION_YAML}
    values:
      global:
        # ARM64 nodes: inject glibc jemalloc compiled with --with-lg-page=14
        # The DaemonSet above places it at /opt/gitlab-jemalloc/libjemalloc.so.2
        extraEnv:
          LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<GNODE
        nodeSelector:
          ${NODE_SELECTOR_YAML}
GNODE
fi)
        # Explicit allowedHosts as flat strings — chart 9.x renders list-of-maps
        # by default which breaks URI::InvalidComponentError in 7_gitlab_http.rb
        allowedHosts:
          - ${GITLAB_DOMAIN}
          - ${GITLAB_DOMAIN#*.}
        hosts:
          domain: prole.org
          gitlab:
            name: ${GITLAB_DOMAIN}
          ssh:
            name: ${GITLAB_DOMAIN}
        ingress:
          class: kong
          configureCertmanager: false
          tls:
            enabled: false
        psql:
          host: ${DB_HOST}
          port: ${DB_PORT}
          username: ${GITLAB_DB_USER}
          database: ${GITLAB_DB_NAME}
          password:
            secret: gitlab-db-password
            key: password
        redis:
          host: ${REDIS_HOST}
          port: ${REDIS_PORT}
          auth:
            enabled: false
        # chart v9+ requires the SA to be pre-created when a custom name is used
        serviceAccount:
          create: false
        # Disable bundled minio globally — object storage is provided by garage
        minio:
          enabled: false
        # Object storage via knoe-system/garage (S3-compatible)
        appConfig:
          object_store:
            enabled: true
            proxy_download: true
            connection:
              secret: gitlab-object-storage
              key: connection
        registry:
          bucket: registry
$(if [[ -n "$FRONTDOOR_HOST" ]]; then
cat <<OMNIAUTH
        appConfig:
          omniauth:
            enabled: true
            autoSignInWithProvider: openid_connect
            allowSingleSignOn:
              - openid_connect
            blockAutoCreatedUsers: false
            providers:
              - secret: gitlab-google-oidc
                key: provider
OMNIAUTH
fi)
      # Disable bundled sub-charts — we provide our own data stores
      postgresql:
        install: false
      redis:
        install: false
      certmanager:
        install: false
      prometheus:
        install: false
      gitlab-runner:
        install: false
      # minio sub-chart is disabled via global.minio.enabled: false (set above under global:).
      # Do NOT set minio.enabled here — chart v9 removed the sub-chart level property
      # and treats it as a hard validation error (NOTES.txt:234).
      gitlab:
        webservice:
          # Single replica + reduced memory — RPi nodes have limited RAM
          replicaCount: 1
          minReplicas: 1
          maxReplicas: 1
          resources:
            requests:
              cpu: 200m
              memory: 1500M
            limits:
              memory: 2000M
          # Rails preloading takes 25-40min on RPi; protect against liveness kills
          livenessProbe:
            initialDelaySeconds: 3600
            periodSeconds: 60
            timeoutSeconds: 30
            failureThreshold: 5
          readinessProbe:
            initialDelaySeconds: 120
            periodSeconds: 30
            timeoutSeconds: 30
            failureThreshold: 60
          ingress:
            enabled: true
            annotations:
              kubernetes.io/ingress.class: kong
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
          extraVolumes: |
            - name: jemalloc-16k
              hostPath:
                path: /opt/gitlab-jemalloc
                type: Directory
          extraVolumeMounts: |
            - name: jemalloc-16k
              mountPath: /opt/gitlab-jemalloc
              readOnly: true
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
          nodeSelector:
            ${NODE_SELECTOR_YAML}
NODE
fi)
        sidekiq:
          resources:
            requests:
              cpu: 100m
              memory: 800M
            limits:
              memory: 1200M
          # Rails loading takes 25-40min on RPi; protect against liveness kills
          livenessProbe:
            initialDelaySeconds: 3600
            periodSeconds: 60
            timeoutSeconds: 30
            failureThreshold: 5
          readinessProbe:
            initialDelaySeconds: 120
            periodSeconds: 30
            timeoutSeconds: 30
            failureThreshold: 60
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
          extraVolumes: |
            - name: jemalloc-16k
              hostPath:
                path: /opt/gitlab-jemalloc
                type: Directory
          extraVolumeMounts: |
            - name: jemalloc-16k
              mountPath: /opt/gitlab-jemalloc
              readOnly: true
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
          nodeSelector:
            ${NODE_SELECTOR_YAML}
NODE
fi)
        gitaly:
          # Named no-provisioner StorageClass — empty string is falsy in helm templates
          # and gets omitted, causing the PVC to use the cluster default (local-path).
          # gitlab-gitaly-static is created by setup_gitlab_storage() above.
          persistence:
            storageClass: gitlab-gitaly-static
            size: ${GITLAB_GITALY_PV_SIZE:-50Gi}
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
          extraVolumes: |
            - name: jemalloc-16k
              hostPath:
                path: /opt/gitlab-jemalloc
                type: Directory
          extraVolumeMounts: |
            - name: jemalloc-16k
              mountPath: /opt/gitlab-jemalloc
              readOnly: true
          # Give gitaly gRPC health check time to initialise before probes start
          livenessProbe:
            initialDelaySeconds: 30
          readinessProbe:
            initialDelaySeconds: 15
          # Always pin gitaly to the storage node (local PV)
          nodeSelector:
            ${STORAGE_NODE_SELECTOR_YAML}
        toolbox:
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
          extraVolumes: |
            - name: jemalloc-16k
              hostPath:
                path: /opt/gitlab-jemalloc
                type: Directory
          extraVolumeMounts: |
            - name: jemalloc-16k
              mountPath: /opt/gitlab-jemalloc
              readOnly: true
        migrations:
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
          extraVolumes: |
            - name: jemalloc-16k
              hostPath:
                path: /opt/gitlab-jemalloc
                type: Directory
          extraVolumeMounts: |
            - name: jemalloc-16k
              mountPath: /opt/gitlab-jemalloc
              readOnly: true
        gitlab-exporter:
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
          extraVolumes: |
            - name: jemalloc-16k
              hostPath:
                path: /opt/gitlab-jemalloc
                type: Directory
          extraVolumeMounts: |
            - name: jemalloc-16k
              mountPath: /opt/gitlab-jemalloc
              readOnly: true
      registry:
        storage:
          secret: gitlab-registry-storage
          key: config
        ingress:
          enabled: false
EOF

# minio is disabled (global.minio.enabled: false) — no ARM64 credential fix needed.

# ---------------------------------------------------------------------------
# ARM64 configure check: GitLab v17+ (chart v8+) ships multi-arch CNG images
# that include ARM64 variants. The old workaround of replacing the configure
# init container with alpine:latest is no longer needed and breaks things
# because /templates/configure is baked into the gitlab-base image, not
# mounted from a ConfigMap.
# This function is kept as a no-op to avoid breaking callers; remove entirely
# once the chart version floor is confirmed stable at v9+.
# ---------------------------------------------------------------------------
fix_registry_arm64_configure() {
  log "Skipping ARM64 configure patch — GitLab v18 CNG images are multi-arch natively."
}

fix_registry_arm64_configure

# ---------------------------------------------------------------------------
# Image migration: registry:2 (knoe-system, port 5000) → gitlab-registry
# Enumerates repositories from the old registry via its HTTP API and emits
# skopeo copy commands.  Executes them automatically if skopeo is on PATH.
# Run once after gitlab-registry is Running and the old registry:2 is still up.
# ---------------------------------------------------------------------------
migrate_registry_images_to_gitlab() {
  local src_registry="${REGISTRY_MIGRATE_SRC:-registry.${REGISTRY_NAMESPACE:-knoe-system}.svc.cluster.local:5000}"
  local dst_registry="${REGISTRY_MIGRATE_DST:-localhost:5000}"  # port-forward to gitlab-registry

  log "Checking for images to migrate from registry:2 (${src_registry}) to gitlab-registry..."

  # Fetch repository catalog via the v2 API using a temporary pod on the cluster
  local repos
  repos=$(kubectl -n "${REGISTRY_NAMESPACE:-knoe-system}" run registry-catalog \
    --image=alpine/curl:latest --restart=Never --rm --attach --quiet \
    --overrides="{\"spec\":{\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"c\",\"image\":\"alpine/curl:latest\",\"command\":[\"sh\",\"-c\",\"curl -s http://${src_registry}/v2/_catalog\"]}]}}" \
    2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); [print(r) for r in d.get('repositories',[])]" 2>/dev/null || true)

  if [[ -z "$repos" ]]; then
    log "No repositories found in registry:2 (or registry unreachable) — skipping migration."
    return 0
  fi

  log "Found repositories in registry:2:"
  printf '%s\n' "$repos" | sed 's/^/  /'

  log "Migration commands (copy each image to gitlab-registry):"
  log "  NOTE: run these from a host with 'skopeo' and access to both registries,"
  log "  or after port-forwarding: kubectl -n gitlab port-forward svc/gitlab-registry 5001:5000"
  while IFS= read -r repo; do
    [[ -z "$repo" ]] && continue
    echo "  skopeo copy --all docker://${src_registry}/${repo} docker://${dst_registry}/${repo} --dest-tls-verify=false --src-tls-verify=false"
  done <<< "$repos"

  if command -v skopeo >/dev/null 2>&1; then
    log "skopeo detected — attempting automated migration..."
    while IFS= read -r repo; do
      [[ -z "$repo" ]] && continue
      log "  Copying ${repo}..."
      skopeo copy --all \
        "docker://${src_registry}/${repo}" \
        "docker://${dst_registry}/${repo}" \
        --dest-tls-verify=false --src-tls-verify=false 2>&1 || \
        warn "  Could not copy ${repo} (may require manual migration)."
    done <<< "$repos"
    log "Image migration complete."
  else
    log "skopeo not found — run the commands above manually to complete migration."
  fi
}

migrate_registry_images_to_gitlab

log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)."
log "Monitor progress:  kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w"
log "Watch pods:        kubectl -n ${NAMESPACE} get pods -w"

# ---------------------------------------------------------------------------
# Optionally wait for GitLab to become available
# ---------------------------------------------------------------------------
WAIT_TIMEOUT="${GITLAB_WAIT_TIMEOUT:-1200}"   # default 20 minutes
if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
  log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status..."
  if ! kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \
       --for=condition=Available \
       --timeout="${WAIT_TIMEOUT}s" 2>/dev/null; then
    warn "Timed out waiting for GitLab CR condition=Available."
    warn "The deployment may still be in progress. Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}"
  fi
fi

# ---------------------------------------------------------------------------
# Kong ingress for git.prole.org -> gitlab-webservice
# The GitLab Operator creates the Ingress; this block patches it to Kong class
# or creates a fallback Kong-native ingress if the CR-managed one is absent.
# ---------------------------------------------------------------------------
WEBSERVICE_SVC="${GITLAB_RELEASE}-webservice-default"

log "Ensuring Kong ingress for ${GITLAB_DOMAIN} -> ${WEBSERVICE_SVC}:8181 ..."

kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: gitlab-kong-ingress
  namespace: ${NAMESPACE}
  annotations:
    kubernetes.io/ingress.class: kong
    konghq.com/strip-path: "false"
spec:
  ingressClassName: kong
  rules:
    - host: ${GITLAB_DOMAIN}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: ${WEBSERVICE_SVC}
                port:
                  number: 8181
EOF

log "Done."
log "GitLab will be reachable at http://${GITLAB_DOMAIN}/ once pods are Running."
log "Initial root password:  kubectl -n ${NAMESPACE} get secret ${GITLAB_RELEASE}-gitlab-initial-root-password -o jsonpath='{.data.password}' | base64 -d"