#!/usr/bin/env bash
set -u

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"

# Inject default config if not provided
_has_config=0
for _arg in "$@"; do
  [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
done
if [[ $_has_config -eq 0 ]]; then
  _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)"
  if [[ -n "$_default_cfg" ]]; then
    set -- "-c" "$_default_cfg" "$@"
  fi
fi
unset _has_config _arg _default_cfg

common_core_preparse_config "$@"

# shellcheck disable=SC1090
source "$SCRIPT_DIR/knoe_cfg.sh"

usage() {
  cat <<EOF
Usage: init_common_services.sh [-n|--namespace NS] [-k|--kerberos] <update|start|status|verify>

Deploys common infrastructure services (Registry, Garage, OpenBao, Kong, Cert-Manager;
OpenTofu on non-k8s modes)
into the given Kubernetes namespace. Use -k to include the Kerberos/KDC service.
EOF
}

NS=""
ACTION="update"
ENABLE_KERBEROS=0

while [ $# -gt 0 ]; do
  case "$1" in
    -c|--config)
      shift
      shift
      ;;
    -m|--mode)
      shift
      knoe_set_mode "${1:-}"
      shift
      ;;
    -m=*|--mode=*)
      knoe_set_mode "${1#*=}"
      shift
      ;;
    -n|--namespace)
      shift
      NS="${1:-}"
      shift
      ;;
    -n=*|--namespace=*)
      NS="${1#*=}"
      shift
      ;;
    -k|--kerberos)
      ENABLE_KERBEROS=1
      shift
      ;;
    update|start|status|verify|repair|reload|restart)
      ACTION="$1"
      [[ "$ACTION" == "repair" ]] && ACTION="update"
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      usage
      exit 2
      ;;
  esac
done

echo "DEBUG: Entering init_common_services.sh main" >&2
if [ -z "$NS" ]; then
  NS="${SERVICE_NAMESPACE:-${NAMESPACE:-}}"
fi
if [ -z "$NS" ]; then
  echo "ERROR: No namespace specified. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2
  exit 1
fi

# Registry should live in the common-core/service namespace unless explicitly overridden.
REGISTRY_NS="${REGISTRY_NAMESPACE:-${NS}}"

knoe_ensure_kubeconfig >/dev/null 2>&1 || true
echo "DEBUG: knoe_ensure_kubeconfig finished" >&2
ensure_kube_context || exit 1
echo "DEBUG: ensure_kube_context finished" >&2

# Check cluster reachability early to fail fast
if [[ "$ACTION" != "status" ]]; then
  echo "DEBUG: KUBECONFIG='${KUBECONFIG:-<not-set>}'" >&2
  if [[ -n "${KUBECONFIG:-}" ]]; then
    ls -l "$KUBECONFIG" 2>&1 || true
    cat "$KUBECONFIG" 2>/dev/null | grep -i server || true
  fi
  if ! kubectl cluster-info >/dev/null 2>&1; then
    echo "ERROR: Cluster not reachable. Check KUBECONFIG and cluster status." >&2
    exit 1
  fi
fi

_verify_common_services() {
  local status_script="$SCRIPT_DIR/status_common_services.sh"
  if [[ ! -x "$status_script" ]]; then
    echo "ERROR: status_common_services.sh not found at $status_script" >&2
    return 2
  fi

  local args=(-n "$NS")
  if [[ "$ENABLE_KERBEROS" == "1" ]]; then
    args+=(-k)
  fi

  echo "Verifying common services in namespace '$NS' ..."
  "$status_script" "${args[@]}"
}

# Short-circuit verify before any deploy/reload/preload work
if [[ -n "${COMMON_SERVICES_INIT_LOG:-}" ]]; then
  mkdir -p "$(dirname "$COMMON_SERVICES_INIT_LOG")"
  exec > >(tee -a "$COMMON_SERVICES_INIT_LOG") 2>&1
fi

if [[ "$ACTION" == "verify" ]]; then
  _verify_common_services
  exit $?
fi

# ---------------------------------------------------------------------------
# Pre-import container images from $PROLE_DATA/docker-import/*.tar
# ---------------------------------------------------------------------------
# final_deployment.sh exports images as
#   $PROLE_DATA/docker-import/<safe_name>.tar
# where safe_name = image ref with / and : replaced by _.
# Loading these tars into the local container runtime before the sub-scripts
# deploy pods avoids slow image pulls from remote registries.
#
# NOTE: This is opt-in. Set `DOCKER_PRELOAD=1` (or `PROLE_DOCKER_PRELOAD=1`) to
# enable pre-loading from `DOCKER_IMPORT_DIR` / `$PROLE_DATA/docker-import`.
# ---------------------------------------------------------------------------

_resolve_knoe_data() {
  local d="${PROLE_DATA:-}"
  if [[ -z "$d" ]]; then
    if [[ -f "$SCRIPT_DIR/../env.sh" ]]; then
      # shellcheck disable=SC1090
      source "$SCRIPT_DIR/../env.sh"
      d="${PROLE_DATA:-}"
    fi
  fi
  echo "${d:-}"
}

# Standard console execution element.
# Runs a command silently (output captured), displays a live elapsed-time footer
# on stderr when attached to a TTY, prints elapsed time on completion, and tails
# captured output for diagnostics on failure.
_exec_with_progress_footer() {
  local label="$1"
  shift

  local start_time
  start_time=$(date +%s)

  local tmp_out
  tmp_out=$(mktemp 2>/dev/null || echo "/tmp/_knoe_exec_$$.out")

  # Spawn live footer only when stderr is a real TTY
  local spinner_pid=""
  if [[ -t 2 ]]; then
    (
      while true; do
        now=$(date +%s)
        elapsed=$(( now - start_time ))
        h=$(( elapsed / 3600 ))
        m=$(( (elapsed % 3600) / 60 ))
        s=$(( elapsed % 60 ))
        printf "\r\033[2K  [%02d:%02d:%02d elapsed] %s" "$h" "$m" "$s" "$label" >&2
        sleep 1
      done
    ) &
    spinner_pid=$!
    disown "$spinner_pid" 2>/dev/null || true
  fi

  "$@" >"$tmp_out" 2>&1
  local rc=$?

  if [[ -n "$spinner_pid" ]]; then
    kill "$spinner_pid" 2>/dev/null || true
    wait "$spinner_pid" 2>/dev/null || true
    printf "\r\033[2K" >&2
  fi

  local end_time elapsed_total h m s
  end_time=$(date +%s)
  elapsed_total=$(( end_time - start_time ))
  h=$(( elapsed_total / 3600 ))
  m=$(( (elapsed_total % 3600) / 60 ))
  s=$(( elapsed_total % 60 ))
  printf "    [%02dh:%02dm:%02ds elapsed] %s\n" "$h" "$m" "$s" "$label"

  if [[ $rc -ne 0 && -s "$tmp_out" ]]; then
    echo "    [Last output]:" >&2
    tail -n 10 "$tmp_out" >&2
  fi
  rm -f "$tmp_out"
  return $rc
}

# Extract the primary image ref from a Docker tarball's manifest.json.
_image_name_from_tar() {
  local tar_file="$1"
  if command -v python3 >/dev/null 2>&1; then
    python3 - "$tar_file" <<'PYEOF' 2>/dev/null
import json, sys, tarfile
try:
    with tarfile.open(sys.argv[1]) as t:
        tags = json.load(t.extractfile("manifest.json"))[0].get("RepoTags") or []
        if tags:
            print(tags[0])
except Exception:
    pass
PYEOF
  fi
}

_preload_docker_images() {
  local knoe_data
  knoe_data=$(_resolve_knoe_data)
  local import_dir="${DOCKER_IMPORT_DIR:-${knoe_data:+${knoe_data}/docker-import}}"

  if [[ -z "$import_dir" || ! -d "$import_dir" ]]; then
    echo "[SKIP] No docker-import directory found; images will be pulled from registries."
    return 0
  fi

  local tar_files
  tar_files=$(find "$import_dir" -maxdepth 1 -name '*.tar' -type f 2>/dev/null || true)
  if [[ -z "$tar_files" ]]; then
    echo "[SKIP] docker-import directory exists but contains no .tar files."
    return 0
  fi

  local mode="${KNOE_MODE:-k3d}"
  echo "Pre-loading container images from $import_dir (mode=$mode) ..."

  case "$mode" in
    k3d)
      if ! command -v k3d >/dev/null 2>&1; then
        echo "WARN: k3d not available; skipping docker-import pre-load." >&2
        return 0
      fi

      # Prefer explicit K3D_CLUSTER_NAME, otherwise derive from the active kubectl context.
      local cluster_name="${K3D_CLUSTER_NAME:-}"
      if [[ -z "${cluster_name:-}" ]]; then
        local _ctx
        _ctx=$(kubectl config current-context 2>/dev/null || true)
        if [[ "${_ctx:-}" == k3d-* ]]; then
          cluster_name="${_ctx#k3d-}"
        fi
        unset _ctx
      fi
      cluster_name="${cluster_name:-knoe-dev-cluster}"

      # ── Step 1: build indexed arrays of tar paths + resolved image names ──
      local -a tar_paths=()
      local -a tar_images=()
      local _tf
      while IFS= read -r _tf; do
        [[ -z "$_tf" ]] && continue
        tar_paths+=("$_tf")
        tar_images+=("$(_image_name_from_tar "$_tf")")
      done <<< "$tar_files"
      local total_tars=${#tar_paths[@]}

      if [[ $total_tars -eq 0 ]]; then
        echo "  [SKIP] No tar files to process."
        return 0
      fi

      # ── Step 2: query images already in the k3d cluster ───────────────────
      echo "  Checking k3d cluster '$cluster_name' for existing images ..."
      local existing_images=""
      local node_name="k3d-${cluster_name}-server-0"
      if docker inspect "$node_name" >/dev/null 2>&1; then
        existing_images=$(docker exec "$node_name" \
          ctr --address /run/k3s/containerd/containerd.sock \
              --namespace k8s.io images ls -q 2>/dev/null || true)
      else
        echo "    [WARN] k3d node '$node_name' not reachable; will import all images."
      fi

      # ── Step 3: delta — split into already-present vs needs-import ─────────
      local -a to_import_tars=()
      local -a to_import_images=()
      local -a already_present=()
      local _i
      for (( _i=0; _i<total_tars; _i++ )); do
        local _tar="${tar_paths[$_i]}"
        local _img="${tar_images[$_i]}"
        local _found=0
        if [[ -n "$_img" && -n "$existing_images" ]]; then
          # Strip digest suffix before matching
          local _img_ref="${_img%%@*}"
          if echo "$existing_images" | grep -qF "$_img_ref"; then
            _found=1
          fi
        fi
        if [[ $_found -eq 1 ]]; then
          already_present+=("$(basename "$_tar")${_img:+ → $_img}")
        else
          to_import_tars+=("$_tar")
          to_import_images+=("${_img:-}")
        fi
      done

      # ── Step 4: report delta to user ───────────────────────────────────────
      if [[ ${#already_present[@]} -gt 0 ]]; then
        echo "  Already in cluster — skipping ${#already_present[@]} image(s):"
        local _entry
        for _entry in "${already_present[@]}"; do
          echo "    ✓ $_entry"
        done
      fi

      local import_count=${#to_import_tars[@]}
      if [[ $import_count -eq 0 ]]; then
        echo "  [OK] All images already present in k3d cluster '$cluster_name'; nothing to import."
        return 0
      fi

      echo "  Bulk uploading $import_count image(s) into k3d cluster '$cluster_name':"
      for (( _i=0; _i<import_count; _i++ )); do
        local _lbl="${to_import_images[$_i]:-$(basename "${to_import_tars[$_i]}")}"
        echo "    → $(basename "${to_import_tars[$_i]}")${_lbl:+ ($_lbl)}"
      done

      # ── Step 5: bulk import with live progress footer ──────────────────────
      local overall_start elapsed_bulk
      overall_start=$(date +%s)

      if _exec_with_progress_footer \
            "k3d image import — bulk ($import_count images) → $cluster_name" \
            k3d image import "${to_import_tars[@]}" -c "$cluster_name"; then
        elapsed_bulk=$(( $(date +%s) - overall_start ))
        local bm=$(( elapsed_bulk / 60 )) bs=$(( elapsed_bulk % 60 ))
        echo "  [OK] Bulk import of $import_count image(s) complete (${bm}m${bs}s)."
      else
        echo "  [WARN] Bulk import failed; falling back to sequential import." >&2
        local _idx
        for (( _i=0; _i<import_count; _i++ )); do
          _idx=$(( _i + 1 ))
          local _tar="${to_import_tars[$_i]}"
          local _img="${to_import_images[$_i]:-$(basename "$_tar")}"

          # ETA based on average time per completed image
          local _now _elapsed_so_far _eta_str=""
          _now=$(date +%s)
          _elapsed_so_far=$(( _now - overall_start ))
          if [[ $_i -gt 0 ]]; then
            local _avg=$(( _elapsed_so_far / _i ))
            local _remaining=$(( _avg * (import_count - _i) ))
            local _rm=$(( _remaining / 60 )) _rs=$(( _remaining % 60 ))
            _eta_str=" | ETA ~${_rm}m${_rs}s"
          fi

          echo "  [$_idx/$import_count] $(basename "$_tar")${_img:+ ($_img)}${_eta_str}"
          local _item_start
          _item_start=$(date +%s)
          if _exec_with_progress_footer \
                "k3d image import [$_idx/$import_count] $(basename "$_tar")" \
                k3d image import "$_tar" -c "$cluster_name"; then
            local _item_elapsed=$(( $(date +%s) - _item_start ))
            echo "    [OK] $(basename "$_tar") (${_item_elapsed}s)"
          else
            echo "    [WARN] Failed to import $(basename "$_tar"); will pull at deploy time." >&2
          fi
        done
        local _total_elapsed=$(( $(date +%s) - overall_start ))
        local _te_m=$(( _total_elapsed / 60 )) _te_s=$(( _total_elapsed % 60 ))
        echo "  [DONE] Sequential import complete. Total: ${_te_m}m${_te_s}s"
      fi
      ;;
    k3s)
      # For k3s: publish images to the in-cluster registry so nodes can pull them.
      # Avoid heavy repeated I/O by skipping pushes when the tag already exists.

      local registry="${LOCAL_REGISTRY:-}"
      if [[ -z "$registry" ]]; then
        local host
        host="$(_knoe_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")"
        if [[ -n "${host:-}" ]]; then
          registry="${host}:5000"
        fi
      fi
      if [[ -z "$registry" ]]; then
        echo "  [WARN] Could not resolve registry host (set LOCAL_REGISTRY or PROLE_K3S_SERVER)." >&2
        echo "  Images will be pulled at deploy time." >&2
        return 0
      fi

      _registry_ref_exists() {
        local ref="$1"
        local reg="${ref%%/*}"
        local rest="${ref#*/}"
        local repo="$rest" tag="latest"
        case "$rest" in
          *@*) repo="${rest%@*}"; tag="${rest#*@}" ;;
          *:*) repo="${rest%:*}"; tag="${rest##*:}" ;;
        esac
        local accept_header='Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json'

        # Prefer HTTPS (TLS-enabled registry); fall back to HTTP for older/insecure setups.
        local url="https://${reg}/v2/${repo}/manifests/${tag}"
        if curl -k -fsS -I \
          -H "$accept_header" \
          "$url" >/dev/null 2>&1; then
          return 0
        fi

        url="http://${reg}/v2/${repo}/manifests/${tag}"
        curl -fsS -I \
          -H 'Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json' \
          "$url" >/dev/null 2>&1
      }

      if ! curl -k -fsS "https://${registry}/v2/" >/dev/null 2>&1 && ! curl -fsS "http://${registry}/v2/" >/dev/null 2>&1; then
        echo "  [WARN] Registry not reachable at https://${registry}/v2/ (or http://${registry}/v2/)." >&2
        echo "  Images will be pulled at deploy time." >&2
        return 0
      fi

      local tar_file
      while IFS= read -r tar_file; do
        [[ -z "$tar_file" ]] && continue
        echo "  Loading $(basename "$tar_file") ..."
        local out
        out=$(docker load -i "$tar_file" 2>&1 || true)
        if [[ -z "$out" ]]; then
          echo "    [WARN] docker load produced no output for $(basename "$tar_file")." >&2
          continue
        fi

        # `docker load` can load multiple tags; push each tag to the registry.
        printf '%s\n' "$out" | sed -n 's/^Loaded image: //p' | while IFS= read -r img; do
          [[ -z "$img" ]] && continue
          local remote="${registry}/${img}"
          if _registry_ref_exists "$remote"; then
            echo "    [SKIP] ${remote} already exists; skipping push."
            continue
          fi
          docker tag "$img" "$remote" >/dev/null 2>&1 || true
          echo "    Pushing ${remote} ..."
          if docker push "$remote" >/dev/null 2>&1; then
            echo "    [OK] Pushed ${remote}"
          else
            echo "    [WARN] Failed to push ${remote}; will pull at deploy time." >&2
          fi
        done
      done <<< "$tar_files"
      ;;
    *)
      echo "WARN: Unknown mode '$mode'; skipping docker-import pre-load." >&2
      ;;
  esac

  echo "Pre-load complete."
}

case "$ACTION" in
  start|update|reload|initialize|restart|repair)
    if _knoe_bool_true "${DOCKER_PRELOAD:-${PROLE_DOCKER_PRELOAD:-}}"; then
      _preload_docker_images
    else
      echo "[SKIP] Docker image pre-load disabled (set DOCKER_PRELOAD=1 to enable)."
    fi
    ;;
esac

if [ -x "$SCRIPT_DIR/init_service_layer.sh" ]; then
  if [[ "$ENABLE_KERBEROS" == "1" ]]; then
    "$SCRIPT_DIR/init_service_layer.sh" -n "$NS" -k "$ACTION"
  else
    "$SCRIPT_DIR/init_service_layer.sh" -n "$NS" "$ACTION"
  fi
  exit $?
fi


OPENTOFU_NAME=${OPENTOFU_NAME:-opentofu}
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
REGISTRY_NAME=${REGISTRY_NAME:-registry}
GARAGE_NAME=${GARAGE_NAME:-garage}
REDIS_NAME=${REDIS_NAME:-redis}
KONG_NAME=${KONG_NAME:-knoe-svc-kong}
KONG_CONFIG_NAME=${KONG_CONFIG_NAME:-knoe-svc-kong-config}
# In k8s (GKE/prod) mode use the knoe-svc-kong naming
if [ "${KNOE_MODE:-}" = "k8s" ]; then
  KONG_NAME=${KONG_NAME/knoe-svc-kong/knoe-svc-kong}
  KONG_CONFIG_NAME=${KONG_CONFIG_NAME/knoe-svc-kong/knoe-svc-kong}
fi
export KONG_NAME KONG_CONFIG_NAME KNOE_MODE
OPENTOFU_CONFIGMAP=${OPENTOFU_CONFIGMAP:-opentofu-nginx}
OPENTOFU_SECRET=${OPENTOFU_SECRET:-opentofu-admin}
GARAGE_CONFIGMAP=${GARAGE_CONFIGMAP:-garage-config}
GARAGE_SECRET_NAME=${GARAGE_SECRET_NAME:-garage-secrets}

find_namespaces() {
  local kind="$1"
  local name="$2"
  kubectl get "$kind" -A --no-headers 2>/dev/null | awk -v n="$name" '$2==n {print $1}' | sort -u
}

collect_other_namespaces() {
  local target="$1"
  shift
  local name="$1"
  shift
  local kinds=("$@")
  local found=""
  local kind
  for kind in "${kinds[@]}"; do
    found+=$(find_namespaces "$kind" "$name")
    found+=$'\n'
  done
  printf '%s\n' "$found" | awk -v target="$target" 'NF && $1 != target {print $1}' | sort -u
}

migrate_common_services() {
  local old_ns

  if [ "${KNOE_MODE:-}" != "k8s" ]; then
    for old_ns in $(collect_other_namespaces "$NS" "$OPENTOFU_NAME" deployment service); do
      echo "Found OpenTofu in namespace '$old_ns'; removing before deploy to '$NS' ..."
      if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then
        "$SCRIPT_DIR/init_opentofu.sh" -n "$old_ns" stop || true
      else
        kubectl delete -n "$old_ns" deploy "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true
        kubectl delete -n "$old_ns" svc "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true
      fi
      kubectl delete -n "$old_ns" configmap "$OPENTOFU_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true
      kubectl delete -n "$old_ns" secret "$OPENTOFU_SECRET" --ignore-not-found >/dev/null 2>&1 || true
    done
  fi

  for old_ns in $(collect_other_namespaces "$REGISTRY_NS" "$REGISTRY_NAME" deployment service); do
    echo "Found Registry ($REGISTRY_NAME) in namespace '$old_ns'; removing before deploy to '$REGISTRY_NS' ..."
    if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then
      "$SCRIPT_DIR/init_registry.sh" -n "$old_ns" stop || true
    else
      kubectl delete -n "$old_ns" deploy "$REGISTRY_NAME" --ignore-not-found >/dev/null 2>&1 || true
      kubectl delete -n "$old_ns" svc "$REGISTRY_NAME" --ignore-not-found >/dev/null 2>&1 || true
    fi
  done

  for old_ns in $(collect_other_namespaces "$NS" "$GARAGE_NAME" statefulset service); do
    echo "Found Garage in namespace '$old_ns'; removing before deploy to '$NS' ..."
    kubectl delete -n "$old_ns" statefulset "$GARAGE_NAME" --ignore-not-found >/dev/null 2>&1 || true
    kubectl delete -n "$old_ns" svc "$GARAGE_NAME" --ignore-not-found >/dev/null 2>&1 || true
    kubectl delete -n "$old_ns" configmap "$GARAGE_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true
    kubectl delete -n "$old_ns" secret "$GARAGE_SECRET_NAME" --ignore-not-found >/dev/null 2>&1 || true
  done

  for old_ns in $(collect_other_namespaces "$NS" "$OPENBAO_NAME" deployment service statefulset); do
    echo "Found OpenBao in namespace '$old_ns'; removing before deploy to '$NS' ..."
    if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then
      "$SCRIPT_DIR/init_openbao.sh" -n "$old_ns" stop || true
    else
      kubectl delete -n "$old_ns" deploy "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
      kubectl delete -n "$old_ns" statefulset "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
      kubectl delete -n "$old_ns" svc "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
    fi
  done

  for old_ns in $(collect_other_namespaces "$NS" "$REDIS_NAME" statefulset deployment service); do
    echo "Found Redis in namespace '$old_ns'; removing before deploy to '$NS' ..."
    if [ -x "$SCRIPT_DIR/init_redis.sh" ]; then
      "$SCRIPT_DIR/init_redis.sh" -n "$old_ns" stop || true
    else
      kubectl delete -n "$old_ns" statefulset "${REDIS_NAME}-master" --ignore-not-found >/dev/null 2>&1 || true
      kubectl delete -n "$old_ns" svc "${REDIS_NAME}-master" "${REDIS_NAME}-headless" --ignore-not-found >/dev/null 2>&1 || true
    fi
  done

  # Kong API gateway (service layer)
  local kong_old_namespaces=""
  kong_old_namespaces+=$(collect_other_namespaces "$NS" "$KONG_NAME" deployment service)
  kong_old_namespaces+=$'\n'
  kong_old_namespaces+=$(collect_other_namespaces "$NS" "$KONG_CONFIG_NAME" configmap)
  for old_ns in $(printf '%s\n' "$kong_old_namespaces" | awk 'NF{print $1}' | sort -u); do
    echo "Found Kong ($KONG_NAME) in namespace '$old_ns'; removing before deploy to '$NS' ..."
    kubectl delete -n "$old_ns" deploy "$KONG_NAME" --ignore-not-found >/dev/null 2>&1 || true
    kubectl delete -n "$old_ns" svc "$KONG_NAME" --ignore-not-found >/dev/null 2>&1 || true
    kubectl delete -n "$old_ns" configmap "$KONG_CONFIG_NAME" --ignore-not-found >/dev/null 2>&1 || true
  done

  # Same-namespace: remove the alternate Kong deployment name if present.
  # In k8s/GKE mode KONG_NAME=knoe-svc-kong; a leftover knoe-svc-kong (or vice
  # versa) in the same namespace causes duplicate pods that the deployer won't
  # clean up on its own.
  local kong_alt_name kong_alt_config
  case "$KONG_NAME" in
    *knoe-svc-kong*)  kong_alt_name="${KONG_NAME//knoe-svc-kong/knoe-svc-kong}"
                      kong_alt_config="${KONG_CONFIG_NAME//knoe-svc-kong/knoe-svc-kong}" ;;
    *knoe-svc-kong*) kong_alt_name="${KONG_NAME//knoe-svc-kong/knoe-svc-kong}"
                      kong_alt_config="${KONG_CONFIG_NAME//knoe-svc-kong/knoe-svc-kong}" ;;
    *)                kong_alt_name="" ; kong_alt_config="" ;;
  esac
  if [[ -n "$kong_alt_name" && "$kong_alt_name" != "$KONG_NAME" ]]; then
    if kubectl -n "$NS" get deploy "$kong_alt_name" >/dev/null 2>&1; then
      echo "Found stale alternate Kong deployment '$kong_alt_name' in '$NS'; removing ..."
      kubectl -n "$NS" delete deploy "$kong_alt_name" --ignore-not-found >/dev/null 2>&1 || true
      kubectl -n "$NS" delete svc "$kong_alt_name" --ignore-not-found >/dev/null 2>&1 || true
      [[ -n "$kong_alt_config" ]] && \
        kubectl -n "$NS" delete configmap "$kong_alt_config" --ignore-not-found >/dev/null 2>&1 || true
    fi
  fi

  # Same-namespace: prune excess unhealthy Kong pods from a stuck rolling update.
  # Happens when the old pod (e.g. OOMKilled) does not terminate cleanly before
  # the new pod comes up, leaving the deployment with more pods than desired.
  if kubectl -n "$NS" get deploy "$KONG_NAME" >/dev/null 2>&1; then
    local _kong_desired _kong_pod_count _stale_pod
    _kong_desired=$(kubectl -n "$NS" get deploy "$KONG_NAME" \
      -o jsonpath='{.spec.replicas}' 2>/dev/null)
    _kong_desired="${_kong_desired:-1}"
    _kong_pod_count=$(kubectl -n "$NS" get pods \
      -l "app=${KONG_NAME}" --no-headers 2>/dev/null | wc -l | tr -d '[:space:]')
    if [[ "${_kong_pod_count:-0}" -gt "${_kong_desired:-1}" ]]; then
      echo "Kong ($KONG_NAME) has ${_kong_pod_count} pod(s) but desired=${_kong_desired};" \
           "removing unhealthy pods ..."
      while IFS= read -r _stale_pod; do
        [[ -z "$_stale_pod" ]] && continue
        echo "  Removing unhealthy pod: $_stale_pod"
        kubectl -n "$NS" delete pod "$_stale_pod" --force --grace-period=0 \
          >/dev/null 2>&1 || true
      done < <(
        kubectl -n "$NS" get pods -l "app=${KONG_NAME}" --no-headers 2>/dev/null \
          | awk '{split($2,r,"/"); ok=($3=="Running" && r[1]==r[2] && r[1]~/^[0-9]+$/); if(!ok) print $1}'
      )
    fi
  fi
}

echo "Deploying common services (namespace=$NS, action=$ACTION)"

rc=0

case "$ACTION" in
  start|update|reload|initialize|restart|status)
    migrate_common_services
    ;;
esac

# ---------------------------------------------------------------------------
# Deploy services in dependency order:
#   1. Registry   – no dependencies; other services pull images from it
#   2. OpenBao    – secrets vault; needed by downstream services
#   3. Garage     – object storage
#   4. OpenTofu   – IaC engine; depends on registry + secrets (non-k8s)
# ---------------------------------------------------------------------------

if [ "${KNOE_MODE:-}" = "k8s" ]; then
  echo "[INFO] Registry update namespace=${REGISTRY_NS}"
  echo "[INFO] GKE/prod mode: skipping in-cluster Docker registry (using GCP Artifact Registry)."
else
  if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then
    REGISTRY_NAMESPACE="$REGISTRY_NS" SERVICE_NAMESPACE="$NS" \
      "$SCRIPT_DIR/init_registry.sh" -n "$REGISTRY_NS" "$ACTION" || rc=$?
  else
    echo "WARN: init_registry.sh not found; registry deploy skipped."
  fi
fi

if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then
  OPENBAO_NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" \
    "$SCRIPT_DIR/init_openbao.sh" -n "$NS" "$ACTION" || rc=$?
else
  echo "WARN: init_openbao.sh not found; skipping OpenBao."
fi

if [ -x "$SCRIPT_DIR/init_redis.sh" ]; then
  REDIS_NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" \
    "$SCRIPT_DIR/init_redis.sh" -n "$NS" "$ACTION" || rc=$?
else
  echo "WARN: init_redis.sh not found; Redis deploy skipped."
fi

if [ -x "$SCRIPT_DIR/init_garage_store.sh" ]; then
  garage_action="$ACTION"
  case "$garage_action" in
    update|reload|initialize) garage_action="start" ;;
  esac
  NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" GARAGE_NAMESPACE="$NS" ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" \
    "$SCRIPT_DIR/init_garage_store.sh" "$garage_action" || rc=$?
else
  echo "WARN: init_garage_store.sh not found; garage deploy skipped."
fi

if [ "${KNOE_MODE:-}" = "k8s" ]; then
  echo "[INFO] k8s mode: skipping OpenTofu deploy."
elif [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then
  ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" "$SCRIPT_DIR/init_opentofu.sh" -n "$NS" "$ACTION" || rc=$?
else
  echo "WARN: init_opentofu.sh not found; skipping OpenTofu."
fi

if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  # KDC is now embedded in the `knoe-auth` pod (multi-container) by default.
  # Only deploy a standalone KDC when explicitly requested.
  if [[ "${PROLE_KDC_STANDALONE:-0}" == "1" ]]; then
    if [ -x "$SCRIPT_DIR/init_kdc.sh" ]; then
      kdc_action="$ACTION"
      case "$kdc_action" in
        stop) kdc_action="cleanup" ;;
        status) kdc_action="status" ;;
        *) kdc_action="update" ;;
      esac
      SERVICE_NAMESPACE="$NS" PROLE_KDC_NAMESPACE="$NS" \
        "$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$?
    else
      echo "WARN: init_kdc.sh not found; standalone KDC deploy skipped."
    fi
  else
    echo "[INFO] Kerberos enabled: skipping standalone KDC deploy (KDC runs as sidecar in knoe-auth)."
  fi
fi

exit "$rc"