prole/scripts/validation/check_kerberos.sh

#!/usr/bin/env bash

set -euo pipefail

# check_kerberos.sh
# Purpose:
# - Run Kerberos authentication checks inside the in-cluster KDC pod

# Initialize SCRIPT_DIR
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
ETC_DIR=$(cd "$SCRIPT_DIR/../../etc" && pwd)
LIB_SHELL_DIR=$(cd "$ETC_DIR/lib/shell" && pwd)
INIT_KDC_SCRIPT="$ETC_DIR/init_kdc.sh"
if [[ ! -x "$INIT_KDC_SCRIPT" && -x "$SCRIPT_DIR/init_kdc.sh" ]]; then
  INIT_KDC_SCRIPT="$SCRIPT_DIR/init_kdc.sh"
fi

ORIG_ARGS=("$@")

# Load env
if [[ -n "${KNOE_HOME:-}" && -f "$KNOE_HOME/env.sh" ]]; then
  set --
  # shellcheck disable=SC1090
  source "$KNOE_HOME/env.sh"
  set -- "${ORIG_ARGS[@]}"
elif [[ -f "$HOME/.knoe/env.sh" ]]; then
  set --
  # shellcheck disable=SC1090
  source "$HOME/.knoe/env.sh"
  set -- "${ORIG_ARGS[@]}"
fi

# Load config values from knoe.cfg before applying defaults/CLI overrides.
# shellcheck disable=SC1090
source "$ETC_DIR/knoe_cfg.sh"

# shellcheck disable=SC1090
source "$LIB_SHELL_DIR/knoe_env.sh"
# shellcheck disable=SC1090
source "$LIB_SHELL_DIR/knoe_secrets.sh"
# shellcheck disable=SC1090
source "$LIB_SHELL_DIR/knoe_string.sh"
# shellcheck disable=SC1090
source "$LIB_SHELL_DIR/knoe_yaml.sh"

if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
  knoe_set_mode "${2:-}"
  shift 2
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
  knoe_set_mode "${1#*=}"
  shift
fi

ACTION=${1:-test}

NAMESPACE="${PROLE_NAMESPACE:-${NAMESPACE:-${SERVICE_NAMESPACE:-default}}}"
SERVICE_NAMESPACE=${SERVICE_NAMESPACE:-${NAMESPACE}}
KRB5_REALM=${KRB5_REALM:-${REALM:-}}
KRB5_KDC=${KRB5_KDC:-}
KRB5_USER=${KRB5_USER:-${KRB5_USERNAME:-}}
KRB5_PASSWORD=${KRB5_PASSWORD:-}
_default_kdc_name() {
  local _mode
  _mode=$(knoe_normalize_mode "${KNOE_MODE:-${DEPLOYMENT_MODE:-${CLUSTER_ENV:-}}}" 2>/dev/null || printf '%s' "${KNOE_MODE:-}")
  case "$_mode" in
    k8s) printf 'authority-gcp-auth' ;;
    *)   printf 'authority-knoe-auth' ;;
  esac
}
PROLE_KDC_NAME=${PROLE_KDC_NAME:-$(_default_kdc_name)}
PROLE_KDC_SERVICE=${PROLE_KDC_SERVICE:-auth}
SAMBA_ADMIN_USER=${SAMBA_ADMIN_USER:-}
SAMBA_ADMIN_PASSWORD=${SAMBA_ADMIN_PASSWORD:-}
SAMBA_DNS_SERVER=${SAMBA_DNS_SERVER:-}
KDC_POD_READY_TIMEOUT=${KDC_POD_READY_TIMEOUT:-60}
KDC_POD_READY_RETRIES=${KDC_POD_READY_RETRIES:-5}
PROLE_KDC_PRECHECK_TIMEOUT=${PROLE_KDC_PRECHECK_TIMEOUT:-5}

KRB5_REALM=$(knoe_normalize_realm "${KRB5_REALM:-}")

ensure_tools() {
  for t in kubectl curl jq; do
    command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
  done
}

ensure_namespace() {
  if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
    echo "Creating namespace '$NAMESPACE' ..."
    kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true
  fi
}

write_krb5_conf_to_pod() {
  local kdc_ns="$1"
  local pod_name="$2"
  if [[ -z "$kdc_ns" || -z "$pod_name" ]]; then
    return 0
  fi
  if [[ -z "$KRB5_REALM" || -z "$KRB5_KDC" ]]; then
    return 0
  fi

  local admin_val domain_val tmp
  admin_val="${KRB5_ADMIN:-$KRB5_KDC}"
  domain_val=$(printf '%s' "$KRB5_REALM" | tr '[:upper:]' '[:lower:]')

  tmp=$(mktemp)
  cat >"$tmp" <<EOF
[libdefaults]
 default_realm = $KRB5_REALM
 dns_lookup_realm = false
 dns_lookup_kdc = false
 udp_preference_limit = 1

[realms]
 $KRB5_REALM = {
  kdc = $KRB5_KDC
  admin_server = $admin_val
 }

[domain_realm]
 .$domain_val = $KRB5_REALM
 $domain_val = $KRB5_REALM
EOF

  echo "Updating /etc/krb5.conf in test pod ${pod_name} for realm ${KRB5_REALM} ..."
  if ! kubectl -n "$kdc_ns" exec -i "$pod_name" -- sh -c 'cat > /etc/krb5.conf' <"$tmp"; then
    echo "WARN: Unable to write /etc/krb5.conf in ${pod_name}." >&2
  fi
  rm -f "$tmp"
}

get_ready_kdc_pod() {
  local kdc_ns="$1"
  kubectl -n "$kdc_ns" wait --for=condition=Ready pod -l "app=${PROLE_KDC_NAME}" --timeout="${KDC_POD_READY_TIMEOUT}s" >/dev/null 2>&1 || true
  kubectl -n "$kdc_ns" get pod -l "app=${PROLE_KDC_NAME}" --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true
}

debug_kdc_service_target() {
  local host="${KRB5_KDC:-}"
  if [[ -z "$host" ]]; then
    return 0
  fi

  # Only attempt Kubernetes Service diagnostics for in-cluster service FQDNs.
  if [[ "$host" != *".svc.cluster.local" ]]; then
    return 0
  fi

  local base svc ns
  base="${host%.svc.cluster.local}"
  svc="${base%%.*}"
  ns="${base#*.}"
  ns="${ns%%.*}"

  if [[ -z "$svc" || -z "$ns" || "$svc" == "$base" ]]; then
    return 0
  fi

  echo "--- KDC service diagnostics for ${svc}.${ns} ---" >&2
  kubectl -n "$ns" get svc "$svc" -o wide 1>&2 || true
  kubectl -n "$ns" get endpoints "$svc" -o yaml 1>&2 || true
}

wait_for_kdc_pod() {
  local kdc_ns="$1"
  local attempt pod
  attempt=1
  while [[ $attempt -le $KDC_POD_READY_RETRIES ]]; do
    pod=$(get_ready_kdc_pod "$kdc_ns")
    if [[ -n "$pod" ]]; then
      printf '%s' "$pod"
      return 0
    fi
    echo "Waiting for KDC pod '${PROLE_KDC_NAME}' to be ready... (${attempt}/${KDC_POD_READY_RETRIES})" >&2
    sleep 2
    attempt=$((attempt + 1))
  done
  return 1
}

# Return 0 if pod is in CrashLoopBackOff (any container), else 1
is_pod_crashloop() {
  local kdc_ns="$1"
  local pod_name="$2"
  if [[ -z "$pod_name" ]]; then
    return 1
  fi
  local reason
  reason=$(kubectl -n "$kdc_ns" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.state.waiting.reason}{"\n"}{end}' 2>/dev/null | tr -d '\r' | tr -s '\n' ' ' || true)
  if echo "${reason}" | grep -q "CrashLoopBackOff"; then
    return 0
  fi
  return 1
}

debug_kdc_pod() {
  local kdc_ns="$1"
  local pod_name="$2"
  if [[ -z "$pod_name" ]]; then
    return 0
  fi
  echo "--- describe pod ${pod_name} ---" >&2
  kubectl -n "$kdc_ns" describe pod "$pod_name" 1>&2 || true
  echo "--- last 200 log lines from ${pod_name} ---" >&2
  kubectl -n "$kdc_ns" logs "$pod_name" --tail=200 1>&2 || true
}

kdc_exec() {
  local kdc_ns="$1"
  local pod_name="$2"
  shift 2
  local output=""
  if output=$(kubectl -n "$kdc_ns" exec "$pod_name" -- "$@" 2>&1); then
    printf '%s' "$output"
    return 0
  fi
  if echo "$output" | grep -qi "completed pod"; then
    return 2
  fi
  if echo "$output" | grep -qi "not found"; then
    return 2
  fi
  echo "$output" >&2
  return 1
}

run_samba_join() {
  local kdc_ns="$1"
  local pod_name="$2"
  local domain="$3"
  local output=""
  if output=$(kubectl -n "$kdc_ns" exec "$pod_name" -- \
    env SAMBA_ADMIN_USER="$SAMBA_ADMIN_USER" SAMBA_ADMIN_PASSWORD="$SAMBA_ADMIN_PASSWORD" \
    SAMBA_DNS_SERVER="$SAMBA_DNS_SERVER" SAMBA_REALM="${KRB5_REALM:-}" \
    /bin/sh -c 'realm_flag=""; [ -n "${SAMBA_REALM}" ] && realm_flag="--realm=${SAMBA_REALM}"; samba-tool domain join "'"$domain"'" MEMBER -U"${SAMBA_ADMIN_USER}%${SAMBA_ADMIN_PASSWORD}" --server="${SAMBA_DNS_SERVER}" ${realm_flag}' 2>&1); then
    printf '%s' "$output"
    return 0
  fi
  if echo "$output" | grep -qi "completed pod"; then
    return 2
  fi
  if echo "$output" | grep -qi "not found"; then
    return 2
  fi
  if [[ -n "$output" ]]; then
    echo "$output" >&2
  else
    echo "samba-tool produced no output." >&2
  fi
  return 1
}

run_kinit() {
  local kdc_ns="$1"
  local pod_name="$2"
  local output=""

  # Prefer /etc/krb5.conf if it exists, otherwise fallback to /opt/knoe-kdc/krb5.conf
  local krb5_config="/etc/krb5.conf"
  if ! kubectl -n "$kdc_ns" exec "$pod_name" -- ls /etc/krb5.conf >/dev/null 2>&1; then
    krb5_config="/opt/knoe-kdc/krb5.conf"
  fi

  echo "Running kinit for ${KRB5_USER}@${KRB5_REALM} inside ${PROLE_KDC_NAME} (${pod_name}) using ${krb5_config} ..."
  if output=$(printf '%s\n' "$KRB5_PASSWORD" | kubectl -n "$kdc_ns" exec -i "$pod_name" -- \
      env KRB5_CONFIG="$krb5_config" kinit "${KRB5_USER}@${KRB5_REALM}" 2>&1); then
    printf '%s' "$output"
    return 0
  fi
  if echo "$output" | grep -qi "completed pod"; then
    return 2
  fi
  if echo "$output" | grep -qi "not found"; then
    return 2
  fi
  if [[ -n "$output" ]]; then
    echo "$output" >&2
  else
    echo "kinit produced no output." >&2
  fi

  echo "--- Debug: kinit failure context ---" >&2
  echo "KRB5_REALM=${KRB5_REALM:-<unset>}" >&2
  echo "KRB5_KDC=${KRB5_KDC:-<unset>}" >&2
  echo "KRB5_CONFIG=${krb5_config}" >&2
  # Print the relevant realm stanza (best-effort).
  kubectl -n "$kdc_ns" exec "$pod_name" -- sh -c "grep -nE '^(\[realms\]|\[libdefaults\]|\[domain_realm\]|[[:space:]]*(kdc|admin_server)[[:space:]]*=)' '$krb5_config' || true" 1>&2 || true
  debug_kdc_service_target

  return 1
}

check_samba_connectivity() {
  local kdc_ns="$1"
  local pod_name="$2"
  if [[ -z "$SAMBA_DNS_SERVER" ]]; then
    return 0
  fi

  echo "Checking connectivity from ${PROLE_KDC_NAME} to ${SAMBA_DNS_SERVER} ..."
  local ports="88 389 445 464"
  local port
  for port in $ports; do
    local out=""
    out=$(kubectl -n "$kdc_ns" exec "$pod_name" -- /bin/sh -c "timeout 3 bash -c '</dev/tcp/${SAMBA_DNS_SERVER}/${port}'" 2>&1) || true
    if [[ -z "$out" ]]; then
      echo "OK: ${SAMBA_DNS_SERVER}:${port}"
    else
      echo "WARN: ${SAMBA_DNS_SERVER}:${port} unreachable: ${out}"
    fi
  done
}

check_kdc_direct_connectivity() {
  local kdc_ns="$1"
  local pod_name="$2"
  local kdc_ip="${AD_DC_IP:-${KDC_ANSIBLE_DETECTED:-${KDC_AUTO_DETECTED:-}}}"
  if [[ -z "$kdc_ip" ]]; then
    return 0
  fi

  echo "Checking connectivity from ${PROLE_KDC_NAME} to KDC IP ${kdc_ip} ..."
  local ports="88 389 445 464"
  local port
  for port in $ports; do
    local out=""
    out=$(kubectl -n "$kdc_ns" exec "$pod_name" -- /bin/sh -c "timeout 3 bash -c '</dev/tcp/${kdc_ip}/${port}'" 2>&1) || true
    if [[ -z "$out" ]]; then
      echo "OK: ${kdc_ip}:${port}"
    else
      echo "WARN: ${kdc_ip}:${port} unreachable: ${out}"
    fi
  done
}

openbao_url() {
  if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
    echo "$PROLE_OPENBAO_URL"
    return 0
  fi
  if knoe_is_in_cluster; then
    echo "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE:-default}}.svc.cluster.local:8200"
    return 0
  fi
  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    echo "http://127.0.0.1:8200"
    return 0
  elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    echo "http://127.0.0.1:8200"
    return 0
  else
    echo ""
    return 0
  fi
}

openbao_token() {
  if [[ -f "$KNOE_SERVICE/secrets/openbao-root-token" ]]; then
    cat "$KNOE_SERVICE/secrets/openbao-root-token"
  else
    echo "${OPENBAO_ROOT_TOKEN:-}"
  fi
}

fetch_openbao_secret() {
  local path="$1"
  local key="$2"
  local token url
  token=$(openbao_token)
  url=$(openbao_url)
  if [[ -z "$token" || -z "$url" ]]; then
    echo ""
    return 0
  fi
  curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
}

resolve_krb5_password() {
  if knoe_secret_needs_resolution "${KRB5_PASSWORD:-}"; then
    local path="knoe/${NAMESPACE:-default}/kerberos"
    local fetched
    fetched=$(fetch_openbao_secret "$path" "password")
    KRB5_PASSWORD=$(knoe_secret_choose_value "${KRB5_PASSWORD:-}" "$fetched")
  fi
}

resolve_samba_vars() {
  if [[ -n "${SAMBA_ADMIN_USER}" && -n "${SAMBA_ADMIN_PASSWORD}" && -n "${SAMBA_DNS_SERVER}" ]]; then
    return 0
  fi

  # 1. Fallback for DNS server from knoe.cfg env (AD_DC_HOST)
  if [[ -z "$SAMBA_DNS_SERVER" && -n "${AD_DC_HOST:-}" ]]; then
    SAMBA_DNS_SERVER="$AD_DC_HOST"
    echo "Using SAMBA_DNS_SERVER=${SAMBA_DNS_SERVER} from AD_DC_HOST"
  fi

  # 2. Try to resolve from Ansible inventory if possible
  local root
  if [[ -n "${KNOE_HOME:-}" && -d "${KNOE_HOME}" ]]; then
    root="${KNOE_HOME}"
  else
    root="${SCRIPT_DIR}/.."
  fi

  local vars_file="${root}/infrastructure/inventory/group_vars/ad_dc/vars.yml"
  if [[ -f "$vars_file" ]]; then
    if [[ -z "$SAMBA_DNS_SERVER" ]]; then
      SAMBA_DNS_SERVER=$(knoe_yaml_get_scalar "$vars_file" "samba_dns_server")
    fi
    if [[ -z "$SAMBA_ADMIN_USER" ]]; then
      SAMBA_ADMIN_USER=$(knoe_yaml_get_scalar "$vars_file" "samba_dns_admin_user")
    fi
  fi

  # 3. Fallback for Admin User
  if [[ -z "$SAMBA_ADMIN_USER" ]]; then
    if [[ -n "${AD_DC_USER:-}" ]]; then
      SAMBA_ADMIN_USER="$AD_DC_USER"
      echo "Using SAMBA_ADMIN_USER=${SAMBA_ADMIN_USER} from AD_DC_USER"
    else
      SAMBA_ADMIN_USER="administrator"
      echo "Defaulting SAMBA_ADMIN_USER to administrator"
    fi
  fi

  # 4. Try to resolve password from 1Password
  if [[ -z "$SAMBA_ADMIN_PASSWORD" ]]; then
    if command -v op >/dev/null 2>&1 && op whoami >/dev/null 2>&1; then
      SAMBA_ADMIN_PASSWORD=$(op item get "samba-dns-admin" --vault knoey --fields password --reveal 2>/dev/null || true)
      [[ -n "$SAMBA_ADMIN_PASSWORD" ]] && echo "Fetched SAMBA_ADMIN_PASSWORD from 1Password knoey vault"
    fi
  fi

  # 5. Fallback for Password from OpenBao
  if [[ -z "$SAMBA_ADMIN_PASSWORD" ]]; then
    local path="knoe/${NAMESPACE:-default}/ad_dc"
    local fetched
    fetched=$(fetch_openbao_secret "$path" "password")
    if [[ -n "$fetched" && "$fetched" != "null" ]]; then
      SAMBA_ADMIN_PASSWORD="$fetched"
      echo "Fetched SAMBA_ADMIN_PASSWORD from OpenBao (${path})"
    else
        # Try kerberos path as fallback
        path="knoe/${NAMESPACE:-default}/kerberos"
        fetched=$(fetch_openbao_secret "$path" "samba_password")
        if [[ -n "$fetched" && "$fetched" != "null" ]]; then
          SAMBA_ADMIN_PASSWORD="$fetched"
          echo "Fetched SAMBA_ADMIN_PASSWORD from OpenBao (${path}#samba_password)"
        fi
    fi
  fi
}

join_samba_realm_if_needed() {
  local kdc_ns="$1"
  local pod_name="$2"
  local domain=""

  resolve_samba_vars

  if [[ -z "$SAMBA_ADMIN_USER" || -z "$SAMBA_ADMIN_PASSWORD" || -z "$SAMBA_DNS_SERVER" ]]; then
    echo "WARN: Missing Samba join info; skipping samba-tool realm join."
    return 0
  fi

  check_samba_connectivity "$kdc_ns" "$pod_name"

  if [[ -n "$KRB5_REALM" ]]; then
    domain=$(printf '%s' "$KRB5_REALM" | tr '[:upper:]' '[:lower:]')
  fi
  if [[ -z "$domain" ]]; then
    domain="$SAMBA_DNS_SERVER"
  fi

  if kubectl -n "$kdc_ns" exec "$pod_name" -- test -f /var/lib/samba/private/secrets.tdb >/dev/null 2>&1; then
    echo "Samba already joined; skipping join."
    return 0
  fi

  echo "Joining Samba realm '${domain}' (server ${SAMBA_DNS_SERVER}) ..."
  local attempt=1
  while true; do
    if run_samba_join "$kdc_ns" "$pod_name" "$domain"; then
      return 0
    fi
    if [[ $? -eq 2 ]]; then
      if [[ $attempt -ge $KDC_POD_READY_RETRIES ]]; then
        echo "WARN: KDC pod completed before join could run; skipping samba-tool join." >&2
        return 0
      fi
      pod_name=$(wait_for_kdc_pod "$kdc_ns" || true)
      if [[ -z "$pod_name" ]]; then
        echo "WARN: KDC pod unavailable for samba-tool join." >&2
        return 0
      fi
      attempt=$((attempt + 1))
      continue
    fi
    echo "WARN: samba-tool domain join failed; continuing to Kerberos test." >&2
    return 0
  done
}

run_test() {
  ensure_tools
  ensure_namespace
  resolve_krb5_password

  # Support both names for the toggle from knoe.cfg
  local enabled="${KERBEROS_ENABLED:-${ENABLED:-false}}"
  if [[ "$enabled" == "false" || "$enabled" == "0" || "$enabled" == "False" ]]; then
    echo "Kerberos is disabled (KERBEROS_ENABLED=$enabled). Skipping test."
    return 0
  fi

  if [[ -z "$KRB5_REALM" || -z "$KRB5_USER" || -z "$KRB5_PASSWORD" || -z "$KRB5_KDC" ]]; then
    echo "ERROR: Missing Kerberos configuration. Ensure KRB5_REALM, KRB5_KDC, KRB5_USER, KRB5_PASSWORD are set." >&2
    exit 1
  fi

  # Quick pre-check: if an existing 'auth' deployment is present but not Available, remove it
  # to avoid starting a rollout that is likely to time out on a stale/broken replica set.
  local kdc_ns
  kdc_ns="${SERVICE_NAMESPACE:-${NAMESPACE:-default}}"
  if kubectl -n "$kdc_ns" get deploy "$PROLE_KDC_NAME" >/dev/null 2>&1; then
    # Try a fast availability wait; if it fails, treat as unhealthy and clean up before proceeding.
    if ! kubectl -n "$kdc_ns" wait --for=condition=Available deploy/"$PROLE_KDC_NAME" --timeout="${PROLE_KDC_PRECHECK_TIMEOUT}s" >/dev/null 2>&1; then
      echo "Detected unhealthy '$PROLE_KDC_NAME' deployment in namespace '$kdc_ns'; removing before Kerberos test rollout to avoid timeout..." >&2
      if [[ -x "$INIT_KDC_SCRIPT" ]]; then
        SERVICE_NAMESPACE="$SERVICE_NAMESPACE" "$INIT_KDC_SCRIPT" cleanup || true
      else
        kubectl -n "$kdc_ns" delete deploy "$PROLE_KDC_NAME" --ignore-not-found || true
        kubectl -n "$kdc_ns" delete svc "$PROLE_KDC_SERVICE" --ignore-not-found || true
        kubectl -n "$kdc_ns" delete configmap knoe-kdc-config --ignore-not-found || true
      fi
    fi
  fi

  local kdc_mode="${KNOE_MODE:-${DEPLOYMENT_MODE:-}}"
  local kdc_mode_args=()
  if [[ -n "$kdc_mode" ]]; then
    kdc_mode_args=(--mode "$kdc_mode")
  fi

  if [[ -x "$INIT_KDC_SCRIPT" ]]; then
    if ! SERVICE_NAMESPACE="$SERVICE_NAMESPACE" KRB5_REALM="$KRB5_REALM" KRB5_KDC="$KRB5_KDC" \
      KRB5_ADMIN="$KRB5_KDC" KRB5_USER="$KRB5_USER" KRB5_PASSWORD="$KRB5_PASSWORD" \
      "$INIT_KDC_SCRIPT" "${kdc_mode_args[@]}" update; then
      echo "ERROR: Failed to ensure KDC deployment '${PROLE_KDC_NAME}' in namespace '$kdc_ns'." >&2
      exit 1
    fi
  fi

  local pod_name
  pod_name=$(wait_for_kdc_pod "$kdc_ns" || true)
  if [[ -z "$pod_name" ]]; then
    echo "ERROR: KDC pod '${PROLE_KDC_NAME}' not found in namespace '$kdc_ns'. Run init_kdc.sh update first." >&2
    exit 1
  fi

  # If the pod is crashlooping, dump logs and fail fast
  if is_pod_crashloop "$kdc_ns" "$pod_name"; then
    echo "ERROR: KDC pod '${pod_name}' is in CrashLoopBackOff. Dumping diagnostics..." >&2
    debug_kdc_pod "$kdc_ns" "$pod_name"
    exit 1
  fi

  write_krb5_conf_to_pod "$kdc_ns" "$pod_name"

  check_kdc_direct_connectivity "$kdc_ns" "$pod_name"

  join_samba_realm_if_needed "$kdc_ns" "$pod_name"

  echo "Running kinit for ${KRB5_USER}@${KRB5_REALM} inside ${PROLE_KDC_NAME} (${pod_name}) ..."
  local attempt=1
  while true; do
    if run_kinit "$kdc_ns" "$pod_name"; then
      break
    fi
    if [[ $? -eq 2 ]] || kubectl -n "$kdc_ns" get pod "$pod_name" -o jsonpath='{.status.phase}' 2>/dev/null | grep -qi "Succeeded"; then
      if [[ $attempt -ge $KDC_POD_READY_RETRIES ]]; then
        echo "ERROR: KDC pod completed before kinit could run." >&2
        exit 1
      fi
      pod_name=$(wait_for_kdc_pod "$kdc_ns" || true)
      if [[ -z "$pod_name" ]]; then
        echo "ERROR: KDC pod unavailable for kinit." >&2
        exit 1
      fi
      attempt=$((attempt + 1))
      continue
    fi
    # If the pod is crashlooping, show logs to aid troubleshooting
    if is_pod_crashloop "$kdc_ns" "$pod_name"; then
      echo "ERROR: KDC pod '${pod_name}' entered CrashLoopBackOff during kinit. Dumping diagnostics..." >&2
      debug_kdc_pod "$kdc_ns" "$pod_name"
    fi
    echo "ERROR: kinit failed for ${KRB5_USER}@${KRB5_REALM}." >&2
    exit 1
  done

  echo "Kerberos ticket cache:"
  # Determine krb5 config path in the pod similar to run_kinit()
  local krb5_config_in_pod="/etc/krb5.conf"
  if ! kubectl -n "$kdc_ns" exec "$pod_name" -- ls /etc/krb5.conf >/dev/null 2>&1; then
    krb5_config_in_pod="/opt/knoe-kdc/krb5.conf"
  fi
  kubectl -n "$kdc_ns" exec "$pod_name" -- env KRB5_CONFIG="$krb5_config_in_pod" klist || true
}

case "$ACTION" in
  test)
    run_test
    ;;
  cleanup)
    echo "No test pods to cleanup (KDC-based test)."
    ;;
  *)
    echo "Usage: $0 {test|cleanup}" >&2
    exit 2
    ;;
esac