prole/mock_val/init_knoe_users.sh

#!/usr/bin/env bash

set -euo pipefail

# init_knoe_users.sh
# Purpose:
# - Provision Kerberos principals and database accounts for knoe-system users
# - Creates: admin@KNOE.LOCAL (master password), guest@KNOE.LOCAL (read-only),
#   postgres service principal (keytab for GSS auth), developer group role
# - Cross-realm trust with myrddin.prole.org PROLE.ORG is referenced via
#   PROLE_KDC_TRUST_REALM=PROLE.ORG in init_kdc.sh / init_knoe_auth.sh
# - Sets up service admin access: ArgoCD RBAC, Gitea, GitLab
# - Ensures Gitea SPNEGO keytab (HTTP/git.prole.org@PROLE.ORG) is provisioned
#
# Prerequisites:
#   init_kdc.sh initialize  (with PROLE_KDC_REALM=PROLE.LOCAL,
#                             PROLE_KDC_TRUST_REALM=PROLE.ORG,
#                             KRB5_KDC=myrddin.prole.org)
#   init_cnpg_backup.sh     (CNPG cluster must exist)
#   init_argocd.sh          (ArgoCD must be running)
#
# Usage:
#   ./init_knoe_users.sh initialize           # create/update all user resources
#   ./init_knoe_users.sh onboard <user> <email> [role]  # add a single new user
#   ./init_knoe_users.sh status               # show current state
#   ./init_knoe_users.sh cleanup              # remove managed secrets/patches
#
# 1Password authentication (required for recovery credential generation):
#   Preferred — run from your laptop with 1Password desktop open:
#     op run -- bash etc/init_knoe_users.sh --mode k3s initialize
#   Headless server — store a service account token in k8s first:
#     kubectl -n knoe-system create secret generic op-service-account-token \
#       --from-literal=token=<op service-account token>
#   Manual — sign in interactively, then run normally:
#     op signin && bash etc/init_knoe_users.sh --mode k3s initialize
#
# Browser SPNEGO (Kerberos SSO in Chrome/Edge):
#   Deploy to all managed endpoints via Ansible — NOT a per-user step:
#     ansible-playbook infrastructure/playbooks/workstation_kerberos.yml
#   This deploys Chrome managed policy + krb5.conf.  New-hire laptops get it
#   automatically as part of the standard provisioning run.

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

# Pre-scan argv for --mode before sourcing knoe_cfg.sh.
# knoe_cfg.sh selects the .cfg file based on KNOE_MODE at source time;
# without this, KNOE_MODE is empty and k3d.cfg wins the fallback loop even
# when --mode k3s is passed — causing k3s.cfg values (e.g. KNOE_ADMIN_PRINCIPAL)
# to never be loaded.  knoe_normalize_mode is not yet available here so we
# accept k3d/k3s/k8s verbatim; the canonical normalisation runs in knoe_set_mode.
_knoe_prescan_mode() {
  local _prev=""
  for _arg in "$@"; do
    case "$_prev" in --mode|-m) export KNOE_MODE="$_arg"; return ;; esac
    case "$_arg" in --mode=*|-m=*) export KNOE_MODE="${_arg#*=}"; return ;; esac
    _prev="$_arg"
  done
}
_knoe_prescan_mode "$@"
unset -f _knoe_prescan_mode

# shellcheck disable=SC1090
source "$SCRIPT_DIR/knoe_cfg.sh"

if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
  knoe_set_mode "${2:-}"
  shift 2
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
  knoe_set_mode "${1#*=}"
  shift
fi

knoe_ensure_kubeconfig >/dev/null 2>&1 || true
ensure_kube_context || exit 1

# Guard: reject the 'default' context in k3d mode — it means the desired
# k3d context was not found and a placeholder was silently accepted.
# In k3s/k8s mode the k3s-generated kubeconfig legitimately uses 'default'
# as the context name (e.g. on the cluster-server node itself), so allow it.
_active_ctx=$(kubectl config current-context 2>/dev/null || true)
_active_mode="${KNOE_MODE:-}"
if [[ "${_active_ctx:-}" == "default" && "$_active_mode" != "k3s" && "$_active_mode" != "k8s" ]]; then
  printf '[ERROR] Active kubectl context is '\''default'\'' — the required context was not found.\n' >&2
  printf '        In k3d mode this indicates a missed context switch.\n' >&2
  printf '        Pass --mode k3s if you intend to target a k3s cluster, e.g.:\n' >&2
  printf '          %s --mode k3s initialize\n' "$(basename "$0")" >&2
  printf '        Or add the correct kubeconfig: export KUBECONFIG=%s:~/.kube/config\n' "${KUBECONFIG:-/path/to/knoe-k3s.kubeconfig}" >&2
  exit 1
fi
unset _active_ctx _active_mode

ACTION="${1:-initialize}"

resolve_knoe_mode() {
  local raw_mode="${KNOE_MODE:-${DEPLOYMENT_MODE:-${CLUSTER_ENV:-k3s}}}"
  if command -v knoe_normalize_mode >/dev/null 2>&1; then
    knoe_normalize_mode "$raw_mode"
    return 0
  fi
  raw_mode=$(printf '%s' "$raw_mode" | tr 'A-Z' 'a-z')
  case "$raw_mode" in
    prod|production)
      printf 'k8s'
      ;;
    *)
      printf '%s' "$raw_mode"
      ;;
  esac
}

default_knoe_auth_deployment() {
  case "$(resolve_knoe_mode)" in
    k8s)
      printf 'authority-gcp-auth'
      ;;
    *)
      printf 'authority-knoe-auth'
      ;;
  esac
}

# ── Configurable variables ─────────────────────────────────────────────────
# PROLE_KDC_NAMESPACE is the canonical env set by init_kdc.sh; honour it here
# so both scripts look at the same namespace for knoe-kdc-secrets.
# Never fall through to "default" — use knoe-system as the last resort.
KNOE_USERS_NAMESPACE="${KNOE_USERS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}"
KNOE_KDC_NAMESPACE="${KNOE_KDC_NAMESPACE:-${PROLE_KDC_NAMESPACE:-${KNOE_USERS_NAMESPACE}}}"
KNOE_DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
KNOE_DB_CLUSTER="${KNOE_DB_CLUSTER:-knoe-db}"
KNOE_DEPLOYMENT_MODE="${KNOE_DEPLOYMENT_MODE:-$(resolve_knoe_mode)}"
KNOE_AUTH_DEPLOYMENT="${KNOE_AUTH_DEPLOYMENT:-${PROLE_KDC_NAME:-$(default_knoe_auth_deployment)}}"
KNOE_ADMIN_PRINCIPAL="${KNOE_ADMIN_PRINCIPAL:-admin}"
PROLE_KDC_REALM="${PROLE_KDC_REALM:-KNOE.LOCAL}"
GITEA_HOST="${GITEA_HOST:-git.prole.org}"
GITEA_SPNEGO_HOST="${GITEA_SPNEGO_HOST:-${GITEA_HOST}}"
GITEA_KRB5_AD_REALM="${GITEA_KRB5_AD_REALM:-PROLE.ORG}"
GITEA_KRB5_AD_USER="${GITEA_KRB5_AD_USER:-gitea-http}"
GITLAB_HOST="${GITLAB_HOST:-gitlab.prole.org}"
GITEA_NAMESPACE="${GITEA_NAMESPACE:-gitea}"
GITLAB_NAMESPACE="${GITLAB_NAMESPACE:-gitlab}"
ARGOCD_NAMESPACE="${ARGOCD_NAMESPACE:-argocd}"
# Provided externally or resolved from k8s secrets / 1Password
PROLE_KDC_MASTER_PASSWORD="${PROLE_KDC_MASTER_PASSWORD:-}"
KNOE_GUEST_PASSWORD="${KNOE_GUEST_PASSWORD:-}"
# 1Password secret references (op:// URIs). Set in conf/k3s.cfg or via env.
# When set and op CLI is available, these are used as fallback if the k8s
# secret is absent or empty.
OP_KDC_MASTER_PASSWORD_REF="${OP_KDC_MASTER_PASSWORD_REF:-}"
OP_KDC_GUEST_PASSWORD_REF="${OP_KDC_GUEST_PASSWORD_REF:-}"
# UI-supplied credentials (set by knoe_users.py from the Database / Kerberos Auth screens)
PROLE_LOCAL_ADMIN_PASSWORD="${PROLE_LOCAL_ADMIN_PASSWORD:-}"
PROLE_ORG_ADMIN_PASSWORD="${PROLE_ORG_ADMIN_PASSWORD:-}"
# Optional: personal access tokens for service admin promotion
GITEA_ADMIN_TOKEN="${GITEA_ADMIN_TOKEN:-}"
GITLAB_ADMIN_TOKEN="${GITLAB_ADMIN_TOKEN:-}"
# 1Password vault for Gitea/service credentials (default: Personal)
GITEA_OP_VAULT="${GITEA_OP_VAULT:-Personal}"

log()  { printf '[INFO]  %s\n' "$*"; }

# _op_ensure_auth
# Establishes a usable 1Password session for headless/automated use.
# Priority order:
#   1. OP_SERVICE_ACCOUNT_TOKEN already in env (Teams/Business service account,
#      or injected by CI/CD).
#   2. K8s Secret "op-service-account-token" in SERVICE_NAMESPACE — for servers
#      that run the script directly (e.g. myrddin running as the admin user).
#      Create once:
#        op service-account create "prole-init" --expires-in 1y \
#          --vault "${GITEA_OP_VAULT}"
#        kubectl -n knoe-system create secret generic op-service-account-token \
#          --from-literal=token=<printed-token>
#      Requires 1Password Teams/Business.
#   3. Existing interactive session — covers admin running the script from their
#      laptop with 1Password desktop app open, or after a manual "op signin".
#   4. "op run --" inheritance — when the script is invoked as:
#        op run -- bash etc/init_knoe_users.sh --mode k3s initialize
#      op run sets OP_SESSION_* in the environment, which the op CLI picks up
#      automatically. No extra code needed; this case falls through to (3).
#
# Returns 0 when 1Password is usable, 1 otherwise (non-fatal; callers warn).
_op_ensure_auth() {
  command -v op >/dev/null 2>&1 || { warn "1Password CLI (op) not installed — skipping credential steps"; return 1; }

  # Already set (service account token, CI env, or op run --)
  if [[ -n "${OP_SERVICE_ACCOUNT_TOKEN:-}" ]]; then
    op account list >/dev/null 2>&1 && return 0
    # Token set but op still rejects — fall through to other methods
  fi

  # Try k8s-stored service account token (headless server running this script)
  local _ns="${SERVICE_NAMESPACE:-knoe-system}"
  local _tok
  _tok=$(kubectl -n "$_ns" get secret op-service-account-token \
    -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null || true)
  if [[ -n "$_tok" ]]; then
    export OP_SERVICE_ACCOUNT_TOKEN="$_tok"
    log "1Password: using service account token from k8s secret op-service-account-token" >&2
    return 0
  fi

  # Try existing interactive session (op signin already done, or desktop app running)
  if op account list >/dev/null 2>&1; then
    return 0
  fi

  warn "1Password: not authenticated. Choose one of:"
  warn "  A) Run from your laptop (1Password desktop app handles auth automatically):"
  warn "       op run -- bash etc/init_knoe_users.sh --mode k3s initialize"
  warn "  B) Sign in once on this host, then re-run:"
  warn "       op signin && bash etc/init_knoe_users.sh --mode k3s initialize"
  warn "  C) Create a service account (1Password Teams/Business) and store token:"
  warn "       op service-account create 'prole-init' --expires-in 1y --vault '${GITEA_OP_VAULT}'"
  warn "       kubectl -n ${_ns} create secret generic op-service-account-token --from-literal=token=<token>"
  return 1
}
err()  { printf '[ERROR] %s\n' "$*" >&2; }
warn() { printf '[WARN]  %s\n' "$*" >&2; }
die()  { err "$*"; exit 1; }

ensure_tools() {
  for t in kubectl curl python3; do
    command -v "$t" >/dev/null || die "Missing required tool: $t"
  done
}

b64_decode() {
  if base64 --decode </dev/null >/dev/null 2>&1; then
    base64 --decode
  elif base64 -d </dev/null >/dev/null 2>&1; then
    base64 -d
  else
    base64 -D
  fi
}

gen_password() {
  if command -v openssl >/dev/null 2>&1; then
    openssl rand -base64 18
    return 0
  fi
  python3 - <<'PY'
import secrets, string
alphabet = string.ascii_letters + string.digits
print(''.join(secrets.choice(alphabet) for _ in range(24)))
PY
}

# ── Secret helpers ─────────────────────────────────────────────────────────

# Read a value from 1Password using the op CLI.
# Returns 1 (empty output) if op is not installed or the ref is empty.
# Usage: try_op_read "op://vault/item/field"
try_op_read() {
  local ref="${1:-}"
  [[ -n "$ref" ]] || return 1
  command -v op >/dev/null 2>&1 || return 1
  local val
  val=$(op read "$ref" 2>/dev/null) || return 1
  [[ -n "$val" ]] || return 1
  printf '%s' "$val"
}

# Read a base64-encoded key from a k8s Secret in KNOE_KDC_NAMESPACE.
# Mirrors get_secret_value() in init_kdc.sh (which reads from KDC_NAMESPACE).
get_secret_value() {
  local secret="$1" key="$2"
  kubectl -n "$KNOE_KDC_NAMESPACE" get secret "$secret" \
    -o "jsonpath={.data.${key}}" 2>/dev/null | b64_decode 2>/dev/null || true
}

# ── KDC helpers ────────────────────────────────────────────────────────────

get_kdc_pod() {
  # The kdc container lives inside the authority deployment pod selected by mode.
  # Only return Running pods — Terminating pods from a prior rollout must be excluded.
  local pod
  pod=$(kubectl -n "$KNOE_KDC_NAMESPACE" get pods \
    -l "app=${KNOE_AUTH_DEPLOYMENT}" \
    --field-selector=status.phase=Running \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -n "$pod" ]]; then
    printf '%s' "$pod"
    return 0
  fi
  return 0
}

# _resolve_kdc_registry — sets _kdc_reg_internal and _kdc_reg_host in the
# caller's scope for use as env-var prefixes on init_kdc.sh invocations.
# In k3s mode derives correct values from KNOE_IMAGE_REGISTRY / PROLE_K3S_SERVER
# so stale k3d LOCAL_REGISTRY_INTERNAL values cannot leak into subprocesses.
_resolve_kdc_registry() {
  _kdc_reg_internal=""
  _kdc_reg_host=""
  if [[ "$(resolve_knoe_mode)" == "k3s" ]]; then
    if [[ -n "${KNOE_IMAGE_REGISTRY:-}" ]]; then
      _kdc_reg_internal="${KNOE_IMAGE_REGISTRY}"
    else
      local _rns="${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-${KNOE_KDC_NAMESPACE}}}"
      _kdc_reg_internal="registry.${_rns}.svc.cluster.local:5000"
    fi
    local _k3s_url="${PROLE_K3S_SERVER:-${K3S_SERVER:-${K3S_SERVER_URL:-}}}"
    if [[ -n "${_k3s_url:-}" ]]; then
      _kdc_reg_host="${_k3s_url#https://}"
      _kdc_reg_host="${_kdc_reg_host#http://}"
      _kdc_reg_host="${_kdc_reg_host%%:*}"
      _kdc_reg_host="${_kdc_reg_host}:5000"
    fi
  fi
}

ensure_kdc_pod() {
  local kdc_pod init_kdc_script
  kdc_pod=$(get_kdc_pod)
  if [[ -n "$kdc_pod" ]]; then
    printf '%s' "$kdc_pod"
    return 0
  fi

  init_kdc_script="$SCRIPT_DIR/init_kdc.sh"
  [[ -f "$init_kdc_script" ]] || die "Required script not found: $init_kdc_script"

  log "No authority pod found in namespace ${KNOE_KDC_NAMESPACE}; bootstrapping ${KNOE_AUTH_DEPLOYMENT} via init_kdc.sh ..." >&2

  _resolve_kdc_registry
  LOCAL_REGISTRY_INTERNAL="${_kdc_reg_internal}" \
  PROLE_KDC_REGISTRY_INTERNAL="${_kdc_reg_internal}" \
  PROLE_KDC_REGISTRY_HOST="${_kdc_reg_host}" \
  PROLE_KDC_NAMESPACE="$KNOE_KDC_NAMESPACE" \
  PROLE_KDC_MASTER_PASSWORD="$PROLE_KDC_MASTER_PASSWORD" \
  PROLE_KDC_NAME="$KNOE_AUTH_DEPLOYMENT" \
    bash "$init_kdc_script" initialize >&2

  kdc_pod=$(get_kdc_pod)
  [[ -n "$kdc_pod" ]] || die "No KDC pod found in namespace ${KNOE_KDC_NAMESPACE} (label app=${KNOE_AUTH_DEPLOYMENT})."
  printf '%s' "$kdc_pod"
}

kdc_principal_exists() {
  local pod="$1" principal="$2"
  kubectl -n "$KNOE_KDC_NAMESPACE" exec "$pod" -c kdc -- \
    kadmin.local -q "get_principal ${principal}" >/dev/null 2>&1
}

kdc_addprinc() {
  local pod="$1" principal="$2" password="$3"
  kubectl -n "$KNOE_KDC_NAMESPACE" exec "$pod" -c kdc -- \
    kadmin.local -q "addprinc -pw ${password} ${principal}"
}

kdc_addprinc_randkey() {
  local pod="$1" principal="$2"
  kubectl -n "$KNOE_KDC_NAMESPACE" exec "$pod" -c kdc -- \
    kadmin.local -q "addprinc -randkey ${principal}"
}

# ── CNPG helpers ───────────────────────────────────────────────────────────

get_cnpg_primary() {
  local pod=""
  # CNPG 1.20+: instanceRole label
  pod=$(kubectl -n "$KNOE_DB_NAMESPACE" get pods \
    -l "cnpg.io/cluster=${KNOE_DB_CLUSTER},cnpg.io/instanceRole=primary" \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -z "$pod" ]]; then
    # Older label set
    pod=$(kubectl -n "$KNOE_DB_NAMESPACE" get pods \
      -l "cnpg.io/cluster=${KNOE_DB_CLUSTER},role=primary" \
      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  fi
  printf '%s' "$pod"
}

cnpg_psql() {
  local pod="$1"; shift
  kubectl -n "$KNOE_DB_NAMESPACE" exec "$pod" -- \
    psql -U postgres -d postgres "$@"
}

# ── Main: initialize ───────────────────────────────────────────────────────

initialize() {
  ensure_tools

  log "=== knoe-system user provisioning ==="
  log "Namespaces: kdc/authority=${KNOE_KDC_NAMESPACE}  db=${KNOE_DB_NAMESPACE}"
  log "Mode: ${KNOE_DEPLOYMENT_MODE}  authority deployment: ${KNOE_AUTH_DEPLOYMENT}"

  # 1. Resolve KDC master password — try multiple sources in order:
  #    a) env var (already set)           b) knoe-kdc-secrets k8s Secret
  #    c) prole-kdc-secrets k8s Secret    d) 1Password (OP_KDC_MASTER_PASSWORD_REF)
  #    e) PROLE_LOCAL_ADMIN_PASSWORD      f) die
  if [[ -z "$PROLE_KDC_MASTER_PASSWORD" ]]; then
    PROLE_KDC_MASTER_PASSWORD=$(get_secret_value "knoe-kdc-secrets" "master_password")
  fi
  if [[ -z "$PROLE_KDC_MASTER_PASSWORD" ]]; then
    # The live prole deployment may use a differently-named secret
    PROLE_KDC_MASTER_PASSWORD=$(get_secret_value "prole-kdc-secrets" "master_password")
    [[ -n "$PROLE_KDC_MASTER_PASSWORD" ]] && log "Resolved KDC master password from prole-kdc-secrets"
  fi
  if [[ -z "$PROLE_KDC_MASTER_PASSWORD" && -n "${OP_KDC_MASTER_PASSWORD_REF:-}" ]]; then
    log "Resolving KDC master password from 1Password (${OP_KDC_MASTER_PASSWORD_REF}) ..."
    PROLE_KDC_MASTER_PASSWORD=$(try_op_read "$OP_KDC_MASTER_PASSWORD_REF" || true)
    [[ -n "$PROLE_KDC_MASTER_PASSWORD" ]] && log "KDC master password resolved from 1Password"
  fi
  # Allow the UI-supplied PROLE_LOCAL_ADMIN_PASSWORD (DB master password field) to
  # override when the secret is absent or empty.
  if [[ -z "$PROLE_KDC_MASTER_PASSWORD" && -n "${PROLE_LOCAL_ADMIN_PASSWORD:-}" ]]; then
    PROLE_KDC_MASTER_PASSWORD="$PROLE_LOCAL_ADMIN_PASSWORD"
    log "Using PROLE_LOCAL_ADMIN_PASSWORD as PROLE_KDC_MASTER_PASSWORD"
  fi
  if [[ -z "$PROLE_KDC_MASTER_PASSWORD" ]]; then
    err "PROLE_KDC_MASTER_PASSWORD not found."
    err "The KDC master password must match the running KDC database and cannot be auto-generated."
    err "Resolve one of:"
    err "  1. Store in k8s secret:"
    err "       kubectl -n ${KNOE_KDC_NAMESPACE} patch secret knoe-kdc-secrets \\"
    err "         --type=merge -p '{\"data\":{\"master_password\":\"BASE64PW\"}}'"
    err "  2. Store in 1Password and set in conf/k3s.cfg:"
    err "       OP_KDC_MASTER_PASSWORD_REF = op://personal/<item>/password"
    err "     then: op run -- bash etc/init_knoe_users.sh --mode k3s initialize"
    err "  3. Export directly: export PROLE_KDC_MASTER_PASSWORD=<password>"
    err "  4. Check existing: kubectl -n ${KNOE_KDC_NAMESPACE} get secrets"
    exit 1
  fi

  if [[ -z "$KNOE_GUEST_PASSWORD" ]]; then
    KNOE_GUEST_PASSWORD=$(get_secret_value "knoe-kdc-secrets" "guest_password")
  fi
  if [[ -z "$KNOE_GUEST_PASSWORD" ]]; then
    KNOE_GUEST_PASSWORD=$(get_secret_value "prole-kdc-secrets" "guest_password")
  fi
  if [[ -z "$KNOE_GUEST_PASSWORD" && -n "${OP_KDC_GUEST_PASSWORD_REF:-}" ]]; then
    KNOE_GUEST_PASSWORD=$(try_op_read "$OP_KDC_GUEST_PASSWORD_REF" || true)
  fi
  if [[ -z "$KNOE_GUEST_PASSWORD" ]]; then
    KNOE_GUEST_PASSWORD=$(gen_password)
    log "Auto-generated guest password — storing in knoe-kdc-secrets ..."
    if kubectl -n "$KNOE_KDC_NAMESPACE" get secret knoe-kdc-secrets >/dev/null 2>&1; then
      local guest_b64
      guest_b64=$(printf '%s' "$KNOE_GUEST_PASSWORD" | base64 | tr -d '\n')
      kubectl -n "$KNOE_KDC_NAMESPACE" patch secret knoe-kdc-secrets \
        --type=merge \
        --patch "{\"data\":{\"guest_password\":\"${guest_b64}\"}}" >/dev/null
    else
      kubectl -n "$KNOE_KDC_NAMESPACE" create secret generic knoe-kdc-secrets \
        --from-literal=guest_password="$KNOE_GUEST_PASSWORD" \
        --dry-run=client -o yaml | kubectl apply -f - >/dev/null
    fi
  fi

  # 2. Locate (or bootstrap) KDC pod
  local kdc_pod
  kdc_pod=$(ensure_kdc_pod)
  log "KDC pod: $kdc_pod"

  # 2a. Pre-flight: verify the KDC database stash file is present and kadmin.local
  #     can access the database.  A missing stash means init_kdc.sh has not been
  #     run (or the pod restarted with ephemeral storage — EmptyDir volumes are
  #     wiped on pod restart).  Auto-recover by re-initialising the database.
  log "Verifying KDC database accessibility ..."
  if ! kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \
      sh -c 'kadmin.local -q listprincs' >/dev/null 2>&1; then
    log "KDC database not accessible — pod may have restarted with ephemeral storage."
    log "Re-initialising KDC database via init_kdc.sh ..."
    local init_kdc_script="$SCRIPT_DIR/init_kdc.sh"
    [[ -f "$init_kdc_script" ]] || die "Required script not found: $init_kdc_script"
    _resolve_kdc_registry
    LOCAL_REGISTRY_INTERNAL="${_kdc_reg_internal}" \
    PROLE_KDC_REGISTRY_INTERNAL="${_kdc_reg_internal}" \
    PROLE_KDC_REGISTRY_HOST="${_kdc_reg_host}" \
    PROLE_KDC_NAMESPACE="$KNOE_KDC_NAMESPACE" \
    PROLE_KDC_MASTER_PASSWORD="$PROLE_KDC_MASTER_PASSWORD" \
    PROLE_KDC_NAME="$KNOE_AUTH_DEPLOYMENT" \
      bash "$init_kdc_script" initialize >&2
    # Poll until the new pod's entrypoint finishes kdb5_util create.
    # The rollout completes before the in-container DB init finishes (no readiness probe).
    local _reinit_timeout=120 _reinit_interval=5 _reinit_elapsed=0
    log "Waiting up to ${_reinit_timeout}s for KDC database to become accessible ..."
    while (( _reinit_elapsed < _reinit_timeout )); do
      kdc_pod=$(get_kdc_pod)
      if [[ -n "$kdc_pod" ]] && kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \
          sh -c 'kadmin.local -q listprincs' >/dev/null 2>&1; then
        log "KDC database re-initialised successfully (pod: $kdc_pod)"
        break
      fi
      sleep "$_reinit_interval"
      _reinit_elapsed=$(( _reinit_elapsed + _reinit_interval ))
    done
    if (( _reinit_elapsed >= _reinit_timeout )); then
      die "KDC database still not accessible after ${_reinit_timeout}s (pod: ${kdc_pod:-<none>}).
  Check init_kdc.sh logs above for errors."
    fi
  else
    log "KDC database accessible"
  fi

  # 3. Create admin@PROLE.LOCAL (UI login with master password)
  local admin_princ="${KNOE_ADMIN_PRINCIPAL}@${PROLE_KDC_REALM}"
  if kdc_principal_exists "$kdc_pod" "$admin_princ"; then
    log "${admin_princ} already exists"
  else
    log "Creating ${admin_princ} ..."
    kdc_addprinc "$kdc_pod" "$admin_princ" "$PROLE_KDC_MASTER_PASSWORD"
  fi

  # 4. Create guest@PROLE.LOCAL (read-only)
  local guest_princ="guest@${PROLE_KDC_REALM}"
  if kdc_principal_exists "$kdc_pod" "$guest_princ"; then
    log "${guest_princ} already exists"
  else
    log "Creating ${guest_princ} ..."
    kdc_addprinc "$kdc_pod" "$guest_princ" "$KNOE_GUEST_PASSWORD"
  fi

  # 5. Ensure BOTH cross-realm trust principals exist in the MIT KDC
  #    Direction A: krbtgt/PROLE.ORG@KNOE.LOCAL  — cluster users → AD services
  #                 (created by init_kdc.sh PROLE_KDC_TRUST_REALM mechanism)
  #    Direction B: krbtgt/KNOE.LOCAL@PROLE.ORG  — AD users → cluster services ← THIS IS WHAT WE NEED
  local trust_princ_in="krbtgt/PROLE.ORG@${PROLE_KDC_REALM}"
  local trust_princ_out="krbtgt/${PROLE_KDC_REALM}@PROLE.ORG"
  local trust_shared_pw
  trust_shared_pw=$(kubectl -n "$KNOE_KDC_NAMESPACE" get secret knoe-kdc-secrets \
    -o jsonpath='{.data.trust_shared_password}' 2>/dev/null | b64_decode || true)
  if [[ -z "$trust_shared_pw" ]]; then
    warn "trust_shared_password not set in knoe-kdc-secrets; cross-realm trust principals will not be created"
    warn "Add trust_shared_password to knoe-kdc-secrets and re-run"
  else
    # Direction A (init_kdc.sh may already have created this)
    if kdc_principal_exists "$kdc_pod" "$trust_princ_in"; then
      log "${trust_princ_in} already exists"
    else
      log "Creating cross-realm principal ${trust_princ_in} ..."
      kdc_addprinc "$kdc_pod" "$trust_princ_in" "$trust_shared_pw"
    fi
    # Direction B — the principal that allows AD users to reach cluster services
    if kdc_principal_exists "$kdc_pod" "$trust_princ_out"; then
      log "${trust_princ_out} already exists"
    else
      log "Creating cross-realm principal ${trust_princ_out} ..."
      kdc_addprinc "$kdc_pod" "$trust_princ_out" "$trust_shared_pw"
    fi
  fi

  # 6. Create postgres service principal + export keytab.
  #    Always (re)create the SPN with a fresh random key — the old key is gone when
  #    the KDC pod restarts on ephemeral storage.  kadmin.local -q exits 0 even on
  #    failure, so we verify success by checking the keytab file was written.
  local pg_spn="postgres/knoe-db-rw.${KNOE_DB_NAMESPACE}.svc.cluster.local@${PROLE_KDC_REALM}"
  log "Creating/refreshing service principal ${pg_spn} ..."
  kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \
    sh -c "kadmin.local -q 'delprinc -force ${pg_spn}' 2>/dev/null; kadmin.local -q 'addprinc -randkey ${pg_spn}'"

  log "Exporting keytab for ${pg_spn} ..."
  kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \
    sh -c "rm -f /tmp/pg.keytab; kadmin.local -q 'ktadd -k /tmp/pg.keytab ${pg_spn}'"

  # kadmin.local -q exits 0 even on failure — verify the file was actually written
  kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \
    sh -c '[ -s /tmp/pg.keytab ]' \
    || die "Keytab export failed for ${pg_spn} — /tmp/pg.keytab is missing or empty. Check KDC logs."

  log "Storing keytab in Secret knoe-db-pg-keytab ..."
  local keytab_b64
  keytab_b64=$(kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \
    sh -c 'base64 /tmp/pg.keytab | tr -d "\n"')
  [[ -n "$keytab_b64" ]] || die "base64 encoding of keytab produced empty output."
  # Write secret into the DB namespace where CNPG mounts it
  kubectl -n "$KNOE_DB_NAMESPACE" create secret generic knoe-db-pg-keytab \
    --from-literal=pg.keytab="$(printf '%s' "$keytab_b64" | b64_decode)" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null
  log "knoe-db-pg-keytab stored in namespace ${KNOE_DB_NAMESPACE}"

  # 6. Patch CNPG cluster with managed.roles (idempotent merge)
  log "Patching CNPG cluster ${KNOE_DB_CLUSTER} managed.roles ..."
  kubectl -n "$KNOE_DB_NAMESPACE" patch cluster "$KNOE_DB_CLUSTER" \
    --type=merge --patch '{
      "spec": {
        "managed": {
          "roles": [
            {
              "name": "admin",
              "ensure": "present",
              "login": true,
              "superuser": true,
              "comment": "Kerberos admin principal — full cluster access"
            },
            {
              "name": "guest",
              "ensure": "present",
              "login": true,
              "superuser": false,
              "comment": "Kerberos guest principal — read-only access to demo schema"
            },
            {
              "name": "developer",
              "ensure": "present",
              "login": false,
              "superuser": false,
              "comment": "Developer group role — granted to knoe-system user accounts"
            }
          ]
        }
      }
    }'

  # 7. Locate CNPG primary and run SQL
  local primary
  primary=$(get_cnpg_primary)
  if [[ -z "$primary" ]]; then
    warn "CNPG primary pod not found — skipping SQL setup (run again once cluster is ready)"
  else
    log "CNPG primary pod: $primary"
    log "Setting up demo schema and role grants ..."
    cnpg_psql "$primary" <<'SQL'
-- demo schema for guest read-only access (evolves over time)
CREATE SCHEMA IF NOT EXISTS demo;

-- Ensure roles exist (CNPG managed.roles may not have reconciled yet)
DO $$
BEGIN
  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'admin') THEN
    CREATE ROLE admin LOGIN SUPERUSER;
  ELSE
    ALTER ROLE admin SUPERUSER LOGIN;
  END IF;
END $$;

DO $$
BEGIN
  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'guest') THEN
    CREATE ROLE guest LOGIN;
  END IF;
END $$;

DO $$
BEGIN
  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'developer') THEN
    CREATE ROLE developer NOLOGIN;
  END IF;
END $$;

-- guest: read-only on demo schema
GRANT USAGE ON SCHEMA demo TO guest;
GRANT SELECT ON ALL TABLES IN SCHEMA demo TO guest;
ALTER DEFAULT PRIVILEGES IN SCHEMA demo GRANT SELECT ON TABLES TO guest;

-- developer: read/write access on public and demo schemas
GRANT USAGE ON SCHEMA public TO developer;
GRANT USAGE ON SCHEMA demo TO developer;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO developer;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA demo TO developer;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO developer;
ALTER DEFAULT PRIVILEGES IN SCHEMA demo GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO developer;
SQL
    log "SQL setup complete"
  fi

  # 8. Create knoe.user schema tables and bootstrap users
  if [[ -n "$primary" ]]; then
    create_knoe_schema "$primary"
    provision_user "$primary" "chrisfu" "chrisfu@prole.org" "Chris Fu" "admin"
    # ron has no prole.org Google Workspace account; identity is ron@KNOE.LOCAL (Kerberos only)
    provision_user "$primary" "ron"     ""                  "Ron"      "developer"
  else
    warn "CNPG primary not found — skipping knoe.user schema and user provisioning"
  fi

  # 9. ArgoCD RBAC patch
  patch_argocd_rbac

  # 10. Gitea SPNEGO keytab (HTTP/git.prole.org@PROLE.ORG)
  ensure_gitea_spnego_keytab

  # 11. Gitea admin promotion
  # Set KNOE_ADMIN_EMAIL so promote_gitea_admin can update the UUID placeholder
  # email that Gitea assigns on SPNEGO auto-registration.
  # Must match the email passed to provision_user for KNOE_ADMIN_PRINCIPAL above.
  KNOE_ADMIN_EMAIL="${KNOE_ADMIN_EMAIL:-chrisfu@prole.org}"
  promote_gitea_admin

  # 12. GitLab admin promotion
  promote_gitlab_admin

  log "=== Initialization complete ==="
  log ""
  log "Next steps:"
  if [[ "$KNOE_DEPLOYMENT_MODE" != "k3d" ]]; then
    log "  1. KDC cross-realm principals: ensure krbtgt/PROLE.ORG@KNOE.LOCAL and krbtgt/KNOE.LOCAL@PROLE.ORG"
    log "     exist in the MIT KDC (created by init_kdc.sh or manually via kadmin.local)"
    log ""
    log "  2. Browser Kerberos SSO — deploy to ALL managed endpoints via Ansible (one-time, not per-user):"
    log "       ansible-playbook infrastructure/playbooks/workstation_kerberos.yml"
    log "     This deploys Chrome managed policy + krb5.conf pointing to ${KRB5_KDC:-myrddin.prole.org}."
    log "     New-hire laptops get it as part of standard provisioning.  No per-user browser configuration."
    log ""
    log "  3. Onboard each new engineer:"
    log "       op run -- bash etc/init_knoe_users.sh --mode k3s onboard <username> <email>"
    log "     Creates Kerberos principal, sets 1Password recovery credential, and sends invite."
    log ""
    log "  4. 1Password recovery credentials — run this script with 1Password authenticated:"
    log "       op run -- bash etc/init_knoe_users.sh --mode k3s initialize"
    log "     Or store a service account token first (see script header for setup instructions)."
  else
    log "  1. After admin logs in to Gitea/GitLab for the first time, re-run: $0 initialize"
  fi
}

# ── knoe.user schema + user provisioning ──────────────────────────────────

create_knoe_schema() {
  local primary="$1"
  log "Creating knoe.user schema tables (idempotent)..."
  cnpg_psql "$primary" <<'SQL'
CREATE TABLE IF NOT EXISTS knoe.user (
    id             SERIAL PRIMARY KEY,
    username       TEXT NOT NULL UNIQUE,
    realm          TEXT NOT NULL DEFAULT 'KNOE.LOCAL',
    email          TEXT,
    display_name   TEXT,
    tenant_realm   TEXT,
    is_realm_admin BOOLEAN DEFAULT false,
    created_at     TIMESTAMPTZ DEFAULT now(),
    updated_at     TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE IF NOT EXISTS knoe.user_role (
    user_id    INT NOT NULL REFERENCES knoe.user(id) ON DELETE CASCADE,
    role       TEXT NOT NULL,
    granted_at TIMESTAMPTZ DEFAULT now(),
    PRIMARY KEY (user_id, role)
);
GRANT SELECT, INSERT, UPDATE ON knoe.user      TO knoe;
GRANT SELECT, INSERT, UPDATE ON knoe.user_role TO knoe;
GRANT USAGE, SELECT ON SEQUENCE knoe.user_id_seq TO knoe;
SQL
  log "knoe.user schema ready."
}

# provision_user <primary_pod> <username> <email> <display_name> <role>
# role: admin | developer | guest  (default: developer)
# Idempotent — safe to re-run.
provision_user() {
  local primary="$1" username="$2" email="${3:-}" display_name="${4:-$2}" role="${5:-developer}"

  # 1. Kerberos principal (randkey — user sets own password via kpasswd / kadmin)
  local user_princ="${username}@${PROLE_KDC_REALM}"
  if [[ -n "$email" ]]; then
    log "Provisioning user: ${username}  kerberos=${user_princ}  email=${email}  role=${role}"
  else
    log "Provisioning user: ${username}  kerberos=${user_princ}  (no contact email)  role=${role}"
  fi
  local kdc_pod
  kdc_pod=$(get_kdc_pod)
  if [[ -n "$kdc_pod" ]]; then
    if kdc_principal_exists "$kdc_pod" "$user_princ"; then
      log "  Kerberos: ${user_princ} already exists"
    else
      log "  Kerberos: creating ${user_princ} ..."
      kdc_addprinc_randkey "$kdc_pod" "$user_princ"
    fi
  else
    warn "  No KDC pod — skipping Kerberos principal for ${username}"
  fi

  # 2. PostgreSQL role (GSS-API: username@PROLE.LOCAL → pg role 'username')
  cnpg_psql "$primary" -v username="$username" -v role="$role" <<SQL
DO \$\$
BEGIN
  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${username}') THEN
    CREATE ROLE ${username} LOGIN;
  END IF;
END \$\$;
GRANT ${role} TO ${username};
SQL

  # 3. knoe.user record (upsert)
  cnpg_psql "$primary" <<SQL
INSERT INTO knoe.user (username, realm, email, display_name)
  VALUES ('${username}', '${PROLE_KDC_REALM}', NULLIF('${email}', ''), '${display_name}')
  ON CONFLICT (username) DO UPDATE SET
    email        = EXCLUDED.email,
    display_name = EXCLUDED.display_name,
    updated_at   = now();
INSERT INTO knoe.user_role (user_id, role)
  SELECT id, '${role}' FROM knoe.user WHERE username = '${username}'
  ON CONFLICT DO NOTHING;
SQL

  log "  User ${username} provisioned."
}

# ── ArgoCD RBAC ────────────────────────────────────────────────────────────

patch_argocd_rbac() {
  if ! kubectl -n "$ARGOCD_NAMESPACE" get configmap argocd-rbac-cm >/dev/null 2>&1; then
    warn "argocd-rbac-cm not found in namespace ${ARGOCD_NAMESPACE}; skipping ArgoCD RBAC patch"
    return 0
  fi

  local admin_line="g, ${KNOE_ADMIN_PRINCIPAL}, role:admin"
  local current_csv
  current_csv=$(kubectl -n "$ARGOCD_NAMESPACE" get configmap argocd-rbac-cm \
    -o jsonpath='{.data.policy\.csv}' 2>/dev/null || true)

  if printf '%s' "$current_csv" | grep -qF "$admin_line"; then
    log "ArgoCD RBAC already contains admin entry"
    return 0
  fi

  local new_csv="${current_csv}"$'\n'"${admin_line}"
  local escaped_csv
  escaped_csv=$(python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<< "$new_csv")

  kubectl -n "$ARGOCD_NAMESPACE" patch configmap argocd-rbac-cm \
    --type=merge \
    --patch "{\"data\":{\"policy.csv\":${escaped_csv}}}" >/dev/null
  log "ArgoCD RBAC: added '${admin_line}'"
}

# ── Gitea SPNEGO keytab ─────────────────────────────────────────────────────

# Ensures the Secret gitea-krb5-keytab exists in GITEA_NAMESPACE.
# If samba-tool is available (i.e. running on myrddin), creates the service
# account, SPN, and keytab automatically.  Otherwise prints the manual steps.
ensure_gitea_spnego_keytab() {
  local spnego_principal="HTTP/${GITEA_SPNEGO_HOST}"
  local spn="${spnego_principal}@${GITEA_KRB5_AD_REALM}"
  local keytab_tmp="/tmp/gitea-http-$$.keytab"

  log "=== Gitea SPNEGO keytab (${GITEA_NAMESPACE}/gitea-krb5-keytab) ==="

  if kubectl -n "$GITEA_NAMESPACE" get secret gitea-krb5-keytab >/dev/null 2>&1; then
    log "gitea-krb5-keytab already exists in namespace ${GITEA_NAMESPACE} — OK"
    return 0
  fi

  log "gitea-krb5-keytab not found — attempting to provision ..."

  if ! command -v samba-tool >/dev/null 2>&1; then
    warn "samba-tool not available — run the following on myrddin.prole.org as root:"
    warn "  samba-tool user create ${GITEA_KRB5_AD_USER} --random-password 2>/dev/null || true"
    warn "  samba-tool spn add ${spnego_principal} ${GITEA_KRB5_AD_USER}"
    warn "  samba-tool domain exportkeytab /tmp/http.keytab --principal ${spnego_principal}"
    warn "  klist -k /tmp/http.keytab  # verify"
    warn "  kubectl -n ${GITEA_NAMESPACE} create secret generic gitea-krb5-keytab \\"
    warn "    --from-file=http.keytab=/tmp/http.keytab"
    warn "  rm -f /tmp/http.keytab"
    warn "  kubectl -n ${GITEA_NAMESPACE} rollout restart deployment gitea-spnego-proxy"
    return 0
  fi

  log "Creating AD service account '${GITEA_KRB5_AD_USER}' (idempotent) ..."
  samba-tool user create "$GITEA_KRB5_AD_USER" --random-password 2>/dev/null \
    && log "Created ${GITEA_KRB5_AD_USER}" \
    || log "${GITEA_KRB5_AD_USER} already exists"

  log "Adding SPN ${spnego_principal} → ${GITEA_KRB5_AD_USER} ..."
  samba-tool spn add "$spnego_principal" "$GITEA_KRB5_AD_USER" \
    || warn "SPN add returned non-zero (may already exist)"

  log "Exporting keytab for ${spn} ..."
  rm -f "$keytab_tmp"
  samba-tool domain exportkeytab "$keytab_tmp" --principal "$spnego_principal"

  if [[ ! -s "$keytab_tmp" ]]; then
    rm -f "$keytab_tmp"
    die "Keytab export failed — ${keytab_tmp} is missing or empty. Check samba-tool output."
  fi

  log "Storing keytab in Secret gitea-krb5-keytab ..."
  kubectl -n "$GITEA_NAMESPACE" create secret generic gitea-krb5-keytab \
    --from-file=http.keytab="$keytab_tmp" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null
  rm -f "$keytab_tmp"
  log "gitea-krb5-keytab provisioned — ${spn}"

  log "Restarting gitea-spnego-proxy to load new keytab ..."
  kubectl -n "$GITEA_NAMESPACE" rollout restart deployment gitea-spnego-proxy 2>/dev/null || true
}

# ── Gitea admin promotion ───────────────────────────────────────────────────

# ── Gitea credential management (1Password-backed) ────────────────────────
#
# Design: Gitea passwords are vault credentials — strong, random, generated
# by 1Password, never logged, never stored in k8s secrets. Primary auth is
# always Kerberos SPNEGO; the password is an emergency recovery path only.
# API tokens (automation) are stored in k8s secret and are revocable.

# gitea_op_item_title <username> — canonical 1Password item title
gitea_op_item_title() {
  printf 'Gitea — %s@%s' "$1" "${GITEA_HOST}"
}

# gitea_ensure_password <username>
# Prints a strong password to stdout (never to stderr/log).
# If a 1Password item exists: retrieves it (idempotent re-runs safe).
# If not: creates a new item with a generated password.
# Returns 1 if op CLI is unavailable or unauthenticated.
gitea_ensure_password() {
  local username="$1"
  local title
  title=$(gitea_op_item_title "$username")

  # Establish 1Password session before any op commands.
  _op_ensure_auth || return 1

  # Retrieve existing password (silent on error = item not found yet)
  local pw
  pw=$(op item get "$title" --vault="${GITEA_OP_VAULT}" --fields=password 2>/dev/null) || true
  if [[ -n "$pw" ]]; then
    log "  Gitea: retrieved existing credential for '${username}' from 1Password (vault: ${GITEA_OP_VAULT})" >&2
    printf '%s' "$pw"
    return 0
  fi

  # Generate and store a new credential in 1Password.
  # --generate-password uses 1Password's generator — no plaintext secret
  # ever appears in the process environment or shell history.
  log "  Gitea: generating credential for '${username}' → 1Password vault '${GITEA_OP_VAULT}'" >&2
  op item create \
    --category=Login \
    --title="$title" \
    --vault="${GITEA_OP_VAULT}" \
    --url="https://${GITEA_HOST}" \
    --generate-password='letters,digits,32' \
    "username=${username}" \
    "notesPlain=Generated by init_knoe_users.sh. Primary auth is Kerberos SPNEGO — this password is for emergency recovery only." \
    >/dev/null 2>&1 || { warn "  Gitea: failed to create 1Password item '${title}' (op signed in?)"; return 1; }

  pw=$(op item get "$title" --vault="${GITEA_OP_VAULT}" --fields=password 2>/dev/null) || true
  if [[ -z "$pw" ]]; then
    err "  Gitea: 1Password item created but password could not be retrieved"
    return 1
  fi
  log "  Gitea: credential stored as '${title}' in vault '${GITEA_OP_VAULT}'" >&2
  printf '%s' "$pw"
}

# gitea_set_password <username> <password>
# Sets the Gitea user's password via kubectl exec (no prior Gitea auth needed).
gitea_set_password() {
  local username="$1" pw="$2"
  local pod
  pod=$(kubectl -n "$GITEA_NAMESPACE" get pods \
    -l "app.kubernetes.io/name=gitea" \
    --field-selector=status.phase=Running \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  [[ -n "$pod" ]] || { err "  Gitea: no running pod in namespace ${GITEA_NAMESPACE}"; return 1; }
  # gitea refuses to run as root; su to the git user (uid 1000).
  # Capture stderr so we can surface it on failure without ever logging the password.
  local _out
  if ! _out=$(kubectl -n "$GITEA_NAMESPACE" exec "$pod" -c gitea -- \
      su git -s /bin/bash -c \
      "gitea admin user change-password --username '${username}' --password '${pw}'" \
      2>&1); then
    warn "  Gitea: change-password failed for '${username}': ${_out}"
    return 1
  fi
}

# gitea_api_token <username> <password> [token_name]
# Obtains a fresh Gitea API token via REST basic-auth.
# Deletes any existing token with the same name first (idempotency).
# Prints the token sha1 to stdout; never logs it.
gitea_api_token() {
  local username="$1" pw="$2" token_name="${3:-knoe-installer}"
  # Remove stale token if present (ignore errors — may not exist)
  curl -s -o /dev/null \
    -u "${username}:${pw}" \
    -X DELETE \
    "https://${GITEA_HOST}/api/v1/users/${username}/tokens/${token_name}" 2>/dev/null || true
  # Capture body + HTTP status separately so we can log failures without
  # exposing the password.  The password never appears in the log/stderr.
  # Gitea ≥1.22 requires explicit scopes; request all admin + user scopes.
  local resp http_code
  resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \
    -u "${username}:${pw}" \
    -X POST \
    -H "Content-Type: application/json" \
    -d "{\"name\":\"${token_name}\",\"scopes\":[\"read:admin\",\"write:admin\",\"read:user\",\"write:user\",\"read:issue\",\"write:issue\",\"read:repository\",\"write:repository\"]}" \
    "https://${GITEA_HOST}/api/v1/users/${username}/tokens" 2>/dev/null) || true
  http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true)
  resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__')
  if [[ "$http_code" != "201" ]]; then
    warn "  Gitea: token POST returned HTTP ${http_code:-???} for '${username}' (body: ${resp:0:120})"
  fi
  # Extract sha1 from {"id":...,"name":...,"sha1":"<token>",...}
  printf '%s' "$resp" | grep -o '"sha1":"[^"]*"' | cut -d'"' -f4
}

# gitea_helm_admin_token [token_name]
# Reads the Helm bootstrap admin credentials from the Gitea Deployment's
# configure-gitea init container environment variables (GITEA_ADMIN_USERNAME /
# GITEA_ADMIN_PASSWORD).  The Helm chart stores these as plain values in the
# pod spec — there is no k8s Secret holding the admin password.
#
# This works even when the gitea admin CLI is broken (e.g. wrong DB hostname
# in app.ini) because the request goes through the running web server which
# has the correct GITEA__database__HOST env var.
# Prints the token sha1 to stdout; returns 1 on failure.
gitea_helm_admin_token() {
  local token_name="${1:-knoe-installer-helm}"
  local release="${GITEA_HELM_RELEASE:-gitea}"

  # The Helm chart renders admin credentials as plain values (not secretKeyRefs)
  # in the configure-gitea init container.  However, if the Deployment has been
  # updated (e.g. a failed upgrade) but the new pod never became Ready, the
  # RUNNING pod still carries the old init container's password.
  #
  # Strategy: build a list of candidate passwords by reading from:
  #   1. The ReplicaSet that owns the currently RUNNING pod (highest priority —
  #      this is what was actually applied to the live admin account)
  #   2. The current Deployment spec (fallback — used when everything is in sync)
  local admin_user admin_pw=""

  admin_user=$(kubectl -n "$GITEA_NAMESPACE" get deployment "$release" \
    -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="configure-gitea")].env[?(@.name=="GITEA_ADMIN_USERNAME")].value}' \
    2>/dev/null || true)
  # Env-var override takes precedence (useful for testing / manual bootstrap)
  admin_user="${GITEA_HELM_ADMIN_USER:-${admin_user}}"

  if [[ -z "$admin_user" ]]; then
    warn "  Gitea: could not determine Helm admin username from Deployment '${release}'"
    return 1
  fi

  # Gather candidate passwords: running-pod RS first, then current deployment.
  local _running_rs _rs_pw _deploy_pw _tok
  _running_rs=$(kubectl -n "$GITEA_NAMESPACE" get pod \
    -l "app.kubernetes.io/name=${release}" \
    --field-selector=status.phase=Running \
    -o jsonpath='{.items[0].metadata.ownerReferences[?(@.kind=="ReplicaSet")].name}' \
    2>/dev/null || true)
  _rs_pw=""
  if [[ -n "$_running_rs" ]]; then
    _rs_pw=$(kubectl -n "$GITEA_NAMESPACE" get rs "$_running_rs" \
      -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="configure-gitea")].env[?(@.name=="GITEA_ADMIN_PASSWORD")].value}' \
      2>/dev/null || true)
  fi
  _deploy_pw=$(kubectl -n "$GITEA_NAMESPACE" get deployment "$release" \
    -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="configure-gitea")].env[?(@.name=="GITEA_ADMIN_PASSWORD")].value}' \
    2>/dev/null || true)

  local tok=""
  for _pw in "$_rs_pw" "$_deploy_pw"; do
    [[ -z "$_pw" ]] && continue
    _tok=$(gitea_api_token "$admin_user" "$_pw" "$token_name")
    if [[ -n "$_tok" ]]; then
      tok="$_tok"
      break
    fi
  done

  if [[ -z "$tok" ]]; then
    warn "  Gitea: could not obtain API token for Helm admin '${admin_user}'"
    return 1
  fi
  log "  Gitea: obtained bootstrap token from Helm admin '${admin_user}'" >&2
  printf '%s' "$tok"
}

# gitea_api_set_password <admin_token> <username> <new_password>
# Changes a user's password via the Gitea admin REST API.
# Uses the running web server (not the CLI), so it works even when app.ini
# has a wrong database hostname (the server overrides via GITEA__ env vars).
#
# NOTE: Gitea's AdminEditUser PATCH requires 'email' alongside login_name/source_id.
# Without it the server returns 200 but silently discards the password field.
# We fetch the current email first; if that fails we fall back to a configured default.
gitea_api_set_password() {
  local token="$1" username="$2" pw="$3"

  # Fetch the user's current email — required by Gitea's admin PATCH endpoint.
  local user_resp user_email
  user_resp=$(curl -s \
    -H "Authorization: token ${token}" \
    "https://${GITEA_HOST}/api/v1/users/${username}" 2>/dev/null) || true
  user_email=$(printf '%s' "$user_resp" | grep -o '"email":"[^"]*"' | cut -d'"' -f4 || true)

  # If the stored email is a UUID placeholder (SPNEGO auto-reg), substitute a real one.
  if [[ -z "$user_email" ]] || printf '%s' "$user_email" | grep -qE '^[0-9a-f-]{36}@localhost$'; then
    user_email="${KNOE_ADMIN_EMAIL:-${username}@${GITEA_HOST}}"
  fi

  local resp http_code
  resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \
    -X PATCH \
    -H "Authorization: token ${token}" \
    -H "Content-Type: application/json" \
    -d "{\"login_name\":\"${username}\",\"source_id\":0,\"password\":\"${pw}\",\"email\":\"${user_email}\",\"must_change_password\":false}" \
    "https://${GITEA_HOST}/api/v1/admin/users/${username}" 2>/dev/null) || true
  http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true)
  resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__')

  if [[ "$http_code" =~ ^2 ]]; then
    return 0
  else
    warn "  Gitea: admin API set-password returned HTTP ${http_code:-???} for '${username}': ${resp:0:200}"
    return 1
  fi
}

promote_gitea_admin() {
  local token="$GITEA_ADMIN_TOKEN"

  # 1. Try GITEA_ADMIN_TOKEN from env or persisted k8s secret
  if [[ -z "$token" ]]; then
    token=$(kubectl -n "$GITEA_NAMESPACE" get secret gitea-admin-token \
      -o jsonpath='{.data.token}' 2>/dev/null | b64_decode || true)
  fi

  # 2. Bootstrap via the Helm admin account (k8s secret → REST API token).
  #    This path works even when the gitea admin CLI is broken (app.ini has wrong
  #    DB hostname) because REST calls go through the running server which has the
  #    correct GITEA__database__HOST env var.
  local _helm_token=""
  if [[ -z "$token" ]]; then
    if _helm_token=$(gitea_helm_admin_token "knoe-installer-helm"); then
      token="$_helm_token"
    fi
  fi

  # 3. Try 1Password → set password via CLI → exchange for token.
  #    Kept as fallback for when the Helm admin secret is absent (e.g. non-Helm deploy).
  if [[ -z "$token" ]]; then
    local _pw=""
    if _pw=$(gitea_ensure_password "${KNOE_ADMIN_PRINCIPAL}"); then
      log "Gitea: setting password for '${KNOE_ADMIN_PRINCIPAL}' via kubectl exec ..."
      if gitea_set_password "${KNOE_ADMIN_PRINCIPAL}" "$_pw"; then
        token=$(gitea_api_token "${KNOE_ADMIN_PRINCIPAL}" "$_pw" "knoe-installer")
        [[ -n "$token" ]] && log "Gitea: API token obtained for '${KNOE_ADMIN_PRINCIPAL}'"
      else
        warn "Gitea: CLI password change failed (DB host mismatch in app.ini?)"
      fi
      _pw=""
    fi
  fi

  # 4. Last resort: kubectl exec generate-access-token (CLI path, may fail if DB broken).
  if [[ -z "$token" ]]; then
    local _gitea_pod
    _gitea_pod=$(kubectl -n "$GITEA_NAMESPACE" get pods \
      -l "app.kubernetes.io/name=gitea" \
      --field-selector=status.phase=Running \
      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
    if [[ -n "$_gitea_pod" ]]; then
      log "Gitea: attempting token generation via kubectl exec (last resort) ..."
      local _u
      for _u in "gitea_admin" "${KNOE_ADMIN_PRINCIPAL}"; do
        token=$(kubectl -n "$GITEA_NAMESPACE" exec "$_gitea_pod" -c gitea -- \
          su git -s /bin/bash -c \
          "gitea admin user generate-access-token --username ${_u} --token-name knoe-installer --scopes write:admin,read:user --raw 2>/dev/null" \
          2>/dev/null | tail -1 || true)
        [[ -n "$token" ]] && { log "Gitea: token generated for user '${_u}'"; break; }
      done
    fi
  fi

  if [[ -z "$token" ]]; then
    warn "No Gitea admin token available — skipping admin promotion"
    warn "  Ensure 'op signin' is active and the Helm admin secret exists, then re-run."
    warn "  Or set GITEA_ADMIN_TOKEN and re-run."
    return 0
  fi

  # Persist token in k8s secret for automation reuse (tokens are revocable)
  kubectl -n "$GITEA_NAMESPACE" create secret generic gitea-admin-token \
    --from-literal=token="$token" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null 2>&1 || true

  local http_code resp
  # Check whether the Gitea account for KNOE_ADMIN_PRINCIPAL exists yet.
  # Gitea auto-registers users on first SPNEGO login via the reverse proxy.
  resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \
    -H "Authorization: token ${token}" \
    "https://${GITEA_HOST}/api/v1/users/${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null || true)
  http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true)
  resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__')

  log "  Gitea: user check for '${KNOE_ADMIN_PRINCIPAL}' → HTTP ${http_code:-???}"

  if [[ "$http_code" == "404" ]]; then
    warn "Gitea: user '${KNOE_ADMIN_PRINCIPAL}' not found — SPNEGO auto-registration not yet triggered"
    warn "  Visit https://${GITEA_HOST} with a valid Kerberos ticket, then re-run: $0 initialize"
    return 0
  fi

  # Fetch existing email — Gitea's edit-user API requires it to avoid resetting.
  # Prefer GITEA_ADMIN_EMAIL (may be set by caller or env), then the current
  # Gitea value (skip UUID placeholders from SPNEGO auto-registration),
  # then fall back to the user's configured email from the users list.
  local existing_email current_email
  current_email=$(printf '%s' "$resp" | grep -o '"email":"[^"]*"' | cut -d'"' -f4 || true)
  if [[ -n "${GITEA_ADMIN_EMAIL:-}" ]]; then
    existing_email="${GITEA_ADMIN_EMAIL}"
  elif [[ -n "$current_email" ]] && ! printf '%s' "$current_email" | grep -qE '^[0-9a-f-]{36}@localhost$'; then
    existing_email="$current_email"
  else
    # Current email is a UUID placeholder; use provisioned email if available,
    # otherwise fall back to a reasonable default.
    existing_email="${KNOE_ADMIN_EMAIL:-${KNOE_ADMIN_PRINCIPAL}@${GITEA_HOST}}"
  fi

  # Admin promotion — sets admin:true.  Must happen BEFORE the recovery password
  # is set, because Gitea's AdminEditUser with source_id:0 on a SPNEGO-registered
  # user can reset the password field as a side effect of re-initialising the auth
  # record.  Setting the password last guarantees it survives the promotion PATCH.
  resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \
    -X PATCH \
    -H "Authorization: token ${token}" \
    -H "Content-Type: application/json" \
    -d "{\"admin\":true,\"login_name\":\"${KNOE_ADMIN_PRINCIPAL}\",\"source_id\":0,\"email\":\"${existing_email}\"}" \
    "https://${GITEA_HOST}/api/v1/admin/users/${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null || true)
  http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true)
  resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__')

  if [[ "$http_code" =~ ^2 ]]; then
    log "Gitea: '${KNOE_ADMIN_PRINCIPAL}' promoted to admin"
  else
    warn "Gitea admin promotion returned HTTP ${http_code:-???} (${resp:0:200})"
  fi

  # Set recovery password AFTER admin promotion.  The promotion PATCH (source_id:0)
  # can reset the password as a side effect on SPNEGO-registered users; setting it
  # last ensures 1Password and Gitea agree on the same value.
  # Skipped gracefully if op is unavailable (automated CI, no 1Password session).
  if _op_ensure_auth 2>/dev/null; then
    local _pw=""
    if _pw=$(gitea_ensure_password "${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null); then
      log "Gitea: setting recovery password for '${KNOE_ADMIN_PRINCIPAL}' via admin REST API ..."
      gitea_api_set_password "$token" "${KNOE_ADMIN_PRINCIPAL}" "$_pw" \
        && log "Gitea: recovery password set for '${KNOE_ADMIN_PRINCIPAL}'" \
        || true
      _pw=""
    else
      warn "  Gitea: 1Password unavailable (op not signed in?) — skipping recovery password set"
      warn "    Run 'op signin' on $(hostname) and re-run to store recovery credentials"
    fi
  fi
}

# ── GitLab admin promotion ─────────────────────────────────────────────────

promote_gitlab_admin() {
  local token="$GITLAB_ADMIN_TOKEN"
  if [[ -z "$token" ]]; then
    warn "No GITLAB_ADMIN_TOKEN (personal access token) available — skipping GitLab admin promotion"
    warn "  Set GITLAB_ADMIN_TOKEN and re-run, or use:"
    warn "  kubectl exec -n ${GITLAB_NAMESPACE} <rails-pod> -- gitlab-rails runner \\"
    warn "    \"User.find_by_username('${KNOE_ADMIN_PRINCIPAL}')&.update(admin: true)\""
    return 0
  fi

  # Look up user ID by username
  local user_json user_id
  user_json=$(curl -s \
    -H "PRIVATE-TOKEN: ${token}" \
    "https://${GITLAB_HOST}/api/v4/users?username=${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null || true)
  user_id=$(python3 -c \
    'import sys,json; u=json.load(sys.stdin); print(u[0]["id"] if u else "")' \
    <<< "$user_json" 2>/dev/null || true)

  if [[ -z "$user_id" ]]; then
    warn "GitLab: user '${KNOE_ADMIN_PRINCIPAL}' not found (must log in first)"
    warn "  Re-run '$0 initialize' after first login"
    return 0
  fi

  local http_code
  http_code=$(curl -s -o /dev/null -w '%{http_code}' \
    -X PUT \
    -H "PRIVATE-TOKEN: ${token}" \
    -H "Content-Type: application/json" \
    -d '{"admin":true}' \
    "https://${GITLAB_HOST}/api/v4/users/${user_id}" 2>/dev/null || true)

  if [[ "$http_code" =~ ^2 ]]; then
    log "GitLab: '${KNOE_ADMIN_PRINCIPAL}' promoted to admin"
  else
    warn "GitLab admin promotion returned HTTP ${http_code}"
  fi
}

# ── Status ─────────────────────────────────────────────────────────────────

show_status() {
  log "=== KDC Principals ==="
  local kdc_pod
  kdc_pod=$(get_kdc_pod)
  if [[ -n "$kdc_pod" ]]; then
    kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \
      kadmin.local -q "list_principals" 2>/dev/null \
      | grep -E "^(admin|guest|developer|postgres/|krbtgt/)" || true
  else
    warn "No KDC pod found"
  fi

  log ""
  log "=== CNPG Managed Roles ==="
  kubectl -n "$KNOE_DB_NAMESPACE" get cluster "$KNOE_DB_CLUSTER" \
    -o jsonpath='{.spec.managed.roles}' 2>/dev/null \
    | python3 -m json.tool 2>/dev/null || warn "Could not read managed.roles"

  log ""
  log "=== PostgreSQL Roles ==="
  local primary
  primary=$(get_cnpg_primary)
  if [[ -n "$primary" ]]; then
    cnpg_psql "$primary" -c "\\du admin guest developer" 2>/dev/null || true
  else
    warn "No CNPG primary pod found"
  fi

  log ""
  log "=== Keytab Secrets ==="
  if kubectl -n "$KNOE_DB_NAMESPACE" get secret knoe-db-pg-keytab >/dev/null 2>&1; then
    log "knoe-db-pg-keytab exists in namespace ${KNOE_DB_NAMESPACE}"
  else
    warn "knoe-db-pg-keytab NOT FOUND in namespace ${KNOE_DB_NAMESPACE}"
  fi
  if kubectl -n "$GITEA_NAMESPACE" get secret gitea-krb5-keytab >/dev/null 2>&1; then
    log "gitea-krb5-keytab exists in namespace ${GITEA_NAMESPACE} (HTTP/${GITEA_SPNEGO_HOST}@${GITEA_KRB5_AD_REALM})"
  else
    warn "gitea-krb5-keytab NOT FOUND in namespace ${GITEA_NAMESPACE}"
  fi

  log ""
  log "=== ArgoCD RBAC ==="
  kubectl -n "$ARGOCD_NAMESPACE" get configmap argocd-rbac-cm \
    -o jsonpath='{.data.policy\.csv}' 2>/dev/null || warn "argocd-rbac-cm not found"
}

# ── Cleanup ────────────────────────────────────────────────────────────────

cleanup() {
  warn "cleanup removes the pg keytab secret and clears managed.roles from CNPG."
  warn "KDC principals are NOT deleted (use kadmin.local to remove them manually)."

  kubectl -n "$KNOE_DB_NAMESPACE" delete secret knoe-db-pg-keytab \
    --ignore-not-found >/dev/null && log "Deleted knoe-db-pg-keytab"

  kubectl -n "$KNOE_DB_NAMESPACE" patch cluster "$KNOE_DB_CLUSTER" \
    --type=merge --patch '{"spec":{"managed":null}}' >/dev/null 2>&1 \
    && log "Cleared managed.roles from cluster ${KNOE_DB_CLUSTER}" || true
}

# ── Dispatch ───────────────────────────────────────────────────────────────

case "$ACTION" in
  initialize|init) initialize ;;
  status)          show_status ;;
  cleanup)         cleanup ;;
  *)
    err "Unknown action: ${ACTION}"
    err "Usage: $(basename "$0") [initialize|status|cleanup]"
    exit 1
    ;;
esac