prole/etc/onboard_tenant.sh

#!/usr/bin/env bash
# etc/onboard_tenant.sh — Tenant secret provisioning and Kubernetes wiring
#
# Usage:
#   ./etc/onboard_tenant.sh provision <tenant-id> [OPTIONS]
#   ./etc/onboard_tenant.sh apply     <tenant-id> [OPTIONS]
#   ./etc/onboard_tenant.sh rotate    <tenant-id> [OPTIONS]
#   ./etc/onboard_tenant.sh status    <tenant-id>
#
# Subcommands:
#   provision   Create 1Password vault, generate all auto-generatable secrets,
#               store PLACEHOLDERs for items that need external input (Google OAuth,
#               Samba keytab). For 'tenant' type: fully automated, no PLACEHOLDERs.
#
#   apply       Read all secrets from the 1P vault and create/update Kubernetes
#               secrets in the tenant namespace. Fails if any PLACEHOLDERs remain.
#
#   rotate      Post-install password rotation. For each user-rotatable secret,
#               offers keep / enter custom / regenerate. Updates 1P + k8s + DB.
#
#   status      Print a table of what is provisioned vs. pending in the 1P vault.
#
# Options:
#   --domain DOMAIN             Tenant's primary domain (e.g. prole.org). Defaults to tenant-id.
#   --type tenant|enterprise    Tenant type. Default: enterprise.
#     tenant      Hosted tenant: uses knoe-auth at api.knoe.dev/auth, has data but no
#                 infrastructure. Keytab bootstrapped by initContainer. Fully automated.
#     enterprise  Brings own Kerberos infrastructure (Samba AD / MIT KDC) and a domain.
#                 knoe-auth integrates with their KDC. Requires Google OAuth + keytab
#                 from external AD. PLACEHOLDERs remain until human steps complete.
#   --context CTX           kubectl context to use. Default: current context.
#   --namespace NS          Kubernetes namespace for knoe-auth. Default: knoe-system.
#   --db-host HOST          PostgreSQL host for ALTER ROLE during rotate. Default: pg.<domain>.
#   --db-context CTX        kubectl context of the DB cluster. Default: same as --context.
#   --db-namespace NS       Namespace where the CNPG cluster lives. Default: knoe-db.
#   --kdc-host HOST         SSH host for Samba AD / KDC keytab extraction (enterprise only).
#                           Default: myrddin.<domain>.
#   --ci-google-client ID   Shared CI Google OAuth client ID (tenant type only).
#   --ci-google-secret SEC  Shared CI Google OAuth client secret (tenant type only).
#   --yes                   Non-interactive: accept suggested values without prompting.
#
# 1Password vault layout:  "Knoe Tenant - <tenant-id>"
#   knoe-db             Login    — DB role 'knoe' password + superuser password + URL
#   knoe-auth           Password — session secret
#   knoe-oidc           Password — RS256 signing key (base64 PEM)
#   knoe-google-tenant  Login    — per-tenant Google OAuth client (enrollment flow)
#   knoe-kerberos       Note     — KDC master/admin passwords + HTTP keytab (base64)
#
# Kubernetes secrets created (namespace: knoe-system):
#   knoe-db-user            CNPG bootstrap secret (username + password for 'knoe' role)
#   knoe-auth-db            jdbc URL + user + password
#   knoe-auth-secrets       sessionSecret
#   knoe-auth-oidc-signing  signing-key
#   knoe-auth-google-<tid>  client_id + client_secret  (enrollment Google OAuth)
#   knoe-auth-http-keytab   http.keytab file
#   knoe-kdc-secrets        master_password + admin_password  (in-cluster KDC, tenant type)
#
# For enterprise tenants: after 'provision', the status command will show which items
# need external completion (Google Cloud Console, Samba AD keytab) before 'apply'.
#
# Rotate targets (prompted in order):
#   1. knoe DB role password   — ALTER ROLE knoe + k8s secret + 1P
#   2. session secret          — k8s secret + 1P (no DB op needed)
#   3. KDC master password     — k8s secret + 1P (in-cluster KDC restart required)
#   4. KDC admin password      — k8s secret + 1P

set -euo pipefail

# ── Constants ──────────────────────────────────────────────────────────────────

PLACEHOLDER="__PLACEHOLDER__"
VAULT_PREFIX="Knoe Tenant - "
ITEM_DB="knoe-db"
ITEM_AUTH="knoe-auth"
ITEM_OIDC="knoe-oidc"
ITEM_GOOGLE="knoe-google-tenant"
ITEM_KERBEROS="knoe-kerberos"

# ── Logging ────────────────────────────────────────────────────────────────────

_info()  { echo "  [onboard] $*"; }
_ok()    { echo "  [onboard] ✓ $*"; }
_warn()  { echo "  [onboard] ⚠ $*" >&2; }
_err()   { echo "  [onboard] ✗ $*" >&2; }
_die()   { _err "$*"; exit 1; }
_sep()   { echo "  ──────────────────────────────────────────────────────────"; }
_head()  { echo; echo "  ══ $* ══"; }

# ── Argument parsing ──────────────────────────────────────────────────────────

SUBCOMMAND="${1:-}"
[[ -z "$SUBCOMMAND" ]] && { echo "Usage: $0 {provision|apply|rotate|status} <tenant-id> [OPTIONS]" >&2; exit 1; }
shift

TENANT_ID="${1:-}"
[[ -z "$TENANT_ID" ]] && _die "tenant-id required as second argument"
shift

# Defaults
TENANT_DOMAIN=""
TENANT_TYPE="enterprise"
KUBE_CONTEXT=""
KUBE_NAMESPACE="knoe-system"
DB_CONTEXT=""
DB_NAMESPACE="knoe-db"
DB_HOST=""
KDC_HOST=""
CI_GOOGLE_CLIENT_ID=""
CI_GOOGLE_CLIENT_SECRET=""
YES=false

while [[ $# -gt 0 ]]; do
  case "$1" in
    --domain)           TENANT_DOMAIN="$2";           shift 2 ;;
    --type)             TENANT_TYPE="$2";              shift 2 ;;
    --context)          KUBE_CONTEXT="$2";             shift 2 ;;
    --namespace)        KUBE_NAMESPACE="$2";           shift 2 ;;
    --db-context)       DB_CONTEXT="$2";               shift 2 ;;
    --db-namespace)     DB_NAMESPACE="$2";             shift 2 ;;
    --db-host)          DB_HOST="$2";                  shift 2 ;;
    --kdc-host)         KDC_HOST="$2";                 shift 2 ;;
    --ci-google-client) CI_GOOGLE_CLIENT_ID="$2";      shift 2 ;;
    --ci-google-secret) CI_GOOGLE_CLIENT_SECRET="$2";  shift 2 ;;
    --yes)              YES=true;                      shift   ;;
    *) _die "Unknown option: $1" ;;
  esac
done

# Apply defaults
[[ -z "$TENANT_DOMAIN" ]] && TENANT_DOMAIN="$TENANT_ID"
[[ -z "$DB_HOST" ]]       && DB_HOST="pg.${TENANT_DOMAIN}"
[[ -z "$KDC_HOST" ]]      && KDC_HOST="myrddin.${TENANT_DOMAIN}"
[[ -z "$DB_CONTEXT" ]]    && DB_CONTEXT="${KUBE_CONTEXT}"

VAULT="${VAULT_PREFIX}${TENANT_ID}"
K8S_GOOGLE_SECRET="knoe-auth-google-${TENANT_ID//[^a-z0-9-]/-}"

# kubectl wrapper respecting --context
_kubectl() {
  if [[ -n "$KUBE_CONTEXT" ]]; then
    kubectl --context="$KUBE_CONTEXT" "$@"
  else
    kubectl "$@"
  fi
}

_kubectl_db() {
  if [[ -n "$DB_CONTEXT" ]]; then
    kubectl --context="$DB_CONTEXT" "$@"
  else
    kubectl "$@"
  fi
}

# ── Preflight ──────────────────────────────────────────────────────────────────

require_op() {
  command -v op >/dev/null 2>&1 || _die "1Password CLI (op) not found. Install: brew install 1password-cli"
  op whoami >/dev/null 2>&1     || _die "Not signed in to 1Password. Run: op signin"
}

require_openssl() {
  command -v openssl >/dev/null 2>&1 || _die "openssl not found"
}

# ── Secret generation ──────────────────────────────────────────────────────────

gen_password() {
  # 32-char alphanumeric+symbol password, URL-safe
  openssl rand -base64 32 | tr -dc 'A-Za-z0-9!@#%^&*_+=' | head -c 32
}

gen_secret() {
  # 48-byte random secret, base64-encoded (for session secrets etc.)
  openssl rand -base64 48 | tr -d '\n'
}

gen_rsa_key_b64() {
  # RSA-2048 private key in PKCS#8 DER format, base64-encoded (no newlines).
  # Must be raw DER (not PEM): OidcTokenService does Base64.decode() → PKCS8EncodedKeySpec,
  # which requires raw DER bytes. PEM headers cause a DER parse error ("extra data at the end").
  openssl genrsa 2048 2>/dev/null \
    | openssl pkcs8 -topk8 -nocrypt -outform DER 2>/dev/null \
    | base64 | tr -d '\n'
}

# ── 1Password helpers ──────────────────────────────────────────────────────────

op_vault_exists() {
  op vault get "$VAULT" >/dev/null 2>&1
}

op_item_exists() {
  local title="$1"
  op item get "$title" --vault "$VAULT" >/dev/null 2>&1
}

op_get_field() {
  local title="$1" field="$2"
  op item get "$title" --vault "$VAULT" --fields "label=$field" --reveal 2>/dev/null \
    | tr -d '\n'
}

op_update_field() {
  local title="$1" field="$2" value="$3"
  op item edit "$title" --vault "$VAULT" "${field}[concealed]=${value}" >/dev/null
}

is_placeholder() {
  [[ "${1:-}" == "$PLACEHOLDER" ]]
}

# ── Provision ─────────────────────────────────────────────────────────────────

cmd_provision() {
  require_op
  require_openssl

  _head "Provisioning tenant: ${TENANT_ID} (type: ${TENANT_TYPE}, domain: ${TENANT_DOMAIN})"

  # ── Create vault ──────────────────────────────────────────────────────────
  if op_vault_exists; then
    _warn "Vault '${VAULT}' already exists — skipping vault creation."
  else
    _info "Creating 1Password vault: ${VAULT}"
    op vault create "$VAULT" >/dev/null
    _ok "Vault created."
  fi

  # ── Item: knoe-db ──────────────────────────────────────────────────────────
  _sep
  _info "Item: knoe-db (database credentials)"
  if op_item_exists "$ITEM_DB"; then
    _warn "Item '${ITEM_DB}' already exists — skipping."
  else
    local knoe_pw superuser_pw
    knoe_pw="$(gen_password)"
    superuser_pw="$(gen_password)"
    local db_url="jdbc:postgresql://knoe-db-rw.knoe-db.svc.cluster.local:5432/knoe-db"

    op item create \
      --vault "$VAULT" \
      --category=Login \
      --title="$ITEM_DB" \
      --url "$db_url" \
      "username[text]=knoe" \
      "password[concealed]=${knoe_pw}" \
      "superuser_password[concealed]=${superuser_pw}" \
      "db_url[text]=${db_url}" \
      "bootstrap_username[text]=knoe" \
      >/dev/null
    _ok "knoe-db: DB role password and superuser password generated."
  fi

  # ── Item: knoe-auth ────────────────────────────────────────────────────────
  _sep
  _info "Item: knoe-auth (session secret)"
  if op_item_exists "$ITEM_AUTH"; then
    _warn "Item '${ITEM_AUTH}' already exists — skipping."
  else
    local session_secret
    session_secret="$(gen_secret)"
    op item create \
      --vault "$VAULT" \
      --category=Password \
      --title="$ITEM_AUTH" \
      "password[concealed]=${session_secret}" \
      >/dev/null
    _ok "knoe-auth: session secret generated (${#session_secret} chars)."
  fi

  # ── Item: knoe-oidc ────────────────────────────────────────────────────────
  _sep
  _info "Item: knoe-oidc (OIDC RS256 signing key)"
  if op_item_exists "$ITEM_OIDC"; then
    _warn "Item '${ITEM_OIDC}' already exists — skipping."
  else
    _info "  Generating RSA-2048 key (this takes a moment)..."
    local signing_key
    signing_key="$(gen_rsa_key_b64)"
    op item create \
      --vault "$VAULT" \
      --category=Password \
      --title="$ITEM_OIDC" \
      "password[concealed]=${signing_key}" \
      >/dev/null
    _ok "knoe-oidc: RS256 signing key generated."
  fi

  # ── Item: knoe-google-tenant ───────────────────────────────────────────────
  _sep
  _info "Item: knoe-google-tenant (Google OAuth for enrollment flow)"
  if op_item_exists "$ITEM_GOOGLE"; then
    _warn "Item '${ITEM_GOOGLE}' already exists — skipping."
  else
    local g_client_id g_client_secret
    if [[ "$TENANT_TYPE" == "tenant" ]]; then
      # Hosted tenant: use shared CI client (points at api.knoe.dev/auth)
      g_client_id="${CI_GOOGLE_CLIENT_ID:-$PLACEHOLDER}"
      g_client_secret="${CI_GOOGLE_CLIENT_SECRET:-$PLACEHOLDER}"
      if [[ "$g_client_id" == "$PLACEHOLDER" ]]; then
        _warn "tenant: No CI Google client provided (--ci-google-client / --ci-google-secret)."
        _warn "        Set PLACEHOLDER now and fill before 'apply', OR pass flags to skip."
      else
        _ok "tenant: Using provided CI Google client."
      fi
    else
      # Enterprise tenant: placeholder — customer must create OAuth app in their Cloud Console
      g_client_id="$PLACEHOLDER"
      g_client_secret="$PLACEHOLDER"
      _warn "PLACEHOLDER set for Google OAuth client."
      _warn "  Complete in Google Cloud Console (enterprise's GCP project):"
      _warn "    App type: Web application"
      _warn "    Redirect URIs:"
      _warn "      https://api.${TENANT_DOMAIN}/auth/auth/enroll/google-callback"
      _warn "      http://localhost:8080/auth/auth/enroll/google-callback  (k3d)"
      _warn "  Then run:"
      _warn "    op item edit '${ITEM_GOOGLE}' --vault '${VAULT}' \\"
      _warn "      'client_id[text]=<ID>' 'password[concealed]=<SECRET>'"
    fi
    op item create \
      --vault "$VAULT" \
      --category=Login \
      --title="$ITEM_GOOGLE" \
      "username[text]=knoe-${TENANT_DOMAIN}" \
      "password[concealed]=${g_client_secret}" \
      "client_id[text]=${g_client_id}" \
      "hosted_domain[text]=${TENANT_DOMAIN}" \
      >/dev/null
    _ok "knoe-google-tenant: item created."
  fi

  # ── Item: knoe-kerberos ────────────────────────────────────────────────────
  _sep
  _info "Item: knoe-kerberos (KDC passwords + HTTP service keytab)"
  if op_item_exists "$ITEM_KERBEROS"; then
    _warn "Item '${ITEM_KERBEROS}' already exists — skipping."
  else
    local master_pw admin_pw http_keytab_b64
    master_pw="$(gen_password)"
    admin_pw="$(gen_password)"

    if [[ "$TENANT_TYPE" == "tenant" ]]; then
      # Hosted tenant: uses an in-cluster KDC — keytab is bootstrapped at deploy time
      # by the keytab-bootstrap initContainer; store a sentinel so apply knows
      # to skip the keytab secret (initContainer handles it).
      http_keytab_b64="__INITCONTAINER__"
      _ok "tenant: Keytab will be bootstrapped by initContainer at deploy time."
    else
      # Enterprise tenant: keytab must be extracted from the enterprise's Samba AD / MIT KDC
      http_keytab_b64="$PLACEHOLDER"
      _warn "PLACEHOLDER set for HTTP service keytab (enterprise Kerberos infrastructure required)."
      _warn "  On ${KDC_HOST} (enterprise AD/KDC), run:"
      _warn "    sudo samba-tool user create knoe-auth-http --random-password"
      _warn "    sudo samba-tool spn add HTTP/api.${TENANT_DOMAIN} knoe-auth-http"
      _warn "    sudo samba-tool domain exportkeytab /tmp/http.keytab \\"
      _warn "      --principal=HTTP/api.${TENANT_DOMAIN}"
      _warn "    klist -k /tmp/http.keytab   # verify"
      _warn "  Then store the keytab in 1Password:"
      _warn "    KEYTAB_B64=\$(ssh ${KDC_HOST} 'base64 -w0 /tmp/http.keytab')"
      _warn "    op item edit '${ITEM_KERBEROS}' --vault '${VAULT}' \\"
      _warn "      'http_keytab_b64[concealed]='\"\${KEYTAB_B64}\""
    fi

    op item create \
      --vault "$VAULT" \
      --category=Password \
      --title="$ITEM_KERBEROS" \
      "password[concealed]=${master_pw}" \
      "master_password[concealed]=${master_pw}" \
      "admin_password[concealed]=${admin_pw}" \
      "http_keytab_b64[concealed]=${http_keytab_b64}" \
      "realm[text]=${TENANT_DOMAIN^^}" \
      "service_principal[text]=HTTP/api.${TENANT_DOMAIN}@${TENANT_DOMAIN^^}" \
      >/dev/null
    _ok "knoe-kerberos: master + admin passwords generated."
  fi

  _sep
  _head "Provision complete"
  echo
  echo "  Vault: ${VAULT}"
  echo
  echo "  Next steps:"
  echo "    1. Run 'status ${TENANT_ID}' to see what still needs external input."
  if [[ "$TENANT_TYPE" == "enterprise" ]]; then
    echo "    2. Complete PLACEHOLDERs (Google OAuth + enterprise Kerberos keytab — see warnings above)."
    echo "    3. Run 'apply ${TENANT_ID} --context <CTX>' to create Kubernetes secrets."
  else
    echo "    2. Run 'apply ${TENANT_ID} --context <CTX>' to create Kubernetes secrets."
  fi
  echo "    4. After install: run 'rotate ${TENANT_ID}' to set user-chosen passwords."
  echo
}

# ── Status ────────────────────────────────────────────────────────────────────

cmd_status() {
  require_op

  _head "Secret status: ${TENANT_ID}"
  echo "  Vault: ${VAULT}"
  echo

  if ! op_vault_exists; then
    _err "Vault '${VAULT}' not found. Run: $0 provision ${TENANT_ID}"
    return 1
  fi

  _check_item() {
    local item="$1" field="$2" label="$3"
    local val
    val="$(op_get_field "$item" "$field" 2>/dev/null || echo "$PLACEHOLDER")"
    if [[ -z "$val" || "$val" == "$PLACEHOLDER" ]]; then
      printf "  %-40s  %s\n" "$label" "⚠  PLACEHOLDER — needs external input"
    elif [[ "$val" == "__INITCONTAINER__" ]]; then
      printf "  %-40s  %s\n" "$label" "✓  (initContainer)"
    else
      printf "  %-40s  %s\n" "$label" "✓  set (${#val} chars)"
    fi
  }

  printf "  %-40s  %s\n" "SECRET" "STATUS"
  printf "  %-40s  %s\n" "──────────────────────────────────────" "──────────────────────────────────"

  _check_item "$ITEM_DB"       "password"         "DB role 'knoe' password"
  _check_item "$ITEM_DB"       "superuser_password" "DB superuser password"
  _check_item "$ITEM_AUTH"     "password"         "Session secret"
  _check_item "$ITEM_OIDC"     "password"         "OIDC RS256 signing key"
  _check_item "$ITEM_GOOGLE"   "client_id"        "Google OAuth client_id"
  _check_item "$ITEM_GOOGLE"   "password"         "Google OAuth client_secret"
  _check_item "$ITEM_KERBEROS" "master_password"  "KDC master password"
  _check_item "$ITEM_KERBEROS" "admin_password"   "KDC admin password"
  _check_item "$ITEM_KERBEROS" "http_keytab_b64"  "HTTP service keytab"
  echo
}

# ── Apply ─────────────────────────────────────────────────────────────────────

cmd_apply() {
  require_op

  _head "Applying secrets to Kubernetes: ${TENANT_ID}"
  _info "Context:   ${KUBE_CONTEXT:-<current>}"
  _info "Namespace: ${KUBE_NAMESPACE}"
  echo

  if ! op_vault_exists; then
    _die "Vault '${VAULT}' not found. Run: $0 provision ${TENANT_ID}"
  fi

  # ── Read all secrets ───────────────────────────────────────────────────────
  local knoe_pw superuser_pw db_url session_secret signing_key
  local g_client_id g_client_secret hosted_domain
  local kdc_master kdc_admin http_keytab_b64

  _info "Reading secrets from vault..."
  knoe_pw=$(op_get_field "$ITEM_DB" "password")
  superuser_pw=$(op_get_field "$ITEM_DB" "superuser_password")
  db_url=$(op_get_field "$ITEM_DB" "db_url")
  session_secret=$(op_get_field "$ITEM_AUTH" "password")
  signing_key=$(op_get_field "$ITEM_OIDC" "password")
  g_client_id=$(op_get_field "$ITEM_GOOGLE" "client_id")
  g_client_secret=$(op_get_field "$ITEM_GOOGLE" "password")
  hosted_domain=$(op_get_field "$ITEM_GOOGLE" "hosted_domain")
  kdc_master=$(op_get_field "$ITEM_KERBEROS" "master_password")
  kdc_admin=$(op_get_field "$ITEM_KERBEROS" "admin_password")
  http_keytab_b64=$(op_get_field "$ITEM_KERBEROS" "http_keytab_b64")

  # ── Validate: no PLACEHOLDERs ─────────────────────────────────────────────
  local has_placeholder=false
  _check_placeholder() {
    local name="$1" val="$2"
    if is_placeholder "$val"; then
      _err "PLACEHOLDERs remain: ${name} — complete before running apply."
      has_placeholder=true
    fi
  }
  _check_placeholder "Google OAuth client_id"     "$g_client_id"
  _check_placeholder "Google OAuth client_secret" "$g_client_secret"
  _check_placeholder "HTTP service keytab"        "$http_keytab_b64"
  if $has_placeholder; then
    _die "Resolve all PLACEHOLDERs first. Use 'status ${TENANT_ID}' to check."
  fi

  _ok "All secrets validated — no PLACEHOLDERs."
  _sep

  # ── Create namespace if needed ─────────────────────────────────────────────
  if ! _kubectl get namespace "$KUBE_NAMESPACE" >/dev/null 2>&1; then
    _info "Creating namespace: ${KUBE_NAMESPACE}"
    _kubectl create namespace "$KUBE_NAMESPACE"
  fi

  _apply_secret() {
    local name="$1"; shift
    if _kubectl -n "$KUBE_NAMESPACE" get secret "$name" >/dev/null 2>&1; then
      _kubectl -n "$KUBE_NAMESPACE" delete secret "$name" >/dev/null
    fi
    _kubectl -n "$KUBE_NAMESPACE" create secret generic "$name" "$@" >/dev/null
    _ok "Secret applied: ${name}"
  }

  # knoe-db-user  (CNPG bootstrap — must match spec.bootstrap.initdb.owner)
  _apply_secret knoe-db-user \
    --from-literal=username="knoe" \
    --from-literal=password="$knoe_pw"

  # knoe-auth-db
  _apply_secret knoe-auth-db \
    --from-literal=db-url="$db_url" \
    --from-literal=db-user="knoe" \
    --from-literal=db-password="$knoe_pw"

  # knoe-auth-secrets
  _apply_secret knoe-auth-secrets \
    --from-literal=sessionSecret="$session_secret"

  # knoe-auth-oidc-signing
  _apply_secret knoe-auth-oidc-signing \
    --from-literal=signing-key="$signing_key"

  # knoe-auth-google-<tenant>  (enrollment flow)
  _apply_secret "$K8S_GOOGLE_SECRET" \
    --from-literal=client_id="$g_client_id" \
    --from-literal=client_secret="$g_client_secret"

  # knoe-auth-http-keytab  (skip if initContainer-managed)
  if [[ "$http_keytab_b64" == "__INITCONTAINER__" ]]; then
    _info "Keytab: managed by initContainer — skipping knoe-auth-http-keytab secret."
  else
    local tmpkeytab
    tmpkeytab="$(mktemp /tmp/http_keytab_XXXX)"
    echo "$http_keytab_b64" | base64 -d > "$tmpkeytab"
    _apply_secret knoe-auth-http-keytab \
      --from-file=http.keytab="$tmpkeytab"
    rm -f "$tmpkeytab"
  fi

  # knoe-kdc-secrets  (always — initContainer uses these even when also generating keytab)
  _apply_secret knoe-kdc-secrets \
    --from-literal=master_password="$kdc_master" \
    --from-literal=admin_password="$kdc_admin"

  _sep
  _head "Apply complete"
  echo
  echo "  All Kubernetes secrets created in ${KUBE_NAMESPACE}."
  echo
  _warn "DB role sync required for existing clusters:"
  _warn "  The k8s secrets now hold the 1Password-generated password, but if the"
  _warn "  PostgreSQL 'knoe' role already exists with a different password, knoe-auth"
  _warn "  will fail to connect. Sync now:"
  echo
  echo "    NEW_PW=\$(op item get 'knoe-db' --vault '${VAULT}' --fields 'label=password' --reveal)"
  echo "    kubectl --context=${KUBE_CONTEXT:-<CTX>} -n ${DB_NAMESPACE} \\"
  echo "      exec \$(kubectl --context=${KUBE_CONTEXT:-<CTX>} -n ${DB_NAMESPACE} get pod \\"
  echo "        -l 'cnpg.io/cluster=knoe-db,role=primary' -o jsonpath='{.items[0].metadata.name}') \\"
  echo "      -c postgres -- psql -U postgres -c \"ALTER ROLE knoe WITH PASSWORD '\${NEW_PW}';\""
  echo
  echo "  Or use 'rotate' which handles this automatically:"
  echo "    $0 rotate ${TENANT_ID} --context ${KUBE_CONTEXT:-<CTX>}"
  echo
  echo "  Then restart knoe-auth to pick up new secrets:"
  echo "    kubectl --context=${KUBE_CONTEXT:-<CTX>} -n ${KUBE_NAMESPACE} \\"
  echo "      rollout restart deployment/knoe-auth"
  echo
}

# ── Rotate ────────────────────────────────────────────────────────────────────

cmd_rotate() {
  require_op
  require_openssl

  _head "Post-install password rotation: ${TENANT_ID}"
  _info "Cluster:   ${KUBE_CONTEXT:-<current>}"
  _info "Namespace: ${KUBE_NAMESPACE}"
  _info "DB host:   ${DB_HOST}"
  echo

  if ! op_vault_exists; then
    _die "Vault '${VAULT}' not found."
  fi

  # Helper: prompt for new value or generate
  _rotate_secret() {
    local label="$1" current="$2"
    local masked="${current:0:4}****"

    if $YES; then
      # Non-interactive: regenerate
      gen_password
      return
    fi

    echo
    echo "  ┌─ ${label}"
    echo "  │  Current: ${masked}  (${#current} chars)"
    echo "  │"
    echo "  │  [1] Keep current"
    echo "  │  [2] Enter custom"
    echo "  │  [3] Regenerate (strong random)"
    printf "  └─ Choice [1/2/3]: "
    read -r choice
    case "$choice" in
      2)
        printf "  Enter new value: "
        read -rs new_val; echo
        echo "$new_val"
        ;;
      3)
        local new; new="$(gen_password)"
        echo "  Generated: ${new:0:4}****" >&2
        echo "$new"
        ;;
      *)
        echo "$current"  # keep
        ;;
    esac
  }

  # ── Rotate: DB role 'knoe' password ───────────────────────────────────────
  _sep
  _info "Rotating: DB role 'knoe' password"
  local current_knoe_pw new_knoe_pw
  current_knoe_pw="$(op_get_field "$ITEM_DB" "password")"
  new_knoe_pw="$(_rotate_secret "DB role 'knoe' password" "$current_knoe_pw")"

  if [[ "$new_knoe_pw" != "$current_knoe_pw" ]]; then
    _info "  Updating DB role 'knoe' via ALTER ROLE..."
    local superuser_pw
    superuser_pw="$(op_get_field "$ITEM_DB" "superuser_password")"

    # Find CNPG primary pod
    local primary_pod
    primary_pod="$(_kubectl_db -n "$DB_NAMESPACE" get pod \
      -l "cnpg.io/cluster=knoe-db,role=primary" \
      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)"
    [[ -z "$primary_pod" ]] && _die "Cannot find CNPG primary pod in ${DB_NAMESPACE}."

    _kubectl_db -n "$DB_NAMESPACE" exec "$primary_pod" -c postgres -- \
      psql -U postgres -c "ALTER ROLE knoe WITH PASSWORD '${new_knoe_pw}';" >/dev/null
    _ok "  DB role updated."

    # Update 1P
    op item edit "$ITEM_DB" --vault "$VAULT" \
      "password[concealed]=${new_knoe_pw}" >/dev/null
    _ok "  1Password updated."

    # Update k8s secrets
    local db_url
    db_url="$(op_get_field "$ITEM_DB" "db_url")"
    _kubectl -n "$KUBE_NAMESPACE" delete secret knoe-auth-db >/dev/null 2>&1 || true
    _kubectl -n "$KUBE_NAMESPACE" create secret generic knoe-auth-db \
      --from-literal=db-url="$db_url" \
      --from-literal=db-user="knoe" \
      --from-literal=db-password="$new_knoe_pw" >/dev/null
    _kubectl -n "$KUBE_NAMESPACE" delete secret knoe-db-user >/dev/null 2>&1 || true
    _kubectl -n "$KUBE_NAMESPACE" create secret generic knoe-db-user \
      --from-literal=username="knoe" \
      --from-literal=password="$new_knoe_pw" >/dev/null
    _ok "  Kubernetes secrets updated."
  else
    _ok "  Keeping current DB password."
  fi

  # ── Rotate: session secret ─────────────────────────────────────────────────
  _sep
  _info "Rotating: session secret"
  local current_session new_session
  current_session="$(op_get_field "$ITEM_AUTH" "password")"
  new_session="$(_rotate_secret "Session secret" "$current_session")"

  if [[ "$new_session" != "$current_session" ]]; then
    op item edit "$ITEM_AUTH" --vault "$VAULT" \
      "password[concealed]=${new_session}" >/dev/null
    _ok "  1Password updated."
    _kubectl -n "$KUBE_NAMESPACE" delete secret knoe-auth-secrets >/dev/null 2>&1 || true
    _kubectl -n "$KUBE_NAMESPACE" create secret generic knoe-auth-secrets \
      --from-literal=sessionSecret="$new_session" >/dev/null
    _ok "  Kubernetes secret updated."
  else
    _ok "  Keeping current session secret."
  fi

  # ── Rotate: KDC master password ────────────────────────────────────────────
  _sep
  _info "Rotating: KDC master password"
  local current_master new_master
  current_master="$(op_get_field "$ITEM_KERBEROS" "master_password")"
  new_master="$(_rotate_secret "KDC master password" "$current_master")"

  if [[ "$new_master" != "$current_master" ]]; then
    op item edit "$ITEM_KERBEROS" --vault "$VAULT" \
      "master_password[concealed]=${new_master}" \
      "password[concealed]=${new_master}" >/dev/null
    _ok "  1Password updated."
    # Update the composite kdc-secrets secret
    local current_admin
    current_admin="$(op_get_field "$ITEM_KERBEROS" "admin_password")"
    _kubectl -n "$KUBE_NAMESPACE" delete secret knoe-kdc-secrets >/dev/null 2>&1 || true
    _kubectl -n "$KUBE_NAMESPACE" create secret generic knoe-kdc-secrets \
      --from-literal=master_password="$new_master" \
      --from-literal=admin_password="$current_admin" >/dev/null
    _ok "  Kubernetes secret updated."
    _warn "  KDC master password changed — the in-cluster KDC pod must be restarted."
    _warn "  Rolling restart will re-run the keytab-bootstrap initContainer."
  else
    _ok "  Keeping current KDC master password."
  fi

  # ── Rotate: KDC admin password ─────────────────────────────────────────────
  _sep
  _info "Rotating: KDC admin password"
  local current_kadmin new_kadmin
  current_kadmin="$(op_get_field "$ITEM_KERBEROS" "admin_password")"
  new_kadmin="$(_rotate_secret "KDC admin password" "$current_kadmin")"

  if [[ "$new_kadmin" != "$current_kadmin" ]]; then
    op item edit "$ITEM_KERBEROS" --vault "$VAULT" \
      "admin_password[concealed]=${new_kadmin}" >/dev/null
    _ok "  1Password updated."
    local current_master_now
    current_master_now="$(op_get_field "$ITEM_KERBEROS" "master_password")"
    _kubectl -n "$KUBE_NAMESPACE" delete secret knoe-kdc-secrets >/dev/null 2>&1 || true
    _kubectl -n "$KUBE_NAMESPACE" create secret generic knoe-kdc-secrets \
      --from-literal=master_password="$current_master_now" \
      --from-literal=admin_password="$new_kadmin" >/dev/null
    _ok "  Kubernetes secret updated."
  else
    _ok "  Keeping current KDC admin password."
  fi

  # ── Roll deployment to pick up new secrets ─────────────────────────────────
  _sep
  _info "Rolling knoe-auth deployment to pick up updated secrets..."
  _kubectl -n "$KUBE_NAMESPACE" rollout restart deployment/knoe-auth >/dev/null 2>&1 || \
    _warn "Could not restart deployment — do it manually: kubectl rollout restart deployment/knoe-auth"
  _ok "Rollout triggered."

  _sep
  _head "Rotation complete"
  echo
  echo "  All selected secrets have been:"
  echo "    • Updated in 1Password vault '${VAULT}'"
  echo "    • Re-applied as Kubernetes secrets in ${KUBE_NAMESPACE}"
  echo "    • DB role updated via ALTER ROLE (if changed)"
  echo
  echo "  knoe-auth deployment is rolling — run:"
  echo "    kubectl -n ${KUBE_NAMESPACE} rollout status deployment/knoe-auth"
  echo
}

# ── Dispatch ──────────────────────────────────────────────────────────────────

case "$SUBCOMMAND" in
  provision) cmd_provision ;;
  apply)     cmd_apply     ;;
  rotate)    cmd_rotate    ;;
  status)    cmd_status    ;;
  *)
    echo "Usage: $0 {provision|apply|rotate|status} <tenant-id> [OPTIONS]" >&2
    echo
    echo "  provision   Create 1Password vault and pre-generate all secrets"
    echo "  apply       Create Kubernetes secrets from 1Password vault"
    echo "  rotate      Post-install: update passwords, re-apply secrets, roll deployment"
    echo "  status      Show what is provisioned vs. pending in the vault"
    echo
    echo "Examples:"
    echo "  $0 provision prole.org --domain prole.org --type enterprise"
    echo "  $0 provision acme      --type tenant --ci-google-client <ID> --ci-google-secret <S>"
    echo "  $0 status    prole.org"
    echo "  $0 apply     prole.org --context prole-service-cluster"
    echo "  $0 rotate    prole.org --context prole-service-cluster --yes"
    exit 1
    ;;
esac