#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ROOT_DIR=$(cd "$SCRIPT_DIR/.." && pwd) # shellcheck disable=SC1090 source "$SCRIPT_DIR/prole_cfg.sh" usage() { cat <] Options: -k, --kdc Authenticate current password against a running KDC -f, --force Skip current password verification -c, --config Path to prole.cfg (default: detected) USAGE } USE_KDC=0 FORCE=0 CFG_PATH="" while [[ $# -gt 0 ]]; do case "$1" in -k|--kdc) USE_KDC=1 shift ;; -f|--force) FORCE=1 shift ;; -c|--config) CFG_PATH="${2:-}" shift 2 ;; -h|--help) usage exit 0 ;; *) echo "Unknown argument: $1" >&2 usage exit 2 ;; esac done if [[ -z "$CFG_PATH" ]]; then if [[ -n "${PROLE_CONF:-}" && -f "$PROLE_CONF/prole.cfg" ]]; then CFG_PATH="$PROLE_CONF/prole.cfg" elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/conf/prole.cfg" ]]; then CFG_PATH="$PROLE_HOME/conf/prole.cfg" elif [[ -f "$ROOT_DIR/conf/prole.cfg" ]]; then CFG_PATH="$ROOT_DIR/conf/prole.cfg" fi fi if [[ -z "$CFG_PATH" || ! -f "$CFG_PATH" ]]; then echo "ERROR: prole.cfg not found. Use -c to specify path." >&2 exit 1 fi cfg_get() { python3 - "$CFG_PATH" "$1" <<'PY' import configparser import sys cfg_path = sys.argv[1] key = sys.argv[2] cfg = configparser.ConfigParser(interpolation=None) cfg.optionxform = str cfg.read(cfg_path) for section in cfg.sections(): if cfg.has_option(section, key): val = cfg.get(section, key, fallback="").strip() if val: print(val) break PY } # Resolve core identity values DB_USER=$(cfg_get "init_password.db_username") if [[ -z "$DB_USER" ]]; then DB_USER=$(cfg_get "KNOE_DB_USER") fi if [[ -z "$DB_USER" ]]; then DB_USER="prole" fi KRB5_USER=$(cfg_get "kerberos_config.user") if [[ -z "$KRB5_USER" ]]; then KRB5_USER=$(cfg_get "USER") fi if [[ -z "$KRB5_USER" ]]; then KRB5_USER="$DB_USER" fi KRB5_REALM=$(cfg_get "kerberos_config.realm") if [[ -z "$KRB5_REALM" ]]; then KRB5_REALM=$(cfg_get "REALM") fi KRB5_KDC=$(cfg_get "kerberos_config.kdc") if [[ -z "$KRB5_KDC" ]]; then KRB5_KDC=$(cfg_get "KDC") fi NAMESPACE="${PROLE_NAMESPACE}" if command -v kubectl >/dev/null 2>&1; then if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then existing_user_b64=$(kubectl -n "$NAMESPACE" get secret knoe-db-user -o jsonpath='{.data.username}' 2>/dev/null || true) if [[ -n "$existing_user_b64" ]]; then existing_user=$(printf '%s' "$existing_user_b64" | base64 -d 2>/dev/null || true) if [[ -n "$existing_user" ]]; then DB_USER="$existing_user" fi fi fi fi openbao_ref() { local leaf="$1" key="$2" printf '%s' "\${OPENBAO:kv/prole/${NAMESPACE}/${leaf}#${key}}" } openbao_available() { local probe if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then local url if url=$(openbao_url); then export PROLE_OPENBAO_URL="$url" fi fi probe=$(_prole_resolve_openbao_ref "$(openbao_ref "db" "password")") if [[ -n "$probe" && "$probe" != "null" ]]; then return 0 fi return 1 } openbao_token() { if [[ -n "${OPENBAO_ROOT_TOKEN:-}" ]]; then printf '%s' "$OPENBAO_ROOT_TOKEN" return 0 fi if [[ -n "${PROLE_SERVICE:-}" && -f "$PROLE_SERVICE/secrets/openbao-root-token" ]]; then cat "$PROLE_SERVICE/secrets/openbao-root-token" return 0 fi return 1 } openbao_url() { if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then printf '%s' "$PROLE_OPENBAO_URL" return 0 fi if [[ -f "$ROOT_DIR/etc/prole_cfg.sh" ]]; then # We use the python version if it exists url=$(python3 -c 'import os, sys; sys.path.append("."); import etc.prole_cfg as cfg; print(cfg.openbao_url())' 2>/dev/null || true) if [[ -n "$url" ]]; then printf '%s' "$url"; return 0; fi fi if prole_is_in_cluster; then printf '%s' "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE}}.svc.cluster.local:8200" return 0 fi if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then printf '%s' "http://127.0.0.1:8200" return 0 fi if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then printf '%s' "http://127.0.0.1:8200" return 0 fi return 1 } verify_current_password() { local current_pw="$1" if [[ "$USE_KDC" -eq 1 ]]; then if [[ -z "$KRB5_USER" || -z "$KRB5_REALM" ]]; then echo "ERROR: KRB5_USER or KRB5_REALM missing; cannot verify via KDC." >&2 return 1 fi if command -v kubectl >/dev/null 2>&1; then local kdc_ns kdc_name pod krb5_config kdc_ns=${PROLE_KDC_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}} kdc_name=${PROLE_KDC_NAME:-auth} pod=$(kubectl -n "$kdc_ns" get pod -l "app=${kdc_name}" --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "$pod" ]]; then krb5_config="/etc/krb5.conf" if ! kubectl -n "$kdc_ns" exec "$pod" -- ls /etc/krb5.conf >/dev/null 2>&1; then krb5_config="/opt/prole-kdc/krb5.conf" fi if printf '%s\n' "$current_pw" | kubectl -n "$kdc_ns" exec -i "$pod" -- env KRB5_CONFIG="$krb5_config" kinit "${KRB5_USER}@${KRB5_REALM}" >/dev/null 2>&1; then return 0 fi echo "ERROR: KDC authentication failed via pod ${pod}." >&2 return 1 fi fi if command -v kinit >/dev/null 2>&1 && [[ -n "$KRB5_KDC" ]]; then local tmp_cfg tmp_cfg=$(mktemp) cat >"$tmp_cfg" </dev/null 2>&1; then rm -f "$tmp_cfg" return 0 fi rm -f "$tmp_cfg" fi echo "ERROR: Unable to authenticate against KDC (pod/local kinit unavailable or failed)." >&2 return 1 fi if openbao_available; then local stored stored=$(_prole_resolve_openbao_ref "$(openbao_ref "db" "password")") if [[ -n "$stored" && "$stored" == "$current_pw" ]]; then return 0 fi echo "ERROR: Current password does not match OpenBao value." >&2 return 1 fi local cfg_pw cfg_pw=$(cfg_get "init_password.db_password") if [[ "$cfg_pw" == '${PROLE_SECRET:'* ]]; then cfg_pw=$(_prole_decrypt_prole_secret "$cfg_pw") else cfg_pw="" fi if [[ -n "$cfg_pw" && "$cfg_pw" == "$current_pw" ]]; then return 0 fi echo "ERROR: Unable to verify current password (no OpenBao/KDC and no at-rest encrypted password in prole.cfg)." >&2 return 1 } prompt_password() { local prompt="$1" out="" if [[ ! -t 0 ]]; then read -r out printf '%s' "$out" return 0 fi local char printf "%s" "$prompt" >&2 while IFS= read -r -s -n1 char; do if [[ -z "$char" ]]; then break fi if [[ "$char" == $'\x7f' || "$char" == $'\b' ]]; then if [[ ${#out} -gt 0 ]]; then out="${out%?}" printf '\b \b' >&2 fi else out+="$char" printf '*' >&2 fi done printf '\n' >&2 printf '%s' "$out" } restart_pods() { local mon_ns="${MONITORING_NAMESPACE:-monitoring}" local ns="$NAMESPACE" echo "Restarting service pods to pick up new secrets..." if kubectl get namespace "$mon_ns" >/dev/null 2>&1; then for deploy in kps-grafana grafana; do if kubectl -n "$mon_ns" get deployment "$deploy" >/dev/null 2>&1; then echo " Restarting $deploy..." kubectl -n "$mon_ns" rollout restart deployment/"$deploy" >/dev/null 2>&1 || true kubectl -n "$mon_ns" rollout status deployment/"$deploy" --timeout=120s >/dev/null 2>&1 || true fi done # After rollout, do CLI reset to ensure DB is updated update_grafana_cli_reset fi if kubectl get namespace "$ns" >/dev/null 2>&1; then # For CNPG we might need to rollout the cluster if not using hot-reload secrets # But usually CNPG picks up secrets. If not, restart. if kubectl -n "$ns" get statefulset knoe-db >/dev/null 2>&1; then echo " Restarting knoe-db..." kubectl -n "$ns" rollout restart statefulset knoe-db >/dev/null 2>&1 || true kubectl -n "$ns" rollout status statefulset knoe-db --timeout=120s >/dev/null 2>&1 || true fi fi } verify_access() { local pw="$1" local ns="$NAMESPACE" local mon_ns="${MONITORING_NAMESPACE:-monitoring}" echo "Verifying service access..." # 1. DB check if kubectl get namespace "$ns" >/dev/null 2>&1; then local db_pod db_pod=$(kubectl -n "$ns" get pod -l "cnpg.io/cluster=knoe-db,role=primary" --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true) if [[ -z "$db_pod" ]]; then db_pod=$(kubectl -n "$ns" get pod -l "app=knoe-db" --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true) fi if [[ -n "$db_pod" ]]; then # Wait for DB to be ready and check via service to avoid Peer auth issues local i for i in {1..10}; do # Use -h knoe-db-rw if it exists, otherwise localhost (though localhost often uses Peer) local host="localhost" if kubectl -n "$ns" get svc knoe-db-rw >/dev/null 2>&1; then host="knoe-db-rw" fi if kubectl -n "$ns" exec "$db_pod" -c postgres -- env PGPASSWORD="$pw" psql -h "$host" -U prole -d postgres -c "SELECT 1" >/dev/null 2>&1; then echo " [PASS] Database access (user: prole)" break fi sleep 2 done if [[ $i -eq 10 ]]; then echo " [FAIL] Database access (user: prole)" fi fi fi # 2. Grafana check if kubectl get namespace "$mon_ns" >/dev/null 2>&1; then local grafana_pod grafana_pod=$(kubectl -n "$mon_ns" get pod -l app.kubernetes.io/name=grafana --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true) if [[ -n "$grafana_pod" ]]; then # Get admin user from secret local admin_user admin_user=$(kubectl -n "$mon_ns" get secret kps-grafana -o jsonpath='{.data.admin-user}' 2>/dev/null | base64 -d 2>/dev/null || echo "admin") # Wait for Grafana to be ready local i for i in {1..15}; do # Check if it's even up first if kubectl -n "$mon_ns" exec "$grafana_pod" -c grafana -- curl -s http://localhost:3000/api/health >/dev/null 2>&1; then # Try one auth if kubectl -n "$mon_ns" exec "$grafana_pod" -c grafana -- curl -s -f -u "${admin_user}:${pw}" http://localhost:3000/api/org >/dev/null 2>&1; then echo " [PASS] Grafana access (user: ${admin_user})" break else # If 401, maybe wait longer or it failed echo " [WAIT] Grafana login failed (i=$i), retrying..." fi fi sleep 5 done if [[ $i -eq 15 ]]; then echo " [FAIL] Grafana access (user: ${admin_user})" fi fi fi # 3. Prometheus check if kubectl get namespace "$mon_ns" >/dev/null 2>&1; then # Try from Grafana pod if it has curl if [[ -n "$grafana_pod" ]]; then if kubectl -n "$mon_ns" exec "$grafana_pod" -c grafana -- curl -s -f http://kps-kube-prometheus-stack-prometheus:9090/-/healthy >/dev/null 2>&1; then echo " [PASS] Prometheus health check (via Grafana pod)" else echo " [FAIL] Prometheus health check" fi fi fi # 4. Supabase check if kubectl get namespace "supabase" >/dev/null 2>&1; then local sb_pod sb_pod=$(kubectl -n supabase get pod -l app=kong -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "$sb_pod" ]]; then # Supabase uses JWT often, but we can check if we can connect to the DB via kong if exposed # Or just check if secrets are set correctly if verify_k8s_secret "supabase" "supabase-db-password" "password" "$pw"; then echo " [PASS] Supabase secret verification" else # Try generic check for any secret containing the password local found=0 local s for s in $(kubectl -n supabase get secret -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do if verify_k8s_secret "supabase" "$s" "password" "$pw" || verify_k8s_secret "supabase" "$s" "POSTGRES_PASSWORD" "$pw"; then found=1; break fi done if [[ $found -eq 1 ]]; then echo " [PASS] Supabase secret verification (found in $s)" else echo " [FAIL] Supabase secret verification" fi fi fi fi } update_kdc_password() { local user="$1" realm="$2" new_pw="$3" local kdc_ns kdc_name pod kdc_ns=${PROLE_KDC_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}} kdc_name=${PROLE_KDC_NAME:-auth} pod=$(kubectl -n "$kdc_ns" get pod -l "app=${kdc_name}" --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "$pod" ]]; then echo "Updating password in internal KDC for ${user}@${realm}..." # Attempt to change password via kadmin.local if kubectl -n "$kdc_ns" exec "$pod" -- kadmin.local -q "cpw -pw ${new_pw} ${user}@${realm}" >/dev/null 2>&1; then echo " [PASS] KDC password updated." return 0 fi # If realm in config doesn't match the one in KDC, it might fail. # Try with default realm if it fails? local default_realm default_realm=$(kubectl -n "$kdc_ns" exec "$pod" -- cat /etc/krb5.conf | grep default_realm | awk '{print $3}' || true) if [[ -n "$default_realm" && "$default_realm" != "$realm" ]]; then if kubectl -n "$kdc_ns" exec "$pod" -- kadmin.local -q "cpw -pw ${new_pw} ${user}@${default_realm}" >/dev/null 2>&1; then echo " [PASS] KDC password updated (using default realm ${default_realm})." return 0 fi fi echo " [FAIL] KDC password update failed." fi return 1 } update_cfg_keys() { local value="$1" python3 - "$CFG_PATH" "$value" <<'PY' import re import sys cfg_path = sys.argv[1] value = sys.argv[2] keys = { "init_password.db_password", "init_password.db_password_confirm", "kerberos_config.password", "DB_PASSWORD", "PASSWORD", "GRAFANA_ADMIN_PASSWORD", } pattern = re.compile(r"^(?P\s*)(?P[^=]+?)(?P
\s*)=(?P\s*).*$")

out_lines = []
with open(cfg_path, "r", encoding="utf-8") as f:
    for line in f.read().splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith((';', '#')) or '=' not in line:
            out_lines.append(line)
            continue
        if stripped.startswith('[') and stripped.endswith(']'):
            out_lines.append(line)
            continue
        m = pattern.match(line)
        if not m:
            out_lines.append(line)
            continue
        key = m.group("key").strip()
        if key in keys:
            lead = m.group("lead")
            pre = m.group("pre")
            post = m.group("post")
            out_lines.append(f"{lead}{key}{pre}={post}{value}")
        else:
            out_lines.append(line)

with open(cfg_path, "w", encoding="utf-8") as f:
    f.write("\n".join(out_lines) + "\n")
PY
}

update_cfg_openbao_refs() {
  python3 - "$CFG_PATH" "$NAMESPACE" <<'PY'
import re
import sys

cfg_path = sys.argv[1]
ns = sys.argv[2]

def ref(leaf: str, key: str) -> str:
    return f"${{OPENBAO:kv/prole/{ns}/{leaf}#{key}}}"

secret_map = {
    "init_password.db_password": ref("db", "password"),
    "init_password.db_password_confirm": ref("db", "password"),
    "kerberos_config.password": ref("kerberos", "password"),
    "DB_PASSWORD": ref("db", "password"),
    "PASSWORD": ref("kerberos", "password"),
    "GRAFANA_ADMIN_PASSWORD": ref("monitoring", "grafana_admin_password"),
}

pattern = re.compile(r"^(?P\s*)(?P[^=]+?)(?P
\s*)=(?P\s*).*$")

out_lines = []
with open(cfg_path, "r", encoding="utf-8") as f:
    for line in f.read().splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith((';', '#')) or '=' not in line:
            out_lines.append(line)
            continue
        if stripped.startswith('[') and stripped.endswith(']'):
            out_lines.append(line)
            continue
        m = pattern.match(line)
        if not m:
            out_lines.append(line)
            continue
        key = m.group("key").strip()
        if key in secret_map:
            lead = m.group("lead")
            pre = m.group("pre")
            post = m.group("post")
            out_lines.append(f"{lead}{key}{pre}={post}{secret_map[key]}")
        else:
            out_lines.append(line)

with open(cfg_path, "w", encoding="utf-8") as f:
    f.write("\n".join(out_lines) + "\n")
PY
}

write_openbao_secrets() {
  local url token
  if ! token=$(openbao_token); then
    echo "WARN: OpenBao token not available; skipping OpenBao update." >&2
    return 1
  fi
  if ! url=$(openbao_url); then
    echo "WARN: OpenBao not reachable; skipping OpenBao update." >&2
    return 1
  fi

  if ! curl -sS -H "X-Vault-Token: $token" "$url/v1/sys/mounts" | grep -q '\"kv/\"'; then
    curl -sS -H "X-Vault-Token: $token" -X POST "$url/v1/sys/mounts/kv" \
      -d '{"type":"kv","options":{"version":"2"}}' >/dev/null 2>&1 || true
  fi

  local db_payload krb_payload grafana_payload
  db_payload=$(python3 - "$DB_USER" "$NEW_PASSWORD" <<'PY'
import json
import sys
print(json.dumps({"data": {"username": sys.argv[1], "password": sys.argv[2]}}))
PY
)
  krb_payload=$(python3 - "$KRB5_USER" "$KRB5_REALM" "$KRB5_KDC" "$NEW_PASSWORD" <<'PY'
import json
import sys
print(json.dumps({"data": {"username": sys.argv[1], "realm": sys.argv[2], "kdc": sys.argv[3], "password": sys.argv[4]}}))
PY
)
  grafana_payload=$(python3 - "$NEW_PASSWORD" <<'PY'
import json
import sys
print(json.dumps({"data": {"grafana_admin_password": sys.argv[1]}}))
PY
)

  curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
    -X POST "$url/v1/kv/data/prole/${NAMESPACE}/db" -d "$db_payload" >/dev/null
  curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
    -X POST "$url/v1/kv/data/prole/${NAMESPACE}/kerberos" -d "$krb_payload" >/dev/null
  curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
    -X POST "$url/v1/kv/data/prole/${NAMESPACE}/monitoring" -d "$grafana_payload" >/dev/null

  return 0
}

update_k8s_db_secrets() {
  local ns="$NAMESPACE"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    return 0
  fi
  kubectl create secret generic knoe-db-user -n "$ns" \
    --from-literal=username="$DB_USER" \
    --from-literal=password="$NEW_PASSWORD" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null

  kubectl create secret generic knoe-db-superuser -n "$ns" \
    --from-literal=username=postgres \
    --from-literal=password="$NEW_PASSWORD" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null
}

update_grafana_secret() {
  local ns="${MONITORING_NAMESPACE:-monitoring}"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    return 0
  fi

  local pw_b64
  pw_b64=$(printf '%s' "$NEW_PASSWORD" | base64 | tr -d '\n')

  local secret
  for secret in kps-grafana grafana; do
    if kubectl -n "$ns" get secret "$secret" >/dev/null 2>&1; then
      if kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.admin-password}' >/dev/null 2>&1; then
        kubectl -n "$ns" patch secret "$secret" --type merge \
          -p "{\"data\":{\"admin-password\":\"${pw_b64}\"}}" >/dev/null
      fi
      if kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.adminPassword}' >/dev/null 2>&1; then
        kubectl -n "$ns" patch secret "$secret" --type merge \
          -p "{\"data\":{\"adminPassword\":\"${pw_b64}\"}}" >/dev/null
      fi
      return 0
    fi
  done
}

update_grafana_cli_reset() {
  local ns="${MONITORING_NAMESPACE:-monitoring}"
  local pod
  # Get the newest running pod
  pod=$(kubectl -n "$ns" get pod -l app.kubernetes.io/name=grafana --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
  if [[ -n "$pod" ]]; then
     echo "  Updating Grafana admin password via CLI in pod $pod..."
     # Wait for DB to be ready inside the pod and retry if locked
     local i
     for i in {1..5}; do
       if kubectl -n "$ns" exec "$pod" -c grafana -- grafana cli admin reset-admin-password "$NEW_PASSWORD" >/dev/null 2>&1; then
         echo "  [PASS] Grafana CLI reset successful."
         return 0
       elif kubectl -n "$ns" exec "$pod" -c grafana -- grafana-cli admin reset-admin-password "$NEW_PASSWORD" >/dev/null 2>&1; then
         echo "  [PASS] Grafana-CLI reset successful."
         return 0
       fi
       echo "  [WAIT] Grafana CLI reset failed (possibly DB locked), retrying (i=$i)..."
       sleep 5
     done
     echo "  [FAIL] Grafana CLI reset failed after 5 attempts."
  fi
  return 1
}

update_supabase_secrets() {
  local ns="supabase"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    return 0
  fi

  local pw_b64
  pw_b64=$(printf '%s' "$NEW_PASSWORD" | base64 | tr -d '\n')

  local secrets
  secrets=$(kubectl -n "$ns" get secret -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
  local s
  for s in $secrets; do
    local updated=0
    for key in POSTGRES_PASSWORD PGPASSWORD DB_PASSWORD SUPABASE_DB_PASSWORD SUPABASE_PASSWORD; do
      if kubectl -n "$ns" get secret "$s" -o jsonpath="{.data.${key}}" >/dev/null 2>&1; then
        kubectl -n "$ns" patch secret "$s" --type merge \
          -p "{\"data\":{\"${key}\":\"${pw_b64}\"}}" >/dev/null
        updated=1
      fi
    done
    if [[ $updated -eq 1 ]]; then
      :
    fi
  done
}

verify_k8s_secret() {
  local ns="$1" secret="$2" key="$3" expected="$4"
  local val
  val=$(kubectl -n "$ns" get secret "$secret" -o jsonpath="{.data.${key}}" 2>/dev/null || true)
  if [[ -z "$val" ]]; then
    return 1
  fi
  val=$(printf '%s' "$val" | base64 -d 2>/dev/null || true)
  if [[ "$val" == '${'* ]]; then
    echo "  [FAIL] Secret $ns/$secret key $key contains a variable: $val" >&2
    return 1
  fi
  [[ "$val" == "$expected" ]]
}

# 1) Verify current password (unless forced)
if [[ "$FORCE" -eq 0 ]]; then
  current_pw=$(prompt_password "Current password: ")
  if ! verify_current_password "$current_pw"; then
    exit 1
  fi
fi

# 2) Prompt for new password
NEW_PASSWORD=$(prompt_password "New password: ")
confirm_pw=$(prompt_password "Repeat new password: ")
if [[ -z "$NEW_PASSWORD" ]]; then
  echo "ERROR: New password cannot be empty." >&2
  exit 1
fi
if [[ "$NEW_PASSWORD" != "$confirm_pw" ]]; then
  echo "ERROR: Passwords do not match." >&2
  exit 1
fi

# 3) Store at-rest encrypted in prole.cfg
ENC_PW=$(python3 - "$ROOT_DIR" "$NEW_PASSWORD" <<'PY'
import sys

root = sys.argv[1]
pw = sys.argv[2]

sys.path.insert(0, root)
from knoe import config as inst_config

print(inst_config._encrypt_prole_secret(pw))
PY
)

update_cfg_keys "$ENC_PW"

# 4) Push to OpenBao (if available) and replace prole.cfg refs
OPENBAO_UPDATED=0
if write_openbao_secrets; then
  OPENBAO_UPDATED=1
  update_cfg_openbao_refs
fi

# 5) Update Kubernetes secrets (if available)
if command -v kubectl >/dev/null 2>&1; then
  prole_ensure_kubeconfig >/dev/null 2>&1 || true
  update_k8s_db_secrets
  update_grafana_secret
  update_supabase_secrets
  update_kdc_password "$KRB5_USER" "$KRB5_REALM" "$NEW_PASSWORD"
  restart_pods
fi

# 6) Verification
verify_access "$NEW_PASSWORD"

if [[ "$OPENBAO_UPDATED" -eq 1 ]]; then
  if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then
    if url=$(openbao_url); then
      export PROLE_OPENBAO_URL="$url"
    fi
  fi
  resolved=$(_prole_resolve_openbao_ref "$(openbao_ref "db" "password")")
  if [[ "$resolved" != "$NEW_PASSWORD" ]]; then
    echo "WARN: OpenBao value did not match expected password." >&2
  fi
fi

if command -v kubectl >/dev/null 2>&1; then
  if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
    if ! verify_k8s_secret "$NAMESPACE" "knoe-db-user" "password" "$NEW_PASSWORD"; then
      echo "WARN: knoe-db-user secret does not match expected password." >&2
    fi
    if ! verify_k8s_secret "$NAMESPACE" "knoe-db-superuser" "password" "$NEW_PASSWORD"; then
      echo "WARN: knoe-db-superuser secret does not match expected password." >&2
    fi
  fi
  if kubectl get namespace "${MONITORING_NAMESPACE:-monitoring}" >/dev/null 2>&1; then
    if kubectl -n "${MONITORING_NAMESPACE:-monitoring}" get secret kps-grafana >/dev/null 2>&1; then
      if ! verify_k8s_secret "${MONITORING_NAMESPACE:-monitoring}" "kps-grafana" "admin-password" "$NEW_PASSWORD"; then
        echo "WARN: Grafana admin secret does not match expected password." >&2
      fi
    elif kubectl -n "${MONITORING_NAMESPACE:-monitoring}" get secret grafana >/dev/null 2>&1; then
      if ! verify_k8s_secret "${MONITORING_NAMESPACE:-monitoring}" "grafana" "admin-password" "$NEW_PASSWORD"; then
        echo "WARN: Grafana admin secret does not match expected password." >&2
      fi
    fi
  fi
fi

echo "Password update complete."