prole/mock_val/knoe-db-passwwd.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
ROOT_DIR=$(cd "$SCRIPT_DIR/.." && pwd)

# shellcheck disable=SC1090
source "$SCRIPT_DIR/knoe_cfg.sh"

usage() {
  cat <<USAGE
Usage: $0 [-k|--kdc] [-f|--force] [-c|--config <path>]

Options:
  -k, --kdc     Authenticate current password against a running KDC
  -f, --force   Skip current password verification
  -c, --config  Path to config file (default: detected)
USAGE
}

USE_KDC=0
FORCE=0
CFG_PATH=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    -k|--kdc)
      USE_KDC=1
      shift
      ;;
    -f|--force)
      FORCE=1
      shift
      ;;
    -c|--config)
      CFG_PATH="${2:-}"
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage
      exit 2
      ;;
  esac
done

if [[ -z "$CFG_PATH" ]]; then
  if [[ -n "${KNOE_CONF:-}" ]]; then
    CFG_PATH="$(_knoe_cfg_select_cfg_file "$KNOE_CONF")"
  elif [[ -n "${KNOE_HOME:-}" ]]; then
    CFG_PATH="$(_knoe_cfg_select_cfg_file "$KNOE_HOME/conf")"
  else
    CFG_PATH="$(_knoe_cfg_select_cfg_file "$ROOT_DIR/conf")"
  fi
fi

if [[ -z "$CFG_PATH" || ! -f "$CFG_PATH" ]]; then
  echo "ERROR: config file not found. Use -c to specify path." >&2
  exit 1
fi

cfg_get() {
  python3 - "$CFG_PATH" "$1" <<'PY'
import configparser
import sys

cfg_path = sys.argv[1]
key = sys.argv[2]

cfg = configparser.ConfigParser(interpolation=None)
cfg.optionxform = str
cfg.read(cfg_path)

for section in cfg.sections():
    if cfg.has_option(section, key):
        val = cfg.get(section, key, fallback="").strip()
        if val:
            print(val)
            break
PY
}

# Resolve core identity values
DB_USER=$(cfg_get "init_password.db_username")
if [[ -z "$DB_USER" ]]; then
  DB_USER=$(cfg_get "KNOE_DB_USER")
fi
if [[ -z "$DB_USER" ]]; then
  DB_USER="knoe"
fi

KRB5_USER=$(cfg_get "kerberos_config.user")
if [[ -z "$KRB5_USER" ]]; then
  KRB5_USER=$(cfg_get "USER")
fi
if [[ -z "$KRB5_USER" ]]; then
  KRB5_USER="$DB_USER"
fi

KRB5_REALM=$(cfg_get "kerberos_config.realm")
if [[ -z "$KRB5_REALM" ]]; then
  KRB5_REALM=$(cfg_get "REALM")
fi

KRB5_KDC=$(cfg_get "kerberos_config.kdc")
if [[ -z "$KRB5_KDC" ]]; then
  KRB5_KDC=$(cfg_get "KDC")
fi

NAMESPACE="${PROLE_NAMESPACE}"

if command -v kubectl >/dev/null 2>&1; then
  if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
    existing_user_b64=$(kubectl -n "$NAMESPACE" get secret knoe-db-user -o jsonpath='{.data.username}' 2>/dev/null || true)
    if [[ -n "$existing_user_b64" ]]; then
      existing_user=$(printf '%s' "$existing_user_b64" | base64 -d 2>/dev/null || true)
      if [[ -n "$existing_user" ]]; then
        DB_USER="$existing_user"
      fi
    fi
  fi
fi

openbao_ref() {
  local leaf="$1" key="$2"
  printf '%s' "\${OPENBAO:kv/knoe/${NAMESPACE}/${leaf}#${key}}"
}

openbao_available() {
  local probe
  if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then
    local url
    if url=$(openbao_url); then
      export PROLE_OPENBAO_URL="$url"
    fi
  fi
  probe=$(_knoe_resolve_openbao_ref "$(openbao_ref "db" "password")")
  if [[ -n "$probe" && "$probe" != "null" ]]; then
    return 0
  fi
  return 1
}

openbao_token() {
  if [[ -n "${OPENBAO_ROOT_TOKEN:-}" ]]; then
    printf '%s' "$OPENBAO_ROOT_TOKEN"
    return 0
  fi
  if [[ -n "${KNOE_SERVICE:-}" && -f "$KNOE_SERVICE/secrets/openbao-root-token" ]]; then
    cat "$KNOE_SERVICE/secrets/openbao-root-token"
    return 0
  fi
  return 1
}

openbao_url() {
  if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
    printf '%s' "$PROLE_OPENBAO_URL"
    return 0
  fi
  if [[ -f "$ROOT_DIR/etc/knoe_cfg.sh" ]]; then
     # We use the python version if it exists
     url=$(python3 -c 'import os, sys; sys.path.append("."); import etc.knoe_cfg as cfg; print(cfg.openbao_url())' 2>/dev/null || true)
     if [[ -n "$url" ]]; then printf '%s' "$url"; return 0; fi
  fi
  if knoe_is_in_cluster; then
    printf '%s' "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE}}.svc.cluster.local:8200"
    return 0
  fi
  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    printf '%s' "http://127.0.0.1:8200"
    return 0
  fi
  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    printf '%s' "http://127.0.0.1:8200"
    return 0
  fi
  return 1
}

verify_current_password() {
  local current_pw="$1"
  if [[ "$USE_KDC" -eq 1 ]]; then
    if [[ -z "$KRB5_USER" || -z "$KRB5_REALM" ]]; then
      echo "ERROR: KRB5_USER or KRB5_REALM missing; cannot verify via KDC." >&2
      return 1
    fi

    if command -v kubectl >/dev/null 2>&1; then
      local kdc_ns kdc_name pod krb5_config
      kdc_ns=${PROLE_KDC_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}}
      kdc_name=${PROLE_KDC_NAME:-auth}
      pod=$(kubectl -n "$kdc_ns" get pod -l "app=${kdc_name}" --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
      if [[ -n "$pod" ]]; then
        krb5_config="/etc/krb5.conf"
        if ! kubectl -n "$kdc_ns" exec "$pod" -- ls /etc/krb5.conf >/dev/null 2>&1; then
          krb5_config="/opt/knoe-kdc/krb5.conf"
        fi
        if printf '%s\n' "$current_pw" | kubectl -n "$kdc_ns" exec -i "$pod" -- env KRB5_CONFIG="$krb5_config" kinit "${KRB5_USER}@${KRB5_REALM}" >/dev/null 2>&1; then
          return 0
        fi
        echo "ERROR: KDC authentication failed via pod ${pod}." >&2
        return 1
      fi
    fi

    if command -v kinit >/dev/null 2>&1 && [[ -n "$KRB5_KDC" ]]; then
      local tmp_cfg
      tmp_cfg=$(mktemp)
      cat >"$tmp_cfg" <<KRB5CONF
[libdefaults]
  default_realm = ${KRB5_REALM}
[realms]
  ${KRB5_REALM} = {
    kdc = ${KRB5_KDC}
  }
KRB5CONF
      if printf '%s\n' "$current_pw" | env KRB5_CONFIG="$tmp_cfg" kinit "${KRB5_USER}@${KRB5_REALM}" >/dev/null 2>&1; then
        rm -f "$tmp_cfg"
        return 0
      fi
      rm -f "$tmp_cfg"
    fi

    echo "ERROR: Unable to authenticate against KDC (pod/local kinit unavailable or failed)." >&2
    return 1
  fi

  if openbao_available; then
    local stored
    stored=$(_knoe_resolve_openbao_ref "$(openbao_ref "db" "password")")
    if [[ -n "$stored" && "$stored" == "$current_pw" ]]; then
      return 0
    fi
    echo "ERROR: Current password does not match OpenBao value." >&2
    return 1
  fi

  local cfg_pw
  cfg_pw=$(cfg_get "init_password.db_password")
  if [[ "$cfg_pw" == '${KNOE_SECRET:'* ]]; then
    cfg_pw=$(_knoe_decrypt_knoe_secret "$cfg_pw")
  else
    cfg_pw=""
  fi

  if [[ -n "$cfg_pw" && "$cfg_pw" == "$current_pw" ]]; then
    return 0
  fi

  echo "ERROR: Unable to verify current password (no OpenBao/KDC and no at-rest encrypted password in knoe.cfg)." >&2
  return 1
}

prompt_password() {
  local prompt="$1" out=""
  if [[ ! -t 0 ]]; then
    read -r out
    printf '%s' "$out"
    return 0
  fi
  local char
  printf "%s" "$prompt" >&2
  while IFS= read -r -s -n1 char; do
    if [[ -z "$char" ]]; then
      break
    fi
    if [[ "$char" == $'\x7f' || "$char" == $'\b' ]]; then
      if [[ ${#out} -gt 0 ]]; then
        out="${out%?}"
        printf '\b \b' >&2
      fi
    else
      out+="$char"
      printf '*' >&2
    fi
  done
  printf '\n' >&2
  printf '%s' "$out"
}

restart_pods() {
  local mon_ns="${MONITORING_NAMESPACE:-monitoring}"
  local ns="$NAMESPACE"
  echo "Restarting service pods to pick up new secrets..."
  if kubectl get namespace "$mon_ns" >/dev/null 2>&1; then
    for deploy in kps-grafana grafana; do
      if kubectl -n "$mon_ns" get deployment "$deploy" >/dev/null 2>&1; then
        echo "  Restarting $deploy..."
        kubectl -n "$mon_ns" rollout restart deployment/"$deploy" >/dev/null 2>&1 || true
        kubectl -n "$mon_ns" rollout status deployment/"$deploy" --timeout=120s >/dev/null 2>&1 || true
      fi
    done
    # After rollout, do CLI reset to ensure DB is updated
    update_grafana_cli_reset
  fi
  if kubectl get namespace "$ns" >/dev/null 2>&1; then
    # For CNPG we might need to rollout the cluster if not using hot-reload secrets
    # But usually CNPG picks up secrets. If not, restart.
    if kubectl -n "$ns" get statefulset knoe-db >/dev/null 2>&1; then
      echo "  Restarting knoe-db..."
      kubectl -n "$ns" rollout restart statefulset knoe-db >/dev/null 2>&1 || true
      kubectl -n "$ns" rollout status statefulset knoe-db --timeout=120s >/dev/null 2>&1 || true
    fi
  fi
}

verify_access() {
  local pw="$1"
  local ns="$NAMESPACE"
  local mon_ns="${MONITORING_NAMESPACE:-monitoring}"
  
  echo "Verifying service access..."
  
  # 1. DB check
  if kubectl get namespace "$ns" >/dev/null 2>&1; then
    local db_pod
    db_pod=$(kubectl -n "$ns" get pod -l "cnpg.io/cluster=knoe-db,role=primary" --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
    if [[ -z "$db_pod" ]]; then
       db_pod=$(kubectl -n "$ns" get pod -l "app=knoe-db" --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
    fi
    if [[ -n "$db_pod" ]]; then
      # Wait for DB to be ready and check via service to avoid Peer auth issues
      local i
      for i in {1..10}; do
        # Use -h knoe-db-rw if it exists, otherwise localhost (though localhost often uses Peer)
        local host="localhost"
        if kubectl -n "$ns" get svc knoe-db-rw >/dev/null 2>&1; then
           host="knoe-db-rw"
        fi
        if kubectl -n "$ns" exec "$db_pod" -c postgres -- env PGPASSWORD="$pw" psql -h "$host" -U knoe -d postgres -c "SELECT 1" >/dev/null 2>&1; then
          echo "  [PASS] Database access (user: knoe)"
          break
        fi
        sleep 2
      done
      if [[ $i -eq 10 ]]; then
        echo "  [FAIL] Database access (user: knoe)"
      fi
    fi
  fi
  
  # 2. Grafana check
  if kubectl get namespace "$mon_ns" >/dev/null 2>&1; then
    local grafana_pod
    grafana_pod=$(kubectl -n "$mon_ns" get pod -l app.kubernetes.io/name=grafana --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
    if [[ -n "$grafana_pod" ]]; then
      # Get admin user from secret
      local admin_user
      admin_user=$(kubectl -n "$mon_ns" get secret kps-grafana -o jsonpath='{.data.admin-user}' 2>/dev/null | base64 -d 2>/dev/null || echo "admin")
      
      # Wait for Grafana to be ready
      local i
      for i in {1..15}; do
        # Check if it's even up first
        if kubectl -n "$mon_ns" exec "$grafana_pod" -c grafana -- curl -s http://localhost:3000/api/health >/dev/null 2>&1; then
          # Try one auth
          if kubectl -n "$mon_ns" exec "$grafana_pod" -c grafana -- curl -s -f -u "${admin_user}:${pw}" http://localhost:3000/api/org >/dev/null 2>&1; then
            echo "  [PASS] Grafana access (user: ${admin_user})"
            break
          else
            # If 401, maybe wait longer or it failed
            echo "  [WAIT] Grafana login failed (i=$i), retrying..."
          fi
        fi
        sleep 5
      done
      if [[ $i -eq 15 ]]; then
        echo "  [FAIL] Grafana access (user: ${admin_user})"
      fi
    fi
  fi
  
  # 3. Prometheus check
  if kubectl get namespace "$mon_ns" >/dev/null 2>&1; then
     # Try from Grafana pod if it has curl
     if [[ -n "$grafana_pod" ]]; then
       if kubectl -n "$mon_ns" exec "$grafana_pod" -c grafana -- curl -s -f http://kps-kube-prometheus-stack-prometheus:9090/-/healthy >/dev/null 2>&1; then
         echo "  [PASS] Prometheus health check (via Grafana pod)"
       else
         echo "  [FAIL] Prometheus health check"
       fi
     fi
  fi
  
  # 4. Supabase check
  if kubectl get namespace "supabase" >/dev/null 2>&1; then
     local sb_pod
     sb_pod=$(kubectl -n supabase get pod -l app=kong -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
     if [[ -n "$sb_pod" ]]; then
        # Supabase uses JWT often, but we can check if we can connect to the DB via kong if exposed
        # Or just check if secrets are set correctly
        if verify_k8s_secret "supabase" "supabase-db-password" "password" "$pw"; then
           echo "  [PASS] Supabase secret verification"
        else
           # Try generic check for any secret containing the password
           local found=0
           local s
           for s in $(kubectl -n supabase get secret -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
              if verify_k8s_secret "supabase" "$s" "password" "$pw" || verify_k8s_secret "supabase" "$s" "POSTGRES_PASSWORD" "$pw"; then
                 found=1; break
              fi
           done
           if [[ $found -eq 1 ]]; then
              echo "  [PASS] Supabase secret verification (found in $s)"
           else
              echo "  [FAIL] Supabase secret verification"
           fi
        fi
     fi
  fi
}

update_kdc_password() {
  local user="$1" realm="$2" new_pw="$3"
  local kdc_ns kdc_name pod
  kdc_ns=${PROLE_KDC_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}}
  kdc_name=${PROLE_KDC_NAME:-auth}
  pod=$(kubectl -n "$kdc_ns" get pod -l "app=${kdc_name}" --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -n "$pod" ]]; then
    echo "Updating password in internal KDC for ${user}@${realm}..."
    # Attempt to change password via kadmin.local
    if kubectl -n "$kdc_ns" exec "$pod" -- kadmin.local -q "cpw -pw ${new_pw} ${user}@${realm}" >/dev/null 2>&1; then
      echo "  [PASS] KDC password updated."
      return 0
    fi
    # If realm in config doesn't match the one in KDC, it might fail.
    # Try with default realm if it fails?
    local default_realm
    default_realm=$(kubectl -n "$kdc_ns" exec "$pod" -- cat /etc/krb5.conf | grep default_realm | awk '{print $3}' || true)
    if [[ -n "$default_realm" && "$default_realm" != "$realm" ]]; then
       if kubectl -n "$kdc_ns" exec "$pod" -- kadmin.local -q "cpw -pw ${new_pw} ${user}@${default_realm}" >/dev/null 2>&1; then
         echo "  [PASS] KDC password updated (using default realm ${default_realm})."
         return 0
       fi
    fi
    echo "  [FAIL] KDC password update failed."
  fi
  return 1
}

update_cfg_keys() {
  local value="$1"
  python3 - "$CFG_PATH" "$value" <<'PY'
import re
import sys

cfg_path = sys.argv[1]
value = sys.argv[2]
keys = {
    "init_password.db_password",
    "init_password.db_password_confirm",
    "kerberos_config.password",
    "DB_PASSWORD",
    "PASSWORD",
    "GRAFANA_ADMIN_PASSWORD",
}
pattern = re.compile(r"^(?P<lead>\s*)(?P<key>[^=]+?)(?P<pre>\s*)=(?P<post>\s*).*$")

out_lines = []
with open(cfg_path, "r", encoding="utf-8") as f:
    for line in f.read().splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith((';', '#')) or '=' not in line:
            out_lines.append(line)
            continue
        if stripped.startswith('[') and stripped.endswith(']'):
            out_lines.append(line)
            continue
        m = pattern.match(line)
        if not m:
            out_lines.append(line)
            continue
        key = m.group("key").strip()
        if key in keys:
            lead = m.group("lead")
            pre = m.group("pre")
            post = m.group("post")
            out_lines.append(f"{lead}{key}{pre}={post}{value}")
        else:
            out_lines.append(line)

with open(cfg_path, "w", encoding="utf-8") as f:
    f.write("\n".join(out_lines) + "\n")
PY
}

update_cfg_openbao_refs() {
  python3 - "$CFG_PATH" "$NAMESPACE" <<'PY'
import re
import sys

cfg_path = sys.argv[1]
ns = sys.argv[2]

def ref(leaf: str, key: str) -> str:
    return f"${{OPENBAO:kv/knoe/{ns}/{leaf}#{key}}}"

secret_map = {
    "init_password.db_password": ref("db", "password"),
    "init_password.db_password_confirm": ref("db", "password"),
    "kerberos_config.password": ref("kerberos", "password"),
    "DB_PASSWORD": ref("db", "password"),
    "PASSWORD": ref("kerberos", "password"),
    "GRAFANA_ADMIN_PASSWORD": ref("monitoring", "grafana_admin_password"),
}

pattern = re.compile(r"^(?P<lead>\s*)(?P<key>[^=]+?)(?P<pre>\s*)=(?P<post>\s*).*$")

out_lines = []
with open(cfg_path, "r", encoding="utf-8") as f:
    for line in f.read().splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith((';', '#')) or '=' not in line:
            out_lines.append(line)
            continue
        if stripped.startswith('[') and stripped.endswith(']'):
            out_lines.append(line)
            continue
        m = pattern.match(line)
        if not m:
            out_lines.append(line)
            continue
        key = m.group("key").strip()
        if key in secret_map:
            lead = m.group("lead")
            pre = m.group("pre")
            post = m.group("post")
            out_lines.append(f"{lead}{key}{pre}={post}{secret_map[key]}")
        else:
            out_lines.append(line)

with open(cfg_path, "w", encoding="utf-8") as f:
    f.write("\n".join(out_lines) + "\n")
PY
}

write_openbao_secrets() {
  local url token
  if ! token=$(openbao_token); then
    echo "WARN: OpenBao token not available; skipping OpenBao update." >&2
    return 1
  fi
  if ! url=$(openbao_url); then
    echo "WARN: OpenBao not reachable; skipping OpenBao update." >&2
    return 1
  fi

  if ! curl -sS -H "X-Vault-Token: $token" "$url/v1/sys/mounts" | grep -q '\"kv/\"'; then
    curl -sS -H "X-Vault-Token: $token" -X POST "$url/v1/sys/mounts/kv" \
      -d '{"type":"kv","options":{"version":"2"}}' >/dev/null 2>&1 || true
  fi

  local db_payload krb_payload grafana_payload
  db_payload=$(python3 - "$DB_USER" "$NEW_PASSWORD" <<'PY'
import json
import sys
print(json.dumps({"data": {"username": sys.argv[1], "password": sys.argv[2]}}))
PY
)
  krb_payload=$(python3 - "$KRB5_USER" "$KRB5_REALM" "$KRB5_KDC" "$NEW_PASSWORD" <<'PY'
import json
import sys
print(json.dumps({"data": {"username": sys.argv[1], "realm": sys.argv[2], "kdc": sys.argv[3], "password": sys.argv[4]}}))
PY
)
  grafana_payload=$(python3 - "$NEW_PASSWORD" <<'PY'
import json
import sys
print(json.dumps({"data": {"grafana_admin_password": sys.argv[1]}}))
PY
)

  curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
    -X POST "$url/v1/kv/data/knoe/${NAMESPACE}/db" -d "$db_payload" >/dev/null
  curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
    -X POST "$url/v1/kv/data/knoe/${NAMESPACE}/kerberos" -d "$krb_payload" >/dev/null
  curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
    -X POST "$url/v1/kv/data/knoe/${NAMESPACE}/monitoring" -d "$grafana_payload" >/dev/null

  return 0
}

update_k8s_db_secrets() {
  local ns="$NAMESPACE"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    return 0
  fi
  kubectl create secret generic knoe-db-user -n "$ns" \
    --from-literal=username="$DB_USER" \
    --from-literal=password="$NEW_PASSWORD" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null

  kubectl create secret generic knoe-db-superuser -n "$ns" \
    --from-literal=username=postgres \
    --from-literal=password="$NEW_PASSWORD" \
    --dry-run=client -o yaml | kubectl apply -f - >/dev/null
}

update_grafana_secret() {
  local ns="${MONITORING_NAMESPACE:-monitoring}"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    return 0
  fi

  local pw_b64
  pw_b64=$(printf '%s' "$NEW_PASSWORD" | base64 | tr -d '\n')

  local secret
  for secret in kps-grafana grafana; do
    if kubectl -n "$ns" get secret "$secret" >/dev/null 2>&1; then
      if kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.admin-password}' >/dev/null 2>&1; then
        kubectl -n "$ns" patch secret "$secret" --type merge \
          -p "{\"data\":{\"admin-password\":\"${pw_b64}\"}}" >/dev/null
      fi
      if kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.adminPassword}' >/dev/null 2>&1; then
        kubectl -n "$ns" patch secret "$secret" --type merge \
          -p "{\"data\":{\"adminPassword\":\"${pw_b64}\"}}" >/dev/null
      fi
      return 0
    fi
  done
}

update_grafana_cli_reset() {
  local ns="${MONITORING_NAMESPACE:-monitoring}"
  local pod
  # Get the newest running pod
  pod=$(kubectl -n "$ns" get pod -l app.kubernetes.io/name=grafana --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
  if [[ -n "$pod" ]]; then
     echo "  Updating Grafana admin password via CLI in pod $pod..."
     # Wait for DB to be ready inside the pod and retry if locked
     local i
     for i in {1..5}; do
       if kubectl -n "$ns" exec "$pod" -c grafana -- grafana cli admin reset-admin-password "$NEW_PASSWORD" >/dev/null 2>&1; then
         echo "  [PASS] Grafana CLI reset successful."
         return 0
       elif kubectl -n "$ns" exec "$pod" -c grafana -- grafana-cli admin reset-admin-password "$NEW_PASSWORD" >/dev/null 2>&1; then
         echo "  [PASS] Grafana-CLI reset successful."
         return 0
       fi
       echo "  [WAIT] Grafana CLI reset failed (possibly DB locked), retrying (i=$i)..."
       sleep 5
     done
     echo "  [FAIL] Grafana CLI reset failed after 5 attempts."
  fi
  return 1
}

update_supabase_secrets() {
  local ns="supabase"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    return 0
  fi

  local pw_b64
  pw_b64=$(printf '%s' "$NEW_PASSWORD" | base64 | tr -d '\n')

  local secrets
  secrets=$(kubectl -n "$ns" get secret -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
  local s
  for s in $secrets; do
    local updated=0
    for key in POSTGRES_PASSWORD PGPASSWORD DB_PASSWORD SUPABASE_DB_PASSWORD SUPABASE_PASSWORD; do
      if kubectl -n "$ns" get secret "$s" -o jsonpath="{.data.${key}}" >/dev/null 2>&1; then
        kubectl -n "$ns" patch secret "$s" --type merge \
          -p "{\"data\":{\"${key}\":\"${pw_b64}\"}}" >/dev/null
        updated=1
      fi
    done
    if [[ $updated -eq 1 ]]; then
      :
    fi
  done
}

verify_k8s_secret() {
  local ns="$1" secret="$2" key="$3" expected="$4"
  local val
  val=$(kubectl -n "$ns" get secret "$secret" -o jsonpath="{.data.${key}}" 2>/dev/null || true)
  if [[ -z "$val" ]]; then
    return 1
  fi
  val=$(printf '%s' "$val" | base64 -d 2>/dev/null || true)
  if [[ "$val" == '${'* ]]; then
    echo "  [FAIL] Secret $ns/$secret key $key contains a variable: $val" >&2
    return 1
  fi
  [[ "$val" == "$expected" ]]
}

# 1) Verify current password (unless forced)
if [[ "$FORCE" -eq 0 ]]; then
  current_pw=$(prompt_password "Current password: ")
  if ! verify_current_password "$current_pw"; then
    exit 1
  fi
fi

# 2) Prompt for new password
NEW_PASSWORD=$(prompt_password "New password: ")
confirm_pw=$(prompt_password "Repeat new password: ")
if [[ -z "$NEW_PASSWORD" ]]; then
  echo "ERROR: New password cannot be empty." >&2
  exit 1
fi
if [[ "$NEW_PASSWORD" != "$confirm_pw" ]]; then
  echo "ERROR: Passwords do not match." >&2
  exit 1
fi

# 3) Store at-rest encrypted in knoe.cfg
ENC_PW=$(python3 - "$ROOT_DIR" "$NEW_PASSWORD" <<'PY'
import sys

root = sys.argv[1]
pw = sys.argv[2]

sys.path.insert(0, root)
from knoe import config as inst_config

print(inst_config._encrypt_knoe_secret(pw))
PY
)

update_cfg_keys "$ENC_PW"

# 4) Push to OpenBao (if available) and replace knoe.cfg refs
OPENBAO_UPDATED=0
if write_openbao_secrets; then
  OPENBAO_UPDATED=1
  update_cfg_openbao_refs
fi

# 5) Update Kubernetes secrets (if available)
if command -v kubectl >/dev/null 2>&1; then
  knoe_ensure_kubeconfig >/dev/null 2>&1 || true
  update_k8s_db_secrets
  update_grafana_secret
  update_supabase_secrets
  update_kdc_password "$KRB5_USER" "$KRB5_REALM" "$NEW_PASSWORD"
  restart_pods
fi

# 6) Verification
verify_access "$NEW_PASSWORD"

if [[ "$OPENBAO_UPDATED" -eq 1 ]]; then
  if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then
    if url=$(openbao_url); then
      export PROLE_OPENBAO_URL="$url"
    fi
  fi
  resolved=$(_knoe_resolve_openbao_ref "$(openbao_ref "db" "password")")
  if [[ "$resolved" != "$NEW_PASSWORD" ]]; then
    echo "WARN: OpenBao value did not match expected password." >&2
  fi
fi

if command -v kubectl >/dev/null 2>&1; then
  if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
    if ! verify_k8s_secret "$NAMESPACE" "knoe-db-user" "password" "$NEW_PASSWORD"; then
      echo "WARN: knoe-db-user secret does not match expected password." >&2
    fi
    if ! verify_k8s_secret "$NAMESPACE" "knoe-db-superuser" "password" "$NEW_PASSWORD"; then
      echo "WARN: knoe-db-superuser secret does not match expected password." >&2
    fi
  fi
  if kubectl get namespace "${MONITORING_NAMESPACE:-monitoring}" >/dev/null 2>&1; then
    if kubectl -n "${MONITORING_NAMESPACE:-monitoring}" get secret kps-grafana >/dev/null 2>&1; then
      if ! verify_k8s_secret "${MONITORING_NAMESPACE:-monitoring}" "kps-grafana" "admin-password" "$NEW_PASSWORD"; then
        echo "WARN: Grafana admin secret does not match expected password." >&2
      fi
    elif kubectl -n "${MONITORING_NAMESPACE:-monitoring}" get secret grafana >/dev/null 2>&1; then
      if ! verify_k8s_secret "${MONITORING_NAMESPACE:-monitoring}" "grafana" "admin-password" "$NEW_PASSWORD"; then
        echo "WARN: Grafana admin secret does not match expected password." >&2
      fi
    fi
  fi
fi

echo "Password update complete."