prole/mock_val/init_openbao.sh

#!/usr/bin/env bash

set -euo pipefail

# init_openbao.sh
# Purpose:
# - Run OpenBao inside Kubernetes and store admin key pair
# - Optionally generate Kerberos krb5.conf ConfigMap for external realm (legacy k8s flow)
# - Store secrets in OpenBao kv (admin keys, user keys, db password, optional TDE keys)

# Initialize SCRIPT_DIR
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"

# Inject default config if not provided
_has_config=0
for _arg in "$@"; do
  [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
done
if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then
  set -- "-c" "$SCRIPT_DIR/../conf/prole.cfg" "$@"
fi
unset _has_config _arg

common_core_preparse_config "$@"

# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

set -- "${COMMON_CORE_ARGS[@]}"
common_core_parse_args "$@"

if [[ -z "${PROLE_MODE:-}" ]]; then
  export PROLE_MODE="k3s"
fi

if [[ "${COMMON_CORE_HELP:-0}" == 1 ]]; then
  common_core_usage "$0"
  exit 0
fi

if [[ -n "${COMMON_CORE_PARSE_ERROR:-}" ]]; then
  echo "ERROR: ${COMMON_CORE_PARSE_ERROR}" >&2
  common_core_usage "$0"
  exit 2
fi

PRIMARY_SERVICE_NAMESPACE="${SERVICE_NAMESPACE:-default}"
ACTION="$COMMON_CORE_ACTION"
NAMESPACE="$(common_core_resolve_namespace "default")"
common_core_apply_namespace "$NAMESPACE"

if [[ -z "${PROLE_SERVICE:-}" ]]; then
  echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2
  exit 1
fi

# Defaults
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
OPENBAO_IMAGE=${OPENBAO_IMAGE:-ghcr.io/openbao/openbao:latest}
OPENBAO_PORT_FORWARD_LOCAL=${OPENBAO_PORT_FORWARD_LOCAL:-8200}
OPENBAO_PF_ADDR="${OPENBAO_PF_ADDR:-}"
if [[ -z "$OPENBAO_PF_ADDR" ]]; then
  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
    OPENBAO_PF_ADDR="0.0.0.0"
  else
    OPENBAO_PF_ADDR="127.0.0.1"
  fi
fi

# Support both PROLE_HOME/k8s and sibling k8s directory
if [[ -d "$SCRIPT_DIR/../k8s/openbao" ]]; then
  OPENBAO_MANIFEST_DIR="$SCRIPT_DIR/../k8s/openbao"
elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/openbao" ]]; then
  OPENBAO_MANIFEST_DIR="$PROLE_HOME/k8s/openbao"
else
  OPENBAO_MANIFEST_DIR="$SCRIPT_DIR/../k8s/openbao"
fi

# Kerberos realm defaults
KRB5_REALM=${KRB5_REALM:-PROLE.ORG}
DOMAIN=${DOMAIN:-$(echo "$KRB5_REALM" | tr 'A-Z' 'a-z')}
KRB5_KDC=${KRB5_KDC:-kdc.$DOMAIN}
KRB5_ADMIN=${KRB5_ADMIN:-}

# Secrets and token locations
# Use PROLE_SERVICE if writable locally, otherwise fallback to ~/.prole
if [[ -n "${PROLE_SERVICE:-}" ]] && _prole_usable_dir "${PROLE_SERVICE}/secrets" >/dev/null; then
  SECRETS_DIR="$PROLE_SERVICE/secrets"
else
  SECRETS_DIR="$HOME/.prole/etc/secrets"
fi
mkdir -p "$SECRETS_DIR"
admin_key_priv="$SECRETS_DIR/admin_ed25519.key"
admin_key_pub="$SECRETS_DIR/admin_ed25519.pub"
root_token_file="$SECRETS_DIR/openbao-root-token"

OPENBAO_NAMESPACE=${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}}
# OpenBao resources (StatefulSet/Service) live in `OPENBAO_NAMESPACE` (usually the service namespace),
# but secret *paths* must be keyed by the Prole workload namespace from `conf/prole.cfg` so that
# `${OPENBAO:kv/prole/<PROLE_NAMESPACE>/...}` references resolve consistently across scripts.
OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${PROLE_NAMESPACE:-${NAMESPACE:-default}}}
OPENBAO_RESOURCE_NAMESPACE="$OPENBAO_NAMESPACE"

BAO_NAMESPACE="${OPENBAO_PATH_NAMESPACE:-default}"
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
BAO_PATH_USER="${BAO_PATH_PREFIX}/user"
BAO_PATH_DB="${BAO_PATH_PREFIX}/db"
BAO_PATH_TDE="${BAO_PATH_PREFIX}/tde"
BAO_PATH_KERBEROS="${BAO_PATH_PREFIX}/kerberos"
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"

# Default Grafana password to the database root password when not explicitly set.
GRAFANA_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-${DB_PASSWORD:-}}

ensure_tools() {
  for t in curl openssl base64 jq; do
    command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
  done
}

ensure_k8s_tools() {
  command -v kubectl >/dev/null || { echo "Missing required tool: kubectl" >&2; exit 1; }
}

ensure_namespace() {
  if ! kubectl get namespace "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
    echo "Creating namespace '$OPENBAO_RESOURCE_NAMESPACE' ..."
    kubectl create namespace "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1 || true
  fi
}


ensure_admin_keypair() {
  # Ensure admin keypair exists and a root token is available
  mkdir -p "$SECRETS_DIR"
  if [[ ! -f "$admin_key_priv" || ! -f "$admin_key_pub" ]]; then
    echo "Generating admin keypair in $SECRETS_DIR ..."
    # Attempt ed25519, fallback to rsa if not available
    if ! openssl genpkey -algorithm ED25519 -out "$admin_key_priv" 2>/dev/null; then
      echo "ED25519 not supported by openssl, falling back to RSA 4096..."
      openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out "$admin_key_priv"
    fi
    openssl pkey -in "$admin_key_priv" -pubout -out "$admin_key_pub"
    chmod 0600 "$admin_key_priv"
  fi
  # Ensure a root token exists for OpenBao dev server interactions.
  # If we can detect a token from manifests, prefer that even if a stale token file exists.
  local token=""
  token="${OPENBAO_ROOT_TOKEN:-}"
  if [[ -z "$token" ]]; then
    token=$(detect_openbao_root_token || true)
  fi
  if [[ -z "$token" ]]; then
    if [[ -f "$root_token_file" && -s "$root_token_file" ]]; then
      token=$(cat "$root_token_file" 2>/dev/null || true)
    fi
  fi
  if [[ -z "$token" ]]; then
    token="root"
  fi

  # Sync the token file if missing or mismatched.
  local current=""
  if [[ -f "$root_token_file" && -s "$root_token_file" ]]; then
    current=$(cat "$root_token_file" 2>/dev/null || true)
  fi
  if [[ -z "$current" || "$current" != "$token" ]]; then
    printf "%s" "$token" >"$root_token_file"
    chmod 0600 "$root_token_file"
  fi
}

generate_openbao_manifests() {
  mkdir -p "$OPENBAO_MANIFEST_DIR"
  # Only create a simple deployment if none exists; otherwise respect current file
  local deploy="$OPENBAO_MANIFEST_DIR/deployment.yaml"
  if [[ ! -f "$deploy" ]]; then
    cat >"$deploy" <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $OPENBAO_NAME
  namespace: $OPENBAO_RESOURCE_NAMESPACE
spec:
  replicas: 1
  selector:
    matchLabels:
      app: $OPENBAO_NAME
  template:
    metadata:
      labels:
        app: $OPENBAO_NAME
    spec:
      containers:
      - name: $OPENBAO_NAME
        image: $OPENBAO_IMAGE
        args: ["server","-dev","-dev-listen-address=0.0.0.0:8200"]
        ports:
        - containerPort: 8200
---
apiVersion: v1
kind: Service
metadata:
  name: $OPENBAO_NAME
  namespace: $OPENBAO_RESOURCE_NAMESPACE
spec:
  selector:
    app: $OPENBAO_NAME
  ports:
  - name: http
    port: 8200
    targetPort: 8200
EOF
  fi
}

bao_service_url() {
  if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
    echo "$PROLE_OPENBAO_URL"
    return 0
  fi

  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
    if prole_is_in_cluster; then
      echo "http://openbao.${OPENBAO_RESOURCE_NAMESPACE}.svc.cluster.local:8200"
      return 0
    fi
    # Use k3s server host if available
    local k3s_host
    k3s_host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-}")
    if [[ -n "$k3s_host" ]]; then
      echo "http://${k3s_host}:${OPENBAO_PORT_FORWARD_LOCAL:-8200}"
      return 0
    fi
  fi

  echo "http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}"
}


_OPENBAO_PF_PID=""
_OPENBAO_PF_STARTED=0

should_skip_deployment() {
  # If the current namespace is NOT the primary service namespace,
  # we assume it's a sub-namespace (like knoe-db) and we should use the common service.
  if [[ "$NAMESPACE" != "$PRIMARY_SERVICE_NAMESPACE" ]]; then
    return 0
  fi
  return 1
}

expose_common_openbao() {
  local target_ns="$OPENBAO_RESOURCE_NAMESPACE"
  local source_ns="$PRIMARY_SERVICE_NAMESPACE"

  if [[ "$target_ns" == "$source_ns" ]]; then
    return 0
  fi

  echo "Exposing common OpenBao service from namespace '$source_ns' to '$target_ns' ..."

  # Use ExternalName to point to the common OpenBao service
  cat <<EOF | kubectl apply -n "$target_ns" -f -
apiVersion: v1
kind: Service
metadata:
  name: $OPENBAO_NAME
spec:
  type: ExternalName
  externalName: $OPENBAO_NAME.$source_ns.svc.cluster.local
EOF
}

# Use PRIMARY_SERVICE_NAMESPACE if skipping deployment (we port-forward to the common svc)
start_openbao_port_forward() {
  # In k3s mode we should never rely on local kubectl port-forwards.
  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
    return 1
  fi

  local pf_ns="$OPENBAO_RESOURCE_NAMESPACE"
  if should_skip_deployment; then
    pf_ns="$PRIMARY_SERVICE_NAMESPACE"
  fi

  local port="${OPENBAO_PORT_FORWARD_LOCAL}"
  local url="http://127.0.0.1:${port}"

  if curl -sS "$url/v1/sys/health" >/dev/null 2>&1; then
    return 0
  fi

  kubectl -n "$pf_ns" port-forward --address "${OPENBAO_PF_ADDR:-127.0.0.1}" "svc/${OPENBAO_NAME}" "${port}:8200" >/dev/null 2>&1 &
  _OPENBAO_PF_PID=$!
  _OPENBAO_PF_STARTED=1

  for _ in {1..20}; do
    if curl -sS "$url/v1/sys/health" >/dev/null 2>&1; then
      return 0
    fi
    if ! kill -0 "$_OPENBAO_PF_PID" >/dev/null 2>&1; then
      break
    fi
    sleep 1
  done
  return 1
}

stop_openbao_port_forward() {
  if [[ "${_OPENBAO_PF_STARTED}" == "1" && -n "${_OPENBAO_PF_PID:-}" ]]; then
    kill "$_OPENBAO_PF_PID" >/dev/null 2>&1 || true
    wait "$_OPENBAO_PF_PID" >/dev/null 2>&1 || true
  fi
  _OPENBAO_PF_STARTED=0
  _OPENBAO_PF_PID=""
}

wait_for_openbao_http() {
  local max_retries=${1:-20}
  local svc="${2:-}"
  if [[ -z "$svc" ]]; then
    svc="$(bao_service_url)"
  fi
  local count=0
  while ! curl -sS "$svc/v1/sys/health" >/dev/null 2>&1; do
    if [[ $count -ge $max_retries ]]; then
      echo "ERROR: OpenBao not reachable at $svc after waiting." >&2
      return 1
    fi
    echo "Waiting for OpenBao to be ready... ($count/$max_retries)"
    sleep 2
    count=$((count + 1))
  done
  return 0
}

detect_openbao_root_token() {
  local token=""
  local file
  for file in "$OPENBAO_MANIFEST_DIR/deployment.yaml" \
              "$OPENBAO_MANIFEST_DIR/statefulset.yaml" \
              "$SCRIPT_DIR/../k8s/prole/openbao-statefulset.yaml"; do
    if [[ -f "$file" ]]; then
      token=$(grep -Eo 'dev-root-token-id=[^" ]+' "$file" 2>/dev/null | head -n1 | cut -d= -f2 || true)
      if [[ -z "$token" ]]; then
        token=$(grep -n "VAULT_DEV_ROOT_TOKEN_ID" -A2 "$file" 2>/dev/null \
          | grep -Eo 'value:[[:space:]]*[^[:space:]]+' | head -n1 | awk '{print $2}' || true)
      fi
      if [[ -n "$token" ]]; then
        echo "$token"
        return 0
      fi
    fi
  done
  return 1
}

generate_kerberos_configmap() {
  mkdir -p "$OPENBAO_MANIFEST_DIR"
  local domain_lower
  domain_lower=$(echo "$DOMAIN" | tr 'A-Z' 'a-z')

  # Split KRB5_KDC on comma into bash array
  local IFS=','
  read -r -a __kdcs <<< "$KRB5_KDC"
  local kdc_lines=""
  local kdc
  for kdc in "${__kdcs[@]}"; do
    kdc_lines+=$'        kdc = '"$kdc"$'\n'
  done
  local admin_server admin_line
  admin_server=${KRB5_ADMIN:-"${__kdcs[0]:-}"}
  admin_line=""
  if [[ -n "$admin_server" ]]; then
    # 8 spaces to remain within the YAML literal block indentation
    admin_line=$'        admin_server = '"$admin_server"$'\n'
  fi

  cat >"$OPENBAO_MANIFEST_DIR/kerberos-configmap.yaml" <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: prole-krb5-conf
  namespace: $OPENBAO_RESOURCE_NAMESPACE
data:
  krb5.conf: |
    [libdefaults]
      default_realm = $KRB5_REALM
      dns_lookup_realm = true
      dns_lookup_kdc = true

    [realms]
      $KRB5_REALM = {
${kdc_lines}${admin_line}      }

    [domain_realm]
      .$domain_lower = $KRB5_REALM
      $domain_lower = $KRB5_REALM
EOF
}

should_apply_kerberos_configmap() {
  local flag="${OPENBAO_APPLY_KRB5_CONFIGMAP:-}"
  if [[ "$flag" == "1" || "$flag" == "true" || "$flag" == "True" ]]; then
    if [[ -n "${KRB5_REALM:-}" && -n "${KRB5_KDC:-}" ]]; then
      return 0
    fi
    echo "WARN: OPENBAO_APPLY_KRB5_CONFIGMAP set but KRB5_REALM/KRB5_KDC missing; skipping Kerberos ConfigMap."
  fi
  return 1
}

apply_k8s() {
  echo "Applying OpenBao manifest to namespace '$OPENBAO_RESOURCE_NAMESPACE' ..."
  local use_statefulset=0
  if [[ "${PROLE_MODE:-}" != "k3d" && -f "$SCRIPT_DIR/../k8s/prole/openbao-statefulset.yaml" ]]; then
    use_statefulset=1
  fi
  if [[ "$use_statefulset" == "1" ]]; then
    # Ensure storage class and PVs exist for k3s
    if [[ -f "$SCRIPT_DIR/../k8s/prole/storageclass-synology-iscsi.yaml" ]]; then
      echo "Applying StorageClass 'synology-iscsi' ..."
      kubectl apply -f "$SCRIPT_DIR/../k8s/prole/storageclass-synology-iscsi.yaml"
    fi
    if [[ -f "$SCRIPT_DIR/../k8s/prole/iscsi-pvs.yaml" ]]; then
      echo "Applying iSCSI PersistentVolumes ..."
      kubectl apply -f "$SCRIPT_DIR/../k8s/prole/iscsi-pvs.yaml"
    fi

    local output=""
    if output=$(prole_render_manifest "$SCRIPT_DIR/../k8s/prole/openbao-statefulset.yaml" \
      | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f - 2>&1); then
      printf '%s\n' "$output"
    else
      if [[ "${PROLE_MODE:-}" == "k3d" && "$output" == *"updates to statefulset spec"* ]]; then
        echo "WARN: OpenBao StatefulSet immutable in k3d; skipping apply."
      else
        echo "$output" >&2
        return 1
      fi
    fi
    # Always apply the Service manifest, then enforce the desired exposure mode.
    prole_render_manifest "$SCRIPT_DIR/../k8s/prole/openbao-service.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f -

    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
      # Expose OpenBao externally in k3s (no kubectl port-forward). k3s `servicelb` will bind this
      # on the node, making it reachable from the installer host.
      kubectl patch svc "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --type merge \
        -p '{"spec":{"type":"LoadBalancer"}}' >/dev/null 2>&1 || true
    fi
  else
    prole_render_manifest "$OPENBAO_MANIFEST_DIR/deployment.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f -
  fi
  if should_apply_kerberos_configmap && [[ -f "$OPENBAO_MANIFEST_DIR/kerberos-configmap.yaml" ]]; then
    echo "Applying Kerberos ConfigMap (external realm) to namespace '$OPENBAO_RESOURCE_NAMESPACE' ..."
    kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f "$OPENBAO_MANIFEST_DIR/kerberos-configmap.yaml"
  fi
}

k3d_cleanup_pending_openbao_pvc() {
  if [[ "${PROLE_MODE:-}" != "k3d" ]]; then
    return 0
  fi
  if ! kubectl get pvc data-openbao-0 -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
    return 0
  fi
  local phase selector
  phase=$(kubectl get pvc data-openbao-0 -n "$OPENBAO_RESOURCE_NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || true)
  if [[ "$phase" == "Bound" ]]; then
    return 0
  fi
  selector=$(kubectl get pvc data-openbao-0 -n "$OPENBAO_RESOURCE_NAMESPACE" -o jsonpath='{.spec.selector}' 2>/dev/null || true)
  if printf '%s' "$selector" | grep -q "synology.storage/"; then
    echo "Removing pending OpenBao PVC with selector for k3d local-path ..."
    kubectl delete statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
    kubectl delete pvc data-openbao-0 -n "$OPENBAO_RESOURCE_NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
  fi
}

recycle_released_prole_iscsi_pv_for_pvc() {
  # StorageClass `synology-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain
  # in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding.
  local ns="$1"
  local pvc="$2"

  if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then
    return 0
  fi

  local pvc_phase pvc_uid
  pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true)
  if [[ "$pvc_phase" != "Pending" ]]; then
    return 0
  fi
  pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)

  local pv sc rp phase claim_ns claim_name claim_uid
  while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do
    [[ "$sc" == "synology-iscsi" ]] || continue
    [[ "$rp" == "Retain" ]] || continue
    [[ "$phase" == "Released" ]] || continue
    [[ "$claim_ns" == "$ns" ]] || continue
    [[ "$claim_name" == "$pvc" ]] || continue

    # If the PV claimRef UID matches the current PVC UID, don't touch it.
    if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then
      continue
    fi

    echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..."
    kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
  done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true)
}

apply_openbao_host_network_patch() {
  if [[ "${OPENBAO_HOST_NETWORK:-0}" != "1" ]]; then
    return
  fi
  echo "Enabling hostNetwork for OpenBao (OPENBAO_HOST_NETWORK=1) ..."
  if kubectl get statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
    kubectl patch statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --type merge \
      -p '{"spec":{"template":{"spec":{"hostNetwork":true,"dnsPolicy":"ClusterFirstWithHostNet"}}}}' >/dev/null
  elif kubectl get deployment "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
    kubectl patch deployment "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --type merge \
      -p '{"spec":{"template":{"spec":{"hostNetwork":true,"dnsPolicy":"ClusterFirstWithHostNet"}}}}' >/dev/null
  fi
}

wait_for_openbao() {
  local wait_ns="$OPENBAO_RESOURCE_NAMESPACE"
  if should_skip_deployment; then
    wait_ns="$PRIMARY_SERVICE_NAMESPACE"
  fi

  local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}"

  echo "Waiting for OpenBao to become ready in namespace '$wait_ns' ..."
  if kubectl get statefulset/$OPENBAO_NAME -n "$wait_ns" >/dev/null 2>&1; then
    if ! kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then
      echo "WARN: OpenBao not ready after $initial_timeout; checking for Released synology-iscsi PVs with stale claimRefs ..." >&2
      recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true
    fi
    kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s}
  else
    if ! kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then
      echo "WARN: OpenBao not ready after $initial_timeout; checking for Released synology-iscsi PVs with stale claimRefs ..." >&2
      recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true
    fi
    kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s}
  fi
}

init_openbao_kv_and_store_admin_key() {
  local token
  token=$(cat "$root_token_file")
  # Enable kv (if not already) and write keys
  local svc="${1:-}"
  if [[ -z "$svc" ]]; then
    svc="$(bao_service_url)"
  fi
  echo "Configuring KV at $svc ..."

  # Check if kv/ is already mounted
  if curl -sS -H "X-Vault-Token: $token" "$svc/v1/sys/mounts" | grep -q '"kv/":'; then
    echo "KV path 'kv/' is already enabled."
  else
    curl -sS -H "X-Vault-Token: $token" -X POST "$svc/v1/sys/mounts/kv" \
      -d '{"type":"kv","options":{"version":"2"}}' >/dev/null 2>&1 || true
  fi

  local priv pub
  priv=$(base64 <"$admin_key_priv" | tr -d '\n')
  pub=$(base64 <"$admin_key_pub" | tr -d '\n')

  # Check if admin key already exists and matches
  local existing_data
  existing_data=$(curl -sS -H "X-Vault-Token: $token" "$svc/v1/kv/data/$BAO_PATH_ADMIN" 2>/dev/null || true)
  if [[ -n "$existing_data" ]]; then
    local ex_priv ex_pub
    ex_priv=$(echo "$existing_data" | grep -o '"admin_private_key_b64":"[^"]*' | cut -d'"' -f4 || true)
    ex_pub=$(echo "$existing_data" | grep -o '"admin_public_key_b64":"[^"]*' | cut -d'"' -f4 || true)

    if [[ "$ex_priv" == "$priv" && "$ex_pub" == "$pub" ]]; then
      echo "Admin key pair in OpenBao kv/$BAO_PATH_ADMIN is already up to date."
    else
      echo "Writing admin key pair to kv/$BAO_PATH_ADMIN ..."
      # Preserve other fields (like root_password or ansible_public_key_b64)
      local current_json
      current_json=$(echo "$existing_data" | jq -c ".data.data")
      local new_json
      new_json=$(echo "$current_json" | jq -c ". + {\"admin_private_key_b64\":\"$priv\",\"admin_public_key_b64\":\"$pub\"}")
      
      curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
        -X POST "$svc/v1/kv/data/$BAO_PATH_ADMIN" \
        -d "{\"data\":$new_json}" >/dev/null
      echo "Stored admin key pair in OpenBao kv/$BAO_PATH_ADMIN."
    fi
  else
    echo "Writing admin key pair to kv/$BAO_PATH_ADMIN ..."
    curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
      -X POST "$svc/v1/kv/data/$BAO_PATH_ADMIN" \
      -d "{\"data\":{\"admin_private_key_b64\":\"$priv\",\"admin_public_key_b64\":\"$pub\"}}" >/dev/null
    echo "Stored admin key pair in OpenBao kv/$BAO_PATH_ADMIN."
  fi

  # Store Ansible public key if it exists
  local ansible_pub_path="$HOME/.ssh/id_ed25519_ansible.pub"
  if [[ -f "$ansible_pub_path" ]]; then
      echo "Storing Ansible public key in OpenBao kv/$BAO_PATH_ADMIN ..."
      local a_pub
      a_pub=$(base64 <"$ansible_pub_path" | tr -d '\n')
      
      local current_admin_data
      current_admin_data=$(curl -sS -H "X-Vault-Token: $token" "$svc/v1/kv/data/$BAO_PATH_ADMIN" | jq -c ".data.data" 2>/dev/null || echo "{}")
      local updated_admin_data
      updated_admin_data=$(echo "$current_admin_data" | jq -c ". + {\"ansible_public_key_b64\":\"$a_pub\"}")
      
      curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
        -X POST "$svc/v1/kv/data/$BAO_PATH_ADMIN" \
        -d "{\"data\":$updated_admin_data}" >/dev/null
      echo "Ansible public key stored."
  fi

  # Store User SSH keys if they exist
  local user_key_priv="$HOME/.ssh/id_prole_ed25519"
  local user_key_pub="$HOME/.ssh/id_prole_ed25519.pub"
  if [[ -f "$user_key_priv" && -f "$user_key_pub" ]]; then
      local u_priv u_pub
      u_priv=$(base64 <"$user_key_priv" | tr -d '\n')
      u_pub=$(base64 <"$user_key_pub" | tr -d '\n')
      local username=${KNOE_DB_USER:-"prole"}
      
      echo "Writing user SSH keys for '$username' to kv/$BAO_PATH_USER ..."
      curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
        -X POST "$svc/v1/kv/data/$BAO_PATH_USER" \
        -d "{\"data\":{\"username\":\"$username\",\"private_key_b64\":\"$u_priv\",\"public_key_b64\":\"$u_pub\"}}" >/dev/null
      echo "Stored user SSH keys in OpenBao kv/$BAO_PATH_USER."
  fi

  if [[ -n "$db_pass" ]]; then
    local username=${KNOE_DB_USER:-"prole"}
    echo "Writing database user password for '$username' to kv/$BAO_PATH_DB ..."
    curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
      -X POST "$svc/v1/kv/data/$BAO_PATH_DB" \
      -d "{\"data\":{\"username\":\"$username\",\"password\":\"$db_pass\"}}" >/dev/null
    echo "Stored database password in OpenBao kv/$BAO_PATH_DB."
  fi

  # Store TDE keypair when at-rest encryption is enabled
  local tde_enabled="${AT_REST_ENCRYPTION_ENABLED:-}"
  case "$tde_enabled" in
    1|true|TRUE|True|yes|YES|on|ON)
      local tde_priv="$SECRETS_DIR/tde_ed25519.key"
      local tde_pub="$SECRETS_DIR/tde_ed25519.pub"
      if [[ ! -f "$tde_priv" || ! -f "$tde_pub" ]]; then
        echo "Generating TDE keypair in $SECRETS_DIR ..."
        if ! openssl genpkey -algorithm ED25519 -out "$tde_priv" 2>/dev/null; then
          echo "ED25519 not supported by openssl, falling back to RSA 4096..."
          openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out "$tde_priv"
        fi
        openssl pkey -in "$tde_priv" -pubout -out "$tde_pub"
        chmod 0600 "$tde_priv"
      fi

      local tde_priv_b64 tde_pub_b64
      tde_priv_b64=$(base64 <"$tde_priv" | tr -d '\n')
      tde_pub_b64=$(base64 <"$tde_pub" | tr -d '\n')

      local tde_existing
      tde_existing=$(curl -sS -H "X-Vault-Token: $token" "$svc/v1/kv/data/$BAO_PATH_TDE" 2>/dev/null || true)
      if [[ -n "$tde_existing" ]]; then
        local ex_priv ex_pub
        ex_priv=$(echo "$tde_existing" | grep -o '"tde_private_key_b64":"[^"]*' | cut -d'"' -f4 || true)
        ex_pub=$(echo "$tde_existing" | grep -o '"tde_public_key_b64":"[^"]*' | cut -d'"' -f4 || true)
        if [[ "$ex_priv" == "$tde_priv_b64" && "$ex_pub" == "$tde_pub_b64" ]]; then
          echo "TDE keypair in OpenBao kv/$BAO_PATH_TDE is already up to date."
        else
          echo "Writing TDE keypair to kv/$BAO_PATH_TDE ..."
          local tde_json
          tde_json=$(echo "$tde_existing" | jq -c ".data.data")
          local new_tde_json
          new_tde_json=$(echo "$tde_json" | jq -c ". + {\"tde_private_key_b64\":\"$tde_priv_b64\",\"tde_public_key_b64\":\"$tde_pub_b64\"}")
          curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
            -X POST "$svc/v1/kv/data/$BAO_PATH_TDE" \
            -d "{\"data\":$new_tde_json}" >/dev/null
          echo "Stored TDE keypair in OpenBao kv/$BAO_PATH_TDE."
        fi
      else
        echo "Writing TDE keypair to kv/$BAO_PATH_TDE ..."
        curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
          -X POST "$svc/v1/kv/data/$BAO_PATH_TDE" \
          -d "{\"data\":{\"tde_private_key_b64\":\"$tde_priv_b64\",\"tde_public_key_b64\":\"$tde_pub_b64\"}}" >/dev/null
        echo "Stored TDE keypair in OpenBao kv/$BAO_PATH_TDE."
      fi
      ;;
  esac

  local krb_user
  krb_user="${KRB5_USER:-${KRB5_USERNAME:-}}"
  if [[ -n "${KRB5_PASSWORD:-}" && -n "${KRB5_REALM:-}" && -n "$krb_user" ]]; then
    echo "Writing Kerberos credentials for '$krb_user@${KRB5_REALM}' to kv/$BAO_PATH_KERBEROS ..."
    curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
      -X POST "$svc/v1/kv/data/$BAO_PATH_KERBEROS" \
      -d "{\"data\":{\"username\":\"$krb_user\",\"realm\":\"$KRB5_REALM\",\"kdc\":\"${KRB5_KDC:-}\",\"password\":\"$KRB5_PASSWORD\"}}" >/dev/null
    echo "Stored Kerberos credentials in OpenBao kv/$BAO_PATH_KERBEROS."
  fi

  if [[ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
    echo "Writing Grafana admin password to kv/$BAO_PATH_MONITORING ..."
    local _clean_grafana_pw
    _clean_grafana_pw="${GRAFANA_ADMIN_PASSWORD//$'\n'/}"
    _clean_grafana_pw="${_clean_grafana_pw//$'\r'/}"
    curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
      -X POST "$svc/v1/kv/data/$BAO_PATH_MONITORING" \
      -d "{\"data\":{\"grafana_admin_password\":\"$_clean_grafana_pw\"}}" >/dev/null
    echo "Stored Grafana admin password in OpenBao kv/$BAO_PATH_MONITORING."
  fi
}

# Removed: prompt_admin_password_and_apply_secret (no AD admin secret required)

# status command implementation wrapped in a function to avoid top-level 'local'
cmd_status() {
  echo "--- openbao status ---"
  echo "Namespace: $OPENBAO_RESOURCE_NAMESPACE"
  echo "Secret namespace: $BAO_NAMESPACE"
  echo "OpenBao name: $OPENBAO_NAME"
  echo "Manifest dir: $OPENBAO_MANIFEST_DIR"

  # Files/manifests present
  local ok=0
  if [[ -f "$OPENBAO_MANIFEST_DIR/deployment.yaml" ]]; then
    echo "[OK] OpenBao deployment manifest exists"
  else
    echo "[MISSING] $OPENBAO_MANIFEST_DIR/deployment.yaml"
    ok=1
  fi
  if [[ -f "$OPENBAO_MANIFEST_DIR/kerberos-configmap.yaml" ]]; then
    echo "[OK] Kerberos ConfigMap manifest exists"
  else
    echo "[MISSING] $OPENBAO_MANIFEST_DIR/kerberos-configmap.yaml"
    ok=1
  fi

  # K8s resources (legacy flow; best-effort only)
  if command -v kubectl >/dev/null 2>&1; then
    if kubectl -n "$OPENBAO_RESOURCE_NAMESPACE" get statefulset "$OPENBAO_NAME" >/dev/null 2>&1; then
      local ready desired
      ready=$(kubectl -n "$OPENBAO_RESOURCE_NAMESPACE" get statefulset "$OPENBAO_NAME" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
      desired=$(kubectl -n "$OPENBAO_RESOURCE_NAMESPACE" get statefulset "$OPENBAO_NAME" -o jsonpath='{.status.replicas}' 2>/dev/null || echo "0")
      ready=${ready:-0}
      desired=${desired:-0}
      if [[ "$ready" == "$desired" && "$ready" != "0" ]]; then
        echo "[OK] OpenBao StatefulSet running ($ready/$desired ready)"
      else
        echo "[WARN] OpenBao StatefulSet not fully ready ($ready/$desired)"
        ok=1
      fi
    elif kubectl -n "$OPENBAO_RESOURCE_NAMESPACE" get deploy "$OPENBAO_NAME" >/dev/null 2>&1; then
      local ready desired
      ready=$(kubectl -n "$OPENBAO_RESOURCE_NAMESPACE" get deploy "$OPENBAO_NAME" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
      desired=$(kubectl -n "$OPENBAO_RESOURCE_NAMESPACE" get deploy "$OPENBAO_NAME" -o jsonpath='{.status.replicas}' 2>/dev/null || echo "0")
      ready=${ready:-0}
      desired=${desired:-0}
      if [[ "$ready" == "$desired" && "$ready" != "0" ]]; then
        echo "[OK] OpenBao Deployment running ($ready/$desired ready)"
      else
        echo "[WARN] OpenBao Deployment not fully ready ($ready/$desired)"
        ok=1
      fi
    fi
    if kubectl -n "$OPENBAO_RESOURCE_NAMESPACE" get configmap prole-krb5-conf >/dev/null 2>&1; then
      echo "[OK] Kerberos ConfigMap 'prole-krb5-conf' present"
    fi
  fi

  # Secrets and tokens
  if [[ -f "$admin_key_priv" && -f "$admin_key_pub" ]]; then
    echo "[OK] Admin keypair present in $SECRETS_DIR"
  else
    echo "[MISSING] Admin keypair files in $SECRETS_DIR"
    ok=1
  fi
  if [[ -s "$root_token_file" ]]; then
    echo "[OK] OpenBao root token file present"
  else
    echo "[MISSING] OpenBao root token file ($root_token_file)"
    ok=1
  fi

  # OpenBao reachability and token validity (best-effort)
  # Prefer explicit override, then localhost port-forward, then cluster DNS
  local svc_url
  svc_url="$(bao_service_url)"
  
  local pf_started=0
  if ! curl -sS "$svc_url/v1/sys/health" >/dev/null 2>&1; then
    if start_openbao_port_forward; then
      pf_started=1
      # If we started a port-forward and the original URL was not local, 
      # we should use the local URL for status verification.
      if [[ "$svc_url" != *"127.0.0.1"* && "$svc_url" != *"localhost"* ]]; then
        local local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}"
        if curl -sS "$local_url/v1/sys/health" >/dev/null 2>&1; then
          svc_url="$local_url"
        fi
      fi
    fi
  fi

  if curl -sS "$svc_url/v1/sys/health" >/dev/null 2>&1; then
    echo "[OK] OpenBao service reachable"
    if [[ -s "$root_token_file" ]]; then
      local code
      code=$(curl -s -o /dev/null -w "%{http_code}" -H "X-Vault-Token: $(cat "$root_token_file")" "$svc_url/v1/sys/mounts" || echo "000")
      if [[ "$code" == "200" ]]; then
        echo "[OK] OpenBao root token authenticates"
      else
        echo "[WARN] OpenBao root token did not authenticate (HTTP $code)"
        ok=1
      fi
    fi
  else
    echo "[WARN] OpenBao service not reachable at $svc_url"
    ok=1
  fi

  if [[ "$pf_started" == "1" ]]; then
    stop_openbao_port_forward
  fi

  echo "-------------------------------------"
  if [[ $ok -eq 0 ]]; then
    echo "Status: OK"
    return 0
  else
    echo "Status: Issues detected"
    return 1
  fi
}

case "$ACTION" in
  start)
    ensure_tools
    ensure_k8s_tools
    ensure_namespace
    if should_skip_deployment; then
      expose_common_openbao
    else
      generate_openbao_manifests
      if should_apply_kerberos_configmap; then
        generate_kerberos_configmap
      fi
      k3d_cleanup_pending_openbao_pvc
      apply_k8s
      apply_openbao_host_network_patch
    fi
    wait_for_openbao
    pf_ns="${OPENBAO_NAMESPACE:-default}"
    if should_skip_deployment; then
      pf_ns="$PRIMARY_SERVICE_NAMESPACE"
    fi
    if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
      prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
    fi
    ;;
  status)
    ensure_tools
    cmd_status
    ;;
  stop)
    ensure_k8s_tools
    if kubectl get statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
      kubectl scale statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --replicas=0 >/dev/null 2>&1 || true
    elif kubectl get deployment "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
      kubectl scale deployment "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --replicas=0 >/dev/null 2>&1 || true
    fi
    ;;
  restart)
    ensure_tools
    ensure_k8s_tools
    if kubectl get statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
      kubectl scale statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --replicas=0 >/dev/null 2>&1 || true
      kubectl scale statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --replicas=1 >/dev/null 2>&1 || true
    elif kubectl get deployment "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then
      kubectl scale deployment "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --replicas=0 >/dev/null 2>&1 || true
      kubectl scale deployment "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --replicas=1 >/dev/null 2>&1 || true
    fi
    wait_for_openbao
    ;;
  initialize)
    ensure_tools
    ensure_k8s_tools
    ensure_namespace
    if should_skip_deployment; then
      expose_common_openbao
    else
      generate_openbao_manifests
      if should_apply_kerberos_configmap; then
        generate_kerberos_configmap
      fi
      k3d_cleanup_pending_openbao_pvc
      apply_k8s
      apply_openbao_host_network_patch
    fi
    wait_for_openbao
    # Read password from stdin if provided
    db_pass=""
    if [[ ! -t 0 ]]; then
      read -r db_pass || true
    fi

    ensure_admin_keypair
    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
      # In k3s, initialize against the externally reachable endpoint (no port-forward).
      svc_url="$(bao_service_url)"
      wait_for_openbao_http 40 "$svc_url"
      init_openbao_kv_and_store_admin_key "$svc_url"
    else
      trap stop_openbao_port_forward EXIT
      if ! start_openbao_port_forward; then
        echo "ERROR: Unable to establish OpenBao port-forward on 127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" >&2
        exit 1
      fi

      # After port-forward, always use the local URL for initialization so we
      # don't depend on OpenBao being externally reachable from this host.
      local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}"
      wait_for_openbao_http 20 "$local_url"
      init_openbao_kv_and_store_admin_key "$local_url"
    fi
    pf_ns="${OPENBAO_NAMESPACE:-default}"
    if should_skip_deployment; then
      pf_ns="$PRIMARY_SERVICE_NAMESPACE"
    fi
    if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
      prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
    fi
    echo "Initialization complete. OpenBao available at $(bao_service_url)"
    ;;
  update|reload)
    ensure_tools
    ensure_k8s_tools
    ensure_namespace
    if should_skip_deployment; then
      expose_common_openbao
    else
      generate_openbao_manifests
      if should_apply_kerberos_configmap; then
        generate_kerberos_configmap
      fi
      k3d_cleanup_pending_openbao_pvc
      apply_k8s
      apply_openbao_host_network_patch
    fi
    pf_ns="${OPENBAO_NAMESPACE:-default}"
    if should_skip_deployment; then
      pf_ns="$PRIMARY_SERVICE_NAMESPACE"
    fi
    if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
      prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
    fi
    echo "Re-applied manifests."
    ;;
  *)
    common_core_usage "$0"
    exit 2
    ;;
esac