prole/prole.sh

#!/usr/bin/env bash
set -euo pipefail

# prole.sh - Unified launcher for Prole infrastructure and installer

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${ROOT_DIR}"

# OpenBao Detection
if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then
  if curl -s -f "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    export PROLE_OPENBAO_URL="http://127.0.0.1:8200"
  elif curl -s -f "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
    export PROLE_OPENBAO_URL="http://127.0.0.1:18200"
  fi
fi

if [[ -z "${OPENBAO_ROOT_TOKEN:-}" ]]; then
  if [[ -f "${ROOT_DIR}/etc/secrets/openbao-root-token" ]]; then
    export OPENBAO_ROOT_TOKEN=$(cat "${ROOT_DIR}/etc/secrets/openbao-root-token")
  fi
fi

# Defaults
VERBOSE=0
DEBUG=0
LOG=0
COVERAGE=0
SILENT=0
RESET=0
EXTRA_ARGS=()

usage() {
  echo "Usage: $0 [options] [command] [command-options]"
  echo
  echo "Options:"
  echo "  -l, --log        Create a datestamped logfile in prole/logs including all stdout/stderr"
  echo "  -v, --verbose    Pass a verbose switch to all functions called by ansible and install.py"
  echo "  -d, --debug      Enable a debug logfile in prole/logs with high verbosity from install.py"
  echo "  -c, --coverage   Create a coverage report for the run of install.py"
  echo "  -s, --silent     Run unattended install (passes -S to install.py)"
  echo "  -r, --reset      Perform a reset before action (site/deploy/install)"
  echo
  echo "Commands:"
  echo "  install    Run the ncurses-based terminal installer"
  echo "  ansible    Run ansible-playbook via ansible.sh"
  echo "  site       Shortcut for running the site deployment"
  echo "  reset      Shortcut for k3s factory reset (full clear)"
  echo "  deploy     Full end-to-end: site run, and silent install (reset optional)"
  echo "  start      Verify/clean environment, ensure cluster availability, run silent install"
  echo "  stop       Validate environment, run full backup+archive, stop services (keep cluster idle)"
  echo "  backup     Run a database backup (-f/full for full, default incremental)"
  echo "  passwd     Change the Prole master password and propagate secrets"
  echo
  echo "Any other arguments are passed directly to ansible.sh"
}

# Parse global options
while [[ $# -gt 0 ]]; do
  case "$1" in
    -l|--log|--logs) LOG=1; shift ;;
    -v|--verbose) VERBOSE=1; shift ;;
    -d|--debug) DEBUG=1; shift ;;
    -c|--coverage) COVERAGE=1; shift ;;
    -s|--silent) SILENT=1; shift ;;
    -r|--reset) RESET=1; shift ;;
    -h|--help) usage; exit 0 ;;
    install|ansible|site|reset|deploy|start|stop|backup|passwd) break ;;
    *) EXTRA_ARGS+=("$1"); shift ;;
  esac
done

if [[ $# -eq 0 && ${#EXTRA_ARGS[@]} -eq 0 ]]; then
  usage
  exit 1
fi

CMD="${1:-ansible}"
[[ $# -gt 0 ]] && shift

# Prepare command arguments
CMD_ARGS=()
if [[ "${VERBOSE}" -eq 1 ]]; then
  CMD_ARGS+=("--verbose")
  export PROLE_VERBOSE=1
  export VERBOSE=1
  export CLICOLOR_FORCE=1
  export PY_COLORS=1
  export ANSIBLE_FORCE_COLOR=1
fi

# We separate arguments that are only for install.py
INSTALL_ONLY_ARGS=()
if [[ "${DEBUG}" -eq 1 ]]; then
  INSTALL_ONLY_ARGS+=("--debug")
fi

if [[ "${SILENT}" -eq 1 ]]; then
  if [[ ! -f "${ROOT_DIR}/conf/prole.cfg" ]]; then
    echo "Error: --silent requires ${ROOT_DIR}/conf/prole.cfg" >&2
    exit 1
  fi
  INSTALL_ONLY_ARGS+=("-S")
fi

if [[ "${RESET}" -eq 1 ]]; then
  INSTALL_ONLY_ARGS+=("--reset")
fi

CMD_ARGS+=("${EXTRA_ARGS[@]}" "$@")

# Helper for logging
run_with_log() {
  local logfile="$1"
  shift
  mkdir -p "$(dirname "${logfile}")"
  echo "Logging to ${logfile}"
  "$@" 2>&1 | tee "${logfile}"
}

# Ensure ansible keeps color output even when piped.
prepare_ansible_color() {
  if [[ -z "${TERM:-}" ]]; then
    export TERM="xterm-256color"
  fi
  unset ANSIBLE_NOCOLOR
  export ANSIBLE_FORCE_COLOR=1
  export PY_COLORS=1
  export CLICOLOR_FORCE=1
}

PROLE_CFG_DEFAULT="${ROOT_DIR}/conf/prole.cfg"

log_msg() {
  printf '%s\n' "$*"
}

err_msg() {
  printf '[error] %s\n' "$*" >&2
}

load_prole_cfg() {
  local cfg="${1:-$PROLE_CFG_DEFAULT}"
  if [[ -f "${ROOT_DIR}/etc/prole_cfg.sh" ]]; then
    # shellcheck disable=SC1090
    source "${ROOT_DIR}/etc/prole_cfg.sh"
  fi
  if [[ -f "$cfg" ]]; then
    export PROLE_CONF
    PROLE_CONF="$(cd "$(dirname "$cfg")" && pwd)"
  fi
  export PROLE_CFG_FILE="$cfg"
}

resolve_prole_mode() {
  local raw="${PROLE_MODE:-${DEPLOYMENT_MODE:-${CLUSTER_ENV:-}}}"
  if command -v prole_normalize_mode >/dev/null 2>&1; then
    prole_normalize_mode "${raw:-}"
  else
    printf '%s' "${raw:-}"
  fi
}

resolve_k3d_cluster_name() {
  local name="${DEPLOYMENT_TARGET:-${DISPLAY_NAME:-}}"
  if [[ -z "$name" && -n "${KUBECTL_CONTEXT:-}" ]]; then
    if [[ "${KUBECTL_CONTEXT}" == k3d-* ]]; then
      name="${KUBECTL_CONTEXT#k3d-}"
    fi
  fi
  if [[ -z "$name" && -n "${CLUSTER_ENV:-}" ]]; then
    if [[ "${CLUSTER_ENV}" == k3d-* ]]; then
      name="${CLUSTER_ENV#k3d-}"
    else
      name="${CLUSTER_ENV}"
    fi
  fi
  printf '%s' "${name:-prole-dev-cluster}"
}

fix_k3d_kubeconfig_server() {
  local kubeconfig="$1"
  local context="$2"
  local desired_server="https://0.0.0.0:6443"
  if [[ -z "$kubeconfig" || ! -f "$kubeconfig" ]]; then
    return 0
  fi
  if ! kubectl --kubeconfig "$kubeconfig" config view >/dev/null 2>&1; then
    return 0
  fi
  local ctx_cluster=""
  ctx_cluster=$(kubectl --kubeconfig "$kubeconfig" config view -o jsonpath="{.contexts[?(@.name=='${context}')].context.cluster}" 2>/dev/null || true)
  if [[ -z "$ctx_cluster" ]]; then
    ctx_cluster="$context"
  fi
  local server=""
  server=$(kubectl --kubeconfig "$kubeconfig" config view -o jsonpath="{.clusters[?(@.name=='${ctx_cluster}')].cluster.server}" 2>/dev/null || true)
  if [[ -z "$server" || "$server" != "https://0.0.0.0:6443" ]]; then
    kubectl --kubeconfig "$kubeconfig" config set-cluster "${ctx_cluster}" --server="${desired_server}" >/dev/null 2>&1 || true
  fi
}

ensure_tools() {
  local mode="$1"
  case "$mode" in
    k3d)
      command -v k3d >/dev/null 2>&1 || { err_msg "Missing required tool: k3d"; exit 1; }
      command -v docker >/dev/null 2>&1 || { err_msg "Missing required tool: docker"; exit 1; }
      command -v kubectl >/dev/null 2>&1 || { err_msg "Missing required tool: kubectl"; exit 1; }
      ;;
    k3s|k8s)
      command -v kubectl >/dev/null 2>&1 || { err_msg "Missing required tool: kubectl"; exit 1; }
      ;;
  esac
}

stop_port_forwards() {
  if [[ -x "${ROOT_DIR}/etc/init_port_forwards.sh" ]]; then
    log_msg "Stopping port forwards..."
    "${ROOT_DIR}/etc/init_port_forwards.sh" -c "${PROLE_CFG_FILE}" stop >/dev/null 2>&1 || true
  fi
}

ensure_k3d_cluster_ready() {
  local cluster_name="$1"
  ensure_tools "k3d"
  if [[ -x "${ROOT_DIR}/etc/init_k8s.sh" ]]; then
    "${ROOT_DIR}/etc/init_k8s.sh" -m k3d -n "${cluster_name}" initialize
    "${ROOT_DIR}/etc/init_k8s.sh" -m k3d -n "${cluster_name}" start
  else
    err_msg "Missing ${ROOT_DIR}/etc/init_k8s.sh"
    exit 1
  fi
  local context="${KUBECTL_CONTEXT:-k3d-${cluster_name}}"
  k3d kubeconfig merge "${cluster_name}" --kubeconfig-switch-context >/dev/null 2>&1 || true

  # Normalize the default kubeconfig to point at the LAN-friendly port.
  if [[ -f "${HOME}/.kube/config" ]]; then
    fix_k3d_kubeconfig_server "${HOME}/.kube/config" "${context}"
  fi

  local k3d_kubeconfig=""
  k3d_kubeconfig="$(mktemp)"
  if k3d kubeconfig get "${cluster_name}" > "${k3d_kubeconfig}" 2>/dev/null; then
    export KUBECONFIG="${k3d_kubeconfig}"
    kubectl --kubeconfig "${k3d_kubeconfig}" config use-context "${context}" >/dev/null 2>&1 || true
    fix_k3d_kubeconfig_server "${k3d_kubeconfig}" "${context}"
  else
    rm -f "${k3d_kubeconfig}" >/dev/null 2>&1 || true
    k3d_kubeconfig=""
  fi

  local attempts=60
  local i
  for i in $(seq 1 "$attempts"); do
    if kubectl --context "${context}" get nodes >/dev/null 2>&1; then
      return 0
    fi
    if [[ -n "${k3d_kubeconfig}" ]] && kubectl --kubeconfig "${k3d_kubeconfig}" get nodes >/dev/null 2>&1; then
      return 0
    fi
    if (( i % 5 == 0 )); then
      k3d kubeconfig merge "${cluster_name}" --kubeconfig-switch-context >/dev/null 2>&1 || true
      if [[ -f "${HOME}/.kube/config" ]]; then
        fix_k3d_kubeconfig_server "${HOME}/.kube/config" "${context}"
      fi
      if [[ -z "${k3d_kubeconfig}" ]]; then
        k3d_kubeconfig="$(mktemp)"
        if k3d kubeconfig get "${cluster_name}" > "${k3d_kubeconfig}" 2>/dev/null; then
          export KUBECONFIG="${k3d_kubeconfig}"
          fix_k3d_kubeconfig_server "${k3d_kubeconfig}" "${context}"
        else
          rm -f "${k3d_kubeconfig}" >/dev/null 2>&1 || true
          k3d_kubeconfig=""
        fi
      fi
    fi
    sleep 2
  done

  err_msg "k3d cluster '${cluster_name}' not reachable after ${attempts} attempts"
  exit 2
}

ensure_k3s_cluster_ready() {
  ensure_tools "k3s"
  if command -v prole_ensure_kubeconfig >/dev/null 2>&1; then
    prole_ensure_kubeconfig >/dev/null 2>&1 || true
  fi
  if ! kubectl get nodes >/dev/null 2>&1; then
    err_msg "k3s cluster not reachable (kubectl get nodes failed)"
    exit 2
  fi
}

filter_install_args() {
  local out=()
  local arg
  for arg in "$@"; do
    case "$arg" in
      -s|-S|--silent|-c|--config|--no-gui|--gui) ;;
      *) out+=("$arg") ;;
    esac
  done
  printf '%s\0' "${out[@]}"
}

wait_for_backup() {
  local ns="$1" backup_name="$2" timeout="${3:-1800}"
  local start_time now phase phase_lc
  start_time=$(date +%s)
  while true; do
    phase=$(kubectl -n "$ns" get backup "$backup_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
    phase_lc=$(printf '%s' "$phase" | tr '[:upper:]' '[:lower:]')
    case "$phase_lc" in
      completed|succeeded)
        log_msg "Backup ${backup_name} completed."
        return 0
        ;;
      failed|error)
        err_msg "Backup ${backup_name} failed (phase=${phase})."
        return 1
        ;;
    esac
    now=$(date +%s)
    if (( now - start_time > timeout )); then
      err_msg "Timed out waiting for backup ${backup_name}."
      return 1
    fi
    log_msg "Waiting for backup ${backup_name} to complete (phase=${phase:-unknown}) ..."
    sleep 10
  done
}

start_garage_port_forward() {
  local ns="$1"
  local svc="$2"
  local local_port="$3"
  local pid_var="$4"
  local port_open="0"
  if python3 - "$local_port" <<'PY' >/dev/null 2>&1; then
import socket, sys
port=int(sys.argv[1])
s=socket.socket()
s.settimeout(0.2)
try:
    s.connect(("127.0.0.1", port))
    sys.exit(0)
except Exception:
    sys.exit(1)
finally:
    s.close()
PY
    port_open="1"
  fi

  if [[ "$port_open" == "1" ]]; then
    printf -v "$pid_var" '%s' ""
    return 0
  fi

  kubectl -n "$ns" port-forward "svc/${svc}" "${local_port}:3900" >/dev/null 2>&1 &
  local pf_pid=$!
  printf -v "$pid_var" '%s' "$pf_pid"

  local i
  for i in {1..30}; do
    if python3 - "$local_port" <<'PY' >/dev/null 2>&1; then
import socket, sys
port=int(sys.argv[1])
s=socket.socket()
s.settimeout(0.2)
try:
    s.connect(("127.0.0.1", port))
    sys.exit(0)
except Exception:
    sys.exit(1)
finally:
    s.close()
PY
      return 0
    fi
    sleep 0.5
  done
  err_msg "Garage port-forward failed to come up on 127.0.0.1:${local_port}"
  return 1
}

archive_backup_from_garage() {
  local ns="$1"
  local backup_name="$2"
  local archive_root="$3"
  local garage_ns="${GARAGE_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}}"
  local garage_svc="${GARAGE_NAME:-garage}"
  local bucket="${GARAGE_BACKUP_BUCKET:-prole-db-backups}"
  local secret="${GARAGE_BACKUP_SECRET_NAME:-prole-db-barman-s3}"

  local access_key secret_key region
  access_key=$(kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.ACCESS_KEY_ID}' 2>/dev/null | base64 -d || true)
  secret_key=$(kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.SECRET_ACCESS_KEY}' 2>/dev/null | base64 -d || true)
  region=$(kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.REGION}' 2>/dev/null | base64 -d || true)

  if [[ -z "$access_key" || -z "$secret_key" ]]; then
    err_msg "Unable to read Garage backup credentials from secret '$secret' in namespace '$ns'"
    return 1
  fi
  if [[ -z "$region" ]]; then
    region="garage"
  fi

  local dest_path backup_id server_name
  dest_path=$(kubectl -n "$ns" get backup "$backup_name" -o jsonpath='{.status.destinationPath}' 2>/dev/null || true)
  backup_id=$(kubectl -n "$ns" get backup "$backup_name" -o jsonpath='{.status.backupName}' 2>/dev/null || true)
  server_name=$(kubectl -n "$ns" get backup "$backup_name" -o jsonpath='{.status.serverName}' 2>/dev/null || true)

  local prefix=""
  if [[ -n "$dest_path" && "$dest_path" == s3://* ]]; then
    local rest="${dest_path#s3://}"
    bucket="${rest%%/*}"
    prefix="${rest#*/}"
    if [[ "$prefix" == "$rest" ]]; then
      prefix=""
    fi
  fi
  if [[ -z "$backup_id" ]]; then
    backup_id="$backup_name"
  fi
  if [[ -z "$prefix" ]]; then
    if [[ -n "$server_name" && "$server_name" != "$backup_id" ]]; then
      prefix="${server_name}/"
    fi
  fi

  local contains="$backup_id"
  if [[ -n "$prefix" && -n "$backup_id" && "$prefix" == *"$backup_id"* ]]; then
    contains=""
  fi
  local archive_target="${archive_root}/${backup_name}"
  local staging_dir="${archive_target}.partial"

  mkdir -p "$archive_root"
  rm -rf "$staging_dir"
  mkdir -p "$staging_dir"

  local pf_pid=""
  start_garage_port_forward "$garage_ns" "$garage_svc" "3900" pf_pid
  trap '[[ -n "${pf_pid:-}" ]] && kill "${pf_pid}" >/dev/null 2>&1 || true' EXIT

  if ! python3 "${ROOT_DIR}/etc/archive_garage_backup.py" \
    --endpoint "http://127.0.0.1:3900" \
    --region "$region" \
    --access-key "$access_key" \
    --secret-key "$secret_key" \
    --bucket "$bucket" \
    --prefix "$prefix" \
    --contains "$contains" \
    --dest "$staging_dir"; then
    if [[ -n "$contains" ]]; then
      log_msg "No objects matched contains filter; retrying with prefix only..."
      python3 "${ROOT_DIR}/etc/archive_garage_backup.py" \
        --endpoint "http://127.0.0.1:3900" \
        --region "$region" \
        --access-key "$access_key" \
        --secret-key "$secret_key" \
        --bucket "$bucket" \
        --prefix "$prefix" \
        --contains "" \
        --dest "$staging_dir"
    else
      return 1
    fi
  fi

  rm -rf "$archive_target"
  mv "$staging_dir" "$archive_target"
  log_msg "Archived backup to ${archive_target}"

  if [[ -n "${pf_pid:-}" ]]; then
    kill "${pf_pid}" >/dev/null 2>&1 || true
    wait "${pf_pid}" >/dev/null 2>&1 || true
  fi
  trap - EXIT
}

stop_monitoring_pods() {
  local ns="${MONITORING_NAMESPACE:-monitoring}"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    return 0
  fi
  log_msg "Stopping monitoring pods in namespace ${ns} ..."
  kubectl -n "$ns" scale deploy --all --replicas=0 >/dev/null 2>&1 || true
  kubectl -n "$ns" scale statefulset --all --replicas=0 >/dev/null 2>&1 || true
  kubectl -n "$ns" delete daemonset --all --ignore-not-found >/dev/null 2>&1 || true
  kubectl -n "$ns" delete pod --all --ignore-not-found >/dev/null 2>&1 || true
}

# Run command
TS="$(date +%Y%m%d-%H%M%S)"

# Prep for Ansible truthfulness
if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
  # We use the same vault pass logic as ansible.sh if available
  export PYTHONPATH="${PYTHONPATH:-}:${ROOT_DIR}"
  python3 "${ROOT_DIR}/etc/sync-prole-cfg.py" || true
fi

case "${CMD}" in
  install)
    LOGFILE="${ROOT_DIR}/logs/install-${TS}.log"

    # Re-parse install-specific flags that may appear after the command word
    _INSTALL_PASSTHRU=()
    _skip_next=0
    _next_is_config=0
    for _arg in "${CMD_ARGS[@]}"; do
      if [[ $_skip_next -eq 1 ]]; then
        _skip_next=0
        continue
      fi
      if [[ $_next_is_config -eq 1 ]]; then
        _next_is_config=0
        # Pass -c <path> through to install.py
        _INSTALL_PASSTHRU+=("-c" "$_arg")
        continue
      fi
      case "$_arg" in
        -s|--silent|-S)
          SILENT=1
          ;;
        -v|--verbose)
          VERBOSE=1
          export PROLE_VERBOSE=1 VERBOSE=1 CLICOLOR_FORCE=1 PY_COLORS=1
          ;;
        -c|--config)
          _next_is_config=1
          ;;
        -r|--reset)
          RESET=1
          ;;
        *)
          _INSTALL_PASSTHRU+=("$_arg")
          ;;
      esac
    done
    # Rebuild INSTALL_ONLY_ARGS from (possibly updated) flags
    if [[ "${SILENT}" -eq 1 ]]; then
      # Ensure -S is present exactly once
      _has_S=0
      for _a in "${INSTALL_ONLY_ARGS[@]}"; do [[ "$_a" == "-S" ]] && _has_S=1; done
      [[ $_has_S -eq 0 ]] && INSTALL_ONLY_ARGS+=("-S")
    fi
    if [[ "${RESET}" -eq 1 ]]; then
      _has_reset=0
      for _a in "${INSTALL_ONLY_ARGS[@]}"; do [[ "$_a" == "--reset" ]] && _has_reset=1; done
      [[ $_has_reset -eq 0 ]] && INSTALL_ONLY_ARGS+=("--reset")
    fi
    CMD_ARGS=("${_INSTALL_PASSTHRU[@]}")

    PY_CMD=("python3")
    if [[ "${COVERAGE}" -eq 1 ]]; then
      PY_CMD=("coverage" "run" "--source=installer")
    fi
    PY_CMD+=("${ROOT_DIR}/install.py" "--no-gui")
    
    if [[ "${LOG}" -eq 1 ]]; then
      run_with_log "${LOGFILE}" "${PY_CMD[@]}" "${INSTALL_ONLY_ARGS[@]}" "${CMD_ARGS[@]}"
    elif [[ "${COVERAGE}" -eq 1 ]]; then
      "${PY_CMD[@]}" "${INSTALL_ONLY_ARGS[@]}" "${CMD_ARGS[@]}"
    else
      exec "${PY_CMD[@]}" "${INSTALL_ONLY_ARGS[@]}" "${CMD_ARGS[@]}"
    fi
    
    if [[ "${COVERAGE}" -eq 1 ]]; then
      echo "Generating coverage report..."
      coverage report -m
      coverage html
      echo "HTML report generated in ${ROOT_DIR}/htmlcov/index.html"
    fi
    ;;
  ansible)
    LOGFILE="${ROOT_DIR}/logs/ansible-${TS}.log"
    prepare_ansible_color
    if [[ "${LOG}" -eq 1 ]]; then
      run_with_log "${LOGFILE}" "${ROOT_DIR}/ansible.sh" "${CMD_ARGS[@]}"
    else
      "${ROOT_DIR}/ansible.sh" "${CMD_ARGS[@]}"
    fi
    # SYNC AFTER ANSIBLE
    if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
       python3 "${ROOT_DIR}/etc/sync-prole-cfg.py" || true
    fi
    ;;
  site)
    LOGFILE="${ROOT_DIR}/logs/site-${TS}.log"
    (
      set -e
      prepare_ansible_color
      if [[ "${RESET}" -eq 1 ]]; then
          echo "Resetting k3s before site run..."
          "${ROOT_DIR}/ansible.sh" -p infrastructure/playbooks/k3s_delete.yml "${CMD_ARGS[@]}"
      fi
      
      if [[ "${LOG}" -eq 1 ]]; then
        run_with_log "${LOGFILE}" "${ROOT_DIR}/ansible.sh" -p infrastructure/playbooks/site.yml "${CMD_ARGS[@]}"
      else
        "${ROOT_DIR}/ansible.sh" -p infrastructure/playbooks/site.yml "${CMD_ARGS[@]}"
      fi
      # SYNC AFTER SITE
      if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
         python3 "${ROOT_DIR}/etc/sync-prole-cfg.py" || true
      fi
    )
    ;;
  reset)
    LOGFILE="${ROOT_DIR}/logs/reset-${TS}.log"
    prepare_ansible_color
    if [[ "${LOG}" -eq 1 ]]; then
      run_with_log "${LOGFILE}" "${ROOT_DIR}/ansible.sh" -p infrastructure/playbooks/k3s_delete.yml "${CMD_ARGS[@]}"
    else
      "${ROOT_DIR}/ansible.sh" -p infrastructure/playbooks/k3s_delete.yml "${CMD_ARGS[@]}"
    fi
    # SYNC AFTER RESET (to clear token)
    if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
       python3 "${ROOT_DIR}/etc/sync-prole-cfg.py" || true
    fi
    ;;
  deploy)
    LOGFILE="${ROOT_DIR}/logs/deploy-${TS}.log"
    echo "Starting full end-to-end deployment..."
    (
      set -e
      prepare_ansible_color
      if [[ "${RESET}" -eq 1 ]]; then
        echo "Step 1/3: Resetting k3s (full)..."
        "${ROOT_DIR}/ansible.sh" -p infrastructure/playbooks/k3s_delete.yml "${CMD_ARGS[@]}"
      else
        echo "Step 1/3: Skipping k3s reset (use -r to force reset)"
      fi
      
      echo "Step 2/3: Running site deployment..."
      "${ROOT_DIR}/ansible.sh" -p infrastructure/playbooks/site.yml "${CMD_ARGS[@]}"
      
      if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
         python3 "${ROOT_DIR}/etc/sync-prole-cfg.py" || true
      fi
      
      echo "Step 3/3: Running silent installer..."
      # INSTALL_ONLY_ARGS includes -S and --reset if flags were set
      python3 "${ROOT_DIR}/install.py" "--no-gui" "-c" "${ROOT_DIR}/conf/prole.cfg" "${INSTALL_ONLY_ARGS[@]}" "${CMD_ARGS[@]}"
      
      echo "Deployment completed successfully!"
    ) | tee "${LOGFILE}"
    ;;
  start)
    if [[ ! -f "${PROLE_CFG_DEFAULT}" ]]; then
      err_msg "Missing ${PROLE_CFG_DEFAULT}"
      exit 1
    fi
    load_prole_cfg "${PROLE_CFG_DEFAULT}"
    stop_port_forwards

    local_mode=$(resolve_prole_mode)
    if [[ -z "${local_mode:-}" ]]; then
      local_mode="k3d"
    fi

    if [[ -n "${PROLE_DATA:-}" ]]; then
      mkdir -p "${PROLE_DATA}" >/dev/null 2>&1 || true
      if [[ -d "${PROLE_DATA}/archive" ]]; then
        rm -rf "${PROLE_DATA}/archive"/*.partial >/dev/null 2>&1 || true
      fi
    fi
    if [[ -n "${PROLE_LOGS:-}" ]]; then
      mkdir -p "${PROLE_LOGS}" >/dev/null 2>&1 || true
    fi

    log_msg "Starting Prole environment (mode=${local_mode})..."

    case "$local_mode" in
      k3d)
        cluster_name=$(resolve_k3d_cluster_name)
        ensure_k3d_cluster_ready "$cluster_name"
        ;;
      k3s|k8s)
        ensure_k3s_cluster_ready
        ;;
      *)
        err_msg "Unsupported mode '${local_mode}'"
        exit 2
        ;;
    esac

    log_msg "Running silent installer..."
    install_args=()
    if [[ "${VERBOSE}" -eq 1 ]]; then
      install_args+=("-v")
    fi
    if [[ "${DEBUG}" -eq 1 ]]; then
      install_args+=("--debug")
    fi
    if [[ "${RESET}" -eq 1 ]]; then
      install_args+=("--reset")
    fi

    PY_CMD=("python3")
    if [[ "${COVERAGE}" -eq 1 ]]; then
      PY_CMD=("coverage" "run" "--source=installer")
    fi
    "${PY_CMD[@]}" "${ROOT_DIR}/install.py" --no-gui -S -c "${PROLE_CFG_DEFAULT}" "${install_args[@]}"

    if [[ "${COVERAGE}" -eq 1 ]]; then
      echo ""
      echo "=== Coverage Report ==="
      coverage report -m
      coverage html
      echo "HTML report generated in ${ROOT_DIR}/htmlcov/index.html"
    fi

    if [[ "$local_mode" == "k3d" && -x "${ROOT_DIR}/etc/init_port_forwards.sh" ]]; then
      log_msg "Starting port forwards..."
      "${ROOT_DIR}/etc/init_port_forwards.sh" -c "${PROLE_CFG_DEFAULT}" start
    fi
    ;;
  stop)
    if [[ ! -f "${PROLE_CFG_DEFAULT}" ]]; then
      err_msg "Missing ${PROLE_CFG_DEFAULT}"
      exit 1
    fi
    load_prole_cfg "${PROLE_CFG_DEFAULT}"

    local_mode=$(resolve_prole_mode)
    if [[ -z "${local_mode:-}" ]]; then
      local_mode="k3d"
    fi

    log_msg "Validating environment (mode=${local_mode})..."
    case "$local_mode" in
      k3d)
        ensure_tools "k3d"
        cluster_name=$(resolve_k3d_cluster_name)
        if ! k3d cluster list "$cluster_name" >/dev/null 2>&1; then
          err_msg "k3d cluster '${cluster_name}' not found"
          exit 2
        fi
        if ! kubectl --context "${KUBECTL_CONTEXT:-k3d-${cluster_name}}" get nodes >/dev/null 2>&1; then
          err_msg "k3d cluster '${cluster_name}' not reachable"
          exit 2
        fi
        ;;
      k3s|k8s)
        ensure_k3s_cluster_ready
        ;;
      *)
        err_msg "Unsupported mode '${local_mode}'"
        exit 2
        ;;
    esac

    log_msg "Validating Garage and Barman backup configuration..."
    if [[ -x "${ROOT_DIR}/etc/init_garage_store.sh" ]]; then
      "${ROOT_DIR}/etc/init_garage_store.sh" start
    fi
    if [[ -x "${ROOT_DIR}/etc/init_cloudnative_pg.sh" ]]; then
      "${ROOT_DIR}/etc/init_cloudnative_pg.sh" install-barman-plugin
    fi
    if [[ -x "${ROOT_DIR}/etc/init_prole-db-backup.sh" ]]; then
      RUN_FIRST_BACKUP=0 "${ROOT_DIR}/etc/init_prole-db-backup.sh" start
    fi

    log_msg "Running full backup..."
    backup_before=$(kubectl -n "${NAMESPACE:-default}" get backup \
      --sort-by=.metadata.creationTimestamp \
      -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
    "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup

    backup_name=""
    for i in {1..30}; do
      backup_name=$(kubectl -n "${NAMESPACE:-default}" get backup \
        --sort-by=.metadata.creationTimestamp \
        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
      if [[ -n "$backup_name" && "$backup_name" != "$backup_before" ]]; then
        break
      fi
      sleep 2
    done

    if [[ -z "$backup_name" || "$backup_name" == "$backup_before" ]]; then
      err_msg "Unable to detect newly created backup"
      exit 1
    fi

    wait_for_backup "${NAMESPACE:-default}" "$backup_name" "${BACKUP_WAIT_TIMEOUT:-1800}"

    archive_root="${PROLE_DATA:-${ROOT_DIR}/data}/archive"
    archive_backup_from_garage "${NAMESPACE:-default}" "$backup_name" "$archive_root"

    stop_monitoring_pods

    if [[ -x "${ROOT_DIR}/etc/init_cloudnative_pg.sh" ]]; then
      log_msg "Stopping CNPG pods..."
      "${ROOT_DIR}/etc/init_cloudnative_pg.sh" stop
    fi

    if [[ -x "${ROOT_DIR}/etc/init_common_services.sh" ]]; then
      log_msg "Stopping common services..."
      if [[ "${KERBEROS_ENABLED:-}" == "1" || "${KERBEROS_ENABLED:-}" == "true" || "${KERBEROS_ENABLED:-}" == "True" ]]; then
        "${ROOT_DIR}/etc/init_common_services.sh" -n "${SERVICE_NAMESPACE:-${NAMESPACE:-default}}" -k stop || true
      else
        "${ROOT_DIR}/etc/init_common_services.sh" -n "${SERVICE_NAMESPACE:-${NAMESPACE:-default}}" stop || true
      fi
    fi
    ;;
  backup)
    if [[ ! -f "${PROLE_CFG_DEFAULT}" ]]; then
      err_msg "Missing ${PROLE_CFG_DEFAULT}"
      exit 1
    fi
    load_prole_cfg "${PROLE_CFG_DEFAULT}"

    local_mode=$(resolve_prole_mode)
    if [[ -z "${local_mode:-}" ]]; then
      local_mode="k3d"
    fi

    # Determine backup type: -f / --full / full → full, otherwise incremental
    backup_type="incremental"
    for arg in "${CMD_ARGS[@]}"; do
      case "$arg" in
        -f|--full|full) backup_type="full" ;;
      esac
    done

    log_msg "Validating environment (mode=${local_mode})..."
    case "$local_mode" in
      k3d)
        ensure_tools "k3d"
        cluster_name=$(resolve_k3d_cluster_name)
        if ! kubectl --context "${KUBECTL_CONTEXT:-k3d-${cluster_name}}" get nodes >/dev/null 2>&1; then
          err_msg "k3d cluster '${cluster_name}' not reachable"
          exit 2
        fi
        ;;
      k3s|k8s)
        ensure_k3s_cluster_ready
        ;;
      *)
        err_msg "Unsupported mode '${local_mode}'"
        exit 2
        ;;
    esac

    # Ensure backup infrastructure is ready
    if [[ -x "${ROOT_DIR}/etc/init_garage_store.sh" ]]; then
      "${ROOT_DIR}/etc/init_garage_store.sh" start
    fi
    if [[ -x "${ROOT_DIR}/etc/init_cloudnative_pg.sh" ]]; then
      "${ROOT_DIR}/etc/init_cloudnative_pg.sh" install-barman-plugin
    fi
    if [[ -x "${ROOT_DIR}/etc/init_prole-db-backup.sh" ]]; then
      RUN_FIRST_BACKUP=0 "${ROOT_DIR}/etc/init_prole-db-backup.sh" start
    fi

    log_msg "Running ${backup_type} backup..."
    backup_before=$(kubectl -n "${NAMESPACE:-default}" get backup \
      --sort-by=.metadata.creationTimestamp \
      -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)

    if [[ "$backup_type" == "full" ]]; then
      "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup full
    else
      "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup incr
    fi

    backup_name=""
    for i in {1..30}; do
      backup_name=$(kubectl -n "${NAMESPACE:-default}" get backup \
        --sort-by=.metadata.creationTimestamp \
        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
      if [[ -n "$backup_name" && "$backup_name" != "$backup_before" ]]; then
        break
      fi
      sleep 2
    done

    if [[ -z "$backup_name" || "$backup_name" == "$backup_before" ]]; then
      err_msg "Unable to detect newly created backup"
      exit 1
    fi

    wait_for_backup "${NAMESPACE:-default}" "$backup_name" "${BACKUP_WAIT_TIMEOUT:-1800}"
    log_msg "Backup ${backup_name} (${backup_type}) completed successfully."
    ;;
  passwd)
    if [[ -x "${ROOT_DIR}/etc/prole-db-passwwd.sh" ]]; then
      "${ROOT_DIR}/etc/prole-db-passwwd.sh" "${CMD_ARGS[@]}"
    else
      err_msg "Missing ${ROOT_DIR}/etc/prole-db-passwwd.sh"
      exit 1
    fi
    ;;
  *)
    # Default to ansible if command not matched
    LOGFILE="${ROOT_DIR}/logs/${CMD}-${TS}.log"
    if [[ "${LOG}" -eq 1 ]]; then
      run_with_log "${LOGFILE}" "${ROOT_DIR}/ansible.sh" "${CMD}" "${CMD_ARGS[@]}"
    else
      "${ROOT_DIR}/ansible.sh" "${CMD}" "${CMD_ARGS[@]}"
    fi
    # SYNC AFTER ANSIBLE
    if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
       python3 "${ROOT_DIR}/etc/sync-prole-cfg.py" || true
    fi
    ;;
esac