prole/etc/status.sh

#!/usr/bin/env bash
# etc/status.sh — Overall deployment health check for the Prole k3d environment.
# Usage: status.sh [-c conf/prole.cfg] [-v|--verbose]
# Exit 0  = all pods running as expected
# Exit 1  = undesirable state detected
set -u

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── colours (disabled when stdout is not a tty) ─────────────────────────
if [[ -t 1 ]]; then
  RED='\033[0;31m'
  GREEN='\033[0;32m'
  YELLOW='\033[1;33m'
  BLUE='\033[0;34m'
  BOLD='\033[1m'
  NC='\033[0m'
else
  RED='' GREEN='' YELLOW='' BLUE='' BOLD='' NC=''
fi

# ── defaults ─────────────────────────────────────────────────────────────
VERBOSE=0
CFG_PATH=""
KUBECONFIG="${KUBECONFIG:-}"
NAMESPACE=""
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-prole-db}"

# ── helpers ──────────────────────────────────────────────────────────────
usage() {
  cat <<EOF
Usage: $(basename "$0") [-c <prole.cfg>] [-v|--verbose] [-h|--help]

Overall deployment health check.
  -c, --config <path>   Path to prole.cfg (default: conf/prole.cfg)
  -v, --verbose         Show full stdout/stderr from each status check
  -h, --help            Show this help
EOF
}

ok()   { printf "${GREEN}[OK]${NC}    %s\n" "$*"; }
fail() { printf "${RED}[FAIL]${NC}  %s\n" "$*"; }
warn() { printf "${YELLOW}[WARN]${NC}  %s\n" "$*"; }
info() { printf "${BLUE}[INFO]${NC}  %s\n" "$*"; }

# ── parse arguments ──────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
  case "$1" in
    -c|--config)
      shift; CFG_PATH="${1:-}"; shift ;;
    -c=*|--config=*)
      CFG_PATH="${1#*=}"; shift ;;
    -v|--verbose)
      VERBOSE=1; shift ;;
    -h|--help)
      usage; exit 0 ;;
    *)
      usage; exit 2 ;;
  esac
done

# ── load prole.cfg (always) ─────────────────────────────────────────────
if [[ -z "$CFG_PATH" ]]; then
  CFG_PATH="$PROJECT_ROOT/conf/prole.cfg"
fi
if [[ ! -f "$CFG_PATH" ]]; then
  echo "FATAL: prole.cfg not found at $CFG_PATH" >&2
  exit 1
fi

# Minimal INI reader — pull NAMESPACE, KUBECONFIG, KERBEROS_ENABLED,
# SUPABASE_ENABLED from prole.cfg if not already set in the environment.
_read_cfg_value() {
  local key="$1"
  local val=""
  # Grab last occurrence (case-insensitive key match) from INI file
  val="$(grep -i "^[[:space:]]*${key}[[:space:]]*=" "$CFG_PATH" 2>/dev/null \
    | tail -1 | sed 's/^[^=]*=[[:space:]]*//' | sed 's/[[:space:]]*$//')"
  # If the value is a ${VAR} reference, resolve it from the first concrete
  # occurrence of VAR in the same config file.
  if [[ "$val" == '${'*'}' ]]; then
    local ref_key="${val#\$\{}"
    ref_key="${ref_key%\}}"
    # Find first concrete (non-${...}) value for the referenced key
    val="$(grep -i "^[[:space:]]*${ref_key}[[:space:]]*=" "$CFG_PATH" 2>/dev/null \
      | sed 's/^[^=]*=[[:space:]]*//' | sed 's/[[:space:]]*$//' \
      | grep -v '^\$' | head -1)"
  fi
  echo "$val"
}

if [[ -z "$NAMESPACE" ]]; then
  NAMESPACE="$(_read_cfg_value 'NAMESPACE')"
fi
if [[ -z "$NAMESPACE" ]]; then
  NAMESPACE="default"
fi

if [[ -z "$KUBECONFIG" ]]; then
  _kc="$(_read_cfg_value 'KUBECONFIG')"
  if [[ -n "$_kc" && -f "$_kc" ]]; then
    export KUBECONFIG="$_kc"
  fi
fi
# Fallback: project-local kubeconfig (only if it can reach the cluster)
if [[ -z "${KUBECONFIG:-}" && -f "$PROJECT_ROOT/prole-k3s.kubeconfig" ]]; then
  if KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig" kubectl cluster-info >/dev/null 2>&1; then
    export KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig"
  fi
fi

_kerberos_enabled="$(_read_cfg_value 'KERBEROS_ENABLED')"
_supabase_enabled="$(_read_cfg_value 'SUPABASE_ENABLED')"

_is_true() {
  case "${1,,}" in
    true|yes|1) return 0 ;;
    *) return 1 ;;
  esac
}

# ── pre-flight ───────────────────────────────────────────────────────────
if ! command -v kubectl >/dev/null 2>&1; then
  fail "kubectl not found in PATH"
  exit 1
fi

OVERALL_RC=0   # will flip to 1 on any failure

# ── header ───────────────────────────────────────────────────────────────
echo ""
printf "${BOLD}Prole Deployment Status${NC}\n"
echo "───────────────────────────────────────────"
info "Config:     $CFG_PATH"
info "Namespace:  $NAMESPACE"
ctx="$(kubectl config current-context 2>/dev/null || true)"
info "Context:    ${ctx:-<unknown>}"
if [[ -n "${KUBECONFIG:-}" ]]; then
  info "Kubeconfig: $KUBECONFIG"
fi
echo ""

# ── 1. kubectl get pods -o wide -A ──────────────────────────────────────
printf "${BOLD}All Pods${NC}\n"
echo "───────────────────────────────────────────"
pod_output="$(kubectl get pods -o wide -A 2>/dev/null)" || true
if [[ $VERBOSE -eq 1 ]]; then
  echo "$pod_output"
  echo ""
fi

# Evaluate pod health
_bad_pods=""
while IFS= read -r line; do
  # skip header
  [[ "$line" == NAMESPACE* ]] && continue
  [[ -z "$line" ]] && continue
  status_field="$(echo "$line" | awk '{print $4}')"
  case "$status_field" in
    Running|Completed|Succeeded) ;;
    Terminating)
      warn "Terminating pod: $(echo "$line" | awk '{printf "%s/%s", $1, $2}')"
      ;;
    Pending|ContainerCreating|Init:*|PodInitializing)
      warn "Pending/init pod: $(echo "$line" | awk '{printf "%s/%s (%s)", $1, $2, $4}')"
      ;;
    *)
      _bad_pods="yes"
      fail "Unhealthy pod: $(echo "$line" | awk '{printf "%s/%s status=%s", $1, $2, $4}')"
      ;;
  esac
done <<< "$pod_output"

if [[ -z "$_bad_pods" ]]; then
  ok "All pods in expected state"
else
  OVERALL_RC=1
fi
echo ""

# ── 2. CloudNative-PG cluster status ────────────────────────────────────
printf "${BOLD}CloudNative-PG: %s${NC}\n" "$CNPG_CLUSTER_NAME"
echo "───────────────────────────────────────────"
if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
  cnpg_output=""
  if command -v kubectl-cnpg >/dev/null 2>&1 || kubectl cnpg version >/dev/null 2>&1; then
    cnpg_output="$(kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" 2>&1)" || true
    echo "$cnpg_output"
  else
    kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o wide 2>&1 || true
    info "kubectl cnpg plugin not available; showing basic cluster info only"
  fi
  # Simple health: check if cluster phase is healthy
  phase="$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
  if [[ "$phase" == "Cluster in healthy state" || "$phase" == "Healthy" ]]; then
    ok "CNPG cluster $CNPG_CLUSTER_NAME is healthy"
  elif [[ -n "$phase" ]]; then
    warn "CNPG cluster phase: $phase"
  else
    fail "Unable to determine CNPG cluster phase"
    OVERALL_RC=1
  fi
else
  fail "CNPG cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'"
  OVERALL_RC=1
fi
echo ""

# ── 3. Per-script status checks ─────────────────────────────────────────
printf "${BOLD}Service Status Checks${NC}\n"
echo "───────────────────────────────────────────"

# Scripts that were deployed during install.py — order matches deployment sequence.
STATUS_SCRIPTS=(
  "init_common_services.sh"
  "init_cloudnative_pg.sh"
  "init_openbao.sh"
  "init_kong.sh"
  "init_postgrest.sh"
  "init_db_manager.sh"
  "init_monitoring.sh"
  "init_prole-db-backup.sh"
  "init_port_forwards.sh"
)

# Conditional scripts
if _is_true "${_kerberos_enabled:-false}"; then
  STATUS_SCRIPTS+=("init_kerberos.sh")
fi

run_status_check() {
  local script="$1"
  local script_path="$SCRIPT_DIR/$script"
  local label="${script%.sh}"
  label="${label#init_}"

  if [[ ! -x "$script_path" ]]; then
    if [[ -f "$script_path" ]]; then
      chmod +x "$script_path" 2>/dev/null || true
    fi
    if [[ ! -f "$script_path" ]]; then
      warn "$label — script not found ($script_path)"
      return
    fi
  fi

  local output rc
  output="$(bash "$script_path" status 2>&1)" && rc=0 || rc=$?

  if [[ $rc -eq 0 ]]; then
    ok "$label"
  else
    fail "$label (exit $rc)"
    OVERALL_RC=1
  fi

  if [[ $VERBOSE -eq 1 && -n "$output" ]]; then
    echo "$output" | sed 's/^/    /'
    echo ""
  fi
}

for s in "${STATUS_SCRIPTS[@]}"; do
  run_status_check "$s"
done

# ── 4. Supabase (if enabled) ────────────────────────────────────────────
if _is_true "${_supabase_enabled:-false}"; then
  echo ""
  printf "${BOLD}Supabase${NC}\n"
  echo "───────────────────────────────────────────"
  supa_ns="${SUPABASE_NAMESPACE:-supabase}"
  supa_pods="$(kubectl get pods -n "$supa_ns" --no-headers 2>&1)" || true
  if [[ -z "$supa_pods" || "$supa_pods" == *"not found"* || "$supa_pods" == *"No resources"* ]]; then
    fail "No Supabase pods found in namespace '$supa_ns'"
    OVERALL_RC=1
  else
    _supa_bad=""
    while IFS= read -r line; do
      [[ -z "$line" ]] && continue
      st="$(echo "$line" | awk '{print $3}')"
      case "$st" in
        Running|Completed|Succeeded) ;;
        *) _supa_bad="yes" ;;
      esac
    done <<< "$supa_pods"
    if [[ -z "$_supa_bad" ]]; then
      ok "Supabase pods healthy in namespace '$supa_ns'"
    else
      fail "Unhealthy Supabase pods in namespace '$supa_ns'"
      OVERALL_RC=1
    fi
    if [[ $VERBOSE -eq 1 ]]; then
      echo "$supa_pods" | sed 's/^/    /'
      echo ""
    fi
  fi
fi

# ── 5. Status of status_common_services.sh (if present) ─────────────────
if [[ -f "$SCRIPT_DIR/status_common_services.sh" ]]; then
  echo ""
  printf "${BOLD}Common Services${NC}\n"
  echo "───────────────────────────────────────────"
  cs_args=("-n" "$NAMESPACE")
  if _is_true "${_kerberos_enabled:-false}"; then
    cs_args+=("-k")
  fi
  cs_output="$(bash "$SCRIPT_DIR/status_common_services.sh" "${cs_args[@]}" 2>&1)" && cs_rc=0 || cs_rc=$?
  if [[ $cs_rc -eq 0 ]]; then
    ok "Common services"
  else
    fail "Common services (exit $cs_rc)"
    OVERALL_RC=1
  fi
  if [[ $VERBOSE -eq 1 && -n "$cs_output" ]]; then
    echo "$cs_output" | sed 's/^/    /'
    echo ""
  fi
fi

# ── summary ──────────────────────────────────────────────────────────────
echo ""
echo "───────────────────────────────────────────"
if [[ $OVERALL_RC -eq 0 ]]; then
  ok "All checks passed"
else
  fail "One or more checks failed"
fi
echo ""
exit "$OVERALL_RC"