#!/usr/bin/env bash # etc/status.sh — Overall deployment health check for the Prole k3d environment. # Usage: status.sh [-c conf/prole.cfg] [-v|--verbose] # Exit 0 = score of 100% (all components healthy) # Exit 1 = score < 100% set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # ── colours (disabled when stdout is not a tty) ───────────────────────── if [[ -t 1 ]] || [[ "${CLICOLOR_FORCE:-0}" == "1" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' BOLD='\033[1m' NC='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' BOLD='' NC='' fi # ── defaults ───────────────────────────────────────────────────────────── VERBOSE=0 CFG_PATH="" KUBECONFIG="${KUBECONFIG:-}" NAMESPACE="" CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-prole-db}" # ── helpers ────────────────────────────────────────────────────────────── usage() { cat <] [-v|--verbose] [-h|--help] Overall deployment health check. -c, --config Path to prole.cfg (default: conf/prole.cfg) -v, --verbose Show full stdout/stderr from each status check -h, --help Show this help EOF } ok() { printf "${GREEN}[OK]${NC} %s\n" "$*"; } fail() { printf "${RED}[FAIL]${NC} %s\n" "$*"; } warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$*"; } info() { printf "${BLUE}[INFO]${NC} %s\n" "$*"; } # Portable timeout wrapper (macOS lacks coreutils timeout by default). # Usage: run_with_timeout [args...] # Returns 124 on timeout, otherwise the command's exit code. STATUS_CHECK_TIMEOUT=${STATUS_CHECK_TIMEOUT:-30} run_with_timeout() { local secs="$1"; shift "$@" & local cmd_pid=$! ( sleep "$secs" && kill "$cmd_pid" 2>/dev/null ) & local watcher_pid=$! wait "$cmd_pid" 2>/dev/null local rc=$? kill "$watcher_pid" 2>/dev/null wait "$watcher_pid" 2>/dev/null || true # If the process was killed by our watcher, rc is 137 (128+9) or 143 (128+15) if [[ $rc -eq 137 || $rc -eq 143 ]]; then return 124 fi return $rc } # ── parse arguments ────────────────────────────────────────────────────── while [[ $# -gt 0 ]]; do case "$1" in -c|--config) shift; CFG_PATH="${1:-}"; shift ;; -c=*|--config=*) CFG_PATH="${1#*=}"; shift ;; -v|--verbose) VERBOSE=1; shift ;; -h|--help) usage; exit 0 ;; *) usage; exit 2 ;; esac done # ── load prole.cfg (always) ───────────────────────────────────────────── if [[ -z "$CFG_PATH" ]]; then CFG_PATH="$PROJECT_ROOT/conf/prole.cfg" fi if [[ ! -f "$CFG_PATH" ]]; then echo "FATAL: prole.cfg not found at $CFG_PATH" >&2 exit 1 fi # Minimal INI reader — pull NAMESPACE, KUBECONFIG, KERBEROS_ENABLED, # SUPABASE_ENABLED from prole.cfg if not already set in the environment. _read_cfg_value() { local key="$1" local val="" # Grab last occurrence (case-insensitive key match) from INI file val="$(grep -i "^[[:space:]]*${key}[[:space:]]*=" "$CFG_PATH" 2>/dev/null \ | tail -1 | sed 's/^[^=]*=[[:space:]]*//' | sed 's/[[:space:]]*$//')" # If the value is a ${VAR} reference, resolve it from the first concrete # occurrence of VAR in the same config file. if [[ "$val" == '${'*'}' ]]; then local ref_key="${val#\$\{}" ref_key="${ref_key%\}}" # Find first concrete (non-${...}) value for the referenced key val="$(grep -i "^[[:space:]]*${ref_key}[[:space:]]*=" "$CFG_PATH" 2>/dev/null \ | sed 's/^[^=]*=[[:space:]]*//' | sed 's/[[:space:]]*$//' \ | grep -v '^\$' | head -1)" fi echo "$val" } if [[ -z "$NAMESPACE" ]]; then NAMESPACE="$(_read_cfg_value 'NAMESPACE')" fi if [[ -z "$NAMESPACE" ]]; then NAMESPACE="default" fi if [[ -z "$KUBECONFIG" ]]; then _kc="$(_read_cfg_value 'KUBECONFIG')" if [[ -n "$_kc" && -f "$_kc" ]]; then export KUBECONFIG="$_kc" fi fi # Fallback: project-local kubeconfig (only if it can reach the cluster) if [[ -z "${KUBECONFIG:-}" && -f "$PROJECT_ROOT/prole-k3s.kubeconfig" ]]; then if KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig" kubectl cluster-info >/dev/null 2>&1; then export KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig" fi fi _kerberos_enabled="$(_read_cfg_value 'KERBEROS_ENABLED')" _supabase_enabled="$(_read_cfg_value 'SUPABASE_ENABLED')" _is_true() { case "${1,,}" in true|yes|1) return 0 ;; *) return 1 ;; esac } # ── pre-flight ─────────────────────────────────────────────────────────── if ! command -v kubectl >/dev/null 2>&1; then fail "kubectl not found in PATH" exit 1 fi OVERALL_RC=0 # will flip to 1 on any failure # ── Scoring variables ──────────────────────────────────────────────────── # Rubric: 40% Common Core, 50% CNPG, 10% Stack Additions (Supabase) CORE_TOTAL=0 CORE_PASS=0 CNPG_SCORE=0 # 0-100 within its own category SUPABASE_SCORE=0 # 0-100 within its own category SUPABASE_WEIGHT=10 # Only counted if supabase is enabled # ── header ─────────────────────────────────────────────────────────────── echo "" printf "${BOLD}Prole Deployment Status${NC}\n" echo "───────────────────────────────────────────" info "Config: $CFG_PATH" info "Namespace: $NAMESPACE" ctx="$(kubectl config current-context 2>/dev/null || true)" info "Context: ${ctx:-}" if [[ -n "${KUBECONFIG:-}" ]]; then info "Kubeconfig: $KUBECONFIG" fi echo "" # ── 1. kubectl get pods -o wide -A ────────────────────────────────────── printf "${BOLD}All Pods${NC}\n" echo "───────────────────────────────────────────" pod_output="$(kubectl get pods -o wide -A 2>/dev/null)" || true if [[ $VERBOSE -eq 1 ]]; then echo "$pod_output" echo "" fi # Evaluate pod health _bad_pods="" while IFS= read -r line; do # skip header [[ "$line" == NAMESPACE* ]] && continue [[ -z "$line" ]] && continue status_field="$(echo "$line" | awk '{print $4}')" case "$status_field" in Running|Completed|Succeeded) ;; Terminating) warn "Terminating pod: $(echo "$line" | awk '{printf "%s/%s", $1, $2}')" ;; Pending|ContainerCreating|Init:*|PodInitializing) warn "Pending/init pod: $(echo "$line" | awk '{printf "%s/%s (%s)", $1, $2, $4}')" ;; *) _bad_pods="yes" fail "Unhealthy pod: $(echo "$line" | awk '{printf "%s/%s status=%s", $1, $2, $4}')" ;; esac done <<< "$pod_output" if [[ -z "$_bad_pods" ]]; then ok "All pods in expected state" else OVERALL_RC=1 fi echo "" # ── 2. CloudNative-PG cluster status ──────────────────────────────────── printf "${BOLD}CloudNative-PG: %s${NC}\n" "$CNPG_CLUSTER_NAME" echo "───────────────────────────────────────────" if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then cnpg_output="" if command -v kubectl-cnpg >/dev/null 2>&1 || kubectl cnpg version >/dev/null 2>&1; then cnpg_output="$(kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" 2>&1)" || true echo "$cnpg_output" else kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o wide 2>&1 || true info "kubectl cnpg plugin not available; showing basic cluster info only" fi # Simple health: check if cluster phase is healthy phase="$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)" if [[ "$phase" == "Cluster in healthy state" || "$phase" == "Healthy" ]]; then ok "CNPG cluster $CNPG_CLUSTER_NAME is healthy" CNPG_SCORE=100 elif [[ -n "$phase" ]]; then warn "CNPG cluster phase: $phase" # Partial — cluster exists but not fully healthy → 50% of CNPG score CNPG_SCORE=50 else fail "Unable to determine CNPG cluster phase" CNPG_SCORE=0 OVERALL_RC=1 fi else fail "CNPG cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'" CNPG_SCORE=0 OVERALL_RC=1 fi echo "" # ── 3. Per-script status checks (Common Core) ─────────────────────────── printf "${BOLD}Service Status Checks${NC}\n" echo "───────────────────────────────────────────" # Scripts that were deployed during install.py — order matches deployment sequence. STATUS_SCRIPTS=( "init_common_services.sh" "init_cloudnative_pg.sh" "init_openbao.sh" "init_kong.sh" "init_db_manager.sh" "init_monitoring.sh" "init_cnpg_backup.sh" "init_port_forwards.sh" ) # Conditional scripts if _is_true "${_kerberos_enabled:-false}"; then STATUS_SCRIPTS+=("init_kerberos.sh") fi run_status_check() { local script="$1" local script_path="$SCRIPT_DIR/$script" local label="${script%.sh}" label="${label#init_}" if [[ ! -x "$script_path" ]]; then if [[ -f "$script_path" ]]; then chmod +x "$script_path" 2>/dev/null || true fi if [[ ! -f "$script_path" ]]; then warn "$label — script not found ($script_path)" CORE_TOTAL=$((CORE_TOTAL + 1)) return fi fi local _output_file _output_file=$(mktemp "${TMPDIR:-/tmp}/prole_status_XXXXXX") local rc run_with_timeout "$STATUS_CHECK_TIMEOUT" bash "$script_path" status >"$_output_file" 2>&1 && rc=0 || rc=$? local output output=$(cat "$_output_file" 2>/dev/null || true) rm -f "$_output_file" CORE_TOTAL=$((CORE_TOTAL + 1)) if [[ $rc -eq 124 ]]; then fail "$label (timeout after ${STATUS_CHECK_TIMEOUT}s)" OVERALL_RC=1 elif [[ $rc -eq 0 ]]; then ok "$label" CORE_PASS=$((CORE_PASS + 1)) else fail "$label (exit $rc)" OVERALL_RC=1 fi if [[ $VERBOSE -eq 1 && -n "$output" ]]; then echo "$output" | sed 's/^/ /' echo "" fi } for s in "${STATUS_SCRIPTS[@]}"; do run_status_check "$s" done # ── 4. Supabase (if enabled) ──────────────────────────────────────────── if _is_true "${_supabase_enabled:-false}"; then echo "" printf "${BOLD}Supabase${NC}\n" echo "───────────────────────────────────────────" supa_ns="${SUPABASE_NAMESPACE:-supabase}" supa_pods="$(kubectl get pods -n "$supa_ns" --no-headers 2>&1)" || true if [[ -z "$supa_pods" || "$supa_pods" == *"not found"* || "$supa_pods" == *"No resources"* ]]; then fail "No Supabase pods found in namespace '$supa_ns'" SUPABASE_SCORE=0 OVERALL_RC=1 else _supa_bad="" _supa_total=0 _supa_healthy=0 while IFS= read -r line; do [[ -z "$line" ]] && continue _supa_total=$((_supa_total + 1)) st="$(echo "$line" | awk '{print $3}')" case "$st" in Running|Completed|Succeeded) _supa_healthy=$((_supa_healthy + 1)) ;; *) _supa_bad="yes" ;; esac done <<< "$supa_pods" if [[ -z "$_supa_bad" ]]; then ok "Supabase pods healthy in namespace '$supa_ns'" SUPABASE_SCORE=100 else fail "Unhealthy Supabase pods in namespace '$supa_ns'" if [[ $_supa_total -gt 0 ]]; then SUPABASE_SCORE=$(( (_supa_healthy * 100) / _supa_total )) fi OVERALL_RC=1 fi if [[ $VERBOSE -eq 1 ]]; then echo "$supa_pods" | sed 's/^/ /' echo "" fi fi else # Supabase disabled — redistribute its weight to the other two categories SUPABASE_WEIGHT=0 fi # ── 5. Status of status_common_services.sh (if present) ───────────────── if [[ -f "$SCRIPT_DIR/status_common_services.sh" ]]; then echo "" printf "${BOLD}Common Services${NC}\n" echo "───────────────────────────────────────────" cs_args=("-n" "$NAMESPACE") if _is_true "${_kerberos_enabled:-false}"; then cs_args+=("-k") fi _cs_output_file=$(mktemp "${TMPDIR:-/tmp}/prole_cs_status_XXXXXX") run_with_timeout "$STATUS_CHECK_TIMEOUT" bash "$SCRIPT_DIR/status_common_services.sh" "${cs_args[@]}" >"$_cs_output_file" 2>&1 && cs_rc=0 || cs_rc=$? cs_output=$(cat "$_cs_output_file" 2>/dev/null || true) rm -f "$_cs_output_file" if [[ $cs_rc -eq 124 ]]; then fail "Common services (timeout after ${STATUS_CHECK_TIMEOUT}s)" OVERALL_RC=1 elif [[ $cs_rc -eq 0 ]]; then ok "Common services" else fail "Common services (exit $cs_rc)" OVERALL_RC=1 fi if [[ $VERBOSE -eq 1 && -n "$cs_output" ]]; then echo "$cs_output" | sed 's/^/ /' echo "" fi fi # ── Scoring ────────────────────────────────────────────────────────────── # Rubric weights: # Common Core: 40% # CloudNative-PG: 50% # Stack Additions (Supabase): 10% (0% when disabled, redistributed) # # When Supabase is disabled the weights become 44.4% / 55.6% (proportional). echo "" echo "───────────────────────────────────────────" printf "${BOLD}Score${NC}\n" echo "───────────────────────────────────────────" # Common Core percentage (from per-script checks) if [[ $CORE_TOTAL -gt 0 ]]; then CORE_PCT=$(( (CORE_PASS * 100) / CORE_TOTAL )) else CORE_PCT=100 fi # Weights W_CORE=40 W_CNPG=50 W_SUPA=$SUPABASE_WEIGHT # If supabase is disabled, redistribute proportionally if [[ $W_SUPA -eq 0 ]]; then # 40/(40+50) = 44.4… 50/(40+50) = 55.5… → use integer math x10 W_CORE=44 W_CNPG=56 fi # Weighted total (integer arithmetic, scale by 100 for precision) TOTAL_SCORE=$(( (CORE_PCT * W_CORE + CNPG_SCORE * W_CNPG + SUPABASE_SCORE * W_SUPA) / (W_CORE + W_CNPG + W_SUPA) )) # Clamp if [[ $TOTAL_SCORE -gt 100 ]]; then TOTAL_SCORE=100; fi if [[ $TOTAL_SCORE -lt 0 ]]; then TOTAL_SCORE=0; fi # Color helper for a percentage _score_color() { local pct=$1 if [[ $pct -ge 100 ]]; then printf "${GREEN}" elif [[ $pct -ge 75 ]]; then printf "${YELLOW}" else printf "${RED}" fi } # Print section scores printf " Common Core (40%%): " _score_color $CORE_PCT printf "%3d%% " "$CORE_PCT" printf "${NC}(%d/%d checks)\n" "$CORE_PASS" "$CORE_TOTAL" printf " CloudNative-PG (50%%): " _score_color $CNPG_SCORE printf "%3d%%${NC}\n" "$CNPG_SCORE" if _is_true "${_supabase_enabled:-false}"; then printf " Stack Additions (10%%): " _score_color $SUPABASE_SCORE printf "%3d%%${NC}\n" "$SUPABASE_SCORE" else printf " Stack Additions (10%%): ${BLUE}n/a${NC} (supabase disabled)\n" fi echo "───────────────────────────────────────────" printf " ${BOLD}Overall: " _score_color $TOTAL_SCORE printf "%3d%%${NC}\n" "$TOTAL_SCORE" echo "───────────────────────────────────────────" echo "" if [[ $TOTAL_SCORE -ge 100 ]]; then printf " ${GREEN}${BOLD}✓ All systems operational${NC}\n" else printf " ${YELLOW}${BOLD}⚠ Score below 100%% — review failing components above${NC}\n" fi echo "" # Exit 0 only for a perfect score if [[ $TOTAL_SCORE -ge 100 ]]; then exit 0 else exit 1 fi