prole/etc/status.sh

#!/usr/bin/env bash
# etc/status.sh — Overall deployment health check for the Knoe k3d environment.
# Usage: status.sh [-c conf/{k3d|k3s|gke}.cfg] [-v|--verbose]
# Exit 0  = score of 100% (all components healthy)
# Exit 1  = score < 100%
set -u

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# shellcheck disable=SC1090
source "$SCRIPT_DIR/knoe_cfg.sh"

# ── colours (disabled when stdout is not a tty) ─────────────────────────
if [[ -t 1 ]] || [[ "${CLICOLOR_FORCE:-0}" == "1" ]]; then
  RED='\033[0;31m'
  GREEN='\033[0;32m'
  YELLOW='\033[1;33m'
  BLUE='\033[0;34m'
  BOLD='\033[1m'
  NC='\033[0m'
else
  RED='' GREEN='' YELLOW='' BLUE='' BOLD='' NC=''
fi

# ── defaults ─────────────────────────────────────────────────────────────
VERBOSE=0
CFG_PATH=""
KUBECONFIG="${KUBECONFIG:-}"
PROLE_NAMESPACE=""
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-knoe-db}"

# ── helpers ──────────────────────────────────────────────────────────────
usage() {
  cat <<EOF
Usage: $(basename "$0") [-c <config.cfg>] [-v|--verbose] [-h|--help]

Overall deployment health check.
  -c, --config <path>   Path to config file (default: conf/{k3d|k3s|gke}.cfg)
  -v, --verbose         Show full stdout/stderr from each status check
  -h, --help            Show this help
EOF
}

ok()   { printf "${GREEN}[OK]${NC}    %s\n" "$*"; }
fail() { printf "${RED}[FAIL]${NC}  %s\n" "$*"; }
warn() { printf "${YELLOW}[WARN]${NC}  %s\n" "$*"; }
info() { printf "${BLUE}[INFO]${NC}  %s\n" "$*"; }

# Portable timeout wrapper (macOS lacks coreutils timeout by default).
# Usage: run_with_timeout <seconds> <command> [args...]
# Returns 124 on timeout, otherwise the command's exit code.
STATUS_CHECK_TIMEOUT=${STATUS_CHECK_TIMEOUT:-30}
run_with_timeout() {
  local secs="$1"; shift
  "$@" &
  local cmd_pid=$!
  ( sleep "$secs" && kill "$cmd_pid" 2>/dev/null ) &
  local watcher_pid=$!
  wait "$cmd_pid" 2>/dev/null
  local rc=$?
  kill "$watcher_pid" 2>/dev/null
  wait "$watcher_pid" 2>/dev/null || true
  # If the process was killed by our watcher, rc is 137 (128+9) or 143 (128+15)
  if [[ $rc -eq 137 || $rc -eq 143 ]]; then
    return 124
  fi
  return $rc
}

# ── parse arguments ──────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
  case "$1" in
    -c|--config)
      shift; CFG_PATH="${1:-}"; shift ;;
    -c=*|--config=*)
      CFG_PATH="${1#*=}"; shift ;;
    -v|--verbose)
      VERBOSE=1; shift ;;
    -h|--help)
      usage; exit 0 ;;
    *)
      usage; exit 2 ;;
  esac
done

# ── load config (always) ────────────────────────────────────────────────
if [[ -z "$CFG_PATH" ]]; then
  CFG_PATH="$(_knoe_cfg_select_cfg_file "$PROJECT_ROOT/conf")"
fi
[[ -z "$CFG_PATH" ]] && CFG_PATH="$PROJECT_ROOT/conf/k3d.cfg"
if [[ ! -f "$CFG_PATH" ]]; then
  echo "FATAL: config file not found at $CFG_PATH" >&2
  exit 1
fi

# Minimal INI reader — pull NAMESPACE, KUBECONFIG, KERBEROS_ENABLED,
# SUPABASE_ENABLED from config file if not already set in the environment.
_read_cfg_value() {
  local key="$1"
  local val=""
  # Grab last occurrence (case-insensitive key match) from INI file
  val="$(grep -i "^[[:space:]]*${key}[[:space:]]*=" "$CFG_PATH" 2>/dev/null \
    | tail -1 | sed 's/^[^=]*=[[:space:]]*//' | sed 's/[[:space:]]*$//')"
  # If the value is a ${VAR} reference, resolve it from the first concrete
  # occurrence of VAR in the same config file.
  if [[ "$val" == '${'*'}' ]]; then
    local ref_key="${val#\$\{}"
    ref_key="${ref_key%\}}"
    # Find first concrete (non-${...}) value for the referenced key
    val="$(grep -i "^[[:space:]]*${ref_key}[[:space:]]*=" "$CFG_PATH" 2>/dev/null \
      | sed 's/^[^=]*=[[:space:]]*//' | sed 's/[[:space:]]*$//' \
      | grep -v '^\$' | head -1)"
  fi
  echo "$val"
}

if [[ -z "$PROLE_NAMESPACE" ]]; then
  PROLE_NAMESPACE="$(_read_cfg_value 'NAMESPACE')"
fi
if [[ -z "$PROLE_NAMESPACE" ]]; then
  PROLE_NAMESPACE="default"
fi

_deployment_mode="$(_read_cfg_value 'DEPLOYMENT_MODE')"

if [[ -z "$KUBECONFIG" ]]; then
  _kc="$(_read_cfg_value 'KUBECONFIG')"
  if [[ -n "$_kc" && -f "$_kc" ]]; then
    export KUBECONFIG="$_kc"
  fi
fi
# Fallback: project-local kubeconfig (only if it can reach the cluster)
if [[ -z "${KUBECONFIG:-}" && -f "$PROJECT_ROOT/knoe-k3s.kubeconfig" ]]; then
  # Only auto-select the project-local k3s kubeconfig when explicitly running in k3s mode.
  # In k3d mode, this kubeconfig may be reachable but is the wrong cluster.
  if [[ "${_deployment_mode,,}" == "k3s" ]]; then
    if KUBECONFIG="$PROJECT_ROOT/knoe-k3s.kubeconfig" kubectl cluster-info >/dev/null 2>&1; then
      export KUBECONFIG="$PROJECT_ROOT/knoe-k3s.kubeconfig"
    fi
  fi
fi

_kerberos_enabled="$(_read_cfg_value 'KERBEROS_ENABLED')"
_supabase_enabled="$(_read_cfg_value 'SUPABASE_ENABLED')"

_is_true() {
  case "${1,,}" in
    true|yes|1) return 0 ;;
    *) return 1 ;;
  esac
}

# ── pre-flight ───────────────────────────────────────────────────────────
if ! command -v kubectl >/dev/null 2>&1; then
  fail "kubectl not found in PATH"
  exit 1
fi

OVERALL_RC=0   # will flip to 1 on any failure

# ── Scoring variables ────────────────────────────────────────────────────
# Rubric: 40% Common Core, 50% CNPG, 10% Stack Additions (Supabase)
CORE_TOTAL=0
CORE_PASS=0
CNPG_SCORE=0        # 0-100 within its own category
SUPABASE_SCORE=0     # 0-100 within its own category
SUPABASE_WEIGHT=10   # Only counted if supabase is enabled

# ── header ───────────────────────────────────────────────────────────────
echo ""
printf "${BOLD}Knoe Deployment Status${NC}\n"
echo "───────────────────────────────────────────"
info "Config:     $CFG_PATH"
info "Namespace:  $PROLE_NAMESPACE"
ctx="$(kubectl config current-context 2>/dev/null || true)"
info "Context:    ${ctx:-<unknown>}"
if [[ -n "${KUBECONFIG:-}" ]]; then
  info "Kubeconfig: $KUBECONFIG"
fi
echo ""

# ── 1. kubectl get pods -o wide -A ──────────────────────────────────────
printf "${BOLD}All Pods${NC}\n"
echo "───────────────────────────────────────────"
pod_output="$(kubectl get pods -o wide -A 2>/dev/null)" || true
if [[ $VERBOSE -eq 1 ]]; then
  echo "$pod_output"
  echo ""
fi

# Evaluate pod health
_bad_pods=""
while IFS= read -r line; do
  # skip header
  [[ "$line" == NAMESPACE* ]] && continue
  [[ -z "$line" ]] && continue
  status_field="$(echo "$line" | awk '{print $4}')"
  case "$status_field" in
    Running|Completed|Succeeded) ;;
    Terminating)
      warn "Terminating pod: $(echo "$line" | awk '{printf "%s/%s", $1, $2}')"
      ;;
    Pending|ContainerCreating|Init:*|PodInitializing)
      warn "Pending/init pod: $(echo "$line" | awk '{printf "%s/%s (%s)", $1, $2, $4}')"
      ;;
    *)
      _bad_pods="yes"
      fail "Unhealthy pod: $(echo "$line" | awk '{printf "%s/%s status=%s", $1, $2, $4}')"
      ;;
  esac
done <<< "$pod_output"

if [[ -z "$_bad_pods" ]]; then
  ok "All pods in expected state"
else
  OVERALL_RC=1
fi
echo ""

# ── 2. CloudNative-PG cluster status ────────────────────────────────────
printf "${BOLD}CloudNative-PG: %s${NC}\n" "$CNPG_CLUSTER_NAME"
echo "───────────────────────────────────────────"
if kubectl -n "$PROLE_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
  cnpg_output=""
  if command -v kubectl-cnpg >/dev/null 2>&1 || kubectl cnpg version >/dev/null 2>&1; then
    cnpg_output="$(kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$PROLE_NAMESPACE" 2>&1)" || true
    echo "$cnpg_output"
  else
    kubectl -n "$PROLE_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o wide 2>&1 || true
    info "kubectl cnpg plugin not available; showing basic cluster info only"
  fi
  # Simple health: check if cluster phase is healthy
  phase="$(kubectl -n "$PROLE_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
  if [[ "$phase" == "Cluster in healthy state" || "$phase" == "Healthy" ]]; then
    ok "CNPG cluster $CNPG_CLUSTER_NAME is healthy"
    CNPG_SCORE=100
  elif [[ -n "$phase" ]]; then
    warn "CNPG cluster phase: $phase"
    # Partial — cluster exists but not fully healthy → 50% of CNPG score
    CNPG_SCORE=50
  else
    fail "Unable to determine CNPG cluster phase"
    CNPG_SCORE=0
    OVERALL_RC=1
  fi
else
  fail "CNPG cluster '$CNPG_CLUSTER_NAME' not found in namespace '$PROLE_NAMESPACE'"
  CNPG_SCORE=0
  OVERALL_RC=1
fi
echo ""

# ── 3. Per-script status checks (Common Core) ───────────────────────────
printf "${BOLD}Service Status Checks${NC}\n"
echo "───────────────────────────────────────────"

# Scripts that were deployed during install.py — order matches deployment sequence.
STATUS_SCRIPTS=(
  "init_common_services.sh"
  "init_cloudnative_pg.sh"
  "init_openbao.sh"
  "init_kong.sh"
  "init_monitoring.sh"
  "init_cnpg_backup.sh"
  "init_port_forwards.sh"
)

# Conditional scripts
if _is_true "${_kerberos_enabled:-false}"; then
  STATUS_SCRIPTS+=("init_kerberos.sh")
fi

run_status_check() {
  local script="$1"
  local script_path="$SCRIPT_DIR/$script"
  local label="${script%.sh}"
  label="${label#init_}"

  if [[ ! -x "$script_path" ]]; then
    if [[ -f "$script_path" ]]; then
      chmod +x "$script_path" 2>/dev/null || true
    fi
    if [[ ! -f "$script_path" ]]; then
      warn "$label — script not found ($script_path)"
      CORE_TOTAL=$((CORE_TOTAL + 1))
      return
    fi
  fi

  local _output_file
  _output_file=$(mktemp "${TMPDIR:-/tmp}/knoe_status_XXXXXX")

  local rc
  run_with_timeout "$STATUS_CHECK_TIMEOUT" bash "$script_path" status >"$_output_file" 2>&1 && rc=0 || rc=$?

  local output
  output=$(cat "$_output_file" 2>/dev/null || true)
  rm -f "$_output_file"

  CORE_TOTAL=$((CORE_TOTAL + 1))
  if [[ $rc -eq 124 ]]; then
    fail "$label (timeout after ${STATUS_CHECK_TIMEOUT}s)"
    OVERALL_RC=1
  elif [[ $rc -eq 0 ]]; then
    ok "$label"
    CORE_PASS=$((CORE_PASS + 1))
  else
    fail "$label (exit $rc)"
    OVERALL_RC=1
  fi

  if [[ $VERBOSE -eq 1 && -n "$output" ]]; then
    echo "$output" | sed 's/^/    /'
    echo ""
  fi
}

for s in "${STATUS_SCRIPTS[@]}"; do
  run_status_check "$s"
done

# ── 4. Supabase (if enabled) ────────────────────────────────────────────
if _is_true "${_supabase_enabled:-false}"; then
  echo ""
  printf "${BOLD}Supabase${NC}\n"
  echo "───────────────────────────────────────────"
  supa_ns="${SUPABASE_NAMESPACE:-supabase}"
  supa_pods="$(kubectl get pods -n "$supa_ns" --no-headers 2>&1)" || true
  if [[ -z "$supa_pods" || "$supa_pods" == *"not found"* || "$supa_pods" == *"No resources"* ]]; then
    fail "No Supabase pods found in namespace '$supa_ns'"
    SUPABASE_SCORE=0
    OVERALL_RC=1
  else
    _supa_bad=""
    _supa_total=0
    _supa_healthy=0
    while IFS= read -r line; do
      [[ -z "$line" ]] && continue
      _supa_total=$((_supa_total + 1))
      st="$(echo "$line" | awk '{print $3}')"
      case "$st" in
        Running|Completed|Succeeded) _supa_healthy=$((_supa_healthy + 1)) ;;
        *) _supa_bad="yes" ;;
      esac
    done <<< "$supa_pods"
    if [[ -z "$_supa_bad" ]]; then
      ok "Supabase pods healthy in namespace '$supa_ns'"
      SUPABASE_SCORE=100
    else
      fail "Unhealthy Supabase pods in namespace '$supa_ns'"
      if [[ $_supa_total -gt 0 ]]; then
        SUPABASE_SCORE=$(( (_supa_healthy * 100) / _supa_total ))
      fi
      OVERALL_RC=1
    fi
    if [[ $VERBOSE -eq 1 ]]; then
      echo "$supa_pods" | sed 's/^/    /'
      echo ""
    fi
  fi
else
  # Supabase disabled — redistribute its weight to the other two categories
  SUPABASE_WEIGHT=0
fi

# ── 5. Status of status_common_services.sh (if present) ─────────────────
if [[ -f "$SCRIPT_DIR/status_common_services.sh" ]]; then
  echo ""
  printf "${BOLD}Common Services${NC}\n"
  echo "───────────────────────────────────────────"
  _service_ns="$(_read_cfg_value 'SERVICE_NAMESPACE')"
  if [[ -z "${_service_ns:-}" ]]; then
    _service_ns="$PROLE_NAMESPACE"
  fi
  cs_args=("-n" "${_service_ns:-default}")
  if _is_true "${_kerberos_enabled:-false}"; then
    cs_args+=("-k")
  fi
  _cs_output_file=$(mktemp "${TMPDIR:-/tmp}/knoe_cs_status_XXXXXX")
  run_with_timeout "$STATUS_CHECK_TIMEOUT" bash "$SCRIPT_DIR/status_common_services.sh" "${cs_args[@]}" >"$_cs_output_file" 2>&1 && cs_rc=0 || cs_rc=$?
  cs_output=$(cat "$_cs_output_file" 2>/dev/null || true)
  rm -f "$_cs_output_file"
  if [[ $cs_rc -eq 124 ]]; then
    fail "Common services (timeout after ${STATUS_CHECK_TIMEOUT}s)"
    OVERALL_RC=1
  elif [[ $cs_rc -eq 0 ]]; then
    ok "Common services"
  else
    fail "Common services (exit $cs_rc)"
    OVERALL_RC=1
  fi
  if [[ $VERBOSE -eq 1 && -n "$cs_output" ]]; then
    echo "$cs_output" | sed 's/^/    /'
    echo ""
  fi
fi

# ── Scoring ──────────────────────────────────────────────────────────────
# Rubric weights:
#   Common Core:   40%
#   CloudNative-PG: 50%
#   Stack Additions (Supabase): 10%  (0% when disabled, redistributed)
#
# When Supabase is disabled the weights become 44.4% / 55.6% (proportional).

echo ""
echo "───────────────────────────────────────────"
printf "${BOLD}Score${NC}\n"
echo "───────────────────────────────────────────"

# Common Core percentage (from per-script checks)
if [[ $CORE_TOTAL -gt 0 ]]; then
  CORE_PCT=$(( (CORE_PASS * 100) / CORE_TOTAL ))
else
  CORE_PCT=100
fi

# Weights
W_CORE=40
W_CNPG=50
W_SUPA=$SUPABASE_WEIGHT

# If supabase is disabled, redistribute proportionally
if [[ $W_SUPA -eq 0 ]]; then
  # 40/(40+50) = 44.4…  50/(40+50) = 55.5…  → use integer math x10
  W_CORE=44
  W_CNPG=56
fi

# Weighted total (integer arithmetic, scale by 100 for precision)
TOTAL_SCORE=$(( (CORE_PCT * W_CORE + CNPG_SCORE * W_CNPG + SUPABASE_SCORE * W_SUPA) / (W_CORE + W_CNPG + W_SUPA) ))

# Clamp
if [[ $TOTAL_SCORE -gt 100 ]]; then TOTAL_SCORE=100; fi
if [[ $TOTAL_SCORE -lt 0 ]]; then TOTAL_SCORE=0; fi

# Color helper for a percentage
_score_color() {
  local pct=$1
  if [[ $pct -ge 100 ]]; then
    printf "${GREEN}"
  elif [[ $pct -ge 75 ]]; then
    printf "${YELLOW}"
  else
    printf "${RED}"
  fi
}

# Print section scores
printf "  Common Core (40%%):       "
_score_color $CORE_PCT
printf "%3d%% " "$CORE_PCT"
printf "${NC}(%d/%d checks)\n" "$CORE_PASS" "$CORE_TOTAL"

printf "  CloudNative-PG (50%%):    "
_score_color $CNPG_SCORE
printf "%3d%%${NC}\n" "$CNPG_SCORE"

if _is_true "${_supabase_enabled:-false}"; then
  printf "  Stack Additions (10%%):   "
  _score_color $SUPABASE_SCORE
  printf "%3d%%${NC}\n" "$SUPABASE_SCORE"
else
  printf "  Stack Additions (10%%):   ${BLUE}n/a${NC} (supabase disabled)\n"
fi

echo "───────────────────────────────────────────"
printf "  ${BOLD}Overall: "
_score_color $TOTAL_SCORE
printf "%3d%%${NC}\n" "$TOTAL_SCORE"
echo "───────────────────────────────────────────"
echo ""

if [[ $TOTAL_SCORE -ge 100 ]]; then
  printf "  ${GREEN}${BOLD}✓ All systems operational${NC}\n"
else
  printf "  ${YELLOW}${BOLD}⚠ Score below 100%% — review failing components above${NC}\n"
fi
echo ""

# Exit 0 only for a perfect score
if [[ $TOTAL_SCORE -ge 100 ]]; then
  exit 0
else
  exit 1
fi