prole/etc/status_common_services.sh

#!/usr/bin/env bash
set -u

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

usage() {
  cat <<EOF
Usage: status_common_services.sh [-n|--namespace NS] [-k|--kerberos]

Checks common infrastructure services (ArgoCD, OpenBao, OpenTofu, Garage).
Use -k to include the Prole KDC (auth) checks.
EOF
}

NS=""
ENABLE_KERBEROS=0
while [ $# -gt 0 ]; do
  case "$1" in
    -n|--namespace)
      shift
      NS="${1:-}"
      shift
      ;;
    -n=*|--namespace=*)
      NS="${1#*=}"
      shift
      ;;
    -k|--kerberos)
      ENABLE_KERBEROS=1
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      usage
      exit 2
      ;;
  esac
done

if [ -z "$NS" ]; then
  NS="${SERVICE_NAMESPACE:-${NAMESPACE:-}}"
fi
if [ -z "$NS" ]; then
  NS="default"
fi

if ! command -v kubectl >/dev/null 2>&1; then
  echo "ERROR: kubectl not found in PATH"
  exit 1
fi

timestamp=$(date "+%Y-%m-%d %H:%M:%S")
ctx=$(kubectl config current-context 2>/dev/null || true)
server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true)

echo "Common service status"
echo "Time: $timestamp"
echo "Context: ${ctx:-<unknown>}"
echo "Server: ${server:-<unknown>}"
if [[ -n "${KUBECONFIG:-}" ]]; then
  echo "Kubeconfig: $KUBECONFIG"
fi
echo "Namespace: $NS"
echo ""

run_cmd() {
  echo "\$ $*"
  "$@" 2>&1 || true
  echo ""
}

echo "== Services =="
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  run_cmd kubectl -n "$NS" get svc argocd-server openbao opentofu garage auth
else
  run_cmd kubectl -n "$NS" get svc argocd-server openbao opentofu garage
fi

echo "== Workloads =="
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  run_cmd kubectl -n "$NS" get deploy argocd-server argocd-repo-server argocd-dex-server argocd-applicationset-controller argocd-notifications-controller argocd-redis opentofu auth
else
  run_cmd kubectl -n "$NS" get deploy argocd-server argocd-repo-server argocd-dex-server argocd-applicationset-controller argocd-notifications-controller argocd-redis opentofu
fi
run_cmd kubectl -n "$NS" get statefulset argocd-application-controller openbao garage

echo "== Pods =="
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  run_cmd kubectl -n "$NS" get pods | grep -Ei "argocd|openbao|opentofu|garage|auth" || true
else
  run_cmd kubectl -n "$NS" get pods | grep -Ei "argocd|openbao|opentofu|garage" || true
fi

echo "== Health Check =="
missing=0
blocked=0

analyze_pods() {
  blocked_lines=""
  pod_rows=$(kubectl -n "$NS" get pods --no-headers 2>/dev/null | grep -Ei "$pod_filter" || true)
  if [[ -z "$pod_rows" ]]; then
    return 0
  fi

  pod_matrix=$(printf '%s\n' "$pod_rows" | awk -v re="^(${pod_filter})-" '
    {
      name=$1; ready=$2; status=$3;
      comp="";
      if (match(name, re)) {
        comp=substr(name, RSTART, RLENGTH-1);
      }
      split(ready, a, "/");
      ready_ok=(a[1]==a[2] && a[1] ~ /^[0-9]+$/ && a[2] ~ /^[0-9]+$/);
      healthy=0;
      if (status=="Running") {
        healthy = ready_ok ? 1 : 0;
      } else if (status=="Completed" || status=="Succeeded") {
        healthy = 1;
      }
      if (healthy==0) {
        print comp "|" name "|" ready "|" status "|" healthy;
      } else {
        print comp "|" name "|" ready "|" status "|" healthy;
      }
    }
  ')

  declare -gA HEALTHY_COUNT
  declare -gA BLOCKED_COUNT
  declare -gA BLOCKED_PODS
  HEALTHY_COUNT=()
  BLOCKED_COUNT=()
  BLOCKED_PODS=()
  blocked_lines=""

  while IFS='|' read -r comp name ready status healthy; do
    [[ -z "$comp" ]] && continue
    if [[ "$healthy" == "1" ]]; then
      HEALTHY_COUNT["$comp"]=$(( ${HEALTHY_COUNT["$comp"]:-0} + 1 ))
    else
      BLOCKED_COUNT["$comp"]=$(( ${BLOCKED_COUNT["$comp"]:-0} + 1 ))
      BLOCKED_PODS["$comp"]="${BLOCKED_PODS["$comp"]:-} $name"
      blocked_lines+="$name    $ready   $status\n"
    fi
  done <<< "$pod_matrix"
}

check_resource() {
  local kind=$1
  local name=$2
  if ! kubectl -n "$NS" get "$kind" "$name" >/dev/null 2>&1; then
    echo "[FAIL] $kind/$name is missing"
    missing=$((missing + 1))
  else
    echo "[OK] $kind/$name exists"
  fi
}

check_resource svc argocd-server
check_resource svc openbao
check_resource svc opentofu
check_resource svc garage
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  check_resource svc auth
fi

check_resource deploy argocd-server
check_resource deploy argocd-repo-server
check_resource deploy argocd-dex-server
check_resource deploy argocd-applicationset-controller
check_resource deploy argocd-notifications-controller
check_resource deploy argocd-redis
check_resource deploy opentofu
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  check_resource deploy auth
fi
check_resource statefulset argocd-application-controller
check_resource statefulset openbao
check_resource statefulset garage

pod_filter="argocd|openbao|opentofu|garage"
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  pod_filter="argocd|openbao|opentofu|garage|auth"
fi

analyze_pods

if [[ -n "$blocked_lines" ]]; then
  echo ""
  echo "Blocked pods detected (not Ready or not Running):"
  printf '%b' "$blocked_lines"
  blocked=$((blocked + 1))
fi

if [ $missing -gt 0 ]; then
  echo ""
  echo "Status: FAILED ($missing resources missing)"
  exit 1
fi

if [ $blocked -gt 0 ]; then
  repairable=()
  fail_all=()
  for comp in argocd openbao opentofu garage auth; do
    if [[ "$comp" == "auth" && "$ENABLE_KERBEROS" != "1" ]]; then
      continue
    fi
    hc=${HEALTHY_COUNT["$comp"]:-0}
    bc=${BLOCKED_COUNT["$comp"]:-0}
    if [[ "$bc" -gt 0 && "$hc" -gt 0 ]]; then
      repairable+=("$comp")
    elif [[ "$bc" -gt 0 && "$hc" -eq 0 ]]; then
      fail_all+=("$comp")
    fi
  done

  if [ ${#fail_all[@]} -gt 0 ]; then
    echo ""
    echo "Repair skipped: all pods blocked for ${fail_all[*]}."
    echo "Action: Deploy Missing Services."
    echo ""
    echo "Status: NOTICE (blocked pods detected)"
    exit 2
  fi

  if [ ${#repairable[@]} -gt 0 ]; then
    echo ""
    echo "== Repair =="
    echo "Repairing blocked pods for: ${repairable[*]}"
    for comp in "${repairable[@]}"; do
      pods="${BLOCKED_PODS["$comp"]:-}"
      if [[ -n "$pods" ]]; then
        kubectl -n "$NS" delete pod $pods --wait=false >/dev/null 2>&1 || true
      fi
    done
    echo "Re-checking pods after repair..."
    sleep 2
    analyze_pods
    if [[ -n "$blocked_lines" ]]; then
      echo ""
      echo "Blocked pods still detected after repair:"
      printf '%b' "$blocked_lines"
      echo ""
      echo "Status: NOTICE (blocked pods detected)"
      exit 2
    fi
  fi
fi

echo ""
echo "Status: OK"
exit 0