prole/etc/status_common_services.sh

#!/usr/bin/env bash
set -u

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"

# Inject default config if not provided
_has_config=0
for _arg in "$@"; do
  [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
done
if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then
  set -- "-c" "$SCRIPT_DIR/../conf/prole.cfg" "$@"
fi
unset _has_config _arg

common_core_preparse_config "$@"

# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

usage() {
  cat <<EOF
Usage: status_common_services.sh [-n|--namespace NS] [-k|--kerberos]

Checks common infrastructure services (ArgoCD, OpenTofu, Garage, OpenBao).
Use -k to include the Prole KDC (auth) checks.
EOF
}

NS=""
ENABLE_KERBEROS=0
while [ $# -gt 0 ]; do
  case "$1" in
    -c|--config)
      shift
      shift
      ;;
    -m|--mode)
      shift
      prole_set_mode "${1:-}"
      shift
      ;;
    -m=*|--mode=*)
      prole_set_mode "${1#*=}"
      shift
      ;;
    -n|--namespace)
      shift
      NS="${1:-}"
      shift
      ;;
    -n=*|--namespace=*)
      NS="${1#*=}"
      shift
      ;;
    -k|--kerberos)
      ENABLE_KERBEROS=1
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      usage
      exit 2
      ;;
  esac
done

if [ -z "$NS" ]; then
  NS="${SERVICE_NAMESPACE:-${NAMESPACE:-}}"
fi
if [ -z "$NS" ]; then
  NS="default"
fi

ARGOCD_NS="${ARGOCD_NAMESPACE:-argocd}"
KONG_NS="${KONG_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-}}}"
CERTMGR_NS="${CERTMGR_NAMESPACE:-cert-manager}"
KONG_NAME="${KONG_NAME:-prole-svc-kong}"

if [ -z "$KONG_NS" ]; then
  KONG_NS="default"
fi
if [ -z "$CERTMGR_NS" ]; then
  CERTMGR_NS="cert-manager"
fi

if ! command -v kubectl >/dev/null 2>&1; then
  echo "ERROR: kubectl not found in PATH"
  exit 1
fi

timestamp=$(date "+%Y-%m-%d %H:%M:%S")
ctx=$(kubectl config current-context 2>/dev/null || true)
server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true)

echo "Common service status"
echo "Time: $timestamp"
echo "Context: ${ctx:-<unknown>}"
echo "Server: ${server:-<unknown>}"
if [[ -n "${KUBECONFIG:-}" ]]; then
  echo "Kubeconfig: $KUBECONFIG"
fi
echo "Namespace: $NS"
echo "ArgoCD Namespace: $ARGOCD_NS"
echo "Kong Namespace: $KONG_NS"
echo "Cert-Manager Namespace: $CERTMGR_NS"
echo ""

run_cmd() {
  echo "\$ $*"
  "$@" 2>&1 || true
  echo ""
}

echo "== Services =="
run_cmd kubectl -n "$ARGOCD_NS" get svc argocd-server
run_cmd kubectl -n "$NS" get svc opentofu garage openbao
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  run_cmd kubectl -n "$NS" get svc auth
fi
run_cmd kubectl -n "$KONG_NS" get svc "$KONG_NAME"
run_cmd kubectl -n "$CERTMGR_NS" get svc cert-manager cert-manager-webhook

echo "== Workloads =="
run_cmd kubectl -n "$ARGOCD_NS" get deploy argocd-server argocd-repo-server argocd-dex-server argocd-applicationset-controller argocd-notifications-controller argocd-redis
run_cmd kubectl -n "$ARGOCD_NS" get statefulset argocd-application-controller
run_cmd kubectl -n "$NS" get deploy opentofu
if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then
  run_cmd kubectl -n "$NS" get statefulset openbao
else
  run_cmd kubectl -n "$NS" get deploy openbao
fi
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  run_cmd kubectl -n "$NS" get deploy auth
fi
run_cmd kubectl -n "$NS" get statefulset garage
run_cmd kubectl -n "$KONG_NS" get deploy "$KONG_NAME"
run_cmd kubectl -n "$CERTMGR_NS" get deploy cert-manager cert-manager-cainjector cert-manager-webhook

echo "== Pods =="
run_cmd kubectl -n "$ARGOCD_NS" get pods | grep -Ei "argocd" || true
run_cmd kubectl -n "$NS" get pods | grep -Ei "opentofu|garage|openbao" || true
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  run_cmd kubectl -n "$NS" get pods | grep -Ei "auth" || true
fi
run_cmd kubectl -n "$KONG_NS" get pods | grep -Ei "kong" || true
run_cmd kubectl -n "$CERTMGR_NS" get pods | grep -Ei "cert-manager" || true

echo "== Health Check =="
missing=0
blocked=0
declare -gA HEALTHY_COUNT
declare -gA BLOCKED_COUNT
declare -gA BLOCKED_PODS
declare -gA COMP_NAMESPACE
HEALTHY_COUNT=()
BLOCKED_COUNT=()
BLOCKED_PODS=()
COMP_NAMESPACE=()
blocked_lines=""

analyze_pods() {
  local ns="$1"
  local pod_filter="$2"
  pod_rows=$(kubectl -n "$ns" get pods --no-headers 2>/dev/null | grep -Ei "$pod_filter" || true)
  if [[ -z "$pod_rows" ]]; then
    return 0
  fi

  pod_matrix=$(printf '%s\n' "$pod_rows" | awk -v re="^(${pod_filter})-" '
    {
      name=$1; ready=$2; status=$3;
      comp="";
      if (match(name, re)) {
        comp=substr(name, RSTART, RLENGTH-1);
      }
      split(ready, a, "/");
      ready_ok=(a[1]==a[2] && a[1] ~ /^[0-9]+$/ && a[2] ~ /^[0-9]+$/);
      healthy=0;
      if (status=="Running") {
        healthy = ready_ok ? 1 : 0;
      } else if (status=="Completed" || status=="Succeeded") {
        healthy = 1;
      }
      if (healthy==0) {
        print comp "|" name "|" ready "|" status "|" healthy;
      } else {
        print comp "|" name "|" ready "|" status "|" healthy;
      }
    }
  ')

  while IFS='|' read -r comp name ready status healthy; do
    [[ -z "$comp" ]] && continue
    if [[ "$comp" == cert-manager* ]]; then
      comp="certmgr"
    fi
    COMP_NAMESPACE["$comp"]="$ns"
    if [[ "$healthy" == "1" ]]; then
      HEALTHY_COUNT["$comp"]=$(( ${HEALTHY_COUNT["$comp"]:-0} + 1 ))
    else
      BLOCKED_COUNT["$comp"]=$(( ${BLOCKED_COUNT["$comp"]:-0} + 1 ))
      BLOCKED_PODS["$comp"]="${BLOCKED_PODS["$comp"]:-} $name"
      blocked_lines+="$ns/$name    $ready   $status\n"
    fi
  done <<< "$pod_matrix"
}

check_resource() {
  local kind=$1
  local name=$2
  local ns=$3
  if ! kubectl -n "$ns" get "$kind" "$name" >/dev/null 2>&1; then
    echo "[FAIL] $kind/$name is missing (ns=$ns)"
    missing=$((missing + 1))
  else
    echo "[OK] $kind/$name exists (ns=$ns)"
  fi
}

check_resource svc argocd-server "$ARGOCD_NS"
check_resource svc opentofu "$NS"
check_resource svc garage "$NS"
check_resource svc openbao "$NS"
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  check_resource svc auth "$NS"
fi
check_resource svc "$KONG_NAME" "$KONG_NS"
check_resource svc cert-manager "$CERTMGR_NS"
check_resource svc cert-manager-webhook "$CERTMGR_NS"

check_resource deploy argocd-server "$ARGOCD_NS"
check_resource deploy argocd-repo-server "$ARGOCD_NS"
check_resource deploy argocd-dex-server "$ARGOCD_NS"
check_resource deploy argocd-applicationset-controller "$ARGOCD_NS"
check_resource deploy argocd-notifications-controller "$ARGOCD_NS"
check_resource deploy argocd-redis "$ARGOCD_NS"
check_resource deploy opentofu "$NS"
if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then
  check_resource statefulset openbao "$NS"
else
  check_resource deploy openbao "$NS"
fi
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  check_resource deploy auth "$NS"
fi
check_resource statefulset argocd-application-controller "$ARGOCD_NS"
check_resource statefulset garage "$NS"
check_resource deploy "$KONG_NAME" "$KONG_NS"
check_resource deploy cert-manager "$CERTMGR_NS"
check_resource deploy cert-manager-cainjector "$CERTMGR_NS"
check_resource deploy cert-manager-webhook "$CERTMGR_NS"

pod_filter="opentofu|garage|openbao|kong|cert-manager"
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
  pod_filter="opentofu|garage|openbao|auth|kong|cert-manager"
fi
analyze_pods "$ARGOCD_NS" "argocd"
analyze_pods "$NS" "$pod_filter"
analyze_pods "$KONG_NS" "kong"
analyze_pods "$CERTMGR_NS" "cert-manager"

if [[ -n "$blocked_lines" ]]; then
  echo ""
  echo "Blocked pods detected (not Ready or not Running):"
  printf '%b' "$blocked_lines"
  blocked=$((blocked + 1))
fi

if [ $missing -gt 0 ]; then
  echo ""
  # ── Attempt to repair a completely absent cert-manager before failing ──────
  if ! kubectl get namespace "$CERTMGR_NS" >/dev/null 2>&1 \
      && [[ -x "$SCRIPT_DIR/init_certmgr.sh" ]]; then
    echo "== Repair: cert-manager namespace '$CERTMGR_NS' absent — re-initializing =="
    "$SCRIPT_DIR/init_certmgr.sh" initialize 2>&1 || true
    echo "  Waiting for cert-manager to settle (20s)..."
    sleep 20
    echo "  Re-checking cert-manager resources..."
    _check_cm_resource() {
      local _kind=$1 _name=$2 _ns=$3
      if ! kubectl -n "$_ns" get "$_kind" "$_name" >/dev/null 2>&1; then
        echo "  [FAIL] $_kind/$_name still missing (ns=$_ns)"
      else
        missing=$((missing - 1))
        echo "  [OK] $_kind/$_name present (ns=$_ns)"
      fi
    }
    _check_cm_resource svc cert-manager "$CERTMGR_NS"
    _check_cm_resource svc cert-manager-webhook "$CERTMGR_NS"
    _check_cm_resource deploy cert-manager "$CERTMGR_NS"
    _check_cm_resource deploy cert-manager-cainjector "$CERTMGR_NS"
    _check_cm_resource deploy cert-manager-webhook "$CERTMGR_NS"
  fi

  if [ $missing -gt 0 ]; then
    echo "Status: FAILED ($missing resources missing)"
    exit 1
  fi
  echo "Status: OK (missing resources restored by repair)"
fi

# ── Repair helpers ────────────────────────────────────────────────────────────

# Emit "kind name ns" lines for every workload owned by a component.
_workloads_for_comp() {
  local _comp="$1"
  local _ns="${COMP_NAMESPACE[$_comp]:-$NS}"
  case "$_comp" in
    argocd)
      echo "statefulset argocd-application-controller $ARGOCD_NS"
      for _d in argocd-server argocd-repo-server argocd-dex-server \
                 argocd-applicationset-controller argocd-notifications-controller argocd-redis; do
        echo "deployment $_d $ARGOCD_NS"
      done
      ;;
    openbao)
      if kubectl -n "$_ns" get statefulset openbao >/dev/null 2>&1; then
        echo "statefulset openbao $_ns"
      else
        echo "deployment openbao $_ns"
      fi
      ;;
    opentofu) echo "deployment opentofu $_ns" ;;
    garage)   echo "statefulset garage $_ns" ;;
    auth)     echo "deployment auth $_ns" ;;
    kong)     echo "deployment $KONG_NAME $KONG_NS" ;;
    certmgr)
      for _d in cert-manager cert-manager-cainjector cert-manager-webhook; do
        echo "deployment $_d $CERTMGR_NS"
      done
      ;;
  esac
}

# Reset all pod-tracking state and re-run analysis.
_recheck_pods() {
  HEALTHY_COUNT=()
  BLOCKED_COUNT=()
  BLOCKED_PODS=()
  COMP_NAMESPACE=()
  blocked_lines=""
  blocked=0
  analyze_pods "$ARGOCD_NS" "argocd"
  analyze_pods "$NS" "$pod_filter"
  analyze_pods "$KONG_NS" "kong"
  analyze_pods "$CERTMGR_NS" "cert-manager"
  if [[ -n "$blocked_lines" ]]; then
    blocked=1
  fi
}

# Print the name of each component that currently has blocked pods.
_blocked_comps() {
  for _bc in argocd openbao opentofu garage auth kong certmgr; do
    [[ "$_bc" == "auth" && "$ENABLE_KERBEROS" != "1" ]] && continue
    [[ ${BLOCKED_COUNT["$_bc"]:-0} -gt 0 ]] && echo "$_bc"
  done
}

# ── Escalating repair ─────────────────────────────────────────────────────────

if [ $blocked -gt 0 ]; then

  # ── Round 1: delete blocked pods ──────────────────────────────────────────
  echo ""
  echo "== Repair Round 1: replacing blocked pods =="
  mapfile -t _r1_comps < <(_blocked_comps)
  for _comp in "${_r1_comps[@]}"; do
    _pods="${BLOCKED_PODS[$_comp]:-}"
    _ns="${COMP_NAMESPACE[$_comp]:-$NS}"
    if [[ -n "$_pods" ]]; then
      echo "  Deleting pod(s) for $_comp in $_ns:$_pods"
      # shellcheck disable=SC2086
      kubectl -n "$_ns" delete pod $_pods --wait=false >/dev/null 2>&1 || true
    fi
  done
  echo "  Waiting for pod replacement (15s)..."
  sleep 15
  _recheck_pods

  if [[ $blocked -eq 0 ]]; then
    echo ""
    echo "Status: OK (resolved in repair round 1)"
    exit 0
  fi

  echo ""
  echo "Pods still blocked after Round 1:"
  printf '%b' "$blocked_lines"

  # ── Round 2: rollout restart owning StatefulSet / Deployment ──────────────
  echo ""
  echo "== Repair Round 2: restarting workload controllers =="
  mapfile -t _r2_comps < <(_blocked_comps)
  for _comp in "${_r2_comps[@]}"; do
    while IFS=' ' read -r _kind _name _wns; do
      [[ -z "$_kind" ]] && continue
      if kubectl -n "$_wns" get "$_kind" "$_name" >/dev/null 2>&1; then
        echo "  kubectl rollout restart $_kind/$_name -n $_wns"
        kubectl -n "$_wns" rollout restart "$_kind/$_name" >/dev/null 2>&1 || true
        kubectl -n "$_wns" rollout status "$_kind/$_name" --timeout=60s 2>/dev/null || true
      fi
    done < <(_workloads_for_comp "$_comp")
  done
  echo "  Waiting for rollout to settle (20s)..."
  sleep 20
  _recheck_pods

  if [[ $blocked -eq 0 ]]; then
    echo ""
    echo "Status: OK (resolved in repair round 2)"
    exit 0
  fi

  echo ""
  echo "Pods still blocked after Round 2:"
  printf '%b' "$blocked_lines"

  # ── Round 3: clear affected namespaces and re-initialize ──────────────────
  echo ""
  echo "== Repair Round 3: clearing affected namespaces and re-initializing =="
  mapfile -t _r3_comps < <(_blocked_comps)
  declare -A _cleared_ns=()
  for _comp in "${_r3_comps[@]}"; do
    _ns="${COMP_NAMESPACE[$_comp]:-$NS}"
    [[ -n "${_cleared_ns[$_ns]:-}" ]] && continue
    _cleared_ns["$_ns"]=1
    if [[ "$_ns" == "default" || "$_ns" == "kube-system" ]]; then
      echo "  [Round 3] Removing workload resources in protected namespace '$_ns' for: $_comp"
      while IFS=' ' read -r _kind _name _wns; do
        [[ -z "$_kind" ]] && continue
        echo "    kubectl delete $_kind $_name -n $_wns --ignore-not-found"
        kubectl -n "$_wns" delete "$_kind" "$_name" --ignore-not-found >/dev/null 2>&1 || true
      done < <(_workloads_for_comp "$_comp")
    else
      echo "  [Round 3] Deleting namespace '$_ns' ..."
      kubectl delete namespace "$_ns" --wait=true --timeout=90s >/dev/null 2>&1 || true
      echo "  [Round 3] Recreating namespace '$_ns' ..."
      kubectl create namespace "$_ns" >/dev/null 2>&1 || true
      echo "    [OK] Namespace '$_ns' cleared and recreated."
    fi
  done

  if [[ -x "$SCRIPT_DIR/init_service_layer.sh" ]]; then
    echo "  [Round 3] Re-initializing service layer ..."
    _krb_flag=""
    [[ "$ENABLE_KERBEROS" == "1" ]] && _krb_flag="-k"
    # shellcheck disable=SC2086
    "$SCRIPT_DIR/init_service_layer.sh" -n "$NS" $_krb_flag update 2>&1 || true
  fi

  echo "  [Round 3] Re-checking status after namespace reset (30s)..."
  sleep 30
  _recheck_pods

  if [[ $blocked -eq 0 ]]; then
    echo ""
    echo "Status: OK (resolved in repair round 3)"
    exit 0
  fi

  echo ""
  echo "Pods still blocked after Round 3:"
  printf '%b' "$blocked_lines"

  # ── Round 4: reset k3d cluster, preserve registry ─────────────────────────
  _prole_mode="${PROLE_MODE:-}"
  if [[ "$_prole_mode" == "k3d" ]]; then
    echo ""
    echo "== Repair Round 4: resetting k3d cluster (preserving registry) =="
    _cluster_name="${K3D_CLUSTER_NAME:-knoe-dev-cluster}"

    _registry_args=()
    if command -v k3d >/dev/null 2>&1; then
      _reg_name=$(k3d registry list --no-headers 2>/dev/null | awk '{print $1}' | head -1 || true)
      if [[ -n "$_reg_name" ]]; then
        echo "  Preserving registry: $_reg_name"
        _registry_args=(--registry-use "$_reg_name")
      fi
    fi

    echo "  Deleting k3d cluster '$_cluster_name' ..."
    k3d cluster delete "$_cluster_name" >/dev/null 2>&1 || true

    echo "  Recreating k3d cluster '$_cluster_name' ..."
    k3d cluster create "$_cluster_name" -a 2 \
      "${_registry_args[@]}" \
      --api-port 0.0.0.0:6443 >/dev/null 2>&1 || true

    kubectl config use-context "k3d-${_cluster_name}" >/dev/null 2>&1 || true

    if [[ -x "$SCRIPT_DIR/init_service_layer.sh" ]]; then
      echo "  [Round 4] Re-initializing service layer after cluster reset ..."
      _krb_flag=""
      [[ "$ENABLE_KERBEROS" == "1" ]] && _krb_flag="-k"
      # shellcheck disable=SC2086
      "$SCRIPT_DIR/init_service_layer.sh" -n "$NS" $_krb_flag update 2>&1 || true
    fi

    echo "  [Round 4] Re-checking status after cluster reset (30s)..."
    sleep 30
    _recheck_pods

    if [[ $blocked -eq 0 ]]; then
      echo ""
      echo "Status: OK (resolved in repair round 4 — cluster reset)"
      exit 0
    fi

    echo ""
    echo "Pods still blocked after Round 4 (cluster reset):"
    printf '%b' "$blocked_lines"
    echo ""
    echo "Status: FAILED — unable to repair after full cluster reset. Manual intervention required."
    exit 5
  fi

  echo ""
  echo "Status: NOTICE (blocked pods remain; k3d cluster reset not applicable for mode '${_prole_mode:-unknown}')"
  exit 3

fi

echo ""
echo "Status: OK"
exit 0