#!/usr/bin/env bash set -u SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck disable=SC1090 source "$SCRIPT_DIR/prole_cfg.sh" usage() { cat </dev/null 2>&1; then echo "ERROR: kubectl not found in PATH" exit 1 fi timestamp=$(date "+%Y-%m-%d %H:%M:%S") ctx=$(kubectl config current-context 2>/dev/null || true) server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true) echo "Common service status" echo "Time: $timestamp" echo "Context: ${ctx:-}" echo "Server: ${server:-}" if [[ -n "${KUBECONFIG:-}" ]]; then echo "Kubeconfig: $KUBECONFIG" fi echo "Namespace: $NS" echo "ArgoCD Namespace: $ARGOCD_NS" echo "" run_cmd() { echo "\$ $*" "$@" 2>&1 || true echo "" } echo "== Services ==" run_cmd kubectl -n "$ARGOCD_NS" get svc argocd-server if [[ "$ENABLE_KERBEROS" == "1" ]]; then run_cmd kubectl -n "$NS" get svc opentofu garage auth else run_cmd kubectl -n "$NS" get svc opentofu garage fi echo "== Workloads ==" run_cmd kubectl -n "$ARGOCD_NS" get deploy argocd-server argocd-repo-server argocd-dex-server argocd-applicationset-controller argocd-notifications-controller argocd-redis run_cmd kubectl -n "$ARGOCD_NS" get statefulset argocd-application-controller if [[ "$ENABLE_KERBEROS" == "1" ]]; then run_cmd kubectl -n "$NS" get deploy opentofu auth else run_cmd kubectl -n "$NS" get deploy opentofu fi run_cmd kubectl -n "$NS" get statefulset garage echo "== Pods ==" run_cmd kubectl -n "$ARGOCD_NS" get pods | grep -Ei "argocd" || true if [[ "$ENABLE_KERBEROS" == "1" ]]; then run_cmd kubectl -n "$NS" get pods | grep -Ei "opentofu|garage|auth" || true else run_cmd kubectl -n "$NS" get pods | grep -Ei "opentofu|garage" || true fi echo "== Health Check ==" missing=0 blocked=0 declare -gA HEALTHY_COUNT declare -gA BLOCKED_COUNT declare -gA BLOCKED_PODS declare -gA COMP_NAMESPACE HEALTHY_COUNT=() BLOCKED_COUNT=() BLOCKED_PODS=() COMP_NAMESPACE=() blocked_lines="" analyze_pods() { local ns="$1" local pod_filter="$2" pod_rows=$(kubectl -n "$ns" get pods --no-headers 2>/dev/null | grep -Ei "$pod_filter" || true) if [[ -z "$pod_rows" ]]; then return 0 fi pod_matrix=$(printf '%s\n' "$pod_rows" | awk -v re="^(${pod_filter})-" ' { name=$1; ready=$2; status=$3; comp=""; if (match(name, re)) { comp=substr(name, RSTART, RLENGTH-1); } split(ready, a, "/"); ready_ok=(a[1]==a[2] && a[1] ~ /^[0-9]+$/ && a[2] ~ /^[0-9]+$/); healthy=0; if (status=="Running") { healthy = ready_ok ? 1 : 0; } else if (status=="Completed" || status=="Succeeded") { healthy = 1; } if (healthy==0) { print comp "|" name "|" ready "|" status "|" healthy; } else { print comp "|" name "|" ready "|" status "|" healthy; } } ') while IFS='|' read -r comp name ready status healthy; do [[ -z "$comp" ]] && continue COMP_NAMESPACE["$comp"]="$ns" if [[ "$healthy" == "1" ]]; then HEALTHY_COUNT["$comp"]=$(( ${HEALTHY_COUNT["$comp"]:-0} + 1 )) else BLOCKED_COUNT["$comp"]=$(( ${BLOCKED_COUNT["$comp"]:-0} + 1 )) BLOCKED_PODS["$comp"]="${BLOCKED_PODS["$comp"]:-} $name" blocked_lines+="$ns/$name $ready $status\n" fi done <<< "$pod_matrix" } check_resource() { local kind=$1 local name=$2 local ns=$3 if ! kubectl -n "$ns" get "$kind" "$name" >/dev/null 2>&1; then echo "[FAIL] $kind/$name is missing (ns=$ns)" missing=$((missing + 1)) else echo "[OK] $kind/$name exists (ns=$ns)" fi } check_resource svc argocd-server "$ARGOCD_NS" check_resource svc opentofu "$NS" check_resource svc garage "$NS" if [[ "$ENABLE_KERBEROS" == "1" ]]; then check_resource svc auth "$NS" fi check_resource deploy argocd-server "$ARGOCD_NS" check_resource deploy argocd-repo-server "$ARGOCD_NS" check_resource deploy argocd-dex-server "$ARGOCD_NS" check_resource deploy argocd-applicationset-controller "$ARGOCD_NS" check_resource deploy argocd-notifications-controller "$ARGOCD_NS" check_resource deploy argocd-redis "$ARGOCD_NS" check_resource deploy opentofu "$NS" if [[ "$ENABLE_KERBEROS" == "1" ]]; then check_resource deploy auth "$NS" fi check_resource statefulset argocd-application-controller "$ARGOCD_NS" check_resource statefulset garage "$NS" pod_filter="opentofu|garage" if [[ "$ENABLE_KERBEROS" == "1" ]]; then pod_filter="opentofu|garage|auth" fi analyze_pods "$ARGOCD_NS" "argocd" analyze_pods "$NS" "$pod_filter" if [[ -n "$blocked_lines" ]]; then echo "" echo "Blocked pods detected (not Ready or not Running):" printf '%b' "$blocked_lines" blocked=$((blocked + 1)) fi if [ $missing -gt 0 ]; then echo "" echo "Status: FAILED ($missing resources missing)" exit 1 fi if [ $blocked -gt 0 ]; then repairable=() fail_all=() for comp in argocd openbao opentofu garage auth; do if [[ "$comp" == "auth" && "$ENABLE_KERBEROS" != "1" ]]; then continue fi hc=${HEALTHY_COUNT["$comp"]:-0} bc=${BLOCKED_COUNT["$comp"]:-0} if [[ "$bc" -gt 0 && "$hc" -gt 0 ]]; then repairable+=("$comp") elif [[ "$bc" -gt 0 && "$hc" -eq 0 ]]; then fail_all+=("$comp") fi done if [ ${#fail_all[@]} -gt 0 ]; then echo "" echo "Repair skipped: all pods blocked for ${fail_all[*]}." echo "Action: Deploy Missing Services." echo "" echo "Status: NOTICE (blocked pods detected)" exit 2 fi if [ ${#repairable[@]} -gt 0 ]; then echo "" echo "== Repair ==" echo "Repairing blocked pods for: ${repairable[*]}" for comp in "${repairable[@]}"; do pods="${BLOCKED_PODS["$comp"]:-}" ns="${COMP_NAMESPACE["$comp"]:-$NS}" if [[ -n "$pods" ]]; then kubectl -n "$ns" delete pod $pods --wait=false >/dev/null 2>&1 || true fi done echo "Re-checking pods after repair..." sleep 2 HEALTHY_COUNT=() BLOCKED_COUNT=() BLOCKED_PODS=() COMP_NAMESPACE=() blocked_lines="" analyze_pods "$ARGOCD_NS" "argocd" analyze_pods "$NS" "$pod_filter" if [[ -n "$blocked_lines" ]]; then echo "" echo "Blocked pods still detected after repair:" printf '%b' "$blocked_lines" echo "" echo "Status: NOTICE (blocked pods detected)" exit 2 fi fi fi echo "" echo "Status: OK" exit 0