mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 10:13:58 +00:00
055069c1f8
k3d-prole-registry was squatting on port 5000, leaving k3d-knoe-registry in created state. Two bugs made this invisible: - k3d_registry.py: substring match (name in stdout) matched k3d-knoe-registry as already-exists without checking STATUS=running; add _k3d_registry_running() requiring last column == running, detect and delete prole-registry before creating knoe-registry, recreate if found in non-running state - status_common_services.sh: grep -qx 'knoe-registry' (exact) never matched k3d prefix 'k3d-knoe-registry'; fix with awk suffix match + STATUS==running Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
637 lines
19 KiB
Bash
Executable File
637 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -u
|
|
|
|
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
# Shared option parsing for common core scripts
|
|
# shellcheck disable=SC1090
|
|
source "$SCRIPT_DIR/common_core_lib.sh"
|
|
|
|
# Inject default config if not provided
|
|
_has_config=0
|
|
for _arg in "$@"; do
|
|
[[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
|
|
done
|
|
if [[ $_has_config -eq 0 ]]; then
|
|
_default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)"
|
|
if [[ -n "$_default_cfg" ]]; then
|
|
set -- "-c" "$_default_cfg" "$@"
|
|
fi
|
|
fi
|
|
unset _has_config _arg _default_cfg
|
|
|
|
common_core_preparse_config "$@"
|
|
|
|
# shellcheck disable=SC1090
|
|
source "$SCRIPT_DIR/knoe_cfg.sh"
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: status_common_services.sh [-n|--namespace NS] [-k|--kerberos]
|
|
|
|
Checks common infrastructure services (Registry, OpenTofu, Garage, OpenBao, Kong, Cert-Manager).
|
|
Use -k to include the Knoe KDC (auth) checks.
|
|
EOF
|
|
}
|
|
|
|
NS=""
|
|
ENABLE_KERBEROS=0
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-c|--config)
|
|
shift
|
|
shift
|
|
;;
|
|
-m|--mode)
|
|
shift
|
|
knoe_set_mode "${1:-}"
|
|
shift
|
|
;;
|
|
-m=*|--mode=*)
|
|
knoe_set_mode "${1#*=}"
|
|
shift
|
|
;;
|
|
-n|--namespace)
|
|
shift
|
|
NS="${1:-}"
|
|
shift
|
|
;;
|
|
-n=*|--namespace=*)
|
|
NS="${1#*=}"
|
|
shift
|
|
;;
|
|
-k|--kerberos)
|
|
ENABLE_KERBEROS=1
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
usage
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ -z "$NS" ]; then
|
|
NS="${SERVICE_NAMESPACE:-${NAMESPACE:-}}"
|
|
fi
|
|
if [ -z "$NS" ]; then
|
|
NS="default"
|
|
fi
|
|
|
|
REGISTRY_NS="${REGISTRY_NAMESPACE:-${NS}}"
|
|
KONG_NS="${KONG_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-}}}"
|
|
CERTMGR_NS="${CERTMGR_NAMESPACE:-cert-manager}"
|
|
KONG_NAME="${KONG_NAME:-knoe-svc-kong}"
|
|
# In k8s (GKE/prod) mode Kong is deployed as knoe-svc-kong
|
|
if [ "${KNOE_MODE:-}" = "k8s" ] && [ "$KONG_NAME" = "knoe-svc-kong" ]; then
|
|
KONG_NAME="knoe-svc-kong"
|
|
fi
|
|
|
|
if [ -z "$KONG_NS" ]; then
|
|
KONG_NS="default"
|
|
fi
|
|
if [ -z "$CERTMGR_NS" ]; then
|
|
CERTMGR_NS="cert-manager"
|
|
fi
|
|
|
|
if ! command -v kubectl >/dev/null 2>&1; then
|
|
echo "ERROR: kubectl not found in PATH"
|
|
exit 1
|
|
fi
|
|
|
|
current_mode() {
|
|
if declare -F knoe_normalize_mode >/dev/null 2>&1; then
|
|
knoe_normalize_mode "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}"
|
|
return 0
|
|
fi
|
|
printf '%s' "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}"
|
|
}
|
|
|
|
MODE="$(current_mode)"
|
|
REGISTRY_CHECK=0
|
|
if [[ "$MODE" == "k3s" || "$MODE" == "k3d" ]]; then
|
|
REGISTRY_CHECK=1
|
|
fi
|
|
OPENTOFU_CHECK=1
|
|
if [[ "$MODE" == "k8s" ]]; then
|
|
OPENTOFU_CHECK=0
|
|
fi
|
|
|
|
timestamp=$(date "+%Y-%m-%d %H:%M:%S")
|
|
ctx=$(kubectl config current-context 2>/dev/null || true)
|
|
server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true)
|
|
|
|
echo "Common service status"
|
|
echo "Time: $timestamp"
|
|
echo "Context: ${ctx:-<unknown>}"
|
|
echo "Server: ${server:-<unknown>}"
|
|
echo "Mode: ${MODE:-<unknown>}"
|
|
if [[ -n "${KUBECONFIG:-}" ]]; then
|
|
echo "Kubeconfig: $KUBECONFIG"
|
|
fi
|
|
echo "Namespace: $NS"
|
|
echo "Registry Namespace: $REGISTRY_NS"
|
|
echo "Kong Namespace: $KONG_NS"
|
|
echo "Cert-Manager Namespace: $CERTMGR_NS"
|
|
echo ""
|
|
|
|
run_cmd() {
|
|
echo "\$ $*"
|
|
"$@" 2>&1 || true
|
|
echo ""
|
|
}
|
|
|
|
echo "== Services =="
|
|
if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
|
|
run_cmd kubectl -n "$REGISTRY_NS" get svc registry
|
|
elif [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" == "k3d" ]]; then
|
|
run_cmd k3d registry list
|
|
fi
|
|
if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
|
|
run_cmd kubectl -n "$NS" get svc opentofu garage openbao
|
|
else
|
|
run_cmd kubectl -n "$NS" get svc garage openbao
|
|
echo "[INFO] k8s mode: skipping OpenTofu service check."
|
|
echo ""
|
|
fi
|
|
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
|
|
run_cmd kubectl -n "$NS" get svc auth
|
|
fi
|
|
run_cmd kubectl -n "$KONG_NS" get svc "$KONG_NAME"
|
|
run_cmd kubectl -n "$CERTMGR_NS" get svc cert-manager cert-manager-webhook
|
|
|
|
echo "== Workloads =="
|
|
if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
|
|
run_cmd kubectl -n "$REGISTRY_NS" get deploy registry
|
|
fi
|
|
if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
|
|
run_cmd kubectl -n "$NS" get deploy opentofu
|
|
else
|
|
echo "[INFO] k8s mode: skipping OpenTofu workload check."
|
|
echo ""
|
|
fi
|
|
if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then
|
|
run_cmd kubectl -n "$NS" get statefulset openbao
|
|
else
|
|
run_cmd kubectl -n "$NS" get deploy openbao
|
|
fi
|
|
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
|
|
run_cmd kubectl -n "$NS" get deploy auth
|
|
fi
|
|
run_cmd kubectl -n "$NS" get statefulset garage
|
|
run_cmd kubectl -n "$KONG_NS" get deploy "$KONG_NAME"
|
|
run_cmd kubectl -n "$CERTMGR_NS" get deploy cert-manager cert-manager-cainjector cert-manager-webhook
|
|
|
|
echo "== Pods =="
|
|
if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
|
|
run_cmd kubectl -n "$REGISTRY_NS" get pods | grep -Ei "registry" || true
|
|
fi
|
|
if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
|
|
run_cmd kubectl -n "$NS" get pods | grep -Ei "opentofu|garage|openbao" || true
|
|
else
|
|
run_cmd kubectl -n "$NS" get pods | grep -Ei "garage|openbao" || true
|
|
fi
|
|
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
|
|
run_cmd kubectl -n "$NS" get pods | grep -Ei "auth" || true
|
|
fi
|
|
run_cmd kubectl -n "$KONG_NS" get pods | grep -Ei "kong" || true
|
|
run_cmd kubectl -n "$CERTMGR_NS" get pods | grep -Ei "cert-manager" || true
|
|
|
|
echo "== Health Check =="
|
|
missing=0
|
|
blocked=0
|
|
declare -gA HEALTHY_COUNT
|
|
declare -gA BLOCKED_COUNT
|
|
declare -gA BLOCKED_PODS
|
|
declare -gA COMP_NAMESPACE
|
|
HEALTHY_COUNT=()
|
|
BLOCKED_COUNT=()
|
|
BLOCKED_PODS=()
|
|
COMP_NAMESPACE=()
|
|
blocked_lines=""
|
|
|
|
analyze_pods() {
|
|
local ns="$1"
|
|
local pod_filter="$2"
|
|
pod_rows=$(kubectl -n "$ns" get pods --no-headers 2>/dev/null | grep -Ei "$pod_filter" || true)
|
|
if [[ -z "$pod_rows" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
pod_matrix=$(printf '%s\n' "$pod_rows" | awk -v re="^(${pod_filter})-" '
|
|
{
|
|
name=$1; ready=$2; status=$3;
|
|
comp="";
|
|
if (match(name, re)) {
|
|
comp=substr(name, RSTART, RLENGTH-1);
|
|
}
|
|
split(ready, a, "/");
|
|
ready_ok=(a[1]==a[2] && a[1] ~ /^[0-9]+$/ && a[2] ~ /^[0-9]+$/);
|
|
healthy=0;
|
|
if (status=="Running") {
|
|
healthy = ready_ok ? 1 : 0;
|
|
} else if (status=="Completed" || status=="Succeeded") {
|
|
healthy = 1;
|
|
}
|
|
if (healthy==0) {
|
|
print comp "|" name "|" ready "|" status "|" healthy;
|
|
} else {
|
|
print comp "|" name "|" ready "|" status "|" healthy;
|
|
}
|
|
}
|
|
')
|
|
|
|
while IFS='|' read -r comp name ready status healthy; do
|
|
[[ -z "$comp" ]] && continue
|
|
if [[ "$comp" == cert-manager* ]]; then
|
|
comp="certmgr"
|
|
fi
|
|
COMP_NAMESPACE["$comp"]="$ns"
|
|
if [[ "$healthy" == "1" ]]; then
|
|
HEALTHY_COUNT["$comp"]=$(( ${HEALTHY_COUNT["$comp"]:-0} + 1 ))
|
|
else
|
|
BLOCKED_COUNT["$comp"]=$(( ${BLOCKED_COUNT["$comp"]:-0} + 1 ))
|
|
BLOCKED_PODS["$comp"]="${BLOCKED_PODS["$comp"]:-} $name"
|
|
blocked_lines+="$ns/$name $ready $status\n"
|
|
fi
|
|
done <<< "$pod_matrix"
|
|
}
|
|
|
|
check_resource() {
|
|
local kind=$1
|
|
local name=$2
|
|
local ns=$3
|
|
if ! kubectl -n "$ns" get "$kind" "$name" >/dev/null 2>&1; then
|
|
echo "[FAIL] $kind/$name is missing (ns=$ns)"
|
|
missing=$((missing + 1))
|
|
else
|
|
echo "[OK] $kind/$name exists (ns=$ns)"
|
|
fi
|
|
}
|
|
|
|
check_deploy_image() {
|
|
local name="$1"
|
|
local ns="$2"
|
|
local expected="$3"
|
|
local actual
|
|
actual=$(kubectl -n "$ns" get deploy "$name" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true)
|
|
if [[ -z "${actual:-}" ]]; then
|
|
echo "[FAIL] image/$name is missing (ns=$ns)"
|
|
missing=$((missing + 1))
|
|
return 0
|
|
fi
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
echo "[FAIL] image/$name expected '$expected' got '$actual' (ns=$ns)"
|
|
missing=$((missing + 1))
|
|
return 0
|
|
fi
|
|
echo "[OK] image/$name is '$expected' (ns=$ns)"
|
|
}
|
|
|
|
check_k3d_registry() {
|
|
if command -v k3d >/dev/null 2>&1; then
|
|
# k3d prefixes registry container names with "k3d-"; match by suffix and
|
|
# require STATUS == running (not just "created").
|
|
if k3d registry list --no-headers 2>/dev/null | awk '$1 ~ /knoe-registry$/ && $NF == "running"' | grep -q .; then
|
|
echo "[OK] registry/registry exists (k3d: knoe-registry)"
|
|
return 0
|
|
fi
|
|
fi
|
|
if command -v docker >/dev/null 2>&1; then
|
|
if docker ps --format '{{.Names}}' 2>/dev/null | grep -Eq '^(k3d-knoe-registry|knoe-registry)$'; then
|
|
echo "[OK] registry/registry exists (docker)"
|
|
return 0
|
|
fi
|
|
fi
|
|
echo "[FAIL] registry/registry is missing (k3d/docker)"
|
|
missing=$((missing + 1))
|
|
}
|
|
|
|
if [[ "$REGISTRY_CHECK" -eq 1 ]]; then
|
|
if [[ "$MODE" == "k3d" ]]; then
|
|
check_k3d_registry
|
|
else
|
|
check_resource svc registry "$REGISTRY_NS"
|
|
check_resource deploy registry "$REGISTRY_NS"
|
|
check_deploy_image registry "$REGISTRY_NS" 'registry:2'
|
|
fi
|
|
fi
|
|
|
|
if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
|
|
check_resource svc opentofu "$NS"
|
|
fi
|
|
check_resource svc garage "$NS"
|
|
check_resource svc openbao "$NS"
|
|
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
|
|
check_resource svc auth "$NS"
|
|
fi
|
|
check_resource svc "$KONG_NAME" "$KONG_NS"
|
|
check_resource svc cert-manager "$CERTMGR_NS"
|
|
check_resource svc cert-manager-webhook "$CERTMGR_NS"
|
|
|
|
if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
|
|
check_resource deploy opentofu "$NS"
|
|
fi
|
|
if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then
|
|
check_resource statefulset openbao "$NS"
|
|
else
|
|
check_resource deploy openbao "$NS"
|
|
fi
|
|
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
|
|
check_resource deploy auth "$NS"
|
|
fi
|
|
check_resource statefulset garage "$NS"
|
|
check_resource deploy "$KONG_NAME" "$KONG_NS"
|
|
check_resource deploy cert-manager "$CERTMGR_NS"
|
|
check_resource deploy cert-manager-cainjector "$CERTMGR_NS"
|
|
check_resource deploy cert-manager-webhook "$CERTMGR_NS"
|
|
|
|
pod_filter="garage|openbao|kong|cert-manager"
|
|
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
|
|
pod_filter="garage|openbao|auth|kong|cert-manager"
|
|
fi
|
|
if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
|
|
pod_filter="opentofu|${pod_filter}"
|
|
fi
|
|
analyze_pods "$NS" "$pod_filter"
|
|
analyze_pods "$KONG_NS" "kong"
|
|
analyze_pods "$CERTMGR_NS" "cert-manager"
|
|
if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
|
|
analyze_pods "$REGISTRY_NS" "registry"
|
|
fi
|
|
|
|
if [[ -n "$blocked_lines" ]]; then
|
|
echo ""
|
|
echo "Blocked pods detected (not Ready or not Running):"
|
|
printf '%b' "$blocked_lines"
|
|
blocked=$((blocked + 1))
|
|
fi
|
|
|
|
if [ $missing -gt 0 ]; then
|
|
echo ""
|
|
# ── Attempt to repair a completely absent cert-manager before failing ──────
|
|
if ! kubectl get namespace "$CERTMGR_NS" >/dev/null 2>&1 \
|
|
&& [[ -x "$SCRIPT_DIR/init_certmgr.sh" ]]; then
|
|
echo "== Repair: cert-manager namespace '$CERTMGR_NS' absent — re-initializing =="
|
|
"$SCRIPT_DIR/init_certmgr.sh" initialize 2>&1 || true
|
|
echo " Waiting for cert-manager to settle (20s)..."
|
|
sleep 20
|
|
echo " Re-checking cert-manager resources..."
|
|
_check_cm_resource() {
|
|
local _kind=$1 _name=$2 _ns=$3
|
|
if ! kubectl -n "$_ns" get "$_kind" "$_name" >/dev/null 2>&1; then
|
|
echo " [FAIL] $_kind/$_name still missing (ns=$_ns)"
|
|
else
|
|
missing=$((missing - 1))
|
|
echo " [OK] $_kind/$_name present (ns=$_ns)"
|
|
fi
|
|
}
|
|
_check_cm_resource svc cert-manager "$CERTMGR_NS"
|
|
_check_cm_resource svc cert-manager-webhook "$CERTMGR_NS"
|
|
_check_cm_resource deploy cert-manager "$CERTMGR_NS"
|
|
_check_cm_resource deploy cert-manager-cainjector "$CERTMGR_NS"
|
|
_check_cm_resource deploy cert-manager-webhook "$CERTMGR_NS"
|
|
fi
|
|
|
|
if [ $missing -gt 0 ]; then
|
|
echo "Status: FAILED ($missing resources missing)"
|
|
exit 1
|
|
fi
|
|
echo "Status: OK (missing resources restored by repair)"
|
|
fi
|
|
|
|
# ── Repair helpers ────────────────────────────────────────────────────────────
|
|
|
|
# Emit "kind name ns" lines for every workload owned by a component.
|
|
_workloads_for_comp() {
|
|
local _comp="$1"
|
|
local _ns="${COMP_NAMESPACE[$_comp]:-$NS}"
|
|
case "$_comp" in
|
|
registry)
|
|
if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
|
|
echo "deployment registry $REGISTRY_NS"
|
|
fi
|
|
;;
|
|
openbao)
|
|
if kubectl -n "$_ns" get statefulset openbao >/dev/null 2>&1; then
|
|
echo "statefulset openbao $_ns"
|
|
else
|
|
echo "deployment openbao $_ns"
|
|
fi
|
|
;;
|
|
opentofu)
|
|
if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
|
|
echo "deployment opentofu $_ns"
|
|
fi
|
|
;;
|
|
garage) echo "statefulset garage $_ns" ;;
|
|
auth) echo "deployment auth $_ns" ;;
|
|
kong) echo "deployment $KONG_NAME $KONG_NS" ;;
|
|
certmgr)
|
|
for _d in cert-manager cert-manager-cainjector cert-manager-webhook; do
|
|
echo "deployment $_d $CERTMGR_NS"
|
|
done
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Reset all pod-tracking state and re-run analysis.
|
|
_recheck_pods() {
|
|
HEALTHY_COUNT=()
|
|
BLOCKED_COUNT=()
|
|
BLOCKED_PODS=()
|
|
COMP_NAMESPACE=()
|
|
blocked_lines=""
|
|
blocked=0
|
|
analyze_pods "$NS" "$pod_filter"
|
|
analyze_pods "$KONG_NS" "kong"
|
|
analyze_pods "$CERTMGR_NS" "cert-manager"
|
|
if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
|
|
analyze_pods "$REGISTRY_NS" "registry"
|
|
fi
|
|
if [[ -n "$blocked_lines" ]]; then
|
|
blocked=1
|
|
fi
|
|
}
|
|
|
|
# Print the name of each component that currently has blocked pods.
|
|
_blocked_comps() {
|
|
for _bc in registry openbao garage opentofu auth kong certmgr; do
|
|
[[ "$_bc" == "auth" && "$ENABLE_KERBEROS" != "1" ]] && continue
|
|
[[ "$_bc" == "registry" && ( "$REGISTRY_CHECK" -ne 1 || "$MODE" == "k3d" ) ]] && continue
|
|
[[ "$_bc" == "opentofu" && "$OPENTOFU_CHECK" -ne 1 ]] && continue
|
|
[[ ${BLOCKED_COUNT["$_bc"]:-0} -gt 0 ]] && echo "$_bc"
|
|
done
|
|
}
|
|
|
|
# ── Escalating repair ─────────────────────────────────────────────────────────
|
|
|
|
if [ $blocked -gt 0 ]; then
|
|
|
|
# ── Round 1: delete blocked pods ──────────────────────────────────────────
|
|
echo ""
|
|
echo "== Repair Round 1: replacing blocked pods =="
|
|
mapfile -t _r1_comps < <(_blocked_comps)
|
|
for _comp in "${_r1_comps[@]}"; do
|
|
_pods="${BLOCKED_PODS[$_comp]:-}"
|
|
_ns="${COMP_NAMESPACE[$_comp]:-$NS}"
|
|
if [[ -n "$_pods" ]]; then
|
|
echo " Deleting pod(s) for $_comp in $_ns:$_pods"
|
|
# shellcheck disable=SC2086
|
|
kubectl -n "$_ns" delete pod $_pods --wait=false >/dev/null 2>&1 || true
|
|
fi
|
|
done
|
|
echo " Waiting for pod replacement (15s)..."
|
|
sleep 15
|
|
_recheck_pods
|
|
|
|
if [[ $blocked -eq 0 ]]; then
|
|
echo ""
|
|
echo "Status: OK (resolved in repair round 1)"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "Pods still blocked after Round 1:"
|
|
printf '%b' "$blocked_lines"
|
|
|
|
# ── Round 2: rollout restart owning StatefulSet / Deployment ──────────────
|
|
echo ""
|
|
echo "== Repair Round 2: restarting workload controllers =="
|
|
mapfile -t _r2_comps < <(_blocked_comps)
|
|
for _comp in "${_r2_comps[@]}"; do
|
|
while IFS=' ' read -r _kind _name _wns; do
|
|
[[ -z "$_kind" ]] && continue
|
|
if kubectl -n "$_wns" get "$_kind" "$_name" >/dev/null 2>&1; then
|
|
echo " kubectl rollout restart $_kind/$_name -n $_wns"
|
|
kubectl -n "$_wns" rollout restart "$_kind/$_name" >/dev/null 2>&1 || true
|
|
kubectl -n "$_wns" rollout status "$_kind/$_name" --timeout=60s 2>/dev/null || true
|
|
fi
|
|
done < <(_workloads_for_comp "$_comp")
|
|
done
|
|
echo " Waiting for rollout to settle (20s)..."
|
|
sleep 20
|
|
_recheck_pods
|
|
|
|
if [[ $blocked -eq 0 ]]; then
|
|
echo ""
|
|
echo "Status: OK (resolved in repair round 2)"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "Pods still blocked after Round 2:"
|
|
printf '%b' "$blocked_lines"
|
|
|
|
# ── Round 3: clear affected namespaces and re-initialize ──────────────────
|
|
echo ""
|
|
echo "== Repair Round 3: clearing affected namespaces and re-initializing =="
|
|
mapfile -t _r3_comps < <(_blocked_comps)
|
|
declare -A _cleared_ns=()
|
|
for _comp in "${_r3_comps[@]}"; do
|
|
_ns="${COMP_NAMESPACE[$_comp]:-$NS}"
|
|
[[ -n "${_cleared_ns[$_ns]:-}" ]] && continue
|
|
_cleared_ns["$_ns"]=1
|
|
if [[ "$_ns" == "default" || "$_ns" == "kube-system" ]]; then
|
|
echo " [Round 3] Removing workload resources in protected namespace '$_ns' for: $_comp"
|
|
while IFS=' ' read -r _kind _name _wns; do
|
|
[[ -z "$_kind" ]] && continue
|
|
echo " kubectl delete $_kind $_name -n $_wns --ignore-not-found"
|
|
kubectl -n "$_wns" delete "$_kind" "$_name" --ignore-not-found >/dev/null 2>&1 || true
|
|
done < <(_workloads_for_comp "$_comp")
|
|
else
|
|
echo " [Round 3] Deleting namespace '$_ns' ..."
|
|
kubectl delete namespace "$_ns" --wait=true --timeout=90s >/dev/null 2>&1 || true
|
|
echo " [Round 3] Recreating namespace '$_ns' ..."
|
|
kubectl create namespace "$_ns" >/dev/null 2>&1 || true
|
|
echo " [OK] Namespace '$_ns' cleared and recreated."
|
|
fi
|
|
done
|
|
|
|
if [[ -x "$SCRIPT_DIR/init_service_layer.sh" ]]; then
|
|
echo " [Round 3] Re-initializing service layer ..."
|
|
_krb_flag=""
|
|
[[ "$ENABLE_KERBEROS" == "1" ]] && _krb_flag="-k"
|
|
# shellcheck disable=SC2086
|
|
"$SCRIPT_DIR/init_service_layer.sh" -n "$NS" $_krb_flag update 2>&1 || true
|
|
fi
|
|
|
|
echo " [Round 3] Re-checking status after namespace reset (30s)..."
|
|
sleep 30
|
|
_recheck_pods
|
|
|
|
if [[ $blocked -eq 0 ]]; then
|
|
echo ""
|
|
echo "Status: OK (resolved in repair round 3)"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "Pods still blocked after Round 3:"
|
|
printf '%b' "$blocked_lines"
|
|
|
|
# ── Round 4: reset k3d cluster, preserve registry ─────────────────────────
|
|
_knoe_mode="${KNOE_MODE:-}"
|
|
if [[ "$_knoe_mode" == "k3d" ]]; then
|
|
echo ""
|
|
echo "== Repair Round 4: resetting k3d cluster (preserving registry) =="
|
|
_cluster_name="${K3D_CLUSTER_NAME:-knoe-dev-cluster}"
|
|
|
|
_registry_args=()
|
|
if command -v k3d >/dev/null 2>&1; then
|
|
_reg_name=$(k3d registry list --no-headers 2>/dev/null | awk '{print $1}' | head -1 || true)
|
|
if [[ -n "$_reg_name" ]]; then
|
|
echo " Preserving registry: $_reg_name"
|
|
_registry_args=(--registry-use "$_reg_name")
|
|
fi
|
|
fi
|
|
|
|
echo " Deleting k3d cluster '$_cluster_name' ..."
|
|
k3d cluster delete "$_cluster_name" >/dev/null 2>&1 || true
|
|
|
|
echo " Recreating k3d cluster '$_cluster_name' ..."
|
|
k3d cluster create "$_cluster_name" -a 2 \
|
|
"${_registry_args[@]}" \
|
|
--api-port 0.0.0.0:6443 >/dev/null 2>&1 || true
|
|
|
|
kubectl config use-context "k3d-${_cluster_name}" >/dev/null 2>&1 || true
|
|
|
|
if [[ -x "$SCRIPT_DIR/init_service_layer.sh" ]]; then
|
|
echo " [Round 4] Re-initializing service layer after cluster reset ..."
|
|
_krb_flag=""
|
|
[[ "$ENABLE_KERBEROS" == "1" ]] && _krb_flag="-k"
|
|
# shellcheck disable=SC2086
|
|
"$SCRIPT_DIR/init_service_layer.sh" -n "$NS" $_krb_flag update 2>&1 || true
|
|
fi
|
|
|
|
echo " [Round 4] Re-checking status after cluster reset (30s)..."
|
|
sleep 30
|
|
_recheck_pods
|
|
|
|
if [[ $blocked -eq 0 ]]; then
|
|
echo ""
|
|
echo "Status: OK (resolved in repair round 4 — cluster reset)"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "Pods still blocked after Round 4 (cluster reset):"
|
|
printf '%b' "$blocked_lines"
|
|
echo ""
|
|
echo "Status: FAILED — unable to repair after full cluster reset. Manual intervention required."
|
|
exit 5
|
|
fi
|
|
|
|
echo ""
|
|
echo "Status: NOTICE (blocked pods remain; k3d cluster reset not applicable for mode '${_knoe_mode:-unknown}')"
|
|
exit 3
|
|
|
|
fi
|
|
|
|
echo ""
|
|
echo "Status: OK"
|
|
exit 0
|