mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
chore: add migration diagnostics and improve failure handling
- Introduced `get_gitlab_migrations_diagnostics` for detailed migrations pod diagnostics. - Enhanced active migration failure detection with detailed logs and diagnostics. - Improved job context logging with pod restart, status, and waiting reason details. - Refined stale job repair logic by incorporating diagnostics and failure insights.
This commit is contained in:
chrisfu
parent
e2827c97e6
commit
ff5b73a435
@ -153,6 +153,60 @@ get_gitlab_migrations_diagnostics() {
|
||||
echo "$_diag"
|
||||
}
|
||||
|
||||
check_gitlab_migrations_blocked() {
|
||||
local _latest_job
|
||||
_latest_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
|
||||
[[ -n "$_latest_job" ]] || return 0
|
||||
|
||||
local _job_failed
|
||||
_job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
|
||||
|
||||
# 1. Active Failure Detection
|
||||
local _latest_pod
|
||||
_latest_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
|
||||
if [[ -n "$_latest_pod" ]]; then
|
||||
local _restarts
|
||||
_restarts=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
|
||||
local _waiting_reason
|
||||
_waiting_reason=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
|
||||
local _exit_code
|
||||
_exit_code=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.terminated.exitCode}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_exit_code" && "$_exit_code" != "0" ) ]]; then
|
||||
log "GitLab migrations job is actively failing: ${_latest_job} (pod: ${_latest_pod}, restarts: ${_restarts}, exitCode: ${_exit_code:-unknown})"
|
||||
local _diag
|
||||
_diag=$(get_gitlab_migrations_diagnostics "$_latest_job")
|
||||
repair_blocked "GitLab migrations job is actively failing" \
|
||||
"Job: ${_latest_job}. Pod: ${_latest_pod}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_exit_code:-unknown}. Restarts: ${_restarts}.
|
||||
Diagnostics:
|
||||
${_diag}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Stale Failed Job Repair
|
||||
local _job_creation_ts
|
||||
_job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
|
||||
local _job_pods_running
|
||||
_job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
|
||||
local _job_pods_pending
|
||||
_job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
|
||||
|
||||
local _now_ts
|
||||
_now_ts=$(date +%s)
|
||||
local _creation_ts
|
||||
_creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0")
|
||||
local _age
|
||||
_age=$(( _now_ts - _creation_ts ))
|
||||
|
||||
if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then
|
||||
log "GitLab operator is stalled on stale failed migrations job: ${_latest_job} (age: ${_age}s, no active pods)"
|
||||
log "Deleting stale failed migrations job to trigger repair..."
|
||||
kubectl -n "$NAMESPACE" delete job "$_latest_job" --wait=true 2>/dev/null || true
|
||||
kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_job" --force --grace-period=0 2>/dev/null || true
|
||||
log "Stale migrations job deleted. Operator should recreate it shortly."
|
||||
fi
|
||||
}
|
||||
|
||||
check_host_resolves() {
|
||||
local target_host="$1"
|
||||
# Strip port if present
|
||||
@ -236,9 +290,8 @@ get_gitlab_blocker_context() {
|
||||
local _m_last_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
|
||||
if [[ -n "$_m_last_pod" ]]; then
|
||||
local _m_pod_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
local _m_mig_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")]}' 2>/dev/null || true)
|
||||
local _m_restarts=$(echo "$_m_mig_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('restartCount', 0))" 2>/dev/null || echo "0")
|
||||
local _m_waiting=$(echo "$_m_mig_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('state', {}).get('waiting', {}).get('reason', ''))" 2>/dev/null || echo "")
|
||||
local _m_restarts=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
|
||||
local _m_waiting=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
|
||||
_m_pod_ctx="Pod: ${_m_last_pod} (${_m_pod_status}, restarts: ${_m_restarts}, waiting: ${_m_waiting:-None}). "
|
||||
fi
|
||||
ctx+="Migration Job: ${_last_mig} (Created: ${_m_ts}, Failed: ${_m_failed:-False}, Pods: ${_m_pods}). ${_m_pod_ctx}"
|
||||
@ -268,47 +321,7 @@ get_gitlab_blocker_context() {
|
||||
|
||||
check_gitlab_post_apply_blocked() {
|
||||
# --- Migrations Check ---
|
||||
local _latest_migration_job
|
||||
_latest_migration_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
|
||||
if [[ -n "$_latest_migration_job" ]]; then
|
||||
local _job_failed _job_creation_ts _job_pods_running _job_pods_pending _now_ts _creation_ts _age
|
||||
_job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_migration_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
|
||||
_job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_migration_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
|
||||
_job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_migration_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
|
||||
_job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_migration_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
|
||||
|
||||
# A. Active Failure Detection
|
||||
local _latest_pod_name=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_migration_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
|
||||
if [[ -n "$_latest_pod_name" ]]; then
|
||||
local _mig_container_status=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod_name" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")]}' 2>/dev/null || true)
|
||||
local _restarts=$(echo "$_mig_container_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('restartCount', 0))" 2>/dev/null || echo "0")
|
||||
local _waiting_reason=$(echo "$_mig_container_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('state', {}).get('waiting', {}).get('reason', ''))" 2>/dev/null || echo "")
|
||||
local _terminated_exit_code=$(echo "$_mig_container_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('state', {}).get('terminated', {}).get('exitCode', ''))" 2>/dev/null || echo "")
|
||||
|
||||
if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_terminated_exit_code" && "$_terminated_exit_code" != "0" ) ]]; then
|
||||
log "GitLab migrations job is actively failing: ${_latest_migration_job} (pod: ${_latest_pod_name}, restarts: ${_restarts}, exitCode: ${_terminated_exit_code:-unknown})"
|
||||
local _diag=$(get_gitlab_migrations_diagnostics "$_latest_migration_job")
|
||||
repair_blocked "GitLab migrations job is actively failing" \
|
||||
"Job: ${_latest_migration_job}. Pod: ${_latest_pod_name}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_terminated_exit_code:-unknown}. Restarts: ${_restarts}.
|
||||
Diagnostics:
|
||||
${_diag}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# B. Stale Failed Job Repair (Repairs stalled operator)
|
||||
_now_ts=$(date +%s)
|
||||
# Parse ISO8601 timestamp using python for cross-platform reliability
|
||||
_creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0")
|
||||
_age=$(( _now_ts - _creation_ts ))
|
||||
|
||||
if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then
|
||||
log "GitLab operator is stalled on stale failed migrations job: ${_latest_migration_job} (age: ${_age}s, no active pods)"
|
||||
log "Deleting stale failed migrations job to trigger repair..."
|
||||
kubectl -n "$NAMESPACE" delete job "$_latest_migration_job" --wait=true 2>/dev/null || true
|
||||
kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_migration_job" --force --grace-period=0 2>/dev/null || true
|
||||
log "Stale migrations job deleted. Operator should recreate it shortly."
|
||||
fi
|
||||
fi
|
||||
check_gitlab_migrations_blocked
|
||||
|
||||
# --- Gitaly Check ---
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
@ -459,13 +472,9 @@ ${_diag}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
local _m_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
|
||||
local _m_failed=$(kubectl -n "$NAMESPACE" get job "$_m_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
|
||||
if [[ "$_m_failed" == "True" ]]; then
|
||||
log "PVC ${pvc_name} not found; operator stalled on failed migrations job: ${_m_job}"
|
||||
else
|
||||
log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..."
|
||||
fi
|
||||
# Check migrations first to fail fast
|
||||
check_gitlab_migrations_blocked
|
||||
log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..."
|
||||
fi
|
||||
|
||||
if (( $(date +%s) - pvc_start > pvc_timeout )); then
|
||||
|
||||
Loading…
Reference in New Issue
Block a user