diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh index 63f47e3..c919ae9 100755 --- a/etc/init_gitlab.sh +++ b/etc/init_gitlab.sh @@ -153,6 +153,60 @@ get_gitlab_migrations_diagnostics() { echo "$_diag" } +check_gitlab_migrations_blocked() { + local _latest_job + _latest_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + [[ -n "$_latest_job" ]] || return 0 + + local _job_failed + _job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) + + # 1. Active Failure Detection + local _latest_pod + _latest_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + if [[ -n "$_latest_pod" ]]; then + local _restarts + _restarts=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0") + local _waiting_reason + _waiting_reason=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "") + local _exit_code + _exit_code=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.terminated.exitCode}' 2>/dev/null || echo "") + + if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_exit_code" && "$_exit_code" != "0" ) ]]; then + log "GitLab migrations job is actively failing: ${_latest_job} (pod: ${_latest_pod}, restarts: ${_restarts}, exitCode: ${_exit_code:-unknown})" + local _diag + _diag=$(get_gitlab_migrations_diagnostics "$_latest_job") + repair_blocked "GitLab migrations job is actively failing" \ + "Job: ${_latest_job}. Pod: ${_latest_pod}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_exit_code:-unknown}. Restarts: ${_restarts}. +Diagnostics: +${_diag}" + fi + fi + + # 2. Stale Failed Job Repair + local _job_creation_ts + _job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true) + local _job_pods_running + _job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0") + local _job_pods_pending + _job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0") + + local _now_ts + _now_ts=$(date +%s) + local _creation_ts + _creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0") + local _age + _age=$(( _now_ts - _creation_ts )) + + if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then + log "GitLab operator is stalled on stale failed migrations job: ${_latest_job} (age: ${_age}s, no active pods)" + log "Deleting stale failed migrations job to trigger repair..." + kubectl -n "$NAMESPACE" delete job "$_latest_job" --wait=true 2>/dev/null || true + kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_job" --force --grace-period=0 2>/dev/null || true + log "Stale migrations job deleted. Operator should recreate it shortly." + fi +} + check_host_resolves() { local target_host="$1" # Strip port if present @@ -236,9 +290,8 @@ get_gitlab_blocker_context() { local _m_last_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) if [[ -n "$_m_last_pod" ]]; then local _m_pod_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.phase}' 2>/dev/null || true) - local _m_mig_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")]}' 2>/dev/null || true) - local _m_restarts=$(echo "$_m_mig_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('restartCount', 0))" 2>/dev/null || echo "0") - local _m_waiting=$(echo "$_m_mig_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('state', {}).get('waiting', {}).get('reason', ''))" 2>/dev/null || echo "") + local _m_restarts=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0") + local _m_waiting=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "") _m_pod_ctx="Pod: ${_m_last_pod} (${_m_pod_status}, restarts: ${_m_restarts}, waiting: ${_m_waiting:-None}). " fi ctx+="Migration Job: ${_last_mig} (Created: ${_m_ts}, Failed: ${_m_failed:-False}, Pods: ${_m_pods}). ${_m_pod_ctx}" @@ -268,47 +321,7 @@ get_gitlab_blocker_context() { check_gitlab_post_apply_blocked() { # --- Migrations Check --- - local _latest_migration_job - _latest_migration_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) - if [[ -n "$_latest_migration_job" ]]; then - local _job_failed _job_creation_ts _job_pods_running _job_pods_pending _now_ts _creation_ts _age - _job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_migration_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) - _job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_migration_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true) - _job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_migration_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0") - _job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_migration_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0") - - # A. Active Failure Detection - local _latest_pod_name=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_migration_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) - if [[ -n "$_latest_pod_name" ]]; then - local _mig_container_status=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod_name" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")]}' 2>/dev/null || true) - local _restarts=$(echo "$_mig_container_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('restartCount', 0))" 2>/dev/null || echo "0") - local _waiting_reason=$(echo "$_mig_container_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('state', {}).get('waiting', {}).get('reason', ''))" 2>/dev/null || echo "") - local _terminated_exit_code=$(echo "$_mig_container_status" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('state', {}).get('terminated', {}).get('exitCode', ''))" 2>/dev/null || echo "") - - if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_terminated_exit_code" && "$_terminated_exit_code" != "0" ) ]]; then - log "GitLab migrations job is actively failing: ${_latest_migration_job} (pod: ${_latest_pod_name}, restarts: ${_restarts}, exitCode: ${_terminated_exit_code:-unknown})" - local _diag=$(get_gitlab_migrations_diagnostics "$_latest_migration_job") - repair_blocked "GitLab migrations job is actively failing" \ - "Job: ${_latest_migration_job}. Pod: ${_latest_pod_name}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_terminated_exit_code:-unknown}. Restarts: ${_restarts}. -Diagnostics: -${_diag}" - fi - fi - - # B. Stale Failed Job Repair (Repairs stalled operator) - _now_ts=$(date +%s) - # Parse ISO8601 timestamp using python for cross-platform reliability - _creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0") - _age=$(( _now_ts - _creation_ts )) - - if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then - log "GitLab operator is stalled on stale failed migrations job: ${_latest_migration_job} (age: ${_age}s, no active pods)" - log "Deleting stale failed migrations job to trigger repair..." - kubectl -n "$NAMESPACE" delete job "$_latest_migration_job" --wait=true 2>/dev/null || true - kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_migration_job" --force --grace-period=0 2>/dev/null || true - log "Stale migrations job deleted. Operator should recreate it shortly." - fi - fi + check_gitlab_migrations_blocked # --- Gitaly Check --- if [[ "$MODE" == "k8s" ]]; then @@ -459,13 +472,9 @@ ${_diag}" fi fi else - local _m_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) - local _m_failed=$(kubectl -n "$NAMESPACE" get job "$_m_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) - if [[ "$_m_failed" == "True" ]]; then - log "PVC ${pvc_name} not found; operator stalled on failed migrations job: ${_m_job}" - else - log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..." - fi + # Check migrations first to fail fast + check_gitlab_migrations_blocked + log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..." fi if (( $(date +%s) - pvc_start > pvc_timeout )); then