prole/etc/diag_gitlab_boot.sh

#!/usr/bin/env bash
# diag_gitlab_boot.sh — read-only snapshot of GitLab backend health.
#
# Investigates why the frontdoor returns "Waiting for GitLab to boot" even
# after DNS + ingress are green. Focuses on:
#   - webservice pod states + last-terminated reasons (OOM, probe failures)
#   - webservice + workhorse log tail on the likely-serving pod
#   - Gitaly readiness
#   - recent Warning events
#
# Usage: ./etc/diag_gitlab_boot.sh [--namespace NS] [--kube-context CTX]
#   --namespace defaults to 'gitlab'

set -euo pipefail

NS="gitlab"
KUBE_CONTEXT=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    --namespace) NS="$2"; shift 2 ;;
    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
    *) echo "unknown arg: $1" >&2; exit 2 ;;
  esac
done

kctl() {
  if [[ -n "$KUBE_CONTEXT" ]]; then
    kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@"
  else
    kubectl -n "$NS" "$@"
  fi
}

TMPDIR="$(mktemp -d)"
trap 'rm -rf "$TMPDIR"' EXIT

WEB_JSON="$TMPDIR/web.json"
kctl get pods -l app=webservice -o json > "$WEB_JSON"

echo "=== Namespace: $NS ==="
echo
echo "=== Webservice pods (restarts + last termination) ==="

PY_SCRIPT="$TMPDIR/analyze.py"
cat > "$PY_SCRIPT" <<'PY'
import json, sys

with open(sys.argv[1]) as f:
    data = json.load(f)

pods = data.get("items") or []
if not pods:
    print("  (no webservice pods found)")

for pod in pods:
    name = (pod.get("metadata") or {}).get("name", "")
    phase = (pod.get("status") or {}).get("phase", "")
    print(f"\n-- {name}  phase={phase} --")
    for cs in (pod.get("status") or {}).get("containerStatuses") or []:
        cname = cs.get("name", "")
        ready = cs.get("ready")
        restart = cs.get("restartCount", 0)
        state = cs.get("state") or {}
        last = cs.get("lastState") or {}
        state_key = next(iter(state.keys()), "unknown")
        state_detail = state.get(state_key) or {}
        last_term = (last.get("terminated") or {}) if last else {}
        print(f"  container={cname:<22} ready={ready} restarts={restart} state={state_key}")
        if state_detail.get("reason"):
            print(f"    state.reason={state_detail.get('reason')} msg={(state_detail.get('message') or '')[:180]}")
        if last_term:
            print(f"    last terminated: exit={last_term.get('exitCode')} reason={last_term.get('reason')} "
                  f"signal={last_term.get('signal')}")
            print(f"    last start={last_term.get('startedAt')} finish={last_term.get('finishedAt')}")
            if last_term.get("message"):
                print(f"    last msg: {(last_term.get('message') or '')[:240]}")

# Emit the best-candidate pod name to a sidecar file so the shell can pick it up
def score(p):
    cs = (p.get("status") or {}).get("containerStatuses") or []
    ready = all(c.get("ready") for c in cs) if cs else False
    restarts = max((c.get("restartCount", 0) for c in cs), default=0)
    return (1 if ready else 0, -restarts)

pods.sort(key=score, reverse=True)
target = pods[0]["metadata"]["name"] if pods else ""
with open(sys.argv[2], "w") as f:
    f.write(target)
PY

TARGET_FILE="$TMPDIR/target.txt"
python3 "$PY_SCRIPT" "$WEB_JSON" "$TARGET_FILE"
POD="$(cat "$TARGET_FILE")"

echo
echo "=== Most-likely serving webservice pod: tail 60 lines (webservice container) ==="
if [[ -n "$POD" ]]; then
  echo "Pod: $POD"
  kctl logs "$POD" -c webservice --tail=60 2>&1 | sed 's/^/  /' || true
  echo
  echo "=== ... gitlab-workhorse container (tail 30) ==="
  kctl logs "$POD" -c gitlab-workhorse --tail=30 2>&1 | sed 's/^/  /' || true
else
  echo "  (no webservice pod found)"
fi

echo
echo "=== Gitaly pods ==="
kctl get pods -l app=gitaly -o wide || true

echo
echo "=== Recent Warning events (last 25) ==="
kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 25 || true

echo
echo "=== GitLab CR status (if present) ==="
GL_JSON="$TMPDIR/gl.json"
if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then
  python3 - "$GL_JSON" <<'PY'
import json, sys
with open(sys.argv[1]) as f:
    data = json.load(f)
for it in data.get("items") or []:
    m = it.get("metadata") or {}
    s = it.get("status") or {}
    print(f"  {m.get('name')}: phase={s.get('phase','?')}")
    for c in (s.get("conditions") or [])[:10]:
        print(f"    {c.get('type')}={c.get('status')} reason={c.get('reason')} "
              f"msg={(c.get('message') or '')[:140]}")
PY
else
  echo "  (no GitLab CR or CRD not installed)"
fi