prole/etc/diag_gitlab_webservice_oom.sh

#!/usr/bin/env bash
# diag_gitlab_webservice_oom.sh — memory ceilings + Puma tuning for webservice.
#
# Read-only. Prints:
#   1. webservice container resources (requests/limits)
#   2. Puma worker/thread env vars from pod spec
#   3. node allocatable memory so we know upper bound
#   4. prometheus-style /metrics snapshot from the running pod if available
#
# Usage: ./etc/diag_gitlab_webservice_oom.sh [--namespace NS] [--kube-context CTX]

set -euo pipefail

NS="gitlab"
KUBE_CONTEXT=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    --namespace) NS="$2"; shift 2 ;;
    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
    *) echo "unknown arg: $1" >&2; exit 2 ;;
  esac
done

kctl() {
  if [[ -n "$KUBE_CONTEXT" ]]; then
    kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@"
  else
    kubectl -n "$NS" "$@"
  fi
}

TMPDIR="$(mktemp -d)"
trap 'rm -rf "$TMPDIR"' EXIT

DEPLOY_JSON="$TMPDIR/deploy.json"
POD_JSON="$TMPDIR/pod.json"
NODE_JSON="$TMPDIR/node.json"

kctl get deployment -l app=webservice -o json > "$DEPLOY_JSON"
kctl get pods -l app=webservice -o json > "$POD_JSON"

echo "=== webservice container resources (from deployment) ==="
python3 - "$DEPLOY_JSON" <<'PY'
import json, sys
with open(sys.argv[1]) as f:
    data = json.load(f)
for d in data.get("items") or []:
    name = (d.get("metadata") or {}).get("name", "")
    print(f"\n-- deployment/{name} --")
    spec = ((d.get("spec") or {}).get("template") or {}).get("spec") or {}
    for c in spec.get("containers") or []:
        cname = c.get("name", "")
        res = c.get("resources") or {}
        req = res.get("requests") or {}
        lim = res.get("limits") or {}
        print(f"  container={cname}")
        print(f"    requests: cpu={req.get('cpu','?')}  mem={req.get('memory','?')}")
        print(f"    limits:   cpu={lim.get('cpu','?')}  mem={lim.get('memory','?')}")
        # Pick out Puma-relevant env vars
        env = c.get("env") or []
        interesting = {
            "PUMA_WORKERS", "PUMA_THREADS_MIN", "PUMA_THREADS_MAX",
            "WORKER_PROCESSES", "SIDEKIQ_CONCURRENCY",
            "GITLAB_MEMORY_WATCHDOG_ENABLED", "GITLAB_MEMORY_WATCHDOG_PUMA_ENABLED",
            "GITLAB_MEMORY_WATCHDOG_MAX_HEAP_FRAG_THRESHOLD_MB",
            "GITLAB_MEMORY_WATCHDOG_MAX_STRIKES",
            "MALLOC_ARENA_MAX",
        }
        seen = [e for e in env if e.get("name") in interesting]
        if seen:
            print("    puma/watchdog env:")
            for e in seen:
                val = e.get("value", "<valueFrom>") if "value" in e else "<valueFrom>"
                print(f"      {e.get('name')}={val}")
PY

echo
echo "=== Node memory allocatable ==="
kctl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory,MEM_CAPACITY:.status.capacity.memory 2>/dev/null \
  || kubectl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory

echo
echo "=== Running webservice pod memory usage (kubectl top if metrics-server present) ==="
kctl top pod -l app=webservice --containers 2>&1 || echo "  (metrics-server not installed or not returning data)"

echo
echo "=== Pod QoS class + restart history ==="
python3 - "$POD_JSON" <<'PY'
import json, sys
with open(sys.argv[1]) as f:
    data = json.load(f)
for p in data.get("items") or []:
    m = p.get("metadata") or {}
    s = p.get("status") or {}
    cs = s.get("containerStatuses") or []
    ws = next((c for c in cs if c.get("name") == "webservice"), None)
    print(f"  {m.get('name'):<48}  qos={s.get('qosClass','?'):<10} "
          f"phase={s.get('phase','?'):<10} restarts={ws.get('restartCount', '?') if ws else '?'}")
PY

echo
echo "=== Last 40 Warning events (whole namespace, for noise diagnosis) ==="
kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 40 || true

echo
echo "=== All PVCs in namespace (what the operator might be trying to reconcile) ==="
kctl get pvc -o wide || true

echo
echo "=== GitLab operator CR spec.gitaly (replicas + storage) ==="
GL_JSON="$TMPDIR/gl.json"
if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then
  python3 - "$GL_JSON" <<'PY'
import json, sys
with open(sys.argv[1]) as f:
    data = json.load(f)
for g in data.get("items") or []:
    m = g.get("metadata") or {}
    name = m.get("name", "")
    chart = ((g.get("spec") or {}).get("chart") or {}).get("values") or {}
    gitaly = ((chart.get("global") or {}).get("gitaly") or {})
    wsvc = (chart.get("gitlab") or {}).get("webservice") or {}
    print(f"  gitlab/{name}")
    print(f"    global.gitaly: {json.dumps(gitaly, indent=6)[:800]}")
    print(f"    gitlab.webservice.replicas: {wsvc.get('replicaCount', wsvc.get('minReplicas','?'))}")
    print(f"    gitlab.webservice.resources: {json.dumps(wsvc.get('resources', {}), indent=6)}")
    print(f"    gitlab.webservice.workerProcesses: {wsvc.get('workerProcesses','?')}")
PY
else
  echo "  (no GitLab CR or not readable)"
fi