chore: add split-cluster diagnostics and refine workload convergence checks

- Implemented `gitlab_split_cluster_ownership_diagnostics` to enforce and log split-cluster ownership policies between app and DB contexts.
- Updated workload convergence logic to prioritize readiness detection over historical restart counts for webservice pods.
- Enhanced deployment utilities with additional diagnostics for pod readiness states and replica discrepancies.
- Introduced tests to validate split-cluster enforcement and refined convergence behavior.

etc/init_gitlab.sh

@@ -81,6 +81,61 @@ enforce_app_cluster_targeting() {
   export KUBECONTEXT="$app_ctx"
 }
 
+gitlab_split_cluster_ownership_diagnostics() {
+  if [[ "$MODE" != "k8s" ]]; then
+    return 0
+  fi
+
+  local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
+  local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
+  local gitlab_instance="${GITLAB_RELEASE:-gitlab}"
+
+  if [[ -z "$db_ctx" || "$db_ctx" == "$app_ctx" ]]; then
+    log "GitLab ownership policy: single-cluster mode (APP=${app_ctx:-unknown}); operator and app workloads are expected in this context."
+    return 0
+  fi
+
+  log "GitLab split-cluster ownership policy: APP context '${app_ctx}' is authoritative for GitLab operator + app workloads in namespace '${NAMESPACE}'."
+  log "GitLab split-cluster ownership policy: DB context '${db_ctx}' must not host GitLab app workloads (gitaly/webservice/sidekiq/kas/registry/toolbox)."
+
+  local app_gitlab_objects db_gitlab_app_objects db_gitlab_operator_objects
+  app_gitlab_objects=$(kubectl -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
+    -l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
+  db_gitlab_app_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
+    -l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
+  db_gitlab_operator_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deployment -o name 2>/dev/null \
+    | grep -E 'deployment.apps/(gitlab-controller-manager|gitlab-operator|.*gitlab.*controller-manager)' || true)
+
+  if [[ -n "$app_gitlab_objects" ]]; then
+    log "GitLab APP-context resource snapshot (${app_ctx}):"
+    while IFS= read -r _obj; do
+      [[ -n "$_obj" ]] || continue
+      log "  - ${_obj}"
+    done <<< "$app_gitlab_objects"
+  else
+    warn "GitLab APP-context snapshot has no resources with app.kubernetes.io/instance=${gitlab_instance} yet (this can be transient during first reconcile)."
+  fi
+
+  if [[ -n "$db_gitlab_operator_objects" ]]; then
+    warn "Detected GitLab operator control-plane resources in DB context '${db_ctx}' (stale/legacy install likely):"
+    while IFS= read -r _obj; do
+      [[ -n "$_obj" ]] || continue
+      warn "  - ${_obj}"
+    done <<< "$db_gitlab_operator_objects"
+    warn "DB-context GitLab operator resources are not authoritative for APP GitLab workloads in split-cluster mode."
+  fi
+
+  if [[ -n "$db_gitlab_app_objects" ]]; then
+    local db_app_report=""
+    while IFS= read -r _obj; do
+      [[ -n "$_obj" ]] || continue
+      db_app_report+="- ${_obj}"$'\n'
+    done <<< "$db_gitlab_app_objects"
+    repair_blocked "GitLab split-cluster ownership violation: DB cluster contains GitLab app workloads" \
+      "APP context: ${app_ctx}. DB context: ${db_ctx}. GitLab app resources detected in DB context:\n${db_app_report}Expected policy: GitLab app workloads run only in APP context. Remove stale DB GitLab app workloads and rerun deploy."
+  fi
+}
+
 kubectl() {
   local target_ctx
   target_ctx="$(resolve_explicit_kube_context || true)"
@@ -354,7 +409,10 @@ gitlab_webservice_blocked_reasons() {
   if [[ "$webservice_selector" != *"="* ]]; then
     webservice_selector="app=${webservice_app}"
   fi
-  local restart_threshold="$2"
+  local not_ready_block_s="${GITLAB_WEBSERVICE_NOT_READY_BLOCK_SECONDS:-180}"
+  if [[ -z "$not_ready_block_s" || ! "$not_ready_block_s" =~ ^[0-9]+$ ]]; then
+    not_ready_block_s=180
+  fi
   local pod_names
   pod_names=$(kubectl -n "$NAMESPACE" get pods -l "$webservice_selector" \
     --field-selector=status.phase!=Succeeded,status.phase!=Failed \
@@ -373,39 +431,65 @@ gitlab_webservice_blocked_reasons() {
       printf '%s\n' "pod ${pod_name} status is ${pod_reason:-$pod_phase}"
     fi
 
+    local pod_ready_status pod_ready_transition
+    pod_ready_status=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
+    pod_ready_transition=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].lastTransitionTime}' 2>/dev/null || true)
+
     local container_rows
-    container_rows=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"|"}{.restartCount}{"|"}{.state.waiting.reason}{"|"}{.state.terminated.reason}{"\n"}{end}' 2>/dev/null || true)
+    container_rows=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"|"}{.state.waiting.reason}{"|"}{.state.terminated.reason}{"\n"}{end}' 2>/dev/null || true)
     [[ -n "$container_rows" ]] || continue
 
-    local container_name restart_count waiting_reason terminated_reason
+    local container_name waiting_reason terminated_reason
-    while IFS='|' read -r container_name restart_count waiting_reason terminated_reason; do
+    while IFS='|' read -r container_name waiting_reason terminated_reason; do
       [[ -n "$container_name" ]] || continue
 
       case "$waiting_reason" in
-        CrashLoopBackOff|ImagePullBackOff|Error)
+        CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerError|CreateContainerConfigError|RunContainerError|Error)
           printf '%s\n' "pod ${pod_name}/${container_name} waiting reason=${waiting_reason}"
           ;;
       esac
       case "$terminated_reason" in
-        Error)
+        Error|ContainerCannotRun)
           printf '%s\n' "pod ${pod_name}/${container_name} terminated reason=${terminated_reason}"
           ;;
       esac
-
-      if [[ -z "$restart_count" || ! "$restart_count" =~ ^[0-9]+$ ]]; then
-        restart_count=0
-      fi
-      if (( restart_count >= restart_threshold )); then
-        printf '%s\n' "pod ${pod_name}/${container_name} restartCount=${restart_count}"
-      fi
     done <<< "$container_rows"
+
+    if [[ "$pod_ready_status" != "True" ]]; then
+      local ready_false_age_s=0
+      if [[ -n "$pod_ready_transition" ]]; then
+        ready_false_age_s=$(python3 - "$pod_ready_transition" <<'PY'
+import datetime
+import sys
+
+raw = (sys.argv[1] if len(sys.argv) > 1 else "").strip()
+if not raw:
+    print("0")
+    raise SystemExit(0)
+
+try:
+    ts = datetime.datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    now = datetime.datetime.now(datetime.timezone.utc)
+    age = int((now - ts).total_seconds())
+    print(str(max(age, 0)))
+except Exception:
+    print("0")
+PY
+)
+      fi
+      if [[ -z "$ready_false_age_s" || ! "$ready_false_age_s" =~ ^[0-9]+$ ]]; then
+        ready_false_age_s=0
+      fi
+      if (( ready_false_age_s >= not_ready_block_s )); then
+        printf '%s\n' "pod ${pod_name} Ready=False for ${ready_false_age_s}s (threshold=${not_ready_block_s}s)"
+      fi
+    fi
   done <<< "$pod_names"
 }
 
 wait_for_gitlab_workload_convergence() {
   local timeout_s="${GITLAB_WORKLOAD_CONVERGENCE_TIMEOUT:-420}"
   local poll_interval_s="${GITLAB_WORKLOAD_CONVERGENCE_POLL_INTERVAL:-10}"
-  local restart_threshold="${GITLAB_WEBSERVICE_RESTART_BLOCK_THRESHOLD:-3}"
 
   if [[ -z "$timeout_s" || ! "$timeout_s" =~ ^[0-9]+$ ]]; then
     timeout_s=420
@@ -413,9 +497,6 @@ wait_for_gitlab_workload_convergence() {
   if [[ -z "$poll_interval_s" || ! "$poll_interval_s" =~ ^[0-9]+$ || "$poll_interval_s" == "0" ]]; then
     poll_interval_s=10
   fi
-  if [[ -z "$restart_threshold" || ! "$restart_threshold" =~ ^[0-9]+$ ]]; then
-    restart_threshold=3
-  fi
 
   local start_ts
   start_ts=$(date +%s)
@@ -424,6 +505,7 @@ wait_for_gitlab_workload_convergence() {
   while true; do
     local -a blockers=()
     local dep_suffix dep_name dep_selector desired_replicas live_non_terminal old_rs_summary
+    local ready_replicas available_replicas
     for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
       dep_name="${GITLAB_RELEASE}-${dep_suffix}"
 
@@ -449,6 +531,21 @@ wait_for_gitlab_workload_convergence() {
         blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} < desired=${desired_replicas} (selector=${dep_selector})")
       fi
 
+      ready_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
+      available_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
+      if [[ -z "$ready_replicas" || ! "$ready_replicas" =~ ^[0-9]+$ ]]; then
+        ready_replicas=0
+      fi
+      if [[ -z "$available_replicas" || ! "$available_replicas" =~ ^[0-9]+$ ]]; then
+        available_replicas=0
+      fi
+      if (( ready_replicas < desired_replicas )); then
+        blockers+=("${dep_name}: readyReplicas=${ready_replicas} < desired=${desired_replicas}")
+      fi
+      if (( available_replicas < desired_replicas )); then
+        blockers+=("${dep_name}: availableReplicas=${available_replicas} < desired=${desired_replicas}")
+      fi
+
       old_rs_summary=$(gitlab_old_replicaset_live_summary "$dep_name")
       if [[ -n "$old_rs_summary" ]]; then
         blockers+=("${dep_name}: old ReplicaSet pods still running (${old_rs_summary})")
@@ -456,7 +553,7 @@ wait_for_gitlab_workload_convergence() {
 
       if [[ "$dep_suffix" == "webservice-default" ]]; then
         local webservice_reasons
-        webservice_reasons=$(gitlab_webservice_blocked_reasons "$dep_selector" "$restart_threshold" 2>/dev/null || true)
+        webservice_reasons=$(gitlab_webservice_blocked_reasons "$dep_selector" 2>/dev/null || true)
         if [[ -n "$webservice_reasons" ]]; then
           local webservice_reason
           while IFS= read -r webservice_reason; do
@@ -2397,6 +2494,8 @@ is_unresolved_secret_ref() {
 }
 GITLAB_RELEASE="gitlab"
 
+gitlab_split_cluster_ownership_diagnostics
+
 assert_public_ingress_targeting "$GITLAB_INGRESS_CLASS" "${GITLAB_PUBLIC_HOSTS[@]}"
 
 # ---------------------------------------------------------------------------

tests/test_repair_update_and_supabase_flags.py

@@ -681,6 +681,28 @@ def test_init_gitlab_renders_non_empty_trusted_proxies_for_forwarded_headers():
     assert "trusted_proxies:${GITLAB_TRUSTED_PROXIES_YAML}" in script
 
 
+def test_init_gitlab_split_cluster_ownership_is_explicit_and_db_app_workloads_blocked():
+    """Split-cluster GitLab ownership must be explicit: APP owns app workloads; DB app workloads are blocked."""
+    script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
+
+    assert "gitlab_split_cluster_ownership_diagnostics()" in script
+    assert "GitLab split-cluster ownership policy: APP context" in script
+    assert "DB context '${db_ctx}' must not host GitLab app workloads" in script
+    assert "GitLab split-cluster ownership violation: DB cluster contains GitLab app workloads" in script
+    assert "gitlab_split_cluster_ownership_diagnostics" in script
+
+
+def test_init_gitlab_convergence_no_longer_blocks_on_historical_restart_count_only():
+    """Convergence gate must not fail solely on historical restartCount for currently healthy webservice pods."""
+    script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
+
+    assert "GITLAB_WEBSERVICE_NOT_READY_BLOCK_SECONDS" in script
+    assert "readyReplicas=" in script
+    assert "availableReplicas=" in script
+    assert "restartCount=" not in script
+    assert "gitlab_webservice_blocked_reasons \"$dep_selector\" \"$restart_threshold\"" not in script
+
+
 def test_init_gitlab_old_replicaset_summary_helper_is_non_fatal_when_empty():
     """ReplicaSet summary helper should not return non-zero on healthy empty summary under set -e."""
     script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")