From 3298d5cf2f20ef60daf718ed2431311b188a4769 Mon Sep 17 00:00:00 2001
From: chrisfu <chrisfu@prole.org>
Date: Sat, 18 Apr 2026 13:32:18 -0700
Subject: [PATCH] chore: scale down over-replicated GitLab deployments to
 target replica count of 1

---
 etc/init_gitlab.sh | 285 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 282 insertions(+), 3 deletions(-)

diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh
index 473389b..2921242 100755
--- a/etc/init_gitlab.sh
+++ b/etc/init_gitlab.sh
@@ -402,6 +402,273 @@ is_gke_standard_storage_equivalent() {
   return 1
 }
 
+is_gke_stale_gitaly_storage_class_for_standard_target() {
+  local desired_sc="$1"
+  local candidate_sc="$2"
+  [[ "$desired_sc" == "standard" ]] || return 1
+  case "$candidate_sc" in
+    standard-rwo|premium-rwo)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_gke_blocking_disk_type_for_standard_target() {
+  local disk_type="$1"
+  case "$disk_type" in
+    pd-balanced|pd-ssd)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_gke_cluster_detected() {
+  local provider_ids gke_pool_labels gke_topology_labels
+  provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true)
+  gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true)
+  gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true)
+  if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then
+    return 0
+  fi
+  return 1
+}
+
+gke_parse_disk_ref_from_volume_handle() {
+  local handle="$1"
+  [[ -n "$handle" ]] || return 1
+
+  if [[ "$handle" =~ /zones/([^/]+)/disks/([^/]+)$ ]]; then
+    printf 'zone|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
+    return 0
+  fi
+  if [[ "$handle" =~ /regions/([^/]+)/disks/([^/]+)$ ]]; then
+    printf 'region|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
+    return 0
+  fi
+  if [[ "$handle" =~ ^[^/]+$ ]]; then
+    printf 'name||%s' "$handle"
+    return 0
+  fi
+  return 1
+}
+
+gke_delete_disk_ref_if_present() {
+  local disk_scope="$1"
+  local disk_location="$2"
+  local disk_name="$3"
+  [[ -n "$disk_name" ]] || return 0
+
+  if ! command -v gcloud >/dev/null 2>&1; then
+    warn "gcloud not found; cannot auto-delete stale disk '${disk_name}'."
+    return 1
+  fi
+
+  local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
+  if [[ -z "$project_id" ]]; then
+    project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
+  fi
+
+  local -a common_args=(--quiet)
+  if [[ -n "$project_id" ]]; then
+    common_args+=(--project "$project_id")
+  fi
+
+  if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then
+    if gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
+      log "Deleting stale GKE disk ${disk_name} (zone=${disk_location})..."
+      gcloud compute disks delete "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${disk_location})."
+    else
+      log "Stale disk ${disk_name} already absent in zone ${disk_location}."
+    fi
+    return 0
+  fi
+
+  if [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then
+    if gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
+      log "Deleting stale GKE regional disk ${disk_name} (region=${disk_location})..."
+      gcloud compute disks delete "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${disk_location})."
+    else
+      log "Stale regional disk ${disk_name} already absent in region ${disk_location}."
+    fi
+    return 0
+  fi
+
+  local matched
+  matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](zone.basename(),region.basename())' 2>/dev/null || true)
+  if [[ -z "$matched" ]]; then
+    log "Stale disk ${disk_name} already absent."
+    return 0
+  fi
+
+  while IFS='|' read -r zone_name region_name; do
+    [[ -n "$zone_name" || -n "$region_name" ]] || continue
+    if [[ -n "$zone_name" ]]; then
+      log "Deleting stale GKE disk ${disk_name} (zone=${zone_name})..."
+      gcloud compute disks delete "$disk_name" --zone "$zone_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${zone_name})."
+    elif [[ -n "$region_name" ]]; then
+      log "Deleting stale GKE regional disk ${disk_name} (region=${region_name})..."
+      gcloud compute disks delete "$disk_name" --region "$region_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${region_name})."
+    fi
+  done <<< "$matched"
+}
+
+collect_stale_gke_gitaly_pv_records() {
+  local desired_sc="$1"
+  [[ "$desired_sc" == "standard" ]] || return 0
+
+  local pvc_name="repo-data-gitlab-gitaly-0"
+  local pvc_phase
+  pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
+
+  local pv_rows
+  pv_rows=$(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"|"}{.spec.storageClassName}{"|"}{.spec.claimRef.namespace}{"|"}{.spec.claimRef.name}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"\n"}{end}' 2>/dev/null || true)
+  [[ -n "$pv_rows" ]] || return 0
+
+  while IFS='|' read -r pv_name pv_phase pv_sc pv_claim_ns pv_claim_name pv_handle pv_gce_pd; do
+    [[ -n "$pv_name" ]] || continue
+    [[ "$pv_claim_ns" == "$NAMESPACE" && "$pv_claim_name" == "$pvc_name" ]] || continue
+    is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$pv_sc" || continue
+
+    local stale_reason=""
+    if [[ "$pv_phase" == "Released" || "$pv_phase" == "Failed" ]]; then
+      stale_reason="pv-phase-${pv_phase}"
+    elif [[ -z "$pvc_phase" ]]; then
+      stale_reason="missing-live-pvc"
+    fi
+    [[ -n "$stale_reason" ]] || continue
+
+    printf '%s|%s|%s|%s|%s|%s\n' "$pv_name" "$pv_phase" "$pv_sc" "$pv_handle" "$pv_gce_pd" "$stale_reason"
+  done <<< "$pv_rows"
+}
+
+collect_stale_gke_gitaly_disk_records_without_pv() {
+  local desired_sc="$1"
+  [[ "$desired_sc" == "standard" ]] || return 0
+  command -v gcloud >/dev/null 2>&1 || return 0
+
+  local pvc_name="repo-data-gitlab-gitaly-0"
+  local pvc_phase
+  pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
+  [[ -z "$pvc_phase" ]] || return 0
+
+  local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
+  if [[ -z "$project_id" ]]; then
+    project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
+  fi
+  local -a common_args=(--quiet)
+  if [[ -n "$project_id" ]]; then
+    common_args+=(--project "$project_id")
+  fi
+
+  local rows
+  rows=$(gcloud compute disks list "${common_args[@]}" \
+    --filter="labels.kubernetes-io-created-for-pvc-name=${pvc_name} AND labels.kubernetes-io-created-for-pvc-namespace=${NAMESPACE}" \
+    --format='csv[no-heading,separator="|"](name,zone.basename(),region.basename(),type.basename())' 2>/dev/null || true)
+  [[ -n "$rows" ]] || return 0
+
+  while IFS='|' read -r disk_name disk_zone disk_region disk_type; do
+    [[ -n "$disk_name" ]] || continue
+    is_gke_blocking_disk_type_for_standard_target "$disk_type" || continue
+    if [[ -n "$disk_zone" ]]; then
+      printf 'zone|%s|%s|%s|%s\n' "$disk_zone" "$disk_name" "$disk_type" "label-scan"
+    elif [[ -n "$disk_region" ]]; then
+      printf 'region|%s|%s|%s|%s\n' "$disk_region" "$disk_name" "$disk_type" "label-scan"
+    else
+      printf 'name||%s|%s|%s\n' "$disk_name" "$disk_type" "label-scan"
+    fi
+  done <<< "$rows"
+}
+
+repair_stale_gke_gitaly_dynamic_storage() {
+  local desired_sc="$1"
+  [[ "$MODE" == "k8s" ]] || return 0
+  [[ "$desired_sc" == "standard" ]] || return 0
+  is_gke_cluster_detected || return 0
+
+  local pvc_name="repo-data-gitlab-gitaly-0"
+  local stale_pv_records
+  stale_pv_records=$(collect_stale_gke_gitaly_pv_records "$desired_sc")
+  local stale_disk_records
+  stale_disk_records=$(collect_stale_gke_gitaly_disk_records_without_pv "$desired_sc")
+
+  if [[ -z "$stale_pv_records" && -z "$stale_disk_records" ]]; then
+    return 0
+  fi
+
+  local summary=""
+  if [[ -n "$stale_pv_records" ]]; then
+    while IFS='|' read -r pv_name pv_phase pv_sc _pv_handle _pv_gce_pd stale_reason; do
+      [[ -n "$pv_name" ]] || continue
+      summary+="PV ${pv_name} (phase=${pv_phase:-unknown}, sc=${pv_sc:-unknown}, reason=${stale_reason}). "
+    done <<< "$stale_pv_records"
+  fi
+  if [[ -n "$stale_disk_records" ]]; then
+    while IFS='|' read -r disk_scope disk_location disk_name disk_type disk_source; do
+      [[ -n "$disk_name" ]] || continue
+      summary+="Disk ${disk_name} (${disk_scope}:${disk_location:-n/a}, type=${disk_type:-unknown}, source=${disk_source}). "
+    done <<< "$stale_disk_records"
+  fi
+
+  if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
+    repair_blocked "Detected stale GKE Gitaly dynamic storage artifacts blocking class '${desired_sc}'" \
+      "Artifacts are scoped to PVC ${pvc_name} only: ${summary}Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to auto-clean stale PV/PD leftovers from old standard-rwo/pd-balanced attempts."
+  fi
+
+  export GITALY_AUTOCLEAN_PERFORMED=1
+  log "AUTOCLEAN: repairing stale GKE Gitaly dynamic storage artifacts for PVC ${pvc_name}..."
+  kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
+
+  local live_pvc_sc live_pvc_phase
+  live_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
+  live_pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
+  if [[ -n "$live_pvc_phase" ]] && is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$live_pvc_sc" && [[ "$live_pvc_phase" != "Bound" ]]; then
+    log "Deleting stale live PVC ${pvc_name} (phase=${live_pvc_phase}, sc=${live_pvc_sc}) before retry..."
+    kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true
+  fi
+
+  local processed_disks="|"
+  local _disk_key
+  local _disk_ref _disk_scope _disk_location _disk_name
+  if [[ -n "$stale_pv_records" ]]; then
+    while IFS='|' read -r pv_name _pv_phase _pv_sc pv_handle pv_gce_pd _stale_reason; do
+      [[ -n "$pv_name" ]] || continue
+      log "Deleting stale Gitaly PV ${pv_name}..."
+      kubectl delete pv "$pv_name" --wait=false 2>/dev/null || true
+
+      _disk_ref=""
+      if [[ -n "$pv_handle" ]]; then
+        _disk_ref=$(gke_parse_disk_ref_from_volume_handle "$pv_handle" || true)
+      fi
+      if [[ -z "$_disk_ref" && -n "$pv_gce_pd" ]]; then
+        _disk_ref="name||${pv_gce_pd}"
+      fi
+      if [[ -n "$_disk_ref" ]]; then
+        IFS='|' read -r _disk_scope _disk_location _disk_name <<< "$_disk_ref"
+        _disk_key="${_disk_scope}|${_disk_location}|${_disk_name}"
+        if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
+          processed_disks+="${_disk_key}|"
+          gke_delete_disk_ref_if_present "$_disk_scope" "$_disk_location" "$_disk_name" || true
+        fi
+      fi
+    done <<< "$stale_pv_records"
+  fi
+
+  if [[ -n "$stale_disk_records" ]]; then
+    while IFS='|' read -r disk_scope disk_location disk_name _disk_type _disk_source; do
+      [[ -n "$disk_name" ]] || continue
+      _disk_key="${disk_scope}|${disk_location}|${disk_name}"
+      if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
+        processed_disks+="${_disk_key}|"
+        gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true
+      fi
+    done <<< "$stale_disk_records"
+  fi
+
+  sleep 2
+}
+
 check_gitlab_post_apply_blocked() {
   # --- Migrations Check ---
   check_gitlab_migrations_blocked
@@ -521,11 +788,13 @@ check_gitlab_post_apply_blocked() {
           local pvc_name="repo-data-gitlab-gitaly-0"
           local pvc_start=$(date +%s)
           local pvc_timeout=600
+          local pvc_uid=""
           while true; do
             local pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
             local pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
 
             if [[ -n "$pvc_phase" ]]; then
+               pvc_uid=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
                local _sc_info=""
                if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-standard-normalized)"; fi
                if [[ "$pvc_sc" == "premium-rwo" ]]; then _sc_info=" (pd-ssd)"; fi
@@ -544,8 +813,11 @@ check_gitlab_post_apply_blocked() {
                   fi
                fi
 
-               # Check for provisioning failures (events)
-               local provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pvc_name",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
+               # Check provisioning failures only for the *current* PVC object.
+               local provisioning_fail=""
+               if [[ -n "$pvc_uid" ]]; then
+                  provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
+               fi
                if [[ -n "$provisioning_fail" ]]; then
                   if [[ "$provisioning_fail" == *"quota"* || "$provisioning_fail" == *"QUOTA"* ]]; then
                      repair_blocked "Gitaly PVC provisioning failed (Quota Exceeded)" \
@@ -561,7 +833,12 @@ check_gitlab_post_apply_blocked() {
             fi
 
             if (( $(date +%s) - pvc_start > pvc_timeout )); then
-               local last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pvc_name" --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
+               local last_msg=""
+               if [[ -n "$pvc_uid" ]]; then
+                  last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
+               else
+                  last_msg="No live PVC UID observed yet; ignoring stale historical PVC events from previous claims."
+               fi
                repair_blocked "Gitaly PVC failed to bind after ${pvc_timeout}s" \
                  "PVC: ${pvc_name}. Status: ${pvc_phase:-NotFound}. Last event: ${last_msg}. Fix: Check storage provider and quota."
             fi
@@ -2238,6 +2515,8 @@ kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0
 kubectl delete pv gitlab-gitaly-synology"
        fi
     fi
+
+    repair_stale_gke_gitaly_dynamic_storage "${GITALY_STORAGE_CLASS:-}"
   else
     setup_gitlab_legacy_storage
   fi