chore: improve Gitaly StatefulSet convergence and enhance legacy repair logic

- Introduced a grace period and explicit replacement of legacy StatefulSet after expiry.
- Enhanced logging to provide clearer details on convergence progress and outcomes.
- Added handling for operator recreation and state validation during legacy repairs.
- Ensured robust detection and resolution of legacy nodeSelector and storageClass issues.

etc/init_gitlab.sh

@@ -223,8 +223,9 @@ check_gitlab_post_apply_blocked() {
     # This prevents false positives when the operator hasn't yet updated the stale StatefulSet.
     if [[ "${GITALY_AUTOCLEAN_PERFORMED:-0}" == "1" ]]; then
        log "AUTOCLEAN was performed. Waiting for Gitaly StatefulSet to converge (removing legacy storage)..."
-       local wait_start=$(date +%s)
+       local grace_start=$(date +%s)
-       local wait_timeout=600 # 10 minutes
+       local grace_timeout=60
+       local converged=0
        while true; do
          local live_sts_yaml
          live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
@@ -232,7 +233,7 @@ check_gitlab_post_apply_blocked() {
             log "Gitaly StatefulSet not found (awaiting operator action)..."
          elif [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
             log "Gitaly StatefulSet converged to corrected state (clean nodeSelector/storageClass)."
-            export GITALY_AUTOCLEAN_PERFORMED=0
+            converged=1
             break
          else
             local live_gitaly_node_selector
@@ -242,13 +243,58 @@ check_gitlab_post_apply_blocked() {
             log "Still waiting for Gitaly convergence (live nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}])..."
          fi
 
-         if (( $(date +%s) - wait_start > wait_timeout )); then
+         if (( $(date +%s) - grace_start > grace_timeout )); then
-            repair_blocked "Gitaly failed to converge after ${wait_timeout}s" \
+            log "Grace period (${grace_timeout}s) expired."
-              "Resource: statefulset/${gitaly_sts_name}. Value: Still contains legacy fields. Fix: Check operator logs."
+            break
          fi
          sleep 15
        done
 
+       if [[ "$converged" == "0" ]]; then
+          local live_sts_yaml
+          live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
+          if [[ -n "$live_sts_yaml" && ( "$live_sts_yaml" == *"gandalf.prole.org"* || "$live_sts_yaml" == *"gitlab-gitaly-static"* ) ]]; then
+             log "Live StatefulSet still legacy after grace period. Performing explicit replacement..."
+             # We already confirmed desired CR is corrected at the start of this function.
+             kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true
+             log "Legacy StatefulSet deleted. Waiting for operator recreation..."
+
+             local recreate_start=$(date +%s)
+             local recreate_timeout=300
+             while true; do
+               live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
+               if [[ -n "$live_sts_yaml" ]]; then
+                  local recreated_node_selector
+                  recreated_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
+                  local recreated_storage_class
+                  recreated_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
+                  log "StatefulSet recreated. nodeSelector=[${recreated_node_selector}], storageClass=[${recreated_storage_class}]"
+
+                  if [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
+                     log "Gitaly StatefulSet recreated in corrected state."
+                     converged=1
+                     break
+                  else
+                     log "Recreated StatefulSet STILL contains legacy fields. Waiting for operator to correct it..."
+                  fi
+               fi
+
+               if (( $(date +%s) - recreate_start > recreate_timeout )); then
+                  repair_blocked "Gitaly failed to recreate clean StatefulSet after ${recreate_timeout}s" \
+                    "Resource: statefulset/${gitaly_sts_name}. Fix: Check operator logs and desired GitLab CR."
+               fi
+               sleep 15
+             done
+          else
+             log "No legacy StatefulSet found after grace period (may have been deleted or converged)."
+             converged=1
+          fi
+       fi
+
+       if [[ "$converged" == "1" ]]; then
+          export GITALY_AUTOCLEAN_PERFORMED=0
+       fi
+
        # Also wait for PV deletion to settle if it exists
        if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
           log "Waiting for legacy PV gitlab-gitaly-synology to be removed..."