chore: enhance GitLab init to block misconfigurations and repair legacy setups

- Improved `check_gitlab_pre_apply_blocked` to better detect invalid placeholders, legacy storage configurations, and DNS issues for Redis and Garage endpoints.
- Added auto-repair for Garage endpoint secrets and storage class mismatches in k8s mode.
- Introduced warnings and blocking logic for unsupported configurations like `gitlab-gitaly-static`.
- Refined storage class handling to dynamically align with `standard-rwo` in k8s/GKE.

etc/init_gitlab.sh

@@ -145,12 +145,16 @@ check_host_resolves() {
 check_gitlab_pre_apply_blocked() {
   # --- Registry Endpoint Placeholder Check ---
   if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
-    repair_blocked "Registry endpoint invalid or wrong Garage" \
+    # If the configured value itself is still a placeholder, we MUST block.
-      "Resource: GARAGE_S3_ENDPOINT. Value: contains placeholder. Fix: Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint."
+    # The operator hasn't provided a real value yet.
+    repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
+      "GARAGE_PRIVATE_S3_ENDPOINT is set to the literal placeholder. Set it to the real DB cluster Garage endpoint."
   fi
 
   # --- Redis Host Resolution Check ---
   if ! check_host_resolves "$REDIS_HOST"; then
+     # This is potentially repairable if we can infer a better host,
+     # but for now we block if it's clearly invalid.
      repair_blocked "Redis host does not resolve" \
         "Resource: REDIS_HOST. Value: ${REDIS_HOST} does not resolve. Fix: Ensure Redis is deployed and REDIS_NAMESPACE is correct."
   fi
@@ -165,11 +169,11 @@ check_gitlab_post_apply_blocked() {
     if [[ -n "$gitaly_sts_yaml" ]]; then
       if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* ]]; then
         repair_blocked "Gitaly using legacy Synology storage" \
-          "Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Unset GITLAB_STORAGE_NODE in config."
+          "Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Ensure GITLAB_STORAGE_NODE is not set in k8s mode."
       fi
       if [[ "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
         repair_blocked "Gitaly using legacy Synology storage" \
-          "Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden."
+          "Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden in k8s mode."
       fi
     fi
 
@@ -182,7 +186,7 @@ check_gitlab_post_apply_blocked() {
     unschedulable_gitaly=$(kubectl -n "$NAMESPACE" get pods -l "app=gitaly" -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="PodScheduled")].status=="False")]}{.metadata.name}:{.status.conditions[?(@.type=="PodScheduled")].reason}{"\n"}{end}' 2>/dev/null | grep "Unschedulable" || true)
     if [[ -n "$unschedulable_gitaly" ]]; then
        repair_blocked "Gitaly pod(s) unschedulable" \
-          "Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storage (standard-rwo) and no hostname nodeSelector."
+          "Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storage (standard-rwo) and no hostname nodeSelector. Ensure PV/PVC were repaired."
     fi
   fi
 
@@ -190,8 +194,14 @@ check_gitlab_post_apply_blocked() {
   local registry_secret_config
   registry_secret_config=$(kubectl -n "$NAMESPACE" get secret "${GITLAB_RELEASE}-registry-storage" -o jsonpath='{.data.config}' 2>/dev/null | base64 -d 2>/dev/null || true)
   if [[ "$registry_secret_config" == *"<private-db-cluster-garage-endpoint>"* ]]; then
-    repair_blocked "Registry endpoint invalid or wrong Garage" \
+    if [[ "$GARAGE_S3_ENDPOINT" != *"<private-db-cluster-garage-endpoint>"* ]]; then
-      "Resource: secret/${GITLAB_RELEASE}-registry-storage. Value: contains placeholder. Fix: Set GARAGE_PRIVATE_S3_ENDPOINT correctly."
+       log "Registry secret still contains placeholder but config is updated. Re-applying secret..."
+       # Re-triggering the secret creation (this is safer than just blocking)
+       setup_garage_for_gitlab
+    else
+       repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder in live secret)" \
+         "Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint."
+    fi
   fi
 
   # --- Registry Logs Check ---
@@ -201,8 +211,9 @@ check_gitlab_post_apply_blocked() {
     local logs
     logs=$(kubectl -n "$NAMESPACE" logs "$registry_pod" --tail=100 2>&1 || true)
     if [[ "$logs" == *"DNS failure"* || "$logs" == *"AccessDenied"* || "$logs" == *"No such key"* ]]; then
-       repair_blocked "Registry endpoint invalid or wrong Garage" \
+       # If it's a "No such key" or "AccessDenied", and we have a custom endpoint, it might be the WRONG cluster Garage.
-          "Resource: registry pod logs. Value: S3 error detected. Fix: Verify Garage endpoint and bucket permissions."
+       repair_blocked "Registry endpoint invalid or wrong Garage (S3 error detected)" \
+          "Logs: ${logs}. Fix: Verify GARAGE_S3_ENDPOINT points to the DB cluster Garage (not the APP cluster one)."
     fi
   fi
 
@@ -213,8 +224,8 @@ check_gitlab_post_apply_blocked() {
     local logs
     logs=$(kubectl -n "$NAMESPACE" logs "$kas_pod" --tail=100 2>&1 || true)
     if [[ "$logs" == *"no such host"* && "$logs" == *"redis"* ]]; then
-       repair_blocked "Redis host does not resolve" \
+       repair_blocked "Redis host does not resolve (detected in KAS logs)" \
-          "Resource: kas pod logs. Value: DNS failure for redis. Fix: Verify REDIS_HOST and REDIS_NAMESPACE."
+          "Logs: ${logs}. Fix: Verify REDIS_HOST (${REDIS_HOST}) and ensure Redis service is healthy."
     fi
   fi
 }
@@ -1039,7 +1050,8 @@ fi
 # ---------------------------------------------------------------------------
 # Storage node is only used for legacy/local static local-PV mode.
 if [[ "$MODE" == "k8s" ]]; then
-  STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
+  # In k8s/GKE mode, we do NOT use static storage nodes or nodeSelectors.
+  STORAGE_NODE=""
 else
   STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
 fi
@@ -1059,10 +1071,15 @@ if [[ -n "$STORAGE_NODE" ]]; then
 fi
 
 GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}"
-if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
+if [[ "$MODE" == "k8s" ]]; then
-  if [[ "$MODE" == "k8s" ]]; then
+  if [[ -z "$GITALY_STORAGE_CLASS" || "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
+    if [[ "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
+      warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode; using 'standard-rwo'."
+    fi
     GITALY_STORAGE_CLASS="standard-rwo"
-  else
+  fi
+else
+  if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
     GITALY_STORAGE_CLASS="gitlab-gitaly-static"
   fi
 fi
@@ -1615,14 +1632,33 @@ setup_gitlab_storage() {
     log "Skipping legacy Synology/local-PV gitaly storage prep in k8s mode; using dynamic StorageClass '${GITALY_STORAGE_CLASS}'."
 
     # Check for legacy resources on GKE/k8s
-    if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1 || \
+    local has_legacy=0
-       kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then
+    if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then has_legacy=1; fi
+    if kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then has_legacy=1; fi
 
+    local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
+    local gitaly_sts_yaml
+    gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
+    if [[ -n "$gitaly_sts_yaml" ]]; then
+      if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* || "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
+         has_legacy=1
+      fi
+    fi
+
+    if [[ "$has_legacy" == "1" ]]; then
        if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then
           log "AUTOCLEAN: repairing legacy Gitaly storage (scaling down and deleting legacy PV/PVC)..."
           kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
           kubectl -n "$NAMESPACE" delete pvc repo-data-gitlab-gitaly-0 --wait=false 2>/dev/null || true
           kubectl delete pv gitlab-gitaly-synology --wait=false 2>/dev/null || true
+          # Give it a moment to process deletions
+          sleep 2
+       else
+          repair_blocked "Legacy Gitaly PV/PVC still bound to gitlab-gitaly-static / gandalf.prole.org" \
+            "Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to automatically repair by deleting legacy PVC/PV, or run:
+kubectl -n $NAMESPACE scale sts ${GITLAB_RELEASE}-gitaly --replicas=0
+kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0
+kubectl delete pv gitlab-gitaly-synology"
        fi
     fi
   else