chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-24 18:44:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

chore: improve Gitaly StatefulSet convergence and enhance legacy repair logic

Browse Source 
- Introduced a grace period and explicit replacement of legacy StatefulSet after expiry.
- Enhanced logging to provide clearer details on convergence progress and outcomes.
- Added handling for operator recreation and state validation during legacy repairs.
- Ensured robust detection and resolution of legacy nodeSelector and storageClass issues.
This commit is contained in:
chrisfu 2026-04-14 16:29:55 -07:00
parent 5657ccb4e5
commit 1502527d6f
1 changed files with 52 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
etc/init_gitlab.sh
View File

@ -223,8 +223,9 @@ check_gitlab_post_apply_blocked() {
# This prevents false positives when the operator hasn't yet updated the stale StatefulSet.
if [[ "${GITALY_AUTOCLEAN_PERFORMED:-0}" == "1" ]]; then
log "AUTOCLEAN was performed. Waiting for Gitaly StatefulSet to converge (removing legacy storage)..."
local wait_start=$(date +%s)
local wait_timeout=600 # 10 minutes
local grace_start=$(date +%s)
local grace_timeout=60
local converged=0
while true; do
local live_sts_yaml
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
@ -232,7 +233,7 @@ check_gitlab_post_apply_blocked() {
log "Gitaly StatefulSet not found (awaiting operator action)..."
elif [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
log "Gitaly StatefulSet converged to corrected state (clean nodeSelector/storageClass)."
export GITALY_AUTOCLEAN_PERFORMED=0
converged=1
break
else
local live_gitaly_node_selector
@ -242,13 +243,58 @@ check_gitlab_post_apply_blocked() {
log "Still waiting for Gitaly convergence (live nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}])..."
fi
if (( $(date +%s) - wait_start > wait_timeout )); then
repair_blocked "Gitaly failed to converge after ${wait_timeout}s" \
"Resource: statefulset/${gitaly_sts_name}. Value: Still contains legacy fields. Fix: Check operator logs."
if (( $(date +%s) - grace_start > grace_timeout )); then
log "Grace period (${grace_timeout}s) expired."
break
fi
sleep 15
done
if [[ "$converged" == "0" ]]; then
local live_sts_yaml
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$live_sts_yaml" && ( "$live_sts_yaml" == *"gandalf.prole.org"* || "$live_sts_yaml" == *"gitlab-gitaly-static"* ) ]]; then
log "Live StatefulSet still legacy after grace period. Performing explicit replacement..."
# We already confirmed desired CR is corrected at the start of this function.
kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true
log "Legacy StatefulSet deleted. Waiting for operator recreation..."
local recreate_start=$(date +%s)
local recreate_timeout=300
while true; do
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$live_sts_yaml" ]]; then
local recreated_node_selector
recreated_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local recreated_storage_class
recreated_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
log "StatefulSet recreated. nodeSelector=[${recreated_node_selector}], storageClass=[${recreated_storage_class}]"
if [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
log "Gitaly StatefulSet recreated in corrected state."
converged=1
break
else
log "Recreated StatefulSet STILL contains legacy fields. Waiting for operator to correct it..."
fi
fi
if (( $(date +%s) - recreate_start > recreate_timeout )); then
repair_blocked "Gitaly failed to recreate clean StatefulSet after ${recreate_timeout}s" \
"Resource: statefulset/${gitaly_sts_name}. Fix: Check operator logs and desired GitLab CR."
fi
sleep 15
done
else
log "No legacy StatefulSet found after grace period (may have been deleted or converged)."
converged=1
fi
fi
if [[ "$converged" == "1" ]]; then
export GITALY_AUTOCLEAN_PERFORMED=0
fi
# Also wait for PV deletion to settle if it exists
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
log "Waiting for legacy PV gitlab-gitaly-synology to be removed..."