mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
chore: refine Gitaly convergence checks and auto-repair legacy configs
- Added logic to wait for Gitaly StatefulSet corrections after autoclean actions. - Implemented live state validation for nodeSelector and storageClass updates. - Enhanced logging and error handling for legacy storage repairs in k8s mode. - Updated PV and PVC handling to address legacy resource cleanup.
This commit is contained in:
chrisfu
parent
7999b2f81a
commit
5657ccb4e5
@ -198,10 +198,82 @@ check_gitlab_pre_apply_blocked() {
|
||||
check_gitlab_post_apply_blocked() {
|
||||
# --- Gitaly Check ---
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
local desired_cr="${GITLAB_CR_RENDERED:-}"
|
||||
if [[ -n "$desired_cr" ]]; then
|
||||
local desired_gitaly_node_selector
|
||||
desired_gitaly_node_selector=$(echo "$desired_cr" | sed -n '/gitaly:/,/toolbox:/p' | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
|
||||
local desired_gitaly_storage_class
|
||||
desired_gitaly_storage_class=$(echo "$desired_cr" | sed -n '/gitaly:/,/toolbox:/p' | grep "storageClass:" | cut -d: -f2 | xargs || true)
|
||||
|
||||
log "Desired Gitaly state: nodeSelector=[${desired_gitaly_node_selector}], storageClass=[${desired_gitaly_storage_class}]"
|
||||
|
||||
if [[ "$desired_cr" == *"gandalf.prole.org"* ]]; then
|
||||
repair_blocked "GitLab CR contains legacy nodeSelector" \
|
||||
"Desired GitLab CR still contains gandalf.prole.org. This is a configuration bug."
|
||||
fi
|
||||
if [[ "$desired_cr" == *"gitlab-gitaly-static"* ]]; then
|
||||
repair_blocked "GitLab CR contains legacy storageClass" \
|
||||
"Desired GitLab CR still contains gitlab-gitaly-static. This is a configuration bug."
|
||||
fi
|
||||
fi
|
||||
|
||||
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
|
||||
|
||||
# If autoclean was performed, wait for convergence before checking live state.
|
||||
# This prevents false positives when the operator hasn't yet updated the stale StatefulSet.
|
||||
if [[ "${GITALY_AUTOCLEAN_PERFORMED:-0}" == "1" ]]; then
|
||||
log "AUTOCLEAN was performed. Waiting for Gitaly StatefulSet to converge (removing legacy storage)..."
|
||||
local wait_start=$(date +%s)
|
||||
local wait_timeout=600 # 10 minutes
|
||||
while true; do
|
||||
local live_sts_yaml
|
||||
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
|
||||
if [[ -z "$live_sts_yaml" ]]; then
|
||||
log "Gitaly StatefulSet not found (awaiting operator action)..."
|
||||
elif [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
|
||||
log "Gitaly StatefulSet converged to corrected state (clean nodeSelector/storageClass)."
|
||||
export GITALY_AUTOCLEAN_PERFORMED=0
|
||||
break
|
||||
else
|
||||
local live_gitaly_node_selector
|
||||
live_gitaly_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
|
||||
local live_gitaly_storage_class
|
||||
live_gitaly_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
|
||||
log "Still waiting for Gitaly convergence (live nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}])..."
|
||||
fi
|
||||
|
||||
if (( $(date +%s) - wait_start > wait_timeout )); then
|
||||
repair_blocked "Gitaly failed to converge after ${wait_timeout}s" \
|
||||
"Resource: statefulset/${gitaly_sts_name}. Value: Still contains legacy fields. Fix: Check operator logs."
|
||||
fi
|
||||
sleep 15
|
||||
done
|
||||
|
||||
# Also wait for PV deletion to settle if it exists
|
||||
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
|
||||
log "Waiting for legacy PV gitlab-gitaly-synology to be removed..."
|
||||
local pv_wait_start=$(date +%s)
|
||||
while kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; do
|
||||
if (( $(date +%s) - pv_wait_start > 120 )); then
|
||||
log "Legacy PV still exists after 120s; continuing (it may be stuck in Terminating)."
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# Final live state checks
|
||||
local gitaly_sts_yaml
|
||||
gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
|
||||
if [[ -n "$gitaly_sts_yaml" ]]; then
|
||||
local live_gitaly_node_selector
|
||||
live_gitaly_node_selector=$(echo "$gitaly_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
|
||||
local live_gitaly_storage_class
|
||||
live_gitaly_storage_class=$(echo "$gitaly_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
|
||||
|
||||
log "Live Gitaly state: nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}]"
|
||||
|
||||
if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* ]]; then
|
||||
repair_blocked "Gitaly using legacy Synology storage" \
|
||||
"Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Ensure GITLAB_STORAGE_NODE is not set in k8s mode."
|
||||
@ -1087,6 +1159,10 @@ fi
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
# In k8s/GKE mode, we do NOT use static storage nodes or nodeSelectors.
|
||||
STORAGE_NODE=""
|
||||
if [[ "$NODE_SELECTOR" == *"gandalf.prole.org"* ]]; then
|
||||
warn "Stripping legacy nodeSelector '${NODE_SELECTOR}' in k8s mode."
|
||||
NODE_SELECTOR=""
|
||||
fi
|
||||
else
|
||||
STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
|
||||
fi
|
||||
@ -1682,6 +1758,7 @@ setup_gitlab_storage() {
|
||||
|
||||
if [[ "$has_legacy" == "1" ]]; then
|
||||
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then
|
||||
export GITALY_AUTOCLEAN_PERFORMED=1
|
||||
log "AUTOCLEAN: repairing legacy Gitaly storage (scaling down and deleting legacy PV/PVC)..."
|
||||
kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
|
||||
kubectl -n "$NAMESPACE" delete pvc repo-data-gitlab-gitaly-0 --wait=false 2>/dev/null || true
|
||||
@ -1776,15 +1853,13 @@ else
|
||||
log "Rendering GitLab values without hostPath jemalloc mounts/LD_PRELOAD."
|
||||
fi
|
||||
|
||||
log "Applying GitLab CR (hosts=${_gitlab_hosts_csv}, db=${GITLAB_DB_NAME}@${DB_HOST})..."
|
||||
|
||||
gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
|
||||
gitlab_exists_before=0
|
||||
if [[ -n "$gitlab_generation_before" ]]; then
|
||||
gitlab_exists_before=1
|
||||
fi
|
||||
|
||||
gitlab_apply_output="$(kubectl apply -f - <<EOF
|
||||
GITLAB_CR_RENDERED="$(cat <<EOF
|
||||
apiVersion: apps.gitlab.com/v1beta1
|
||||
kind: GitLab
|
||||
metadata:
|
||||
@ -1972,6 +2047,9 @@ ${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
EOF
|
||||
)"
|
||||
|
||||
log "Applying GitLab CR (hosts=${_gitlab_hosts_csv}, db=${GITLAB_DB_NAME}@${DB_HOST})..."
|
||||
gitlab_apply_output="$(echo "$GITLAB_CR_RENDERED" | kubectl apply -f -)"
|
||||
|
||||
check_gitlab_post_apply_blocked
|
||||
|
||||
gitlab_generation_after="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user