mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
chore: scale down over-replicated GitLab deployments to target replica count of 1
This commit is contained in:
chrisfu
parent
b8a55888be
commit
3298d5cf2f
@ -402,6 +402,273 @@ is_gke_standard_storage_equivalent() {
|
||||
return 1
|
||||
}
|
||||
|
||||
is_gke_stale_gitaly_storage_class_for_standard_target() {
|
||||
local desired_sc="$1"
|
||||
local candidate_sc="$2"
|
||||
[[ "$desired_sc" == "standard" ]] || return 1
|
||||
case "$candidate_sc" in
|
||||
standard-rwo|premium-rwo)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
is_gke_blocking_disk_type_for_standard_target() {
|
||||
local disk_type="$1"
|
||||
case "$disk_type" in
|
||||
pd-balanced|pd-ssd)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
is_gke_cluster_detected() {
|
||||
local provider_ids gke_pool_labels gke_topology_labels
|
||||
provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true)
|
||||
gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true)
|
||||
gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true)
|
||||
if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
gke_parse_disk_ref_from_volume_handle() {
|
||||
local handle="$1"
|
||||
[[ -n "$handle" ]] || return 1
|
||||
|
||||
if [[ "$handle" =~ /zones/([^/]+)/disks/([^/]+)$ ]]; then
|
||||
printf 'zone|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
|
||||
return 0
|
||||
fi
|
||||
if [[ "$handle" =~ /regions/([^/]+)/disks/([^/]+)$ ]]; then
|
||||
printf 'region|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
|
||||
return 0
|
||||
fi
|
||||
if [[ "$handle" =~ ^[^/]+$ ]]; then
|
||||
printf 'name||%s' "$handle"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
gke_delete_disk_ref_if_present() {
|
||||
local disk_scope="$1"
|
||||
local disk_location="$2"
|
||||
local disk_name="$3"
|
||||
[[ -n "$disk_name" ]] || return 0
|
||||
|
||||
if ! command -v gcloud >/dev/null 2>&1; then
|
||||
warn "gcloud not found; cannot auto-delete stale disk '${disk_name}'."
|
||||
return 1
|
||||
fi
|
||||
|
||||
local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
|
||||
if [[ -z "$project_id" ]]; then
|
||||
project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
|
||||
fi
|
||||
|
||||
local -a common_args=(--quiet)
|
||||
if [[ -n "$project_id" ]]; then
|
||||
common_args+=(--project "$project_id")
|
||||
fi
|
||||
|
||||
if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then
|
||||
if gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
|
||||
log "Deleting stale GKE disk ${disk_name} (zone=${disk_location})..."
|
||||
gcloud compute disks delete "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${disk_location})."
|
||||
else
|
||||
log "Stale disk ${disk_name} already absent in zone ${disk_location}."
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then
|
||||
if gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
|
||||
log "Deleting stale GKE regional disk ${disk_name} (region=${disk_location})..."
|
||||
gcloud compute disks delete "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${disk_location})."
|
||||
else
|
||||
log "Stale regional disk ${disk_name} already absent in region ${disk_location}."
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
local matched
|
||||
matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](zone.basename(),region.basename())' 2>/dev/null || true)
|
||||
if [[ -z "$matched" ]]; then
|
||||
log "Stale disk ${disk_name} already absent."
|
||||
return 0
|
||||
fi
|
||||
|
||||
while IFS='|' read -r zone_name region_name; do
|
||||
[[ -n "$zone_name" || -n "$region_name" ]] || continue
|
||||
if [[ -n "$zone_name" ]]; then
|
||||
log "Deleting stale GKE disk ${disk_name} (zone=${zone_name})..."
|
||||
gcloud compute disks delete "$disk_name" --zone "$zone_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${zone_name})."
|
||||
elif [[ -n "$region_name" ]]; then
|
||||
log "Deleting stale GKE regional disk ${disk_name} (region=${region_name})..."
|
||||
gcloud compute disks delete "$disk_name" --region "$region_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${region_name})."
|
||||
fi
|
||||
done <<< "$matched"
|
||||
}
|
||||
|
||||
collect_stale_gke_gitaly_pv_records() {
|
||||
local desired_sc="$1"
|
||||
[[ "$desired_sc" == "standard" ]] || return 0
|
||||
|
||||
local pvc_name="repo-data-gitlab-gitaly-0"
|
||||
local pvc_phase
|
||||
pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
|
||||
local pv_rows
|
||||
pv_rows=$(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"|"}{.spec.storageClassName}{"|"}{.spec.claimRef.namespace}{"|"}{.spec.claimRef.name}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"\n"}{end}' 2>/dev/null || true)
|
||||
[[ -n "$pv_rows" ]] || return 0
|
||||
|
||||
while IFS='|' read -r pv_name pv_phase pv_sc pv_claim_ns pv_claim_name pv_handle pv_gce_pd; do
|
||||
[[ -n "$pv_name" ]] || continue
|
||||
[[ "$pv_claim_ns" == "$NAMESPACE" && "$pv_claim_name" == "$pvc_name" ]] || continue
|
||||
is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$pv_sc" || continue
|
||||
|
||||
local stale_reason=""
|
||||
if [[ "$pv_phase" == "Released" || "$pv_phase" == "Failed" ]]; then
|
||||
stale_reason="pv-phase-${pv_phase}"
|
||||
elif [[ -z "$pvc_phase" ]]; then
|
||||
stale_reason="missing-live-pvc"
|
||||
fi
|
||||
[[ -n "$stale_reason" ]] || continue
|
||||
|
||||
printf '%s|%s|%s|%s|%s|%s\n' "$pv_name" "$pv_phase" "$pv_sc" "$pv_handle" "$pv_gce_pd" "$stale_reason"
|
||||
done <<< "$pv_rows"
|
||||
}
|
||||
|
||||
collect_stale_gke_gitaly_disk_records_without_pv() {
|
||||
local desired_sc="$1"
|
||||
[[ "$desired_sc" == "standard" ]] || return 0
|
||||
command -v gcloud >/dev/null 2>&1 || return 0
|
||||
|
||||
local pvc_name="repo-data-gitlab-gitaly-0"
|
||||
local pvc_phase
|
||||
pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
[[ -z "$pvc_phase" ]] || return 0
|
||||
|
||||
local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
|
||||
if [[ -z "$project_id" ]]; then
|
||||
project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
|
||||
fi
|
||||
local -a common_args=(--quiet)
|
||||
if [[ -n "$project_id" ]]; then
|
||||
common_args+=(--project "$project_id")
|
||||
fi
|
||||
|
||||
local rows
|
||||
rows=$(gcloud compute disks list "${common_args[@]}" \
|
||||
--filter="labels.kubernetes-io-created-for-pvc-name=${pvc_name} AND labels.kubernetes-io-created-for-pvc-namespace=${NAMESPACE}" \
|
||||
--format='csv[no-heading,separator="|"](name,zone.basename(),region.basename(),type.basename())' 2>/dev/null || true)
|
||||
[[ -n "$rows" ]] || return 0
|
||||
|
||||
while IFS='|' read -r disk_name disk_zone disk_region disk_type; do
|
||||
[[ -n "$disk_name" ]] || continue
|
||||
is_gke_blocking_disk_type_for_standard_target "$disk_type" || continue
|
||||
if [[ -n "$disk_zone" ]]; then
|
||||
printf 'zone|%s|%s|%s|%s\n' "$disk_zone" "$disk_name" "$disk_type" "label-scan"
|
||||
elif [[ -n "$disk_region" ]]; then
|
||||
printf 'region|%s|%s|%s|%s\n' "$disk_region" "$disk_name" "$disk_type" "label-scan"
|
||||
else
|
||||
printf 'name||%s|%s|%s\n' "$disk_name" "$disk_type" "label-scan"
|
||||
fi
|
||||
done <<< "$rows"
|
||||
}
|
||||
|
||||
repair_stale_gke_gitaly_dynamic_storage() {
|
||||
local desired_sc="$1"
|
||||
[[ "$MODE" == "k8s" ]] || return 0
|
||||
[[ "$desired_sc" == "standard" ]] || return 0
|
||||
is_gke_cluster_detected || return 0
|
||||
|
||||
local pvc_name="repo-data-gitlab-gitaly-0"
|
||||
local stale_pv_records
|
||||
stale_pv_records=$(collect_stale_gke_gitaly_pv_records "$desired_sc")
|
||||
local stale_disk_records
|
||||
stale_disk_records=$(collect_stale_gke_gitaly_disk_records_without_pv "$desired_sc")
|
||||
|
||||
if [[ -z "$stale_pv_records" && -z "$stale_disk_records" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local summary=""
|
||||
if [[ -n "$stale_pv_records" ]]; then
|
||||
while IFS='|' read -r pv_name pv_phase pv_sc _pv_handle _pv_gce_pd stale_reason; do
|
||||
[[ -n "$pv_name" ]] || continue
|
||||
summary+="PV ${pv_name} (phase=${pv_phase:-unknown}, sc=${pv_sc:-unknown}, reason=${stale_reason}). "
|
||||
done <<< "$stale_pv_records"
|
||||
fi
|
||||
if [[ -n "$stale_disk_records" ]]; then
|
||||
while IFS='|' read -r disk_scope disk_location disk_name disk_type disk_source; do
|
||||
[[ -n "$disk_name" ]] || continue
|
||||
summary+="Disk ${disk_name} (${disk_scope}:${disk_location:-n/a}, type=${disk_type:-unknown}, source=${disk_source}). "
|
||||
done <<< "$stale_disk_records"
|
||||
fi
|
||||
|
||||
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
|
||||
repair_blocked "Detected stale GKE Gitaly dynamic storage artifacts blocking class '${desired_sc}'" \
|
||||
"Artifacts are scoped to PVC ${pvc_name} only: ${summary}Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to auto-clean stale PV/PD leftovers from old standard-rwo/pd-balanced attempts."
|
||||
fi
|
||||
|
||||
export GITALY_AUTOCLEAN_PERFORMED=1
|
||||
log "AUTOCLEAN: repairing stale GKE Gitaly dynamic storage artifacts for PVC ${pvc_name}..."
|
||||
kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
|
||||
|
||||
local live_pvc_sc live_pvc_phase
|
||||
live_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
|
||||
live_pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
if [[ -n "$live_pvc_phase" ]] && is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$live_pvc_sc" && [[ "$live_pvc_phase" != "Bound" ]]; then
|
||||
log "Deleting stale live PVC ${pvc_name} (phase=${live_pvc_phase}, sc=${live_pvc_sc}) before retry..."
|
||||
kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true
|
||||
fi
|
||||
|
||||
local processed_disks="|"
|
||||
local _disk_key
|
||||
local _disk_ref _disk_scope _disk_location _disk_name
|
||||
if [[ -n "$stale_pv_records" ]]; then
|
||||
while IFS='|' read -r pv_name _pv_phase _pv_sc pv_handle pv_gce_pd _stale_reason; do
|
||||
[[ -n "$pv_name" ]] || continue
|
||||
log "Deleting stale Gitaly PV ${pv_name}..."
|
||||
kubectl delete pv "$pv_name" --wait=false 2>/dev/null || true
|
||||
|
||||
_disk_ref=""
|
||||
if [[ -n "$pv_handle" ]]; then
|
||||
_disk_ref=$(gke_parse_disk_ref_from_volume_handle "$pv_handle" || true)
|
||||
fi
|
||||
if [[ -z "$_disk_ref" && -n "$pv_gce_pd" ]]; then
|
||||
_disk_ref="name||${pv_gce_pd}"
|
||||
fi
|
||||
if [[ -n "$_disk_ref" ]]; then
|
||||
IFS='|' read -r _disk_scope _disk_location _disk_name <<< "$_disk_ref"
|
||||
_disk_key="${_disk_scope}|${_disk_location}|${_disk_name}"
|
||||
if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
|
||||
processed_disks+="${_disk_key}|"
|
||||
gke_delete_disk_ref_if_present "$_disk_scope" "$_disk_location" "$_disk_name" || true
|
||||
fi
|
||||
fi
|
||||
done <<< "$stale_pv_records"
|
||||
fi
|
||||
|
||||
if [[ -n "$stale_disk_records" ]]; then
|
||||
while IFS='|' read -r disk_scope disk_location disk_name _disk_type _disk_source; do
|
||||
[[ -n "$disk_name" ]] || continue
|
||||
_disk_key="${disk_scope}|${disk_location}|${disk_name}"
|
||||
if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
|
||||
processed_disks+="${_disk_key}|"
|
||||
gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true
|
||||
fi
|
||||
done <<< "$stale_disk_records"
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
}
|
||||
|
||||
check_gitlab_post_apply_blocked() {
|
||||
# --- Migrations Check ---
|
||||
check_gitlab_migrations_blocked
|
||||
@ -521,11 +788,13 @@ check_gitlab_post_apply_blocked() {
|
||||
local pvc_name="repo-data-gitlab-gitaly-0"
|
||||
local pvc_start=$(date +%s)
|
||||
local pvc_timeout=600
|
||||
local pvc_uid=""
|
||||
while true; do
|
||||
local pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
|
||||
local pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
|
||||
if [[ -n "$pvc_phase" ]]; then
|
||||
pvc_uid=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
|
||||
local _sc_info=""
|
||||
if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-standard-normalized)"; fi
|
||||
if [[ "$pvc_sc" == "premium-rwo" ]]; then _sc_info=" (pd-ssd)"; fi
|
||||
@ -544,8 +813,11 @@ check_gitlab_post_apply_blocked() {
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check for provisioning failures (events)
|
||||
local provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pvc_name",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
|
||||
# Check provisioning failures only for the *current* PVC object.
|
||||
local provisioning_fail=""
|
||||
if [[ -n "$pvc_uid" ]]; then
|
||||
provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
|
||||
fi
|
||||
if [[ -n "$provisioning_fail" ]]; then
|
||||
if [[ "$provisioning_fail" == *"quota"* || "$provisioning_fail" == *"QUOTA"* ]]; then
|
||||
repair_blocked "Gitaly PVC provisioning failed (Quota Exceeded)" \
|
||||
@ -561,7 +833,12 @@ check_gitlab_post_apply_blocked() {
|
||||
fi
|
||||
|
||||
if (( $(date +%s) - pvc_start > pvc_timeout )); then
|
||||
local last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pvc_name" --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
|
||||
local last_msg=""
|
||||
if [[ -n "$pvc_uid" ]]; then
|
||||
last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
|
||||
else
|
||||
last_msg="No live PVC UID observed yet; ignoring stale historical PVC events from previous claims."
|
||||
fi
|
||||
repair_blocked "Gitaly PVC failed to bind after ${pvc_timeout}s" \
|
||||
"PVC: ${pvc_name}. Status: ${pvc_phase:-NotFound}. Last event: ${last_msg}. Fix: Check storage provider and quota."
|
||||
fi
|
||||
@ -2238,6 +2515,8 @@ kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0
|
||||
kubectl delete pv gitlab-gitaly-synology"
|
||||
fi
|
||||
fi
|
||||
|
||||
repair_stale_gke_gitaly_dynamic_storage "${GITALY_STORAGE_CLASS:-}"
|
||||
else
|
||||
setup_gitlab_legacy_storage
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user