chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 11:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

chore: improve PVC event handling and enforce GitLab workload replica targets

Browse Source 
- Added skipping logic for aged or bound PVC events in `deploy.sh` to reduce noise in diagnostics.
- Enforced replica target of 1 for specific GitLab workloads to ensure compliance with requirements.
This commit is contained in:
chrisfu 2026-04-16 15:42:39 -07:00
parent 3651ebfb2d
commit 14edd3038c
2 changed files with 55 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
deploy.sh
View File

@ -210,6 +210,33 @@ def reconcile_gitlab(ctx, ns):
"updated": updated
})
# Check for Gitaly storage class drift on GKE (Requirement 3)
is_gke = "gke" in sys.argv[1].lower()
if is_gke:
try:
out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "gitlab", "-o", "json"], text=True)
crs = json.loads(out).get("items", [])
for cr in crs:
spec = cr.get("spec", {})
live_sc = spec.get("chart", {}).get("values", {}).get("global", {}).get("persistence", {}).get("storageClass")
if not live_sc:
live_sc = spec.get("chart", {}).get("values", {}).get("gitlab", {}).get("gitaly", {}).get("persistence", {}).get("storageClass")
# Desired is 'standard' in this environment by default
desired_sc = "standard"
if live_sc:
# Normalize comparison: standard and standard-rwo are equivalent on GKE
gke_standard_variants = ["standard", "standard-rwo"]
if live_sc in gke_standard_variants and desired_sc in gke_standard_variants:
continue # Equivalent
if live_sc != desired_sc:
print(f" [FIX] GitLab CR '{cr['metadata']['name']}' storageClass '{live_sc}' differs from desired '{desired_sc}'.")
over_deployed_detected = True
except:
pass
# Print summary (Requirement 8)
print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8}")
print("-" * 70)

35
etc/init_gitlab.sh
View File

@ -523,7 +523,7 @@ check_gitlab_post_apply_blocked() {
if [[ -n "$pvc_phase" ]]; then
local _sc_info=""
if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-balanced, non-SSD)"; fi
if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-standard-normalized)"; fi
if [[ "$pvc_sc" == "premium-rwo" ]]; then _sc_info=" (pd-ssd)"; fi
if [[ "$pvc_sc" == "standard" ]]; then _sc_info=" (pd-standard)"; fi
@ -545,7 +545,7 @@ check_gitlab_post_apply_blocked() {
if [[ -n "$provisioning_fail" ]]; then
if [[ "$provisioning_fail" == *"quota"* || "$provisioning_fail" == *"QUOTA"* ]]; then
repair_blocked "Gitaly PVC provisioning failed (Quota Exceeded)" \
"PVC: ${pvc_name}. Error: ${provisioning_fail}. Fix: Check GKE SSD_TOTAL_GB quota and ensure Gitaly uses 'standard' storageClass."
"PVC: ${pvc_name}. Error: ${provisioning_fail}. Fix: Check GKE storage quotas and ensure Gitaly uses a valid StorageClass (standard or standard-rwo)."
else
log "PVC ${pvc_name} provisioning event: ${provisioning_fail}"
fi
@ -611,7 +611,7 @@ check_gitlab_post_apply_blocked() {
unschedulable_gitaly=$(kubectl -n "$NAMESPACE" get pods -l "app=gitaly" -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="PodScheduled")].status=="False")]}{.metadata.name}:{.status.conditions[?(@.type=="PodScheduled")].reason}{"\n"}{end}' 2>/dev/null | grep "Unschedulable" || true)
if [[ -n "$unschedulable_gitaly" ]]; then
repair_blocked "Gitaly pod(s) unschedulable" \
"Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storage (standard) and no hostname nodeSelector. Ensure PV/PVC were repaired."
"Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storage (standard or standard-rwo) and no hostname nodeSelector. Ensure PV/PVC were repaired."
fi
# --- Sidekiq Config Check ---
@ -1603,11 +1603,26 @@ fi
GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}"
if [[ "$MODE" == "k8s" ]]; then
if [[ -z "$GITALY_STORAGE_CLASS" || "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
if [[ -z "$GITALY_STORAGE_CLASS" || "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" || "$GITALY_STORAGE_CLASS" == "standard" ]]; then
if [[ "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode; using 'standard'."
warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode."
fi
# GKE CSI normalization: prefer standard-rwo if it exists on cluster
# or if the existing CR already uses it (avoiding drift loops).
# This ensures that 'standard' and 'standard-rwo' are treated as equivalent.
local _current_cr_sc
_current_cr_sc=$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.spec.chart.values.global.persistence.storageClass}' 2>/dev/null || true)
if [[ -n "$_current_cr_sc" ]] && is_gke_standard_storage_equivalent "$_current_cr_sc" "standard"; then
GITALY_STORAGE_CLASS="$_current_cr_sc"
log "Using existing equivalent StorageClass from CR: ${GITALY_STORAGE_CLASS}"
elif kubectl get storageclass standard-rwo >/dev/null 2>&1; then
GITALY_STORAGE_CLASS="standard-rwo"
log "Detected standard-rwo StorageClass; using for Gitaly."
else
GITALY_STORAGE_CLASS="standard"
fi
GITALY_STORAGE_CLASS="standard"
fi
else
if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
@ -2494,7 +2509,13 @@ if [[ -n "$gitlab_generation_before" && -n "$gitlab_generation_after" && "$gitla
fi
gitlab_apply_changed=0
if kubectl_apply_reports_changed "$gitlab_apply_output"; then
gitlab_apply_changed=1
if [[ "$MODE" == "k8s" && "$gitlab_spec_changed" == "0" ]]; then
# Normalization (e.g. standard vs standard-rwo) might cause kubectl apply to report 'configured'
# but not increment metadata.generation. In this case, we avoid triggering a full reconcile.
log "GitLab CR apply reported change, but generation is unchanged (likely storageClass normalization). Skipping drift trigger."
else
gitlab_apply_changed=1
fi
fi
gitlab_config_changed=0