mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
chore: improve PVC event handling and enforce GitLab workload replica targets
- Added skipping logic for aged or bound PVC events in `deploy.sh` to reduce noise in diagnostics. - Enforced replica target of 1 for specific GitLab workloads to ensure compliance with requirements.
This commit is contained in:
chrisfu
parent
b9047cd2ad
commit
3651ebfb2d
68
deploy.sh
68
deploy.sh
@ -62,8 +62,9 @@ fi
|
||||
# Post-deploy summary
|
||||
echo ""
|
||||
echo "Public Endpoints:"
|
||||
"${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
|
||||
ROOT_DIR="${ROOT_DIR}" "${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
|
||||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
import configparser
|
||||
@ -171,7 +172,7 @@ def reconcile_gitlab(ctx, ns):
|
||||
|
||||
for dep in deploys:
|
||||
name = dep['metadata']['name']
|
||||
# Filter for operator-managed GitLab workloads only (Requirement 7)
|
||||
# Filter for operator-managed GitLab workloads only
|
||||
is_gitlab = False
|
||||
if name in gitlab_workloads:
|
||||
is_gitlab = True
|
||||
@ -184,8 +185,7 @@ def reconcile_gitlab(ctx, ns):
|
||||
spec = dep.get('spec', {})
|
||||
status = dep.get('status', {})
|
||||
|
||||
# Target should be 1 for these specific GitLab workloads in this environment (Requirement 11)
|
||||
# Reading from spec.replicas can drift from the intended target.
|
||||
# Target should be 1 for these specific GitLab workloads (Requirement 11)
|
||||
if name in gitlab_workloads:
|
||||
desired = 1
|
||||
else:
|
||||
@ -194,60 +194,52 @@ def reconcile_gitlab(ctx, ns):
|
||||
live = status.get('replicas', 0)
|
||||
updated = status.get('updatedReplicas', 0)
|
||||
|
||||
# Identify RS owned by this deployment (Requirement 4)
|
||||
dep_uid = dep['metadata']['uid']
|
||||
my_rss = [rs for rs in all_rss if any(o.get('uid') == dep_uid for o in rs['metadata'].get('ownerReferences', []))]
|
||||
|
||||
current_revision = dep['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision')
|
||||
|
||||
stale_rss_with_pods = []
|
||||
for rs in my_rss:
|
||||
rev = rs['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision')
|
||||
rs_replicas = rs.get('status', {}).get('replicas', 0)
|
||||
if rev != current_revision and rs_replicas > 0:
|
||||
stale_rss_with_pods.append(rs)
|
||||
|
||||
# Forced reconciliation if over-deployed (Requirement 3 & 5)
|
||||
if (live > desired or stale_rss_with_pods) and stale_rss_with_pods:
|
||||
print(f" [FIX] Deployment '{name}' is over-deployed (live={live}, desired={desired}). Scaling stale ReplicaSets to 0...")
|
||||
for rs in stale_rss_with_pods:
|
||||
rs_name = rs['metadata']['name']
|
||||
print(f" - Scaling stale RS '{rs_name}' down from {rs.get('status', {}).get('replicas')} to 0")
|
||||
try:
|
||||
subprocess.run(["kubectl", "--context", ctx, "-n", ns, "scale", "rs", rs_name, "--replicas=0"], check=True)
|
||||
except:
|
||||
print(f" FAILED to scale RS {rs_name}")
|
||||
# Enforce desired replica count (Requirement 3 & 4)
|
||||
# Reconciliation compares:
|
||||
# - desired replicas (from Helm values)
|
||||
# - actual replicas (from Deployment.spec.replicas)
|
||||
actual_spec_replicas = spec.get('replicas', 0)
|
||||
if name in gitlab_workloads and actual_spec_replicas != desired:
|
||||
print(f" [FIX] Deployment '{name}' spec.replicas ({actual_spec_replicas}) differs from desired ({desired}).")
|
||||
over_deployed_detected = True
|
||||
|
||||
summary.append({
|
||||
"name": name,
|
||||
"desired": desired,
|
||||
"live": live,
|
||||
"updated": updated,
|
||||
"stale_rs": [rs['metadata']['name'] for rs in stale_rss_with_pods]
|
||||
"updated": updated
|
||||
})
|
||||
|
||||
# Print summary (Requirement 8)
|
||||
print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8} {'Stale RS'}")
|
||||
print("-" * 85)
|
||||
print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8}")
|
||||
print("-" * 70)
|
||||
for s in summary:
|
||||
stale_str = ",".join(s['stale_rs']) if s['stale_rs'] else "none"
|
||||
print(f"{s['name']:<35} {s['desired']:<8} {s['live']:<8} {s['updated']:<8} {stale_str}")
|
||||
print(f"{s['name']:<35} {s['desired']:<8} {s['live']:<8} {s['updated']:<8}")
|
||||
|
||||
if over_deployed_detected:
|
||||
print("\nEnforcing desired configuration by re-applying GitLab Helm release/CR...")
|
||||
# Re-apply the Helm release (Requirement 2 & 4)
|
||||
try:
|
||||
root_dir = os.environ.get("ROOT_DIR", ".")
|
||||
subprocess.run(["/usr/bin/env", "bash", f"{root_dir}/etc/init_gitlab.sh", "deploy"], check=True)
|
||||
except Exception as e:
|
||||
print(f" ERROR: Failed to re-apply GitLab configuration: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Final Convergence Check (Requirement 6)
|
||||
still_over_deployed = False
|
||||
for s in summary:
|
||||
# If we scaled them down, they SHOULD be 0 now, but let's re-verify from current state
|
||||
# For simplicity, if we detect over-deployment in the initial check, we already know it was bad.
|
||||
# Requirement says "Fail clearly if GitLab IS still over-deployed" after the reconciliation pass.
|
||||
# If we detected drift, we re-applied the configuration.
|
||||
# Now we check if it converged.
|
||||
if over_deployed_detected:
|
||||
# Re-fetch live status for the final check
|
||||
try:
|
||||
out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployment", s['name'], "-o", "json"], text=True)
|
||||
d = json.loads(out)
|
||||
st = d.get('status', {})
|
||||
if st.get('replicas', 0) > d.get('spec', {}).get('replicas', 0) or st.get('updatedReplicas', 0) != d.get('spec', {}).get('replicas', 0):
|
||||
print(f"ERROR: GitLab deployment '{s['name']}' failed to converge.")
|
||||
# Compare against the intended 'desired' count (1), not necessarily the (potentially stale) spec.replicas
|
||||
if st.get('replicas', 0) > s['desired'] or st.get('updatedReplicas', 0) != s['desired']:
|
||||
print(f"ERROR: GitLab deployment '{s['name']}' failed to converge (live={st.get('replicas')}, desired={s['desired']}).")
|
||||
still_over_deployed = True
|
||||
except:
|
||||
pass
|
||||
|
||||
@ -2410,6 +2410,7 @@ cat <<NODE
|
||||
NODE
|
||||
fi)
|
||||
sidekiq:
|
||||
replicaCount: 1
|
||||
concurrency: ${GITLAB_SIDEKIQ_CONCURRENCY}
|
||||
resources:
|
||||
requests:
|
||||
@ -2468,7 +2469,12 @@ ${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
gitlab-exporter:
|
||||
${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
gitlab-shell:
|
||||
replicaCount: 1
|
||||
kas:
|
||||
replicaCount: 1
|
||||
registry:
|
||||
replicaCount: 1
|
||||
storage:
|
||||
secret: gitlab-registry-storage
|
||||
key: config
|
||||
@ -2497,7 +2503,19 @@ if [[ "$gitlab_db_secret_changed" == "1" || "$gitlab_object_storage_secret_chang
|
||||
fi
|
||||
|
||||
gitlab_reconcile_required=0
|
||||
if [[ "$gitlab_exists_before" == "0" || "$operator_changed" == "1" || "$gitlab_chart_version_changed" == "1" || "$gitlab_spec_changed" == "1" || "$gitlab_apply_changed" == "1" || "$gitlab_config_changed" == "1" ]]; then
|
||||
# Check actual replicas to ensure they match desired count (1) - Requirement 3
|
||||
gitlab_replica_drift=0
|
||||
for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
|
||||
_dep_name="${GITLAB_RELEASE}-${_dep_suffix}"
|
||||
_actual=$(kubectl -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")
|
||||
if [[ "$_actual" != "1" ]]; then
|
||||
log "Detected replica drift for ${_dep_name}: actual=${_actual}, desired=1"
|
||||
gitlab_replica_drift=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$gitlab_exists_before" == "0" || "$operator_changed" == "1" || "$gitlab_chart_version_changed" == "1" || "$gitlab_spec_changed" == "1" || "$gitlab_apply_changed" == "1" || "$gitlab_config_changed" == "1" || "$gitlab_replica_drift" == "1" ]]; then
|
||||
gitlab_reconcile_required=1
|
||||
fi
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user