diff --git a/deploy.sh b/deploy.sh index d302759..8b6d2de 100755 --- a/deploy.sh +++ b/deploy.sh @@ -62,8 +62,9 @@ fi # Post-deploy summary echo "" echo "Public Endpoints:" -"${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY' +ROOT_DIR="${ROOT_DIR}" "${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY' import sys +import os import subprocess import json import configparser @@ -171,7 +172,7 @@ def reconcile_gitlab(ctx, ns): for dep in deploys: name = dep['metadata']['name'] - # Filter for operator-managed GitLab workloads only (Requirement 7) + # Filter for operator-managed GitLab workloads only is_gitlab = False if name in gitlab_workloads: is_gitlab = True @@ -184,8 +185,7 @@ def reconcile_gitlab(ctx, ns): spec = dep.get('spec', {}) status = dep.get('status', {}) - # Target should be 1 for these specific GitLab workloads in this environment (Requirement 11) - # Reading from spec.replicas can drift from the intended target. + # Target should be 1 for these specific GitLab workloads (Requirement 11) if name in gitlab_workloads: desired = 1 else: @@ -194,60 +194,52 @@ def reconcile_gitlab(ctx, ns): live = status.get('replicas', 0) updated = status.get('updatedReplicas', 0) - # Identify RS owned by this deployment (Requirement 4) - dep_uid = dep['metadata']['uid'] - my_rss = [rs for rs in all_rss if any(o.get('uid') == dep_uid for o in rs['metadata'].get('ownerReferences', []))] - - current_revision = dep['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision') - - stale_rss_with_pods = [] - for rs in my_rss: - rev = rs['metadata'].get('annotations', {}).get('deployment.kubernetes.io/revision') - rs_replicas = rs.get('status', {}).get('replicas', 0) - if rev != current_revision and rs_replicas > 0: - stale_rss_with_pods.append(rs) - - # Forced reconciliation if over-deployed (Requirement 3 & 5) - if (live > desired or stale_rss_with_pods) and stale_rss_with_pods: - print(f" [FIX] Deployment '{name}' is over-deployed (live={live}, desired={desired}). Scaling stale ReplicaSets to 0...") - for rs in stale_rss_with_pods: - rs_name = rs['metadata']['name'] - print(f" - Scaling stale RS '{rs_name}' down from {rs.get('status', {}).get('replicas')} to 0") - try: - subprocess.run(["kubectl", "--context", ctx, "-n", ns, "scale", "rs", rs_name, "--replicas=0"], check=True) - except: - print(f" FAILED to scale RS {rs_name}") + # Enforce desired replica count (Requirement 3 & 4) + # Reconciliation compares: + # - desired replicas (from Helm values) + # - actual replicas (from Deployment.spec.replicas) + actual_spec_replicas = spec.get('replicas', 0) + if name in gitlab_workloads and actual_spec_replicas != desired: + print(f" [FIX] Deployment '{name}' spec.replicas ({actual_spec_replicas}) differs from desired ({desired}).") over_deployed_detected = True summary.append({ "name": name, "desired": desired, "live": live, - "updated": updated, - "stale_rs": [rs['metadata']['name'] for rs in stale_rss_with_pods] + "updated": updated }) # Print summary (Requirement 8) - print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8} {'Stale RS'}") - print("-" * 85) + print(f"\n{'GitLab Deployment':<35} {'Desired':<8} {'Live':<8} {'Updated':<8}") + print("-" * 70) for s in summary: - stale_str = ",".join(s['stale_rs']) if s['stale_rs'] else "none" - print(f"{s['name']:<35} {s['desired']:<8} {s['live']:<8} {s['updated']:<8} {stale_str}") + print(f"{s['name']:<35} {s['desired']:<8} {s['live']:<8} {s['updated']:<8}") + + if over_deployed_detected: + print("\nEnforcing desired configuration by re-applying GitLab Helm release/CR...") + # Re-apply the Helm release (Requirement 2 & 4) + try: + root_dir = os.environ.get("ROOT_DIR", ".") + subprocess.run(["/usr/bin/env", "bash", f"{root_dir}/etc/init_gitlab.sh", "deploy"], check=True) + except Exception as e: + print(f" ERROR: Failed to re-apply GitLab configuration: {e}") + sys.exit(1) # Final Convergence Check (Requirement 6) still_over_deployed = False for s in summary: - # If we scaled them down, they SHOULD be 0 now, but let's re-verify from current state - # For simplicity, if we detect over-deployment in the initial check, we already know it was bad. - # Requirement says "Fail clearly if GitLab IS still over-deployed" after the reconciliation pass. + # If we detected drift, we re-applied the configuration. + # Now we check if it converged. if over_deployed_detected: # Re-fetch live status for the final check try: out = subprocess.check_output(["kubectl", "--context", ctx, "-n", ns, "get", "deployment", s['name'], "-o", "json"], text=True) d = json.loads(out) st = d.get('status', {}) - if st.get('replicas', 0) > d.get('spec', {}).get('replicas', 0) or st.get('updatedReplicas', 0) != d.get('spec', {}).get('replicas', 0): - print(f"ERROR: GitLab deployment '{s['name']}' failed to converge.") + # Compare against the intended 'desired' count (1), not necessarily the (potentially stale) spec.replicas + if st.get('replicas', 0) > s['desired'] or st.get('updatedReplicas', 0) != s['desired']: + print(f"ERROR: GitLab deployment '{s['name']}' failed to converge (live={st.get('replicas')}, desired={s['desired']}).") still_over_deployed = True except: pass diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh index c387453..7b2c320 100755 --- a/etc/init_gitlab.sh +++ b/etc/init_gitlab.sh @@ -2410,6 +2410,7 @@ cat </dev/null || echo "1") + if [[ "$_actual" != "1" ]]; then + log "Detected replica drift for ${_dep_name}: actual=${_actual}, desired=1" + gitlab_replica_drift=1 + break + fi +done + +if [[ "$gitlab_exists_before" == "0" || "$operator_changed" == "1" || "$gitlab_chart_version_changed" == "1" || "$gitlab_spec_changed" == "1" || "$gitlab_apply_changed" == "1" || "$gitlab_config_changed" == "1" || "$gitlab_replica_drift" == "1" ]]; then gitlab_reconcile_required=1 fi