mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
chore: add split-cluster diagnostics and refine workload convergence checks
- Implemented `gitlab_split_cluster_ownership_diagnostics` to enforce and log split-cluster ownership policies between app and DB contexts. - Updated workload convergence logic to prioritize readiness detection over historical restart counts for webservice pods. - Enhanced deployment utilities with additional diagnostics for pod readiness states and replica discrepancies. - Introduced tests to validate split-cluster enforcement and refined convergence behavior.
This commit is contained in:
chrisfu
parent
6532197570
commit
200c6a5e69
@ -81,6 +81,61 @@ enforce_app_cluster_targeting() {
|
||||
export KUBECONTEXT="$app_ctx"
|
||||
}
|
||||
|
||||
gitlab_split_cluster_ownership_diagnostics() {
|
||||
if [[ "$MODE" != "k8s" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
|
||||
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
|
||||
local gitlab_instance="${GITLAB_RELEASE:-gitlab}"
|
||||
|
||||
if [[ -z "$db_ctx" || "$db_ctx" == "$app_ctx" ]]; then
|
||||
log "GitLab ownership policy: single-cluster mode (APP=${app_ctx:-unknown}); operator and app workloads are expected in this context."
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "GitLab split-cluster ownership policy: APP context '${app_ctx}' is authoritative for GitLab operator + app workloads in namespace '${NAMESPACE}'."
|
||||
log "GitLab split-cluster ownership policy: DB context '${db_ctx}' must not host GitLab app workloads (gitaly/webservice/sidekiq/kas/registry/toolbox)."
|
||||
|
||||
local app_gitlab_objects db_gitlab_app_objects db_gitlab_operator_objects
|
||||
app_gitlab_objects=$(kubectl -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
|
||||
-l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
|
||||
db_gitlab_app_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
|
||||
-l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
|
||||
db_gitlab_operator_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deployment -o name 2>/dev/null \
|
||||
| grep -E 'deployment.apps/(gitlab-controller-manager|gitlab-operator|.*gitlab.*controller-manager)' || true)
|
||||
|
||||
if [[ -n "$app_gitlab_objects" ]]; then
|
||||
log "GitLab APP-context resource snapshot (${app_ctx}):"
|
||||
while IFS= read -r _obj; do
|
||||
[[ -n "$_obj" ]] || continue
|
||||
log " - ${_obj}"
|
||||
done <<< "$app_gitlab_objects"
|
||||
else
|
||||
warn "GitLab APP-context snapshot has no resources with app.kubernetes.io/instance=${gitlab_instance} yet (this can be transient during first reconcile)."
|
||||
fi
|
||||
|
||||
if [[ -n "$db_gitlab_operator_objects" ]]; then
|
||||
warn "Detected GitLab operator control-plane resources in DB context '${db_ctx}' (stale/legacy install likely):"
|
||||
while IFS= read -r _obj; do
|
||||
[[ -n "$_obj" ]] || continue
|
||||
warn " - ${_obj}"
|
||||
done <<< "$db_gitlab_operator_objects"
|
||||
warn "DB-context GitLab operator resources are not authoritative for APP GitLab workloads in split-cluster mode."
|
||||
fi
|
||||
|
||||
if [[ -n "$db_gitlab_app_objects" ]]; then
|
||||
local db_app_report=""
|
||||
while IFS= read -r _obj; do
|
||||
[[ -n "$_obj" ]] || continue
|
||||
db_app_report+="- ${_obj}"$'\n'
|
||||
done <<< "$db_gitlab_app_objects"
|
||||
repair_blocked "GitLab split-cluster ownership violation: DB cluster contains GitLab app workloads" \
|
||||
"APP context: ${app_ctx}. DB context: ${db_ctx}. GitLab app resources detected in DB context:\n${db_app_report}Expected policy: GitLab app workloads run only in APP context. Remove stale DB GitLab app workloads and rerun deploy."
|
||||
fi
|
||||
}
|
||||
|
||||
kubectl() {
|
||||
local target_ctx
|
||||
target_ctx="$(resolve_explicit_kube_context || true)"
|
||||
@ -354,7 +409,10 @@ gitlab_webservice_blocked_reasons() {
|
||||
if [[ "$webservice_selector" != *"="* ]]; then
|
||||
webservice_selector="app=${webservice_app}"
|
||||
fi
|
||||
local restart_threshold="$2"
|
||||
local not_ready_block_s="${GITLAB_WEBSERVICE_NOT_READY_BLOCK_SECONDS:-180}"
|
||||
if [[ -z "$not_ready_block_s" || ! "$not_ready_block_s" =~ ^[0-9]+$ ]]; then
|
||||
not_ready_block_s=180
|
||||
fi
|
||||
local pod_names
|
||||
pod_names=$(kubectl -n "$NAMESPACE" get pods -l "$webservice_selector" \
|
||||
--field-selector=status.phase!=Succeeded,status.phase!=Failed \
|
||||
@ -373,39 +431,65 @@ gitlab_webservice_blocked_reasons() {
|
||||
printf '%s\n' "pod ${pod_name} status is ${pod_reason:-$pod_phase}"
|
||||
fi
|
||||
|
||||
local pod_ready_status pod_ready_transition
|
||||
pod_ready_status=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
|
||||
pod_ready_transition=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].lastTransitionTime}' 2>/dev/null || true)
|
||||
|
||||
local container_rows
|
||||
container_rows=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"|"}{.restartCount}{"|"}{.state.waiting.reason}{"|"}{.state.terminated.reason}{"\n"}{end}' 2>/dev/null || true)
|
||||
container_rows=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"|"}{.state.waiting.reason}{"|"}{.state.terminated.reason}{"\n"}{end}' 2>/dev/null || true)
|
||||
[[ -n "$container_rows" ]] || continue
|
||||
|
||||
local container_name restart_count waiting_reason terminated_reason
|
||||
while IFS='|' read -r container_name restart_count waiting_reason terminated_reason; do
|
||||
local container_name waiting_reason terminated_reason
|
||||
while IFS='|' read -r container_name waiting_reason terminated_reason; do
|
||||
[[ -n "$container_name" ]] || continue
|
||||
|
||||
case "$waiting_reason" in
|
||||
CrashLoopBackOff|ImagePullBackOff|Error)
|
||||
CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerError|CreateContainerConfigError|RunContainerError|Error)
|
||||
printf '%s\n' "pod ${pod_name}/${container_name} waiting reason=${waiting_reason}"
|
||||
;;
|
||||
esac
|
||||
case "$terminated_reason" in
|
||||
Error)
|
||||
Error|ContainerCannotRun)
|
||||
printf '%s\n' "pod ${pod_name}/${container_name} terminated reason=${terminated_reason}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ -z "$restart_count" || ! "$restart_count" =~ ^[0-9]+$ ]]; then
|
||||
restart_count=0
|
||||
fi
|
||||
if (( restart_count >= restart_threshold )); then
|
||||
printf '%s\n' "pod ${pod_name}/${container_name} restartCount=${restart_count}"
|
||||
fi
|
||||
done <<< "$container_rows"
|
||||
|
||||
if [[ "$pod_ready_status" != "True" ]]; then
|
||||
local ready_false_age_s=0
|
||||
if [[ -n "$pod_ready_transition" ]]; then
|
||||
ready_false_age_s=$(python3 - "$pod_ready_transition" <<'PY'
|
||||
import datetime
|
||||
import sys
|
||||
|
||||
raw = (sys.argv[1] if len(sys.argv) > 1 else "").strip()
|
||||
if not raw:
|
||||
print("0")
|
||||
raise SystemExit(0)
|
||||
|
||||
try:
|
||||
ts = datetime.datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
age = int((now - ts).total_seconds())
|
||||
print(str(max(age, 0)))
|
||||
except Exception:
|
||||
print("0")
|
||||
PY
|
||||
)
|
||||
fi
|
||||
if [[ -z "$ready_false_age_s" || ! "$ready_false_age_s" =~ ^[0-9]+$ ]]; then
|
||||
ready_false_age_s=0
|
||||
fi
|
||||
if (( ready_false_age_s >= not_ready_block_s )); then
|
||||
printf '%s\n' "pod ${pod_name} Ready=False for ${ready_false_age_s}s (threshold=${not_ready_block_s}s)"
|
||||
fi
|
||||
fi
|
||||
done <<< "$pod_names"
|
||||
}
|
||||
|
||||
wait_for_gitlab_workload_convergence() {
|
||||
local timeout_s="${GITLAB_WORKLOAD_CONVERGENCE_TIMEOUT:-420}"
|
||||
local poll_interval_s="${GITLAB_WORKLOAD_CONVERGENCE_POLL_INTERVAL:-10}"
|
||||
local restart_threshold="${GITLAB_WEBSERVICE_RESTART_BLOCK_THRESHOLD:-3}"
|
||||
|
||||
if [[ -z "$timeout_s" || ! "$timeout_s" =~ ^[0-9]+$ ]]; then
|
||||
timeout_s=420
|
||||
@ -413,9 +497,6 @@ wait_for_gitlab_workload_convergence() {
|
||||
if [[ -z "$poll_interval_s" || ! "$poll_interval_s" =~ ^[0-9]+$ || "$poll_interval_s" == "0" ]]; then
|
||||
poll_interval_s=10
|
||||
fi
|
||||
if [[ -z "$restart_threshold" || ! "$restart_threshold" =~ ^[0-9]+$ ]]; then
|
||||
restart_threshold=3
|
||||
fi
|
||||
|
||||
local start_ts
|
||||
start_ts=$(date +%s)
|
||||
@ -424,6 +505,7 @@ wait_for_gitlab_workload_convergence() {
|
||||
while true; do
|
||||
local -a blockers=()
|
||||
local dep_suffix dep_name dep_selector desired_replicas live_non_terminal old_rs_summary
|
||||
local ready_replicas available_replicas
|
||||
for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
|
||||
dep_name="${GITLAB_RELEASE}-${dep_suffix}"
|
||||
|
||||
@ -449,6 +531,21 @@ wait_for_gitlab_workload_convergence() {
|
||||
blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} < desired=${desired_replicas} (selector=${dep_selector})")
|
||||
fi
|
||||
|
||||
ready_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
|
||||
available_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
|
||||
if [[ -z "$ready_replicas" || ! "$ready_replicas" =~ ^[0-9]+$ ]]; then
|
||||
ready_replicas=0
|
||||
fi
|
||||
if [[ -z "$available_replicas" || ! "$available_replicas" =~ ^[0-9]+$ ]]; then
|
||||
available_replicas=0
|
||||
fi
|
||||
if (( ready_replicas < desired_replicas )); then
|
||||
blockers+=("${dep_name}: readyReplicas=${ready_replicas} < desired=${desired_replicas}")
|
||||
fi
|
||||
if (( available_replicas < desired_replicas )); then
|
||||
blockers+=("${dep_name}: availableReplicas=${available_replicas} < desired=${desired_replicas}")
|
||||
fi
|
||||
|
||||
old_rs_summary=$(gitlab_old_replicaset_live_summary "$dep_name")
|
||||
if [[ -n "$old_rs_summary" ]]; then
|
||||
blockers+=("${dep_name}: old ReplicaSet pods still running (${old_rs_summary})")
|
||||
@ -456,7 +553,7 @@ wait_for_gitlab_workload_convergence() {
|
||||
|
||||
if [[ "$dep_suffix" == "webservice-default" ]]; then
|
||||
local webservice_reasons
|
||||
webservice_reasons=$(gitlab_webservice_blocked_reasons "$dep_selector" "$restart_threshold" 2>/dev/null || true)
|
||||
webservice_reasons=$(gitlab_webservice_blocked_reasons "$dep_selector" 2>/dev/null || true)
|
||||
if [[ -n "$webservice_reasons" ]]; then
|
||||
local webservice_reason
|
||||
while IFS= read -r webservice_reason; do
|
||||
@ -2397,6 +2494,8 @@ is_unresolved_secret_ref() {
|
||||
}
|
||||
GITLAB_RELEASE="gitlab"
|
||||
|
||||
gitlab_split_cluster_ownership_diagnostics
|
||||
|
||||
assert_public_ingress_targeting "$GITLAB_INGRESS_CLASS" "${GITLAB_PUBLIC_HOSTS[@]}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@ -681,6 +681,28 @@ def test_init_gitlab_renders_non_empty_trusted_proxies_for_forwarded_headers():
|
||||
assert "trusted_proxies:${GITLAB_TRUSTED_PROXIES_YAML}" in script
|
||||
|
||||
|
||||
def test_init_gitlab_split_cluster_ownership_is_explicit_and_db_app_workloads_blocked():
|
||||
"""Split-cluster GitLab ownership must be explicit: APP owns app workloads; DB app workloads are blocked."""
|
||||
script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "gitlab_split_cluster_ownership_diagnostics()" in script
|
||||
assert "GitLab split-cluster ownership policy: APP context" in script
|
||||
assert "DB context '${db_ctx}' must not host GitLab app workloads" in script
|
||||
assert "GitLab split-cluster ownership violation: DB cluster contains GitLab app workloads" in script
|
||||
assert "gitlab_split_cluster_ownership_diagnostics" in script
|
||||
|
||||
|
||||
def test_init_gitlab_convergence_no_longer_blocks_on_historical_restart_count_only():
|
||||
"""Convergence gate must not fail solely on historical restartCount for currently healthy webservice pods."""
|
||||
script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "GITLAB_WEBSERVICE_NOT_READY_BLOCK_SECONDS" in script
|
||||
assert "readyReplicas=" in script
|
||||
assert "availableReplicas=" in script
|
||||
assert "restartCount=" not in script
|
||||
assert "gitlab_webservice_blocked_reasons \"$dep_selector\" \"$restart_threshold\"" not in script
|
||||
|
||||
|
||||
def test_init_gitlab_old_replicaset_summary_helper_is_non_fatal_when_empty():
|
||||
"""ReplicaSet summary helper should not return non-zero on healthy empty summary under set -e."""
|
||||
script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user