mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
chore: add diagnostics for rendered CR replica fields and registry-specific key sanitization
- Introduced `gitlab_rendered_replica_source_fields_from_cr` to extract and log replica fields from rendered CR values. - Enhanced logging for replica source-of-truth mismatches, including detailed diagnostics of rendered CR fields. - Updated registry-specific key sanitization to align with chart 9+ authoritative fields. - Added and extended tests to validate replica field rendering, sanitization, and logging enhancements.
This commit is contained in:
chrisfu
parent
e53771af7b
commit
5f701026c6
@ -246,10 +246,70 @@ gitlab_replica_source_of_truth_report() {
|
||||
printf '%s' "$report"
|
||||
}
|
||||
|
||||
gitlab_rendered_replica_source_fields_from_cr() {
|
||||
local desired_cr="$1"
|
||||
[[ -n "$desired_cr" ]] || return 0
|
||||
|
||||
echo "$desired_cr" | awk '
|
||||
BEGIN {
|
||||
in_values = 0
|
||||
component = ""
|
||||
}
|
||||
{
|
||||
line = $0
|
||||
if (line ~ /^ values:[[:space:]]*$/) {
|
||||
in_values = 1
|
||||
next
|
||||
}
|
||||
if (!in_values) {
|
||||
next
|
||||
}
|
||||
|
||||
if (line ~ /^ webservice:[[:space:]]*$/) {
|
||||
component = "gitlab.webservice"
|
||||
next
|
||||
}
|
||||
if (line ~ /^ sidekiq:[[:space:]]*$/) {
|
||||
component = "gitlab.sidekiq"
|
||||
next
|
||||
}
|
||||
if (line ~ /^ gitlab-shell:[[:space:]]*$/) {
|
||||
component = "gitlab.gitlab-shell"
|
||||
next
|
||||
}
|
||||
if (line ~ /^ kas:[[:space:]]*$/) {
|
||||
component = "gitlab.kas"
|
||||
next
|
||||
}
|
||||
if (line ~ /^ registry:[[:space:]]*$/) {
|
||||
component = "registry"
|
||||
next
|
||||
}
|
||||
|
||||
if (line ~ /^ gitaly:[[:space:]]*$/ || line ~ /^ postgresql:[[:space:]]*$/ || line ~ /^ redis:[[:space:]]*$/) {
|
||||
component = ""
|
||||
}
|
||||
if (line ~ /^ gitaly:[[:space:]]*$/ || line ~ /^ toolbox:[[:space:]]*$/) {
|
||||
component = ""
|
||||
}
|
||||
|
||||
if (component != "" && line ~ /^[[:space:]]*(replicaCount|minReplicas|maxReplicas|hpa):[[:space:]]*/) {
|
||||
sub(/^[[:space:]]+/, "", line)
|
||||
print "- " component "." line
|
||||
}
|
||||
}'
|
||||
}
|
||||
|
||||
gitlab_verify_replica_source_of_truth() {
|
||||
local target_replicas="$1"
|
||||
local verify_timeout_s="${GITLAB_SOURCE_REPLICA_VERIFY_TIMEOUT:-180}"
|
||||
local verify_poll_interval_s="${GITLAB_SOURCE_REPLICA_VERIFY_POLL_INTERVAL:-10}"
|
||||
local rendered_replica_fields
|
||||
|
||||
rendered_replica_fields="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")"
|
||||
if [[ -z "$rendered_replica_fields" ]]; then
|
||||
rendered_replica_fields="- (no replica-related fields detected in rendered CR values)"
|
||||
fi
|
||||
|
||||
if [[ -z "$verify_timeout_s" || ! "$verify_timeout_s" =~ ^[0-9]+$ || "$verify_timeout_s" == "0" ]]; then
|
||||
verify_timeout_s=180
|
||||
@ -270,7 +330,7 @@ gitlab_verify_replica_source_of_truth() {
|
||||
verify_now_ts=$(date +%s)
|
||||
if (( verify_now_ts - verify_start_ts >= verify_timeout_s )); then
|
||||
repair_blocked "GitLab operator desired replica source-of-truth mismatch" \
|
||||
"Operator-managed Deployment specs did not converge to desired=${target_replicas} after GitLab CR apply:\n${mismatch_report}This is a source-of-truth issue (CR values still resolve to replicas>1), not a rollout lag issue."
|
||||
"Operator-managed Deployment specs did not converge to desired=${target_replicas} after GitLab CR apply:\n${mismatch_report}Rendered GitLab CR replica source fields:\n${rendered_replica_fields}\nThis is a source-of-truth issue (CR values still resolve to replicas>1), not a rollout lag issue."
|
||||
fi
|
||||
|
||||
log "Waiting for GitLab operator to apply source-of-truth replicas (desired=${target_replicas}) before settle verification..."
|
||||
@ -755,6 +815,8 @@ function leading_spaces(str, i, c, n) {
|
||||
}
|
||||
BEGIN {
|
||||
in_values = 0
|
||||
in_registry = 0
|
||||
registry_indent = -1
|
||||
in_hpa = 0
|
||||
hpa_indent = -1
|
||||
}
|
||||
@ -773,6 +835,19 @@ BEGIN {
|
||||
in_hpa = 0
|
||||
hpa_indent = -1
|
||||
}
|
||||
if (in_registry && indent <= registry_indent && line !~ /^ registry:[[:space:]]*$/) {
|
||||
in_registry = 0
|
||||
registry_indent = -1
|
||||
}
|
||||
|
||||
if (line ~ /^ registry:[[:space:]]*$/) {
|
||||
in_registry = 1
|
||||
registry_indent = indent
|
||||
next
|
||||
}
|
||||
if (!in_registry) {
|
||||
next
|
||||
}
|
||||
|
||||
if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) {
|
||||
in_hpa = 1
|
||||
@ -808,6 +883,8 @@ function leading_spaces(str, i, c, n) {
|
||||
}
|
||||
BEGIN {
|
||||
in_values = 0
|
||||
in_registry = 0
|
||||
registry_indent = -1
|
||||
in_hpa = 0
|
||||
hpa_indent = -1
|
||||
}
|
||||
@ -828,6 +905,21 @@ BEGIN {
|
||||
in_hpa = 0
|
||||
hpa_indent = -1
|
||||
}
|
||||
if (in_registry && indent <= registry_indent && line !~ /^ registry:[[:space:]]*$/) {
|
||||
in_registry = 0
|
||||
registry_indent = -1
|
||||
}
|
||||
|
||||
if (line ~ /^ registry:[[:space:]]*$/) {
|
||||
in_registry = 1
|
||||
registry_indent = indent
|
||||
print line
|
||||
next
|
||||
}
|
||||
if (!in_registry) {
|
||||
print line
|
||||
next
|
||||
}
|
||||
|
||||
if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) {
|
||||
in_hpa = 1
|
||||
@ -3571,6 +3663,8 @@ fi)
|
||||
enabled: true
|
||||
annotations:
|
||||
kubernetes.io/ingress.class: kong
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
|
||||
cat <<NODE
|
||||
@ -3580,9 +3674,8 @@ NODE
|
||||
fi)
|
||||
sidekiq:
|
||||
replicaCount: 1
|
||||
hpa:
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
concurrency: ${GITLAB_SIDEKIQ_CONCURRENCY}
|
||||
resources:
|
||||
requests:
|
||||
@ -3643,14 +3736,12 @@ ${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
${WORKLOAD_JEMALLOC_VALUES_YAML}
|
||||
gitlab-shell:
|
||||
replicaCount: 1
|
||||
hpa:
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
kas:
|
||||
replicaCount: 1
|
||||
hpa:
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
gitaly:
|
||||
persistence:
|
||||
storageClass: ${GITALY_STORAGE_CLASS}
|
||||
@ -3833,8 +3924,12 @@ if [[ "$_replica_drift_found" == "1" ]]; then
|
||||
done
|
||||
|
||||
if [[ -n "$_source_of_truth_mismatch_report" ]]; then
|
||||
_rendered_replica_fields_report="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")"
|
||||
if [[ -z "$_rendered_replica_fields_report" ]]; then
|
||||
_rendered_replica_fields_report="- (no replica-related fields detected in rendered CR values)"
|
||||
fi
|
||||
repair_blocked "GitLab operator desired replica source-of-truth mismatch after CR re-apply" \
|
||||
"Deployment specs still do not match desired=${_gitlab_replica_target}:\n${_source_of_truth_mismatch_report}This indicates CR source-of-truth still resolves to replicas>1."
|
||||
"Deployment specs still do not match desired=${_gitlab_replica_target}:\n${_source_of_truth_mismatch_report}Rendered GitLab CR replica source fields:\n${_rendered_replica_fields_report}\nThis indicates CR source-of-truth still resolves to replicas>1."
|
||||
fi
|
||||
|
||||
log "Waiting for rollout of reconciled deployments..."
|
||||
|
||||
@ -654,13 +654,14 @@ def test_init_gitlab_blocks_wrong_live_gitaly_claim_template_storageclass():
|
||||
|
||||
|
||||
def test_init_gitlab_sanitizes_removed_chart9_top_level_replica_keys_before_apply():
|
||||
"""GitLab chart v9 path must sanitize deprecated top-level min/max replica keys before CR apply."""
|
||||
"""GitLab chart v9 path must sanitize only removed top-level registry min/max replica keys before CR apply."""
|
||||
script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "gitlab_chart9_find_deprecated_top_level_replica_keys" in script
|
||||
assert "gitlab_chart9_strip_deprecated_top_level_replica_keys" in script
|
||||
assert "sanitize_gitlab_cr_rendered_values_for_chart_version" in script
|
||||
assert "preflight_validate_gitlab_cr_rendered_values" in script
|
||||
assert 'if (line ~ /^ registry:[[:space:]]*$/) {' in script
|
||||
assert 'GITLAB_CR_RENDERED="$(sanitize_gitlab_cr_rendered_values_for_chart_version "$GITLAB_CR_RENDERED")"' in script
|
||||
assert 'preflight_validate_gitlab_cr_rendered_values "$GITLAB_CR_RENDERED"' in script
|
||||
|
||||
@ -676,6 +677,34 @@ def test_init_gitlab_rendered_registry_uses_hpa_replica_fields_without_legacy_to
|
||||
assert " minReplicas: 1\n maxReplicas: 1\n hpa:" not in script
|
||||
|
||||
|
||||
def test_init_gitlab_rendered_shell_kas_sidekiq_webservice_use_chart9_authoritative_replica_fields():
|
||||
"""Chart 9.10.3 authoritative replica fields must be rendered per-component for operator-managed GitLab workloads."""
|
||||
script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert " webservice:" in script
|
||||
assert " replicaCount: 1" in script
|
||||
assert " minReplicas: 1" in script
|
||||
assert " maxReplicas: 1" in script
|
||||
|
||||
assert " sidekiq:" in script
|
||||
assert " minReplicas: 1" in script
|
||||
assert " maxReplicas: 1" in script
|
||||
assert " sidekiq:\n replicaCount: 1\n hpa:" not in script
|
||||
|
||||
assert " gitlab-shell:" in script
|
||||
assert " kas:" in script
|
||||
assert " gitlab-shell:\n replicaCount: 1\n hpa:" not in script
|
||||
assert " kas:\n replicaCount: 1\n hpa:" not in script
|
||||
|
||||
|
||||
def test_init_gitlab_replica_source_of_truth_failure_logs_rendered_cr_replica_fields():
|
||||
"""Replica source-of-truth mismatch failures should print rendered CR replica field diagnostics."""
|
||||
script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "gitlab_rendered_replica_source_fields_from_cr" in script
|
||||
assert "Rendered GitLab CR replica source fields:" in script
|
||||
|
||||
|
||||
def test_gitlab_and_gitea_init_storage_node_defaults_are_config_driven():
|
||||
"""GitLab/Gitea init scripts must not hardcode physical host defaults for storage node pinning."""
|
||||
gitlab_script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user