chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 12:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
200c6a5e69
prole/etc/init_gitlab.sh
chrisfu 200c6a5e69 chore: add split-cluster diagnostics and refine workload convergence checks 
- Implemented `gitlab_split_cluster_ownership_diagnostics` to enforce and log split-cluster ownership policies between app and DB contexts.
- Updated workload convergence logic to prioritize readiness detection over historical restart counts for webservice pods.
- Enhanced deployment utilities with additional diagnostics for pod readiness states and replica discrepancies.
- Introduced tests to validate split-cluster enforcement and refined convergence behavior.
2026-04-19 05:32:22 -07:00

4451 lines
185 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env bash
set -euo pipefail
PROG="init_gitlab"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"
MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")"
# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden.
# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps).
NAMESPACE="${GITLAB_NAMESPACE:-gitlab}"
CFG_PATH=""
FORCE=0
# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE).
# Setting this would pin ALL global components to one node, exhausting RAM.
NODE_SELECTOR=""
usage() {
cat <<EOF
Usage:
$PROG [options] [deploy]
Options:
--mode <k3d|k3s|k8s|local> Deployment mode (default: ${MODE:-k3d})
-n, --namespace <name> Target namespace (default: gitlab)
-c, --config <config.cfg> Path to config file (defaults to detected)
--node-selector <node> Node to pin all GitLab workloads (optional)
--force Remove existing GitLab and Gitea releases before deploy
--help Show this help
Behavior:
- Removes any pre-existing GitLab-domain Gitea configurations (helm release,
namespace, Kong routes) when --force is specified or when gitea is detected.
- Installs the GitLab Operator via Helm into the gitlab namespace.
- Provisions a GitLab CR that uses the knoe-db CloudNativePG cluster as its
external PostgreSQL data store.
- Configures the GitLab public hostname as Kubernetes ingress pointing to GitLab.
- Legacy/local modes pin gitaly storage to GITLAB_STORAGE_NODE (required).
EOF
}
die() { echo "[ERROR] $*" >&2; exit 2; }
log() { echo "[INFO] $*" >&2; }
warn() { echo "[WARN] $*" >&2; }
resolve_explicit_kube_context() {
local ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
if [[ -n "$ctx" ]]; then
printf '%s' "$ctx"
return 0
fi
return 1
}
enforce_app_cluster_targeting() {
if [[ "$MODE" != "k8s" ]]; then
return 0
fi
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
local target_ctx
target_ctx="$(resolve_explicit_kube_context || true)"
log "Reconciling GitLab with APP context: ${app_ctx:-"(unset)"}"
[[ -n "$db_ctx" ]] && log "Reconciling GitLab with DB context: ${db_ctx}"
[[ -n "$app_ctx" ]] || die "APP_CLUSTER_KUBECONTEXT is required for k8s GitLab deployment."
[[ -n "$target_ctx" ]] || die "Explicit kubectl context is required for k8s GitLab deployment."
if [[ -n "$db_ctx" && "$target_ctx" == "$db_ctx" ]]; then
die "Refusing GitLab APP step against DB context '$target_ctx'."
fi
if [[ "$target_ctx" != "$app_ctx" ]]; then
die "GitLab APP step must target APP_CLUSTER_KUBECONTEXT='${app_ctx}' (got '${target_ctx}')."
fi
export KUBECTL_CONTEXT="$app_ctx"
export KUBE_CONTEXT_NAME="$app_ctx"
export KUBECONTEXT="$app_ctx"
}
gitlab_split_cluster_ownership_diagnostics() {
if [[ "$MODE" != "k8s" ]]; then
return 0
fi
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
local gitlab_instance="${GITLAB_RELEASE:-gitlab}"
if [[ -z "$db_ctx" || "$db_ctx" == "$app_ctx" ]]; then
log "GitLab ownership policy: single-cluster mode (APP=${app_ctx:-unknown}); operator and app workloads are expected in this context."
return 0
fi
log "GitLab split-cluster ownership policy: APP context '${app_ctx}' is authoritative for GitLab operator + app workloads in namespace '${NAMESPACE}'."
log "GitLab split-cluster ownership policy: DB context '${db_ctx}' must not host GitLab app workloads (gitaly/webservice/sidekiq/kas/registry/toolbox)."
local app_gitlab_objects db_gitlab_app_objects db_gitlab_operator_objects
app_gitlab_objects=$(kubectl -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
-l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
db_gitlab_app_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deploy,statefulset,job,cronjob \
-l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true)
db_gitlab_operator_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deployment -o name 2>/dev/null \
| grep -E 'deployment.apps/(gitlab-controller-manager|gitlab-operator|.*gitlab.*controller-manager)' || true)
if [[ -n "$app_gitlab_objects" ]]; then
log "GitLab APP-context resource snapshot (${app_ctx}):"
while IFS= read -r _obj; do
[[ -n "$_obj" ]] || continue
log " - ${_obj}"
done <<< "$app_gitlab_objects"
else
warn "GitLab APP-context snapshot has no resources with app.kubernetes.io/instance=${gitlab_instance} yet (this can be transient during first reconcile)."
fi
if [[ -n "$db_gitlab_operator_objects" ]]; then
warn "Detected GitLab operator control-plane resources in DB context '${db_ctx}' (stale/legacy install likely):"
while IFS= read -r _obj; do
[[ -n "$_obj" ]] || continue
warn " - ${_obj}"
done <<< "$db_gitlab_operator_objects"
warn "DB-context GitLab operator resources are not authoritative for APP GitLab workloads in split-cluster mode."
fi
if [[ -n "$db_gitlab_app_objects" ]]; then
local db_app_report=""
while IFS= read -r _obj; do
[[ -n "$_obj" ]] || continue
db_app_report+="- ${_obj}"$'\n'
done <<< "$db_gitlab_app_objects"
repair_blocked "GitLab split-cluster ownership violation: DB cluster contains GitLab app workloads" \
"APP context: ${app_ctx}. DB context: ${db_ctx}. GitLab app resources detected in DB context:\n${db_app_report}Expected policy: GitLab app workloads run only in APP context. Remove stale DB GitLab app workloads and rerun deploy."
fi
}
kubectl() {
local target_ctx
target_ctx="$(resolve_explicit_kube_context || true)"
if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then
die "Explicit kubectl context is required in k8s mode."
fi
local arg has_context=0
for arg in "$@"; do
case "$arg" in
--context|--context=*|--server|--server=*)
has_context=1
break
;;
esac
done
if [[ -n "$target_ctx" && $has_context -eq 0 ]]; then
command kubectl --context "$target_ctx" "$@"
else
command kubectl "$@"
fi
}
resolve_db_cluster_context() {
if [[ "$MODE" == "k8s" ]]; then
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
[[ -n "$db_ctx" ]] || die "DB_CLUSTER_KUBECONTEXT is required for GitLab DB setup in k8s mode."
log "Using DB cluster context: ${db_ctx}"
printf '%s' "$db_ctx"
return 0
fi
return 1
}
repair_blocked() {
local reason="$1"
local remediation="${2:-}"
echo "" >&2
echo "[REPAIR_BLOCKED] ${reason}" >&2
if [[ -n "$remediation" ]]; then
echo "Remediation: ${remediation}" >&2
fi
echo "" >&2
exit 1
}
gitlab_selector_for_deployment() {
local deployment_name="$1"
local selector_lines selector=""
selector_lines=$(kubectl -n "$NAMESPACE" get deployment "$deployment_name" \
-o go-template='{{range $k, $v := .spec.selector.matchLabels}}{{printf "%s=%s\n" $k $v}}{{end}}' 2>/dev/null || true)
[[ -n "$selector_lines" ]] || return 1
local selector_line
while IFS= read -r selector_line; do
[[ -n "$selector_line" ]] || continue
if [[ -n "$selector" ]]; then
selector+=",${selector_line}"
else
selector="$selector_line"
fi
done <<< "$selector_lines"
[[ -n "$selector" ]] || return 1
printf '%s' "$selector"
}
gitlab_non_terminal_pod_count_for_app() {
local app_name="$1"
local pod_selector="$app_name"
if [[ -z "$pod_selector" ]]; then
printf '0'
return 0
fi
if [[ "$pod_selector" != *"="* ]]; then
pod_selector="app=${app_name}"
fi
local count
count=$(kubectl -n "$NAMESPACE" get pods -l "$pod_selector" \
--field-selector=status.phase!=Succeeded,status.phase!=Failed \
--no-headers 2>/dev/null | wc -l | xargs || echo "0")
if [[ -z "$count" || ! "$count" =~ ^[0-9]+$ ]]; then
count=0
fi
printf '%s' "$count"
}
gitlab_non_terminal_pod_names_for_app() {
local app_name="$1"
local pod_selector="$app_name"
if [[ -z "$pod_selector" ]]; then
return 0
fi
if [[ "$pod_selector" != *"="* ]]; then
pod_selector="app=${app_name}"
fi
local names
names=$(kubectl -n "$NAMESPACE" get pods -l "$pod_selector" \
--field-selector=status.phase!=Succeeded,status.phase!=Failed \
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}' 2>/dev/null || true)
names="${names% }"
if [[ -n "$names" ]]; then
printf '%s' "$names"
fi
return 0
}
gitlab_old_replicaset_live_summary() {
local deployment_name="$1"
local current_revision rs_rows summary=""
current_revision=$(kubectl -n "$NAMESPACE" get deployment "$deployment_name" \
-o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)
[[ -n "$current_revision" ]] || return 0
rs_rows=$(kubectl -n "$NAMESPACE" get rs -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.ownerReferences[0].kind}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.metadata.annotations.deployment\.kubernetes\.io/revision}{"|"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true)
[[ -n "$rs_rows" ]] || return 0
local rs_name owner_kind owner_name rs_revision rs_replicas
while IFS='|' read -r rs_name owner_kind owner_name rs_revision rs_replicas; do
[[ -n "$rs_name" ]] || continue
[[ "$owner_kind" == "Deployment" && "$owner_name" == "$deployment_name" ]] || continue
[[ "$rs_revision" != "$current_revision" ]] || continue
if [[ -z "$rs_replicas" || ! "$rs_replicas" =~ ^[0-9]+$ ]]; then
rs_replicas=0
fi
if (( rs_replicas > 0 )); then
summary+="${rs_name}:${rs_replicas},"
fi
done <<< "$rs_rows"
summary="${summary%,}"
if [[ -n "$summary" ]]; then
printf '%s' "$summary"
fi
return 0
}
gitlab_replica_source_of_truth_report() {
local target_replicas="$1"
local report=""
local dep_suffix dep_name live_replicas
for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2"; do
dep_name="${GITLAB_RELEASE}-${dep_suffix}"
live_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
if [[ -z "$live_replicas" ]]; then
report+="- ${dep_name}: spec=<missing>, desired=${target_replicas}"$'\n'
continue
fi
if [[ ! "$live_replicas" =~ ^[0-9]+$ ]]; then
report+="- ${dep_name}: spec=${live_replicas}, desired=${target_replicas}"$'\n'
continue
fi
if [[ "$live_replicas" != "$target_replicas" ]]; then
report+="- ${dep_name}: spec=${live_replicas}, desired=${target_replicas}"$'\n'
fi
done
printf '%s' "$report"
}
gitlab_rendered_replica_source_fields_from_cr() {
local desired_cr="$1"
[[ -n "$desired_cr" ]] || return 0
echo "$desired_cr" | awk '
BEGIN {
in_values = 0
component = ""
}
{
line = $0
if (line ~ /^ values:[[:space:]]*$/) {
in_values = 1
next
}
if (!in_values) {
next
}
if (line ~ /^ webservice:[[:space:]]*$/) {
component = "gitlab.webservice"
next
}
if (line ~ /^ sidekiq:[[:space:]]*$/) {
component = "gitlab.sidekiq"
next
}
if (line ~ /^ gitlab-shell:[[:space:]]*$/) {
component = "gitlab.gitlab-shell"
next
}
if (line ~ /^ kas:[[:space:]]*$/) {
component = "gitlab.kas"
next
}
if (line ~ /^ registry:[[:space:]]*$/) {
component = "registry"
next
}
if (line ~ /^ gitaly:[[:space:]]*$/ || line ~ /^ postgresql:[[:space:]]*$/ || line ~ /^ redis:[[:space:]]*$/) {
component = ""
}
if (line ~ /^ gitaly:[[:space:]]*$/ || line ~ /^ toolbox:[[:space:]]*$/) {
component = ""
}
if (component != "" && line ~ /^[[:space:]]*(replicaCount|minReplicas|maxReplicas|hpa):[[:space:]]*/) {
sub(/^[[:space:]]+/, "", line)
print "- " component "." line
}
}'
}
gitlab_verify_replica_source_of_truth() {
local target_replicas="$1"
local verify_timeout_s="${GITLAB_SOURCE_REPLICA_VERIFY_TIMEOUT:-180}"
local verify_poll_interval_s="${GITLAB_SOURCE_REPLICA_VERIFY_POLL_INTERVAL:-10}"
local rendered_replica_fields
rendered_replica_fields="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")"
if [[ -z "$rendered_replica_fields" ]]; then
rendered_replica_fields="- (no replica-related fields detected in rendered CR values)"
fi
if [[ -z "$verify_timeout_s" || ! "$verify_timeout_s" =~ ^[0-9]+$ || "$verify_timeout_s" == "0" ]]; then
verify_timeout_s=180
fi
if [[ -z "$verify_poll_interval_s" || ! "$verify_poll_interval_s" =~ ^[0-9]+$ || "$verify_poll_interval_s" == "0" ]]; then
verify_poll_interval_s=10
fi
local verify_start_ts verify_now_ts mismatch_report
verify_start_ts=$(date +%s)
while true; do
mismatch_report=$(gitlab_replica_source_of_truth_report "$target_replicas")
if [[ -z "$mismatch_report" ]]; then
log "GitLab operator desired replicas are source-of-truth converged (gitlab-shell/kas/registry/sidekiq spec=${target_replicas})."
return 0
fi
verify_now_ts=$(date +%s)
if (( verify_now_ts - verify_start_ts >= verify_timeout_s )); then
repair_blocked "GitLab operator desired replica source-of-truth mismatch" \
"Operator-managed Deployment specs did not converge to desired=${target_replicas} after GitLab CR apply:\n${mismatch_report}Rendered GitLab CR replica source fields:\n${rendered_replica_fields}\nThis is a source-of-truth issue (CR values still resolve to replicas>1), not a rollout lag issue."
fi
log "Waiting for GitLab operator to apply source-of-truth replicas (desired=${target_replicas}) before settle verification..."
while IFS= read -r mismatch_line; do
[[ -n "$mismatch_line" ]] || continue
log " ${mismatch_line}"
done <<< "$mismatch_report"
sleep "$verify_poll_interval_s"
done
}
gitlab_webservice_blocked_reasons() {
local webservice_app="$1"
local webservice_selector="$webservice_app"
if [[ -z "$webservice_selector" ]]; then
return 0
fi
if [[ "$webservice_selector" != *"="* ]]; then
webservice_selector="app=${webservice_app}"
fi
local not_ready_block_s="${GITLAB_WEBSERVICE_NOT_READY_BLOCK_SECONDS:-180}"
if [[ -z "$not_ready_block_s" || ! "$not_ready_block_s" =~ ^[0-9]+$ ]]; then
not_ready_block_s=180
fi
local pod_names
pod_names=$(kubectl -n "$NAMESPACE" get pods -l "$webservice_selector" \
--field-selector=status.phase!=Succeeded,status.phase!=Failed \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
[[ -n "$pod_names" ]] || return 0
local pod_name
while IFS= read -r pod_name; do
[[ -n "$pod_name" ]] || continue
local pod_phase pod_reason
pod_phase=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
pod_reason=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.reason}' 2>/dev/null || true)
if [[ "$pod_reason" == "ContainerStatusUnknown" || "$pod_phase" == "Unknown" ]]; then
printf '%s\n' "pod ${pod_name} status is ${pod_reason:-$pod_phase}"
fi
local pod_ready_status pod_ready_transition
pod_ready_status=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
pod_ready_transition=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].lastTransitionTime}' 2>/dev/null || true)
local container_rows
container_rows=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"|"}{.state.waiting.reason}{"|"}{.state.terminated.reason}{"\n"}{end}' 2>/dev/null || true)
[[ -n "$container_rows" ]] || continue
local container_name waiting_reason terminated_reason
while IFS='|' read -r container_name waiting_reason terminated_reason; do
[[ -n "$container_name" ]] || continue
case "$waiting_reason" in
CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerError|CreateContainerConfigError|RunContainerError|Error)
printf '%s\n' "pod ${pod_name}/${container_name} waiting reason=${waiting_reason}"
;;
esac
case "$terminated_reason" in
Error|ContainerCannotRun)
printf '%s\n' "pod ${pod_name}/${container_name} terminated reason=${terminated_reason}"
;;
esac
done <<< "$container_rows"
if [[ "$pod_ready_status" != "True" ]]; then
local ready_false_age_s=0
if [[ -n "$pod_ready_transition" ]]; then
ready_false_age_s=$(python3 - "$pod_ready_transition" <<'PY'
import datetime
import sys
raw = (sys.argv[1] if len(sys.argv) > 1 else "").strip()
if not raw:
print("0")
raise SystemExit(0)
try:
ts = datetime.datetime.fromisoformat(raw.replace("Z", "+00:00"))
now = datetime.datetime.now(datetime.timezone.utc)
age = int((now - ts).total_seconds())
print(str(max(age, 0)))
except Exception:
print("0")
PY
)
fi
if [[ -z "$ready_false_age_s" || ! "$ready_false_age_s" =~ ^[0-9]+$ ]]; then
ready_false_age_s=0
fi
if (( ready_false_age_s >= not_ready_block_s )); then
printf '%s\n' "pod ${pod_name} Ready=False for ${ready_false_age_s}s (threshold=${not_ready_block_s}s)"
fi
fi
done <<< "$pod_names"
}
wait_for_gitlab_workload_convergence() {
local timeout_s="${GITLAB_WORKLOAD_CONVERGENCE_TIMEOUT:-420}"
local poll_interval_s="${GITLAB_WORKLOAD_CONVERGENCE_POLL_INTERVAL:-10}"
if [[ -z "$timeout_s" || ! "$timeout_s" =~ ^[0-9]+$ ]]; then
timeout_s=420
fi
if [[ -z "$poll_interval_s" || ! "$poll_interval_s" =~ ^[0-9]+$ || "$poll_interval_s" == "0" ]]; then
poll_interval_s=10
fi
local start_ts
start_ts=$(date +%s)
local last_report=""
while true; do
local -a blockers=()
local dep_suffix dep_name dep_selector desired_replicas live_non_terminal old_rs_summary
local ready_replicas available_replicas
for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
dep_name="${GITLAB_RELEASE}-${dep_suffix}"
desired_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
if [[ -z "$desired_replicas" ]]; then
blockers+=("${dep_name}: deployment not found")
continue
fi
if [[ ! "$desired_replicas" =~ ^[0-9]+$ ]]; then
desired_replicas=1
fi
dep_selector=$(gitlab_selector_for_deployment "$dep_name" 2>/dev/null || true)
if [[ -z "$dep_selector" ]]; then
blockers+=("${dep_name}: deployment selector is empty")
continue
fi
live_non_terminal=$(gitlab_non_terminal_pod_count_for_app "$dep_selector")
if (( live_non_terminal > desired_replicas )); then
blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} > desired=${desired_replicas} (selector=${dep_selector})")
elif (( live_non_terminal < desired_replicas )); then
blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} < desired=${desired_replicas} (selector=${dep_selector})")
fi
ready_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
available_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
if [[ -z "$ready_replicas" || ! "$ready_replicas" =~ ^[0-9]+$ ]]; then
ready_replicas=0
fi
if [[ -z "$available_replicas" || ! "$available_replicas" =~ ^[0-9]+$ ]]; then
available_replicas=0
fi
if (( ready_replicas < desired_replicas )); then
blockers+=("${dep_name}: readyReplicas=${ready_replicas} < desired=${desired_replicas}")
fi
if (( available_replicas < desired_replicas )); then
blockers+=("${dep_name}: availableReplicas=${available_replicas} < desired=${desired_replicas}")
fi
old_rs_summary=$(gitlab_old_replicaset_live_summary "$dep_name")
if [[ -n "$old_rs_summary" ]]; then
blockers+=("${dep_name}: old ReplicaSet pods still running (${old_rs_summary})")
fi
if [[ "$dep_suffix" == "webservice-default" ]]; then
local webservice_reasons
webservice_reasons=$(gitlab_webservice_blocked_reasons "$dep_selector" 2>/dev/null || true)
if [[ -n "$webservice_reasons" ]]; then
local webservice_reason
while IFS= read -r webservice_reason; do
[[ -n "$webservice_reason" ]] || continue
blockers+=("${dep_name}: ${webservice_reason}")
done <<< "$webservice_reasons"
fi
fi
done
if (( ${#blockers[@]} == 0 )); then
log "GitLab workloads converged: pod counts, ReplicaSets, and webservice health are clean."
return 0
fi
local report=""
local blocker
for blocker in "${blockers[@]}"; do
report+="- ${blocker}"$'\n'
done
local now_ts
now_ts=$(date +%s)
if (( now_ts - start_ts >= timeout_s )); then
repair_blocked "GitLab workloads are not converged after reconcile" \
"Namespace: ${NAMESPACE}. Remaining blockers:
${report}Check: kubectl -n ${NAMESPACE} get deploy,rs,pods -o wide"
fi
if [[ "$report" != "$last_report" ]]; then
warn "Waiting for GitLab workload convergence..."
local report_line
while IFS= read -r report_line; do
[[ -n "$report_line" ]] || continue
warn " ${report_line}"
done <<< "$report"
last_report="$report"
fi
sleep "$poll_interval_s"
done
}
get_gitlab_migrations_diagnostics() {
local job_name="$1"
[[ -n "$job_name" ]] || return 0
local _pod_name=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$job_name" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -z "$_pod_name" ]]; then
echo "No pods found for migration job $job_name."
return 0
fi
local _diag=""
_diag+="--- Migrations Diagnostics for Pod: $_pod_name ---"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o wide 2>/dev/null || true)"$'\n'
_diag+=$'\n'"[Container Statuses]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}: state={.state.waiting.reason}{.state.terminated.reason}{.state.running.startedAt}, exitCode={.state.terminated.exitCode}, restarts={.restartCount}{"\n"}{end}' 2>/dev/null || true)"$'\n'
_diag+=$'\n'"[Migrations Logs (tail=100)]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --tail=100 2>/dev/null || echo "(no logs available)")"$'\n'
_diag+=$'\n'"[Previous Migrations Logs (if any)]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --previous --tail=100 2>/dev/null || echo "(no previous logs)")"$'\n'
_diag+=$'\n'"[Pod Events]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$_pod_name" --sort-by='.lastTimestamp' 2>/dev/null | tail -n 10 || true)"
echo "$_diag"
}
check_gitlab_migrations_blocked() {
local _latest_job
_latest_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
[[ -n "$_latest_job" ]] || return 0
local _job_failed
_job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
# 1. Active Failure Detection
local _latest_pod
_latest_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -n "$_latest_pod" ]]; then
local _restarts
_restarts=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
local _waiting_reason
_waiting_reason=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
local _exit_code
_exit_code=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.terminated.exitCode}' 2>/dev/null || echo "")
if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_exit_code" && "$_exit_code" != "0" ) ]]; then
log "GitLab migrations job is actively failing: ${_latest_job} (pod: ${_latest_pod}, restarts: ${_restarts}, exitCode: ${_exit_code:-unknown})"
# Enhanced DB connectivity diagnostics for split-cluster visibility
local _db_info="DB_HOST=${DB_HOST}, DB_PORT=${DB_PORT}"
local _split_cluster="No"
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then _split_cluster="Yes (APP: ${app_ctx}, DB: ${db_ctx})"; fi
local _diag
_diag=$(get_gitlab_migrations_diagnostics "$_latest_job")
local _connectivity_hint=""
if echo "$_diag" | grep -qi "connecting with your hostname"; then
_connectivity_hint=" (Likely DNS/resolution failure)"
elif echo "$_diag" | grep -qiE "connection refused|timeout"; then
_connectivity_hint=" (Likely TCP connectivity/firewall failure)"
fi
repair_blocked "GitLab migrations job is actively failing" \
"Job: ${_latest_job}. Pod: ${_latest_pod}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_exit_code:-unknown}. Restarts: ${_restarts}.
DB Config: ${_db_info}${_connectivity_hint}
Split-cluster: ${_split_cluster}
Diagnostics:
${_diag}"
fi
fi
# 2. Stale Failed Job Repair
local _job_creation_ts
_job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
local _job_pods_running
_job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
local _job_pods_pending
_job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
local _now_ts
_now_ts=$(date +%s)
local _creation_ts
_creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0")
local _age
_age=$(( _now_ts - _creation_ts ))
if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then
log "GitLab operator is stalled on stale failed migrations job: ${_latest_job} (age: ${_age}s, no active pods)"
log "Deleting stale failed migrations job to trigger repair..."
kubectl -n "$NAMESPACE" delete job "$_latest_job" --wait=true 2>/dev/null || true
kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_job" --force --grace-period=0 2>/dev/null || true
log "Stale migrations job deleted. Operator should recreate it shortly."
fi
}
check_host_resolves() {
local target_host="$1"
# Strip port if present
local host_only="${target_host%:*}"
local target_ctx
target_ctx="$(resolve_explicit_kube_context || true)"
local is_k8s_svc=0
if [[ "$host_only" == *".svc.cluster.local" ]]; then
is_k8s_svc=1
elif [[ "$host_only" =~ ^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+$ ]]; then
# Simple <service>.<namespace> format (exactly one dot, no numbers to avoid IPs)
is_k8s_svc=1
fi
if [[ $is_k8s_svc -eq 1 ]]; then
# Parse service and namespace (first and second labels)
local svc_name svc_ns
svc_name=$(echo "$host_only" | cut -d. -f1)
svc_ns=$(echo "$host_only" | cut -d. -f2)
log "Redis validation mode: kubernetes-service (cluster: ${target_ctx:-default})"
log "Parsed REDIS_HOST ${target_host} -> service=${svc_name} namespace=${svc_ns}"
if kubectl -n "$svc_ns" get service "$svc_name" >/dev/null 2>&1; then
log "Service ${svc_name} found in namespace ${svc_ns}"
# Preferably verify it has endpoints / ready backing pods
local endpoints_found
endpoints_found=$(kubectl -n "$svc_ns" get endpoints "$svc_name" -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
if [[ -n "$endpoints_found" ]]; then
log "Redis endpoints present for ${svc_name}"
else
warn "Service ${svc_name} found in namespace ${svc_ns} but has NO ready endpoints (yet)."
fi
return 0
else
warn "Service ${svc_name} NOT found in namespace ${svc_ns} (cluster: ${target_ctx:-default})"
return 1
fi
else
log "Redis validation mode: external-host"
if ! host "$host_only" >/dev/null 2>&1; then
# If it's an IP, host might fail. Check if it's an IP.
if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
log "Host ${host_only} is an IP address, skipping DNS check."
return 0
fi
return 1
fi
fi
return 0
}
check_db_connectivity() {
local target_host="$1"
local target_port="${2:-5432}"
local host_only="${target_host%:*}"
log "Validating GitLab database connectivity to ${target_host}:${target_port}..."
# DNS sanity check
if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
log "Host ${host_only} is an IP address, skipping DNS check."
else
# In k8s mode, internal DNS resolution might fail from the script runner
# but work from within the cluster. We do a basic 'host' check as a hint.
if ! host "$host_only" >/dev/null 2>&1; then
warn "DNS resolution (host) failed for database host: ${host_only}. This may be expected if only resolvable in-cluster."
else
log "DNS resolution successful for ${host_only}"
fi
fi
# TCP check via temporary pod in the APP cluster namespace
log "Performing in-cluster TCP connectivity probe to ${host_only}:${target_port} (namespace: ${NAMESPACE})..."
if kubectl -n "$NAMESPACE" run db-probe \
--image=alpine:latest --restart=Never --rm --attach --timeout=30s \
--command -- sh -c "nc -zv -w 5 ${host_only} ${target_port}" >/dev/null 2>&1; then
log "TCP connectivity to ${host_only}:${target_port} SUCCESSFUL."
return 0
else
warn "TCP connectivity probe to ${host_only}:${target_port} FAILED."
return 1
fi
}
check_gitlab_pre_apply_blocked() {
# --- Registry Endpoint Placeholder Check ---
if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
# If the configured value itself is still a placeholder, we MUST block.
# The operator hasn't provided a real value yet.
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
"GARAGE_PRIVATE_S3_ENDPOINT is set to the literal placeholder. Set it to the real DB cluster Garage endpoint."
fi
# --- Redis Host Resolution Check ---
if ! check_host_resolves "$REDIS_HOST"; then
# This is potentially repairable if we can infer a better host,
# but for now we block if it's clearly invalid.
repair_blocked "Redis host does not resolve" \
"Resource: REDIS_HOST. Value: ${REDIS_HOST} does not resolve. Fix: Ensure Redis is deployed and REDIS_NAMESPACE is correct."
fi
# --- Database Connectivity Check ---
if [[ "$MODE" == "k8s" ]]; then
if ! check_db_connectivity "$DB_HOST" "$DB_PORT"; then
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
repair_blocked "GitLab database host is not reachable from APP cluster" \
"Host: ${DB_HOST}. Port: ${DB_PORT}. App Context: ${app_ctx}. DB Context: ${db_ctx}. Fix: Ensure cross-cluster networking (ILB) is functional."
fi
fi
}
get_gitlab_blocker_context() {
local ctx=""
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then
ctx+="Split-cluster (APP: ${app_ctx}, DB: ${db_ctx}). "
fi
ctx+="DB_HOST: ${DB_HOST}. "
local _last_mig=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -n "$_last_mig" ]]; then
local _m_ts=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
local _m_failed=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
local _m_pods=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --no-headers 2>/dev/null | wc -l | xargs || echo "0")
local _m_pod_ctx=""
local _m_last_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -n "$_m_last_pod" ]]; then
local _m_pod_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.phase}' 2>/dev/null || true)
local _m_restarts=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
local _m_waiting=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
_m_pod_ctx="Pod: ${_m_last_pod} (${_m_pod_status}, restarts: ${_m_restarts}, waiting: ${_m_waiting:-None}). "
fi
ctx+="Migration Job: ${_last_mig} (Created: ${_m_ts}, Failed: ${_m_failed:-False}, Pods: ${_m_pods}). ${_m_pod_ctx}"
fi
local _sts_name="${GITLAB_RELEASE}-gitaly"
local _sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$_sts_yaml" ]]; then
local _sts_ns=$(echo "$_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local _sts_sc=$(echo "$_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
ctx+="Gitaly STS: nodeSelector=[${_sts_ns}], storageClass=[${_sts_sc}]. "
fi
local _pvc_name="repo-data-gitlab-gitaly-0"
local _pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$_pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
if [[ -n "$_pvc_phase" ]]; then
ctx+="Gitaly PVC: ${_pvc_phase}. "
fi
local _op_pod=$(kubectl -n "$NAMESPACE" get pods -l "control-plane=controller-manager" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -n "$_op_pod" ]]; then
local _op_log=$(kubectl -n "$NAMESPACE" logs "$_op_pod" -c manager --tail=1 2>/dev/null || true)
ctx+="Operator Log: ${_op_log}. "
fi
echo "$ctx"
}
gitlab_storage_class_matches_expected() {
local live_sc="$1"
local expected_sc="$2"
[[ -z "$expected_sc" ]] && return 0
[[ "$live_sc" == "$expected_sc" ]]
}
gitlab_rendered_storage_fields_from_cr() {
local desired_cr="$1"
local rendered_global_sc=""
local rendered_gitlab_gitaly_sc=""
local rendered_chart_gitaly_sc=""
if [[ -n "$desired_cr" ]]; then
rendered_global_sc=$(echo "$desired_cr" | sed -n '/^ global:/,/^ postgresql:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true)
rendered_gitlab_gitaly_sc=$(echo "$desired_cr" | sed -n '/^ gitaly:/,/^ toolbox:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true)
rendered_chart_gitaly_sc=$(echo "$desired_cr" | sed -n '/^ gitaly:/,/^ registry:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true)
fi
printf '%s|%s|%s\n' "$rendered_global_sc" "$rendered_gitlab_gitaly_sc" "$rendered_chart_gitaly_sc"
}
gitlab_chart_major_version() {
local chart_version="${GITLAB_CHART_VERSION:-}"
chart_version="${chart_version#v}"
chart_version="${chart_version%%-*}"
chart_version="${chart_version%%.*}"
printf '%s' "$chart_version"
}
gitlab_chart_is_v9_or_newer() {
local chart_major
chart_major="$(gitlab_chart_major_version)"
[[ "$chart_major" =~ ^[0-9]+$ ]] || return 1
(( chart_major >= 9 ))
}
gitlab_chart9_find_deprecated_top_level_replica_keys() {
local desired_cr="$1"
[[ -n "$desired_cr" ]] || return 0
echo "$desired_cr" | awk '
function leading_spaces(str, i, c, n) {
n = 0
for (i = 1; i <= length(str); i++) {
c = substr(str, i, 1)
if (c == " ") {
n++
} else {
break
}
}
return n
}
BEGIN {
in_values = 0
in_registry = 0
registry_indent = -1
in_hpa = 0
hpa_indent = -1
}
{
line = $0
if (line ~ /^ values:[[:space:]]*$/) {
in_values = 1
next
}
if (!in_values) {
next
}
indent = leading_spaces(line)
if (in_hpa && indent <= hpa_indent) {
in_hpa = 0
hpa_indent = -1
}
if (in_registry && indent <= registry_indent && line !~ /^ registry:[[:space:]]*$/) {
in_registry = 0
registry_indent = -1
}
if (line ~ /^ registry:[[:space:]]*$/) {
in_registry = 1
registry_indent = indent
next
}
if (!in_registry) {
next
}
if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) {
in_hpa = 1
hpa_indent = indent
next
}
if (!in_hpa && line ~ /^[[:space:]]*(minReplicas|maxReplicas):[[:space:]]*/) {
print NR ":" line
}
}'
}
gitlab_chart9_strip_deprecated_top_level_replica_keys() {
local desired_cr="$1"
[[ -n "$desired_cr" ]] || {
printf '%s' "$desired_cr"
return 0
}
echo "$desired_cr" | awk '
function leading_spaces(str, i, c, n) {
n = 0
for (i = 1; i <= length(str); i++) {
c = substr(str, i, 1)
if (c == " ") {
n++
} else {
break
}
}
return n
}
BEGIN {
in_values = 0
in_registry = 0
registry_indent = -1
in_hpa = 0
hpa_indent = -1
}
{
line = $0
if (line ~ /^ values:[[:space:]]*$/) {
in_values = 1
print line
next
}
if (!in_values) {
print line
next
}
indent = leading_spaces(line)
if (in_hpa && indent <= hpa_indent) {
in_hpa = 0
hpa_indent = -1
}
if (in_registry && indent <= registry_indent && line !~ /^ registry:[[:space:]]*$/) {
in_registry = 0
registry_indent = -1
}
if (line ~ /^ registry:[[:space:]]*$/) {
in_registry = 1
registry_indent = indent
print line
next
}
if (!in_registry) {
print line
next
}
if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) {
in_hpa = 1
hpa_indent = indent
print line
next
}
if (!in_hpa && line ~ /^[[:space:]]*(minReplicas|maxReplicas):[[:space:]]*/) {
next
}
print line
}'
}
sanitize_gitlab_cr_rendered_values_for_chart_version() {
local desired_cr="$1"
[[ -n "$desired_cr" ]] || {
printf '%s' "$desired_cr"
return 0
}
if ! gitlab_chart_is_v9_or_newer; then
printf '%s' "$desired_cr"
return 0
fi
local deprecated_keys_before
deprecated_keys_before="$(gitlab_chart9_find_deprecated_top_level_replica_keys "$desired_cr")"
if [[ -n "$deprecated_keys_before" ]]; then
log "GitLab chart ${GITLAB_CHART_VERSION}: stripping deprecated top-level replica keys before CR apply:"
while IFS= read -r deprecated_key; do
[[ -n "$deprecated_key" ]] || continue
log " - ${deprecated_key}"
done <<< "$deprecated_keys_before"
desired_cr="$(gitlab_chart9_strip_deprecated_top_level_replica_keys "$desired_cr")"
fi
printf '%s' "$desired_cr"
}
preflight_validate_gitlab_cr_rendered_values() {
local desired_cr="$1"
[[ -n "$desired_cr" ]] || return 0
if ! gitlab_chart_is_v9_or_newer; then
return 0
fi
local deprecated_keys_remaining
deprecated_keys_remaining="$(gitlab_chart9_find_deprecated_top_level_replica_keys "$desired_cr")"
if [[ -n "$deprecated_keys_remaining" ]]; then
repair_blocked "Rendered GitLab CR values contain removed chart v9+ keys" \
"Local preflight rejected GitLab CR apply. Deprecated top-level replica keys remain (for example registry.minReplicas/registry.maxReplicas). Remaining keys: ${deprecated_keys_remaining//$'\n'/; }."
fi
}
gitlab_live_gitaly_repo_data_template_storage_class() {
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
local live_template_sc
live_template_sc=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o jsonpath='{range .spec.volumeClaimTemplates[?(@.metadata.name=="repo-data")]}{.spec.storageClassName}{end}' 2>/dev/null || true)
if [[ -z "$live_template_sc" ]]; then
live_template_sc=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o jsonpath='{.spec.volumeClaimTemplates[0].spec.storageClassName}' 2>/dev/null || true)
fi
printf '%s' "$live_template_sc"
}
cleanup_gitlab_wrong_gitaly_template_storage() {
local expected_sc="$1"
local live_template_sc="$2"
local pvc_mismatch_records="$3"
local desired_cr="$4"
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
export GITALY_AUTOCLEAN_PERFORMED=1
log "AUTOCLEAN: repairing wrong-class Gitaly StatefulSet template storageClass (expected='${expected_sc}', live='${live_template_sc:-<unset>}')."
log "Rendered GitLab CR storage fields prior to repair: $(gitlab_rendered_storage_fields_from_cr "$desired_cr")"
kubectl -n "$NAMESPACE" scale statefulset "$gitaly_sts_name" --replicas=0 --timeout=30s 2>/dev/null || true
if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then
log "Deleting StatefulSet ${gitaly_sts_name} to force operator recreation of repo-data claim template..."
kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true
fi
if [[ -n "$pvc_mismatch_records" ]]; then
cleanup_gitlab_wrong_storage_class_records "$pvc_mismatch_records"
fi
# Also clear retained stale GKE dynamic-provisioning leftovers for repo-data claim.
repair_stale_gke_gitaly_dynamic_storage "$expected_sc"
}
is_gke_stale_gitaly_storage_class_for_standard_target() {
local desired_sc="$1"
local candidate_sc="$2"
[[ "$desired_sc" == "standard" ]] || return 1
[[ -n "$candidate_sc" && "$candidate_sc" != "$desired_sc" ]] && return 0
return 1
}
is_gke_blocking_disk_type_for_standard_target() {
local disk_type="$1"
case "$disk_type" in
pd-balanced|pd-ssd)
return 0
;;
esac
return 1
}
gke_fetch_disk_type_from_volume_handle() {
local volume_handle="$1"
[[ -n "$volume_handle" ]] || return 0
command -v gcloud >/dev/null 2>&1 || return 0
local parsed_ref
parsed_ref=$(gke_parse_disk_ref_from_volume_handle "$volume_handle" || true)
[[ -n "$parsed_ref" ]] || return 0
local disk_scope disk_location disk_name
IFS='|' read -r disk_scope disk_location disk_name <<< "$parsed_ref"
[[ -n "$disk_name" ]] || return 0
local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
if [[ -z "$project_id" ]]; then
project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
fi
local -a common_args=()
if [[ -n "$project_id" ]]; then
common_args+=(--project "$project_id")
fi
local disk_type=""
if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then
disk_type=$(gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" --format='value(type.basename())' 2>/dev/null || true)
elif [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then
disk_type=$(gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" --format='value(type.basename())' 2>/dev/null || true)
else
local matched
matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](type.basename())' 2>/dev/null | head -n1 || true)
disk_type="$matched"
fi
printf '%s' "$disk_type"
}
collect_gitlab_storage_mismatch_records() {
local expected_sc="$1"
[[ -n "$expected_sc" ]] || return 0
local pvc_rows
pvc_rows=$(kubectl -n "$NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.storageClassName}{"|"}{.status.phase}{"|"}{.spec.volumeName}{"|"}{.metadata.labels.app\.kubernetes\.io/instance}{"\n"}{end}' 2>/dev/null || true)
[[ -n "$pvc_rows" ]] || return 0
while IFS='|' read -r pvc_name pvc_sc pvc_phase pvc_pv pvc_instance; do
[[ -n "$pvc_name" ]] || continue
local is_gitlab_pvc=0
if [[ "$pvc_instance" == "$GITLAB_RELEASE" || "$pvc_name" == "repo-data-${GITLAB_RELEASE}-"* || "$pvc_name" == "${GITLAB_RELEASE}-"* ]]; then
is_gitlab_pvc=1
fi
[[ "$is_gitlab_pvc" == "1" ]] || continue
local pv_sc="" disk_handle="" disk_type="" disk_scope="" disk_location="" disk_name=""
if [[ -n "$pvc_pv" ]]; then
local pv_details pv_disk_name
pv_details=$(kubectl get pv "$pvc_pv" -o jsonpath='{.spec.storageClassName}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"|"}{.spec.csi.volumeAttributes.type}' 2>/dev/null || true)
if [[ -n "$pv_details" ]]; then
IFS='|' read -r pv_sc disk_handle pv_disk_name disk_type <<< "$pv_details"
fi
if [[ -z "$disk_handle" && -n "$pv_disk_name" ]]; then
disk_handle="$pv_disk_name"
fi
if [[ -n "$disk_handle" ]]; then
local parsed_ref
parsed_ref=$(gke_parse_disk_ref_from_volume_handle "$disk_handle" || true)
if [[ -n "$parsed_ref" ]]; then
IFS='|' read -r disk_scope disk_location disk_name <<< "$parsed_ref"
fi
fi
if [[ -z "$disk_type" && -n "$disk_handle" ]]; then
disk_type=$(gke_fetch_disk_type_from_volume_handle "$disk_handle" || true)
fi
fi
local effective_live_sc="$pvc_sc"
if [[ -z "$effective_live_sc" && -n "$pv_sc" ]]; then
effective_live_sc="$pv_sc"
fi
if ! gitlab_storage_class_matches_expected "$effective_live_sc" "$expected_sc"; then
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
"$pvc_name" "$pvc_sc" "$pvc_phase" "$pvc_pv" "$pv_sc" "$disk_handle" "$disk_scope" "$disk_location" "$disk_name" "$disk_type"
fi
done <<< "$pvc_rows"
}
collect_gitlab_statefulset_template_mismatch_records() {
local expected_sc="$1"
[[ -n "$expected_sc" ]] || return 0
local sts_rows
sts_rows=$(kubectl -n "$NAMESPACE" get statefulset -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.labels.app\.kubernetes\.io/instance}{"|"}{range .spec.volumeClaimTemplates[*]}{.metadata.name}{":"}{.spec.storageClassName}{","}{end}{"\n"}{end}' 2>/dev/null || true)
[[ -n "$sts_rows" ]] || return 0
while IFS='|' read -r sts_name sts_instance claims; do
[[ -n "$sts_name" ]] || continue
local is_gitlab_sts=0
if [[ "$sts_instance" == "$GITLAB_RELEASE" || "$sts_name" == "${GITLAB_RELEASE}-"* ]]; then
is_gitlab_sts=1
fi
[[ "$is_gitlab_sts" == "1" ]] || continue
[[ -n "$claims" ]] || continue
IFS=',' read -ra claim_pairs <<< "$claims"
local pair
for pair in "${claim_pairs[@]}"; do
[[ -n "$pair" ]] || continue
local claim_name claim_sc
claim_name="${pair%%:*}"
claim_sc="${pair#*:}"
if ! gitlab_storage_class_matches_expected "$claim_sc" "$expected_sc"; then
printf '%s|%s|%s\n' "$sts_name" "$claim_name" "$claim_sc"
fi
done
done <<< "$sts_rows"
}
cleanup_gitlab_wrong_storage_class_records() {
local mismatch_records="$1"
[[ -n "$mismatch_records" ]] || return 0
export GITALY_AUTOCLEAN_PERFORMED=1
log "AUTOCLEAN: removing wrong-class GitLab PVC/PV artifacts to enforce storageClass='${GITALY_STORAGE_CLASS:-}'..."
local processed_disks="|"
local _disk_key
while IFS='|' read -r pvc_name _pvc_sc _pvc_phase pvc_pv _pv_sc disk_handle disk_scope disk_location disk_name _disk_type; do
[[ -n "$pvc_name" ]] || continue
if [[ "$pvc_name" == "repo-data-${GITLAB_RELEASE}-gitaly-0" ]]; then
kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
fi
if kubectl -n "$NAMESPACE" get pvc "$pvc_name" >/dev/null 2>&1; then
log "Deleting wrong-class PVC ${pvc_name}..."
kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true
fi
if [[ -n "$pvc_pv" ]] && kubectl get pv "$pvc_pv" >/dev/null 2>&1; then
log "Deleting wrong-class PV ${pvc_pv} (PVC=${pvc_name})..."
kubectl delete pv "$pvc_pv" --wait=false 2>/dev/null || true
fi
local disk_ref=""
if [[ -n "$disk_handle" ]]; then
disk_ref=$(gke_parse_disk_ref_from_volume_handle "$disk_handle" || true)
fi
if [[ -n "$disk_ref" ]]; then
IFS='|' read -r disk_scope disk_location disk_name <<< "$disk_ref"
fi
if [[ -z "$disk_name" && -n "$disk_handle" ]]; then
disk_scope="name"
disk_location=""
disk_name="$disk_handle"
fi
if [[ -n "$disk_name" ]]; then
_disk_key="${disk_scope}|${disk_location}|${disk_name}"
if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
processed_disks+="${_disk_key}|"
gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true
fi
fi
done <<< "$mismatch_records"
sleep 2
}
enforce_gitlab_storage_class_target() {
local expected_sc="$1"
local desired_cr="$2"
[[ "$MODE" == "k8s" ]] || return 0
[[ -n "$expected_sc" ]] || return 0
local rendered_global_sc=""
local rendered_gitlab_gitaly_sc=""
local rendered_chart_gitaly_sc=""
IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")"
log "GitLab storage target: configured='${expected_sc}', rendered.global.persistence.storageClass='${rendered_global_sc:-<unset>}', rendered.gitlab.gitaly.persistence.storageClass='${rendered_gitlab_gitaly_sc:-<unset>}', rendered.gitaly.persistence.storageClass='${rendered_chart_gitaly_sc:-<unset>}'"
if [[ -n "$rendered_global_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_global_sc" "$expected_sc"; then
repair_blocked "GitLab CR global persistence storageClass mismatch" \
"Configured target=[${expected_sc}] but rendered global.persistence.storageClass=[${rendered_global_sc}]. This indicates GitLab CR rendering is wrong."
fi
if [[ -n "$rendered_gitlab_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_gitlab_gitaly_sc" "$expected_sc"; then
repair_blocked "GitLab CR gitaly persistence storageClass mismatch" \
"Configured target=[${expected_sc}] but rendered gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc}]. This indicates GitLab CR rendering is wrong."
fi
if [[ -n "$rendered_chart_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_chart_gitaly_sc" "$expected_sc"; then
repair_blocked "GitLab CR chart gitaly persistence storageClass mismatch" \
"Configured target=[${expected_sc}] but rendered gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc}]. This indicates GitLab CR rendering is wrong."
fi
local template_mismatches
template_mismatches=$(collect_gitlab_statefulset_template_mismatch_records "$expected_sc")
local pvc_mismatches
pvc_mismatches=$(collect_gitlab_storage_mismatch_records "$expected_sc")
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
local gitaly_sts_exists=0
if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then
gitaly_sts_exists=1
fi
local live_gitaly_template_sc
live_gitaly_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class)
local live_gitaly_pvc_sc
live_gitaly_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "repo-data-${GITLAB_RELEASE}-gitaly-0" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
log "GitLab Gitaly live storage snapshot: statefulset=${gitaly_sts_name} claimTemplate.repo-data.storageClassName='${live_gitaly_template_sc:-<unset>}' pvc.repo-data-${GITLAB_RELEASE}-gitaly-0.storageClassName='${live_gitaly_pvc_sc:-<unset>}'"
if [[ "$gitaly_sts_exists" == "1" ]] && ! gitlab_storage_class_matches_expected "$live_gitaly_template_sc" "$expected_sc"; then
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass mismatch" \
"Configured target=[${expected_sc}] rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}] live.statefulset=${GITLAB_RELEASE}-gitaly live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_template_sc:-<unset>}]. Delete statefulset/${GITLAB_RELEASE}-gitaly, pvc/repo-data-${GITLAB_RELEASE}-gitaly-0, and any wrong-class bound/released PV+disk artifacts, or enable GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 for automatic destructive repair."
fi
cleanup_gitlab_wrong_gitaly_template_storage "$expected_sc" "$live_gitaly_template_sc" "$pvc_mismatches" "$desired_cr"
# Re-check after cleanup so subsequent gates reflect live post-repair state.
template_mismatches=$(collect_gitlab_statefulset_template_mismatch_records "$expected_sc")
pvc_mismatches=$(collect_gitlab_storage_mismatch_records "$expected_sc")
gitaly_sts_exists=0
if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then
gitaly_sts_exists=1
fi
live_gitaly_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class)
if [[ "$gitaly_sts_exists" == "1" ]] && ! gitlab_storage_class_matches_expected "$live_gitaly_template_sc" "$expected_sc"; then
repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass remains wrong after repair" \
"Configured target=[${expected_sc}] rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}] live.statefulset=${GITLAB_RELEASE}-gitaly live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_template_sc:-<unset>}]."
fi
fi
if [[ -n "$template_mismatches" ]]; then
log "GitLab StatefulSet volumeClaimTemplate storageClass diagnostics (expected='${expected_sc}')"
while IFS='|' read -r sts_name claim_name claim_sc; do
[[ -n "$sts_name" ]] || continue
log " - workload=statefulset/${sts_name} claimTemplate=${claim_name} storageClass=${claim_sc}"
done <<< "$template_mismatches"
fi
if [[ -n "$pvc_mismatches" ]]; then
log "GitLab PVC/PV storageClass diagnostics (expected='${expected_sc}')"
while IFS='|' read -r pvc_name pvc_sc pvc_phase pvc_pv pv_sc disk_handle disk_scope disk_location disk_name disk_type; do
[[ -n "$pvc_name" ]] || continue
log " - pvc=${pvc_name} phase=${pvc_phase:-unknown} pvc.storageClass=${pvc_sc:-<unset>} pv=${pvc_pv:-<none>} pv.storageClass=${pv_sc:-<unset>} diskHandle=${disk_handle:-<none>} diskRef=${disk_scope:-<none>}:${disk_location:-<none>}/${disk_name:-<none>} diskType=${disk_type:-<unknown>}"
done <<< "$pvc_mismatches"
fi
if [[ -z "$template_mismatches" && -z "$pvc_mismatches" ]]; then
return 0
fi
if [[ -n "$template_mismatches" ]]; then
repair_blocked "GitLab StatefulSet claim-template storageClass mismatch" \
"Configured target=[${expected_sc}] but one or more live GitLab StatefulSet volumeClaimTemplates use a different storageClass. For Gitaly this is a hard mismatch and must be repaired with authoritative CR fields + destructive cleanup gate."
fi
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
repair_blocked "GitLab storageClass mismatch detected" \
"Configured target=[${expected_sc}] but one or more GitLab StatefulSet/PVC/PV resources are on a different storageClass. Enable GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to delete wrong-class GitLab PVC/PV/disk artifacts for reprovision."
fi
cleanup_gitlab_wrong_storage_class_records "$pvc_mismatches"
}
is_gke_cluster_detected() {
local provider_ids gke_pool_labels gke_topology_labels
provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true)
gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true)
gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true)
if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then
return 0
fi
return 1
}
gke_parse_disk_ref_from_volume_handle() {
local handle="$1"
[[ -n "$handle" ]] || return 1
if [[ "$handle" =~ /zones/([^/]+)/disks/([^/]+)$ ]]; then
printf 'zone|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
return 0
fi
if [[ "$handle" =~ /regions/([^/]+)/disks/([^/]+)$ ]]; then
printf 'region|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
return 0
fi
if [[ "$handle" =~ ^[^/]+$ ]]; then
printf 'name||%s' "$handle"
return 0
fi
return 1
}
gke_delete_disk_ref_if_present() {
local disk_scope="$1"
local disk_location="$2"
local disk_name="$3"
[[ -n "$disk_name" ]] || return 0
if ! command -v gcloud >/dev/null 2>&1; then
warn "gcloud not found; cannot auto-delete stale disk '${disk_name}'."
return 1
fi
local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
if [[ -z "$project_id" ]]; then
project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
fi
local -a common_args=(--quiet)
if [[ -n "$project_id" ]]; then
common_args+=(--project "$project_id")
fi
if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then
if gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
log "Deleting stale GKE disk ${disk_name} (zone=${disk_location})..."
gcloud compute disks delete "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${disk_location})."
else
log "Stale disk ${disk_name} already absent in zone ${disk_location}."
fi
return 0
fi
if [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then
if gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then
log "Deleting stale GKE regional disk ${disk_name} (region=${disk_location})..."
gcloud compute disks delete "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${disk_location})."
else
log "Stale regional disk ${disk_name} already absent in region ${disk_location}."
fi
return 0
fi
local matched
matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](zone.basename(),region.basename())' 2>/dev/null || true)
if [[ -z "$matched" ]]; then
log "Stale disk ${disk_name} already absent."
return 0
fi
while IFS='|' read -r zone_name region_name; do
[[ -n "$zone_name" || -n "$region_name" ]] || continue
if [[ -n "$zone_name" ]]; then
log "Deleting stale GKE disk ${disk_name} (zone=${zone_name})..."
gcloud compute disks delete "$disk_name" --zone "$zone_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${zone_name})."
elif [[ -n "$region_name" ]]; then
log "Deleting stale GKE regional disk ${disk_name} (region=${region_name})..."
gcloud compute disks delete "$disk_name" --region "$region_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${region_name})."
fi
done <<< "$matched"
}
collect_stale_gke_gitaly_pv_records() {
local desired_sc="$1"
[[ "$desired_sc" == "standard" ]] || return 0
local pvc_name="repo-data-gitlab-gitaly-0"
local pvc_phase
pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
local pv_rows
pv_rows=$(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"|"}{.spec.storageClassName}{"|"}{.spec.claimRef.namespace}{"|"}{.spec.claimRef.name}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"\n"}{end}' 2>/dev/null || true)
[[ -n "$pv_rows" ]] || return 0
while IFS='|' read -r pv_name pv_phase pv_sc pv_claim_ns pv_claim_name pv_handle pv_gce_pd; do
[[ -n "$pv_name" ]] || continue
[[ "$pv_claim_ns" == "$NAMESPACE" && "$pv_claim_name" == "$pvc_name" ]] || continue
is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$pv_sc" || continue
local stale_reason=""
if [[ "$pv_phase" == "Released" || "$pv_phase" == "Failed" ]]; then
stale_reason="pv-phase-${pv_phase}"
elif [[ -z "$pvc_phase" ]]; then
stale_reason="missing-live-pvc"
fi
[[ -n "$stale_reason" ]] || continue
printf '%s|%s|%s|%s|%s|%s\n' "$pv_name" "$pv_phase" "$pv_sc" "$pv_handle" "$pv_gce_pd" "$stale_reason"
done <<< "$pv_rows"
}
collect_stale_gke_gitaly_disk_records_without_pv() {
local desired_sc="$1"
[[ "$desired_sc" == "standard" ]] || return 0
command -v gcloud >/dev/null 2>&1 || return 0
local pvc_name="repo-data-gitlab-gitaly-0"
local pvc_phase
pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
[[ -z "$pvc_phase" ]] || return 0
local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}"
if [[ -z "$project_id" ]]; then
project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true)
fi
local -a common_args=(--quiet)
if [[ -n "$project_id" ]]; then
common_args+=(--project "$project_id")
fi
local rows
rows=$(gcloud compute disks list "${common_args[@]}" \
--filter="labels.kubernetes-io-created-for-pvc-name=${pvc_name} AND labels.kubernetes-io-created-for-pvc-namespace=${NAMESPACE}" \
--format='csv[no-heading,separator="|"](name,zone.basename(),region.basename(),type.basename())' 2>/dev/null || true)
[[ -n "$rows" ]] || return 0
while IFS='|' read -r disk_name disk_zone disk_region disk_type; do
[[ -n "$disk_name" ]] || continue
is_gke_blocking_disk_type_for_standard_target "$disk_type" || continue
if [[ -n "$disk_zone" ]]; then
printf 'zone|%s|%s|%s|%s\n' "$disk_zone" "$disk_name" "$disk_type" "label-scan"
elif [[ -n "$disk_region" ]]; then
printf 'region|%s|%s|%s|%s\n' "$disk_region" "$disk_name" "$disk_type" "label-scan"
else
printf 'name||%s|%s|%s\n' "$disk_name" "$disk_type" "label-scan"
fi
done <<< "$rows"
}
repair_stale_gke_gitaly_dynamic_storage() {
local desired_sc="$1"
[[ "$MODE" == "k8s" ]] || return 0
[[ "$desired_sc" == "standard" ]] || return 0
is_gke_cluster_detected || return 0
local pvc_name="repo-data-gitlab-gitaly-0"
local stale_pv_records
stale_pv_records=$(collect_stale_gke_gitaly_pv_records "$desired_sc")
local stale_disk_records
stale_disk_records=$(collect_stale_gke_gitaly_disk_records_without_pv "$desired_sc")
if [[ -z "$stale_pv_records" && -z "$stale_disk_records" ]]; then
return 0
fi
local summary=""
if [[ -n "$stale_pv_records" ]]; then
while IFS='|' read -r pv_name pv_phase pv_sc _pv_handle _pv_gce_pd stale_reason; do
[[ -n "$pv_name" ]] || continue
summary+="PV ${pv_name} (phase=${pv_phase:-unknown}, sc=${pv_sc:-unknown}, reason=${stale_reason}). "
done <<< "$stale_pv_records"
fi
if [[ -n "$stale_disk_records" ]]; then
while IFS='|' read -r disk_scope disk_location disk_name disk_type disk_source; do
[[ -n "$disk_name" ]] || continue
summary+="Disk ${disk_name} (${disk_scope}:${disk_location:-n/a}, type=${disk_type:-unknown}, source=${disk_source}). "
done <<< "$stale_disk_records"
fi
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
repair_blocked "Detected stale GKE Gitaly dynamic storage artifacts blocking class '${desired_sc}'" \
"Artifacts are scoped to PVC ${pvc_name} only: ${summary}Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to auto-clean stale PV/PD leftovers from old standard-rwo/pd-balanced attempts."
fi
export GITALY_AUTOCLEAN_PERFORMED=1
log "AUTOCLEAN: repairing stale GKE Gitaly dynamic storage artifacts for PVC ${pvc_name}..."
kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
local live_pvc_sc live_pvc_phase
live_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
live_pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
if [[ -n "$live_pvc_phase" ]] && is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$live_pvc_sc" && [[ "$live_pvc_phase" != "Bound" ]]; then
log "Deleting stale live PVC ${pvc_name} (phase=${live_pvc_phase}, sc=${live_pvc_sc}) before retry..."
kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true
fi
local processed_disks="|"
local _disk_key
local _disk_ref _disk_scope _disk_location _disk_name
if [[ -n "$stale_pv_records" ]]; then
while IFS='|' read -r pv_name _pv_phase _pv_sc pv_handle pv_gce_pd _stale_reason; do
[[ -n "$pv_name" ]] || continue
log "Deleting stale Gitaly PV ${pv_name}..."
kubectl delete pv "$pv_name" --wait=false 2>/dev/null || true
_disk_ref=""
if [[ -n "$pv_handle" ]]; then
_disk_ref=$(gke_parse_disk_ref_from_volume_handle "$pv_handle" || true)
fi
if [[ -z "$_disk_ref" && -n "$pv_gce_pd" ]]; then
_disk_ref="name||${pv_gce_pd}"
fi
if [[ -n "$_disk_ref" ]]; then
IFS='|' read -r _disk_scope _disk_location _disk_name <<< "$_disk_ref"
_disk_key="${_disk_scope}|${_disk_location}|${_disk_name}"
if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
processed_disks+="${_disk_key}|"
gke_delete_disk_ref_if_present "$_disk_scope" "$_disk_location" "$_disk_name" || true
fi
fi
done <<< "$stale_pv_records"
fi
if [[ -n "$stale_disk_records" ]]; then
while IFS='|' read -r disk_scope disk_location disk_name _disk_type _disk_source; do
[[ -n "$disk_name" ]] || continue
_disk_key="${disk_scope}|${disk_location}|${disk_name}"
if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then
processed_disks+="${_disk_key}|"
gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true
fi
done <<< "$stale_disk_records"
fi
sleep 2
}
check_gitlab_post_apply_blocked() {
# --- Migrations Check ---
check_gitlab_migrations_blocked
# --- Gitaly Check ---
if [[ "$MODE" == "k8s" ]]; then
local desired_cr="${GITLAB_CR_RENDERED:-}"
if [[ -n "$desired_cr" ]]; then
local rendered_global_sc=""
local rendered_gitlab_gitaly_sc=""
local rendered_chart_gitaly_sc=""
IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")"
local desired_gitaly_node_selector
desired_gitaly_node_selector=$(echo "$desired_cr" | sed -n '/gitaly:/,/toolbox:/p' | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local desired_gitaly_storage_class="$rendered_gitlab_gitaly_sc"
log "Desired Gitaly state (from CR): nodeSelector=[${desired_gitaly_node_selector}], rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}], rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}], rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}]"
log "Internal script GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]"
if [[ "$desired_cr" == *"gandalf.prole.org"* ]]; then
repair_blocked "GitLab CR contains legacy nodeSelector" \
"Desired GitLab CR still contains gandalf.prole.org. This is a configuration bug."
fi
if [[ "$desired_cr" == *"gitlab-gitaly-static"* ]]; then
repair_blocked "GitLab CR contains legacy storageClass" \
"Desired GitLab CR still contains gitlab-gitaly-static. This is a configuration bug."
fi
if [[ -n "$desired_gitaly_storage_class" ]] && ! gitlab_storage_class_matches_expected "$desired_gitaly_storage_class" "${GITALY_STORAGE_CLASS:-}"; then
repair_blocked "GitLab CR storageClass mismatch" \
"Desired GitLab CR storageClass=[${desired_gitaly_storage_class}] does not match GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]. Configuration bug."
fi
if [[ -n "$rendered_chart_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_chart_gitaly_sc" "${GITALY_STORAGE_CLASS:-}"; then
repair_blocked "GitLab CR chart-level Gitaly storageClass mismatch" \
"Rendered gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc}] does not match GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]. Configuration bug."
fi
enforce_gitlab_storage_class_target "${GITALY_STORAGE_CLASS:-}" "$desired_cr"
fi
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
# If autoclean was performed, wait for convergence before checking live state.
# This prevents false positives when the operator hasn't yet updated the stale StatefulSet.
if [[ "${GITALY_AUTOCLEAN_PERFORMED:-0}" == "1" ]]; then
log "AUTOCLEAN was performed. Waiting for Gitaly StatefulSet to converge (removing legacy storage)..."
local grace_start=$(date +%s)
local grace_timeout=60
local converged=0
while true; do
local live_sts_yaml
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -z "$live_sts_yaml" ]]; then
log "Gitaly StatefulSet not found (awaiting operator action)..."
elif [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
log "Gitaly StatefulSet converged to corrected state (clean nodeSelector/storageClass)."
local _live_sc
_live_sc=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
if [[ -n "$_live_sc" ]] && ! gitlab_storage_class_matches_expected "$_live_sc" "${GITALY_STORAGE_CLASS:-}"; then
log "WARNING: Converged StatefulSet uses storageClass=[${_live_sc}], expected [${GITALY_STORAGE_CLASS:-}]."
fi
converged=1
break
else
local live_gitaly_node_selector
live_gitaly_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local live_gitaly_storage_class
live_gitaly_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
log "Still waiting for Gitaly convergence (live nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}])..."
fi
if (( $(date +%s) - grace_start > grace_timeout )); then
log "Grace period (${grace_timeout}s) expired."
break
fi
sleep 15
done
if [[ "$converged" == "0" ]]; then
local live_sts_yaml
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$live_sts_yaml" && ( "$live_sts_yaml" == *"gandalf.prole.org"* || "$live_sts_yaml" == *"gitlab-gitaly-static"* ) ]]; then
log "Live StatefulSet still legacy after grace period. Performing explicit replacement..."
# We already confirmed desired CR is corrected at the start of this function.
kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true
log "Legacy StatefulSet deleted. Waiting for operator recreation..."
local recreate_start=$(date +%s)
local recreate_timeout=300
while true; do
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$live_sts_yaml" ]]; then
local recreated_node_selector
recreated_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local recreated_storage_class
recreated_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
log "StatefulSet recreated. nodeSelector=[${recreated_node_selector}], storageClass=[${recreated_storage_class}]"
if [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
if [[ -n "$recreated_storage_class" ]] && ! gitlab_storage_class_matches_expected "$recreated_storage_class" "${GITALY_STORAGE_CLASS:-}"; then
repair_blocked "Recreated Gitaly StatefulSet uses wrong storageClass" \
"Resource: statefulset/${gitaly_sts_name}. Value: storageClassName=[${recreated_storage_class}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check operator reconciliation."
fi
log "Gitaly StatefulSet recreated in corrected state."
converged=1
break
else
log "Recreated StatefulSet STILL contains legacy fields. Waiting for operator to correct it..."
fi
fi
if (( $(date +%s) - recreate_start > recreate_timeout )); then
repair_blocked "Gitaly failed to recreate clean StatefulSet after ${recreate_timeout}s" \
"Resource: statefulset/${gitaly_sts_name}. Fix: Check operator logs and desired GitLab CR."
fi
sleep 15
done
else
log "No legacy StatefulSet found after grace period (may have been deleted or converged)."
converged=1
fi
fi
if [[ "$converged" == "1" ]]; then
# Wait for PVC provisioning if autoclean was performed
log "Waiting for PVC repo-data-gitlab-gitaly-0 to be provisioned and Bound..."
local pvc_name="repo-data-gitlab-gitaly-0"
local pvc_start=$(date +%s)
local pvc_timeout=600
local pvc_uid=""
while true; do
local pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
local pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
if [[ -n "$pvc_phase" ]]; then
pvc_uid=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
local _sc_info=""
if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-balanced / wrong-class for strict standard target)"; fi
if [[ "$pvc_sc" == "premium-rwo" ]]; then _sc_info=" (pd-ssd)"; fi
if [[ "$pvc_sc" == "standard" ]]; then _sc_info=" (pd-standard)"; fi
log "PVC ${pvc_name}: storageClass=[${pvc_sc}${_sc_info}], phase=[${pvc_phase}]"
if [[ -n "$pvc_sc" ]] && ! gitlab_storage_class_matches_expected "$pvc_sc" "${GITALY_STORAGE_CLASS:-}"; then
repair_blocked "Gitaly PVC uses wrong storageClass" \
"PVC: ${pvc_name}. Value: storageClass=[${pvc_sc}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check StatefulSet volumeClaimTemplates and operator reconciliation."
fi
if [[ "$pvc_phase" == "Bound" ]]; then
if [[ "$pvc_sc" == "${GITALY_STORAGE_CLASS:-}" ]]; then
log "PVC ${pvc_name} successfully provisioned on '${pvc_sc}' storage."
break
fi
fi
# Check provisioning failures only for the *current* PVC object.
local provisioning_fail=""
if [[ -n "$pvc_uid" ]]; then
provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
fi
if [[ -n "$provisioning_fail" ]]; then
if [[ "$provisioning_fail" == *"quota"* || "$provisioning_fail" == *"QUOTA"* ]]; then
repair_blocked "Gitaly PVC provisioning failed (Quota Exceeded)" \
"PVC: ${pvc_name}. Error: ${provisioning_fail}. Fix: Check GKE storage quotas and ensure Gitaly uses storageClass='${GITALY_STORAGE_CLASS:-standard}'."
else
log "PVC ${pvc_name} provisioning event: ${provisioning_fail}"
fi
fi
else
# Check migrations first to fail fast
check_gitlab_migrations_blocked
log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..."
fi
if (( $(date +%s) - pvc_start > pvc_timeout )); then
local last_msg=""
if [[ -n "$pvc_uid" ]]; then
last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
else
last_msg="No live PVC UID observed yet; ignoring stale historical PVC events from previous claims."
fi
repair_blocked "Gitaly PVC failed to bind after ${pvc_timeout}s" \
"PVC: ${pvc_name}. Status: ${pvc_phase:-NotFound}. Last event: ${last_msg}. Fix: Check storage provider and quota."
fi
sleep 15
done
export GITALY_AUTOCLEAN_PERFORMED=0
fi
# Also wait for PV deletion to settle if it exists
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
log "Waiting for legacy PV gitlab-gitaly-synology to be removed..."
local pv_wait_start=$(date +%s)
while kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; do
if (( $(date +%s) - pv_wait_start > 120 )); then
log "Legacy PV still exists after 120s; continuing (it may be stuck in Terminating)."
break
fi
sleep 5
done
fi
fi
# Final live state checks
local gitaly_sts_yaml
gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$gitaly_sts_yaml" ]]; then
local rendered_global_sc=""
local rendered_gitlab_gitaly_sc=""
local rendered_chart_gitaly_sc=""
IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")"
local live_gitaly_node_selector
live_gitaly_node_selector=$(echo "$gitaly_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local live_gitaly_storage_class
live_gitaly_storage_class=$(echo "$gitaly_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
local live_gitaly_repo_data_template_sc
live_gitaly_repo_data_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class)
log "Live Gitaly state: nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}]"
log "Live Gitaly StatefulSet repo-data claim template storageClassName=[${live_gitaly_repo_data_template_sc:-<unset>}]"
if ! gitlab_storage_class_matches_expected "$live_gitaly_repo_data_template_sc" "${GITALY_STORAGE_CLASS:-}"; then
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then
repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass mismatch" \
"Configured target=[${GITALY_STORAGE_CLASS:-}] rendered.global.persistence.storageClass=[${rendered_global_sc:-<unset>}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-<unset>}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-<unset>}] live.statefulset=${gitaly_sts_name} live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_repo_data_template_sc:-<unset>}]. Delete statefulset/${gitaly_sts_name}, pvc/repo-data-${GITLAB_RELEASE}-gitaly-0, and wrong-class PV+disk artifacts or set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 for automatic destructive repair."
fi
local gitaly_pvc_mismatches
gitaly_pvc_mismatches=$(collect_gitlab_storage_mismatch_records "${GITALY_STORAGE_CLASS:-}")
cleanup_gitlab_wrong_gitaly_template_storage "${GITALY_STORAGE_CLASS:-}" "$live_gitaly_repo_data_template_sc" "$gitaly_pvc_mismatches" "$desired_cr"
fi
if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* ]]; then
repair_blocked "Gitaly using legacy Synology storage" \
"Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Ensure GITLAB_STORAGE_NODE is not set in k8s mode."
fi
if [[ "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
repair_blocked "Gitaly using legacy Synology storage" \
"Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden in k8s mode."
fi
fi
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
repair_blocked "Gitaly using legacy Synology storage" \
"Resource: pv/gitlab-gitaly-synology. Value: exists. Fix: Delete legacy PV or use GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1."
fi
local unschedulable_gitaly
unschedulable_gitaly=$(kubectl -n "$NAMESPACE" get pods -l "app=gitaly" -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="PodScheduled")].status=="False")]}{.metadata.name}:{.status.conditions[?(@.type=="PodScheduled")].reason}{"\n"}{end}' 2>/dev/null | grep "Unschedulable" || true)
if [[ -n "$unschedulable_gitaly" ]]; then
repair_blocked "Gitaly pod(s) unschedulable" \
"Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storageClass='${GITALY_STORAGE_CLASS:-standard}' and no hostname nodeSelector. Ensure wrong-class PV/PVC artifacts were repaired."
fi
# --- Sidekiq Config Check ---
log "Desired Sidekiq state: concurrency=${GITLAB_SIDEKIQ_CONCURRENCY}, requests=[cpu=${GITLAB_SIDEKIQ_REQUESTS_CPU}, mem=${GITLAB_SIDEKIQ_REQUESTS_MEMORY}], limits=[cpu=${GITLAB_SIDEKIQ_LIMITS_CPU}, mem=${GITLAB_SIDEKIQ_LIMITS_MEMORY}]"
if [[ "${gitlab_apply_changed:-0}" == "1" || "${gitlab_spec_changed:-0}" == "1" ]]; then
log "GitLab CR was updated to fix configuration drift (Sidekiq or other fields)."
fi
local sidekiq_deploy_name="${GITLAB_RELEASE}-sidekiq-all-in-1-v2"
local live_sidekiq_cpu_req
live_sidekiq_cpu_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}' 2>/dev/null || true)
local live_sidekiq_mem_req
live_sidekiq_mem_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.memory}' 2>/dev/null || true)
local live_sidekiq_cpu_lim
live_sidekiq_cpu_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.cpu}' 2>/dev/null || true)
local live_sidekiq_mem_lim
live_sidekiq_mem_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.memory}' 2>/dev/null || true)
local live_sidekiq_concurrency
live_sidekiq_concurrency=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="SIDEKIQ_CONCURRENCY")].value}' 2>/dev/null || true)
if [[ -n "$live_sidekiq_cpu_req" ]]; then
log "Live Sidekiq state: concurrency=${live_sidekiq_concurrency}, requests=[cpu=${live_sidekiq_cpu_req}, mem=${live_sidekiq_mem_req}], limits=[cpu=${live_sidekiq_cpu_lim}, mem=${live_sidekiq_mem_lim}]"
fi
fi
# --- Registry Config Check ---
local registry_secret_config
registry_secret_config=$(kubectl -n "$NAMESPACE" get secret "${GITLAB_RELEASE}-registry-storage" -o jsonpath='{.data.config}' 2>/dev/null | base64 -d 2>/dev/null || true)
if [[ "$registry_secret_config" == *"<private-db-cluster-garage-endpoint>"* ]]; then
if [[ "$GARAGE_S3_ENDPOINT" != *"<private-db-cluster-garage-endpoint>"* ]]; then
log "Registry secret still contains placeholder but config is updated. Re-applying secret..."
# Re-triggering the secret creation (this is safer than just blocking)
setup_garage_for_gitlab
else
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder in live secret)" \
"Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint."
fi
fi
# --- Registry Logs Check ---
local registry_pod
registry_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=registry" -o name 2>/dev/null | head -n1 || true)
if [[ -n "$registry_pod" ]]; then
local logs
logs=$(kubectl -n "$NAMESPACE" logs "$registry_pod" --tail=100 2>&1 || true)
if [[ "$logs" == *"DNS failure"* || "$logs" == *"AccessDenied"* || "$logs" == *"No such key"* ]]; then
# If it's a "No such key" or "AccessDenied", and we have a custom endpoint, it might be the WRONG cluster Garage.
repair_blocked "Registry endpoint invalid or wrong Garage (S3 error detected)" \
"Logs: ${logs}. Fix: Verify GARAGE_S3_ENDPOINT points to the DB cluster Garage (not the APP cluster one)."
fi
fi
# --- KAS Logs Check ---
local kas_pod
kas_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=kas" -o name 2>/dev/null | head -n1 || true)
if [[ -n "$kas_pod" ]]; then
local logs
logs=$(kubectl -n "$NAMESPACE" logs "$kas_pod" --tail=100 2>&1 || true)
if [[ "$logs" == *"no such host"* && "$logs" == *"redis"* ]]; then
repair_blocked "Redis host does not resolve (detected in KAS logs)" \
"Logs: ${logs}. Fix: Verify REDIS_HOST (${REDIS_HOST}) and ensure Redis service is healthy."
fi
fi
}
kubectl_db() {
local db_ctx=""
db_ctx="$(resolve_db_cluster_context || true)"
if [[ -n "$db_ctx" ]]; then
command kubectl --context "$db_ctx" "$@"
else
kubectl "$@"
fi
}
resolve_garage_admin_context() {
if [[ "$MODE" == "k8s" ]]; then
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
local explicit_ctx=""
explicit_ctx="$(resolve_explicit_kube_context || true)"
if [[ -n "$db_ctx" && -n "$app_ctx" && "$db_ctx" != "$app_ctx" ]]; then
printf '%s' "$db_ctx"
return 0
fi
if [[ -n "$db_ctx" ]]; then
printf '%s' "$db_ctx"
return 0
fi
if [[ -n "$explicit_ctx" ]]; then
printf '%s' "$explicit_ctx"
return 0
fi
if [[ -n "$app_ctx" ]]; then
printf '%s' "$app_ctx"
return 0
fi
die "Explicit Garage admin context is required in k8s mode."
fi
return 1
}
kubectl_garage_admin() {
local garage_ctx=""
garage_ctx="$(resolve_garage_admin_context || true)"
if [[ -n "$garage_ctx" ]]; then
command kubectl --context "$garage_ctx" "$@"
else
kubectl "$@"
fi
}
db_kubectl() {
local db_ctx
db_ctx="$(resolve_db_cluster_context || true)"
if [[ -n "$db_ctx" ]]; then
command kubectl --context "$db_ctx" "$@"
else
kubectl "$@"
fi
}
ensure_cross_cluster_db_host() {
if [[ "$MODE" != "k8s" ]]; then
return 0
fi
local app_ctx db_ctx
app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
if [[ -z "$app_ctx" ]]; then
app_ctx="$(kubectl config current-context 2>/dev/null || true)"
fi
db_ctx="$(resolve_db_cluster_context || true)"
if [[ -z "$db_ctx" || -z "$app_ctx" || "$db_ctx" == "$app_ctx" ]]; then
return 0
fi
local db_cluster_name="${CNPG_CLUSTER_NAME:-knoe-db}"
local db_ns="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
local ilb_service="${GITLAB_DB_ILB_SERVICE:-knoe-db-rw-ilb}"
log "Split-cluster mode detected (APP: ${app_ctx}, DB: ${db_ctx})"
log "Ensuring cross-cluster Postgres ILB service '${ilb_service}' in namespace '${db_ns}'..."
db_kubectl -n "$db_ns" apply -f - <<EOF >/dev/null
apiVersion: v1
kind: Service
metadata:
name: ${ilb_service}
annotations:
networking.gke.io/load-balancer-type: "Internal"
spec:
type: LoadBalancer
selector:
cnpg.io/cluster: ${db_cluster_name}
cnpg.io/instanceRole: primary
ports:
- name: postgres
port: 5432
targetPort: 5432
protocol: TCP
EOF
local db_host=""
local attempts=0
log "Waiting for cross-cluster DB host (ILB IP/hostname)..."
while (( attempts < 60 )); do
attempts=$((attempts + 1))
db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)"
if [[ -z "$db_host" ]]; then
db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)"
fi
if [[ -n "$db_host" ]]; then
break
fi
sleep 5
done
if [[ -z "$db_host" ]]; then
warn "Cross-cluster Postgres ILB '${ilb_service}' has no ingress address yet after 5 minutes."
return 0
fi
log "Successfully resolved cross-cluster DB host: ${db_host}"
GITLAB_CROSS_CLUSTER_DB_HOST="$db_host"
export GITLAB_CROSS_CLUSTER_DB_HOST
}
enforce_app_cluster_targeting
is_truthy() {
case "${1:-}" in
1|true|TRUE|True|yes|YES|on|ON|y|Y)
return 0
;;
*)
return 1
;;
esac
}
assert_public_ingress_targeting() {
local ingress_class="${1:-}"
shift || true
local hosts=("$@")
if [[ "$MODE" != "k8s" ]]; then
return 0
fi
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
local active_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
if [[ -z "$app_ctx" || -z "$active_ctx" ]]; then
die "Explicit APP cluster context is required for GitLab ingress operations in k8s mode."
fi
local host_count=0
local h
for h in "${hosts[@]}"; do
[[ -n "${h:-}" ]] && host_count=$((host_count + 1))
done
if [[ "$host_count" -gt 0 && -n "$db_ctx" && "$active_ctx" == "$db_ctx" ]]; then
die "Refusing to render/apply GitLab public ingress in DB cluster context '${active_ctx}' (hosts: ${hosts[*]})."
fi
if [[ "$host_count" -gt 0 && -n "$app_ctx" && -n "$active_ctx" && "$active_ctx" != "$app_ctx" ]]; then
die "GitLab public ingress must target APP cluster context '${app_ctx}', active context is '${active_ctx}'."
fi
local normalized_class="${ingress_class,,}"
if [[ "$normalized_class" == traefik* ]] && ! is_truthy "${ALLOW_TRAEFIK_PUBLIC_INGRESS:-${GITLAB_ALLOW_TRAEFIK_INGRESS:-0}}"; then
die "Ingress class '${ingress_class}' is incompatible with k8s mode unless Traefik public ingress is explicitly enabled."
fi
}
assert_unique_ingress_host_claims() {
local ingress_name="${1:-}"
local ingress_namespace="${2:-}"
local host_csv="${3:-}"
local gitlab_release="${4:-}"
local expected_backend_service="${5:-}"
[[ -n "$host_csv" ]] || return 0
local target_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then
die "Explicit kubectl context is required for ingress ownership checks in k8s mode."
fi
INGRESS_HOST_CLAIM_DETAILS=""
local claim_result claim_status
claim_result=$(python3 - "$host_csv" "$ingress_namespace" "$ingress_name" "$target_ctx" "$gitlab_release" "$expected_backend_service" <<'PY'
import json
import subprocess
import sys
requested_hosts = {h.strip().lower() for h in (sys.argv[1] or "").split(",") if h.strip()}
target_ns = sys.argv[2]
target_name = sys.argv[3]
target_ctx = (sys.argv[4] or "").strip()
target_release = (sys.argv[5] or "").strip()
expected_backend_service = (sys.argv[6] or "").strip()
cmd = ["kubectl"]
if target_ctx:
cmd.extend(["--context", target_ctx])
cmd.extend(["get", "ingress", "-A", "-o", "json"])
try:
raw = subprocess.check_output(cmd, text=True)
except Exception:
print("OK|")
raise SystemExit(0)
if not raw.strip():
print("OK|")
raise SystemExit(0)
try:
payload = json.loads(raw)
except Exception:
print("OK|")
raise SystemExit(0)
same_owner_conflicts: list[str] = []
foreign_conflicts: list[str] = []
for item in payload.get("items", []) or []:
md = item.get("metadata", {}) or {}
ns = (md.get("namespace") or "").strip()
name = (md.get("name") or "").strip()
if ns == target_ns and name == target_name:
continue
labels = md.get("labels", {}) or {}
spec = item.get("spec", {}) or {}
rules = spec.get("rules", []) or []
for rule in rules:
host = (rule.get("host") or "").strip().lower()
if not host or host not in requested_hosts:
continue
http = rule.get("http", {}) or {}
paths = http.get("paths", []) or [{"path": "/"}]
for path_item in paths:
path = (path_item.get("path") or "/").strip() or "/"
if path not in {"/", ""}:
continue
backend = path_item.get("backend", {}) or {}
backend_svc = backend.get("service", {}) or {}
backend_name = (backend_svc.get("name") or "").strip()
same_owner = False
if ns == target_ns:
if expected_backend_service and backend_name == expected_backend_service:
same_owner = True
if target_release and backend_name.startswith(f"{target_release}-webservice"):
same_owner = True
instance_label = (labels.get("app.kubernetes.io/instance") or "").strip()
part_of_label = (labels.get("app.kubernetes.io/part-of") or "").strip().lower()
if target_release and (
name == f"{target_release}-webservice-default"
or name.startswith(f"{target_release}-")
or instance_label == target_release
or part_of_label == "gitlab"
):
same_owner = True
owner_desc = f"{host}{path} already owned by {ns}/{name}"
if backend_name:
owner_desc += f" (backend={backend_name})"
if same_owner:
same_owner_conflicts.append(owner_desc)
else:
foreign_conflicts.append(owner_desc)
if foreign_conflicts:
print("CONFLICT|" + "; ".join(sorted(set(foreign_conflicts))))
elif same_owner_conflicts:
print("OWNED_BY_GITLAB|" + "; ".join(sorted(set(same_owner_conflicts))))
else:
print("OK|")
PY
)
claim_status="${claim_result%%|*}"
INGRESS_HOST_CLAIM_DETAILS="${claim_result#*|}"
case "$claim_status" in
OK|"")
return 0
;;
OWNED_BY_GITLAB)
return 10
;;
CONFLICT)
return 11
;;
*)
INGRESS_HOST_CLAIM_DETAILS="$claim_result"
return 11
;;
esac
}
while [[ $# -gt 0 ]]; do
case "$1" in
--mode) MODE="$(prole_normalize_mode "${2:-}")"; shift 2 ;;
--mode=*) MODE="$(prole_normalize_mode "${1#*=}")"; shift 1 ;;
-n|--namespace) NAMESPACE="${2:-}"; shift 2 ;;
--namespace=*) NAMESPACE="${1#*=}"; shift 1 ;;
-c|--config) CFG_PATH="${2:-}"; shift 2 ;;
--config=*) CFG_PATH="${1#*=}"; shift 1 ;;
--node-selector) NODE_SELECTOR="${2:-}"; shift 2 ;;
--node-selector=*) NODE_SELECTOR="${1#*=}"; shift 1 ;;
--force) FORCE=1; shift 1 ;;
-h|--help) usage; exit 0 ;;
--trace-namespace)
# Resolve namespace then exit — used by diagnostic scripts only
_TRACE_NS=1; shift 1 ;;
*) break ;;
esac
done
# ---------------------------------------------------------------------------
# Resolve config path
# ---------------------------------------------------------------------------
if [[ -z "$CFG_PATH" && -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then
CFG_PATH="$(_prole_cfg_select_cfg_file "${PROLE_CONF}")"
fi
if [[ -z "$CFG_PATH" ]]; then
CFG_PATH="$(_prole_cfg_select_cfg_file "$SCRIPT_DIR/../conf")"
fi
# cfg-based namespace override (only GITLAB_NAMESPACE; GITOPS_NAMESPACE intentionally excluded)
if [[ -z "$NAMESPACE" || "$NAMESPACE" == "gitlab" ]] && [[ -n "$CFG_PATH" ]]; then
maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")"
[[ -n "$maybe_ns" ]] && NAMESPACE="$maybe_ns"
fi
# Final safety net — always default to 'gitlab'
NAMESPACE="${NAMESPACE:-gitlab}"
export GITLAB_NAMESPACE="$NAMESPACE"
# Diagnostic exit — used by tmp/sim_installer_ns.sh
if [[ "${_TRACE_NS:-0}" == "1" ]]; then
echo "TRACE: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" >&2
echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-<unset>}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-<unset>}'" >&2
exit 0
fi
case "$MODE" in
k3d|k3s|k8s|local) ;;
*) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;;
esac
export PROLE_MODE="$MODE"
command -v kubectl >/dev/null || die "kubectl not found"
command -v helm >/dev/null || die "helm not found (required for GitLab Operator install)"
# ---------------------------------------------------------------------------
# knoe-db (CNPG) settings — reuse the same cluster gitea uses
# ---------------------------------------------------------------------------
DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-${CLUSTER_NAME:-knoe-db}}"
ensure_cross_cluster_db_host
GITLAB_DB_NAME="${GITLAB_DB_NAME:-gitlabhq_production}"
GITLAB_DB_USER="${GITLAB_DB_USER:-gitlab}"
GITLAB_DB_PASSWORD="${GITLAB_DB_PASSWORD:-}"
DB_HOST="${GITLAB_DB_HOST:-${GITLAB_CROSS_CLUSTER_DB_HOST:-${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local}}"
DB_PORT="${GITLAB_DB_PORT:-5432}"
# ---------------------------------------------------------------------------
# Redis — shared common service (deployed by init_redis.sh in knoe-system)
# ---------------------------------------------------------------------------
REDIS_NAMESPACE="${REDIS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}"
REDIS_HOST="${GITLAB_REDIS_HOST:-redis-master.${REDIS_NAMESPACE}.svc.cluster.local}"
REDIS_PORT="${GITLAB_REDIS_PORT:-6379}"
# GitLab public domain (configurable; default follows deployment mode)
_default_gitlab_domain="git.prole.org"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_domain="git.knoe.dev"
fi
GITLAB_DOMAIN="${GITLAB_DOMAIN:-${GITLAB_HOSTNAME:-$_default_gitlab_domain}}"
GITLAB_PUBLIC_HOSTS_RAW="${GITLAB_PUBLIC_HOSTS:-$GITLAB_DOMAIN}"
trim_csv_token() {
local token="$1"
token="${token#${token%%[![:space:]]*}}"
token="${token%${token##*[![:space:]]}}"
printf '%s' "$token"
}
_gitlab_hosts_csv=""
GITLAB_PUBLIC_HOSTS=()
IFS=',' read -r -a _gitlab_host_candidates <<< "$GITLAB_PUBLIC_HOSTS_RAW"
for _gitlab_host in "${_gitlab_host_candidates[@]}"; do
_gitlab_host="$(trim_csv_token "$_gitlab_host")"
[[ -n "$_gitlab_host" ]] || continue
case ",${_gitlab_hosts_csv}," in
*,"${_gitlab_host}",*) ;;
*)
GITLAB_PUBLIC_HOSTS+=("$_gitlab_host")
_gitlab_hosts_csv="${_gitlab_hosts_csv:+${_gitlab_hosts_csv},}${_gitlab_host}"
;;
esac
done
unset _gitlab_host_candidates _gitlab_host
if [[ ${#GITLAB_PUBLIC_HOSTS[@]} -eq 0 ]]; then
GITLAB_PUBLIC_HOSTS=("$GITLAB_DOMAIN")
fi
PRIMARY_GITLAB_HOST="${GITLAB_PUBLIC_HOSTS[0]}"
GITLAB_DOMAIN="$PRIMARY_GITLAB_HOST"
PRIMARY_GITLAB_DOMAIN_ROOT="${PRIMARY_GITLAB_HOST#*.}"
if [[ "$PRIMARY_GITLAB_DOMAIN_ROOT" == "$PRIMARY_GITLAB_HOST" || -z "$PRIMARY_GITLAB_DOMAIN_ROOT" ]]; then
PRIMARY_GITLAB_DOMAIN_ROOT="prole.org"
fi
# Public ingress class (k8s/GKE defaults to gce; local clusters keep kong)
_default_gitlab_ingress_class="kong"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_ingress_class="gce"
fi
GITLAB_INGRESS_CLASS="${GITLAB_INGRESS_CLASS:-$_default_gitlab_ingress_class}"
# Single GitLab front-door owner model.
# - operator: GitLab chart/operator-managed ingress owns ${GITLAB_PUBLIC_HOSTS}
# - fallback: custom fallback ingress (gitlab-kong-ingress) owns ${GITLAB_PUBLIC_HOSTS}
# In k8s/GKE mode, default to operator ownership to avoid dual-ingress host claims.
_default_gitlab_frontdoor_owner="fallback"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_frontdoor_owner="operator"
fi
GITLAB_FRONTDOOR_OWNER="${GITLAB_FRONTDOOR_OWNER:-$_default_gitlab_frontdoor_owner}"
GITLAB_FRONTDOOR_OWNER="${GITLAB_FRONTDOOR_OWNER,,}"
case "$GITLAB_FRONTDOOR_OWNER" in
operator|fallback)
;;
*)
die "Unsupported GITLAB_FRONTDOOR_OWNER='${GITLAB_FRONTDOOR_OWNER}'. Supported values: operator, fallback."
;;
esac
# Trusted proxies used by GitLab Rails/Workhorse to accept forwarded host/proto
# from ingress/load-balancer hops.
_default_gitlab_trusted_proxies="127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_trusted_proxies+=" ,130.211.0.0/22,35.191.0.0/16"
fi
_default_gitlab_trusted_proxies="${_default_gitlab_trusted_proxies// /}"
GITLAB_TRUSTED_PROXIES_RAW="${GITLAB_TRUSTED_PROXIES:-${_default_gitlab_trusted_proxies}}"
_gitlab_trusted_proxies_csv=""
GITLAB_TRUSTED_PROXIES=()
IFS=',' read -r -a _gitlab_trusted_proxy_candidates <<< "$GITLAB_TRUSTED_PROXIES_RAW"
for _gitlab_trusted_proxy in "${_gitlab_trusted_proxy_candidates[@]}"; do
_gitlab_trusted_proxy="$(trim_csv_token "$_gitlab_trusted_proxy")"
[[ -n "$_gitlab_trusted_proxy" ]] || continue
case ",${_gitlab_trusted_proxies_csv}," in
*,"${_gitlab_trusted_proxy}",*) ;;
*)
GITLAB_TRUSTED_PROXIES+=("$_gitlab_trusted_proxy")
_gitlab_trusted_proxies_csv="${_gitlab_trusted_proxies_csv:+${_gitlab_trusted_proxies_csv},}${_gitlab_trusted_proxy}"
;;
esac
done
unset _gitlab_trusted_proxy_candidates _gitlab_trusted_proxy
if [[ ${#GITLAB_TRUSTED_PROXIES[@]} -eq 0 ]]; then
GITLAB_TRUSTED_PROXIES=("127.0.0.1/32")
_gitlab_trusted_proxies_csv="127.0.0.1/32"
fi
GITLAB_WEBSERVICE_INGRESS_ENABLED="${GITLAB_WEBSERVICE_INGRESS_ENABLED:-true}"
if [[ "$MODE" == "k8s" ]]; then
if [[ "$GITLAB_FRONTDOOR_OWNER" == "operator" ]]; then
GITLAB_WEBSERVICE_INGRESS_ENABLED=true
elif [[ "$GITLAB_FRONTDOOR_OWNER" == "fallback" ]]; then
GITLAB_WEBSERVICE_INGRESS_ENABLED=false
fi
fi
# Google Workspace OIDC — FRONTDOOR_HOST gates OmniAuth configuration.
# On k3s: api.prole.org is the prole-auth SSO gateway (knoe-auth service).
# Requires k8s secret 'gitlab-google-oidc' in GITLAB_NAMESPACE with Google
# OAuth2 client credentials (client_id, client_secret, redirect_uri).
# To disable OIDC: unset FRONTDOOR_HOST before running.
_default_auth_hostname="api.prole.org"
if [[ "$MODE" == "k8s" ]]; then
_default_auth_hostname="api.knoe.dev"
fi
AUTH_HOSTNAME="${AUTH_HOSTNAME:-${FRONTDOOR_HOST:-${_default_auth_hostname}}}"
FRONTDOOR_HOST="${FRONTDOOR_HOST:-$AUTH_HOSTNAME}"
FRONTDOOR_AUTH_ENABLED="${FRONTDOOR_AUTH_ENABLED:-${AUTHORITY_ENABLED:-1}}"
AUTH_VERIFY_PATH="${AUTH_VERIFY_PATH:-/auth/verify}"
AUTH_LOGIN_PATH="${AUTH_LOGIN_PATH:-/auth/login}"
AUTH_RESPONSE_HEADERS="${AUTH_RESPONSE_HEADERS:-X-Prole-User,X-Prole-Email,X-Prole-Groups}"
AUTH_VERIFY_URL="${AUTH_VERIFY_URL:-https://${AUTH_HOSTNAME}${AUTH_VERIFY_PATH}}"
AUTH_SIGNIN_URL="${AUTH_SIGNIN_URL:-https://${AUTH_HOSTNAME}${AUTH_LOGIN_PATH}?next=\$scheme://\$host\$escaped_request_uri}"
_default_gitlab_oidc_issuer="https://${AUTH_HOSTNAME}/auth"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_oidc_issuer="https://api.knoe.dev/auth"
fi
_default_gitlab_oidc_redirect_uri="https://${GITLAB_DOMAIN}/users/auth/openid_connect/callback"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_oidc_redirect_uri="https://git.knoe.dev/users/auth/openid_connect/callback"
fi
GITLAB_OIDC_PROVIDER_NAME="${GITLAB_OIDC_PROVIDER_NAME:-openid_connect}"
GITLAB_OIDC_ISSUER="${GITLAB_OIDC_ISSUER:-${OIDC_ISSUER:-${_default_gitlab_oidc_issuer}}}"
GITLAB_OIDC_REDIRECT_URI="${GITLAB_OIDC_REDIRECT_URI:-${_default_gitlab_oidc_redirect_uri}}"
GITLAB_OIDC_CLIENT_ID="${GITLAB_OIDC_CLIENT_ID:-${OIDC_CLIENT_ID:-${GOOGLE_OIDC_CLIENT_ID:-${GOOGLE_CLIENT_ID:-}}}}"
GITLAB_OIDC_CLIENT_SECRET="${GITLAB_OIDC_CLIENT_SECRET:-${OIDC_CLIENT_SECRET:-${GOOGLE_OIDC_CLIENT_SECRET:-${GOOGLE_CLIENT_SECRET:-}}}}"
is_unresolved_secret_ref() {
local raw="${1:-}"
case "$raw" in
""|secretref://*|'${OPENBAO:'*|'${PROLE_SECRET:'*)
return 0
;;
esac
return 1
}
GITLAB_RELEASE="gitlab"
gitlab_split_cluster_ownership_diagnostics
assert_public_ingress_targeting "$GITLAB_INGRESS_CLASS" "${GITLAB_PUBLIC_HOSTS[@]}"
# ---------------------------------------------------------------------------
# Garage S3 (knoe-system) — replaces embedded minio for object storage
# ---------------------------------------------------------------------------
GARAGE_NAMESPACE="${GARAGE_NAMESPACE:-knoe-system}"
GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-}"
GARAGE_S3_ENDPOINT="${GARAGE_S3_ENDPOINT:-${GARAGE_PRIVATE_S3_ENDPOINT:-${GARAGE_PRIVATE_ENDPOINT:-}}}"
if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
"Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint in your config."
fi
if [[ -z "$GARAGE_S3_ENDPOINT" ]]; then
if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]]; then
die "GARAGE_S3_ENDPOINT (or GARAGE_PRIVATE_S3_ENDPOINT) must be set to an explicit private cross-cluster endpoint when APP and DB clusters differ."
fi
GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-garage.${GARAGE_NAMESPACE}.svc.cluster.local}"
GARAGE_S3_ENDPOINT="http://${GARAGE_SVC_HOST}:3900"
fi
if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]] && [[ "$GARAGE_S3_ENDPOINT" == *".svc.cluster.local"* ]]; then
die "GARAGE_S3_ENDPOINT must use a private cross-cluster endpoint and cannot use cluster-local service DNS (${GARAGE_S3_ENDPOINT})."
fi
if [[ -z "$GARAGE_SVC_HOST" ]]; then
_garage_endpoint_hostport="${GARAGE_S3_ENDPOINT#http://}"
_garage_endpoint_hostport="${_garage_endpoint_hostport#https://}"
_garage_endpoint_hostport="${_garage_endpoint_hostport%%/*}"
GARAGE_SVC_HOST="${_garage_endpoint_hostport%%:*}"
unset _garage_endpoint_hostport
fi
GARAGE_S3_KEY_NAME="${GARAGE_S3_KEY_NAME:-gitlab-s3}"
GITLAB_OBJECT_STORAGE_REQUIRED="${GITLAB_OBJECT_STORAGE_REQUIRED:-1}"
# ---------------------------------------------------------------------------
# Operator chart coordinates
# ---------------------------------------------------------------------------
OPERATOR_REPO_NAME="gitlab-operator"
OPERATOR_REPO_URL="https://gitlab.com/api/v4/projects/18899486/packages/helm/stable"
OPERATOR_CHART="gitlab-operator/gitlab-operator"
OPERATOR_RELEASE="gitlab-operator"
OPERATOR_NAMESPACE="${GITLAB_OPERATOR_NAMESPACE:-${NAMESPACE}}"
GITLAB_OPERATOR_CHART_VERSION="${GITLAB_OPERATOR_CHART_VERSION:-}"
# GitLab Helm chart version required in the CR; auto-detected if not set.
GITLAB_CHART_VERSION="${GITLAB_CHART_VERSION:-}"
GITLAB_CHART_REPO_NAME="gitlab"
GITLAB_CHART_REPO_URL="https://charts.gitlab.io/"
gitlab_db_secret_changed=0
gitlab_object_storage_secret_changed=0
gitlab_registry_storage_secret_changed=0
resolve_helm_release_chart_version() {
local release_namespace="$1"
local release_name="$2"
local chart_version
chart_version=$({ helm -n "$release_namespace" list -f "^${release_name}$" -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys
try:
rows = json.load(sys.stdin)
except Exception:
print("")
raise SystemExit(0)
if not rows:
print("")
raise SystemExit(0)
chart = str(rows[0].get("chart", ""))
if not chart:
print("")
raise SystemExit(0)
if "-" in chart:
print(chart.rsplit("-", 1)[-1])
else:
print("")
PY
)
if [[ -n "$chart_version" ]]; then
printf '%s' "$chart_version"
return 0
fi
{ helm -n "$release_namespace" status "$release_name" -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys
try:
payload = json.load(sys.stdin)
except Exception:
print("")
raise SystemExit(0)
chart = str(payload.get("chart", ""))
if not chart:
print("")
raise SystemExit(0)
if "-" in chart:
print(chart.rsplit("-", 1)[-1])
else:
print("")
PY
}
resolve_operator_watch_namespace() {
local watch_namespace
watch_namespace=$({ helm -n "$OPERATOR_NAMESPACE" get values "$OPERATOR_RELEASE" --all -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys
try:
values = json.load(sys.stdin)
except Exception:
print("")
raise SystemExit(0)
watch_namespace = values.get("watchNamespace", "")
if isinstance(watch_namespace, str):
print(watch_namespace.strip())
else:
print("")
PY
)
if [[ -n "$watch_namespace" ]]; then
printf '%s' "$watch_namespace"
return 0
fi
kubectl -n "$OPERATOR_NAMESPACE" get deploy "$OPERATOR_RELEASE" \
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="WATCH_NAMESPACE")].value}' 2>/dev/null || true
}
kubectl_apply_reports_changed() {
local apply_output="${1:-}"
if [[ -z "${apply_output//[[:space:]]/}" ]]; then
# Be conservative: unknown apply output should force full reconcile checks.
return 0
fi
case "$apply_output" in
*" created"*|*" configured"*|*" patched"*) return 0 ;;
esac
case "$apply_output" in
*" unchanged"*) return 1 ;;
esac
# Unknown token from kubectl apply output -> treat as changed.
return 0
}
# ---------------------------------------------------------------------------
# Helper: resolve knoe-db primary pod (same pattern as init_gitea.sh)
# ---------------------------------------------------------------------------
resolve_knoe_db_primary_pod() {
local primary
primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \
-l "cnpg.io/cluster=${CNPG_CLUSTER_NAME},cnpg.io/instanceRole=primary" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -z "$primary" ]]; then
primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \
-l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
fi
printf '%s' "$primary"
}
sql_escape_literal() {
printf '%s' "${1:-}" | sed "s/'/''/g"
}
# ---------------------------------------------------------------------------
# Resolve GitLab DB password (pull from knoe-db-superuser secret as fallback)
# ---------------------------------------------------------------------------
resolve_gitlab_db_password() {
if [[ -n "${GITLAB_DB_PASSWORD:-}" ]]; then
return 0
fi
local resolved=""
if kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then
resolved=$(kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser \
-o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true)
fi
if [[ -n "$resolved" ]]; then
GITLAB_DB_PASSWORD="$resolved"
else
# Generate a random password when none is available
GITLAB_DB_PASSWORD="$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom 2>/dev/null | head -c 32 || true)"
warn "Generated random GitLab DB password; store it in GITLAB_DB_PASSWORD for future runs."
fi
}
# ---------------------------------------------------------------------------
# Provision GitLab role + database in knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab() {
local primary db_ctx
db_ctx="$(resolve_db_cluster_context || true)"
primary="$(resolve_knoe_db_primary_pod)"
if [[ -z "$primary" ]]; then
if [[ "$MODE" == "k8s" ]]; then
die "No knoe-db pod found in namespace '${DB_NAMESPACE}' on DB_CLUSTER_KUBECONTEXT='${db_ctx}'."
fi
warn "No knoe-db pod found in namespace '${DB_NAMESPACE}'; skipping GitLab DB setup."
return 0
fi
resolve_gitlab_db_password
local admin_user=""
local candidate
for candidate in postgres root; do
if kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$candidate" -d postgres -tAc "SELECT 1" >/dev/null 2>&1; then
admin_user="$candidate"
break
fi
done
if [[ -z "$admin_user" ]]; then
if [[ "$MODE" == "k8s" ]]; then
die "Unable to connect to knoe-db as admin on DB_CLUSTER_KUBECONTEXT='${db_ctx}'."
fi
warn "Unable to connect to knoe-db as admin; skipping GitLab DB setup."
return 0
fi
local esc_pw
esc_pw="$(sql_escape_literal "$GITLAB_DB_PASSWORD")"
if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -c "
DO \$\$ BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${GITLAB_DB_USER}') THEN
CREATE ROLE ${GITLAB_DB_USER} LOGIN PASSWORD '${esc_pw}';
ELSE
ALTER ROLE ${GITLAB_DB_USER} WITH PASSWORD '${esc_pw}';
END IF;
END \$\$;
" >/dev/null 2>&1; then
if [[ "$MODE" == "k8s" ]]; then
die "Could not create/update role '${GITLAB_DB_USER}' on DB cluster context '${db_ctx}'."
fi
warn "Could not create/update role '${GITLAB_DB_USER}'."
fi
local db_exists
db_exists=$(kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -tAc \
"SELECT 1 FROM pg_database WHERE datname='${GITLAB_DB_NAME}';" 2>/dev/null || true)
if [[ "$db_exists" != "1" ]]; then
if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -c \
"CREATE DATABASE ${GITLAB_DB_NAME} OWNER ${GITLAB_DB_USER};" \
>/dev/null 2>&1; then
if [[ "$MODE" == "k8s" ]]; then
die "Could not create database '${GITLAB_DB_NAME}' on DB cluster context '${db_ctx}'."
fi
warn "Could not create database '${GITLAB_DB_NAME}'."
fi
fi
if [[ -n "$db_ctx" ]]; then
log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE}, ctx=${db_ctx})."
else
log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE})."
fi
}
# ---------------------------------------------------------------------------
# Clean up any pre-existing Gitea / git.prole.org configurations
# ---------------------------------------------------------------------------
cleanup_gitea() {
log "Checking for pre-existing Gitea deployment to remove before GitLab install..."
local gitea_ns="${GITEA_NAMESPACE:-gitea}"
# Remove Gitea Helm release
if helm -n "$gitea_ns" status gitea >/dev/null 2>&1; then
log "Uninstalling Gitea Helm release from namespace '$gitea_ns'..."
helm -n "$gitea_ns" uninstall gitea >/dev/null 2>&1 || true
fi
# Remove any leftover raw Gitea resources
kubectl -n "$gitea_ns" delete deploy/gitea svc/gitea-http svc/gitea-ssh \
>/dev/null 2>&1 || true
# Remove Kong routes/services registered for git.prole.org (best-effort)
local kong_ns="${KONG_NAMESPACE:-${NAMESPACE:-kong}}"
for res_type in kongplugins kongingresses; do
kubectl -n "$gitea_ns" delete "$res_type" --all >/dev/null 2>&1 || true
done
# Remove gitea namespace Ingress objects that route git.prole.org
kubectl -n "$gitea_ns" delete ingress \
-l "app.kubernetes.io/name=gitea" >/dev/null 2>&1 || true
kubectl -n "$gitea_ns" delete ingress \
--field-selector="metadata.name=gitea" >/dev/null 2>&1 || true
log "Gitea cleanup complete."
}
# ---------------------------------------------------------------------------
# Ensure gitlab namespace exists
# ---------------------------------------------------------------------------
kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null
# ---------------------------------------------------------------------------
# --force: remove existing GitLab operator and release first
# ---------------------------------------------------------------------------
if [[ "$FORCE" -eq 1 ]]; then
warn "--force: removing existing GitLab resources in namespace '$NAMESPACE'..."
if kubectl get crd gitlabs.apps.gitlab.com >/dev/null 2>&1; then
kubectl -n "$NAMESPACE" delete gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
# Wait briefly for operator to clean up managed resources
sleep 10
fi
helm -n "$NAMESPACE" uninstall "$GITLAB_RELEASE" >/dev/null 2>&1 || true
helm -n "$NAMESPACE" uninstall "$OPERATOR_RELEASE" >/dev/null 2>&1 || true
cleanup_gitea
fi
# ---------------------------------------------------------------------------
# Always remove gitea if its helm release exists (non-destructive path)
# gitea and gitlab both claim git.prole.org; they cannot coexist.
# ---------------------------------------------------------------------------
if helm -n "${GITEA_NAMESPACE:-gitea}" status gitea >/dev/null 2>&1; then
warn "Gitea release detected — removing to free git.prole.org for GitLab..."
cleanup_gitea
fi
# ---------------------------------------------------------------------------
# Prepare knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab
# Persist the DB password as a k8s Secret the operator CR can reference
resolve_gitlab_db_password
gitlab_db_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-db-password \
--from-literal=password="$GITLAB_DB_PASSWORD" \
--dry-run=client -o yaml | kubectl apply -f - 2>&1)"
if kubectl_apply_reports_changed "$gitlab_db_secret_apply_output"; then
gitlab_db_secret_changed=1
fi
log "gitlab-db-password secret applied in namespace '$NAMESPACE' (${gitlab_db_secret_apply_output})."
# ---------------------------------------------------------------------------
# Install / upgrade the GitLab Operator
# ---------------------------------------------------------------------------
operator_changed=0
operator_release_exists=0
operator_requires_upgrade=1
installed_operator_chart_version=""
installed_operator_watch_namespace=""
operator_watch_namespace_matches=1
if helm -n "$OPERATOR_NAMESPACE" status "$OPERATOR_RELEASE" >/dev/null 2>&1; then
operator_release_exists=1
installed_operator_chart_version="$(resolve_helm_release_chart_version "$OPERATOR_NAMESPACE" "$OPERATOR_RELEASE")"
installed_operator_watch_namespace="$(resolve_operator_watch_namespace)"
fi
if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" && -n "$installed_operator_chart_version" ]]; then
GITLAB_OPERATOR_CHART_VERSION="$installed_operator_chart_version"
log "Reusing installed GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
fi
if [[ -n "$installed_operator_watch_namespace" && "$installed_operator_watch_namespace" != "$NAMESPACE" ]]; then
operator_watch_namespace_matches=0
fi
if [[ "$operator_release_exists" == "1" && -n "$installed_operator_chart_version" \
&& "$installed_operator_chart_version" == "$GITLAB_OPERATOR_CHART_VERSION" \
&& "$operator_watch_namespace_matches" == "1" \
&& "${GITLAB_OPERATOR_FORCE_UPGRADE:-0}" != "1" ]]; then
operator_requires_upgrade=0
fi
if [[ "$operator_requires_upgrade" == "1" ]]; then
if [[ "$operator_watch_namespace_matches" == "0" ]]; then
warn "Installed GitLab Operator watchNamespace (${installed_operator_watch_namespace}) differs from target (${NAMESPACE}); forcing operator upgrade."
fi
log "Adding/updating GitLab Operator Helm repo..."
helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true
helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..."
if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then
GITLAB_OPERATOR_CHART_VERSION=$(helm search repo "$OPERATOR_CHART" --output table 2>/dev/null \
| awk 'NR==2{print $2}' || true)
if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then
die "Cannot detect GitLab Operator chart version. Set GITLAB_OPERATOR_CHART_VERSION explicitly and re-run."
fi
log "Auto-detected GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
else
log "Using GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
fi
log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..."
# The operator needs cluster-scoped RBAC; it watches all namespaces by default.
helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \
-n "$OPERATOR_NAMESPACE" \
--create-namespace \
--version "$GITLAB_OPERATOR_CHART_VERSION" \
--timeout 10m \
--wait \
--set watchNamespace="$NAMESPACE"
operator_changed=1
log "GitLab Operator ready."
else
log "GitLab Operator already matches desired chart/watchNamespace; skipping operator upgrade."
fi
# ---------------------------------------------------------------------------
# Resolve GitLab chart version (required by the Operator CR since >= v0.28)
# ---------------------------------------------------------------------------
installed_gitlab_chart_version=""
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
installed_gitlab_chart_version="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.spec.chart.version}' 2>/dev/null || true)"
if [[ -n "$installed_gitlab_chart_version" ]]; then
GITLAB_CHART_VERSION="$installed_gitlab_chart_version"
log "Reusing installed GitLab chart version from existing GitLab CR: ${GITLAB_CHART_VERSION}"
else
log "Auto-detecting latest GitLab chart version..."
helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true
helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..."
GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \
| awk 'NR==2{print $2}' || true)
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run."
fi
log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}"
fi
fi
CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\""
gitlab_chart_version_changed=0
if [[ -z "$installed_gitlab_chart_version" || "$installed_gitlab_chart_version" != "$GITLAB_CHART_VERSION" ]]; then
gitlab_chart_version_changed=1
fi
# ---------------------------------------------------------------------------
# Determine node selector block for the GitLab CR
# ---------------------------------------------------------------------------
# Storage node is only used for legacy/local static local-PV mode.
if [[ "$MODE" == "k8s" ]]; then
# In k8s/GKE mode, we do NOT use static storage nodes or nodeSelectors.
STORAGE_NODE=""
if [[ "$NODE_SELECTOR" == *"gandalf.prole.org"* ]]; then
warn "Stripping legacy nodeSelector '${NODE_SELECTOR}' in k8s mode."
NODE_SELECTOR=""
fi
else
STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
fi
[[ "$MODE" == "k8s" || -n "$STORAGE_NODE" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set in ${MODE} mode."
# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned
NODE_SELECTOR="${NODE_SELECTOR:-}"
NODE_SELECTOR_KEY="${GITLAB_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"
STORAGE_NODE_SELECTOR_KEY="${GITLAB_STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY}}"
NODE_SELECTOR_YAML=""
if [[ -n "$NODE_SELECTOR" ]]; then
log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}"
NODE_SELECTOR_YAML="${NODE_SELECTOR_KEY}: ${NODE_SELECTOR}"
fi
STORAGE_NODE_SELECTOR_YAML=""
if [[ -n "$STORAGE_NODE" ]]; then
STORAGE_NODE_SELECTOR_YAML="${STORAGE_NODE_SELECTOR_KEY}: ${STORAGE_NODE}"
fi
GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}"
if [[ "$MODE" == "k8s" ]]; then
# Strip legacy values that are meaningless in k8s mode — fall through to defaults.
if [[ "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode."
GITALY_STORAGE_CLASS=""
fi
# Default to strict 'standard' (pd-standard / HDD) in k8s mode.
# Never auto-substitute to 'standard-rwo' for GitLab storage.
if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
GITALY_STORAGE_CLASS="standard"
log "Defaulting Gitaly StorageClass to strict target 'standard' (pd-standard / HDD)."
else
log "Honoring explicit GITLAB_GITALY_STORAGE_CLASS=${GITALY_STORAGE_CLASS} (no auto-normalization)."
fi
if [[ "$GITALY_STORAGE_CLASS" == "standard" ]] && ! kubectl get storageclass standard >/dev/null 2>&1; then
repair_blocked "Required GitLab StorageClass is missing" \
"Configured target storageClass is 'standard' but StorageClass/standard does not exist on the cluster. Do not substitute to standard-rwo; create/restore StorageClass 'standard' or set an explicit non-standard class."
fi
else
if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
GITALY_STORAGE_CLASS="gitlab-gitaly-static"
fi
fi
# Sidekiq resources and concurrency
GITLAB_SIDEKIQ_REQUESTS_CPU="${GITLAB_SIDEKIQ_REQUESTS_CPU:-250m}"
GITLAB_SIDEKIQ_REQUESTS_MEMORY="${GITLAB_SIDEKIQ_REQUESTS_MEMORY:-1500Mi}"
GITLAB_SIDEKIQ_LIMITS_CPU="${GITLAB_SIDEKIQ_LIMITS_CPU:-1}"
GITLAB_SIDEKIQ_LIMITS_MEMORY="${GITLAB_SIDEKIQ_LIMITS_MEMORY:-3Gi}"
GITLAB_SIDEKIQ_CONCURRENCY="${GITLAB_SIDEKIQ_CONCURRENCY:-5}"
# Webservice (Puma) resources and concurrency
# Defaults calibrated for e2-standard-2 nodes (~7.1 GB allocatable).
# - workerProcesses=1 keeps steady-state RSS under ~1.1 GB.
# - Memory *limit* must exceed the boot spike (Rails preload + worker fork),
# which peaks around 1.4–1.6 GB. 1800M gives headroom without exceeding
# the node's 1/4-node per-pod budget.
# - puma.threads.{min,max} are the correct chart paths; PUMA_THREADS_{MIN,MAX}
# env vars do NOT propagate through the chart's ERB-rendered puma config.
GITLAB_WEBSERVICE_REQUESTS_CPU="${GITLAB_WEBSERVICE_REQUESTS_CPU:-200m}"
GITLAB_WEBSERVICE_REQUESTS_MEMORY="${GITLAB_WEBSERVICE_REQUESTS_MEMORY:-900M}"
GITLAB_WEBSERVICE_LIMITS_MEMORY="${GITLAB_WEBSERVICE_LIMITS_MEMORY:-1800M}"
GITLAB_WEBSERVICE_WORKER_PROCESSES="${GITLAB_WEBSERVICE_WORKER_PROCESSES:-1}"
GITLAB_WEBSERVICE_PUMA_THREADS_MIN="${GITLAB_WEBSERVICE_PUMA_THREADS_MIN:-2}"
GITLAB_WEBSERVICE_PUMA_THREADS_MAX="${GITLAB_WEBSERVICE_PUMA_THREADS_MAX:-2}"
JEMALLOC_HOSTPATH_DIR="/opt/gitlab-jemalloc"
JEMALLOC_HOSTPATH_LIB="${JEMALLOC_HOSTPATH_DIR}/libjemalloc.so.2"
GITLAB_JEMALLOC_MODE="${GITLAB_JEMALLOC_MODE:-auto}"
if is_truthy "${GITLAB_JEMALLOC_REQUIRED:-0}"; then
GITLAB_JEMALLOC_MODE="force"
fi
JEMALLOC_HOSTPATH_SUPPORTED=1
JEMALLOC_HOSTPATH_REASON=""
JEMALLOC_HOSTPATH_WANTED=0
JEMALLOC_HOSTPATH_ACTIVE=0
detect_jemalloc_hostpath_support() {
local provider_ids gke_pool_labels gke_topology_labels os_images
provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true)
gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true)
gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true)
os_images=$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.nodeInfo.osImage}{"\n"}{end}' 2>/dev/null || true)
if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then
JEMALLOC_HOSTPATH_SUPPORTED=0
JEMALLOC_HOSTPATH_REASON="Detected GKE/GCE node metadata (providerID/labels)."
return 0
fi
if [[ "${os_images,,}" == *"container-optimized os"* ]]; then
JEMALLOC_HOSTPATH_SUPPORTED=0
JEMALLOC_HOSTPATH_REASON="Detected Container-Optimized OS node image."
fi
}
configure_jemalloc_hostpath_mode() {
local mode_normalized="${GITLAB_JEMALLOC_MODE,,}"
detect_jemalloc_hostpath_support
case "$mode_normalized" in
auto|"")
if (( JEMALLOC_HOSTPATH_SUPPORTED == 1 )); then
JEMALLOC_HOSTPATH_WANTED=1
else
JEMALLOC_HOSTPATH_WANTED=0
warn "Skipping hostPath jemalloc optimization: ${JEMALLOC_HOSTPATH_REASON}"
fi
;;
off|false|0|disabled)
JEMALLOC_HOSTPATH_WANTED=0
;;
force|required|on|true|1)
JEMALLOC_HOSTPATH_WANTED=1
if (( JEMALLOC_HOSTPATH_SUPPORTED == 0 )); then
die "HostPath jemalloc mode is forced but unsupported in this cluster: ${JEMALLOC_HOSTPATH_REASON} Disable jemalloc (GITLAB_JEMALLOC_MODE=off) or use a non-hostPath approach."
fi
;;
*)
die "Unsupported GITLAB_JEMALLOC_MODE='${GITLAB_JEMALLOC_MODE}' (expected: auto, off, force)."
;;
esac
}
# ---------------------------------------------------------------------------
# Create the GitLab CR (operator reconciles this into the full deployment)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Apply gitlab-google-oidc secret (OIDC provider config for OmniAuth)
# ---------------------------------------------------------------------------
if [[ -n "$FRONTDOOR_HOST" ]]; then
oidc_secret_tmpl="${SCRIPT_DIR}/../deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
if is_unresolved_secret_ref "$GITLAB_OIDC_CLIENT_ID" || is_unresolved_secret_ref "$GITLAB_OIDC_CLIENT_SECRET"; then
repair_blocked "GitLab OIDC client credentials are missing or unresolved" \
"Set non-empty OIDC client values in installer inputs/config (auth.clientId/auth.clientSecret or GITLAB_OIDC_CLIENT_ID/GITLAB_OIDC_CLIENT_SECRET) and rerun deploy."
fi
if [[ -f "$oidc_secret_tmpl" ]]; then
log "Applying gitlab-google-oidc secret (issuer=${GITLAB_OIDC_ISSUER}, redirect=${GITLAB_OIDC_REDIRECT_URI}) ..."
GITLAB_OIDC_PROVIDER_NAME="$GITLAB_OIDC_PROVIDER_NAME" \
GITLAB_OIDC_ISSUER="$GITLAB_OIDC_ISSUER" \
GITLAB_OIDC_CLIENT_ID="$GITLAB_OIDC_CLIENT_ID" \
GITLAB_OIDC_CLIENT_SECRET="$GITLAB_OIDC_CLIENT_SECRET" \
GITLAB_OIDC_REDIRECT_URI="$GITLAB_OIDC_REDIRECT_URI" \
envsubst < "$oidc_secret_tmpl" | kubectl apply -n "$NAMESPACE" -f - >/dev/null || \
repair_blocked "Could not apply gitlab-google-oidc secret" "GitLab OIDC provider secret apply failed for namespace ${NAMESPACE}."
else
repair_blocked "gitlab-google-oidc-secret.example.yaml not found" "Missing ${oidc_secret_tmpl}; cannot configure GitLab OIDC automatically."
fi
fi
# ---------------------------------------------------------------------------
# ARM64 / 16KB-page jemalloc fix — build glibc jemalloc on every node once
# All RPi nodes use 16KB kernel pages; GitLab's bundled jemalloc is 4KB-only.
# We compile a compatible jemalloc-5.3.0 with --with-lg-page=14 via DaemonSet.
# ---------------------------------------------------------------------------
setup_jemalloc_on_nodes() {
local jlib="${JEMALLOC_HOSTPATH_LIB}"
local ds_name="jemalloc-builder"
if (( JEMALLOC_HOSTPATH_WANTED == 0 )); then
log "HostPath jemalloc optimization disabled for this environment."
if kubectl -n "$NAMESPACE" get ds "$ds_name" >/dev/null 2>&1; then
log "Deleting existing ${ds_name} DaemonSet because hostPath jemalloc is disabled."
kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \
warn "Could not delete ${ds_name} DaemonSet; continuing."
fi
JEMALLOC_HOSTPATH_ACTIVE=0
return 0
fi
# Check if already present on all schedulable nodes
local nodes
nodes=$(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | tr '\n' ' ')
local all_ready=true
for node in $nodes; do
result=$(kubectl -n "$NAMESPACE" run "jcheck-${node//\./-}" \
--image=ubuntu:22.04 --restart=Never --rm --attach --quiet \
--overrides="{\"spec\":{\"nodeName\":\"${node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"jlib\",\"hostPath\":{\"path\":\"${JEMALLOC_HOSTPATH_DIR}\",\"type\":\"DirectoryOrCreate\"}}],\"containers\":[{\"name\":\"c\",\"image\":\"ubuntu:22.04\",\"command\":[\"bash\",\"-c\",\"test -f /jlib/libjemalloc.so.2 && echo OK || echo MISSING\"],\"volumeMounts\":[{\"name\":\"jlib\",\"mountPath\":\"/jlib\"}]}]}}" 2>/dev/null || echo "MISSING")
if [[ "$result" != *"OK"* ]]; then
all_ready=false
break
fi
done
if $all_ready; then
log "jemalloc-16k already present on all nodes — skipping build."
JEMALLOC_HOSTPATH_ACTIVE=1
return 0
fi
log "Deploying jemalloc-builder DaemonSet (compiles jemalloc-5.3.0 with --with-lg-page=14 on each node)..."
local _tmpds
_tmpds=$(mktemp /tmp/jemalloc-ds-XXXXXX.yaml)
cat > "$_tmpds" <<'JEDS'
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: jemalloc-builder
namespace: NAMESPACE_PLACEHOLDER
labels:
app: jemalloc-builder
spec:
selector:
matchLabels:
app: jemalloc-builder
template:
metadata:
labels:
app: jemalloc-builder
spec:
tolerations:
- operator: Exists
initContainers:
- name: build
image: ubuntu:22.04
command:
- bash
- -c
- |
set -e
TARGET=/hostlib/libjemalloc.so.2
if [ -f "$TARGET" ]; then echo "Already built"; exit 0; fi
apt-get update -qq && apt-get install -y -q gcc make wget bzip2 2>&1
cd /tmp
wget -q -O jemalloc.tar.bz2 \
https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
tar xjf jemalloc.tar.bz2 && cd jemalloc-5.3.0
./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1
make -j2 lib/libjemalloc.so.2 2>&1
cp lib/libjemalloc.so.2 "$TARGET" && chmod 755 "$TARGET"
echo "Done: $(ls -lh $TARGET)"
volumeMounts:
- name: hostlib
mountPath: /hostlib
containers:
- name: keepalive
image: ubuntu:22.04
command: [bash, -c, "sleep infinity"]
volumes:
- name: hostlib
hostPath:
path: JEMALLOC_HOSTPATH_DIR_PLACEHOLDER
type: DirectoryOrCreate
JEDS
sed -i.bak "s/NAMESPACE_PLACEHOLDER/${NAMESPACE}/g" "$_tmpds"
sed -i.bak2 "s#JEMALLOC_HOSTPATH_DIR_PLACEHOLDER#${JEMALLOC_HOSTPATH_DIR}#g" "$_tmpds"
kubectl apply -f "$_tmpds"
rm -f "$_tmpds" "${_tmpds}.bak" "${_tmpds}.bak2"
log "Waiting up to 20 minutes for jemalloc to build on all nodes..."
local deadline=$((SECONDS + 1200))
while (( SECONDS < deadline )); do
ready=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
-o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0)
desired=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
-o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo 1)
[[ "$ready" -ge "$desired" && "$desired" -gt 0 ]] && break
log " jemalloc build progress: ${ready}/${desired} nodes ready..."
sleep 30
done
if [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]]; then
JEMALLOC_HOSTPATH_ACTIVE=1
log "jemalloc-16k ready on all nodes."
return 0
fi
JEMALLOC_HOSTPATH_ACTIVE=0
if [[ "${GITLAB_JEMALLOC_MODE,,}" == "force" || "${GITLAB_JEMALLOC_MODE,,}" == "required" || "${GITLAB_JEMALLOC_MODE,,}" == "1" || "${GITLAB_JEMALLOC_MODE,,}" == "true" || "${GITLAB_JEMALLOC_MODE,,}" == "on" ]]; then
die "jemalloc-builder did not become ready (${ready}/${desired}) and jemalloc is required."
fi
kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \
warn "Could not delete ${ds_name} DaemonSet after timeout; continuing without jemalloc."
warn "jemalloc-builder did not become ready (${ready}/${desired}); continuing without hostPath jemalloc optimization."
}
# ---------------------------------------------------------------------------
# Garage S3 key + bucket provisioning for GitLab object storage
# Creates a dedicated "gitlab-s3" key in Garage, provisions required buckets,
# and writes the k8s secrets consumed by the GitLab CR.
# ---------------------------------------------------------------------------
setup_garage_for_gitlab() {
local garage_admin_context garage_admin_context_label
garage_admin_context="$(resolve_garage_admin_context || true)"
if [[ -n "$garage_admin_context" ]]; then
garage_admin_context_label="$garage_admin_context"
else
garage_admin_context_label="(active/default)"
fi
log "Configuring Garage S3 (${GARAGE_NAMESPACE}) as GitLab object storage..."
log "Garage admin context: ${garage_admin_context_label}"
log "Garage namespace: ${GARAGE_NAMESPACE}"
log "GitLab object storage S3 endpoint: ${GARAGE_S3_ENDPOINT}"
garage_setup_failure() {
local reason="$1"
if is_truthy "${GITLAB_OBJECT_STORAGE_REQUIRED:-1}"; then
die "${reason} (object storage is required for GitLab deployment)."
fi
warn "${reason} (continuing because GITLAB_OBJECT_STORAGE_REQUIRED=${GITLAB_OBJECT_STORAGE_REQUIRED})."
return 0
}
local garage_pod
garage_pod=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get pods -l app=garage \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -z "$garage_pod" ]]; then
garage_setup_failure "Garage pod not found in namespace '${GARAGE_NAMESPACE}' on context '${garage_admin_context_label}'"
return 0
fi
local garage_admin_token_b64 garage_admin_token
garage_admin_token_b64=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get secret garage-secrets \
-o jsonpath='{.data.admin_token}' 2>/dev/null || true)
if [[ -z "$garage_admin_token_b64" ]]; then
garage_setup_failure "Garage admin token missing in secret '${GARAGE_NAMESPACE}/garage-secrets' on context '${garage_admin_context_label}'"
return 0
fi
garage_admin_token=$(printf '%s' "$garage_admin_token_b64" | base64 -d 2>/dev/null || true)
garage_admin_token="${garage_admin_token//$'\r'/}"
garage_admin_token="${garage_admin_token//$'\n'/}"
if [[ -z "$garage_admin_token" ]]; then
garage_setup_failure "Garage admin token in secret '${GARAGE_NAMESPACE}/garage-secrets' is malformed or empty on context '${garage_admin_context_label}'"
return 0
fi
local garage_cmd_last_reason=""
local garage_cmd_output=""
garage_exec() {
local cmd_desc="$1"
shift
local out_file err_file stderr_text lower_stderr
out_file=$(mktemp)
err_file=$(mktemp)
if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
/garage --admin-token "$garage_admin_token" "$@" >"$out_file" 2>"$err_file"; then
garage_cmd_output=$(cat "$out_file")
rm -f "$out_file" "$err_file"
garage_cmd_last_reason=""
return 0
fi
stderr_text=$(cat "$err_file" 2>/dev/null || true)
if [[ "$stderr_text" == *"--admin-token"* ]] && [[ "$stderr_text" == *"unknown"* || "$stderr_text" == *"unexpected"* || "$stderr_text" == *"wasn't expected"* ]]; then
if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
/garage "$@" >"$out_file" 2>"$err_file"; then
garage_cmd_output=$(cat "$out_file")
rm -f "$out_file" "$err_file"
garage_cmd_last_reason=""
return 0
fi
stderr_text=$(cat "$err_file" 2>/dev/null || true)
fi
lower_stderr="${stderr_text,,}"
if [[ "$lower_stderr" == *"unauth"* || "$lower_stderr" == *"forbidden"* || "$lower_stderr" == *"invalid token"* || "$lower_stderr" == *"permission denied"* || "$lower_stderr" == *" 401"* || "$lower_stderr" == *" 403"* ]]; then
garage_cmd_last_reason="Unauthenticated admin command while running '${cmd_desc}'"
else
garage_cmd_last_reason="Garage admin command failed while running '${cmd_desc}'"
fi
if [[ -n "$stderr_text" ]]; then
garage_cmd_last_reason+=" (${stderr_text})"
fi
garage_cmd_output=""
rm -f "$out_file" "$err_file"
return 1
}
# ---- Resolve or create the gitlab-s3 access key ----
# garage v1.x outputs plaintext (not JSON) for key list / key info / key create.
# key list format: " GKxxxxxxxxx <name>"
# key create/info format includes lines:
# "Key ID: GKxxxxxxxxx"
# "Secret key: <secret>"
local access_key secret_key existing_id key_output rotated_key_name key_list_output create_failure_reason
if ! garage_exec "garage key list" key list; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
key_list_output="$garage_cmd_output"
existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true)
if [[ -n "$existing_id" ]]; then
log "Garage key '${GARAGE_S3_KEY_NAME}' already exists (id=${existing_id}); fetching info..."
if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
key_output="$garage_cmd_output"
else
log "Creating Garage key '${GARAGE_S3_KEY_NAME}'..."
create_failure_reason=""
if ! garage_exec "garage key create ${GARAGE_S3_KEY_NAME}" key create "$GARAGE_S3_KEY_NAME"; then
create_failure_reason="$garage_cmd_last_reason"
fi
if ! garage_exec "garage key list" key list; then
if [[ -n "$create_failure_reason" ]]; then
garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}"
else
garage_setup_failure "${garage_cmd_last_reason}"
fi
return 0
fi
key_list_output="$garage_cmd_output"
existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true)
if [[ -z "$existing_id" ]]; then
if [[ -n "$create_failure_reason" ]]; then
garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}' (${create_failure_reason})"
else
garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'"
fi
return 0
fi
if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then
if [[ -n "$create_failure_reason" ]]; then
garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}"
else
garage_setup_failure "${garage_cmd_last_reason}"
fi
return 0
fi
key_output="$garage_cmd_output"
fi
if [[ -z "$key_output" ]]; then
garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'"
return 0
fi
# Parse plaintext output — match "Key ID: GKxxx" and "Secret key: xxx" lines.
# Also try JSON in case a future garage version changes format.
access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1)
secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1)
# JSON fallback
if [[ -z "$access_key" ]]; then
access_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true)
fi
if [[ -z "$secret_key" ]]; then
secret_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true)
fi
if [[ "$secret_key" == "(redacted)" ]]; then
warn "Garage key '${GARAGE_S3_KEY_NAME}' secret is redacted; creating a deterministic fallback key for GitLab object storage."
rotated_key_name="${GARAGE_S3_KEY_NAME}-gitlab"
if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then
if ! garage_exec "garage key create ${rotated_key_name}" key create "$rotated_key_name"; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
fi
key_output="$garage_cmd_output"
access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1)
secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1)
if [[ -z "$access_key" ]]; then
access_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true)
fi
if [[ -z "$secret_key" ]]; then
secret_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true)
fi
if [[ -n "$access_key" && -n "$secret_key" && "$secret_key" != "(redacted)" ]]; then
GARAGE_S3_KEY_NAME="$rotated_key_name"
log "Using rotated Garage key '${GARAGE_S3_KEY_NAME}'."
fi
fi
if [[ -z "$access_key" || -z "$secret_key" || "$secret_key" == "(redacted)" ]]; then
garage_setup_failure "Malformed Garage key output for '${GARAGE_S3_KEY_NAME}' (missing access key or secret key)"
warn "Raw output: ${key_output}"
return 0
fi
# ---- Create buckets and grant permissions ----
# --key takes the accessKeyId (not the name) in garage v1.x
local buckets=(
registry
gitlab-artifacts-storage
gitlab-lfs-storage
gitlab-uploads-storage
gitlab-packages-storage
gitlab-dependency-proxy-storage
gitlab-terraform-state
gitlab-ci-secure-files
)
for bucket in "${buckets[@]}"; do
if ! garage_exec "garage bucket create ${bucket}" bucket create "$bucket"; then
local bucket_create_failure_reason lower_bucket_create_failure_reason
bucket_create_failure_reason="${garage_cmd_last_reason}"
lower_bucket_create_failure_reason="${bucket_create_failure_reason,,}"
if [[ "$lower_bucket_create_failure_reason" == *"already exists"* ]]; then
log "Garage bucket '${bucket}' already exists; reusing existing bucket."
else
garage_setup_failure "${bucket_create_failure_reason}"
return 0
fi
fi
# bucket name is positional; --read/--write/--owner are boolean flags
if ! garage_exec "garage bucket allow ${bucket}" bucket allow "$bucket" --key "$access_key" \
--read --write --owner; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
done
log "Garage buckets provisioned for GitLab."
# ---- Write k8s secrets consumed by the GitLab CR ----
# Rails object storage connection (artifacts, LFS, uploads, packages, etc.)
local object_storage_secret_apply_output registry_storage_secret_apply_output
object_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-object-storage \
--from-literal=connection="provider: AWS
region: garage
aws_access_key_id: ${access_key}
aws_secret_access_key: ${secret_key}
endpoint: '${GARAGE_S3_ENDPOINT}'
path_style: true" \
--dry-run=client -o yaml | kubectl apply -f - 2>&1)"
if kubectl_apply_reports_changed "$object_storage_secret_apply_output"; then
gitlab_object_storage_secret_changed=1
fi
log "gitlab-object-storage secret applied (${object_storage_secret_apply_output})."
# Docker registry S3 storage driver config.
# Use s3_v2 (AWS SDK v2) — s3_v1 has signature computation issues with
# non-standard ports (Host: endpoint:3900) against Garage. SDK v2 always
# uses SigV4; v4auth is not a recognised key and must be omitted.
registry_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-registry-storage \
--from-literal=config="s3_v2:
accesskey: ${access_key}
secretkey: ${secret_key}
bucket: registry
regionendpoint: ${GARAGE_S3_ENDPOINT}
region: garage
pathstyle: true" \
--dry-run=client -o yaml | kubectl apply -f - 2>&1)"
if kubectl_apply_reports_changed "$registry_storage_secret_apply_output"; then
gitlab_registry_storage_secret_changed=1
fi
log "gitlab-registry-storage secret applied (${registry_storage_secret_apply_output})."
}
# ---------------------------------------------------------------------------
# Legacy/local static PV + synology directory setup for gitaly
# minio PV removed — object storage is provided by knoe-system/garage
# ---------------------------------------------------------------------------
setup_gitlab_legacy_storage() {
local synology_node="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
local storage_selector_key="${STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"
local base_path="${GITLAB_STORAGE_BASE:-/synology/d005/gitlab}"
local gitaly_size="${GITLAB_GITALY_PV_SIZE:-50Gi}"
[[ -n "$synology_node" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set for legacy GitLab storage setup."
log "Creating synology gitaly directory on ${synology_node}..."
kubectl -n "$NAMESPACE" run gitlab-dirprep \
--image=alpine:latest --restart=Never --rm --attach \
--overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"s\",\"hostPath\":{\"path\":\"$(dirname ${base_path})\",\"type\":\"Directory\"}}],\"containers\":[{\"name\":\"dirprep\",\"image\":\"alpine:latest\",\"command\":[\"sh\",\"-c\",\"mkdir -p ${base_path}/gitaly && chmod 777 ${base_path}/gitaly && echo done\"],\"volumeMounts\":[{\"name\":\"s\",\"mountPath\":\"$(dirname ${base_path})\"}]}]}}" \
2>&1 || warn "Could not create synology dirs (may already exist)"
# A named no-provisioner StorageClass is required so the helm chart renders
# storageClassName in the PVC template (an empty string is treated as falsy
# by the chart and omitted, causing the PVC to use the cluster default).
log "Ensuring gitlab-gitaly-static StorageClass (no-provisioner)..."
kubectl apply -f - <<SCYAML
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: gitlab-gitaly-static
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: Immediate
reclaimPolicy: Retain
SCYAML
log "Creating static PV for gitaly (${gitaly_size})..."
kubectl apply -f - <<PVYAML
apiVersion: v1
kind: PersistentVolume
metadata:
name: gitlab-gitaly-synology
spec:
capacity:
storage: ${gitaly_size}
accessModes: [ReadWriteOnce]
persistentVolumeReclaimPolicy: Retain
storageClassName: gitlab-gitaly-static
local:
path: ${base_path}/gitaly
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: ${storage_selector_key}
operator: In
values: [${synology_node}]
PVYAML
log "GitLab gitaly storage PV ready."
}
setup_gitlab_storage() {
if [[ "$MODE" == "k8s" ]]; then
log "Skipping legacy Synology/local-PV gitaly storage prep in k8s mode; using dynamic StorageClass '${GITALY_STORAGE_CLASS}'."
# Check for legacy resources on GKE/k8s
local has_legacy=0
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then has_legacy=1; fi
if kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then has_legacy=1; fi
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
local gitaly_sts_yaml
gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$gitaly_sts_yaml" ]]; then
if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* || "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
has_legacy=1
fi
fi
if [[ "$has_legacy" == "1" ]]; then
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then
export GITALY_AUTOCLEAN_PERFORMED=1
log "AUTOCLEAN: repairing legacy Gitaly storage (scaling down and deleting legacy PV/PVC)..."
kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
kubectl -n "$NAMESPACE" delete pvc repo-data-gitlab-gitaly-0 --wait=false 2>/dev/null || true
kubectl delete pv gitlab-gitaly-synology --wait=false 2>/dev/null || true
# Give it a moment to process deletions
sleep 2
else
repair_blocked "Legacy Gitaly PV/PVC still bound to gitlab-gitaly-static / gandalf.prole.org" \
"Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to automatically repair by deleting legacy PVC/PV, or run:
kubectl -n $NAMESPACE scale sts ${GITLAB_RELEASE}-gitaly --replicas=0
kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0
kubectl delete pv gitlab-gitaly-synology"
fi
fi
repair_stale_gke_gitaly_dynamic_storage "${GITALY_STORAGE_CLASS:-}"
else
setup_gitlab_legacy_storage
fi
# Object storage (registry, artifacts, LFS, uploads, etc.) is provided by
# knoe-system/garage — credentials and buckets are set up here.
setup_garage_for_gitlab
}
configure_jemalloc_hostpath_mode
setup_jemalloc_on_nodes
check_gitlab_pre_apply_blocked
setup_gitlab_storage
log "Pre-creating gitlab-app-nonroot ServiceAccount (required by chart v9+)..."
kubectl -n "${NAMESPACE}" apply -f - <<SAYAML
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitlab-app-nonroot
namespace: ${NAMESPACE}
SAYAML
GITLAB_ALLOWED_HOSTS_YAML=""
_gitlab_allowed_hosts_csv=""
GITLAB_ALLOWED_HOSTS=()
append_gitlab_allowed_host() {
local candidate="$1"
[[ -n "$candidate" ]] || return 0
case ",${_gitlab_allowed_hosts_csv}," in
*,"${candidate}",*) return 0 ;;
esac
GITLAB_ALLOWED_HOSTS+=("$candidate")
_gitlab_allowed_hosts_csv="${_gitlab_allowed_hosts_csv:+${_gitlab_allowed_hosts_csv},}${candidate}"
return 0
}
for _gitlab_allowed_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
append_gitlab_allowed_host "$_gitlab_allowed_host"
_gitlab_host_root="${_gitlab_allowed_host#*.}"
if [[ -n "$_gitlab_host_root" && "$_gitlab_host_root" != "$_gitlab_allowed_host" ]]; then
append_gitlab_allowed_host "$_gitlab_host_root"
fi
done
unset _gitlab_allowed_host _gitlab_host_root
for _gitlab_allowed_host in "${GITLAB_ALLOWED_HOSTS[@]}"; do
GITLAB_ALLOWED_HOSTS_YAML+=$'\n'" - ${_gitlab_allowed_host}"
done
unset _gitlab_allowed_host
GITLAB_TRUSTED_PROXIES_YAML=""
for _gitlab_trusted_proxy in "${GITLAB_TRUSTED_PROXIES[@]}"; do
GITLAB_TRUSTED_PROXIES_YAML+=$'\n'" - ${_gitlab_trusted_proxy}"
done
unset _gitlab_trusted_proxy
log "GitLab trusted_proxies configured: ${_gitlab_trusted_proxies_csv}"
GLOBAL_JEMALLOC_VALUES_YAML=""
WORKLOAD_JEMALLOC_VALUES_YAML=""
if (( JEMALLOC_HOSTPATH_ACTIVE == 1 )); then
GLOBAL_JEMALLOC_VALUES_YAML=$(cat <<JEMGLOBAL
# ARM64 nodes: inject glibc jemalloc compiled with --with-lg-page=14
# The DaemonSet above places it at ${JEMALLOC_HOSTPATH_LIB}
extraEnv:
LD_PRELOAD: ${JEMALLOC_HOSTPATH_LIB}
JEMGLOBAL
)
WORKLOAD_JEMALLOC_VALUES_YAML=$(cat <<JEMWORKLOAD
extraEnv:
LD_PRELOAD: ${JEMALLOC_HOSTPATH_LIB}
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: ${JEMALLOC_HOSTPATH_DIR}
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: ${JEMALLOC_HOSTPATH_DIR}
readOnly: true
JEMWORKLOAD
)
else
log "Rendering GitLab values without hostPath jemalloc mounts/LD_PRELOAD."
fi
gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_exists_before=0
if [[ -n "$gitlab_generation_before" ]]; then
gitlab_exists_before=1
fi
GITLAB_CR_RENDERED="$(cat <<EOF
apiVersion: apps.gitlab.com/v1beta1
kind: GitLab
metadata:
name: ${GITLAB_RELEASE}
namespace: ${NAMESPACE}
spec:
chart:
${CHART_VERSION_YAML}
values:
global:
persistence:
storageClass: ${GITALY_STORAGE_CLASS}
${GLOBAL_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<GNODE
nodeSelector:
${NODE_SELECTOR_YAML}
GNODE
fi)
# Explicit allowedHosts as flat strings — chart 9.x renders list-of-maps
# by default which breaks URI::InvalidComponentError in 7_gitlab_http.rb
allowedHosts:${GITLAB_ALLOWED_HOSTS_YAML}
hosts:
domain: ${PRIMARY_GITLAB_DOMAIN_ROOT}
gitlab:
name: ${GITLAB_DOMAIN}
ssh: ${GITLAB_DOMAIN}
ingress:
class: ${GITLAB_INGRESS_CLASS}
configureCertmanager: false
tls:
enabled: false
psql:
host: ${DB_HOST}
port: ${DB_PORT}
username: ${GITLAB_DB_USER}
database: ${GITLAB_DB_NAME}
password:
secret: gitlab-db-password
key: password
redis:
host: ${REDIS_HOST}
port: ${REDIS_PORT}
auth:
enabled: false
# chart v9+ requires the SA to be pre-created when a custom name is used
serviceAccount:
create: false
# Disable bundled minio globally — object storage is provided by garage
minio:
enabled: false
# Object storage via knoe-system/garage (S3-compatible)
appConfig:
object_store:
enabled: true
proxy_download: true
connection:
secret: gitlab-object-storage
key: connection
trusted_proxies:${GITLAB_TRUSTED_PROXIES_YAML}
$(if [[ -n "$FRONTDOOR_HOST" ]]; then
cat <<OMNIAUTH
omniauth:
enabled: true
autoSignInWithProvider: openid_connect
allowSingleSignOn:
- openid_connect
blockAutoCreatedUsers: false
providers:
- secret: gitlab-google-oidc
key: provider
OMNIAUTH
fi)
registry:
bucket: registry
# Disable bundled sub-charts — we provide our own data stores
postgresql:
install: false
redis:
install: false
certmanager:
install: false
prometheus:
install: false
gitlab-runner:
install: false
# minio sub-chart is disabled via global.minio.enabled: false (set above under global:).
# Do NOT set minio.enabled here — chart v9 removed the sub-chart level property
# and treats it as a hard validation error (NOTES.txt:234).
gitlab:
webservice:
# Shrunk Puma for modest-sized GKE nodes (currently e2-standard-2,
# ~7.1 GB allocatable — four pods share it).
# Default workerProcesses=2 × 4 threads → ~2.2 GB steady state
# (and ~2.8 GB boot spike) → OOMKilled under a 2 GB limit.
# workerProcesses=1 + threads=2 keeps steady RSS ~0.9–1.1 GB and
# boot spike ~1.4–1.6 GB, which fits within ${GITLAB_WEBSERVICE_LIMITS_MEMORY}.
replicaCount: 1
workerProcesses: ${GITLAB_WEBSERVICE_WORKER_PROCESSES}
puma:
# Canonical chart path — env-var overrides are ignored because the
# chart renders puma.rb from these values via ERB.
threads:
min: ${GITLAB_WEBSERVICE_PUMA_THREADS_MIN}
max: ${GITLAB_WEBSERVICE_PUMA_THREADS_MAX}
resources:
requests:
cpu: ${GITLAB_WEBSERVICE_REQUESTS_CPU}
memory: ${GITLAB_WEBSERVICE_REQUESTS_MEMORY}
limits:
memory: ${GITLAB_WEBSERVICE_LIMITS_MEMORY}
# Rails preloading is slow on shared nodes — protect against
# probe kills during the boot window, and against transient Gitaly
# unavailability (readiness depends on /-/readiness reaching Gitaly).
livenessProbe:
initialDelaySeconds: 3600
periodSeconds: 60
timeoutSeconds: 30
failureThreshold: 5
readinessProbe:
initialDelaySeconds: 120
periodSeconds: 30
timeoutSeconds: 30
failureThreshold: 60
ingress:
enabled: ${GITLAB_WEBSERVICE_INGRESS_ENABLED}
annotations:
kubernetes.io/ingress.class: ${GITLAB_INGRESS_CLASS}
minReplicas: 1
maxReplicas: 1
${WORKLOAD_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
nodeSelector:
${NODE_SELECTOR_YAML}
NODE
fi)
sidekiq:
replicaCount: 1
minReplicas: 1
maxReplicas: 1
concurrency: ${GITLAB_SIDEKIQ_CONCURRENCY}
resources:
requests:
cpu: ${GITLAB_SIDEKIQ_REQUESTS_CPU}
memory: ${GITLAB_SIDEKIQ_REQUESTS_MEMORY}
limits:
cpu: ${GITLAB_SIDEKIQ_LIMITS_CPU}
memory: ${GITLAB_SIDEKIQ_LIMITS_MEMORY}
# Rails loading takes 25-40min on RPi; protect against liveness kills
livenessProbe:
initialDelaySeconds: 3600
periodSeconds: 60
timeoutSeconds: 30
failureThreshold: 5
readinessProbe:
initialDelaySeconds: 120
periodSeconds: 30
timeoutSeconds: 30
failureThreshold: 60
${WORKLOAD_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
nodeSelector:
${NODE_SELECTOR_YAML}
NODE
fi)
gitaly:
# Storage class is mode-specific:
# - k8s: dynamic PVCs on cluster storage class
# - legacy/local: static local-PV storage class created by setup_gitlab_storage()
persistence:
storageClass: ${GITALY_STORAGE_CLASS}
size: ${GITLAB_GITALY_PV_SIZE:-50Gi}
${WORKLOAD_JEMALLOC_VALUES_YAML}
# Give gitaly gRPC health check time to initialise before probes start
livenessProbe:
initialDelaySeconds: 30
readinessProbe:
initialDelaySeconds: 15
$(if [[ -n "$STORAGE_NODE_SELECTOR_YAML" ]]; then
cat <<NODE
# Legacy/local mode pins gitaly to storage node for local PV.
nodeSelector:
${STORAGE_NODE_SELECTOR_YAML}
NODE
fi)
toolbox:
backups:
objectStorage:
backend: s3
config:
secret: gitlab-object-storage
key: connection
${WORKLOAD_JEMALLOC_VALUES_YAML}
migrations:
${WORKLOAD_JEMALLOC_VALUES_YAML}
gitlab-exporter:
${WORKLOAD_JEMALLOC_VALUES_YAML}
gitlab-shell:
replicaCount: 1
minReplicas: 1
maxReplicas: 1
kas:
replicaCount: 1
minReplicas: 1
maxReplicas: 1
gitaly:
persistence:
storageClass: ${GITALY_STORAGE_CLASS}
size: ${GITLAB_GITALY_PV_SIZE:-50Gi}
registry:
replicaCount: 1
hpa:
minReplicas: 1
maxReplicas: 1
storage:
secret: gitlab-registry-storage
key: config
ingress:
enabled: false
EOF
)"
GITLAB_CR_RENDERED="$(sanitize_gitlab_cr_rendered_values_for_chart_version "$GITLAB_CR_RENDERED")"
preflight_validate_gitlab_cr_rendered_values "$GITLAB_CR_RENDERED"
log "Applying GitLab CR (hosts=${_gitlab_hosts_csv}, db=${GITLAB_DB_NAME}@${DB_HOST})..."
gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_apply_output="$(echo "$GITLAB_CR_RENDERED" | kubectl apply -f -)"
gitlab_generation_after="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_spec_changed=1
if [[ -n "$gitlab_generation_before" && -n "$gitlab_generation_after" && "$gitlab_generation_before" == "$gitlab_generation_after" ]]; then
gitlab_spec_changed=0
fi
gitlab_apply_changed=0
if kubectl_apply_reports_changed "$gitlab_apply_output"; then
if [[ "$MODE" == "k8s" && "$gitlab_spec_changed" == "0" ]]; then
# Some apply operations may report "configured" without changing generation.
# In that case, avoid triggering a full reconcile solely from apply output text.
log "GitLab CR apply reported change, but generation is unchanged. Skipping drift trigger."
else
gitlab_apply_changed=1
fi
fi
gitlab_config_changed=0
if [[ "$gitlab_db_secret_changed" == "1" || "$gitlab_object_storage_secret_changed" == "1" || "$gitlab_registry_storage_secret_changed" == "1" ]]; then
gitlab_config_changed=1
fi
gitlab_reconcile_required=0
# Check actual replicas to ensure they match desired count (1) - Requirement 3
gitlab_replica_drift=0
for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
_dep_name="${GITLAB_RELEASE}-${_dep_suffix}"
_actual=$(kubectl -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")
if [[ "$_actual" != "1" ]]; then
log "Detected replica drift for ${_dep_name}: actual=${_actual}, desired=1"
gitlab_replica_drift=1
break
fi
done
if [[ "$gitlab_exists_before" == "0" || "$operator_changed" == "1" || "$gitlab_chart_version_changed" == "1" || "$gitlab_spec_changed" == "1" || "$gitlab_apply_changed" == "1" || "$gitlab_config_changed" == "1" || "$gitlab_replica_drift" == "1" ]]; then
gitlab_reconcile_required=1
fi
check_gitlab_post_apply_blocked
# minio is disabled (global.minio.enabled: false) — no ARM64 credential fix needed.
# ---------------------------------------------------------------------------
# ARM64 configure check: GitLab v17+ (chart v8+) ships multi-arch CNG images
# that include ARM64 variants. The old workaround of replacing the configure
# init container with alpine:latest is no longer needed and breaks things
# because /templates/configure is baked into the gitlab-base image, not
# mounted from a ConfigMap.
# This function is kept as a no-op to avoid breaking callers; remove entirely
# once the chart version floor is confirmed stable at v9+.
# ---------------------------------------------------------------------------
fix_registry_arm64_configure() {
log "Skipping ARM64 configure patch — GitLab v18 CNG images are multi-arch natively."
}
fix_registry_arm64_configure
log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)."
log "Monitor progress: kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w"
log "Watch pods: kubectl -n ${NAMESPACE} get pods -w"
# ---------------------------------------------------------------------------
# Optionally wait for GitLab to become available
# ---------------------------------------------------------------------------
WAIT_TIMEOUT="${GITLAB_WAIT_TIMEOUT:-1200}" # default 20 minutes
if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
if [[ "$gitlab_reconcile_required" == "0" ]]; then
log "Fast-path: no GitLab reconcile triggers detected; skipping long wait."
gitlab_available_status="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{range .status.conditions[?(@.type=="Available")]}{.status}{end}' 2>/dev/null || true)"
if [[ "$gitlab_available_status" == "True" ]]; then
log "Short health check OK: GitLab CR condition Available=True."
check_gitlab_post_apply_blocked
else
warn "Short health check: GitLab CR Available condition is '${gitlab_available_status:-unknown}'."
kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
fi
else
log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status (checking for blocked states every 30s)..."
_wait_start=$(date +%s)
while true; do
if kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \
--for=condition=Available \
--timeout=30s >/dev/null 2>&1; then
log "GitLab CR condition Available=True."
break
fi
check_gitlab_post_apply_blocked
_now=$(date +%s)
if (( _now - _wait_start >= WAIT_TIMEOUT )); then
_ctx=$(get_gitlab_blocker_context)
repair_blocked "Timed out waiting for GitLab CR condition=Available" \
"The deployment is taking too long. Context: ${_ctx}Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}"
fi
done
fi
fi
# ---------------------------------------------------------------------------
# Requirement: Explicitly scale down over-replicated GitLab deployments
# Some components may remain at 2 replicas even after CR is Available.
# ---------------------------------------------------------------------------
log "Verifying GitLab deployment replica counts (desired=1)..."
_gitlab_replica_target=1
if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
gitlab_verify_replica_source_of_truth "$_gitlab_replica_target"
fi
gitlab_post_remediation_success=0
_replica_drift_found=0
_corrected_deployments=()
_drifted_deployments=()
# Explicit list of components to verify and scale if needed (Requirement 2)
for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2"; do
_dep_name="${GITLAB_RELEASE}-${_dep_suffix}"
# Read live spec.replicas (Requirement 2 & 6)
_live_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
if [[ -z "$_live_replicas" || ! "$_live_replicas" =~ ^[0-9]+$ ]]; then
_live_replicas=0
fi
if [[ "$_live_replicas" != "$_gitlab_replica_target" ]]; then
log "Detected replica drift for ${_dep_name}: spec=${_live_replicas}, desired=${_gitlab_replica_target}"
_replica_drift_found=1
_drifted_deployments+=("$_dep_name")
else
log "Deployment ${_dep_name} replica count is correct: spec=${_live_replicas}, desired=${_gitlab_replica_target}"
fi
done
if [[ "$_replica_drift_found" == "1" ]]; then
log "Replica drift detected. Re-applying GitLab CR source-of-truth (gitlab-shell/kas/registry/sidekiq desired=${_gitlab_replica_target})."
if ! echo "$GITLAB_CR_RENDERED" | kubectl --context "$KUBECTL_CONTEXT" apply -f - >/dev/null; then
repair_blocked "Failed to re-apply GitLab CR for replica drift remediation" \
"Could not reconcile source-of-truth replica values."
fi
gitlab_verify_replica_source_of_truth "$_gitlab_replica_target"
_source_of_truth_mismatch_report=""
for _dep_name in "${_drifted_deployments[@]}"; do
_post_reconcile_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
if [[ -z "$_post_reconcile_replicas" || ! "$_post_reconcile_replicas" =~ ^[0-9]+$ ]]; then
_post_reconcile_replicas=0
fi
_corrected_deployments+=("$_dep_name")
if [[ "$_post_reconcile_replicas" != "$_gitlab_replica_target" ]]; then
_source_of_truth_mismatch_report+="- ${_dep_name}: spec=${_post_reconcile_replicas}, desired=${_gitlab_replica_target}"$'\n'
fi
log "Deployment ${_dep_name} after CR reconcile: spec=${_post_reconcile_replicas}, desired=${_gitlab_replica_target}."
done
if [[ -n "$_source_of_truth_mismatch_report" ]]; then
_rendered_replica_fields_report="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")"
if [[ -z "$_rendered_replica_fields_report" ]]; then
_rendered_replica_fields_report="- (no replica-related fields detected in rendered CR values)"
fi
repair_blocked "GitLab operator desired replica source-of-truth mismatch after CR re-apply" \
"Deployment specs still do not match desired=${_gitlab_replica_target}:\n${_source_of_truth_mismatch_report}Rendered GitLab CR replica source fields:\n${_rendered_replica_fields_report}\nThis indicates CR source-of-truth still resolves to replicas>1."
fi
log "Waiting for rollout of reconciled deployments..."
_remediation_rollout_failed=0
for _dep_name in "${_corrected_deployments[@]}"; do
if ! kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" rollout status deployment/"$_dep_name" --timeout=300s; then
warn "Rollout status check did not report success for ${_dep_name}; final replica state will decide pass/fail."
_remediation_rollout_failed=1
fi
done
# Final verification with settle window to tolerate transient old pod linger
_settle_timeout_s="${GITLAB_POST_REMEDIATION_SETTLE_TIMEOUT:-90}"
_settle_poll_interval_s="${GITLAB_POST_REMEDIATION_SETTLE_POLL_INTERVAL:-5}"
if [[ -z "$_settle_timeout_s" || ! "$_settle_timeout_s" =~ ^[0-9]+$ || "$_settle_timeout_s" == "0" ]]; then
_settle_timeout_s=90
fi
if [[ -z "$_settle_poll_interval_s" || ! "$_settle_poll_interval_s" =~ ^[0-9]+$ || "$_settle_poll_interval_s" == "0" ]]; then
_settle_poll_interval_s=5
fi
log "Post-remediation settle verification for corrected deployments (timeout=${_settle_timeout_s}s, poll=${_settle_poll_interval_s}s):"
_remediation_final_failed=0
_remediation_failure_class=""
_remediation_failure_report=""
if (( ${#_corrected_deployments[@]} > 0 )); then
_settle_start_ts=$(date +%s)
_settle_had_errexit=0
if [[ $- == *e* ]]; then
_settle_had_errexit=1
set +e
fi
_settle_iteration=0
while true; do
_settle_iteration=$((_settle_iteration + 1))
_remediation_failure_report=""
_iteration_blocked=0
_source_of_truth_reversion=0
for _dep_name in "${_corrected_deployments[@]}"; do
_target_desired_replicas="${_gitlab_replica_target}"
_desired_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
if [[ -z "$_desired_replicas" || ! "$_desired_replicas" =~ ^[0-9]+$ ]]; then
_desired_replicas=0
fi
_status_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.replicas}' 2>/dev/null || true)
_status_ready=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
_status_available=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
_status_updated=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.updatedReplicas}' 2>/dev/null || true)
_status_replicas=${_status_replicas:-0}
_status_ready=${_status_ready:-0}
_status_available=${_status_available:-0}
_status_updated=${_status_updated:-0}
_dep_selector=$(gitlab_selector_for_deployment "$_dep_name" 2>/dev/null || true)
if [[ -n "$_dep_selector" ]]; then
_final_live=$(gitlab_non_terminal_pod_count_for_app "$_dep_selector" 2>/dev/null || true)
if [[ -z "$_final_live" || ! "$_final_live" =~ ^[0-9]+$ ]]; then
_final_live=0
fi
_matching_pods=$(gitlab_non_terminal_pod_names_for_app "$_dep_selector" 2>/dev/null || true)
else
_final_live=0
_matching_pods=""
fi
if [[ -z "$_matching_pods" ]]; then
_matching_pods="(none)"
fi
log "Settle poll #${_settle_iteration}: deployment ${_dep_name} selector=${_dep_selector:-<unresolved>} spec=${_desired_replicas}, desired=${_target_desired_replicas}, live=${_final_live}, status(replicas/ready/available/updated)=${_status_replicas}/${_status_ready}/${_status_available}/${_status_updated}, pods=[${_matching_pods}]"
if [[ -z "$_final_live" || ! "$_final_live" =~ ^[0-9]+$ || "$_desired_replicas" != "$_target_desired_replicas" || "$_final_live" != "$_target_desired_replicas" ]]; then
_iteration_blocked=1
if [[ "$_desired_replicas" != "$_target_desired_replicas" ]]; then
_source_of_truth_reversion=1
fi
_remediation_failure_report+="- ${_dep_name}: spec=${_desired_replicas}, final_live=${_final_live:-unknown}, desired=${_target_desired_replicas}, selector=${_dep_selector:-<unresolved>}, status=${_status_replicas}/${_status_ready}/${_status_available}/${_status_updated}, pods=[${_matching_pods}]"$'\n'
fi
done
if [[ "$_source_of_truth_reversion" == "1" ]]; then
_remediation_final_failed=1
_remediation_failure_class="source_of_truth"
break
fi
if [[ "$_iteration_blocked" == "0" ]]; then
break
fi
_settle_now_ts=$(date +%s)
if (( _settle_now_ts - _settle_start_ts >= _settle_timeout_s )); then
_remediation_final_failed=1
_remediation_failure_class="convergence"
break
fi
sleep "$_settle_poll_interval_s"
done
if [[ "$_settle_had_errexit" == "1" ]]; then
set -e
fi
fi
if [[ "$_remediation_final_failed" == "1" ]]; then
if [[ "$_remediation_failure_class" == "source_of_truth" ]]; then
repair_blocked "GitLab operator reverted deployment spec.replicas after source-of-truth reconcile" \
"Operator-managed desired state diverged during settle verification:\n${_remediation_failure_report}This is a source-of-truth failure (CR values path), not a rollout lag issue."
else
repair_blocked "GitLab post-remediation replica verification failed" \
"Source-of-truth reconcile ran, but final convergence did not meet required conditions (spec.replicas==desired and live selector pod count==desired):\n${_remediation_failure_report}Check: kubectl -n ${NAMESPACE} get deploy -l app.kubernetes.io/instance=${GITLAB_RELEASE}"
fi
fi
gitlab_replica_drift=0
gitlab_reconcile_required=0
gitlab_post_remediation_success=1
if [[ "$_remediation_rollout_failed" == "1" ]]; then
log "GitLab corrective action converged by final replica state (spec and live selector counts match desired=${_gitlab_replica_target})."
else
log "GitLab corrective action succeeded: rollout completed and all spec/live selector counts match desired=${_gitlab_replica_target}."
fi
fi
if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
if [[ "${gitlab_post_remediation_success:-0}" == "1" ]]; then
log "GitLab corrective action success is authoritative; skipping additional post-remediation convergence gate."
else
wait_for_gitlab_workload_convergence
fi
else
warn "GITLAB_NO_WAIT=1, skipping strict GitLab workload convergence checks."
fi
# ---------------------------------------------------------------------------
# Public ingress for configured GitLab domain -> gitlab-webservice
# The GitLab Operator creates the Ingress; this block ensures a stable public
# class/host mapping if the CR-managed ingress is absent or not usable.
# ---------------------------------------------------------------------------
WEBSERVICE_SVC="${GITLAB_RELEASE}-webservice-default"
GITLAB_INGRESS_RULES_YAML=""
for _gitlab_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
GITLAB_INGRESS_RULES_YAML+=$'\n'" - host: ${_gitlab_host}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: ${WEBSERVICE_SVC}
port:
number: 8181"
done
unset _gitlab_host
GITLAB_AUTH_ANNOTATIONS_YAML=""
case "${FRONTDOOR_AUTH_ENABLED}" in
1|true|TRUE|True|yes|YES|Yes|on|ON|On)
GITLAB_AUTH_ANNOTATIONS_YAML=$(cat <<EOAUTH
nginx.ingress.kubernetes.io/auth-url: "${AUTH_VERIFY_URL}"
nginx.ingress.kubernetes.io/auth-signin: "${AUTH_SIGNIN_URL}"
nginx.ingress.kubernetes.io/auth-response-headers: "${AUTH_RESPONSE_HEADERS}"
EOAUTH
)
;;
esac
# -- GKE-managed TLS (only when ingressClassName=gce) ------------------------
# With GCE/GCLB ingress, the idiomatic TLS story is ManagedCertificate +
# FrontendConfig (HTTP->HTTPS redirect). We create both alongside the
# Ingress and wire them up via annotations. No-op for nginx/kong classes.
#
# We also create a BackendConfig so GCLB's health check hits
# gitlab-workhorse's `/-/readiness` endpoint instead of `/`.
# The default `/` check returns 302 (workhorse redirects to /users/sign_in),
# which GCLB interprets as an unhealthy backend — the backend flaps between
# UNHEALTHY/HEALTHY and the ingress surfaces as HTTP 502.
# The Service is annotated with cloud.google.com/backend-config below so
# GCLB picks the BackendConfig up for the workhorse port (8181).
GITLAB_GCE_TLS_ANNOTATIONS_YAML=""
GITLAB_MANAGED_CERT_NAME="gitlab-managed-cert"
GITLAB_FRONTEND_CONFIG_NAME="gitlab-frontend-config"
GITLAB_BACKEND_CONFIG_NAME="gitlab-webservice-backendconfig"
if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
GITLAB_GCE_TLS_ANNOTATIONS_YAML=$(cat <<EOTLS
networking.gke.io/managed-certificates: "${GITLAB_MANAGED_CERT_NAME}"
networking.gke.io/v1beta1.FrontendConfig: "${GITLAB_FRONTEND_CONFIG_NAME}"
EOTLS
)
# Build the ManagedCertificate domains list from GITLAB_PUBLIC_HOSTS.
_gitlab_mc_domains_yaml=""
for _gitlab_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
_gitlab_mc_domains_yaml+=$'\n'" - ${_gitlab_host}"
done
unset _gitlab_host
log "Applying GKE ManagedCertificate (${GITLAB_MANAGED_CERT_NAME}) + FrontendConfig (${GITLAB_FRONTEND_CONFIG_NAME}) + BackendConfig (${GITLAB_BACKEND_CONFIG_NAME}) in ns=${NAMESPACE}..."
kubectl apply -f - <<EOF
apiVersion: networking.gke.io/v1
kind: ManagedCertificate
metadata:
name: ${GITLAB_MANAGED_CERT_NAME}
namespace: ${NAMESPACE}
spec:
domains:${_gitlab_mc_domains_yaml}
---
apiVersion: networking.gke.io/v1beta1
kind: FrontendConfig
metadata:
name: ${GITLAB_FRONTEND_CONFIG_NAME}
namespace: ${NAMESPACE}
spec:
redirectToHttps:
enabled: true
responseCodeName: MOVED_PERMANENTLY_DEFAULT
---
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
name: ${GITLAB_BACKEND_CONFIG_NAME}
namespace: ${NAMESPACE}
spec:
healthCheck:
type: HTTP
requestPath: /-/readiness
port: 8181
# Puma + Rails boot is slow; be generous before marking backends unhealthy.
checkIntervalSec: 30
timeoutSec: 10
healthyThreshold: 1
unhealthyThreshold: 3
# Drain a little more generously — GitLab requests can be long-running.
connectionDraining:
drainingTimeoutSec: 60
EOF
unset _gitlab_mc_domains_yaml
# Annotate the webservice Service so GCLB picks up the BackendConfig.
# The GitLab Operator owns this Service, so we apply the annotation
# out-of-band. It is additive (not owned by the chart) and survives
# operator reconciles.
_webservice_svc="${GITLAB_RELEASE}-webservice-default"
if kubectl -n "$NAMESPACE" get svc "$_webservice_svc" >/dev/null 2>&1; then
log "Annotating Service ${_webservice_svc} with cloud.google.com/backend-config=${GITLAB_BACKEND_CONFIG_NAME}..."
kubectl -n "$NAMESPACE" annotate svc "$_webservice_svc" \
"cloud.google.com/backend-config={\"default\":\"${GITLAB_BACKEND_CONFIG_NAME}\"}" \
--overwrite >/dev/null || warn "Failed to annotate ${_webservice_svc}; will retry after CR reconcile."
else
log "Service ${_webservice_svc} not present yet; BackendConfig annotation will be (re-)applied after the CR reconcile."
fi
unset _webservice_svc
fi
log "Reconciling GitLab front-door owner='${GITLAB_FRONTDOOR_OWNER}' (ingressClass=${GITLAB_INGRESS_CLASS}) for hosts=${_gitlab_hosts_csv} -> ${WEBSERVICE_SVC}:8181 ..."
if [[ "$GITLAB_FRONTDOOR_OWNER" == "operator" ]]; then
if assert_unique_ingress_host_claims "${GITLAB_RELEASE}-webservice-default" "$NAMESPACE" "$_gitlab_hosts_csv" "$GITLAB_RELEASE" "$WEBSERVICE_SVC"; then
:
else
_ingress_claim_rc=$?
case "$_ingress_claim_rc" in
10)
log "GitLab operator ingress host ownership confirmed in namespace '${NAMESPACE}': ${INGRESS_HOST_CLAIM_DETAILS}"
;;
*)
die "Duplicate ingress host/path claim detected for GitLab operator ingress '${NAMESPACE}/${GITLAB_RELEASE}-webservice-default': ${INGRESS_HOST_CLAIM_DETAILS}"
;;
esac
fi
if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then
_operator_ingress_name="${GITLAB_RELEASE}-webservice-default"
if kubectl -n "$NAMESPACE" get ingress "$_operator_ingress_name" >/dev/null 2>&1; then
log "Annotating ingress ${_operator_ingress_name} with ManagedCertificate=${GITLAB_MANAGED_CERT_NAME} and FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}..."
kubectl -n "$NAMESPACE" annotate ingress "$_operator_ingress_name" \
"networking.gke.io/managed-certificates=${GITLAB_MANAGED_CERT_NAME}" \
"networking.gke.io/v1beta1.FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}" \
--overwrite >/dev/null || warn "Failed to annotate ${_operator_ingress_name} with GCE ingress TLS annotations."
else
warn "Operator ingress ${_operator_ingress_name} not found yet; cannot apply GCE ingress annotations in this pass."
fi
unset _operator_ingress_name
fi
if kubectl -n "$NAMESPACE" get ingress gitlab-kong-ingress >/dev/null 2>&1; then
log "Deleting stale fallback ingress ${NAMESPACE}/gitlab-kong-ingress to enforce single front-door ownership (${GITLAB_FRONTDOOR_OWNER})."
kubectl -n "$NAMESPACE" delete ingress gitlab-kong-ingress --ignore-not-found >/dev/null || \
warn "Failed to delete stale fallback ingress ${NAMESPACE}/gitlab-kong-ingress."
fi
else
_skip_gitlab_fallback_ingress=0
if assert_unique_ingress_host_claims "gitlab-kong-ingress" "$NAMESPACE" "$_gitlab_hosts_csv" "$GITLAB_RELEASE" "$WEBSERVICE_SVC"; then
:
else
_ingress_claim_rc=$?
case "$_ingress_claim_rc" in
10)
_skip_gitlab_fallback_ingress=1
log "Operator-managed ingress already owns GitLab host/path in namespace '${NAMESPACE}': ${INGRESS_HOST_CLAIM_DETAILS}"
log "Skipping fallback ingress creation for GitLab release '${GITLAB_RELEASE}'."
;;
*)
die "Duplicate ingress host/path claim detected for GitLab ingress '${NAMESPACE}/gitlab-kong-ingress': ${INGRESS_HOST_CLAIM_DETAILS}"
;;
esac
fi
if [[ "$_skip_gitlab_fallback_ingress" != "1" ]]; then
kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gitlab-kong-ingress
namespace: ${NAMESPACE}
annotations:
kubernetes.io/ingress.class: ${GITLAB_INGRESS_CLASS}
konghq.com/strip-path: "false"
${GITLAB_GCE_TLS_ANNOTATIONS_YAML}
${GITLAB_AUTH_ANNOTATIONS_YAML}
spec:
ingressClassName: ${GITLAB_INGRESS_CLASS}
rules:${GITLAB_INGRESS_RULES_YAML}
EOF
fi
fi
# ---------------------------------------------------------------------------
# Registry migration: registry:2 (knoe-system) → gitlab-registry
# Runs unattended after GitLab is Ready. Delegates to init_registry.sh
# migrate which uses skopeo when available, otherwise prints commands.
# Skipped if the old registry:2 has no images or is already gone.
# Set SKIP_REGISTRY_MIGRATE=1 to suppress.
# ---------------------------------------------------------------------------
if [[ "${SKIP_REGISTRY_MIGRATE:-0}" != "1" ]]; then
_init_registry_sh="${SCRIPT_DIR}/init_registry.sh"
if [[ -x "$_init_registry_sh" ]]; then
log "--- Registry migration: registry:2 → gitlab-registry ---"
# Pass the gitlab namespace so the migrate action knows where to send images.
GITLAB_NAMESPACE="$NAMESPACE" \
REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}" \
"$_init_registry_sh" migrate || \
warn "Registry migration encountered errors — check output above."
else
warn "init_registry.sh not found at ${_init_registry_sh}; skipping registry migration."
warn "Run manually: ./etc/init_registry.sh migrate"
fi
unset _init_registry_sh
fi
log "Done."
log "GitLab will be reachable at hosts=${_gitlab_hosts_csv} once pods are Running."
log "Initial root password: kubectl -n ${NAMESPACE} get secret ${GITLAB_RELEASE}-gitlab-initial-root-password -o jsonpath='{.data.password}' | base64 -d"
Reference in New Issue View Git Blame Copy Permalink