chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 12:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
9a30a60a2b
prole/etc/init_gitlab.sh
chrisfu 9a30a60a2b chore: enhance GitLab reconciliation with explicit context handling 
- Added detailed logging for APP and DB contexts in `init_gitlab.sh` and `deploy.sh`.
- Enabled explicit passing of contexts and configuration paths to avoid fallbacks.
- Introduced strict environment resolution logic in `prole_cfg.sh` with fail-safe mechanisms.
2026-04-16 16:36:05 -07:00

2703 lines
111 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
PROG="init_gitlab"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"
MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")"
# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden.
# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps).
NAMESPACE="${GITLAB_NAMESPACE:-gitlab}"
CFG_PATH=""
FORCE=0
# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE).
# Setting this would pin ALL global components to one node, exhausting RAM.
NODE_SELECTOR=""
usage() {
cat <<EOF
Usage:
$PROG [options] [deploy]
Options:
--mode <k3d|k3s|k8s|local> Deployment mode (default: ${MODE:-k3d})
-n, --namespace <name> Target namespace (default: gitlab)
-c, --config <config.cfg> Path to config file (defaults to detected)
--node-selector <node> Node to pin all GitLab workloads (optional)
--force Remove existing GitLab and Gitea releases before deploy
--help Show this help
Behavior:
- Removes any pre-existing GitLab-domain Gitea configurations (helm release,
namespace, Kong routes) when --force is specified or when gitea is detected.
- Installs the GitLab Operator via Helm into the gitlab namespace.
- Provisions a GitLab CR that uses the knoe-db CloudNativePG cluster as its
external PostgreSQL data store.
- Configures the GitLab public hostname as Kubernetes ingress pointing to GitLab.
- Legacy/local modes pin gitaly storage to GITLAB_STORAGE_NODE (required).
EOF
}
die() { echo "[ERROR] $*" >&2; exit 2; }
log() { echo "[INFO] $*"; }
warn() { echo "[WARN] $*" >&2; }
resolve_explicit_kube_context() {
local ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
if [[ -n "$ctx" ]]; then
printf '%s' "$ctx"
return 0
fi
return 1
}
enforce_app_cluster_targeting() {
if [[ "$MODE" != "k8s" ]]; then
return 0
fi
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
local target_ctx
target_ctx="$(resolve_explicit_kube_context || true)"
log "Reconciling GitLab with APP context: ${app_ctx:-"(unset)"}"
[[ -n "$db_ctx" ]] && log "Reconciling GitLab with DB context: ${db_ctx}"
[[ -n "$app_ctx" ]] || die "APP_CLUSTER_KUBECONTEXT is required for k8s GitLab deployment."
[[ -n "$target_ctx" ]] || die "Explicit kubectl context is required for k8s GitLab deployment."
if [[ -n "$db_ctx" && "$target_ctx" == "$db_ctx" ]]; then
die "Refusing GitLab APP step against DB context '$target_ctx'."
fi
if [[ "$target_ctx" != "$app_ctx" ]]; then
die "GitLab APP step must target APP_CLUSTER_KUBECONTEXT='${app_ctx}' (got '${target_ctx}')."
fi
export KUBECTL_CONTEXT="$app_ctx"
export KUBE_CONTEXT_NAME="$app_ctx"
export KUBECONTEXT="$app_ctx"
}
kubectl() {
local target_ctx
target_ctx="$(resolve_explicit_kube_context || true)"
if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then
die "Explicit kubectl context is required in k8s mode."
fi
local arg has_context=0
for arg in "$@"; do
case "$arg" in
--context|--context=*|--server|--server=*)
has_context=1
break
;;
esac
done
if [[ -n "$target_ctx" && $has_context -eq 0 ]]; then
command kubectl --context "$target_ctx" "$@"
else
command kubectl "$@"
fi
}
resolve_db_cluster_context() {
if [[ "$MODE" == "k8s" ]]; then
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
[[ -n "$db_ctx" ]] || die "DB_CLUSTER_KUBECONTEXT is required for GitLab DB setup in k8s mode."
log "Using DB cluster context: ${db_ctx}"
printf '%s' "$db_ctx"
return 0
fi
return 1
}
repair_blocked() {
local reason="$1"
local remediation="${2:-}"
echo "" >&2
echo "[REPAIR_BLOCKED] ${reason}" >&2
if [[ -n "$remediation" ]]; then
echo "Remediation: ${remediation}" >&2
fi
echo "" >&2
exit 1
}
get_gitlab_migrations_diagnostics() {
local job_name="$1"
[[ -n "$job_name" ]] || return 0
local _pod_name=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$job_name" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -z "$_pod_name" ]]; then
echo "No pods found for migration job $job_name."
return 0
fi
local _diag=""
_diag+="--- Migrations Diagnostics for Pod: $_pod_name ---"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o wide 2>/dev/null || true)"$'\n'
_diag+=$'\n'"[Container Statuses]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}: state={.state.waiting.reason}{.state.terminated.reason}{.state.running.startedAt}, exitCode={.state.terminated.exitCode}, restarts={.restartCount}{"\n"}{end}' 2>/dev/null || true)"$'\n'
_diag+=$'\n'"[Migrations Logs (tail=100)]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --tail=100 2>/dev/null || echo "(no logs available)")"$'\n'
_diag+=$'\n'"[Previous Migrations Logs (if any)]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --previous --tail=100 2>/dev/null || echo "(no previous logs)")"$'\n'
_diag+=$'\n'"[Pod Events]"$'\n'
_diag+="$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$_pod_name" --sort-by='.lastTimestamp' 2>/dev/null | tail -n 10 || true)"
echo "$_diag"
}
check_gitlab_migrations_blocked() {
local _latest_job
_latest_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
[[ -n "$_latest_job" ]] || return 0
local _job_failed
_job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
# 1. Active Failure Detection
local _latest_pod
_latest_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -n "$_latest_pod" ]]; then
local _restarts
_restarts=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
local _waiting_reason
_waiting_reason=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
local _exit_code
_exit_code=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.terminated.exitCode}' 2>/dev/null || echo "")
if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_exit_code" && "$_exit_code" != "0" ) ]]; then
log "GitLab migrations job is actively failing: ${_latest_job} (pod: ${_latest_pod}, restarts: ${_restarts}, exitCode: ${_exit_code:-unknown})"
# Enhanced DB connectivity diagnostics for split-cluster visibility
local _db_info="DB_HOST=${DB_HOST}, DB_PORT=${DB_PORT}"
local _split_cluster="No"
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then _split_cluster="Yes (APP: ${app_ctx}, DB: ${db_ctx})"; fi
local _diag
_diag=$(get_gitlab_migrations_diagnostics "$_latest_job")
local _connectivity_hint=""
if echo "$_diag" | grep -qi "connecting with your hostname"; then
_connectivity_hint=" (Likely DNS/resolution failure)"
elif echo "$_diag" | grep -qiE "connection refused|timeout"; then
_connectivity_hint=" (Likely TCP connectivity/firewall failure)"
fi
repair_blocked "GitLab migrations job is actively failing" \
"Job: ${_latest_job}. Pod: ${_latest_pod}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_exit_code:-unknown}. Restarts: ${_restarts}.
DB Config: ${_db_info}${_connectivity_hint}
Split-cluster: ${_split_cluster}
Diagnostics:
${_diag}"
fi
fi
# 2. Stale Failed Job Repair
local _job_creation_ts
_job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
local _job_pods_running
_job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
local _job_pods_pending
_job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0")
local _now_ts
_now_ts=$(date +%s)
local _creation_ts
_creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0")
local _age
_age=$(( _now_ts - _creation_ts ))
if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then
log "GitLab operator is stalled on stale failed migrations job: ${_latest_job} (age: ${_age}s, no active pods)"
log "Deleting stale failed migrations job to trigger repair..."
kubectl -n "$NAMESPACE" delete job "$_latest_job" --wait=true 2>/dev/null || true
kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_job" --force --grace-period=0 2>/dev/null || true
log "Stale migrations job deleted. Operator should recreate it shortly."
fi
}
check_host_resolves() {
local target_host="$1"
# Strip port if present
local host_only="${target_host%:*}"
local target_ctx
target_ctx="$(resolve_explicit_kube_context || true)"
local is_k8s_svc=0
if [[ "$host_only" == *".svc.cluster.local" ]]; then
is_k8s_svc=1
elif [[ "$host_only" =~ ^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+$ ]]; then
# Simple <service>.<namespace> format (exactly one dot, no numbers to avoid IPs)
is_k8s_svc=1
fi
if [[ $is_k8s_svc -eq 1 ]]; then
# Parse service and namespace (first and second labels)
local svc_name svc_ns
svc_name=$(echo "$host_only" | cut -d. -f1)
svc_ns=$(echo "$host_only" | cut -d. -f2)
log "Redis validation mode: kubernetes-service (cluster: ${target_ctx:-default})"
log "Parsed REDIS_HOST ${target_host} -> service=${svc_name} namespace=${svc_ns}"
if kubectl -n "$svc_ns" get service "$svc_name" >/dev/null 2>&1; then
log "Service ${svc_name} found in namespace ${svc_ns}"
# Preferably verify it has endpoints / ready backing pods
local endpoints_found
endpoints_found=$(kubectl -n "$svc_ns" get endpoints "$svc_name" -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
if [[ -n "$endpoints_found" ]]; then
log "Redis endpoints present for ${svc_name}"
else
warn "Service ${svc_name} found in namespace ${svc_ns} but has NO ready endpoints (yet)."
fi
return 0
else
warn "Service ${svc_name} NOT found in namespace ${svc_ns} (cluster: ${target_ctx:-default})"
return 1
fi
else
log "Redis validation mode: external-host"
if ! host "$host_only" >/dev/null 2>&1; then
# If it's an IP, host might fail. Check if it's an IP.
if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
log "Host ${host_only} is an IP address, skipping DNS check."
return 0
fi
return 1
fi
fi
return 0
}
check_db_connectivity() {
local target_host="$1"
local target_port="${2:-5432}"
local host_only="${target_host%:*}"
log "Validating GitLab database connectivity to ${target_host}:${target_port}..."
# DNS sanity check
if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
log "Host ${host_only} is an IP address, skipping DNS check."
else
# In k8s mode, internal DNS resolution might fail from the script runner
# but work from within the cluster. We do a basic 'host' check as a hint.
if ! host "$host_only" >/dev/null 2>&1; then
warn "DNS resolution (host) failed for database host: ${host_only}. This may be expected if only resolvable in-cluster."
else
log "DNS resolution successful for ${host_only}"
fi
fi
# TCP check via temporary pod in the APP cluster namespace
log "Performing in-cluster TCP connectivity probe to ${host_only}:${target_port} (namespace: ${NAMESPACE})..."
if kubectl -n "$NAMESPACE" run db-probe \
--image=alpine:latest --restart=Never --rm --attach --timeout=30s \
--command -- sh -c "nc -zv -w 5 ${host_only} ${target_port}" >/dev/null 2>&1; then
log "TCP connectivity to ${host_only}:${target_port} SUCCESSFUL."
return 0
else
warn "TCP connectivity probe to ${host_only}:${target_port} FAILED."
return 1
fi
}
check_gitlab_pre_apply_blocked() {
# --- Registry Endpoint Placeholder Check ---
if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
# If the configured value itself is still a placeholder, we MUST block.
# The operator hasn't provided a real value yet.
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
"GARAGE_PRIVATE_S3_ENDPOINT is set to the literal placeholder. Set it to the real DB cluster Garage endpoint."
fi
# --- Redis Host Resolution Check ---
if ! check_host_resolves "$REDIS_HOST"; then
# This is potentially repairable if we can infer a better host,
# but for now we block if it's clearly invalid.
repair_blocked "Redis host does not resolve" \
"Resource: REDIS_HOST. Value: ${REDIS_HOST} does not resolve. Fix: Ensure Redis is deployed and REDIS_NAMESPACE is correct."
fi
# --- Database Connectivity Check ---
if [[ "$MODE" == "k8s" ]]; then
if ! check_db_connectivity "$DB_HOST" "$DB_PORT"; then
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
repair_blocked "GitLab database host is not reachable from APP cluster" \
"Host: ${DB_HOST}. Port: ${DB_PORT}. App Context: ${app_ctx}. DB Context: ${db_ctx}. Fix: Ensure cross-cluster networking (ILB) is functional."
fi
fi
}
get_gitlab_blocker_context() {
local ctx=""
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then
ctx+="Split-cluster (APP: ${app_ctx}, DB: ${db_ctx}). "
fi
ctx+="DB_HOST: ${DB_HOST}. "
local _last_mig=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -n "$_last_mig" ]]; then
local _m_ts=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
local _m_failed=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
local _m_pods=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --no-headers 2>/dev/null | wc -l | xargs || echo "0")
local _m_pod_ctx=""
local _m_last_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
if [[ -n "$_m_last_pod" ]]; then
local _m_pod_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.phase}' 2>/dev/null || true)
local _m_restarts=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0")
local _m_waiting=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "")
_m_pod_ctx="Pod: ${_m_last_pod} (${_m_pod_status}, restarts: ${_m_restarts}, waiting: ${_m_waiting:-None}). "
fi
ctx+="Migration Job: ${_last_mig} (Created: ${_m_ts}, Failed: ${_m_failed:-False}, Pods: ${_m_pods}). ${_m_pod_ctx}"
fi
local _sts_name="${GITLAB_RELEASE}-gitaly"
local _sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$_sts_yaml" ]]; then
local _sts_ns=$(echo "$_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local _sts_sc=$(echo "$_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
ctx+="Gitaly STS: nodeSelector=[${_sts_ns}], storageClass=[${_sts_sc}]. "
fi
local _pvc_name="repo-data-gitlab-gitaly-0"
local _pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$_pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
if [[ -n "$_pvc_phase" ]]; then
ctx+="Gitaly PVC: ${_pvc_phase}. "
fi
local _op_pod=$(kubectl -n "$NAMESPACE" get pods -l "control-plane=controller-manager" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -n "$_op_pod" ]]; then
local _op_log=$(kubectl -n "$NAMESPACE" logs "$_op_pod" -c manager --tail=1 2>/dev/null || true)
ctx+="Operator Log: ${_op_log}. "
fi
echo "$ctx"
}
is_gke_standard_storage_equivalent() {
local sc1="$1"
local sc2="$2"
[[ "$sc1" == "$sc2" ]] && return 0
if [[ ("$sc1" == "standard" || "$sc1" == "standard-rwo") && \
("$sc2" == "standard" || "$sc2" == "standard-rwo") ]]; then
return 0
fi
return 1
}
check_gitlab_post_apply_blocked() {
# --- Migrations Check ---
check_gitlab_migrations_blocked
# --- Gitaly Check ---
if [[ "$MODE" == "k8s" ]]; then
local desired_cr="${GITLAB_CR_RENDERED:-}"
if [[ -n "$desired_cr" ]]; then
local desired_gitaly_node_selector
desired_gitaly_node_selector=$(echo "$desired_cr" | sed -n '/gitaly:/,/toolbox:/p' | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local desired_gitaly_storage_class
desired_gitaly_storage_class=$(echo "$desired_cr" | sed -n '/gitaly:/,/toolbox:/p' | grep "storageClass:" | head -n 1 | cut -d: -f2 | xargs || true)
log "Desired Gitaly state (from CR): nodeSelector=[${desired_gitaly_node_selector}], storageClass=[${desired_gitaly_storage_class}]"
log "Internal script GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]"
if [[ "$desired_cr" == *"gandalf.prole.org"* ]]; then
repair_blocked "GitLab CR contains legacy nodeSelector" \
"Desired GitLab CR still contains gandalf.prole.org. This is a configuration bug."
fi
if [[ "$desired_cr" == *"gitlab-gitaly-static"* ]]; then
repair_blocked "GitLab CR contains legacy storageClass" \
"Desired GitLab CR still contains gitlab-gitaly-static. This is a configuration bug."
fi
if [[ -n "$desired_gitaly_storage_class" ]] && ! is_gke_standard_storage_equivalent "$desired_gitaly_storage_class" "${GITALY_STORAGE_CLASS:-}"; then
repair_blocked "GitLab CR storageClass mismatch" \
"Desired GitLab CR storageClass=[${desired_gitaly_storage_class}] does not match GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]. Configuration bug."
fi
fi
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
# If autoclean was performed, wait for convergence before checking live state.
# This prevents false positives when the operator hasn't yet updated the stale StatefulSet.
if [[ "${GITALY_AUTOCLEAN_PERFORMED:-0}" == "1" ]]; then
log "AUTOCLEAN was performed. Waiting for Gitaly StatefulSet to converge (removing legacy storage)..."
local grace_start=$(date +%s)
local grace_timeout=60
local converged=0
while true; do
local live_sts_yaml
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -z "$live_sts_yaml" ]]; then
log "Gitaly StatefulSet not found (awaiting operator action)..."
elif [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
log "Gitaly StatefulSet converged to corrected state (clean nodeSelector/storageClass)."
local _live_sc
_live_sc=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
if [[ -n "$_live_sc" ]] && ! is_gke_standard_storage_equivalent "$_live_sc" "${GITALY_STORAGE_CLASS:-}"; then
log "WARNING: Converged StatefulSet uses storageClass=[${_live_sc}], expected [${GITALY_STORAGE_CLASS:-}]."
fi
converged=1
break
else
local live_gitaly_node_selector
live_gitaly_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local live_gitaly_storage_class
live_gitaly_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
log "Still waiting for Gitaly convergence (live nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}])..."
fi
if (( $(date +%s) - grace_start > grace_timeout )); then
log "Grace period (${grace_timeout}s) expired."
break
fi
sleep 15
done
if [[ "$converged" == "0" ]]; then
local live_sts_yaml
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$live_sts_yaml" && ( "$live_sts_yaml" == *"gandalf.prole.org"* || "$live_sts_yaml" == *"gitlab-gitaly-static"* ) ]]; then
log "Live StatefulSet still legacy after grace period. Performing explicit replacement..."
# We already confirmed desired CR is corrected at the start of this function.
kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true
log "Legacy StatefulSet deleted. Waiting for operator recreation..."
local recreate_start=$(date +%s)
local recreate_timeout=300
while true; do
live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$live_sts_yaml" ]]; then
local recreated_node_selector
recreated_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local recreated_storage_class
recreated_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
log "StatefulSet recreated. nodeSelector=[${recreated_node_selector}], storageClass=[${recreated_storage_class}]"
if [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then
if [[ -n "$recreated_storage_class" ]] && ! is_gke_standard_storage_equivalent "$recreated_storage_class" "${GITALY_STORAGE_CLASS:-}"; then
repair_blocked "Recreated Gitaly StatefulSet uses wrong storageClass" \
"Resource: statefulset/${gitaly_sts_name}. Value: storageClassName=[${recreated_storage_class}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check operator reconciliation."
fi
log "Gitaly StatefulSet recreated in corrected state."
converged=1
break
else
log "Recreated StatefulSet STILL contains legacy fields. Waiting for operator to correct it..."
fi
fi
if (( $(date +%s) - recreate_start > recreate_timeout )); then
repair_blocked "Gitaly failed to recreate clean StatefulSet after ${recreate_timeout}s" \
"Resource: statefulset/${gitaly_sts_name}. Fix: Check operator logs and desired GitLab CR."
fi
sleep 15
done
else
log "No legacy StatefulSet found after grace period (may have been deleted or converged)."
converged=1
fi
fi
if [[ "$converged" == "1" ]]; then
# Wait for PVC provisioning if autoclean was performed
log "Waiting for PVC repo-data-gitlab-gitaly-0 to be provisioned and Bound..."
local pvc_name="repo-data-gitlab-gitaly-0"
local pvc_start=$(date +%s)
local pvc_timeout=600
while true; do
local pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
local pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
if [[ -n "$pvc_phase" ]]; then
local _sc_info=""
if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-standard-normalized)"; fi
if [[ "$pvc_sc" == "premium-rwo" ]]; then _sc_info=" (pd-ssd)"; fi
if [[ "$pvc_sc" == "standard" ]]; then _sc_info=" (pd-standard)"; fi
log "PVC ${pvc_name}: storageClass=[${pvc_sc}${_sc_info}], phase=[${pvc_phase}]"
if [[ -n "$pvc_sc" ]] && ! is_gke_standard_storage_equivalent "$pvc_sc" "${GITALY_STORAGE_CLASS:-}"; then
repair_blocked "Gitaly PVC uses wrong storageClass" \
"PVC: ${pvc_name}. Value: storageClass=[${pvc_sc}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check StatefulSet volumeClaimTemplates and operator reconciliation."
fi
if [[ "$pvc_phase" == "Bound" ]]; then
if [[ "$pvc_sc" == "${GITALY_STORAGE_CLASS:-}" ]]; then
log "PVC ${pvc_name} successfully provisioned on '${pvc_sc}' storage."
break
fi
fi
# Check for provisioning failures (events)
local provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pvc_name",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true)
if [[ -n "$provisioning_fail" ]]; then
if [[ "$provisioning_fail" == *"quota"* || "$provisioning_fail" == *"QUOTA"* ]]; then
repair_blocked "Gitaly PVC provisioning failed (Quota Exceeded)" \
"PVC: ${pvc_name}. Error: ${provisioning_fail}. Fix: Check GKE storage quotas and ensure Gitaly uses a valid StorageClass (standard or standard-rwo)."
else
log "PVC ${pvc_name} provisioning event: ${provisioning_fail}"
fi
fi
else
# Check migrations first to fail fast
check_gitlab_migrations_blocked
log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..."
fi
if (( $(date +%s) - pvc_start > pvc_timeout )); then
local last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pvc_name" --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
repair_blocked "Gitaly PVC failed to bind after ${pvc_timeout}s" \
"PVC: ${pvc_name}. Status: ${pvc_phase:-NotFound}. Last event: ${last_msg}. Fix: Check storage provider and quota."
fi
sleep 15
done
export GITALY_AUTOCLEAN_PERFORMED=0
fi
# Also wait for PV deletion to settle if it exists
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
log "Waiting for legacy PV gitlab-gitaly-synology to be removed..."
local pv_wait_start=$(date +%s)
while kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; do
if (( $(date +%s) - pv_wait_start > 120 )); then
log "Legacy PV still exists after 120s; continuing (it may be stuck in Terminating)."
break
fi
sleep 5
done
fi
fi
# Final live state checks
local gitaly_sts_yaml
gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$gitaly_sts_yaml" ]]; then
local live_gitaly_node_selector
live_gitaly_node_selector=$(echo "$gitaly_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true)
local live_gitaly_storage_class
live_gitaly_storage_class=$(echo "$gitaly_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true)
log "Live Gitaly state: nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}]"
if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* ]]; then
repair_blocked "Gitaly using legacy Synology storage" \
"Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Ensure GITLAB_STORAGE_NODE is not set in k8s mode."
fi
if [[ "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
repair_blocked "Gitaly using legacy Synology storage" \
"Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden in k8s mode."
fi
fi
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then
repair_blocked "Gitaly using legacy Synology storage" \
"Resource: pv/gitlab-gitaly-synology. Value: exists. Fix: Delete legacy PV or use GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1."
fi
local unschedulable_gitaly
unschedulable_gitaly=$(kubectl -n "$NAMESPACE" get pods -l "app=gitaly" -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="PodScheduled")].status=="False")]}{.metadata.name}:{.status.conditions[?(@.type=="PodScheduled")].reason}{"\n"}{end}' 2>/dev/null | grep "Unschedulable" || true)
if [[ -n "$unschedulable_gitaly" ]]; then
repair_blocked "Gitaly pod(s) unschedulable" \
"Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storage (standard or standard-rwo) and no hostname nodeSelector. Ensure PV/PVC were repaired."
fi
# --- Sidekiq Config Check ---
log "Desired Sidekiq state: concurrency=${GITLAB_SIDEKIQ_CONCURRENCY}, requests=[cpu=${GITLAB_SIDEKIQ_REQUESTS_CPU}, mem=${GITLAB_SIDEKIQ_REQUESTS_MEMORY}], limits=[cpu=${GITLAB_SIDEKIQ_LIMITS_CPU}, mem=${GITLAB_SIDEKIQ_LIMITS_MEMORY}]"
if [[ "${gitlab_apply_changed:-0}" == "1" || "${gitlab_spec_changed:-0}" == "1" ]]; then
log "GitLab CR was updated to fix configuration drift (Sidekiq or other fields)."
fi
local sidekiq_deploy_name="${GITLAB_RELEASE}-sidekiq-all-in-1-v2"
local live_sidekiq_cpu_req
live_sidekiq_cpu_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}' 2>/dev/null || true)
local live_sidekiq_mem_req
live_sidekiq_mem_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.memory}' 2>/dev/null || true)
local live_sidekiq_cpu_lim
live_sidekiq_cpu_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.cpu}' 2>/dev/null || true)
local live_sidekiq_mem_lim
live_sidekiq_mem_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.memory}' 2>/dev/null || true)
local live_sidekiq_concurrency
live_sidekiq_concurrency=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="SIDEKIQ_CONCURRENCY")].value}' 2>/dev/null || true)
if [[ -n "$live_sidekiq_cpu_req" ]]; then
log "Live Sidekiq state: concurrency=${live_sidekiq_concurrency}, requests=[cpu=${live_sidekiq_cpu_req}, mem=${live_sidekiq_mem_req}], limits=[cpu=${live_sidekiq_cpu_lim}, mem=${live_sidekiq_mem_lim}]"
fi
fi
# --- Registry Config Check ---
local registry_secret_config
registry_secret_config=$(kubectl -n "$NAMESPACE" get secret "${GITLAB_RELEASE}-registry-storage" -o jsonpath='{.data.config}' 2>/dev/null | base64 -d 2>/dev/null || true)
if [[ "$registry_secret_config" == *"<private-db-cluster-garage-endpoint>"* ]]; then
if [[ "$GARAGE_S3_ENDPOINT" != *"<private-db-cluster-garage-endpoint>"* ]]; then
log "Registry secret still contains placeholder but config is updated. Re-applying secret..."
# Re-triggering the secret creation (this is safer than just blocking)
setup_garage_for_gitlab
else
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder in live secret)" \
"Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint."
fi
fi
# --- Registry Logs Check ---
local registry_pod
registry_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=registry" -o name 2>/dev/null | head -n1 || true)
if [[ -n "$registry_pod" ]]; then
local logs
logs=$(kubectl -n "$NAMESPACE" logs "$registry_pod" --tail=100 2>&1 || true)
if [[ "$logs" == *"DNS failure"* || "$logs" == *"AccessDenied"* || "$logs" == *"No such key"* ]]; then
# If it's a "No such key" or "AccessDenied", and we have a custom endpoint, it might be the WRONG cluster Garage.
repair_blocked "Registry endpoint invalid or wrong Garage (S3 error detected)" \
"Logs: ${logs}. Fix: Verify GARAGE_S3_ENDPOINT points to the DB cluster Garage (not the APP cluster one)."
fi
fi
# --- KAS Logs Check ---
local kas_pod
kas_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=kas" -o name 2>/dev/null | head -n1 || true)
if [[ -n "$kas_pod" ]]; then
local logs
logs=$(kubectl -n "$NAMESPACE" logs "$kas_pod" --tail=100 2>&1 || true)
if [[ "$logs" == *"no such host"* && "$logs" == *"redis"* ]]; then
repair_blocked "Redis host does not resolve (detected in KAS logs)" \
"Logs: ${logs}. Fix: Verify REDIS_HOST (${REDIS_HOST}) and ensure Redis service is healthy."
fi
fi
}
kubectl_db() {
local db_ctx=""
db_ctx="$(resolve_db_cluster_context || true)"
if [[ -n "$db_ctx" ]]; then
command kubectl --context "$db_ctx" "$@"
else
kubectl "$@"
fi
}
resolve_garage_admin_context() {
if [[ "$MODE" == "k8s" ]]; then
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
local explicit_ctx=""
explicit_ctx="$(resolve_explicit_kube_context || true)"
if [[ -n "$db_ctx" && -n "$app_ctx" && "$db_ctx" != "$app_ctx" ]]; then
printf '%s' "$db_ctx"
return 0
fi
if [[ -n "$db_ctx" ]]; then
printf '%s' "$db_ctx"
return 0
fi
if [[ -n "$explicit_ctx" ]]; then
printf '%s' "$explicit_ctx"
return 0
fi
if [[ -n "$app_ctx" ]]; then
printf '%s' "$app_ctx"
return 0
fi
die "Explicit Garage admin context is required in k8s mode."
fi
return 1
}
kubectl_garage_admin() {
local garage_ctx=""
garage_ctx="$(resolve_garage_admin_context || true)"
if [[ -n "$garage_ctx" ]]; then
command kubectl --context "$garage_ctx" "$@"
else
kubectl "$@"
fi
}
db_kubectl() {
local db_ctx
db_ctx="$(resolve_db_cluster_context || true)"
if [[ -n "$db_ctx" ]]; then
command kubectl --context "$db_ctx" "$@"
else
kubectl "$@"
fi
}
ensure_cross_cluster_db_host() {
if [[ "$MODE" != "k8s" ]]; then
return 0
fi
local app_ctx db_ctx
app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
if [[ -z "$app_ctx" ]]; then
app_ctx="$(kubectl config current-context 2>/dev/null || true)"
fi
db_ctx="$(resolve_db_cluster_context || true)"
if [[ -z "$db_ctx" || -z "$app_ctx" || "$db_ctx" == "$app_ctx" ]]; then
return 0
fi
local db_cluster_name="${CNPG_CLUSTER_NAME:-knoe-db}"
local db_ns="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
local ilb_service="${GITLAB_DB_ILB_SERVICE:-knoe-db-rw-ilb}"
log "Split-cluster mode detected (APP: ${app_ctx}, DB: ${db_ctx})"
log "Ensuring cross-cluster Postgres ILB service '${ilb_service}' in namespace '${db_ns}'..."
db_kubectl -n "$db_ns" apply -f - <<EOF >/dev/null
apiVersion: v1
kind: Service
metadata:
name: ${ilb_service}
annotations:
networking.gke.io/load-balancer-type: "Internal"
spec:
type: LoadBalancer
selector:
cnpg.io/cluster: ${db_cluster_name}
cnpg.io/instanceRole: primary
ports:
- name: postgres
port: 5432
targetPort: 5432
protocol: TCP
EOF
local db_host=""
local attempts=0
log "Waiting for cross-cluster DB host (ILB IP/hostname)..."
while (( attempts < 60 )); do
attempts=$((attempts + 1))
db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)"
if [[ -z "$db_host" ]]; then
db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)"
fi
if [[ -n "$db_host" ]]; then
break
fi
sleep 5
done
if [[ -z "$db_host" ]]; then
warn "Cross-cluster Postgres ILB '${ilb_service}' has no ingress address yet after 5 minutes."
return 0
fi
log "Successfully resolved cross-cluster DB host: ${db_host}"
GITLAB_CROSS_CLUSTER_DB_HOST="$db_host"
export GITLAB_CROSS_CLUSTER_DB_HOST
}
enforce_app_cluster_targeting
is_truthy() {
case "${1:-}" in
1|true|TRUE|True|yes|YES|on|ON|y|Y)
return 0
;;
*)
return 1
;;
esac
}
assert_public_ingress_targeting() {
local ingress_class="${1:-}"
shift || true
local hosts=("$@")
if [[ "$MODE" != "k8s" ]]; then
return 0
fi
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}"
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
local active_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
if [[ -z "$app_ctx" || -z "$active_ctx" ]]; then
die "Explicit APP cluster context is required for GitLab ingress operations in k8s mode."
fi
local host_count=0
local h
for h in "${hosts[@]}"; do
[[ -n "${h:-}" ]] && host_count=$((host_count + 1))
done
if [[ "$host_count" -gt 0 && -n "$db_ctx" && "$active_ctx" == "$db_ctx" ]]; then
die "Refusing to render/apply GitLab public ingress in DB cluster context '${active_ctx}' (hosts: ${hosts[*]})."
fi
if [[ "$host_count" -gt 0 && -n "$app_ctx" && -n "$active_ctx" && "$active_ctx" != "$app_ctx" ]]; then
die "GitLab public ingress must target APP cluster context '${app_ctx}', active context is '${active_ctx}'."
fi
local normalized_class="${ingress_class,,}"
if [[ "$normalized_class" == traefik* ]] && ! is_truthy "${ALLOW_TRAEFIK_PUBLIC_INGRESS:-${GITLAB_ALLOW_TRAEFIK_INGRESS:-0}}"; then
die "Ingress class '${ingress_class}' is incompatible with k8s mode unless Traefik public ingress is explicitly enabled."
fi
}
assert_unique_ingress_host_claims() {
local ingress_name="${1:-}"
local ingress_namespace="${2:-}"
local host_csv="${3:-}"
local gitlab_release="${4:-}"
local expected_backend_service="${5:-}"
[[ -n "$host_csv" ]] || return 0
local target_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}"
if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then
die "Explicit kubectl context is required for ingress ownership checks in k8s mode."
fi
INGRESS_HOST_CLAIM_DETAILS=""
local claim_result claim_status
claim_result=$(python3 - "$host_csv" "$ingress_namespace" "$ingress_name" "$target_ctx" "$gitlab_release" "$expected_backend_service" <<'PY'
import json
import subprocess
import sys
requested_hosts = {h.strip().lower() for h in (sys.argv[1] or "").split(",") if h.strip()}
target_ns = sys.argv[2]
target_name = sys.argv[3]
target_ctx = (sys.argv[4] or "").strip()
target_release = (sys.argv[5] or "").strip()
expected_backend_service = (sys.argv[6] or "").strip()
cmd = ["kubectl"]
if target_ctx:
cmd.extend(["--context", target_ctx])
cmd.extend(["get", "ingress", "-A", "-o", "json"])
try:
raw = subprocess.check_output(cmd, text=True)
except Exception:
print("OK|")
raise SystemExit(0)
if not raw.strip():
print("OK|")
raise SystemExit(0)
try:
payload = json.loads(raw)
except Exception:
print("OK|")
raise SystemExit(0)
same_owner_conflicts: list[str] = []
foreign_conflicts: list[str] = []
for item in payload.get("items", []) or []:
md = item.get("metadata", {}) or {}
ns = (md.get("namespace") or "").strip()
name = (md.get("name") or "").strip()
if ns == target_ns and name == target_name:
continue
labels = md.get("labels", {}) or {}
spec = item.get("spec", {}) or {}
rules = spec.get("rules", []) or []
for rule in rules:
host = (rule.get("host") or "").strip().lower()
if not host or host not in requested_hosts:
continue
http = rule.get("http", {}) or {}
paths = http.get("paths", []) or [{"path": "/"}]
for path_item in paths:
path = (path_item.get("path") or "/").strip() or "/"
if path not in {"/", ""}:
continue
backend = path_item.get("backend", {}) or {}
backend_svc = backend.get("service", {}) or {}
backend_name = (backend_svc.get("name") or "").strip()
same_owner = False
if ns == target_ns:
if expected_backend_service and backend_name == expected_backend_service:
same_owner = True
if target_release and backend_name.startswith(f"{target_release}-webservice"):
same_owner = True
instance_label = (labels.get("app.kubernetes.io/instance") or "").strip()
part_of_label = (labels.get("app.kubernetes.io/part-of") or "").strip().lower()
if target_release and (
name == f"{target_release}-webservice-default"
or name.startswith(f"{target_release}-")
or instance_label == target_release
or part_of_label == "gitlab"
):
same_owner = True
owner_desc = f"{host}{path} already owned by {ns}/{name}"
if backend_name:
owner_desc += f" (backend={backend_name})"
if same_owner:
same_owner_conflicts.append(owner_desc)
else:
foreign_conflicts.append(owner_desc)
if foreign_conflicts:
print("CONFLICT|" + "; ".join(sorted(set(foreign_conflicts))))
elif same_owner_conflicts:
print("OWNED_BY_GITLAB|" + "; ".join(sorted(set(same_owner_conflicts))))
else:
print("OK|")
PY
)
claim_status="${claim_result%%|*}"
INGRESS_HOST_CLAIM_DETAILS="${claim_result#*|}"
case "$claim_status" in
OK|"")
return 0
;;
OWNED_BY_GITLAB)
return 10
;;
CONFLICT)
return 11
;;
*)
INGRESS_HOST_CLAIM_DETAILS="$claim_result"
return 11
;;
esac
}
while [[ $# -gt 0 ]]; do
case "$1" in
--mode) MODE="$(prole_normalize_mode "${2:-}")"; shift 2 ;;
--mode=*) MODE="$(prole_normalize_mode "${1#*=}")"; shift 1 ;;
-n|--namespace) NAMESPACE="${2:-}"; shift 2 ;;
--namespace=*) NAMESPACE="${1#*=}"; shift 1 ;;
-c|--config) CFG_PATH="${2:-}"; shift 2 ;;
--config=*) CFG_PATH="${1#*=}"; shift 1 ;;
--node-selector) NODE_SELECTOR="${2:-}"; shift 2 ;;
--node-selector=*) NODE_SELECTOR="${1#*=}"; shift 1 ;;
--force) FORCE=1; shift 1 ;;
-h|--help) usage; exit 0 ;;
--trace-namespace)
# Resolve namespace then exit — used by diagnostic scripts only
_TRACE_NS=1; shift 1 ;;
*) break ;;
esac
done
# ---------------------------------------------------------------------------
# Resolve config path
# ---------------------------------------------------------------------------
if [[ -z "$CFG_PATH" && -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then
CFG_PATH="$(_prole_cfg_select_cfg_file "${PROLE_CONF}")"
fi
if [[ -z "$CFG_PATH" ]]; then
CFG_PATH="$(_prole_cfg_select_cfg_file "$SCRIPT_DIR/../conf")"
fi
# cfg-based namespace override (only GITLAB_NAMESPACE; GITOPS_NAMESPACE intentionally excluded)
if [[ -z "$NAMESPACE" || "$NAMESPACE" == "gitlab" ]] && [[ -n "$CFG_PATH" ]]; then
maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")"
[[ -n "$maybe_ns" ]] && NAMESPACE="$maybe_ns"
fi
# Final safety net — always default to 'gitlab'
NAMESPACE="${NAMESPACE:-gitlab}"
export GITLAB_NAMESPACE="$NAMESPACE"
# Diagnostic exit — used by tmp/sim_installer_ns.sh
if [[ "${_TRACE_NS:-0}" == "1" ]]; then
echo "TRACE: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" >&2
echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-<unset>}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-<unset>}'" >&2
exit 0
fi
case "$MODE" in
k3d|k3s|k8s|local) ;;
*) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;;
esac
export PROLE_MODE="$MODE"
command -v kubectl >/dev/null || die "kubectl not found"
command -v helm >/dev/null || die "helm not found (required for GitLab Operator install)"
# ---------------------------------------------------------------------------
# knoe-db (CNPG) settings — reuse the same cluster gitea uses
# ---------------------------------------------------------------------------
DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-${CLUSTER_NAME:-knoe-db}}"
ensure_cross_cluster_db_host
GITLAB_DB_NAME="${GITLAB_DB_NAME:-gitlabhq_production}"
GITLAB_DB_USER="${GITLAB_DB_USER:-gitlab}"
GITLAB_DB_PASSWORD="${GITLAB_DB_PASSWORD:-}"
DB_HOST="${GITLAB_DB_HOST:-${GITLAB_CROSS_CLUSTER_DB_HOST:-${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local}}"
DB_PORT="${GITLAB_DB_PORT:-5432}"
# ---------------------------------------------------------------------------
# Redis — shared common service (deployed by init_redis.sh in knoe-system)
# ---------------------------------------------------------------------------
REDIS_NAMESPACE="${REDIS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}"
REDIS_HOST="${GITLAB_REDIS_HOST:-redis-master.${REDIS_NAMESPACE}.svc.cluster.local}"
REDIS_PORT="${GITLAB_REDIS_PORT:-6379}"
# GitLab public domain (configurable; default follows deployment mode)
_default_gitlab_domain="git.prole.org"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_domain="git.knoe.dev"
fi
GITLAB_DOMAIN="${GITLAB_DOMAIN:-${GITLAB_HOSTNAME:-$_default_gitlab_domain}}"
GITLAB_PUBLIC_HOSTS_RAW="${GITLAB_PUBLIC_HOSTS:-$GITLAB_DOMAIN}"
trim_csv_token() {
local token="$1"
token="${token#${token%%[![:space:]]*}}"
token="${token%${token##*[![:space:]]}}"
printf '%s' "$token"
}
_gitlab_hosts_csv=""
GITLAB_PUBLIC_HOSTS=()
IFS=',' read -r -a _gitlab_host_candidates <<< "$GITLAB_PUBLIC_HOSTS_RAW"
for _gitlab_host in "${_gitlab_host_candidates[@]}"; do
_gitlab_host="$(trim_csv_token "$_gitlab_host")"
[[ -n "$_gitlab_host" ]] || continue
case ",${_gitlab_hosts_csv}," in
*,"${_gitlab_host}",*) ;;
*)
GITLAB_PUBLIC_HOSTS+=("$_gitlab_host")
_gitlab_hosts_csv="${_gitlab_hosts_csv:+${_gitlab_hosts_csv},}${_gitlab_host}"
;;
esac
done
unset _gitlab_host_candidates _gitlab_host
if [[ ${#GITLAB_PUBLIC_HOSTS[@]} -eq 0 ]]; then
GITLAB_PUBLIC_HOSTS=("$GITLAB_DOMAIN")
fi
PRIMARY_GITLAB_HOST="${GITLAB_PUBLIC_HOSTS[0]}"
GITLAB_DOMAIN="$PRIMARY_GITLAB_HOST"
PRIMARY_GITLAB_DOMAIN_ROOT="${PRIMARY_GITLAB_HOST#*.}"
if [[ "$PRIMARY_GITLAB_DOMAIN_ROOT" == "$PRIMARY_GITLAB_HOST" || -z "$PRIMARY_GITLAB_DOMAIN_ROOT" ]]; then
PRIMARY_GITLAB_DOMAIN_ROOT="prole.org"
fi
# Public ingress class (k8s/GKE defaults to gce; local clusters keep kong)
_default_gitlab_ingress_class="kong"
if [[ "$MODE" == "k8s" ]]; then
_default_gitlab_ingress_class="gce"
fi
GITLAB_INGRESS_CLASS="${GITLAB_INGRESS_CLASS:-$_default_gitlab_ingress_class}"
# Google Workspace OIDC — FRONTDOOR_HOST gates OmniAuth configuration.
# On k3s: api.prole.org is the prole-auth SSO gateway (knoe-auth service).
# Requires k8s secret 'gitlab-google-oidc' in GITLAB_NAMESPACE with Google
# OAuth2 client credentials (client_id, client_secret, redirect_uri).
# To disable OIDC: unset FRONTDOOR_HOST before running.
_default_auth_hostname="api.prole.org"
if [[ "$MODE" == "k8s" ]]; then
_default_auth_hostname="api.knoe.dev"
fi
AUTH_HOSTNAME="${AUTH_HOSTNAME:-${FRONTDOOR_HOST:-${_default_auth_hostname}}}"
FRONTDOOR_HOST="${FRONTDOOR_HOST:-$AUTH_HOSTNAME}"
FRONTDOOR_AUTH_ENABLED="${FRONTDOOR_AUTH_ENABLED:-${AUTHORITY_ENABLED:-1}}"
AUTH_VERIFY_PATH="${AUTH_VERIFY_PATH:-/auth/verify}"
AUTH_LOGIN_PATH="${AUTH_LOGIN_PATH:-/auth/login}"
AUTH_RESPONSE_HEADERS="${AUTH_RESPONSE_HEADERS:-X-Prole-User,X-Prole-Email,X-Prole-Groups}"
AUTH_VERIFY_URL="${AUTH_VERIFY_URL:-https://${AUTH_HOSTNAME}${AUTH_VERIFY_PATH}}"
AUTH_SIGNIN_URL="${AUTH_SIGNIN_URL:-https://${AUTH_HOSTNAME}${AUTH_LOGIN_PATH}?next=\$scheme://\$host\$escaped_request_uri}"
GITLAB_RELEASE="gitlab"
assert_public_ingress_targeting "$GITLAB_INGRESS_CLASS" "${GITLAB_PUBLIC_HOSTS[@]}"
# ---------------------------------------------------------------------------
# Garage S3 (knoe-system) — replaces embedded minio for object storage
# ---------------------------------------------------------------------------
GARAGE_NAMESPACE="${GARAGE_NAMESPACE:-knoe-system}"
GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-}"
GARAGE_S3_ENDPOINT="${GARAGE_S3_ENDPOINT:-${GARAGE_PRIVATE_S3_ENDPOINT:-${GARAGE_PRIVATE_ENDPOINT:-}}}"
if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
"Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint in your config."
fi
if [[ -z "$GARAGE_S3_ENDPOINT" ]]; then
if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]]; then
die "GARAGE_S3_ENDPOINT (or GARAGE_PRIVATE_S3_ENDPOINT) must be set to an explicit private cross-cluster endpoint when APP and DB clusters differ."
fi
GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-garage.${GARAGE_NAMESPACE}.svc.cluster.local}"
GARAGE_S3_ENDPOINT="http://${GARAGE_SVC_HOST}:3900"
fi
if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]] && [[ "$GARAGE_S3_ENDPOINT" == *".svc.cluster.local"* ]]; then
die "GARAGE_S3_ENDPOINT must use a private cross-cluster endpoint and cannot use cluster-local service DNS (${GARAGE_S3_ENDPOINT})."
fi
if [[ -z "$GARAGE_SVC_HOST" ]]; then
_garage_endpoint_hostport="${GARAGE_S3_ENDPOINT#http://}"
_garage_endpoint_hostport="${_garage_endpoint_hostport#https://}"
_garage_endpoint_hostport="${_garage_endpoint_hostport%%/*}"
GARAGE_SVC_HOST="${_garage_endpoint_hostport%%:*}"
unset _garage_endpoint_hostport
fi
GARAGE_S3_KEY_NAME="${GARAGE_S3_KEY_NAME:-gitlab-s3}"
GITLAB_OBJECT_STORAGE_REQUIRED="${GITLAB_OBJECT_STORAGE_REQUIRED:-1}"
# ---------------------------------------------------------------------------
# Operator chart coordinates
# ---------------------------------------------------------------------------
OPERATOR_REPO_NAME="gitlab-operator"
OPERATOR_REPO_URL="https://gitlab.com/api/v4/projects/18899486/packages/helm/stable"
OPERATOR_CHART="gitlab-operator/gitlab-operator"
OPERATOR_RELEASE="gitlab-operator"
OPERATOR_NAMESPACE="${GITLAB_OPERATOR_NAMESPACE:-${NAMESPACE}}"
GITLAB_OPERATOR_CHART_VERSION="${GITLAB_OPERATOR_CHART_VERSION:-}"
# GitLab Helm chart version required in the CR; auto-detected if not set.
GITLAB_CHART_VERSION="${GITLAB_CHART_VERSION:-}"
GITLAB_CHART_REPO_NAME="gitlab"
GITLAB_CHART_REPO_URL="https://charts.gitlab.io/"
gitlab_db_secret_changed=0
gitlab_object_storage_secret_changed=0
gitlab_registry_storage_secret_changed=0
resolve_helm_release_chart_version() {
local release_namespace="$1"
local release_name="$2"
local chart_version
chart_version=$({ helm -n "$release_namespace" list -f "^${release_name}$" -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys
try:
rows = json.load(sys.stdin)
except Exception:
print("")
raise SystemExit(0)
if not rows:
print("")
raise SystemExit(0)
chart = str(rows[0].get("chart", ""))
if not chart:
print("")
raise SystemExit(0)
if "-" in chart:
print(chart.rsplit("-", 1)[-1])
else:
print("")
PY
)
if [[ -n "$chart_version" ]]; then
printf '%s' "$chart_version"
return 0
fi
{ helm -n "$release_namespace" status "$release_name" -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys
try:
payload = json.load(sys.stdin)
except Exception:
print("")
raise SystemExit(0)
chart = str(payload.get("chart", ""))
if not chart:
print("")
raise SystemExit(0)
if "-" in chart:
print(chart.rsplit("-", 1)[-1])
else:
print("")
PY
}
resolve_operator_watch_namespace() {
local watch_namespace
watch_namespace=$({ helm -n "$OPERATOR_NAMESPACE" get values "$OPERATOR_RELEASE" --all -o json 2>/dev/null || true; } | python3 - <<'PY'
import json
import sys
try:
values = json.load(sys.stdin)
except Exception:
print("")
raise SystemExit(0)
watch_namespace = values.get("watchNamespace", "")
if isinstance(watch_namespace, str):
print(watch_namespace.strip())
else:
print("")
PY
)
if [[ -n "$watch_namespace" ]]; then
printf '%s' "$watch_namespace"
return 0
fi
kubectl -n "$OPERATOR_NAMESPACE" get deploy "$OPERATOR_RELEASE" \
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="WATCH_NAMESPACE")].value}' 2>/dev/null || true
}
kubectl_apply_reports_changed() {
local apply_output="${1:-}"
if [[ -z "${apply_output//[[:space:]]/}" ]]; then
# Be conservative: unknown apply output should force full reconcile checks.
return 0
fi
case "$apply_output" in
*" created"*|*" configured"*|*" patched"*) return 0 ;;
esac
case "$apply_output" in
*" unchanged"*) return 1 ;;
esac
# Unknown token from kubectl apply output -> treat as changed.
return 0
}
# ---------------------------------------------------------------------------
# Helper: resolve knoe-db primary pod (same pattern as init_gitea.sh)
# ---------------------------------------------------------------------------
resolve_knoe_db_primary_pod() {
local primary
primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \
-l "cnpg.io/cluster=${CNPG_CLUSTER_NAME},cnpg.io/instanceRole=primary" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -z "$primary" ]]; then
primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \
-l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
fi
printf '%s' "$primary"
}
sql_escape_literal() {
printf '%s' "${1:-}" | sed "s/'/''/g"
}
# ---------------------------------------------------------------------------
# Resolve GitLab DB password (pull from knoe-db-superuser secret as fallback)
# ---------------------------------------------------------------------------
resolve_gitlab_db_password() {
if [[ -n "${GITLAB_DB_PASSWORD:-}" ]]; then
return 0
fi
local resolved=""
if kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then
resolved=$(kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser \
-o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true)
fi
if [[ -n "$resolved" ]]; then
GITLAB_DB_PASSWORD="$resolved"
else
# Generate a random password when none is available
GITLAB_DB_PASSWORD="$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom 2>/dev/null | head -c 32 || true)"
warn "Generated random GitLab DB password; store it in GITLAB_DB_PASSWORD for future runs."
fi
}
# ---------------------------------------------------------------------------
# Provision GitLab role + database in knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab() {
local primary db_ctx
db_ctx="$(resolve_db_cluster_context || true)"
primary="$(resolve_knoe_db_primary_pod)"
if [[ -z "$primary" ]]; then
if [[ "$MODE" == "k8s" ]]; then
die "No knoe-db pod found in namespace '${DB_NAMESPACE}' on DB_CLUSTER_KUBECONTEXT='${db_ctx}'."
fi
warn "No knoe-db pod found in namespace '${DB_NAMESPACE}'; skipping GitLab DB setup."
return 0
fi
resolve_gitlab_db_password
local admin_user=""
local candidate
for candidate in postgres root; do
if kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$candidate" -d postgres -tAc "SELECT 1" >/dev/null 2>&1; then
admin_user="$candidate"
break
fi
done
if [[ -z "$admin_user" ]]; then
if [[ "$MODE" == "k8s" ]]; then
die "Unable to connect to knoe-db as admin on DB_CLUSTER_KUBECONTEXT='${db_ctx}'."
fi
warn "Unable to connect to knoe-db as admin; skipping GitLab DB setup."
return 0
fi
local esc_pw
esc_pw="$(sql_escape_literal "$GITLAB_DB_PASSWORD")"
if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -c "
DO \$\$ BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${GITLAB_DB_USER}') THEN
CREATE ROLE ${GITLAB_DB_USER} LOGIN PASSWORD '${esc_pw}';
ELSE
ALTER ROLE ${GITLAB_DB_USER} WITH PASSWORD '${esc_pw}';
END IF;
END \$\$;
" >/dev/null 2>&1; then
if [[ "$MODE" == "k8s" ]]; then
die "Could not create/update role '${GITLAB_DB_USER}' on DB cluster context '${db_ctx}'."
fi
warn "Could not create/update role '${GITLAB_DB_USER}'."
fi
local db_exists
db_exists=$(kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -tAc \
"SELECT 1 FROM pg_database WHERE datname='${GITLAB_DB_NAME}';" 2>/dev/null || true)
if [[ "$db_exists" != "1" ]]; then
if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -c \
"CREATE DATABASE ${GITLAB_DB_NAME} OWNER ${GITLAB_DB_USER};" \
>/dev/null 2>&1; then
if [[ "$MODE" == "k8s" ]]; then
die "Could not create database '${GITLAB_DB_NAME}' on DB cluster context '${db_ctx}'."
fi
warn "Could not create database '${GITLAB_DB_NAME}'."
fi
fi
if [[ -n "$db_ctx" ]]; then
log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE}, ctx=${db_ctx})."
else
log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE})."
fi
}
# ---------------------------------------------------------------------------
# Clean up any pre-existing Gitea / git.prole.org configurations
# ---------------------------------------------------------------------------
cleanup_gitea() {
log "Checking for pre-existing Gitea deployment to remove before GitLab install..."
local gitea_ns="${GITEA_NAMESPACE:-gitea}"
# Remove Gitea Helm release
if helm -n "$gitea_ns" status gitea >/dev/null 2>&1; then
log "Uninstalling Gitea Helm release from namespace '$gitea_ns'..."
helm -n "$gitea_ns" uninstall gitea >/dev/null 2>&1 || true
fi
# Remove any leftover raw Gitea resources
kubectl -n "$gitea_ns" delete deploy/gitea svc/gitea-http svc/gitea-ssh \
>/dev/null 2>&1 || true
# Remove Kong routes/services registered for git.prole.org (best-effort)
local kong_ns="${KONG_NAMESPACE:-${NAMESPACE:-kong}}"
for res_type in kongplugins kongingresses; do
kubectl -n "$gitea_ns" delete "$res_type" --all >/dev/null 2>&1 || true
done
# Remove gitea namespace Ingress objects that route git.prole.org
kubectl -n "$gitea_ns" delete ingress \
-l "app.kubernetes.io/name=gitea" >/dev/null 2>&1 || true
kubectl -n "$gitea_ns" delete ingress \
--field-selector="metadata.name=gitea" >/dev/null 2>&1 || true
log "Gitea cleanup complete."
}
# ---------------------------------------------------------------------------
# Ensure gitlab namespace exists
# ---------------------------------------------------------------------------
kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null
# ---------------------------------------------------------------------------
# --force: remove existing GitLab operator and release first
# ---------------------------------------------------------------------------
if [[ "$FORCE" -eq 1 ]]; then
warn "--force: removing existing GitLab resources in namespace '$NAMESPACE'..."
if kubectl get crd gitlabs.apps.gitlab.com >/dev/null 2>&1; then
kubectl -n "$NAMESPACE" delete gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
# Wait briefly for operator to clean up managed resources
sleep 10
fi
helm -n "$NAMESPACE" uninstall "$GITLAB_RELEASE" >/dev/null 2>&1 || true
helm -n "$NAMESPACE" uninstall "$OPERATOR_RELEASE" >/dev/null 2>&1 || true
cleanup_gitea
fi
# ---------------------------------------------------------------------------
# Always remove gitea if its helm release exists (non-destructive path)
# gitea and gitlab both claim git.prole.org; they cannot coexist.
# ---------------------------------------------------------------------------
if helm -n "${GITEA_NAMESPACE:-gitea}" status gitea >/dev/null 2>&1; then
warn "Gitea release detected — removing to free git.prole.org for GitLab..."
cleanup_gitea
fi
# ---------------------------------------------------------------------------
# Prepare knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab
# Persist the DB password as a k8s Secret the operator CR can reference
resolve_gitlab_db_password
gitlab_db_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-db-password \
--from-literal=password="$GITLAB_DB_PASSWORD" \
--dry-run=client -o yaml | kubectl apply -f - 2>&1)"
if kubectl_apply_reports_changed "$gitlab_db_secret_apply_output"; then
gitlab_db_secret_changed=1
fi
log "gitlab-db-password secret applied in namespace '$NAMESPACE' (${gitlab_db_secret_apply_output})."
# ---------------------------------------------------------------------------
# Install / upgrade the GitLab Operator
# ---------------------------------------------------------------------------
operator_changed=0
operator_release_exists=0
operator_requires_upgrade=1
installed_operator_chart_version=""
installed_operator_watch_namespace=""
operator_watch_namespace_matches=1
if helm -n "$OPERATOR_NAMESPACE" status "$OPERATOR_RELEASE" >/dev/null 2>&1; then
operator_release_exists=1
installed_operator_chart_version="$(resolve_helm_release_chart_version "$OPERATOR_NAMESPACE" "$OPERATOR_RELEASE")"
installed_operator_watch_namespace="$(resolve_operator_watch_namespace)"
fi
if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" && -n "$installed_operator_chart_version" ]]; then
GITLAB_OPERATOR_CHART_VERSION="$installed_operator_chart_version"
log "Reusing installed GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
fi
if [[ -n "$installed_operator_watch_namespace" && "$installed_operator_watch_namespace" != "$NAMESPACE" ]]; then
operator_watch_namespace_matches=0
fi
if [[ "$operator_release_exists" == "1" && -n "$installed_operator_chart_version" \
&& "$installed_operator_chart_version" == "$GITLAB_OPERATOR_CHART_VERSION" \
&& "$operator_watch_namespace_matches" == "1" \
&& "${GITLAB_OPERATOR_FORCE_UPGRADE:-0}" != "1" ]]; then
operator_requires_upgrade=0
fi
if [[ "$operator_requires_upgrade" == "1" ]]; then
if [[ "$operator_watch_namespace_matches" == "0" ]]; then
warn "Installed GitLab Operator watchNamespace (${installed_operator_watch_namespace}) differs from target (${NAMESPACE}); forcing operator upgrade."
fi
log "Adding/updating GitLab Operator Helm repo..."
helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true
helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..."
if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then
GITLAB_OPERATOR_CHART_VERSION=$(helm search repo "$OPERATOR_CHART" --output table 2>/dev/null \
| awk 'NR==2{print $2}' || true)
if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then
die "Cannot detect GitLab Operator chart version. Set GITLAB_OPERATOR_CHART_VERSION explicitly and re-run."
fi
log "Auto-detected GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
else
log "Using GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}"
fi
log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..."
# The operator needs cluster-scoped RBAC; it watches all namespaces by default.
helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \
-n "$OPERATOR_NAMESPACE" \
--create-namespace \
--version "$GITLAB_OPERATOR_CHART_VERSION" \
--timeout 10m \
--wait \
--set watchNamespace="$NAMESPACE"
operator_changed=1
log "GitLab Operator ready."
else
log "GitLab Operator already matches desired chart/watchNamespace; skipping operator upgrade."
fi
# ---------------------------------------------------------------------------
# Resolve GitLab chart version (required by the Operator CR since >= v0.28)
# ---------------------------------------------------------------------------
installed_gitlab_chart_version=""
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
installed_gitlab_chart_version="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.spec.chart.version}' 2>/dev/null || true)"
if [[ -n "$installed_gitlab_chart_version" ]]; then
GITLAB_CHART_VERSION="$installed_gitlab_chart_version"
log "Reusing installed GitLab chart version from existing GitLab CR: ${GITLAB_CHART_VERSION}"
else
log "Auto-detecting latest GitLab chart version..."
helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true
helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..."
GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \
| awk 'NR==2{print $2}' || true)
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run."
fi
log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}"
fi
fi
CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\""
gitlab_chart_version_changed=0
if [[ -z "$installed_gitlab_chart_version" || "$installed_gitlab_chart_version" != "$GITLAB_CHART_VERSION" ]]; then
gitlab_chart_version_changed=1
fi
# ---------------------------------------------------------------------------
# Determine node selector block for the GitLab CR
# ---------------------------------------------------------------------------
# Storage node is only used for legacy/local static local-PV mode.
if [[ "$MODE" == "k8s" ]]; then
# In k8s/GKE mode, we do NOT use static storage nodes or nodeSelectors.
STORAGE_NODE=""
if [[ "$NODE_SELECTOR" == *"gandalf.prole.org"* ]]; then
warn "Stripping legacy nodeSelector '${NODE_SELECTOR}' in k8s mode."
NODE_SELECTOR=""
fi
else
STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
fi
[[ "$MODE" == "k8s" || -n "$STORAGE_NODE" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set in ${MODE} mode."
# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned
NODE_SELECTOR="${NODE_SELECTOR:-}"
NODE_SELECTOR_KEY="${GITLAB_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"
STORAGE_NODE_SELECTOR_KEY="${GITLAB_STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY}}"
NODE_SELECTOR_YAML=""
if [[ -n "$NODE_SELECTOR" ]]; then
log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}"
NODE_SELECTOR_YAML="${NODE_SELECTOR_KEY}: ${NODE_SELECTOR}"
fi
STORAGE_NODE_SELECTOR_YAML=""
if [[ -n "$STORAGE_NODE" ]]; then
STORAGE_NODE_SELECTOR_YAML="${STORAGE_NODE_SELECTOR_KEY}: ${STORAGE_NODE}"
fi
GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}"
if [[ "$MODE" == "k8s" ]]; then
if [[ -z "$GITALY_STORAGE_CLASS" || "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" || "$GITALY_STORAGE_CLASS" == "standard" ]]; then
if [[ "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode."
fi
# GKE CSI normalization: prefer standard-rwo if it exists on cluster
# or if the existing CR already uses it (avoiding drift loops).
# This ensures that 'standard' and 'standard-rwo' are treated as equivalent.
_current_cr_sc=$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.spec.chart.values.global.persistence.storageClass}' 2>/dev/null || true)
if [[ -n "$_current_cr_sc" ]] && is_gke_standard_storage_equivalent "$_current_cr_sc" "standard"; then
GITALY_STORAGE_CLASS="$_current_cr_sc"
log "Using existing equivalent StorageClass from CR: ${GITALY_STORAGE_CLASS}"
elif kubectl get storageclass standard-rwo >/dev/null 2>&1; then
GITALY_STORAGE_CLASS="standard-rwo"
log "Detected standard-rwo StorageClass; using for Gitaly."
else
GITALY_STORAGE_CLASS="standard"
fi
fi
else
if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
GITALY_STORAGE_CLASS="gitlab-gitaly-static"
fi
fi
# Sidekiq resources and concurrency
GITLAB_SIDEKIQ_REQUESTS_CPU="${GITLAB_SIDEKIQ_REQUESTS_CPU:-250m}"
GITLAB_SIDEKIQ_REQUESTS_MEMORY="${GITLAB_SIDEKIQ_REQUESTS_MEMORY:-1500Mi}"
GITLAB_SIDEKIQ_LIMITS_CPU="${GITLAB_SIDEKIQ_LIMITS_CPU:-1}"
GITLAB_SIDEKIQ_LIMITS_MEMORY="${GITLAB_SIDEKIQ_LIMITS_MEMORY:-3Gi}"
GITLAB_SIDEKIQ_CONCURRENCY="${GITLAB_SIDEKIQ_CONCURRENCY:-5}"
JEMALLOC_HOSTPATH_DIR="/opt/gitlab-jemalloc"
JEMALLOC_HOSTPATH_LIB="${JEMALLOC_HOSTPATH_DIR}/libjemalloc.so.2"
GITLAB_JEMALLOC_MODE="${GITLAB_JEMALLOC_MODE:-auto}"
if is_truthy "${GITLAB_JEMALLOC_REQUIRED:-0}"; then
GITLAB_JEMALLOC_MODE="force"
fi
JEMALLOC_HOSTPATH_SUPPORTED=1
JEMALLOC_HOSTPATH_REASON=""
JEMALLOC_HOSTPATH_WANTED=0
JEMALLOC_HOSTPATH_ACTIVE=0
detect_jemalloc_hostpath_support() {
local provider_ids gke_pool_labels gke_topology_labels os_images
provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true)
gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true)
gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true)
os_images=$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.nodeInfo.osImage}{"\n"}{end}' 2>/dev/null || true)
if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then
JEMALLOC_HOSTPATH_SUPPORTED=0
JEMALLOC_HOSTPATH_REASON="Detected GKE/GCE node metadata (providerID/labels)."
return 0
fi
if [[ "${os_images,,}" == *"container-optimized os"* ]]; then
JEMALLOC_HOSTPATH_SUPPORTED=0
JEMALLOC_HOSTPATH_REASON="Detected Container-Optimized OS node image."
fi
}
configure_jemalloc_hostpath_mode() {
local mode_normalized="${GITLAB_JEMALLOC_MODE,,}"
detect_jemalloc_hostpath_support
case "$mode_normalized" in
auto|"")
if (( JEMALLOC_HOSTPATH_SUPPORTED == 1 )); then
JEMALLOC_HOSTPATH_WANTED=1
else
JEMALLOC_HOSTPATH_WANTED=0
warn "Skipping hostPath jemalloc optimization: ${JEMALLOC_HOSTPATH_REASON}"
fi
;;
off|false|0|disabled)
JEMALLOC_HOSTPATH_WANTED=0
;;
force|required|on|true|1)
JEMALLOC_HOSTPATH_WANTED=1
if (( JEMALLOC_HOSTPATH_SUPPORTED == 0 )); then
die "HostPath jemalloc mode is forced but unsupported in this cluster: ${JEMALLOC_HOSTPATH_REASON} Disable jemalloc (GITLAB_JEMALLOC_MODE=off) or use a non-hostPath approach."
fi
;;
*)
die "Unsupported GITLAB_JEMALLOC_MODE='${GITLAB_JEMALLOC_MODE}' (expected: auto, off, force)."
;;
esac
}
# ---------------------------------------------------------------------------
# Create the GitLab CR (operator reconciles this into the full deployment)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Apply gitlab-google-oidc secret (OIDC provider config for OmniAuth)
# ---------------------------------------------------------------------------
if [[ -n "$FRONTDOOR_HOST" ]]; then
PLATFORM_DOMAIN="${PLATFORM_DOMAIN:-}"
oidc_secret_tmpl="${SCRIPT_DIR}/../deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
if [[ -f "$oidc_secret_tmpl" ]]; then
log "Applying gitlab-google-oidc secret (issuer=https://${FRONTDOOR_HOST}/auth) ..."
FRONTDOOR_HOST="$FRONTDOOR_HOST" PLATFORM_DOMAIN="$PLATFORM_DOMAIN" \
envsubst < "$oidc_secret_tmpl" | kubectl apply -n "$NAMESPACE" -f - >/dev/null || \
warn "Could not apply gitlab-google-oidc secret; create it manually from deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
else
warn "gitlab-google-oidc-secret.example.yaml not found; skipping OIDC secret (create it manually)."
fi
fi
# ---------------------------------------------------------------------------
# ARM64 / 16KB-page jemalloc fix — build glibc jemalloc on every node once
# All RPi nodes use 16KB kernel pages; GitLab's bundled jemalloc is 4KB-only.
# We compile a compatible jemalloc-5.3.0 with --with-lg-page=14 via DaemonSet.
# ---------------------------------------------------------------------------
setup_jemalloc_on_nodes() {
local jlib="${JEMALLOC_HOSTPATH_LIB}"
local ds_name="jemalloc-builder"
if (( JEMALLOC_HOSTPATH_WANTED == 0 )); then
log "HostPath jemalloc optimization disabled for this environment."
if kubectl -n "$NAMESPACE" get ds "$ds_name" >/dev/null 2>&1; then
log "Deleting existing ${ds_name} DaemonSet because hostPath jemalloc is disabled."
kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \
warn "Could not delete ${ds_name} DaemonSet; continuing."
fi
JEMALLOC_HOSTPATH_ACTIVE=0
return 0
fi
# Check if already present on all schedulable nodes
local nodes
nodes=$(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | tr '\n' ' ')
local all_ready=true
for node in $nodes; do
result=$(kubectl -n "$NAMESPACE" run "jcheck-${node//\./-}" \
--image=ubuntu:22.04 --restart=Never --rm --attach --quiet \
--overrides="{\"spec\":{\"nodeName\":\"${node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"jlib\",\"hostPath\":{\"path\":\"${JEMALLOC_HOSTPATH_DIR}\",\"type\":\"DirectoryOrCreate\"}}],\"containers\":[{\"name\":\"c\",\"image\":\"ubuntu:22.04\",\"command\":[\"bash\",\"-c\",\"test -f /jlib/libjemalloc.so.2 && echo OK || echo MISSING\"],\"volumeMounts\":[{\"name\":\"jlib\",\"mountPath\":\"/jlib\"}]}]}}" 2>/dev/null || echo "MISSING")
if [[ "$result" != *"OK"* ]]; then
all_ready=false
break
fi
done
if $all_ready; then
log "jemalloc-16k already present on all nodes — skipping build."
JEMALLOC_HOSTPATH_ACTIVE=1
return 0
fi
log "Deploying jemalloc-builder DaemonSet (compiles jemalloc-5.3.0 with --with-lg-page=14 on each node)..."
local _tmpds
_tmpds=$(mktemp /tmp/jemalloc-ds-XXXXXX.yaml)
cat > "$_tmpds" <<'JEDS'
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: jemalloc-builder
namespace: NAMESPACE_PLACEHOLDER
labels:
app: jemalloc-builder
spec:
selector:
matchLabels:
app: jemalloc-builder
template:
metadata:
labels:
app: jemalloc-builder
spec:
tolerations:
- operator: Exists
initContainers:
- name: build
image: ubuntu:22.04
command:
- bash
- -c
- |
set -e
TARGET=/hostlib/libjemalloc.so.2
if [ -f "$TARGET" ]; then echo "Already built"; exit 0; fi
apt-get update -qq && apt-get install -y -q gcc make wget bzip2 2>&1
cd /tmp
wget -q -O jemalloc.tar.bz2 \
https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
tar xjf jemalloc.tar.bz2 && cd jemalloc-5.3.0
./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1
make -j2 lib/libjemalloc.so.2 2>&1
cp lib/libjemalloc.so.2 "$TARGET" && chmod 755 "$TARGET"
echo "Done: $(ls -lh $TARGET)"
volumeMounts:
- name: hostlib
mountPath: /hostlib
containers:
- name: keepalive
image: ubuntu:22.04
command: [bash, -c, "sleep infinity"]
volumes:
- name: hostlib
hostPath:
path: JEMALLOC_HOSTPATH_DIR_PLACEHOLDER
type: DirectoryOrCreate
JEDS
sed -i.bak "s/NAMESPACE_PLACEHOLDER/${NAMESPACE}/g" "$_tmpds"
sed -i.bak2 "s#JEMALLOC_HOSTPATH_DIR_PLACEHOLDER#${JEMALLOC_HOSTPATH_DIR}#g" "$_tmpds"
kubectl apply -f "$_tmpds"
rm -f "$_tmpds" "${_tmpds}.bak" "${_tmpds}.bak2"
log "Waiting up to 20 minutes for jemalloc to build on all nodes..."
local deadline=$((SECONDS + 1200))
while (( SECONDS < deadline )); do
ready=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
-o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0)
desired=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
-o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo 1)
[[ "$ready" -ge "$desired" && "$desired" -gt 0 ]] && break
log " jemalloc build progress: ${ready}/${desired} nodes ready..."
sleep 30
done
if [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]]; then
JEMALLOC_HOSTPATH_ACTIVE=1
log "jemalloc-16k ready on all nodes."
return 0
fi
JEMALLOC_HOSTPATH_ACTIVE=0
if [[ "${GITLAB_JEMALLOC_MODE,,}" == "force" || "${GITLAB_JEMALLOC_MODE,,}" == "required" || "${GITLAB_JEMALLOC_MODE,,}" == "1" || "${GITLAB_JEMALLOC_MODE,,}" == "true" || "${GITLAB_JEMALLOC_MODE,,}" == "on" ]]; then
die "jemalloc-builder did not become ready (${ready}/${desired}) and jemalloc is required."
fi
kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \
warn "Could not delete ${ds_name} DaemonSet after timeout; continuing without jemalloc."
warn "jemalloc-builder did not become ready (${ready}/${desired}); continuing without hostPath jemalloc optimization."
}
# ---------------------------------------------------------------------------
# Garage S3 key + bucket provisioning for GitLab object storage
# Creates a dedicated "gitlab-s3" key in Garage, provisions required buckets,
# and writes the k8s secrets consumed by the GitLab CR.
# ---------------------------------------------------------------------------
setup_garage_for_gitlab() {
local garage_admin_context garage_admin_context_label
garage_admin_context="$(resolve_garage_admin_context || true)"
if [[ -n "$garage_admin_context" ]]; then
garage_admin_context_label="$garage_admin_context"
else
garage_admin_context_label="(active/default)"
fi
log "Configuring Garage S3 (${GARAGE_NAMESPACE}) as GitLab object storage..."
log "Garage admin context: ${garage_admin_context_label}"
log "Garage namespace: ${GARAGE_NAMESPACE}"
log "GitLab object storage S3 endpoint: ${GARAGE_S3_ENDPOINT}"
garage_setup_failure() {
local reason="$1"
if is_truthy "${GITLAB_OBJECT_STORAGE_REQUIRED:-1}"; then
die "${reason} (object storage is required for GitLab deployment)."
fi
warn "${reason} (continuing because GITLAB_OBJECT_STORAGE_REQUIRED=${GITLAB_OBJECT_STORAGE_REQUIRED})."
return 0
}
local garage_pod
garage_pod=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get pods -l app=garage \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -z "$garage_pod" ]]; then
garage_setup_failure "Garage pod not found in namespace '${GARAGE_NAMESPACE}' on context '${garage_admin_context_label}'"
return 0
fi
local garage_admin_token_b64 garage_admin_token
garage_admin_token_b64=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get secret garage-secrets \
-o jsonpath='{.data.admin_token}' 2>/dev/null || true)
if [[ -z "$garage_admin_token_b64" ]]; then
garage_setup_failure "Garage admin token missing in secret '${GARAGE_NAMESPACE}/garage-secrets' on context '${garage_admin_context_label}'"
return 0
fi
garage_admin_token=$(printf '%s' "$garage_admin_token_b64" | base64 -d 2>/dev/null || true)
garage_admin_token="${garage_admin_token//$'\r'/}"
garage_admin_token="${garage_admin_token//$'\n'/}"
if [[ -z "$garage_admin_token" ]]; then
garage_setup_failure "Garage admin token in secret '${GARAGE_NAMESPACE}/garage-secrets' is malformed or empty on context '${garage_admin_context_label}'"
return 0
fi
local garage_cmd_last_reason=""
local garage_cmd_output=""
garage_exec() {
local cmd_desc="$1"
shift
local out_file err_file stderr_text lower_stderr
out_file=$(mktemp)
err_file=$(mktemp)
if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
/garage --admin-token "$garage_admin_token" "$@" >"$out_file" 2>"$err_file"; then
garage_cmd_output=$(cat "$out_file")
rm -f "$out_file" "$err_file"
garage_cmd_last_reason=""
return 0
fi
stderr_text=$(cat "$err_file" 2>/dev/null || true)
if [[ "$stderr_text" == *"--admin-token"* ]] && [[ "$stderr_text" == *"unknown"* || "$stderr_text" == *"unexpected"* || "$stderr_text" == *"wasn't expected"* ]]; then
if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \
/garage "$@" >"$out_file" 2>"$err_file"; then
garage_cmd_output=$(cat "$out_file")
rm -f "$out_file" "$err_file"
garage_cmd_last_reason=""
return 0
fi
stderr_text=$(cat "$err_file" 2>/dev/null || true)
fi
lower_stderr="${stderr_text,,}"
if [[ "$lower_stderr" == *"unauth"* || "$lower_stderr" == *"forbidden"* || "$lower_stderr" == *"invalid token"* || "$lower_stderr" == *"permission denied"* || "$lower_stderr" == *" 401"* || "$lower_stderr" == *" 403"* ]]; then
garage_cmd_last_reason="Unauthenticated admin command while running '${cmd_desc}'"
else
garage_cmd_last_reason="Garage admin command failed while running '${cmd_desc}'"
fi
if [[ -n "$stderr_text" ]]; then
garage_cmd_last_reason+=" (${stderr_text})"
fi
garage_cmd_output=""
rm -f "$out_file" "$err_file"
return 1
}
# ---- Resolve or create the gitlab-s3 access key ----
# garage v1.x outputs plaintext (not JSON) for key list / key info / key create.
# key list format: " GKxxxxxxxxx <name>"
# key create/info format includes lines:
# "Key ID: GKxxxxxxxxx"
# "Secret key: <secret>"
local access_key secret_key existing_id key_output rotated_key_name key_list_output create_failure_reason
if ! garage_exec "garage key list" key list; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
key_list_output="$garage_cmd_output"
existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true)
if [[ -n "$existing_id" ]]; then
log "Garage key '${GARAGE_S3_KEY_NAME}' already exists (id=${existing_id}); fetching info..."
if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
key_output="$garage_cmd_output"
else
log "Creating Garage key '${GARAGE_S3_KEY_NAME}'..."
create_failure_reason=""
if ! garage_exec "garage key create ${GARAGE_S3_KEY_NAME}" key create "$GARAGE_S3_KEY_NAME"; then
create_failure_reason="$garage_cmd_last_reason"
fi
if ! garage_exec "garage key list" key list; then
if [[ -n "$create_failure_reason" ]]; then
garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}"
else
garage_setup_failure "${garage_cmd_last_reason}"
fi
return 0
fi
key_list_output="$garage_cmd_output"
existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true)
if [[ -z "$existing_id" ]]; then
if [[ -n "$create_failure_reason" ]]; then
garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}' (${create_failure_reason})"
else
garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'"
fi
return 0
fi
if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then
if [[ -n "$create_failure_reason" ]]; then
garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}"
else
garage_setup_failure "${garage_cmd_last_reason}"
fi
return 0
fi
key_output="$garage_cmd_output"
fi
if [[ -z "$key_output" ]]; then
garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'"
return 0
fi
# Parse plaintext output — match "Key ID: GKxxx" and "Secret key: xxx" lines.
# Also try JSON in case a future garage version changes format.
access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1)
secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1)
# JSON fallback
if [[ -z "$access_key" ]]; then
access_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true)
fi
if [[ -z "$secret_key" ]]; then
secret_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true)
fi
if [[ "$secret_key" == "(redacted)" ]]; then
warn "Garage key '${GARAGE_S3_KEY_NAME}' secret is redacted; creating a deterministic fallback key for GitLab object storage."
rotated_key_name="${GARAGE_S3_KEY_NAME}-gitlab"
if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then
if ! garage_exec "garage key create ${rotated_key_name}" key create "$rotated_key_name"; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
fi
key_output="$garage_cmd_output"
access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1)
secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1)
if [[ -z "$access_key" ]]; then
access_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true)
fi
if [[ -z "$secret_key" ]]; then
secret_key=$(printf '%s' "$key_output" | python3 -c \
"import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true)
fi
if [[ -n "$access_key" && -n "$secret_key" && "$secret_key" != "(redacted)" ]]; then
GARAGE_S3_KEY_NAME="$rotated_key_name"
log "Using rotated Garage key '${GARAGE_S3_KEY_NAME}'."
fi
fi
if [[ -z "$access_key" || -z "$secret_key" || "$secret_key" == "(redacted)" ]]; then
garage_setup_failure "Malformed Garage key output for '${GARAGE_S3_KEY_NAME}' (missing access key or secret key)"
warn "Raw output: ${key_output}"
return 0
fi
# ---- Create buckets and grant permissions ----
# --key takes the accessKeyId (not the name) in garage v1.x
local buckets=(
registry
gitlab-artifacts-storage
gitlab-lfs-storage
gitlab-uploads-storage
gitlab-packages-storage
gitlab-dependency-proxy-storage
gitlab-terraform-state
gitlab-ci-secure-files
)
for bucket in "${buckets[@]}"; do
if ! garage_exec "garage bucket create ${bucket}" bucket create "$bucket"; then
local bucket_create_failure_reason lower_bucket_create_failure_reason
bucket_create_failure_reason="${garage_cmd_last_reason}"
lower_bucket_create_failure_reason="${bucket_create_failure_reason,,}"
if [[ "$lower_bucket_create_failure_reason" == *"already exists"* ]]; then
log "Garage bucket '${bucket}' already exists; reusing existing bucket."
else
garage_setup_failure "${bucket_create_failure_reason}"
return 0
fi
fi
# bucket name is positional; --read/--write/--owner are boolean flags
if ! garage_exec "garage bucket allow ${bucket}" bucket allow "$bucket" --key "$access_key" \
--read --write --owner; then
garage_setup_failure "${garage_cmd_last_reason}"
return 0
fi
done
log "Garage buckets provisioned for GitLab."
# ---- Write k8s secrets consumed by the GitLab CR ----
# Rails object storage connection (artifacts, LFS, uploads, packages, etc.)
local object_storage_secret_apply_output registry_storage_secret_apply_output
object_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-object-storage \
--from-literal=connection="provider: AWS
region: garage
aws_access_key_id: ${access_key}
aws_secret_access_key: ${secret_key}
endpoint: '${GARAGE_S3_ENDPOINT}'
path_style: true" \
--dry-run=client -o yaml | kubectl apply -f - 2>&1)"
if kubectl_apply_reports_changed "$object_storage_secret_apply_output"; then
gitlab_object_storage_secret_changed=1
fi
log "gitlab-object-storage secret applied (${object_storage_secret_apply_output})."
# Docker registry S3 storage driver config.
# Use s3_v2 (AWS SDK v2) — s3_v1 has signature computation issues with
# non-standard ports (Host: endpoint:3900) against Garage. SDK v2 always
# uses SigV4; v4auth is not a recognised key and must be omitted.
registry_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-registry-storage \
--from-literal=config="s3_v2:
accesskey: ${access_key}
secretkey: ${secret_key}
bucket: registry
regionendpoint: ${GARAGE_S3_ENDPOINT}
region: garage
pathstyle: true" \
--dry-run=client -o yaml | kubectl apply -f - 2>&1)"
if kubectl_apply_reports_changed "$registry_storage_secret_apply_output"; then
gitlab_registry_storage_secret_changed=1
fi
log "gitlab-registry-storage secret applied (${registry_storage_secret_apply_output})."
}
# ---------------------------------------------------------------------------
# Legacy/local static PV + synology directory setup for gitaly
# minio PV removed — object storage is provided by knoe-system/garage
# ---------------------------------------------------------------------------
setup_gitlab_legacy_storage() {
local synology_node="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
local storage_selector_key="${STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"
local base_path="${GITLAB_STORAGE_BASE:-/synology/d005/gitlab}"
local gitaly_size="${GITLAB_GITALY_PV_SIZE:-50Gi}"
[[ -n "$synology_node" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set for legacy GitLab storage setup."
log "Creating synology gitaly directory on ${synology_node}..."
kubectl -n "$NAMESPACE" run gitlab-dirprep \
--image=alpine:latest --restart=Never --rm --attach \
--overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"s\",\"hostPath\":{\"path\":\"$(dirname ${base_path})\",\"type\":\"Directory\"}}],\"containers\":[{\"name\":\"dirprep\",\"image\":\"alpine:latest\",\"command\":[\"sh\",\"-c\",\"mkdir -p ${base_path}/gitaly && chmod 777 ${base_path}/gitaly && echo done\"],\"volumeMounts\":[{\"name\":\"s\",\"mountPath\":\"$(dirname ${base_path})\"}]}]}}" \
2>&1 || warn "Could not create synology dirs (may already exist)"
# A named no-provisioner StorageClass is required so the helm chart renders
# storageClassName in the PVC template (an empty string is treated as falsy
# by the chart and omitted, causing the PVC to use the cluster default).
log "Ensuring gitlab-gitaly-static StorageClass (no-provisioner)..."
kubectl apply -f - <<SCYAML
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: gitlab-gitaly-static
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: Immediate
reclaimPolicy: Retain
SCYAML
log "Creating static PV for gitaly (${gitaly_size})..."
kubectl apply -f - <<PVYAML
apiVersion: v1
kind: PersistentVolume
metadata:
name: gitlab-gitaly-synology
spec:
capacity:
storage: ${gitaly_size}
accessModes: [ReadWriteOnce]
persistentVolumeReclaimPolicy: Retain
storageClassName: gitlab-gitaly-static
local:
path: ${base_path}/gitaly
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: ${storage_selector_key}
operator: In
values: [${synology_node}]
PVYAML
log "GitLab gitaly storage PV ready."
}
setup_gitlab_storage() {
if [[ "$MODE" == "k8s" ]]; then
log "Skipping legacy Synology/local-PV gitaly storage prep in k8s mode; using dynamic StorageClass '${GITALY_STORAGE_CLASS}'."
# Check for legacy resources on GKE/k8s
local has_legacy=0
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then has_legacy=1; fi
if kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then has_legacy=1; fi
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
local gitaly_sts_yaml
gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
if [[ -n "$gitaly_sts_yaml" ]]; then
if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* || "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
has_legacy=1
fi
fi
if [[ "$has_legacy" == "1" ]]; then
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then
export GITALY_AUTOCLEAN_PERFORMED=1
log "AUTOCLEAN: repairing legacy Gitaly storage (scaling down and deleting legacy PV/PVC)..."
kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
kubectl -n "$NAMESPACE" delete pvc repo-data-gitlab-gitaly-0 --wait=false 2>/dev/null || true
kubectl delete pv gitlab-gitaly-synology --wait=false 2>/dev/null || true
# Give it a moment to process deletions
sleep 2
else
repair_blocked "Legacy Gitaly PV/PVC still bound to gitlab-gitaly-static / gandalf.prole.org" \
"Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to automatically repair by deleting legacy PVC/PV, or run:
kubectl -n $NAMESPACE scale sts ${GITLAB_RELEASE}-gitaly --replicas=0
kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0
kubectl delete pv gitlab-gitaly-synology"
fi
fi
else
setup_gitlab_legacy_storage
fi
# Object storage (registry, artifacts, LFS, uploads, etc.) is provided by
# knoe-system/garage — credentials and buckets are set up here.
setup_garage_for_gitlab
}
configure_jemalloc_hostpath_mode
setup_jemalloc_on_nodes
check_gitlab_pre_apply_blocked
setup_gitlab_storage
log "Pre-creating gitlab-app-nonroot ServiceAccount (required by chart v9+)..."
kubectl -n "${NAMESPACE}" apply -f - <<SAYAML
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitlab-app-nonroot
namespace: ${NAMESPACE}
SAYAML
GITLAB_ALLOWED_HOSTS_YAML=""
_gitlab_allowed_hosts_csv=""
GITLAB_ALLOWED_HOSTS=()
append_gitlab_allowed_host() {
local candidate="$1"
[[ -n "$candidate" ]] || return 0
case ",${_gitlab_allowed_hosts_csv}," in
*,"${candidate}",*) return 0 ;;
esac
GITLAB_ALLOWED_HOSTS+=("$candidate")
_gitlab_allowed_hosts_csv="${_gitlab_allowed_hosts_csv:+${_gitlab_allowed_hosts_csv},}${candidate}"
return 0
}
for _gitlab_allowed_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
append_gitlab_allowed_host "$_gitlab_allowed_host"
_gitlab_host_root="${_gitlab_allowed_host#*.}"
if [[ -n "$_gitlab_host_root" && "$_gitlab_host_root" != "$_gitlab_allowed_host" ]]; then
append_gitlab_allowed_host "$_gitlab_host_root"
fi
done
unset _gitlab_allowed_host _gitlab_host_root
for _gitlab_allowed_host in "${GITLAB_ALLOWED_HOSTS[@]}"; do
GITLAB_ALLOWED_HOSTS_YAML+=$'\n'" - ${_gitlab_allowed_host}"
done
unset _gitlab_allowed_host
GLOBAL_JEMALLOC_VALUES_YAML=""
WORKLOAD_JEMALLOC_VALUES_YAML=""
if (( JEMALLOC_HOSTPATH_ACTIVE == 1 )); then
GLOBAL_JEMALLOC_VALUES_YAML=$(cat <<JEMGLOBAL
# ARM64 nodes: inject glibc jemalloc compiled with --with-lg-page=14
# The DaemonSet above places it at ${JEMALLOC_HOSTPATH_LIB}
extraEnv:
LD_PRELOAD: ${JEMALLOC_HOSTPATH_LIB}
JEMGLOBAL
)
WORKLOAD_JEMALLOC_VALUES_YAML=$(cat <<JEMWORKLOAD
extraEnv:
LD_PRELOAD: ${JEMALLOC_HOSTPATH_LIB}
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: ${JEMALLOC_HOSTPATH_DIR}
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: ${JEMALLOC_HOSTPATH_DIR}
readOnly: true
JEMWORKLOAD
)
else
log "Rendering GitLab values without hostPath jemalloc mounts/LD_PRELOAD."
fi
gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_exists_before=0
if [[ -n "$gitlab_generation_before" ]]; then
gitlab_exists_before=1
fi
GITLAB_CR_RENDERED="$(cat <<EOF
apiVersion: apps.gitlab.com/v1beta1
kind: GitLab
metadata:
name: ${GITLAB_RELEASE}
namespace: ${NAMESPACE}
spec:
chart:
${CHART_VERSION_YAML}
values:
global:
persistence:
storageClass: ${GITALY_STORAGE_CLASS}
${GLOBAL_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<GNODE
nodeSelector:
${NODE_SELECTOR_YAML}
GNODE
fi)
# Explicit allowedHosts as flat strings — chart 9.x renders list-of-maps
# by default which breaks URI::InvalidComponentError in 7_gitlab_http.rb
allowedHosts:${GITLAB_ALLOWED_HOSTS_YAML}
hosts:
domain: ${PRIMARY_GITLAB_DOMAIN_ROOT}
gitlab:
name: ${GITLAB_DOMAIN}
ssh: ${GITLAB_DOMAIN}
ingress:
class: kong
configureCertmanager: false
tls:
enabled: false
psql:
host: ${DB_HOST}
port: ${DB_PORT}
username: ${GITLAB_DB_USER}
database: ${GITLAB_DB_NAME}
password:
secret: gitlab-db-password
key: password
redis:
host: ${REDIS_HOST}
port: ${REDIS_PORT}
auth:
enabled: false
# chart v9+ requires the SA to be pre-created when a custom name is used
serviceAccount:
create: false
# Disable bundled minio globally — object storage is provided by garage
minio:
enabled: false
# Object storage via knoe-system/garage (S3-compatible)
appConfig:
object_store:
enabled: true
proxy_download: true
connection:
secret: gitlab-object-storage
key: connection
$(if [[ -n "$FRONTDOOR_HOST" ]]; then
cat <<OMNIAUTH
omniauth:
enabled: true
autoSignInWithProvider: openid_connect
allowSingleSignOn:
- openid_connect
blockAutoCreatedUsers: false
providers:
- secret: gitlab-google-oidc
key: provider
OMNIAUTH
fi)
registry:
bucket: registry
# Disable bundled sub-charts — we provide our own data stores
postgresql:
install: false
redis:
install: false
certmanager:
install: false
prometheus:
install: false
gitlab-runner:
install: false
# minio sub-chart is disabled via global.minio.enabled: false (set above under global:).
# Do NOT set minio.enabled here — chart v9 removed the sub-chart level property
# and treats it as a hard validation error (NOTES.txt:234).
gitlab:
webservice:
# Single replica + reduced memory — RPi nodes have limited RAM
replicaCount: 1
minReplicas: 1
maxReplicas: 1
resources:
requests:
cpu: 200m
memory: 1500M
limits:
memory: 2000M
# Rails preloading takes 25-40min on RPi; protect against liveness kills
livenessProbe:
initialDelaySeconds: 3600
periodSeconds: 60
timeoutSeconds: 30
failureThreshold: 5
readinessProbe:
initialDelaySeconds: 120
periodSeconds: 30
timeoutSeconds: 30
failureThreshold: 60
ingress:
enabled: true
annotations:
kubernetes.io/ingress.class: kong
${WORKLOAD_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
nodeSelector:
${NODE_SELECTOR_YAML}
NODE
fi)
sidekiq:
replicaCount: 1
concurrency: ${GITLAB_SIDEKIQ_CONCURRENCY}
resources:
requests:
cpu: ${GITLAB_SIDEKIQ_REQUESTS_CPU}
memory: ${GITLAB_SIDEKIQ_REQUESTS_MEMORY}
limits:
cpu: ${GITLAB_SIDEKIQ_LIMITS_CPU}
memory: ${GITLAB_SIDEKIQ_LIMITS_MEMORY}
# Rails loading takes 25-40min on RPi; protect against liveness kills
livenessProbe:
initialDelaySeconds: 3600
periodSeconds: 60
timeoutSeconds: 30
failureThreshold: 5
readinessProbe:
initialDelaySeconds: 120
periodSeconds: 30
timeoutSeconds: 30
failureThreshold: 60
${WORKLOAD_JEMALLOC_VALUES_YAML}
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
nodeSelector:
${NODE_SELECTOR_YAML}
NODE
fi)
gitaly:
# Storage class is mode-specific:
# - k8s: dynamic PVCs on cluster storage class
# - legacy/local: static local-PV storage class created by setup_gitlab_storage()
persistence:
storageClass: ${GITALY_STORAGE_CLASS}
size: ${GITLAB_GITALY_PV_SIZE:-50Gi}
${WORKLOAD_JEMALLOC_VALUES_YAML}
# Give gitaly gRPC health check time to initialise before probes start
livenessProbe:
initialDelaySeconds: 30
readinessProbe:
initialDelaySeconds: 15
$(if [[ -n "$STORAGE_NODE_SELECTOR_YAML" ]]; then
cat <<NODE
# Legacy/local mode pins gitaly to storage node for local PV.
nodeSelector:
${STORAGE_NODE_SELECTOR_YAML}
NODE
fi)
toolbox:
backups:
objectStorage:
backend: s3
config:
secret: gitlab-object-storage
key: connection
${WORKLOAD_JEMALLOC_VALUES_YAML}
migrations:
${WORKLOAD_JEMALLOC_VALUES_YAML}
gitlab-exporter:
${WORKLOAD_JEMALLOC_VALUES_YAML}
gitlab-shell:
replicaCount: 1
kas:
replicaCount: 1
registry:
replicaCount: 1
storage:
secret: gitlab-registry-storage
key: config
ingress:
enabled: false
EOF
)"
log "Applying GitLab CR (hosts=${_gitlab_hosts_csv}, db=${GITLAB_DB_NAME}@${DB_HOST})..."
gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_apply_output="$(echo "$GITLAB_CR_RENDERED" | kubectl apply -f -)"
gitlab_generation_after="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)"
gitlab_spec_changed=1
if [[ -n "$gitlab_generation_before" && -n "$gitlab_generation_after" && "$gitlab_generation_before" == "$gitlab_generation_after" ]]; then
gitlab_spec_changed=0
fi
gitlab_apply_changed=0
if kubectl_apply_reports_changed "$gitlab_apply_output"; then
if [[ "$MODE" == "k8s" && "$gitlab_spec_changed" == "0" ]]; then
# Normalization (e.g. standard vs standard-rwo) might cause kubectl apply to report 'configured'
# but not increment metadata.generation. In this case, we avoid triggering a full reconcile.
log "GitLab CR apply reported change, but generation is unchanged (likely storageClass normalization). Skipping drift trigger."
else
gitlab_apply_changed=1
fi
fi
gitlab_config_changed=0
if [[ "$gitlab_db_secret_changed" == "1" || "$gitlab_object_storage_secret_changed" == "1" || "$gitlab_registry_storage_secret_changed" == "1" ]]; then
gitlab_config_changed=1
fi
gitlab_reconcile_required=0
# Check actual replicas to ensure they match desired count (1) - Requirement 3
gitlab_replica_drift=0
for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do
_dep_name="${GITLAB_RELEASE}-${_dep_suffix}"
_actual=$(kubectl -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")
if [[ "$_actual" != "1" ]]; then
log "Detected replica drift for ${_dep_name}: actual=${_actual}, desired=1"
gitlab_replica_drift=1
break
fi
done
if [[ "$gitlab_exists_before" == "0" || "$operator_changed" == "1" || "$gitlab_chart_version_changed" == "1" || "$gitlab_spec_changed" == "1" || "$gitlab_apply_changed" == "1" || "$gitlab_config_changed" == "1" || "$gitlab_replica_drift" == "1" ]]; then
gitlab_reconcile_required=1
fi
check_gitlab_post_apply_blocked
# minio is disabled (global.minio.enabled: false) — no ARM64 credential fix needed.
# ---------------------------------------------------------------------------
# ARM64 configure check: GitLab v17+ (chart v8+) ships multi-arch CNG images
# that include ARM64 variants. The old workaround of replacing the configure
# init container with alpine:latest is no longer needed and breaks things
# because /templates/configure is baked into the gitlab-base image, not
# mounted from a ConfigMap.
# This function is kept as a no-op to avoid breaking callers; remove entirely
# once the chart version floor is confirmed stable at v9+.
# ---------------------------------------------------------------------------
fix_registry_arm64_configure() {
log "Skipping ARM64 configure patch — GitLab v18 CNG images are multi-arch natively."
}
fix_registry_arm64_configure
log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)."
log "Monitor progress: kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w"
log "Watch pods: kubectl -n ${NAMESPACE} get pods -w"
# ---------------------------------------------------------------------------
# Optionally wait for GitLab to become available
# ---------------------------------------------------------------------------
WAIT_TIMEOUT="${GITLAB_WAIT_TIMEOUT:-1200}" # default 20 minutes
if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
if [[ "$gitlab_reconcile_required" == "0" ]]; then
log "Fast-path: no GitLab reconcile triggers detected; skipping long wait."
gitlab_available_status="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{range .status.conditions[?(@.type=="Available")]}{.status}{end}' 2>/dev/null || true)"
if [[ "$gitlab_available_status" == "True" ]]; then
log "Short health check OK: GitLab CR condition Available=True."
check_gitlab_post_apply_blocked
else
warn "Short health check: GitLab CR Available condition is '${gitlab_available_status:-unknown}'."
kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
fi
else
log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status (checking for blocked states every 30s)..."
_wait_start=$(date +%s)
while true; do
if kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \
--for=condition=Available \
--timeout=30s >/dev/null 2>&1; then
log "GitLab CR condition Available=True."
break
fi
check_gitlab_post_apply_blocked
_now=$(date +%s)
if (( _now - _wait_start >= WAIT_TIMEOUT )); then
_ctx=$(get_gitlab_blocker_context)
repair_blocked "Timed out waiting for GitLab CR condition=Available" \
"The deployment is taking too long. Context: ${_ctx}Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}"
fi
done
fi
fi
# ---------------------------------------------------------------------------
# Public ingress for configured GitLab domain -> gitlab-webservice
# The GitLab Operator creates the Ingress; this block ensures a stable public
# class/host mapping if the CR-managed ingress is absent or not usable.
# ---------------------------------------------------------------------------
WEBSERVICE_SVC="${GITLAB_RELEASE}-webservice-default"
GITLAB_INGRESS_RULES_YAML=""
for _gitlab_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do
GITLAB_INGRESS_RULES_YAML+=$'\n'" - host: ${_gitlab_host}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: ${WEBSERVICE_SVC}
port:
number: 8181"
done
unset _gitlab_host
GITLAB_AUTH_ANNOTATIONS_YAML=""
case "${FRONTDOOR_AUTH_ENABLED}" in
1|true|TRUE|True|yes|YES|Yes|on|ON|On)
GITLAB_AUTH_ANNOTATIONS_YAML=$(cat <<EOAUTH
nginx.ingress.kubernetes.io/auth-url: "${AUTH_VERIFY_URL}"
nginx.ingress.kubernetes.io/auth-signin: "${AUTH_SIGNIN_URL}"
nginx.ingress.kubernetes.io/auth-response-headers: "${AUTH_RESPONSE_HEADERS}"
EOAUTH
)
;;
esac
log "Ensuring ingress (${GITLAB_INGRESS_CLASS}) for hosts=${_gitlab_hosts_csv} -> ${WEBSERVICE_SVC}:8181 ..."
_skip_gitlab_fallback_ingress=0
if assert_unique_ingress_host_claims "gitlab-kong-ingress" "$NAMESPACE" "$_gitlab_hosts_csv" "$GITLAB_RELEASE" "$WEBSERVICE_SVC"; then
:
else
_ingress_claim_rc=$?
case "$_ingress_claim_rc" in
10)
_skip_gitlab_fallback_ingress=1
log "Operator-managed ingress already owns GitLab host/path in namespace '${NAMESPACE}': ${INGRESS_HOST_CLAIM_DETAILS}"
log "Skipping fallback ingress creation for GitLab release '${GITLAB_RELEASE}'."
;;
*)
die "Duplicate ingress host/path claim detected for GitLab ingress '${NAMESPACE}/gitlab-kong-ingress': ${INGRESS_HOST_CLAIM_DETAILS}"
;;
esac
fi
if [[ "$_skip_gitlab_fallback_ingress" != "1" ]]; then
kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gitlab-kong-ingress
namespace: ${NAMESPACE}
annotations:
kubernetes.io/ingress.class: ${GITLAB_INGRESS_CLASS}
konghq.com/strip-path: "false"
${GITLAB_AUTH_ANNOTATIONS_YAML}
spec:
ingressClassName: ${GITLAB_INGRESS_CLASS}
rules:${GITLAB_INGRESS_RULES_YAML}
EOF
fi
# ---------------------------------------------------------------------------
# Registry migration: registry:2 (knoe-system) → gitlab-registry
# Runs unattended after GitLab is Ready. Delegates to init_registry.sh
# migrate which uses skopeo when available, otherwise prints commands.
# Skipped if the old registry:2 has no images or is already gone.
# Set SKIP_REGISTRY_MIGRATE=1 to suppress.
# ---------------------------------------------------------------------------
if [[ "${SKIP_REGISTRY_MIGRATE:-0}" != "1" ]]; then
_init_registry_sh="${SCRIPT_DIR}/init_registry.sh"
if [[ -x "$_init_registry_sh" ]]; then
log "--- Registry migration: registry:2 → gitlab-registry ---"
# Pass the gitlab namespace so the migrate action knows where to send images.
GITLAB_NAMESPACE="$NAMESPACE" \
REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}" \
"$_init_registry_sh" migrate || \
warn "Registry migration encountered errors — check output above."
else
warn "init_registry.sh not found at ${_init_registry_sh}; skipping registry migration."
warn "Run manually: ./etc/init_registry.sh migrate"
fi
unset _init_registry_sh
fi
log "Done."
log "GitLab will be reachable at hosts=${_gitlab_hosts_csv} once pods are Running."
log "Initial root password: kubectl -n ${NAMESPACE} get secret ${GITLAB_RELEASE}-gitlab-initial-root-password -o jsonpath='{.data.password}' | base64 -d"
Reference in New Issue View Git Blame Copy Permalink