prole/etc/init_gitlab.sh
chrisfu eb9430df9d fix(gitlab,infra): ARM64 RPi service cluster – GitLab deploy in gitlab ns on gandalf
Namespace & routing
- milestones.py: GitOpsMilestone now resolves namespace from
  gitops.gitlab_namespace (new) → Global.GITLAB_NAMESPACE → 'gitlab'
  hardcoded; never falls through to gitops.namespace (was 'gitea')
- prole.cfg: add gitops.gitlab_namespace=gitlab + GITLAB_NAMESPACE=gitlab
- init_gitlab.sh: NAMESPACE defaults to gitlab, NODE_SELECTOR blanked so
  only gitaly+minio are node-pinned; GITOPS_NAMESPACE fallback removed

GitLab on ARM64 RPi (16 KB kernel pages)
- init_gitlab.sh: DaemonSet compiles jemalloc-5.3.0 with --with-lg-page=14
  (glibc/Ubuntu) on every node; LD_PRELOAD injected per Ruby component
- Minio: quay.io 2022 image (ARM64); configure init container replaced
  with ARM64 alpine that writes credential files; MINIO_ROOT_USER/PASSWORD
  injected directly into main container env via secretKeyRef
- Minio buckets auto-created post-deploy (registry, lfs, artifacts, etc.)
- webservice/sidekiq: replicaCount=1, reduced memory (1500M/800M),
  liveness probe initialDelaySeconds=3600 (Rails loads 25-40min on RPi)
- allowedHosts set as flat string list (chart 9.x default is list-of-maps
  which breaks URI initializer in 7_gitlab_http.rb)
- gitaly+minio always pinned to gandalf (local PV); other workloads spread

Storage
- Static PVs created for gitaly (50Gi) + minio (10Gi) on synology d005
- Synology dirs created before PVs; bucket creation idempotent

Redis (shared for GitLab KAS)
- init_redis.sh: persistence disabled (no dynamic provisioner); Redis used
  as pub/sub broker only

Infrastructure / pi.prole.org
- Removed pi.prole.org from [k3s_agents] – dedicated pihole node, OOM
- host_vars: k3s_enabled=false, k3s_state=absent (storage preserved)
- New playbook: infrastructure/playbooks/disable_pi_k3s.yml (drain + disable)
- monitoring.py: node-exporter DaemonSet excludes pi.prole.org
- init_monitoring.sh: pi.prole.org excluded from node-exporter affinity
- kong-deployment.yaml: affinity rule prevents scheduling on pi (pihole owns 80/443)

Co-authored-by: Junie <junie@jetbrains.com>
2026-04-02 20:15:11 -07:00

898 lines
36 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
PROG="init_gitlab"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"
MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")"
# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden.
# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps).
NAMESPACE="${GITLAB_NAMESPACE:-gitlab}"
CFG_PATH=""
FORCE=0
# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE).
# Setting this would pin ALL global components to one node, exhausting RAM.
NODE_SELECTOR=""
usage() {
cat <<EOF
Usage:
$PROG [options] [deploy]
Options:
--mode <k3d|k3s|k8s|local> Deployment mode (default: ${MODE:-k3d})
-n, --namespace <name> Target namespace (default: gitlab)
-c, --config <prole.cfg> Path to prole.cfg (defaults to detected)
--node-selector <node> Node to pin GitLab workloads (default: gandalf.prole.org)
--force Remove existing GitLab and Gitea releases before deploy
--help Show this help
Behavior:
- Removes any pre-existing git.prole.org Gitea configurations (helm release,
namespace, Kong routes) when --force is specified or when gitea is detected.
- Installs the GitLab Operator via Helm into the gitlab namespace.
- Provisions a GitLab CR that uses the knoe-db CloudNativePG cluster as its
external PostgreSQL data store.
- Configures git.prole.org as a Kong-managed ingress pointing to GitLab.
- Pins all GitLab workloads to the selected node (default: gandalf.prole.org).
EOF
}
die() { echo "[ERROR] $*" >&2; exit 2; }
log() { echo "[INFO] $*"; }
warn() { echo "[WARN] $*" >&2; }
while [[ $# -gt 0 ]]; do
case "$1" in
--mode) MODE="$(prole_normalize_mode "${2:-}")"; shift 2 ;;
--mode=*) MODE="$(prole_normalize_mode "${1#*=}")"; shift 1 ;;
-n|--namespace) NAMESPACE="${2:-}"; shift 2 ;;
--namespace=*) NAMESPACE="${1#*=}"; shift 1 ;;
-c|--config) CFG_PATH="${2:-}"; shift 2 ;;
--config=*) CFG_PATH="${1#*=}"; shift 1 ;;
--node-selector) NODE_SELECTOR="${2:-}"; shift 2 ;;
--node-selector=*) NODE_SELECTOR="${1#*=}"; shift 1 ;;
--force) FORCE=1; shift 1 ;;
-h|--help) usage; exit 0 ;;
--trace-namespace)
# Resolve namespace then exit — used by diagnostic scripts only
_TRACE_NS=1; shift 1 ;;
*) break ;;
esac
done
# ---------------------------------------------------------------------------
# Resolve config path
# ---------------------------------------------------------------------------
if [[ -z "$CFG_PATH" && -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then
CFG_PATH="${PROLE_CONF}/prole.cfg"
elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then
CFG_PATH="$SCRIPT_DIR/../conf/prole.cfg"
fi
# cfg-based namespace override (only GITLAB_NAMESPACE; GITOPS_NAMESPACE intentionally excluded)
if [[ -z "$NAMESPACE" || "$NAMESPACE" == "gitlab" ]] && [[ -n "$CFG_PATH" ]]; then
maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")"
[[ -n "$maybe_ns" ]] && NAMESPACE="$maybe_ns"
fi
# Final safety net — always default to 'gitlab'
NAMESPACE="${NAMESPACE:-gitlab}"
export GITLAB_NAMESPACE="$NAMESPACE"
# Diagnostic exit — used by tmp/sim_installer_ns.sh
if [[ "${_TRACE_NS:-0}" == "1" ]]; then
echo "TRACE: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" >&2
echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-<unset>}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-<unset>}'" >&2
exit 0
fi
case "$MODE" in
k3d|k3s|k8s|local) ;;
*) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;;
esac
export PROLE_MODE="$MODE"
command -v kubectl >/dev/null || die "kubectl not found"
command -v helm >/dev/null || die "helm not found (required for GitLab Operator install)"
# ---------------------------------------------------------------------------
# knoe-db (CNPG) settings — reuse the same cluster gitea uses
# ---------------------------------------------------------------------------
DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-${CLUSTER_NAME:-knoe-db}}"
GITLAB_DB_NAME="${GITLAB_DB_NAME:-gitlabhq_production}"
GITLAB_DB_USER="${GITLAB_DB_USER:-gitlab}"
GITLAB_DB_PASSWORD="${GITLAB_DB_PASSWORD:-}"
DB_HOST="${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local"
DB_PORT="${GITLAB_DB_PORT:-5432}"
# ---------------------------------------------------------------------------
# Redis — shared common service (deployed by init_redis.sh in knoe-system)
# ---------------------------------------------------------------------------
REDIS_NAMESPACE="${REDIS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}"
REDIS_HOST="${GITLAB_REDIS_HOST:-redis-master.${REDIS_NAMESPACE}.svc.cluster.local}"
REDIS_PORT="${GITLAB_REDIS_PORT:-6379}"
# GitLab domain (reuses the git.prole.org shared hostname)
GITLAB_DOMAIN="${GITLAB_DOMAIN:-git.prole.org}"
# Google Workspace OIDC via knoe-auth — set FRONTDOOR_HOST to enable
FRONTDOOR_HOST="${FRONTDOOR_HOST:-}"
GITLAB_RELEASE="gitlab"
# ---------------------------------------------------------------------------
# Operator chart coordinates
# ---------------------------------------------------------------------------
OPERATOR_REPO_NAME="gitlab-operator"
OPERATOR_REPO_URL="https://gitlab.com/api/v4/projects/18899486/packages/helm/stable"
OPERATOR_CHART="gitlab-operator/gitlab-operator"
OPERATOR_RELEASE="gitlab-operator"
OPERATOR_NAMESPACE="${GITLAB_OPERATOR_NAMESPACE:-${NAMESPACE}}"
# GitLab Helm chart version required in the CR; auto-detected if not set.
GITLAB_CHART_VERSION="${GITLAB_CHART_VERSION:-}"
GITLAB_CHART_REPO_NAME="gitlab"
GITLAB_CHART_REPO_URL="https://charts.gitlab.io/"
# ---------------------------------------------------------------------------
# Helper: resolve knoe-db primary pod (same pattern as init_gitea.sh)
# ---------------------------------------------------------------------------
resolve_knoe_db_primary_pod() {
local primary
primary=$(kubectl -n "$DB_NAMESPACE" get pods \
-l "cnpg.io/cluster=${CNPG_CLUSTER_NAME},cnpg.io/instanceRole=primary" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -z "$primary" ]]; then
primary=$(kubectl -n "$DB_NAMESPACE" get pods \
-l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
fi
printf '%s' "$primary"
}
sql_escape_literal() {
printf '%s' "${1:-}" | sed "s/'/''/g"
}
# ---------------------------------------------------------------------------
# Resolve GitLab DB password (pull from knoe-db-superuser secret as fallback)
# ---------------------------------------------------------------------------
resolve_gitlab_db_password() {
if [[ -n "${GITLAB_DB_PASSWORD:-}" ]]; then
return 0
fi
local resolved=""
if kubectl -n "$DB_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then
resolved=$(kubectl -n "$DB_NAMESPACE" get secret knoe-db-superuser \
-o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true)
fi
if [[ -n "$resolved" ]]; then
GITLAB_DB_PASSWORD="$resolved"
else
# Generate a random password when none is available
GITLAB_DB_PASSWORD="$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom 2>/dev/null | head -c 32 || true)"
warn "Generated random GitLab DB password; store it in GITLAB_DB_PASSWORD for future runs."
fi
}
# ---------------------------------------------------------------------------
# Provision GitLab role + database in knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab() {
local primary
primary="$(resolve_knoe_db_primary_pod)"
if [[ -z "$primary" ]]; then
warn "No knoe-db pod found in namespace '${DB_NAMESPACE}'; skipping GitLab DB setup."
return 0
fi
resolve_gitlab_db_password
local admin_user=""
local candidate
for candidate in postgres root; do
if kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$candidate" -d postgres -tAc "SELECT 1" >/dev/null 2>&1; then
admin_user="$candidate"
break
fi
done
if [[ -z "$admin_user" ]]; then
warn "Unable to connect to knoe-db as admin; skipping GitLab DB setup."
return 0
fi
local esc_pw
esc_pw="$(sql_escape_literal "$GITLAB_DB_PASSWORD")"
kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -c "
DO \$\$ BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${GITLAB_DB_USER}') THEN
CREATE ROLE ${GITLAB_DB_USER} LOGIN PASSWORD '${esc_pw}';
ELSE
ALTER ROLE ${GITLAB_DB_USER} WITH PASSWORD '${esc_pw}';
END IF;
END \$\$;
" >/dev/null 2>&1 || warn "Could not create/update role '${GITLAB_DB_USER}'."
local db_exists
db_exists=$(kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -tAc \
"SELECT 1 FROM pg_database WHERE datname='${GITLAB_DB_NAME}';" 2>/dev/null || true)
if [[ "$db_exists" != "1" ]]; then
kubectl -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \
psql -U "$admin_user" -d postgres -c \
"CREATE DATABASE ${GITLAB_DB_NAME} OWNER ${GITLAB_DB_USER};" \
>/dev/null 2>&1 || warn "Could not create database '${GITLAB_DB_NAME}'."
fi
log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE})."
}
# ---------------------------------------------------------------------------
# Clean up any pre-existing Gitea / git.prole.org configurations
# ---------------------------------------------------------------------------
cleanup_gitea() {
log "Checking for pre-existing Gitea deployment to remove before GitLab install..."
local gitea_ns="${GITEA_NAMESPACE:-gitea}"
# Remove Gitea Helm release
if helm -n "$gitea_ns" status gitea >/dev/null 2>&1; then
log "Uninstalling Gitea Helm release from namespace '$gitea_ns'..."
helm -n "$gitea_ns" uninstall gitea >/dev/null 2>&1 || true
fi
# Remove any leftover raw Gitea resources
kubectl -n "$gitea_ns" delete deploy/gitea svc/gitea-http svc/gitea-ssh \
>/dev/null 2>&1 || true
# Remove Kong routes/services registered for git.prole.org (best-effort)
local kong_ns="${KONG_NAMESPACE:-${NAMESPACE:-kong}}"
for res_type in kongplugins kongingresses; do
kubectl -n "$gitea_ns" delete "$res_type" --all >/dev/null 2>&1 || true
done
# Remove gitea namespace Ingress objects that route git.prole.org
kubectl -n "$gitea_ns" delete ingress \
-l "app.kubernetes.io/name=gitea" >/dev/null 2>&1 || true
kubectl -n "$gitea_ns" delete ingress \
--field-selector="metadata.name=gitea" >/dev/null 2>&1 || true
log "Gitea cleanup complete."
}
# ---------------------------------------------------------------------------
# Ensure gitlab namespace exists
# ---------------------------------------------------------------------------
kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null
# ---------------------------------------------------------------------------
# --force: remove existing GitLab operator and release first
# ---------------------------------------------------------------------------
if [[ "$FORCE" -eq 1 ]]; then
warn "--force: removing existing GitLab resources in namespace '$NAMESPACE'..."
if kubectl get crd gitlabs.apps.gitlab.com >/dev/null 2>&1; then
kubectl -n "$NAMESPACE" delete gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true
# Wait briefly for operator to clean up managed resources
sleep 10
fi
helm -n "$NAMESPACE" uninstall "$GITLAB_RELEASE" >/dev/null 2>&1 || true
helm -n "$NAMESPACE" uninstall "$OPERATOR_RELEASE" >/dev/null 2>&1 || true
cleanup_gitea
fi
# ---------------------------------------------------------------------------
# Always remove gitea if its helm release exists (non-destructive path)
# gitea and gitlab both claim git.prole.org; they cannot coexist.
# ---------------------------------------------------------------------------
if helm -n "${GITEA_NAMESPACE:-gitea}" status gitea >/dev/null 2>&1; then
warn "Gitea release detected — removing to free git.prole.org for GitLab..."
cleanup_gitea
fi
# ---------------------------------------------------------------------------
# Prepare knoe-db
# ---------------------------------------------------------------------------
setup_knoe_db_for_gitlab
# Persist the DB password as a k8s Secret the operator CR can reference
resolve_gitlab_db_password
kubectl -n "$NAMESPACE" create secret generic gitlab-db-password \
--from-literal=password="$GITLAB_DB_PASSWORD" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
log "gitlab-db-password secret applied in namespace '$NAMESPACE'."
# ---------------------------------------------------------------------------
# Install / upgrade the GitLab Operator
# ---------------------------------------------------------------------------
log "Adding/updating GitLab Operator Helm repo..."
helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true
helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..."
log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..."
# The operator needs cluster-scoped RBAC; it watches all namespaces by default.
helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \
-n "$OPERATOR_NAMESPACE" \
--create-namespace \
--timeout 10m \
--wait \
--set watchNamespace="$NAMESPACE"
log "GitLab Operator ready."
# ---------------------------------------------------------------------------
# Resolve GitLab chart version (required by the Operator CR since >= v0.28)
# ---------------------------------------------------------------------------
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
log "Auto-detecting latest GitLab chart version..."
helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true
helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..."
GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \
| awk 'NR==2{print $2}' || true)
if [[ -z "$GITLAB_CHART_VERSION" ]]; then
die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run."
fi
log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}"
fi
CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\""
# ---------------------------------------------------------------------------
# Determine node selector block for the GitLab CR
# ---------------------------------------------------------------------------
# Storage node for gitaly + minio only (local PVs require co-location)
# Other components spread across the cluster via default scheduler.
STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-gandalf.prole.org}}"
# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned
NODE_SELECTOR="${NODE_SELECTOR:-}"
NODE_SELECTOR_YAML=""
if [[ -n "$NODE_SELECTOR" ]]; then
log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}"
NODE_SELECTOR_YAML="kubernetes.io/hostname: ${NODE_SELECTOR}"
fi
STORAGE_NODE_SELECTOR_YAML="kubernetes.io/hostname: ${STORAGE_NODE}"
# ---------------------------------------------------------------------------
# Create the GitLab CR (operator reconciles this into the full deployment)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Apply gitlab-google-oidc secret (OIDC provider config for OmniAuth)
# ---------------------------------------------------------------------------
if [[ -n "$FRONTDOOR_HOST" ]]; then
PLATFORM_DOMAIN="${PLATFORM_DOMAIN:-}"
oidc_secret_tmpl="${SCRIPT_DIR}/../deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
if [[ -f "$oidc_secret_tmpl" ]]; then
log "Applying gitlab-google-oidc secret (issuer=https://${FRONTDOOR_HOST}/auth) ..."
FRONTDOOR_HOST="$FRONTDOOR_HOST" PLATFORM_DOMAIN="$PLATFORM_DOMAIN" \
envsubst < "$oidc_secret_tmpl" | kubectl apply -n "$NAMESPACE" -f - >/dev/null || \
warn "Could not apply gitlab-google-oidc secret; create it manually from deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml"
else
warn "gitlab-google-oidc-secret.example.yaml not found; skipping OIDC secret (create it manually)."
fi
fi
# ---------------------------------------------------------------------------
# ARM64 / 16KB-page jemalloc fix — build glibc jemalloc on every node once
# All RPi nodes use 16KB kernel pages; GitLab's bundled jemalloc is 4KB-only.
# We compile a compatible jemalloc-5.3.0 with --with-lg-page=14 via DaemonSet.
# ---------------------------------------------------------------------------
setup_jemalloc_on_nodes() {
local jlib="/opt/gitlab-jemalloc/libjemalloc.so.2"
local ds_name="jemalloc-builder"
# Check if already present on all schedulable nodes
local nodes
nodes=$(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | tr '\n' ' ')
local all_ready=true
for node in $nodes; do
result=$(kubectl -n "$NAMESPACE" run "jcheck-${node//\./-}" \
--image=ubuntu:22.04 --restart=Never --rm --attach --quiet \
--overrides="{\"spec\":{\"nodeName\":\"${node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"jlib\",\"hostPath\":{\"path\":\"/opt/gitlab-jemalloc\",\"type\":\"DirectoryOrCreate\"}}],\"containers\":[{\"name\":\"c\",\"image\":\"ubuntu:22.04\",\"command\":[\"bash\",\"-c\",\"test -f /jlib/libjemalloc.so.2 && echo OK || echo MISSING\"],\"volumeMounts\":[{\"name\":\"jlib\",\"mountPath\":\"/jlib\"}]}]}}" 2>/dev/null || echo "MISSING")
if [[ "$result" != *"OK"* ]]; then
all_ready=false
break
fi
done
if $all_ready; then
log "jemalloc-16k already present on all nodes — skipping build."
return 0
fi
log "Deploying jemalloc-builder DaemonSet (compiles jemalloc-5.3.0 with --with-lg-page=14 on each node)..."
local _tmpds
_tmpds=$(mktemp /tmp/jemalloc-ds-XXXXXX.yaml)
cat > "$_tmpds" <<'JEDS'
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: jemalloc-builder
namespace: NAMESPACE_PLACEHOLDER
labels:
app: jemalloc-builder
spec:
selector:
matchLabels:
app: jemalloc-builder
template:
metadata:
labels:
app: jemalloc-builder
spec:
tolerations:
- operator: Exists
initContainers:
- name: build
image: ubuntu:22.04
command:
- bash
- -c
- |
set -e
TARGET=/hostlib/libjemalloc.so.2
if [ -f "$TARGET" ]; then echo "Already built"; exit 0; fi
apt-get update -qq && apt-get install -y -q gcc make wget bzip2 2>&1
cd /tmp
wget -q -O jemalloc.tar.bz2 \
https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
tar xjf jemalloc.tar.bz2 && cd jemalloc-5.3.0
./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1
make -j2 lib/libjemalloc.so.2 2>&1
cp lib/libjemalloc.so.2 "$TARGET" && chmod 755 "$TARGET"
echo "Done: $(ls -lh $TARGET)"
volumeMounts:
- name: hostlib
mountPath: /hostlib
containers:
- name: keepalive
image: ubuntu:22.04
command: [bash, -c, "sleep infinity"]
volumes:
- name: hostlib
hostPath:
path: /opt/gitlab-jemalloc
type: DirectoryOrCreate
JEDS
sed -i.bak "s/NAMESPACE_PLACEHOLDER/${NAMESPACE}/g" "$_tmpds"
kubectl apply -f "$_tmpds"
rm -f "$_tmpds" "${_tmpds}.bak"
log "Waiting up to 20 minutes for jemalloc to build on all nodes..."
local deadline=$((SECONDS + 1200))
while (( SECONDS < deadline )); do
ready=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
-o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0)
desired=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \
-o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo 1)
[[ "$ready" -ge "$desired" && "$desired" -gt 0 ]] && break
log " jemalloc build progress: ${ready}/${desired} nodes ready..."
sleep 30
done
log "jemalloc-16k ready on all nodes."
}
# ---------------------------------------------------------------------------
# Static PV + synology directory setup for gitaly and minio on gandalf
# (no dynamic provisioner is available in the service cluster)
# ---------------------------------------------------------------------------
setup_gitlab_storage() {
local synology_node="${GITLAB_STORAGE_NODE:-gandalf.prole.org}"
local base_path="${GITLAB_STORAGE_BASE:-/synology/d005/gitlab}"
local gitaly_size="${GITLAB_GITALY_PV_SIZE:-50Gi}"
local minio_size="${GITLAB_MINIO_PV_SIZE:-10Gi}"
log "Creating synology directories on ${synology_node}..."
kubectl -n "$NAMESPACE" run gitlab-dirprep \
--image=alpine:latest --restart=Never --rm --attach \
--overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"s\",\"hostPath\":{\"path\":\"$(dirname ${base_path})\",\"type\":\"Directory\"}}],\"containers\":[{\"name\":\"dirprep\",\"image\":\"alpine:latest\",\"command\":[\"sh\",\"-c\",\"mkdir -p ${base_path}/minio ${base_path}/gitaly && chmod 777 ${base_path}/minio ${base_path}/gitaly && echo done\"],\"volumeMounts\":[{\"name\":\"s\",\"mountPath\":\"$(dirname ${base_path})\"}]}]}}" \
2>&1 || warn "Could not create synology dirs (may already exist)"
log "Creating static PVs for gitaly (${gitaly_size}) and minio (${minio_size})..."
kubectl apply -f - <<PVYAML
apiVersion: v1
kind: PersistentVolume
metadata:
name: gitlab-gitaly-synology
spec:
capacity:
storage: ${gitaly_size}
accessModes: [ReadWriteOnce]
persistentVolumeReclaimPolicy: Retain
storageClassName: ""
local:
path: ${base_path}/gitaly
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: [${synology_node}]
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: gitlab-minio-synology
spec:
capacity:
storage: ${minio_size}
accessModes: [ReadWriteOnce]
persistentVolumeReclaimPolicy: Retain
storageClassName: ""
local:
path: ${base_path}/minio
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: [${synology_node}]
PVYAML
log "GitLab storage PVs ready."
# Create minio buckets — runs idempotently (--ignore-existing)
# bucket names must match chart defaults (registry uses short name "registry")
log "Creating minio buckets in namespace ${NAMESPACE}..."
local _ak _sk
_ak=$(kubectl -n "$NAMESPACE" get secret gitlab-minio-secret \
-o jsonpath='{.data.accesskey}' 2>/dev/null | base64 -d || true)
_sk=$(kubectl -n "$NAMESPACE" get secret gitlab-minio-secret \
-o jsonpath='{.data.secretkey}' 2>/dev/null | base64 -d || true)
if [[ -n "$_ak" && -n "$_sk" ]]; then
kubectl -n "$NAMESPACE" delete pod gitlab-minio-init --ignore-not-found 2>/dev/null || true
kubectl -n "$NAMESPACE" run gitlab-minio-init \
--image=minio/mc:latest \
--restart=Never \
--overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"gitlab-minio-init\",\"image\":\"minio/mc:latest\",\"command\":[\"sh\",\"-c\",\"mc alias set gl http://gitlab-minio-svc.${NAMESPACE}.svc.cluster.local:9000 ${_ak} ${_sk} && for b in registry gitlab-artifacts-storage gitlab-lfs-storage gitlab-uploads-storage gitlab-packages-storage gitlab-dependency-proxy-storage gitlab-terraform-state gitlab-ci-secure-files; do mc mb --ignore-existing gl/\$b; done && mc ls gl && echo BUCKETS_DONE\"]}]}}" 2>&1 || \
warn "minio-init pod failed to start; buckets may need to be created manually"
log "minio-init pod started — buckets will be ready in ~1 minute."
else
warn "gitlab-minio-secret not yet available; skipping bucket creation (run again after minio is up)."
fi
}
setup_jemalloc_on_nodes
setup_gitlab_storage
log "Pre-creating gitlab-app-nonroot ServiceAccount (required by chart v9+)..."
kubectl -n "${NAMESPACE}" apply -f - <<SAYAML
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitlab-app-nonroot
namespace: ${NAMESPACE}
SAYAML
log "Applying GitLab CR (domain=${GITLAB_DOMAIN}, db=${GITLAB_DB_NAME}@${DB_HOST})..."
kubectl apply -f - <<EOF
apiVersion: apps.gitlab.com/v1beta1
kind: GitLab
metadata:
name: ${GITLAB_RELEASE}
namespace: ${NAMESPACE}
spec:
chart:
${CHART_VERSION_YAML}
values:
global:
# ARM64 nodes: inject glibc jemalloc compiled with --with-lg-page=14
# The DaemonSet above places it at /opt/gitlab-jemalloc/libjemalloc.so.2
extraEnv:
LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<GNODE
nodeSelector:
${NODE_SELECTOR_YAML}
GNODE
fi)
# Explicit allowedHosts as flat strings — chart 9.x renders list-of-maps
# by default which breaks URI::InvalidComponentError in 7_gitlab_http.rb
allowedHosts:
- ${GITLAB_DOMAIN}
- ${GITLAB_DOMAIN#*.}
hosts:
domain: prole.org
gitlab:
name: ${GITLAB_DOMAIN}
ssh:
name: ${GITLAB_DOMAIN}
ingress:
class: kong
configureCertmanager: false
tls:
enabled: false
psql:
host: ${DB_HOST}
port: ${DB_PORT}
username: ${GITLAB_DB_USER}
database: ${GITLAB_DB_NAME}
password:
secret: gitlab-db-password
key: password
redis:
host: ${REDIS_HOST}
port: ${REDIS_PORT}
auth:
enabled: false
# chart v9+ requires the SA to be pre-created when a custom name is used
serviceAccount:
create: false
$(if [[ -n "$FRONTDOOR_HOST" ]]; then
cat <<OMNIAUTH
appConfig:
omniauth:
enabled: true
autoSignInWithProvider: openid_connect
allowSingleSignOn:
- openid_connect
blockAutoCreatedUsers: false
providers:
- secret: gitlab-google-oidc
key: provider
OMNIAUTH
fi)
# Disable bundled sub-charts — we provide our own data stores
postgresql:
install: false
redis:
install: false
certmanager:
install: false
prometheus:
install: false
gitlab-runner:
install: false
# ARM64 minio: quay.io 2022 image supports MINIO_ACCESS_KEY + has ARM64 variant
minio:
image: quay.io/minio/minio
imageTag: "RELEASE.2022-10-08T20-11-00Z"
mcImage:
repository: minio/mc
tag: "RELEASE.2022-10-20T23-30-35Z"
# Always pin minio to the storage node (local PV)
nodeSelector:
${STORAGE_NODE_SELECTOR_YAML}
gitlab:
webservice:
# Single replica + reduced memory — RPi nodes have limited RAM
replicaCount: 1
minReplicas: 1
maxReplicas: 1
resources:
requests:
cpu: 200m
memory: 1500M
limits:
memory: 2000M
# Rails preloading takes 25-40min on RPi; protect against liveness kills
livenessProbe:
initialDelaySeconds: 3600
periodSeconds: 60
timeoutSeconds: 30
failureThreshold: 5
readinessProbe:
initialDelaySeconds: 120
periodSeconds: 30
timeoutSeconds: 30
failureThreshold: 60
ingress:
enabled: true
annotations:
kubernetes.io/ingress.class: kong
extraEnv:
LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: /opt/gitlab-jemalloc
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: /opt/gitlab-jemalloc
readOnly: true
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
nodeSelector:
${NODE_SELECTOR_YAML}
NODE
fi)
sidekiq:
resources:
requests:
cpu: 100m
memory: 800M
limits:
memory: 1200M
# Rails loading takes 25-40min on RPi; protect against liveness kills
livenessProbe:
initialDelaySeconds: 3600
periodSeconds: 60
timeoutSeconds: 30
failureThreshold: 5
readinessProbe:
initialDelaySeconds: 120
periodSeconds: 30
timeoutSeconds: 30
failureThreshold: 60
extraEnv:
LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: /opt/gitlab-jemalloc
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: /opt/gitlab-jemalloc
readOnly: true
$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
cat <<NODE
nodeSelector:
${NODE_SELECTOR_YAML}
NODE
fi)
gitaly:
extraEnv:
LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: /opt/gitlab-jemalloc
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: /opt/gitlab-jemalloc
readOnly: true
# Give gitaly gRPC health check time to initialise before probes start
livenessProbe:
initialDelaySeconds: 30
readinessProbe:
initialDelaySeconds: 15
# Always pin gitaly to the storage node (local PV)
nodeSelector:
${STORAGE_NODE_SELECTOR_YAML}
toolbox:
extraEnv:
LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: /opt/gitlab-jemalloc
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: /opt/gitlab-jemalloc
readOnly: true
migrations:
extraEnv:
LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: /opt/gitlab-jemalloc
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: /opt/gitlab-jemalloc
readOnly: true
gitlab-exporter:
extraEnv:
LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
extraVolumes: |
- name: jemalloc-16k
hostPath:
path: /opt/gitlab-jemalloc
type: Directory
extraVolumeMounts: |
- name: jemalloc-16k
mountPath: /opt/gitlab-jemalloc
readOnly: true
registry:
ingress:
enabled: false
EOF
# ---------------------------------------------------------------------------
# ARM64 fix: minio configure init container uses x86-only gitlab-base image.
# Patch the Deployment to (a) replace init container with ARM64 alpine that
# writes the credential files, and (b) inject MINIO_ROOT_USER/PASSWORD
# directly into the main container so the 2022 quay.io image accepts them.
# This runs after CR apply so the Deployment exists.
# ---------------------------------------------------------------------------
fix_minio_arm64_credentials() {
local deadline=$((SECONDS + 120))
log "Waiting for minio Deployment to appear (post-CR apply)..."
while (( SECONDS < deadline )); do
kubectl -n "$NAMESPACE" get deployment gitlab-minio >/dev/null 2>&1 && break
sleep 10
done
kubectl -n "$NAMESPACE" get deployment gitlab-minio >/dev/null 2>&1 || {
warn "gitlab-minio Deployment not found — skipping ARM64 minio credential fix."
return 0
}
log "Patching minio Deployment: ARM64 init container + direct credential env vars..."
kubectl -n "$NAMESPACE" patch deployment gitlab-minio --type=json -p '[
{"op":"replace","path":"/spec/template/spec/initContainers/0","value":{
"name":"configure",
"image":"alpine:latest",
"command":["sh","-c","mkdir -p /tmp/.minio && printf \"%s\" \"$MINIO_ACCESS_KEY\" > /tmp/.minio/access_key && printf \"%s\" \"$MINIO_SECRET_KEY\" > /tmp/.minio/secret_key && chmod 600 /tmp/.minio/access_key /tmp/.minio/secret_key && echo done"],
"env":[
{"name":"MINIO_ACCESS_KEY","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"accesskey"}}},
{"name":"MINIO_SECRET_KEY","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"secretkey"}}}
],
"volumeMounts":[{"name":"minio-configuration","mountPath":"/tmp/.minio"}]
}},
{"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"MINIO_ROOT_USER","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"accesskey"}}}},
{"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"MINIO_ROOT_PASSWORD","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"secretkey"}}}}
]' 2>/dev/null || warn "Could not patch minio Deployment (operator may reconcile it back)."
log "Minio ARM64 credential fix applied."
}
fix_minio_arm64_credentials
log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)."
log "Monitor progress: kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w"
log "Watch pods: kubectl -n ${NAMESPACE} get pods -w"
# ---------------------------------------------------------------------------
# Optionally wait for GitLab to become available
# ---------------------------------------------------------------------------
WAIT_TIMEOUT="${GITLAB_WAIT_TIMEOUT:-1200}" # default 20 minutes
if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then
log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status..."
if ! kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \
--for=condition=Available \
--timeout="${WAIT_TIMEOUT}s" 2>/dev/null; then
warn "Timed out waiting for GitLab CR condition=Available."
warn "The deployment may still be in progress. Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}"
fi
fi
# ---------------------------------------------------------------------------
# Kong ingress for git.prole.org -> gitlab-webservice
# The GitLab Operator creates the Ingress; this block patches it to Kong class
# or creates a fallback Kong-native ingress if the CR-managed one is absent.
# ---------------------------------------------------------------------------
WEBSERVICE_SVC="${GITLAB_RELEASE}-webservice-default"
log "Ensuring Kong ingress for ${GITLAB_DOMAIN} -> ${WEBSERVICE_SVC}:8181 ..."
kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gitlab-kong-ingress
namespace: ${NAMESPACE}
annotations:
kubernetes.io/ingress.class: kong
konghq.com/strip-path: "false"
spec:
ingressClassName: kong
rules:
- host: ${GITLAB_DOMAIN}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: ${WEBSERVICE_SVC}
port:
number: 8181
EOF
log "Done."
log "GitLab will be reachable at http://${GITLAB_DOMAIN}/ once pods are Running."
log "Initial root password: kubectl -n ${NAMESPACE} get secret ${GITLAB_RELEASE}-gitlab-initial-root-password -o jsonpath='{.data.password}' | base64 -d"