prole/archive/2026-04-cnpg-and-gitlab-cleanup/rebuild_jemalloc_glibc.sh

#!/usr/bin/env bash
set -euo pipefail
export KUBECONFIG=/Users/chrisfu/dev/knoe/knoe-k3s.kubeconfig

echo "==> Deleting old musl-based DaemonSet and cleaning nodes..."
kubectl -n gitlab delete daemonset jemalloc-builder --ignore-not-found

# Clean old musl binary from all nodes via a cleanup DaemonSet
kubectl apply -f - <<'CLEANUP'
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: jemalloc-cleanup
  namespace: gitlab
spec:
  selector:
    matchLabels:
      app: jemalloc-cleanup
  template:
    metadata:
      labels:
        app: jemalloc-cleanup
    spec:
      tolerations:
      - operator: Exists
      initContainers:
      - name: clean
        image: alpine:latest
        command: [sh, -c, "rm -f /hostlib/libjemalloc.so.2 && echo cleaned"]
        volumeMounts:
        - name: hostlib
          mountPath: /hostlib
      containers:
      - name: done
        image: alpine:latest
        command: [sh, -c, "echo done; sleep infinity"]
      volumes:
      - name: hostlib
        hostPath:
          path: /opt/gitlab-jemalloc
          type: DirectoryOrCreate
CLEANUP

echo "Waiting 30s for cleanup to complete..."
sleep 30
kubectl -n gitlab delete daemonset jemalloc-cleanup --ignore-not-found

echo "==> Deploying Ubuntu-based (glibc) jemalloc builder DaemonSet..."
kubectl apply -f - <<'DS'
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: jemalloc-builder
  namespace: gitlab
  labels:
    app: jemalloc-builder
spec:
  selector:
    matchLabels:
      app: jemalloc-builder
  template:
    metadata:
      labels:
        app: jemalloc-builder
    spec:
      tolerations:
      - operator: Exists
      initContainers:
      - name: build
        image: ubuntu:22.04
        command:
        - bash
        - -c
        - |
          set -e
          TARGET=/hostlib/libjemalloc.so.2
          if [ -f "$TARGET" ]; then echo "Already built — skipping."; exit 0; fi
          echo "Installing build tools (glibc-based)..."
          apt-get update -qq && apt-get install -y -q gcc make wget autoconf 2>&1
          cd /tmp
          echo "Downloading jemalloc 5.3.0..."
          wget -q -O jemalloc.tar.bz2 \
            https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
          tar xf jemalloc.tar.bz2 && cd jemalloc-5.3.0
          echo "Configuring with --with-lg-page=14 (16KB, glibc)..."
          ./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1
          make -j2 lib/libjemalloc.so.2 2>&1
          cp lib/libjemalloc.so.2 "$TARGET"
          chmod 755 "$TARGET"
          file "$TARGET"
          echo "Done: $(ls -lh $TARGET)"
        volumeMounts:
        - name: hostlib
          mountPath: /hostlib
      containers:
      - name: keepalive
        image: ubuntu:22.04
        command: [bash, -c, "echo 'glibc jemalloc-16k ready'; sleep infinity"]
      volumes:
      - name: hostlib
        hostPath:
          path: /opt/gitlab-jemalloc
          type: DirectoryOrCreate
DS

echo "==> Builder DaemonSet created. This will take ~10-15min on RPi. Checking in 5min..."
sleep 300
kubectl -n gitlab get pods -l app=jemalloc-builder