2026-09-23 10:13:58 +00:00 · 2026-04-02 20:15:11 -07:00 · 2026-04-02 20:15:11 -07:00 · eb9430df9d
commit eb9430df9d
parent 2bcc40fd50
9 changed files with 328 additions and 37 deletions
--- a/conf/service/prole.cfg
+++ b/conf/service/prole.cfg
@ -1,5 +1,5 @@
 ; Prole Master Configuration File
-; Generated by install.py on 2026-04-01 20:19:32
+; Generated by install.py on 2026-04-02 15:08:15
 ; This file is used as input for Ansible deployment and k8s cluster creation.

 [User]
@ -73,8 +73,8 @@ init_db_build.run_build = true
 init_password.cluster_name = ${CLUSTER_NAME}
 init_password.db_host_port = 5432
 init_password.db_namespace = ${DATABASE_NAMESPACE}
-init_password.db_password = ${PROLE_SECRET:v1:ES-jFEjo8Uc9MEi3:hg0JXudLh1Gv6jtYkKPsh5CR5hKhDXNO}
-init_password.db_password_confirm = ${PROLE_SECRET:v1:ES-jFEjo8Uc9MEi3:hg0JXudLh1Gv6jtYkKPsh5CR5hKhDXNO}
+init_password.db_password = ${PROLE_SECRET:v1:xBDGgaaL55pMR2kj:wm_BCtknYSCwNksQZHLqDf5IweNirlsu}
+init_password.db_password_confirm = ${PROLE_SECRET:v1:xBDGgaaL55pMR2kj:wm_BCtknYSCwNksQZHLqDf5IweNirlsu}
 init_password.db_username = root
 init_password.generate_ssh_key = true
 init_scripts.run_scripts = true
@ -201,7 +201,7 @@ KERBEROS_ENABLED = true
 SUPABASE_ENABLED = true

 [GitOps]
-STATUS = Deployed
+STATUS = Attempted

 [Database Creation]
 DB_USER = root
--- a/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml
+++ b/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml
@ -16,6 +16,24 @@ spec:
      labels:
        app: prole-svc-kong
    spec:
+      affinity:
+        nodeAffinity:
+          # Never schedule on pi.prole.org — pihole-FTL owns ports 80/443 there
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                - pi.prole.org
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 80
+            preference:
+              matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: In
+                values:
+                - gandalf.prole.org
      containers:
        - name: kong
          image: kong:3.9
--- a/etc/init_gitlab.sh
+++ b/etc/init_gitlab.sh
@ -8,10 +8,14 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "$SCRIPT_DIR/prole_cfg.sh"

 MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")"
-NAMESPACE="${GITLAB_NAMESPACE:-}"
+# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden.
+# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps).
+NAMESPACE="${GITLAB_NAMESPACE:-gitlab}"
 CFG_PATH=""
 FORCE=0
-NODE_SELECTOR="${GITLAB_NODE_SELECTOR:-${NODE_SELECTOR:-gandalf.prole.org}}"
+# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE).
+# Setting this would pin ALL global components to one node, exhausting RAM.
+NODE_SELECTOR=""

 usage() {
  cat <<EOF
@ -53,6 +57,9 @@ while [[ $# -gt 0 ]]; do
    --node-selector=*) NODE_SELECTOR="${1#*=}"; shift 1 ;;
    --force)         FORCE=1; shift 1 ;;
    -h|--help)       usage; exit 0 ;;
+    --trace-namespace)
+      # Resolve namespace then exit — used by diagnostic scripts only
+      _TRACE_NS=1; shift 1 ;;
    *) break ;;
  esac
 done
@ -66,14 +73,22 @@ elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then
  CFG_PATH="$SCRIPT_DIR/../conf/prole.cfg"
 fi

-if [[ -z "$NAMESPACE" && -n "$CFG_PATH" ]]; then
+# cfg-based namespace override (only GITLAB_NAMESPACE; GITOPS_NAMESPACE intentionally excluded)
+if [[ -z "$NAMESPACE" || "$NAMESPACE" == "gitlab" ]] && [[ -n "$CFG_PATH" ]]; then
  maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")"
-  [[ -z "$maybe_ns" ]] && maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITOPS_NAMESPACE")"
-  NAMESPACE="$maybe_ns"
+  [[ -n "$maybe_ns" ]] && NAMESPACE="$maybe_ns"
 fi
+# Final safety net — always default to 'gitlab'
 NAMESPACE="${NAMESPACE:-gitlab}"
 export GITLAB_NAMESPACE="$NAMESPACE"

+# Diagnostic exit — used by tmp/sim_installer_ns.sh
+if [[ "${_TRACE_NS:-0}" == "1" ]]; then
+  echo "TRACE: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" >&2
+  echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-<unset>}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-<unset>}'" >&2
+  exit 0
+fi
+
 case "$MODE" in
  k3d|k3s|k8s|local) ;;
  *) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;;
@ -331,11 +346,17 @@ CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\""
 # ---------------------------------------------------------------------------
 # Determine node selector block for the GitLab CR
 # ---------------------------------------------------------------------------
+# Storage node for gitaly + minio only (local PVs require co-location)
+# Other components spread across the cluster via default scheduler.
+STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-gandalf.prole.org}}"
+# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned
+NODE_SELECTOR="${NODE_SELECTOR:-}"
 NODE_SELECTOR_YAML=""
 if [[ -n "$NODE_SELECTOR" ]]; then
-  log "Pinning GitLab workloads to node: ${NODE_SELECTOR}"
+  log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}"
  NODE_SELECTOR_YAML="kubernetes.io/hostname: ${NODE_SELECTOR}"
 fi
+STORAGE_NODE_SELECTOR_YAML="kubernetes.io/hostname: ${STORAGE_NODE}"

 # ---------------------------------------------------------------------------
 # Create the GitLab CR  (operator reconciles this into the full deployment)
@ -515,6 +536,27 @@ spec:
          values: [${synology_node}]
 PVYAML
  log "GitLab storage PVs ready."
+
+  # Create minio buckets — runs idempotently (--ignore-existing)
+  # bucket names must match chart defaults (registry uses short name "registry")
+  log "Creating minio buckets in namespace ${NAMESPACE}..."
+  local _ak _sk
+  _ak=$(kubectl -n "$NAMESPACE" get secret gitlab-minio-secret \
+    -o jsonpath='{.data.accesskey}' 2>/dev/null | base64 -d || true)
+  _sk=$(kubectl -n "$NAMESPACE" get secret gitlab-minio-secret \
+    -o jsonpath='{.data.secretkey}' 2>/dev/null | base64 -d || true)
+
+  if [[ -n "$_ak" && -n "$_sk" ]]; then
+    kubectl -n "$NAMESPACE" delete pod gitlab-minio-init --ignore-not-found 2>/dev/null || true
+    kubectl -n "$NAMESPACE" run gitlab-minio-init \
+      --image=minio/mc:latest \
+      --restart=Never \
+      --overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"gitlab-minio-init\",\"image\":\"minio/mc:latest\",\"command\":[\"sh\",\"-c\",\"mc alias set gl http://gitlab-minio-svc.${NAMESPACE}.svc.cluster.local:9000 ${_ak} ${_sk} && for b in registry gitlab-artifacts-storage gitlab-lfs-storage gitlab-uploads-storage gitlab-packages-storage gitlab-dependency-proxy-storage gitlab-terraform-state gitlab-ci-secure-files; do mc mb --ignore-existing gl/\$b; done && mc ls gl && echo BUCKETS_DONE\"]}]}}" 2>&1 || \
+      warn "minio-init pod failed to start; buckets may need to be created manually"
+    log "minio-init pod started — buckets will be ready in ~1 minute."
+  else
+    warn "gitlab-minio-secret not yet available; skipping bucket creation (run again after minio is up)."
+  fi
 }

 setup_jemalloc_on_nodes
@ -546,6 +588,17 @@ spec:
        # The DaemonSet above places it at /opt/gitlab-jemalloc/libjemalloc.so.2
        extraEnv:
          LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
+$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
+cat <<GNODE
+        nodeSelector:
+          ${NODE_SELECTOR_YAML}
+GNODE
+fi)
+        # Explicit allowedHosts as flat strings — chart 9.x renders list-of-maps
+        # by default which breaks URI::InvalidComponentError in 7_gitlab_http.rb
+        allowedHosts:
+          - ${GITLAB_DOMAIN}
+          - ${GITLAB_DOMAIN#*.}
        hosts:
          domain: prole.org
          gitlab:
@ -603,12 +656,34 @@ fi)
        image: quay.io/minio/minio
        imageTag: "RELEASE.2022-10-08T20-11-00Z"
        mcImage:
-          repository: quay.io/minio/mc
+          repository: minio/mc
          tag: "RELEASE.2022-10-20T23-30-35Z"
+        # Always pin minio to the storage node (local PV)
+        nodeSelector:
+          ${STORAGE_NODE_SELECTOR_YAML}
      gitlab:
        webservice:
-          # Single replica — RPi nodes have limited RAM
+          # Single replica + reduced memory — RPi nodes have limited RAM
          replicaCount: 1
+          minReplicas: 1
+          maxReplicas: 1
+          resources:
+            requests:
+              cpu: 200m
+              memory: 1500M
+            limits:
+              memory: 2000M
+          # Rails preloading takes 25-40min on RPi; protect against liveness kills
+          livenessProbe:
+            initialDelaySeconds: 3600
+            periodSeconds: 60
+            timeoutSeconds: 30
+            failureThreshold: 5
+          readinessProbe:
+            initialDelaySeconds: 120
+            periodSeconds: 30
+            timeoutSeconds: 30
+            failureThreshold: 60
          ingress:
            enabled: true
            annotations:
@ -631,6 +706,23 @@ cat <<NODE
 NODE
 fi)
        sidekiq:
+          resources:
+            requests:
+              cpu: 100m
+              memory: 800M
+            limits:
+              memory: 1200M
+          # Rails loading takes 25-40min on RPi; protect against liveness kills
+          livenessProbe:
+            initialDelaySeconds: 3600
+            periodSeconds: 60
+            timeoutSeconds: 30
+            failureThreshold: 5
+          readinessProbe:
+            initialDelaySeconds: 120
+            periodSeconds: 30
+            timeoutSeconds: 30
+            failureThreshold: 60
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
          extraVolumes: |
@ -665,12 +757,9 @@ fi)
            initialDelaySeconds: 30
          readinessProbe:
            initialDelaySeconds: 15
-$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then
-cat <<NODE
+          # Always pin gitaly to the storage node (local PV)
          nodeSelector:
-            ${NODE_SELECTOR_YAML}
-NODE
-fi)
+            ${STORAGE_NODE_SELECTOR_YAML}
        toolbox:
          extraEnv:
            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
@ -695,11 +784,63 @@ fi)
            - name: jemalloc-16k
              mountPath: /opt/gitlab-jemalloc
              readOnly: true
+        gitlab-exporter:
+          extraEnv:
+            LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2
+          extraVolumes: |
+            - name: jemalloc-16k
+              hostPath:
+                path: /opt/gitlab-jemalloc
+                type: Directory
+          extraVolumeMounts: |
+            - name: jemalloc-16k
+              mountPath: /opt/gitlab-jemalloc
+              readOnly: true
      registry:
        ingress:
          enabled: false
 EOF

+# ---------------------------------------------------------------------------
+# ARM64 fix: minio configure init container uses x86-only gitlab-base image.
+# Patch the Deployment to (a) replace init container with ARM64 alpine that
+# writes the credential files, and (b) inject MINIO_ROOT_USER/PASSWORD
+# directly into the main container so the 2022 quay.io image accepts them.
+# This runs after CR apply so the Deployment exists.
+# ---------------------------------------------------------------------------
+fix_minio_arm64_credentials() {
+  local deadline=$((SECONDS + 120))
+  log "Waiting for minio Deployment to appear (post-CR apply)..."
+  while (( SECONDS < deadline )); do
+    kubectl -n "$NAMESPACE" get deployment gitlab-minio >/dev/null 2>&1 && break
+    sleep 10
+  done
+
+  kubectl -n "$NAMESPACE" get deployment gitlab-minio >/dev/null 2>&1 || {
+    warn "gitlab-minio Deployment not found — skipping ARM64 minio credential fix."
+    return 0
+  }
+
+  log "Patching minio Deployment: ARM64 init container + direct credential env vars..."
+  kubectl -n "$NAMESPACE" patch deployment gitlab-minio --type=json -p '[
+    {"op":"replace","path":"/spec/template/spec/initContainers/0","value":{
+      "name":"configure",
+      "image":"alpine:latest",
+      "command":["sh","-c","mkdir -p /tmp/.minio && printf \"%s\" \"$MINIO_ACCESS_KEY\" > /tmp/.minio/access_key && printf \"%s\" \"$MINIO_SECRET_KEY\" > /tmp/.minio/secret_key && chmod 600 /tmp/.minio/access_key /tmp/.minio/secret_key && echo done"],
+      "env":[
+        {"name":"MINIO_ACCESS_KEY","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"accesskey"}}},
+        {"name":"MINIO_SECRET_KEY","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"secretkey"}}}
+      ],
+      "volumeMounts":[{"name":"minio-configuration","mountPath":"/tmp/.minio"}]
+    }},
+    {"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"MINIO_ROOT_USER","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"accesskey"}}}},
+    {"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"MINIO_ROOT_PASSWORD","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"secretkey"}}}}
+  ]' 2>/dev/null || warn "Could not patch minio Deployment (operator may reconcile it back)."
+  log "Minio ARM64 credential fix applied."
+}
+
+fix_minio_arm64_credentials
+
 log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)."
 log "Monitor progress:  kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w"
 log "Watch pods:        kubectl -n ${NAMESPACE} get pods -w"
--- a/img/KnoeLogo.png
+++ b/img/KnoeLogo.png
--- a/infrastructure/inventory/host_vars/pi.prole.org.yml
+++ b/infrastructure/inventory/host_vars/pi.prole.org.yml
@ -26,6 +26,11 @@ iscsi_targets:
        opts: "_netdev,noatime"
        src: "UUID=56b21ec3-2826-4171-b909-a6715223f9a4"

+# k3s intentionally disabled — pi.prole.org is a dedicated pihole node;
+# insufficient RAM for k3s workloads; agent manually stopped 2026-04-01.
+k3s_enabled: false
+k3s_state: absent
+
 k3s_rancher_mount_required: true
 k3s_rancher_mount_src: /synology/d003/rancher
 k3s_rancher_mount_fstype: none
--- a/infrastructure/inventory/hosts.ini
+++ b/infrastructure/inventory/hosts.ini
@ -17,7 +17,6 @@ myrddin.prole.org
 myrddin.prole.org

 [k3s_agents]
-pi.prole.org
 merlin.prole.org
 gandalf.prole.org

--- a/infrastructure/playbooks/disable_pi_k3s.yml
+++ b/infrastructure/playbooks/disable_pi_k3s.yml
@ -0,0 +1,86 @@
+---
+# Disable k3s-agent on pi.prole.org and remove it from the cluster.
+# Safe to run against a live cluster — only touches pi.prole.org.
+# Storage (iSCSI) and pihole configuration are intentionally preserved.
+#
+# Usage:
+#   ansible-playbook -i inventory/hosts.ini playbooks/disable_pi_k3s.yml
+#   ansible-playbook -i inventory/hosts.ini playbooks/disable_pi_k3s.yml --check
+
+- name: Drain and delete pi node from k3s control-plane
+  hosts: k3s_servers
+  gather_facts: false
+  become: true
+  tasks:
+    - name: Drain pi.prole.org (evict pods, ignore DaemonSets)
+      ansible.builtin.command:
+        cmd: kubectl drain pi.prole.org
+              --ignore-daemonsets
+              --delete-emptydir-data
+              --force
+              --timeout=120s
+      environment:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      register: _drain
+      changed_when: _drain.rc == 0
+      failed_when: false
+
+    - name: Delete pi node from cluster
+      ansible.builtin.command:
+        cmd: kubectl delete node pi.prole.org --ignore-not-found
+      environment:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      register: _delete_node
+      changed_when: "'deleted' in _delete_node.stdout"
+      failed_when: false
+
+- name: Stop and disable k3s-agent on pi.prole.org
+  hosts: pi.prole.org
+  gather_facts: false
+  become: true
+  tasks:
+    - name: Gather minimal facts
+      ansible.builtin.setup:
+        gather_subset:
+          - min
+
+    - name: Check for k3s-agent systemd unit
+      ansible.builtin.stat:
+        path: /etc/systemd/system/k3s-agent.service
+      register: _agent_unit
+
+    - name: Stop k3s-agent service
+      ansible.builtin.systemd:
+        name: k3s-agent
+        state: stopped
+        enabled: false
+      when:
+        - _agent_unit.stat.exists | default(false)
+        - not ansible_check_mode
+
+    - name: Check for generic k3s systemd unit (agent variant)
+      ansible.builtin.stat:
+        path: /etc/systemd/system/k3s.service
+      register: _k3s_unit
+
+    - name: Stop k3s service if present (some installs use k3s not k3s-agent)
+      ansible.builtin.systemd:
+        name: k3s
+        state: stopped
+        enabled: false
+      when:
+        - _k3s_unit.stat.exists | default(false)
+        - not ansible_check_mode
+      failed_when: false
+
+    - name: Confirm k3s processes are not running
+      ansible.builtin.command: pgrep -c k3s
+      register: _k3s_procs
+      changed_when: false
+      failed_when: false
+
+    - name: Report k3s process status
+      ansible.builtin.debug:
+        msg: >-
+          k3s processes on pi.prole.org:
+          {{ 'NONE (clean)' if _k3s_procs.rc != 0 else _k3s_procs.stdout + ' process(es) still running' }}
--- a/knoe/core/milestones.py
+++ b/knoe/core/milestones.py
@ -963,9 +963,16 @@ class GitOpsMilestone(Milestone):
        env = self._get_script_env(state)

        if is_gitlab:
+            # Always prefer the explicit gitlab_namespace key; never fall back to
+            # gitops.namespace which may be 'gitea' from a prior Gitea run.
            ns = (
                str(state.inputs.get("gitops.gitlab_namespace", "") or "").strip()
-                or str(state.inputs.get("gitops.namespace", "") or "").strip()
+                or str(
+                    (state.config_data.get("Global") or {}).get(
+                        "GITLAB_NAMESPACE", ""
+                    )
+                    or ""
+                ).strip()
                or "gitlab"
            )
            env["GITLAB_NAMESPACE"] = ns
--- a/knoe/core/ops/monitoring.py
+++ b/knoe/core/ops/monitoring.py
@ -1,5 +1,8 @@
 from __future__ import annotations

+import os
+import tempfile
+
 from ._services_common import _LogFn, _detect_mode, _helm, _kubectl, _log, _namespace, _to_bool


@ -54,25 +57,57 @@ def update(

    _kubectl(["create", "namespace", ns], env=env, timeout=60)

-    _log(log, f"[MONITORING] Deploying {release} in namespace {ns}")
-    _helm(
-        [
-            "upgrade",
-            "--install",
-            release,
-            chart,
-            "--namespace",
-            ns,
-            "--set",
-            "grafana.adminUser=admin",
-            "--set",
-            f"grafana.adminPassword={grafana_password}",
-            "--wait",
-        ],
-        env=env,
-        timeout=600,
-        check=True,
+    # Nodes with broken kubelet (e.g. pi.prole.org returning 502) must be excluded
+    # from the node-exporter DaemonSet so helm --wait can succeed.
+    excluded_nodes = str((env or {}).get("MONITORING_NODE_EXPORTER_EXCLUDE_NODES") or "pi.prole.org")
+    excluded_list = [n.strip() for n in excluded_nodes.split(",") if n.strip()]
+
+    values_yaml = "prometheus-node-exporter:\n"
+    if excluded_list:
+        values_yaml += (
+            "  affinity:\n"
+            "    nodeAffinity:\n"
+            "      requiredDuringSchedulingIgnoredDuringExecution:\n"
+            "        nodeSelectorTerms:\n"
+            "        - matchExpressions:\n"
+            "          - key: kubernetes.io/hostname\n"
+            "            operator: NotIn\n"
+            "            values:\n"
+        )
+        for node in excluded_list:
+            values_yaml += f"            - {node}\n"
+
+    tmp_values = tempfile.NamedTemporaryFile(
+        mode="w", suffix=".yaml", prefix="monitoring-values-", delete=False
    )
+    try:
+        tmp_values.write(values_yaml)
+        tmp_values.flush()
+        tmp_values.close()
+
+        _log(log, f"[MONITORING] Deploying {release} in namespace {ns}")
+        _helm(
+            [
+                "upgrade",
+                "--install",
+                release,
+                chart,
+                "--namespace",
+                ns,
+                "--set",
+                "grafana.adminUser=admin",
+                "--set",
+                f"grafana.adminPassword={grafana_password}",
+                "-f",
+                tmp_values.name,
+                "--wait",
+            ],
+            env=env,
+            timeout=600,
+            check=True,
+        )
+    finally:
+        os.unlink(tmp_values.name)


 def restart(