diff --git a/conf/service/prole.cfg b/conf/service/prole.cfg index b61f999..d6a8bd2 100644 --- a/conf/service/prole.cfg +++ b/conf/service/prole.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-01 20:19:32 +; Generated by install.py on 2026-04-02 15:08:15 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -73,8 +73,8 @@ init_db_build.run_build = true init_password.cluster_name = ${CLUSTER_NAME} init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password = ${PROLE_SECRET:v1:ES-jFEjo8Uc9MEi3:hg0JXudLh1Gv6jtYkKPsh5CR5hKhDXNO} -init_password.db_password_confirm = ${PROLE_SECRET:v1:ES-jFEjo8Uc9MEi3:hg0JXudLh1Gv6jtYkKPsh5CR5hKhDXNO} +init_password.db_password = ${PROLE_SECRET:v1:xBDGgaaL55pMR2kj:wm_BCtknYSCwNksQZHLqDf5IweNirlsu} +init_password.db_password_confirm = ${PROLE_SECRET:v1:xBDGgaaL55pMR2kj:wm_BCtknYSCwNksQZHLqDf5IweNirlsu} init_password.db_username = root init_password.generate_ssh_key = true init_scripts.run_scripts = true @@ -201,7 +201,7 @@ KERBEROS_ENABLED = true SUPABASE_ENABLED = true [GitOps] -STATUS = Deployed +STATUS = Attempted [Database Creation] DB_USER = root diff --git a/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml b/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml index 4c1662f..dfe421c 100644 --- a/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml +++ b/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml @@ -16,6 +16,24 @@ spec: labels: app: prole-svc-kong spec: + affinity: + nodeAffinity: + # Never schedule on pi.prole.org — pihole-FTL owns ports 80/443 there + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - pi.prole.org + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 80 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - gandalf.prole.org containers: - name: kong image: kong:3.9 diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh old mode 100644 new mode 100755 index 12f98ef..a9c7eb8 --- a/etc/init_gitlab.sh +++ b/etc/init_gitlab.sh @@ -8,10 +8,14 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/prole_cfg.sh" MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")" -NAMESPACE="${GITLAB_NAMESPACE:-}" +# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden. +# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps). +NAMESPACE="${GITLAB_NAMESPACE:-gitlab}" CFG_PATH="" FORCE=0 -NODE_SELECTOR="${GITLAB_NODE_SELECTOR:-${NODE_SELECTOR:-gandalf.prole.org}}" +# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE). +# Setting this would pin ALL global components to one node, exhausting RAM. +NODE_SELECTOR="" usage() { cat <&2 + echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-}'" >&2 + exit 0 +fi + case "$MODE" in k3d|k3s|k8s|local) ;; *) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;; @@ -331,11 +346,17 @@ CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\"" # --------------------------------------------------------------------------- # Determine node selector block for the GitLab CR # --------------------------------------------------------------------------- +# Storage node for gitaly + minio only (local PVs require co-location) +# Other components spread across the cluster via default scheduler. +STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-gandalf.prole.org}}" +# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned +NODE_SELECTOR="${NODE_SELECTOR:-}" NODE_SELECTOR_YAML="" if [[ -n "$NODE_SELECTOR" ]]; then - log "Pinning GitLab workloads to node: ${NODE_SELECTOR}" + log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}" NODE_SELECTOR_YAML="kubernetes.io/hostname: ${NODE_SELECTOR}" fi +STORAGE_NODE_SELECTOR_YAML="kubernetes.io/hostname: ${STORAGE_NODE}" # --------------------------------------------------------------------------- # Create the GitLab CR (operator reconciles this into the full deployment) @@ -515,6 +536,27 @@ spec: values: [${synology_node}] PVYAML log "GitLab storage PVs ready." + + # Create minio buckets — runs idempotently (--ignore-existing) + # bucket names must match chart defaults (registry uses short name "registry") + log "Creating minio buckets in namespace ${NAMESPACE}..." + local _ak _sk + _ak=$(kubectl -n "$NAMESPACE" get secret gitlab-minio-secret \ + -o jsonpath='{.data.accesskey}' 2>/dev/null | base64 -d || true) + _sk=$(kubectl -n "$NAMESPACE" get secret gitlab-minio-secret \ + -o jsonpath='{.data.secretkey}' 2>/dev/null | base64 -d || true) + + if [[ -n "$_ak" && -n "$_sk" ]]; then + kubectl -n "$NAMESPACE" delete pod gitlab-minio-init --ignore-not-found 2>/dev/null || true + kubectl -n "$NAMESPACE" run gitlab-minio-init \ + --image=minio/mc:latest \ + --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"gitlab-minio-init\",\"image\":\"minio/mc:latest\",\"command\":[\"sh\",\"-c\",\"mc alias set gl http://gitlab-minio-svc.${NAMESPACE}.svc.cluster.local:9000 ${_ak} ${_sk} && for b in registry gitlab-artifacts-storage gitlab-lfs-storage gitlab-uploads-storage gitlab-packages-storage gitlab-dependency-proxy-storage gitlab-terraform-state gitlab-ci-secure-files; do mc mb --ignore-existing gl/\$b; done && mc ls gl && echo BUCKETS_DONE\"]}]}}" 2>&1 || \ + warn "minio-init pod failed to start; buckets may need to be created manually" + log "minio-init pod started — buckets will be ready in ~1 minute." + else + warn "gitlab-minio-secret not yet available; skipping bucket creation (run again after minio is up)." + fi } setup_jemalloc_on_nodes @@ -546,6 +588,17 @@ spec: # The DaemonSet above places it at /opt/gitlab-jemalloc/libjemalloc.so.2 extraEnv: LD_PRELOAD: /opt/gitlab-jemalloc/libjemalloc.so.2 +$(if [[ -n "$NODE_SELECTOR_YAML" ]]; then +cat </dev/null 2>&1 && break + sleep 10 + done + + kubectl -n "$NAMESPACE" get deployment gitlab-minio >/dev/null 2>&1 || { + warn "gitlab-minio Deployment not found — skipping ARM64 minio credential fix." + return 0 + } + + log "Patching minio Deployment: ARM64 init container + direct credential env vars..." + kubectl -n "$NAMESPACE" patch deployment gitlab-minio --type=json -p '[ + {"op":"replace","path":"/spec/template/spec/initContainers/0","value":{ + "name":"configure", + "image":"alpine:latest", + "command":["sh","-c","mkdir -p /tmp/.minio && printf \"%s\" \"$MINIO_ACCESS_KEY\" > /tmp/.minio/access_key && printf \"%s\" \"$MINIO_SECRET_KEY\" > /tmp/.minio/secret_key && chmod 600 /tmp/.minio/access_key /tmp/.minio/secret_key && echo done"], + "env":[ + {"name":"MINIO_ACCESS_KEY","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"accesskey"}}}, + {"name":"MINIO_SECRET_KEY","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"secretkey"}}} + ], + "volumeMounts":[{"name":"minio-configuration","mountPath":"/tmp/.minio"}] + }}, + {"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"MINIO_ROOT_USER","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"accesskey"}}}}, + {"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"MINIO_ROOT_PASSWORD","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"secretkey"}}}} + ]' 2>/dev/null || warn "Could not patch minio Deployment (operator may reconcile it back)." + log "Minio ARM64 credential fix applied." +} + +fix_minio_arm64_credentials + log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)." log "Monitor progress: kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w" log "Watch pods: kubectl -n ${NAMESPACE} get pods -w" diff --git a/img/KnoeLogo.png b/img/KnoeLogo.png new file mode 100644 index 0000000..7b7f93e Binary files /dev/null and b/img/KnoeLogo.png differ diff --git a/infrastructure/inventory/host_vars/pi.prole.org.yml b/infrastructure/inventory/host_vars/pi.prole.org.yml index 94270d7..ac75b4e 100644 --- a/infrastructure/inventory/host_vars/pi.prole.org.yml +++ b/infrastructure/inventory/host_vars/pi.prole.org.yml @@ -26,6 +26,11 @@ iscsi_targets: opts: "_netdev,noatime" src: "UUID=56b21ec3-2826-4171-b909-a6715223f9a4" +# k3s intentionally disabled — pi.prole.org is a dedicated pihole node; +# insufficient RAM for k3s workloads; agent manually stopped 2026-04-01. +k3s_enabled: false +k3s_state: absent + k3s_rancher_mount_required: true k3s_rancher_mount_src: /synology/d003/rancher k3s_rancher_mount_fstype: none diff --git a/infrastructure/inventory/hosts.ini b/infrastructure/inventory/hosts.ini index 3748f0f..d76061e 100644 --- a/infrastructure/inventory/hosts.ini +++ b/infrastructure/inventory/hosts.ini @@ -17,7 +17,6 @@ myrddin.prole.org myrddin.prole.org [k3s_agents] -pi.prole.org merlin.prole.org gandalf.prole.org diff --git a/infrastructure/playbooks/disable_pi_k3s.yml b/infrastructure/playbooks/disable_pi_k3s.yml new file mode 100644 index 0000000..ccc37eb --- /dev/null +++ b/infrastructure/playbooks/disable_pi_k3s.yml @@ -0,0 +1,86 @@ +--- +# Disable k3s-agent on pi.prole.org and remove it from the cluster. +# Safe to run against a live cluster — only touches pi.prole.org. +# Storage (iSCSI) and pihole configuration are intentionally preserved. +# +# Usage: +# ansible-playbook -i inventory/hosts.ini playbooks/disable_pi_k3s.yml +# ansible-playbook -i inventory/hosts.ini playbooks/disable_pi_k3s.yml --check + +- name: Drain and delete pi node from k3s control-plane + hosts: k3s_servers + gather_facts: false + become: true + tasks: + - name: Drain pi.prole.org (evict pods, ignore DaemonSets) + ansible.builtin.command: + cmd: kubectl drain pi.prole.org + --ignore-daemonsets + --delete-emptydir-data + --force + --timeout=120s + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: _drain + changed_when: _drain.rc == 0 + failed_when: false + + - name: Delete pi node from cluster + ansible.builtin.command: + cmd: kubectl delete node pi.prole.org --ignore-not-found + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: _delete_node + changed_when: "'deleted' in _delete_node.stdout" + failed_when: false + +- name: Stop and disable k3s-agent on pi.prole.org + hosts: pi.prole.org + gather_facts: false + become: true + tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + + - name: Check for k3s-agent systemd unit + ansible.builtin.stat: + path: /etc/systemd/system/k3s-agent.service + register: _agent_unit + + - name: Stop k3s-agent service + ansible.builtin.systemd: + name: k3s-agent + state: stopped + enabled: false + when: + - _agent_unit.stat.exists | default(false) + - not ansible_check_mode + + - name: Check for generic k3s systemd unit (agent variant) + ansible.builtin.stat: + path: /etc/systemd/system/k3s.service + register: _k3s_unit + + - name: Stop k3s service if present (some installs use k3s not k3s-agent) + ansible.builtin.systemd: + name: k3s + state: stopped + enabled: false + when: + - _k3s_unit.stat.exists | default(false) + - not ansible_check_mode + failed_when: false + + - name: Confirm k3s processes are not running + ansible.builtin.command: pgrep -c k3s + register: _k3s_procs + changed_when: false + failed_when: false + + - name: Report k3s process status + ansible.builtin.debug: + msg: >- + k3s processes on pi.prole.org: + {{ 'NONE (clean)' if _k3s_procs.rc != 0 else _k3s_procs.stdout + ' process(es) still running' }} diff --git a/knoe/core/milestones.py b/knoe/core/milestones.py index 00359dc..3e6e3b9 100644 --- a/knoe/core/milestones.py +++ b/knoe/core/milestones.py @@ -963,9 +963,16 @@ class GitOpsMilestone(Milestone): env = self._get_script_env(state) if is_gitlab: + # Always prefer the explicit gitlab_namespace key; never fall back to + # gitops.namespace which may be 'gitea' from a prior Gitea run. ns = ( str(state.inputs.get("gitops.gitlab_namespace", "") or "").strip() - or str(state.inputs.get("gitops.namespace", "") or "").strip() + or str( + (state.config_data.get("Global") or {}).get( + "GITLAB_NAMESPACE", "" + ) + or "" + ).strip() or "gitlab" ) env["GITLAB_NAMESPACE"] = ns diff --git a/knoe/core/ops/monitoring.py b/knoe/core/ops/monitoring.py index 5c7c7c5..71b0755 100644 --- a/knoe/core/ops/monitoring.py +++ b/knoe/core/ops/monitoring.py @@ -1,5 +1,8 @@ from __future__ import annotations +import os +import tempfile + from ._services_common import _LogFn, _detect_mode, _helm, _kubectl, _log, _namespace, _to_bool @@ -54,25 +57,57 @@ def update( _kubectl(["create", "namespace", ns], env=env, timeout=60) - _log(log, f"[MONITORING] Deploying {release} in namespace {ns}") - _helm( - [ - "upgrade", - "--install", - release, - chart, - "--namespace", - ns, - "--set", - "grafana.adminUser=admin", - "--set", - f"grafana.adminPassword={grafana_password}", - "--wait", - ], - env=env, - timeout=600, - check=True, + # Nodes with broken kubelet (e.g. pi.prole.org returning 502) must be excluded + # from the node-exporter DaemonSet so helm --wait can succeed. + excluded_nodes = str((env or {}).get("MONITORING_NODE_EXPORTER_EXCLUDE_NODES") or "pi.prole.org") + excluded_list = [n.strip() for n in excluded_nodes.split(",") if n.strip()] + + values_yaml = "prometheus-node-exporter:\n" + if excluded_list: + values_yaml += ( + " affinity:\n" + " nodeAffinity:\n" + " requiredDuringSchedulingIgnoredDuringExecution:\n" + " nodeSelectorTerms:\n" + " - matchExpressions:\n" + " - key: kubernetes.io/hostname\n" + " operator: NotIn\n" + " values:\n" + ) + for node in excluded_list: + values_yaml += f" - {node}\n" + + tmp_values = tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", prefix="monitoring-values-", delete=False ) + try: + tmp_values.write(values_yaml) + tmp_values.flush() + tmp_values.close() + + _log(log, f"[MONITORING] Deploying {release} in namespace {ns}") + _helm( + [ + "upgrade", + "--install", + release, + chart, + "--namespace", + ns, + "--set", + "grafana.adminUser=admin", + "--set", + f"grafana.adminPassword={grafana_password}", + "-f", + tmp_values.name, + "--wait", + ], + env=env, + timeout=600, + check=True, + ) + finally: + os.unlink(tmp_values.name) def restart(