diff --git a/conf/port-mapping.cfg b/conf/port-mapping.cfg index 92d8287..986985d 100644 --- a/conf/port-mapping.cfg +++ b/conf/port-mapping.cfg @@ -1,11 +1,3 @@ # Port mappings for Prole Tools (generated). # Format: key: local=... remote=... ns=... svc=... address=... -argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0 -garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0 -openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0 -opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0 -dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1 -postgres: local=5432 remote=5432 ns=knoey-db-0 svc=prole-db-rw address=0.0.0.0 -prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1 -grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0 diff --git a/conf/prole.cfg b/conf/prole.cfg index fabc351..e06b123 100644 --- a/conf/prole.cfg +++ b/conf/prole.cfg @@ -96,7 +96,7 @@ PROLE_DB_USER = root PROLE_HOME = ${PROLE_HOME} PROLE_K3S_SERVER = https://myrddin.prole.org:6443 PROLE_K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token} -PROLE_OPENTOFU_URL = http://127.0.0.1:8080 +PROLE_OPENTOFU_URL = http://myrddin.prole.org:8080 REGISTRY_NAMESPACE = default SERVICE_NAMESPACE = knoe-system @@ -175,18 +175,18 @@ DISPLAY_NAME = prole-service-cluster K3S_SERVER_URL = https://myrddin.prole.org:6443 K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token} MODE = k3s -PIPELINE_URL = http://127.0.0.1:8080 +PIPELINE_URL = http://myrddin.prole.org:8080 [Prod Cluster (k8s)] ARTIFACTS_DIR = ${PROLE_DATA}/staging CLUSTER_ENV = prole-prod-cluster DISPLAY_NAME = prole-prod-cluster MODE = k8s -PIPELINE_URL = http://127.0.0.1:8080 +PIPELINE_URL = http://myrddin.prole.org:8080 [Docker Build] -LOCAL_REGISTRY = localhost:5000 -LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000 +LOCAL_REGISTRY = myrddin.prole.org:5000 +LOCAL_REGISTRY_INTERNAL = myrddin.prole.org:5000 [Initialization Scripts] STATUS = Attempted diff --git a/deploy/opentofu/k3s/manifests/prole/prole-db.yaml b/deploy/opentofu/k3s/manifests/prole/prole-db.yaml index 3049c11..92fb9e3 100644 --- a/deploy/opentofu/k3s/manifests/prole/prole-db.yaml +++ b/deploy/opentofu/k3s/manifests/prole/prole-db.yaml @@ -4,7 +4,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: k3d-prole-registry:5000/prole-db:17.7-059 + imageName: myrddin.prole.org:5000/prole-db:17.7-059 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 diff --git a/etc/common_core_lib.sh b/etc/common_core_lib.sh index 1cf5b63..b962b63 100644 --- a/etc/common_core_lib.sh +++ b/etc/common_core_lib.sh @@ -118,6 +118,17 @@ common_core_resolve_namespace() { [[ -z "$ns" && -n "${SERVICE_NAMESPACE:-}" ]] && ns="$SERVICE_NAMESPACE" [[ -z "$ns" && -n "${NAMESPACE:-}" ]] && ns="$NAMESPACE" [[ -z "$ns" ]] && ns="$default_ns" + + # In k3s mode, common core services must not default into the `default` namespace. + # If the caller didn't supply an explicit namespace, normalize `default` -> `knoe-system`. + # (Explicit `-n default` is treated as a legacy/mistake and will be corrected.) + local mode="${PROLE_MODE:-}" + if declare -F prole_normalize_mode >/dev/null 2>&1; then + mode="$(prole_normalize_mode "$mode")" + fi + if [[ "$mode" == "k3s" && "$ns" == "default" ]]; then + ns="knoe-system" + fi echo "$ns" } diff --git a/etc/init_registry.sh b/etc/init_registry.sh index e6a10d9..4ede0b8 100755 --- a/etc/init_registry.sh +++ b/etc/init_registry.sh @@ -5,7 +5,7 @@ set -euo pipefail # init_registry.sh # Purpose: # - Deploy ArgoCD into Kubernetes -# - Deploy registry:2 into k3s clusters (default namespace) when enabled +# - Deploy registry:2 into k3s clusters (service namespace) when enabled SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -63,7 +63,7 @@ usage() { Usage: init_registry.sh [-n|--namespace NS] [-r|--registry-namespace NS] Deploys ArgoCD into the target namespace and, for k3s, deploys registry:2 -into the registry namespace (default: default). +into the registry namespace (default: SERVICE_NAMESPACE / knoe-system). Note: The local Docker registry (port 5000) is managed separately for k3d. EOF } @@ -144,6 +144,17 @@ else REGISTRY_NAMESPACE="default" fi +# In k3s mode, the in-cluster registry is a common core service and should live +# in the service namespace (not `default`). +_mode_resolved="${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" +if declare -F prole_normalize_mode >/dev/null 2>&1; then + _mode_resolved="$(prole_normalize_mode "$_mode_resolved")" +fi +if [[ "$_mode_resolved" == "k3s" && ( -z "${REGISTRY_NAMESPACE:-}" || "${REGISTRY_NAMESPACE}" == "default" ) ]]; then + REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-knoe-system}" +fi +unset _mode_resolved + export ARGOCD_NAMESPACE export REGISTRY_NAMESPACE export NAMESPACE="$ARGOCD_NAMESPACE" diff --git a/etc/repair_pipeline.sh b/etc/repair_pipeline.sh index ffb7ac5..330f1b9 100755 --- a/etc/repair_pipeline.sh +++ b/etc/repair_pipeline.sh @@ -72,6 +72,16 @@ else SERVICE_NAMESPACE="default" fi +# In k3s mode, common core services must live in knoe-system by default. +_mode_resolved="${PROLE_MODE:-}" +if declare -F prole_normalize_mode >/dev/null 2>&1; then + _mode_resolved="$(prole_normalize_mode "$_mode_resolved")" +fi +if [[ "$_mode_resolved" == "k3s" && ( -z "${SERVICE_NAMESPACE:-}" || "${SERVICE_NAMESPACE}" == "default" ) ]]; then + SERVICE_NAMESPACE="knoe-system" +fi +unset _mode_resolved + if [[ -n "$DB_NAMESPACE_OVERRIDE" ]]; then DB_NAMESPACE="$DB_NAMESPACE_OVERRIDE" elif [[ -n "${PROLE_DB_NAMESPACE:-}" ]]; then @@ -234,6 +244,8 @@ repair_dashboard() { repair_argocd() { local ns="$1" + local reg_ns + reg_ns="${REGISTRY_NAMESPACE:-$SERVICE_NAMESPACE}" local need_fix=0 local items=( "deploy argocd-server" @@ -256,15 +268,115 @@ repair_argocd() { done if [[ "$need_fix" -eq 1 ]]; then warn "ArgoCD not ready; re-deploying" - REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-default}" \ - "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "${REGISTRY_NAMESPACE:-default}" stop || true - REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-default}" \ - "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "${REGISTRY_NAMESPACE:-default}" update || true + REGISTRY_NAMESPACE="$reg_ns" \ + "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "$reg_ns" stop || true + REGISTRY_NAMESPACE="$reg_ns" \ + "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "$reg_ns" update || true else log "ArgoCD: OK" fi } +delete_if_exists() { + local ns="$1" kind="$2" name="$3" + kubectl -n "$ns" delete "$kind" "$name" --ignore-not-found --wait=false >/dev/null 2>&1 || true +} + +delete_pods_by_label() { + local ns="$1" selector="$2" + kubectl -n "$ns" delete pod -l "$selector" --ignore-not-found --wait=false >/dev/null 2>&1 || true +} + +has_openbao() { + local ns="$1" + resource_exists deploy openbao "$ns" || resource_exists statefulset openbao "$ns" +} + +has_opentofu() { + local ns="$1" + resource_exists deploy opentofu "$ns" +} + +has_garage() { + local ns="$1" + resource_exists statefulset garage "$ns" +} + +has_registry() { + local ns="$1" + resource_exists deploy registry "$ns" +} + +dedupe_common_core_default_namespace() { + # The only known/expected accidental duplicate location is `default`. + local mode + mode="$(detect_mode)" + if [[ "$mode" != "k3s" ]]; then + return 0 + fi + + local canonical="${SERVICE_NAMESPACE:-knoe-system}" + if [[ "$canonical" == "default" ]]; then + canonical="knoe-system" + fi + local legacy="default" + if [[ "$canonical" == "$legacy" ]]; then + return 0 + fi + + if ! kubectl get ns "$legacy" >/dev/null 2>&1; then + return 0 + fi + + # OpenBao + if has_openbao "$legacy"; then + warn "Detected OpenBao in legacy namespace '$legacy'" + if ! has_openbao "$canonical"; then + warn "Canonical OpenBao missing in '$canonical'; deploying before cleanup" + "$SCRIPT_DIR/init_openbao.sh" -n "$canonical" update || true + fi + delete_pods_by_label "$legacy" "app=openbao" + delete_if_exists "$legacy" deploy openbao + delete_if_exists "$legacy" statefulset openbao + delete_if_exists "$legacy" svc openbao + delete_if_exists "$legacy" pvc data-openbao-0 + fi + + # OpenTofu + if has_opentofu "$legacy"; then + warn "Detected OpenTofu in legacy namespace '$legacy'" + if ! has_opentofu "$canonical"; then + warn "Canonical OpenTofu missing in '$canonical'; deploying before cleanup" + "$SCRIPT_DIR/init_opentofu.sh" -n "$canonical" update || true + fi + delete_if_exists "$legacy" deploy opentofu + delete_if_exists "$legacy" svc opentofu + fi + + # Garage + if has_garage "$legacy"; then + warn "Detected Garage in legacy namespace '$legacy'" + if ! has_garage "$canonical"; then + warn "Canonical Garage missing in '$canonical'; deploying before cleanup" + "$SCRIPT_DIR/init_garage_store.sh" -n "$canonical" start || true + fi + delete_pods_by_label "$legacy" "app=garage" + delete_if_exists "$legacy" statefulset garage + delete_if_exists "$legacy" svc garage + fi + + # Registry + if has_registry "$legacy"; then + warn "Detected registry in legacy namespace '$legacy'" + if ! has_registry "$canonical"; then + warn "Canonical registry missing in '$canonical'; deploying before cleanup" + REGISTRY_NAMESPACE="$canonical" "$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NAMESPACE" --registry-namespace "$canonical" update || true + fi + delete_if_exists "$legacy" deploy registry + delete_if_exists "$legacy" svc registry + fi +} + repair_openbao() { local ns="$1" local status @@ -392,6 +504,9 @@ case "${ACTION}" in log "-- Dashboard (Kong) --" repair_dashboard || true + log "-- Dedupe common core (k3s) --" + dedupe_common_core_default_namespace || true + log "-- ArgoCD --" repair_argocd "$ARGOCD_NAMESPACE" @@ -404,6 +519,10 @@ case "${ACTION}" in log "-- Garage --" repair_garage "$SERVICE_NAMESPACE" + # One more pass after repairs in case the canonical deployments were missing + # and we needed to deploy them first. + dedupe_common_core_default_namespace || true + if [[ "${KERBEROS_ENABLED:-}" == "0" || "${KERBEROS_ENABLED:-}" == "false" || "${KERBEROS_ENABLED:-}" == "False" ]]; then log "KDC (auth): skipped (Kerberos disabled)" elif [[ "${PROLE_KDC_ENABLED:-1}" != "0" ]]; then diff --git a/infrastructure/inventory/host_vars/myrddin.prole.org.yml b/infrastructure/inventory/host_vars/myrddin.prole.org.yml index ecdbfce..95ff0c9 100644 --- a/infrastructure/inventory/host_vars/myrddin.prole.org.yml +++ b/infrastructure/inventory/host_vars/myrddin.prole.org.yml @@ -70,6 +70,13 @@ k3s_enabled: true k3s_cluster_init: false k3s_role: server k3s_token: "{{ vault_k3s_token | default('') }}" # store this in vault +k3s_datastore_mariadb_host: synology.prole.org +k3s_datastore_mariadb_port: 3306 +k3s_datastore_mariadb_db: k3s +k3s_datastore_mariadb_user: prole_k3s +# Password is the same as the Samba administrator password and must remain in Ansible Vault. +k3s_datastore_mariadb_password: "{{ vault_samba_dns_admin_pass }}" +k3s_datastore_endpoint: "mysql://{{ k3s_datastore_mariadb_user }}:{{ k3s_datastore_mariadb_password | urlencode }}@tcp({{ k3s_datastore_mariadb_host }}:{{ k3s_datastore_mariadb_port }})/{{ k3s_datastore_mariadb_db }}" k3s_tls_sans: - myrddin.prole.org k3s_node_labels: @@ -90,6 +97,11 @@ k3s_kube_scheduler_args: - "leader-elect-renew-deadline=100s" - "leader-elect-retry-period=20s" +k3s_cloud_controller_manager_args: + - "leader-elect-lease-duration=300s" + - "leader-elect-renew-deadline=200s" + - "leader-elect-retry-period=40s" + k3s_kubelet_args: - "housekeeping-interval=10s" diff --git a/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml b/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml new file mode 100644 index 0000000..af0460e --- /dev/null +++ b/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml @@ -0,0 +1,102 @@ +--- +- name: Prepare MariaDB datastore for k3s (create DB/user/grants) + hosts: k3s_hosts + gather_facts: false + become: false + serial: 1 + vars: + k3s_mariadb_host: "{{ k3s_datastore_mariadb_host | default('synology.prole.org') }}" + k3s_mariadb_port: "{{ k3s_datastore_mariadb_port | default(3306) }}" + k3s_mariadb_db: "{{ k3s_datastore_mariadb_db | default('k3s') }}" + k3s_mariadb_user: "{{ k3s_datastore_mariadb_user | default('prole_k3s') }}" + k3s_mariadb_user_password: "{{ k3s_datastore_mariadb_password | default('') }}" + + # MariaDB admin credentials (must be stored in Ansible Vault by the operator) + k3s_mariadb_admin_user: "{{ k3s_mariadb_admin_user | default('root') }}" + k3s_mariadb_admin_password: "{{ k3s_mariadb_admin_password | default('') }}" + + # Where to allow the k3s user to connect from (tighten this if possible) + k3s_mariadb_user_host: "{{ k3s_mariadb_user_host | default('%') }}" + + pre_tasks: + - name: Only run on k3s server nodes + ansible.builtin.meta: end_host + when: k3s_role | default('') != 'server' + + - name: Require k3s MariaDB user password (vaulted) + ansible.builtin.assert: + that: + - k3s_mariadb_user_password | length > 0 + fail_msg: >- + k3s_mariadb_user_password is empty. Ensure k3s_datastore_mariadb_password is set + (and sourced from Ansible Vault, e.g. vault_samba_dns_admin_pass). + + - name: Require MariaDB admin password for non-check runs + ansible.builtin.assert: + that: + - k3s_mariadb_admin_password | length > 0 + fail_msg: >- + k3s_mariadb_admin_password is empty. + Provide a vaulted MariaDB admin password (e.g. in an encrypted group_vars/*/vault.yml). + when: not ansible_check_mode + + - name: Verify mysql client is available on this host + ansible.builtin.command: mysql --version + changed_when: false + + tasks: + - name: Create k3s database (if missing) + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="CREATE DATABASE IF NOT EXISTS `{{ k3s_mariadb_db }}`" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true + + - name: Create/ensure k3s MariaDB user exists + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="CREATE USER IF NOT EXISTS '{{ k3s_mariadb_user }}'@'{{ k3s_mariadb_user_host }}' IDENTIFIED BY '{{ k3s_mariadb_user_password }}'" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true + + - name: Grant admin privileges on k3s database to k3s MariaDB user + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="GRANT ALL PRIVILEGES ON `{{ k3s_mariadb_db }}`.* TO '{{ k3s_mariadb_user }}'@'{{ k3s_mariadb_user_host }}'" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true + + - name: Flush privileges + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="FLUSH PRIVILEGES" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true diff --git a/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml b/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml new file mode 100644 index 0000000..0e0a1c9 --- /dev/null +++ b/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml @@ -0,0 +1,54 @@ +--- +- name: Roll out k3s config for external MariaDB datastore (servers) + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Require explicit acknowledgement (this changes the k3s datastore) + ansible.builtin.assert: + that: + - k3s_datastore_migration_ack | default(false) | bool + fail_msg: >- + Refusing to proceed without k3s_datastore_migration_ack=true. + Switching k3s datastore backends may require a rebuild/migration plan. + + - name: Only run on k3s server nodes + ansible.builtin.meta: end_host + when: k3s_role | default('') != 'server' + + - name: Require k3s_datastore_endpoint for MariaDB datastore + ansible.builtin.assert: + that: + - (k3s_datastore_endpoint | default('')) | length > 0 + fail_msg: "k3s_datastore_endpoint is empty on {{ inventory_hostname }}" + + tasks: + - name: Apply k3s install/config changes (server) + ansible.builtin.import_role: + name: k3s + tasks_from: install + +- name: Roll out k3s config for external MariaDB datastore (agents) + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Require explicit acknowledgement (this changes the k3s datastore) + ansible.builtin.assert: + that: + - k3s_datastore_migration_ack | default(false) | bool + fail_msg: >- + Refusing to proceed without k3s_datastore_migration_ack=true. + Switching k3s datastore backends may require a rebuild/migration plan. + + - name: Only run on k3s agent nodes + ansible.builtin.meta: end_host + when: k3s_role | default('') != 'agent' + + tasks: + - name: Apply k3s install/config changes (agent) + ansible.builtin.import_role: + name: k3s + tasks_from: install diff --git a/infrastructure/playbooks/k3s_server_refresh.yml b/infrastructure/playbooks/k3s_server_refresh.yml new file mode 100644 index 0000000..3fe6a79 --- /dev/null +++ b/infrastructure/playbooks/k3s_server_refresh.yml @@ -0,0 +1,19 @@ +--- +- name: Refresh k3s server (blank slate except token/CA) + hosts: k3s_hosts + become: true + serial: 1 + vars: + # Safety guard: you must also pass `-e k3s_refresh_confirm=YES`. + # Example: + # ./ansible.sh -v .vault_pass -p infrastructure/playbooks/k3s_server_refresh.yml -l myrddin.prole.org -- -e k3s_refresh_confirm=YES + k3s_refresh_run_configure: true + tasks: + - name: Ensure cgroup kernel params are set + ansible.builtin.import_role: + name: cgroups + + - name: Refresh k3s server (preserve token/CA) + ansible.builtin.import_role: + name: k3s + tasks_from: refresh_server diff --git a/infrastructure/roles/k3s/defaults/main.yml b/infrastructure/roles/k3s/defaults/main.yml index 0513a04..43adf41 100644 --- a/infrastructure/roles/k3s/defaults/main.yml +++ b/infrastructure/roles/k3s/defaults/main.yml @@ -4,6 +4,13 @@ k3s_state: present k3s_version: "" k3s_guard_token_drift: true +# Destructive repair/refresh workflow (server only) +# A refresh wipes the k3s installation/state to a blank slate but preserves the +# join token and CA material so existing agents can rejoin without changing the token. +k3s_refresh_backup_root: /root/k3s-refresh-backups +k3s_refresh_confirm: "" +k3s_refresh_run_configure: true + # CloudNative-PG # Pin operator/plugin version by default to avoid unanticipated upgrades and load spikes. # Upgrades must be explicitly enabled via `k3s_cnpg_upgrade: true`. @@ -14,6 +21,9 @@ k3s_role: agent k3s_cluster_init: false k3s_server_url: "" k3s_token: "{{ vault_k3s_token | default('') }}" +# Optional: if `k3s_token` is a full node-token (`K10...::server:...`), set this +# to the raw secret (`...` after `::server:`) for the server config. +k3s_server_token: "" k3s_disable: [] k3s_tls_sans: [] k3s_node_labels: [] diff --git a/infrastructure/roles/k3s/tasks/cleanup.yml b/infrastructure/roles/k3s/tasks/cleanup.yml index 91243e9..3c360fd 100644 --- a/infrastructure/roles/k3s/tasks/cleanup.yml +++ b/infrastructure/roles/k3s/tasks/cleanup.yml @@ -1,4 +1,9 @@ --- + +- name: Set effective k3s data-dir + ansible.builtin.set_fact: + k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}" + - name: Best-effort purge kube-system workloads before uninstall ansible.builtin.command: "kubectl -n kube-system delete all --all --ignore-not-found" failed_when: false @@ -51,15 +56,15 @@ - /etc/rancher/k3s - /etc/cni - /opt/cni - - /var/lib/rancher/k3s - - /var/lib/rancher/k3s/server/tls - - /var/lib/rancher/k3s/server/cred - - /var/lib/rancher/k3s/server/token - - /var/lib/rancher/k3s/server/node-token - - /var/lib/rancher/k3s/server/agent-token - - /var/lib/rancher/k3s/server/db - - /var/lib/rancher/k3s/agent - - /var/lib/rancher/k3s/data + - "{{ k3s_effective_data_dir }}" + - "{{ k3s_effective_data_dir }}/server/tls" + - "{{ k3s_effective_data_dir }}/server/cred" + - "{{ k3s_effective_data_dir }}/server/token" + - "{{ k3s_effective_data_dir }}/server/node-token" + - "{{ k3s_effective_data_dir }}/server/agent-token" + - "{{ k3s_effective_data_dir }}/server/db" + - "{{ k3s_effective_data_dir }}/agent" + - "{{ k3s_effective_data_dir }}/data" - /var/lib/kubelet - /var/lib/etcd - /var/lib/cni diff --git a/infrastructure/roles/k3s/tasks/install.yml b/infrastructure/roles/k3s/tasks/install.yml index ef06189..45df0ae 100644 --- a/infrastructure/roles/k3s/tasks/install.yml +++ b/infrastructure/roles/k3s/tasks/install.yml @@ -8,6 +8,18 @@ ansible.builtin.set_fact: k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}" +- name: Derive server token secret from node-token (server only) + ansible.builtin.set_fact: + k3s_server_token: "{{ k3s_token | regex_replace('^K10[0-9a-f]+::server:', '') }}" + when: + - k3s_role == 'server' + - (k3s_token | default('')) is match('^K10[0-9a-f]+::server:.+') + no_log: true + +- name: Set effective k3s data-dir + ansible.builtin.set_fact: + k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}" + - name: Check if k3s service is installed ansible.builtin.stat: path: "/etc/systemd/system/{{ k3s_service_name }}.service" @@ -162,7 +174,8 @@ - (['cgroup_memory=1', 'cgroup_enable=memory'] | reject('in', k3s_proc_cmdline.stdout | default('')) | list) | length > 0 - name: Guardrail - ensure rancher mountpoint is not on SD/rootfs - ansible.builtin.command: "findmnt -n -o SOURCE {{ k3s_rancher_mountpoint | default('/var/lib/rancher') }}" + ansible.builtin.shell: | + awk -v mp="{{ k3s_rancher_mountpoint | default('/var/lib/rancher') }}" '$5==mp {for(i=1;i<=NF;i++) if($i=="-") {print $(i+2); exit}}' /proc/self/mountinfo register: rancher_source changed_when: false check_mode: no @@ -175,7 +188,8 @@ - (rancher_source.stdout | default('')) is search("mmcblk0") or (rancher_source.stdout | default('')) is search("/dev/mmc") - name: Preflight - ensure required mountpoints are mounted - ansible.builtin.command: "findmnt -n {{ item }}" + ansible.builtin.shell: | + awk -v mp="{{ item }}" '$5==mp {found=1} END {exit(found?0:1)}' /proc/self/mountinfo register: k3s_required_mounts_present changed_when: false failed_when: k3s_required_mounts_present.rc != 0 @@ -184,7 +198,8 @@ when: (k3s_required_mounts | default([])) | length > 0 - name: Guardrail - ensure required mountpoints are not on SD - ansible.builtin.command: "findmnt -n -o SOURCE {{ item }}" + ansible.builtin.shell: | + awk -v mp="{{ item }}" '$5==mp {for(i=1;i<=NF;i++) if($i=="-") {print $(i+2); exit}}' /proc/self/mountinfo register: k3s_required_mounts_sources changed_when: false check_mode: no @@ -201,7 +216,7 @@ - name: Read k3s node token for sharing ansible.builtin.slurp: - src: /var/lib/rancher/k3s/server/node-token + src: "{{ k3s_effective_data_dir }}/server/node-token" register: k3s_node_token_slurp when: - k3s_role == "server" @@ -250,8 +265,8 @@ block: - name: Get local agent CA cert hash (DER) ansible.builtin.shell: | - if [ -f /var/lib/rancher/k3s/agent/server-ca.crt ]; then - openssl x509 -in /var/lib/rancher/k3s/agent/server-ca.crt -outform DER | openssl dgst -sha256 | awk '{print $NF}' + if [ -f {{ k3s_effective_data_dir }}/agent/server-ca.crt ]; then + openssl x509 -in {{ k3s_effective_data_dir }}/agent/server-ca.crt -outform DER | openssl dgst -sha256 | awk '{print $NF}' else echo "MISSING" fi @@ -285,7 +300,7 @@ - name: Wipe stale agent state ansible.builtin.file: - path: /var/lib/rancher/k3s/agent + path: "{{ k3s_effective_data_dir }}/agent" state: absent - name: Force k3s re-installation after CA wipe @@ -309,7 +324,7 @@ - name: Check for existing k3s server node token ansible.builtin.stat: - path: /var/lib/rancher/k3s/server/node-token + path: "{{ k3s_effective_data_dir }}/server/node-token" register: k3s_server_node_token when: - k3s_guard_token_drift | bool @@ -318,7 +333,7 @@ - name: Read existing k3s server node token ansible.builtin.slurp: - src: /var/lib/rancher/k3s/server/node-token + src: "{{ k3s_effective_data_dir }}/server/node-token" register: k3s_server_node_token_raw when: - k3s_guard_token_drift | bool @@ -329,7 +344,7 @@ - name: Fail when vault k3s token is missing for existing cluster ansible.builtin.fail: msg: >- - vault_k3s_token is empty but /var/lib/rancher/k3s/server/node-token exists. + vault_k3s_token is empty but {{ k3s_effective_data_dir }}/server/node-token exists. Refusing to overwrite k3s config. Update vault_k3s.yml or run the k3s sync playbook. when: - k3s_guard_token_drift | bool @@ -504,23 +519,31 @@ changed_when: "'modified' in _taint_result.stdout or 'tainted' in _taint_result.stdout" failed_when: false # Avoid failing if node is not yet ready -- name: Wait for k3s to be ready - ansible.builtin.shell: k3s kubectl get nodes | grep -w "Ready" - register: k3s_ready - until: k3s_ready.rc == 0 - retries: 20 +- name: Wait for k3s API readiness + ansible.builtin.command: k3s kubectl get --raw=/readyz + register: k3s_readyz + until: + - k3s_readyz.rc == 0 + - (k3s_readyz.stdout | default('') | trim) in ['ok', 'OK'] + retries: 60 delay: 5 changed_when: false when: k3s_state == "present" and k3s_role == "server" -- name: Pause to allow k3s API to stabilize - ansible.builtin.pause: - seconds: 10 - when: k3s_state == "present" and k3s_role == "server" and k3s_ready.changed | default(true) +- name: Wait for local node to report Ready + ansible.builtin.shell: | + k3s kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true + register: k3s_node_ready + until: (k3s_node_ready.stdout | default('') | trim) == 'True' + retries: 60 + delay: 5 + changed_when: false + failed_when: false + when: k3s_state == "present" and k3s_role == "server" - name: Read k3s node token after install for sharing ansible.builtin.slurp: - src: /var/lib/rancher/k3s/server/node-token + src: "{{ k3s_effective_data_dir }}/server/node-token" register: k3s_node_token_slurp_post when: - k3s_role == "server" diff --git a/infrastructure/roles/k3s/tasks/refresh_server.yml b/infrastructure/roles/k3s/tasks/refresh_server.yml new file mode 100644 index 0000000..9034ccd --- /dev/null +++ b/infrastructure/roles/k3s/tasks/refresh_server.yml @@ -0,0 +1,195 @@ +--- + +- name: Guard - require explicit confirmation for k3s server refresh + ansible.builtin.assert: + that: + - k3s_role == 'server' + - k3s_state == 'present' + - (k3s_refresh_confirm | default('')) == 'YES' + fail_msg: >- + Refusing to refresh k3s server without explicit confirmation. + Set k3s_refresh_confirm=YES and re-run (this is destructive: it wipes k3s state). + +- name: Read current k3s config (to detect current data-dir) + ansible.builtin.stat: + path: /etc/rancher/k3s/config.yaml + register: _k3s_refresh_config_stat + +- name: Slurp current k3s config + ansible.builtin.slurp: + src: /etc/rancher/k3s/config.yaml + register: _k3s_refresh_config_slurp + when: _k3s_refresh_config_stat.stat.exists + +- name: Set refresh source/target data-dir + ansible.builtin.set_fact: + _k3s_refresh_cfg: >- + {{ + (_k3s_refresh_config_slurp.content | b64decode | from_yaml) + if (_k3s_refresh_config_stat.stat.exists | bool) + else {} + }} + k3s_refresh_source_data_dir: >- + {{ + _k3s_refresh_cfg.get('data-dir') + | default((k3s_data_dir | default('/var/lib/rancher/k3s')), true) + }} + k3s_refresh_target_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}" + +- name: Preflight - check external datastore TCP reachability (best-effort) + ansible.builtin.wait_for: + host: "{{ k3s_datastore_mariadb_host }}" + port: "{{ k3s_datastore_mariadb_port | default(3306) }}" + connect_timeout: 2 + timeout: 5 + state: started + when: + - (k3s_datastore_endpoint | default('') | length) > 0 + - k3s_datastore_mariadb_host is defined + changed_when: false + failed_when: false + +- name: Create refresh backup directory + ansible.builtin.file: + path: "{{ k3s_refresh_backup_root }}/{{ inventory_hostname }}/{{ ansible_date_time.iso8601_basic_short }}" + state: directory + mode: "0700" + register: _k3s_refresh_backup_dir + +- name: Stat k3s token/tls artifacts (source data-dir) + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ k3s_refresh_source_data_dir }}/server/token" + - "{{ k3s_refresh_source_data_dir }}/server/node-token" + # Minimal TLS set to preserve agent trust (CA hash) while allowing k3s to + # regenerate component creds and `server/cred/*.kubeconfig`. + - "{{ k3s_refresh_source_data_dir }}/server/tls/server-ca.crt" + - "{{ k3s_refresh_source_data_dir }}/server/tls/server-ca.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/client-ca.crt" + - "{{ k3s_refresh_source_data_dir }}/server/tls/client-ca.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/request-header-ca.crt" + - "{{ k3s_refresh_source_data_dir }}/server/tls/request-header-ca.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/service.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/service.current.key" + register: _k3s_refresh_artifacts + +- name: Back up k3s token/tls artifacts (remote->remote) + ansible.builtin.copy: + remote_src: true + src: "{{ item.item }}" + dest: "{{ _k3s_refresh_backup_dir.path }}/" + mode: preserve + loop: "{{ _k3s_refresh_artifacts.results }}" + when: item.stat.exists + +- name: Read preserved server token (source data-dir) + ansible.builtin.slurp: + src: "{{ k3s_refresh_source_data_dir }}/server/token" + register: _k3s_refresh_token_slurp + when: >- + {{ + ( + _k3s_refresh_artifacts.results + | selectattr('item', 'equalto', (k3s_refresh_source_data_dir ~ '/server/token')) + | map(attribute='stat.exists') + | first + ) + | default(false) + }} + no_log: true + +- name: Ensure we have a token to preserve + ansible.builtin.assert: + that: + - (k3s_token | default('') | length) > 0 or (_k3s_refresh_token_slurp is defined) + fail_msg: >- + No existing token file was found under {{ k3s_refresh_source_data_dir }}/server/token, + and k3s_token is empty. Refusing to proceed because this would change the join token. + +- name: Force k3s_token to preserved token (so rendered config matches what we restore) + ansible.builtin.set_fact: + k3s_token: "{{ (_k3s_refresh_token_slurp.content | b64decode | trim) if (_k3s_refresh_token_slurp is defined) else k3s_token }}" + no_log: true + +- name: Stop k3s services before refresh + ansible.builtin.service: + name: "{{ item }}" + state: stopped + enabled: false + loop: + - k3s + - k3s-agent + failed_when: false + +- name: Uninstall and wipe k3s state (blank slate) + ansible.builtin.import_tasks: cleanup.yml + +- name: Wipe target k3s data-dir (blank slate except restored token/CA) + ansible.builtin.file: + path: "{{ k3s_refresh_target_data_dir }}" + state: absent + +- name: Recreate target server directory + ansible.builtin.file: + path: "{{ k3s_refresh_target_data_dir }}/server" + state: directory + mode: "0700" + +- name: Recreate target tls directory + ansible.builtin.file: + path: "{{ k3s_refresh_target_data_dir }}/server/tls" + state: directory + mode: "0700" + +- name: Restore preserved minimal tls artifacts into target data-dir + ansible.builtin.copy: + remote_src: true + src: "{{ item }}" + dest: "{{ k3s_refresh_target_data_dir }}/server/tls/" + mode: preserve + loop: + - "{{ _k3s_refresh_backup_dir.path }}/server-ca.crt" + - "{{ _k3s_refresh_backup_dir.path }}/server-ca.key" + - "{{ _k3s_refresh_backup_dir.path }}/client-ca.crt" + - "{{ _k3s_refresh_backup_dir.path }}/client-ca.key" + - "{{ _k3s_refresh_backup_dir.path }}/request-header-ca.crt" + - "{{ _k3s_refresh_backup_dir.path }}/request-header-ca.key" + - "{{ _k3s_refresh_backup_dir.path }}/service.key" + - "{{ _k3s_refresh_backup_dir.path }}/service.current.key" + failed_when: false + +- name: Install and start k3s + ansible.builtin.import_tasks: install.yml + +- name: Configure k3s add-ons + ansible.builtin.import_tasks: configure.yml + when: k3s_refresh_run_configure | bool + +- name: Verify k3s API is responsive after refresh + ansible.builtin.command: k3s kubectl get nodes + register: _k3s_refresh_nodes + changed_when: false + until: _k3s_refresh_nodes.rc == 0 + retries: 18 + delay: 10 + +- name: Read k3s node-token after refresh + ansible.builtin.slurp: + src: "{{ k3s_refresh_target_data_dir }}/server/node-token" + register: _k3s_refresh_node_token + changed_when: false + until: _k3s_refresh_node_token is succeeded + retries: 18 + delay: 10 + no_log: true + +- name: Assert node-token is preserved (when token is in node-token format) + ansible.builtin.assert: + that: + - (k3s_token | default('')) == (_k3s_refresh_node_token.content | b64decode | trim) + fail_msg: >- + Refusing to continue: expected regenerated node-token to match preserved k3s_token, + but it differs. This indicates the join token changed. + when: + - (k3s_token | default('')) is match('^K10[0-9a-f]+::server:.+') diff --git a/infrastructure/roles/k3s/tasks/sync.yml b/infrastructure/roles/k3s/tasks/sync.yml index cbbb5c6..aa39a9f 100644 --- a/infrastructure/roles/k3s/tasks/sync.yml +++ b/infrastructure/roles/k3s/tasks/sync.yml @@ -1,4 +1,9 @@ --- + +- name: Set effective k3s data-dir + ansible.builtin.set_fact: + k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}" + - name: Ensure k3s config exists ansible.builtin.stat: path: /etc/rancher/k3s/config.yaml @@ -87,21 +92,21 @@ - name: Ensure k3s server directory exists ansible.builtin.file: - path: /var/lib/rancher/k3s/server + path: "{{ k3s_effective_data_dir }}/server" state: directory mode: "0755" when: k3s_sync_tls_bundle_present | default(false) - name: Remove existing k3s tls directory before sync ansible.builtin.file: - path: /var/lib/rancher/k3s/server/tls + path: "{{ k3s_effective_data_dir }}/server/tls" state: absent when: k3s_sync_tls_bundle_present | default(false) - name: Restore k3s tls bundle ansible.builtin.unarchive: src: "{{ k3s_sync_tls_bundle }}" - dest: /var/lib/rancher/k3s/server + dest: "{{ k3s_effective_data_dir }}/server" owner: root group: root when: k3s_sync_tls_bundle_present | default(false) diff --git a/infrastructure/roles/k3s/tasks/validate_args.yml b/infrastructure/roles/k3s/tasks/validate_args.yml index a82e49b..dac8bc4 100644 --- a/infrastructure/roles/k3s/tasks/validate_args.yml +++ b/infrastructure/roles/k3s/tasks/validate_args.yml @@ -15,3 +15,12 @@ | select('search', '^etcd-timeout=') | list }} + +- name: Validate datastore configuration (embedded etcd vs external datastore) + ansible.builtin.assert: + that: + - not ((k3s_cluster_init | default(false) | bool) and ((k3s_datastore_endpoint | default('')) | length > 0)) + fail_msg: >- + Invalid k3s datastore config: k3s_cluster_init=true cannot be used with an external datastore. + Set k3s_cluster_init: false (or unset k3s_datastore_endpoint). + when: k3s_role == 'server' diff --git a/infrastructure/roles/k3s/templates/config.yaml.j2 b/infrastructure/roles/k3s/templates/config.yaml.j2 index d634b3f..de618b6 100644 --- a/infrastructure/roles/k3s/templates/config.yaml.j2 +++ b/infrastructure/roles/k3s/templates/config.yaml.j2 @@ -1,14 +1,28 @@ -{% if k3s_role == 'server' %} -{% if k3s_cluster_init | bool %} +{% set _k3s_server_line_needed = (k3s_role != 'server') or ((k3s_role == 'server') and (not (k3s_cluster_init | bool)) and (k3s_server_url is defined) and k3s_server_url) %} +{% if k3s_role == 'server' and k3s_cluster_init | bool %} cluster-init: true -{% elif k3s_server_url %} +{% endif %} +{% if _k3s_server_line_needed %} server: "{{ k3s_server_url }}" {% endif %} -{% else %} -server: "{{ k3s_server_url }}" +{% set _k3s_effective_token = k3s_token %} +{% if k3s_role == 'server' and (k3s_server_token is defined) and k3s_server_token %} +{% set _k3s_effective_token = k3s_server_token %} +{% endif %} +{% if _k3s_effective_token %} +token: "{{ _k3s_effective_token }}" +{% endif %} +{% if k3s_role == 'server' and k3s_datastore_endpoint is defined and k3s_datastore_endpoint | length %} +datastore-endpoint: "{{ k3s_datastore_endpoint }}" +{% if k3s_datastore_cafile is defined and k3s_datastore_cafile | length %} +datastore-cafile: "{{ k3s_datastore_cafile }}" +{% endif %} +{% if k3s_datastore_certfile is defined and k3s_datastore_certfile | length %} +datastore-certfile: "{{ k3s_datastore_certfile }}" +{% endif %} +{% if k3s_datastore_keyfile is defined and k3s_datastore_keyfile | length %} +datastore-keyfile: "{{ k3s_datastore_keyfile }}" {% endif %} -{% if k3s_token %} -token: "{{ k3s_token }}" {% endif %} {% if k3s_role == 'server' and k3s_write_kubeconfig_mode %} write-kubeconfig-mode: "{{ k3s_write_kubeconfig_mode }}" diff --git a/installer/core/actions.py b/installer/core/actions.py index 1dd13e9..cf90bce 100644 --- a/installer/core/actions.py +++ b/installer/core/actions.py @@ -48,6 +48,7 @@ from installer.core.env import ( _deployment_target_label, _default_opentofu_pipeline_url, _format_ollama_host, + _host_from_url, _render_prole_cfg, _k3d_prole_data_volume_args, _sync_opentofu_pipeline, @@ -173,7 +174,18 @@ class ProleInstallerBase: ) if not ns: ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip() - return ns or "default" + + if ns: + return ns + + # In k3s service clusters, core services live in knoe-system by default. + try: + if self._deployment_mode() == "k3s": + return "knoe-system" + except Exception: + pass + + return "default" # ------------------------------------------ k3s connection helpers def _read_k3s_cfg_values( @@ -1831,9 +1843,29 @@ class ProleSilentInstaller(ProleInstallerBase): def _ensure_local_registry_available(self): """Ensure a local k3d registry is running and return (host_registry, cluster_registry).""" - if not _local_registry_enabled(self._get_input("init_cluster.cluster_env", "")): + env_hint = self._get_input("init_cluster.cluster_env", "") + mode = _deployment_mode_from_env(env_hint) + if not _local_registry_enabled(env_hint): self.log("[SKIP] Local registry disabled; skipping registry setup.") return None + + if mode == "k3s": + # In k3s mode we must not use localhost registries or k3d registry containers. + server, _token = _resolve_k3s_connection_fn(cfg_path=self.cfg_path) + host = _host_from_url(server) + if not host: + self.err( + "[WARN] k3s mode but K3S_SERVER_URL/PROLE_K3S_SERVER not set; cannot resolve registry." + ) + return None + host_registry = f"{host}:5000" + cluster_registry = host_registry + self.local_registry_url = host_registry + self.local_registry_internal = cluster_registry + self.prole_cfg_data.setdefault("Docker Build", {})["LOCAL_REGISTRY"] = host_registry + self.prole_cfg_data["Docker Build"]["LOCAL_REGISTRY_INTERNAL"] = cluster_registry + return host_registry, cluster_registry + reg_name = "prole-registry" registry_container = f"k3d-{reg_name}" host_registry = "localhost:5000" @@ -3974,7 +4006,15 @@ class ProleSilentInstaller(ProleInstallerBase): needs_prompt = True if needs_prompt: - new_pw = self._prompt_for_master_password() + # In true silent/non-interactive runs (e.g. CI/tests), prompting will fail. + # Prefer generating a strong password automatically when stdin is not a TTY. + if getattr(sys.__stdin__, "isatty", lambda: False)(): + new_pw = self._prompt_for_master_password() + else: + self.err( + "[WARN] Silent install requires a database master password but stdin is non-interactive; generating one automatically." + ) + new_pw = secrets.token_urlsafe(24) self.inputs["init_password.db_password"] = new_pw self.inputs["init_password.db_password_confirm"] = new_pw # Save immediately to prole.cfg so subsequent steps/scripts see it. diff --git a/installer/core/env.py b/installer/core/env.py index 0dc785d..06fea51 100644 --- a/installer/core/env.py +++ b/installer/core/env.py @@ -614,6 +614,20 @@ def _expand_path(val: str | None) -> str: return os.path.expandvars(os.path.expanduser(str(val))) +def _host_from_url(val: str | None) -> str: + raw = (val or "").strip() + if not raw: + return "" + # urlparse requires a scheme to reliably detect hostnames. + if "://" not in raw: + raw = f"https://{raw}" + try: + parsed = urllib.parse.urlparse(raw) + except Exception: + return "" + return (parsed.hostname or "").strip() + + _CFG_VAR_PATTERN = re.compile(r"\$(\w+)|\$\{(\w+)\}") @@ -874,7 +888,7 @@ def _cluster_env_radio_value(env: str | None) -> str: def _deployment_target_label(env: str | None) -> str: key = _normalize_cluster_env(env) if key == "dev": - return "knoe-dev-cluster" + return "prole-dev-cluster" if key == "service": return "prole-service-cluster" if key == "prod": @@ -1243,6 +1257,30 @@ def _default_opentofu_pipeline_url() -> str: url = ( os.environ.get("PROLE_OPENTOFU_URL") or os.environ.get("OPENTOFU_URL") or "" ).strip() + mode = ( + os.environ.get("PROLE_MODE") + or os.environ.get("DEPLOYMENT_MODE") + or os.environ.get("DEPLOY_MODE") + or "" + ).strip().lower() + + # In k3s mode we must not rely on localhost/port-forwards. + if mode == "k3s": + if url: + host = _host_from_url(url) + if host and host not in ("localhost", "127.0.0.1"): + return url + # If user provided a localhost URL in k3s, ignore it and try to derive. + url = "" + + k3s_server = ( + os.environ.get("PROLE_K3S_SERVER") or os.environ.get("K3S_SERVER_URL") or "" + ).strip() + k3s_host = _host_from_url(k3s_server) + if k3s_host: + return f"http://{k3s_host}:8080" + return "" + return url or "http://127.0.0.1:8080" diff --git a/installer/milestone.py b/installer/milestone.py index c8c8503..79c1265 100644 --- a/installer/milestone.py +++ b/installer/milestone.py @@ -79,8 +79,8 @@ class Milestone(ABC): project_root = inst_config.PROJECT_ROOT env["PROLE_HOME"] = str(project_root) env["PROLE_SERVICE"] = str(project_root) - # Namespace is read from prole.cfg by scripts — never set in subprocess env - namespace = state.inputs.get("init_password.db_namespace", "") + namespace = (state.inputs.get("init_password.db_namespace", "") or "").strip() + env["NAMESPACE"] = namespace or "default" # Paths env["PROLE_CONF"] = state.inputs.get( @@ -104,7 +104,7 @@ class Milestone(ABC): if not service_ns: service_ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip() if not service_ns: - service_ns = namespace or "default" + service_ns = env["NAMESPACE"] env["SERVICE_NAMESPACE"] = service_ns from installer.core.env import _deployment_mode_from_env diff --git a/installer/ui/screens/cluster.py b/installer/ui/screens/cluster.py index 294ce42..d2236d6 100644 --- a/installer/ui/screens/cluster.py +++ b/installer/ui/screens/cluster.py @@ -309,7 +309,7 @@ class ClusterScreenMixin: self, x_label, y, - "Common Services Namespace:", + "Common Core Services Namespace:", fill="black", font=("SF Pro Text", 12, "bold"), ) @@ -523,7 +523,15 @@ class ClusterScreenMixin: ns = "" if not ns: ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip() - return ns or "default" + + default_ns = "default" + try: + if self._cluster_env_key() == "service": + default_ns = "knoe-system" + except Exception: + pass + + return ns or default_ns def _on_cluster_env_change(self, *args): self._set_deploy_target_from_cluster_env() diff --git a/installer/ui/screens/docker.py b/installer/ui/screens/docker.py index 1083890..4a98eac 100644 --- a/installer/ui/screens/docker.py +++ b/installer/ui/screens/docker.py @@ -8,10 +8,13 @@ import subprocess import sys import time from pathlib import Path + +from installer.config import get_docker_build_platform_args from installer.core.env import ( PROJECT_ROOT, _collect_images_from_files, _deployment_mode_from_env, + _host_from_url, _http_ping_registry, _k3d_prole_data_volume_args, _local_registry_enabled, @@ -24,6 +27,24 @@ from installer.core.env import ( class DockerScreenMixin: """Docker image build, registry management and image pre-pull.""" + def _k3s_registry_hostport(self) -> str: + # For k3s, registry access must never rely on localhost. + # Derive the registry host from the configured k3s API server URL. + cfg = self.prole_cfg_data or {} + server = ( + (cfg.get("Global", {}) or {}).get("PROLE_K3S_SERVER") + or (cfg.get("Global", {}) or {}).get("K3S_SERVER_URL") + or (cfg.get("Initialize Cluster", {}) or {}).get("K3S_SERVER_URL") + or (cfg.get("Service Cluster (k3s)", {}) or {}).get("K3S_SERVER_URL") + or os.environ.get("PROLE_K3S_SERVER") + or os.environ.get("K3S_SERVER_URL") + or "" + ) + host = _host_from_url(server) + if not host: + return "" + return f"{host}:5000" + def check_docker_running(self): """Check if Docker is running""" try: @@ -37,6 +58,14 @@ class DockerScreenMixin: Returns the registry URL (host:port) to be used for tagging/pushing. For Dev, may create a local k3d-managed registry; for other envs, still prefer the external if reachable. """ + mode = _deployment_mode_from_env( + (self.prole_cfg_data.get("Global", {}) or {}).get("DEPLOYMENT_MODE") + or env + ) + if mode == "k3s": + # k3s must use the k3s registry (never localhost/k3d registries). + return self._k3s_registry_hostport() + # Prefer the shared registry if reachable if _http_ping_registry("k8s.prole.org", 5000): return "k8s.prole.org:5000" @@ -127,6 +156,36 @@ class DockerScreenMixin: mode = _deployment_mode_from_env( self.prole_cfg_data.get("Global", {}).get("DEPLOYMENT_MODE") ) + if mode == "k3s": + k3s_registry = self._k3s_registry_hostport() + if not k3s_registry: + _log("k3s mode but K3S_SERVER_URL/PROLE_K3S_SERVER not set; cannot resolve registry.\n") + if log_fp: + log_fp.close() + return None + host_registry = k3s_registry + cluster_registry = k3s_registry + if not self.check_docker_running(): + _log("Docker not running; cannot push/prepull images to k3s registry.\n") + if log_fp: + log_fp.close() + return None + self.local_registry_url = host_registry + self.local_registry_internal = cluster_registry + try: + if "Docker Build" not in self.prole_cfg_data: + self.prole_cfg_data["Docker Build"] = {} + self.prole_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry + self.prole_cfg_data["Docker Build"][ + "LOCAL_REGISTRY_INTERNAL" + ] = cluster_registry + self.safe_after(self._save_prole_cfg) + except Exception: + pass + if log_fp: + log_fp.close() + return host_registry, cluster_registry + if mode == "k3d": cluster_registry = f"{registry_container}.localhost:5000" else: diff --git a/k8s/argocd/install.yaml b/k8s/argocd/install.yaml index e22eb38..2a975ac 100644 --- a/k8s/argocd/install.yaml +++ b/k8s/argocd/install.yaml @@ -32496,7 +32496,7 @@ spec: initContainers: - args: - /bin/cp --update=none /usr/local/bin/argocd /var/run/argocd/argocd && /bin/ln - -s /var/run/argocd/argocd /var/run/argocd/argocd-cmp-server + -sf /var/run/argocd/argocd /var/run/argocd/argocd-cmp-server command: - sh - -c diff --git a/k8s/opentofu/deployment.yaml b/k8s/opentofu/deployment.yaml index 3a1e849..70f629a 100644 --- a/k8s/opentofu/deployment.yaml +++ b/k8s/opentofu/deployment.yaml @@ -105,6 +105,7 @@ kind: Service metadata: name: opentofu spec: + type: LoadBalancer selector: app: opentofu ports: diff --git a/k8s/prole/prole-db-recovery.yaml.tpl b/k8s/prole/prole-db-recovery.yaml.tpl index 0491bbf..5f5eab2 100644 --- a/k8s/prole/prole-db-recovery.yaml.tpl +++ b/k8s/prole/prole-db-recovery.yaml.tpl @@ -5,7 +5,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: k3d-prole-registry.localhost:5000/prole-db:18-101 + imageName: prole-db:18-102 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 diff --git a/k8s/prole/prole-db.yaml b/k8s/prole/prole-db.yaml index 8817dd6..91b730e 100644 --- a/k8s/prole/prole-db.yaml +++ b/k8s/prole/prole-db.yaml @@ -4,7 +4,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: k3d-prole-registry.localhost:5000/prole-db:18-101 + imageName: prole-db:18-102 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 diff --git a/prole-db/.version b/prole-db/.version index 97a55e1..0aede4a 100644 --- a/prole-db/.version +++ b/prole-db/.version @@ -1 +1 @@ -101 \ No newline at end of file +102 \ No newline at end of file diff --git a/tests/test_argocd_manifest_idempotency.py b/tests/test_argocd_manifest_idempotency.py new file mode 100644 index 0000000..02870e8 --- /dev/null +++ b/tests/test_argocd_manifest_idempotency.py @@ -0,0 +1,29 @@ +import re +from pathlib import Path + + +def test_argocd_repo_server_copyutil_symlink_is_idempotent(): + manifest_path = Path(__file__).resolve().parents[1] / "k8s" / "argocd" / "install.yaml" + text = manifest_path.read_text(encoding="utf-8") + + # Ensure the repo-server initContainer makes the cmp-server symlink idempotently. + # The repo-server can be recreated while `/var/run/argocd` is persisted (hostPath), + # so an unconditional `ln -s` will fail with "File exists". + assert "argocd-cmp-server" in text + + safe_link_re = re.compile( + r"/bin/ln\s+-sf\s+/var/run/argocd/argocd\s+/var/run/argocd/argocd-cmp-server", + re.MULTILINE, + ) + assert safe_link_re.search(text), ( + "Expected ArgoCD repo-server copyutil initContainer to use `ln -sf` when creating " + "`/var/run/argocd/argocd-cmp-server`." + ) + + unsafe_link_re = re.compile( + r"/bin/ln\s+-s\s+/var/run/argocd/argocd\s+/var/run/argocd/argocd-cmp-server", + re.MULTILINE, + ) + assert not unsafe_link_re.search(text), ( + "Found an unconditional `ln -s` for `argocd-cmp-server` which is not idempotent." + ) diff --git a/tests/test_k3s_common_core_namespace_and_repair.py b/tests/test_k3s_common_core_namespace_and_repair.py new file mode 100644 index 0000000..f9df33d --- /dev/null +++ b/tests/test_k3s_common_core_namespace_and_repair.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from pathlib import Path + +from installer.core.actions import ProleSilentInstaller +from installer.core.controller import ProleController + + +def test_silent_installer_service_namespace_defaults_to_knoe_system_for_k3s(tmp_path, monkeypatch): + """Regression: k3s service clusters must not default core services into `default`.""" + + monkeypatch.delenv("SERVICE_NAMESPACE", raising=False) + + c = ProleController(tmp_path) + installer = ProleSilentInstaller(c) + installer.inputs["init_cluster.cluster_env"] = "service" + installer.prole_cfg_data = {"Global": {}} + + assert installer._service_namespace() == "knoe-system" + + +def test_repair_pipeline_has_k3s_dedupe_guard(): + """Repair pipeline should reconcile accidental `default` deployments in k3s.""" + + text = Path("etc/repair_pipeline.sh").read_text(encoding="utf-8") + + # k3s default namespace normalization + assert "SERVICE_NAMESPACE=\"knoe-system\"" in text + + # cross-namespace dedupe pass + assert "dedupe_common_core_default_namespace" in text