k3s: eliminate localhost registry + dedupe common-core

- Ensure k3s mode uses the k3s registry endpoint and avoid localhost/k3d image prefixes. - Make ArgoCD repo-server cmp symlink creation idempotent. - Normalize common-core provisioning to knoe-system and add repair-time dedupe of stray default-namespace installs. - Add k3s MariaDB datastore/refresh playbooks and regression tests.
2026-09-23 10:13:58 +00:00 · 2026-03-05 14:31:33 -08:00 · 2026-03-05 14:31:33 -08:00 · a43aed7134
commit a43aed7134
parent 60820b2b7c
29 changed files with 859 additions and 72 deletions
--- a/conf/port-mapping.cfg
+++ b/conf/port-mapping.cfg
@ -1,11 +1,3 @@
 # Port mappings for Prole Tools (generated).
 # Format: key: local=... remote=... ns=... svc=... address=...

-argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0
-garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0
-openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0
-opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0
-dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1
-postgres: local=5432 remote=5432 ns=knoey-db-0 svc=prole-db-rw address=0.0.0.0
-prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1
-grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0
--- a/conf/prole.cfg
+++ b/conf/prole.cfg
@ -96,7 +96,7 @@ PROLE_DB_USER = root
 PROLE_HOME = ${PROLE_HOME}
 PROLE_K3S_SERVER = https://myrddin.prole.org:6443
 PROLE_K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
-PROLE_OPENTOFU_URL = http://127.0.0.1:8080
+PROLE_OPENTOFU_URL = http://myrddin.prole.org:8080
 REGISTRY_NAMESPACE = default
 SERVICE_NAMESPACE = knoe-system

@ -175,18 +175,18 @@ DISPLAY_NAME = prole-service-cluster
 K3S_SERVER_URL = https://myrddin.prole.org:6443
 K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
 MODE = k3s
-PIPELINE_URL = http://127.0.0.1:8080
+PIPELINE_URL = http://myrddin.prole.org:8080

 [Prod Cluster (k8s)]
 ARTIFACTS_DIR = ${PROLE_DATA}/staging
 CLUSTER_ENV = prole-prod-cluster
 DISPLAY_NAME = prole-prod-cluster
 MODE = k8s
-PIPELINE_URL = http://127.0.0.1:8080
+PIPELINE_URL = http://myrddin.prole.org:8080

 [Docker Build]
-LOCAL_REGISTRY = localhost:5000
-LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000
+LOCAL_REGISTRY = myrddin.prole.org:5000
+LOCAL_REGISTRY_INTERNAL = myrddin.prole.org:5000

 [Initialization Scripts]
 STATUS = Attempted
--- a/deploy/opentofu/k3s/manifests/prole/prole-db.yaml
+++ b/deploy/opentofu/k3s/manifests/prole/prole-db.yaml
@ -4,7 +4,7 @@ metadata:
  name: prole-db
 spec:
  instances: 3
-  imageName: k3d-prole-registry:5000/prole-db:17.7-059
+  imageName: myrddin.prole.org:5000/prole-db:17.7-059
  postgresUID: 100
  postgresGID: 101
  maxSyncReplicas: 1
--- a/etc/common_core_lib.sh
+++ b/etc/common_core_lib.sh
@ -118,6 +118,17 @@ common_core_resolve_namespace() {
  [[ -z "$ns" && -n "${SERVICE_NAMESPACE:-}" ]] && ns="$SERVICE_NAMESPACE"
  [[ -z "$ns" && -n "${NAMESPACE:-}" ]] && ns="$NAMESPACE"
  [[ -z "$ns" ]] && ns="$default_ns"
+
+  # In k3s mode, common core services must not default into the `default` namespace.
+  # If the caller didn't supply an explicit namespace, normalize `default` -> `knoe-system`.
+  # (Explicit `-n default` is treated as a legacy/mistake and will be corrected.)
+  local mode="${PROLE_MODE:-}"
+  if declare -F prole_normalize_mode >/dev/null 2>&1; then
+    mode="$(prole_normalize_mode "$mode")"
+  fi
+  if [[ "$mode" == "k3s" && "$ns" == "default" ]]; then
+    ns="knoe-system"
+  fi
  echo "$ns"
 }

--- a/etc/init_registry.sh
+++ b/etc/init_registry.sh
@ -5,7 +5,7 @@ set -euo pipefail
 # init_registry.sh
 # Purpose:
 # - Deploy ArgoCD into Kubernetes
-# - Deploy registry:2 into k3s clusters (default namespace) when enabled
+# - Deploy registry:2 into k3s clusters (service namespace) when enabled

 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

@ -63,7 +63,7 @@ usage() {
 Usage: init_registry.sh [-n|--namespace NS] [-r|--registry-namespace NS] <start|stop|status|restart|initialize|update|reload>

 Deploys ArgoCD into the target namespace and, for k3s, deploys registry:2
-into the registry namespace (default: default).
+into the registry namespace (default: SERVICE_NAMESPACE / knoe-system).
 Note: The local Docker registry (port 5000) is managed separately for k3d.
 EOF
 }
@ -144,6 +144,17 @@ else
  REGISTRY_NAMESPACE="default"
 fi

+# In k3s mode, the in-cluster registry is a common core service and should live
+# in the service namespace (not `default`).
+_mode_resolved="${PROLE_MODE:-${DEPLOYMENT_MODE:-}}"
+if declare -F prole_normalize_mode >/dev/null 2>&1; then
+  _mode_resolved="$(prole_normalize_mode "$_mode_resolved")"
+fi
+if [[ "$_mode_resolved" == "k3s" && ( -z "${REGISTRY_NAMESPACE:-}" || "${REGISTRY_NAMESPACE}" == "default" ) ]]; then
+  REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-knoe-system}"
+fi
+unset _mode_resolved
+
 export ARGOCD_NAMESPACE
 export REGISTRY_NAMESPACE
 export NAMESPACE="$ARGOCD_NAMESPACE"
--- a/etc/repair_pipeline.sh
+++ b/etc/repair_pipeline.sh
@ -72,6 +72,16 @@ else
  SERVICE_NAMESPACE="default"
 fi

+# In k3s mode, common core services must live in knoe-system by default.
+_mode_resolved="${PROLE_MODE:-}"
+if declare -F prole_normalize_mode >/dev/null 2>&1; then
+  _mode_resolved="$(prole_normalize_mode "$_mode_resolved")"
+fi
+if [[ "$_mode_resolved" == "k3s" && ( -z "${SERVICE_NAMESPACE:-}" || "${SERVICE_NAMESPACE}" == "default" ) ]]; then
+  SERVICE_NAMESPACE="knoe-system"
+fi
+unset _mode_resolved
+
 if [[ -n "$DB_NAMESPACE_OVERRIDE" ]]; then
  DB_NAMESPACE="$DB_NAMESPACE_OVERRIDE"
 elif [[ -n "${PROLE_DB_NAMESPACE:-}" ]]; then
@ -234,6 +244,8 @@ repair_dashboard() {

 repair_argocd() {
  local ns="$1"
+  local reg_ns
+  reg_ns="${REGISTRY_NAMESPACE:-$SERVICE_NAMESPACE}"
  local need_fix=0
  local items=(
    "deploy argocd-server"
@ -256,15 +268,115 @@ repair_argocd() {
  done
  if [[ "$need_fix" -eq 1 ]]; then
    warn "ArgoCD not ready; re-deploying"
-    REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-default}" \
-      "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "${REGISTRY_NAMESPACE:-default}" stop || true
-    REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-default}" \
-      "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "${REGISTRY_NAMESPACE:-default}" update || true
+    REGISTRY_NAMESPACE="$reg_ns" \
+      "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "$reg_ns" stop || true
+    REGISTRY_NAMESPACE="$reg_ns" \
+      "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "$reg_ns" update || true
  else
    log "ArgoCD: OK"
  fi
 }

+delete_if_exists() {
+  local ns="$1" kind="$2" name="$3"
+  kubectl -n "$ns" delete "$kind" "$name" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+}
+
+delete_pods_by_label() {
+  local ns="$1" selector="$2"
+  kubectl -n "$ns" delete pod -l "$selector" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+}
+
+has_openbao() {
+  local ns="$1"
+  resource_exists deploy openbao "$ns" || resource_exists statefulset openbao "$ns"
+}
+
+has_opentofu() {
+  local ns="$1"
+  resource_exists deploy opentofu "$ns"
+}
+
+has_garage() {
+  local ns="$1"
+  resource_exists statefulset garage "$ns"
+}
+
+has_registry() {
+  local ns="$1"
+  resource_exists deploy registry "$ns"
+}
+
+dedupe_common_core_default_namespace() {
+  # The only known/expected accidental duplicate location is `default`.
+  local mode
+  mode="$(detect_mode)"
+  if [[ "$mode" != "k3s" ]]; then
+    return 0
+  fi
+
+  local canonical="${SERVICE_NAMESPACE:-knoe-system}"
+  if [[ "$canonical" == "default" ]]; then
+    canonical="knoe-system"
+  fi
+  local legacy="default"
+  if [[ "$canonical" == "$legacy" ]]; then
+    return 0
+  fi
+
+  if ! kubectl get ns "$legacy" >/dev/null 2>&1; then
+    return 0
+  fi
+
+  # OpenBao
+  if has_openbao "$legacy"; then
+    warn "Detected OpenBao in legacy namespace '$legacy'"
+    if ! has_openbao "$canonical"; then
+      warn "Canonical OpenBao missing in '$canonical'; deploying before cleanup"
+      "$SCRIPT_DIR/init_openbao.sh" -n "$canonical" update || true
+    fi
+    delete_pods_by_label "$legacy" "app=openbao"
+    delete_if_exists "$legacy" deploy openbao
+    delete_if_exists "$legacy" statefulset openbao
+    delete_if_exists "$legacy" svc openbao
+    delete_if_exists "$legacy" pvc data-openbao-0
+  fi
+
+  # OpenTofu
+  if has_opentofu "$legacy"; then
+    warn "Detected OpenTofu in legacy namespace '$legacy'"
+    if ! has_opentofu "$canonical"; then
+      warn "Canonical OpenTofu missing in '$canonical'; deploying before cleanup"
+      "$SCRIPT_DIR/init_opentofu.sh" -n "$canonical" update || true
+    fi
+    delete_if_exists "$legacy" deploy opentofu
+    delete_if_exists "$legacy" svc opentofu
+  fi
+
+  # Garage
+  if has_garage "$legacy"; then
+    warn "Detected Garage in legacy namespace '$legacy'"
+    if ! has_garage "$canonical"; then
+      warn "Canonical Garage missing in '$canonical'; deploying before cleanup"
+      "$SCRIPT_DIR/init_garage_store.sh" -n "$canonical" start || true
+    fi
+    delete_pods_by_label "$legacy" "app=garage"
+    delete_if_exists "$legacy" statefulset garage
+    delete_if_exists "$legacy" svc garage
+  fi
+
+  # Registry
+  if has_registry "$legacy"; then
+    warn "Detected registry in legacy namespace '$legacy'"
+    if ! has_registry "$canonical"; then
+      warn "Canonical registry missing in '$canonical'; deploying before cleanup"
+      REGISTRY_NAMESPACE="$canonical" "$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NAMESPACE" --registry-namespace "$canonical" update || true
+    fi
+    delete_if_exists "$legacy" deploy registry
+    delete_if_exists "$legacy" svc registry
+  fi
+}
+
 repair_openbao() {
  local ns="$1"
  local status
@ -392,6 +504,9 @@ case "${ACTION}" in
    log "-- Dashboard (Kong) --"
    repair_dashboard || true

+    log "-- Dedupe common core (k3s) --"
+    dedupe_common_core_default_namespace || true
+
    log "-- ArgoCD --"
    repair_argocd "$ARGOCD_NAMESPACE"

@ -404,6 +519,10 @@ case "${ACTION}" in
    log "-- Garage --"
    repair_garage "$SERVICE_NAMESPACE"

+    # One more pass after repairs in case the canonical deployments were missing
+    # and we needed to deploy them first.
+    dedupe_common_core_default_namespace || true
+
    if [[ "${KERBEROS_ENABLED:-}" == "0" || "${KERBEROS_ENABLED:-}" == "false" || "${KERBEROS_ENABLED:-}" == "False" ]]; then
      log "KDC (auth): skipped (Kerberos disabled)"
    elif [[ "${PROLE_KDC_ENABLED:-1}" != "0" ]]; then
--- a/infrastructure/inventory/host_vars/myrddin.prole.org.yml
+++ b/infrastructure/inventory/host_vars/myrddin.prole.org.yml
@ -70,6 +70,13 @@ k3s_enabled: true
 k3s_cluster_init: false
 k3s_role: server
 k3s_token: "{{ vault_k3s_token | default('') }}"   # store this in vault
+k3s_datastore_mariadb_host: synology.prole.org
+k3s_datastore_mariadb_port: 3306
+k3s_datastore_mariadb_db: k3s
+k3s_datastore_mariadb_user: prole_k3s
+# Password is the same as the Samba administrator password and must remain in Ansible Vault.
+k3s_datastore_mariadb_password: "{{ vault_samba_dns_admin_pass }}"
+k3s_datastore_endpoint: "mysql://{{ k3s_datastore_mariadb_user }}:{{ k3s_datastore_mariadb_password | urlencode }}@tcp({{ k3s_datastore_mariadb_host }}:{{ k3s_datastore_mariadb_port }})/{{ k3s_datastore_mariadb_db }}"
 k3s_tls_sans:
  - myrddin.prole.org
 k3s_node_labels:
@ -90,6 +97,11 @@ k3s_kube_scheduler_args:
  - "leader-elect-renew-deadline=100s"
  - "leader-elect-retry-period=20s"

+k3s_cloud_controller_manager_args:
+  - "leader-elect-lease-duration=300s"
+  - "leader-elect-renew-deadline=200s"
+  - "leader-elect-retry-period=40s"
+
 k3s_kubelet_args:
  - "housekeeping-interval=10s"

--- a/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml
+++ b/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml
@ -0,0 +1,102 @@
+---
+- name: Prepare MariaDB datastore for k3s (create DB/user/grants)
+  hosts: k3s_hosts
+  gather_facts: false
+  become: false
+  serial: 1
+  vars:
+    k3s_mariadb_host: "{{ k3s_datastore_mariadb_host | default('synology.prole.org') }}"
+    k3s_mariadb_port: "{{ k3s_datastore_mariadb_port | default(3306) }}"
+    k3s_mariadb_db: "{{ k3s_datastore_mariadb_db | default('k3s') }}"
+    k3s_mariadb_user: "{{ k3s_datastore_mariadb_user | default('prole_k3s') }}"
+    k3s_mariadb_user_password: "{{ k3s_datastore_mariadb_password | default('') }}"
+
+    # MariaDB admin credentials (must be stored in Ansible Vault by the operator)
+    k3s_mariadb_admin_user: "{{ k3s_mariadb_admin_user | default('root') }}"
+    k3s_mariadb_admin_password: "{{ k3s_mariadb_admin_password | default('') }}"
+
+    # Where to allow the k3s user to connect from (tighten this if possible)
+    k3s_mariadb_user_host: "{{ k3s_mariadb_user_host | default('%') }}"
+
+  pre_tasks:
+    - name: Only run on k3s server nodes
+      ansible.builtin.meta: end_host
+      when: k3s_role | default('') != 'server'
+
+    - name: Require k3s MariaDB user password (vaulted)
+      ansible.builtin.assert:
+        that:
+          - k3s_mariadb_user_password | length > 0
+        fail_msg: >-
+          k3s_mariadb_user_password is empty. Ensure k3s_datastore_mariadb_password is set
+          (and sourced from Ansible Vault, e.g. vault_samba_dns_admin_pass).
+
+    - name: Require MariaDB admin password for non-check runs
+      ansible.builtin.assert:
+        that:
+          - k3s_mariadb_admin_password | length > 0
+        fail_msg: >-
+          k3s_mariadb_admin_password is empty.
+          Provide a vaulted MariaDB admin password (e.g. in an encrypted group_vars/*/vault.yml).
+      when: not ansible_check_mode
+
+    - name: Verify mysql client is available on this host
+      ansible.builtin.command: mysql --version
+      changed_when: false
+
+  tasks:
+    - name: Create k3s database (if missing)
+      ansible.builtin.command: >-
+        mysql
+        --protocol=tcp
+        --host={{ k3s_mariadb_host }}
+        --port={{ k3s_mariadb_port }}
+        --user={{ k3s_mariadb_admin_user }}
+        --execute="CREATE DATABASE IF NOT EXISTS `{{ k3s_mariadb_db }}`"
+      environment:
+        MYSQL_PWD: "{{ k3s_mariadb_admin_password }}"
+      changed_when: false
+      when: not ansible_check_mode
+      no_log: true
+
+    - name: Create/ensure k3s MariaDB user exists
+      ansible.builtin.command: >-
+        mysql
+        --protocol=tcp
+        --host={{ k3s_mariadb_host }}
+        --port={{ k3s_mariadb_port }}
+        --user={{ k3s_mariadb_admin_user }}
+        --execute="CREATE USER IF NOT EXISTS '{{ k3s_mariadb_user }}'@'{{ k3s_mariadb_user_host }}' IDENTIFIED BY '{{ k3s_mariadb_user_password }}'"
+      environment:
+        MYSQL_PWD: "{{ k3s_mariadb_admin_password }}"
+      changed_when: false
+      when: not ansible_check_mode
+      no_log: true
+
+    - name: Grant admin privileges on k3s database to k3s MariaDB user
+      ansible.builtin.command: >-
+        mysql
+        --protocol=tcp
+        --host={{ k3s_mariadb_host }}
+        --port={{ k3s_mariadb_port }}
+        --user={{ k3s_mariadb_admin_user }}
+        --execute="GRANT ALL PRIVILEGES ON `{{ k3s_mariadb_db }}`.* TO '{{ k3s_mariadb_user }}'@'{{ k3s_mariadb_user_host }}'"
+      environment:
+        MYSQL_PWD: "{{ k3s_mariadb_admin_password }}"
+      changed_when: false
+      when: not ansible_check_mode
+      no_log: true
+
+    - name: Flush privileges
+      ansible.builtin.command: >-
+        mysql
+        --protocol=tcp
+        --host={{ k3s_mariadb_host }}
+        --port={{ k3s_mariadb_port }}
+        --user={{ k3s_mariadb_admin_user }}
+        --execute="FLUSH PRIVILEGES"
+      environment:
+        MYSQL_PWD: "{{ k3s_mariadb_admin_password }}"
+      changed_when: false
+      when: not ansible_check_mode
+      no_log: true
--- a/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml
+++ b/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml
@ -0,0 +1,54 @@
+---
+- name: Roll out k3s config for external MariaDB datastore (servers)
+  hosts: k3s_hosts
+  gather_facts: false
+  become: true
+  serial: 1
+  pre_tasks:
+    - name: Require explicit acknowledgement (this changes the k3s datastore)
+      ansible.builtin.assert:
+        that:
+          - k3s_datastore_migration_ack | default(false) | bool
+        fail_msg: >-
+          Refusing to proceed without k3s_datastore_migration_ack=true.
+          Switching k3s datastore backends may require a rebuild/migration plan.
+
+    - name: Only run on k3s server nodes
+      ansible.builtin.meta: end_host
+      when: k3s_role | default('') != 'server'
+
+    - name: Require k3s_datastore_endpoint for MariaDB datastore
+      ansible.builtin.assert:
+        that:
+          - (k3s_datastore_endpoint | default('')) | length > 0
+        fail_msg: "k3s_datastore_endpoint is empty on {{ inventory_hostname }}"
+
+  tasks:
+    - name: Apply k3s install/config changes (server)
+      ansible.builtin.import_role:
+        name: k3s
+        tasks_from: install
+
+- name: Roll out k3s config for external MariaDB datastore (agents)
+  hosts: k3s_hosts
+  gather_facts: false
+  become: true
+  serial: 1
+  pre_tasks:
+    - name: Require explicit acknowledgement (this changes the k3s datastore)
+      ansible.builtin.assert:
+        that:
+          - k3s_datastore_migration_ack | default(false) | bool
+        fail_msg: >-
+          Refusing to proceed without k3s_datastore_migration_ack=true.
+          Switching k3s datastore backends may require a rebuild/migration plan.
+
+    - name: Only run on k3s agent nodes
+      ansible.builtin.meta: end_host
+      when: k3s_role | default('') != 'agent'
+
+  tasks:
+    - name: Apply k3s install/config changes (agent)
+      ansible.builtin.import_role:
+        name: k3s
+        tasks_from: install
--- a/infrastructure/playbooks/k3s_server_refresh.yml
+++ b/infrastructure/playbooks/k3s_server_refresh.yml
@ -0,0 +1,19 @@
+---
+- name: Refresh k3s server (blank slate except token/CA)
+  hosts: k3s_hosts
+  become: true
+  serial: 1
+  vars:
+    # Safety guard: you must also pass `-e k3s_refresh_confirm=YES`.
+    # Example:
+    #   ./ansible.sh -v .vault_pass -p infrastructure/playbooks/k3s_server_refresh.yml -l myrddin.prole.org -- -e k3s_refresh_confirm=YES
+    k3s_refresh_run_configure: true
+  tasks:
+    - name: Ensure cgroup kernel params are set
+      ansible.builtin.import_role:
+        name: cgroups
+
+    - name: Refresh k3s server (preserve token/CA)
+      ansible.builtin.import_role:
+        name: k3s
+        tasks_from: refresh_server
--- a/infrastructure/roles/k3s/defaults/main.yml
+++ b/infrastructure/roles/k3s/defaults/main.yml
@ -4,6 +4,13 @@ k3s_state: present
 k3s_version: ""
 k3s_guard_token_drift: true

+# Destructive repair/refresh workflow (server only)
+# A refresh wipes the k3s installation/state to a blank slate but preserves the
+# join token and CA material so existing agents can rejoin without changing the token.
+k3s_refresh_backup_root: /root/k3s-refresh-backups
+k3s_refresh_confirm: ""
+k3s_refresh_run_configure: true
+
 # CloudNative-PG
 # Pin operator/plugin version by default to avoid unanticipated upgrades and load spikes.
 # Upgrades must be explicitly enabled via `k3s_cnpg_upgrade: true`.
@ -14,6 +21,9 @@ k3s_role: agent
 k3s_cluster_init: false
 k3s_server_url: ""
 k3s_token: "{{ vault_k3s_token | default('') }}"
+# Optional: if `k3s_token` is a full node-token (`K10...::server:...`), set this
+# to the raw secret (`...` after `::server:`) for the server config.
+k3s_server_token: ""
 k3s_disable: []
 k3s_tls_sans: []
 k3s_node_labels: []
--- a/infrastructure/roles/k3s/tasks/cleanup.yml
+++ b/infrastructure/roles/k3s/tasks/cleanup.yml
@ -1,4 +1,9 @@
 ---
+
+- name: Set effective k3s data-dir
+  ansible.builtin.set_fact:
+    k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}"
+
 - name: Best-effort purge kube-system workloads before uninstall
  ansible.builtin.command: "kubectl -n kube-system delete all --all --ignore-not-found"
  failed_when: false
@ -51,15 +56,15 @@
    - /etc/rancher/k3s
    - /etc/cni
    - /opt/cni
-    - /var/lib/rancher/k3s
-    - /var/lib/rancher/k3s/server/tls
-    - /var/lib/rancher/k3s/server/cred
-    - /var/lib/rancher/k3s/server/token
-    - /var/lib/rancher/k3s/server/node-token
-    - /var/lib/rancher/k3s/server/agent-token
-    - /var/lib/rancher/k3s/server/db
-    - /var/lib/rancher/k3s/agent
-    - /var/lib/rancher/k3s/data
+    - "{{ k3s_effective_data_dir }}"
+    - "{{ k3s_effective_data_dir }}/server/tls"
+    - "{{ k3s_effective_data_dir }}/server/cred"
+    - "{{ k3s_effective_data_dir }}/server/token"
+    - "{{ k3s_effective_data_dir }}/server/node-token"
+    - "{{ k3s_effective_data_dir }}/server/agent-token"
+    - "{{ k3s_effective_data_dir }}/server/db"
+    - "{{ k3s_effective_data_dir }}/agent"
+    - "{{ k3s_effective_data_dir }}/data"
    - /var/lib/kubelet
    - /var/lib/etcd
    - /var/lib/cni
--- a/infrastructure/roles/k3s/tasks/install.yml
+++ b/infrastructure/roles/k3s/tasks/install.yml
@ -8,6 +8,18 @@
  ansible.builtin.set_fact:
    k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}"

+- name: Derive server token secret from node-token (server only)
+  ansible.builtin.set_fact:
+    k3s_server_token: "{{ k3s_token | regex_replace('^K10[0-9a-f]+::server:', '') }}"
+  when:
+    - k3s_role == 'server'
+    - (k3s_token | default('')) is match('^K10[0-9a-f]+::server:.+')
+  no_log: true
+
+- name: Set effective k3s data-dir
+  ansible.builtin.set_fact:
+    k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}"
+
 - name: Check if k3s service is installed
  ansible.builtin.stat:
    path: "/etc/systemd/system/{{ k3s_service_name }}.service"
@ -162,7 +174,8 @@
    - (['cgroup_memory=1', 'cgroup_enable=memory'] | reject('in', k3s_proc_cmdline.stdout | default('')) | list) | length > 0

 - name: Guardrail - ensure rancher mountpoint is not on SD/rootfs
-  ansible.builtin.command: "findmnt -n -o SOURCE {{ k3s_rancher_mountpoint | default('/var/lib/rancher') }}"
+  ansible.builtin.shell: |
+    awk -v mp="{{ k3s_rancher_mountpoint | default('/var/lib/rancher') }}" '$5==mp {for(i=1;i<=NF;i++) if($i=="-") {print $(i+2); exit}}' /proc/self/mountinfo
  register: rancher_source
  changed_when: false
  check_mode: no
@ -175,7 +188,8 @@
    - (rancher_source.stdout | default('')) is search("mmcblk0") or (rancher_source.stdout | default('')) is search("/dev/mmc")

 - name: Preflight - ensure required mountpoints are mounted
-  ansible.builtin.command: "findmnt -n {{ item }}"
+  ansible.builtin.shell: |
+    awk -v mp="{{ item }}" '$5==mp {found=1} END {exit(found?0:1)}' /proc/self/mountinfo
  register: k3s_required_mounts_present
  changed_when: false
  failed_when: k3s_required_mounts_present.rc != 0
@ -184,7 +198,8 @@
  when: (k3s_required_mounts | default([])) | length > 0

 - name: Guardrail - ensure required mountpoints are not on SD
-  ansible.builtin.command: "findmnt -n -o SOURCE {{ item }}"
+  ansible.builtin.shell: |
+    awk -v mp="{{ item }}" '$5==mp {for(i=1;i<=NF;i++) if($i=="-") {print $(i+2); exit}}' /proc/self/mountinfo
  register: k3s_required_mounts_sources
  changed_when: false
  check_mode: no
@ -201,7 +216,7 @@

 - name: Read k3s node token for sharing
  ansible.builtin.slurp:
-    src: /var/lib/rancher/k3s/server/node-token
+    src: "{{ k3s_effective_data_dir }}/server/node-token"
  register: k3s_node_token_slurp
  when:
    - k3s_role == "server"
@ -250,8 +265,8 @@
  block:
    - name: Get local agent CA cert hash (DER)
      ansible.builtin.shell: |
-        if [ -f /var/lib/rancher/k3s/agent/server-ca.crt ]; then
-          openssl x509 -in /var/lib/rancher/k3s/agent/server-ca.crt -outform DER | openssl dgst -sha256 | awk '{print $NF}'
+        if [ -f {{ k3s_effective_data_dir }}/agent/server-ca.crt ]; then
+          openssl x509 -in {{ k3s_effective_data_dir }}/agent/server-ca.crt -outform DER | openssl dgst -sha256 | awk '{print $NF}'
        else
          echo "MISSING"
        fi
@ -285,7 +300,7 @@

    - name: Wipe stale agent state
      ansible.builtin.file:
-        path: /var/lib/rancher/k3s/agent
+        path: "{{ k3s_effective_data_dir }}/agent"
        state: absent

    - name: Force k3s re-installation after CA wipe
@ -309,7 +324,7 @@

 - name: Check for existing k3s server node token
  ansible.builtin.stat:
-    path: /var/lib/rancher/k3s/server/node-token
+    path: "{{ k3s_effective_data_dir }}/server/node-token"
  register: k3s_server_node_token
  when:
    - k3s_guard_token_drift | bool
@ -318,7 +333,7 @@

 - name: Read existing k3s server node token
  ansible.builtin.slurp:
-    src: /var/lib/rancher/k3s/server/node-token
+    src: "{{ k3s_effective_data_dir }}/server/node-token"
  register: k3s_server_node_token_raw
  when:
    - k3s_guard_token_drift | bool
@ -329,7 +344,7 @@
 - name: Fail when vault k3s token is missing for existing cluster
  ansible.builtin.fail:
    msg: >-
-      vault_k3s_token is empty but /var/lib/rancher/k3s/server/node-token exists.
+      vault_k3s_token is empty but {{ k3s_effective_data_dir }}/server/node-token exists.
      Refusing to overwrite k3s config. Update vault_k3s.yml or run the k3s sync playbook.
  when:
    - k3s_guard_token_drift | bool
@ -504,23 +519,31 @@
  changed_when: "'modified' in _taint_result.stdout or 'tainted' in _taint_result.stdout"
  failed_when: false # Avoid failing if node is not yet ready

- name: Wait for k3s to be ready
-  ansible.builtin.shell: k3s kubectl get nodes | grep -w "Ready"
-  register: k3s_ready
-  until: k3s_ready.rc == 0
-  retries: 20
+- name: Wait for k3s API readiness
+  ansible.builtin.command: k3s kubectl get --raw=/readyz
+  register: k3s_readyz
+  until:
+    - k3s_readyz.rc == 0
+    - (k3s_readyz.stdout | default('') | trim) in ['ok', 'OK']
+  retries: 60
  delay: 5
  changed_when: false
  when: k3s_state == "present" and k3s_role == "server"

- name: Pause to allow k3s API to stabilize
-  ansible.builtin.pause:
-    seconds: 10
-  when: k3s_state == "present" and k3s_role == "server" and k3s_ready.changed | default(true)
+- name: Wait for local node to report Ready
+  ansible.builtin.shell: |
+    k3s kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true
+  register: k3s_node_ready
+  until: (k3s_node_ready.stdout | default('') | trim) == 'True'
+  retries: 60
+  delay: 5
+  changed_when: false
+  failed_when: false
+  when: k3s_state == "present" and k3s_role == "server"

 - name: Read k3s node token after install for sharing
  ansible.builtin.slurp:
-    src: /var/lib/rancher/k3s/server/node-token
+    src: "{{ k3s_effective_data_dir }}/server/node-token"
  register: k3s_node_token_slurp_post
  when:
    - k3s_role == "server"
--- a/infrastructure/roles/k3s/tasks/refresh_server.yml
+++ b/infrastructure/roles/k3s/tasks/refresh_server.yml
@ -0,0 +1,195 @@
+---
+
+- name: Guard - require explicit confirmation for k3s server refresh
+  ansible.builtin.assert:
+    that:
+      - k3s_role == 'server'
+      - k3s_state == 'present'
+      - (k3s_refresh_confirm | default('')) == 'YES'
+    fail_msg: >-
+      Refusing to refresh k3s server without explicit confirmation.
+      Set k3s_refresh_confirm=YES and re-run (this is destructive: it wipes k3s state).
+
+- name: Read current k3s config (to detect current data-dir)
+  ansible.builtin.stat:
+    path: /etc/rancher/k3s/config.yaml
+  register: _k3s_refresh_config_stat
+
+- name: Slurp current k3s config
+  ansible.builtin.slurp:
+    src: /etc/rancher/k3s/config.yaml
+  register: _k3s_refresh_config_slurp
+  when: _k3s_refresh_config_stat.stat.exists
+
+- name: Set refresh source/target data-dir
+  ansible.builtin.set_fact:
+    _k3s_refresh_cfg: >-
+      {{
+        (_k3s_refresh_config_slurp.content | b64decode | from_yaml)
+        if (_k3s_refresh_config_stat.stat.exists | bool)
+        else {}
+      }}
+    k3s_refresh_source_data_dir: >-
+      {{
+        _k3s_refresh_cfg.get('data-dir')
+        | default((k3s_data_dir | default('/var/lib/rancher/k3s')), true)
+      }}
+    k3s_refresh_target_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}"
+
+- name: Preflight - check external datastore TCP reachability (best-effort)
+  ansible.builtin.wait_for:
+    host: "{{ k3s_datastore_mariadb_host }}"
+    port: "{{ k3s_datastore_mariadb_port | default(3306) }}"
+    connect_timeout: 2
+    timeout: 5
+    state: started
+  when:
+    - (k3s_datastore_endpoint | default('') | length) > 0
+    - k3s_datastore_mariadb_host is defined
+  changed_when: false
+  failed_when: false
+
+- name: Create refresh backup directory
+  ansible.builtin.file:
+    path: "{{ k3s_refresh_backup_root }}/{{ inventory_hostname }}/{{ ansible_date_time.iso8601_basic_short }}"
+    state: directory
+    mode: "0700"
+  register: _k3s_refresh_backup_dir
+
+- name: Stat k3s token/tls artifacts (source data-dir)
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  loop:
+    - "{{ k3s_refresh_source_data_dir }}/server/token"
+    - "{{ k3s_refresh_source_data_dir }}/server/node-token"
+    # Minimal TLS set to preserve agent trust (CA hash) while allowing k3s to
+    # regenerate component creds and `server/cred/*.kubeconfig`.
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/server-ca.crt"
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/server-ca.key"
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/client-ca.crt"
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/client-ca.key"
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/request-header-ca.crt"
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/request-header-ca.key"
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/service.key"
+    - "{{ k3s_refresh_source_data_dir }}/server/tls/service.current.key"
+  register: _k3s_refresh_artifacts
+
+- name: Back up k3s token/tls artifacts (remote->remote)
+  ansible.builtin.copy:
+    remote_src: true
+    src: "{{ item.item }}"
+    dest: "{{ _k3s_refresh_backup_dir.path }}/"
+    mode: preserve
+  loop: "{{ _k3s_refresh_artifacts.results }}"
+  when: item.stat.exists
+
+- name: Read preserved server token (source data-dir)
+  ansible.builtin.slurp:
+    src: "{{ k3s_refresh_source_data_dir }}/server/token"
+  register: _k3s_refresh_token_slurp
+  when: >-
+    {{
+      (
+        _k3s_refresh_artifacts.results
+        | selectattr('item', 'equalto', (k3s_refresh_source_data_dir ~ '/server/token'))
+        | map(attribute='stat.exists')
+        | first
+      )
+      | default(false)
+    }}
+  no_log: true
+
+- name: Ensure we have a token to preserve
+  ansible.builtin.assert:
+    that:
+      - (k3s_token | default('') | length) > 0 or (_k3s_refresh_token_slurp is defined)
+    fail_msg: >-
+      No existing token file was found under {{ k3s_refresh_source_data_dir }}/server/token,
+      and k3s_token is empty. Refusing to proceed because this would change the join token.
+
+- name: Force k3s_token to preserved token (so rendered config matches what we restore)
+  ansible.builtin.set_fact:
+    k3s_token: "{{ (_k3s_refresh_token_slurp.content | b64decode | trim) if (_k3s_refresh_token_slurp is defined) else k3s_token }}"
+  no_log: true
+
+- name: Stop k3s services before refresh
+  ansible.builtin.service:
+    name: "{{ item }}"
+    state: stopped
+    enabled: false
+  loop:
+    - k3s
+    - k3s-agent
+  failed_when: false
+
+- name: Uninstall and wipe k3s state (blank slate)
+  ansible.builtin.import_tasks: cleanup.yml
+
+- name: Wipe target k3s data-dir (blank slate except restored token/CA)
+  ansible.builtin.file:
+    path: "{{ k3s_refresh_target_data_dir }}"
+    state: absent
+
+- name: Recreate target server directory
+  ansible.builtin.file:
+    path: "{{ k3s_refresh_target_data_dir }}/server"
+    state: directory
+    mode: "0700"
+
+- name: Recreate target tls directory
+  ansible.builtin.file:
+    path: "{{ k3s_refresh_target_data_dir }}/server/tls"
+    state: directory
+    mode: "0700"
+
+- name: Restore preserved minimal tls artifacts into target data-dir
+  ansible.builtin.copy:
+    remote_src: true
+    src: "{{ item }}"
+    dest: "{{ k3s_refresh_target_data_dir }}/server/tls/"
+    mode: preserve
+  loop:
+    - "{{ _k3s_refresh_backup_dir.path }}/server-ca.crt"
+    - "{{ _k3s_refresh_backup_dir.path }}/server-ca.key"
+    - "{{ _k3s_refresh_backup_dir.path }}/client-ca.crt"
+    - "{{ _k3s_refresh_backup_dir.path }}/client-ca.key"
+    - "{{ _k3s_refresh_backup_dir.path }}/request-header-ca.crt"
+    - "{{ _k3s_refresh_backup_dir.path }}/request-header-ca.key"
+    - "{{ _k3s_refresh_backup_dir.path }}/service.key"
+    - "{{ _k3s_refresh_backup_dir.path }}/service.current.key"
+  failed_when: false
+
+- name: Install and start k3s
+  ansible.builtin.import_tasks: install.yml
+
+- name: Configure k3s add-ons
+  ansible.builtin.import_tasks: configure.yml
+  when: k3s_refresh_run_configure | bool
+
+- name: Verify k3s API is responsive after refresh
+  ansible.builtin.command: k3s kubectl get nodes
+  register: _k3s_refresh_nodes
+  changed_when: false
+  until: _k3s_refresh_nodes.rc == 0
+  retries: 18
+  delay: 10
+
+- name: Read k3s node-token after refresh
+  ansible.builtin.slurp:
+    src: "{{ k3s_refresh_target_data_dir }}/server/node-token"
+  register: _k3s_refresh_node_token
+  changed_when: false
+  until: _k3s_refresh_node_token is succeeded
+  retries: 18
+  delay: 10
+  no_log: true
+
+- name: Assert node-token is preserved (when token is in node-token format)
+  ansible.builtin.assert:
+    that:
+      - (k3s_token | default('')) == (_k3s_refresh_node_token.content | b64decode | trim)
+    fail_msg: >-
+      Refusing to continue: expected regenerated node-token to match preserved k3s_token,
+      but it differs. This indicates the join token changed.
+  when:
+    - (k3s_token | default('')) is match('^K10[0-9a-f]+::server:.+')
--- a/infrastructure/roles/k3s/tasks/sync.yml
+++ b/infrastructure/roles/k3s/tasks/sync.yml
@ -1,4 +1,9 @@
 ---
+
+- name: Set effective k3s data-dir
+  ansible.builtin.set_fact:
+    k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}"
+
 - name: Ensure k3s config exists
  ansible.builtin.stat:
    path: /etc/rancher/k3s/config.yaml
@ -87,21 +92,21 @@

 - name: Ensure k3s server directory exists
  ansible.builtin.file:
-    path: /var/lib/rancher/k3s/server
+    path: "{{ k3s_effective_data_dir }}/server"
    state: directory
    mode: "0755"
  when: k3s_sync_tls_bundle_present | default(false)

 - name: Remove existing k3s tls directory before sync
  ansible.builtin.file:
-    path: /var/lib/rancher/k3s/server/tls
+    path: "{{ k3s_effective_data_dir }}/server/tls"
    state: absent
  when: k3s_sync_tls_bundle_present | default(false)

 - name: Restore k3s tls bundle
  ansible.builtin.unarchive:
    src: "{{ k3s_sync_tls_bundle }}"
-    dest: /var/lib/rancher/k3s/server
+    dest: "{{ k3s_effective_data_dir }}/server"
    owner: root
    group: root
  when: k3s_sync_tls_bundle_present | default(false)
--- a/infrastructure/roles/k3s/tasks/validate_args.yml
+++ b/infrastructure/roles/k3s/tasks/validate_args.yml
@ -15,3 +15,12 @@
        | select('search', '^etcd-timeout=')
        | list
      }}
+
+- name: Validate datastore configuration (embedded etcd vs external datastore)
+  ansible.builtin.assert:
+    that:
+      - not ((k3s_cluster_init | default(false) | bool) and ((k3s_datastore_endpoint | default('')) | length > 0))
+    fail_msg: >-
+      Invalid k3s datastore config: k3s_cluster_init=true cannot be used with an external datastore.
+      Set k3s_cluster_init: false (or unset k3s_datastore_endpoint).
+  when: k3s_role == 'server'
--- a/infrastructure/roles/k3s/templates/config.yaml.j2
+++ b/infrastructure/roles/k3s/templates/config.yaml.j2
@ -1,14 +1,28 @@
-{% if k3s_role == 'server' %}
-{% if k3s_cluster_init | bool %}
+{% set _k3s_server_line_needed = (k3s_role != 'server') or ((k3s_role == 'server') and (not (k3s_cluster_init | bool)) and (k3s_server_url is defined) and k3s_server_url) %}
+{% if k3s_role == 'server' and k3s_cluster_init | bool %}
 cluster-init: true
-{% elif k3s_server_url %}
+{% endif %}
+{% if _k3s_server_line_needed %}
 server: "{{ k3s_server_url }}"
 {% endif %}
-{% else %}
-server: "{{ k3s_server_url }}"
+{% set _k3s_effective_token = k3s_token %}
+{% if k3s_role == 'server' and (k3s_server_token is defined) and k3s_server_token %}
+{% set _k3s_effective_token = k3s_server_token %}
+{% endif %}
+{% if _k3s_effective_token %}
+token: "{{ _k3s_effective_token }}"
+{% endif %}
+{% if k3s_role == 'server' and k3s_datastore_endpoint is defined and k3s_datastore_endpoint | length %}
+datastore-endpoint: "{{ k3s_datastore_endpoint }}"
+{% if k3s_datastore_cafile is defined and k3s_datastore_cafile | length %}
+datastore-cafile: "{{ k3s_datastore_cafile }}"
+{% endif %}
+{% if k3s_datastore_certfile is defined and k3s_datastore_certfile | length %}
+datastore-certfile: "{{ k3s_datastore_certfile }}"
+{% endif %}
+{% if k3s_datastore_keyfile is defined and k3s_datastore_keyfile | length %}
+datastore-keyfile: "{{ k3s_datastore_keyfile }}"
 {% endif %}
-{% if k3s_token %}
-token: "{{ k3s_token }}"
 {% endif %}
 {% if k3s_role == 'server' and k3s_write_kubeconfig_mode %}
 write-kubeconfig-mode: "{{ k3s_write_kubeconfig_mode }}"
--- a/installer/core/actions.py
+++ b/installer/core/actions.py
@ -48,6 +48,7 @@ from installer.core.env import (
    _deployment_target_label,
    _default_opentofu_pipeline_url,
    _format_ollama_host,
+    _host_from_url,
    _render_prole_cfg,
    _k3d_prole_data_volume_args,
    _sync_opentofu_pipeline,
@ -173,7 +174,18 @@ class ProleInstallerBase:
        )
        if not ns:
            ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip()
-        return ns or "default"
+
+        if ns:
+            return ns
+
+        # In k3s service clusters, core services live in knoe-system by default.
+        try:
+            if self._deployment_mode() == "k3s":
+                return "knoe-system"
+        except Exception:
+            pass
+
+        return "default"

    # ------------------------------------------ k3s connection helpers
    def _read_k3s_cfg_values(
@ -1831,9 +1843,29 @@ class ProleSilentInstaller(ProleInstallerBase):

    def _ensure_local_registry_available(self):
        """Ensure a local k3d registry is running and return (host_registry, cluster_registry)."""
-        if not _local_registry_enabled(self._get_input("init_cluster.cluster_env", "")):
+        env_hint = self._get_input("init_cluster.cluster_env", "")
+        mode = _deployment_mode_from_env(env_hint)
+        if not _local_registry_enabled(env_hint):
            self.log("[SKIP] Local registry disabled; skipping registry setup.")
            return None
+
+        if mode == "k3s":
+            # In k3s mode we must not use localhost registries or k3d registry containers.
+            server, _token = _resolve_k3s_connection_fn(cfg_path=self.cfg_path)
+            host = _host_from_url(server)
+            if not host:
+                self.err(
+                    "[WARN] k3s mode but K3S_SERVER_URL/PROLE_K3S_SERVER not set; cannot resolve registry."
+                )
+                return None
+            host_registry = f"{host}:5000"
+            cluster_registry = host_registry
+            self.local_registry_url = host_registry
+            self.local_registry_internal = cluster_registry
+            self.prole_cfg_data.setdefault("Docker Build", {})["LOCAL_REGISTRY"] = host_registry
+            self.prole_cfg_data["Docker Build"]["LOCAL_REGISTRY_INTERNAL"] = cluster_registry
+            return host_registry, cluster_registry
+
        reg_name = "prole-registry"
        registry_container = f"k3d-{reg_name}"
        host_registry = "localhost:5000"
@ -3974,7 +4006,15 @@ class ProleSilentInstaller(ProleInstallerBase):
                    needs_prompt = True

            if needs_prompt:
-                new_pw = self._prompt_for_master_password()
+                # In true silent/non-interactive runs (e.g. CI/tests), prompting will fail.
+                # Prefer generating a strong password automatically when stdin is not a TTY.
+                if getattr(sys.__stdin__, "isatty", lambda: False)():
+                    new_pw = self._prompt_for_master_password()
+                else:
+                    self.err(
+                        "[WARN] Silent install requires a database master password but stdin is non-interactive; generating one automatically."
+                    )
+                    new_pw = secrets.token_urlsafe(24)
                self.inputs["init_password.db_password"] = new_pw
                self.inputs["init_password.db_password_confirm"] = new_pw
                # Save immediately to prole.cfg so subsequent steps/scripts see it.
--- a/installer/core/env.py
+++ b/installer/core/env.py
@ -614,6 +614,20 @@ def _expand_path(val: str | None) -> str:
    return os.path.expandvars(os.path.expanduser(str(val)))


+def _host_from_url(val: str | None) -> str:
+    raw = (val or "").strip()
+    if not raw:
+        return ""
+    # urlparse requires a scheme to reliably detect hostnames.
+    if "://" not in raw:
+        raw = f"https://{raw}"
+    try:
+        parsed = urllib.parse.urlparse(raw)
+    except Exception:
+        return ""
+    return (parsed.hostname or "").strip()
+
+
 _CFG_VAR_PATTERN = re.compile(r"\$(\w+)|\$\{(\w+)\}")


@ -874,7 +888,7 @@ def _cluster_env_radio_value(env: str | None) -> str:
 def _deployment_target_label(env: str | None) -> str:
    key = _normalize_cluster_env(env)
    if key == "dev":
-        return "knoe-dev-cluster"
+        return "prole-dev-cluster"
    if key == "service":
        return "prole-service-cluster"
    if key == "prod":
@ -1243,6 +1257,30 @@ def _default_opentofu_pipeline_url() -> str:
    url = (
        os.environ.get("PROLE_OPENTOFU_URL") or os.environ.get("OPENTOFU_URL") or ""
    ).strip()
+    mode = (
+        os.environ.get("PROLE_MODE")
+        or os.environ.get("DEPLOYMENT_MODE")
+        or os.environ.get("DEPLOY_MODE")
+        or ""
+    ).strip().lower()
+
+    # In k3s mode we must not rely on localhost/port-forwards.
+    if mode == "k3s":
+        if url:
+            host = _host_from_url(url)
+            if host and host not in ("localhost", "127.0.0.1"):
+                return url
+            # If user provided a localhost URL in k3s, ignore it and try to derive.
+            url = ""
+
+        k3s_server = (
+            os.environ.get("PROLE_K3S_SERVER") or os.environ.get("K3S_SERVER_URL") or ""
+        ).strip()
+        k3s_host = _host_from_url(k3s_server)
+        if k3s_host:
+            return f"http://{k3s_host}:8080"
+        return ""
+
    return url or "http://127.0.0.1:8080"


--- a/installer/milestone.py
+++ b/installer/milestone.py
@ -79,8 +79,8 @@ class Milestone(ABC):
        project_root = inst_config.PROJECT_ROOT
        env["PROLE_HOME"] = str(project_root)
        env["PROLE_SERVICE"] = str(project_root)
-        # Namespace is read from prole.cfg by scripts — never set in subprocess env
-        namespace = state.inputs.get("init_password.db_namespace", "")
+        namespace = (state.inputs.get("init_password.db_namespace", "") or "").strip()
+        env["NAMESPACE"] = namespace or "default"

        # Paths
        env["PROLE_CONF"] = state.inputs.get(
@ -104,7 +104,7 @@ class Milestone(ABC):
        if not service_ns:
            service_ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip()
        if not service_ns:
-            service_ns = namespace or "default"
+            service_ns = env["NAMESPACE"]
        env["SERVICE_NAMESPACE"] = service_ns

        from installer.core.env import _deployment_mode_from_env
--- a/installer/ui/screens/cluster.py
+++ b/installer/ui/screens/cluster.py
@ -309,7 +309,7 @@ class ClusterScreenMixin:
                self,
                x_label,
                y,
-                "Common Services Namespace:",
+                "Common Core Services Namespace:",
                fill="black",
                font=("SF Pro Text", 12, "bold"),
            )
@ -523,7 +523,15 @@ class ClusterScreenMixin:
                ns = ""
        if not ns:
            ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip()
-        return ns or "default"
+
+        default_ns = "default"
+        try:
+            if self._cluster_env_key() == "service":
+                default_ns = "knoe-system"
+        except Exception:
+            pass
+
+        return ns or default_ns

    def _on_cluster_env_change(self, *args):
        self._set_deploy_target_from_cluster_env()
--- a/installer/ui/screens/docker.py
+++ b/installer/ui/screens/docker.py
@ -8,10 +8,13 @@ import subprocess
 import sys
 import time
 from pathlib import Path
+
+from installer.config import get_docker_build_platform_args
 from installer.core.env import (
    PROJECT_ROOT,
    _collect_images_from_files,
    _deployment_mode_from_env,
+    _host_from_url,
    _http_ping_registry,
    _k3d_prole_data_volume_args,
    _local_registry_enabled,
@ -24,6 +27,24 @@ from installer.core.env import (
 class DockerScreenMixin:
    """Docker image build, registry management and image pre-pull."""

+    def _k3s_registry_hostport(self) -> str:
+        # For k3s, registry access must never rely on localhost.
+        # Derive the registry host from the configured k3s API server URL.
+        cfg = self.prole_cfg_data or {}
+        server = (
+            (cfg.get("Global", {}) or {}).get("PROLE_K3S_SERVER")
+            or (cfg.get("Global", {}) or {}).get("K3S_SERVER_URL")
+            or (cfg.get("Initialize Cluster", {}) or {}).get("K3S_SERVER_URL")
+            or (cfg.get("Service Cluster (k3s)", {}) or {}).get("K3S_SERVER_URL")
+            or os.environ.get("PROLE_K3S_SERVER")
+            or os.environ.get("K3S_SERVER_URL")
+            or ""
+        )
+        host = _host_from_url(server)
+        if not host:
+            return ""
+        return f"{host}:5000"
+
    def check_docker_running(self):
        """Check if Docker is running"""
        try:
@ -37,6 +58,14 @@ class DockerScreenMixin:
        Returns the registry URL (host:port) to be used for tagging/pushing.
        For Dev, may create a local k3d-managed registry; for other envs, still prefer the external if reachable.
        """
+        mode = _deployment_mode_from_env(
+            (self.prole_cfg_data.get("Global", {}) or {}).get("DEPLOYMENT_MODE")
+            or env
+        )
+        if mode == "k3s":
+            # k3s must use the k3s registry (never localhost/k3d registries).
+            return self._k3s_registry_hostport()
+
        # Prefer the shared registry if reachable
        if _http_ping_registry("k8s.prole.org", 5000):
            return "k8s.prole.org:5000"
@ -127,6 +156,36 @@ class DockerScreenMixin:
        mode = _deployment_mode_from_env(
            self.prole_cfg_data.get("Global", {}).get("DEPLOYMENT_MODE")
        )
+        if mode == "k3s":
+            k3s_registry = self._k3s_registry_hostport()
+            if not k3s_registry:
+                _log("k3s mode but K3S_SERVER_URL/PROLE_K3S_SERVER not set; cannot resolve registry.\n")
+                if log_fp:
+                    log_fp.close()
+                return None
+            host_registry = k3s_registry
+            cluster_registry = k3s_registry
+            if not self.check_docker_running():
+                _log("Docker not running; cannot push/prepull images to k3s registry.\n")
+                if log_fp:
+                    log_fp.close()
+                return None
+            self.local_registry_url = host_registry
+            self.local_registry_internal = cluster_registry
+            try:
+                if "Docker Build" not in self.prole_cfg_data:
+                    self.prole_cfg_data["Docker Build"] = {}
+                self.prole_cfg_data["Docker Build"]["LOCAL_REGISTRY"] = host_registry
+                self.prole_cfg_data["Docker Build"][
+                    "LOCAL_REGISTRY_INTERNAL"
+                ] = cluster_registry
+                self.safe_after(self._save_prole_cfg)
+            except Exception:
+                pass
+            if log_fp:
+                log_fp.close()
+            return host_registry, cluster_registry
+
        if mode == "k3d":
            cluster_registry = f"{registry_container}.localhost:5000"
        else:
--- a/k8s/argocd/install.yaml
+++ b/k8s/argocd/install.yaml
@ -32496,7 +32496,7 @@ spec:
      initContainers:
      - args:
        - /bin/cp --update=none /usr/local/bin/argocd /var/run/argocd/argocd && /bin/ln
-          -s /var/run/argocd/argocd /var/run/argocd/argocd-cmp-server
+          -sf /var/run/argocd/argocd /var/run/argocd/argocd-cmp-server
        command:
        - sh
        - -c
--- a/k8s/opentofu/deployment.yaml
+++ b/k8s/opentofu/deployment.yaml
@ -105,6 +105,7 @@ kind: Service
 metadata:
  name: opentofu
 spec:
+  type: LoadBalancer
  selector:
    app: opentofu
  ports:
--- a/k8s/prole/prole-db-recovery.yaml.tpl
+++ b/k8s/prole/prole-db-recovery.yaml.tpl
@ -5,7 +5,7 @@ metadata:
  name: prole-db
 spec:
  instances: 3
-  imageName: k3d-prole-registry.localhost:5000/prole-db:18-101
+  imageName: prole-db:18-102
  postgresUID: 100
  postgresGID: 101
  maxSyncReplicas: 1
--- a/k8s/prole/prole-db.yaml
+++ b/k8s/prole/prole-db.yaml
@ -4,7 +4,7 @@ metadata:
  name: prole-db
 spec:
  instances: 3
-  imageName: k3d-prole-registry.localhost:5000/prole-db:18-101
+  imageName: prole-db:18-102
  postgresUID: 100
  postgresGID: 101
  maxSyncReplicas: 1
--- a/prole-db/.version
+++ b/prole-db/.version
@ -1 +1 @@
-101
+102
--- a/tests/test_argocd_manifest_idempotency.py
+++ b/tests/test_argocd_manifest_idempotency.py
@ -0,0 +1,29 @@
+import re
+from pathlib import Path
+
+
+def test_argocd_repo_server_copyutil_symlink_is_idempotent():
+    manifest_path = Path(__file__).resolve().parents[1] / "k8s" / "argocd" / "install.yaml"
+    text = manifest_path.read_text(encoding="utf-8")
+
+    # Ensure the repo-server initContainer makes the cmp-server symlink idempotently.
+    # The repo-server can be recreated while `/var/run/argocd` is persisted (hostPath),
+    # so an unconditional `ln -s` will fail with "File exists".
+    assert "argocd-cmp-server" in text
+
+    safe_link_re = re.compile(
+        r"/bin/ln\s+-sf\s+/var/run/argocd/argocd\s+/var/run/argocd/argocd-cmp-server",
+        re.MULTILINE,
+    )
+    assert safe_link_re.search(text), (
+        "Expected ArgoCD repo-server copyutil initContainer to use `ln -sf` when creating "
+        "`/var/run/argocd/argocd-cmp-server`."
+    )
+
+    unsafe_link_re = re.compile(
+        r"/bin/ln\s+-s\s+/var/run/argocd/argocd\s+/var/run/argocd/argocd-cmp-server",
+        re.MULTILINE,
+    )
+    assert not unsafe_link_re.search(text), (
+        "Found an unconditional `ln -s` for `argocd-cmp-server` which is not idempotent."
+    )
--- a/tests/test_k3s_common_core_namespace_and_repair.py
+++ b/tests/test_k3s_common_core_namespace_and_repair.py
@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from installer.core.actions import ProleSilentInstaller
+from installer.core.controller import ProleController
+
+
+def test_silent_installer_service_namespace_defaults_to_knoe_system_for_k3s(tmp_path, monkeypatch):
+    """Regression: k3s service clusters must not default core services into `default`."""
+
+    monkeypatch.delenv("SERVICE_NAMESPACE", raising=False)
+
+    c = ProleController(tmp_path)
+    installer = ProleSilentInstaller(c)
+    installer.inputs["init_cluster.cluster_env"] = "service"
+    installer.prole_cfg_data = {"Global": {}}
+
+    assert installer._service_namespace() == "knoe-system"
+
+
+def test_repair_pipeline_has_k3s_dedupe_guard():
+    """Repair pipeline should reconcile accidental `default` deployments in k3s."""
+
+    text = Path("etc/repair_pipeline.sh").read_text(encoding="utf-8")
+
+    # k3s default namespace normalization
+    assert "SERVICE_NAMESPACE=\"knoe-system\"" in text
+
+    # cross-namespace dedupe pass
+    assert "dedupe_common_core_default_namespace" in text