Add node management workflow and installer config persistence updates

- add tools/manage-node.sh for k3s node add/remove automation with inventory reconciliation

- include kubectl drain flag compatibility and --ssh-host targeting for uninstall workflows

- persist installer cfg/env values and update deploy/database/cfg UI handling

- refresh related inventory/network artifacts and expand installer/database test coverage

Co-authored-by: Junie <junie@jetbrains.com>

conf/service/prole.cfg

@@ -1,5 +1,5 @@
 ; Prole Master Configuration File
-; Generated by install.py on 2026-03-27 22:43:14
+; Generated by install.py on 2026-03-29 10:17:03
 ; This file is used as input for Ansible deployment and k8s cluster creation.
 
 [User]
@@ -13,10 +13,12 @@ build.run_build = false
 database_options.distribution = percona
 database_options.ext.dblink = true
 database_options.ext.pg_buffercache = true
+database_options.ext.pg_cron = true
 database_options.ext.pg_freespacemap = true
 database_options.ext.pg_repack = true
 database_options.ext.pg_stat_monitor = true
 database_options.ext.pg_stat_statements = true
+database_options.ext.pg_tde = true
 database_options.ext.pgaudit = true
 database_options.ext.pgbadger = true
 database_options.ext.pgcrypto = true
@@ -36,22 +38,22 @@ dependencies.opentofu.install = true
 dependencies.python.install = true
 dependencies.verify_all = false
 disk_selection.disk_type = local
-disk_selection.local_path = $HOME/dev/prole/prole-tools-app/dist
+disk_selection.local_path = /Users/chrisfu/dev/prole/prole-tools-app/dist
 disk_selection.removable_mount = 
 env_setup.CLUSTER_NAME = ${CLUSTER_NAME}
 env_setup.DATABASE_NAMESPACE = ${DATABASE_NAMESPACE}
-env_setup.PROLE_CONF = $HOME/dev/prole/conf
+env_setup.PROLE_CONF = /Users/chrisfu/dev/prole/conf
-env_setup.PROLE_DATA = $HOME/dev/prole/data
+env_setup.PROLE_DATA = /Users/chrisfu/dev/prole/data
-env_setup.PROLE_HOME = $HOME/dev/prole
+env_setup.PROLE_HOME = /Users/chrisfu/dev/prole
-env_setup.PROLE_LOGS = $HOME/dev/prole/logs
+env_setup.PROLE_LOGS = /Users/chrisfu/dev/prole/logs
-env_setup.PROLE_SERVICE = $HOME/dev/prole/etc
+env_setup.PROLE_SERVICE = /Users/chrisfu/dev/prole/etc
 init_cluster.argocd_enabled = false
 init_cluster.at_rest_encryption_enabled = true
 init_cluster.cluster_env = service
 init_cluster.deployment_target = prole-service-cluster
 init_cluster.gitops_enabled = false
 init_cluster.k3s_server_url = https://myrddin.prole.org:6443
-init_cluster.k3s_token = ${PROLE_SECRET:v1:PrN9vxJojFfMVhPP:IkJHdXS8MLxJQmJwfoLhXYHHu3mWKNZ526Iwnby5MUQ041IZF_jrFLnjCWLpS9TiKpnrX_QqAJBFaUnFwEQQ-lKh-jylMNDIFRfwPrPu1is1CwACtbS8ShxedO2t9KWlmxdMgioVDJIqQwvEDx9H_OisARTzxRcoo9iM6Q==}
+init_cluster.k3s_token = ${PROLE_SECRET:v1:uRoZnuXD_SOTEa5u:TaOfxmViSVZzc2usW_csMhqsGEfCHse_dstrJMSTwZ1C5b-s7rlsrpledht-m8FqQX6AqpxzTudZTvhtcd1u-7n082sDcdwcUhdGCj3c0Ve1kGk7lIXA1Sl0RUYL4DNST1u2k_riFFOPWZgyk3GKUFvD-OB2K8_hC5jtxQ==}
 init_cluster.kerberos_enabled = true
 init_cluster.mode = k3s
 init_cluster.start_cluster = true
@@ -62,14 +64,14 @@ init_db_build.run_build = true
 init_password.cluster_name = ${CLUSTER_NAME}
 init_password.db_host_port = 5432
 init_password.db_namespace = ${DATABASE_NAMESPACE}
-init_password.db_password = ${PROLE_SECRET:v1:V2IgV0LfqFnw5jXl:dme7psI1WBaA82guzmxgZXQeq1ZfqSjLSPcYYc0Q77ryYHhUYhMcjojyHQq5lriz}
+init_password.db_password = ${PROLE_SECRET:v1:c3HtcAjZ2N3bbX0L:4M-580OJh6mlNbT5taYiLLiEd2WpD_iyxv-p4A4sqHSb_sXwe1zM9-qEPhE8xS0Q}
-init_password.db_password_confirm = ${PROLE_SECRET:v1:V2IgV0LfqFnw5jXl:dme7psI1WBaA82guzmxgZXQeq1ZfqSjLSPcYYc0Q77ryYHhUYhMcjojyHQq5lriz}
+init_password.db_password_confirm = ${PROLE_SECRET:v1:c3HtcAjZ2N3bbX0L:4M-580OJh6mlNbT5taYiLLiEd2WpD_iyxv-p4A4sqHSb_sXwe1zM9-qEPhE8xS0Q}
 init_password.db_username = root
 init_password.generate_ssh_key = true
 init_scripts.run_scripts = true
 kerberos_config.enabled = true
 kerberos_config.kdc = 10.0.0.3
-kerberos_config.password = ${PROLE_SECRET:v1:9cfA9AAQDSQ-c-xk:hBOKrfdBv_sPlWuF-FHwL5D6qrEjvB5fC10j6Yy3_oYK3H-B}
+kerberos_config.password = ${PROLE_SECRET:v1:HboDpQsTUGZBuddn:IS55-zeRNwRzK5jEmNZb_GCyMIQav_OvNBTH4NeQe9EO8rB7}
 kerberos_config.realm = PROLE.ORG
 kerberos_config.test_connection = false
 kerberos_config.user = administrator
@@ -85,19 +87,17 @@ CLUSTER_ENV = service
 CLUSTER_NAME = knoe-db
 CNPG_ELIGIBLE_NODES = merlin.prole.org,myrddin.prole.org,pi.prole.org
 CNPG_STAGE1_NODE = merlin.prole.org
-DATABASE_NAMESPACE = knoe-db-17-7-043-18-140-17-7-043-18-140
+DATABASE_NAMESPACE = knoe-db-17-7-043-18-140-17-7-043-18-140-17-7-043-18-140
 DB_HOST_PORT = 5432
-DB_PASSWORD = ${PROLE_SECRET:v1:_nC-mrjMrpmQ7G8p:CWA664RTEld0LhNcW8O43vT4OnVPE6DHruWJ-CQFlM-zKp5TveTzw2Zrw3nbyBo6}
+DB_PASSWORD = ${PROLE_SECRET:v1:UqH_FpRvoVe1XdFj:d0p6Gdbr7V1jImZQgYUFiDoHfaCoHK_cRonvQUsCPHM2BfTzHJgeJdLUdS5eZYKl}
 DEPLOYMENT_MODE = k3s
 DEPLOYMENT_TARGET = prole-service-cluster
-DOCKER_IMPORT_DIR = 
 DOCKER_PRELOAD = false
 KNOE_DB_USER = root
 KUBECONTEXT = knoe.dev.prole.org
 OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
 PROLE_K3S_SERVER = https://myrddin.prole.org:6443
-PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:PjwYLtK16-ti4xUA:L_LZPKg3JPktDa5H_LIwW0EAuHUd_mGKQBAOMidD38-qCLHzgzRmftKACmtKcvlcFmMV5D9yzVfqBREcZ285vtjtbj7yDSNO0gcrldH30G2OOvlr3rJ2LrkbqSU4shsiyzS0H4i-y8btCQ8wo9YDsSnAt017CNorDeLk2A==}
+PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:RSjajq3-2i4bM1nh:Ji7aiKn6TnNcrNd7kPL6Hv4nRwQBUIbDcsnZSCmqLq8QDd9evxZeXNzjedJ6KILEUkHTtEpUTOMF0OWY7EJoKmXzkcanCMCtxQd_C_KBX-TvBXCwoP54cvIXOJo2KTK2R06yaFMqaY4jKEGhzkDLeHSaqVfeiutEtDn_Qw==}
-PROLE_OPENTOFU_URL = http://127.0.0.1:8080
 SERVICE_NAMESPACE = knoe-system
 SYNOLOGY_ROOTS = /synology/d001,/synology/d002,/synology/d003,/synology/d004
 
@@ -111,10 +111,10 @@ SYNOLOGY_ROOTS = /synology/d001,/synology/d002,/synology/d003,/synology/d004
 AD_DC_HOST = myrddin.prole.org
 AD_DC_IP = 10.0.0.3
 ANSIBLE_DOMAIN = prole.org
-ANSIBLE_INFRASTRUCTURE = $HOME/dev/prole/infrastructure
+ANSIBLE_INFRASTRUCTURE = /Users/chrisfu/dev/prole/infrastructure
-ANSIBLE_INVENTORY = $HOME/dev/prole/infrastructure/inventory
+ANSIBLE_INVENTORY = /Users/chrisfu/dev/prole/infrastructure/inventory
 ANSIBLE_REALM = PROLE.ORG
-ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
+ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org","gandalf.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org","gandalf.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org","gandalf.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"],"k3s_hosts":["gandalf.prole.org"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["gandalf.prole.org","k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
 KDC_ANSIBLE_DETECTED = 10.0.0.3
 KDC_AUTO_DETECTED = 10.0.0.3
 KERBEROS_AUTO_ENABLED = True
@@ -123,11 +123,11 @@ KERBEROS_AUTO_ENABLED = True
 ; No configuration values captured yet for this section.
 
 [System Environment]
-PROLE_CONF = $HOME/dev/prole/conf
+PROLE_CONF = /Users/chrisfu/dev/prole/conf
-PROLE_DATA = $HOME/dev/prole/data
+PROLE_DATA = /Users/chrisfu/dev/prole/data
-PROLE_HOME = $HOME/dev/prole
+PROLE_HOME = /Users/chrisfu/dev/prole
-PROLE_LOGS = $HOME/dev/prole/logs
+PROLE_LOGS = /Users/chrisfu/dev/prole/logs
-PROLE_SERVICE = $HOME/dev/prole/etc
+PROLE_SERVICE = /Users/chrisfu/dev/prole/etc
 
 [Monitoring]
 ; No configuration values captured yet for this section.
@@ -137,7 +137,6 @@ PROLE_SERVICE = $HOME/dev/prole/etc
 
 [Ollama]
 OLLAMA_HOST = http://fairyland.prole.org:11434
-OLLAMA_SERVERS = fairyland.prole.org:11434,k3d.localhost:11434,morgoth.prole.org:11434
 OLLAMA_SERVER_HOST = fairyland.prole.org
 OLLAMA_SERVER_PORT = 11434
 
@@ -156,7 +155,7 @@ SUPABASE_ENABLED = False
 [Initialize Cluster]
 ENVIRONMENT = service
 K3S_SERVER_URL = https://myrddin.prole.org:6443
-K3S_TOKEN = ${PROLE_SECRET:v1:dxqbZ1XsfonPQ5Ux:Osu-4qhl_6o0od85N49d_BXu5PsOQm9mzwsoRrsjCQzKqH30p5L8sUnbEy2jUw1VyLiR3-oP8P1H0lU5TMgSZS89vcTrNMc2aYfcZwRNvHZLpQyxn-3zx81DGKQ8yNJE3MYmQAVYkhDqbash1ekSNctl3noWtjU93GnCNw==}
+K3S_TOKEN = ${PROLE_SECRET:v1:8jsokjD9t68PVO_4:pEQenJTlcY1lM985bnfKh2OyrsakuK2iKI_lqeHFFds9CNoGXF9oRIfcpWgJFzv1b7_w-xGgMSMipr8pL1x_nZA_WllKI8Jg52bFDhR2TrwIk5it0yykU4KsGg0n7sIpatVfGnjBqsscwi73DE8Y-sB_o4tZPnfH4La11w==}
 
 [Dev Cluster (k3d)]
 CLUSTER_ENV = dev
@@ -168,16 +167,14 @@ MODE = k3d
 CLUSTER_ENV = prole-service-cluster
 DISPLAY_NAME = prole-service-cluster
 K3S_SERVER_URL = https://myrddin.prole.org:6443
-K3S_TOKEN = ${PROLE_SECRET:v1:ChNsNodubwNnQG0I:h3ZC5FcPKzpqIuV1n2Sp22eaS5RgYeFHSQm0gGW1GpV_Ry_oLING88f9kFUIPbipfRJmepW5730HlfmZsZ0NlwLg3VKUyel-M0gYCOuak4SHY7FOg5mSKGvhCyuMGsKNEmNi1QwwmFFP8P3bVAPFKk9vedAFUzjGZbnX_A==}
+K3S_TOKEN = ${PROLE_SECRET:v1:Z0rjW966HyWI1T-O:AO7EJcZvX1OwMryjwvQLdo6shi0_pAUVO-BBVLlcAOj06XoT5gJvXPsDVcTHkCc8_EwWeZfButvKBkoqhhTHqTQeq4DqG62lBZO2b5sxaK8WlfduKE23Ej-8SxXvyH3luUuif6pLgImfyloRGCKqqqcF9NMPdNRVILpRBA==}
 MODE = k3s
-PIPELINE_URL = http://127.0.0.1:8080
 
 [Prod Cluster (k8s)]
-ARTIFACTS_DIR = $HOME/dev/prole/data/staging
+ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging
 CLUSTER_ENV = prole-prod-cluster
 DISPLAY_NAME = prole-prod-cluster
 MODE = k8s
-PIPELINE_URL = http://127.0.0.1:8080
 
 [Docker Build]
 ; No configuration values captured yet for this section.

docs/installer-config-persistence.md

@@ -0,0 +1,15 @@
+### Installer-managed configuration scope
+
+- The installer persists only explicit operator inputs and explicit connection handoff values.
+- For service/k3s-hosts workflows, values like `PROLE_K3S_SERVER` and other explicit user-entered settings are saved.
+- Generated local/dev endpoint mappings, generated service hostnames, and placeholder-derived values are not persisted as installer-owned facts.
+
+### Live service endpoints
+
+- Live k3s service endpoints are not installer-managed configuration.
+- Endpoint host/port discovery for running services is performed against the live cluster rather than persisted as static installer defaults.
+
+### Downstream behavior
+
+- Later Ansible phases are expected to discover current service endpoints from the running cluster.
+- Installer replay/config files remain focused on explicit intent and handoff values instead of dynamic runtime topology.

infrastructure/inventory/hosts.ini

@@ -48,3 +48,6 @@ mariadb_replica
 
 [merlin_bootstrap]
 merlin ansible_host=10.0.0.36 ansible_user=ansible
+
+[k3s_hosts]
+gandalf.prole.org

knoe-db/Dockerfile

@@ -81,7 +81,6 @@ RUN set -eux; \
   apt-get install -y --no-install-recommends percona-pg-stat-monitor18; \
   apt-get install -y --no-install-recommends percona-pgbadger; \
   apt-get install -y --no-install-recommends percona-pg-tde18; \
-  apt-get install -y --no-install-recommends percona-postgresql-18-cron; \
   apt-get install -y --no-install-recommends \
     percona-postgresql-contrib \
     freetds-dev \

knoe-db/Dockerfile.percona.template

@@ -76,7 +76,6 @@ RUN set -eux; \
   apt-get update; \
 {{EXTENSION_INSTALL_STEPS}}
   apt-get install -y --no-install-recommends percona-pg-tde{{MAJOR_VERSION}}; \
-  apt-get install -y --no-install-recommends percona-postgresql-{{MAJOR_VERSION}}-cron; \
   apt-get install -y --no-install-recommends \
     percona-postgresql-contrib \
     freetds-dev \

knoe/config.py

@@ -19,7 +19,7 @@ import subprocess
 import sys
 import urllib.request
 from pathlib import Path
-from typing import Tuple, Optional
+from typing import Optional, Tuple
 
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 
@@ -444,8 +444,127 @@ def _expand_path(val: str | None) -> str:
     return out
 
 
-def _collect_cfg_vars(cfg: any) -> dict:
+def _normalize_persistence_mode(mode: str | None) -> str:
+    raw = (mode or "").strip().lower()
+    if raw in {"dev", "k3d", "local"}:
+        return "dev"
+    if raw in {"service", "k3s", "k3s-hosts", "prole-service-cluster"}:
+        return "k3s-hosts"
+    if raw in {"prod", "k8s", "prole-prod-cluster"}:
+        return "prod"
+    return raw
+
+
+def _is_simple_template_placeholder(value: str | None) -> bool:
+    v = (value or "").strip()
+    return bool(re.match(r"^\$\{[A-Za-z_][A-Za-z0-9_]*\}$", v))
+
+
+def _looks_like_localhost_value(value: str | None) -> bool:
+    v = (value or "").strip().lower()
+    if not v:
+        return False
+    return "localhost" in v or "127.0.0.1" in v
+
+
+def _looks_like_generated_service_endpoint_key(key: str | None) -> bool:
+    k = (key or "").strip().upper()
+    if not k:
+        return False
+    return (
+        k == "SERVICE_HOSTNAME"
+        or k.endswith("_SERVICE_ENDPOINT")
+        or k.endswith("_SERVICE_HOSTNAME")
+        or (k.startswith("SERVICE_") and k.endswith("_HOSTNAME"))
+    )
+
+
+def _normalize_cfg_value_for_persistence(
+    section: str,
+    key: str,
+    value: str | None,
+    *,
+    mode: str | None = None,
+    explicit: bool = False,
+) -> str | None:
+    """Return a persistable value, or `None` if it must not be persisted.
+
+    Policy:
+    - Dev/k3d: retain existing behavior.
+    - k3s-hosts/Service and Prod: keep explicit inputs/handoff values, but drop
+      derived placeholders, generated endpoint hostnames, localhost-based
+      derived values, and k3d/dev endpoint assumptions.
+    """
+
+    raw_value = "" if value is None else str(value)
+    trimmed = raw_value.strip()
+    if not trimmed:
+        return None
+
+    normalized_mode = _normalize_persistence_mode(mode)
+    if normalized_mode not in {"k3s-hosts", "prod"}:
+        return trimmed
+
+    normalized_key = (key or "").strip().upper()
+    if _is_simple_template_placeholder(trimmed):
+        return None
+
+    if normalized_key == "SERVICE_NAMESPACE" and not explicit:
+        return None
+
+    if _looks_like_generated_service_endpoint_key(normalized_key) and not explicit:
+        return None
+
+    if _looks_like_localhost_value(trimmed) and not explicit:
+        return None
+
+    if "k3d.localhost" in trimmed.lower() and not explicit:
+        return None
+
+    return trimmed
+
+
+def _filter_cfg_values_for_persistence(
+    section: str,
+    values: dict | None,
+    *,
+    mode: str | None = None,
+    explicit_keys: set[str] | None = None,
+) -> dict[str, str]:
+    """Filter section values using mode-aware persistence rules."""
+
+    if not values or not isinstance(values, dict):
+        return {}
+
+    explicit_keys_upper = {str(k).strip().upper() for k in (explicit_keys or set())}
+    filtered: dict[str, str] = {}
+    for key, value in values.items():
+        key_str = str(key)
+        normalized = _normalize_cfg_value_for_persistence(
+            section,
+            key_str,
+            "" if value is None else str(value),
+            mode=mode,
+            explicit=key_str.strip().upper() in explicit_keys_upper,
+        )
+        if normalized is None:
+            continue
+        filtered[key_str] = normalized
+
+    return filtered
+
+
+def _collect_cfg_vars(
+    cfg: any,
+    *,
+    mode: str | None = None,
+    explicit_keys: set[tuple[str, str]] | None = None,
+) -> dict:
     variables: dict[str, str] = {}
+    explicit_pairs = {
+        (str(sec).strip().upper(), str(key).strip().upper())
+        for sec, key in (explicit_keys or set())
+    }
 
     def _is_placeholder(v: str) -> bool:
         vv = (v or "").strip()
@@ -459,6 +578,17 @@ def _collect_cfg_vars(cfg: any) -> dict:
             # Avoid polluting the variable map with empty/self-referencing placeholders.
             if not s or _is_placeholder(s):
                 continue
+            if mode:
+                norm = _normalize_cfg_value_for_persistence(
+                    section,
+                    str(k),
+                    s,
+                    mode=mode,
+                    explicit=(section.upper(), str(k).strip().upper()) in explicit_pairs,
+                )
+                if norm is None:
+                    continue
+                s = norm
             variables[k] = s
 
     # Global is the baseline. System and User can override, but only with concrete values.
@@ -468,7 +598,12 @@ def _collect_cfg_vars(cfg: any) -> dict:
     return variables
 
 
-def _collect_cfg_vars_from_data(cfg_data: dict | None) -> dict[str, str]:
+def _collect_cfg_vars_from_data(
+    cfg_data: dict | None,
+    *,
+    mode: str | None = None,
+    explicit_keys: set[tuple[str, str]] | None = None,
+) -> dict[str, str]:
     """Collect variable names from a `prole_cfg_data`-style dict.
 
     This mirrors `_collect_cfg_vars()` but operates on the UI/controller's in-memory
@@ -477,6 +612,10 @@ def _collect_cfg_vars_from_data(cfg_data: dict | None) -> dict[str, str]:
     variables: dict[str, str] = {}
     if not cfg_data or not isinstance(cfg_data, dict):
         return variables
+    explicit_pairs = {
+        (str(sec).strip().upper(), str(key).strip().upper())
+        for sec, key in (explicit_keys or set())
+    }
 
     def _is_placeholder(v: str) -> bool:
         vv = (v or "").strip()
@@ -490,6 +629,17 @@ def _collect_cfg_vars_from_data(cfg_data: dict | None) -> dict[str, str]:
             s = str(v or "").strip()
             if not s or _is_placeholder(s):
                 continue
+            if mode:
+                norm = _normalize_cfg_value_for_persistence(
+                    section,
+                    str(k),
+                    s,
+                    mode=mode,
+                    explicit=(section.upper(), str(k).strip().upper()) in explicit_pairs,
+                )
+                if norm is None:
+                    continue
+                s = norm
             variables[str(k)] = s
     return variables
 
@@ -672,7 +822,14 @@ def _try_read_ansible_vault_value(vault_path: Path, key: str) -> str:
     return ""
 
 
-def _update_prole_cfg_value(section: str, key: str, value: str):
+def _update_prole_cfg_value(
+    section: str,
+    key: str,
+    value: str,
+    *,
+    mode: str | None = None,
+    explicit: bool = False,
+):
     # Prefer `$PROLE_CONF/prole.cfg` (single entrypoint) and follow symlink so we
     # update the active environment base file without mutating other environments.
     try:
@@ -703,7 +860,31 @@ def _update_prole_cfg_value(section: str, key: str, value: str):
     if not cfg.has_section(section):
         cfg.add_section(section)
 
-    cfg.set(section, key, value)
+    effective_mode = mode
+    if effective_mode is None:
+        effective_mode = (
+            os.environ.get("DEPLOYMENT_MODE")
+            or os.environ.get("CLUSTER_ENV")
+            or os.environ.get("MODE")
+            or ""
+        )
+
+    normalized_value = _normalize_cfg_value_for_persistence(
+        section,
+        key,
+        value,
+        mode=effective_mode,
+        explicit=explicit,
+    )
+
+    if normalized_value is None:
+        if cfg.has_option(section, key):
+            cfg.remove_option(section, key)
+        with open(write_path, "w") as f:
+            cfg.write(f)
+        return
+
+    cfg.set(section, key, normalized_value)
 
     with open(write_path, "w") as f:
         cfg.write(f)

knoe/core/env.py

@@ -2003,6 +2003,28 @@ def _render_prole_cfg(
     def get_section(section: str, key: str) -> str:
         return str((sections.get(section, {}) or {}).get(key, "") or "").strip()
 
+    def normalize_runtime_mode(value: str) -> str:
+        v = str(value or "").strip().lower()
+        if v in {"dev", "k3d"}:
+            return "dev"
+        if v in {"service", "k3s", "k3s-hosts", "prole-service-cluster"}:
+            return "service"
+        if v in {"prod", "k8s", "prole-prod-cluster"}:
+            return "prod"
+        return v
+
+    runtime_mode = normalize_runtime_mode(
+        first_non_empty(
+            str(globals_to_save.get("DEPLOYMENT_MODE", "")).strip(),
+            get_section("Deployment", "MODE"),
+            get_input("init_cluster.mode"),
+            get_input("build.deploy_env"),
+            str(globals_to_save.get("CLUSTER_ENV", "")).strip(),
+            get_section("Initialize Cluster", "ENVIRONMENT"),
+        )
+    )
+    remote_mode = runtime_mode in {"service", "prod"}
+
     base_home = first_non_empty(
         str(globals_to_save.get("PROLE_HOME", "")).strip(),
         get_section("System Environment", "PROLE_HOME"),
@@ -2050,22 +2072,21 @@ def _render_prole_cfg(
     )
     if not base_cluster_name:
         base_cluster_name = "knoe-db"
-    base_service_namespace = (
+    base_service_namespace = first_non_empty(
-        first_non_empty(
         str(globals_to_save.get("SERVICE_NAMESPACE", "")).strip(),
         get_section("Global", "SERVICE_NAMESPACE"),
     )
-        or base_database_namespace
-    )
     if base_service_namespace == "${SERVICE_NAMESPACE}":
-        base_service_namespace = "default"
+        base_service_namespace = ""
+    if not base_service_namespace and not remote_mode:
+        base_service_namespace = base_database_namespace
 
     base_service_hostname = first_non_empty(
         str(globals_to_save.get("SERVICE_HOSTNAME", "")).strip(),
         get_section("User", "SERVICE_HOSTNAME"),
         get_section("Global", "SERVICE_HOSTNAME"),
     )
-    if not base_service_hostname:
+    if not base_service_hostname and not remote_mode:
         base_service_hostname = "svc.prole.org"
 
     base_supabase_hostname = first_non_empty(

knoe/ui/screens/cfg.py

@@ -28,7 +28,12 @@ from knoe.core.env import (
     _read_k3s_cfg,
     _render_prole_cfg,
 )
-from knoe.config import _collect_cfg_vars, _encrypt_cfg_secret, _expand_cfg_value
+from knoe.config import (
+    _collect_cfg_vars,
+    _encrypt_cfg_secret,
+    _expand_cfg_value,
+    _filter_cfg_values_for_persistence,
+)
 
 
 class ConfigMixin:
@@ -80,6 +85,7 @@ class ConfigMixin:
 
             # Identify global candidates
             mode = self._deployment_mode()
+            persistence_mode = mode or self.cluster_env.get()
             target_label = _deployment_target_label(self.cluster_env.get())
 
             # New policy: do not persist default filesystem paths into prole.cfg.
@@ -142,15 +148,51 @@ class ConfigMixin:
             globals_to_save["DEPLOYMENT_TARGET"] = target_label
             if not (globals_to_save.get("CLUSTER_NAME") or "").strip():
                 globals_to_save["CLUSTER_NAME"] = "knoe-db"
-            globals_to_save["PROLE_K3S_SERVER"] = (
+            globals_to_save["PROLE_K3S_SERVER"] = (self.k3s_server_url.get() or "").strip()
-                self.k3s_server_url.get() or ""
-            ).strip()
             globals_to_save["PROLE_K3S_TOKEN"] = _encrypt_cfg_secret(
                 self.k3s_token.get() or ""
             )
-            globals_to_save["SERVICE_NAMESPACE"] = self._get_service_namespace()
+            service_namespace = (self._get_service_namespace() or "").strip()
+            if service_namespace:
+                globals_to_save["SERVICE_NAMESPACE"] = service_namespace
             globals_to_save.pop("NAMESPACE", None)
 
+            explicit_global_keys = {
+                "KNOE_DB_USER",
+                "DB_PASSWORD",
+                "CLUSTER_ENV",
+                "DEPLOYMENT_MODE",
+                "DEPLOYMENT_TARGET",
+                "DATABASE_NAMESPACE",
+                "CLUSTER_NAME",
+                "DB_HOST_PORT",
+                "PROLE_K3S_SERVER",
+                "PROLE_K3S_TOKEN",
+                "KUBECONTEXT",
+            }
+            if prole_home_override:
+                explicit_global_keys.add("PROLE_HOME")
+            explicit_service_namespace = False
+            try:
+                explicit_service_namespace = bool(
+                    (self.service_namespace.get() or "").strip()
+                )
+            except Exception:
+                explicit_service_namespace = False
+            if not explicit_service_namespace:
+                explicit_service_namespace = bool(
+                    (existing_global.get("SERVICE_NAMESPACE") or "").strip()
+                )
+            if explicit_service_namespace:
+                explicit_global_keys.add("SERVICE_NAMESPACE")
+
+            globals_to_save = _filter_cfg_values_for_persistence(
+                "Global",
+                globals_to_save,
+                mode=persistence_mode,
+                explicit_keys=explicit_global_keys,
+            )
+
             self._sync_port_forward_mappings()
 
             # Inputs section (replayable UI inputs)
@@ -210,6 +252,20 @@ class ConfigMixin:
                 "PIPELINE_URL": _default_opentofu_pipeline_url(),
             }
             sections = self._sanitize_sections_for_cfg(sections)
+            section_explicit_keys = {
+                "Initialize Cluster": {"ENVIRONMENT", "K3S_SERVER_URL", "K3S_TOKEN"},
+                "Service Cluster (k3s)": {"K3S_SERVER_URL", "K3S_TOKEN"},
+                "Dev Cluster (k3d)": {"KUBECTL_CONTEXT"},
+            }
+            sections = {
+                section_name: _filter_cfg_values_for_persistence(
+                    section_name,
+                    section_values,
+                    mode=persistence_mode,
+                    explicit_keys=section_explicit_keys.get(section_name, set()),
+                )
+                for section_name, section_values in sections.items()
+            }
 
             # Sync to Ansible Prole Vault
             self._save_ansible_prole_vault(self.db_password.get())

knoe/ui/screens/database_options.py

@@ -67,6 +67,7 @@ class DatabaseOptionsScreenMixin:
         self.extensions_list = [
             {"id": "postgis", "name": "PostGIS", "description": "Spatial and geographic objects for PostgreSQL"},
             {"id": "pgvector", "name": "pgvector", "description": "Vector similarity search"},
+            {"id": "pg_cron", "name": "pg_cron", "description": "Job scheduler for PostgreSQL"},
             {"id": "pgcrypto", "name": "pgcrypto", "description": "Cryptographic functions", "in_contrib": True},
             {"id": "pgaudit", "name": "pgAudit", "description": "Audit logging"},
             {"id": "pg_repack", "name": "pg_repack", "description": "Reorganize tables with minimal locks"},
@@ -77,11 +78,12 @@ class DatabaseOptionsScreenMixin:
             {"id": "postgres_fdw", "name": "postgres_fdw", "description": "Foreign-data wrapper for PostgreSQL", "in_contrib": True},
             {"id": "dblink", "name": "dblink", "description": "Connect to other databases", "in_contrib": True},
             {"id": "pg_stat_monitor", "name": "pg_stat_monitor", "description": "Statistics collector (Percona only)", "percona_only": True},
+            {"id": "pg_tde", "name": "pg_tde", "description": "Transparent Data Encryption dependency (pre-installed)", "percona_only": True, "preinstalled": True, "locked": True, "no_extension": True},
             {"id": "pgbadger", "name": "pgBadger", "description": "Log analyzer (Package only, no extension)", "no_extension": True},
         ]
 
         # Default enabled extensions (all on for percona v18)
-        default_enabled = ["postgis", "pgvector", "pgcrypto", "pgaudit", "pg_repack", "pg_stat_statements", "pg_buffercache", "pg_freespacemap", "pgrowlocks", "postgres_fdw", "dblink", "pg_stat_monitor", "pgbadger"]
+        default_enabled = ["postgis", "pgvector", "pg_cron", "pgcrypto", "pgaudit", "pg_repack", "pg_stat_statements", "pg_buffercache", "pg_freespacemap", "pgrowlocks", "postgres_fdw", "dblink", "pg_stat_monitor", "pg_tde", "pgbadger"]
 
         for ext in self.extensions_list:
             saved_key = f"database_options.ext.{ext['id']}"
@@ -262,14 +264,16 @@ class DatabaseOptionsScreenMixin:
         self._overlay_widgets.append(ext_frame)
 
         # Extension table (browser style)
-        columns = ("enabled", "name", "description")
+        columns = ("enabled", "name", "description", "notes")
         self.ext_tree = ttk.Treeview(ext_frame, columns=columns, show="headings", height=10)
         self.ext_tree.heading("enabled", text="Enabled")
         self.ext_tree.heading("name", text="Extension Name")
         self.ext_tree.heading("description", text="Description")
+        self.ext_tree.heading("notes", text="Notes")
         self.ext_tree.column("enabled", width=80, anchor="center")
         self.ext_tree.column("name", width=200, anchor="w")
-        self.ext_tree.column("description", width=600, anchor="w")
+        self.ext_tree.column("description", width=420, anchor="w")
+        self.ext_tree.column("notes", width=200, anchor="w")
         self.ext_tree.pack(side="left", fill="both", expand=True)
 
         scroll = ttk.Scrollbar(ext_frame, orient="vertical", command=self.ext_tree.yview)
@@ -280,12 +284,13 @@ class DatabaseOptionsScreenMixin:
 
         def on_ext_click(event):
             item = self.ext_tree.identify_row(event.y)
-            if item:
+            if item and item in self.db_extensions:
-                vals = self.ext_tree.item(item, "values")
+                ext = next((e for e in self.extensions_list if e["id"] == item), {})
-                ext_id = vals[1].lower()
+                if ext.get("locked"):
-                if ext_id in self.db_extensions:
+                    return
-                    current = self.db_extensions[ext_id].get()
+                current = self.db_extensions[item].get()
-                    self.db_extensions[ext_id].set(not current)
+                self.db_extensions[item].set(not current)
+                self._save_prole_cfg()
                 self._refresh_extensions_table()
 
         self.ext_tree.bind("<ButtonRelease-1>", on_ext_click)
@@ -298,8 +303,26 @@ class DatabaseOptionsScreenMixin:
             self.ext_tree.delete(item)
         
         for ext in self.extensions_list:
-            enabled = "[ ✓ ]" if self.db_extensions.get(ext["id"], tk.BooleanVar()).get() else "[   ]"
+            ext_var = self.db_extensions.get(ext["id"])
-            self.ext_tree.insert("", "end", values=(enabled, ext["name"], ext["description"]))
+            is_enabled = ext_var.get() if ext_var else False
+            enabled = "[ ✓ ]" if is_enabled else "[   ]"
+            notes = []
+            if ext.get("preinstalled"):
+                notes.append("Pre-installed")
+            if ext.get("locked"):
+                notes.append("Always enabled")
+            if ext.get("in_contrib"):
+                notes.append("In contrib")
+            if ext.get("percona_only"):
+                notes.append("Percona only")
+            if ext.get("no_extension"):
+                notes.append("Package only")
+            self.ext_tree.insert(
+                "",
+                "end",
+                iid=ext["id"],
+                values=(enabled, ext["name"], ext["description"], "; ".join(notes)),
+            )
 
     def _refresh_database_options_ui(self):
         dist = self.db_distribution.get()
@@ -376,6 +399,9 @@ class DatabaseOptionsScreenMixin:
                         elif ext_id == "pg_repack":
                             # Percona uses 'repack' instead of 'pg_repack' in package name
                             pkg = f"percona-postgresql-{version}-repack"
+                        elif ext_id == "pg_cron":
+                            # pg_cron is bundled with Percona PG packages in this build flow.
+                            pkg = ""
                         elif ext_id == "pgbadger":
                             pkg = "percona-pgbadger"
                         elif ext_id == "pg_stat_monitor":
@@ -386,6 +412,8 @@ class DatabaseOptionsScreenMixin:
                         # Standard PostgreSQL
                         if ext_id == "postgis":
                             pkg = f"postgresql-{version}-postgis-3"
+                        elif ext_id == "pg_cron":
+                            pkg = f"postgresql-{version}-cron"
                         elif ext_id == "pg_repack":
                             pkg = f"postgresql-{version}-repack"
                         elif ext_id == "pgbadger":

knoe/ui/screens/deploy.py

@@ -655,9 +655,8 @@ class DeployScreenMixin:
         """Dynamically generate prole-tools-app/prole.properties based on environment"""
         props_path = Path("prole-tools-app/prole.properties")
 
-        # Determine host based on environment
+        env_key = (env or "").strip().lower()
-        # For now use localhost as a placeholder for Service/Prod
+        dev_mode = env_key == "dev"
-        host = "localhost"
 
         content = f"""# Prole default endpoints (dynamically generated by install.py)
 # UI assets
@@ -665,38 +664,48 @@ icon=img/proleIcon.png
 background=img/proleLogoSepia.png
 
 # Dev port-forward supervision
-pf.enabled=true
+pf.enabled={str(dev_mode).lower()}
 
-# Service endpoints (7 traffic lights)
+"""
+
+        if dev_mode:
+            content += """# Service endpoints (7 traffic lights)
 svc.1.name=K3D
-svc.1.host={host}
+svc.1.host=localhost
 svc.1.port=6443
 
 svc.2.name=Prometheus
-svc.2.host={host}
+svc.2.host=localhost
 svc.2.port=9090
 
 svc.3.name=Grafana
-svc.3.host={host}
+svc.3.host=localhost
 svc.3.port=3000
 
 svc.4.name=OpenBAO
-svc.4.host={host}
+svc.4.host=localhost
 svc.4.port=8200
 
 svc.5.name=PostgreSQL
-svc.5.host={host}
+svc.5.host=localhost
 svc.5.port=5432
 
 svc.6.name=Kong
-svc.6.host={host}
+svc.6.host=localhost
 svc.6.port=8000
 
 svc.7.name=CertManager
-svc.7.host={host}
+svc.7.host=localhost
 svc.7.port=9402
 
-# Kerberos configuration
+"""
+        else:
+            content += """# Service/Prod endpoint mappings are intentionally not persisted here.
+# Live service endpoints are discovered from the running cluster.
+
+"""
+
+        content += f"""# Kerberos configuration
 kerberos.enabled={str(self.kerberos_enabled.get()).lower()}
 kerberos.realm={self.kerberos_realm.get()}
 kerberos.user={self.kerberos_user.get()}
@@ -709,9 +718,10 @@ kerberos.kdc={self.kerberos_kdc.get()}
         """Run the deployment steps"""
         try:
             # Capture environment selection and prepare dynamic labels
-            env = self.deploy_environment.get().strip()
+            env_selected = (self.deploy_environment.get() or "").strip().lower()
-            if env not in ("Dev", "Service", "Prod"):
+            env = {"dev": "Dev", "service": "Service", "prod": "Prod"}.get(
-                env = "Dev"
+                env_selected, "Dev"
+            )
 
             # Update step labels to reflect environment
             self.deploy_widgets["Ensure target cluster"]["step"][

modes/k3s/knoe-db/.version

@@ -1 +1 @@
-141
+143

scan/ansible_inventory_flat.txt

@@ -1,4 +1,5 @@
 host,ip,roles
+gandalf.prole.org,,iscsi,k3s_agents,k3s_hosts,linux_hosts
 merlin,10.0.0.36,merlin_bootstrap
 merlin.prole.org,,iscsi,k3s_agents,k3s_hosts,linux_hosts,mariadb,mariadb_primary
 myrddin.prole.org,,ad_dc,iscsi,k3s_hosts,k3s_servers,linux_hosts,ssl_hosts

scan/network_description.txt

@@ -1,60 +1,58 @@
 Network Discovery Summary:
 Primary Router: 10.0.0.1 (eero_5d:50:f2)
-DNS Servers: 10.0.0.4, 10.0.0.5, 100.100.100.100
+DNS Servers: 10.0.0.5, 100.100.100.100, 10.0.0.4
 Detected Devices:
-- 10.0.0.31 [2e:c1:c1:e9:87:b4] (2e:c1:c1:e9:87:b4): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
-- 10.0.0.5 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
-- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
 - 10.0.0.3 [2c:cf:67:8c:2b:47] (RaspberryPi_8c:2b:47): Ports [22, 53, 80, 443, 2049, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
 - 10.0.0.32 [48:a6:b8:a4:8e:cc] (Sonos_a4:8e:cc): Ports [], Services: []
-- 10.0.0.30 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH']
+- 10.0.0.22 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
+- 10.0.0.35 [7e:19:9a:fe:9f:ed] (7e:19:9a:fe:9f:ed): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
 - 10.0.0.26 [c8:db:26:08:52:f8] (Logitech_08:52:f8): Ports [], Services: []
 - 10.0.0.4 [b8:27:eb:b3:6f:2b] (RaspberryPiF_b3:6f:2b): Ports [22, 53, 443, 2049], Services: ['DNS', 'SSH', 'NFS', 'Web Server']
-- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
 - 10.0.0.2 [dc:a4:ca:ea:1a:2f] (Apple_ea:1a:2f): Ports [], Services: []
 - 10.0.0.1 [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [53], Services: ['DNS']
+- 10.0.0.7 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH']
+- 10.0.0.5 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC']
 - 10.0.0.6 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
 - 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server']
-- 10.0.0.60 [c6:d0:1a:97:9f:e9] (c6:d0:1a:97:9f:e9): Ports [], Services: []
-- 10.0.0.45 [f8:b4:6a:30:a4:1f] (HewlettPacka_30:a4:1f): Ports [80, 443], Services: ['Web Server']
 - 10.0.0.37 [54:07:7d:22:c0:b2] (Netgear_22:c0:b2): Ports [80], Services: ['Web Server']
-- 10.0.0.46 [d4:f7:d5:40:ab:17] (SonyInteract_40:ab:17): Ports [], Services: []
+- 10.0.0.45 [f8:b4:6a:30:a4:1f] (HewlettPacka_30:a4:1f): Ports [80, 443], Services: ['Web Server']
 - 10.0.0.48 [0c:fe:45:53:f4:3c] (SonyInteract_53:f4:3c): Ports [], Services: []
 - 10.0.0.52 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH']
 - 10.0.0.41 [b8:27:eb:57:10:d9] (RaspberryPiF_57:10:d9): Ports [22], Services: ['SSH']
-- 10.0.0.34 [3a:b4:ce:79:b0:55] (3a:b4:ce:79:b0:55): Ports [], Services: []
+- 10.0.0.46 [d4:f7:d5:40:ab:17] (SonyInteract_40:ab:17): Ports [], Services: []
-- 10.0.0.58 [7e:46:74:2a:b0:b2] (7e:46:74:2a:b0:b2): Ports [], Services: []
 - 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: []
-- 10.0.0.100 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: []
+- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server']
-- 10.0.0.95 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
+- 10.0.0.95 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC']
+- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
 - 10.0.0.99 [20:28:bc:f1:dc:ba] (Visionscape_f1:dc:ba): Ports [], Services: []
 - 10.0.0.106 [48:a6:b8:a7:50:60] (Sonos_a7:50:60): Ports [], Services: []
-- 10.0.0.117 [40:f5:20:1e:5e:91] (Espressif_1e:5e:91): Ports [80], Services: ['Web Server']
 - 10.0.0.123 [3c:ef:8c:96:e3:3c] (ZhejiangDahu_96:e3:3c): Ports [80], Services: ['Web Server']
 - 10.0.0.124 [9c:8e:cd:02:73:c4] (AmcrestTechn_02:73:c4): Ports [80], Services: ['Web Server']
+- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
+- 10.0.0.113 [60:81:10:92:02:4e] (Apple_92:02:4e): Ports [], Services: []
+- 10.0.0.111 [ca:0b:6c:5d:0a:e5] (ca:0b:6c:5d:0a:e5): Ports [], Services: []
+- 10.0.0.117 [40:f5:20:1e:5e:91] (Espressif_1e:5e:91): Ports [80], Services: ['Web Server']
 - 10.0.0.125 [a0:60:32:04:73:ba] (AmcrestTechn_04:73:ba): Ports [80], Services: ['Web Server']
-- 10.0.0.189 [00:17:88:a3:2f:cc] (PhilipsLight_a3:2f:cc): Ports [80, 443], Services: ['Web Server']
 - 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH']
-- 10.0.0.128 [20:c9:d0:94:30:dd] (Apple_94:30:dd): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
+- 10.0.0.112 [f8:33:31:40:58:c0] (TexasInstrum_40:58:c0): Ports [80, 443], Services: ['Web Server']
-- 10.0.0.130 [f8:bb:bf:a3:84:eb] (eero_a3:84:eb): Ports [53], Services: ['DNS']
 - 10.0.0.143 [60:5f:8d:7a:aa:32] (eero_7a:aa:32): Ports [53], Services: ['DNS']
+- 10.0.0.145 [7c:a6:b0:04:db:1c] (7c:a6:b0:04:db:1c): Ports [], Services: []
 - 10.0.0.204 [50:eb:f6:56:f3:26] (ASUSTekCOMPU_56:f3:26): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama']
-- 10.0.0.155 [62:17:97:b3:06:a1] (62:17:97:b3:06:a1): Ports [], Services: []
+- 10.0.0.163 [b0:8b:a8:f8:96:92] (AmazonTechno_f8:96:92): Ports [], Services: []
-- 10.0.0.205 [a8:20:66:28:12:e7] (Apple_28:12:e7): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
+- 10.0.0.205 [a8:20:66:28:12:e7] (Apple_28:12:e7): Ports [22, 53, 445, 5900, 88], Services: ['DNS', 'SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
 - 10.0.0.170 [b0:ee:7b:ca:73:99] (Roku_ca:73:99): Ports [], Services: []
-- 10.0.0.175 [28:80:88:e4:10:1a] (Netgear_e4:10:1a): Ports [80], Services: ['Web Server']
 - 10.0.0.180 [4c:e1:73:42:1a:be] (HuizhouDehon_2:1a:be): Ports [445], Services: ['SMB/CIFS (Possible Windows/AD)']
+- 10.0.0.189 [00:17:88:a3:2f:cc] (PhilipsLight_a3:2f:cc): Ports [80, 443], Services: ['Web Server']
 - 10.0.0.188 [00:16:6c:c5:1f:54] (SamsungElect_c5:1f:54): Ports [80, 443], Services: ['Web Server']
+- 10.0.0.175 [28:80:88:e4:10:1a] (Netgear_e4:10:1a): Ports [80], Services: ['Web Server']
 - 10.0.0.196 [60:5f:8d:88:08:f2] (eero_88:08:f2): Ports [53], Services: ['DNS']
 - 10.0.0.199 [7c:a6:b0:01:a7:33] (7c:a6:b0:01:a7:33): Ports [], Services: []
-- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server']
+Ollama Instances found at: 10.0.0.35, 10.0.0.204
-- 10.0.0.111 [ca:0b:6c:5d:0a:e5] (ca:0b:6c:5d:0a:e5): Ports [], Services: []
-- 10.0.0.1\ [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: []
-Ollama Instances found at: 10.0.0.31, 10.0.0.204
 
 Ansible Inventory Summary:
 Inventory directory: /Users/chrisfu/dev/prole/infrastructure
-Hosts detected: 6
+Hosts detected: 7
+- gandalf.prole.org (unknown): roles [iscsi, k3s_agents, k3s_hosts, linux_hosts]
 - merlin (10.0.0.36): roles [merlin_bootstrap]
 - merlin.prole.org (unknown): roles [iscsi, k3s_agents, k3s_hosts, linux_hosts, mariadb, mariadb_primary]
 - myrddin.prole.org (unknown): roles [ad_dc, iscsi, k3s_hosts, k3s_servers, linux_hosts, ssl_hosts]

tests/installer/test_cfg_save_kubecontext.py

@@ -16,25 +16,38 @@ class _Var:
 
 
 class _DummyCfgApp(ConfigMixin):
-    def __init__(self, conf_dir: Path, kubectx: str):
+    def __init__(
+        self,
+        conf_dir: Path,
+        kubectx: str,
+        *,
+        mode: str = "k3d",
+        cluster_env: str = "dev",
+        service_namespace_input: str = "knoe-system",
+        resolved_service_namespace: str = "knoe-system",
+        existing_global: dict | None = None,
+    ):
         self._conf_dir = conf_dir
         self._cfg_path_override = conf_dir / "prole.cfg"
+        self._mode = mode
+        self._resolved_service_namespace = resolved_service_namespace
 
         # Minimal surface required by ConfigMixin._save_prole_cfg
-        self.cluster_env = _Var("dev")
+        self.cluster_env = _Var(cluster_env)
         self.selected_kubectx = _Var(kubectx)
         self.db_username = _Var("knoe-db")
         self.db_password = _Var("secret")
         self.db_namespace = _Var("knoe-system")
+        self.cnpg_cluster_name = _Var("knoe-db")
         self.db_host_port = _Var("5432")
         self.k3s_server_url = _Var("")
         self.k3s_token = _Var("")
         self.prod_artifacts_path = _Var(str(conf_dir))
 
-        self.service_namespace = _Var("knoe-system")
+        self.service_namespace = _Var(service_namespace_input)
 
         self.prole_cfg_data = {
-            "Global": {},
+            "Global": dict(existing_global or {}),
             "Initialize Cluster": {},
             "Dev Cluster (k3d)": {},
             "Service Cluster (k3s)": {},
@@ -45,10 +58,10 @@ class _DummyCfgApp(ConfigMixin):
         return self._conf_dir
 
     def _deployment_mode(self) -> str:
-        return "k3d"
+        return self._mode
 
     def _get_service_namespace(self) -> str:
-        return "knoe-system"
+        return self._resolved_service_namespace
 
     def _secret_cfg_value(self, _section: str, _key: str, value: str, *_args) -> str:
         return value
@@ -79,3 +92,82 @@ def test_save_prole_cfg_persists_kubecontext(tmp_path, monkeypatch):
     text = (tmp_path / "prole.cfg").read_text(encoding="utf-8")
     assert "KUBECONTEXT" in text
     assert "k3d-knoe-system" in text
+
+
+def test_save_prole_cfg_service_mode_drops_generated_localhost_values(
+    tmp_path, monkeypatch
+):
+    import knoe.ui.screens.cfg as cfg_mod
+
+    monkeypatch.setattr(cfg_mod.prole_conf, "activate_environment", lambda *_a, **_k: None)
+
+    app = _DummyCfgApp(
+        conf_dir=tmp_path,
+        kubectx="",
+        mode="k3s",
+        cluster_env="service",
+        service_namespace_input="",
+        resolved_service_namespace="knoe-system",
+        existing_global={
+            "SERVICE_HOSTNAME": "k3d.localhost",
+            "API_SERVICE_ENDPOINT": "http://localhost:8081",
+            "PROLE_OPENTOFU_URL": "http://127.0.0.1:8080",
+        },
+    )
+    app._save_prole_cfg()
+
+    text = (tmp_path / "prole.cfg").read_text(encoding="utf-8")
+    assert "SERVICE_HOSTNAME" not in text
+    assert "API_SERVICE_ENDPOINT" not in text
+    assert "localhost" not in text
+    assert "SERVICE_NAMESPACE" not in text
+
+
+def test_save_prole_cfg_service_mode_keeps_explicit_service_namespace(
+    tmp_path, monkeypatch
+):
+    import knoe.ui.screens.cfg as cfg_mod
+
+    monkeypatch.setattr(cfg_mod.prole_conf, "activate_environment", lambda *_a, **_k: None)
+
+    app = _DummyCfgApp(
+        conf_dir=tmp_path,
+        kubectx="",
+        mode="k3s",
+        cluster_env="service",
+        service_namespace_input="explicit-ns",
+        resolved_service_namespace="explicit-ns",
+    )
+    app._save_prole_cfg()
+
+    text = (tmp_path / "prole.cfg").read_text(encoding="utf-8")
+    assert "SERVICE_NAMESPACE = explicit-ns" in text
+
+
+def test_save_prole_cfg_service_mode_repeated_saves_remain_clean(tmp_path, monkeypatch):
+    import knoe.ui.screens.cfg as cfg_mod
+
+    monkeypatch.setattr(cfg_mod.prole_conf, "activate_environment", lambda *_a, **_k: None)
+
+    app = _DummyCfgApp(
+        conf_dir=tmp_path,
+        kubectx="",
+        mode="k3s",
+        cluster_env="service",
+        service_namespace_input="",
+        resolved_service_namespace="knoe-system",
+        existing_global={
+            "SERVICE_HOSTNAME": "k3d.localhost",
+            "API_SERVICE_ENDPOINT": "http://localhost:8081",
+        },
+    )
+
+    app._save_prole_cfg()
+    first = (tmp_path / "prole.cfg").read_text(encoding="utf-8")
+    app._save_prole_cfg()
+    second = (tmp_path / "prole.cfg").read_text(encoding="utf-8")
+
+    for text in (first, second):
+        assert "SERVICE_HOSTNAME" not in text
+        assert "API_SERVICE_ENDPOINT" not in text
+        assert "localhost" not in text

tests/installer/test_deploy_screen_properties.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import knoe.ui.screens.deploy as deploy_mod
+from knoe.ui.screens.deploy import DeployScreenMixin
+
+
+class _Var:
+    def __init__(self, value):
+        self._value = value
+
+    def get(self):
+        return self._value
+
+
+class _PropsApp(DeployScreenMixin):
+    def __init__(self):
+        self.kerberos_enabled = _Var(True)
+        self.kerberos_realm = _Var("PROLE.ORG")
+        self.kerberos_user = _Var("administrator")
+        self.kerberos_kdc = _Var("10.0.0.3")
+
+
+def test_generate_prole_properties_dev_writes_local_endpoints(tmp_path, monkeypatch):
+    (tmp_path / "prole-tools-app").mkdir(parents=True)
+    monkeypatch.chdir(tmp_path)
+
+    app = _PropsApp()
+    app.generate_prole_properties("Dev")
+
+    content = (tmp_path / "prole-tools-app" / "prole.properties").read_text()
+    assert "pf.enabled=true" in content
+    assert "svc.1.host=localhost" in content
+    assert "svc.7.port=9402" in content
+
+
+def test_generate_prole_properties_service_and_prod_skip_service_endpoints(
+    tmp_path, monkeypatch
+):
+    (tmp_path / "prole-tools-app").mkdir(parents=True)
+    monkeypatch.chdir(tmp_path)
+
+    app = _PropsApp()
+    for env in ("Service", "Prod"):
+        app.generate_prole_properties(env)
+        content = (tmp_path / "prole-tools-app" / "prole.properties").read_text()
+        assert "pf.enabled=false" in content
+        assert "svc.1.host=" not in content
+        assert "Service/Prod endpoint mappings are intentionally not persisted here." in content
+
+
+class _LabelChild:
+    def __init__(self):
+        self.text = ""
+
+    def configure(self, **kwargs):
+        self.text = kwargs.get("text", self.text)
+
+
+class _LabelMaster:
+    def __init__(self):
+        self._children = [object(), _LabelChild()]
+
+    def winfo_children(self):
+        return self._children
+
+
+class _Label:
+    def __init__(self):
+        self.master = _LabelMaster()
+
+
+class _RunDeploymentApp(DeployScreenMixin):
+    def __init__(self, selected_env: str):
+        self.deploy_environment = _Var(selected_env)
+        self.generated_env = ""
+        self.deploy_widgets = {
+            "Ensure target cluster": {
+                "step": {"name": "Ensure target cluster"},
+                "label": _Label(),
+            }
+        }
+
+    def generate_prole_properties(self, env: str):
+        self.generated_env = env
+
+    def update_deploy_step_status(self, *_args, **_kwargs):
+        return None
+
+    def build_prole_app(self):
+        return None
+
+    def check_docker_running(self):
+        return True
+
+    def ensure_registry_available(self, _env):
+        return "registry.local"
+
+    def create_or_select_cluster(self, _env):
+        return None
+
+    def build_docker_image(self):
+        return None
+
+    def tag_docker_image(self):
+        return None
+
+    def push_docker_image(self):
+        return None
+
+    def import_k3d_image(self, cluster_name=""):
+        return None
+
+    def install_launchagent_port_forwards(self) -> bool:
+        return True
+
+
+def test_run_deployment_normalizes_environment_before_generation(monkeypatch):
+    app = _RunDeploymentApp("service")
+    monkeypatch.setattr(deploy_mod.messagebox, "showinfo", lambda *_a, **_k: None)
+    monkeypatch.setattr(deploy_mod.messagebox, "showerror", lambda *_a, **_k: None)
+
+    app.run_deployment()
+
+    assert app.generated_env == "Service"
+    ensure_label = app.deploy_widgets["Ensure target cluster"]["label"].master.winfo_children()[1]
+    assert "Ensure target cluster (Service)" == ensure_label.text

tests/installer/test_env_helpers.py

@@ -63,6 +63,8 @@ from knoe.config import (
     _expand_path as cfg_expand_path,
     _expand_cfg_value as cfg_expand_cfg_value,
     _collect_cfg_vars as cfg_collect_cfg_vars,
+    _filter_cfg_values_for_persistence,
+    _normalize_cfg_value_for_persistence,
     _extract_yaml_scalar_from_text as cfg_extract_yaml,
     _extract_inline_vault_block as cfg_extract_vault,
     _load_properties,
@@ -928,6 +930,60 @@ class TestUpdateProleCfgValue:
         assert "FOO = updated" in dev_base.read_text()
         assert svc_base.read_text() == "[Global]\nFOO = service\n"
 
+    def test_service_mode_drops_placeholder_endpoint_value(self, tmp_path):
+        cfg_file = tmp_path / "conf" / "prole.cfg"
+        cfg_file.parent.mkdir(parents=True)
+        cfg_file.write_text("[Global]\nSERVICE_HOSTNAME = old-host\n")
+
+        with mock.patch("knoe.config.PROJECT_ROOT", tmp_path):
+            _update_prole_cfg_value(
+                "Global",
+                "SERVICE_HOSTNAME",
+                "${SERVICE_HOSTNAME}",
+                mode="k3s",
+            )
+
+        content = cfg_file.read_text()
+        assert "SERVICE_HOSTNAME" not in content
+
+
+class TestPersistenceFilters:
+    def test_service_mode_drops_generated_endpoints_and_localhost(self):
+        values = {
+            "API_SERVICE_ENDPOINT": "http://localhost:8081",
+            "SERVICE_HOSTNAME": "k3d.localhost",
+            "PROLE_K3S_SERVER": "https://myrddin.prole.org:6443",
+            "SERVICE_NAMESPACE": "knoe-system",
+        }
+
+        filtered = _filter_cfg_values_for_persistence(
+            "Global",
+            values,
+            mode="k3s",
+            explicit_keys={"PROLE_K3S_SERVER"},
+        )
+
+        assert filtered == {"PROLE_K3S_SERVER": "https://myrddin.prole.org:6443"}
+
+    def test_service_mode_keeps_explicit_service_namespace(self):
+        filtered = _filter_cfg_values_for_persistence(
+            "Global",
+            {"SERVICE_NAMESPACE": "knoe-system"},
+            mode="k3s",
+            explicit_keys={"SERVICE_NAMESPACE"},
+        )
+
+        assert filtered == {"SERVICE_NAMESPACE": "knoe-system"}
+
+    def test_dev_mode_keeps_localhost_endpoint(self):
+        value = _normalize_cfg_value_for_persistence(
+            "Global",
+            "API_SERVICE_ENDPOINT",
+            "http://localhost:8081",
+            mode="k3d",
+        )
+        assert value == "http://localhost:8081"
+
 
 # ===== is_apple_silicon / get_docker_build_platform_args =====
 

tests/test_database_options.py

@@ -28,6 +28,10 @@ sys.modules["PIL.ImageTk"] = MagicMock()
 import install
 from install import ProleInstaller
 
+
+def _generated_dockerfile_path(installer, tmp_path):
+    return tmp_path / "build" / installer._db_mode_key() / "knoe-db" / "Dockerfile"
+
 @pytest.fixture
 def mock_installer(tmp_path):
     root = MagicMock()
@@ -74,6 +78,10 @@ def test_database_options_state_init(mock_installer):
     assert mock_installer.db_distribution.get() == "percona"
     # Default is Percona 18
     assert mock_installer.db_version_type.get() == "v18"
+    assert "pg_cron" in mock_installer.db_extensions
+    assert "pg_tde" in mock_installer.db_extensions
+    assert mock_installer.db_extensions["pg_cron"].get() is True
+    assert mock_installer.db_extensions["pg_tde"].get() is True
 
 def test_encryption_toggle(mock_installer):
     # Initial state: Encryption ON -> Percona
@@ -127,13 +135,14 @@ def test_dockerfile_generation(mock_installer, tmp_path):
     template_path = tmp_path / "knoe-db" / "Dockerfile.percona.template"
     template_path.write_text("FROM percona:{{MAJOR_VERSION}}\n{{EXTENSION_INSTALL_STEPS}}\n{{EXTENSION_CREATE_STEPS}}")
     
+    for ext in mock_installer.db_extensions.values():
+        ext.set(False)
     mock_installer.db_extensions["postgis"].set(True)
-    mock_installer.db_extensions["pgvector"].set(False)
     
     success = mock_installer._generate_knoe_db_dockerfile()
     assert success is True
     
-    dockerfile = tmp_path / "build" / "k3d" / "knoe-db" / "Dockerfile"
+    dockerfile = _generated_dockerfile_path(mock_installer, tmp_path)
     assert dockerfile.exists()
     content = dockerfile.read_text()
     assert "FROM percona:17" in content
@@ -158,7 +167,7 @@ def test_dockerfile_generation_percona_18(mock_installer, tmp_path):
     success = mock_installer._generate_knoe_db_dockerfile()
     assert success is True
     
-    dockerfile = tmp_path / "build" / "k3d" / "knoe-db" / "Dockerfile"
+    dockerfile = _generated_dockerfile_path(mock_installer, tmp_path)
     content = dockerfile.read_text()
     
     assert "FROM percona:18" in content
@@ -168,11 +177,37 @@ def test_dockerfile_generation_percona_18(mock_installer, tmp_path):
     assert "percona-postgresql-18-repack" in content
     # pgvector should use 'pgvector' for package
     assert "percona-postgresql-18-pgvector" in content
+    # Percona build flow should not install non-existent cron package variants
+    assert "percona-postgresql-18-cron" not in content
+    assert "percona-postgresql-18-pg_cron" not in content
     
     # SQL creation steps
     assert "CREATE EXTENSION IF NOT EXISTS pgcrypto;" in content
     assert "CREATE EXTENSION IF NOT EXISTS pg_repack;" in content
     assert "CREATE EXTENSION IF NOT EXISTS vector;" in content
+    assert "CREATE EXTENSION IF NOT EXISTS pg_cron;" in content
+
+
+def test_pg_cron_package_mapping_for_postgresql(mock_installer, tmp_path):
+    mock_installer.db_distribution.set("postgresql")
+    mock_installer.db_selected_version.set("17 (Latest)")
+
+    template_path = tmp_path / "knoe-db" / "Dockerfile.postgresql.template"
+    template_path.write_text("FROM postgres:{{MAJOR_VERSION}}\n{{EXTENSION_INSTALL_STEPS}}\n{{EXTENSION_CREATE_STEPS}}")
+
+    # Keep scenario focused on pg_cron mapping
+    for ext in mock_installer.db_extensions.values():
+        ext.set(False)
+    mock_installer.db_extensions["pg_cron"].set(True)
+
+    success = mock_installer._generate_knoe_db_dockerfile()
+    assert success is True
+
+    dockerfile = _generated_dockerfile_path(mock_installer, tmp_path)
+    content = dockerfile.read_text()
+
+    assert "postgresql-17-cron" in content
+    assert "postgresql-17-pg_cron" not in content
 
 def test_pgbadger_handling(mock_installer, tmp_path):
     mock_installer.db_distribution.set("percona")
@@ -181,12 +216,14 @@ def test_pgbadger_handling(mock_installer, tmp_path):
     template_path = tmp_path / "knoe-db" / "Dockerfile.percona.template"
     template_path.write_text("{{EXTENSION_INSTALL_STEPS}}\n{{EXTENSION_CREATE_STEPS}}")
     
+    for ext in mock_installer.db_extensions.values():
+        ext.set(False)
     mock_installer.db_extensions["pgbadger"].set(True)
     
     success = mock_installer._generate_knoe_db_dockerfile()
     assert success is True
     
-    dockerfile = tmp_path / "build" / "k3d" / "knoe-db" / "Dockerfile"
+    dockerfile = _generated_dockerfile_path(mock_installer, tmp_path)
     content = dockerfile.read_text()
     
     assert "apt-get install -y --no-install-recommends percona-pgbadger" in content

tools/manage-node.sh

@@ -0,0 +1,371 @@
+#!/usr/bin/env bash
+# manage-node.sh
+# Usage: manage-node.sh <add|remove> --node NODE [--inventory PATH] [--k3s-server SERVER]
+# Options:
+#   --no-ansible     : skip running ./ansible.sh
+#   --no-k8s         : skip k3s cluster operations (drain/delete / wait-for-ready)
+#   --ssh-user USER  : SSH user for remote commands (default: ansible)
+#   --ssh-key PATH   : SSH key to use (default: ~/.ssh/id_ed25519_ansible)
+#   --ssh-host HOST  : SSH host to use for node-side uninstall (default: same as --node)
+#   --vault PASSFILE : ansible vault password file path (default: .vault_pass)
+#   --dry-run        : print actions but do not execute destructive steps
+#   --force          : allow destructive actions (delete node, rm -rf)
+#   --timeout N      : wait timeout in seconds for node readiness (default: 300)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Defaults (override with env or flags)
+SSH_USER="${SSH_USER:-ansible}"
+SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519_ansible}"
+SSH_HOST="${SSH_HOST:-}"
+K3S_SERVER="${K3S_SERVER:-myrddin.prole.org}"
+ANSIBLE_SCRIPT="${ANSIBLE_SCRIPT:-$REPO_ROOT/ansible.sh}"
+VAULT_PASS="${VAULT_PASS:-$REPO_ROOT/.vault_pass}"
+DRY_RUN=false
+FORCE=false
+NO_ANSIBLE=false
+NO_K8S=false
+TIMEOUT=300
+
+# candidate inventory locations (script will pick first that contains k3s_hosts)
+INVENTORIES=(
+  "$REPO_ROOT/inventory"
+  "$REPO_ROOT/inventory/hosts.ini"
+  "$REPO_ROOT/infrastructure/hosts"
+  "$REPO_ROOT/infrastructure/ansible/hosts"
+  "$REPO_ROOT/infrastructure/ansible/inventory"
+  "$REPO_ROOT/infrastructure/ansible/inventory.ini"
+  "$REPO_ROOT/hosts.ini"
+  "$REPO_ROOT/ansible/inventory"
+)
+
+usage() {
+  cat <<EOF
+Usage: $0 <add|remove> --node NODE [options]
+
+Examples:
+  # dry-run removal
+  $0 remove --node pi.prole.org --dry-run
+
+  # remove node (do k8s drain + ansible update)
+  $0 remove --node pi.prole.org --force
+
+  # add node
+  $0 add --node gandalf.prole.org
+
+Options:
+  --no-ansible     : skip running ansible
+  --no-k8s         : skip k8s drain/delete / wait-for-ready
+  --ssh-user USER
+  --ssh-key PATH
+  --ssh-host HOST  : SSH host for node uninstall (default: --node value)
+  --k3s-server HOST : default myrddin.prole.org
+  --vault PASSFILE
+  --inventory PATH
+  --dry-run
+  --force
+  --timeout N
+EOF
+  exit 1
+}
+
+# parse args
+if [ $# -lt 1 ]; then usage; fi
+ACTION="$1"; shift
+NODE=""
+INVENTORY_OVERRIDE=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --node) NODE="$2"; shift 2;;
+    --ssh-user) SSH_USER="$2"; shift 2;;
+    --ssh-key) SSH_KEY="$2"; shift 2;;
+    --ssh-host) SSH_HOST="$2"; shift 2;;
+    --k3s-server) K3S_SERVER="$2"; shift 2;;
+    --inventory) INVENTORY_OVERRIDE="$2"; shift 2;;
+    --vault) VAULT_PASS="$2"; shift 2;;
+    --dry-run) DRY_RUN=true; shift;;
+    --force) FORCE=true; shift;;
+    --no-ansible) NO_ANSIBLE=true; shift;;
+    --no-k8s) NO_K8S=true; shift;;
+    --timeout) TIMEOUT="$2"; shift 2;;
+    -h|--help) usage;;
+    *) echo "Unknown arg: $1"; usage;;
+  esac
+done
+
+[ -n "$NODE" ] || { echo "ERROR: --node required"; usage; }
+if [ -z "$SSH_HOST" ]; then
+  SSH_HOST="$NODE"
+fi
+
+echoinfo() { echo "==> $*"; }
+echowarn() { echo "WARN: $*" >&2; }
+echofatal() { echo "FATAL: $*" >&2; exit 2; }
+
+run_or_echo() {
+  if [ "$DRY_RUN" = true ]; then
+    echo "[DRY-RUN] $*"
+  else
+    echo "+ $*"
+    eval "$*"
+  fi
+}
+
+# find inventory file that contains group 'k3s_hosts'
+find_inventory() {
+  if [ -n "$INVENTORY_OVERRIDE" ]; then
+    echo "$INVENTORY_OVERRIDE"
+    return 0
+  fi
+  for f in "${INVENTORIES[@]}"; do
+    [ -e "$f" ] || continue
+    # check whether it looks like an ini and has [k3s_hosts] or a YAML with k3s_hosts
+    if grep -q -E '^\s*\[k3s_hosts\]' "$f" 2>/dev/null || grep -q 'k3s_hosts' "$f" 2>/dev/null; then
+      echo "$f"
+      return 0
+    fi
+  done
+  # fallback: repo root inventory file
+  if [ -e "$REPO_ROOT/inventory" ]; then
+    echo "$REPO_ROOT/inventory"
+    return 0
+  fi
+  echowarn "No inventory file found in standard locations. Use --inventory to specify."
+  echo ""
+  return 1
+}
+
+backup_file() {
+  local file="$1"
+  if [ -e "$file" ]; then
+    local ts; ts="$(date -u +"%Y%m%dT%H%M%SZ")"
+    local bak="${file}.manage-node.bak.${ts}"
+    run_or_echo "cp -a \"$file\" \"$bak\""
+    echo "$bak"
+  fi
+}
+
+# Remove node from INI-style inventory group [k3s_hosts]
+remove_node_from_ini_group() {
+  local inventory="$1"
+  local node="$2"
+  local group="k3s_hosts"
+  backup_file "$inventory" >/dev/null || true
+
+  # Use awk to remove lines matching the node only when within the k3s_hosts group
+  local tmp="${inventory}.tmp.$$"
+  awk -v node="$node" -v grp="$group" '
+    BEGIN{in_grp=0}
+    /^[[:space:]]*\[[^]]+\][[:space:]]*$/ {
+      header=$0
+      sub(/^[[:space:]]*\[/, "", header)
+      sub(/\][[:space:]]*$/, "", header)
+      in_grp = (header == grp) ? 1 : 0
+      print; next
+    }
+    {
+      if(in_grp){
+        line=$0
+        sub(/^[[:space:]]+/, "", line)
+        sub(/[[:space:]]+$/, "", line)
+        if(line == node){ next } # skip exact node line
+      }
+      print
+    }
+  ' "$inventory" > "$tmp"
+  run_or_echo "mv \"$tmp\" \"$inventory\""
+  echoinfo "Removed $node from $inventory ($group)"
+}
+
+# Add node to INI-style inventory group [k3s_hosts]
+add_node_to_ini_group() {
+  local inventory="$1"
+  local node="$2"
+  local group="k3s_hosts"
+  backup_file "$inventory" >/dev/null || true
+
+  # if group exists, append node if not present
+  if grep -q -E "^[[:space:]]*\[$group\]" "$inventory"; then
+    if grep -q -E "^[[:space:]]*$node[[:space:]]*$" "$inventory"; then
+      echoinfo "$node already present in $inventory"
+      return 0
+    fi
+    # insert node after group header or at end of group
+    awk -v node="$node" -v grp="$group" '
+      BEGIN{in_grp=0; printed=0}
+      /^[[:space:]]*\[[^]]+\][[:space:]]*$/ {
+        if(in_grp && !printed){ print node; printed=1 } # add before next group
+        header=$0
+        sub(/^[[:space:]]*\[/, "", header)
+        sub(/\][[:space:]]*$/, "", header)
+        in_grp = (header == grp) ? 1 : 0
+        print; next
+      }
+      { print }
+      END {
+        if(!printed && in_grp){ print node; printed=1 } # if file ended in group
+        if(!printed && !in_grp){ print ""; print "[" grp "]"; print node } # group not found anywhere
+      }
+    ' "$inventory" > "${inventory}.tmp.$$"
+    run_or_echo "mv \"${inventory}.tmp.$$\" \"$inventory\""
+  else
+    # group doesn't exist, append group + node
+    cat >> "$inventory" <<EOF
+
+[$group]
+$node
+EOF
+    echoinfo "Appended group [$group] and added $node to $inventory"
+  fi
+}
+
+# run ansible playbook for the provided limit or full k3s_hosts
+run_ansible() {
+  local limit="$1"
+  if [ "$NO_ANSIBLE" = true ]; then
+    echoinfo "Skipping ansible as requested (--no-ansible)"
+    return 0
+  fi
+  if [ ! -x "$ANSIBLE_SCRIPT" ]; then
+    echowarn "Ansible script $ANSIBLE_SCRIPT not found or not executable; skipping ansible step"
+    return 0
+  fi
+  local cmd
+  # Use vault pass file if exists
+  if [ -f "$VAULT_PASS" ]; then
+    cmd="$ANSIBLE_SCRIPT -v $VAULT_PASS --limit $limit"
+  else
+    cmd="$ANSIBLE_SCRIPT -v --limit $limit"
+  fi
+  run_or_echo "$cmd"
+}
+
+# k8s: cordon, drain, delete node
+drain_and_delete_node() {
+  local server="$1"
+  local node="$2"
+  echoinfo "Will cordon & drain $node via k3s server $server"
+  if [ "$NO_K8S" = true ]; then
+    echoinfo "Skipping k8s operations (--no-k8s)"
+    return 0
+  fi
+
+  # Step 1: show pods on node
+  echoinfo "Listing pods scheduled on $node"
+  run_or_echo "ssh -o ControlMaster=no -i \"$SSH_KEY\" $SSH_USER@$server \"sudo k3s kubectl get pods -A --field-selector spec.nodeName=$node -o wide || true\""
+
+  echoinfo "Cordoning $node"
+  run_or_echo "ssh -o ControlMaster=no -i \"$SSH_KEY\" $SSH_USER@$server \"sudo k3s kubectl cordon $node\""
+
+  echoinfo "Draining $node (this may take a while)"
+  run_or_echo "ssh -o ControlMaster=no -i \"$SSH_KEY\" $SSH_USER@$server \"if sudo k3s kubectl drain --help 2>/dev/null | grep -q -- '--delete-emptydir-data'; then sudo k3s kubectl drain $node --ignore-daemonsets --delete-emptydir-data --force --grace-period=30 --timeout=10m || true; else sudo k3s kubectl drain $node --ignore-daemonsets --delete-local-data --force --grace-period=30 --timeout=10m || true; fi\""
+
+  echoinfo "Deleting node object from k8s"
+  run_or_echo "ssh -o ControlMaster=no -i \"$SSH_KEY\" $SSH_USER@$server \"sudo k3s kubectl delete node $node || true\""
+}
+
+# wait for node ready
+wait_for_node_ready() {
+  local server="$1"
+  local node="$2"
+  local timeout_secs="$3"
+  local start_ts=$(date +%s)
+  echoinfo "Waiting up to ${timeout_secs}s for $node to become Ready"
+  while true; do
+    if ssh -o ControlMaster=no -i "$SSH_KEY" "$SSH_USER@$server" "sudo k3s kubectl get node $node -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' 2>/dev/null" | grep -q "True"; then
+      echoinfo "$node is Ready"
+      return 0
+    fi
+    now=$(date +%s)
+    if [ $((now - start_ts)) -gt "$timeout_secs" ]; then
+      echofatal "Timeout waiting for $node to become Ready"
+    fi
+    sleep 5
+  done
+}
+
+# uninstall k3s agent on node
+uninstall_k3s_agent_on_node() {
+  local ssh_host="$1"
+  echoinfo "Attempting to run k3s-agent uninstall on SSH host $ssh_host"
+  if [ "$DRY_RUN" = true ]; then
+    echo "[DRY-RUN] ssh -i $SSH_KEY $SSH_USER@$ssh_host sudo /usr/local/bin/k3s-agent-uninstall.sh || sudo /usr/local/bin/k3s-uninstall.sh"
+    return 0
+  fi
+
+  ssh -o ControlMaster=no -i "$SSH_KEY" "$SSH_USER@$ssh_host" "sudo /usr/local/bin/k3s-agent-uninstall.sh || sudo /usr/local/bin/k3s-uninstall.sh || true"
+  # cleanup known drop-ins introduced by iscsi role
+  ssh -o ControlMaster=no -i "$SSH_KEY" "$SSH_USER@$ssh_host" "sudo rm -f /etc/systemd/system/k3s-agent.service.d/open-iscsi.conf || true; sudo systemctl daemon-reload || true"
+}
+
+# MAIN: add/remove
+INVENTORY="$(find_inventory || true)"
+if [ -z "$INVENTORY" ]; then
+  echowarn "No inventory auto-detected. Use --inventory to set the path. Exiting."
+  exit 3
+fi
+
+case "$ACTION" in
+  remove)
+    echoinfo "REMOVE node: $NODE"
+    # 1) k8s: cordon/drain/delete
+    if [ "$NO_K8S" = false ]; then
+      drain_and_delete_node "$K3S_SERVER" "$NODE"
+    fi
+
+    # 2) run ansible (optional) to remove any node-specific config (we run for remaining k3s_hosts)
+    if [ "$NO_ANSIBLE" = false ]; then
+      echoinfo "Running ansible on remaining k3s_hosts to reconcile config"
+      run_ansible "k3s_hosts"
+    fi
+
+    # 3) remove node from inventory
+    echoinfo "Removing $NODE from inventory $INVENTORY"
+    if grep -q -E "^[[:space:]]*\[k3s_hosts\]" "$INVENTORY" 2>/dev/null; then
+      remove_node_from_ini_group "$INVENTORY" "$NODE"
+      echoinfo "Inventory updated; commit changes if desired:"
+      echoinfo "  git add \"$INVENTORY\" && git commit -m \"Remove $NODE from k3s_hosts\""
+    else
+      echowarn "k3s_hosts group not found in $INVENTORY - manual edit required"
+    fi
+
+    # 4) optionally uninstall k3s agent on node
+    if [ "$FORCE" = true ]; then
+      echoinfo "FORCE requested: uninstalling k3s agent for $NODE via SSH host $SSH_HOST"
+      uninstall_k3s_agent_on_node "$SSH_HOST"
+    else
+      echoinfo "Skipping k3s-agent uninstall on $NODE (use --force to remove agent)"
+    fi
+
+    echoinfo "REMOVE workflow finished. Verify cluster health and inventory changes."
+    ;;
+
+  add)
+    echoinfo "ADD node: $NODE"
+    # 1) Add node to inventory
+    echoinfo "Adding $NODE to inventory $INVENTORY"
+    add_node_to_ini_group "$INVENTORY" "$NODE"
+    echoinfo "Inventory updated; commit changes if desired:"
+    echoinfo "  git add \"$INVENTORY\" && git commit -m \"Add $NODE to k3s_hosts\""
+
+    # 2) run ansible to provision node (only that node)
+    if [ "$NO_ANSIBLE" = false ]; then
+      echoinfo "Provisioning node with ansible (limit: $NODE)"
+      run_ansible "$NODE"
+    fi
+
+    # 3) wait for k3s agent to register with server
+    if [ "$NO_K8S" = false ]; then
+      echoinfo "Waiting for node to join the k3s cluster and become Ready"
+      wait_for_node_ready "$K3S_SERVER" "$NODE" "$TIMEOUT" || echowarn "Node did not become Ready within timeout"
+    fi
+
+    echoinfo "ADD workflow finished. Verify cluster health and inventory changes."
+    ;;
+
+  *)
+    echofatal "Unknown action: $ACTION"
+    ;;
+esac