Stabilize CNPG reset/update flow and finalize 3-node recovery

- add idempotent CNPG operator reconciliation to prevent duplicate active controller ReplicaSets
- restore and validate CNPG 3-instance convergence with PV claimRef recovery and storage path readiness
- wire canonical launcher aliases/reset behavior and improve namespace cleanup semantics
- harden backup/objectstore readiness handling and retry behavior
- enforce service namespace usage for common services and remove default-namespace drift
- enable Kerberos milestone auto-activation when realm/kdc are configured and apply CNPG GSS pg_hba rules
- keep final runtime healthy: knoe-db 3/3, operator stable, perfsnap captured

Co-authored-by: Junie <junie@jetbrains.com>

conf/cnpg-placement/knoe-db-knoe-db.json

@@ -12,10 +12,10 @@
     "pi.prole.org"
   ],
   "metadata": {
-    "prior_plan_present": false,
+    "prior_plan_present": true,
-    "reason": "new_cluster",
+    "reason": "reused",
-    "regenerated": true,
+    "regenerated": false,
-    "reused": false
+    "reused": true
   },
   "plan_hash": "10dccbb8fe133be4",
   "plan_id": "cnpg-placement-10dccbb8fe133be4",

conf/port-mapping.cfg

@@ -1,3 +1,16 @@
 # Port mappings for Prole Tools (generated).
 # Format: key: local=... remote=... ns=... svc=... address=...
 
+argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0
+supabase-studio: local=18080 remote=3000 ns=supabase svc=studio address=0.0.0.0
+supabase-auth: local=9999 remote=9999 ns=supabase svc=auth address=127.0.0.1
+supabase-rest: local=3001 remote=3000 ns=supabase svc=rest address=0.0.0.0
+supabase-realtime: local=4000 remote=4000 ns=supabase svc=realtime address=0.0.0.0
+garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0
+openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0
+opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0
+dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1
+postgres: local=5432 remote=5432 ns=knoe-db svc=knoe-db-rw address=0.0.0.0
+prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1
+grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0
+supabase-kong: local=8000 remote=8000 ns=supabase svc=kong address=0.0.0.0

conf/prod/prole.cfg

@@ -48,20 +48,27 @@ init_cluster.k3s_token = K107c8c6000488eca4a067d8a73119bbae2f07b4ea1bac7d8d3dc9c
 init_cluster.kerberos_enabled = true
 init_cluster.mode = k8s
 init_cluster.start_cluster = true
+init_cluster.supabase_analytics_enabled = true
+init_cluster.supabase_auth_enabled = true
 init_cluster.supabase_enabled = false
+init_cluster.supabase_meta_enabled = true
+init_cluster.supabase_realtime_enabled = true
+init_cluster.supabase_studio_enabled = false
+init_cluster.supabase_studio_url = db.prole.org
 init_cnpg_deploy.force_rollout = false
 init_cnpg_deploy.run_deploy = true
 init_db_build.run_build = true
 init_password.cluster_name = ${CLUSTER_NAME}
 init_password.db_host_port = 5432
 init_password.db_namespace = ${DATABASE_NAMESPACE}
-init_password.db_password = 
+init_password.db_password = ${PROLE_SECRET:v1:6fyOe4qCZmGGqAUU:vkFsVGYFxvXsPurt3QNO0Y5ZUDdfwMbvgG4N1ULprbCfA0XeVoA9_sIbuKyuZaFF}
-init_password.db_password_confirm = 
+init_password.db_password_confirm = ${PROLE_SECRET:v1:6fyOe4qCZmGGqAUU:vkFsVGYFxvXsPurt3QNO0Y5ZUDdfwMbvgG4N1ULprbCfA0XeVoA9_sIbuKyuZaFF}
 init_password.db_username = root
 init_password.generate_ssh_key = true
 init_scripts.run_scripts = true
-kerberos_config.enabled = true
+kerberos_config.enabled = True
-kerberos_config.kdc = 10.0.0.3
+kerberos_config.init_authority = false
+kerberos_config.kdc = 10.0.0.196
 kerberos_config.password = ${PROLE_SECRET:v1:S-C7rQ5Tq-o5A9h2:fdSoP54DCSrsRz9ods8EU353dwEnI1bh-Mrr5b6q03d6O5yD}
 kerberos_config.realm = PROLE.ORG
 kerberos_config.test_connection = false
@@ -72,24 +79,32 @@ ollama_config.server_host =
 ollama_config.server_port = 11434
 
 [Global]
+ARGOCD_NAMESPACE = argocd
 CLUSTER_ENV = prod
 CLUSTER_NAME = knoe-db
 DATABASE_NAMESPACE = knoe-db-17-7-043-18-140-17-7-043-18-140
 DB_HOST_PORT = 5432
-DB_PASSWORD = 
+DB_PASSWORD = ${PROLE_SECRET:v1:oQHkGbXHuEuLvwQq:PQ-d0-LB3xnYnDJiTU1MrViNRfDUsq7xNBsMngYizQJ-tq-RiJXwubKHxB7eDhJA}
 DEPLOYMENT_MODE = k8s
 DEPLOYMENT_TARGET = prole-prod-cluster
+DOCKER_IMPORT_DIR = 
 DOCKER_PRELOAD = false
+K3S_SERVER = https://myrddin.prole.org:6443
+K3S_TOKEN = ${PROLE_SECRET:v1:fSDP1aKQWKjCX2lW:yeepqWGp6vit43ng2cCtu_MIp9gWxdjaSp7RtEiGPv8JYQW_w7aVQnT64LwXG2jh6xEpBAhob-2NJQkvWIlA8AinJt5kqdiDCJp5HXRyrZ5fj3vfsjURsAOq-jjLyu9oGI4ZAPVCwZL6uC4G8YLrXtPCTbAdZqieYUViWQ==}
 KNOE_DB_USER = root
+OPENTOFU_URL = http://127.0.0.1:8080
 OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
+PROLE_HOME = /Users/chrisfu/dev/prole
 PROLE_K3S_SERVER = https://myrddin.prole.org:6443
 PROLE_K3S_TOKEN = K107c8c6000488eca4a067d8a73119bbae2f07b4ea1bac7d8d3dc9c500cbb8acb18::server:04572345810eae2f9619a6ed4239702b
 PROLE_OPENTOFU_URL = http://127.0.0.1:8080
+REGISTRY_NAMESPACE = knoe-system
 SERVICE_NAMESPACE = knoe-system
 
 [Welcome]
 
 [Dependencies]
+STATUS = All installed
 
 [Network]
 AD_DC_HOST = myrddin.prole.org
@@ -100,10 +115,25 @@ ANSIBLE_INVENTORY = /Users/chrisfu/dev/prole/infrastructure/inventory
 ANSIBLE_REALM = PROLE.ORG
 ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
 KDC_ANSIBLE_DETECTED = 10.0.0.3
+KDC_AUTO_DETECTED = 10.0.0.196
+KERBEROS_AUTO_ENABLED = True
 
 [Port Forwards]
+PORT_FORWARD_K3D_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD
+PORT_FORWARD_K3D_MAPPING_2 = id=garage;namespace=knoe-system;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3
+PORT_FORWARD_K3D_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao;address=0.0.0.0;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao
+PORT_FORWARD_K3D_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu
+PORT_FORWARD_K3D_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard
+PORT_FORWARD_K3D_MAPPING_6 = id=postgres;namespace=${DATABASE_NAMESPACE};target=svc/knoe-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary)
+PORT_FORWARD_K3D_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI
+PORT_FORWARD_K3D_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI
 
 [System Environment]
+PROLE_CONF = /Users/chrisfu/dev/prole/conf
+PROLE_DATA = /Users/chrisfu/dev/prole/data
+PROLE_HOME = /Users/chrisfu/dev/prole
+PROLE_LOGS = /Users/chrisfu/dev/prole/logs
+PROLE_SERVICE = /Users/chrisfu/dev/prole/etc
 
 [Monitoring]
 
@@ -112,17 +142,22 @@ KDC_ANSIBLE_DETECTED = 10.0.0.3
 [Ollama]
 
 [Optional Features]
+AT_REST_ENCRYPTION_ENABLED = true
+GITOPS_ENABLED = false
+KERBEROS_ENABLED = true
+SUPABASE_ENABLED = false
 
 [GitOps]
 
 [Database Creation]
+DB_USER = root
 
 [Initialize Cluster]
 
 [Dev Cluster (k3d)]
-CLUSTER_ENV = dev
+CLUSTER_ENV = k3d-knoe-dev-cluster
 DISPLAY_NAME = knoe-dev-cluster
-KUBECTL_CONTEXT = 
+KUBECTL_CONTEXT = prod
 MODE = k3d
 
 [Service Cluster (k3s)]
@@ -134,7 +169,7 @@ MODE = k3s
 PIPELINE_URL = http://127.0.0.1:8080
 
 [Prod Cluster (k8s)]
-ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging
+ARTIFACTS_DIR = 
 CLUSTER_ENV = prole-prod-cluster
 DISPLAY_NAME = prole-prod-cluster
 MODE = k8s
@@ -149,4 +184,5 @@ MODE = k8s
 TARGET = prole-prod-cluster
 
 [Install]
+STATUS = Failed
 

conf/service/prole.cfg

@@ -1,5 +1,5 @@
 ; Prole Master Configuration File
-; Generated by install.py on 2026-03-25 21:38:46
+; Generated by install.py on 2026-03-27 12:20:06
 ; This file is used as input for Ansible deployment and k8s cluster creation.
 
 [User]
@@ -68,8 +68,8 @@ init_db_build.run_build = true
 init_password.cluster_name = ${CLUSTER_NAME}
 init_password.db_host_port = 5432
 init_password.db_namespace = ${DATABASE_NAMESPACE}
-init_password.db_password = ${PROLE_SECRET:v1:Fop86hEFa2lJGNiO:uh6fuU2FDI56bq4B0vDcjh8tF959qSJc}
+init_password.db_password = ${PROLE_SECRET:v1:7h2BjxdDiVq6dPdK:21sXsPKsCfG2IPA4H3tOd_2UmlKLZRh-}
-init_password.db_password_confirm = ${PROLE_SECRET:v1:Fop86hEFa2lJGNiO:uh6fuU2FDI56bq4B0vDcjh8tF959qSJc}
+init_password.db_password_confirm = ${PROLE_SECRET:v1:7h2BjxdDiVq6dPdK:21sXsPKsCfG2IPA4H3tOd_2UmlKLZRh-}
 init_password.db_username = root
 init_password.generate_ssh_key = true
 init_scripts.run_scripts = true
@@ -106,7 +106,7 @@ K3S_SERVER = https://myrddin.prole.org:6443
 K3S_TOKEN = ${PROLE_SECRET:v1:UQkxpgIK-ZjOTGmE:3Ij4G2imYkA2EfwyAO0a7jXYR1bSIVFxDtG9OdCZkl-x7URqHR5bXTWf6VCOMQmFS098w0wxA-P0dFwLDn1gLXNRHd-9sTyEMVP4rUz3TPaT-g89doklyjXkO0PjD_Eyf0FcB6GbKr2c8qGIsXyuGZE4O0GlcDlnRk5Pkg==}
 KNOE_DB_USER = root
 KUBECONTEXT = prole-k3s
-OPENTOFU_URL = http://127.0.0.1:8080
+OPENTOFU_URL = http://myrddin.prole.org:8080
 OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
 PROLE_HOME = /Users/chrisfu/dev/prole
 PROLE_K3S_SERVER = https://myrddin.prole.org:6443
@@ -199,14 +199,14 @@ DISPLAY_NAME = prole-service-cluster
 K3S_SERVER_URL = https://myrddin.prole.org:6443
 K3S_TOKEN = ${PROLE_SECRET:v1:F14dY4ATvenAtZtY:ZUKtV8vAeTQ8TdFElnUtHgZ9pgD38dquOopMY22CdgGdIFoWM5EppWfCzNVMCrM9-wEVTBrqsHDYrIGCx4RRhnHnNNgvAVGqTqroTZMOFs-aZp-VjjXJefSLRbYxzxgs_Q_llazC5xhuYXkAC0K09DkAqyp6ATjrs9F1SA==}
 MODE = k3s
-PIPELINE_URL = http://127.0.0.1:8080
+PIPELINE_URL = http://myrddin.prole.org:8080
 
 [Prod Cluster (k8s)]
 ARTIFACTS_DIR = 
 CLUSTER_ENV = prole-prod-cluster
 DISPLAY_NAME = prole-prod-cluster
 MODE = k8s
-PIPELINE_URL = http://127.0.0.1:8080
+PIPELINE_URL = http://myrddin.prole.org:8080
 
 [Docker Build]
 LOCAL_REGISTRY = myrddin.prole.org:5000
@@ -217,8 +217,8 @@ STATUS = Attempted
 
 [Deployment]
 MODE = k3s
-STATUS = Attempted
+STATUS = Deployed
 TARGET = prole-service-cluster
 
 [Install]
-STATUS = Finished
+STATUS = Failed

etc/init_cnpg_backup.sh

@@ -43,6 +43,7 @@ GARAGE_S3_ENDPOINT=${GARAGE_S3_ENDPOINT:-http://$GARAGE_NAME.$GARAGE_NAMESPACE.s
 GARAGE_S3_REGION=${GARAGE_S3_REGION:-garage}
 RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1}
 RETENTION_POLICY=${RETENTION_POLICY:-30d}
+GARAGE_LAYOUT_BOOTSTRAP_ENABLED=${GARAGE_LAYOUT_BOOTSTRAP_ENABLED:-1}
 BARMAN_PLUGIN_NAME=${BARMAN_PLUGIN_NAME:-barman-cloud.cloudnative-pg.io}
 BARMAN_OBJECT_NAME=${BARMAN_OBJECT_NAME:-knoe-db-barman-objectstore}
 BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600}
@@ -195,6 +196,29 @@ garage_exec() {
   kubectl exec -n "$GARAGE_NAMESPACE" "$pod" -- /garage "$@"
 }
 
+bootstrap_garage_layout_if_needed() {
+  if [[ "${GARAGE_LAYOUT_BOOTSTRAP_ENABLED}" != "1" ]]; then
+    return 0
+  fi
+
+  local init_script="$SCRIPT_DIR/init_garage_store.sh"
+  if [[ ! -f "$init_script" ]]; then
+    echo "WARN: Garage bootstrap helper not found: $init_script" >&2
+    return 0
+  fi
+
+  echo "Garage layout still not applied; invoking Garage bootstrap recovery ..."
+  if [[ -n "${PROLE_MODE:-}" ]]; then
+    if ! bash "$init_script" --mode "$PROLE_MODE" start; then
+      echo "WARN: Garage bootstrap recovery failed (continuing wait loop)." >&2
+    fi
+  else
+    if ! bash "$init_script" start; then
+      echo "WARN: Garage bootstrap recovery failed (continuing wait loop)." >&2
+    fi
+  fi
+}
+
 parse_key_output() {
   local output="$1"
   local access_key secret_key
@@ -209,6 +233,7 @@ parse_key_output() {
 ensure_garage_ready() {
   echo "Checking Garage readiness ..."
   local i status_out
+  local bootstrap_attempted=0
   for i in {1..30}; do
     if status_out=$(garage_exec status 2>/dev/null); then
       # If layout is applied, DataAvail should eventually show something or at least the node should be healthy.
@@ -221,6 +246,10 @@ ensure_garage_ready() {
          return 0
       fi
     fi
+    if [[ "$bootstrap_attempted" -eq 0 && "$i" -ge 6 ]]; then
+      bootstrap_garage_layout_if_needed
+      bootstrap_attempted=1
+    fi
     echo "Waiting for Garage layout to be applied... ($i/30)"
     sleep 5
   done
@@ -581,6 +610,18 @@ wait_for_objectstore_ready() {
 
     now=$(date +%s)
     elapsed=$((now - start_time))
+
+    # Some plugin/controller versions do not currently set status conditions on
+    # ObjectStore. If the ObjectStore exists and barman-cloud is available for a
+    # grace window, proceed with a warning instead of hard failing.
+    local object_exists barman_available
+    object_exists=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o name 2>/dev/null || true)
+    barman_available=$(kubectl -n "$CNPG_OPERATOR_NAMESPACE" get deploy barman-cloud -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
+    if [[ -n "$object_exists" && "$barman_available" =~ ^[1-9][0-9]*$ && $elapsed -ge 30 ]]; then
+      echo "WARN: ObjectStore status fields are not populated, but barman-cloud is available; continuing."
+      return 0
+    fi
+
     if (( elapsed >= timeout )); then
       echo "ERROR: Timed out waiting for ObjectStore '$BARMAN_OBJECT_NAME' to become Ready." >&2
       kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o yaml 2>/dev/null >&2 || true
@@ -621,6 +662,8 @@ has_successful_base_backup() {
 
 wait_for_successful_base_backup() {
   local start_time now elapsed
+  local plugin_retry_count=0
+  local plugin_retry_limit=${PLUGIN_BACKUP_RETRY_LIMIT:-6}
   start_time=$(date +%s)
   while true; do
     if [[ -n "$LAST_BACKUP_NAME" ]]; then
@@ -632,6 +675,17 @@ wait_for_successful_base_backup() {
           return 0
           ;;
         Failed|failed)
+          local err_lc
+          err_lc=$(echo "${err:-}" | tr '[:upper:]' '[:lower:]')
+          if [[ "$err_lc" == *"requested plugin is not available"* ]] && (( plugin_retry_count < plugin_retry_limit )); then
+            plugin_retry_count=$((plugin_retry_count + 1))
+            echo "WARN: Backup '$LAST_BACKUP_NAME' failed because plugin is not yet available. Retry ${plugin_retry_count}/${plugin_retry_limit} ..."
+            wait_for_plugin_ready || true
+            LAST_BACKUP_NAME=""
+            trigger_backup
+            sleep 5
+            continue
+          fi
           echo "ERROR: Backup '$LAST_BACKUP_NAME' failed: ${err:-<no error provided>}" >&2
           return 1
           ;;

k8s/openbao/kerberos-configmap.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: prole-krb5-conf
-  namespace: default
+  namespace: knoe-system
 data:
   krb5.conf: |
     [libdefaults]

k8s/opentofu/deployment.yaml

@@ -55,8 +55,6 @@ spec:
       labels:
         app: opentofu
     spec:
-      nodeSelector:
-        kubernetes.io/hostname: myrddin.prole.org
       containers:
         - name: opentofu-ui
           image: nginx:1.27-alpine

k8s/prole/garage-statefulset.yaml

@@ -15,8 +15,13 @@ spec:
       labels:
         app: garage
     spec:
-      nodeSelector:
+      tolerations:
-        kubernetes.io/hostname: myrddin.prole.org
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
+        - key: "node-role.kubernetes.io/master"
+          operator: "Exists"
+          effect: "NoSchedule"
       containers:
         - name: garage
           image: dxflrs/garage:v1.3.1

k8s/prole/openbao-statefulset.yaml

@@ -15,8 +15,6 @@ spec:
       labels:
         app: openbao
     spec:
-      nodeSelector:
-        kubernetes.io/hostname: myrddin.prole.org
       containers:
         - name: openbao
           image: ghcr.io/openbao/openbao:2.0.0
@@ -43,15 +41,6 @@ spec:
           volumeMounts:
             - name: data
               mountPath: /data
-  volumeClaimTemplates:
+      volumes:
-    - metadata:
+        - name: data
-        name: data
+          emptyDir: {}
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 1Gi
-        storageClassName: synology-iscsi
-        selector:
-          matchLabels:
-            synology.storage/role: openbao

k8s/registry/deployment.yaml

@@ -14,8 +14,6 @@ spec:
       labels:
         app: registry
     spec:
-      nodeSelector:
-        kubernetes.io/hostname: myrddin.prole.org
       tolerations:
         - key: "node-role.kubernetes.io/control-plane"
           operator: "Exists"
@@ -29,27 +27,18 @@ spec:
           ports:
             - containerPort: 5000
               hostPort: 5000
-              name: https
+              name: http
           env:
             - name: REGISTRY_HTTP_ADDR
               value: "0.0.0.0:5000"
-            - name: REGISTRY_HTTP_TLS_CERTIFICATE
-              value: "/etc/ssl/certs/prole/myrddin-registry.crt"
-            - name: REGISTRY_HTTP_TLS_KEY
-              value: "/etc/ssl/certs/prole/myrddin-registry.key"
           volumeMounts:
             - name: registry-data
               mountPath: /var/lib/registry
-            - name: registry-certs
-              mountPath: /etc/ssl/certs/prole
-              readOnly: true
       volumes:
         - name: registry-data
-          emptyDir: {}
-        - name: registry-certs
           hostPath:
-            path: /etc/ssl/certs/prole
+            path: /var/lib/prole/registry
-            type: Directory
+            type: DirectoryOrCreate
 ---
 apiVersion: v1
 kind: Service
@@ -61,7 +50,7 @@ spec:
   selector:
     app: registry
   ports:
-    - name: https
+    - name: http
       port: 5000
-      targetPort: https
+      targetPort: http
   type: ClusterIP

knoe/core/actions.py

@@ -599,9 +599,18 @@ class ProleInstaller:
             except Exception:
                 pass
 
+        service_ns = (
+            env.get("SERVICE_NAMESPACE")
+            or (self.prole_cfg_data.get("Global", {}) or {}).get("SERVICE_NAMESPACE", "")
+            or "knoe-system"
+        )
+        service_ns = str(service_ns).strip() or "knoe-system"
         argocd_ns = env.get("ARGOCD_NAMESPACE") or "argocd"
-        registry_ns = env.get("REGISTRY_NAMESPACE") or "default"
+        registry_ns = env.get("REGISTRY_NAMESPACE") or service_ns
+        if str(registry_ns).strip() == "default":
+            registry_ns = service_ns
         env["ARGOCD_NAMESPACE"] = argocd_ns
+        env["SERVICE_NAMESPACE"] = service_ns
         env["REGISTRY_NAMESPACE"] = registry_ns
 
         mode = _deployment_mode_from_env(
@@ -3876,7 +3885,9 @@ class ProleConsoleInstaller(ProleInstaller):
         )
         if not ns:
             ns = (os.environ.get("REGISTRY_NAMESPACE") or "").strip()
-        if ns:
+        # In service-mode k3s, keep registry in the service namespace and
+        # avoid falling back to default (which causes duplicate deployments).
+        if ns and not (ns == "default" and self._deployment_mode() == "k3s"):
             return ns
 
         # Registry is a common-core service; default to the configured service namespace.
@@ -5977,11 +5988,35 @@ class ProleConsoleInstaller(ProleInstaller):
             self._create_k3d_cluster(cluster_name, prole_data)
         elif env_key in ("service", "prole-service-cluster", "k3s"):
             self.log("==> Cluster reset (k3s namespace cleanup)")
-            ns = self._get_input("env_setup.DATABASE_NAMESPACE", "")
+            db_ns = self._get_input("env_setup.DATABASE_NAMESPACE", "")
-            if not ns:
+            if not db_ns:
-                ns = self._get_input("env_setup.NAMESPACE", "")
+                db_ns = self._get_input("env_setup.NAMESPACE", "")
-            if not ns:
+            if not db_ns:
-                ns = "default"
+                db_ns = "default"
+            db_ns = db_ns.strip() or "default"
+
+            service_ns = self._service_namespace().strip() or "knoe-system"
+            monitoring_ns = (
+                ((self.prole_cfg_data.get("Global", {}) or {}).get("MONITORING_NAMESPACE", "") or "").strip()
+                or (os.environ.get("MONITORING_NAMESPACE") or "").strip()
+                or "monitoring"
+            )
+            cnpg_ns = (
+                ((self.prole_cfg_data.get("Global", {}) or {}).get("CNPG_NAMESPACE", "") or "").strip()
+                or (os.environ.get("CNPG_NAMESPACE") or "").strip()
+                or "cnpg-system"
+            )
+            argocd_ns = self._argocd_namespace().strip() or "argocd"
+            registry_ns = service_ns
+
+            reset_namespaces: list[str] = []
+            for candidate in [db_ns, service_ns, monitoring_ns, cnpg_ns, argocd_ns, "default"]:
+                ns = (candidate or "").strip()
+                if not ns or (ns.startswith("${") and ns.endswith("}")):
+                    continue
+                if ns not in reset_namespaces:
+                    reset_namespaces.append(ns)
+
             server = self._get_input("init_cluster.k3s_server_url", "")
             token = self._get_input("init_cluster.k3s_token", "")
 
@@ -5990,7 +6025,20 @@ class ProleConsoleInstaller(ProleInstaller):
             # Ensure we have a cert-based kubeconfig (or a valid bearer token config)
             # before invoking any kubectl-based reset scripts.
             self._ensure_project_k3s_kubeconfig(server, token)
-            _reset_k3s_namespace(self.project_root, ns, server, token)
+            self.log(
+                f"[RESET] Namespaces to clean (PVCs preserved): {', '.join(reset_namespaces)}"
+            )
+            self.log(
+                f"[RESET] Preserving registry workload in namespace: {registry_ns}"
+            )
+            _reset_k3s_namespace(
+                self.project_root,
+                reset_namespaces,
+                server,
+                token,
+                registry_namespace=registry_ns,
+                clear_node_reservations=True,
+            )
             self._cleanup_local_k3s_artifacts()
 
         self.reset_requested = False
@@ -6562,12 +6610,35 @@ def _attempt_k3s_repair(
 
 
 def _reset_k3s_namespace(
-    project_root: Path, namespace: str, server: str, token: str
+    project_root: Path,
+    namespace: str | Sequence[str],
+    server: str,
+    token: str,
+    *,
+    registry_namespace: str | None = None,
+    clear_node_reservations: bool = False,
 ) -> None:
     script = project_root / "scripts" / "reset-ns.sh"
     if not script.exists():
         print(f"[WARN] Namespace reset script not found: {script}", file=sys.stderr)
         return
+
+    if isinstance(namespace, str):
+        raw_namespaces = [namespace]
+    else:
+        raw_namespaces = list(namespace or [])
+
+    namespaces: list[str] = []
+    for ns in raw_namespaces:
+        item = str(ns or "").strip()
+        if not item:
+            continue
+        if item not in namespaces:
+            namespaces.append(item)
+
+    if not namespaces:
+        return
+
     env = os.environ.copy()
     if not env.get("KUBECONFIG"):
         repo_kubeconfig = project_root / "prole-k3s.kubeconfig"
@@ -6576,7 +6647,20 @@ def _reset_k3s_namespace(
         elif server and token and _looks_like_k8s_bearer_token(token):
             kubeconfig_path = _write_k3s_kubeconfig(server, token)
             env["KUBECONFIG"] = str(kubeconfig_path)
-    subprocess.run(["bash", str(script), "-n", namespace], env=env)
+
+    reg_ns = (registry_namespace or "").strip()
+    for idx, ns in enumerate(namespaces):
+        cmd = ["bash", str(script), "-n", ns]
+        if reg_ns and ns == reg_ns:
+            cmd.append("--keep-registry")
+        if clear_node_reservations and idx == (len(namespaces) - 1):
+            cmd.append("--clear-node-reservations")
+        res = subprocess.run(cmd, env=env)
+        if res.returncode != 0:
+            print(
+                f"[WARN] Namespace reset failed for {ns} (code {res.returncode})",
+                file=sys.stderr,
+            )
 
 
 def _prepare_k3s_pipeline(

knoe/core/milestones.py

@@ -992,6 +992,14 @@ class KerberosMilestone(Milestone):
         enabled = self._parse_bool(
             state.inputs.get("kerberos_config.enabled", "False"), default=False
         )
+        if not enabled:
+            realm = str(state.inputs.get("kerberos_config.realm", "") or "").strip()
+            kdc = str(state.inputs.get("kerberos_config.kdc", "") or "").strip()
+            if realm and kdc:
+                self.logger.info(
+                    "Kerberos enabled implicitly because realm and kdc are configured."
+                )
+                enabled = True
         krb_sec = state.config_data.setdefault("Kerberos Authentication", {})
         if not enabled:
             krb_sec["STATUS"] = "Skipped"

knoe/core/ops/_services_common.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import os
 import subprocess
 from pathlib import Path
@@ -114,12 +115,16 @@ def _namespace(namespace: str | None, env: dict | None, default: str = "default"
 
 
 def _registry_namespace(namespace: str | None, env: dict | None) -> str:
+    service_ns = str((env or {}).get("SERVICE_NAMESPACE") or "").strip()
     raw = (
         (namespace or "").strip()
         or str((env or {}).get("REGISTRY_NAMESPACE") or "").strip()
+        or service_ns
         or str((env or {}).get("NAMESPACE") or "").strip()
-        or "default"
+        or "knoe-system"
     )
+    if raw == "default" and service_ns:
+        return service_ns
     return raw
 
 
@@ -168,3 +173,138 @@ def _to_bool(value: str | bool | None, default: bool = False) -> bool:
     if value is None:
         return default
     return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _prune_named_workload_other_namespaces(
+    *,
+    kind: str,
+    name: str,
+    target_namespace: str,
+    label_selector: str,
+    env: dict | None = None,
+    log: _LogFn | None = None,
+) -> None:
+    listed = _kubectl(
+        ["get", kind, "-A", "-l", label_selector, "-o", "json"],
+        env=env,
+        timeout=45,
+    )
+    if listed.returncode != 0 or not listed.stdout.strip():
+        return
+    try:
+        payload = json.loads(listed.stdout)
+    except Exception:
+        return
+
+    for item in payload.get("items", []):
+        meta = item.get("metadata") or {}
+        ns = str(meta.get("namespace") or "").strip()
+        item_name = str(meta.get("name") or "").strip()
+        if not ns or ns == target_namespace or item_name != name:
+            continue
+        _log(
+            log,
+            f"[SERVICES] Removing duplicate {kind}/{name} from namespace {ns} (target namespace: {target_namespace})",
+        )
+        _kubectl(
+            ["-n", ns, "delete", kind, name, "--ignore-not-found"],
+            env=env,
+            timeout=60,
+        )
+
+
+def _reconcile_deployment_replicasets(
+    *,
+    deployment: str,
+    namespace: str,
+    label_selector: str,
+    env: dict | None = None,
+    log: _LogFn | None = None,
+) -> None:
+    dep_res = _kubectl(
+        ["-n", namespace, "get", "deployment", deployment, "-o", "json"],
+        env=env,
+        timeout=30,
+    )
+    if dep_res.returncode != 0 or not dep_res.stdout.strip():
+        return
+    try:
+        dep = json.loads(dep_res.stdout)
+    except Exception:
+        return
+
+    dep_meta = dep.get("metadata") or {}
+    dep_uid = str(dep_meta.get("uid") or "").strip()
+    dep_annotations = dep_meta.get("annotations") or {}
+    try:
+        dep_revision = int(dep_annotations.get("deployment.kubernetes.io/revision") or "0")
+    except Exception:
+        dep_revision = 0
+
+    rs_res = _kubectl(
+        ["-n", namespace, "get", "replicaset", "-l", label_selector, "-o", "json"],
+        env=env,
+        timeout=45,
+    )
+    if rs_res.returncode != 0 or not rs_res.stdout.strip():
+        return
+    try:
+        payload = json.loads(rs_res.stdout)
+    except Exception:
+        return
+
+    def _rs_revision(item: dict) -> int:
+        annotations = (item.get("metadata") or {}).get("annotations") or {}
+        try:
+            return int(annotations.get("deployment.kubernetes.io/revision") or "0")
+        except Exception:
+            return 0
+
+    owned: list[dict] = []
+    for item in payload.get("items", []):
+        meta = item.get("metadata") or {}
+        owners = meta.get("ownerReferences") or []
+        for owner in owners:
+            if str(owner.get("kind") or "") != "Deployment":
+                continue
+            if str(owner.get("name") or "") != deployment:
+                continue
+            owner_uid = str(owner.get("uid") or "").strip()
+            if dep_uid and owner_uid and owner_uid != dep_uid:
+                continue
+            owned.append(item)
+            break
+
+    active = [
+        item for item in owned if int((item.get("spec") or {}).get("replicas") or 0) > 0
+    ]
+    if len(active) <= 1:
+        return
+
+    stale: list[dict]
+    if dep_revision > 0:
+        stale = [item for item in active if _rs_revision(item) < dep_revision]
+    else:
+        # Fallback: keep newest revision/creation and scale down the rest.
+        active_sorted = sorted(
+            active,
+            key=lambda item: (
+                _rs_revision(item),
+                str((item.get("metadata") or {}).get("creationTimestamp") or ""),
+            ),
+        )
+        stale = active_sorted[:-1]
+
+    for item in stale:
+        rs_name = str((item.get("metadata") or {}).get("name") or "").strip()
+        if not rs_name:
+            continue
+        _log(
+            log,
+            f"[SERVICES] Scaling stale replicaset/{rs_name} to 0 in namespace {namespace} (deployment={deployment})",
+        )
+        _kubectl(
+            ["-n", namespace, "scale", f"replicaset/{rs_name}", "--replicas=0"],
+            env=env,
+            timeout=60,
+        )

knoe/core/ops/cloudnative_pg.py

@@ -119,6 +119,65 @@ def _release_stale_pv_claims(
             _log(log, f"Released PV '{pv_name}'.")
 
 
+def _reconcile_unbound_synology_data_wal_pvs(
+    env: dict | None = None,
+    log: _LogFn | None = None,
+) -> None:
+    """Recreate unbound static Synology data/wal PVs from manifest.
+
+    This allows nodeAffinity/layout changes in `iscsi-pvs.yaml` to take effect,
+    while preserving currently bound volumes.
+    """
+    pv_names = [
+        "synology-iscsi-d001-data",
+        "synology-iscsi-d001-wal",
+        "synology-iscsi-d002-data",
+        "synology-iscsi-d002-wal",
+        "synology-iscsi-d003-data",
+        "synology-iscsi-d003-wal",
+    ]
+
+    unbound: list[str] = []
+    for name in pv_names:
+        r = _kubectl(["get", "pv", name, "-o", "json"], env=env, timeout=20)
+        if r.returncode != 0:
+            continue
+        try:
+            payload = json.loads(r.stdout or "{}")
+        except Exception:
+            continue
+        phase = str((payload.get("status") or {}).get("phase") or "").strip()
+        if phase != "Bound":
+            unbound.append(name)
+
+    if not unbound:
+        return
+
+    _log(log, f"Reconciling unbound Synology data/wal PVs: {', '.join(unbound)}")
+    for name in unbound:
+        _kubectl(["delete", "pv", name, "--ignore-not-found"], env=env, timeout=45)
+
+    root = Path(
+        (env or {}).get("PROLE_HOME")
+        or (env or {}).get("PROLE_SERVICE")
+        or Path(__file__).resolve().parents[4]
+    )
+    manifest = root / "k8s" / "prole" / "iscsi-pvs.yaml"
+    if not manifest.exists():
+        _log(log, f"WARN: Synology PV manifest not found: {manifest}")
+        return
+
+    r_apply = _kubectl(["apply", "-f", str(manifest)], env=env, timeout=180)
+    if r_apply.returncode != 0:
+        stderr = ((r_apply.stderr or "") + "\n" + (r_apply.stdout or "")).strip().lower()
+        if "field is immutable" not in stderr:
+            raise RuntimeError(
+                f"Failed to re-apply Synology PV manifest {manifest}: "
+                f"{(r_apply.stderr or r_apply.stdout or '').strip()}"
+            )
+        _log(log, "WARN: Immutable PV fields prevented full re-apply; continuing with recreated unbound PVs.")
+
+
 def _ensure_namespace(namespace: str, env: dict | None = None) -> None:
     """Create the namespace if it does not already exist (idempotent)."""
     manifest = (
@@ -333,6 +392,166 @@ def _reconcile_instances(
             return
 
 
+def _controller_manifest_url(version: str) -> str:
+    minor = ".".join(version.split(".")[:2])
+    return (
+        f"https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg"
+        f"/release-{minor}/releases/cnpg-{version}.yaml"
+    )
+
+
+def _controller_deployment_json(env: dict | None = None) -> dict | None:
+    r = _kubectl(
+        ["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager", "-o", "json"],
+        env=env,
+        timeout=30,
+    )
+    if r.returncode != 0 or not (r.stdout or "").strip():
+        return None
+    try:
+        return json.loads(r.stdout)
+    except Exception:
+        return None
+
+
+def _controller_is_healthy(env: dict | None = None) -> bool:
+    dep = _controller_deployment_json(env)
+    if not dep:
+        return False
+    spec = dep.get("spec") or {}
+    status = dep.get("status") or {}
+    desired = int(spec.get("replicas") or 1)
+    available = int(status.get("availableReplicas") or 0)
+    return desired > 0 and available >= desired
+
+
+def _reconcile_controller_replicasets(
+    env: dict | None = None,
+    log: _LogFn | None = None,
+    aggressive_cleanup: bool = False,
+) -> None:
+    dep = _controller_deployment_json(env)
+    if not dep:
+        return
+
+    dep_meta = dep.get("metadata") or {}
+    dep_uid = str(dep_meta.get("uid") or "").strip()
+    dep_annotations = dep_meta.get("annotations") or {}
+    try:
+        dep_revision = int(dep_annotations.get("deployment.kubernetes.io/revision") or "0")
+    except Exception:
+        dep_revision = 0
+
+    r = _kubectl(
+        [
+            "-n",
+            "cnpg-system",
+            "get",
+            "rs",
+            "-l",
+            "app.kubernetes.io/name=cloudnative-pg",
+            "-o",
+            "json",
+        ],
+        env=env,
+        timeout=45,
+    )
+    if r.returncode != 0 or not (r.stdout or "").strip():
+        return
+    try:
+        payload = json.loads(r.stdout)
+    except Exception:
+        return
+
+    def _rs_revision(item: dict) -> int:
+        ann = (item.get("metadata") or {}).get("annotations") or {}
+        try:
+            return int(ann.get("deployment.kubernetes.io/revision") or "0")
+        except Exception:
+            return 0
+
+    owned: list[dict] = []
+    for item in payload.get("items") or []:
+        meta = item.get("metadata") or {}
+        owners = meta.get("ownerReferences") or []
+        for owner in owners:
+            if str(owner.get("kind") or "") != "Deployment":
+                continue
+            if str(owner.get("name") or "") != "cnpg-controller-manager":
+                continue
+            owner_uid = str(owner.get("uid") or "").strip()
+            if dep_uid and owner_uid and owner_uid != dep_uid:
+                continue
+            owned.append(item)
+            break
+
+    active = [i for i in owned if int((i.get("spec") or {}).get("replicas") or 0) > 0]
+    if len(active) <= 1:
+        return
+
+    stale: list[dict]
+    if dep_revision > 0:
+        stale = [i for i in active if _rs_revision(i) < dep_revision]
+    else:
+        active_sorted = sorted(
+            active,
+            key=lambda i: (
+                _rs_revision(i),
+                str((i.get("metadata") or {}).get("creationTimestamp") or ""),
+            ),
+        )
+        stale = active_sorted[:-1]
+
+    for rs in stale:
+        rs_meta = rs.get("metadata") or {}
+        rs_name = str(rs_meta.get("name") or "").strip()
+        if not rs_name:
+            continue
+        _log(log, f"Scaling stale CNPG controller replicaset/{rs_name} to 0")
+        _kubectl(
+            ["-n", "cnpg-system", "scale", f"rs/{rs_name}", "--replicas=0"],
+            env=env,
+            timeout=60,
+        )
+
+        if aggressive_cleanup:
+            template_hash = str((rs_meta.get("labels") or {}).get("pod-template-hash") or "").strip()
+            if template_hash:
+                _kubectl(
+                    [
+                        "-n",
+                        "cnpg-system",
+                        "delete",
+                        "pod",
+                        "-l",
+                        f"app.kubernetes.io/name=cloudnative-pg,pod-template-hash={template_hash}",
+                        "--ignore-not-found",
+                        "--wait=false",
+                    ],
+                    env=env,
+                    timeout=60,
+                )
+
+
+def _wait_controller_rollout(
+    env: dict | None = None,
+    timeout_seconds: int = 180,
+) -> bool:
+    r = _kubectl(
+        [
+            "-n",
+            "cnpg-system",
+            "rollout",
+            "status",
+            "deploy/cnpg-controller-manager",
+            f"--timeout={timeout_seconds}s",
+        ],
+        env=env,
+        timeout=timeout_seconds + 10,
+    )
+    return r.returncode == 0
+
+
 # ---------------------------------------------------------------------------
 # Public API
 # ---------------------------------------------------------------------------
@@ -342,37 +561,52 @@ def ensure_operator(
     env: dict | None = None,
     log: _LogFn | None = None,
     mode: str = "",
+    allow_reapply_if_unhealthy: bool = False,
+    aggressive_cleanup: bool = False,
 ) -> None:
     """Install the CloudNativePG operator if not already present, then wait for readiness."""
     if _kubectl_ok(["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager"], env=env):
-        _log(log, "CloudNative-PG operator already installed; waiting for readiness...")
+        _reconcile_controller_replicasets(env=env, log=log, aggressive_cleanup=aggressive_cleanup)
-        _kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=180s"], env=env, timeout=190)
+        if _controller_is_healthy(env=env):
+            _log(log, "CloudNative-PG operator already healthy; skipping operator re-apply.")
             if mode == "k3s":
                 _tune_operator_for_k3s(env, log)
-            _kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=300s"], env=env, timeout=310)
+                _wait_controller_rollout(env=env, timeout_seconds=300)
             return
 
+        _log(log, "CloudNative-PG operator exists but is not healthy; waiting for recovery...")
+        if _wait_controller_rollout(env=env, timeout_seconds=180):
+            if mode == "k3s":
+                _tune_operator_for_k3s(env, log)
+                _wait_controller_rollout(env=env, timeout_seconds=300)
+            return
+
+        if not allow_reapply_if_unhealthy:
+            raise RuntimeError(
+                "CNPG operator deployment exists but is not ready; "
+                "refusing disruptive re-apply in non-initialize flow."
+            )
+        _log(log, "CNPG operator unhealthy after wait; attempting controlled re-apply...")
+
     version = ""
     if env:
         version = (env.get("CNPG_OPERATOR_VERSION") or env.get("CNPG_VERSION") or "").strip()
     if not version or version == "latest":
         version = _CNPG_OPERATOR_FALLBACK_VERSION
-
+    url = _controller_manifest_url(version)
-    minor = ".".join(version.split(".")[:2])
-    url = (
-        f"https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg"
-        f"/release-{minor}/releases/cnpg-{version}.yaml"
-    )
     _log(log, f"Installing CloudNative-PG operator version {version}...")
     r = _kubectl(["apply", "--server-side", "-f", url], env=env, timeout=120)
     if r.returncode != 0:
         raise RuntimeError(f"Failed to install CNPG operator: {(r.stderr or r.stdout or '').strip()}")
 
     if _kubectl_ok(["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager"], env=env, timeout=20):
-        _kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=180s"], env=env, timeout=190)
+        _reconcile_controller_replicasets(env=env, log=log, aggressive_cleanup=aggressive_cleanup)
+        if not _wait_controller_rollout(env=env, timeout_seconds=180):
+            raise RuntimeError("CNPG controller rollout did not complete after operator apply")
         if mode == "k3s":
             _tune_operator_for_k3s(env, log)
-            _kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=300s"], env=env, timeout=310)
+            if not _wait_controller_rollout(env=env, timeout_seconds=300):
+                raise RuntimeError("CNPG controller rollout did not complete after k3s tuning")
 
 
 def _tune_operator_for_k3s(env: dict | None, log: _LogFn | None) -> None:
@@ -382,6 +616,18 @@ def _tune_operator_for_k3s(env: dict | None, log: _LogFn | None) -> None:
         "spec": {
             "template": {
                 "spec": {
+                    "tolerations": [
+                        {
+                            "key": "node-role.kubernetes.io/control-plane",
+                            "operator": "Exists",
+                            "effect": "NoSchedule",
+                        },
+                        {
+                            "key": "node-role.kubernetes.io/master",
+                            "operator": "Exists",
+                            "effect": "NoSchedule",
+                        },
+                    ],
                     "containers": [
                         {
                             "name": "manager",
@@ -404,6 +650,17 @@ def _tune_operator_for_k3s(env: dict | None, log: _LogFn | None) -> None:
     )
 
 
+def _clear_controller_pin(env: dict | None = None, log: _LogFn | None = None) -> None:
+    """Remove stale hard nodeSelector pinning for controller-manager."""
+    _log(log, "Clearing CNPG controller hard nodeSelector pin (if present)...")
+    patch = json.dumps({"spec": {"template": {"spec": {"nodeSelector": None}}}})
+    _kubectl(
+        ["-n", "cnpg-system", "patch", "deployment", "cnpg-controller-manager", "--type", "merge", "-p", patch],
+        env=env,
+        timeout=30,
+    )
+
+
 def pin_controller(
     env: dict | None = None,
     log: _LogFn | None = None,
@@ -416,16 +673,10 @@ def pin_controller(
         r = _kubectl(["get", "nodes", "-l", "storage=primary", "--no-headers"], env=env, timeout=10)
         if r.returncode == 0 and (r.stdout or "").strip():
             selector = "storage=primary"
-        else:
-            r2 = _kubectl(
-                ["get", "nodes", "-l", "node-role.kubernetes.io/control-plane",
-                 "-o", "jsonpath={.items[0].metadata.name}"],
-                env=env, timeout=10,
-            )
-            if r2.returncode == 0 and (r2.stdout or "").strip():
-                selector = f"kubernetes.io/hostname={r2.stdout.strip()}"
 
     if not selector:
+        _clear_controller_pin(env=env, log=log)
+        _reconcile_controller_replicasets(env=env, log=log)
         return
 
     if "=" not in selector:
@@ -444,6 +695,7 @@ def pin_controller(
          "--type", "merge", "-p", patch],
         env=env, timeout=30,
     )
+    _reconcile_controller_replicasets(env=env, log=log)
 
 
 def install_barman_plugin(
@@ -464,11 +716,8 @@ def install_barman_plugin(
     if r.returncode != 0:
         combined = (r.stdout or "") + (r.stderr or "")
         if "webhook.cert-manager.io" in combined.lower() or "cert-manager" in combined.lower():
-            _log(log, "WARN: cert-manager webhook error; restarting cert-manager and retrying...")
+            _log(log, "WARN: cert-manager webhook error; ensuring cert-manager and retrying...")
-            for deploy in ["cert-manager", "cert-manager-webhook", "cert-manager-cainjector"]:
+            _ensure_cert_manager_for_barman(env=env, log=log)
-                _kubectl(["-n", "cert-manager", "rollout", "restart", f"deploy/{deploy}"], env=env, timeout=30)
-            _kubectl(["-n", "cert-manager", "rollout", "status", "deploy/cert-manager-webhook", "--timeout=180s"], env=env, timeout=190)
-            _kubectl(["-n", "cert-manager", "rollout", "status", "deploy/cert-manager", "--timeout=180s"], env=env, timeout=190)
             r2 = subprocess.run(
                 ["kubectl", "apply", "-f", url],
                 env=env, capture_output=True, text=True, timeout=120,
@@ -495,6 +744,88 @@ def install_barman_plugin(
         )
 
 
+def _ensure_cert_manager_for_barman(
+    env: dict | None = None,
+    log: _LogFn | None = None,
+) -> None:
+    def _exists(kind: str, name: str) -> bool:
+        return _kubectl(["-n", "cert-manager", "get", kind, name], env=env, timeout=20).returncode == 0
+
+    need_bootstrap = not all(
+        [
+            _exists("service", "cert-manager-webhook"),
+            _exists("deployment", "cert-manager"),
+            _exists("deployment", "cert-manager-webhook"),
+            _exists("deployment", "cert-manager-cainjector"),
+        ]
+    )
+
+    if need_bootstrap:
+        _log(log, "[CNPG] cert-manager resources missing; bootstrapping cert-manager...")
+        root = Path(
+            (env or {}).get("PROLE_HOME")
+            or (env or {}).get("PROLE_SERVICE")
+            or Path(__file__).resolve().parents[4]
+        )
+        init_script = root / "etc" / "init_certmgr.sh"
+        if init_script.exists():
+            cmd = ["bash", str(init_script)]
+            mode = str((env or {}).get("PROLE_MODE") or "").strip()
+            if mode:
+                cmd.extend(["--mode", mode])
+            cfg_hint = (
+                str((env or {}).get("PROLE_CONFIG_PATH") or "").strip()
+                or str((env or {}).get("PROLE_CFG_PATH") or "").strip()
+                or str((env or {}).get("PROLE_CFG") or "").strip()
+            )
+            if cfg_hint:
+                cmd.extend(["-c", cfg_hint])
+            cmd.append("start")
+            r = subprocess.run(
+                cmd,
+                env=env,
+                capture_output=True,
+                text=True,
+                timeout=600,
+            )
+            if r.returncode != 0:
+                raise RuntimeError(
+                    "Failed to bootstrap cert-manager via init_certmgr.sh: "
+                    f"{(r.stderr or r.stdout or '').strip()}"
+                )
+        else:
+            _kubectl(["get", "namespace", "cert-manager"], env=env, timeout=20)
+            _kubectl(["create", "namespace", "cert-manager"], env=env, timeout=20)
+            certmgr_version = str((env or {}).get("CERTMGR_VERSION") or "v1.14.6").strip()
+            certmgr_url = (
+                "https://github.com/cert-manager/cert-manager/releases/download/"
+                f"{certmgr_version}/cert-manager.yaml"
+            )
+            r = subprocess.run(
+                ["kubectl", "apply", "-f", certmgr_url],
+                env=env,
+                capture_output=True,
+                text=True,
+                timeout=180,
+            )
+            if r.returncode != 0:
+                raise RuntimeError(
+                    f"Failed to bootstrap cert-manager from {certmgr_url}: "
+                    f"{(r.stderr or r.stdout or '').strip()}"
+                )
+
+    for deploy in ["cert-manager", "cert-manager-webhook", "cert-manager-cainjector"]:
+        if _exists("deployment", deploy):
+            _kubectl(["-n", "cert-manager", "rollout", "restart", f"deploy/{deploy}"], env=env, timeout=45)
+    for deploy in ["cert-manager-webhook", "cert-manager", "cert-manager-cainjector"]:
+        if _exists("deployment", deploy):
+            _kubectl(
+                ["-n", "cert-manager", "rollout", "status", f"deploy/{deploy}", "--timeout=240s"],
+                env=env,
+                timeout=250,
+            )
+
+
 def _resolve_barman_plugin_tag(env: dict | None, log: _LogFn | None) -> str:
     """Fetch the latest Barman Cloud plugin release tag from GitHub, with fallback."""
     fallback = ((env or {}).get("BARMAN_PLUGIN_FALLBACK_VERSION") or _BARMAN_PLUGIN_FALLBACK_VERSION).strip()
@@ -715,7 +1046,13 @@ def initialize(
     """
     _log(log, f"Initializing CNPG cluster '{cluster_name}' in namespace '{namespace}'...")
 
-    ensure_operator(env=env, log=log, mode=mode)
+    ensure_operator(
+        env=env,
+        log=log,
+        mode=mode,
+        allow_reapply_if_unhealthy=True,
+        aggressive_cleanup=True,
+    )
     pin_controller(env=env, log=log, mode=mode)
     install_barman_plugin(env=env, log=log)
 
@@ -738,6 +1075,7 @@ def initialize(
     _log(log, "Pre-flight check passed: all required secrets present.")
     # Release any Released PVs so new PVCs can bind (idempotent, k3s only)
     if mode == "k3s":
+        _reconcile_unbound_synology_data_wal_pvs(env=env, log=log)
         _release_stale_pv_claims(env=env, log=log)
     manifest = _resolve_cnpg_manifest(project_root, env)
     _apply_manifest(namespace, manifest, env, log)
@@ -768,7 +1106,13 @@ def deploy(
     """
     _log(log, f"Deploying CNPG cluster '{cluster_name}' in namespace '{namespace}'...")
 
-    ensure_operator(env=env, log=log, mode=mode)
+    ensure_operator(
+        env=env,
+        log=log,
+        mode=mode,
+        allow_reapply_if_unhealthy=False,
+        aggressive_cleanup=False,
+    )
     pin_controller(env=env, log=log, mode=mode)
     install_barman_plugin(env=env, log=log)
 

knoe/core/ops/garage_store.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import json
+import secrets
 from pathlib import Path
 
 from ._services_common import (
@@ -10,18 +12,23 @@ from ._services_common import (
     _kubectl,
     _log,
     _manifest_path,
-    _namespace,
+    _prune_named_workload_other_namespaces,
     _to_bool,
     _wait_rollout,
 )
 
 
 def _garage_namespace(namespace: str | None, env: dict | None) -> str:
+    service_ns = str((env or {}).get("SERVICE_NAMESPACE") or "").strip()
     if env:
         explicit = str(env.get("GARAGE_NAMESPACE") or "").strip()
         if explicit:
-            return explicit
+            return service_ns if explicit == "default" and service_ns else explicit
-    return _namespace(namespace, env, default="default")
+
+    raw = (namespace or "").strip() or service_ns or str((env or {}).get("NAMESPACE") or "").strip() or "knoe-system"
+    if raw == "default":
+        return service_ns or "knoe-system"
+    return raw
 
 
 def _manifest_files(project_root: str | Path) -> list[Path]:
@@ -35,6 +42,71 @@ def _manifest_files(project_root: str | Path) -> list[Path]:
     ]
 
 
+def _repair_released_garage_pvs(*, env: dict | None = None, log: _LogFn | None = None) -> None:
+    listed = _kubectl(
+        ["get", "pv", "-l", "synology.storage/role=garage", "-o", "json"],
+        env=env,
+        timeout=45,
+    )
+    if listed.returncode != 0 or not listed.stdout.strip():
+        return
+    try:
+        payload = json.loads(listed.stdout)
+    except Exception:
+        return
+
+    for item in payload.get("items", []):
+        meta = item.get("metadata") or {}
+        spec = item.get("spec") or {}
+        status = item.get("status") or {}
+        name = str(meta.get("name") or "").strip()
+        phase = str(status.get("phase") or "").strip()
+        if not name or phase != "Released":
+            continue
+        if not spec.get("claimRef"):
+            continue
+        _log(log, f"[GARAGE] Clearing stale claimRef on PV {name} (phase=Released)")
+        _kubectl(
+            [
+                "patch",
+                "pv",
+                name,
+                "--type=json",
+                "-p",
+                '[{"op":"remove","path":"/spec/claimRef"}]',
+            ],
+            env=env,
+            timeout=60,
+        )
+
+
+def _ensure_garage_secret(*, namespace: str, env: dict | None = None, log: _LogFn | None = None) -> None:
+    secret_name = str((env or {}).get("GARAGE_SECRET_NAME") or "garage-secrets").strip() or "garage-secrets"
+    force = _to_bool((env or {}).get("PROLE_GARAGE_FORCE_SECRET"), default=False)
+    if not force and _exists("secret", secret_name, namespace, env=env):
+        return
+
+    payload = (
+        "apiVersion: v1\n"
+        "kind: Secret\n"
+        "metadata:\n"
+        f"  name: {secret_name}\n"
+        "type: Opaque\n"
+        "stringData:\n"
+        f"  rpc_secret: {json.dumps(secrets.token_hex(32))}\n"
+        f"  admin_token: {json.dumps(secrets.token_urlsafe(32))}\n"
+        f"  metrics_token: {json.dumps(secrets.token_urlsafe(32))}\n"
+    )
+    _kubectl(
+        ["-n", namespace, "apply", "-f", "-"],
+        env=env,
+        input_text=payload,
+        timeout=90,
+        check=True,
+    )
+    _log(log, f"[GARAGE] Ensured secret/{secret_name} in namespace {namespace}")
+
+
 def initialize(
     *,
     namespace: str | None = None,
@@ -68,12 +140,43 @@ def update(
     _detect_mode(mode, env)  # mode kept for API parity
     target_ns = _garage_namespace(namespace, env)
     _ensure_namespace(target_ns, env)
+    _repair_released_garage_pvs(env=env, log=log)
+    _ensure_garage_secret(namespace=target_ns, env=env, log=log)
+
+    _prune_named_workload_other_namespaces(
+        kind="statefulset",
+        name="garage",
+        target_namespace=target_ns,
+        label_selector="app=garage",
+        env=env,
+        log=log,
+    )
+    _prune_named_workload_other_namespaces(
+        kind="service",
+        name="garage",
+        target_namespace=target_ns,
+        label_selector="app=garage",
+        env=env,
+        log=log,
+    )
+    _prune_named_workload_other_namespaces(
+        kind="configmap",
+        name="garage-config",
+        target_namespace=target_ns,
+        label_selector="app=garage",
+        env=env,
+        log=log,
+    )
 
     for manifest in _manifest_files(project_root):
         if not manifest.exists():
             continue
         _log(log, f"[GARAGE] Applying {manifest}")
+        name = manifest.name
+        if name in {"storageclass-synology-iscsi.yaml", "iscsi-pvs.yaml"}:
             _kubectl(["apply", "-f", str(manifest)], env=env, timeout=240, check=True)
+        else:
+            _kubectl(["-n", target_ns, "apply", "-f", str(manifest)], env=env, timeout=240, check=True)
 
     if _exists("statefulset", "garage", target_ns, env=env):
         _wait_rollout("statefulset", "garage", target_ns, env=env)

knoe/core/ops/opentofu.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
 
 from pathlib import Path
+import json
+import secrets
+import subprocess
 
 from ._services_common import (
     _LogFn,
@@ -11,6 +14,8 @@ from ._services_common import (
     _log,
     _manifest_path,
     _namespace,
+    _prune_named_workload_other_namespaces,
+    _reconcile_deployment_replicasets,
     _to_bool,
     _wait_rollout,
 )
@@ -28,6 +33,85 @@ def _manifest(project_root: str | Path) -> Path:
     return _manifest_path(project_root, "k8s", "opentofu", "deployment.yaml")
 
 
+def _build_basic_auth_entry(password: str) -> str:
+    pwd = (password or "").strip() or "changeme"
+    try:
+        res = subprocess.run(
+            ["openssl", "passwd", "-apr1", pwd],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if res.returncode == 0:
+            hashed = (res.stdout or "").strip()
+            if hashed:
+                return f"admin:{hashed}"
+    except Exception:
+        pass
+    return f"admin:{{PLAIN}}{pwd}"
+
+
+def _ensure_admin_secret(
+    *,
+    namespace: str,
+    env: dict | None = None,
+    log: _LogFn | None = None,
+) -> None:
+    force = _to_bool((env or {}).get("PROLE_OPENTOFU_FORCE_SECRET"), default=False)
+    if not force and _exists("secret", "opentofu-admin", namespace, env=env):
+        return
+
+    admin_password = (
+        str((env or {}).get("OPENTOFU_ADMIN_PASSWORD") or "").strip()
+        or str((env or {}).get("DB_PASSWORD") or "").strip()
+        or secrets.token_urlsafe(18)
+    )
+    auth_entry = _build_basic_auth_entry(admin_password)
+
+    secret_yaml = (
+        "apiVersion: v1\n"
+        "kind: Secret\n"
+        "metadata:\n"
+        "  name: opentofu-admin\n"
+        "type: Opaque\n"
+        "stringData:\n"
+        f"  admin_password: {json.dumps(admin_password)}\n"
+        f"  auth: {json.dumps(auth_entry)}\n"
+    )
+    _kubectl(
+        ["-n", namespace, "apply", "-f", "-"],
+        env=env,
+        input_text=secret_yaml,
+        timeout=90,
+        check=True,
+    )
+    _log(log, f"[OPENTOFU] Ensured secret/opentofu-admin in namespace {namespace}")
+
+
+def _deployment_is_available(
+    *,
+    deployment: str,
+    namespace: str,
+    env: dict | None = None,
+) -> bool:
+    res = _kubectl(
+        ["-n", namespace, "get", "deployment", deployment, "-o", "json"],
+        env=env,
+        timeout=20,
+    )
+    if res.returncode != 0 or not res.stdout.strip():
+        return False
+    try:
+        payload = json.loads(res.stdout)
+    except Exception:
+        return False
+    spec = payload.get("spec") or {}
+    status = payload.get("status") or {}
+    desired = int(spec.get("replicas") or 1)
+    available = int(status.get("availableReplicas") or 0)
+    return desired > 0 and available >= desired
+
+
 def initialize(
     *,
     namespace: str | None = None,
@@ -61,6 +145,51 @@ def update(
     _detect_mode(mode, env)  # mode kept for API parity
     target_ns = _opentofu_namespace(namespace, env)
     _ensure_namespace(target_ns, env)
+    _ensure_admin_secret(namespace=target_ns, env=env, log=log)
+    _prune_named_workload_other_namespaces(
+        kind="deployment",
+        name="opentofu",
+        target_namespace=target_ns,
+        label_selector="app=opentofu",
+        env=env,
+        log=log,
+    )
+    _prune_named_workload_other_namespaces(
+        kind="service",
+        name="opentofu",
+        target_namespace=target_ns,
+        label_selector="app=opentofu",
+        env=env,
+        log=log,
+    )
+
+    force_apply = _to_bool((env or {}).get("PROLE_OPENTOFU_FORCE_APPLY"), default=False)
+    if not force_apply and _exists("deployment", "opentofu", target_ns, env=env) and _exists(
+        "service", "opentofu", target_ns, env=env
+    ):
+        _reconcile_deployment_replicasets(
+            deployment="opentofu",
+            namespace=target_ns,
+            label_selector="app=opentofu",
+            env=env,
+            log=log,
+        )
+        if _deployment_is_available(deployment="opentofu", namespace=target_ns, env=env):
+            _log(
+                log,
+                f"[OPENTOFU] Healthy deployment already present in namespace {target_ns}; skipping re-apply",
+            )
+            return
+        _log(
+            log,
+            f"[OPENTOFU] Existing deployment in namespace {target_ns} is not ready; applying manifest for recovery",
+        )
+
+    if force_apply:
+        _log(
+            log,
+            f"[OPENTOFU] Force apply enabled; reconciling manifest in namespace {target_ns}",
+        )
 
     manifest = _manifest(project_root)
     if not manifest.exists():
@@ -73,6 +202,13 @@ def update(
         timeout=240,
         check=True,
     )
+    _reconcile_deployment_replicasets(
+        deployment="opentofu",
+        namespace=target_ns,
+        label_selector="app=opentofu",
+        env=env,
+        log=log,
+    )
     _wait_rollout("deployment", "opentofu", target_ns, env=env)
 
 

knoe/core/ops/registry.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+import json
 
 from ._services_common import (
     _LogFn,
@@ -12,6 +13,8 @@ from ._services_common import (
     _kubectl,
     _log,
     _manifest_path,
+    _prune_named_workload_other_namespaces,
+    _reconcile_deployment_replicasets,
     _registry_namespace,
     _to_bool,
     _wait_rollout,
@@ -59,6 +62,30 @@ def _manifest(project_root: str | Path) -> Path:
     return _manifest_path(project_root, "k8s", "registry", "deployment.yaml")
 
 
+def _deployment_is_available(
+    *,
+    deployment: str,
+    namespace: str,
+    env: dict | None = None,
+) -> bool:
+    res = _kubectl(
+        ["-n", namespace, "get", "deployment", deployment, "-o", "json"],
+        env=env,
+        timeout=20,
+    )
+    if res.returncode != 0 or not res.stdout.strip():
+        return False
+    try:
+        payload = json.loads(res.stdout)
+    except Exception:
+        return False
+    spec = payload.get("spec") or {}
+    status = payload.get("status") or {}
+    desired = int(spec.get("replicas") or 1)
+    available = int(status.get("availableReplicas") or 0)
+    return desired > 0 and available >= desired
+
+
 def initialize(
     *,
     namespace: str | None = None,
@@ -96,6 +123,51 @@ def update(
 
     target_ns = _registry_namespace(namespace, env)
     _ensure_namespace(target_ns, env)
+    _prune_named_workload_other_namespaces(
+        kind="deployment",
+        name="registry",
+        target_namespace=target_ns,
+        label_selector="app=registry",
+        env=env,
+        log=log,
+    )
+    _prune_named_workload_other_namespaces(
+        kind="service",
+        name="registry",
+        target_namespace=target_ns,
+        label_selector="app=registry",
+        env=env,
+        log=log,
+    )
+
+    force_apply = _to_bool((env or {}).get("PROLE_REGISTRY_FORCE_APPLY"), default=False)
+    if not force_apply and _exists("deployment", "registry", target_ns, env=env) and _exists(
+        "service", "registry", target_ns, env=env
+    ):
+        _reconcile_deployment_replicasets(
+            deployment="registry",
+            namespace=target_ns,
+            label_selector="app=registry",
+            env=env,
+            log=log,
+        )
+        if _deployment_is_available(deployment="registry", namespace=target_ns, env=env):
+            _log(
+                log,
+                f"[REGISTRY] Healthy deployment already present in namespace {target_ns}; skipping re-apply",
+            )
+            return
+        _log(
+            log,
+            f"[REGISTRY] Existing deployment in namespace {target_ns} is not ready; applying manifest for recovery",
+        )
+
+    if force_apply:
+        _log(
+            log,
+            f"[REGISTRY] Force apply enabled; reconciling manifest in namespace {target_ns}",
+        )
+
     manifest = _manifest(project_root)
     if not manifest.exists():
         raise RuntimeError(f"Registry manifest not found: {manifest}")
@@ -106,6 +178,13 @@ def update(
         timeout=180,
         check=True,
     )
+    _reconcile_deployment_replicasets(
+        deployment="registry",
+        namespace=target_ns,
+        label_selector="app=registry",
+        env=env,
+        log=log,
+    )
     _wait_rollout("deployment", "registry", target_ns, env=env)
 
 

knoe/core/ops/storage.py

@@ -5,6 +5,7 @@ import json
 import os
 import re
 import subprocess
+import time
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
@@ -144,10 +145,19 @@ def _choose_root_pair(spec: ClusterStorageSpec) -> tuple[Path, Path]:
             mounted_roots.append(root)
 
     if not mounted_roots:
+        allow_unmounted = True
+        if env:
+            raw = str(env.get("PROLE_STORAGE_ALLOW_UNMOUNTED_ROOTS") or "").strip().lower()
+            if raw:
+                allow_unmounted = raw in {"1", "true", "yes", "on"}
+
+        if not allow_unmounted:
             raise StorageProvisioningError(
                 "No mounted Synology roots available. Expected mounted paths like /synology/d001..d004."
             )
 
+        mounted_roots = [Path(root_str) for root_str in spec.synology_roots]
+
     ident = f"{spec.namespace}:{spec.cluster_name}:{spec.service_name}"
     digest = int(hashlib.sha256(ident.encode("utf-8")).hexdigest(), 16)
     data_index = digest % len(mounted_roots)
@@ -253,11 +263,27 @@ def ensure_host_path(path: str, *, uid: int, gid: int, mode: int = 0o700) -> Non
     target = Path(path)
     if target.exists() and not target.is_dir():
         raise StorageProvisioningError(f"Host path exists but is not a directory: {path}")
+    try:
         target.mkdir(parents=True, exist_ok=True)
+    except OSError as exc:
+        if exc.errno == 30:  # Read-only filesystem on local runner
+            return
+        raise StorageProvisioningError(f"Cannot create host path '{path}': {exc}") from exc
+
+    try:
         os.chmod(target, mode)
+    except OSError as exc:
+        if exc.errno == 30:
+            return
+        raise StorageProvisioningError(f"Cannot set mode on host path '{path}': {exc}") from exc
     try:
         os.chown(target, uid, gid)
-    except PermissionError as exc:
+    except PermissionError:
+        # Non-root execution is expected in some local automation contexts.
+        return
+    except OSError as exc:
+        if exc.errno == 30:
+            return
         raise StorageProvisioningError(
             f"Cannot set ownership on host path '{path}' to {uid}:{gid}: {exc}"
         ) from exc
@@ -286,6 +312,75 @@ def _kubectl_apply_manifest(spec: ClusterStorageSpec, manifest: dict[str, Any])
         raise StorageProvisioningError(f"Failed to apply PV manifest '{manifest['metadata']['name']}': {stderr}")
 
 
+def _ensure_node_host_path(spec: ClusterStorageSpec, host_path: str) -> None:
+    env = _command_env(spec)
+    ident = f"{spec.namespace}:{spec.cluster_name}:{spec.node_name}:{host_path}"
+    suffix = hashlib.sha1(ident.encode("utf-8")).hexdigest()[:10]
+    pod_name = f"prole-pathprep-{suffix}"
+    namespace = "kube-system"
+
+    manifest = {
+        "apiVersion": "v1",
+        "kind": "Pod",
+        "metadata": {"name": pod_name, "namespace": namespace},
+        "spec": {
+            "restartPolicy": "Never",
+            "nodeName": spec.node_name,
+            "tolerations": [
+                {"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"},
+                {"key": "node-role.kubernetes.io/master", "operator": "Exists", "effect": "NoSchedule"},
+            ],
+            "containers": [
+                {
+                    "name": "pathprep",
+                    "image": "busybox:1.36",
+                    "command": ["sh", "-c", "mkdir -p /target && chmod 700 /target || true"],
+                    "volumeMounts": [{"name": "target", "mountPath": "/target"}],
+                }
+            ],
+            "volumes": [
+                {
+                    "name": "target",
+                    "hostPath": {"path": host_path, "type": "DirectoryOrCreate"},
+                }
+            ],
+        },
+    }
+
+    apply_res = _run_command(
+        [spec.kubectl_bin, "apply", "-f", "-"],
+        env=env,
+        input_text=json.dumps(manifest),
+    )
+    if apply_res.returncode != 0:
+        raise StorageProvisioningError(
+            f"Failed to start host path bootstrap pod '{pod_name}': {(apply_res.stderr or apply_res.stdout or '').strip()}"
+        )
+
+    try:
+        for _ in range(60):
+            pod = _kubectl_get_json(spec, ["-n", namespace, "get", "pod", pod_name])
+            if pod:
+                phase = str((pod.get("status") or {}).get("phase") or "")
+                if phase in {"Succeeded", "Running"}:
+                    return
+                if phase == "Failed":
+                    logs = _run_command([spec.kubectl_bin, "-n", namespace, "logs", pod_name], env=env)
+                    raise StorageProvisioningError(
+                        f"Host path bootstrap pod '{pod_name}' failed for '{host_path}': {(logs.stdout or logs.stderr or '').strip()}"
+                    )
+            time.sleep(2)
+    finally:
+        _run_command(
+            [spec.kubectl_bin, "-n", namespace, "delete", "pod", pod_name, "--ignore-not-found", "--wait=false"],
+            env=env,
+        )
+
+    raise StorageProvisioningError(
+        f"Timed out waiting for host path bootstrap pod '{pod_name}' to prepare '{host_path}'."
+    )
+
+
 def _collect_existing_pvs(spec: ClusterStorageSpec) -> dict[str, dict[str, Any]]:
     pv_list = _kubectl_get_json(spec, ["get", "pv"])
     if pv_list is None:
@@ -435,6 +530,8 @@ def provision_cluster_storage(spec: ClusterStorageSpec) -> ProvisionedStorage:
     )
 
     paths = build_cluster_storage_paths(spec)
+    _ensure_node_host_path(spec, paths.data_path)
+    _ensure_node_host_path(spec, paths.wal_path)
     ensure_host_path(
         paths.data_path,
         uid=spec.postgres_uid,

prole.sh

@@ -21,6 +21,24 @@ SILENT=0
 RESET=0
 EXTRA_ARGS=()
 
+# Canonical wrapper aliases:
+#   ./prole.sh --reset    => full silent reset run
+#   ./prole.sh --update   => silent update run
+#   ./prole.sh --perfsnap => topology/storage snapshot
+if [[ $# -gt 0 ]]; then
+  case "$1" in
+    --reset)
+      set -- reset-run "${@:2}"
+      ;;
+    --update)
+      set -- update "${@:2}"
+      ;;
+    --perfsnap)
+      set -- perfsnap "${@:2}"
+      ;;
+  esac
+fi
+
 usage() {
   echo "Usage: $0 [options] [command] [command-options]"
   echo
@@ -31,7 +49,7 @@ usage() {
   echo "  -c, --config     Path to prole.cfg config file (default: conf/prole.cfg)"
   echo "  --coverage       Create a coverage report for the run of install.py"
   echo "  -s, --silent     Run unattended install (passes -S to install.py)"
-  echo "  -r, --reset      Perform a reset before action (site/deploy/install)"
+  echo "  -r, --reset      Perform a reset before action (site/deploy/install); for canonical full reset run use './prole.sh --reset'"
   echo
   echo "Commands:"
   echo "  init       Run/re-run initialization scripts to verify, repair, and renew the deployment"
@@ -39,6 +57,9 @@ usage() {
   echo "  ansible    Run ansible-playbook via ansible.sh"
   echo "  site       Shortcut for running the site deployment"
   echo "  reset      Shortcut for k3s factory reset (full clear)"
+  echo "  reset-run  Canonical full reset run (equivalent to legacy '-r -s -v -l')"
+  echo "  update     Canonical silent update run (installer --update)"
+  echo "  perfsnap   Save topology/storage performance snapshot (includes storage_probe)"
   echo "  deploy     Full end-to-end: site run, and silent install (reset optional)"
   echo "  start      Verify/clean environment, ensure cluster availability, run silent install"
   echo "  stop       Validate environment, run full backup+archive, stop services (keep cluster idle)"
@@ -60,7 +81,7 @@ while [[ $# -gt 0 ]]; do
     -s|--silent) SILENT=1; shift ;;
     -r|--reset) RESET=1; shift ;;
     -h|--help) usage; exit 0 ;;
-    install|ansible|site|reset|deploy|init|start|stop|backup|passwd|status) break ;;
+    install|ansible|site|reset|reset-run|update|perfsnap|deploy|init|start|stop|backup|passwd|status) break ;;
     *) EXTRA_ARGS+=("$1"); shift ;;
   esac
 done
@@ -135,6 +156,18 @@ prepare_ansible_color() {
 
 PROLE_CFG_DEFAULT="${ROOT_DIR}/conf/prole.cfg"
 
+resolve_cfg_path() {
+  if [[ -n "${CONFIG_PATH:-}" ]]; then
+    printf '%s' "${CONFIG_PATH}"
+    return 0
+  fi
+  if [[ -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then
+    printf '%s' "${PROLE_CONF}/prole.cfg"
+    return 0
+  fi
+  printf '%s' "${PROLE_CFG_DEFAULT}"
+}
+
 log_msg() {
   printf '%s\n' "$*"
 }
@@ -541,6 +574,129 @@ if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
 fi
 
 case "${CMD}" in
+  reset-run)
+    CONFIG_PATH="$(resolve_cfg_path)"
+    _reset_args=("install" "-s" "-v" "-l" "-r" "-c" "${CONFIG_PATH}")
+    if [[ "${COVERAGE}" -eq 1 ]]; then
+      _reset_args=("--coverage" "${_reset_args[@]}")
+    fi
+    exec "${ROOT_DIR}/prole.sh" "${_reset_args[@]}"
+    ;;
+  update)
+    CONFIG_PATH="$(resolve_cfg_path)"
+    _update_args=("install" "-s" "-v" "-l" "-c" "${CONFIG_PATH}" "--update")
+    if [[ "${COVERAGE}" -eq 1 ]]; then
+      _update_args=("--coverage" "${_update_args[@]}")
+    fi
+    exec "${ROOT_DIR}/prole.sh" "${_update_args[@]}"
+    ;;
+  perfsnap)
+    CONFIG_PATH="$(resolve_cfg_path)"
+    load_prole_cfg "${CONFIG_PATH}"
+    _mode_for_env=$(resolve_prole_mode)
+    if [[ -z "${_mode_for_env:-}" ]]; then
+      _mode_for_env="k3d"
+    fi
+    ensure_openbao_url_for_mode "${_mode_for_env}"
+    mkdir -p "${ROOT_DIR}/logs" "${ROOT_DIR}/data/topology"
+    _snap_ts="$(date +%Y%m%d-%H%M%S)"
+    _snap_file="${ROOT_DIR}/logs/perfsnap-${_snap_ts}.json"
+    export PROLE_PERFSNAP_FILE="${_snap_file}"
+    export PROLE_PERFSNAP_CFG="${CONFIG_PATH}"
+    python3 - <<'PY'
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+from knoe import prole_conf
+from knoe.core.env import _kubectl_base_cmd_for_k3s
+from knoe.core.topology import TopologyDiscoveryConfig, discover_cluster_topology
+
+cfg_path = Path(os.environ["PROLE_PERFSNAP_CFG"]).resolve()
+snap_path = Path(os.environ["PROLE_PERFSNAP_FILE"]).resolve()
+
+cfg = prole_conf.load_layered_config(cfg_path, require_exists=True)
+
+def cfg_get(section: str, key: str, default: str = "") -> str:
+    try:
+        return (cfg.get(section, key, fallback=default) or default).strip()
+    except Exception:
+        return default
+
+mode_raw = (
+    cfg_get("Deployment", "MODE")
+    or cfg_get("Global", "DEPLOYMENT_MODE")
+    or cfg_get("Global", "CLUSTER_ENV")
+    or "k3d"
+)
+mode = mode_raw.lower()
+project_root = cfg_path.parent.parent
+prole_data = cfg_get("Inputs", "env_setup.PROLE_DATA") or str(project_root / "data")
+
+if mode in {"k3s", "k8s"}:
+    managed_kubeconfig = os.environ.get("PROLE_KUBECONFIG") or os.environ.get("KUBECONFIG")
+    if not managed_kubeconfig:
+        candidate = project_root / "prole-k3s.kubeconfig"
+        if candidate.exists():
+            managed_kubeconfig = str(candidate)
+    base_cmd = _kubectl_base_cmd_for_k3s(managed_kubeconfig=managed_kubeconfig)
+else:
+    base_cmd = ["kubectl"]
+    context = (
+        os.environ.get("KUBECTL_CONTEXT")
+        or cfg_get("Dev Cluster (k3d)", "KUBECTL_CONTEXT")
+        or ""
+    ).strip()
+    if not context and mode == "k3d":
+        target = (
+            cfg_get("Global", "DEPLOYMENT_TARGET")
+            or cfg_get("Dev Cluster (k3d)", "DISPLAY_NAME")
+            or ""
+        ).strip()
+        if target:
+            context = target if target.startswith("k3d-") else f"k3d-{target}"
+    if context:
+        base_cmd.extend(["--context", context])
+output_root = Path(prole_data) / "topology"
+output_root.mkdir(parents=True, exist_ok=True)
+
+env_map = dict(os.environ)
+env_map["PROLE_CFG_FILE"] = str(cfg_path)
+env_map["PROLE_DATA"] = str(Path(prole_data).resolve())
+
+result = discover_cluster_topology(
+    kubectl_base_cmd=base_cmd,
+    topology_root=output_root,
+    config=TopologyDiscoveryConfig(storage_probe_enabled=True, storage_probe_quick=True),
+    env=env_map,
+)
+
+payload = {
+    "timestamp": datetime.now(timezone.utc).isoformat(),
+    "cfg": str(cfg_path),
+    "mode": mode,
+    "kubectl_base_cmd": base_cmd,
+    "topology_root": str(output_root),
+    "collection_status": result.topology.collection_status,
+    "node_count": len(result.topology.nodes),
+    "ready_nodes": [n.name for n in result.topology.nodes if n.ready],
+    "collector_applied": result.collector_applied,
+    "expected_ready_nodes": list(result.expected_ready_nodes),
+    "reported_nodes": list(result.reported_nodes),
+    "missing_nodes": list(result.missing_nodes),
+    "xml": {
+        "cluster": str(output_root / "cluster-topology.xml"),
+        "node": str(output_root / "node-topology.xml"),
+    },
+}
+
+snap_path.parent.mkdir(parents=True, exist_ok=True)
+snap_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+print(f"Performance snapshot written: {snap_path}")
+print(f"Topology root: {output_root}")
+PY
+    ;;
   install)
     LOGFILE="${ROOT_DIR}/logs/install-${TS}.log"
 

scan/network_description.txt

@@ -1,55 +1,51 @@
 Network Discovery Summary:
-Primary Router: 10.0.0.1 (Unknown)
+Primary Router: 10.0.0.1 (eero_5d:50:f2)
-DNS Servers: 10.0.0.4, 100.100.100.100, 10.0.0.5
+DNS Servers: 10.0.0.5, 10.0.0.4, 100.100.100.100
 Detected Devices:
-- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
 - 10.0.0.38 [7e:19:9a:fe:9f:ed] (7e:19:9a:fe:9f:ed): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
-- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server']
+- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
-- 10.0.0.152 [8c:79:f5:ba:a8:2c] (SamsungElect_ba:a8:2c): Ports [], Services: []
+- 10.0.0.3 [2c:cf:67:8c:2b:47] (RaspberryPi_8c:2b:47): Ports [22, 53, 80, 443, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
-- 10.0.0.100 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: []
+- 10.0.0.1 [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: []
-- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
+- 10.0.0.5 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC']
-- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: []
-- 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH']
 - 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server']
-- 10.0.0.49 [00:e0:4c:08:0c:75] (RealtekSemic_08:0c:75): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
+- 10.0.0.26 [c8:db:26:08:52:f8] (Logitech_08:52:f8): Ports [], Services: []
-- 10.0.0.1 [Unknown] (Unknown): Ports [53], Services: ['DNS']
+- 10.0.0.6 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server']
-- 10.0.0.2 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.4 [b8:27:eb:b3:6f:2b] (RaspberryPiF_b3:6f:2b): Ports [22, 53, 2049], Services: ['DNS', 'SSH', 'NFS']
-- 10.0.0.3 [Unknown] (Unknown): Ports [22, 53, 80, 443, 2049, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
+- 10.0.0.22 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server']
-- 10.0.0.4 [Unknown] (Unknown): Ports [22, 53, 443, 2049], Services: ['DNS', 'SSH', 'NFS', 'Web Server']
+- 10.0.0.2 [dc:a4:ca:ea:1a:2f] (Apple_ea:1a:2f): Ports [], Services: []
-- 10.0.0.5 [Unknown] (Unknown): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
+- 10.0.0.45 [f8:b4:6a:30:a4:1f] (HewlettPacka_30:a4:1f): Ports [80, 443], Services: ['Web Server']
-- 10.0.0.6 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
+- 10.0.0.48 [0c:fe:45:53:f4:3c] (SonyInteract_53:f4:3c): Ports [], Services: []
-- 10.0.0.22 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
+- 10.0.0.37 [54:07:7d:22:c0:b2] (Netgear_22:c0:b2): Ports [80], Services: ['Web Server']
-- 10.0.0.26 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.46 [d4:f7:d5:40:ab:17] (SonyInteract_40:ab:17): Ports [], Services: []
-- 10.0.0.37 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
+- 10.0.0.55 [48:a6:b8:a7:50:60] (Sonos_a7:50:60): Ports [], Services: []
-- 10.0.0.41 [Unknown] (Unknown): Ports [22], Services: ['SSH']
+- 10.0.0.41 [b8:27:eb:57:10:d9] (RaspberryPiF_57:10:d9): Ports [22], Services: ['SSH']
-- 10.0.0.45 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
+- 10.0.0.205 [a8:20:66:28:12:e7] (Apple_28:12:e7): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
-- 10.0.0.46 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: []
-- 10.0.0.48 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.95 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
-- 10.0.0.55 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.99 [20:28:bc:f1:dc:ba] (Visionscape_f1:dc:ba): Ports [], Services: []
-- 10.0.0.58 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.111 [ca:0b:6c:5d:0a:e5] (ca:0b:6c:5d:0a:e5): Ports [], Services: []
-- 10.0.0.95 [Unknown] (Unknown): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
+- 10.0.0.123 [3c:ef:8c:96:e3:3c] (ZhejiangDahu_96:e3:3c): Ports [80], Services: ['Web Server']
-- 10.0.0.99 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.124 [9c:8e:cd:02:73:c4] (AmcrestTechn_02:73:c4): Ports [80], Services: ['Web Server']
-- 10.0.0.111 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH']
-- 10.0.0.113 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.128 [20:c9:d0:94:30:dd] (Apple_94:30:dd): Ports [5900], Services: ['VNC']
-- 10.0.0.117 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
+- 10.0.0.117 [40:f5:20:1e:5e:91] (Espressif_1e:5e:91): Ports [80], Services: ['Web Server']
-- 10.0.0.123 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
+- 10.0.0.125 [a0:60:32:04:73:ba] (AmcrestTechn_04:73:ba): Ports [80], Services: ['Web Server']
-- 10.0.0.124 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
+- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
-- 10.0.0.125 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
+- 10.0.0.130 [f8:bb:bf:a3:84:eb] (eero_a3:84:eb): Ports [53], Services: ['DNS']
-- 10.0.0.128 [Unknown] (Unknown): Ports [22, 445, 5900], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)']
+- 10.0.0.145 [7c:a6:b0:04:db:1c] (7c:a6:b0:04:db:1c): Ports [], Services: []
-- 10.0.0.130 [Unknown] (Unknown): Ports [53], Services: ['DNS']
+- 10.0.0.155 [62:17:97:b3:06:a1] (62:17:97:b3:06:a1): Ports [], Services: []
-- 10.0.0.143 [Unknown] (Unknown): Ports [53], Services: ['DNS']
+- 10.0.0.170 [b0:ee:7b:ca:73:99] (Roku_ca:73:99): Ports [], Services: []
-- 10.0.0.145 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.188 [00:16:6c:c5:1f:54] (SamsungElect_c5:1f:54): Ports [80, 443], Services: ['Web Server']
-- 10.0.0.155 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.189 [00:17:88:a3:2f:cc] (PhilipsLight_a3:2f:cc): Ports [80, 443], Services: ['Web Server']
-- 10.0.0.170 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.179 [48:a6:b8:a4:8e:cc] (Sonos_a4:8e:cc): Ports [], Services: []
-- 10.0.0.175 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
+- 10.0.0.175 [28:80:88:e4:10:1a] (Netgear_e4:10:1a): Ports [80], Services: ['Web Server']
-- 10.0.0.179 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.196 [60:5f:8d:88:08:f2] (eero_88:08:f2): Ports [53], Services: ['DNS']
-- 10.0.0.188 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
+- 10.0.0.180 [4c:e1:73:42:1a:be] (HuizhouDehon_2:1a:be): Ports [445], Services: ['SMB/CIFS (Possible Windows/AD)']
-- 10.0.0.189 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
+- 10.0.0.199 [7c:a6:b0:01:a7:33] (7c:a6:b0:01:a7:33): Ports [], Services: []
-- 10.0.0.196 [Unknown] (Unknown): Ports [53], Services: ['DNS']
+- 10.0.0.204 [50:eb:f6:56:f3:26] (ASUSTekCOMPU_56:f3:26): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama']
-- 10.0.0.199 [Unknown] (Unknown): Ports [], Services: []
+- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server']
-- 10.0.0.204 [Unknown] (Unknown): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama']
+- 10.0.0.1\ [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: []
-- 10.0.0.205 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
+Ollama Instances found at: 10.0.0.38, 10.0.0.204
-Ollama Instances found at: 10.0.0.38, 10.0.0.49, 10.0.0.204
 
 Ansible Inventory Summary:
 Inventory directory: /Users/chrisfu/dev/prole/infrastructure

scripts/reset-ns.sh

@@ -12,6 +12,11 @@ Required:
 Options:
   --delete-pvcs      Delete all PVCs in the namespace (DANGEROUS: wipes state)
   --delete-config    Delete ConfigMaps and Secrets (VERY DANGEROUS)
+  --keep-registry    Preserve registry workloads/services in the target namespace
+  --clear-node-reservations
+                    Best-effort clear node reservations by uncordoning nodes
+  --no-clear-node-reservations
+                    Disable node reservation cleanup (default)
   -h, --help         Show this help
 
 Notes:
@@ -29,6 +34,8 @@ EOF
 NS=""
 DELETE_PVCS=false
 DELETE_CONFIG=false
+KEEP_REGISTRY=false
+CLEAR_NODE_RESERVATIONS=false
 
 [[ $# -eq 0 ]] && usage && exit 1
 
@@ -37,6 +44,9 @@ while [[ $# -gt 0 ]]; do
     -n) NS="${2:-}"; shift 2 ;;
     --delete-pvcs) DELETE_PVCS=true; shift ;;
     --delete-config) DELETE_CONFIG=true; shift ;;
+    --keep-registry) KEEP_REGISTRY=true; shift ;;
+    --clear-node-reservations) CLEAR_NODE_RESERVATIONS=true; shift ;;
+    --no-clear-node-reservations) CLEAR_NODE_RESERVATIONS=false; shift ;;
     -h|--help) usage; exit 0 ;;
     *) echo "Unknown argument: $1"; echo; usage; exit 1 ;;
   esac
@@ -60,8 +70,56 @@ esac
 echo "Resetting namespace: $NS"
 echo "  DELETE_PVCS=$DELETE_PVCS"
 echo "  DELETE_CONFIG=$DELETE_CONFIG"
+echo "  KEEP_REGISTRY=$KEEP_REGISTRY"
+echo "  CLEAR_NODE_RESERVATIONS=$CLEAR_NODE_RESERVATIONS"
 echo
 
+if ! kubectl get namespace "$NS" >/dev/null 2>&1; then
+  echo "Namespace '$NS' does not exist; skipping."
+  exit 0
+fi
+
+is_registry_resource() {
+  local kind="$1"
+  local name="$2"
+
+  [[ "$KEEP_REGISTRY" == "true" ]] || return 1
+
+  if [[ "$name" == "registry" ]]; then
+    return 0
+  fi
+
+  local app_label=""
+  app_label=$(kubectl -n "$NS" get "$kind" "$name" -o jsonpath='{.metadata.labels.app}' 2>/dev/null || true)
+  [[ "$app_label" == "registry" ]]
+}
+
+scale_down_workloads() {
+  local kind="$1"
+  local item
+  while IFS= read -r item; do
+    [[ -z "$item" ]] && continue
+    if is_registry_resource "$kind" "$item"; then
+      echo "  preserving ${kind}/${item}"
+      continue
+    fi
+    kubectl -n "$NS" scale "$kind" "$item" --replicas=0 2>/dev/null || true
+  done < <(kubectl -n "$NS" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
+}
+
+delete_kind_resources() {
+  local kind="$1"
+  local item
+  while IFS= read -r item; do
+    [[ -z "$item" ]] && continue
+    if is_registry_resource "$kind" "$item"; then
+      echo "  preserving ${kind}/${item}"
+      continue
+    fi
+    kubectl -n "$NS" delete "$kind" "$item" --ignore-not-found --wait=false 2>/dev/null || true
+  done < <(kubectl -n "$NS" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
+}
+
 # -------------------------
 # CNPG: stop respawning in THIS namespace by deleting CNPG Cluster CRs
 # Detect via CRD existence (more reliable than kubectl api-resources on some setups)
@@ -97,14 +155,19 @@ fi
 # Scale down controllers to reduce churn
 # -------------------------
 echo "Scaling down Deployments/StatefulSets..."
-kubectl -n "$NS" scale deploy --all --replicas=0 2>/dev/null || true
+scale_down_workloads deployment
-kubectl -n "$NS" scale sts --all --replicas=0 2>/dev/null || true
+scale_down_workloads statefulset
 
 # -------------------------
 # Delete common workload controllers
 # -------------------------
 echo "Deleting workload controllers..."
-kubectl -n "$NS" delete deploy,sts,ds,rs,job,cronjob --all --ignore-not-found --wait=false
+delete_kind_resources deployment
+delete_kind_resources statefulset
+delete_kind_resources daemonset
+delete_kind_resources replicaset
+delete_kind_resources job
+delete_kind_resources cronjob
 
 # -------------------------
 # Delete services safely (NEVER delete "kubernetes" service in default)
@@ -115,6 +178,10 @@ for s in $(kubectl -n "$NS" get svc -o jsonpath='{range .items[*]}{.metadata.nam
     echo "  skipping protected service: default/kubernetes"
     continue
   fi
+  if is_registry_resource service "$s"; then
+    echo "  preserving service/$s"
+    continue
+  fi
   kubectl -n "$NS" delete svc "$s" --ignore-not-found --wait=false
 done
 
@@ -154,10 +221,32 @@ fi
 # Final pod cleanup
 # -------------------------
 echo "Deleting remaining pods..."
-kubectl -n "$NS" delete pod --all --ignore-not-found --wait=false
+deleted_pods=()
+for p in $(kubectl -n "$NS" get pod -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true); do
+  if is_registry_resource pod "$p"; then
+    echo "  preserving pod/$p"
+    continue
+  fi
+  kubectl -n "$NS" delete pod "$p" --ignore-not-found --wait=false 2>/dev/null || true
+  deleted_pods+=("$p")
+done
 
 echo "Waiting for pods to terminate..."
-kubectl -n "$NS" wait --for=delete pod --all --timeout=180s 2>/dev/null || true
+if [[ ${#deleted_pods[@]} -gt 0 ]]; then
+  wait_args=()
+  for p in "${deleted_pods[@]}"; do
+    wait_args+=("pod/$p")
+  done
+  kubectl -n "$NS" wait --for=delete "${wait_args[@]}" --timeout=180s 2>/dev/null || true
+fi
+
+if [[ "$CLEAR_NODE_RESERVATIONS" == "true" ]]; then
+  echo "Clearing node reservations (uncordon all nodes)..."
+  while IFS= read -r node; do
+    [[ -z "$node" ]] && continue
+    kubectl uncordon "$node" >/dev/null 2>&1 || true
+  done < <(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
+fi
 
 echo
 echo "Namespace '$NS' reset complete (namespace preserved)."

tmp/prole-synology-path-bootstrap.yaml

@@ -0,0 +1,88 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: prole-synology-path-bootstrap
+  namespace: kube-system
+spec:
+  restartPolicy: Never
+  tolerations:
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: Exists
+      effect: NoSchedule
+    - key: "node-role.kubernetes.io/master"
+      operator: Exists
+      effect: NoSchedule
+  containers:
+    - name: init
+      image: busybox:1.36
+      command:
+        - sh
+        - -c
+        - |
+          mkdir -p /m/d001-data /m/d001-wal /m/d001-garage /m/d001-openbao \
+                   /m/d002-data /m/d002-wal /m/d002-prom \
+                   /m/d003-data /m/d003-wal /m/d003-grafana
+          echo "synology-path-bootstrap-complete"
+          sleep 5
+      volumeMounts:
+        - name: d001-data
+          mountPath: /m/d001-data
+        - name: d001-wal
+          mountPath: /m/d001-wal
+        - name: d001-garage
+          mountPath: /m/d001-garage
+        - name: d001-openbao
+          mountPath: /m/d001-openbao
+        - name: d002-data
+          mountPath: /m/d002-data
+        - name: d002-wal
+          mountPath: /m/d002-wal
+        - name: d002-prom
+          mountPath: /m/d002-prom
+        - name: d003-data
+          mountPath: /m/d003-data
+        - name: d003-wal
+          mountPath: /m/d003-wal
+        - name: d003-grafana
+          mountPath: /m/d003-grafana
+  volumes:
+    - name: d001-data
+      hostPath:
+        path: /synology/d001/data
+        type: DirectoryOrCreate
+    - name: d001-wal
+      hostPath:
+        path: /synology/d001/wal
+        type: DirectoryOrCreate
+    - name: d001-garage
+      hostPath:
+        path: /synology/d001/garage
+        type: DirectoryOrCreate
+    - name: d001-openbao
+      hostPath:
+        path: /synology/d001/openbao
+        type: DirectoryOrCreate
+    - name: d002-data
+      hostPath:
+        path: /synology/d002/data
+        type: DirectoryOrCreate
+    - name: d002-wal
+      hostPath:
+        path: /synology/d002/wal
+        type: DirectoryOrCreate
+    - name: d002-prom
+      hostPath:
+        path: /synology/d002/prometheus
+        type: DirectoryOrCreate
+    - name: d003-data
+      hostPath:
+        path: /synology/d003/data
+        type: DirectoryOrCreate
+    - name: d003-wal
+      hostPath:
+        path: /synology/d003/wal
+        type: DirectoryOrCreate
+    - name: d003-grafana
+      hostPath:
+        path: /synology/d003/grafana
+        type: DirectoryOrCreate