milestone(k3s+cnpg): end-to-end deploy run via standalone installer

We can now complete a full K3s deployment run and deliver a working CloudNativePG (CNPG) stack from the standalone installer.\n\nHighlights:\n- Installer scripts hardened for CNPG deploy + recovery templating + rollout control\n- OpenTofu/K3s manifests aligned with in-repo k8s templates for prole-db\n- Supporting init flows updated (OpenBao, monitoring, Kerberos, OpenTofu)

Co-authored-by: Junie <junie@jetbrains.com>

.gitignore

@@ -42,6 +42,7 @@
 
 # Secrets and local config
 *.kubeconfig
+*.kubeconfig*
 etc/secrets/
 *-password.txt
 *secret.yaml

conf/prole.cfg

@@ -1,5 +1,5 @@
 ; Prole Master Configuration File
-; Generated by install.py on 2026-03-02 18:39:26
+; Generated by install.py on 2026-03-03 16:30:57
 ; This file is used as input for Ansible deployment and k8s cluster creation.
 
 [User]
@@ -42,7 +42,7 @@ dependencies.opentofu.install = true
 dependencies.python.install = true
 dependencies.verify_all = false
 disk_selection.disk_type = local
-disk_selection.local_path = /Users/chrisfu
+disk_selection.local_path = ${PROLE_HOME}/prole-tools-app/dist
 disk_selection.removable_mount = 
 env_setup.NAMESPACE = ${NAMESPACE}
 env_setup.PROLE_CONF = ${PROLE_CONF}
@@ -51,13 +51,13 @@ env_setup.PROLE_HOME = ${PROLE_HOME}
 env_setup.PROLE_LOGS = ${PROLE_LOGS}
 env_setup.PROLE_SERVICE = ${PROLE_SERVICE}
 init_cluster.at_rest_encryption_enabled = true
-init_cluster.cluster_env = dev
+init_cluster.cluster_env = service
-init_cluster.deployment_target = knoe-dev-cluster
+init_cluster.deployment_target = prole-service-cluster
 init_cluster.gitops_enabled = false
-init_cluster.k3s_server_url = 
+init_cluster.k3s_server_url = https://myrddin.prole.org:6443
-init_cluster.k3s_token = 
+init_cluster.k3s_token = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
-init_cluster.kerberos_enabled = false
+init_cluster.kerberos_enabled = true
-init_cluster.mode = k3d
+init_cluster.mode = k3s
 init_cluster.start_cluster = true
 init_cluster.supabase_enabled = false
 init_cnpg_deploy.force_rollout = false
@@ -70,10 +70,11 @@ init_password.db_password_confirm = ${OPENBAO:kv/prole/knoey-db-0/db#password}
 init_password.db_username = root
 init_password.generate_ssh_key = true
 init_scripts.run_scripts = true
-kerberos_config.enabled = false
+kerberos_config.enabled = true
-kerberos_config.kdc = 
+kerberos_config.init_authority = false
+kerberos_config.kdc = 10.0.0.3
 kerberos_config.password = ${OPENBAO:kv/prole/knoey-db-0/kerberos#password}
-kerberos_config.realm = 
+kerberos_config.realm = PROLE.ORG
 kerberos_config.test_connection = false
 kerberos_config.user = administrator
 network_scan.run = true
@@ -82,26 +83,30 @@ ollama_config.server_host =
 ollama_config.server_port = 11434
 
 [Global]
+CNPG_VERSION = 1.28.1
+CNPG_RELEASE_SERIES = 1.28
 ; Variables used by name in more than one place or assumed global scope
-CLUSTER_ENV = dev
+ARGOCD_NAMESPACE = argocd
+CLUSTER_ENV = service
 DB_HOST_PORT = 5432
 DB_PASSWORD = ${OPENBAO:kv/prole/knoey-db-0/db#password}
-DEPLOYMENT_MODE = k3d
+DEPLOYMENT_MODE = k3s
-DEPLOYMENT_TARGET = knoe-dev-cluster
+DEPLOYMENT_TARGET = prole-service-cluster
 DOCKER_IMPORT_DIR = 
 NAMESPACE = ${NAMESPACE}
 PROLE_DB_USER = root
 PROLE_HOME = ${PROLE_HOME}
-PROLE_K3S_SERVER = 
+PROLE_K3S_SERVER = https://myrddin.prole.org:6443
-PROLE_K3S_TOKEN = 
+PROLE_K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
 PROLE_OPENTOFU_URL = http://127.0.0.1:8080
+REGISTRY_NAMESPACE = default
 SERVICE_NAMESPACE = knoe-system
 
 [Welcome]
 ; No configuration values captured yet for this section.
 
 [Dependencies]
-; No configuration values captured yet for this section.
+STATUS = All installed
 
 [Network]
 AD_DC_HOST = myrddin.prole.org
@@ -112,11 +117,21 @@ ANSIBLE_INVENTORY = ${PROLE_HOME}/infrastructure/inventory
 ANSIBLE_REALM = PROLE.ORG
 ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_hosts":["myrddin.prole.org","pi.prole.org","retropie.prole.org"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"]},"hosts":{"myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":[]}
 KDC_ANSIBLE_DETECTED = 10.0.0.3
+KDC_AUTO_DETECTED = 10.0.0.3
+KERBEROS_AUTO_ENABLED = True
 
 [Port Forwards]
-; No configuration values captured yet for this section.
+PORT_FORWARD_K3S_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD
+PORT_FORWARD_K3S_MAPPING_2 = id=garage;namespace=knoe-system;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3
+PORT_FORWARD_K3S_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao;address=0.0.0.0;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao
+PORT_FORWARD_K3S_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu
+PORT_FORWARD_K3S_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard
+PORT_FORWARD_K3S_MAPPING_6 = id=postgres;namespace=${NAMESPACE};target=svc/prole-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary)
+PORT_FORWARD_K3S_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI
+PORT_FORWARD_K3S_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI
 
 [System Environment]
+NAMESPACE = knoey-db-0
 PROLE_CONF = ${PROLE_CONF}
 PROLE_DATA = ${PROLE_DATA}
 PROLE_HOME = ${PROLE_HOME}
@@ -133,47 +148,54 @@ PASSWORD = ${OPENBAO:kv/prole/knoey-db-0/kerberos#password}
 ; No configuration values captured yet for this section.
 
 [Optional Features]
-GITOPS_ENABLED = False
+AT_REST_ENCRYPTION_ENABLED = true
+GITOPS_ENABLED = false
+KERBEROS_ENABLED = true
+SUPABASE_ENABLED = false
 
 [GitOps]
-; No configuration values captured yet for this section.
 
 [Database Creation]
+DB_NAME = knoey-db-0
+DB_USER = root
 NAMESPACE = ${NAMESPACE}
 
 [Initialize Cluster]
-; No configuration values captured yet for this section.
+ENVIRONMENT = service
+K3S_SERVER_URL = https://myrddin.prole.org:6443
+K3S_TOKEN = ${PROLE_SECRET:v1:7AxJwJb0PoEoe0a0:4P77NxrvZejAdfZlZlrQ7m8zC-ENEaInKxb43uzSGFl5nLpSjjuGsx_x1oPB38qijvDx59O-2jqp-nId3H6JJSdy7g1U01I6P4xiZ9xrUDMd5SazXzrPhQCE4ztPo-M1-czz7BpaelffQxkhk0gdUrEHoFzXLiGamX7S9Q==}
 
 [Dev Cluster (k3d)]
-CLUSTER_ENV = dev
+CLUSTER_ENV = k3d-knoe-dev-cluster
 DISPLAY_NAME = knoe-dev-cluster
-KUBECTL_CONTEXT = 
+KUBECTL_CONTEXT = service
 MODE = k3d
 
 [Service Cluster (k3s)]
 CLUSTER_ENV = prole-service-cluster
 DISPLAY_NAME = prole-service-cluster
-K3S_SERVER_URL = 
+K3S_SERVER_URL = https://myrddin.prole.org:6443
-K3S_TOKEN = 
+K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
 MODE = k3s
 PIPELINE_URL = http://127.0.0.1:8080
 
 [Prod Cluster (k8s)]
-ARTIFACTS_DIR = /private/var/folders/rt/pywlnmxj3dn7t5552vwdcpp80000gn/T/pytest-of-chrisfu/pytest-94/test_encryption_toggle0/data/staging
+ARTIFACTS_DIR = ${PROLE_DATA}/staging
 CLUSTER_ENV = prole-prod-cluster
 DISPLAY_NAME = prole-prod-cluster
 MODE = k8s
 PIPELINE_URL = http://127.0.0.1:8080
 
 [Docker Build]
-; No configuration values captured yet for this section.
+LOCAL_REGISTRY = myrddin.prole.org:5000
+LOCAL_REGISTRY_INTERNAL = myrddin.prole.org:5000
 
 [Initialization Scripts]
-; No configuration values captured yet for this section.
+STATUS = Attempted
 
 [Deployment]
-MODE = k3d
+MODE = k3s
-TARGET = knoe-dev-cluster
+TARGET = prole-service-cluster
 
 [Install]
-; No configuration values captured yet for this section.
+STATUS = Failed

deploy/opentofu/k3s/manifests/prole/prole-db.yaml

@@ -102,21 +102,3 @@ spec:
           prole.storage/role: wal
   monitoring:
     enablePodMonitor: false
-
-  # managed rw service
-  managed:
-    services:
-      additional:
-        - selectorType: rw
-          serviceTemplate:
-            metadata:
-              name: "prole-db-001"
-              labels:
-                test-label: "true"
-              annotations:
-                test-annotation: "true"
-                prometheus.io/scrape: "true"
-                prometheus.io/port: "9187"
-                prometheus.io/path: "/metrics"
-            spec:
-              type: LoadBalancer

etc/init_cloudnative_pg.sh

@@ -49,6 +49,11 @@ CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-}
 PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}
 BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup}
 BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800}
+if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+  CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-900}
+else
+  CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-300}
+fi
 RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl"
 BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-}
 BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0}
@@ -203,16 +208,28 @@ openbao_url() {
     echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200"
     return 0
   fi
-  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+
-    echo "http://127.0.0.1:8200"
+  # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
-    return 0
+  # (never via localhost or kubectl port-forward).
-  elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
-    echo "http://127.0.0.1:8200"
+    if command -v _prole_host_from_url >/dev/null 2>&1; then
-    return 0
+      local host
-  else
+      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
+      if [[ -n "${host:-}" ]]; then
+        echo "http://${host}:8200"
+        return 0
+      fi
+    fi
     echo ""
     return 0
   fi
+
+  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+    echo "http://127.0.0.1:8200"
+    return 0
+  fi
+  echo ""
+  return 0
 }
 
 openbao_token() {
@@ -253,6 +270,11 @@ ensure_cnpg_operator() {
   if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
     kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
     wait_for_cnpg_webhook 180 || true
+    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+      tune_cnpg_operator_for_k3s || true
+      kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true
+      wait_for_cnpg_webhook 300 || true
+    fi
     return 0
   fi
 
@@ -267,9 +289,42 @@ ensure_cnpg_operator() {
   if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
     kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
     wait_for_cnpg_webhook 180 || true
+    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+      tune_cnpg_operator_for_k3s || true
+      kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true
+      wait_for_cnpg_webhook 300 || true
+    fi
   fi
 }
 
+tune_cnpg_operator_for_k3s() {
+  # k3s on a single node can experience short API/server or scheduling stalls under load.
+  # CNPG's default probes are very aggressive (timeoutSeconds=1), which can cause flapping
+  # readiness and webhook endpoints disappearing mid-apply.
+  echo "Tuning CNPG operator deployment probes/resources for k3s ..."
+  kubectl -n cnpg-system patch deploy cnpg-controller-manager --type merge -p '
+  {
+    "spec": {
+      "template": {
+        "spec": {
+          "containers": [
+            {
+              "name": "manager",
+              "resources": {
+                "requests": {"cpu": "250m", "memory": "512Mi"},
+                "limits": {"cpu": "500m", "memory": "1Gi"}
+              },
+              "livenessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
+              "readinessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
+              "startupProbe": {"timeoutSeconds": 5, "failureThreshold": 60}
+            }
+          ]
+        }
+      }
+    }
+  }' >/dev/null 2>&1 || return 1
+}
+
 get_latest_barman_plugin_version() {
   local version tag
   tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "")
@@ -579,6 +634,31 @@ apply_prole_manifest_file() {
   return 1
 }
 
+_push_to_k3s_registry() {
+  local image="$1"
+  local push_host="${LOCAL_REGISTRY:-${LOCAL_REGISTRY_INTERNAL:-myrddin.prole.org:5000}}"
+  local plain_image="${image##*/}"
+
+  if command -v skopeo >/dev/null 2>&1; then
+    echo "  Pushing to k3s registry at '${push_host}' using skopeo ..."
+    if skopeo copy --dest-tls-verify=false docker-daemon:"$image" docker://"${push_host}/${plain_image}"; then
+      echo "  ✓ Image pushed to registry at '${push_host}'."
+      return 0
+    fi
+  fi
+
+  # Fallback to docker push if skopeo is missing or fails (might fail if daemon not configured)
+  local push_ref="${push_host}/${plain_image}"
+  docker tag "$image" "$push_ref" 2>/dev/null || true
+  if docker push "$push_ref" 2>/dev/null; then
+    echo "  ✓ Image pushed to registry at '${push_host}' via docker push."
+    return 0
+  fi
+
+  echo "ERROR: Failed to push image '$image' to k3s registry at '${push_host}'." >&2
+  return 1
+}
+
 # Push image to local registry (via LOCAL_REGISTRY host address) with k3d import fallback.
 _push_to_k3d_registry() {
   local image="$1"
@@ -609,7 +689,9 @@ _push_to_k3d_registry() {
 # Steps: containerd cache → Docker daemon (registry tag) → Docker daemon (plain tag)
 #        → tar import → docker build + push/import.
 _ensure_prole_db_image() {
-  [[ "${PROLE_MODE:-}" != "k3d" ]] && return 0
+  if [[ "${PROLE_MODE:-}" != "k3d" && "${PROLE_MODE:-}" != "k3s" ]]; then
+    return 0
+  fi
 
   local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
   local image=""
@@ -656,7 +738,11 @@ _ensure_prole_db_image() {
   # Step 2a: registry-tagged image in local Docker daemon → push + import
   if docker image inspect "$image" >/dev/null 2>&1; then
     echo "  Image '$image' found in Docker daemon; pushing to registry ..."
-    _push_to_k3d_registry "$image" "$cluster_name"
+    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+      _push_to_k3s_registry "$image"
+    else
+      _push_to_k3d_registry "$image" "$cluster_name"
+    fi
     return $?
   fi
 
@@ -664,7 +750,11 @@ _ensure_prole_db_image() {
   if docker image inspect "$plain_image" >/dev/null 2>&1; then
     echo "  Plain image '$plain_image' found in Docker daemon; tagging as '$image' and pushing ..."
     docker tag "$plain_image" "$image"
-    _push_to_k3d_registry "$image" "$cluster_name"
+    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+      _push_to_k3s_registry "$image"
+    else
+      _push_to_k3d_registry "$image" "$cluster_name"
+    fi
     return $?
   fi
 
@@ -689,7 +779,11 @@ _ensure_prole_db_image() {
     echo "  Loading tar '$(basename "$found_tar")' into Docker daemon ..."
     docker load -i "$found_tar"
     docker tag "$plain_image" "$image" 2>/dev/null || true
-    _push_to_k3d_registry "$image" "$cluster_name"
+    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+      _push_to_k3s_registry "$image"
+    else
+      _push_to_k3d_registry "$image" "$cluster_name"
+    fi
     return $?
   fi
 
@@ -706,12 +800,28 @@ _ensure_prole_db_image() {
   fi
   docker tag "$plain_image" "$image"
   echo "  Build complete. Pushing '$image' to registry ..."
-  _push_to_k3d_registry "$image" "$cluster_name"
+  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+    _push_to_k3s_registry "$image"
+  else
+    _push_to_k3d_registry "$image" "$cluster_name"
+  fi
   return $?
 }
 
+cleanup_unintended_cnpg_services() {
+  # Historical manifest bug: we used to create a CNPG-managed Service named `prole-db-001`
+  # as `type: LoadBalancer`. In k3s this spawns `svclb-prole-db-001` pods, and it can
+  # interfere with CNPG startup/reconciliation. Ensure it is removed if it exists.
+  local svc_name="prole-db-001"
+  if kubectl -n "$NAMESPACE" get svc "$svc_name" >/dev/null 2>&1; then
+    echo "Removing unintended Service '$svc_name' from namespace '$NAMESPACE' ..."
+    kubectl -n "$NAMESPACE" delete svc "$svc_name" --ignore-not-found >/dev/null 2>&1 || true
+  fi
+}
+
 ensure_prole_stack_resources() {
   echo "Applying CloudNative-PG cluster and related resources ..."
+  cleanup_unintended_cnpg_services || true
   local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
   local image=""
   if [[ -n "$image_override" ]]; then
@@ -876,9 +986,9 @@ wait_for_cnpg_pods() {
         return 1
       fi
 
-      # Check for image pull failures — attempt remediation in k3d mode
+      # Check for image pull failures — attempt remediation in k3d/k3s mode
       if echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >/dev/null; then
-        if [[ "${PROLE_MODE:-}" == "k3d" ]]; then
+        if [[ "${PROLE_MODE:-}" == "k3d" || "${PROLE_MODE:-}" == "k3s" ]]; then
           if [[ -z "$_image_pull_remediated" ]]; then
             _image_pull_remediated=1
             local fail_pod fail_image
@@ -1278,6 +1388,8 @@ reset_and_reinit() {
   kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
   kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true
 
+  recycle_released_prole_iscsi_pvs || true
+
   if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then
     tmp_manifest=$(mktemp)
     sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest"
@@ -1294,10 +1406,35 @@ reset_and_reinit() {
     ensure_prole_stack_resources
   fi
 
-  wait_for_cnpg_pods 300
+  wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"
 }
 
-# Resolve OpenBao URL: prefer explicit env, then in-cluster service, then localhost port-forward
+recycle_released_prole_iscsi_pvs() {
+  # StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. After a CNPG reset, PVs can remain
+  # in `Released` with a stale `claimRef`, which prevents new PVCs (same names) from binding.
+  # In that case, CNPG init jobs remain `Pending` with "didn't find available persistent volumes".
+  echo "Recycling Released prole-iscsi PVs (clearing stale claimRefs) for namespace '$NAMESPACE' ..."
+  local pvs
+  pvs=$(kubectl get pv -o json \
+    | jq -r --arg ns "$NAMESPACE" '.items[]
+      | select(.spec.storageClassName == "prole-iscsi")
+      | select(.status.phase == "Released")
+      | select((.spec.claimRef.namespace // "") == $ns)
+      | .metadata.name' 2>/dev/null || true)
+
+  if [[ -z "${pvs:-}" ]]; then
+    return 0
+  fi
+
+  local pv
+  for pv in $pvs; do
+    echo "  - Clearing claimRef on PV: $pv"
+    kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
+  done
+}
+
+# Resolve OpenBao URL: prefer explicit env, then in-cluster service.
+# In k3s mode, out-of-cluster runs must use the k3s host (no localhost/port-forward).
 bao_service_url() {
   if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
     echo "$PROLE_OPENBAO_URL"
@@ -1305,13 +1442,23 @@ bao_service_url() {
   fi
   if prole_is_in_cluster; then
     local ns="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}"
-    # Troubleshoot: if the primary namespace is not found, check knoey-system as a common variant
-    if [[ "$ns" == "knoe-system" ]] && ! kubectl get ns knoe-system >/dev/null 2>&1 && kubectl get ns knoey-system >/dev/null 2>&1; then
-      ns="knoey-system"
-    fi
     echo "http://$OPENBAO_NAME.$ns.svc.cluster.local:8200"
     return 0
   fi
+
+  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
+    if command -v _prole_host_from_url >/dev/null 2>&1; then
+      local host
+      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
+      if [[ -n "${host:-}" ]]; then
+        echo "http://${host}:8200"
+        return 0
+      fi
+    fi
+    echo ""
+    return 0
+  fi
+
   if curl -sS --connect-timeout 2 "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
     echo "http://127.0.0.1:8200"
     return 0
@@ -1600,11 +1747,11 @@ initialize() {
 
   ensure_prole_stack_resources
 
-  if ! wait_for_cnpg_pods 300; then
+  if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then
     if cluster_has_pods; then
       echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2
       if force_rollout; then
-        if ! wait_for_cnpg_pods 300; then
+        if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then
           echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2
           if ! reset_and_reinit; then
             return 1

etc/init_kerberos.sh

@@ -234,22 +234,38 @@ openbao_url() {
     echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200"
     return 0
   fi
-  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+
-    echo "http://127.0.0.1:8200"
+  # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
-    return 0
+  # (never via localhost or kubectl port-forward).
-  elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
-    echo "http://127.0.0.1:8200"
+    if command -v _prole_host_from_url >/dev/null 2>&1; then
-    return 0
+      local host
-  else
+      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
+      if [[ -n "${host:-}" ]]; then
+        echo "http://${host}:8200"
+        return 0
+      fi
+    fi
     echo ""
     return 0
   fi
+
+  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+    echo "http://127.0.0.1:8200"
+    return 0
+  fi
+  echo ""
+  return 0
 }
 
 _OPENBAO_PF_PID=""
 _OPENBAO_PF_STARTED=0
 
 start_openbao_port_forward() {
+  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
+    return 1
+  fi
+
   local port="${OPENBAO_PORT_FORWARD_LOCAL}"
   local url="http://127.0.0.1:${port}"
 
@@ -289,6 +305,13 @@ ensure_openbao_url() {
     printf '%s' "$url"
     return 0
   fi
+
+  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
+    # In k3s, do not attempt to synthesize connectivity via local port-forwards.
+    printf ''
+    return 0
+  fi
+
   if start_openbao_port_forward; then
     trap stop_openbao_port_forward EXIT
     printf 'http://127.0.0.1:%s' "${OPENBAO_PORT_FORWARD_LOCAL}"

etc/init_monitoring.sh

@@ -287,16 +287,28 @@ openbao_url() {
     echo "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE:-default}}.svc.cluster.local:8200"
     return 0
   fi
-  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+
-    echo "http://127.0.0.1:8200"
+  # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
-    return 0
+  # (never via localhost or kubectl port-forward).
-  elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
-    echo "http://127.0.0.1:8200"
+    if command -v _prole_host_from_url >/dev/null 2>&1; then
-    return 0
+      local host
-  else
+      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
+      if [[ -n "${host:-}" ]]; then
+        echo "http://${host}:8200"
+        return 0
+      fi
+    fi
     echo ""
     return 0
   fi
+
+  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+    echo "http://127.0.0.1:8200"
+    return 0
+  fi
+  echo ""
+  return 0
 }
 
 openbao_token() {

etc/init_openbao.sh

@@ -99,7 +99,10 @@ admin_key_pub="$SECRETS_DIR/admin_ed25519.pub"
 root_token_file="$SECRETS_DIR/openbao-root-token"
 
 OPENBAO_NAMESPACE=${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}}
-OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${NAMESPACE:-default}}
+# OpenBao resources (StatefulSet/Service) live in `OPENBAO_NAMESPACE` (usually the service namespace),
+# but secret *paths* must be keyed by the Prole workload namespace from `conf/prole.cfg` so that
+# `${OPENBAO:kv/prole/<PROLE_NAMESPACE>/...}` references resolve consistently across scripts.
+OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${PROLE_NAMESPACE:-${NAMESPACE:-default}}}
 OPENBAO_RESOURCE_NAMESPACE="$OPENBAO_NAMESPACE"
 
 BAO_NAMESPACE="${OPENBAO_PATH_NAMESPACE:-default}"
@@ -276,6 +279,11 @@ EOF
 
 # Use PRIMARY_SERVICE_NAMESPACE if skipping deployment (we port-forward to the common svc)
 start_openbao_port_forward() {
+  # In k3s mode we should never rely on local kubectl port-forwards.
+  if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+    return 1
+  fi
+
   local pf_ns="$OPENBAO_RESOURCE_NAMESPACE"
   if should_skip_deployment; then
     pf_ns="$PRIMARY_SERVICE_NAMESPACE"
@@ -437,7 +445,15 @@ apply_k8s() {
         return 1
       fi
     fi
+    # Always apply the Service manifest, then enforce the desired exposure mode.
     prole_render_manifest "$SCRIPT_DIR/../k8s/prole/openbao-service.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f -
+
+    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
+      # Expose OpenBao externally in k3s (no kubectl port-forward). k3s `servicelb` will bind this
+      # on the node, making it reachable from the installer host.
+      kubectl patch svc "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --type merge \
+        -p '{"spec":{"type":"LoadBalancer"}}' >/dev/null 2>&1 || true
+    fi
   else
     prole_render_manifest "$OPENBAO_MANIFEST_DIR/deployment.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f -
   fi
@@ -805,7 +821,9 @@ case "$ACTION" in
     if should_skip_deployment; then
       pf_ns="$PRIMARY_SERVICE_NAMESPACE"
     fi
-    prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
+    if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
+      prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
+    fi
     ;;
   status)
     ensure_tools
@@ -854,22 +872,31 @@ case "$ACTION" in
     fi
 
     ensure_admin_keypair
-    trap stop_openbao_port_forward EXIT
+    if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
-    if ! start_openbao_port_forward; then
+      # In k3s, initialize against the externally reachable endpoint (no port-forward).
-      echo "ERROR: Unable to establish OpenBao port-forward on 127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" >&2
+      svc_url="$(bao_service_url)"
-      exit 1
+      wait_for_openbao_http 40 "$svc_url"
-    fi
+      init_openbao_kv_and_store_admin_key "$svc_url"
+    else
+      trap stop_openbao_port_forward EXIT
+      if ! start_openbao_port_forward; then
+        echo "ERROR: Unable to establish OpenBao port-forward on 127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" >&2
+        exit 1
+      fi
 
-    # After port-forward, always use the local URL for initialization so we
+      # After port-forward, always use the local URL for initialization so we
-    # don't depend on OpenBao being externally reachable from this host.
+      # don't depend on OpenBao being externally reachable from this host.
-    local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}"
+      local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}"
-    wait_for_openbao_http 20 "$local_url"
+      wait_for_openbao_http 20 "$local_url"
-    init_openbao_kv_and_store_admin_key "$local_url"
+      init_openbao_kv_and_store_admin_key "$local_url"
+    fi
     pf_ns="${OPENBAO_NAMESPACE:-default}"
     if should_skip_deployment; then
       pf_ns="$PRIMARY_SERVICE_NAMESPACE"
     fi
-    prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
+    if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
+      prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
+    fi
     echo "Initialization complete. OpenBao available at $(bao_service_url)"
     ;;
   update|reload)
@@ -891,7 +918,9 @@ case "$ACTION" in
     if should_skip_deployment; then
       pf_ns="$PRIMARY_SERVICE_NAMESPACE"
     fi
-    prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
+    if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
+      prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
+    fi
     echo "Re-applied manifests."
     ;;
   *)

etc/init_opentofu.sh

@@ -78,16 +78,28 @@ openbao_url() {
     echo "http://openbao.${OPENTOFU_OPENBAO_NAMESPACE}.svc.cluster.local:8200"
     return 0
   fi
-  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+
-    echo "http://127.0.0.1:8200"
+  # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
-    return 0
+  # (never via localhost or kubectl port-forward).
-  elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
-    echo "http://127.0.0.1:8200"
+    if command -v _prole_host_from_url >/dev/null 2>&1; then
-    return 0
+      local host
-  else
+      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
+      if [[ -n "${host:-}" ]]; then
+        echo "http://${host}:8200"
+        return 0
+      fi
+    fi
     echo ""
     return 0
   fi
+
+  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+    echo "http://127.0.0.1:8200"
+    return 0
+  fi
+  echo ""
+  return 0
 }
 
 openbao_token() {

etc/prole_cfg.sh

@@ -155,7 +155,7 @@ _prole_cfg_extract_key() {
   value=$(awk -F= -v k="$key" '
     /^[[:space:]]*;/ {next}
     /^[[:space:]]*#/ {next}
-    /^[[:space:]]*\\[/ {next}
+    /^[[:space:]]*\[/ {next}
     $1 ~ "^[[:space:]]*" k "[[:space:]]*$" {
       v=$2
       sub(/^[[:space:]]+/, "", v)
@@ -433,6 +433,7 @@ import json
 import os
 import sys
 import urllib.request
+import urllib.parse
 
 OPENBAO_PREFIX = "${OPENBAO:"
 OPENBAO_SUFFIX = "}"
@@ -465,6 +466,21 @@ def resolve_openbao_ref(value: str) -> str:
         return ""
 
     url = os.environ.get("PROLE_OPENBAO_URL")
+    if not url:
+        # In k3s mode, do NOT fall back to localhost/port-forward. Derive the external
+        # OpenBao URL from the k3s API server URL when available.
+        mode = (os.environ.get("PROLE_MODE") or os.environ.get("DEPLOYMENT_MODE") or "").strip().lower()
+        if mode == "k3s":
+            k3s_server = (os.environ.get("PROLE_K3S_SERVER") or os.environ.get("K3S_SERVER_URL") or "").strip()
+            if k3s_server:
+                try:
+                    parsed = urllib.parse.urlparse(k3s_server)
+                    host = parsed.hostname or ""
+                    if host:
+                        url = f"http://{host}:8200"
+                except Exception:
+                    url = ""
+
     if not url:
         for p in ["8200", "8200"]:
             try:
@@ -475,7 +491,11 @@ def resolve_openbao_ref(value: str) -> str:
             except Exception:
                 pass
     if not url:
-        url = "http://127.0.0.1:8200"
+        mode = (os.environ.get("PROLE_MODE") or os.environ.get("DEPLOYMENT_MODE") or "").strip().lower()
+        if mode != "k3s":
+            url = "http://127.0.0.1:8200"
+        else:
+            return ""
     url = url.rstrip("/")
     try:
         req = urllib.request.Request(f"{url}/v1/{mount}/data/{secret_path}")

k8s/prole/prole-db-recovery.yaml.tpl

@@ -5,7 +5,7 @@ metadata:
   name: prole-db
 spec:
   instances: 3
-  imageName: k3d-prole-registry.localhost:5000/prole-db:18-095
+  imageName: myrddin.prole.org:5000/prole-db:18-100
   postgresUID: 100
   postgresGID: 101
   maxSyncReplicas: 1
@@ -66,21 +66,3 @@ spec:
 
   monitoring:
     enablePodMonitor: false
-
-  # managed rw service
-  managed:
-    services:
-      additional:
-        - selectorType: rw
-          serviceTemplate:
-            metadata:
-              name: "prole-db-001"
-              labels:
-                test-label: "true"
-              annotations:
-                test-annotation: "true"
-                prometheus.io/scrape: "true"
-                prometheus.io/port: "9187"
-                prometheus.io/path: "/metrics"
-            spec:
-              type: LoadBalancer

k8s/prole/prole-db.yaml

@@ -4,7 +4,7 @@ metadata:
   name: prole-db
 spec:
   instances: 3
-  imageName: k3d-prole-registry.localhost:5000/prole-db:18-095
+  imageName: myrddin.prole.org:5000/prole-db:18-100
   postgresUID: 100
   postgresGID: 101
   maxSyncReplicas: 1
@@ -102,21 +102,3 @@ spec:
           prole.storage/role: wal
   monitoring:
     enablePodMonitor: false
-
-  # managed rw service
-  managed:
-    services:
-      additional:
-        - selectorType: rw
-          serviceTemplate:
-            metadata:
-              name: "prole-db-001"
-              labels:
-                test-label: "true"
-              annotations:
-                test-annotation: "true"
-                prometheus.io/scrape: "true"
-                prometheus.io/port: "9187"
-                prometheus.io/path: "/metrics"
-            spec:
-              type: LoadBalancer

prole-db/.version

@@ -1 +1 @@
-98
+100

prole.sh

@@ -6,15 +6,6 @@ set -euo pipefail
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "${ROOT_DIR}"
 
-# OpenBao Detection
-if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then
-  if curl -s -f "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
-    export PROLE_OPENBAO_URL="http://127.0.0.1:8200"
-  elif curl -s -f "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
-    export PROLE_OPENBAO_URL="http://127.0.0.1:18200"
-  fi
-fi
-
 if [[ -z "${OPENBAO_ROOT_TOKEN:-}" ]]; then
   if [[ -f "${ROOT_DIR}/etc/secrets/openbao-root-token" ]]; then
     export OPENBAO_ROOT_TOKEN=$(cat "${ROOT_DIR}/etc/secrets/openbao-root-token")
@@ -174,6 +165,34 @@ resolve_prole_mode() {
   fi
 }
 
+ensure_openbao_url_for_mode() {
+  local mode="${1:-}"
+  if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
+    return 0
+  fi
+
+  case "$mode" in
+    k3d)
+      if curl -s -f "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
+        export PROLE_OPENBAO_URL="http://127.0.0.1:8200"
+      elif curl -s -f "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
+        export PROLE_OPENBAO_URL="http://127.0.0.1:18200"
+      fi
+      ;;
+    k3s|k8s)
+      # In k3s/k8s, the installer runs outside the cluster and must reach OpenBao via a real
+      # externally reachable endpoint (no kubectl port-forward / localhost defaults).
+      if command -v _prole_host_from_url >/dev/null 2>&1; then
+        local host
+        host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
+        if [[ -n "${host:-}" ]]; then
+          export PROLE_OPENBAO_URL="http://${host}:8200"
+        fi
+      fi
+      ;;
+  esac
+}
+
 resolve_k3d_cluster_name() {
   local name="${DEPLOYMENT_TARGET:-${DISPLAY_NAME:-}}"
   if [[ -z "$name" && -n "${KUBECTL_CONTEXT:-}" ]]; then
@@ -574,6 +593,23 @@ case "${CMD}" in
     fi
     CMD_ARGS=("${_INSTALL_PASSTHRU[@]}")
 
+    # Ensure OpenBao URL is set appropriately for the active mode before running install.py.
+    _cfg_for_env="${PROLE_CFG_DEFAULT}"
+    for ((i=0; i<${#CMD_ARGS[@]}; i++)); do
+      if [[ "${CMD_ARGS[$i]}" == "-c" || "${CMD_ARGS[$i]}" == "--config" ]]; then
+        if [[ $((i+1)) -lt ${#CMD_ARGS[@]} ]]; then
+          _cfg_for_env="${CMD_ARGS[$((i+1))]}"
+        fi
+      fi
+    done
+    load_prole_cfg "${_cfg_for_env}"
+    _mode_for_env=$(resolve_prole_mode)
+    if [[ -z "${_mode_for_env:-}" ]]; then
+      _mode_for_env="k3d"
+    fi
+    ensure_openbao_url_for_mode "${_mode_for_env}"
+    unset _cfg_for_env _mode_for_env
+
     PY_CMD=("python3")
     if [[ "${COVERAGE}" -eq 1 ]]; then
       PY_CMD=("coverage" "run" "--source=installer")