diff --git a/.gitignore b/.gitignore index 33d0bfb..ca3180e 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ # Secrets and local config *.kubeconfig +*.kubeconfig* etc/secrets/ *-password.txt *secret.yaml diff --git a/conf/prole.cfg b/conf/prole.cfg index 80a1178..c80f97f 100644 --- a/conf/prole.cfg +++ b/conf/prole.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-03-02 18:39:26 +; Generated by install.py on 2026-03-03 16:30:57 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -42,7 +42,7 @@ dependencies.opentofu.install = true dependencies.python.install = true dependencies.verify_all = false disk_selection.disk_type = local -disk_selection.local_path = /Users/chrisfu +disk_selection.local_path = ${PROLE_HOME}/prole-tools-app/dist disk_selection.removable_mount = env_setup.NAMESPACE = ${NAMESPACE} env_setup.PROLE_CONF = ${PROLE_CONF} @@ -51,13 +51,13 @@ env_setup.PROLE_HOME = ${PROLE_HOME} env_setup.PROLE_LOGS = ${PROLE_LOGS} env_setup.PROLE_SERVICE = ${PROLE_SERVICE} init_cluster.at_rest_encryption_enabled = true -init_cluster.cluster_env = dev -init_cluster.deployment_target = knoe-dev-cluster +init_cluster.cluster_env = service +init_cluster.deployment_target = prole-service-cluster init_cluster.gitops_enabled = false -init_cluster.k3s_server_url = -init_cluster.k3s_token = -init_cluster.kerberos_enabled = false -init_cluster.mode = k3d +init_cluster.k3s_server_url = https://myrddin.prole.org:6443 +init_cluster.k3s_token = ${OPENBAO:kv/prole/knoey-db-0/k3s#token} +init_cluster.kerberos_enabled = true +init_cluster.mode = k3s init_cluster.start_cluster = true init_cluster.supabase_enabled = false init_cnpg_deploy.force_rollout = false @@ -70,10 +70,11 @@ init_password.db_password_confirm = ${OPENBAO:kv/prole/knoey-db-0/db#password} init_password.db_username = root init_password.generate_ssh_key = true init_scripts.run_scripts = true -kerberos_config.enabled = false -kerberos_config.kdc = +kerberos_config.enabled = true +kerberos_config.init_authority = false +kerberos_config.kdc = 10.0.0.3 kerberos_config.password = ${OPENBAO:kv/prole/knoey-db-0/kerberos#password} -kerberos_config.realm = +kerberos_config.realm = PROLE.ORG kerberos_config.test_connection = false kerberos_config.user = administrator network_scan.run = true @@ -82,26 +83,30 @@ ollama_config.server_host = ollama_config.server_port = 11434 [Global] +CNPG_VERSION = 1.28.1 +CNPG_RELEASE_SERIES = 1.28 ; Variables used by name in more than one place or assumed global scope -CLUSTER_ENV = dev +ARGOCD_NAMESPACE = argocd +CLUSTER_ENV = service DB_HOST_PORT = 5432 DB_PASSWORD = ${OPENBAO:kv/prole/knoey-db-0/db#password} -DEPLOYMENT_MODE = k3d -DEPLOYMENT_TARGET = knoe-dev-cluster +DEPLOYMENT_MODE = k3s +DEPLOYMENT_TARGET = prole-service-cluster DOCKER_IMPORT_DIR = NAMESPACE = ${NAMESPACE} PROLE_DB_USER = root PROLE_HOME = ${PROLE_HOME} -PROLE_K3S_SERVER = -PROLE_K3S_TOKEN = +PROLE_K3S_SERVER = https://myrddin.prole.org:6443 +PROLE_K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token} PROLE_OPENTOFU_URL = http://127.0.0.1:8080 +REGISTRY_NAMESPACE = default SERVICE_NAMESPACE = knoe-system [Welcome] ; No configuration values captured yet for this section. [Dependencies] -; No configuration values captured yet for this section. +STATUS = All installed [Network] AD_DC_HOST = myrddin.prole.org @@ -112,11 +117,21 @@ ANSIBLE_INVENTORY = ${PROLE_HOME}/infrastructure/inventory ANSIBLE_REALM = PROLE.ORG ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_hosts":["myrddin.prole.org","pi.prole.org","retropie.prole.org"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"]},"hosts":{"myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":[]} KDC_ANSIBLE_DETECTED = 10.0.0.3 +KDC_AUTO_DETECTED = 10.0.0.3 +KERBEROS_AUTO_ENABLED = True [Port Forwards] -; No configuration values captured yet for this section. +PORT_FORWARD_K3S_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD +PORT_FORWARD_K3S_MAPPING_2 = id=garage;namespace=knoe-system;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3 +PORT_FORWARD_K3S_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao;address=0.0.0.0;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao +PORT_FORWARD_K3S_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu +PORT_FORWARD_K3S_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard +PORT_FORWARD_K3S_MAPPING_6 = id=postgres;namespace=${NAMESPACE};target=svc/prole-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary) +PORT_FORWARD_K3S_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI +PORT_FORWARD_K3S_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI [System Environment] +NAMESPACE = knoey-db-0 PROLE_CONF = ${PROLE_CONF} PROLE_DATA = ${PROLE_DATA} PROLE_HOME = ${PROLE_HOME} @@ -133,47 +148,54 @@ PASSWORD = ${OPENBAO:kv/prole/knoey-db-0/kerberos#password} ; No configuration values captured yet for this section. [Optional Features] -GITOPS_ENABLED = False +AT_REST_ENCRYPTION_ENABLED = true +GITOPS_ENABLED = false +KERBEROS_ENABLED = true +SUPABASE_ENABLED = false [GitOps] -; No configuration values captured yet for this section. [Database Creation] +DB_NAME = knoey-db-0 +DB_USER = root NAMESPACE = ${NAMESPACE} [Initialize Cluster] -; No configuration values captured yet for this section. +ENVIRONMENT = service +K3S_SERVER_URL = https://myrddin.prole.org:6443 +K3S_TOKEN = ${PROLE_SECRET:v1:7AxJwJb0PoEoe0a0:4P77NxrvZejAdfZlZlrQ7m8zC-ENEaInKxb43uzSGFl5nLpSjjuGsx_x1oPB38qijvDx59O-2jqp-nId3H6JJSdy7g1U01I6P4xiZ9xrUDMd5SazXzrPhQCE4ztPo-M1-czz7BpaelffQxkhk0gdUrEHoFzXLiGamX7S9Q==} [Dev Cluster (k3d)] -CLUSTER_ENV = dev +CLUSTER_ENV = k3d-knoe-dev-cluster DISPLAY_NAME = knoe-dev-cluster -KUBECTL_CONTEXT = +KUBECTL_CONTEXT = service MODE = k3d [Service Cluster (k3s)] CLUSTER_ENV = prole-service-cluster DISPLAY_NAME = prole-service-cluster -K3S_SERVER_URL = -K3S_TOKEN = +K3S_SERVER_URL = https://myrddin.prole.org:6443 +K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token} MODE = k3s PIPELINE_URL = http://127.0.0.1:8080 [Prod Cluster (k8s)] -ARTIFACTS_DIR = /private/var/folders/rt/pywlnmxj3dn7t5552vwdcpp80000gn/T/pytest-of-chrisfu/pytest-94/test_encryption_toggle0/data/staging +ARTIFACTS_DIR = ${PROLE_DATA}/staging CLUSTER_ENV = prole-prod-cluster DISPLAY_NAME = prole-prod-cluster MODE = k8s PIPELINE_URL = http://127.0.0.1:8080 [Docker Build] -; No configuration values captured yet for this section. +LOCAL_REGISTRY = myrddin.prole.org:5000 +LOCAL_REGISTRY_INTERNAL = myrddin.prole.org:5000 [Initialization Scripts] -; No configuration values captured yet for this section. +STATUS = Attempted [Deployment] -MODE = k3d -TARGET = knoe-dev-cluster +MODE = k3s +TARGET = prole-service-cluster [Install] -; No configuration values captured yet for this section. +STATUS = Failed diff --git a/deploy/opentofu/k3s/manifests/prole/prole-db.yaml b/deploy/opentofu/k3s/manifests/prole/prole-db.yaml index 5b1171e..3049c11 100644 --- a/deploy/opentofu/k3s/manifests/prole/prole-db.yaml +++ b/deploy/opentofu/k3s/manifests/prole/prole-db.yaml @@ -102,21 +102,3 @@ spec: prole.storage/role: wal monitoring: enablePodMonitor: false - - # managed rw service - managed: - services: - additional: - - selectorType: rw - serviceTemplate: - metadata: - name: "prole-db-001" - labels: - test-label: "true" - annotations: - test-annotation: "true" - prometheus.io/scrape: "true" - prometheus.io/port: "9187" - prometheus.io/path: "/metrics" - spec: - type: LoadBalancer diff --git a/etc/init_cloudnative_pg.sh b/etc/init_cloudnative_pg.sh index 604a1d5..cdbab0e 100755 --- a/etc/init_cloudnative_pg.sh +++ b/etc/init_cloudnative_pg.sh @@ -49,6 +49,11 @@ CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-} PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup} BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800} +if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-900} +else + CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-300} +fi RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl" BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-} BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0} @@ -203,16 +208,28 @@ openbao_url() { echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200" return 0 fi - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - else + + # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host + # (never via localhost or kubectl port-forward). + if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then + if command -v _prole_host_from_url >/dev/null 2>&1; then + local host + host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") + if [[ -n "${host:-}" ]]; then + echo "http://${host}:8200" + return 0 + fi + fi echo "" return 0 fi + + if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then + echo "http://127.0.0.1:8200" + return 0 + fi + echo "" + return 0 } openbao_token() { @@ -253,6 +270,11 @@ ensure_cnpg_operator() { if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true wait_for_cnpg_webhook 180 || true + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + tune_cnpg_operator_for_k3s || true + kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true + wait_for_cnpg_webhook 300 || true + fi return 0 fi @@ -267,9 +289,42 @@ ensure_cnpg_operator() { if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true wait_for_cnpg_webhook 180 || true + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + tune_cnpg_operator_for_k3s || true + kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true + wait_for_cnpg_webhook 300 || true + fi fi } +tune_cnpg_operator_for_k3s() { + # k3s on a single node can experience short API/server or scheduling stalls under load. + # CNPG's default probes are very aggressive (timeoutSeconds=1), which can cause flapping + # readiness and webhook endpoints disappearing mid-apply. + echo "Tuning CNPG operator deployment probes/resources for k3s ..." + kubectl -n cnpg-system patch deploy cnpg-controller-manager --type merge -p ' + { + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "manager", + "resources": { + "requests": {"cpu": "250m", "memory": "512Mi"}, + "limits": {"cpu": "500m", "memory": "1Gi"} + }, + "livenessProbe": {"timeoutSeconds": 5, "failureThreshold": 6}, + "readinessProbe": {"timeoutSeconds": 5, "failureThreshold": 6}, + "startupProbe": {"timeoutSeconds": 5, "failureThreshold": 60} + } + ] + } + } + } + }' >/dev/null 2>&1 || return 1 +} + get_latest_barman_plugin_version() { local version tag tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "") @@ -579,6 +634,31 @@ apply_prole_manifest_file() { return 1 } +_push_to_k3s_registry() { + local image="$1" + local push_host="${LOCAL_REGISTRY:-${LOCAL_REGISTRY_INTERNAL:-myrddin.prole.org:5000}}" + local plain_image="${image##*/}" + + if command -v skopeo >/dev/null 2>&1; then + echo " Pushing to k3s registry at '${push_host}' using skopeo ..." + if skopeo copy --dest-tls-verify=false docker-daemon:"$image" docker://"${push_host}/${plain_image}"; then + echo " ✓ Image pushed to registry at '${push_host}'." + return 0 + fi + fi + + # Fallback to docker push if skopeo is missing or fails (might fail if daemon not configured) + local push_ref="${push_host}/${plain_image}" + docker tag "$image" "$push_ref" 2>/dev/null || true + if docker push "$push_ref" 2>/dev/null; then + echo " ✓ Image pushed to registry at '${push_host}' via docker push." + return 0 + fi + + echo "ERROR: Failed to push image '$image' to k3s registry at '${push_host}'." >&2 + return 1 +} + # Push image to local registry (via LOCAL_REGISTRY host address) with k3d import fallback. _push_to_k3d_registry() { local image="$1" @@ -609,7 +689,9 @@ _push_to_k3d_registry() { # Steps: containerd cache → Docker daemon (registry tag) → Docker daemon (plain tag) # → tar import → docker build + push/import. _ensure_prole_db_image() { - [[ "${PROLE_MODE:-}" != "k3d" ]] && return 0 + if [[ "${PROLE_MODE:-}" != "k3d" && "${PROLE_MODE:-}" != "k3s" ]]; then + return 0 + fi local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}" local image="" @@ -656,7 +738,11 @@ _ensure_prole_db_image() { # Step 2a: registry-tagged image in local Docker daemon → push + import if docker image inspect "$image" >/dev/null 2>&1; then echo " Image '$image' found in Docker daemon; pushing to registry ..." - _push_to_k3d_registry "$image" "$cluster_name" + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + _push_to_k3s_registry "$image" + else + _push_to_k3d_registry "$image" "$cluster_name" + fi return $? fi @@ -664,7 +750,11 @@ _ensure_prole_db_image() { if docker image inspect "$plain_image" >/dev/null 2>&1; then echo " Plain image '$plain_image' found in Docker daemon; tagging as '$image' and pushing ..." docker tag "$plain_image" "$image" - _push_to_k3d_registry "$image" "$cluster_name" + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + _push_to_k3s_registry "$image" + else + _push_to_k3d_registry "$image" "$cluster_name" + fi return $? fi @@ -689,7 +779,11 @@ _ensure_prole_db_image() { echo " Loading tar '$(basename "$found_tar")' into Docker daemon ..." docker load -i "$found_tar" docker tag "$plain_image" "$image" 2>/dev/null || true - _push_to_k3d_registry "$image" "$cluster_name" + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + _push_to_k3s_registry "$image" + else + _push_to_k3d_registry "$image" "$cluster_name" + fi return $? fi @@ -706,12 +800,28 @@ _ensure_prole_db_image() { fi docker tag "$plain_image" "$image" echo " Build complete. Pushing '$image' to registry ..." - _push_to_k3d_registry "$image" "$cluster_name" + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + _push_to_k3s_registry "$image" + else + _push_to_k3d_registry "$image" "$cluster_name" + fi return $? } +cleanup_unintended_cnpg_services() { + # Historical manifest bug: we used to create a CNPG-managed Service named `prole-db-001` + # as `type: LoadBalancer`. In k3s this spawns `svclb-prole-db-001` pods, and it can + # interfere with CNPG startup/reconciliation. Ensure it is removed if it exists. + local svc_name="prole-db-001" + if kubectl -n "$NAMESPACE" get svc "$svc_name" >/dev/null 2>&1; then + echo "Removing unintended Service '$svc_name' from namespace '$NAMESPACE' ..." + kubectl -n "$NAMESPACE" delete svc "$svc_name" --ignore-not-found >/dev/null 2>&1 || true + fi +} + ensure_prole_stack_resources() { echo "Applying CloudNative-PG cluster and related resources ..." + cleanup_unintended_cnpg_services || true local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}" local image="" if [[ -n "$image_override" ]]; then @@ -876,9 +986,9 @@ wait_for_cnpg_pods() { return 1 fi - # Check for image pull failures — attempt remediation in k3d mode + # Check for image pull failures — attempt remediation in k3d/k3s mode if echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >/dev/null; then - if [[ "${PROLE_MODE:-}" == "k3d" ]]; then + if [[ "${PROLE_MODE:-}" == "k3d" || "${PROLE_MODE:-}" == "k3s" ]]; then if [[ -z "$_image_pull_remediated" ]]; then _image_pull_remediated=1 local fail_pod fail_image @@ -1278,6 +1388,8 @@ reset_and_reinit() { kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true + recycle_released_prole_iscsi_pvs || true + if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then tmp_manifest=$(mktemp) sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest" @@ -1294,10 +1406,35 @@ reset_and_reinit() { ensure_prole_stack_resources fi - wait_for_cnpg_pods 300 + wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}" } -# Resolve OpenBao URL: prefer explicit env, then in-cluster service, then localhost port-forward +recycle_released_prole_iscsi_pvs() { + # StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. After a CNPG reset, PVs can remain + # in `Released` with a stale `claimRef`, which prevents new PVCs (same names) from binding. + # In that case, CNPG init jobs remain `Pending` with "didn't find available persistent volumes". + echo "Recycling Released prole-iscsi PVs (clearing stale claimRefs) for namespace '$NAMESPACE' ..." + local pvs + pvs=$(kubectl get pv -o json \ + | jq -r --arg ns "$NAMESPACE" '.items[] + | select(.spec.storageClassName == "prole-iscsi") + | select(.status.phase == "Released") + | select((.spec.claimRef.namespace // "") == $ns) + | .metadata.name' 2>/dev/null || true) + + if [[ -z "${pvs:-}" ]]; then + return 0 + fi + + local pv + for pv in $pvs; do + echo " - Clearing claimRef on PV: $pv" + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true + done +} + +# Resolve OpenBao URL: prefer explicit env, then in-cluster service. +# In k3s mode, out-of-cluster runs must use the k3s host (no localhost/port-forward). bao_service_url() { if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then echo "$PROLE_OPENBAO_URL" @@ -1305,13 +1442,23 @@ bao_service_url() { fi if prole_is_in_cluster; then local ns="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}" - # Troubleshoot: if the primary namespace is not found, check knoey-system as a common variant - if [[ "$ns" == "knoe-system" ]] && ! kubectl get ns knoe-system >/dev/null 2>&1 && kubectl get ns knoey-system >/dev/null 2>&1; then - ns="knoey-system" - fi echo "http://$OPENBAO_NAME.$ns.svc.cluster.local:8200" return 0 fi + + if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then + if command -v _prole_host_from_url >/dev/null 2>&1; then + local host + host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") + if [[ -n "${host:-}" ]]; then + echo "http://${host}:8200" + return 0 + fi + fi + echo "" + return 0 + fi + if curl -sS --connect-timeout 2 "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then echo "http://127.0.0.1:8200" return 0 @@ -1600,11 +1747,11 @@ initialize() { ensure_prole_stack_resources - if ! wait_for_cnpg_pods 300; then + if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then if cluster_has_pods; then echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2 if force_rollout; then - if ! wait_for_cnpg_pods 300; then + if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2 if ! reset_and_reinit; then return 1 diff --git a/etc/init_kerberos.sh b/etc/init_kerberos.sh index dbe489c..9f0e78c 100755 --- a/etc/init_kerberos.sh +++ b/etc/init_kerberos.sh @@ -234,22 +234,38 @@ openbao_url() { echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200" return 0 fi - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - else + + # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host + # (never via localhost or kubectl port-forward). + if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then + if command -v _prole_host_from_url >/dev/null 2>&1; then + local host + host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") + if [[ -n "${host:-}" ]]; then + echo "http://${host}:8200" + return 0 + fi + fi echo "" return 0 fi + + if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then + echo "http://127.0.0.1:8200" + return 0 + fi + echo "" + return 0 } _OPENBAO_PF_PID="" _OPENBAO_PF_STARTED=0 start_openbao_port_forward() { + if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then + return 1 + fi + local port="${OPENBAO_PORT_FORWARD_LOCAL}" local url="http://127.0.0.1:${port}" @@ -289,6 +305,13 @@ ensure_openbao_url() { printf '%s' "$url" return 0 fi + + if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then + # In k3s, do not attempt to synthesize connectivity via local port-forwards. + printf '' + return 0 + fi + if start_openbao_port_forward; then trap stop_openbao_port_forward EXIT printf 'http://127.0.0.1:%s' "${OPENBAO_PORT_FORWARD_LOCAL}" diff --git a/etc/init_monitoring.sh b/etc/init_monitoring.sh index b660bce..15f2705 100755 --- a/etc/init_monitoring.sh +++ b/etc/init_monitoring.sh @@ -287,16 +287,28 @@ openbao_url() { echo "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE:-default}}.svc.cluster.local:8200" return 0 fi - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - else + + # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host + # (never via localhost or kubectl port-forward). + if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then + if command -v _prole_host_from_url >/dev/null 2>&1; then + local host + host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") + if [[ -n "${host:-}" ]]; then + echo "http://${host}:8200" + return 0 + fi + fi echo "" return 0 fi + + if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then + echo "http://127.0.0.1:8200" + return 0 + fi + echo "" + return 0 } openbao_token() { diff --git a/etc/init_openbao.sh b/etc/init_openbao.sh index afef4e3..3ea1d73 100755 --- a/etc/init_openbao.sh +++ b/etc/init_openbao.sh @@ -99,7 +99,10 @@ admin_key_pub="$SECRETS_DIR/admin_ed25519.pub" root_token_file="$SECRETS_DIR/openbao-root-token" OPENBAO_NAMESPACE=${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}} -OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${NAMESPACE:-default}} +# OpenBao resources (StatefulSet/Service) live in `OPENBAO_NAMESPACE` (usually the service namespace), +# but secret *paths* must be keyed by the Prole workload namespace from `conf/prole.cfg` so that +# `${OPENBAO:kv/prole//...}` references resolve consistently across scripts. +OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${PROLE_NAMESPACE:-${NAMESPACE:-default}}} OPENBAO_RESOURCE_NAMESPACE="$OPENBAO_NAMESPACE" BAO_NAMESPACE="${OPENBAO_PATH_NAMESPACE:-default}" @@ -276,6 +279,11 @@ EOF # Use PRIMARY_SERVICE_NAMESPACE if skipping deployment (we port-forward to the common svc) start_openbao_port_forward() { + # In k3s mode we should never rely on local kubectl port-forwards. + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + return 1 + fi + local pf_ns="$OPENBAO_RESOURCE_NAMESPACE" if should_skip_deployment; then pf_ns="$PRIMARY_SERVICE_NAMESPACE" @@ -437,7 +445,15 @@ apply_k8s() { return 1 fi fi + # Always apply the Service manifest, then enforce the desired exposure mode. prole_render_manifest "$SCRIPT_DIR/../k8s/prole/openbao-service.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f - + + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + # Expose OpenBao externally in k3s (no kubectl port-forward). k3s `servicelb` will bind this + # on the node, making it reachable from the installer host. + kubectl patch svc "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --type merge \ + -p '{"spec":{"type":"LoadBalancer"}}' >/dev/null 2>&1 || true + fi else prole_render_manifest "$OPENBAO_MANIFEST_DIR/deployment.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f - fi @@ -805,7 +821,9 @@ case "$ACTION" in if should_skip_deployment; then pf_ns="$PRIMARY_SERVICE_NAMESPACE" fi - prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao" + if [[ "${PROLE_MODE:-}" != "k3s" ]]; then + prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao" + fi ;; status) ensure_tools @@ -854,22 +872,31 @@ case "$ACTION" in fi ensure_admin_keypair - trap stop_openbao_port_forward EXIT - if ! start_openbao_port_forward; then - echo "ERROR: Unable to establish OpenBao port-forward on 127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" >&2 - exit 1 - fi + if [[ "${PROLE_MODE:-}" == "k3s" ]]; then + # In k3s, initialize against the externally reachable endpoint (no port-forward). + svc_url="$(bao_service_url)" + wait_for_openbao_http 40 "$svc_url" + init_openbao_kv_and_store_admin_key "$svc_url" + else + trap stop_openbao_port_forward EXIT + if ! start_openbao_port_forward; then + echo "ERROR: Unable to establish OpenBao port-forward on 127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" >&2 + exit 1 + fi - # After port-forward, always use the local URL for initialization so we - # don't depend on OpenBao being externally reachable from this host. - local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" - wait_for_openbao_http 20 "$local_url" - init_openbao_kv_and_store_admin_key "$local_url" + # After port-forward, always use the local URL for initialization so we + # don't depend on OpenBao being externally reachable from this host. + local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" + wait_for_openbao_http 20 "$local_url" + init_openbao_kv_and_store_admin_key "$local_url" + fi pf_ns="${OPENBAO_NAMESPACE:-default}" if should_skip_deployment; then pf_ns="$PRIMARY_SERVICE_NAMESPACE" fi - prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao" + if [[ "${PROLE_MODE:-}" != "k3s" ]]; then + prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao" + fi echo "Initialization complete. OpenBao available at $(bao_service_url)" ;; update|reload) @@ -891,7 +918,9 @@ case "$ACTION" in if should_skip_deployment; then pf_ns="$PRIMARY_SERVICE_NAMESPACE" fi - prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao" + if [[ "${PROLE_MODE:-}" != "k3s" ]]; then + prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao" + fi echo "Re-applied manifests." ;; *) diff --git a/etc/init_opentofu.sh b/etc/init_opentofu.sh index 91ec0f4..07fdbde 100755 --- a/etc/init_opentofu.sh +++ b/etc/init_opentofu.sh @@ -78,16 +78,28 @@ openbao_url() { echo "http://openbao.${OPENTOFU_OPENBAO_NAMESPACE}.svc.cluster.local:8200" return 0 fi - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - else + + # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host + # (never via localhost or kubectl port-forward). + if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then + if command -v _prole_host_from_url >/dev/null 2>&1; then + local host + host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") + if [[ -n "${host:-}" ]]; then + echo "http://${host}:8200" + return 0 + fi + fi echo "" return 0 fi + + if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then + echo "http://127.0.0.1:8200" + return 0 + fi + echo "" + return 0 } openbao_token() { diff --git a/etc/prole_cfg.sh b/etc/prole_cfg.sh index 34d09dc..dc11a77 100644 --- a/etc/prole_cfg.sh +++ b/etc/prole_cfg.sh @@ -155,7 +155,7 @@ _prole_cfg_extract_key() { value=$(awk -F= -v k="$key" ' /^[[:space:]]*;/ {next} /^[[:space:]]*#/ {next} - /^[[:space:]]*\\[/ {next} + /^[[:space:]]*\[/ {next} $1 ~ "^[[:space:]]*" k "[[:space:]]*$" { v=$2 sub(/^[[:space:]]+/, "", v) @@ -433,6 +433,7 @@ import json import os import sys import urllib.request +import urllib.parse OPENBAO_PREFIX = "${OPENBAO:" OPENBAO_SUFFIX = "}" @@ -465,6 +466,21 @@ def resolve_openbao_ref(value: str) -> str: return "" url = os.environ.get("PROLE_OPENBAO_URL") + if not url: + # In k3s mode, do NOT fall back to localhost/port-forward. Derive the external + # OpenBao URL from the k3s API server URL when available. + mode = (os.environ.get("PROLE_MODE") or os.environ.get("DEPLOYMENT_MODE") or "").strip().lower() + if mode == "k3s": + k3s_server = (os.environ.get("PROLE_K3S_SERVER") or os.environ.get("K3S_SERVER_URL") or "").strip() + if k3s_server: + try: + parsed = urllib.parse.urlparse(k3s_server) + host = parsed.hostname or "" + if host: + url = f"http://{host}:8200" + except Exception: + url = "" + if not url: for p in ["8200", "8200"]: try: @@ -475,7 +491,11 @@ def resolve_openbao_ref(value: str) -> str: except Exception: pass if not url: - url = "http://127.0.0.1:8200" + mode = (os.environ.get("PROLE_MODE") or os.environ.get("DEPLOYMENT_MODE") or "").strip().lower() + if mode != "k3s": + url = "http://127.0.0.1:8200" + else: + return "" url = url.rstrip("/") try: req = urllib.request.Request(f"{url}/v1/{mount}/data/{secret_path}") diff --git a/k8s/prole/prole-db-recovery.yaml.tpl b/k8s/prole/prole-db-recovery.yaml.tpl index 58b80e8..c999bc7 100644 --- a/k8s/prole/prole-db-recovery.yaml.tpl +++ b/k8s/prole/prole-db-recovery.yaml.tpl @@ -5,7 +5,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: k3d-prole-registry.localhost:5000/prole-db:18-095 + imageName: myrddin.prole.org:5000/prole-db:18-100 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 @@ -66,21 +66,3 @@ spec: monitoring: enablePodMonitor: false - - # managed rw service - managed: - services: - additional: - - selectorType: rw - serviceTemplate: - metadata: - name: "prole-db-001" - labels: - test-label: "true" - annotations: - test-annotation: "true" - prometheus.io/scrape: "true" - prometheus.io/port: "9187" - prometheus.io/path: "/metrics" - spec: - type: LoadBalancer diff --git a/k8s/prole/prole-db.yaml b/k8s/prole/prole-db.yaml index 3a9b70e..7106663 100644 --- a/k8s/prole/prole-db.yaml +++ b/k8s/prole/prole-db.yaml @@ -4,7 +4,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: k3d-prole-registry.localhost:5000/prole-db:18-095 + imageName: myrddin.prole.org:5000/prole-db:18-100 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 @@ -102,21 +102,3 @@ spec: prole.storage/role: wal monitoring: enablePodMonitor: false - - # managed rw service - managed: - services: - additional: - - selectorType: rw - serviceTemplate: - metadata: - name: "prole-db-001" - labels: - test-label: "true" - annotations: - test-annotation: "true" - prometheus.io/scrape: "true" - prometheus.io/port: "9187" - prometheus.io/path: "/metrics" - spec: - type: LoadBalancer diff --git a/prole-db/.version b/prole-db/.version index d7f3668..105d7d9 100644 --- a/prole-db/.version +++ b/prole-db/.version @@ -1 +1 @@ -98 \ No newline at end of file +100 \ No newline at end of file diff --git a/prole.sh b/prole.sh index e222d9a..10dda7b 100755 --- a/prole.sh +++ b/prole.sh @@ -6,15 +6,6 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "${ROOT_DIR}" -# OpenBao Detection -if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then - if curl -s -f "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - export PROLE_OPENBAO_URL="http://127.0.0.1:8200" - elif curl -s -f "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then - export PROLE_OPENBAO_URL="http://127.0.0.1:18200" - fi -fi - if [[ -z "${OPENBAO_ROOT_TOKEN:-}" ]]; then if [[ -f "${ROOT_DIR}/etc/secrets/openbao-root-token" ]]; then export OPENBAO_ROOT_TOKEN=$(cat "${ROOT_DIR}/etc/secrets/openbao-root-token") @@ -174,6 +165,34 @@ resolve_prole_mode() { fi } +ensure_openbao_url_for_mode() { + local mode="${1:-}" + if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then + return 0 + fi + + case "$mode" in + k3d) + if curl -s -f "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then + export PROLE_OPENBAO_URL="http://127.0.0.1:8200" + elif curl -s -f "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then + export PROLE_OPENBAO_URL="http://127.0.0.1:18200" + fi + ;; + k3s|k8s) + # In k3s/k8s, the installer runs outside the cluster and must reach OpenBao via a real + # externally reachable endpoint (no kubectl port-forward / localhost defaults). + if command -v _prole_host_from_url >/dev/null 2>&1; then + local host + host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") + if [[ -n "${host:-}" ]]; then + export PROLE_OPENBAO_URL="http://${host}:8200" + fi + fi + ;; + esac +} + resolve_k3d_cluster_name() { local name="${DEPLOYMENT_TARGET:-${DISPLAY_NAME:-}}" if [[ -z "$name" && -n "${KUBECTL_CONTEXT:-}" ]]; then @@ -574,6 +593,23 @@ case "${CMD}" in fi CMD_ARGS=("${_INSTALL_PASSTHRU[@]}") + # Ensure OpenBao URL is set appropriately for the active mode before running install.py. + _cfg_for_env="${PROLE_CFG_DEFAULT}" + for ((i=0; i<${#CMD_ARGS[@]}; i++)); do + if [[ "${CMD_ARGS[$i]}" == "-c" || "${CMD_ARGS[$i]}" == "--config" ]]; then + if [[ $((i+1)) -lt ${#CMD_ARGS[@]} ]]; then + _cfg_for_env="${CMD_ARGS[$((i+1))]}" + fi + fi + done + load_prole_cfg "${_cfg_for_env}" + _mode_for_env=$(resolve_prole_mode) + if [[ -z "${_mode_for_env:-}" ]]; then + _mode_for_env="k3d" + fi + ensure_openbao_url_for_mode "${_mode_for_env}" + unset _cfg_for_env _mode_for_env + PY_CMD=("python3") if [[ "${COVERAGE}" -eq 1 ]]; then PY_CMD=("coverage" "run" "--source=installer")