chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 11:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

milestone(k3s+cnpg): end-to-end deploy run via standalone installer

Browse Source 
We can now complete a full K3s deployment run and deliver a working CloudNativePG (CNPG) stack from the standalone installer.\n\nHighlights:\n- Installer scripts hardened for CNPG deploy + recovery templating + rollout control\n- OpenTofu/K3s manifests aligned with in-repo k8s templates for prole-db\n- Supporting init flows updated (OpenBao, monitoring, Kerberos, OpenTofu)

Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
chrisfu 2026-03-03 20:03:15 -08:00
parent c0c7c5539c
commit 86cf99cf76
13 changed files with 404 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -42,6 +42,7 @@
# Secrets and local config
*.kubeconfig
*.kubeconfig*
etc/secrets/
*-password.txt
*secret.yaml

84
conf/prole.cfg
View File

@ -1,5 +1,5 @@
; Prole Master Configuration File
; Generated by install.py on 2026-03-02 18:39:26
; Generated by install.py on 2026-03-03 16:30:57
; This file is used as input for Ansible deployment and k8s cluster creation.
[User]
@ -42,7 +42,7 @@ dependencies.opentofu.install = true
dependencies.python.install = true
dependencies.verify_all = false
disk_selection.disk_type = local
disk_selection.local_path = /Users/chrisfu
disk_selection.local_path = ${PROLE_HOME}/prole-tools-app/dist
disk_selection.removable_mount =
env_setup.NAMESPACE = ${NAMESPACE}
env_setup.PROLE_CONF = ${PROLE_CONF}
@ -51,13 +51,13 @@ env_setup.PROLE_HOME = ${PROLE_HOME}
env_setup.PROLE_LOGS = ${PROLE_LOGS}
env_setup.PROLE_SERVICE = ${PROLE_SERVICE}
init_cluster.at_rest_encryption_enabled = true
init_cluster.cluster_env = dev
init_cluster.deployment_target = knoe-dev-cluster
init_cluster.cluster_env = service
init_cluster.deployment_target = prole-service-cluster
init_cluster.gitops_enabled = false
init_cluster.k3s_server_url =
init_cluster.k3s_token =
init_cluster.kerberos_enabled = false
init_cluster.mode = k3d
init_cluster.k3s_server_url = https://myrddin.prole.org:6443
init_cluster.k3s_token = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
init_cluster.kerberos_enabled = true
init_cluster.mode = k3s
init_cluster.start_cluster = true
init_cluster.supabase_enabled = false
init_cnpg_deploy.force_rollout = false
@ -70,10 +70,11 @@ init_password.db_password_confirm = ${OPENBAO:kv/prole/knoey-db-0/db#password}
init_password.db_username = root
init_password.generate_ssh_key = true
init_scripts.run_scripts = true
kerberos_config.enabled = false
kerberos_config.kdc =
kerberos_config.enabled = true
kerberos_config.init_authority = false
kerberos_config.kdc = 10.0.0.3
kerberos_config.password = ${OPENBAO:kv/prole/knoey-db-0/kerberos#password}
kerberos_config.realm =
kerberos_config.realm = PROLE.ORG
kerberos_config.test_connection = false
kerberos_config.user = administrator
network_scan.run = true
@ -82,26 +83,30 @@ ollama_config.server_host =
ollama_config.server_port = 11434
[Global]
CNPG_VERSION = 1.28.1
CNPG_RELEASE_SERIES = 1.28
; Variables used by name in more than one place or assumed global scope
CLUSTER_ENV = dev
ARGOCD_NAMESPACE = argocd
CLUSTER_ENV = service
DB_HOST_PORT = 5432
DB_PASSWORD = ${OPENBAO:kv/prole/knoey-db-0/db#password}
DEPLOYMENT_MODE = k3d
DEPLOYMENT_TARGET = knoe-dev-cluster
DEPLOYMENT_MODE = k3s
DEPLOYMENT_TARGET = prole-service-cluster
DOCKER_IMPORT_DIR =
NAMESPACE = ${NAMESPACE}
PROLE_DB_USER = root
PROLE_HOME = ${PROLE_HOME}
PROLE_K3S_SERVER =
PROLE_K3S_TOKEN =
PROLE_K3S_SERVER = https://myrddin.prole.org:6443
PROLE_K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
PROLE_OPENTOFU_URL = http://127.0.0.1:8080
REGISTRY_NAMESPACE = default
SERVICE_NAMESPACE = knoe-system
[Welcome]
; No configuration values captured yet for this section.
[Dependencies]
; No configuration values captured yet for this section.
STATUS = All installed
[Network]
AD_DC_HOST = myrddin.prole.org
@ -112,11 +117,21 @@ ANSIBLE_INVENTORY = ${PROLE_HOME}/infrastructure/inventory
ANSIBLE_REALM = PROLE.ORG
ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_hosts":["myrddin.prole.org","pi.prole.org","retropie.prole.org"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"]},"hosts":{"myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":[]}
KDC_ANSIBLE_DETECTED = 10.0.0.3
KDC_AUTO_DETECTED = 10.0.0.3
KERBEROS_AUTO_ENABLED = True
[Port Forwards]
; No configuration values captured yet for this section.
PORT_FORWARD_K3S_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD
PORT_FORWARD_K3S_MAPPING_2 = id=garage;namespace=knoe-system;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3
PORT_FORWARD_K3S_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao;address=0.0.0.0;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao
PORT_FORWARD_K3S_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu
PORT_FORWARD_K3S_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard
PORT_FORWARD_K3S_MAPPING_6 = id=postgres;namespace=${NAMESPACE};target=svc/prole-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary)
PORT_FORWARD_K3S_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI
PORT_FORWARD_K3S_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI
[System Environment]
NAMESPACE = knoey-db-0
PROLE_CONF = ${PROLE_CONF}
PROLE_DATA = ${PROLE_DATA}
PROLE_HOME = ${PROLE_HOME}
@ -133,47 +148,54 @@ PASSWORD = ${OPENBAO:kv/prole/knoey-db-0/kerberos#password}
; No configuration values captured yet for this section.
[Optional Features]
GITOPS_ENABLED = False
AT_REST_ENCRYPTION_ENABLED = true
GITOPS_ENABLED = false
KERBEROS_ENABLED = true
SUPABASE_ENABLED = false
[GitOps]
; No configuration values captured yet for this section.
[Database Creation]
DB_NAME = knoey-db-0
DB_USER = root
NAMESPACE = ${NAMESPACE}
[Initialize Cluster]
; No configuration values captured yet for this section.
ENVIRONMENT = service
K3S_SERVER_URL = https://myrddin.prole.org:6443
K3S_TOKEN = ${PROLE_SECRET:v1:7AxJwJb0PoEoe0a0:4P77NxrvZejAdfZlZlrQ7m8zC-ENEaInKxb43uzSGFl5nLpSjjuGsx_x1oPB38qijvDx59O-2jqp-nId3H6JJSdy7g1U01I6P4xiZ9xrUDMd5SazXzrPhQCE4ztPo-M1-czz7BpaelffQxkhk0gdUrEHoFzXLiGamX7S9Q==}
[Dev Cluster (k3d)]
CLUSTER_ENV = dev
CLUSTER_ENV = k3d-knoe-dev-cluster
DISPLAY_NAME = knoe-dev-cluster
KUBECTL_CONTEXT =
KUBECTL_CONTEXT = service
MODE = k3d
[Service Cluster (k3s)]
CLUSTER_ENV = prole-service-cluster
DISPLAY_NAME = prole-service-cluster
K3S_SERVER_URL =
K3S_TOKEN =
K3S_SERVER_URL = https://myrddin.prole.org:6443
K3S_TOKEN = ${OPENBAO:kv/prole/knoey-db-0/k3s#token}
MODE = k3s
PIPELINE_URL = http://127.0.0.1:8080
[Prod Cluster (k8s)]
ARTIFACTS_DIR = /private/var/folders/rt/pywlnmxj3dn7t5552vwdcpp80000gn/T/pytest-of-chrisfu/pytest-94/test_encryption_toggle0/data/staging
ARTIFACTS_DIR = ${PROLE_DATA}/staging
CLUSTER_ENV = prole-prod-cluster
DISPLAY_NAME = prole-prod-cluster
MODE = k8s
PIPELINE_URL = http://127.0.0.1:8080
[Docker Build]
; No configuration values captured yet for this section.
LOCAL_REGISTRY = myrddin.prole.org:5000
LOCAL_REGISTRY_INTERNAL = myrddin.prole.org:5000
[Initialization Scripts]
; No configuration values captured yet for this section.
STATUS = Attempted
[Deployment]
MODE = k3d
TARGET = knoe-dev-cluster
MODE = k3s
TARGET = prole-service-cluster
[Install]
; No configuration values captured yet for this section.
STATUS = Failed

18
deploy/opentofu/k3s/manifests/prole/prole-db.yaml
View File

@ -102,21 +102,3 @@ spec:
prole.storage/role: wal
monitoring:
enablePodMonitor: false
# managed rw service
managed:
services:
additional:
- selectorType: rw
serviceTemplate:
metadata:
name: "prole-db-001"
labels:
test-label: "true"
annotations:
test-annotation: "true"
prometheus.io/scrape: "true"
prometheus.io/port: "9187"
prometheus.io/path: "/metrics"
spec:
type: LoadBalancer

191
etc/init_cloudnative_pg.sh
View File

@ -49,6 +49,11 @@ CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-}
PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}
BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup}
BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800}
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-900}
else
CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-300}
fi
RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl"
BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-}
BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0}
@ -203,16 +208,28 @@ openbao_url() {
echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200"
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
else
# In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
# (never via localhost or kubectl port-forward).
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
if command -v _prole_host_from_url >/dev/null 2>&1; then
local host
host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
if [[ -n "${host:-}" ]]; then
echo "http://${host}:8200"
return 0
fi
fi
echo ""
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
fi
echo ""
return 0
}
openbao_token() {
@ -253,6 +270,11 @@ ensure_cnpg_operator() {
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
wait_for_cnpg_webhook 180 || true
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
tune_cnpg_operator_for_k3s || true
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true
wait_for_cnpg_webhook 300 || true
fi
return 0
fi
@ -267,9 +289,42 @@ ensure_cnpg_operator() {
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
wait_for_cnpg_webhook 180 || true
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
tune_cnpg_operator_for_k3s || true
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true
wait_for_cnpg_webhook 300 || true
fi
fi
}
tune_cnpg_operator_for_k3s() {
# k3s on a single node can experience short API/server or scheduling stalls under load.
# CNPG's default probes are very aggressive (timeoutSeconds=1), which can cause flapping
# readiness and webhook endpoints disappearing mid-apply.
echo "Tuning CNPG operator deployment probes/resources for k3s ..."
kubectl -n cnpg-system patch deploy cnpg-controller-manager --type merge -p '
{
"spec": {
"template": {
"spec": {
"containers": [
{
"name": "manager",
"resources": {
"requests": {"cpu": "250m", "memory": "512Mi"},
"limits": {"cpu": "500m", "memory": "1Gi"}
},
"livenessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
"readinessProbe": {"timeoutSeconds": 5, "failureThreshold": 6},
"startupProbe": {"timeoutSeconds": 5, "failureThreshold": 60}
}
]
}
}
}
}' >/dev/null 2>&1 || return 1
}
get_latest_barman_plugin_version() {
local version tag
tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "")
@ -579,6 +634,31 @@ apply_prole_manifest_file() {
return 1
}
_push_to_k3s_registry() {
local image="$1"
local push_host="${LOCAL_REGISTRY:-${LOCAL_REGISTRY_INTERNAL:-myrddin.prole.org:5000}}"
local plain_image="${image##*/}"
if command -v skopeo >/dev/null 2>&1; then
echo " Pushing to k3s registry at '${push_host}' using skopeo ..."
if skopeo copy --dest-tls-verify=false docker-daemon:"$image" docker://"${push_host}/${plain_image}"; then
echo " ✓ Image pushed to registry at '${push_host}'."
return 0
fi
fi
# Fallback to docker push if skopeo is missing or fails (might fail if daemon not configured)
local push_ref="${push_host}/${plain_image}"
docker tag "$image" "$push_ref" 2>/dev/null || true
if docker push "$push_ref" 2>/dev/null; then
echo " ✓ Image pushed to registry at '${push_host}' via docker push."
return 0
fi
echo "ERROR: Failed to push image '$image' to k3s registry at '${push_host}'." >&2
return 1
}
# Push image to local registry (via LOCAL_REGISTRY host address) with k3d import fallback.
_push_to_k3d_registry() {
local image="$1"
@ -609,7 +689,9 @@ _push_to_k3d_registry() {
# Steps: containerd cache → Docker daemon (registry tag) → Docker daemon (plain tag)
# → tar import → docker build + push/import.
_ensure_prole_db_image() {
[[ "${PROLE_MODE:-}" != "k3d" ]] && return 0
if [[ "${PROLE_MODE:-}" != "k3d" && "${PROLE_MODE:-}" != "k3s" ]]; then
return 0
fi
local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
local image=""
@ -656,7 +738,11 @@ _ensure_prole_db_image() {
# Step 2a: registry-tagged image in local Docker daemon → push + import
if docker image inspect "$image" >/dev/null 2>&1; then
echo " Image '$image' found in Docker daemon; pushing to registry ..."
_push_to_k3d_registry "$image" "$cluster_name"
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
_push_to_k3s_registry "$image"
else
_push_to_k3d_registry "$image" "$cluster_name"
fi
return $?
fi
@ -664,7 +750,11 @@ _ensure_prole_db_image() {
if docker image inspect "$plain_image" >/dev/null 2>&1; then
echo " Plain image '$plain_image' found in Docker daemon; tagging as '$image' and pushing ..."
docker tag "$plain_image" "$image"
_push_to_k3d_registry "$image" "$cluster_name"
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
_push_to_k3s_registry "$image"
else
_push_to_k3d_registry "$image" "$cluster_name"
fi
return $?
fi
@ -689,7 +779,11 @@ _ensure_prole_db_image() {
echo " Loading tar '$(basename "$found_tar")' into Docker daemon ..."
docker load -i "$found_tar"
docker tag "$plain_image" "$image" 2>/dev/null || true
_push_to_k3d_registry "$image" "$cluster_name"
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
_push_to_k3s_registry "$image"
else
_push_to_k3d_registry "$image" "$cluster_name"
fi
return $?
fi
@ -706,12 +800,28 @@ _ensure_prole_db_image() {
fi
docker tag "$plain_image" "$image"
echo " Build complete. Pushing '$image' to registry ..."
_push_to_k3d_registry "$image" "$cluster_name"
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
_push_to_k3s_registry "$image"
else
_push_to_k3d_registry "$image" "$cluster_name"
fi
return $?
}
cleanup_unintended_cnpg_services() {
# Historical manifest bug: we used to create a CNPG-managed Service named `prole-db-001`
# as `type: LoadBalancer`. In k3s this spawns `svclb-prole-db-001` pods, and it can
# interfere with CNPG startup/reconciliation. Ensure it is removed if it exists.
local svc_name="prole-db-001"
if kubectl -n "$NAMESPACE" get svc "$svc_name" >/dev/null 2>&1; then
echo "Removing unintended Service '$svc_name' from namespace '$NAMESPACE' ..."
kubectl -n "$NAMESPACE" delete svc "$svc_name" --ignore-not-found >/dev/null 2>&1 || true
fi
}
ensure_prole_stack_resources() {
echo "Applying CloudNative-PG cluster and related resources ..."
cleanup_unintended_cnpg_services || true
local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
local image=""
if [[ -n "$image_override" ]]; then
@ -876,9 +986,9 @@ wait_for_cnpg_pods() {
return 1
fi
# Check for image pull failures — attempt remediation in k3d mode
# Check for image pull failures — attempt remediation in k3d/k3s mode
if echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >/dev/null; then
if [[ "${PROLE_MODE:-}" == "k3d" ]]; then
if [[ "${PROLE_MODE:-}" == "k3d" || "${PROLE_MODE:-}" == "k3s" ]]; then
if [[ -z "$_image_pull_remediated" ]]; then
_image_pull_remediated=1
local fail_pod fail_image
@ -1278,6 +1388,8 @@ reset_and_reinit() {
kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true
recycle_released_prole_iscsi_pvs || true
if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then
tmp_manifest=$(mktemp)
sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest"
@ -1294,10 +1406,35 @@ reset_and_reinit() {
ensure_prole_stack_resources
fi
wait_for_cnpg_pods 300
wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"
}
# Resolve OpenBao URL: prefer explicit env, then in-cluster service, then localhost port-forward
recycle_released_prole_iscsi_pvs() {
# StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. After a CNPG reset, PVs can remain
# in `Released` with a stale `claimRef`, which prevents new PVCs (same names) from binding.
# In that case, CNPG init jobs remain `Pending` with "didn't find available persistent volumes".
echo "Recycling Released prole-iscsi PVs (clearing stale claimRefs) for namespace '$NAMESPACE' ..."
local pvs
pvs=$(kubectl get pv -o json \
| jq -r --arg ns "$NAMESPACE" '.items[]
| select(.spec.storageClassName == "prole-iscsi")
| select(.status.phase == "Released")
| select((.spec.claimRef.namespace // "") == $ns)
| .metadata.name' 2>/dev/null || true)
if [[ -z "${pvs:-}" ]]; then
return 0
fi
local pv
for pv in $pvs; do
echo " - Clearing claimRef on PV: $pv"
kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
done
}
# Resolve OpenBao URL: prefer explicit env, then in-cluster service.
# In k3s mode, out-of-cluster runs must use the k3s host (no localhost/port-forward).
bao_service_url() {
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
echo "$PROLE_OPENBAO_URL"
@ -1305,13 +1442,23 @@ bao_service_url() {
fi
if prole_is_in_cluster; then
local ns="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}"
# Troubleshoot: if the primary namespace is not found, check knoey-system as a common variant
if [[ "$ns" == "knoe-system" ]] && ! kubectl get ns knoe-system >/dev/null 2>&1 && kubectl get ns knoey-system >/dev/null 2>&1; then
ns="knoey-system"
fi
echo "http://$OPENBAO_NAME.$ns.svc.cluster.local:8200"
return 0
fi
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
if command -v _prole_host_from_url >/dev/null 2>&1; then
local host
host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
if [[ -n "${host:-}" ]]; then
echo "http://${host}:8200"
return 0
fi
fi
echo ""
return 0
fi
if curl -sS --connect-timeout 2 "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
@ -1600,11 +1747,11 @@ initialize() {
ensure_prole_stack_resources
if ! wait_for_cnpg_pods 300; then
if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then
if cluster_has_pods; then
echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2
if force_rollout; then
if ! wait_for_cnpg_pods 300; then
if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then
echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2
if ! reset_and_reinit; then
return 1

37
etc/init_kerberos.sh
View File

@ -234,22 +234,38 @@ openbao_url() {
echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200"
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
else
# In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
# (never via localhost or kubectl port-forward).
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
if command -v _prole_host_from_url >/dev/null 2>&1; then
local host
host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
if [[ -n "${host:-}" ]]; then
echo "http://${host}:8200"
return 0
fi
fi
echo ""
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
fi
echo ""
return 0
}
_OPENBAO_PF_PID=""
_OPENBAO_PF_STARTED=0
start_openbao_port_forward() {
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
return 1
fi
local port="${OPENBAO_PORT_FORWARD_LOCAL}"
local url="http://127.0.0.1:${port}"
@ -289,6 +305,13 @@ ensure_openbao_url() {
printf '%s' "$url"
return 0
fi
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
# In k3s, do not attempt to synthesize connectivity via local port-forwards.
printf ''
return 0
fi
if start_openbao_port_forward; then
trap stop_openbao_port_forward EXIT
printf 'http://127.0.0.1:%s' "${OPENBAO_PORT_FORWARD_LOCAL}"

26
etc/init_monitoring.sh
View File

@ -287,16 +287,28 @@ openbao_url() {
echo "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE:-default}}.svc.cluster.local:8200"
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
else
# In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
# (never via localhost or kubectl port-forward).
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
if command -v _prole_host_from_url >/dev/null 2>&1; then
local host
host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
if [[ -n "${host:-}" ]]; then
echo "http://${host}:8200"
return 0
fi
fi
echo ""
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
fi
echo ""
return 0
}
openbao_token() {

57
etc/init_openbao.sh
View File

@ -99,7 +99,10 @@ admin_key_pub="$SECRETS_DIR/admin_ed25519.pub"
root_token_file="$SECRETS_DIR/openbao-root-token"
OPENBAO_NAMESPACE=${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}}
OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${NAMESPACE:-default}}
# OpenBao resources (StatefulSet/Service) live in `OPENBAO_NAMESPACE` (usually the service namespace),
# but secret *paths* must be keyed by the Prole workload namespace from `conf/prole.cfg` so that
# `${OPENBAO:kv/prole/<PROLE_NAMESPACE>/...}` references resolve consistently across scripts.
OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${PROLE_NAMESPACE:-${NAMESPACE:-default}}}
OPENBAO_RESOURCE_NAMESPACE="$OPENBAO_NAMESPACE"
BAO_NAMESPACE="${OPENBAO_PATH_NAMESPACE:-default}"
@ -276,6 +279,11 @@ EOF
# Use PRIMARY_SERVICE_NAMESPACE if skipping deployment (we port-forward to the common svc)
start_openbao_port_forward() {
# In k3s mode we should never rely on local kubectl port-forwards.
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
return 1
fi
local pf_ns="$OPENBAO_RESOURCE_NAMESPACE"
if should_skip_deployment; then
pf_ns="$PRIMARY_SERVICE_NAMESPACE"
@ -437,7 +445,15 @@ apply_k8s() {
return 1
fi
fi
# Always apply the Service manifest, then enforce the desired exposure mode.
prole_render_manifest "$SCRIPT_DIR/../k8s/prole/openbao-service.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f -
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
# Expose OpenBao externally in k3s (no kubectl port-forward). k3s `servicelb` will bind this
# on the node, making it reachable from the installer host.
kubectl patch svc "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --type merge \
-p '{"spec":{"type":"LoadBalancer"}}' >/dev/null 2>&1 || true
fi
else
prole_render_manifest "$OPENBAO_MANIFEST_DIR/deployment.yaml" | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f -
fi
@ -805,7 +821,9 @@ case "$ACTION" in
if should_skip_deployment; then
pf_ns="$PRIMARY_SERVICE_NAMESPACE"
fi
prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
fi
;;
status)
ensure_tools
@ -854,22 +872,31 @@ case "$ACTION" in
fi
ensure_admin_keypair
trap stop_openbao_port_forward EXIT
if ! start_openbao_port_forward; then
echo "ERROR: Unable to establish OpenBao port-forward on 127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" >&2
exit 1
fi
if [[ "${PROLE_MODE:-}" == "k3s" ]]; then
# In k3s, initialize against the externally reachable endpoint (no port-forward).
svc_url="$(bao_service_url)"
wait_for_openbao_http 40 "$svc_url"
init_openbao_kv_and_store_admin_key "$svc_url"
else
trap stop_openbao_port_forward EXIT
if ! start_openbao_port_forward; then
echo "ERROR: Unable to establish OpenBao port-forward on 127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}" >&2
exit 1
fi
# After port-forward, always use the local URL for initialization so we
# don't depend on OpenBao being externally reachable from this host.
local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}"
wait_for_openbao_http 20 "$local_url"
init_openbao_kv_and_store_admin_key "$local_url"
# After port-forward, always use the local URL for initialization so we
# don't depend on OpenBao being externally reachable from this host.
local_url="http://127.0.0.1:${OPENBAO_PORT_FORWARD_LOCAL}"
wait_for_openbao_http 20 "$local_url"
init_openbao_kv_and_store_admin_key "$local_url"
fi
pf_ns="${OPENBAO_NAMESPACE:-default}"
if should_skip_deployment; then
pf_ns="$PRIMARY_SERVICE_NAMESPACE"
fi
prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
fi
echo "Initialization complete. OpenBao available at $(bao_service_url)"
;;
update|reload)
@ -891,7 +918,9 @@ case "$ACTION" in
if should_skip_deployment; then
pf_ns="$PRIMARY_SERVICE_NAMESPACE"
fi
prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
prole_register_port_forward "openbao" "$pf_ns" "svc/openbao" "${OPENBAO_PORT_FORWARD_LOCAL}" "8200" "${OPENBAO_PF_ADDR}" "TCP" "OpenBao"
fi
echo "Re-applied manifests."
;;
*)

26
etc/init_opentofu.sh
View File

@ -78,16 +78,28 @@ openbao_url() {
echo "http://openbao.${OPENTOFU_OPENBAO_NAMESPACE}.svc.cluster.local:8200"
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
else
# In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
# (never via localhost or kubectl port-forward).
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
if command -v _prole_host_from_url >/dev/null 2>&1; then
local host
host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
if [[ -n "${host:-}" ]]; then
echo "http://${host}:8200"
return 0
fi
fi
echo ""
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
fi
echo ""
return 0
}
openbao_token() {

24
etc/prole_cfg.sh
View File

@ -155,7 +155,7 @@ _prole_cfg_extract_key() {
value=$(awk -F= -v k="$key" '
/^[[:space:]]*;/ {next}
/^[[:space:]]*#/ {next}
/^[[:space:]]*\\[/ {next}
/^[[:space:]]*\[/ {next}
$1 ~ "^[[:space:]]*" k "[[:space:]]*$" {
v=$2
sub(/^[[:space:]]+/, "", v)
@ -433,6 +433,7 @@ import json
import os
import sys
import urllib.request
import urllib.parse
OPENBAO_PREFIX = "${OPENBAO:"
OPENBAO_SUFFIX = "}"
@ -465,6 +466,21 @@ def resolve_openbao_ref(value: str) -> str:
return ""
url = os.environ.get("PROLE_OPENBAO_URL")
if not url:
# In k3s mode, do NOT fall back to localhost/port-forward. Derive the external
# OpenBao URL from the k3s API server URL when available.
mode = (os.environ.get("PROLE_MODE") or os.environ.get("DEPLOYMENT_MODE") or "").strip().lower()
if mode == "k3s":
k3s_server = (os.environ.get("PROLE_K3S_SERVER") or os.environ.get("K3S_SERVER_URL") or "").strip()
if k3s_server:
try:
parsed = urllib.parse.urlparse(k3s_server)
host = parsed.hostname or ""
if host:
url = f"http://{host}:8200"
except Exception:
url = ""
if not url:
for p in ["8200", "8200"]:
try:
@ -475,7 +491,11 @@ def resolve_openbao_ref(value: str) -> str:
except Exception:
pass
if not url:
url = "http://127.0.0.1:8200"
mode = (os.environ.get("PROLE_MODE") or os.environ.get("DEPLOYMENT_MODE") or "").strip().lower()
if mode != "k3s":
url = "http://127.0.0.1:8200"
else:
return ""
url = url.rstrip("/")
try:
req = urllib.request.Request(f"{url}/v1/{mount}/data/{secret_path}")

20
k8s/prole/prole-db-recovery.yaml.tpl
View File

@ -5,7 +5,7 @@ metadata:
name: prole-db
spec:
instances: 3
imageName: k3d-prole-registry.localhost:5000/prole-db:18-095
imageName: myrddin.prole.org:5000/prole-db:18-100
postgresUID: 100
postgresGID: 101
maxSyncReplicas: 1
@ -66,21 +66,3 @@ spec:
monitoring:
enablePodMonitor: false
# managed rw service
managed:
services:
additional:
- selectorType: rw
serviceTemplate:
metadata:
name: "prole-db-001"
labels:
test-label: "true"
annotations:
test-annotation: "true"
prometheus.io/scrape: "true"
prometheus.io/port: "9187"
prometheus.io/path: "/metrics"
spec:
type: LoadBalancer

20
k8s/prole/prole-db.yaml
View File

@ -4,7 +4,7 @@ metadata:
name: prole-db
spec:
instances: 3
imageName: k3d-prole-registry.localhost:5000/prole-db:18-095
imageName: myrddin.prole.org:5000/prole-db:18-100
postgresUID: 100
postgresGID: 101
maxSyncReplicas: 1
@ -102,21 +102,3 @@ spec:
prole.storage/role: wal
monitoring:
enablePodMonitor: false
# managed rw service
managed:
services:
additional:
- selectorType: rw
serviceTemplate:
metadata:
name: "prole-db-001"
labels:
test-label: "true"
annotations:
test-annotation: "true"
prometheus.io/scrape: "true"
prometheus.io/port: "9187"
prometheus.io/path: "/metrics"
spec:
type: LoadBalancer

2
prole-db/.version
View File

@ -1 +1 @@
98
100

54
prole.sh
View File

@ -6,15 +6,6 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${ROOT_DIR}"
# OpenBao Detection
if [[ -z "${PROLE_OPENBAO_URL:-}" ]]; then
if curl -s -f "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
export PROLE_OPENBAO_URL="http://127.0.0.1:8200"
elif curl -s -f "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
export PROLE_OPENBAO_URL="http://127.0.0.1:18200"
fi
fi
if [[ -z "${OPENBAO_ROOT_TOKEN:-}" ]]; then
if [[ -f "${ROOT_DIR}/etc/secrets/openbao-root-token" ]]; then
export OPENBAO_ROOT_TOKEN=$(cat "${ROOT_DIR}/etc/secrets/openbao-root-token")
@ -174,6 +165,34 @@ resolve_prole_mode() {
fi
}
ensure_openbao_url_for_mode() {
local mode="${1:-}"
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
return 0
fi
case "$mode" in
k3d)
if curl -s -f "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
export PROLE_OPENBAO_URL="http://127.0.0.1:8200"
elif curl -s -f "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
export PROLE_OPENBAO_URL="http://127.0.0.1:18200"
fi
;;
k3s|k8s)
# In k3s/k8s, the installer runs outside the cluster and must reach OpenBao via a real
# externally reachable endpoint (no kubectl port-forward / localhost defaults).
if command -v _prole_host_from_url >/dev/null 2>&1; then
local host
host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
if [[ -n "${host:-}" ]]; then
export PROLE_OPENBAO_URL="http://${host}:8200"
fi
fi
;;
esac
}
resolve_k3d_cluster_name() {
local name="${DEPLOYMENT_TARGET:-${DISPLAY_NAME:-}}"
if [[ -z "$name" && -n "${KUBECTL_CONTEXT:-}" ]]; then
@ -574,6 +593,23 @@ case "${CMD}" in
fi
CMD_ARGS=("${_INSTALL_PASSTHRU[@]}")
# Ensure OpenBao URL is set appropriately for the active mode before running install.py.
_cfg_for_env="${PROLE_CFG_DEFAULT}"
for ((i=0; i<${#CMD_ARGS[@]}; i++)); do
if [[ "${CMD_ARGS[$i]}" == "-c" || "${CMD_ARGS[$i]}" == "--config" ]]; then
if [[ $((i+1)) -lt ${#CMD_ARGS[@]} ]]; then
_cfg_for_env="${CMD_ARGS[$((i+1))]}"
fi
fi
done
load_prole_cfg "${_cfg_for_env}"
_mode_for_env=$(resolve_prole_mode)
if [[ -z "${_mode_for_env:-}" ]]; then
_mode_for_env="k3d"
fi
ensure_openbao_url_for_mode "${_mode_for_env}"
unset _cfg_for_env _mode_for_env
PY_CMD=("python3")
if [[ "${COVERAGE}" -eq 1 ]]; then
PY_CMD=("coverage" "run" "--source=installer")