feat: pre-provision cluster-scoped CNPG Synology PVs

Add storage ops for deterministic namespace+cluster local PV paths, labels/selectors, host-path preparation, and idempotent reconciliation before CNPG cluster apply.

Wire selector injection and validation into CNPG deploy flow/script, and extend installer tests for provisioning orchestration and failure handling.

Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
chrisfu 2026-03-23 00:32:35 -07:00
parent 59c8ea3f8f
commit b09319ffaf
10 changed files with 950 additions and 27 deletions

1
.gitignore vendored
View File

@ -76,3 +76,4 @@ htmlcov/
/mock_val/secrets/
/prole-auth/target/surefire-reports/org.prole.auth.session.SessionTokenServiceTest.txt
/prole-auth/target/surefire-reports/org.prole.auth.web.VerifyControllerTest.txt
/prole-db.iml

View File

@ -1,5 +1,5 @@
; Prole Master Configuration File
; Generated by install.py on 2026-03-22 22:05:39
; Generated by install.py on 2026-03-23 00:22:54
; This file is used as input for Ansible deployment and k8s cluster creation.
[User]
@ -36,7 +36,7 @@ dependencies.opentofu.install = true
dependencies.python.install = true
dependencies.verify_all = false
disk_selection.disk_type = local
disk_selection.local_path = /Users/chrisfu
disk_selection.local_path = /Users/chrisfu/dev/prole/prole-tools-app/dist
disk_selection.removable_mount =
env_setup.NAMESPACE = ${NAMESPACE}
env_setup.PROLE_CONF = /Users/chrisfu/dev/prole/conf
@ -49,9 +49,9 @@ init_cluster.at_rest_encryption_enabled = true
init_cluster.cluster_env = dev
init_cluster.deployment_target = prole-dev-cluster
init_cluster.gitops_enabled = false
init_cluster.k3s_server_url =
init_cluster.k3s_token =
init_cluster.kerberos_enabled = false
init_cluster.k3s_server_url = https://myrddin.prole.org:6443
init_cluster.k3s_token = ${PROLE_SECRET:v1:ieZKRwYG_3X8vbBc:nQXXeKdQOvurVrt3nhCVFVVbqZ1dnOkGfuAp9V0R3-Ja11csMEum9VKBCSQe0GmxVW6LagYb2o7JOjlfVFwxo5X7PVWLHr4OjJuXVzcPxcFvgtd8YmTmd7G32dEif9wFiJdn8mplO2N5164O3xU8kQXhABZAlZiYdpK-qw==}
init_cluster.kerberos_enabled = true
init_cluster.mode = k3d
init_cluster.start_cluster = true
init_cluster.supabase_enabled = false
@ -60,35 +60,35 @@ init_cnpg_deploy.run_deploy = true
init_db_build.run_build = true
init_password.db_host_port = 5432
init_password.db_namespace = ${NAMESPACE}
init_password.db_password = ${PROLE_SECRET:v1:E5nbxsZ--UVdRi5n:NkN4_olrIDaryH3Oj6wC7vxfCgqEydRQ}
init_password.db_password_confirm = ${PROLE_SECRET:v1:E5nbxsZ--UVdRi5n:NkN4_olrIDaryH3Oj6wC7vxfCgqEydRQ}
init_password.db_password = ${PROLE_SECRET:v1:gvTjU-AbhUM7wJkS:OgxaEBO6RolGjSYrp2Up9FfqPZYQp7m1}
init_password.db_password_confirm = ${PROLE_SECRET:v1:gvTjU-AbhUM7wJkS:OgxaEBO6RolGjSYrp2Up9FfqPZYQp7m1}
init_password.db_username = root
init_password.generate_ssh_key = true
init_scripts.run_scripts = true
kerberos_config.enabled = false
kerberos_config.kdc =
kerberos_config.password = ${PROLE_SECRET:v1:jytT_kzMGIUznT7m:P34n5fDmRDmoYSBjZNaPQHR3ikuuT-9t7XnJLmEsTa3IznBJ}
kerberos_config.realm =
kerberos_config.enabled = true
kerberos_config.kdc = 10.0.0.3
kerberos_config.password = ${PROLE_SECRET:v1:995AGhbJI31IMHSa:DxdKatrMw70ZRxwPZeyvMpaom_Im9Nm-tFTg1iLzN-E9cu9V}
kerberos_config.realm = PROLE.ORG
kerberos_config.test_connection = false
kerberos_config.user = administrator
network_scan.run = true
ollama_config.model =
ollama_config.server_host =
ollama_config.server_host = fairyland.prole.org
ollama_config.server_port = 11434
[Global]
; Variables used by name in more than one place or assumed global scope
CLUSTER_ENV = dev
DB_HOST_PORT = 5432
DB_PASSWORD = ${PROLE_SECRET:v1:K4zAWM5pKjHVl1eN:IBLdjB7qsfbNWKsKdrMgzxy4gPp5UoyI}
DB_PASSWORD = ${PROLE_SECRET:v1:4yDE4I_OLZ1yXDAw:JGnP7OKExvxN6CC7GUn95wbm5nXirdEV}
DEPLOYMENT_MODE = k3d
DEPLOYMENT_TARGET = prole-dev-cluster
DOCKER_PRELOAD = false
KNOE_DB_USER = root
NAMESPACE = knoe-db-17-7-043
NAMESPACE = knoe-db-17-7-043-18-140
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
PROLE_K3S_SERVER =
PROLE_K3S_TOKEN =
PROLE_K3S_SERVER = https://myrddin.prole.org:6443
PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:bDmf-jKDTyBHJeKW:LN48-b1j7A9QzN3PG__U-UBtO7sJJEsdSER5YhNAX0AtNfS0e7YHQi9JwUfd-OAf02bmblvypaHnQLtSUmkoR85aEq9nFKsZKfWJncGfp38YIgQHskjyoz25IShamqJe1JYWuw2didiGix5EX70823_KGpcArxQdSnc8vA==}
PROLE_OPENTOFU_URL = http://127.0.0.1:8080
SERVICE_NAMESPACE = knoe-system
@ -107,12 +107,18 @@ ANSIBLE_INVENTORY = /Users/chrisfu/dev/prole/infrastructure/inventory
ANSIBLE_REALM = PROLE.ORG
ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
KDC_ANSIBLE_DETECTED = 10.0.0.3
KDC_AUTO_DETECTED = 10.0.0.3
KERBEROS_AUTO_ENABLED = True
[Port Forwards]
; No configuration values captured yet for this section.
[System Environment]
; No configuration values captured yet for this section.
PROLE_CONF = /Users/chrisfu/dev/prole/conf
PROLE_DATA = /Users/chrisfu/dev/prole/data
PROLE_HOME = /Users/chrisfu/dev/prole
PROLE_LOGS = /Users/chrisfu/dev/prole/logs
PROLE_SERVICE = /Users/chrisfu/dev/prole/etc
[Monitoring]
; No configuration values captured yet for this section.
@ -121,10 +127,16 @@ KDC_ANSIBLE_DETECTED = 10.0.0.3
; No configuration values captured yet for this section.
[Ollama]
; No configuration values captured yet for this section.
OLLAMA_HOST = http://fairyland.prole.org:11434
OLLAMA_SERVERS = fairyland.prole.org:11434,k3d.localhost:11434,morgoth.prole.org:11434
OLLAMA_SERVER_HOST = fairyland.prole.org
OLLAMA_SERVER_PORT = 11434
[Optional Features]
AT_REST_ENCRYPTION_ENABLED = True
GITOPS_ENABLED = False
KERBEROS_ENABLED = True
SUPABASE_ENABLED = False
[GitOps]
; No configuration values captured yet for this section.
@ -133,7 +145,9 @@ GITOPS_ENABLED = False
; No configuration values captured yet for this section.
[Initialize Cluster]
; No configuration values captured yet for this section.
ENVIRONMENT = dev
K3S_SERVER_URL = https://myrddin.prole.org:6443
K3S_TOKEN = ${PROLE_SECRET:v1:_K1lCt2nPLJUd00r:ri7UOTkKGSCpASmswpQsVYcKwcW3LtyXkVWCs1HkW0UBv5OWkArCO7A2CBru514YwTZDNPjbjvk064rpNYI3xg6DxnjK6NiwMa8iKXT7h2tU9dQ8LJ9aOy5N3tETBWqCgaLQuie7brzPPW7CfIjcH_kTolZ3CWzSx53qFQ==}
[Dev Cluster (k3d)]
CLUSTER_ENV = dev
@ -144,23 +158,24 @@ MODE = k3d
[Service Cluster (k3s)]
CLUSTER_ENV = prole-service-cluster
DISPLAY_NAME = prole-service-cluster
K3S_SERVER_URL =
K3S_TOKEN =
K3S_SERVER_URL = https://myrddin.prole.org:6443
K3S_TOKEN = ${PROLE_SECRET:v1:mYAO5AuJd_xh2Eaj:HbkWk_iLARDTEXlsidb5TBjEhMEqo0tzTZB9Hx7uaBi8sLqxUiEsABQBIEmNBJMapz5F3n9KloNfoNLcPvYDeEKI3B7SlyOVWpq6CVfUkLZn3Ndq4bPlrdoyc9v-5TdD_vGHUSAFJ-kgCdzKRRsNmbHtrT0V3vd9hfUZJQ==}
MODE = k3s
PIPELINE_URL = http://127.0.0.1:8080
[Prod Cluster (k8s)]
ARTIFACTS_DIR = /private/var/folders/rt/pywlnmxj3dn7t5552vwdcpp80000gn/T/pytest-of-chrisfu/pytest-69/test_encryption_toggle0/data/staging
ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging
CLUSTER_ENV = prole-prod-cluster
DISPLAY_NAME = prole-prod-cluster
MODE = k8s
PIPELINE_URL = http://127.0.0.1:8080
[Docker Build]
; No configuration values captured yet for this section.
LOCAL_REGISTRY = localhost:5000
LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000
[Initialization Scripts]
; No configuration values captured yet for this section.
STATUS = Attempted
[Deployment]
MODE = k3d

View File

@ -1,3 +1,11 @@
# Port mappings for Prole Tools (generated).
# Format: key: local=... remote=... ns=... svc=... address=...
argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0
garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0
openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=127.0.0.1
opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0
dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1
postgres: local=5432 remote=5432 ns=knoe-db-17-7-043-18-140 svc=knoe-db-rw address=0.0.0.0
prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1
grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0

View File

@ -214,6 +214,39 @@ ensure_prole_protected_storage() {
return 0
}
apply_cnpg_selector_overrides() {
local rendered_manifest="$1"
if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
return 0
fi
if [[ ! -f "$rendered_manifest" ]]; then
echo "ERROR: Rendered CNPG manifest not found for selector override: $rendered_manifest" >&2
return 1
fi
local data_selector_json="${PROLE_CNPG_DATA_SELECTOR_JSON:-}"
local wal_selector_json="${PROLE_CNPG_WAL_SELECTOR_JSON:-}"
if [[ -z "$data_selector_json" || -z "$wal_selector_json" ]]; then
return 0
fi
local patch_json patched
if ! patch_json=$(jq -cn --argjson data "$data_selector_json" --argjson wal "$wal_selector_json" '{"spec":{"storage":{"pvcTemplate":{"selector":{"matchLabels":$data}}},"walStorage":{"pvcTemplate":{"selector":{"matchLabels":$wal}}}}}'); then
echo "ERROR: Invalid CNPG selector JSON provided by installer (data/wal)." >&2
return 1
fi
if ! patched=$(kubectl patch --local -f "$rendered_manifest" --type merge -p "$patch_json" -o yaml 2>/dev/null); then
echo "ERROR: Failed to inject CNPG PVC selector overrides into rendered manifest." >&2
return 1
fi
printf '%s\n' "$patched" > "$rendered_manifest"
return 0
}
validate_cnpg_manifest_storage() {
local rendered_manifest="$1"
@ -232,7 +265,9 @@ validate_cnpg_manifest_storage() {
return 1
fi
local expected_ns="$NAMESPACE" expected_cluster="$CNPG_CLUSTER_NAME"
local in_cluster=0 block="" data_sc="" wal_sc="" data_role="" wal_role=""
local data_ns="" wal_ns="" data_cluster="" wal_cluster=""
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$line" =~ ^---[[:space:]]*$ ]]; then
in_cluster=0
@ -280,6 +315,24 @@ validate_cnpg_manifest_storage() {
wal_role="$r"
fi
fi
if [[ -n "$block" && "$line" =~ prole\.io/namespace: ]]; then
local ns_v
ns_v=$(printf '%s' "$line" | sed -E 's/^.*prole\.io\/namespace:[[:space:]]*//; s/[[:space:]]+$//')
if [[ "$block" == "storage" && -z "$data_ns" ]]; then
data_ns="$ns_v"
elif [[ "$block" == "wal" && -z "$wal_ns" ]]; then
wal_ns="$ns_v"
fi
fi
if [[ -n "$block" && "$line" =~ prole\.io/cluster: ]]; then
local c_v
c_v=$(printf '%s' "$line" | sed -E 's/^.*prole\.io\/cluster:[[:space:]]*//; s/[[:space:]]+$//')
if [[ "$block" == "storage" && -z "$data_cluster" ]]; then
data_cluster="$c_v"
elif [[ "$block" == "wal" && -z "$wal_cluster" ]]; then
wal_cluster="$c_v"
fi
fi
done < "$rendered_manifest"
if [[ "$data_sc" != "$PROLE_CNPG_STORAGE_CLASS" || "$wal_sc" != "$PROLE_CNPG_STORAGE_CLASS" ]]; then
@ -294,6 +347,18 @@ validate_cnpg_manifest_storage() {
return 1
fi
if [[ "$data_ns" != "$expected_ns" || "$wal_ns" != "$expected_ns" ]]; then
echo "ERROR: CNPG Cluster manifest must include namespace-scoped selectors (prole.io/namespace=${expected_ns}) for both data and wal." >&2
echo "Got selector namespaces: data='${data_ns:-<missing>}', wal='${wal_ns:-<missing>}'" >&2
return 1
fi
if [[ "$data_cluster" != "$expected_cluster" || "$wal_cluster" != "$expected_cluster" ]]; then
echo "ERROR: CNPG Cluster manifest must include cluster-scoped selectors (prole.io/cluster=${expected_cluster}) for both data and wal." >&2
echo "Got selector clusters: data='${data_cluster:-<missing>}', wal='${wal_cluster:-<missing>}'" >&2
return 1
fi
return 0
}
@ -310,11 +375,15 @@ validate_cnpg_runtime_storage() {
return 1
fi
local data_sc wal_sc data_role wal_role
local data_sc wal_sc data_role wal_role data_ns wal_ns data_cluster wal_cluster
data_sc=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.storageClassName // empty')
wal_sc=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.storageClassName // empty')
data_role=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.selector.matchLabels["synology.storage/role"] // empty')
wal_role=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.selector.matchLabels["synology.storage/role"] // empty')
data_ns=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.selector.matchLabels["prole.io/namespace"] // empty')
wal_ns=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.selector.matchLabels["prole.io/namespace"] // empty')
data_cluster=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.selector.matchLabels["prole.io/cluster"] // empty')
wal_cluster=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.selector.matchLabels["prole.io/cluster"] // empty')
if [[ "$data_sc" != "$sc" || "$wal_sc" != "$sc" ]]; then
echo "ERROR: Live CNPG Cluster storageClassName must be '${sc}' for both data and walStorage (got data='${data_sc:-<missing>}', wal='${wal_sc:-<missing>}')." >&2
@ -324,6 +393,14 @@ validate_cnpg_runtime_storage() {
echo "ERROR: Live CNPG Cluster must select Prole PVs via selector labels (got data role='${data_role:-<missing>}', wal role='${wal_role:-<missing>}')." >&2
return 1
fi
if [[ "$data_ns" != "$NAMESPACE" || "$wal_ns" != "$NAMESPACE" ]]; then
echo "ERROR: Live CNPG Cluster must use namespace-scoped selectors prole.io/namespace='${NAMESPACE}' (got data='${data_ns:-<missing>}', wal='${wal_ns:-<missing>}')." >&2
return 1
fi
if [[ "$data_cluster" != "$CNPG_CLUSTER_NAME" || "$wal_cluster" != "$CNPG_CLUSTER_NAME" ]]; then
echo "ERROR: Live CNPG Cluster must use cluster-scoped selectors prole.io/cluster='${CNPG_CLUSTER_NAME}' (got data='${data_cluster:-<missing>}', wal='${wal_cluster:-<missing>}')." >&2
return 1
fi
# Wait for CNPG PVCs to exist so we can validate binding and PV paths.
local timeout=${CNPG_STORAGE_VALIDATE_TIMEOUT:-120}
@ -2255,6 +2332,10 @@ apply_cnpg_cluster_manifest() {
prole_render_manifest "$manifest" >"$tmp"
patch_cnpg_manifest_for_topology "$tmp" || true
if ! apply_cnpg_selector_overrides "$tmp"; then
rm -f "$tmp"
return 1
fi
if ! validate_cnpg_manifest_storage "$tmp"; then
rm -f "$tmp"

View File

@ -92,6 +92,11 @@ from knoe.core.policy import (
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES,
evaluate_optional_workloads_allowed,
)
from knoe.core.ops.storage import (
ClusterStorageSpec,
StorageProvisioningError,
provision_cluster_storage,
)
def _configure_unbuffered_io():
@ -1736,6 +1741,66 @@ class ProleInstaller:
return placement_plan, plan_path
def _cnpg_storage_node(self, env: dict[str, str]) -> str:
stage1 = str(env.get("CNPG_STAGE1_NODE") or "").strip()
if stage1:
return stage1
eligible_raw = str(env.get("CNPG_PLACEMENT_ELIGIBLE_NODES") or "").strip()
if eligible_raw:
for part in eligible_raw.split(","):
node = str(part or "").strip()
if node:
return node
candidates = self._cnpg_candidate_nodes()
if candidates:
return candidates[0]
raise StorageProvisioningError(
"Unable to determine CNPG node for local PV provisioning. "
"Expected CNPG placement env (CNPG_STAGE1_NODE/CNPG_PLACEMENT_ELIGIBLE_NODES) "
"or configured CNPG candidate nodes."
)
def _ensure_cnpg_storage_provisioned(self, namespace: str, env: dict[str, str]) -> None:
if self._deployment_mode() != "k3s":
return
cluster_name = str(env.get("CNPG_CLUSTER_NAME") or self._cnpg_cluster_name()).strip()
if not cluster_name:
cluster_name = "knoe-db"
node_name = self._cnpg_storage_node(env)
spec = ClusterStorageSpec(
namespace=namespace,
cluster_name=cluster_name,
node_name=node_name,
storage_class_name="synology-iscsi",
command_env=env,
)
provisioned = provision_cluster_storage(spec)
env["PROLE_CNPG_STORAGE_CLASS"] = spec.storage_class_name
env["PROLE_CNPG_DATA_SELECTOR_JSON"] = json.dumps(
provisioned.data_selector,
separators=(",", ":"),
sort_keys=True,
)
env["PROLE_CNPG_WAL_SELECTOR_JSON"] = json.dumps(
provisioned.wal_selector,
separators=(",", ":"),
sort_keys=True,
)
env["PROLE_CNPG_DATA_PV_NAME"] = provisioned.data_pv_name
env["PROLE_CNPG_WAL_PV_NAME"] = provisioned.wal_pv_name
self.log(
"[CNPG] Provisioned cluster storage "
f"namespace={namespace} cluster={cluster_name} node={node_name} "
f"data_pv={provisioned.data_pv_name} wal_pv={provisioned.wal_pv_name}"
)
# --------------------------------------------- authority / repair
def _authority_context_missing(self) -> bool:
enabled = self._get_input_bool(
@ -5347,6 +5412,17 @@ class ProleConsoleInstaller(ProleInstaller):
self._get_input("init_password.db_namespace", "") or ""
).strip() or "default"
env = self._script_env_for_namespace(ns)
cluster_name = str(env.get("CNPG_CLUSTER_NAME") or self._cnpg_cluster_name()).strip()
if not cluster_name:
cluster_name = "knoe-db"
try:
self._ensure_cnpg_storage_provisioned(ns, env)
except Exception as exc:
self._cnpg_success = False
self.prole_cfg_data["Deployment"]["STATUS"] = "Attempted"
self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}")
return
rc = self._run_cmd(
["bash", str(etc_dir / "init_cloudnative_pg.sh"), "deploy", "latest"],
@ -5373,7 +5449,7 @@ class ProleConsoleInstaller(ProleInstaller):
"-n",
ns,
"-l",
f"cnpg.io/cluster=knoe-db",
f"cnpg.io/cluster={cluster_name}",
"--no-headers",
],
capture_output=True,

View File

@ -0,0 +1 @@
"""Core operational helpers for installer actions."""

476
knoe/core/ops/storage.py Normal file
View File

@ -0,0 +1,476 @@
from __future__ import annotations
import hashlib
import json
import os
import re
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import Any
DEFAULT_SYNOLOGY_ROOTS: tuple[str, ...] = (
"/synology/d001",
"/synology/d002",
"/synology/d003",
"/synology/d004",
)
class StorageProvisioningError(RuntimeError):
pass
@dataclass(slots=True, frozen=True)
class ClusterStorageSpec:
namespace: str
cluster_name: str
node_name: str
storage_class_name: str = "synology-iscsi"
service_name: str = "knoe-db"
root_namespace: str = "knoe"
postgres_uid: int = 100
postgres_gid: int = 101
directory_mode: int = 0o700
volume_capacity: str = "29Gi"
synology_roots: tuple[str, ...] = DEFAULT_SYNOLOGY_ROOTS
kubectl_bin: str = "kubectl"
mount_check_bin: str = "findmnt"
command_env: dict[str, str] | None = None
@dataclass(slots=True, frozen=True)
class ClusterStoragePaths:
data_root: str
wal_root: str
data_volume: str
wal_volume: str
data_path: str
wal_path: str
@dataclass(slots=True, frozen=True)
class ProvisionedStorage:
data_pv_name: str
wal_pv_name: str
data_path: str
wal_path: str
data_selector: dict[str, str]
wal_selector: dict[str, str]
def _sanitize_dns_label(value: str, max_len: int = 253) -> str:
lowered = (value or "").strip().lower()
cleaned = re.sub(r"[^a-z0-9.-]+", "-", lowered)
cleaned = re.sub(r"-+", "-", cleaned).strip("-.")
if not cleaned:
cleaned = "pv"
if len(cleaned) <= max_len:
return cleaned
digest = hashlib.sha1(cleaned.encode("utf-8")).hexdigest()[:8]
head = cleaned[: max_len - 9].rstrip("-.")
return f"{head}-{digest}"
def _normalize_posix_path(path: str) -> str:
parts = [p for p in path.split("/") if p]
return "/" + "/".join(parts)
def _paths_overlap(path_a: str, path_b: str) -> bool:
a = _normalize_posix_path(path_a)
b = _normalize_posix_path(path_b)
return a == b or a.startswith(f"{b}/") or b.startswith(f"{a}/")
def _command_env(spec: ClusterStorageSpec) -> dict[str, str] | None:
return dict(spec.command_env) if spec.command_env else None
def _run_command(
args: list[str],
*,
env: dict[str, str] | None = None,
input_text: str | None = None,
) -> subprocess.CompletedProcess[str]:
return subprocess.run(
args,
capture_output=True,
text=True,
env=env,
input=input_text,
)
def _is_non_root_mount(path: Path, *, mount_check_bin: str, env: dict[str, str] | None) -> bool:
if not path.is_dir():
return False
check = _run_command([mount_check_bin, "-T", str(path), "-n", "-o", "TARGET,SOURCE"], env=env)
if check.returncode != 0:
return bool(os.path.ismount(path))
root_check = _run_command([mount_check_bin, "-T", "/", "-n", "-o", "SOURCE"], env=env)
root_source = (root_check.stdout or "").strip() if root_check.returncode == 0 else ""
parts = (check.stdout or "").strip().split()
if len(parts) < 2:
return bool(os.path.ismount(path))
target, source = parts[0], parts[1]
if target == "/":
return False
if root_source and source == root_source:
return False
return True
def _ensure_valid_identity(value: str, field_name: str) -> str:
val = (value or "").strip()
if not val:
raise StorageProvisioningError(f"Missing required storage identity field: {field_name}")
return val
def _volume_id_from_root(root: Path) -> str:
return _sanitize_dns_label(root.name, max_len=32)
def _choose_root_pair(spec: ClusterStorageSpec) -> tuple[Path, Path]:
env = _command_env(spec)
mounted_roots: list[Path] = []
for root_str in spec.synology_roots:
root = Path(root_str)
if _is_non_root_mount(root, mount_check_bin=spec.mount_check_bin, env=env):
mounted_roots.append(root)
if not mounted_roots:
raise StorageProvisioningError(
"No mounted Synology roots available. Expected mounted paths like /synology/d001..d004."
)
ident = f"{spec.namespace}:{spec.cluster_name}:{spec.service_name}"
digest = int(hashlib.sha256(ident.encode("utf-8")).hexdigest(), 16)
data_index = digest % len(mounted_roots)
data_root = mounted_roots[data_index]
if len(mounted_roots) == 1:
wal_root = data_root
else:
wal_root = mounted_roots[(data_index + 1) % len(mounted_roots)]
return data_root, wal_root
def build_cluster_storage_paths(spec: ClusterStorageSpec) -> ClusterStoragePaths:
namespace = _ensure_valid_identity(spec.namespace, "namespace")
cluster_name = _ensure_valid_identity(spec.cluster_name, "cluster_name")
data_root, wal_root = _choose_root_pair(spec)
base_rel = Path(spec.root_namespace) / spec.service_name / namespace / cluster_name
data_path = data_root / base_rel / "data"
wal_path = wal_root / base_rel / "wal"
if _paths_overlap(str(data_path), str(wal_path)):
raise StorageProvisioningError(
f"Refusing overlapping storage paths: data='{data_path}', wal='{wal_path}'"
)
return ClusterStoragePaths(
data_root=str(data_root),
wal_root=str(wal_root),
data_volume=_volume_id_from_root(data_root),
wal_volume=_volume_id_from_root(wal_root),
data_path=str(data_path),
wal_path=str(wal_path),
)
def build_pv_name(storage_class_name: str, namespace: str, cluster_name: str, role: str) -> str:
base = f"{storage_class_name}-{namespace}-{cluster_name}-{role}"
return _sanitize_dns_label(base)
def build_pv_labels(
*,
namespace: str,
cluster_name: str,
role: str,
volume: str,
service_name: str = "knoe-db",
) -> dict[str, str]:
return {
"synology.storage/role": role,
"synology.storage/volume": volume,
"prole.io/namespace": namespace,
"prole.io/cluster": cluster_name,
"prole.io/service": service_name,
}
def render_local_pv(
*,
pv_name: str,
host_path: str,
node_name: str,
storage_class_name: str,
capacity: str,
labels: dict[str, str],
) -> dict[str, Any]:
return {
"apiVersion": "v1",
"kind": "PersistentVolume",
"metadata": {
"name": pv_name,
"labels": labels,
},
"spec": {
"capacity": {"storage": capacity},
"volumeMode": "Filesystem",
"accessModes": ["ReadWriteOnce"],
"storageClassName": storage_class_name,
"persistentVolumeReclaimPolicy": "Retain",
"local": {"path": host_path},
"nodeAffinity": {
"required": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "kubernetes.io/hostname",
"operator": "In",
"values": [node_name],
}
]
}
]
}
},
},
}
def ensure_host_path(path: str, *, uid: int, gid: int, mode: int = 0o700) -> None:
target = Path(path)
if target.exists() and not target.is_dir():
raise StorageProvisioningError(f"Host path exists but is not a directory: {path}")
target.mkdir(parents=True, exist_ok=True)
os.chmod(target, mode)
try:
os.chown(target, uid, gid)
except PermissionError as exc:
raise StorageProvisioningError(
f"Cannot set ownership on host path '{path}' to {uid}:{gid}: {exc}"
) from exc
def _kubectl_get_json(spec: ClusterStorageSpec, args: list[str]) -> dict[str, Any] | None:
env = _command_env(spec)
cmd = [spec.kubectl_bin, *args, "-o", "json"]
res = _run_command(cmd, env=env)
if res.returncode != 0:
return None
try:
return json.loads(res.stdout or "{}")
except json.JSONDecodeError as exc:
raise StorageProvisioningError(
f"Failed to parse kubectl JSON output for command: {' '.join(cmd)}"
) from exc
def _kubectl_apply_manifest(spec: ClusterStorageSpec, manifest: dict[str, Any]) -> None:
env = _command_env(spec)
payload = json.dumps(manifest)
res = _run_command([spec.kubectl_bin, "apply", "-f", "-"], env=env, input_text=payload)
if res.returncode != 0:
stderr = (res.stderr or "").strip()
raise StorageProvisioningError(f"Failed to apply PV manifest '{manifest['metadata']['name']}': {stderr}")
def _collect_existing_pvs(spec: ClusterStorageSpec) -> dict[str, dict[str, Any]]:
pv_list = _kubectl_get_json(spec, ["get", "pv"])
if pv_list is None:
raise StorageProvisioningError("Unable to list existing PersistentVolumes via kubectl.")
items = pv_list.get("items") or []
return {str(item.get("metadata", {}).get("name") or ""): item for item in items}
def _extract_node_names(pv_obj: dict[str, Any]) -> set[str]:
names: set[str] = set()
terms = (
pv_obj.get("spec", {})
.get("nodeAffinity", {})
.get("required", {})
.get("nodeSelectorTerms", [])
)
for term in terms:
for expr in term.get("matchExpressions", []) or []:
if expr.get("key") != "kubernetes.io/hostname":
continue
if expr.get("operator") != "In":
continue
for val in expr.get("values") or []:
if val:
names.add(str(val))
return names
def _validate_existing_pv(
*,
existing_pv: dict[str, Any],
expected_name: str,
expected_path: str,
expected_storage_class: str,
expected_node_name: str,
expected_labels: dict[str, str],
) -> None:
existing_name = str(existing_pv.get("metadata", {}).get("name") or "")
if existing_name != expected_name:
raise StorageProvisioningError(
f"Internal PV validation error: expected '{expected_name}', got '{existing_name}'"
)
current_path = str(existing_pv.get("spec", {}).get("local", {}).get("path") or "")
if _normalize_posix_path(current_path) != _normalize_posix_path(expected_path):
raise StorageProvisioningError(
f"PV '{expected_name}' exists with unexpected path '{current_path}' (expected '{expected_path}')."
)
current_sc = str(existing_pv.get("spec", {}).get("storageClassName") or "")
if current_sc != expected_storage_class:
raise StorageProvisioningError(
f"PV '{expected_name}' exists with unexpected storageClassName '{current_sc}' (expected '{expected_storage_class}')."
)
current_labels = existing_pv.get("metadata", {}).get("labels") or {}
for key, expected in expected_labels.items():
actual = str(current_labels.get(key) or "")
if actual != expected:
raise StorageProvisioningError(
f"PV '{expected_name}' label mismatch for '{key}': got '{actual}', expected '{expected}'."
)
node_names = _extract_node_names(existing_pv)
if expected_node_name not in node_names:
raise StorageProvisioningError(
f"PV '{expected_name}' node affinity mismatch: expected node '{expected_node_name}', got {sorted(node_names)}."
)
def _ensure_no_path_overlap(
*,
existing_pvs: dict[str, dict[str, Any]],
pv_name: str,
target_path: str,
) -> None:
for existing_name, existing_pv in existing_pvs.items():
if existing_name == pv_name:
continue
existing_path = str(existing_pv.get("spec", {}).get("local", {}).get("path") or "").strip()
if not existing_path:
continue
if _paths_overlap(target_path, existing_path):
raise StorageProvisioningError(
f"PV path overlap detected: target '{target_path}' conflicts with existing PV '{existing_name}' path '{existing_path}'."
)
def _provision_one_pv(
*,
spec: ClusterStorageSpec,
existing_pvs: dict[str, dict[str, Any]],
role: str,
path: str,
volume: str,
) -> tuple[str, dict[str, str]]:
pv_name = build_pv_name(spec.storage_class_name, spec.namespace, spec.cluster_name, role)
labels = build_pv_labels(
namespace=spec.namespace,
cluster_name=spec.cluster_name,
role=role,
volume=volume,
service_name=spec.service_name,
)
_ensure_no_path_overlap(existing_pvs=existing_pvs, pv_name=pv_name, target_path=path)
existing = existing_pvs.get(pv_name)
if existing:
_validate_existing_pv(
existing_pv=existing,
expected_name=pv_name,
expected_path=path,
expected_storage_class=spec.storage_class_name,
expected_node_name=spec.node_name,
expected_labels=labels,
)
return pv_name, labels
manifest = render_local_pv(
pv_name=pv_name,
host_path=path,
node_name=spec.node_name,
storage_class_name=spec.storage_class_name,
capacity=spec.volume_capacity,
labels=labels,
)
_kubectl_apply_manifest(spec, manifest)
return pv_name, labels
def provision_cluster_storage(spec: ClusterStorageSpec) -> ProvisionedStorage:
spec = ClusterStorageSpec(
namespace=_ensure_valid_identity(spec.namespace, "namespace"),
cluster_name=_ensure_valid_identity(spec.cluster_name, "cluster_name"),
node_name=_ensure_valid_identity(spec.node_name, "node_name"),
storage_class_name=_ensure_valid_identity(spec.storage_class_name, "storage_class_name"),
service_name=_ensure_valid_identity(spec.service_name, "service_name"),
root_namespace=_ensure_valid_identity(spec.root_namespace, "root_namespace"),
postgres_uid=spec.postgres_uid,
postgres_gid=spec.postgres_gid,
directory_mode=spec.directory_mode,
volume_capacity=_ensure_valid_identity(spec.volume_capacity, "volume_capacity"),
synology_roots=spec.synology_roots,
kubectl_bin=spec.kubectl_bin,
mount_check_bin=spec.mount_check_bin,
command_env=spec.command_env,
)
paths = build_cluster_storage_paths(spec)
ensure_host_path(
paths.data_path,
uid=spec.postgres_uid,
gid=spec.postgres_gid,
mode=spec.directory_mode,
)
ensure_host_path(
paths.wal_path,
uid=spec.postgres_uid,
gid=spec.postgres_gid,
mode=spec.directory_mode,
)
existing_pvs = _collect_existing_pvs(spec)
data_pv_name, data_labels = _provision_one_pv(
spec=spec,
existing_pvs=existing_pvs,
role="data",
path=paths.data_path,
volume=paths.data_volume,
)
existing_pvs = _collect_existing_pvs(spec)
wal_pv_name, wal_labels = _provision_one_pv(
spec=spec,
existing_pvs=existing_pvs,
role="wal",
path=paths.wal_path,
volume=paths.wal_volume,
)
return ProvisionedStorage(
data_pv_name=data_pv_name,
wal_pv_name=wal_pv_name,
data_path=paths.data_path,
wal_path=paths.wal_path,
data_selector=dict(data_labels),
wal_selector=dict(wal_labels),
)

View File

@ -1 +1 @@
4
5

View File

@ -756,6 +756,82 @@ class TestStepInitScripts:
assert inst._scripts_success is True
# ---------------------------------------------------------------------------
# _step_cnpg_deploy
# ---------------------------------------------------------------------------
class TestStepCnpgDeploy:
def test_provisions_storage_before_cnpg_deploy(self, monkeypatch):
inst = _TestableSilentInstaller(
inputs={
"init_cnpg_deploy.run_deploy": "true",
"init_cnpg_deploy.force_rollout": "false",
"init_password.db_namespace": "knoe-db",
}
)
inst.prole_cfg_data.setdefault("Deployment", {})
env = {"CNPG_CLUSTER_NAME": "cluster-a"}
monkeypatch.setattr(inst, "_script_env_for_namespace", lambda _ns: env)
calls = {"provision": 0, "run": 0}
def _fake_provision(ns, env_arg):
calls["provision"] += 1
assert ns == "knoe-db"
assert env_arg is env
env_arg["PROLE_CNPG_DATA_SELECTOR_JSON"] = '{"synology.storage/role":"data"}'
env_arg["PROLE_CNPG_WAL_SELECTOR_JSON"] = '{"synology.storage/role":"wal"}'
def _fake_run(cmd, env=None, **_kwargs):
calls["run"] += 1
assert "init_cloudnative_pg.sh" in str(cmd[1])
assert env is not None
assert env["PROLE_CNPG_DATA_SELECTOR_JSON"]
assert env["PROLE_CNPG_WAL_SELECTOR_JSON"]
return 0
monkeypatch.setattr(inst, "_ensure_cnpg_storage_provisioned", _fake_provision)
monkeypatch.setattr(inst, "_run_cmd", _fake_run)
inst._step_cnpg_deploy()
assert calls["provision"] == 1
assert calls["run"] == 1
assert inst._cnpg_success is True
assert inst.prole_cfg_data["Deployment"]["STATUS"] == "Deployed"
def test_stops_deploy_on_storage_provision_failure(self, monkeypatch):
inst = _TestableSilentInstaller(
inputs={
"init_cnpg_deploy.run_deploy": "true",
"init_cnpg_deploy.force_rollout": "false",
"init_password.db_namespace": "knoe-db",
}
)
inst.prole_cfg_data.setdefault("Deployment", {})
monkeypatch.setattr(
inst,
"_script_env_for_namespace",
lambda _ns: {"CNPG_CLUSTER_NAME": "cluster-a"},
)
monkeypatch.setattr(
inst,
"_ensure_cnpg_storage_provisioned",
lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("boom")),
)
run_mock = MagicMock(return_value=0)
monkeypatch.setattr(inst, "_run_cmd", run_mock)
inst._step_cnpg_deploy()
run_mock.assert_not_called()
assert inst._cnpg_success is False
assert inst.prole_cfg_data["Deployment"]["STATUS"] == "Attempted"
# ---------------------------------------------------------------------------
# _prepare_opentofu_pipeline
# ---------------------------------------------------------------------------

View File

@ -0,0 +1,189 @@
from __future__ import annotations
import pytest
from knoe.core.ops import storage as storage_ops
def test_build_cluster_storage_paths_prefers_distinct_roots(monkeypatch):
monkeypatch.setattr(
storage_ops,
"_is_non_root_mount",
lambda path, **_kwargs: str(path) in {"/synology/d001", "/synology/d002"},
)
spec = storage_ops.ClusterStorageSpec(
namespace="knoe-db",
cluster_name="cluster-a",
node_name="myrddin.prole.org",
synology_roots=("/synology/d001", "/synology/d002"),
)
paths = storage_ops.build_cluster_storage_paths(spec)
assert paths.data_root != paths.wal_root
assert paths.data_path.endswith("/knoe/knoe-db/knoe-db/cluster-a/data")
assert paths.wal_path.endswith("/knoe/knoe-db/knoe-db/cluster-a/wal")
def test_build_cluster_storage_paths_falls_back_to_same_root(monkeypatch):
monkeypatch.setattr(
storage_ops,
"_is_non_root_mount",
lambda path, **_kwargs: str(path) == "/synology/d001",
)
spec = storage_ops.ClusterStorageSpec(
namespace="knoe-db",
cluster_name="cluster-a",
node_name="myrddin.prole.org",
synology_roots=("/synology/d001", "/synology/d002"),
)
paths = storage_ops.build_cluster_storage_paths(spec)
assert paths.data_root == "/synology/d001"
assert paths.wal_root == "/synology/d001"
assert paths.data_path != paths.wal_path
def test_build_pv_labels_include_cluster_scope():
labels = storage_ops.build_pv_labels(
namespace="knoe-db",
cluster_name="cluster-a",
role="data",
volume="d001",
)
assert labels["synology.storage/role"] == "data"
assert labels["synology.storage/volume"] == "d001"
assert labels["prole.io/namespace"] == "knoe-db"
assert labels["prole.io/cluster"] == "cluster-a"
assert labels["prole.io/service"] == "knoe-db"
def test_provision_cluster_storage_fails_on_existing_name_mismatch(monkeypatch):
spec = storage_ops.ClusterStorageSpec(
namespace="knoe-db",
cluster_name="cluster-a",
node_name="myrddin.prole.org",
)
paths = storage_ops.ClusterStoragePaths(
data_root="/synology/d001",
wal_root="/synology/d002",
data_volume="d001",
wal_volume="d002",
data_path="/synology/d001/knoe/knoe-db/knoe-db/cluster-a/data",
wal_path="/synology/d002/knoe/knoe-db/knoe-db/cluster-a/wal",
)
data_name = storage_ops.build_pv_name("synology-iscsi", "knoe-db", "cluster-a", "data")
monkeypatch.setattr(storage_ops, "build_cluster_storage_paths", lambda _spec: paths)
monkeypatch.setattr(storage_ops, "ensure_host_path", lambda *args, **kwargs: None)
existing = {
data_name: {
"metadata": {
"name": data_name,
"labels": storage_ops.build_pv_labels(
namespace="knoe-db",
cluster_name="cluster-a",
role="data",
volume="d001",
),
},
"spec": {
"storageClassName": "synology-iscsi",
"local": {"path": "/synology/d001/other-path"},
"nodeAffinity": {
"required": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "kubernetes.io/hostname",
"operator": "In",
"values": ["myrddin.prole.org"],
}
]
}
]
}
},
},
}
}
monkeypatch.setattr(storage_ops, "_collect_existing_pvs", lambda _spec: existing)
with pytest.raises(storage_ops.StorageProvisioningError, match="unexpected path"):
storage_ops.provision_cluster_storage(spec)
def test_provision_cluster_storage_applies_and_returns_selectors(monkeypatch):
spec = storage_ops.ClusterStorageSpec(
namespace="knoe-db",
cluster_name="cluster-a",
node_name="myrddin.prole.org",
)
paths = storage_ops.ClusterStoragePaths(
data_root="/synology/d001",
wal_root="/synology/d002",
data_volume="d001",
wal_volume="d002",
data_path="/synology/d001/knoe/knoe-db/knoe-db/cluster-a/data",
wal_path="/synology/d002/knoe/knoe-db/knoe-db/cluster-a/wal",
)
monkeypatch.setattr(storage_ops, "build_cluster_storage_paths", lambda _spec: paths)
monkeypatch.setattr(storage_ops, "ensure_host_path", lambda *args, **kwargs: None)
monkeypatch.setattr(storage_ops, "_collect_existing_pvs", lambda _spec: {})
applied: list[dict] = []
def _capture_apply(_spec, manifest):
applied.append(manifest)
monkeypatch.setattr(storage_ops, "_kubectl_apply_manifest", _capture_apply)
provisioned = storage_ops.provision_cluster_storage(spec)
assert len(applied) == 2
assert provisioned.data_pv_name == "synology-iscsi-knoe-db-cluster-a-data"
assert provisioned.wal_pv_name == "synology-iscsi-knoe-db-cluster-a-wal"
assert provisioned.data_selector["prole.io/namespace"] == "knoe-db"
assert provisioned.data_selector["prole.io/cluster"] == "cluster-a"
assert provisioned.wal_selector["synology.storage/role"] == "wal"
def test_provision_cluster_storage_fails_on_path_overlap(monkeypatch):
spec = storage_ops.ClusterStorageSpec(
namespace="knoe-db",
cluster_name="cluster-a",
node_name="myrddin.prole.org",
)
paths = storage_ops.ClusterStoragePaths(
data_root="/synology/d001",
wal_root="/synology/d002",
data_volume="d001",
wal_volume="d002",
data_path="/synology/d001/knoe/knoe-db/knoe-db/cluster-a/data",
wal_path="/synology/d002/knoe/knoe-db/knoe-db/cluster-a/wal",
)
monkeypatch.setattr(storage_ops, "build_cluster_storage_paths", lambda _spec: paths)
monkeypatch.setattr(storage_ops, "ensure_host_path", lambda *args, **kwargs: None)
existing = {
"some-other-pv": {
"metadata": {"name": "some-other-pv", "labels": {}},
"spec": {
"storageClassName": "synology-iscsi",
"local": {"path": "/synology/d001/knoe/knoe-db/knoe-db/cluster-a"},
"nodeAffinity": {"required": {"nodeSelectorTerms": []}},
},
}
}
monkeypatch.setattr(storage_ops, "_collect_existing_pvs", lambda _spec: existing)
with pytest.raises(storage_ops.StorageProvisioningError, match="overlap"):
storage_ops.provision_cluster_storage(spec)