mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
feat: pre-provision cluster-scoped CNPG Synology PVs
Add storage ops for deterministic namespace+cluster local PV paths, labels/selectors, host-path preparation, and idempotent reconciliation before CNPG cluster apply. Wire selector injection and validation into CNPG deploy flow/script, and extend installer tests for provisioning orchestration and failure handling. Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
parent
59c8ea3f8f
commit
b09319ffaf
1
.gitignore
vendored
1
.gitignore
vendored
@ -76,3 +76,4 @@ htmlcov/
|
||||
/mock_val/secrets/
|
||||
/prole-auth/target/surefire-reports/org.prole.auth.session.SessionTokenServiceTest.txt
|
||||
/prole-auth/target/surefire-reports/org.prole.auth.web.VerifyControllerTest.txt
|
||||
/prole-db.iml
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
; Prole Master Configuration File
|
||||
; Generated by install.py on 2026-03-22 22:05:39
|
||||
; Generated by install.py on 2026-03-23 00:22:54
|
||||
; This file is used as input for Ansible deployment and k8s cluster creation.
|
||||
|
||||
[User]
|
||||
@ -36,7 +36,7 @@ dependencies.opentofu.install = true
|
||||
dependencies.python.install = true
|
||||
dependencies.verify_all = false
|
||||
disk_selection.disk_type = local
|
||||
disk_selection.local_path = /Users/chrisfu
|
||||
disk_selection.local_path = /Users/chrisfu/dev/prole/prole-tools-app/dist
|
||||
disk_selection.removable_mount =
|
||||
env_setup.NAMESPACE = ${NAMESPACE}
|
||||
env_setup.PROLE_CONF = /Users/chrisfu/dev/prole/conf
|
||||
@ -49,9 +49,9 @@ init_cluster.at_rest_encryption_enabled = true
|
||||
init_cluster.cluster_env = dev
|
||||
init_cluster.deployment_target = prole-dev-cluster
|
||||
init_cluster.gitops_enabled = false
|
||||
init_cluster.k3s_server_url =
|
||||
init_cluster.k3s_token =
|
||||
init_cluster.kerberos_enabled = false
|
||||
init_cluster.k3s_server_url = https://myrddin.prole.org:6443
|
||||
init_cluster.k3s_token = ${PROLE_SECRET:v1:ieZKRwYG_3X8vbBc:nQXXeKdQOvurVrt3nhCVFVVbqZ1dnOkGfuAp9V0R3-Ja11csMEum9VKBCSQe0GmxVW6LagYb2o7JOjlfVFwxo5X7PVWLHr4OjJuXVzcPxcFvgtd8YmTmd7G32dEif9wFiJdn8mplO2N5164O3xU8kQXhABZAlZiYdpK-qw==}
|
||||
init_cluster.kerberos_enabled = true
|
||||
init_cluster.mode = k3d
|
||||
init_cluster.start_cluster = true
|
||||
init_cluster.supabase_enabled = false
|
||||
@ -60,35 +60,35 @@ init_cnpg_deploy.run_deploy = true
|
||||
init_db_build.run_build = true
|
||||
init_password.db_host_port = 5432
|
||||
init_password.db_namespace = ${NAMESPACE}
|
||||
init_password.db_password = ${PROLE_SECRET:v1:E5nbxsZ--UVdRi5n:NkN4_olrIDaryH3Oj6wC7vxfCgqEydRQ}
|
||||
init_password.db_password_confirm = ${PROLE_SECRET:v1:E5nbxsZ--UVdRi5n:NkN4_olrIDaryH3Oj6wC7vxfCgqEydRQ}
|
||||
init_password.db_password = ${PROLE_SECRET:v1:gvTjU-AbhUM7wJkS:OgxaEBO6RolGjSYrp2Up9FfqPZYQp7m1}
|
||||
init_password.db_password_confirm = ${PROLE_SECRET:v1:gvTjU-AbhUM7wJkS:OgxaEBO6RolGjSYrp2Up9FfqPZYQp7m1}
|
||||
init_password.db_username = root
|
||||
init_password.generate_ssh_key = true
|
||||
init_scripts.run_scripts = true
|
||||
kerberos_config.enabled = false
|
||||
kerberos_config.kdc =
|
||||
kerberos_config.password = ${PROLE_SECRET:v1:jytT_kzMGIUznT7m:P34n5fDmRDmoYSBjZNaPQHR3ikuuT-9t7XnJLmEsTa3IznBJ}
|
||||
kerberos_config.realm =
|
||||
kerberos_config.enabled = true
|
||||
kerberos_config.kdc = 10.0.0.3
|
||||
kerberos_config.password = ${PROLE_SECRET:v1:995AGhbJI31IMHSa:DxdKatrMw70ZRxwPZeyvMpaom_Im9Nm-tFTg1iLzN-E9cu9V}
|
||||
kerberos_config.realm = PROLE.ORG
|
||||
kerberos_config.test_connection = false
|
||||
kerberos_config.user = administrator
|
||||
network_scan.run = true
|
||||
ollama_config.model =
|
||||
ollama_config.server_host =
|
||||
ollama_config.server_host = fairyland.prole.org
|
||||
ollama_config.server_port = 11434
|
||||
|
||||
[Global]
|
||||
; Variables used by name in more than one place or assumed global scope
|
||||
CLUSTER_ENV = dev
|
||||
DB_HOST_PORT = 5432
|
||||
DB_PASSWORD = ${PROLE_SECRET:v1:K4zAWM5pKjHVl1eN:IBLdjB7qsfbNWKsKdrMgzxy4gPp5UoyI}
|
||||
DB_PASSWORD = ${PROLE_SECRET:v1:4yDE4I_OLZ1yXDAw:JGnP7OKExvxN6CC7GUn95wbm5nXirdEV}
|
||||
DEPLOYMENT_MODE = k3d
|
||||
DEPLOYMENT_TARGET = prole-dev-cluster
|
||||
DOCKER_PRELOAD = false
|
||||
KNOE_DB_USER = root
|
||||
NAMESPACE = knoe-db-17-7-043
|
||||
NAMESPACE = knoe-db-17-7-043-18-140
|
||||
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
|
||||
PROLE_K3S_SERVER =
|
||||
PROLE_K3S_TOKEN =
|
||||
PROLE_K3S_SERVER = https://myrddin.prole.org:6443
|
||||
PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:bDmf-jKDTyBHJeKW:LN48-b1j7A9QzN3PG__U-UBtO7sJJEsdSER5YhNAX0AtNfS0e7YHQi9JwUfd-OAf02bmblvypaHnQLtSUmkoR85aEq9nFKsZKfWJncGfp38YIgQHskjyoz25IShamqJe1JYWuw2didiGix5EX70823_KGpcArxQdSnc8vA==}
|
||||
PROLE_OPENTOFU_URL = http://127.0.0.1:8080
|
||||
SERVICE_NAMESPACE = knoe-system
|
||||
|
||||
@ -107,12 +107,18 @@ ANSIBLE_INVENTORY = /Users/chrisfu/dev/prole/infrastructure/inventory
|
||||
ANSIBLE_REALM = PROLE.ORG
|
||||
ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
|
||||
KDC_ANSIBLE_DETECTED = 10.0.0.3
|
||||
KDC_AUTO_DETECTED = 10.0.0.3
|
||||
KERBEROS_AUTO_ENABLED = True
|
||||
|
||||
[Port Forwards]
|
||||
; No configuration values captured yet for this section.
|
||||
|
||||
[System Environment]
|
||||
; No configuration values captured yet for this section.
|
||||
PROLE_CONF = /Users/chrisfu/dev/prole/conf
|
||||
PROLE_DATA = /Users/chrisfu/dev/prole/data
|
||||
PROLE_HOME = /Users/chrisfu/dev/prole
|
||||
PROLE_LOGS = /Users/chrisfu/dev/prole/logs
|
||||
PROLE_SERVICE = /Users/chrisfu/dev/prole/etc
|
||||
|
||||
[Monitoring]
|
||||
; No configuration values captured yet for this section.
|
||||
@ -121,10 +127,16 @@ KDC_ANSIBLE_DETECTED = 10.0.0.3
|
||||
; No configuration values captured yet for this section.
|
||||
|
||||
[Ollama]
|
||||
; No configuration values captured yet for this section.
|
||||
OLLAMA_HOST = http://fairyland.prole.org:11434
|
||||
OLLAMA_SERVERS = fairyland.prole.org:11434,k3d.localhost:11434,morgoth.prole.org:11434
|
||||
OLLAMA_SERVER_HOST = fairyland.prole.org
|
||||
OLLAMA_SERVER_PORT = 11434
|
||||
|
||||
[Optional Features]
|
||||
AT_REST_ENCRYPTION_ENABLED = True
|
||||
GITOPS_ENABLED = False
|
||||
KERBEROS_ENABLED = True
|
||||
SUPABASE_ENABLED = False
|
||||
|
||||
[GitOps]
|
||||
; No configuration values captured yet for this section.
|
||||
@ -133,7 +145,9 @@ GITOPS_ENABLED = False
|
||||
; No configuration values captured yet for this section.
|
||||
|
||||
[Initialize Cluster]
|
||||
; No configuration values captured yet for this section.
|
||||
ENVIRONMENT = dev
|
||||
K3S_SERVER_URL = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:_K1lCt2nPLJUd00r:ri7UOTkKGSCpASmswpQsVYcKwcW3LtyXkVWCs1HkW0UBv5OWkArCO7A2CBru514YwTZDNPjbjvk064rpNYI3xg6DxnjK6NiwMa8iKXT7h2tU9dQ8LJ9aOy5N3tETBWqCgaLQuie7brzPPW7CfIjcH_kTolZ3CWzSx53qFQ==}
|
||||
|
||||
[Dev Cluster (k3d)]
|
||||
CLUSTER_ENV = dev
|
||||
@ -144,23 +158,24 @@ MODE = k3d
|
||||
[Service Cluster (k3s)]
|
||||
CLUSTER_ENV = prole-service-cluster
|
||||
DISPLAY_NAME = prole-service-cluster
|
||||
K3S_SERVER_URL =
|
||||
K3S_TOKEN =
|
||||
K3S_SERVER_URL = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:mYAO5AuJd_xh2Eaj:HbkWk_iLARDTEXlsidb5TBjEhMEqo0tzTZB9Hx7uaBi8sLqxUiEsABQBIEmNBJMapz5F3n9KloNfoNLcPvYDeEKI3B7SlyOVWpq6CVfUkLZn3Ndq4bPlrdoyc9v-5TdD_vGHUSAFJ-kgCdzKRRsNmbHtrT0V3vd9hfUZJQ==}
|
||||
MODE = k3s
|
||||
PIPELINE_URL = http://127.0.0.1:8080
|
||||
|
||||
[Prod Cluster (k8s)]
|
||||
ARTIFACTS_DIR = /private/var/folders/rt/pywlnmxj3dn7t5552vwdcpp80000gn/T/pytest-of-chrisfu/pytest-69/test_encryption_toggle0/data/staging
|
||||
ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging
|
||||
CLUSTER_ENV = prole-prod-cluster
|
||||
DISPLAY_NAME = prole-prod-cluster
|
||||
MODE = k8s
|
||||
PIPELINE_URL = http://127.0.0.1:8080
|
||||
|
||||
[Docker Build]
|
||||
; No configuration values captured yet for this section.
|
||||
LOCAL_REGISTRY = localhost:5000
|
||||
LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000
|
||||
|
||||
[Initialization Scripts]
|
||||
; No configuration values captured yet for this section.
|
||||
STATUS = Attempted
|
||||
|
||||
[Deployment]
|
||||
MODE = k3d
|
||||
|
||||
@ -1,3 +1,11 @@
|
||||
# Port mappings for Prole Tools (generated).
|
||||
# Format: key: local=... remote=... ns=... svc=... address=...
|
||||
|
||||
argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0
|
||||
garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0
|
||||
openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=127.0.0.1
|
||||
opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0
|
||||
dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1
|
||||
postgres: local=5432 remote=5432 ns=knoe-db-17-7-043-18-140 svc=knoe-db-rw address=0.0.0.0
|
||||
prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1
|
||||
grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0
|
||||
|
||||
@ -214,6 +214,39 @@ ensure_prole_protected_storage() {
|
||||
return 0
|
||||
}
|
||||
|
||||
apply_cnpg_selector_overrides() {
|
||||
local rendered_manifest="$1"
|
||||
|
||||
if [[ "${PROLE_MODE:-}" != "k3s" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ ! -f "$rendered_manifest" ]]; then
|
||||
echo "ERROR: Rendered CNPG manifest not found for selector override: $rendered_manifest" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local data_selector_json="${PROLE_CNPG_DATA_SELECTOR_JSON:-}"
|
||||
local wal_selector_json="${PROLE_CNPG_WAL_SELECTOR_JSON:-}"
|
||||
if [[ -z "$data_selector_json" || -z "$wal_selector_json" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local patch_json patched
|
||||
if ! patch_json=$(jq -cn --argjson data "$data_selector_json" --argjson wal "$wal_selector_json" '{"spec":{"storage":{"pvcTemplate":{"selector":{"matchLabels":$data}}},"walStorage":{"pvcTemplate":{"selector":{"matchLabels":$wal}}}}}'); then
|
||||
echo "ERROR: Invalid CNPG selector JSON provided by installer (data/wal)." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! patched=$(kubectl patch --local -f "$rendered_manifest" --type merge -p "$patch_json" -o yaml 2>/dev/null); then
|
||||
echo "ERROR: Failed to inject CNPG PVC selector overrides into rendered manifest." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "$patched" > "$rendered_manifest"
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_cnpg_manifest_storage() {
|
||||
local rendered_manifest="$1"
|
||||
|
||||
@ -232,7 +265,9 @@ validate_cnpg_manifest_storage() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
local expected_ns="$NAMESPACE" expected_cluster="$CNPG_CLUSTER_NAME"
|
||||
local in_cluster=0 block="" data_sc="" wal_sc="" data_role="" wal_role=""
|
||||
local data_ns="" wal_ns="" data_cluster="" wal_cluster=""
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$line" =~ ^---[[:space:]]*$ ]]; then
|
||||
in_cluster=0
|
||||
@ -280,6 +315,24 @@ validate_cnpg_manifest_storage() {
|
||||
wal_role="$r"
|
||||
fi
|
||||
fi
|
||||
if [[ -n "$block" && "$line" =~ prole\.io/namespace: ]]; then
|
||||
local ns_v
|
||||
ns_v=$(printf '%s' "$line" | sed -E 's/^.*prole\.io\/namespace:[[:space:]]*//; s/[[:space:]]+$//')
|
||||
if [[ "$block" == "storage" && -z "$data_ns" ]]; then
|
||||
data_ns="$ns_v"
|
||||
elif [[ "$block" == "wal" && -z "$wal_ns" ]]; then
|
||||
wal_ns="$ns_v"
|
||||
fi
|
||||
fi
|
||||
if [[ -n "$block" && "$line" =~ prole\.io/cluster: ]]; then
|
||||
local c_v
|
||||
c_v=$(printf '%s' "$line" | sed -E 's/^.*prole\.io\/cluster:[[:space:]]*//; s/[[:space:]]+$//')
|
||||
if [[ "$block" == "storage" && -z "$data_cluster" ]]; then
|
||||
data_cluster="$c_v"
|
||||
elif [[ "$block" == "wal" && -z "$wal_cluster" ]]; then
|
||||
wal_cluster="$c_v"
|
||||
fi
|
||||
fi
|
||||
done < "$rendered_manifest"
|
||||
|
||||
if [[ "$data_sc" != "$PROLE_CNPG_STORAGE_CLASS" || "$wal_sc" != "$PROLE_CNPG_STORAGE_CLASS" ]]; then
|
||||
@ -294,6 +347,18 @@ validate_cnpg_manifest_storage() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$data_ns" != "$expected_ns" || "$wal_ns" != "$expected_ns" ]]; then
|
||||
echo "ERROR: CNPG Cluster manifest must include namespace-scoped selectors (prole.io/namespace=${expected_ns}) for both data and wal." >&2
|
||||
echo "Got selector namespaces: data='${data_ns:-<missing>}', wal='${wal_ns:-<missing>}'" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$data_cluster" != "$expected_cluster" || "$wal_cluster" != "$expected_cluster" ]]; then
|
||||
echo "ERROR: CNPG Cluster manifest must include cluster-scoped selectors (prole.io/cluster=${expected_cluster}) for both data and wal." >&2
|
||||
echo "Got selector clusters: data='${data_cluster:-<missing>}', wal='${wal_cluster:-<missing>}'" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -310,11 +375,15 @@ validate_cnpg_runtime_storage() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
local data_sc wal_sc data_role wal_role
|
||||
local data_sc wal_sc data_role wal_role data_ns wal_ns data_cluster wal_cluster
|
||||
data_sc=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.storageClassName // empty')
|
||||
wal_sc=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.storageClassName // empty')
|
||||
data_role=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.selector.matchLabels["synology.storage/role"] // empty')
|
||||
wal_role=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.selector.matchLabels["synology.storage/role"] // empty')
|
||||
data_ns=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.selector.matchLabels["prole.io/namespace"] // empty')
|
||||
wal_ns=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.selector.matchLabels["prole.io/namespace"] // empty')
|
||||
data_cluster=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.selector.matchLabels["prole.io/cluster"] // empty')
|
||||
wal_cluster=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.selector.matchLabels["prole.io/cluster"] // empty')
|
||||
|
||||
if [[ "$data_sc" != "$sc" || "$wal_sc" != "$sc" ]]; then
|
||||
echo "ERROR: Live CNPG Cluster storageClassName must be '${sc}' for both data and walStorage (got data='${data_sc:-<missing>}', wal='${wal_sc:-<missing>}')." >&2
|
||||
@ -324,6 +393,14 @@ validate_cnpg_runtime_storage() {
|
||||
echo "ERROR: Live CNPG Cluster must select Prole PVs via selector labels (got data role='${data_role:-<missing>}', wal role='${wal_role:-<missing>}')." >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$data_ns" != "$NAMESPACE" || "$wal_ns" != "$NAMESPACE" ]]; then
|
||||
echo "ERROR: Live CNPG Cluster must use namespace-scoped selectors prole.io/namespace='${NAMESPACE}' (got data='${data_ns:-<missing>}', wal='${wal_ns:-<missing>}')." >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$data_cluster" != "$CNPG_CLUSTER_NAME" || "$wal_cluster" != "$CNPG_CLUSTER_NAME" ]]; then
|
||||
echo "ERROR: Live CNPG Cluster must use cluster-scoped selectors prole.io/cluster='${CNPG_CLUSTER_NAME}' (got data='${data_cluster:-<missing>}', wal='${wal_cluster:-<missing>}')." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Wait for CNPG PVCs to exist so we can validate binding and PV paths.
|
||||
local timeout=${CNPG_STORAGE_VALIDATE_TIMEOUT:-120}
|
||||
@ -2255,6 +2332,10 @@ apply_cnpg_cluster_manifest() {
|
||||
prole_render_manifest "$manifest" >"$tmp"
|
||||
|
||||
patch_cnpg_manifest_for_topology "$tmp" || true
|
||||
if ! apply_cnpg_selector_overrides "$tmp"; then
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! validate_cnpg_manifest_storage "$tmp"; then
|
||||
rm -f "$tmp"
|
||||
|
||||
@ -92,6 +92,11 @@ from knoe.core.policy import (
|
||||
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES,
|
||||
evaluate_optional_workloads_allowed,
|
||||
)
|
||||
from knoe.core.ops.storage import (
|
||||
ClusterStorageSpec,
|
||||
StorageProvisioningError,
|
||||
provision_cluster_storage,
|
||||
)
|
||||
|
||||
|
||||
def _configure_unbuffered_io():
|
||||
@ -1736,6 +1741,66 @@ class ProleInstaller:
|
||||
|
||||
return placement_plan, plan_path
|
||||
|
||||
def _cnpg_storage_node(self, env: dict[str, str]) -> str:
|
||||
stage1 = str(env.get("CNPG_STAGE1_NODE") or "").strip()
|
||||
if stage1:
|
||||
return stage1
|
||||
|
||||
eligible_raw = str(env.get("CNPG_PLACEMENT_ELIGIBLE_NODES") or "").strip()
|
||||
if eligible_raw:
|
||||
for part in eligible_raw.split(","):
|
||||
node = str(part or "").strip()
|
||||
if node:
|
||||
return node
|
||||
|
||||
candidates = self._cnpg_candidate_nodes()
|
||||
if candidates:
|
||||
return candidates[0]
|
||||
|
||||
raise StorageProvisioningError(
|
||||
"Unable to determine CNPG node for local PV provisioning. "
|
||||
"Expected CNPG placement env (CNPG_STAGE1_NODE/CNPG_PLACEMENT_ELIGIBLE_NODES) "
|
||||
"or configured CNPG candidate nodes."
|
||||
)
|
||||
|
||||
def _ensure_cnpg_storage_provisioned(self, namespace: str, env: dict[str, str]) -> None:
|
||||
if self._deployment_mode() != "k3s":
|
||||
return
|
||||
|
||||
cluster_name = str(env.get("CNPG_CLUSTER_NAME") or self._cnpg_cluster_name()).strip()
|
||||
if not cluster_name:
|
||||
cluster_name = "knoe-db"
|
||||
node_name = self._cnpg_storage_node(env)
|
||||
|
||||
spec = ClusterStorageSpec(
|
||||
namespace=namespace,
|
||||
cluster_name=cluster_name,
|
||||
node_name=node_name,
|
||||
storage_class_name="synology-iscsi",
|
||||
command_env=env,
|
||||
)
|
||||
provisioned = provision_cluster_storage(spec)
|
||||
|
||||
env["PROLE_CNPG_STORAGE_CLASS"] = spec.storage_class_name
|
||||
env["PROLE_CNPG_DATA_SELECTOR_JSON"] = json.dumps(
|
||||
provisioned.data_selector,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
env["PROLE_CNPG_WAL_SELECTOR_JSON"] = json.dumps(
|
||||
provisioned.wal_selector,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
env["PROLE_CNPG_DATA_PV_NAME"] = provisioned.data_pv_name
|
||||
env["PROLE_CNPG_WAL_PV_NAME"] = provisioned.wal_pv_name
|
||||
|
||||
self.log(
|
||||
"[CNPG] Provisioned cluster storage "
|
||||
f"namespace={namespace} cluster={cluster_name} node={node_name} "
|
||||
f"data_pv={provisioned.data_pv_name} wal_pv={provisioned.wal_pv_name}"
|
||||
)
|
||||
|
||||
# --------------------------------------------- authority / repair
|
||||
def _authority_context_missing(self) -> bool:
|
||||
enabled = self._get_input_bool(
|
||||
@ -5347,6 +5412,17 @@ class ProleConsoleInstaller(ProleInstaller):
|
||||
self._get_input("init_password.db_namespace", "") or ""
|
||||
).strip() or "default"
|
||||
env = self._script_env_for_namespace(ns)
|
||||
cluster_name = str(env.get("CNPG_CLUSTER_NAME") or self._cnpg_cluster_name()).strip()
|
||||
if not cluster_name:
|
||||
cluster_name = "knoe-db"
|
||||
|
||||
try:
|
||||
self._ensure_cnpg_storage_provisioned(ns, env)
|
||||
except Exception as exc:
|
||||
self._cnpg_success = False
|
||||
self.prole_cfg_data["Deployment"]["STATUS"] = "Attempted"
|
||||
self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}")
|
||||
return
|
||||
|
||||
rc = self._run_cmd(
|
||||
["bash", str(etc_dir / "init_cloudnative_pg.sh"), "deploy", "latest"],
|
||||
@ -5373,7 +5449,7 @@ class ProleConsoleInstaller(ProleInstaller):
|
||||
"-n",
|
||||
ns,
|
||||
"-l",
|
||||
f"cnpg.io/cluster=knoe-db",
|
||||
f"cnpg.io/cluster={cluster_name}",
|
||||
"--no-headers",
|
||||
],
|
||||
capture_output=True,
|
||||
|
||||
1
knoe/core/ops/__init__.py
Normal file
1
knoe/core/ops/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""Core operational helpers for installer actions."""
|
||||
476
knoe/core/ops/storage.py
Normal file
476
knoe/core/ops/storage.py
Normal file
@ -0,0 +1,476 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_SYNOLOGY_ROOTS: tuple[str, ...] = (
|
||||
"/synology/d001",
|
||||
"/synology/d002",
|
||||
"/synology/d003",
|
||||
"/synology/d004",
|
||||
)
|
||||
|
||||
|
||||
class StorageProvisioningError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class ClusterStorageSpec:
|
||||
namespace: str
|
||||
cluster_name: str
|
||||
node_name: str
|
||||
storage_class_name: str = "synology-iscsi"
|
||||
service_name: str = "knoe-db"
|
||||
root_namespace: str = "knoe"
|
||||
postgres_uid: int = 100
|
||||
postgres_gid: int = 101
|
||||
directory_mode: int = 0o700
|
||||
volume_capacity: str = "29Gi"
|
||||
synology_roots: tuple[str, ...] = DEFAULT_SYNOLOGY_ROOTS
|
||||
kubectl_bin: str = "kubectl"
|
||||
mount_check_bin: str = "findmnt"
|
||||
command_env: dict[str, str] | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class ClusterStoragePaths:
|
||||
data_root: str
|
||||
wal_root: str
|
||||
data_volume: str
|
||||
wal_volume: str
|
||||
data_path: str
|
||||
wal_path: str
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class ProvisionedStorage:
|
||||
data_pv_name: str
|
||||
wal_pv_name: str
|
||||
data_path: str
|
||||
wal_path: str
|
||||
data_selector: dict[str, str]
|
||||
wal_selector: dict[str, str]
|
||||
|
||||
|
||||
def _sanitize_dns_label(value: str, max_len: int = 253) -> str:
|
||||
lowered = (value or "").strip().lower()
|
||||
cleaned = re.sub(r"[^a-z0-9.-]+", "-", lowered)
|
||||
cleaned = re.sub(r"-+", "-", cleaned).strip("-.")
|
||||
if not cleaned:
|
||||
cleaned = "pv"
|
||||
if len(cleaned) <= max_len:
|
||||
return cleaned
|
||||
digest = hashlib.sha1(cleaned.encode("utf-8")).hexdigest()[:8]
|
||||
head = cleaned[: max_len - 9].rstrip("-.")
|
||||
return f"{head}-{digest}"
|
||||
|
||||
|
||||
def _normalize_posix_path(path: str) -> str:
|
||||
parts = [p for p in path.split("/") if p]
|
||||
return "/" + "/".join(parts)
|
||||
|
||||
|
||||
def _paths_overlap(path_a: str, path_b: str) -> bool:
|
||||
a = _normalize_posix_path(path_a)
|
||||
b = _normalize_posix_path(path_b)
|
||||
return a == b or a.startswith(f"{b}/") or b.startswith(f"{a}/")
|
||||
|
||||
|
||||
def _command_env(spec: ClusterStorageSpec) -> dict[str, str] | None:
|
||||
return dict(spec.command_env) if spec.command_env else None
|
||||
|
||||
|
||||
def _run_command(
|
||||
args: list[str],
|
||||
*,
|
||||
env: dict[str, str] | None = None,
|
||||
input_text: str | None = None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
input=input_text,
|
||||
)
|
||||
|
||||
|
||||
def _is_non_root_mount(path: Path, *, mount_check_bin: str, env: dict[str, str] | None) -> bool:
|
||||
if not path.is_dir():
|
||||
return False
|
||||
check = _run_command([mount_check_bin, "-T", str(path), "-n", "-o", "TARGET,SOURCE"], env=env)
|
||||
if check.returncode != 0:
|
||||
return bool(os.path.ismount(path))
|
||||
|
||||
root_check = _run_command([mount_check_bin, "-T", "/", "-n", "-o", "SOURCE"], env=env)
|
||||
root_source = (root_check.stdout or "").strip() if root_check.returncode == 0 else ""
|
||||
|
||||
parts = (check.stdout or "").strip().split()
|
||||
if len(parts) < 2:
|
||||
return bool(os.path.ismount(path))
|
||||
target, source = parts[0], parts[1]
|
||||
if target == "/":
|
||||
return False
|
||||
if root_source and source == root_source:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _ensure_valid_identity(value: str, field_name: str) -> str:
|
||||
val = (value or "").strip()
|
||||
if not val:
|
||||
raise StorageProvisioningError(f"Missing required storage identity field: {field_name}")
|
||||
return val
|
||||
|
||||
|
||||
def _volume_id_from_root(root: Path) -> str:
|
||||
return _sanitize_dns_label(root.name, max_len=32)
|
||||
|
||||
|
||||
def _choose_root_pair(spec: ClusterStorageSpec) -> tuple[Path, Path]:
|
||||
env = _command_env(spec)
|
||||
mounted_roots: list[Path] = []
|
||||
for root_str in spec.synology_roots:
|
||||
root = Path(root_str)
|
||||
if _is_non_root_mount(root, mount_check_bin=spec.mount_check_bin, env=env):
|
||||
mounted_roots.append(root)
|
||||
|
||||
if not mounted_roots:
|
||||
raise StorageProvisioningError(
|
||||
"No mounted Synology roots available. Expected mounted paths like /synology/d001..d004."
|
||||
)
|
||||
|
||||
ident = f"{spec.namespace}:{spec.cluster_name}:{spec.service_name}"
|
||||
digest = int(hashlib.sha256(ident.encode("utf-8")).hexdigest(), 16)
|
||||
data_index = digest % len(mounted_roots)
|
||||
data_root = mounted_roots[data_index]
|
||||
|
||||
if len(mounted_roots) == 1:
|
||||
wal_root = data_root
|
||||
else:
|
||||
wal_root = mounted_roots[(data_index + 1) % len(mounted_roots)]
|
||||
|
||||
return data_root, wal_root
|
||||
|
||||
|
||||
def build_cluster_storage_paths(spec: ClusterStorageSpec) -> ClusterStoragePaths:
|
||||
namespace = _ensure_valid_identity(spec.namespace, "namespace")
|
||||
cluster_name = _ensure_valid_identity(spec.cluster_name, "cluster_name")
|
||||
|
||||
data_root, wal_root = _choose_root_pair(spec)
|
||||
base_rel = Path(spec.root_namespace) / spec.service_name / namespace / cluster_name
|
||||
data_path = data_root / base_rel / "data"
|
||||
wal_path = wal_root / base_rel / "wal"
|
||||
|
||||
if _paths_overlap(str(data_path), str(wal_path)):
|
||||
raise StorageProvisioningError(
|
||||
f"Refusing overlapping storage paths: data='{data_path}', wal='{wal_path}'"
|
||||
)
|
||||
|
||||
return ClusterStoragePaths(
|
||||
data_root=str(data_root),
|
||||
wal_root=str(wal_root),
|
||||
data_volume=_volume_id_from_root(data_root),
|
||||
wal_volume=_volume_id_from_root(wal_root),
|
||||
data_path=str(data_path),
|
||||
wal_path=str(wal_path),
|
||||
)
|
||||
|
||||
|
||||
def build_pv_name(storage_class_name: str, namespace: str, cluster_name: str, role: str) -> str:
|
||||
base = f"{storage_class_name}-{namespace}-{cluster_name}-{role}"
|
||||
return _sanitize_dns_label(base)
|
||||
|
||||
|
||||
def build_pv_labels(
|
||||
*,
|
||||
namespace: str,
|
||||
cluster_name: str,
|
||||
role: str,
|
||||
volume: str,
|
||||
service_name: str = "knoe-db",
|
||||
) -> dict[str, str]:
|
||||
return {
|
||||
"synology.storage/role": role,
|
||||
"synology.storage/volume": volume,
|
||||
"prole.io/namespace": namespace,
|
||||
"prole.io/cluster": cluster_name,
|
||||
"prole.io/service": service_name,
|
||||
}
|
||||
|
||||
|
||||
def render_local_pv(
|
||||
*,
|
||||
pv_name: str,
|
||||
host_path: str,
|
||||
node_name: str,
|
||||
storage_class_name: str,
|
||||
capacity: str,
|
||||
labels: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"apiVersion": "v1",
|
||||
"kind": "PersistentVolume",
|
||||
"metadata": {
|
||||
"name": pv_name,
|
||||
"labels": labels,
|
||||
},
|
||||
"spec": {
|
||||
"capacity": {"storage": capacity},
|
||||
"volumeMode": "Filesystem",
|
||||
"accessModes": ["ReadWriteOnce"],
|
||||
"storageClassName": storage_class_name,
|
||||
"persistentVolumeReclaimPolicy": "Retain",
|
||||
"local": {"path": host_path},
|
||||
"nodeAffinity": {
|
||||
"required": {
|
||||
"nodeSelectorTerms": [
|
||||
{
|
||||
"matchExpressions": [
|
||||
{
|
||||
"key": "kubernetes.io/hostname",
|
||||
"operator": "In",
|
||||
"values": [node_name],
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def ensure_host_path(path: str, *, uid: int, gid: int, mode: int = 0o700) -> None:
|
||||
target = Path(path)
|
||||
if target.exists() and not target.is_dir():
|
||||
raise StorageProvisioningError(f"Host path exists but is not a directory: {path}")
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(target, mode)
|
||||
try:
|
||||
os.chown(target, uid, gid)
|
||||
except PermissionError as exc:
|
||||
raise StorageProvisioningError(
|
||||
f"Cannot set ownership on host path '{path}' to {uid}:{gid}: {exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _kubectl_get_json(spec: ClusterStorageSpec, args: list[str]) -> dict[str, Any] | None:
|
||||
env = _command_env(spec)
|
||||
cmd = [spec.kubectl_bin, *args, "-o", "json"]
|
||||
res = _run_command(cmd, env=env)
|
||||
if res.returncode != 0:
|
||||
return None
|
||||
try:
|
||||
return json.loads(res.stdout or "{}")
|
||||
except json.JSONDecodeError as exc:
|
||||
raise StorageProvisioningError(
|
||||
f"Failed to parse kubectl JSON output for command: {' '.join(cmd)}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _kubectl_apply_manifest(spec: ClusterStorageSpec, manifest: dict[str, Any]) -> None:
|
||||
env = _command_env(spec)
|
||||
payload = json.dumps(manifest)
|
||||
res = _run_command([spec.kubectl_bin, "apply", "-f", "-"], env=env, input_text=payload)
|
||||
if res.returncode != 0:
|
||||
stderr = (res.stderr or "").strip()
|
||||
raise StorageProvisioningError(f"Failed to apply PV manifest '{manifest['metadata']['name']}': {stderr}")
|
||||
|
||||
|
||||
def _collect_existing_pvs(spec: ClusterStorageSpec) -> dict[str, dict[str, Any]]:
|
||||
pv_list = _kubectl_get_json(spec, ["get", "pv"])
|
||||
if pv_list is None:
|
||||
raise StorageProvisioningError("Unable to list existing PersistentVolumes via kubectl.")
|
||||
items = pv_list.get("items") or []
|
||||
return {str(item.get("metadata", {}).get("name") or ""): item for item in items}
|
||||
|
||||
|
||||
def _extract_node_names(pv_obj: dict[str, Any]) -> set[str]:
|
||||
names: set[str] = set()
|
||||
terms = (
|
||||
pv_obj.get("spec", {})
|
||||
.get("nodeAffinity", {})
|
||||
.get("required", {})
|
||||
.get("nodeSelectorTerms", [])
|
||||
)
|
||||
for term in terms:
|
||||
for expr in term.get("matchExpressions", []) or []:
|
||||
if expr.get("key") != "kubernetes.io/hostname":
|
||||
continue
|
||||
if expr.get("operator") != "In":
|
||||
continue
|
||||
for val in expr.get("values") or []:
|
||||
if val:
|
||||
names.add(str(val))
|
||||
return names
|
||||
|
||||
|
||||
def _validate_existing_pv(
|
||||
*,
|
||||
existing_pv: dict[str, Any],
|
||||
expected_name: str,
|
||||
expected_path: str,
|
||||
expected_storage_class: str,
|
||||
expected_node_name: str,
|
||||
expected_labels: dict[str, str],
|
||||
) -> None:
|
||||
existing_name = str(existing_pv.get("metadata", {}).get("name") or "")
|
||||
if existing_name != expected_name:
|
||||
raise StorageProvisioningError(
|
||||
f"Internal PV validation error: expected '{expected_name}', got '{existing_name}'"
|
||||
)
|
||||
|
||||
current_path = str(existing_pv.get("spec", {}).get("local", {}).get("path") or "")
|
||||
if _normalize_posix_path(current_path) != _normalize_posix_path(expected_path):
|
||||
raise StorageProvisioningError(
|
||||
f"PV '{expected_name}' exists with unexpected path '{current_path}' (expected '{expected_path}')."
|
||||
)
|
||||
|
||||
current_sc = str(existing_pv.get("spec", {}).get("storageClassName") or "")
|
||||
if current_sc != expected_storage_class:
|
||||
raise StorageProvisioningError(
|
||||
f"PV '{expected_name}' exists with unexpected storageClassName '{current_sc}' (expected '{expected_storage_class}')."
|
||||
)
|
||||
|
||||
current_labels = existing_pv.get("metadata", {}).get("labels") or {}
|
||||
for key, expected in expected_labels.items():
|
||||
actual = str(current_labels.get(key) or "")
|
||||
if actual != expected:
|
||||
raise StorageProvisioningError(
|
||||
f"PV '{expected_name}' label mismatch for '{key}': got '{actual}', expected '{expected}'."
|
||||
)
|
||||
|
||||
node_names = _extract_node_names(existing_pv)
|
||||
if expected_node_name not in node_names:
|
||||
raise StorageProvisioningError(
|
||||
f"PV '{expected_name}' node affinity mismatch: expected node '{expected_node_name}', got {sorted(node_names)}."
|
||||
)
|
||||
|
||||
|
||||
def _ensure_no_path_overlap(
|
||||
*,
|
||||
existing_pvs: dict[str, dict[str, Any]],
|
||||
pv_name: str,
|
||||
target_path: str,
|
||||
) -> None:
|
||||
for existing_name, existing_pv in existing_pvs.items():
|
||||
if existing_name == pv_name:
|
||||
continue
|
||||
existing_path = str(existing_pv.get("spec", {}).get("local", {}).get("path") or "").strip()
|
||||
if not existing_path:
|
||||
continue
|
||||
if _paths_overlap(target_path, existing_path):
|
||||
raise StorageProvisioningError(
|
||||
f"PV path overlap detected: target '{target_path}' conflicts with existing PV '{existing_name}' path '{existing_path}'."
|
||||
)
|
||||
|
||||
|
||||
def _provision_one_pv(
|
||||
*,
|
||||
spec: ClusterStorageSpec,
|
||||
existing_pvs: dict[str, dict[str, Any]],
|
||||
role: str,
|
||||
path: str,
|
||||
volume: str,
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
pv_name = build_pv_name(spec.storage_class_name, spec.namespace, spec.cluster_name, role)
|
||||
labels = build_pv_labels(
|
||||
namespace=spec.namespace,
|
||||
cluster_name=spec.cluster_name,
|
||||
role=role,
|
||||
volume=volume,
|
||||
service_name=spec.service_name,
|
||||
)
|
||||
_ensure_no_path_overlap(existing_pvs=existing_pvs, pv_name=pv_name, target_path=path)
|
||||
|
||||
existing = existing_pvs.get(pv_name)
|
||||
if existing:
|
||||
_validate_existing_pv(
|
||||
existing_pv=existing,
|
||||
expected_name=pv_name,
|
||||
expected_path=path,
|
||||
expected_storage_class=spec.storage_class_name,
|
||||
expected_node_name=spec.node_name,
|
||||
expected_labels=labels,
|
||||
)
|
||||
return pv_name, labels
|
||||
|
||||
manifest = render_local_pv(
|
||||
pv_name=pv_name,
|
||||
host_path=path,
|
||||
node_name=spec.node_name,
|
||||
storage_class_name=spec.storage_class_name,
|
||||
capacity=spec.volume_capacity,
|
||||
labels=labels,
|
||||
)
|
||||
_kubectl_apply_manifest(spec, manifest)
|
||||
return pv_name, labels
|
||||
|
||||
|
||||
def provision_cluster_storage(spec: ClusterStorageSpec) -> ProvisionedStorage:
|
||||
spec = ClusterStorageSpec(
|
||||
namespace=_ensure_valid_identity(spec.namespace, "namespace"),
|
||||
cluster_name=_ensure_valid_identity(spec.cluster_name, "cluster_name"),
|
||||
node_name=_ensure_valid_identity(spec.node_name, "node_name"),
|
||||
storage_class_name=_ensure_valid_identity(spec.storage_class_name, "storage_class_name"),
|
||||
service_name=_ensure_valid_identity(spec.service_name, "service_name"),
|
||||
root_namespace=_ensure_valid_identity(spec.root_namespace, "root_namespace"),
|
||||
postgres_uid=spec.postgres_uid,
|
||||
postgres_gid=spec.postgres_gid,
|
||||
directory_mode=spec.directory_mode,
|
||||
volume_capacity=_ensure_valid_identity(spec.volume_capacity, "volume_capacity"),
|
||||
synology_roots=spec.synology_roots,
|
||||
kubectl_bin=spec.kubectl_bin,
|
||||
mount_check_bin=spec.mount_check_bin,
|
||||
command_env=spec.command_env,
|
||||
)
|
||||
|
||||
paths = build_cluster_storage_paths(spec)
|
||||
ensure_host_path(
|
||||
paths.data_path,
|
||||
uid=spec.postgres_uid,
|
||||
gid=spec.postgres_gid,
|
||||
mode=spec.directory_mode,
|
||||
)
|
||||
ensure_host_path(
|
||||
paths.wal_path,
|
||||
uid=spec.postgres_uid,
|
||||
gid=spec.postgres_gid,
|
||||
mode=spec.directory_mode,
|
||||
)
|
||||
|
||||
existing_pvs = _collect_existing_pvs(spec)
|
||||
|
||||
data_pv_name, data_labels = _provision_one_pv(
|
||||
spec=spec,
|
||||
existing_pvs=existing_pvs,
|
||||
role="data",
|
||||
path=paths.data_path,
|
||||
volume=paths.data_volume,
|
||||
)
|
||||
existing_pvs = _collect_existing_pvs(spec)
|
||||
wal_pv_name, wal_labels = _provision_one_pv(
|
||||
spec=spec,
|
||||
existing_pvs=existing_pvs,
|
||||
role="wal",
|
||||
path=paths.wal_path,
|
||||
volume=paths.wal_volume,
|
||||
)
|
||||
|
||||
return ProvisionedStorage(
|
||||
data_pv_name=data_pv_name,
|
||||
wal_pv_name=wal_pv_name,
|
||||
data_path=paths.data_path,
|
||||
wal_path=paths.wal_path,
|
||||
data_selector=dict(data_labels),
|
||||
wal_selector=dict(wal_labels),
|
||||
)
|
||||
@ -1 +1 @@
|
||||
4
|
||||
5
|
||||
@ -756,6 +756,82 @@ class TestStepInitScripts:
|
||||
assert inst._scripts_success is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _step_cnpg_deploy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStepCnpgDeploy:
|
||||
def test_provisions_storage_before_cnpg_deploy(self, monkeypatch):
|
||||
inst = _TestableSilentInstaller(
|
||||
inputs={
|
||||
"init_cnpg_deploy.run_deploy": "true",
|
||||
"init_cnpg_deploy.force_rollout": "false",
|
||||
"init_password.db_namespace": "knoe-db",
|
||||
}
|
||||
)
|
||||
inst.prole_cfg_data.setdefault("Deployment", {})
|
||||
|
||||
env = {"CNPG_CLUSTER_NAME": "cluster-a"}
|
||||
monkeypatch.setattr(inst, "_script_env_for_namespace", lambda _ns: env)
|
||||
|
||||
calls = {"provision": 0, "run": 0}
|
||||
|
||||
def _fake_provision(ns, env_arg):
|
||||
calls["provision"] += 1
|
||||
assert ns == "knoe-db"
|
||||
assert env_arg is env
|
||||
env_arg["PROLE_CNPG_DATA_SELECTOR_JSON"] = '{"synology.storage/role":"data"}'
|
||||
env_arg["PROLE_CNPG_WAL_SELECTOR_JSON"] = '{"synology.storage/role":"wal"}'
|
||||
|
||||
def _fake_run(cmd, env=None, **_kwargs):
|
||||
calls["run"] += 1
|
||||
assert "init_cloudnative_pg.sh" in str(cmd[1])
|
||||
assert env is not None
|
||||
assert env["PROLE_CNPG_DATA_SELECTOR_JSON"]
|
||||
assert env["PROLE_CNPG_WAL_SELECTOR_JSON"]
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(inst, "_ensure_cnpg_storage_provisioned", _fake_provision)
|
||||
monkeypatch.setattr(inst, "_run_cmd", _fake_run)
|
||||
|
||||
inst._step_cnpg_deploy()
|
||||
|
||||
assert calls["provision"] == 1
|
||||
assert calls["run"] == 1
|
||||
assert inst._cnpg_success is True
|
||||
assert inst.prole_cfg_data["Deployment"]["STATUS"] == "Deployed"
|
||||
|
||||
def test_stops_deploy_on_storage_provision_failure(self, monkeypatch):
|
||||
inst = _TestableSilentInstaller(
|
||||
inputs={
|
||||
"init_cnpg_deploy.run_deploy": "true",
|
||||
"init_cnpg_deploy.force_rollout": "false",
|
||||
"init_password.db_namespace": "knoe-db",
|
||||
}
|
||||
)
|
||||
inst.prole_cfg_data.setdefault("Deployment", {})
|
||||
|
||||
monkeypatch.setattr(
|
||||
inst,
|
||||
"_script_env_for_namespace",
|
||||
lambda _ns: {"CNPG_CLUSTER_NAME": "cluster-a"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
inst,
|
||||
"_ensure_cnpg_storage_provisioned",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("boom")),
|
||||
)
|
||||
run_mock = MagicMock(return_value=0)
|
||||
monkeypatch.setattr(inst, "_run_cmd", run_mock)
|
||||
|
||||
inst._step_cnpg_deploy()
|
||||
|
||||
run_mock.assert_not_called()
|
||||
assert inst._cnpg_success is False
|
||||
assert inst.prole_cfg_data["Deployment"]["STATUS"] == "Attempted"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _prepare_opentofu_pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
189
tests/installer/test_storage_ops.py
Normal file
189
tests/installer/test_storage_ops.py
Normal file
@ -0,0 +1,189 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from knoe.core.ops import storage as storage_ops
|
||||
|
||||
|
||||
def test_build_cluster_storage_paths_prefers_distinct_roots(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
storage_ops,
|
||||
"_is_non_root_mount",
|
||||
lambda path, **_kwargs: str(path) in {"/synology/d001", "/synology/d002"},
|
||||
)
|
||||
|
||||
spec = storage_ops.ClusterStorageSpec(
|
||||
namespace="knoe-db",
|
||||
cluster_name="cluster-a",
|
||||
node_name="myrddin.prole.org",
|
||||
synology_roots=("/synology/d001", "/synology/d002"),
|
||||
)
|
||||
|
||||
paths = storage_ops.build_cluster_storage_paths(spec)
|
||||
|
||||
assert paths.data_root != paths.wal_root
|
||||
assert paths.data_path.endswith("/knoe/knoe-db/knoe-db/cluster-a/data")
|
||||
assert paths.wal_path.endswith("/knoe/knoe-db/knoe-db/cluster-a/wal")
|
||||
|
||||
|
||||
def test_build_cluster_storage_paths_falls_back_to_same_root(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
storage_ops,
|
||||
"_is_non_root_mount",
|
||||
lambda path, **_kwargs: str(path) == "/synology/d001",
|
||||
)
|
||||
|
||||
spec = storage_ops.ClusterStorageSpec(
|
||||
namespace="knoe-db",
|
||||
cluster_name="cluster-a",
|
||||
node_name="myrddin.prole.org",
|
||||
synology_roots=("/synology/d001", "/synology/d002"),
|
||||
)
|
||||
|
||||
paths = storage_ops.build_cluster_storage_paths(spec)
|
||||
|
||||
assert paths.data_root == "/synology/d001"
|
||||
assert paths.wal_root == "/synology/d001"
|
||||
assert paths.data_path != paths.wal_path
|
||||
|
||||
|
||||
def test_build_pv_labels_include_cluster_scope():
|
||||
labels = storage_ops.build_pv_labels(
|
||||
namespace="knoe-db",
|
||||
cluster_name="cluster-a",
|
||||
role="data",
|
||||
volume="d001",
|
||||
)
|
||||
|
||||
assert labels["synology.storage/role"] == "data"
|
||||
assert labels["synology.storage/volume"] == "d001"
|
||||
assert labels["prole.io/namespace"] == "knoe-db"
|
||||
assert labels["prole.io/cluster"] == "cluster-a"
|
||||
assert labels["prole.io/service"] == "knoe-db"
|
||||
|
||||
|
||||
def test_provision_cluster_storage_fails_on_existing_name_mismatch(monkeypatch):
|
||||
spec = storage_ops.ClusterStorageSpec(
|
||||
namespace="knoe-db",
|
||||
cluster_name="cluster-a",
|
||||
node_name="myrddin.prole.org",
|
||||
)
|
||||
paths = storage_ops.ClusterStoragePaths(
|
||||
data_root="/synology/d001",
|
||||
wal_root="/synology/d002",
|
||||
data_volume="d001",
|
||||
wal_volume="d002",
|
||||
data_path="/synology/d001/knoe/knoe-db/knoe-db/cluster-a/data",
|
||||
wal_path="/synology/d002/knoe/knoe-db/knoe-db/cluster-a/wal",
|
||||
)
|
||||
|
||||
data_name = storage_ops.build_pv_name("synology-iscsi", "knoe-db", "cluster-a", "data")
|
||||
monkeypatch.setattr(storage_ops, "build_cluster_storage_paths", lambda _spec: paths)
|
||||
monkeypatch.setattr(storage_ops, "ensure_host_path", lambda *args, **kwargs: None)
|
||||
|
||||
existing = {
|
||||
data_name: {
|
||||
"metadata": {
|
||||
"name": data_name,
|
||||
"labels": storage_ops.build_pv_labels(
|
||||
namespace="knoe-db",
|
||||
cluster_name="cluster-a",
|
||||
role="data",
|
||||
volume="d001",
|
||||
),
|
||||
},
|
||||
"spec": {
|
||||
"storageClassName": "synology-iscsi",
|
||||
"local": {"path": "/synology/d001/other-path"},
|
||||
"nodeAffinity": {
|
||||
"required": {
|
||||
"nodeSelectorTerms": [
|
||||
{
|
||||
"matchExpressions": [
|
||||
{
|
||||
"key": "kubernetes.io/hostname",
|
||||
"operator": "In",
|
||||
"values": ["myrddin.prole.org"],
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
monkeypatch.setattr(storage_ops, "_collect_existing_pvs", lambda _spec: existing)
|
||||
|
||||
with pytest.raises(storage_ops.StorageProvisioningError, match="unexpected path"):
|
||||
storage_ops.provision_cluster_storage(spec)
|
||||
|
||||
|
||||
def test_provision_cluster_storage_applies_and_returns_selectors(monkeypatch):
|
||||
spec = storage_ops.ClusterStorageSpec(
|
||||
namespace="knoe-db",
|
||||
cluster_name="cluster-a",
|
||||
node_name="myrddin.prole.org",
|
||||
)
|
||||
paths = storage_ops.ClusterStoragePaths(
|
||||
data_root="/synology/d001",
|
||||
wal_root="/synology/d002",
|
||||
data_volume="d001",
|
||||
wal_volume="d002",
|
||||
data_path="/synology/d001/knoe/knoe-db/knoe-db/cluster-a/data",
|
||||
wal_path="/synology/d002/knoe/knoe-db/knoe-db/cluster-a/wal",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(storage_ops, "build_cluster_storage_paths", lambda _spec: paths)
|
||||
monkeypatch.setattr(storage_ops, "ensure_host_path", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(storage_ops, "_collect_existing_pvs", lambda _spec: {})
|
||||
|
||||
applied: list[dict] = []
|
||||
|
||||
def _capture_apply(_spec, manifest):
|
||||
applied.append(manifest)
|
||||
|
||||
monkeypatch.setattr(storage_ops, "_kubectl_apply_manifest", _capture_apply)
|
||||
|
||||
provisioned = storage_ops.provision_cluster_storage(spec)
|
||||
|
||||
assert len(applied) == 2
|
||||
assert provisioned.data_pv_name == "synology-iscsi-knoe-db-cluster-a-data"
|
||||
assert provisioned.wal_pv_name == "synology-iscsi-knoe-db-cluster-a-wal"
|
||||
assert provisioned.data_selector["prole.io/namespace"] == "knoe-db"
|
||||
assert provisioned.data_selector["prole.io/cluster"] == "cluster-a"
|
||||
assert provisioned.wal_selector["synology.storage/role"] == "wal"
|
||||
|
||||
|
||||
def test_provision_cluster_storage_fails_on_path_overlap(monkeypatch):
|
||||
spec = storage_ops.ClusterStorageSpec(
|
||||
namespace="knoe-db",
|
||||
cluster_name="cluster-a",
|
||||
node_name="myrddin.prole.org",
|
||||
)
|
||||
paths = storage_ops.ClusterStoragePaths(
|
||||
data_root="/synology/d001",
|
||||
wal_root="/synology/d002",
|
||||
data_volume="d001",
|
||||
wal_volume="d002",
|
||||
data_path="/synology/d001/knoe/knoe-db/knoe-db/cluster-a/data",
|
||||
wal_path="/synology/d002/knoe/knoe-db/knoe-db/cluster-a/wal",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(storage_ops, "build_cluster_storage_paths", lambda _spec: paths)
|
||||
monkeypatch.setattr(storage_ops, "ensure_host_path", lambda *args, **kwargs: None)
|
||||
|
||||
existing = {
|
||||
"some-other-pv": {
|
||||
"metadata": {"name": "some-other-pv", "labels": {}},
|
||||
"spec": {
|
||||
"storageClassName": "synology-iscsi",
|
||||
"local": {"path": "/synology/d001/knoe/knoe-db/knoe-db/cluster-a"},
|
||||
"nodeAffinity": {"required": {"nodeSelectorTerms": []}},
|
||||
},
|
||||
}
|
||||
}
|
||||
monkeypatch.setattr(storage_ops, "_collect_existing_pvs", lambda _spec: existing)
|
||||
|
||||
with pytest.raises(storage_ops.StorageProvisioningError, match="overlap"):
|
||||
storage_ops.provision_cluster_storage(spec)
|
||||
Loading…
Reference in New Issue
Block a user