mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
Stabilize CNPG reset/update flow and finalize 3-node recovery
- add idempotent CNPG operator reconciliation to prevent duplicate active controller ReplicaSets - restore and validate CNPG 3-instance convergence with PV claimRef recovery and storage path readiness - wire canonical launcher aliases/reset behavior and improve namespace cleanup semantics - harden backup/objectstore readiness handling and retry behavior - enforce service namespace usage for common services and remove default-namespace drift - enable Kerberos milestone auto-activation when realm/kdc are configured and apply CNPG GSS pg_hba rules - keep final runtime healthy: knoe-db 3/3, operator stable, perfsnap captured Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
parent
761d80486b
commit
15890187a3
@ -12,10 +12,10 @@
|
||||
"pi.prole.org"
|
||||
],
|
||||
"metadata": {
|
||||
"prior_plan_present": false,
|
||||
"reason": "new_cluster",
|
||||
"regenerated": true,
|
||||
"reused": false
|
||||
"prior_plan_present": true,
|
||||
"reason": "reused",
|
||||
"regenerated": false,
|
||||
"reused": true
|
||||
},
|
||||
"plan_hash": "10dccbb8fe133be4",
|
||||
"plan_id": "cnpg-placement-10dccbb8fe133be4",
|
||||
|
||||
@ -1,3 +1,16 @@
|
||||
# Port mappings for Prole Tools (generated).
|
||||
# Format: key: local=... remote=... ns=... svc=... address=...
|
||||
|
||||
argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0
|
||||
supabase-studio: local=18080 remote=3000 ns=supabase svc=studio address=0.0.0.0
|
||||
supabase-auth: local=9999 remote=9999 ns=supabase svc=auth address=127.0.0.1
|
||||
supabase-rest: local=3001 remote=3000 ns=supabase svc=rest address=0.0.0.0
|
||||
supabase-realtime: local=4000 remote=4000 ns=supabase svc=realtime address=0.0.0.0
|
||||
garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0
|
||||
openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0
|
||||
opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0
|
||||
dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1
|
||||
postgres: local=5432 remote=5432 ns=knoe-db svc=knoe-db-rw address=0.0.0.0
|
||||
prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1
|
||||
grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0
|
||||
supabase-kong: local=8000 remote=8000 ns=supabase svc=kong address=0.0.0.0
|
||||
|
||||
@ -48,20 +48,27 @@ init_cluster.k3s_token = K107c8c6000488eca4a067d8a73119bbae2f07b4ea1bac7d8d3dc9c
|
||||
init_cluster.kerberos_enabled = true
|
||||
init_cluster.mode = k8s
|
||||
init_cluster.start_cluster = true
|
||||
init_cluster.supabase_analytics_enabled = true
|
||||
init_cluster.supabase_auth_enabled = true
|
||||
init_cluster.supabase_enabled = false
|
||||
init_cluster.supabase_meta_enabled = true
|
||||
init_cluster.supabase_realtime_enabled = true
|
||||
init_cluster.supabase_studio_enabled = false
|
||||
init_cluster.supabase_studio_url = db.prole.org
|
||||
init_cnpg_deploy.force_rollout = false
|
||||
init_cnpg_deploy.run_deploy = true
|
||||
init_db_build.run_build = true
|
||||
init_password.cluster_name = ${CLUSTER_NAME}
|
||||
init_password.db_host_port = 5432
|
||||
init_password.db_namespace = ${DATABASE_NAMESPACE}
|
||||
init_password.db_password =
|
||||
init_password.db_password_confirm =
|
||||
init_password.db_password = ${PROLE_SECRET:v1:6fyOe4qCZmGGqAUU:vkFsVGYFxvXsPurt3QNO0Y5ZUDdfwMbvgG4N1ULprbCfA0XeVoA9_sIbuKyuZaFF}
|
||||
init_password.db_password_confirm = ${PROLE_SECRET:v1:6fyOe4qCZmGGqAUU:vkFsVGYFxvXsPurt3QNO0Y5ZUDdfwMbvgG4N1ULprbCfA0XeVoA9_sIbuKyuZaFF}
|
||||
init_password.db_username = root
|
||||
init_password.generate_ssh_key = true
|
||||
init_scripts.run_scripts = true
|
||||
kerberos_config.enabled = true
|
||||
kerberos_config.kdc = 10.0.0.3
|
||||
kerberos_config.enabled = True
|
||||
kerberos_config.init_authority = false
|
||||
kerberos_config.kdc = 10.0.0.196
|
||||
kerberos_config.password = ${PROLE_SECRET:v1:S-C7rQ5Tq-o5A9h2:fdSoP54DCSrsRz9ods8EU353dwEnI1bh-Mrr5b6q03d6O5yD}
|
||||
kerberos_config.realm = PROLE.ORG
|
||||
kerberos_config.test_connection = false
|
||||
@ -72,24 +79,32 @@ ollama_config.server_host =
|
||||
ollama_config.server_port = 11434
|
||||
|
||||
[Global]
|
||||
ARGOCD_NAMESPACE = argocd
|
||||
CLUSTER_ENV = prod
|
||||
CLUSTER_NAME = knoe-db
|
||||
DATABASE_NAMESPACE = knoe-db-17-7-043-18-140-17-7-043-18-140
|
||||
DB_HOST_PORT = 5432
|
||||
DB_PASSWORD =
|
||||
DB_PASSWORD = ${PROLE_SECRET:v1:oQHkGbXHuEuLvwQq:PQ-d0-LB3xnYnDJiTU1MrViNRfDUsq7xNBsMngYizQJ-tq-RiJXwubKHxB7eDhJA}
|
||||
DEPLOYMENT_MODE = k8s
|
||||
DEPLOYMENT_TARGET = prole-prod-cluster
|
||||
DOCKER_IMPORT_DIR =
|
||||
DOCKER_PRELOAD = false
|
||||
K3S_SERVER = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:fSDP1aKQWKjCX2lW:yeepqWGp6vit43ng2cCtu_MIp9gWxdjaSp7RtEiGPv8JYQW_w7aVQnT64LwXG2jh6xEpBAhob-2NJQkvWIlA8AinJt5kqdiDCJp5HXRyrZ5fj3vfsjURsAOq-jjLyu9oGI4ZAPVCwZL6uC4G8YLrXtPCTbAdZqieYUViWQ==}
|
||||
KNOE_DB_USER = root
|
||||
OPENTOFU_URL = http://127.0.0.1:8080
|
||||
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
|
||||
PROLE_HOME = /Users/chrisfu/dev/prole
|
||||
PROLE_K3S_SERVER = https://myrddin.prole.org:6443
|
||||
PROLE_K3S_TOKEN = K107c8c6000488eca4a067d8a73119bbae2f07b4ea1bac7d8d3dc9c500cbb8acb18::server:04572345810eae2f9619a6ed4239702b
|
||||
PROLE_OPENTOFU_URL = http://127.0.0.1:8080
|
||||
REGISTRY_NAMESPACE = knoe-system
|
||||
SERVICE_NAMESPACE = knoe-system
|
||||
|
||||
[Welcome]
|
||||
|
||||
[Dependencies]
|
||||
STATUS = All installed
|
||||
|
||||
[Network]
|
||||
AD_DC_HOST = myrddin.prole.org
|
||||
@ -100,10 +115,25 @@ ANSIBLE_INVENTORY = /Users/chrisfu/dev/prole/infrastructure/inventory
|
||||
ANSIBLE_REALM = PROLE.ORG
|
||||
ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["pi.prole.org","merlin.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]}
|
||||
KDC_ANSIBLE_DETECTED = 10.0.0.3
|
||||
KDC_AUTO_DETECTED = 10.0.0.196
|
||||
KERBEROS_AUTO_ENABLED = True
|
||||
|
||||
[Port Forwards]
|
||||
PORT_FORWARD_K3D_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD
|
||||
PORT_FORWARD_K3D_MAPPING_2 = id=garage;namespace=knoe-system;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3
|
||||
PORT_FORWARD_K3D_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao;address=0.0.0.0;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao
|
||||
PORT_FORWARD_K3D_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu
|
||||
PORT_FORWARD_K3D_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard
|
||||
PORT_FORWARD_K3D_MAPPING_6 = id=postgres;namespace=${DATABASE_NAMESPACE};target=svc/knoe-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary)
|
||||
PORT_FORWARD_K3D_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI
|
||||
PORT_FORWARD_K3D_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI
|
||||
|
||||
[System Environment]
|
||||
PROLE_CONF = /Users/chrisfu/dev/prole/conf
|
||||
PROLE_DATA = /Users/chrisfu/dev/prole/data
|
||||
PROLE_HOME = /Users/chrisfu/dev/prole
|
||||
PROLE_LOGS = /Users/chrisfu/dev/prole/logs
|
||||
PROLE_SERVICE = /Users/chrisfu/dev/prole/etc
|
||||
|
||||
[Monitoring]
|
||||
|
||||
@ -112,17 +142,22 @@ KDC_ANSIBLE_DETECTED = 10.0.0.3
|
||||
[Ollama]
|
||||
|
||||
[Optional Features]
|
||||
AT_REST_ENCRYPTION_ENABLED = true
|
||||
GITOPS_ENABLED = false
|
||||
KERBEROS_ENABLED = true
|
||||
SUPABASE_ENABLED = false
|
||||
|
||||
[GitOps]
|
||||
|
||||
[Database Creation]
|
||||
DB_USER = root
|
||||
|
||||
[Initialize Cluster]
|
||||
|
||||
[Dev Cluster (k3d)]
|
||||
CLUSTER_ENV = dev
|
||||
CLUSTER_ENV = k3d-knoe-dev-cluster
|
||||
DISPLAY_NAME = knoe-dev-cluster
|
||||
KUBECTL_CONTEXT =
|
||||
KUBECTL_CONTEXT = prod
|
||||
MODE = k3d
|
||||
|
||||
[Service Cluster (k3s)]
|
||||
@ -134,7 +169,7 @@ MODE = k3s
|
||||
PIPELINE_URL = http://127.0.0.1:8080
|
||||
|
||||
[Prod Cluster (k8s)]
|
||||
ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging
|
||||
ARTIFACTS_DIR =
|
||||
CLUSTER_ENV = prole-prod-cluster
|
||||
DISPLAY_NAME = prole-prod-cluster
|
||||
MODE = k8s
|
||||
@ -149,4 +184,5 @@ MODE = k8s
|
||||
TARGET = prole-prod-cluster
|
||||
|
||||
[Install]
|
||||
STATUS = Failed
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
; Prole Master Configuration File
|
||||
; Generated by install.py on 2026-03-25 21:38:46
|
||||
; Generated by install.py on 2026-03-27 12:20:06
|
||||
; This file is used as input for Ansible deployment and k8s cluster creation.
|
||||
|
||||
[User]
|
||||
@ -68,8 +68,8 @@ init_db_build.run_build = true
|
||||
init_password.cluster_name = ${CLUSTER_NAME}
|
||||
init_password.db_host_port = 5432
|
||||
init_password.db_namespace = ${DATABASE_NAMESPACE}
|
||||
init_password.db_password = ${PROLE_SECRET:v1:Fop86hEFa2lJGNiO:uh6fuU2FDI56bq4B0vDcjh8tF959qSJc}
|
||||
init_password.db_password_confirm = ${PROLE_SECRET:v1:Fop86hEFa2lJGNiO:uh6fuU2FDI56bq4B0vDcjh8tF959qSJc}
|
||||
init_password.db_password = ${PROLE_SECRET:v1:7h2BjxdDiVq6dPdK:21sXsPKsCfG2IPA4H3tOd_2UmlKLZRh-}
|
||||
init_password.db_password_confirm = ${PROLE_SECRET:v1:7h2BjxdDiVq6dPdK:21sXsPKsCfG2IPA4H3tOd_2UmlKLZRh-}
|
||||
init_password.db_username = root
|
||||
init_password.generate_ssh_key = true
|
||||
init_scripts.run_scripts = true
|
||||
@ -106,7 +106,7 @@ K3S_SERVER = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:UQkxpgIK-ZjOTGmE:3Ij4G2imYkA2EfwyAO0a7jXYR1bSIVFxDtG9OdCZkl-x7URqHR5bXTWf6VCOMQmFS098w0wxA-P0dFwLDn1gLXNRHd-9sTyEMVP4rUz3TPaT-g89doklyjXkO0PjD_Eyf0FcB6GbKr2c8qGIsXyuGZE4O0GlcDlnRk5Pkg==}
|
||||
KNOE_DB_USER = root
|
||||
KUBECONTEXT = prole-k3s
|
||||
OPENTOFU_URL = http://127.0.0.1:8080
|
||||
OPENTOFU_URL = http://myrddin.prole.org:8080
|
||||
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
|
||||
PROLE_HOME = /Users/chrisfu/dev/prole
|
||||
PROLE_K3S_SERVER = https://myrddin.prole.org:6443
|
||||
@ -199,14 +199,14 @@ DISPLAY_NAME = prole-service-cluster
|
||||
K3S_SERVER_URL = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:F14dY4ATvenAtZtY:ZUKtV8vAeTQ8TdFElnUtHgZ9pgD38dquOopMY22CdgGdIFoWM5EppWfCzNVMCrM9-wEVTBrqsHDYrIGCx4RRhnHnNNgvAVGqTqroTZMOFs-aZp-VjjXJefSLRbYxzxgs_Q_llazC5xhuYXkAC0K09DkAqyp6ATjrs9F1SA==}
|
||||
MODE = k3s
|
||||
PIPELINE_URL = http://127.0.0.1:8080
|
||||
PIPELINE_URL = http://myrddin.prole.org:8080
|
||||
|
||||
[Prod Cluster (k8s)]
|
||||
ARTIFACTS_DIR =
|
||||
CLUSTER_ENV = prole-prod-cluster
|
||||
DISPLAY_NAME = prole-prod-cluster
|
||||
MODE = k8s
|
||||
PIPELINE_URL = http://127.0.0.1:8080
|
||||
PIPELINE_URL = http://myrddin.prole.org:8080
|
||||
|
||||
[Docker Build]
|
||||
LOCAL_REGISTRY = myrddin.prole.org:5000
|
||||
@ -217,8 +217,8 @@ STATUS = Attempted
|
||||
|
||||
[Deployment]
|
||||
MODE = k3s
|
||||
STATUS = Attempted
|
||||
STATUS = Deployed
|
||||
TARGET = prole-service-cluster
|
||||
|
||||
[Install]
|
||||
STATUS = Finished
|
||||
STATUS = Failed
|
||||
|
||||
@ -43,6 +43,7 @@ GARAGE_S3_ENDPOINT=${GARAGE_S3_ENDPOINT:-http://$GARAGE_NAME.$GARAGE_NAMESPACE.s
|
||||
GARAGE_S3_REGION=${GARAGE_S3_REGION:-garage}
|
||||
RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1}
|
||||
RETENTION_POLICY=${RETENTION_POLICY:-30d}
|
||||
GARAGE_LAYOUT_BOOTSTRAP_ENABLED=${GARAGE_LAYOUT_BOOTSTRAP_ENABLED:-1}
|
||||
BARMAN_PLUGIN_NAME=${BARMAN_PLUGIN_NAME:-barman-cloud.cloudnative-pg.io}
|
||||
BARMAN_OBJECT_NAME=${BARMAN_OBJECT_NAME:-knoe-db-barman-objectstore}
|
||||
BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600}
|
||||
@ -195,6 +196,29 @@ garage_exec() {
|
||||
kubectl exec -n "$GARAGE_NAMESPACE" "$pod" -- /garage "$@"
|
||||
}
|
||||
|
||||
bootstrap_garage_layout_if_needed() {
|
||||
if [[ "${GARAGE_LAYOUT_BOOTSTRAP_ENABLED}" != "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local init_script="$SCRIPT_DIR/init_garage_store.sh"
|
||||
if [[ ! -f "$init_script" ]]; then
|
||||
echo "WARN: Garage bootstrap helper not found: $init_script" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Garage layout still not applied; invoking Garage bootstrap recovery ..."
|
||||
if [[ -n "${PROLE_MODE:-}" ]]; then
|
||||
if ! bash "$init_script" --mode "$PROLE_MODE" start; then
|
||||
echo "WARN: Garage bootstrap recovery failed (continuing wait loop)." >&2
|
||||
fi
|
||||
else
|
||||
if ! bash "$init_script" start; then
|
||||
echo "WARN: Garage bootstrap recovery failed (continuing wait loop)." >&2
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
parse_key_output() {
|
||||
local output="$1"
|
||||
local access_key secret_key
|
||||
@ -209,6 +233,7 @@ parse_key_output() {
|
||||
ensure_garage_ready() {
|
||||
echo "Checking Garage readiness ..."
|
||||
local i status_out
|
||||
local bootstrap_attempted=0
|
||||
for i in {1..30}; do
|
||||
if status_out=$(garage_exec status 2>/dev/null); then
|
||||
# If layout is applied, DataAvail should eventually show something or at least the node should be healthy.
|
||||
@ -221,6 +246,10 @@ ensure_garage_ready() {
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
if [[ "$bootstrap_attempted" -eq 0 && "$i" -ge 6 ]]; then
|
||||
bootstrap_garage_layout_if_needed
|
||||
bootstrap_attempted=1
|
||||
fi
|
||||
echo "Waiting for Garage layout to be applied... ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
@ -581,6 +610,18 @@ wait_for_objectstore_ready() {
|
||||
|
||||
now=$(date +%s)
|
||||
elapsed=$((now - start_time))
|
||||
|
||||
# Some plugin/controller versions do not currently set status conditions on
|
||||
# ObjectStore. If the ObjectStore exists and barman-cloud is available for a
|
||||
# grace window, proceed with a warning instead of hard failing.
|
||||
local object_exists barman_available
|
||||
object_exists=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o name 2>/dev/null || true)
|
||||
barman_available=$(kubectl -n "$CNPG_OPERATOR_NAMESPACE" get deploy barman-cloud -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
|
||||
if [[ -n "$object_exists" && "$barman_available" =~ ^[1-9][0-9]*$ && $elapsed -ge 30 ]]; then
|
||||
echo "WARN: ObjectStore status fields are not populated, but barman-cloud is available; continuing."
|
||||
return 0
|
||||
fi
|
||||
|
||||
if (( elapsed >= timeout )); then
|
||||
echo "ERROR: Timed out waiting for ObjectStore '$BARMAN_OBJECT_NAME' to become Ready." >&2
|
||||
kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o yaml 2>/dev/null >&2 || true
|
||||
@ -621,6 +662,8 @@ has_successful_base_backup() {
|
||||
|
||||
wait_for_successful_base_backup() {
|
||||
local start_time now elapsed
|
||||
local plugin_retry_count=0
|
||||
local plugin_retry_limit=${PLUGIN_BACKUP_RETRY_LIMIT:-6}
|
||||
start_time=$(date +%s)
|
||||
while true; do
|
||||
if [[ -n "$LAST_BACKUP_NAME" ]]; then
|
||||
@ -632,6 +675,17 @@ wait_for_successful_base_backup() {
|
||||
return 0
|
||||
;;
|
||||
Failed|failed)
|
||||
local err_lc
|
||||
err_lc=$(echo "${err:-}" | tr '[:upper:]' '[:lower:]')
|
||||
if [[ "$err_lc" == *"requested plugin is not available"* ]] && (( plugin_retry_count < plugin_retry_limit )); then
|
||||
plugin_retry_count=$((plugin_retry_count + 1))
|
||||
echo "WARN: Backup '$LAST_BACKUP_NAME' failed because plugin is not yet available. Retry ${plugin_retry_count}/${plugin_retry_limit} ..."
|
||||
wait_for_plugin_ready || true
|
||||
LAST_BACKUP_NAME=""
|
||||
trigger_backup
|
||||
sleep 5
|
||||
continue
|
||||
fi
|
||||
echo "ERROR: Backup '$LAST_BACKUP_NAME' failed: ${err:-<no error provided>}" >&2
|
||||
return 1
|
||||
;;
|
||||
|
||||
@ -2,7 +2,7 @@ apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: prole-krb5-conf
|
||||
namespace: default
|
||||
namespace: knoe-system
|
||||
data:
|
||||
krb5.conf: |
|
||||
[libdefaults]
|
||||
|
||||
@ -55,8 +55,6 @@ spec:
|
||||
labels:
|
||||
app: opentofu
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: myrddin.prole.org
|
||||
containers:
|
||||
- name: opentofu-ui
|
||||
image: nginx:1.27-alpine
|
||||
|
||||
@ -15,8 +15,13 @@ spec:
|
||||
labels:
|
||||
app: garage
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: myrddin.prole.org
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Exists"
|
||||
effect: "NoSchedule"
|
||||
- key: "node-role.kubernetes.io/master"
|
||||
operator: "Exists"
|
||||
effect: "NoSchedule"
|
||||
containers:
|
||||
- name: garage
|
||||
image: dxflrs/garage:v1.3.1
|
||||
|
||||
@ -15,8 +15,6 @@ spec:
|
||||
labels:
|
||||
app: openbao
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: myrddin.prole.org
|
||||
containers:
|
||||
- name: openbao
|
||||
image: ghcr.io/openbao/openbao:2.0.0
|
||||
@ -43,15 +41,6 @@ spec:
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
storageClassName: synology-iscsi
|
||||
selector:
|
||||
matchLabels:
|
||||
synology.storage/role: openbao
|
||||
volumes:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
|
||||
@ -14,8 +14,6 @@ spec:
|
||||
labels:
|
||||
app: registry
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: myrddin.prole.org
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Exists"
|
||||
@ -29,27 +27,18 @@ spec:
|
||||
ports:
|
||||
- containerPort: 5000
|
||||
hostPort: 5000
|
||||
name: https
|
||||
name: http
|
||||
env:
|
||||
- name: REGISTRY_HTTP_ADDR
|
||||
value: "0.0.0.0:5000"
|
||||
- name: REGISTRY_HTTP_TLS_CERTIFICATE
|
||||
value: "/etc/ssl/certs/prole/myrddin-registry.crt"
|
||||
- name: REGISTRY_HTTP_TLS_KEY
|
||||
value: "/etc/ssl/certs/prole/myrddin-registry.key"
|
||||
volumeMounts:
|
||||
- name: registry-data
|
||||
mountPath: /var/lib/registry
|
||||
- name: registry-certs
|
||||
mountPath: /etc/ssl/certs/prole
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: registry-data
|
||||
emptyDir: {}
|
||||
- name: registry-certs
|
||||
hostPath:
|
||||
path: /etc/ssl/certs/prole
|
||||
type: Directory
|
||||
path: /var/lib/prole/registry
|
||||
type: DirectoryOrCreate
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
@ -61,7 +50,7 @@ spec:
|
||||
selector:
|
||||
app: registry
|
||||
ports:
|
||||
- name: https
|
||||
- name: http
|
||||
port: 5000
|
||||
targetPort: https
|
||||
targetPort: http
|
||||
type: ClusterIP
|
||||
|
||||
@ -599,9 +599,18 @@ class ProleInstaller:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
service_ns = (
|
||||
env.get("SERVICE_NAMESPACE")
|
||||
or (self.prole_cfg_data.get("Global", {}) or {}).get("SERVICE_NAMESPACE", "")
|
||||
or "knoe-system"
|
||||
)
|
||||
service_ns = str(service_ns).strip() or "knoe-system"
|
||||
argocd_ns = env.get("ARGOCD_NAMESPACE") or "argocd"
|
||||
registry_ns = env.get("REGISTRY_NAMESPACE") or "default"
|
||||
registry_ns = env.get("REGISTRY_NAMESPACE") or service_ns
|
||||
if str(registry_ns).strip() == "default":
|
||||
registry_ns = service_ns
|
||||
env["ARGOCD_NAMESPACE"] = argocd_ns
|
||||
env["SERVICE_NAMESPACE"] = service_ns
|
||||
env["REGISTRY_NAMESPACE"] = registry_ns
|
||||
|
||||
mode = _deployment_mode_from_env(
|
||||
@ -3876,7 +3885,9 @@ class ProleConsoleInstaller(ProleInstaller):
|
||||
)
|
||||
if not ns:
|
||||
ns = (os.environ.get("REGISTRY_NAMESPACE") or "").strip()
|
||||
if ns:
|
||||
# In service-mode k3s, keep registry in the service namespace and
|
||||
# avoid falling back to default (which causes duplicate deployments).
|
||||
if ns and not (ns == "default" and self._deployment_mode() == "k3s"):
|
||||
return ns
|
||||
|
||||
# Registry is a common-core service; default to the configured service namespace.
|
||||
@ -5977,11 +5988,35 @@ class ProleConsoleInstaller(ProleInstaller):
|
||||
self._create_k3d_cluster(cluster_name, prole_data)
|
||||
elif env_key in ("service", "prole-service-cluster", "k3s"):
|
||||
self.log("==> Cluster reset (k3s namespace cleanup)")
|
||||
ns = self._get_input("env_setup.DATABASE_NAMESPACE", "")
|
||||
if not ns:
|
||||
ns = self._get_input("env_setup.NAMESPACE", "")
|
||||
if not ns:
|
||||
ns = "default"
|
||||
db_ns = self._get_input("env_setup.DATABASE_NAMESPACE", "")
|
||||
if not db_ns:
|
||||
db_ns = self._get_input("env_setup.NAMESPACE", "")
|
||||
if not db_ns:
|
||||
db_ns = "default"
|
||||
db_ns = db_ns.strip() or "default"
|
||||
|
||||
service_ns = self._service_namespace().strip() or "knoe-system"
|
||||
monitoring_ns = (
|
||||
((self.prole_cfg_data.get("Global", {}) or {}).get("MONITORING_NAMESPACE", "") or "").strip()
|
||||
or (os.environ.get("MONITORING_NAMESPACE") or "").strip()
|
||||
or "monitoring"
|
||||
)
|
||||
cnpg_ns = (
|
||||
((self.prole_cfg_data.get("Global", {}) or {}).get("CNPG_NAMESPACE", "") or "").strip()
|
||||
or (os.environ.get("CNPG_NAMESPACE") or "").strip()
|
||||
or "cnpg-system"
|
||||
)
|
||||
argocd_ns = self._argocd_namespace().strip() or "argocd"
|
||||
registry_ns = service_ns
|
||||
|
||||
reset_namespaces: list[str] = []
|
||||
for candidate in [db_ns, service_ns, monitoring_ns, cnpg_ns, argocd_ns, "default"]:
|
||||
ns = (candidate or "").strip()
|
||||
if not ns or (ns.startswith("${") and ns.endswith("}")):
|
||||
continue
|
||||
if ns not in reset_namespaces:
|
||||
reset_namespaces.append(ns)
|
||||
|
||||
server = self._get_input("init_cluster.k3s_server_url", "")
|
||||
token = self._get_input("init_cluster.k3s_token", "")
|
||||
|
||||
@ -5990,7 +6025,20 @@ class ProleConsoleInstaller(ProleInstaller):
|
||||
# Ensure we have a cert-based kubeconfig (or a valid bearer token config)
|
||||
# before invoking any kubectl-based reset scripts.
|
||||
self._ensure_project_k3s_kubeconfig(server, token)
|
||||
_reset_k3s_namespace(self.project_root, ns, server, token)
|
||||
self.log(
|
||||
f"[RESET] Namespaces to clean (PVCs preserved): {', '.join(reset_namespaces)}"
|
||||
)
|
||||
self.log(
|
||||
f"[RESET] Preserving registry workload in namespace: {registry_ns}"
|
||||
)
|
||||
_reset_k3s_namespace(
|
||||
self.project_root,
|
||||
reset_namespaces,
|
||||
server,
|
||||
token,
|
||||
registry_namespace=registry_ns,
|
||||
clear_node_reservations=True,
|
||||
)
|
||||
self._cleanup_local_k3s_artifacts()
|
||||
|
||||
self.reset_requested = False
|
||||
@ -6562,12 +6610,35 @@ def _attempt_k3s_repair(
|
||||
|
||||
|
||||
def _reset_k3s_namespace(
|
||||
project_root: Path, namespace: str, server: str, token: str
|
||||
project_root: Path,
|
||||
namespace: str | Sequence[str],
|
||||
server: str,
|
||||
token: str,
|
||||
*,
|
||||
registry_namespace: str | None = None,
|
||||
clear_node_reservations: bool = False,
|
||||
) -> None:
|
||||
script = project_root / "scripts" / "reset-ns.sh"
|
||||
if not script.exists():
|
||||
print(f"[WARN] Namespace reset script not found: {script}", file=sys.stderr)
|
||||
return
|
||||
|
||||
if isinstance(namespace, str):
|
||||
raw_namespaces = [namespace]
|
||||
else:
|
||||
raw_namespaces = list(namespace or [])
|
||||
|
||||
namespaces: list[str] = []
|
||||
for ns in raw_namespaces:
|
||||
item = str(ns or "").strip()
|
||||
if not item:
|
||||
continue
|
||||
if item not in namespaces:
|
||||
namespaces.append(item)
|
||||
|
||||
if not namespaces:
|
||||
return
|
||||
|
||||
env = os.environ.copy()
|
||||
if not env.get("KUBECONFIG"):
|
||||
repo_kubeconfig = project_root / "prole-k3s.kubeconfig"
|
||||
@ -6576,7 +6647,20 @@ def _reset_k3s_namespace(
|
||||
elif server and token and _looks_like_k8s_bearer_token(token):
|
||||
kubeconfig_path = _write_k3s_kubeconfig(server, token)
|
||||
env["KUBECONFIG"] = str(kubeconfig_path)
|
||||
subprocess.run(["bash", str(script), "-n", namespace], env=env)
|
||||
|
||||
reg_ns = (registry_namespace or "").strip()
|
||||
for idx, ns in enumerate(namespaces):
|
||||
cmd = ["bash", str(script), "-n", ns]
|
||||
if reg_ns and ns == reg_ns:
|
||||
cmd.append("--keep-registry")
|
||||
if clear_node_reservations and idx == (len(namespaces) - 1):
|
||||
cmd.append("--clear-node-reservations")
|
||||
res = subprocess.run(cmd, env=env)
|
||||
if res.returncode != 0:
|
||||
print(
|
||||
f"[WARN] Namespace reset failed for {ns} (code {res.returncode})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_k3s_pipeline(
|
||||
|
||||
@ -992,6 +992,14 @@ class KerberosMilestone(Milestone):
|
||||
enabled = self._parse_bool(
|
||||
state.inputs.get("kerberos_config.enabled", "False"), default=False
|
||||
)
|
||||
if not enabled:
|
||||
realm = str(state.inputs.get("kerberos_config.realm", "") or "").strip()
|
||||
kdc = str(state.inputs.get("kerberos_config.kdc", "") or "").strip()
|
||||
if realm and kdc:
|
||||
self.logger.info(
|
||||
"Kerberos enabled implicitly because realm and kdc are configured."
|
||||
)
|
||||
enabled = True
|
||||
krb_sec = state.config_data.setdefault("Kerberos Authentication", {})
|
||||
if not enabled:
|
||||
krb_sec["STATUS"] = "Skipped"
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
@ -114,12 +115,16 @@ def _namespace(namespace: str | None, env: dict | None, default: str = "default"
|
||||
|
||||
|
||||
def _registry_namespace(namespace: str | None, env: dict | None) -> str:
|
||||
service_ns = str((env or {}).get("SERVICE_NAMESPACE") or "").strip()
|
||||
raw = (
|
||||
(namespace or "").strip()
|
||||
or str((env or {}).get("REGISTRY_NAMESPACE") or "").strip()
|
||||
or service_ns
|
||||
or str((env or {}).get("NAMESPACE") or "").strip()
|
||||
or "default"
|
||||
or "knoe-system"
|
||||
)
|
||||
if raw == "default" and service_ns:
|
||||
return service_ns
|
||||
return raw
|
||||
|
||||
|
||||
@ -168,3 +173,138 @@ def _to_bool(value: str | bool | None, default: bool = False) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _prune_named_workload_other_namespaces(
|
||||
*,
|
||||
kind: str,
|
||||
name: str,
|
||||
target_namespace: str,
|
||||
label_selector: str,
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
) -> None:
|
||||
listed = _kubectl(
|
||||
["get", kind, "-A", "-l", label_selector, "-o", "json"],
|
||||
env=env,
|
||||
timeout=45,
|
||||
)
|
||||
if listed.returncode != 0 or not listed.stdout.strip():
|
||||
return
|
||||
try:
|
||||
payload = json.loads(listed.stdout)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
for item in payload.get("items", []):
|
||||
meta = item.get("metadata") or {}
|
||||
ns = str(meta.get("namespace") or "").strip()
|
||||
item_name = str(meta.get("name") or "").strip()
|
||||
if not ns or ns == target_namespace or item_name != name:
|
||||
continue
|
||||
_log(
|
||||
log,
|
||||
f"[SERVICES] Removing duplicate {kind}/{name} from namespace {ns} (target namespace: {target_namespace})",
|
||||
)
|
||||
_kubectl(
|
||||
["-n", ns, "delete", kind, name, "--ignore-not-found"],
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def _reconcile_deployment_replicasets(
|
||||
*,
|
||||
deployment: str,
|
||||
namespace: str,
|
||||
label_selector: str,
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
) -> None:
|
||||
dep_res = _kubectl(
|
||||
["-n", namespace, "get", "deployment", deployment, "-o", "json"],
|
||||
env=env,
|
||||
timeout=30,
|
||||
)
|
||||
if dep_res.returncode != 0 or not dep_res.stdout.strip():
|
||||
return
|
||||
try:
|
||||
dep = json.loads(dep_res.stdout)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
dep_meta = dep.get("metadata") or {}
|
||||
dep_uid = str(dep_meta.get("uid") or "").strip()
|
||||
dep_annotations = dep_meta.get("annotations") or {}
|
||||
try:
|
||||
dep_revision = int(dep_annotations.get("deployment.kubernetes.io/revision") or "0")
|
||||
except Exception:
|
||||
dep_revision = 0
|
||||
|
||||
rs_res = _kubectl(
|
||||
["-n", namespace, "get", "replicaset", "-l", label_selector, "-o", "json"],
|
||||
env=env,
|
||||
timeout=45,
|
||||
)
|
||||
if rs_res.returncode != 0 or not rs_res.stdout.strip():
|
||||
return
|
||||
try:
|
||||
payload = json.loads(rs_res.stdout)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
def _rs_revision(item: dict) -> int:
|
||||
annotations = (item.get("metadata") or {}).get("annotations") or {}
|
||||
try:
|
||||
return int(annotations.get("deployment.kubernetes.io/revision") or "0")
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
owned: list[dict] = []
|
||||
for item in payload.get("items", []):
|
||||
meta = item.get("metadata") or {}
|
||||
owners = meta.get("ownerReferences") or []
|
||||
for owner in owners:
|
||||
if str(owner.get("kind") or "") != "Deployment":
|
||||
continue
|
||||
if str(owner.get("name") or "") != deployment:
|
||||
continue
|
||||
owner_uid = str(owner.get("uid") or "").strip()
|
||||
if dep_uid and owner_uid and owner_uid != dep_uid:
|
||||
continue
|
||||
owned.append(item)
|
||||
break
|
||||
|
||||
active = [
|
||||
item for item in owned if int((item.get("spec") or {}).get("replicas") or 0) > 0
|
||||
]
|
||||
if len(active) <= 1:
|
||||
return
|
||||
|
||||
stale: list[dict]
|
||||
if dep_revision > 0:
|
||||
stale = [item for item in active if _rs_revision(item) < dep_revision]
|
||||
else:
|
||||
# Fallback: keep newest revision/creation and scale down the rest.
|
||||
active_sorted = sorted(
|
||||
active,
|
||||
key=lambda item: (
|
||||
_rs_revision(item),
|
||||
str((item.get("metadata") or {}).get("creationTimestamp") or ""),
|
||||
),
|
||||
)
|
||||
stale = active_sorted[:-1]
|
||||
|
||||
for item in stale:
|
||||
rs_name = str((item.get("metadata") or {}).get("name") or "").strip()
|
||||
if not rs_name:
|
||||
continue
|
||||
_log(
|
||||
log,
|
||||
f"[SERVICES] Scaling stale replicaset/{rs_name} to 0 in namespace {namespace} (deployment={deployment})",
|
||||
)
|
||||
_kubectl(
|
||||
["-n", namespace, "scale", f"replicaset/{rs_name}", "--replicas=0"],
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
@ -119,6 +119,65 @@ def _release_stale_pv_claims(
|
||||
_log(log, f"Released PV '{pv_name}'.")
|
||||
|
||||
|
||||
def _reconcile_unbound_synology_data_wal_pvs(
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
) -> None:
|
||||
"""Recreate unbound static Synology data/wal PVs from manifest.
|
||||
|
||||
This allows nodeAffinity/layout changes in `iscsi-pvs.yaml` to take effect,
|
||||
while preserving currently bound volumes.
|
||||
"""
|
||||
pv_names = [
|
||||
"synology-iscsi-d001-data",
|
||||
"synology-iscsi-d001-wal",
|
||||
"synology-iscsi-d002-data",
|
||||
"synology-iscsi-d002-wal",
|
||||
"synology-iscsi-d003-data",
|
||||
"synology-iscsi-d003-wal",
|
||||
]
|
||||
|
||||
unbound: list[str] = []
|
||||
for name in pv_names:
|
||||
r = _kubectl(["get", "pv", name, "-o", "json"], env=env, timeout=20)
|
||||
if r.returncode != 0:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(r.stdout or "{}")
|
||||
except Exception:
|
||||
continue
|
||||
phase = str((payload.get("status") or {}).get("phase") or "").strip()
|
||||
if phase != "Bound":
|
||||
unbound.append(name)
|
||||
|
||||
if not unbound:
|
||||
return
|
||||
|
||||
_log(log, f"Reconciling unbound Synology data/wal PVs: {', '.join(unbound)}")
|
||||
for name in unbound:
|
||||
_kubectl(["delete", "pv", name, "--ignore-not-found"], env=env, timeout=45)
|
||||
|
||||
root = Path(
|
||||
(env or {}).get("PROLE_HOME")
|
||||
or (env or {}).get("PROLE_SERVICE")
|
||||
or Path(__file__).resolve().parents[4]
|
||||
)
|
||||
manifest = root / "k8s" / "prole" / "iscsi-pvs.yaml"
|
||||
if not manifest.exists():
|
||||
_log(log, f"WARN: Synology PV manifest not found: {manifest}")
|
||||
return
|
||||
|
||||
r_apply = _kubectl(["apply", "-f", str(manifest)], env=env, timeout=180)
|
||||
if r_apply.returncode != 0:
|
||||
stderr = ((r_apply.stderr or "") + "\n" + (r_apply.stdout or "")).strip().lower()
|
||||
if "field is immutable" not in stderr:
|
||||
raise RuntimeError(
|
||||
f"Failed to re-apply Synology PV manifest {manifest}: "
|
||||
f"{(r_apply.stderr or r_apply.stdout or '').strip()}"
|
||||
)
|
||||
_log(log, "WARN: Immutable PV fields prevented full re-apply; continuing with recreated unbound PVs.")
|
||||
|
||||
|
||||
def _ensure_namespace(namespace: str, env: dict | None = None) -> None:
|
||||
"""Create the namespace if it does not already exist (idempotent)."""
|
||||
manifest = (
|
||||
@ -333,6 +392,166 @@ def _reconcile_instances(
|
||||
return
|
||||
|
||||
|
||||
def _controller_manifest_url(version: str) -> str:
|
||||
minor = ".".join(version.split(".")[:2])
|
||||
return (
|
||||
f"https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg"
|
||||
f"/release-{minor}/releases/cnpg-{version}.yaml"
|
||||
)
|
||||
|
||||
|
||||
def _controller_deployment_json(env: dict | None = None) -> dict | None:
|
||||
r = _kubectl(
|
||||
["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager", "-o", "json"],
|
||||
env=env,
|
||||
timeout=30,
|
||||
)
|
||||
if r.returncode != 0 or not (r.stdout or "").strip():
|
||||
return None
|
||||
try:
|
||||
return json.loads(r.stdout)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _controller_is_healthy(env: dict | None = None) -> bool:
|
||||
dep = _controller_deployment_json(env)
|
||||
if not dep:
|
||||
return False
|
||||
spec = dep.get("spec") or {}
|
||||
status = dep.get("status") or {}
|
||||
desired = int(spec.get("replicas") or 1)
|
||||
available = int(status.get("availableReplicas") or 0)
|
||||
return desired > 0 and available >= desired
|
||||
|
||||
|
||||
def _reconcile_controller_replicasets(
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
aggressive_cleanup: bool = False,
|
||||
) -> None:
|
||||
dep = _controller_deployment_json(env)
|
||||
if not dep:
|
||||
return
|
||||
|
||||
dep_meta = dep.get("metadata") or {}
|
||||
dep_uid = str(dep_meta.get("uid") or "").strip()
|
||||
dep_annotations = dep_meta.get("annotations") or {}
|
||||
try:
|
||||
dep_revision = int(dep_annotations.get("deployment.kubernetes.io/revision") or "0")
|
||||
except Exception:
|
||||
dep_revision = 0
|
||||
|
||||
r = _kubectl(
|
||||
[
|
||||
"-n",
|
||||
"cnpg-system",
|
||||
"get",
|
||||
"rs",
|
||||
"-l",
|
||||
"app.kubernetes.io/name=cloudnative-pg",
|
||||
"-o",
|
||||
"json",
|
||||
],
|
||||
env=env,
|
||||
timeout=45,
|
||||
)
|
||||
if r.returncode != 0 or not (r.stdout or "").strip():
|
||||
return
|
||||
try:
|
||||
payload = json.loads(r.stdout)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
def _rs_revision(item: dict) -> int:
|
||||
ann = (item.get("metadata") or {}).get("annotations") or {}
|
||||
try:
|
||||
return int(ann.get("deployment.kubernetes.io/revision") or "0")
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
owned: list[dict] = []
|
||||
for item in payload.get("items") or []:
|
||||
meta = item.get("metadata") or {}
|
||||
owners = meta.get("ownerReferences") or []
|
||||
for owner in owners:
|
||||
if str(owner.get("kind") or "") != "Deployment":
|
||||
continue
|
||||
if str(owner.get("name") or "") != "cnpg-controller-manager":
|
||||
continue
|
||||
owner_uid = str(owner.get("uid") or "").strip()
|
||||
if dep_uid and owner_uid and owner_uid != dep_uid:
|
||||
continue
|
||||
owned.append(item)
|
||||
break
|
||||
|
||||
active = [i for i in owned if int((i.get("spec") or {}).get("replicas") or 0) > 0]
|
||||
if len(active) <= 1:
|
||||
return
|
||||
|
||||
stale: list[dict]
|
||||
if dep_revision > 0:
|
||||
stale = [i for i in active if _rs_revision(i) < dep_revision]
|
||||
else:
|
||||
active_sorted = sorted(
|
||||
active,
|
||||
key=lambda i: (
|
||||
_rs_revision(i),
|
||||
str((i.get("metadata") or {}).get("creationTimestamp") or ""),
|
||||
),
|
||||
)
|
||||
stale = active_sorted[:-1]
|
||||
|
||||
for rs in stale:
|
||||
rs_meta = rs.get("metadata") or {}
|
||||
rs_name = str(rs_meta.get("name") or "").strip()
|
||||
if not rs_name:
|
||||
continue
|
||||
_log(log, f"Scaling stale CNPG controller replicaset/{rs_name} to 0")
|
||||
_kubectl(
|
||||
["-n", "cnpg-system", "scale", f"rs/{rs_name}", "--replicas=0"],
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
if aggressive_cleanup:
|
||||
template_hash = str((rs_meta.get("labels") or {}).get("pod-template-hash") or "").strip()
|
||||
if template_hash:
|
||||
_kubectl(
|
||||
[
|
||||
"-n",
|
||||
"cnpg-system",
|
||||
"delete",
|
||||
"pod",
|
||||
"-l",
|
||||
f"app.kubernetes.io/name=cloudnative-pg,pod-template-hash={template_hash}",
|
||||
"--ignore-not-found",
|
||||
"--wait=false",
|
||||
],
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def _wait_controller_rollout(
|
||||
env: dict | None = None,
|
||||
timeout_seconds: int = 180,
|
||||
) -> bool:
|
||||
r = _kubectl(
|
||||
[
|
||||
"-n",
|
||||
"cnpg-system",
|
||||
"rollout",
|
||||
"status",
|
||||
"deploy/cnpg-controller-manager",
|
||||
f"--timeout={timeout_seconds}s",
|
||||
],
|
||||
env=env,
|
||||
timeout=timeout_seconds + 10,
|
||||
)
|
||||
return r.returncode == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
@ -342,37 +561,52 @@ def ensure_operator(
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
mode: str = "",
|
||||
allow_reapply_if_unhealthy: bool = False,
|
||||
aggressive_cleanup: bool = False,
|
||||
) -> None:
|
||||
"""Install the CloudNativePG operator if not already present, then wait for readiness."""
|
||||
if _kubectl_ok(["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager"], env=env):
|
||||
_log(log, "CloudNative-PG operator already installed; waiting for readiness...")
|
||||
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=180s"], env=env, timeout=190)
|
||||
if mode == "k3s":
|
||||
_tune_operator_for_k3s(env, log)
|
||||
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=300s"], env=env, timeout=310)
|
||||
return
|
||||
_reconcile_controller_replicasets(env=env, log=log, aggressive_cleanup=aggressive_cleanup)
|
||||
if _controller_is_healthy(env=env):
|
||||
_log(log, "CloudNative-PG operator already healthy; skipping operator re-apply.")
|
||||
if mode == "k3s":
|
||||
_tune_operator_for_k3s(env, log)
|
||||
_wait_controller_rollout(env=env, timeout_seconds=300)
|
||||
return
|
||||
|
||||
_log(log, "CloudNative-PG operator exists but is not healthy; waiting for recovery...")
|
||||
if _wait_controller_rollout(env=env, timeout_seconds=180):
|
||||
if mode == "k3s":
|
||||
_tune_operator_for_k3s(env, log)
|
||||
_wait_controller_rollout(env=env, timeout_seconds=300)
|
||||
return
|
||||
|
||||
if not allow_reapply_if_unhealthy:
|
||||
raise RuntimeError(
|
||||
"CNPG operator deployment exists but is not ready; "
|
||||
"refusing disruptive re-apply in non-initialize flow."
|
||||
)
|
||||
_log(log, "CNPG operator unhealthy after wait; attempting controlled re-apply...")
|
||||
|
||||
version = ""
|
||||
if env:
|
||||
version = (env.get("CNPG_OPERATOR_VERSION") or env.get("CNPG_VERSION") or "").strip()
|
||||
if not version or version == "latest":
|
||||
version = _CNPG_OPERATOR_FALLBACK_VERSION
|
||||
|
||||
minor = ".".join(version.split(".")[:2])
|
||||
url = (
|
||||
f"https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg"
|
||||
f"/release-{minor}/releases/cnpg-{version}.yaml"
|
||||
)
|
||||
url = _controller_manifest_url(version)
|
||||
_log(log, f"Installing CloudNative-PG operator version {version}...")
|
||||
r = _kubectl(["apply", "--server-side", "-f", url], env=env, timeout=120)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError(f"Failed to install CNPG operator: {(r.stderr or r.stdout or '').strip()}")
|
||||
|
||||
if _kubectl_ok(["-n", "cnpg-system", "get", "deploy", "cnpg-controller-manager"], env=env, timeout=20):
|
||||
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=180s"], env=env, timeout=190)
|
||||
_reconcile_controller_replicasets(env=env, log=log, aggressive_cleanup=aggressive_cleanup)
|
||||
if not _wait_controller_rollout(env=env, timeout_seconds=180):
|
||||
raise RuntimeError("CNPG controller rollout did not complete after operator apply")
|
||||
if mode == "k3s":
|
||||
_tune_operator_for_k3s(env, log)
|
||||
_kubectl(["-n", "cnpg-system", "rollout", "status", "deploy/cnpg-controller-manager", "--timeout=300s"], env=env, timeout=310)
|
||||
if not _wait_controller_rollout(env=env, timeout_seconds=300):
|
||||
raise RuntimeError("CNPG controller rollout did not complete after k3s tuning")
|
||||
|
||||
|
||||
def _tune_operator_for_k3s(env: dict | None, log: _LogFn | None) -> None:
|
||||
@ -382,6 +616,18 @@ def _tune_operator_for_k3s(env: dict | None, log: _LogFn | None) -> None:
|
||||
"spec": {
|
||||
"template": {
|
||||
"spec": {
|
||||
"tolerations": [
|
||||
{
|
||||
"key": "node-role.kubernetes.io/control-plane",
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule",
|
||||
},
|
||||
{
|
||||
"key": "node-role.kubernetes.io/master",
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule",
|
||||
},
|
||||
],
|
||||
"containers": [
|
||||
{
|
||||
"name": "manager",
|
||||
@ -404,6 +650,17 @@ def _tune_operator_for_k3s(env: dict | None, log: _LogFn | None) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _clear_controller_pin(env: dict | None = None, log: _LogFn | None = None) -> None:
|
||||
"""Remove stale hard nodeSelector pinning for controller-manager."""
|
||||
_log(log, "Clearing CNPG controller hard nodeSelector pin (if present)...")
|
||||
patch = json.dumps({"spec": {"template": {"spec": {"nodeSelector": None}}}})
|
||||
_kubectl(
|
||||
["-n", "cnpg-system", "patch", "deployment", "cnpg-controller-manager", "--type", "merge", "-p", patch],
|
||||
env=env,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def pin_controller(
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
@ -416,16 +673,10 @@ def pin_controller(
|
||||
r = _kubectl(["get", "nodes", "-l", "storage=primary", "--no-headers"], env=env, timeout=10)
|
||||
if r.returncode == 0 and (r.stdout or "").strip():
|
||||
selector = "storage=primary"
|
||||
else:
|
||||
r2 = _kubectl(
|
||||
["get", "nodes", "-l", "node-role.kubernetes.io/control-plane",
|
||||
"-o", "jsonpath={.items[0].metadata.name}"],
|
||||
env=env, timeout=10,
|
||||
)
|
||||
if r2.returncode == 0 and (r2.stdout or "").strip():
|
||||
selector = f"kubernetes.io/hostname={r2.stdout.strip()}"
|
||||
|
||||
if not selector:
|
||||
_clear_controller_pin(env=env, log=log)
|
||||
_reconcile_controller_replicasets(env=env, log=log)
|
||||
return
|
||||
|
||||
if "=" not in selector:
|
||||
@ -444,6 +695,7 @@ def pin_controller(
|
||||
"--type", "merge", "-p", patch],
|
||||
env=env, timeout=30,
|
||||
)
|
||||
_reconcile_controller_replicasets(env=env, log=log)
|
||||
|
||||
|
||||
def install_barman_plugin(
|
||||
@ -464,11 +716,8 @@ def install_barman_plugin(
|
||||
if r.returncode != 0:
|
||||
combined = (r.stdout or "") + (r.stderr or "")
|
||||
if "webhook.cert-manager.io" in combined.lower() or "cert-manager" in combined.lower():
|
||||
_log(log, "WARN: cert-manager webhook error; restarting cert-manager and retrying...")
|
||||
for deploy in ["cert-manager", "cert-manager-webhook", "cert-manager-cainjector"]:
|
||||
_kubectl(["-n", "cert-manager", "rollout", "restart", f"deploy/{deploy}"], env=env, timeout=30)
|
||||
_kubectl(["-n", "cert-manager", "rollout", "status", "deploy/cert-manager-webhook", "--timeout=180s"], env=env, timeout=190)
|
||||
_kubectl(["-n", "cert-manager", "rollout", "status", "deploy/cert-manager", "--timeout=180s"], env=env, timeout=190)
|
||||
_log(log, "WARN: cert-manager webhook error; ensuring cert-manager and retrying...")
|
||||
_ensure_cert_manager_for_barman(env=env, log=log)
|
||||
r2 = subprocess.run(
|
||||
["kubectl", "apply", "-f", url],
|
||||
env=env, capture_output=True, text=True, timeout=120,
|
||||
@ -495,6 +744,88 @@ def install_barman_plugin(
|
||||
)
|
||||
|
||||
|
||||
def _ensure_cert_manager_for_barman(
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
) -> None:
|
||||
def _exists(kind: str, name: str) -> bool:
|
||||
return _kubectl(["-n", "cert-manager", "get", kind, name], env=env, timeout=20).returncode == 0
|
||||
|
||||
need_bootstrap = not all(
|
||||
[
|
||||
_exists("service", "cert-manager-webhook"),
|
||||
_exists("deployment", "cert-manager"),
|
||||
_exists("deployment", "cert-manager-webhook"),
|
||||
_exists("deployment", "cert-manager-cainjector"),
|
||||
]
|
||||
)
|
||||
|
||||
if need_bootstrap:
|
||||
_log(log, "[CNPG] cert-manager resources missing; bootstrapping cert-manager...")
|
||||
root = Path(
|
||||
(env or {}).get("PROLE_HOME")
|
||||
or (env or {}).get("PROLE_SERVICE")
|
||||
or Path(__file__).resolve().parents[4]
|
||||
)
|
||||
init_script = root / "etc" / "init_certmgr.sh"
|
||||
if init_script.exists():
|
||||
cmd = ["bash", str(init_script)]
|
||||
mode = str((env or {}).get("PROLE_MODE") or "").strip()
|
||||
if mode:
|
||||
cmd.extend(["--mode", mode])
|
||||
cfg_hint = (
|
||||
str((env or {}).get("PROLE_CONFIG_PATH") or "").strip()
|
||||
or str((env or {}).get("PROLE_CFG_PATH") or "").strip()
|
||||
or str((env or {}).get("PROLE_CFG") or "").strip()
|
||||
)
|
||||
if cfg_hint:
|
||||
cmd.extend(["-c", cfg_hint])
|
||||
cmd.append("start")
|
||||
r = subprocess.run(
|
||||
cmd,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError(
|
||||
"Failed to bootstrap cert-manager via init_certmgr.sh: "
|
||||
f"{(r.stderr or r.stdout or '').strip()}"
|
||||
)
|
||||
else:
|
||||
_kubectl(["get", "namespace", "cert-manager"], env=env, timeout=20)
|
||||
_kubectl(["create", "namespace", "cert-manager"], env=env, timeout=20)
|
||||
certmgr_version = str((env or {}).get("CERTMGR_VERSION") or "v1.14.6").strip()
|
||||
certmgr_url = (
|
||||
"https://github.com/cert-manager/cert-manager/releases/download/"
|
||||
f"{certmgr_version}/cert-manager.yaml"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["kubectl", "apply", "-f", certmgr_url],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=180,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to bootstrap cert-manager from {certmgr_url}: "
|
||||
f"{(r.stderr or r.stdout or '').strip()}"
|
||||
)
|
||||
|
||||
for deploy in ["cert-manager", "cert-manager-webhook", "cert-manager-cainjector"]:
|
||||
if _exists("deployment", deploy):
|
||||
_kubectl(["-n", "cert-manager", "rollout", "restart", f"deploy/{deploy}"], env=env, timeout=45)
|
||||
for deploy in ["cert-manager-webhook", "cert-manager", "cert-manager-cainjector"]:
|
||||
if _exists("deployment", deploy):
|
||||
_kubectl(
|
||||
["-n", "cert-manager", "rollout", "status", f"deploy/{deploy}", "--timeout=240s"],
|
||||
env=env,
|
||||
timeout=250,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_barman_plugin_tag(env: dict | None, log: _LogFn | None) -> str:
|
||||
"""Fetch the latest Barman Cloud plugin release tag from GitHub, with fallback."""
|
||||
fallback = ((env or {}).get("BARMAN_PLUGIN_FALLBACK_VERSION") or _BARMAN_PLUGIN_FALLBACK_VERSION).strip()
|
||||
@ -715,7 +1046,13 @@ def initialize(
|
||||
"""
|
||||
_log(log, f"Initializing CNPG cluster '{cluster_name}' in namespace '{namespace}'...")
|
||||
|
||||
ensure_operator(env=env, log=log, mode=mode)
|
||||
ensure_operator(
|
||||
env=env,
|
||||
log=log,
|
||||
mode=mode,
|
||||
allow_reapply_if_unhealthy=True,
|
||||
aggressive_cleanup=True,
|
||||
)
|
||||
pin_controller(env=env, log=log, mode=mode)
|
||||
install_barman_plugin(env=env, log=log)
|
||||
|
||||
@ -738,6 +1075,7 @@ def initialize(
|
||||
_log(log, "Pre-flight check passed: all required secrets present.")
|
||||
# Release any Released PVs so new PVCs can bind (idempotent, k3s only)
|
||||
if mode == "k3s":
|
||||
_reconcile_unbound_synology_data_wal_pvs(env=env, log=log)
|
||||
_release_stale_pv_claims(env=env, log=log)
|
||||
manifest = _resolve_cnpg_manifest(project_root, env)
|
||||
_apply_manifest(namespace, manifest, env, log)
|
||||
@ -768,7 +1106,13 @@ def deploy(
|
||||
"""
|
||||
_log(log, f"Deploying CNPG cluster '{cluster_name}' in namespace '{namespace}'...")
|
||||
|
||||
ensure_operator(env=env, log=log, mode=mode)
|
||||
ensure_operator(
|
||||
env=env,
|
||||
log=log,
|
||||
mode=mode,
|
||||
allow_reapply_if_unhealthy=False,
|
||||
aggressive_cleanup=False,
|
||||
)
|
||||
pin_controller(env=env, log=log, mode=mode)
|
||||
install_barman_plugin(env=env, log=log)
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from ._services_common import (
|
||||
@ -10,18 +12,23 @@ from ._services_common import (
|
||||
_kubectl,
|
||||
_log,
|
||||
_manifest_path,
|
||||
_namespace,
|
||||
_prune_named_workload_other_namespaces,
|
||||
_to_bool,
|
||||
_wait_rollout,
|
||||
)
|
||||
|
||||
|
||||
def _garage_namespace(namespace: str | None, env: dict | None) -> str:
|
||||
service_ns = str((env or {}).get("SERVICE_NAMESPACE") or "").strip()
|
||||
if env:
|
||||
explicit = str(env.get("GARAGE_NAMESPACE") or "").strip()
|
||||
if explicit:
|
||||
return explicit
|
||||
return _namespace(namespace, env, default="default")
|
||||
return service_ns if explicit == "default" and service_ns else explicit
|
||||
|
||||
raw = (namespace or "").strip() or service_ns or str((env or {}).get("NAMESPACE") or "").strip() or "knoe-system"
|
||||
if raw == "default":
|
||||
return service_ns or "knoe-system"
|
||||
return raw
|
||||
|
||||
|
||||
def _manifest_files(project_root: str | Path) -> list[Path]:
|
||||
@ -35,6 +42,71 @@ def _manifest_files(project_root: str | Path) -> list[Path]:
|
||||
]
|
||||
|
||||
|
||||
def _repair_released_garage_pvs(*, env: dict | None = None, log: _LogFn | None = None) -> None:
|
||||
listed = _kubectl(
|
||||
["get", "pv", "-l", "synology.storage/role=garage", "-o", "json"],
|
||||
env=env,
|
||||
timeout=45,
|
||||
)
|
||||
if listed.returncode != 0 or not listed.stdout.strip():
|
||||
return
|
||||
try:
|
||||
payload = json.loads(listed.stdout)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
for item in payload.get("items", []):
|
||||
meta = item.get("metadata") or {}
|
||||
spec = item.get("spec") or {}
|
||||
status = item.get("status") or {}
|
||||
name = str(meta.get("name") or "").strip()
|
||||
phase = str(status.get("phase") or "").strip()
|
||||
if not name or phase != "Released":
|
||||
continue
|
||||
if not spec.get("claimRef"):
|
||||
continue
|
||||
_log(log, f"[GARAGE] Clearing stale claimRef on PV {name} (phase=Released)")
|
||||
_kubectl(
|
||||
[
|
||||
"patch",
|
||||
"pv",
|
||||
name,
|
||||
"--type=json",
|
||||
"-p",
|
||||
'[{"op":"remove","path":"/spec/claimRef"}]',
|
||||
],
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_garage_secret(*, namespace: str, env: dict | None = None, log: _LogFn | None = None) -> None:
|
||||
secret_name = str((env or {}).get("GARAGE_SECRET_NAME") or "garage-secrets").strip() or "garage-secrets"
|
||||
force = _to_bool((env or {}).get("PROLE_GARAGE_FORCE_SECRET"), default=False)
|
||||
if not force and _exists("secret", secret_name, namespace, env=env):
|
||||
return
|
||||
|
||||
payload = (
|
||||
"apiVersion: v1\n"
|
||||
"kind: Secret\n"
|
||||
"metadata:\n"
|
||||
f" name: {secret_name}\n"
|
||||
"type: Opaque\n"
|
||||
"stringData:\n"
|
||||
f" rpc_secret: {json.dumps(secrets.token_hex(32))}\n"
|
||||
f" admin_token: {json.dumps(secrets.token_urlsafe(32))}\n"
|
||||
f" metrics_token: {json.dumps(secrets.token_urlsafe(32))}\n"
|
||||
)
|
||||
_kubectl(
|
||||
["-n", namespace, "apply", "-f", "-"],
|
||||
env=env,
|
||||
input_text=payload,
|
||||
timeout=90,
|
||||
check=True,
|
||||
)
|
||||
_log(log, f"[GARAGE] Ensured secret/{secret_name} in namespace {namespace}")
|
||||
|
||||
|
||||
def initialize(
|
||||
*,
|
||||
namespace: str | None = None,
|
||||
@ -68,12 +140,43 @@ def update(
|
||||
_detect_mode(mode, env) # mode kept for API parity
|
||||
target_ns = _garage_namespace(namespace, env)
|
||||
_ensure_namespace(target_ns, env)
|
||||
_repair_released_garage_pvs(env=env, log=log)
|
||||
_ensure_garage_secret(namespace=target_ns, env=env, log=log)
|
||||
|
||||
_prune_named_workload_other_namespaces(
|
||||
kind="statefulset",
|
||||
name="garage",
|
||||
target_namespace=target_ns,
|
||||
label_selector="app=garage",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
_prune_named_workload_other_namespaces(
|
||||
kind="service",
|
||||
name="garage",
|
||||
target_namespace=target_ns,
|
||||
label_selector="app=garage",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
_prune_named_workload_other_namespaces(
|
||||
kind="configmap",
|
||||
name="garage-config",
|
||||
target_namespace=target_ns,
|
||||
label_selector="app=garage",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
|
||||
for manifest in _manifest_files(project_root):
|
||||
if not manifest.exists():
|
||||
continue
|
||||
_log(log, f"[GARAGE] Applying {manifest}")
|
||||
_kubectl(["apply", "-f", str(manifest)], env=env, timeout=240, check=True)
|
||||
name = manifest.name
|
||||
if name in {"storageclass-synology-iscsi.yaml", "iscsi-pvs.yaml"}:
|
||||
_kubectl(["apply", "-f", str(manifest)], env=env, timeout=240, check=True)
|
||||
else:
|
||||
_kubectl(["-n", target_ns, "apply", "-f", str(manifest)], env=env, timeout=240, check=True)
|
||||
|
||||
if _exists("statefulset", "garage", target_ns, env=env):
|
||||
_wait_rollout("statefulset", "garage", target_ns, env=env)
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
import secrets
|
||||
import subprocess
|
||||
|
||||
from ._services_common import (
|
||||
_LogFn,
|
||||
@ -11,6 +14,8 @@ from ._services_common import (
|
||||
_log,
|
||||
_manifest_path,
|
||||
_namespace,
|
||||
_prune_named_workload_other_namespaces,
|
||||
_reconcile_deployment_replicasets,
|
||||
_to_bool,
|
||||
_wait_rollout,
|
||||
)
|
||||
@ -28,6 +33,85 @@ def _manifest(project_root: str | Path) -> Path:
|
||||
return _manifest_path(project_root, "k8s", "opentofu", "deployment.yaml")
|
||||
|
||||
|
||||
def _build_basic_auth_entry(password: str) -> str:
|
||||
pwd = (password or "").strip() or "changeme"
|
||||
try:
|
||||
res = subprocess.run(
|
||||
["openssl", "passwd", "-apr1", pwd],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if res.returncode == 0:
|
||||
hashed = (res.stdout or "").strip()
|
||||
if hashed:
|
||||
return f"admin:{hashed}"
|
||||
except Exception:
|
||||
pass
|
||||
return f"admin:{{PLAIN}}{pwd}"
|
||||
|
||||
|
||||
def _ensure_admin_secret(
|
||||
*,
|
||||
namespace: str,
|
||||
env: dict | None = None,
|
||||
log: _LogFn | None = None,
|
||||
) -> None:
|
||||
force = _to_bool((env or {}).get("PROLE_OPENTOFU_FORCE_SECRET"), default=False)
|
||||
if not force and _exists("secret", "opentofu-admin", namespace, env=env):
|
||||
return
|
||||
|
||||
admin_password = (
|
||||
str((env or {}).get("OPENTOFU_ADMIN_PASSWORD") or "").strip()
|
||||
or str((env or {}).get("DB_PASSWORD") or "").strip()
|
||||
or secrets.token_urlsafe(18)
|
||||
)
|
||||
auth_entry = _build_basic_auth_entry(admin_password)
|
||||
|
||||
secret_yaml = (
|
||||
"apiVersion: v1\n"
|
||||
"kind: Secret\n"
|
||||
"metadata:\n"
|
||||
" name: opentofu-admin\n"
|
||||
"type: Opaque\n"
|
||||
"stringData:\n"
|
||||
f" admin_password: {json.dumps(admin_password)}\n"
|
||||
f" auth: {json.dumps(auth_entry)}\n"
|
||||
)
|
||||
_kubectl(
|
||||
["-n", namespace, "apply", "-f", "-"],
|
||||
env=env,
|
||||
input_text=secret_yaml,
|
||||
timeout=90,
|
||||
check=True,
|
||||
)
|
||||
_log(log, f"[OPENTOFU] Ensured secret/opentofu-admin in namespace {namespace}")
|
||||
|
||||
|
||||
def _deployment_is_available(
|
||||
*,
|
||||
deployment: str,
|
||||
namespace: str,
|
||||
env: dict | None = None,
|
||||
) -> bool:
|
||||
res = _kubectl(
|
||||
["-n", namespace, "get", "deployment", deployment, "-o", "json"],
|
||||
env=env,
|
||||
timeout=20,
|
||||
)
|
||||
if res.returncode != 0 or not res.stdout.strip():
|
||||
return False
|
||||
try:
|
||||
payload = json.loads(res.stdout)
|
||||
except Exception:
|
||||
return False
|
||||
spec = payload.get("spec") or {}
|
||||
status = payload.get("status") or {}
|
||||
desired = int(spec.get("replicas") or 1)
|
||||
available = int(status.get("availableReplicas") or 0)
|
||||
return desired > 0 and available >= desired
|
||||
|
||||
|
||||
def initialize(
|
||||
*,
|
||||
namespace: str | None = None,
|
||||
@ -61,6 +145,51 @@ def update(
|
||||
_detect_mode(mode, env) # mode kept for API parity
|
||||
target_ns = _opentofu_namespace(namespace, env)
|
||||
_ensure_namespace(target_ns, env)
|
||||
_ensure_admin_secret(namespace=target_ns, env=env, log=log)
|
||||
_prune_named_workload_other_namespaces(
|
||||
kind="deployment",
|
||||
name="opentofu",
|
||||
target_namespace=target_ns,
|
||||
label_selector="app=opentofu",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
_prune_named_workload_other_namespaces(
|
||||
kind="service",
|
||||
name="opentofu",
|
||||
target_namespace=target_ns,
|
||||
label_selector="app=opentofu",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
|
||||
force_apply = _to_bool((env or {}).get("PROLE_OPENTOFU_FORCE_APPLY"), default=False)
|
||||
if not force_apply and _exists("deployment", "opentofu", target_ns, env=env) and _exists(
|
||||
"service", "opentofu", target_ns, env=env
|
||||
):
|
||||
_reconcile_deployment_replicasets(
|
||||
deployment="opentofu",
|
||||
namespace=target_ns,
|
||||
label_selector="app=opentofu",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
if _deployment_is_available(deployment="opentofu", namespace=target_ns, env=env):
|
||||
_log(
|
||||
log,
|
||||
f"[OPENTOFU] Healthy deployment already present in namespace {target_ns}; skipping re-apply",
|
||||
)
|
||||
return
|
||||
_log(
|
||||
log,
|
||||
f"[OPENTOFU] Existing deployment in namespace {target_ns} is not ready; applying manifest for recovery",
|
||||
)
|
||||
|
||||
if force_apply:
|
||||
_log(
|
||||
log,
|
||||
f"[OPENTOFU] Force apply enabled; reconciling manifest in namespace {target_ns}",
|
||||
)
|
||||
|
||||
manifest = _manifest(project_root)
|
||||
if not manifest.exists():
|
||||
@ -73,6 +202,13 @@ def update(
|
||||
timeout=240,
|
||||
check=True,
|
||||
)
|
||||
_reconcile_deployment_replicasets(
|
||||
deployment="opentofu",
|
||||
namespace=target_ns,
|
||||
label_selector="app=opentofu",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
_wait_rollout("deployment", "opentofu", target_ns, env=env)
|
||||
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
from ._services_common import (
|
||||
_LogFn,
|
||||
@ -12,6 +13,8 @@ from ._services_common import (
|
||||
_kubectl,
|
||||
_log,
|
||||
_manifest_path,
|
||||
_prune_named_workload_other_namespaces,
|
||||
_reconcile_deployment_replicasets,
|
||||
_registry_namespace,
|
||||
_to_bool,
|
||||
_wait_rollout,
|
||||
@ -59,6 +62,30 @@ def _manifest(project_root: str | Path) -> Path:
|
||||
return _manifest_path(project_root, "k8s", "registry", "deployment.yaml")
|
||||
|
||||
|
||||
def _deployment_is_available(
|
||||
*,
|
||||
deployment: str,
|
||||
namespace: str,
|
||||
env: dict | None = None,
|
||||
) -> bool:
|
||||
res = _kubectl(
|
||||
["-n", namespace, "get", "deployment", deployment, "-o", "json"],
|
||||
env=env,
|
||||
timeout=20,
|
||||
)
|
||||
if res.returncode != 0 or not res.stdout.strip():
|
||||
return False
|
||||
try:
|
||||
payload = json.loads(res.stdout)
|
||||
except Exception:
|
||||
return False
|
||||
spec = payload.get("spec") or {}
|
||||
status = payload.get("status") or {}
|
||||
desired = int(spec.get("replicas") or 1)
|
||||
available = int(status.get("availableReplicas") or 0)
|
||||
return desired > 0 and available >= desired
|
||||
|
||||
|
||||
def initialize(
|
||||
*,
|
||||
namespace: str | None = None,
|
||||
@ -96,6 +123,51 @@ def update(
|
||||
|
||||
target_ns = _registry_namespace(namespace, env)
|
||||
_ensure_namespace(target_ns, env)
|
||||
_prune_named_workload_other_namespaces(
|
||||
kind="deployment",
|
||||
name="registry",
|
||||
target_namespace=target_ns,
|
||||
label_selector="app=registry",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
_prune_named_workload_other_namespaces(
|
||||
kind="service",
|
||||
name="registry",
|
||||
target_namespace=target_ns,
|
||||
label_selector="app=registry",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
|
||||
force_apply = _to_bool((env or {}).get("PROLE_REGISTRY_FORCE_APPLY"), default=False)
|
||||
if not force_apply and _exists("deployment", "registry", target_ns, env=env) and _exists(
|
||||
"service", "registry", target_ns, env=env
|
||||
):
|
||||
_reconcile_deployment_replicasets(
|
||||
deployment="registry",
|
||||
namespace=target_ns,
|
||||
label_selector="app=registry",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
if _deployment_is_available(deployment="registry", namespace=target_ns, env=env):
|
||||
_log(
|
||||
log,
|
||||
f"[REGISTRY] Healthy deployment already present in namespace {target_ns}; skipping re-apply",
|
||||
)
|
||||
return
|
||||
_log(
|
||||
log,
|
||||
f"[REGISTRY] Existing deployment in namespace {target_ns} is not ready; applying manifest for recovery",
|
||||
)
|
||||
|
||||
if force_apply:
|
||||
_log(
|
||||
log,
|
||||
f"[REGISTRY] Force apply enabled; reconciling manifest in namespace {target_ns}",
|
||||
)
|
||||
|
||||
manifest = _manifest(project_root)
|
||||
if not manifest.exists():
|
||||
raise RuntimeError(f"Registry manifest not found: {manifest}")
|
||||
@ -106,6 +178,13 @@ def update(
|
||||
timeout=180,
|
||||
check=True,
|
||||
)
|
||||
_reconcile_deployment_replicasets(
|
||||
deployment="registry",
|
||||
namespace=target_ns,
|
||||
label_selector="app=registry",
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
_wait_rollout("deployment", "registry", target_ns, env=env)
|
||||
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@ -144,9 +145,18 @@ def _choose_root_pair(spec: ClusterStorageSpec) -> tuple[Path, Path]:
|
||||
mounted_roots.append(root)
|
||||
|
||||
if not mounted_roots:
|
||||
raise StorageProvisioningError(
|
||||
"No mounted Synology roots available. Expected mounted paths like /synology/d001..d004."
|
||||
)
|
||||
allow_unmounted = True
|
||||
if env:
|
||||
raw = str(env.get("PROLE_STORAGE_ALLOW_UNMOUNTED_ROOTS") or "").strip().lower()
|
||||
if raw:
|
||||
allow_unmounted = raw in {"1", "true", "yes", "on"}
|
||||
|
||||
if not allow_unmounted:
|
||||
raise StorageProvisioningError(
|
||||
"No mounted Synology roots available. Expected mounted paths like /synology/d001..d004."
|
||||
)
|
||||
|
||||
mounted_roots = [Path(root_str) for root_str in spec.synology_roots]
|
||||
|
||||
ident = f"{spec.namespace}:{spec.cluster_name}:{spec.service_name}"
|
||||
digest = int(hashlib.sha256(ident.encode("utf-8")).hexdigest(), 16)
|
||||
@ -253,11 +263,27 @@ def ensure_host_path(path: str, *, uid: int, gid: int, mode: int = 0o700) -> Non
|
||||
target = Path(path)
|
||||
if target.exists() and not target.is_dir():
|
||||
raise StorageProvisioningError(f"Host path exists but is not a directory: {path}")
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(target, mode)
|
||||
try:
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as exc:
|
||||
if exc.errno == 30: # Read-only filesystem on local runner
|
||||
return
|
||||
raise StorageProvisioningError(f"Cannot create host path '{path}': {exc}") from exc
|
||||
|
||||
try:
|
||||
os.chmod(target, mode)
|
||||
except OSError as exc:
|
||||
if exc.errno == 30:
|
||||
return
|
||||
raise StorageProvisioningError(f"Cannot set mode on host path '{path}': {exc}") from exc
|
||||
try:
|
||||
os.chown(target, uid, gid)
|
||||
except PermissionError as exc:
|
||||
except PermissionError:
|
||||
# Non-root execution is expected in some local automation contexts.
|
||||
return
|
||||
except OSError as exc:
|
||||
if exc.errno == 30:
|
||||
return
|
||||
raise StorageProvisioningError(
|
||||
f"Cannot set ownership on host path '{path}' to {uid}:{gid}: {exc}"
|
||||
) from exc
|
||||
@ -286,6 +312,75 @@ def _kubectl_apply_manifest(spec: ClusterStorageSpec, manifest: dict[str, Any])
|
||||
raise StorageProvisioningError(f"Failed to apply PV manifest '{manifest['metadata']['name']}': {stderr}")
|
||||
|
||||
|
||||
def _ensure_node_host_path(spec: ClusterStorageSpec, host_path: str) -> None:
|
||||
env = _command_env(spec)
|
||||
ident = f"{spec.namespace}:{spec.cluster_name}:{spec.node_name}:{host_path}"
|
||||
suffix = hashlib.sha1(ident.encode("utf-8")).hexdigest()[:10]
|
||||
pod_name = f"prole-pathprep-{suffix}"
|
||||
namespace = "kube-system"
|
||||
|
||||
manifest = {
|
||||
"apiVersion": "v1",
|
||||
"kind": "Pod",
|
||||
"metadata": {"name": pod_name, "namespace": namespace},
|
||||
"spec": {
|
||||
"restartPolicy": "Never",
|
||||
"nodeName": spec.node_name,
|
||||
"tolerations": [
|
||||
{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"},
|
||||
{"key": "node-role.kubernetes.io/master", "operator": "Exists", "effect": "NoSchedule"},
|
||||
],
|
||||
"containers": [
|
||||
{
|
||||
"name": "pathprep",
|
||||
"image": "busybox:1.36",
|
||||
"command": ["sh", "-c", "mkdir -p /target && chmod 700 /target || true"],
|
||||
"volumeMounts": [{"name": "target", "mountPath": "/target"}],
|
||||
}
|
||||
],
|
||||
"volumes": [
|
||||
{
|
||||
"name": "target",
|
||||
"hostPath": {"path": host_path, "type": "DirectoryOrCreate"},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
apply_res = _run_command(
|
||||
[spec.kubectl_bin, "apply", "-f", "-"],
|
||||
env=env,
|
||||
input_text=json.dumps(manifest),
|
||||
)
|
||||
if apply_res.returncode != 0:
|
||||
raise StorageProvisioningError(
|
||||
f"Failed to start host path bootstrap pod '{pod_name}': {(apply_res.stderr or apply_res.stdout or '').strip()}"
|
||||
)
|
||||
|
||||
try:
|
||||
for _ in range(60):
|
||||
pod = _kubectl_get_json(spec, ["-n", namespace, "get", "pod", pod_name])
|
||||
if pod:
|
||||
phase = str((pod.get("status") or {}).get("phase") or "")
|
||||
if phase in {"Succeeded", "Running"}:
|
||||
return
|
||||
if phase == "Failed":
|
||||
logs = _run_command([spec.kubectl_bin, "-n", namespace, "logs", pod_name], env=env)
|
||||
raise StorageProvisioningError(
|
||||
f"Host path bootstrap pod '{pod_name}' failed for '{host_path}': {(logs.stdout or logs.stderr or '').strip()}"
|
||||
)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
_run_command(
|
||||
[spec.kubectl_bin, "-n", namespace, "delete", "pod", pod_name, "--ignore-not-found", "--wait=false"],
|
||||
env=env,
|
||||
)
|
||||
|
||||
raise StorageProvisioningError(
|
||||
f"Timed out waiting for host path bootstrap pod '{pod_name}' to prepare '{host_path}'."
|
||||
)
|
||||
|
||||
|
||||
def _collect_existing_pvs(spec: ClusterStorageSpec) -> dict[str, dict[str, Any]]:
|
||||
pv_list = _kubectl_get_json(spec, ["get", "pv"])
|
||||
if pv_list is None:
|
||||
@ -435,6 +530,8 @@ def provision_cluster_storage(spec: ClusterStorageSpec) -> ProvisionedStorage:
|
||||
)
|
||||
|
||||
paths = build_cluster_storage_paths(spec)
|
||||
_ensure_node_host_path(spec, paths.data_path)
|
||||
_ensure_node_host_path(spec, paths.wal_path)
|
||||
ensure_host_path(
|
||||
paths.data_path,
|
||||
uid=spec.postgres_uid,
|
||||
|
||||
160
prole.sh
160
prole.sh
@ -21,6 +21,24 @@ SILENT=0
|
||||
RESET=0
|
||||
EXTRA_ARGS=()
|
||||
|
||||
# Canonical wrapper aliases:
|
||||
# ./prole.sh --reset => full silent reset run
|
||||
# ./prole.sh --update => silent update run
|
||||
# ./prole.sh --perfsnap => topology/storage snapshot
|
||||
if [[ $# -gt 0 ]]; then
|
||||
case "$1" in
|
||||
--reset)
|
||||
set -- reset-run "${@:2}"
|
||||
;;
|
||||
--update)
|
||||
set -- update "${@:2}"
|
||||
;;
|
||||
--perfsnap)
|
||||
set -- perfsnap "${@:2}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [options] [command] [command-options]"
|
||||
echo
|
||||
@ -31,7 +49,7 @@ usage() {
|
||||
echo " -c, --config Path to prole.cfg config file (default: conf/prole.cfg)"
|
||||
echo " --coverage Create a coverage report for the run of install.py"
|
||||
echo " -s, --silent Run unattended install (passes -S to install.py)"
|
||||
echo " -r, --reset Perform a reset before action (site/deploy/install)"
|
||||
echo " -r, --reset Perform a reset before action (site/deploy/install); for canonical full reset run use './prole.sh --reset'"
|
||||
echo
|
||||
echo "Commands:"
|
||||
echo " init Run/re-run initialization scripts to verify, repair, and renew the deployment"
|
||||
@ -39,6 +57,9 @@ usage() {
|
||||
echo " ansible Run ansible-playbook via ansible.sh"
|
||||
echo " site Shortcut for running the site deployment"
|
||||
echo " reset Shortcut for k3s factory reset (full clear)"
|
||||
echo " reset-run Canonical full reset run (equivalent to legacy '-r -s -v -l')"
|
||||
echo " update Canonical silent update run (installer --update)"
|
||||
echo " perfsnap Save topology/storage performance snapshot (includes storage_probe)"
|
||||
echo " deploy Full end-to-end: site run, and silent install (reset optional)"
|
||||
echo " start Verify/clean environment, ensure cluster availability, run silent install"
|
||||
echo " stop Validate environment, run full backup+archive, stop services (keep cluster idle)"
|
||||
@ -60,7 +81,7 @@ while [[ $# -gt 0 ]]; do
|
||||
-s|--silent) SILENT=1; shift ;;
|
||||
-r|--reset) RESET=1; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
install|ansible|site|reset|deploy|init|start|stop|backup|passwd|status) break ;;
|
||||
install|ansible|site|reset|reset-run|update|perfsnap|deploy|init|start|stop|backup|passwd|status) break ;;
|
||||
*) EXTRA_ARGS+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
@ -135,6 +156,18 @@ prepare_ansible_color() {
|
||||
|
||||
PROLE_CFG_DEFAULT="${ROOT_DIR}/conf/prole.cfg"
|
||||
|
||||
resolve_cfg_path() {
|
||||
if [[ -n "${CONFIG_PATH:-}" ]]; then
|
||||
printf '%s' "${CONFIG_PATH}"
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then
|
||||
printf '%s' "${PROLE_CONF}/prole.cfg"
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "${PROLE_CFG_DEFAULT}"
|
||||
}
|
||||
|
||||
log_msg() {
|
||||
printf '%s\n' "$*"
|
||||
}
|
||||
@ -541,6 +574,129 @@ if [[ -f "${ROOT_DIR}/etc/sync-prole-cfg.py" ]]; then
|
||||
fi
|
||||
|
||||
case "${CMD}" in
|
||||
reset-run)
|
||||
CONFIG_PATH="$(resolve_cfg_path)"
|
||||
_reset_args=("install" "-s" "-v" "-l" "-r" "-c" "${CONFIG_PATH}")
|
||||
if [[ "${COVERAGE}" -eq 1 ]]; then
|
||||
_reset_args=("--coverage" "${_reset_args[@]}")
|
||||
fi
|
||||
exec "${ROOT_DIR}/prole.sh" "${_reset_args[@]}"
|
||||
;;
|
||||
update)
|
||||
CONFIG_PATH="$(resolve_cfg_path)"
|
||||
_update_args=("install" "-s" "-v" "-l" "-c" "${CONFIG_PATH}" "--update")
|
||||
if [[ "${COVERAGE}" -eq 1 ]]; then
|
||||
_update_args=("--coverage" "${_update_args[@]}")
|
||||
fi
|
||||
exec "${ROOT_DIR}/prole.sh" "${_update_args[@]}"
|
||||
;;
|
||||
perfsnap)
|
||||
CONFIG_PATH="$(resolve_cfg_path)"
|
||||
load_prole_cfg "${CONFIG_PATH}"
|
||||
_mode_for_env=$(resolve_prole_mode)
|
||||
if [[ -z "${_mode_for_env:-}" ]]; then
|
||||
_mode_for_env="k3d"
|
||||
fi
|
||||
ensure_openbao_url_for_mode "${_mode_for_env}"
|
||||
mkdir -p "${ROOT_DIR}/logs" "${ROOT_DIR}/data/topology"
|
||||
_snap_ts="$(date +%Y%m%d-%H%M%S)"
|
||||
_snap_file="${ROOT_DIR}/logs/perfsnap-${_snap_ts}.json"
|
||||
export PROLE_PERFSNAP_FILE="${_snap_file}"
|
||||
export PROLE_PERFSNAP_CFG="${CONFIG_PATH}"
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from knoe import prole_conf
|
||||
from knoe.core.env import _kubectl_base_cmd_for_k3s
|
||||
from knoe.core.topology import TopologyDiscoveryConfig, discover_cluster_topology
|
||||
|
||||
cfg_path = Path(os.environ["PROLE_PERFSNAP_CFG"]).resolve()
|
||||
snap_path = Path(os.environ["PROLE_PERFSNAP_FILE"]).resolve()
|
||||
|
||||
cfg = prole_conf.load_layered_config(cfg_path, require_exists=True)
|
||||
|
||||
def cfg_get(section: str, key: str, default: str = "") -> str:
|
||||
try:
|
||||
return (cfg.get(section, key, fallback=default) or default).strip()
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
mode_raw = (
|
||||
cfg_get("Deployment", "MODE")
|
||||
or cfg_get("Global", "DEPLOYMENT_MODE")
|
||||
or cfg_get("Global", "CLUSTER_ENV")
|
||||
or "k3d"
|
||||
)
|
||||
mode = mode_raw.lower()
|
||||
project_root = cfg_path.parent.parent
|
||||
prole_data = cfg_get("Inputs", "env_setup.PROLE_DATA") or str(project_root / "data")
|
||||
|
||||
if mode in {"k3s", "k8s"}:
|
||||
managed_kubeconfig = os.environ.get("PROLE_KUBECONFIG") or os.environ.get("KUBECONFIG")
|
||||
if not managed_kubeconfig:
|
||||
candidate = project_root / "prole-k3s.kubeconfig"
|
||||
if candidate.exists():
|
||||
managed_kubeconfig = str(candidate)
|
||||
base_cmd = _kubectl_base_cmd_for_k3s(managed_kubeconfig=managed_kubeconfig)
|
||||
else:
|
||||
base_cmd = ["kubectl"]
|
||||
context = (
|
||||
os.environ.get("KUBECTL_CONTEXT")
|
||||
or cfg_get("Dev Cluster (k3d)", "KUBECTL_CONTEXT")
|
||||
or ""
|
||||
).strip()
|
||||
if not context and mode == "k3d":
|
||||
target = (
|
||||
cfg_get("Global", "DEPLOYMENT_TARGET")
|
||||
or cfg_get("Dev Cluster (k3d)", "DISPLAY_NAME")
|
||||
or ""
|
||||
).strip()
|
||||
if target:
|
||||
context = target if target.startswith("k3d-") else f"k3d-{target}"
|
||||
if context:
|
||||
base_cmd.extend(["--context", context])
|
||||
output_root = Path(prole_data) / "topology"
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env_map = dict(os.environ)
|
||||
env_map["PROLE_CFG_FILE"] = str(cfg_path)
|
||||
env_map["PROLE_DATA"] = str(Path(prole_data).resolve())
|
||||
|
||||
result = discover_cluster_topology(
|
||||
kubectl_base_cmd=base_cmd,
|
||||
topology_root=output_root,
|
||||
config=TopologyDiscoveryConfig(storage_probe_enabled=True, storage_probe_quick=True),
|
||||
env=env_map,
|
||||
)
|
||||
|
||||
payload = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"cfg": str(cfg_path),
|
||||
"mode": mode,
|
||||
"kubectl_base_cmd": base_cmd,
|
||||
"topology_root": str(output_root),
|
||||
"collection_status": result.topology.collection_status,
|
||||
"node_count": len(result.topology.nodes),
|
||||
"ready_nodes": [n.name for n in result.topology.nodes if n.ready],
|
||||
"collector_applied": result.collector_applied,
|
||||
"expected_ready_nodes": list(result.expected_ready_nodes),
|
||||
"reported_nodes": list(result.reported_nodes),
|
||||
"missing_nodes": list(result.missing_nodes),
|
||||
"xml": {
|
||||
"cluster": str(output_root / "cluster-topology.xml"),
|
||||
"node": str(output_root / "node-topology.xml"),
|
||||
},
|
||||
}
|
||||
|
||||
snap_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
snap_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
print(f"Performance snapshot written: {snap_path}")
|
||||
print(f"Topology root: {output_root}")
|
||||
PY
|
||||
;;
|
||||
install)
|
||||
LOGFILE="${ROOT_DIR}/logs/install-${TS}.log"
|
||||
|
||||
|
||||
@ -1,55 +1,51 @@
|
||||
Network Discovery Summary:
|
||||
Primary Router: 10.0.0.1 (Unknown)
|
||||
DNS Servers: 10.0.0.4, 100.100.100.100, 10.0.0.5
|
||||
Primary Router: 10.0.0.1 (eero_5d:50:f2)
|
||||
DNS Servers: 10.0.0.5, 10.0.0.4, 100.100.100.100
|
||||
Detected Devices:
|
||||
- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
|
||||
- 10.0.0.38 [7e:19:9a:fe:9f:ed] (7e:19:9a:fe:9f:ed): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
|
||||
- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.152 [8c:79:f5:ba:a8:2c] (SamsungElect_ba:a8:2c): Ports [], Services: []
|
||||
- 10.0.0.100 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: []
|
||||
- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
|
||||
- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: []
|
||||
- 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
|
||||
- 10.0.0.3 [2c:cf:67:8c:2b:47] (RaspberryPi_8c:2b:47): Ports [22, 53, 80, 443, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
- 10.0.0.1 [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: []
|
||||
- 10.0.0.5 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC']
|
||||
- 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.49 [00:e0:4c:08:0c:75] (RealtekSemic_08:0c:75): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
|
||||
- 10.0.0.1 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.2 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.3 [Unknown] (Unknown): Ports [22, 53, 80, 443, 2049, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
- 10.0.0.4 [Unknown] (Unknown): Ports [22, 53, 443, 2049], Services: ['DNS', 'SSH', 'NFS', 'Web Server']
|
||||
- 10.0.0.5 [Unknown] (Unknown): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
|
||||
- 10.0.0.6 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.22 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.26 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.37 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.41 [Unknown] (Unknown): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.45 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.46 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.48 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.55 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.58 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.95 [Unknown] (Unknown): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
|
||||
- 10.0.0.99 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.111 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.113 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.117 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.123 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.124 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.125 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.128 [Unknown] (Unknown): Ports [22, 445, 5900], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)']
|
||||
- 10.0.0.130 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.143 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.145 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.155 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.170 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.175 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.179 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.188 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.189 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.196 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.199 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.204 [Unknown] (Unknown): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama']
|
||||
- 10.0.0.205 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
Ollama Instances found at: 10.0.0.38, 10.0.0.49, 10.0.0.204
|
||||
- 10.0.0.26 [c8:db:26:08:52:f8] (Logitech_08:52:f8): Ports [], Services: []
|
||||
- 10.0.0.6 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.4 [b8:27:eb:b3:6f:2b] (RaspberryPiF_b3:6f:2b): Ports [22, 53, 2049], Services: ['DNS', 'SSH', 'NFS']
|
||||
- 10.0.0.22 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.2 [dc:a4:ca:ea:1a:2f] (Apple_ea:1a:2f): Ports [], Services: []
|
||||
- 10.0.0.45 [f8:b4:6a:30:a4:1f] (HewlettPacka_30:a4:1f): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.48 [0c:fe:45:53:f4:3c] (SonyInteract_53:f4:3c): Ports [], Services: []
|
||||
- 10.0.0.37 [54:07:7d:22:c0:b2] (Netgear_22:c0:b2): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.46 [d4:f7:d5:40:ab:17] (SonyInteract_40:ab:17): Ports [], Services: []
|
||||
- 10.0.0.55 [48:a6:b8:a7:50:60] (Sonos_a7:50:60): Ports [], Services: []
|
||||
- 10.0.0.41 [b8:27:eb:57:10:d9] (RaspberryPiF_57:10:d9): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.205 [a8:20:66:28:12:e7] (Apple_28:12:e7): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: []
|
||||
- 10.0.0.95 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
|
||||
- 10.0.0.99 [20:28:bc:f1:dc:ba] (Visionscape_f1:dc:ba): Ports [], Services: []
|
||||
- 10.0.0.111 [ca:0b:6c:5d:0a:e5] (ca:0b:6c:5d:0a:e5): Ports [], Services: []
|
||||
- 10.0.0.123 [3c:ef:8c:96:e3:3c] (ZhejiangDahu_96:e3:3c): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.124 [9c:8e:cd:02:73:c4] (AmcrestTechn_02:73:c4): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.128 [20:c9:d0:94:30:dd] (Apple_94:30:dd): Ports [5900], Services: ['VNC']
|
||||
- 10.0.0.117 [40:f5:20:1e:5e:91] (Espressif_1e:5e:91): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.125 [a0:60:32:04:73:ba] (AmcrestTechn_04:73:ba): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
|
||||
- 10.0.0.130 [f8:bb:bf:a3:84:eb] (eero_a3:84:eb): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.145 [7c:a6:b0:04:db:1c] (7c:a6:b0:04:db:1c): Ports [], Services: []
|
||||
- 10.0.0.155 [62:17:97:b3:06:a1] (62:17:97:b3:06:a1): Ports [], Services: []
|
||||
- 10.0.0.170 [b0:ee:7b:ca:73:99] (Roku_ca:73:99): Ports [], Services: []
|
||||
- 10.0.0.188 [00:16:6c:c5:1f:54] (SamsungElect_c5:1f:54): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.189 [00:17:88:a3:2f:cc] (PhilipsLight_a3:2f:cc): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.179 [48:a6:b8:a4:8e:cc] (Sonos_a4:8e:cc): Ports [], Services: []
|
||||
- 10.0.0.175 [28:80:88:e4:10:1a] (Netgear_e4:10:1a): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.196 [60:5f:8d:88:08:f2] (eero_88:08:f2): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.180 [4c:e1:73:42:1a:be] (HuizhouDehon_2:1a:be): Ports [445], Services: ['SMB/CIFS (Possible Windows/AD)']
|
||||
- 10.0.0.199 [7c:a6:b0:01:a7:33] (7c:a6:b0:01:a7:33): Ports [], Services: []
|
||||
- 10.0.0.204 [50:eb:f6:56:f3:26] (ASUSTekCOMPU_56:f3:26): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama']
|
||||
- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.1\ [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: []
|
||||
Ollama Instances found at: 10.0.0.38, 10.0.0.204
|
||||
|
||||
Ansible Inventory Summary:
|
||||
Inventory directory: /Users/chrisfu/dev/prole/infrastructure
|
||||
|
||||
@ -12,6 +12,11 @@ Required:
|
||||
Options:
|
||||
--delete-pvcs Delete all PVCs in the namespace (DANGEROUS: wipes state)
|
||||
--delete-config Delete ConfigMaps and Secrets (VERY DANGEROUS)
|
||||
--keep-registry Preserve registry workloads/services in the target namespace
|
||||
--clear-node-reservations
|
||||
Best-effort clear node reservations by uncordoning nodes
|
||||
--no-clear-node-reservations
|
||||
Disable node reservation cleanup (default)
|
||||
-h, --help Show this help
|
||||
|
||||
Notes:
|
||||
@ -29,6 +34,8 @@ EOF
|
||||
NS=""
|
||||
DELETE_PVCS=false
|
||||
DELETE_CONFIG=false
|
||||
KEEP_REGISTRY=false
|
||||
CLEAR_NODE_RESERVATIONS=false
|
||||
|
||||
[[ $# -eq 0 ]] && usage && exit 1
|
||||
|
||||
@ -37,6 +44,9 @@ while [[ $# -gt 0 ]]; do
|
||||
-n) NS="${2:-}"; shift 2 ;;
|
||||
--delete-pvcs) DELETE_PVCS=true; shift ;;
|
||||
--delete-config) DELETE_CONFIG=true; shift ;;
|
||||
--keep-registry) KEEP_REGISTRY=true; shift ;;
|
||||
--clear-node-reservations) CLEAR_NODE_RESERVATIONS=true; shift ;;
|
||||
--no-clear-node-reservations) CLEAR_NODE_RESERVATIONS=false; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown argument: $1"; echo; usage; exit 1 ;;
|
||||
esac
|
||||
@ -60,8 +70,56 @@ esac
|
||||
echo "Resetting namespace: $NS"
|
||||
echo " DELETE_PVCS=$DELETE_PVCS"
|
||||
echo " DELETE_CONFIG=$DELETE_CONFIG"
|
||||
echo " KEEP_REGISTRY=$KEEP_REGISTRY"
|
||||
echo " CLEAR_NODE_RESERVATIONS=$CLEAR_NODE_RESERVATIONS"
|
||||
echo
|
||||
|
||||
if ! kubectl get namespace "$NS" >/dev/null 2>&1; then
|
||||
echo "Namespace '$NS' does not exist; skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
is_registry_resource() {
|
||||
local kind="$1"
|
||||
local name="$2"
|
||||
|
||||
[[ "$KEEP_REGISTRY" == "true" ]] || return 1
|
||||
|
||||
if [[ "$name" == "registry" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local app_label=""
|
||||
app_label=$(kubectl -n "$NS" get "$kind" "$name" -o jsonpath='{.metadata.labels.app}' 2>/dev/null || true)
|
||||
[[ "$app_label" == "registry" ]]
|
||||
}
|
||||
|
||||
scale_down_workloads() {
|
||||
local kind="$1"
|
||||
local item
|
||||
while IFS= read -r item; do
|
||||
[[ -z "$item" ]] && continue
|
||||
if is_registry_resource "$kind" "$item"; then
|
||||
echo " preserving ${kind}/${item}"
|
||||
continue
|
||||
fi
|
||||
kubectl -n "$NS" scale "$kind" "$item" --replicas=0 2>/dev/null || true
|
||||
done < <(kubectl -n "$NS" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
||||
}
|
||||
|
||||
delete_kind_resources() {
|
||||
local kind="$1"
|
||||
local item
|
||||
while IFS= read -r item; do
|
||||
[[ -z "$item" ]] && continue
|
||||
if is_registry_resource "$kind" "$item"; then
|
||||
echo " preserving ${kind}/${item}"
|
||||
continue
|
||||
fi
|
||||
kubectl -n "$NS" delete "$kind" "$item" --ignore-not-found --wait=false 2>/dev/null || true
|
||||
done < <(kubectl -n "$NS" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
||||
}
|
||||
|
||||
# -------------------------
|
||||
# CNPG: stop respawning in THIS namespace by deleting CNPG Cluster CRs
|
||||
# Detect via CRD existence (more reliable than kubectl api-resources on some setups)
|
||||
@ -97,14 +155,19 @@ fi
|
||||
# Scale down controllers to reduce churn
|
||||
# -------------------------
|
||||
echo "Scaling down Deployments/StatefulSets..."
|
||||
kubectl -n "$NS" scale deploy --all --replicas=0 2>/dev/null || true
|
||||
kubectl -n "$NS" scale sts --all --replicas=0 2>/dev/null || true
|
||||
scale_down_workloads deployment
|
||||
scale_down_workloads statefulset
|
||||
|
||||
# -------------------------
|
||||
# Delete common workload controllers
|
||||
# -------------------------
|
||||
echo "Deleting workload controllers..."
|
||||
kubectl -n "$NS" delete deploy,sts,ds,rs,job,cronjob --all --ignore-not-found --wait=false
|
||||
delete_kind_resources deployment
|
||||
delete_kind_resources statefulset
|
||||
delete_kind_resources daemonset
|
||||
delete_kind_resources replicaset
|
||||
delete_kind_resources job
|
||||
delete_kind_resources cronjob
|
||||
|
||||
# -------------------------
|
||||
# Delete services safely (NEVER delete "kubernetes" service in default)
|
||||
@ -115,6 +178,10 @@ for s in $(kubectl -n "$NS" get svc -o jsonpath='{range .items[*]}{.metadata.nam
|
||||
echo " skipping protected service: default/kubernetes"
|
||||
continue
|
||||
fi
|
||||
if is_registry_resource service "$s"; then
|
||||
echo " preserving service/$s"
|
||||
continue
|
||||
fi
|
||||
kubectl -n "$NS" delete svc "$s" --ignore-not-found --wait=false
|
||||
done
|
||||
|
||||
@ -154,10 +221,32 @@ fi
|
||||
# Final pod cleanup
|
||||
# -------------------------
|
||||
echo "Deleting remaining pods..."
|
||||
kubectl -n "$NS" delete pod --all --ignore-not-found --wait=false
|
||||
deleted_pods=()
|
||||
for p in $(kubectl -n "$NS" get pod -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true); do
|
||||
if is_registry_resource pod "$p"; then
|
||||
echo " preserving pod/$p"
|
||||
continue
|
||||
fi
|
||||
kubectl -n "$NS" delete pod "$p" --ignore-not-found --wait=false 2>/dev/null || true
|
||||
deleted_pods+=("$p")
|
||||
done
|
||||
|
||||
echo "Waiting for pods to terminate..."
|
||||
kubectl -n "$NS" wait --for=delete pod --all --timeout=180s 2>/dev/null || true
|
||||
if [[ ${#deleted_pods[@]} -gt 0 ]]; then
|
||||
wait_args=()
|
||||
for p in "${deleted_pods[@]}"; do
|
||||
wait_args+=("pod/$p")
|
||||
done
|
||||
kubectl -n "$NS" wait --for=delete "${wait_args[@]}" --timeout=180s 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [[ "$CLEAR_NODE_RESERVATIONS" == "true" ]]; then
|
||||
echo "Clearing node reservations (uncordon all nodes)..."
|
||||
while IFS= read -r node; do
|
||||
[[ -z "$node" ]] && continue
|
||||
kubectl uncordon "$node" >/dev/null 2>&1 || true
|
||||
done < <(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Namespace '$NS' reset complete (namespace preserved)."
|
||||
|
||||
88
tmp/prole-synology-path-bootstrap.yaml
Normal file
88
tmp/prole-synology-path-bootstrap.yaml
Normal file
@ -0,0 +1,88 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: prole-synology-path-bootstrap
|
||||
namespace: kube-system
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: "node-role.kubernetes.io/master"
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: init
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
mkdir -p /m/d001-data /m/d001-wal /m/d001-garage /m/d001-openbao \
|
||||
/m/d002-data /m/d002-wal /m/d002-prom \
|
||||
/m/d003-data /m/d003-wal /m/d003-grafana
|
||||
echo "synology-path-bootstrap-complete"
|
||||
sleep 5
|
||||
volumeMounts:
|
||||
- name: d001-data
|
||||
mountPath: /m/d001-data
|
||||
- name: d001-wal
|
||||
mountPath: /m/d001-wal
|
||||
- name: d001-garage
|
||||
mountPath: /m/d001-garage
|
||||
- name: d001-openbao
|
||||
mountPath: /m/d001-openbao
|
||||
- name: d002-data
|
||||
mountPath: /m/d002-data
|
||||
- name: d002-wal
|
||||
mountPath: /m/d002-wal
|
||||
- name: d002-prom
|
||||
mountPath: /m/d002-prom
|
||||
- name: d003-data
|
||||
mountPath: /m/d003-data
|
||||
- name: d003-wal
|
||||
mountPath: /m/d003-wal
|
||||
- name: d003-grafana
|
||||
mountPath: /m/d003-grafana
|
||||
volumes:
|
||||
- name: d001-data
|
||||
hostPath:
|
||||
path: /synology/d001/data
|
||||
type: DirectoryOrCreate
|
||||
- name: d001-wal
|
||||
hostPath:
|
||||
path: /synology/d001/wal
|
||||
type: DirectoryOrCreate
|
||||
- name: d001-garage
|
||||
hostPath:
|
||||
path: /synology/d001/garage
|
||||
type: DirectoryOrCreate
|
||||
- name: d001-openbao
|
||||
hostPath:
|
||||
path: /synology/d001/openbao
|
||||
type: DirectoryOrCreate
|
||||
- name: d002-data
|
||||
hostPath:
|
||||
path: /synology/d002/data
|
||||
type: DirectoryOrCreate
|
||||
- name: d002-wal
|
||||
hostPath:
|
||||
path: /synology/d002/wal
|
||||
type: DirectoryOrCreate
|
||||
- name: d002-prom
|
||||
hostPath:
|
||||
path: /synology/d002/prometheus
|
||||
type: DirectoryOrCreate
|
||||
- name: d003-data
|
||||
hostPath:
|
||||
path: /synology/d003/data
|
||||
type: DirectoryOrCreate
|
||||
- name: d003-wal
|
||||
hostPath:
|
||||
path: /synology/d003/wal
|
||||
type: DirectoryOrCreate
|
||||
- name: d003-grafana
|
||||
hostPath:
|
||||
path: /synology/d003/grafana
|
||||
type: DirectoryOrCreate
|
||||
Loading…
Reference in New Issue
Block a user