prole/mock_val/init_cloudnative_pg.sh
chrisfu ce5ff83eb4 checkpoint: improve init scripts, installer flows, and kube context handling
- Fix kube context switching for k3s single-context kubeconfigs and k3d shorthand prefixes

- Update common init/status scripts (registry, kerberos, cnpg backup, service layer, common services)

- Add Gitea init script and installer ArgoCD screen

- Add Supabase realtime probe patching plus regression tests

- Extend installer core/UI test coverage
2026-03-14 20:07:54 -07:00

471 lines
18 KiB
Bash
Executable File

local token url
if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
token=$(cat "$OPENBAO_TOKEN_FILE")
else
token=""
fi
url=$(bao_service_url)
if [[ -n "$token" && -n "$url" ]]; then
echo "Attempting to read admin key pair from OpenBao kv/$BAO_PATH_ADMIN ..."
if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -e '.data.data' >/dev/null 2>&1; then
local priv_b64 pub_b64
priv_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_private_key_b64')
pub_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_public_key_b64')
# Use a temporary file to determine where to save based on existing legacy or generic preference
local target_priv="$ADMIN_PRIV_GENERIC"
local target_pub="$ADMIN_PUB_GENERIC"
# If legacy keys exist, we might want to overwrite them too for compatibility
printf "%s" "$priv_b64" | base64 -d >"$target_priv"
printf "%s" "$pub_b64" | base64 -d >"$target_pub"
chmod 0600 "$target_priv"
# Mirror to legacy path if it was expected by other scripts
cp "$target_priv" "$ADMIN_PRIV_ED25519" 2>/dev/null || true
cp "$target_pub" "$ADMIN_PUB_ED25519" 2>/dev/null || true
fi
echo "Attempting to read database password from OpenBao kv/$BAO_PATH_DB ..."
if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -e '.data.data' >/dev/null 2>&1; then
local db_pass
db_pass=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -r '.data.data.password')
if [[ -n "$db_pass" ]]; then
echo "Updating database user secret 'prole-db-user' from OpenBao ..."
kubectl create secret generic prole-db-user -n "$NAMESPACE" \
--from-literal=username=prole \
--from-literal=password="$db_pass" \
--dry-run=client -o yaml | kubectl apply -f -
echo "Updating database superuser secret 'prole-db-superuser' from OpenBao ..."
kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \
--from-literal=username=postgres \
--from-literal=password="$db_pass" \
--dry-run=client -o yaml | kubectl apply -f -
fi
fi
fi
# Fallback: use DB_PASSWORD from env if OpenBao is unreachable or empty
if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
local db_pass="${DB_PASSWORD:-}"
if [[ -n "$db_pass" && "$db_pass" != '${OPENBAO:'* && "$db_pass" != '${PROLE_SECRET:'* ]]; then
echo "Updating database user secret 'prole-db-user' from DB_PASSWORD env ..."
kubectl create secret generic prole-db-user -n "$NAMESPACE" \
--from-literal=username=prole \
--from-literal=password="$db_pass" \
--dry-run=client -o yaml | kubectl apply -f -
echo "Updating database superuser secret 'prole-db-superuser' from DB_PASSWORD env ..."
kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \
--from-literal=username=postgres \
--from-literal=password="$db_pass" \
--dry-run=client -o yaml | kubectl apply -f -
fi
fi
if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
echo "ERROR: 'prole-db-user' secret is missing in namespace '$NAMESPACE' and could not be resolved from OpenBao or DB_PASSWORD env." >&2
return 1
fi
ensure_local_admin_keypair
if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then
echo "Using local admin key pair at $SECRETS_DIR"
return 0
fi
echo "ERROR: Could not obtain admin key pair from OpenBao and could not generate local keys." >&2
return 1
}
apply_cnpg_admin_secret() {
echo "Creating/updating Secret cnpg-admin-key ..."
kubectl create secret generic cnpg-admin-key -n "$NAMESPACE" \
--from-file=admin.key="$ADMIN_PRIV_GENERIC" \
--from-file=admin.pub="$ADMIN_PUB_GENERIC" \
--dry-run=client -o yaml | kubectl apply -f -
}
detect_and_reprovision_unencrypted_cluster() {
local encryption_enabled="${AT_REST_ENCRYPTION_ENABLED:-false}"
if [[ "$encryption_enabled" != "true" && "$encryption_enabled" != "True" && "$encryption_enabled" != "1" ]]; then
return 0
fi
if ! kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
return 0
fi
local server_tls
server_tls=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.certificates.serverTLSSecret}' 2>/dev/null || echo "")
if [[ -n "$server_tls" ]]; then
echo "Existing cluster '$CNPG_CLUSTER_NAME' already has TLS configured ($server_tls); no conversion needed."
return 0
fi
echo "WARN: Existing cluster '$CNPG_CLUSTER_NAME' does not have TLS/at-rest-encryption configured."
echo "At-rest encryption is now enabled; the cluster must be reprovisioned."
echo "NOTE: External full backup and restore from barman backup is not yet available."
echo " Removing existing cluster to allow re-creation with encryption enabled ..."
attempt_backup_if_active
kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true
echo "Existing unencrypted cluster removed. Will be re-created with TLS and encryption."
}
validate_ca_secret() {
local secret_name="$1"
local ns="$2"
if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then
return 1
fi
local crt key
crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true)
key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.key}' 2>/dev/null || true)
if [[ -z "$crt" || -z "$key" ]]; then
echo "WARN: CA secret '$secret_name' exists but is missing ca.crt or ca.key data." >&2
return 1
fi
if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then
echo "WARN: CA secret '$secret_name' has invalid certificate data." >&2
return 1
fi
# CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8
local key_header
key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1)
if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then
echo "WARN: CA secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2
return 1
fi
return 0
}
validate_tls_secret() {
local secret_name="$1"
local ns="$2"
if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then
return 1
fi
local crt key
crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true)
key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.key}' 2>/dev/null || true)
if [[ -z "$crt" || -z "$key" ]]; then
echo "WARN: TLS secret '$secret_name' exists but is missing tls.crt or tls.key data." >&2
return 1
fi
if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then
echo "WARN: TLS secret '$secret_name' has invalid certificate data." >&2
return 1
fi
# CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8
local key_header
key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1)
if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then
echo "WARN: TLS secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2
return 1
fi
return 0
}
generate_tls_if_missing() {
local ca_secret_name="${CNPG_CLUSTER_NAME}-ca"
local tls_secret_name="${CNPG_CLUSTER_NAME}-tls"
local need_ca=0
local need_tls=0
if validate_ca_secret "$ca_secret_name" "$NAMESPACE"; then
echo "CA secret $ca_secret_name already exists and is valid."
else
if kubectl -n "$NAMESPACE" get secret "$ca_secret_name" >/dev/null 2>&1; then
echo "Removing invalid CA secret '$ca_secret_name' ..."
kubectl -n "$NAMESPACE" delete secret "$ca_secret_name" --ignore-not-found
fi
need_ca=1
fi
if validate_tls_secret "$tls_secret_name" "$NAMESPACE"; then
echo "TLS secret $tls_secret_name already exists and is valid."
else
if kubectl -n "$NAMESPACE" get secret "$tls_secret_name" >/dev/null 2>&1; then
echo "Removing invalid TLS secret '$tls_secret_name' ..."
kubectl -n "$NAMESPACE" delete secret "$tls_secret_name" --ignore-not-found
fi
need_tls=1
fi
if [[ "$need_ca" -eq 0 && "$need_tls" -eq 0 ]]; then
echo "TLS secrets already present and valid; skipping generation."
return 0
fi
local TMPD
TMPD=$(mktemp -d)
if [[ "$need_ca" -eq 1 ]]; then
echo "Generating self-signed CA (EC P-256) for CNPG ..."
openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/ca.key"
openssl req -x509 -new -key "$TMPD/ca.key" -out "$TMPD/ca.crt" -days 3650 -subj "/CN=Prole CNPG CA"
kubectl -n "$NAMESPACE" create secret generic "$ca_secret_name" \
--from-file=ca.crt="$TMPD/ca.crt" \
--from-file=ca.key="$TMPD/ca.key" \
--dry-run=client -o yaml | kubectl apply -f -
else
# Extract existing CA for signing the server certificate
kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.crt}' | base64 -d > "$TMPD/ca.crt"
kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.key}' | base64 -d > "$TMPD/ca.key"
fi
if [[ "$need_tls" -eq 1 ]]; then
echo "Generating server TLS certificate for CNPG ($tls_secret_name) ..."
openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/tls.key"
openssl req -new -key "$TMPD/tls.key" -out "$TMPD/tls.csr" \
-subj "/CN=${CNPG_CLUSTER_NAME}.${NAMESPACE}.svc"
openssl x509 -req -in "$TMPD/tls.csr" -CA "$TMPD/ca.crt" -CAkey "$TMPD/ca.key" \
-CAcreateserial -out "$TMPD/tls.crt" -days 365 \
-extfile <(printf "subjectAltName=DNS:%s,DNS:%s-rw,DNS:%s-rw.%s.svc,DNS:%s-r,DNS:%s-ro" \
"$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$NAMESPACE" \
"$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME")
kubectl -n "$NAMESPACE" create secret tls "$tls_secret_name" \
--cert="$TMPD/tls.crt" \
--key="$TMPD/tls.key" \
--dry-run=client -o yaml | kubectl apply -f -
fi
rm -rf "$TMPD"
}
# Resolve latest CNPG version from GitHub if possible, fallback to a sensible default.
get_latest_cnpg_version() {
local version
version=$(curl -s "https://api.github.com/repos/cloudnative-pg/cloudnative-pg/releases/latest" | jq -r '.tag_name' | sed 's/^v//' || echo "")
if [[ -z "$version" || "$version" == "null" ]]; then
echo "1.27.0"
else
echo "$version"
fi
}
initialize() {
ensure_tools
ensure_namespace
attempt_backup_if_active
ensure_cnpg_operator
pin_cnpg_controller
ensure_barman_plugin
echo "Using namespace: $NAMESPACE"
if ! fetch_admin_keys_and_db_pass_from_bao_or_local; then
echo "ERROR: Failed to fetch/ensure database secrets and admin keys." >&2
return 1
fi
apply_cnpg_admin_secret
detect_and_reprovision_unencrypted_cluster
generate_tls_if_missing
# Pre-flight: verify required secrets exist before deploying cluster
local _missing_secrets=""
if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
_missing_secrets="${_missing_secrets} prole-db-user"
fi
if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-tls" >/dev/null 2>&1; then
_missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-tls"
fi
if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-ca" >/dev/null 2>&1; then
_missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-ca"
fi
if [[ -n "$_missing_secrets" ]]; then
echo "ERROR: Required secrets missing in namespace '$NAMESPACE':${_missing_secrets}" >&2
echo "The CNPG cluster cannot start without these secrets." >&2
return 1
fi
echo "Pre-flight check passed: all required secrets present."
if ! _ensure_prole_db_image; then
echo "ERROR: Pre-flight image check failed; aborting cluster initialization." >&2
return 1
fi
ensure_prole_stack_resources
if ! wait_for_cnpg_pods 300; then
if cluster_has_pods; then
echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2
if force_rollout; then
if ! wait_for_cnpg_pods 300; then
echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2
if ! reset_and_reinit; then
return 1
fi
fi
else
echo "WARN: Force rollout failed; attempting reset and re-init ..." >&2
if ! reset_and_reinit; then
return 1
fi
fi
else
echo "WARN: No CNPG pods found — this is a new installation." >&2
echo "The CNPG operator was unable to create cluster pods." >&2
echo "Check operator logs: kubectl -n cnpg-system logs -l app.kubernetes.io/name=cloudnative-pg" >&2
echo "Check cluster status: kubectl -n $NAMESPACE describe cluster $CNPG_CLUSTER_NAME" >&2
return 1
fi
fi
echo "Initialization complete for CNPG + cert artifacts."
prole_register_port_forward "postgres" "${NAMESPACE:-default}" "svc/${CNPG_CLUSTER_NAME}-rw" "5432" "5432" "0.0.0.0" "TCP" "PostgreSQL"
}
update_reload() {
initialize
}
deploy_cluster() {
ensure_tools
ensure_namespace
ensure_cnpg_operator
pin_cnpg_controller
ensure_barman_plugin
local image
if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then
image=$(get_latest_image)
else
image="prole-db:$VERSION"
fi
image=$(resolve_cnpg_image "$image")
sync_manifest_image "$image"
generate_tls_if_missing
echo "Deploying $image to cluster $CNPG_CLUSTER_NAME in namespace $NAMESPACE..."
_ensure_prole_db_image || true
apply_cnpg_cluster_manifest "$CNPG_MANIFEST"
local current_image
current_image=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "")
if [[ -n "$current_image" && "$current_image" != "$image" ]]; then
echo "Patching cluster to use image '$image' (was '$current_image')..."
kubectl -n "$NAMESPACE" patch cluster "$CNPG_CLUSTER_NAME" --type merge -p "{\"spec\": {\"imageName\": \"$image\"}}"
if kubectl cnpg version >/dev/null 2>&1; then
kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
fi
fi
}
case "$ACTION" in
recreate)
ensure_tools
"$0" delete "$CNPG_CLUSTER_NAME"
# Clean up PVCs left behind by the CNPG operator (not in static manifests)
echo "Cleaning up PVCs in namespace '$NAMESPACE' ..."
kubectl -n "$NAMESPACE" delete pvc --all --ignore-not-found 2>/dev/null || true
# Wait briefly for pods to fully terminate before re-creating
echo "Waiting for pods to terminate in namespace '$NAMESPACE' ..."
_wait_term=0
while kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null | grep -qv '^No resources'; do
sleep 3
_wait_term=$(( _wait_term + 3 ))
if (( _wait_term >= 60 )); then
echo "WARN: Pods still present after 60s; proceeding anyway." >&2
break
fi
done
"$0" create "$CNPG_CLUSTER_NAME"
;;
create)
ensure_tools
ensure_namespace
initialize
;;
delete)
ensure_tools
echo "Deleting all resources for '$CNPG_CLUSTER_NAME' ..."
# Delete each manifest individually, mirroring the apply pattern:
# ingress.yaml contains multi-namespace resources and must be deleted without -n.
for _del_f in "$SCRIPT_DIR/../k8s/prole"/*.yaml; do
_del_base=$(basename "$_del_f")
case "$_del_base" in
kustomization.yaml|ingress.yaml) continue ;;
*) kubectl delete -n "$NAMESPACE" -f "$_del_f" --ignore-not-found 2>&1 \
| grep -v "^Error from server (NotFound)" || true ;;
esac
done
if [[ -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" ]]; then
kubectl delete -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" --ignore-not-found 2>&1 \
| grep -v "^Error from server (NotFound)" || true
fi
;;
start)
ensure_tools
ensure_namespace
ensure_cnpg_operator
pin_cnpg_controller
ensure_barman_plugin
if [[ ! -f "$CNPG_MANIFEST" ]]; then
echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
exit 1
fi
echo "Starting CloudNative-PG cluster from $CNPG_MANIFEST in namespace $NAMESPACE..."
prole_render_manifest "$CNPG_MANIFEST" | kubectl apply -n "$NAMESPACE" -f -
;;
stop)
ensure_tools
if [[ ! -f "$CNPG_MANIFEST" ]]; then
echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
exit 1
fi
echo "Stopping CloudNative-PG cluster using $CNPG_MANIFEST ..."
kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found
;;
status)
ensure_tools
echo "--- CloudNative-PG Cluster Status ($CNPG_CLUSTER_NAME) ---"
if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME"
echo ""
echo "CNPG Plugin Status:"
if kubectl cnpg version >/dev/null 2>&1; then
kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE"
else
echo "Note: 'kubectl cnpg' plugin not found; skipping detailed status."
fi
else
echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'."
fi
;;
restart)
ensure_tools
"$0" stop
"$0" start
;;
initialize)
if ! initialize; then
exit 1
fi
;;
update|reload)
if ! update_reload; then
exit 1
fi
;;
deploy)
deploy_cluster
;;
rollout|force-rollout)
rollout_cluster
;;
install-barman-plugin)
ensure_tools
ensure_cnpg_operator
pin_cnpg_controller
ensure_barman_plugin
;;
*)
echo "Usage: $0 {create|delete|recreate|start|stop|status|restart|initialize|update|reload|deploy|rollout|install-barman-plugin} [cluster] [version]" >&2
exit 2
;;
esac