mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 10:13:58 +00:00
5618b662dd
Summary: Removed the prole-db-manager microservice and simplified deployment to use prole-authority as the internal management and authorization point. Fixed two blocking bugs that prevented silent install from completing on knoe-dev-cluster. Removed: prole-db-manager - Deleted db-manager-deployment.yaml and db-manager-service.yaml from opentofu manifests - Deleted src/db-manager/ (Dockerfile, server.js, package.json, tests) - Removed prole-db-manager port-forward mapping from installer/core/env.py - Removed init_db_manager.sh from Initialization Scripts (milestones.py, actions.py) - Removed init_certmgr.sh and init_db_manager.sh tabs from services screen (services.py) - Removed live k8s Deployment/Service from knoe-dev-cluster Fixed: PostgreSQL version downgrade error (pg17 -> pg18) - Created conf/postgresql/.version with value 18 - Updated k8s/prole/prole-db.yaml and prole-db-recovery.yaml.tpl imageName to prole-db:18-089 - Fixed _init_database_options_state() to restore saved version_type from prole.cfg so db_version_type defaults to v18 (pg18) instead of silently reverting to pg17 - Added database_options.* keys to _collect_input_snapshot() in cfg.py so distribution, version_type, and all extension toggles persist to prole.cfg Fixed: Cluster name inconsistency - Removed stale prole-dev-cluster references; all scripts now use knoe-dev-cluster - Added knoe-dev-cluster to mode-detection case in etc/prole_cfg.sh Config: conf/prole.cfg - Set kerberos_config.enabled = False, KERBEROS_AUTO_ENABLED = False - Added database_options.distribution = percona, version_type = v18 - Added all 13 extension flags set to True (postgis, pgvector, pgcrypto, pgaudit, pg_repack, pg_stat_statements, pg_buffercache, pg_freespacemap, pgrowlocks, postgres_fdw, dblink, pg_stat_monitor, pgbadger) Verification: ./install.py -s -l -v -c conf/prole.cfg completed successfully. CNPG deployed prole-db:18-089 to knoe-dev-cluster; all milestones passed. Co-authored-by: Junie <junie@jetbrains.com>
1787 lines
64 KiB
Bash
Executable File
1787 lines
64 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
|
||
set -euo pipefail
|
||
|
||
# init_cloudnative_pg.sh
|
||
# Purpose:
|
||
# - Distribute administrator ed25519 key pair to CloudNative‑PG as a Kubernetes Secret for cert auth
|
||
# - Patch CNPG cluster to enable TLS where possible
|
||
#
|
||
# Usage:
|
||
# ./init_cloudnative_pg.sh start|stop|status|restart
|
||
# ./init_cloudnative_pg.sh initialize # install CNPG, create cluster, configure secrets
|
||
# ./init_cloudnative_pg.sh update|reload # re-apply/patch
|
||
# ./init_cloudnative_pg.sh deploy [version] # apply CNPG manifest and update image
|
||
# ./init_cloudnative_pg.sh rollout # rolling restart of CNPG pods
|
||
#
|
||
# Requirements:
|
||
# - init_openbao.sh has been run (OpenBao running as a local container)
|
||
# - $PROLE_HOME/env.sh or $HOME/.prole/env.sh defining PROLE_SERVICE
|
||
# - Optional: CNPG_MANIFEST_OVERRIDE to apply a recovery manifest instead of kustomize
|
||
|
||
# Initialize SCRIPT_DIR
|
||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||
|
||
# Load environment and config via prole_cfg.sh
|
||
# shellcheck disable=SC1090
|
||
source "$SCRIPT_DIR/prole_cfg.sh"
|
||
|
||
if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
|
||
prole_set_mode "${2:-}"
|
||
shift 2
|
||
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
|
||
prole_set_mode "${1#*=}"
|
||
shift
|
||
fi
|
||
|
||
if [[ -z "${PROLE_SERVICE:-}" ]]; then
|
||
echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2
|
||
exit 1
|
||
fi
|
||
|
||
ACTION=${1:-}
|
||
CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-prole-db}
|
||
VERSION=${2:-latest}
|
||
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
|
||
REALM=${REALM:-PROLE.ORG}
|
||
DOMAIN=${DOMAIN:-prole.org}
|
||
CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-}
|
||
PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}
|
||
BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup}
|
||
BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800}
|
||
RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl"
|
||
BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-}
|
||
BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0}
|
||
CERT_MANAGER_MANIFEST_URL=${CERT_MANAGER_MANIFEST_URL:-}
|
||
CERT_MANAGER_FALLBACK_VERSION=${CERT_MANAGER_FALLBACK_VERSION:-1.19.3}
|
||
|
||
if [[ "$ACTION" != "deploy" && "$ACTION" != "rollout" && "$ACTION" != "force-rollout" ]]; then
|
||
if [[ -n "${2:-}" ]]; then
|
||
CNPG_CLUSTER_NAME="${2}"
|
||
fi
|
||
VERSION=${3:-latest}
|
||
fi
|
||
|
||
# Support both PROLE_HOME/k8s and sibling k8s directory
|
||
if [[ -d "$SCRIPT_DIR/../k8s/prole" ]]; then
|
||
K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole"
|
||
elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/prole" ]]; then
|
||
K8S_PROLE_DIR="$PROLE_HOME/k8s/prole"
|
||
else
|
||
K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole"
|
||
fi
|
||
CNPG_MANIFEST="$K8S_PROLE_DIR/prole-db.yaml"
|
||
BARMAN_OBJECTSTORE_MANIFEST="$K8S_PROLE_DIR/prole-db-barman-objectstore.yaml"
|
||
|
||
SECRETS_DIR="$PROLE_SERVICE/secrets"
|
||
# Resolving CNPG admin keys.
|
||
# We prefer names without algorithm suffixes to be more generic, matching install.py fallback strategy.
|
||
ADMIN_PRIV_ED25519="$SECRETS_DIR/admin_ed25519.key"
|
||
ADMIN_PUB_ED25519="$SECRETS_DIR/admin_ed25519.pub"
|
||
ADMIN_PRIV_GENERIC="$SECRETS_DIR/admin.key"
|
||
ADMIN_PUB_GENERIC="$SECRETS_DIR/admin.pub"
|
||
OPENBAO_TOKEN_FILE="$SECRETS_DIR/openbao-root-token"
|
||
BAO_NAMESPACE="${NAMESPACE:-default}"
|
||
OPENBAO_NAMESPACE="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}"
|
||
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
|
||
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
|
||
BAO_PATH_DB="${BAO_PATH_PREFIX}/db"
|
||
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"
|
||
|
||
ensure_tools() {
|
||
for t in kubectl curl openssl base64 jq; do
|
||
command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
|
||
done
|
||
}
|
||
|
||
ensure_namespace() {
|
||
if [[ -z "${NAMESPACE:-}" ]]; then
|
||
echo "ERROR: NAMESPACE is empty. Check env.sh or conf/prole.cfg." >&2
|
||
exit 1
|
||
fi
|
||
if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
|
||
echo "Creating namespace '$NAMESPACE' ..."
|
||
kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true
|
||
fi
|
||
}
|
||
|
||
get_latest_image() {
|
||
local pg_version_file release_file pg_version release
|
||
if [[ -f "$SCRIPT_DIR/../conf/postgresql/.version" ]]; then
|
||
pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version"
|
||
elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/conf/postgresql/.version" ]]; then
|
||
pg_version_file="$PROLE_HOME/conf/postgresql/.version"
|
||
else
|
||
pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version"
|
||
fi
|
||
|
||
if [[ -f "$SCRIPT_DIR/../prole-db/.version" ]]; then
|
||
release_file="$SCRIPT_DIR/../prole-db/.version"
|
||
elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/prole-db/.version" ]]; then
|
||
release_file="$PROLE_HOME/prole-db/.version"
|
||
else
|
||
release_file="$SCRIPT_DIR/../prole-db/.version"
|
||
fi
|
||
|
||
if [[ -f "$pg_version_file" ]]; then
|
||
pg_version=$(tr -d '[:space:]' < "$pg_version_file")
|
||
else
|
||
pg_version="17.7"
|
||
fi
|
||
|
||
if [[ -f "$release_file" ]]; then
|
||
release=$(tr -d '[:space:]' < "$release_file")
|
||
else
|
||
release="43"
|
||
fi
|
||
|
||
if [[ "$release" =~ ^[0-9]+$ ]]; then
|
||
release=$(printf "%03d" "$release")
|
||
fi
|
||
|
||
echo "prole-db:${pg_version}-${release}"
|
||
}
|
||
|
||
resolve_cnpg_image() {
|
||
local image="$1"
|
||
if _prole_local_registry_enabled; then
|
||
# Prefer the in-cluster/internal registry for k3d/k3s nodes pulling images.
|
||
# Using LOCAL_REGISTRY (e.g., localhost:5000) breaks inside cluster and may
|
||
# also prefer IPv6 ::1, leading to connection refused. Avoid it.
|
||
local registry=""
|
||
if [[ -n "${LOCAL_REGISTRY_INTERNAL:-}" ]]; then
|
||
registry="${LOCAL_REGISTRY_INTERNAL}"
|
||
elif [[ -n "${LOCAL_REGISTRY:-}" ]]; then
|
||
# Only fall back to LOCAL_REGISTRY when INTERNAL is not available
|
||
# and it's not pointing at localhost (which is invalid for cluster pulls).
|
||
if [[ "${LOCAL_REGISTRY}" != "localhost:5000" && "${LOCAL_REGISTRY}" != "127.0.0.1:5000" ]]; then
|
||
registry="${LOCAL_REGISTRY}"
|
||
fi
|
||
fi
|
||
|
||
if [[ -n "$registry" ]]; then
|
||
local first="${image%%/*}"
|
||
# If the image is unqualified (no registry), prefix it with the chosen registry
|
||
if [[ "$image" != */* ]]; then
|
||
image="${registry}/${image}"
|
||
# If the first path segment has no dot/colon, it's still unqualified (e.g., prole-db:TAG)
|
||
elif [[ "$first" != *"."* && "$first" != *":"* ]]; then
|
||
image="${registry}/${image}"
|
||
fi
|
||
fi
|
||
fi
|
||
printf '%s' "$image"
|
||
}
|
||
|
||
sync_manifest_image() {
|
||
local image="$1"
|
||
local files=()
|
||
if [[ -n "$CNPG_MANIFEST" ]]; then
|
||
files+=("$CNPG_MANIFEST")
|
||
fi
|
||
if [[ -f "$RECOVERY_TEMPLATE" ]]; then
|
||
files+=("$RECOVERY_TEMPLATE")
|
||
fi
|
||
local f tmp
|
||
for f in "${files[@]}"; do
|
||
if [[ -f "$f" ]] && grep -qE '^[[:space:]]*imageName:' "$f"; then
|
||
tmp=$(mktemp)
|
||
sed -E "s|^([[:space:]]*imageName:).*|\\1 ${image}|" "$f" > "$tmp"
|
||
mv "$tmp" "$f"
|
||
fi
|
||
done
|
||
}
|
||
|
||
openbao_url() {
|
||
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
|
||
echo "$PROLE_OPENBAO_URL"
|
||
return 0
|
||
fi
|
||
if prole_is_in_cluster; then
|
||
echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200"
|
||
return 0
|
||
fi
|
||
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
|
||
echo "http://127.0.0.1:8200"
|
||
return 0
|
||
elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
|
||
echo "http://127.0.0.1:8200"
|
||
return 0
|
||
else
|
||
echo ""
|
||
return 0
|
||
fi
|
||
}
|
||
|
||
openbao_token() {
|
||
if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
|
||
cat "$OPENBAO_TOKEN_FILE"
|
||
else
|
||
echo "${OPENBAO_ROOT_TOKEN:-}"
|
||
fi
|
||
}
|
||
|
||
fetch_openbao_secret() {
|
||
local path="$1"
|
||
local key="$2"
|
||
local token url
|
||
token=$(openbao_token)
|
||
url=$(openbao_url)
|
||
if [[ -z "$token" || -z "$url" ]]; then
|
||
echo ""
|
||
return 0
|
||
fi
|
||
curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
|
||
}
|
||
|
||
resolve_db_password() {
|
||
local db_pw="${DB_PASSWORD:-}"
|
||
if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
|
||
local fetched_db
|
||
fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password")
|
||
if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
|
||
db_pw="$fetched_db"
|
||
fi
|
||
fi
|
||
printf '%s' "$db_pw"
|
||
}
|
||
|
||
|
||
ensure_cnpg_operator() {
|
||
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
|
||
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
|
||
wait_for_cnpg_webhook 180 || true
|
||
return 0
|
||
fi
|
||
|
||
local latest_version minor_version yaml_url
|
||
latest_version=$(get_latest_cnpg_version)
|
||
minor_version=$(echo "$latest_version" | cut -d. -f1,2)
|
||
yaml_url="https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${minor_version}/releases/cnpg-${latest_version}.yaml"
|
||
|
||
echo "Installing CloudNative-PG operator version ${latest_version} ..."
|
||
kubectl apply --server-side -f "$yaml_url"
|
||
|
||
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
|
||
kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true
|
||
wait_for_cnpg_webhook 180 || true
|
||
fi
|
||
}
|
||
|
||
get_latest_barman_plugin_version() {
|
||
local version tag
|
||
tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "")
|
||
if [[ -z "$tag" || "$tag" == "null" ]]; then
|
||
echo "v${BARMAN_PLUGIN_FALLBACK_VERSION}"
|
||
return 0
|
||
fi
|
||
echo "$tag"
|
||
}
|
||
|
||
resolve_barman_plugin_manifest_url() {
|
||
if [[ -n "$BARMAN_PLUGIN_MANIFEST_URL" ]]; then
|
||
echo "$BARMAN_PLUGIN_MANIFEST_URL"
|
||
return 0
|
||
fi
|
||
local tag
|
||
tag=$(get_latest_barman_plugin_version)
|
||
echo "https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/${tag}/manifest.yaml"
|
||
}
|
||
|
||
cert_manager_ready() {
|
||
if ! kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
|
||
return 1
|
||
fi
|
||
if ! kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
resolve_control_plane_selector() {
|
||
local node=""
|
||
node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||
if [[ -n "$node" ]]; then
|
||
printf 'kubernetes.io/hostname=%s' "$node"
|
||
fi
|
||
}
|
||
|
||
pin_cert_manager() {
|
||
local selector="${CERT_MANAGER_NODE_SELECTOR:-}"
|
||
if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then
|
||
selector="$(resolve_control_plane_selector)"
|
||
fi
|
||
if [[ -z "$selector" ]]; then
|
||
return 0
|
||
fi
|
||
local key value
|
||
key="${selector%%=*}"
|
||
value="${selector#*=}"
|
||
if [[ -z "$key" || -z "$value" ]]; then
|
||
echo "WARN: CERT_MANAGER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2
|
||
return 0
|
||
fi
|
||
if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then
|
||
echo "WARN: No nodes match CERT_MANAGER_NODE_SELECTOR=${selector}. Skipping pin." >&2
|
||
return 0
|
||
fi
|
||
echo "Pinning cert-manager deployments to nodes with ${selector} ..."
|
||
for dep in cert-manager cert-manager-webhook cert-manager-cainjector; do
|
||
kubectl -n cert-manager patch deployment "$dep" --type merge \
|
||
-p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true
|
||
done
|
||
kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true
|
||
}
|
||
|
||
pin_barman_cloud() {
|
||
local selector="${BARMAN_NODE_SELECTOR:-}"
|
||
if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then
|
||
selector="$(resolve_control_plane_selector)"
|
||
fi
|
||
if [[ -z "$selector" ]]; then
|
||
return 0
|
||
fi
|
||
local key value
|
||
key="${selector%%=*}"
|
||
value="${selector#*=}"
|
||
if [[ -z "$key" || -z "$value" ]]; then
|
||
echo "WARN: BARMAN_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2
|
||
return 0
|
||
fi
|
||
if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then
|
||
echo "WARN: No nodes match BARMAN_NODE_SELECTOR=${selector}. Skipping pin." >&2
|
||
return 0
|
||
fi
|
||
echo "Pinning barman-cloud deployment to nodes with ${selector} ..."
|
||
kubectl -n cnpg-system patch deployment barman-cloud --type merge \
|
||
-p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true
|
||
kubectl -n cnpg-system rollout restart deploy/barman-cloud >/dev/null 2>&1 || true
|
||
}
|
||
|
||
get_latest_cert_manager_version() {
|
||
local tag
|
||
tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cert-manager/cert-manager/releases/latest" | jq -r '.tag_name' || echo "")
|
||
if [[ -z "$tag" || "$tag" == "null" ]]; then
|
||
echo "v${CERT_MANAGER_FALLBACK_VERSION}"
|
||
return 0
|
||
fi
|
||
echo "$tag"
|
||
}
|
||
|
||
resolve_cert_manager_manifest_url() {
|
||
if [[ -n "$CERT_MANAGER_MANIFEST_URL" ]]; then
|
||
echo "$CERT_MANAGER_MANIFEST_URL"
|
||
return 0
|
||
fi
|
||
local tag
|
||
tag=$(get_latest_cert_manager_version)
|
||
echo "https://github.com/cert-manager/cert-manager/releases/download/${tag}/cert-manager.yaml"
|
||
}
|
||
|
||
ensure_cert_manager() {
|
||
if cert_manager_ready; then
|
||
pin_cert_manager
|
||
if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
|
||
kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true
|
||
kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true
|
||
kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true
|
||
fi
|
||
return 0
|
||
fi
|
||
|
||
local cm_url
|
||
cm_url=$(resolve_cert_manager_manifest_url)
|
||
echo "Installing cert-manager from ${cm_url} ..."
|
||
kubectl apply -f "$cm_url"
|
||
pin_cert_manager
|
||
|
||
if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then
|
||
kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true
|
||
kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true
|
||
kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true
|
||
fi
|
||
}
|
||
|
||
wait_for_barman_crd() {
|
||
local timeout=${1:-120}
|
||
local start_time
|
||
start_time=$(date +%s)
|
||
echo "Waiting for Barman Cloud CRD (timeout: ${timeout}s)..."
|
||
while true; do
|
||
if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then
|
||
echo "Barman Cloud CRD is available."
|
||
return 0
|
||
fi
|
||
local elapsed=$(( $(date +%s) - start_time ))
|
||
if (( elapsed > timeout )); then
|
||
echo "Barman Cloud CRD not ready after ${elapsed}s." >&2
|
||
return 1
|
||
fi
|
||
if (( elapsed % 30 < 6 && elapsed > 5 )); then
|
||
echo " [${elapsed}s/${timeout}s] Still waiting for Barman Cloud CRD..."
|
||
fi
|
||
sleep 5
|
||
done
|
||
}
|
||
|
||
wait_for_barman_tls_secrets() {
|
||
local timeout=${1:-180}
|
||
local start_time
|
||
start_time=$(date +%s)
|
||
echo "Waiting for Barman Cloud TLS secrets (timeout: ${timeout}s)..."
|
||
while true; do
|
||
if kubectl -n cnpg-system get secret barman-cloud-client-tls >/dev/null 2>&1 \
|
||
&& kubectl -n cnpg-system get secret barman-cloud-server-tls >/dev/null 2>&1; then
|
||
echo "Barman Cloud TLS secrets are available."
|
||
return 0
|
||
fi
|
||
local elapsed=$(( $(date +%s) - start_time ))
|
||
if (( elapsed > timeout )); then
|
||
echo "Barman Cloud TLS secrets not ready after ${elapsed}s." >&2
|
||
return 1
|
||
fi
|
||
if (( elapsed % 30 < 6 && elapsed > 5 )); then
|
||
echo " [${elapsed}s/${timeout}s] Still waiting for Barman Cloud TLS secrets..."
|
||
fi
|
||
sleep 5
|
||
done
|
||
}
|
||
|
||
ensure_barman_plugin() {
|
||
ensure_cert_manager
|
||
|
||
local plugin_url
|
||
plugin_url=$(resolve_barman_plugin_manifest_url)
|
||
echo "Installing Barman Cloud plugin from ${plugin_url} ..."
|
||
local apply_out=""
|
||
if ! apply_out=$(kubectl apply -f "$plugin_url" 2>&1); then
|
||
echo "$apply_out" >&2
|
||
if echo "$apply_out" | grep -qi "webhook.cert-manager.io"; then
|
||
echo "WARN: cert-manager webhook error detected; restarting cert-manager components and retrying..." >&2
|
||
kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true
|
||
kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s >/dev/null 2>&1 || true
|
||
kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s >/dev/null 2>&1 || true
|
||
kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s >/dev/null 2>&1 || true
|
||
kubectl apply -f "$plugin_url" || true
|
||
fi
|
||
else
|
||
printf '%s\n' "$apply_out"
|
||
fi
|
||
|
||
if ! wait_for_barman_crd 120; then
|
||
echo "WARN: Barman Cloud ObjectStore CRD not ready after install." >&2
|
||
fi
|
||
|
||
if ! wait_for_barman_tls_secrets 180; then
|
||
echo "WARN: Barman Cloud TLS secrets not ready after install." >&2
|
||
fi
|
||
|
||
pin_barman_cloud
|
||
|
||
if kubectl -n cnpg-system get deploy barman-cloud >/dev/null 2>&1; then
|
||
kubectl -n cnpg-system rollout status deploy/barman-cloud --timeout=180s || true
|
||
fi
|
||
}
|
||
|
||
pin_cnpg_controller() {
|
||
local selector="${CNPG_CONTROLLER_NODE_SELECTOR:-}"
|
||
if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then
|
||
if kubectl get nodes -l "storage=primary" >/dev/null 2>&1; then
|
||
selector="storage=primary"
|
||
else
|
||
local cp_node=""
|
||
cp_node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||
if [[ -n "$cp_node" ]]; then
|
||
selector="kubernetes.io/hostname=${cp_node}"
|
||
fi
|
||
fi
|
||
fi
|
||
if [[ -z "$selector" ]]; then
|
||
return 0
|
||
fi
|
||
local key value
|
||
key="${selector%%=*}"
|
||
value="${selector#*=}"
|
||
if [[ -z "$key" || -z "$value" ]]; then
|
||
echo "WARN: CNPG_CONTROLLER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2
|
||
return 0
|
||
fi
|
||
if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then
|
||
echo "WARN: No nodes match CNPG_CONTROLLER_NODE_SELECTOR=${selector}. Skipping pin." >&2
|
||
return 0
|
||
fi
|
||
echo "Pinning cnpg-controller-manager to nodes with ${selector} ..."
|
||
kubectl -n cnpg-system patch deployment cnpg-controller-manager --type merge \
|
||
-p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true
|
||
}
|
||
|
||
wait_for_cnpg_webhook() {
|
||
local timeout=${1:-120}
|
||
local start_time
|
||
start_time=$(date +%s)
|
||
|
||
echo "Waiting for CNPG webhook service endpoints to be ready (timeout: ${timeout}s)..."
|
||
while true; do
|
||
local endpoints
|
||
endpoints=$(kubectl -n cnpg-system get endpoints cnpg-webhook-service -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
|
||
if [[ -n "$endpoints" ]]; then
|
||
echo "CNPG webhook service has endpoints."
|
||
return 0
|
||
fi
|
||
local elapsed=$(( $(date +%s) - start_time ))
|
||
if (( elapsed > timeout )); then
|
||
echo "WARN: CNPG webhook endpoints not ready after ${elapsed}s." >&2
|
||
kubectl -n cnpg-system get pods 2>/dev/null >&2 || true
|
||
return 1
|
||
fi
|
||
if (( elapsed % 30 < 6 && elapsed > 5 )); then
|
||
echo " [${elapsed}s/${timeout}s] Waiting for CNPG webhook endpoints..."
|
||
kubectl -n cnpg-system get pods --no-headers 2>/dev/null | sed 's/^/ /' || true
|
||
fi
|
||
sleep 5
|
||
done
|
||
}
|
||
|
||
apply_barman_objectstore_if_present() {
|
||
if [[ ! -f "$BARMAN_OBJECTSTORE_MANIFEST" ]]; then
|
||
return 0
|
||
fi
|
||
if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then
|
||
if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
|
||
local endpoint
|
||
endpoint="http://garage.${SERVICE_NAMESPACE}.svc.cluster.local:3900"
|
||
prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" \
|
||
| sed -E "s|^[[:space:]]*endpointURL:.*| endpointURL: ${endpoint}|" \
|
||
| kubectl apply -n "$NAMESPACE" -f -
|
||
else
|
||
prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" | kubectl apply -n "$NAMESPACE" -f -
|
||
fi
|
||
else
|
||
echo "WARN: Barman Cloud ObjectStore CRD not found; skipping $BARMAN_OBJECTSTORE_MANIFEST."
|
||
fi
|
||
}
|
||
|
||
apply_prole_manifest_file() {
|
||
local file="$1"
|
||
local output=""
|
||
if output=$(prole_render_manifest "$file" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then
|
||
printf '%s\n' "$output"
|
||
return 0
|
||
fi
|
||
if [[ "${PROLE_MODE:-}" == "k3d" && "$(basename "$file")" == "garage-statefulset.yaml" ]] \
|
||
&& echo "$output" | grep -q "updates to statefulset spec"; then
|
||
echo "WARN: Garage StatefulSet immutable in k3d; skipping apply."
|
||
return 0
|
||
fi
|
||
echo "$output" >&2
|
||
return 1
|
||
}
|
||
|
||
# Push image to local registry (via LOCAL_REGISTRY host address) with k3d import fallback.
|
||
_push_to_k3d_registry() {
|
||
local image="$1"
|
||
local cluster_name="$2"
|
||
local push_host="${LOCAL_REGISTRY:-localhost:5000}"
|
||
local plain_image="${image##*/}" # strip registry prefix, e.g. prole-db:18-088
|
||
|
||
if [[ -n "$push_host" ]]; then
|
||
local push_ref="${push_host}/${plain_image}"
|
||
docker tag "$image" "$push_ref" 2>/dev/null || true
|
||
if docker push "$push_ref" 2>/dev/null; then
|
||
echo " ✓ Image pushed to registry at '${push_host}'."
|
||
return 0
|
||
fi
|
||
echo " WARN: push to '${push_host}' failed; falling back to k3d image import ..." >&2
|
||
fi
|
||
|
||
if k3d image import "$image" -c "$cluster_name" 2>/dev/null; then
|
||
echo " ✓ Image '$image' imported directly into k3d cluster '$cluster_name'."
|
||
return 0
|
||
fi
|
||
echo "ERROR: Failed to push or import image '$image'." >&2
|
||
return 1
|
||
}
|
||
|
||
# Pre-flight: ensure the prole-db image is available in the k3d cluster before
|
||
# the CNPG operator ever tries to pull it, avoiding ErrImagePull backoff loops.
|
||
# Steps: containerd cache → Docker daemon (registry tag) → Docker daemon (plain tag)
|
||
# → tar import → docker build + push/import.
|
||
_ensure_prole_db_image() {
|
||
[[ "${PROLE_MODE:-}" != "k3d" ]] && return 0
|
||
|
||
local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
|
||
local image=""
|
||
if [[ -n "$image_override" ]]; then
|
||
image="$image_override"
|
||
else
|
||
if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then
|
||
image=$(get_latest_image)
|
||
else
|
||
image="prole-db:$VERSION"
|
||
fi
|
||
fi
|
||
image=$(resolve_cnpg_image "$image")
|
||
|
||
# Auto-detect active k3d cluster name
|
||
local cluster_name="${K3D_CLUSTER_NAME:-}"
|
||
if [[ -z "$cluster_name" ]]; then
|
||
cluster_name=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | head -1)
|
||
fi
|
||
cluster_name="${cluster_name:-knoe-dev-cluster}"
|
||
|
||
local prole_db_dir="${PROLE_HOME:-$SCRIPT_DIR/..}/prole-db"
|
||
local plain_image="${image##*/}" # e.g. prole-db:18-088
|
||
|
||
echo "Pre-flight: verifying image '$image' is available in k3d cluster '$cluster_name' ..."
|
||
|
||
# Step 1: check if already present in k3d containerd with matching digest
|
||
local containerd_digest local_digest containerd_sha
|
||
containerd_digest=$(docker exec "k3d-${cluster_name}-server-0" \
|
||
ctr images ls -q 2>/dev/null | grep -F "$image" | head -1 || true)
|
||
if [[ -n "$containerd_digest" ]]; then
|
||
local_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$image" 2>/dev/null \
|
||
| awk -F@ '{print $2}' || true)
|
||
containerd_sha=$(docker exec "k3d-${cluster_name}-server-0" \
|
||
ctr images ls 2>/dev/null | grep -F "$image" | awk '{print $3}' | head -1 || true)
|
||
if [[ -z "$local_digest" || "$containerd_sha" == "$local_digest" ]]; then
|
||
echo " ✓ Image '$image' already in k3d containerd (digest match); no import needed."
|
||
return 0
|
||
fi
|
||
echo " Image '$image' in containerd but digest mismatch (local: ${local_digest:-unknown}, containerd: ${containerd_sha:-unknown}); re-importing ..."
|
||
docker exec "k3d-${cluster_name}-server-0" ctr images rm "$image" 2>/dev/null || true
|
||
fi
|
||
|
||
# Step 2a: registry-tagged image in local Docker daemon → push + import
|
||
if docker image inspect "$image" >/dev/null 2>&1; then
|
||
echo " Image '$image' found in Docker daemon; pushing to registry ..."
|
||
_push_to_k3d_registry "$image" "$cluster_name"
|
||
return $?
|
||
fi
|
||
|
||
# Step 2b: plain-tagged image in local Docker daemon → tag + push + import
|
||
if docker image inspect "$plain_image" >/dev/null 2>&1; then
|
||
echo " Plain image '$plain_image' found in Docker daemon; tagging as '$image' and pushing ..."
|
||
docker tag "$plain_image" "$image"
|
||
_push_to_k3d_registry "$image" "$cluster_name"
|
||
return $?
|
||
fi
|
||
|
||
# Step 3: look for a matching tar in the docker-import directory
|
||
local prole_data="${PROLE_DATA:-$HOME/.prole/data}"
|
||
local docker_import_dir="${DOCKER_IMPORT_DIR:-${prole_data}/docker-import}"
|
||
local name_part="${plain_image%%:*}" # e.g. prole-db
|
||
local tag_part="${plain_image##*:}" # e.g. 18-088
|
||
local found_tar=""
|
||
if [[ -d "$docker_import_dir" ]]; then
|
||
for _t in "$docker_import_dir"/*.tar; do
|
||
[[ -f "$_t" ]] || continue
|
||
local _bn
|
||
_bn=$(basename "$_t" .tar)
|
||
if [[ "$_bn" == *"$name_part"* && "$_bn" == *"$tag_part"* ]]; then
|
||
found_tar="$_t"
|
||
break
|
||
fi
|
||
done
|
||
fi
|
||
if [[ -n "$found_tar" ]]; then
|
||
echo " Loading tar '$(basename "$found_tar")' into Docker daemon ..."
|
||
docker load -i "$found_tar"
|
||
docker tag "$plain_image" "$image" 2>/dev/null || true
|
||
_push_to_k3d_registry "$image" "$cluster_name"
|
||
return $?
|
||
fi
|
||
|
||
# Step 4: image not found anywhere — build from source then push + import
|
||
if [[ ! -f "$prole_db_dir/Dockerfile" ]]; then
|
||
echo "ERROR: Dockerfile not found in '$prole_db_dir'; cannot build prole-db image." >&2
|
||
return 1
|
||
fi
|
||
echo " Image '$image' not found in k3d, Docker daemon, or docker-import dir."
|
||
echo " Building prole-db image from '$prole_db_dir' ..."
|
||
if ! docker build -t "$plain_image" "$prole_db_dir"; then
|
||
echo "ERROR: docker build failed for image '$plain_image'." >&2
|
||
return 1
|
||
fi
|
||
docker tag "$plain_image" "$image"
|
||
echo " Build complete. Pushing '$image' to registry ..."
|
||
_push_to_k3d_registry "$image" "$cluster_name"
|
||
return $?
|
||
}
|
||
|
||
ensure_prole_stack_resources() {
|
||
echo "Applying CloudNative-PG cluster and related resources ..."
|
||
local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}"
|
||
local image=""
|
||
if [[ -n "$image_override" ]]; then
|
||
image="$image_override"
|
||
else
|
||
if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then
|
||
image=$(get_latest_image)
|
||
else
|
||
image="prole-db:$VERSION"
|
||
fi
|
||
fi
|
||
image=$(resolve_cnpg_image "$image")
|
||
sync_manifest_image "$image"
|
||
if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then
|
||
if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then
|
||
echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2
|
||
return 1
|
||
fi
|
||
local dir file
|
||
dir="$K8S_PROLE_DIR"
|
||
for file in "$dir"/*.yaml; do
|
||
case "$(basename "$file")" in
|
||
prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml|ingress.yaml)
|
||
continue
|
||
;;
|
||
openbao-statefulset.yaml|openbao-service.yaml)
|
||
if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
|
||
continue
|
||
fi
|
||
;;
|
||
garage-*.yaml|grafana-*.yaml|prometheus-*.yaml)
|
||
if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
|
||
continue
|
||
fi
|
||
;;
|
||
esac
|
||
apply_prole_manifest_file "$file"
|
||
done
|
||
# Apply ingress.yaml without -n flag so each document targets its own namespace
|
||
if [[ -f "$dir/ingress.yaml" ]]; then
|
||
# Ensure referenced namespaces exist before applying multi-namespace ingress
|
||
for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do
|
||
if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then
|
||
echo "Creating namespace '$_ing_ns' for ingress resource ..."
|
||
kubectl create namespace "$_ing_ns" 2>/dev/null || true
|
||
fi
|
||
done
|
||
prole_render_manifest "$dir/ingress.yaml" | kubectl apply -f - 2>&1 || true
|
||
fi
|
||
apply_barman_objectstore_if_present
|
||
apply_cnpg_cluster_manifest "$CNPG_MANIFEST_OVERRIDE"
|
||
else
|
||
local dir file
|
||
dir="$K8S_PROLE_DIR"
|
||
for file in "$dir"/*.yaml; do
|
||
case "$(basename "$file")" in
|
||
prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml|ingress.yaml)
|
||
continue
|
||
;;
|
||
openbao-statefulset.yaml|openbao-service.yaml)
|
||
if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
|
||
continue
|
||
fi
|
||
;;
|
||
garage-*.yaml|grafana-*.yaml|prometheus-*.yaml)
|
||
if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then
|
||
continue
|
||
fi
|
||
;;
|
||
esac
|
||
apply_prole_manifest_file "$file"
|
||
done
|
||
# Apply ingress.yaml without -n flag so each document targets its own namespace
|
||
if [[ -f "$dir/ingress.yaml" ]]; then
|
||
# Ensure referenced namespaces exist before applying multi-namespace ingress
|
||
for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do
|
||
if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then
|
||
echo "Creating namespace '$_ing_ns' for ingress resource ..."
|
||
kubectl create namespace "$_ing_ns" 2>/dev/null || true
|
||
fi
|
||
done
|
||
prole_render_manifest "$dir/ingress.yaml" | kubectl apply -f - 2>&1 || true
|
||
fi
|
||
apply_barman_objectstore_if_present
|
||
apply_cnpg_cluster_manifest "$CNPG_MANIFEST"
|
||
fi
|
||
|
||
# Ensure prole-index-html exists for prole deployment readiness probe
|
||
if ! kubectl get configmap prole-index-html -n "$NAMESPACE" >/dev/null 2>&1; then
|
||
echo "Creating prole-index-html configmap..."
|
||
printf "<html><body><h1>Prole</h1></body></html>" > /tmp/index.html
|
||
kubectl create configmap prole-index-html --from-file=/tmp/index.html -n "$NAMESPACE"
|
||
rm /tmp/index.html
|
||
fi
|
||
|
||
# Ensure prole-nginx-tls exists (self-signed for dev)
|
||
if ! kubectl get secret prole-nginx-tls -n "$NAMESPACE" >/dev/null 2>&1; then
|
||
echo "Generating self-signed prole-nginx-tls for development..."
|
||
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
|
||
-keyout /tmp/nginx-tls.key -out /tmp/nginx-tls.crt \
|
||
-subj "/CN=prole.org" >/dev/null 2>&1
|
||
kubectl create secret tls prole-nginx-tls --key /tmp/nginx-tls.key --cert /tmp/nginx-tls.crt -n "$NAMESPACE"
|
||
rm /tmp/nginx-tls.key /tmp/nginx-tls.crt
|
||
fi
|
||
}
|
||
|
||
apply_cnpg_cluster_manifest() {
|
||
local manifest="$1"
|
||
local attempts=${CNPG_APPLY_RETRIES:-6}
|
||
local i out
|
||
|
||
for ((i=1; i<=attempts; i++)); do
|
||
if out=$(prole_render_manifest "$manifest" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then
|
||
printf '%s\n' "$out"
|
||
return 0
|
||
fi
|
||
if echo "$out" | grep -q "cnpg-webhook-service"; then
|
||
echo "CNPG webhook not ready yet (attempt $i/$attempts). Retrying..."
|
||
sleep 5
|
||
continue
|
||
fi
|
||
echo "$out" >&2
|
||
return 1
|
||
done
|
||
|
||
echo "ERROR: Failed to apply CNPG manifest after $attempts attempts." >&2
|
||
echo "$out" >&2
|
||
return 1
|
||
}
|
||
|
||
wait_for_cnpg_pods() {
|
||
local timeout=${1:-${CNPG_WAIT_TIMEOUT:-900}}
|
||
local start_time
|
||
start_time=$(date +%s)
|
||
|
||
local target_pods="${CNPG_TARGET_PODS:-}"
|
||
if [[ -z "$target_pods" ]]; then
|
||
target_pods=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.instances}' 2>/dev/null || true)
|
||
fi
|
||
if [[ -z "$target_pods" ]]; then
|
||
target_pods=3
|
||
fi
|
||
|
||
echo "Waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' to be Running..."
|
||
echo " Timeout: ${timeout}s"
|
||
local last_feedback=0
|
||
local feedback_interval=30 # print detailed status every 30s
|
||
local _image_pull_remediated="" # set after first k3d import attempt to avoid loops
|
||
while true; do
|
||
local now elapsed
|
||
now=$(date +%s)
|
||
elapsed=$(( now - start_time ))
|
||
|
||
local pods
|
||
pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null || true)
|
||
|
||
if [[ -n "$pods" ]]; then
|
||
# Check for Error or CrashLoopBackOff (terminal failures)
|
||
if echo "$pods" | grep -E "Error|CrashLoopBackOff" >/dev/null; then
|
||
echo "ERROR: Some CNPG pods are in Error or CrashLoopBackOff state:" >&2
|
||
echo "$pods" | grep -E "Error|CrashLoopBackOff" >&2
|
||
return 1
|
||
fi
|
||
|
||
# Check for image pull failures — attempt remediation in k3d mode
|
||
if echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >/dev/null; then
|
||
if [[ "${PROLE_MODE:-}" == "k3d" ]]; then
|
||
if [[ -z "$_image_pull_remediated" ]]; then
|
||
_image_pull_remediated=1
|
||
local fail_pod fail_image
|
||
fail_pod=$(echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" | awk '{print $1}' | head -1)
|
||
# Try containers then initContainers
|
||
fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \
|
||
-o jsonpath='{.spec.containers[0].image}' 2>/dev/null || true)
|
||
if [[ -z "$fail_image" ]]; then
|
||
fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \
|
||
-o jsonpath='{.spec.initContainers[0].image}' 2>/dev/null || true)
|
||
fi
|
||
echo "WARN: Pod '$fail_pod' cannot pull image '${fail_image:-unknown}' (ErrImagePull/ImagePullBackOff in k3d mode)." >&2
|
||
import_dir="${DOCKER_IMPORT_DIR:-${PROLE_DATA:+${PROLE_DATA}/docker-import}}"
|
||
cluster_name="${K3D_CLUSTER_NAME:-}"
|
||
if [[ -z "$cluster_name" ]]; then
|
||
cluster_name=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | head -1)
|
||
fi
|
||
cluster_name="${cluster_name:-knoe-dev-cluster}"
|
||
# Re-run full image ensure logic to push/import the failing image
|
||
echo " Attempting image remediation via _ensure_prole_db_image ..." >&2
|
||
_ensure_prole_db_image >&2 || true
|
||
echo " Remediation complete; resuming wait ..." >&2
|
||
fi
|
||
# Continue the wait loop — do not return 1
|
||
else
|
||
echo "ERROR: Image pull failure in non-k3d mode — cannot auto-recover:" >&2
|
||
echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >&2
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
# Count Running pods by name (exclude initdb)
|
||
local running_pods
|
||
running_pods=$(kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null \
|
||
| grep "^${CNPG_CLUSTER_NAME}-" \
|
||
| grep -v "initdb" \
|
||
| grep "Running" \
|
||
| wc -l | xargs)
|
||
|
||
if [[ "$running_pods" -ge "$target_pods" ]]; then
|
||
echo "All $running_pods/$target_pods pods are Running."
|
||
return 0
|
||
fi
|
||
|
||
# Periodic detailed feedback
|
||
if (( now - last_feedback >= feedback_interval )); then
|
||
last_feedback=$now
|
||
echo ""
|
||
echo " [${elapsed}s/${timeout}s] Waiting — ${running_pods:-0}/${target_pods} pods Running"
|
||
echo " Pod status:"
|
||
echo "$pods" | sed 's/^/ /'
|
||
|
||
# Show cluster status if available
|
||
local cluster_phase
|
||
cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||
if [[ -n "$cluster_phase" ]]; then
|
||
echo " Cluster phase: ${cluster_phase}"
|
||
fi
|
||
|
||
# Show recent events (last 3)
|
||
local events
|
||
events=$(kubectl -n "$NAMESPACE" get events \
|
||
--sort-by='.lastTimestamp' \
|
||
--field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \
|
||
-o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \
|
||
--no-headers 2>/dev/null | tail -3 || true)
|
||
if [[ -n "$events" ]]; then
|
||
echo " Recent cluster events:"
|
||
echo "$events" | sed 's/^/ /'
|
||
fi
|
||
fi
|
||
else
|
||
# No pods yet — give feedback
|
||
if (( now - last_feedback >= feedback_interval )); then
|
||
last_feedback=$now
|
||
echo " [${elapsed}s/${timeout}s] No CNPG pods found yet for cluster '$CNPG_CLUSTER_NAME' — waiting for operator to create them..."
|
||
local cluster_phase
|
||
cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||
if [[ -n "$cluster_phase" ]]; then
|
||
echo " Cluster phase: ${cluster_phase}"
|
||
fi
|
||
# If stuck in "Unable to create required cluster objects", show conditions and events for diagnosis
|
||
if [[ "$cluster_phase" == *"Unable to create"* ]]; then
|
||
echo " Cluster conditions:"
|
||
kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \
|
||
-o jsonpath='{range .status.conditions[*]} {.type}: {.status} — {.message}{"\n"}{end}' 2>/dev/null || true
|
||
echo " Recent events:"
|
||
kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \
|
||
--field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \
|
||
-o custom-columns='TYPE:.type,REASON:.reason,MESSAGE:.message' \
|
||
--no-headers 2>/dev/null | tail -5 | sed 's/^/ /' || true
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( elapsed > timeout )); then
|
||
echo ""
|
||
echo "ERROR: Timed out after ${elapsed}s waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE'." >&2
|
||
echo "Final pod status:" >&2
|
||
kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" 2>/dev/null >&2 || true
|
||
echo "Recent events:" >&2
|
||
kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \
|
||
--field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \
|
||
-o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \
|
||
--no-headers 2>/dev/null | tail -5 >&2 || true
|
||
return 1
|
||
fi
|
||
sleep 5
|
||
done
|
||
}
|
||
|
||
cluster_exists() {
|
||
kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1
|
||
}
|
||
|
||
cluster_has_pods() {
|
||
kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null | grep -q .
|
||
}
|
||
|
||
cluster_has_ready_pods() {
|
||
kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" \
|
||
-o jsonpath='{.items[?(@.status.conditions[?(@.type=="Ready")].status=="True")].metadata.name}' 2>/dev/null | grep -q .
|
||
}
|
||
|
||
latest_backup_name() {
|
||
kubectl -n "$NAMESPACE" get backup \
|
||
--sort-by=.metadata.creationTimestamp \
|
||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1
|
||
}
|
||
|
||
latest_completed_backup_name() {
|
||
kubectl -n "$NAMESPACE" get backup \
|
||
--sort-by=.metadata.creationTimestamp \
|
||
-o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null | \
|
||
awk -F'|' '{p=tolower($2); if (p=="completed" || p=="succeeded") {name=$1}} END {print name}'
|
||
}
|
||
|
||
wait_for_backup() {
|
||
local backup_name="$1"
|
||
local start_time now phase phase_lc
|
||
start_time=$(date +%s)
|
||
|
||
while true; do
|
||
phase=$(kubectl -n "$NAMESPACE" get backup "$backup_name" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||
phase_lc=$(printf '%s' "$phase" | tr '[:upper:]' '[:lower:]')
|
||
|
||
case "$phase_lc" in
|
||
completed|succeeded)
|
||
echo "Backup $backup_name completed."
|
||
return 0
|
||
;;
|
||
failed|error)
|
||
echo "Backup $backup_name failed (phase=$phase)." >&2
|
||
return 1
|
||
;;
|
||
esac
|
||
|
||
now=$(date +%s)
|
||
if (( now - start_time > BACKUP_WAIT_TIMEOUT )); then
|
||
echo "Timed out waiting for backup $backup_name." >&2
|
||
return 1
|
||
fi
|
||
|
||
echo "Waiting for backup $backup_name to complete (phase=${phase:-unknown}) ..."
|
||
sleep 10
|
||
done
|
||
}
|
||
|
||
run_garage_backup() {
|
||
if [[ ! -x "$SCRIPT_DIR/init_prole-db-backup.sh" ]]; then
|
||
echo "WARN: init_prole-db-backup.sh not found; skipping Garage backup." >&2
|
||
return 1
|
||
fi
|
||
|
||
echo "Running Garage backup via init_prole-db-backup.sh ..."
|
||
if ! "$SCRIPT_DIR/init_prole-db-backup.sh" start; then
|
||
echo "Garage backup script failed." >&2
|
||
return 1
|
||
fi
|
||
|
||
sleep 2
|
||
local backup_name
|
||
backup_name=$(latest_backup_name)
|
||
if [[ -z "$backup_name" ]]; then
|
||
echo "No backup resource detected after triggering backup." >&2
|
||
return 1
|
||
fi
|
||
|
||
wait_for_backup "$backup_name"
|
||
}
|
||
|
||
get_primary_pod() {
|
||
local primary
|
||
primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
|
||
if [[ -z "$primary" ]]; then
|
||
primary=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||
fi
|
||
printf '%s' "$primary"
|
||
}
|
||
|
||
decode_b64() {
|
||
local data="$1"
|
||
if [[ -z "$data" ]]; then
|
||
return 1
|
||
fi
|
||
printf '%s' "$data" | base64 -d 2>/dev/null
|
||
}
|
||
|
||
resolve_db_credentials() {
|
||
local user_b64 pass_b64 user pass
|
||
|
||
user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.username}' 2>/dev/null || true)
|
||
pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.password}' 2>/dev/null || true)
|
||
user=$(decode_b64 "$user_b64" || true)
|
||
pass=$(decode_b64 "$pass_b64" || true)
|
||
|
||
if [[ -z "$user" || -z "$pass" ]]; then
|
||
user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.username}' 2>/dev/null || true)
|
||
pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.password}' 2>/dev/null || true)
|
||
user=$(decode_b64 "$user_b64" || true)
|
||
pass=$(decode_b64 "$pass_b64" || true)
|
||
fi
|
||
|
||
if [[ -z "$user" || -z "$pass" ]]; then
|
||
return 1
|
||
fi
|
||
|
||
printf '%s\n%s' "$user" "$pass"
|
||
}
|
||
|
||
pgdump_local() {
|
||
local pod user pass db_name dump_dir dump_file timestamp
|
||
pod=$(get_primary_pod)
|
||
if [[ -z "$pod" ]]; then
|
||
echo "ERROR: No CNPG pod available for pg_dump." >&2
|
||
return 1
|
||
fi
|
||
|
||
local creds
|
||
if ! creds=$(resolve_db_credentials); then
|
||
echo "ERROR: Unable to resolve database credentials for pg_dump." >&2
|
||
return 1
|
||
fi
|
||
user=$(printf '%s' "$creds" | sed -n '1p')
|
||
pass=$(printf '%s' "$creds" | sed -n '2p')
|
||
|
||
db_name=${PROLE_DB_NAME:-prole-db}
|
||
dump_dir="$BACKUP_DIR"
|
||
mkdir -p "$dump_dir"
|
||
|
||
timestamp=$(date +%Y%m%d%H%M%S)
|
||
dump_file="$dump_dir/${CNPG_CLUSTER_NAME}-pgdump-${timestamp}.dump"
|
||
|
||
echo "Running pg_dump against pod $pod (db=$db_name) ..."
|
||
if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- env PGPASSWORD="$pass" \
|
||
pg_dump -U "$user" -d "$db_name" -Fc > "$dump_file"; then
|
||
echo "pg_dump saved to $dump_file"
|
||
return 0
|
||
fi
|
||
|
||
echo "pg_dump failed; removing partial file." >&2
|
||
rm -f "$dump_file"
|
||
return 1
|
||
}
|
||
|
||
attempt_backup_if_active() {
|
||
if ! cluster_exists; then
|
||
return 0
|
||
fi
|
||
|
||
if ! cluster_has_pods; then
|
||
echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no pods detected; skipping backup." >&2
|
||
return 0
|
||
fi
|
||
|
||
if ! cluster_has_ready_pods; then
|
||
echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no ready pods detected; skipping backup." >&2
|
||
return 0
|
||
fi
|
||
|
||
echo "CNPG cluster '$CNPG_CLUSTER_NAME' detected; attempting Garage backup ..."
|
||
if run_garage_backup; then
|
||
return 0
|
||
fi
|
||
|
||
echo "Garage backup failed; attempting local pg_dump ..." >&2
|
||
if pgdump_local; then
|
||
return 0
|
||
fi
|
||
|
||
echo "WARN: Both Garage backup and local pg_dump failed." >&2
|
||
return 0
|
||
}
|
||
|
||
wait_for_pod_ready() {
|
||
local pod="$1"
|
||
if ! kubectl -n "$NAMESPACE" wait --for=condition=Ready pod "$pod" --timeout=300s >/dev/null 2>&1; then
|
||
echo "WARN: pod $pod did not become Ready within timeout." >&2
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
rollout_cluster() {
|
||
ensure_tools
|
||
ensure_namespace
|
||
echo "Starting manual recreate rollout for $CNPG_CLUSTER_NAME in $NAMESPACE..."
|
||
|
||
local has_cnpg_plugin="0"
|
||
if kubectl cnpg version >/dev/null 2>&1; then
|
||
has_cnpg_plugin="1"
|
||
kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
|
||
fi
|
||
|
||
local primary
|
||
primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
|
||
if [[ -z "$primary" ]]; then
|
||
echo "ERROR: Could not identify primary instance." >&2
|
||
return 1
|
||
fi
|
||
echo "Primary instance: $primary"
|
||
|
||
local instances
|
||
instances=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)
|
||
if [[ -z "$instances" ]]; then
|
||
echo "ERROR: No pods found for cluster $CNPG_CLUSTER_NAME." >&2
|
||
return 1
|
||
fi
|
||
|
||
local pod
|
||
for pod in $instances; do
|
||
if [[ "$pod" != "$primary" ]]; then
|
||
echo "Recreating non-primary pod: $pod..."
|
||
kubectl delete pod -n "$NAMESPACE" "$pod"
|
||
wait_for_pod_ready "$pod" || true
|
||
if [[ "$has_cnpg_plugin" == "1" ]]; then
|
||
kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
|
||
fi
|
||
fi
|
||
done
|
||
|
||
local new_primary=""
|
||
for pod in $instances; do
|
||
if [[ "$pod" != "$primary" ]]; then
|
||
new_primary="$pod"
|
||
break
|
||
fi
|
||
done
|
||
|
||
if [[ -n "$new_primary" && "$has_cnpg_plugin" == "1" ]]; then
|
||
echo "Promoting $new_primary..."
|
||
kubectl cnpg promote "$CNPG_CLUSTER_NAME" "$new_primary" -n "$NAMESPACE" || true
|
||
echo "Waiting for $new_primary to become primary..."
|
||
for _ in {1..60}; do
|
||
local current_primary
|
||
current_primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
|
||
if [[ "$current_primary" == "$new_primary" ]]; then
|
||
echo "$new_primary is now the primary."
|
||
break
|
||
fi
|
||
sleep 5
|
||
done
|
||
else
|
||
if [[ -n "$new_primary" ]]; then
|
||
echo "WARN: kubectl cnpg plugin not available; skipping explicit promotion."
|
||
fi
|
||
fi
|
||
|
||
echo "Recreating the old primary pod: $primary..."
|
||
kubectl delete pod -n "$NAMESPACE" "$primary"
|
||
wait_for_pod_ready "$primary" || true
|
||
|
||
if [[ "$has_cnpg_plugin" == "1" ]]; then
|
||
echo "Rollout complete. Final cluster status:"
|
||
kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
|
||
fi
|
||
}
|
||
|
||
force_rollout() {
|
||
echo "Attempting force rollout of CNPG pods ..."
|
||
rollout_cluster
|
||
}
|
||
|
||
reset_and_reinit() {
|
||
local backup_name tmp_manifest
|
||
backup_name=$(latest_completed_backup_name)
|
||
|
||
echo "Resetting CNPG cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' ..."
|
||
kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
|
||
kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true
|
||
|
||
if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then
|
||
tmp_manifest=$(mktemp)
|
||
sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest"
|
||
echo "Re-initializing from backup $backup_name ..."
|
||
CNPG_MANIFEST_OVERRIDE="$tmp_manifest" ensure_prole_stack_resources
|
||
rm -f "$tmp_manifest"
|
||
else
|
||
if [[ -n "$backup_name" && ! -f "$RECOVERY_TEMPLATE" ]]; then
|
||
echo "WARN: Recovery template not found: $RECOVERY_TEMPLATE" >&2
|
||
fi
|
||
if [[ -z "$backup_name" ]]; then
|
||
echo "No completed backups found; starting fresh initialization." >&2
|
||
fi
|
||
ensure_prole_stack_resources
|
||
fi
|
||
|
||
wait_for_cnpg_pods 300
|
||
}
|
||
|
||
# Resolve OpenBao URL: prefer explicit env, then in-cluster service, then localhost port-forward
|
||
bao_service_url() {
|
||
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
|
||
echo "$PROLE_OPENBAO_URL"
|
||
return 0
|
||
fi
|
||
if prole_is_in_cluster; then
|
||
echo "http://$OPENBAO_NAME.${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}.svc.cluster.local:8200"
|
||
return 0
|
||
fi
|
||
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
|
||
echo "http://127.0.0.1:8200"
|
||
return 0
|
||
fi
|
||
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
|
||
echo "http://127.0.0.1:8200"
|
||
return 0
|
||
fi
|
||
echo ""
|
||
}
|
||
|
||
fetch_admin_keys_and_db_pass_from_bao_or_local() {
|
||
local token url
|
||
if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
|
||
token=$(cat "$OPENBAO_TOKEN_FILE")
|
||
else
|
||
token=""
|
||
fi
|
||
url=$(bao_service_url)
|
||
if [[ -n "$token" && -n "$url" ]]; then
|
||
echo "Attempting to read admin key pair from OpenBao kv/$BAO_PATH_ADMIN ..."
|
||
if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -e '.data.data' >/dev/null 2>&1; then
|
||
local priv_b64 pub_b64
|
||
priv_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_private_key_b64')
|
||
pub_b64=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_ADMIN" | jq -r '.data.data.admin_public_key_b64')
|
||
# Use a temporary file to determine where to save based on existing legacy or generic preference
|
||
local target_priv="$ADMIN_PRIV_GENERIC"
|
||
local target_pub="$ADMIN_PUB_GENERIC"
|
||
|
||
# If legacy keys exist, we might want to overwrite them too for compatibility
|
||
printf "%s" "$priv_b64" | base64 -d >"$target_priv"
|
||
printf "%s" "$pub_b64" | base64 -d >"$target_pub"
|
||
chmod 0600 "$target_priv"
|
||
|
||
# Mirror to legacy path if it was expected by other scripts
|
||
cp "$target_priv" "$ADMIN_PRIV_ED25519" 2>/dev/null || true
|
||
cp "$target_pub" "$ADMIN_PUB_ED25519" 2>/dev/null || true
|
||
fi
|
||
|
||
echo "Attempting to read database password from OpenBao kv/$BAO_PATH_DB ..."
|
||
if curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -e '.data.data' >/dev/null 2>&1; then
|
||
local db_pass
|
||
db_pass=$(curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$BAO_PATH_DB" | jq -r '.data.data.password')
|
||
if [[ -n "$db_pass" ]]; then
|
||
echo "Updating database user secret 'prole-db-user' from OpenBao ..."
|
||
kubectl create secret generic prole-db-user -n "$NAMESPACE" \
|
||
--from-literal=username=prole \
|
||
--from-literal=password="$db_pass" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
|
||
echo "Updating database superuser secret 'prole-db-superuser' from OpenBao ..."
|
||
kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \
|
||
--from-literal=username=postgres \
|
||
--from-literal=password="$db_pass" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# Fallback: use DB_PASSWORD from env if OpenBao is unreachable or empty
|
||
if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
|
||
local db_pass="${DB_PASSWORD:-}"
|
||
if [[ -n "$db_pass" && "$db_pass" != '${OPENBAO:'* && "$db_pass" != '${PROLE_SECRET:'* ]]; then
|
||
echo "Updating database user secret 'prole-db-user' from DB_PASSWORD env ..."
|
||
kubectl create secret generic prole-db-user -n "$NAMESPACE" \
|
||
--from-literal=username=prole \
|
||
--from-literal=password="$db_pass" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
|
||
echo "Updating database superuser secret 'prole-db-superuser' from DB_PASSWORD env ..."
|
||
kubectl create secret generic prole-db-superuser -n "$NAMESPACE" \
|
||
--from-literal=username=postgres \
|
||
--from-literal=password="$db_pass" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
fi
|
||
fi
|
||
|
||
if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
|
||
echo "ERROR: 'prole-db-user' secret is missing in namespace '$NAMESPACE' and could not be resolved from OpenBao or DB_PASSWORD env." >&2
|
||
return 1
|
||
fi
|
||
|
||
if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then
|
||
echo "Using local admin key pair at $SECRETS_DIR"
|
||
return 0
|
||
fi
|
||
if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then
|
||
echo "Using local legacy admin key pair at $SECRETS_DIR"
|
||
# Link or copy to generic for consistent use below
|
||
cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC"
|
||
cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC"
|
||
return 0
|
||
fi
|
||
echo "ERROR: Could not obtain admin key pair from OpenBao and no local files found." >&2
|
||
exit 1
|
||
}
|
||
|
||
apply_cnpg_admin_secret() {
|
||
echo "Creating/updating Secret cnpg-admin-key ..."
|
||
kubectl create secret generic cnpg-admin-key -n "$NAMESPACE" \
|
||
--from-file=admin.key="$ADMIN_PRIV_GENERIC" \
|
||
--from-file=admin.pub="$ADMIN_PUB_GENERIC" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
}
|
||
|
||
detect_and_reprovision_unencrypted_cluster() {
|
||
local encryption_enabled="${AT_REST_ENCRYPTION_ENABLED:-false}"
|
||
if [[ "$encryption_enabled" != "true" && "$encryption_enabled" != "True" && "$encryption_enabled" != "1" ]]; then
|
||
return 0
|
||
fi
|
||
|
||
if ! kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
|
||
return 0
|
||
fi
|
||
|
||
local server_tls
|
||
server_tls=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.certificates.serverTLSSecret}' 2>/dev/null || echo "")
|
||
if [[ -n "$server_tls" ]]; then
|
||
echo "Existing cluster '$CNPG_CLUSTER_NAME' already has TLS configured ($server_tls); no conversion needed."
|
||
return 0
|
||
fi
|
||
|
||
echo "WARN: Existing cluster '$CNPG_CLUSTER_NAME' does not have TLS/at-rest-encryption configured."
|
||
echo "At-rest encryption is now enabled; the cluster must be reprovisioned."
|
||
echo "NOTE: External full backup and restore from barman backup is not yet available."
|
||
echo " Removing existing cluster to allow re-creation with encryption enabled ..."
|
||
|
||
attempt_backup_if_active
|
||
|
||
kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found
|
||
kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true
|
||
echo "Existing unencrypted cluster removed. Will be re-created with TLS and encryption."
|
||
}
|
||
|
||
validate_ca_secret() {
|
||
local secret_name="$1"
|
||
local ns="$2"
|
||
if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then
|
||
return 1
|
||
fi
|
||
local crt key
|
||
crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true)
|
||
key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.key}' 2>/dev/null || true)
|
||
if [[ -z "$crt" || -z "$key" ]]; then
|
||
echo "WARN: CA secret '$secret_name' exists but is missing ca.crt or ca.key data." >&2
|
||
return 1
|
||
fi
|
||
if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then
|
||
echo "WARN: CA secret '$secret_name' has invalid certificate data." >&2
|
||
return 1
|
||
fi
|
||
# CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8
|
||
local key_header
|
||
key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1)
|
||
if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then
|
||
echo "WARN: CA secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
validate_tls_secret() {
|
||
local secret_name="$1"
|
||
local ns="$2"
|
||
if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then
|
||
return 1
|
||
fi
|
||
local crt key
|
||
crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true)
|
||
key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.key}' 2>/dev/null || true)
|
||
if [[ -z "$crt" || -z "$key" ]]; then
|
||
echo "WARN: TLS secret '$secret_name' exists but is missing tls.crt or tls.key data." >&2
|
||
return 1
|
||
fi
|
||
if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then
|
||
echo "WARN: TLS secret '$secret_name' has invalid certificate data." >&2
|
||
return 1
|
||
fi
|
||
# CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8
|
||
local key_header
|
||
key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1)
|
||
if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then
|
||
echo "WARN: TLS secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
generate_tls_if_missing() {
|
||
local ca_secret_name="${CNPG_CLUSTER_NAME}-ca"
|
||
local tls_secret_name="${CNPG_CLUSTER_NAME}-tls"
|
||
local need_ca=0
|
||
local need_tls=0
|
||
|
||
if validate_ca_secret "$ca_secret_name" "$NAMESPACE"; then
|
||
echo "CA secret $ca_secret_name already exists and is valid."
|
||
else
|
||
if kubectl -n "$NAMESPACE" get secret "$ca_secret_name" >/dev/null 2>&1; then
|
||
echo "Removing invalid CA secret '$ca_secret_name' ..."
|
||
kubectl -n "$NAMESPACE" delete secret "$ca_secret_name" --ignore-not-found
|
||
fi
|
||
need_ca=1
|
||
fi
|
||
|
||
if validate_tls_secret "$tls_secret_name" "$NAMESPACE"; then
|
||
echo "TLS secret $tls_secret_name already exists and is valid."
|
||
else
|
||
if kubectl -n "$NAMESPACE" get secret "$tls_secret_name" >/dev/null 2>&1; then
|
||
echo "Removing invalid TLS secret '$tls_secret_name' ..."
|
||
kubectl -n "$NAMESPACE" delete secret "$tls_secret_name" --ignore-not-found
|
||
fi
|
||
need_tls=1
|
||
fi
|
||
|
||
if [[ "$need_ca" -eq 0 && "$need_tls" -eq 0 ]]; then
|
||
echo "TLS secrets already present and valid; skipping generation."
|
||
return 0
|
||
fi
|
||
|
||
local TMPD
|
||
TMPD=$(mktemp -d)
|
||
|
||
if [[ "$need_ca" -eq 1 ]]; then
|
||
echo "Generating self-signed CA (EC P-256) for CNPG ..."
|
||
openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/ca.key"
|
||
openssl req -x509 -new -key "$TMPD/ca.key" -out "$TMPD/ca.crt" -days 3650 -subj "/CN=Prole CNPG CA"
|
||
kubectl -n "$NAMESPACE" create secret generic "$ca_secret_name" \
|
||
--from-file=ca.crt="$TMPD/ca.crt" \
|
||
--from-file=ca.key="$TMPD/ca.key" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
else
|
||
# Extract existing CA for signing the server certificate
|
||
kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.crt}' | base64 -d > "$TMPD/ca.crt"
|
||
kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.key}' | base64 -d > "$TMPD/ca.key"
|
||
fi
|
||
|
||
if [[ "$need_tls" -eq 1 ]]; then
|
||
echo "Generating server TLS certificate for CNPG ($tls_secret_name) ..."
|
||
openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/tls.key"
|
||
openssl req -new -key "$TMPD/tls.key" -out "$TMPD/tls.csr" \
|
||
-subj "/CN=${CNPG_CLUSTER_NAME}.${NAMESPACE}.svc"
|
||
openssl x509 -req -in "$TMPD/tls.csr" -CA "$TMPD/ca.crt" -CAkey "$TMPD/ca.key" \
|
||
-CAcreateserial -out "$TMPD/tls.crt" -days 365 \
|
||
-extfile <(printf "subjectAltName=DNS:%s,DNS:%s-rw,DNS:%s-rw.%s.svc,DNS:%s-r,DNS:%s-ro" \
|
||
"$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$NAMESPACE" \
|
||
"$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME")
|
||
kubectl -n "$NAMESPACE" create secret tls "$tls_secret_name" \
|
||
--cert="$TMPD/tls.crt" \
|
||
--key="$TMPD/tls.key" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
fi
|
||
|
||
rm -rf "$TMPD"
|
||
}
|
||
|
||
# Resolve latest CNPG version from GitHub if possible, fallback to a sensible default.
|
||
get_latest_cnpg_version() {
|
||
local version
|
||
version=$(curl -s "https://api.github.com/repos/cloudnative-pg/cloudnative-pg/releases/latest" | jq -r '.tag_name' | sed 's/^v//' || echo "")
|
||
if [[ -z "$version" || "$version" == "null" ]]; then
|
||
echo "1.27.0"
|
||
else
|
||
echo "$version"
|
||
fi
|
||
}
|
||
|
||
initialize() {
|
||
ensure_tools
|
||
ensure_namespace
|
||
attempt_backup_if_active
|
||
ensure_cnpg_operator
|
||
pin_cnpg_controller
|
||
ensure_barman_plugin
|
||
echo "Using namespace: $NAMESPACE"
|
||
|
||
if ! fetch_admin_keys_and_db_pass_from_bao_or_local; then
|
||
echo "ERROR: Failed to fetch/ensure database secrets and admin keys." >&2
|
||
return 1
|
||
fi
|
||
apply_cnpg_admin_secret
|
||
detect_and_reprovision_unencrypted_cluster
|
||
generate_tls_if_missing
|
||
|
||
# Pre-flight: verify required secrets exist before deploying cluster
|
||
local _missing_secrets=""
|
||
if ! kubectl -n "$NAMESPACE" get secret prole-db-user >/dev/null 2>&1; then
|
||
_missing_secrets="${_missing_secrets} prole-db-user"
|
||
fi
|
||
if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-tls" >/dev/null 2>&1; then
|
||
_missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-tls"
|
||
fi
|
||
if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-ca" >/dev/null 2>&1; then
|
||
_missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-ca"
|
||
fi
|
||
if [[ -n "$_missing_secrets" ]]; then
|
||
echo "ERROR: Required secrets missing in namespace '$NAMESPACE':${_missing_secrets}" >&2
|
||
echo "The CNPG cluster cannot start without these secrets." >&2
|
||
return 1
|
||
fi
|
||
echo "Pre-flight check passed: all required secrets present."
|
||
|
||
if ! _ensure_prole_db_image; then
|
||
echo "ERROR: Pre-flight image check failed; aborting cluster initialization." >&2
|
||
return 1
|
||
fi
|
||
|
||
ensure_prole_stack_resources
|
||
|
||
if ! wait_for_cnpg_pods 300; then
|
||
if cluster_has_pods; then
|
||
echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2
|
||
if force_rollout; then
|
||
if ! wait_for_cnpg_pods 300; then
|
||
echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2
|
||
if ! reset_and_reinit; then
|
||
return 1
|
||
fi
|
||
fi
|
||
else
|
||
echo "WARN: Force rollout failed; attempting reset and re-init ..." >&2
|
||
if ! reset_and_reinit; then
|
||
return 1
|
||
fi
|
||
fi
|
||
else
|
||
echo "WARN: No CNPG pods found — this is a new installation." >&2
|
||
echo "The CNPG operator was unable to create cluster pods." >&2
|
||
echo "Check operator logs: kubectl -n cnpg-system logs -l app.kubernetes.io/name=cloudnative-pg" >&2
|
||
echo "Check cluster status: kubectl -n $NAMESPACE describe cluster $CNPG_CLUSTER_NAME" >&2
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
echo "Initialization complete for CNPG + cert artifacts."
|
||
prole_register_port_forward "postgres" "${NAMESPACE:-default}" "svc/${CNPG_CLUSTER_NAME}-rw" "5432" "5432" "0.0.0.0" "TCP" "PostgreSQL"
|
||
}
|
||
|
||
update_reload() {
|
||
initialize
|
||
}
|
||
|
||
deploy_cluster() {
|
||
ensure_tools
|
||
ensure_namespace
|
||
ensure_cnpg_operator
|
||
pin_cnpg_controller
|
||
ensure_barman_plugin
|
||
|
||
local image
|
||
if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then
|
||
image=$(get_latest_image)
|
||
else
|
||
image="prole-db:$VERSION"
|
||
fi
|
||
image=$(resolve_cnpg_image "$image")
|
||
sync_manifest_image "$image"
|
||
|
||
generate_tls_if_missing
|
||
|
||
echo "Deploying $image to cluster $CNPG_CLUSTER_NAME in namespace $NAMESPACE..."
|
||
_ensure_prole_db_image || true
|
||
apply_cnpg_cluster_manifest "$CNPG_MANIFEST"
|
||
|
||
local current_image
|
||
current_image=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "")
|
||
if [[ -n "$current_image" && "$current_image" != "$image" ]]; then
|
||
echo "Patching cluster to use image '$image' (was '$current_image')..."
|
||
kubectl -n "$NAMESPACE" patch cluster "$CNPG_CLUSTER_NAME" --type merge -p "{\"spec\": {\"imageName\": \"$image\"}}"
|
||
if kubectl cnpg version >/dev/null 2>&1; then
|
||
kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true
|
||
fi
|
||
fi
|
||
}
|
||
|
||
case "$ACTION" in
|
||
recreate)
|
||
ensure_tools
|
||
"$0" delete "$CNPG_CLUSTER_NAME"
|
||
# Clean up PVCs left behind by the CNPG operator (not in static manifests)
|
||
echo "Cleaning up PVCs in namespace '$NAMESPACE' ..."
|
||
kubectl -n "$NAMESPACE" delete pvc --all --ignore-not-found 2>/dev/null || true
|
||
# Wait briefly for pods to fully terminate before re-creating
|
||
echo "Waiting for pods to terminate in namespace '$NAMESPACE' ..."
|
||
_wait_term=0
|
||
while kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null | grep -qv '^No resources'; do
|
||
sleep 3
|
||
_wait_term=$(( _wait_term + 3 ))
|
||
if (( _wait_term >= 60 )); then
|
||
echo "WARN: Pods still present after 60s; proceeding anyway." >&2
|
||
break
|
||
fi
|
||
done
|
||
"$0" create "$CNPG_CLUSTER_NAME"
|
||
;;
|
||
create)
|
||
ensure_tools
|
||
ensure_namespace
|
||
initialize
|
||
;;
|
||
delete)
|
||
ensure_tools
|
||
echo "Deleting all resources for '$CNPG_CLUSTER_NAME' ..."
|
||
# Delete each manifest individually, mirroring the apply pattern:
|
||
# ingress.yaml contains multi-namespace resources and must be deleted without -n.
|
||
for _del_f in "$SCRIPT_DIR/../k8s/prole"/*.yaml; do
|
||
_del_base=$(basename "$_del_f")
|
||
case "$_del_base" in
|
||
kustomization.yaml|ingress.yaml) continue ;;
|
||
*) kubectl delete -n "$NAMESPACE" -f "$_del_f" --ignore-not-found 2>&1 \
|
||
| grep -v "^Error from server (NotFound)" || true ;;
|
||
esac
|
||
done
|
||
if [[ -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" ]]; then
|
||
kubectl delete -f "$SCRIPT_DIR/../k8s/prole/ingress.yaml" --ignore-not-found 2>&1 \
|
||
| grep -v "^Error from server (NotFound)" || true
|
||
fi
|
||
;;
|
||
start)
|
||
ensure_tools
|
||
ensure_namespace
|
||
ensure_cnpg_operator
|
||
pin_cnpg_controller
|
||
ensure_barman_plugin
|
||
if [[ ! -f "$CNPG_MANIFEST" ]]; then
|
||
echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
|
||
exit 1
|
||
fi
|
||
echo "Starting CloudNative-PG cluster from $CNPG_MANIFEST in namespace $NAMESPACE..."
|
||
prole_render_manifest "$CNPG_MANIFEST" | kubectl apply -n "$NAMESPACE" -f -
|
||
;;
|
||
stop)
|
||
ensure_tools
|
||
if [[ ! -f "$CNPG_MANIFEST" ]]; then
|
||
echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2
|
||
exit 1
|
||
fi
|
||
echo "Stopping CloudNative-PG cluster using $CNPG_MANIFEST ..."
|
||
kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found
|
||
;;
|
||
status)
|
||
ensure_tools
|
||
echo "--- CloudNative-PG Cluster Status ($CNPG_CLUSTER_NAME) ---"
|
||
if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then
|
||
kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME"
|
||
echo ""
|
||
echo "CNPG Plugin Status:"
|
||
if kubectl cnpg version >/dev/null 2>&1; then
|
||
kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE"
|
||
else
|
||
echo "Note: 'kubectl cnpg' plugin not found; skipping detailed status."
|
||
fi
|
||
else
|
||
echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'."
|
||
fi
|
||
;;
|
||
restart)
|
||
ensure_tools
|
||
"$0" stop
|
||
"$0" start
|
||
;;
|
||
initialize)
|
||
if ! initialize; then
|
||
exit 1
|
||
fi
|
||
;;
|
||
update|reload)
|
||
if ! update_reload; then
|
||
exit 1
|
||
fi
|
||
;;
|
||
deploy)
|
||
deploy_cluster
|
||
;;
|
||
rollout|force-rollout)
|
||
rollout_cluster
|
||
;;
|
||
install-barman-plugin)
|
||
ensure_tools
|
||
ensure_cnpg_operator
|
||
pin_cnpg_controller
|
||
ensure_barman_plugin
|
||
;;
|
||
*)
|
||
echo "Usage: $0 {create|delete|recreate|start|stop|status|restart|initialize|update|reload|deploy|rollout|install-barman-plugin} [cluster] [version]" >&2
|
||
exit 2
|
||
;;
|
||
esac
|