mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
644 lines
22 KiB
Bash
Executable File
644 lines
22 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
# init_cnpg_backup.sh
|
|
# Purpose:
|
|
# - Configure CloudNative-PG to backup to Garage (S3-compatible) via Barman Cloud Plugin
|
|
# - Create initial backup
|
|
|
|
# Initialize SCRIPT_DIR
|
|
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
|
|
# Load environment and config via knoe_cfg.sh
|
|
# shellcheck disable=SC1090
|
|
source "$SCRIPT_DIR/knoe_cfg.sh"
|
|
|
|
if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
|
|
knoe_set_mode "${2:-}"
|
|
shift 2
|
|
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
|
|
knoe_set_mode "${1#*=}"
|
|
shift
|
|
fi
|
|
|
|
ACTION=${1:-start}
|
|
|
|
NAMESPACE="${PROLE_NAMESPACE}"
|
|
CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-knoe-db}
|
|
CNPG_OPERATOR_NAMESPACE=${CNPG_OPERATOR_NAMESPACE:-cnpg-system}
|
|
GARAGE_NAME=${GARAGE_NAME:-garage}
|
|
SERVICE_NAMESPACE=${SERVICE_NAMESPACE:-}
|
|
if [[ -z "${GARAGE_NAMESPACE:-}" ]]; then
|
|
if [[ -n "$SERVICE_NAMESPACE" ]]; then
|
|
GARAGE_NAMESPACE="$SERVICE_NAMESPACE"
|
|
else
|
|
GARAGE_NAMESPACE="$NAMESPACE"
|
|
fi
|
|
fi
|
|
GARAGE_BACKUP_BUCKET=${GARAGE_BACKUP_BUCKET:-knoe-db-backups}
|
|
GARAGE_BACKUP_KEY_NAME=${GARAGE_BACKUP_KEY_NAME:-knoe-db-backup}
|
|
GARAGE_BACKUP_SECRET_NAME=${GARAGE_BACKUP_SECRET_NAME:-knoe-db-barman-s3}
|
|
GARAGE_S3_ENDPOINT=${GARAGE_S3_ENDPOINT:-http://$GARAGE_NAME.$GARAGE_NAMESPACE.svc.cluster.local:3900}
|
|
GARAGE_S3_REGION=${GARAGE_S3_REGION:-garage}
|
|
RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1}
|
|
RETENTION_POLICY=${RETENTION_POLICY:-30d}
|
|
BARMAN_PLUGIN_NAME=${BARMAN_PLUGIN_NAME:-barman-cloud.cloudnative-pg.io}
|
|
BARMAN_OBJECT_NAME=${BARMAN_OBJECT_NAME:-knoe-db-barman-objectstore}
|
|
BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600}
|
|
BACKUP_STATUS_INTERVAL=${BACKUP_STATUS_INTERVAL:-10}
|
|
PLUGIN_READY_TIMEOUT=${PLUGIN_READY_TIMEOUT:-180}
|
|
|
|
SCHEDULED_BACKUP_NAME=${SCHEDULED_BACKUP_NAME:-knoe-db-scheduled-backup}
|
|
SCHEDULED_BACKUP_CRON=${SCHEDULED_BACKUP_CRON:-"0 3 * * *"}
|
|
SCHEDULED_BACKUP_IMMEDIATE=${SCHEDULED_BACKUP_IMMEDIATE:-true}
|
|
SCHEDULED_BACKUP_METHOD=${SCHEDULED_BACKUP_METHOD:-plugin}
|
|
SCHEDULED_BACKUP_INCREMENTAL=${SCHEDULED_BACKUP_INCREMENTAL:-false}
|
|
|
|
PLUGIN_SOCKET_DIR=${PLUGIN_SOCKET_DIR:-/plugins}
|
|
LAST_BACKUP_NAME=""
|
|
|
|
usage() {
|
|
cat <<USAGE
|
|
Usage: $0 [start|backup|status]
|
|
|
|
Actions:
|
|
start Configure Garage-backed backups and run initial backup
|
|
backup [full|incr] Trigger a new backup now (default: full)
|
|
status Show backup resources and verify at least one successful base backup
|
|
USAGE
|
|
exit 1
|
|
}
|
|
|
|
ensure_tools() {
|
|
for t in kubectl; do
|
|
command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
|
|
done
|
|
}
|
|
|
|
_kubectl_is_transient_error() {
|
|
local msg="${1:-}"
|
|
printf '%s' "$msg" | grep -Eqi \
|
|
'(failed to download openapi|apiserver not ready|the server is currently unable to handle the request|connect: connection refused|context deadline exceeded|i/o timeout|no endpoints available for service "cnpg-webhook-service"|failed calling webhook)'
|
|
}
|
|
|
|
wait_for_apiserver_ready() {
|
|
local timeout=${1:-120}
|
|
local start_time elapsed
|
|
start_time=$(date +%s)
|
|
|
|
while true; do
|
|
if kubectl --request-timeout=5s get --raw='/readyz' >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
if kubectl --request-timeout=5s version --short >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
elapsed=$(( $(date +%s) - start_time ))
|
|
if (( elapsed > timeout )); then
|
|
echo "ERROR: Kubernetes API server not ready after ${elapsed}s." >&2
|
|
return 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
}
|
|
|
|
wait_for_cnpg_webhook() {
|
|
local timeout=${1:-180}
|
|
local start_time elapsed
|
|
start_time=$(date +%s)
|
|
|
|
echo "Waiting for CNPG webhook service endpoints to be ready (timeout: ${timeout}s)..."
|
|
while true; do
|
|
local endpoints
|
|
endpoints=$(kubectl -n "$CNPG_OPERATOR_NAMESPACE" get endpoints cnpg-webhook-service -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
|
|
if [[ -n "$endpoints" ]]; then
|
|
return 0
|
|
fi
|
|
elapsed=$(( $(date +%s) - start_time ))
|
|
if (( elapsed > timeout )); then
|
|
echo "ERROR: CNPG webhook service has no endpoints after ${elapsed}s." >&2
|
|
kubectl -n "$CNPG_OPERATOR_NAMESPACE" get pods 2>/dev/null >&2 || true
|
|
return 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
}
|
|
|
|
kubectl_apply_retry() {
|
|
local namespace="${1:-}"
|
|
local attempts=${2:-8}
|
|
local sleep_s=${3:-5}
|
|
local i out
|
|
|
|
for ((i=1; i<=attempts; i++)); do
|
|
if [[ -n "$namespace" ]]; then
|
|
if out=$(kubectl apply -n "$namespace" -f - 2>&1); then
|
|
printf '%s\n' "$out"
|
|
return 0
|
|
fi
|
|
else
|
|
if out=$(kubectl apply -f - 2>&1); then
|
|
printf '%s\n' "$out"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if _kubectl_is_transient_error "$out"; then
|
|
echo "WARN: kubectl apply failed due to transient API/webhook issue (attempt $i/$attempts); retrying..." >&2
|
|
echo "$out" >&2
|
|
sleep "$sleep_s"
|
|
continue
|
|
fi
|
|
|
|
echo "$out" >&2
|
|
return 1
|
|
done
|
|
|
|
echo "ERROR: kubectl apply failed after $attempts attempts." >&2
|
|
echo "$out" >&2
|
|
return 1
|
|
}
|
|
|
|
ensure_namespace() {
|
|
if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
|
|
echo "Creating namespace '$NAMESPACE' ..."
|
|
kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true
|
|
fi
|
|
}
|
|
|
|
ensure_cluster() {
|
|
if ! kubectl get cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
|
|
echo "ERROR: CNPG cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
barman_crd_ready() {
|
|
kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1
|
|
}
|
|
|
|
get_garage_pod() {
|
|
kubectl get pods -n "$GARAGE_NAMESPACE" -l "app=$GARAGE_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true
|
|
}
|
|
|
|
garage_exec() {
|
|
local pod
|
|
pod=$(get_garage_pod)
|
|
if [[ -z "$pod" ]]; then
|
|
echo "ERROR: Garage pod not found in namespace '$GARAGE_NAMESPACE'." >&2
|
|
exit 1
|
|
fi
|
|
kubectl exec -n "$GARAGE_NAMESPACE" "$pod" -- /garage "$@"
|
|
}
|
|
|
|
parse_key_output() {
|
|
local output="$1"
|
|
local access_key secret_key
|
|
access_key=$(echo "$output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1)
|
|
secret_key=$(echo "$output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1)
|
|
if [[ -z "$access_key" || -z "$secret_key" ]]; then
|
|
return 1
|
|
fi
|
|
printf "%s\n%s" "$access_key" "$secret_key"
|
|
}
|
|
|
|
ensure_garage_ready() {
|
|
echo "Checking Garage readiness ..."
|
|
local i status_out
|
|
for i in {1..30}; do
|
|
if status_out=$(garage_exec status 2>/dev/null); then
|
|
# If layout is applied, DataAvail should eventually show something or at least the node should be healthy.
|
|
# A better check is 'layout show' to see if current layout version > 0
|
|
local layout_out version
|
|
layout_out=$(garage_exec layout show 2>/dev/null | grep -v "INFO" || true)
|
|
version=$(echo "$layout_out" | awk -F: '/Current cluster layout version/ {gsub(/[[:space:]]/,"",$2); print $2; exit}' || true)
|
|
if [[ -n "$version" && "$version" -gt 0 ]]; then
|
|
echo "Garage layout version $version is applied and ready."
|
|
return 0
|
|
fi
|
|
fi
|
|
echo "Waiting for Garage layout to be applied... ($i/30)"
|
|
sleep 5
|
|
done
|
|
echo "ERROR: Garage not ready (layout not applied) after 30 attempts." >&2
|
|
exit 1
|
|
}
|
|
|
|
ensure_garage_bucket_and_key() {
|
|
ensure_garage_ready
|
|
echo "Ensuring Garage bucket and access key for backups ..."
|
|
local key_info parsed access_key secret_key
|
|
if key_info=$(garage_exec key info --show-secret "$GARAGE_BACKUP_KEY_NAME" 2>/dev/null); then
|
|
:
|
|
else
|
|
key_info=$(garage_exec key create "$GARAGE_BACKUP_KEY_NAME")
|
|
fi
|
|
|
|
if ! parsed=$(parse_key_output "$key_info"); then
|
|
echo "ERROR: Unable to parse Garage key output." >&2
|
|
echo "$key_info" >&2
|
|
exit 1
|
|
fi
|
|
access_key=$(echo "$parsed" | sed -n '1p')
|
|
secret_key=$(echo "$parsed" | sed -n '2p')
|
|
|
|
if ! garage_exec bucket info "$GARAGE_BACKUP_BUCKET" >/dev/null 2>&1; then
|
|
garage_exec bucket create "$GARAGE_BACKUP_BUCKET"
|
|
fi
|
|
|
|
garage_exec bucket allow --read --write --owner --key "$GARAGE_BACKUP_KEY_NAME" "$GARAGE_BACKUP_BUCKET" || true
|
|
|
|
echo "Creating/updating Kubernetes secret '$GARAGE_BACKUP_SECRET_NAME' ..."
|
|
kubectl create secret generic "$GARAGE_BACKUP_SECRET_NAME" -n "$NAMESPACE" \
|
|
--from-literal=ACCESS_KEY_ID="$access_key" \
|
|
--from-literal=SECRET_ACCESS_KEY="$secret_key" \
|
|
--from-literal=REGION="$GARAGE_S3_REGION" \
|
|
--dry-run=client -o yaml | kubectl_apply_retry "$NAMESPACE"
|
|
}
|
|
|
|
apply_barman_object_store() {
|
|
if ! barman_crd_ready; then
|
|
echo "ERROR: Barman Cloud plugin CRD not found (objectstores.barmancloud.cnpg.io). Install the plugin first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Applying Barman Cloud ObjectStore '$BARMAN_OBJECT_NAME' ..."
|
|
kubectl_apply_retry "$NAMESPACE" <<OBJECTSTORE
|
|
apiVersion: barmancloud.cnpg.io/v1
|
|
kind: ObjectStore
|
|
metadata:
|
|
name: $BARMAN_OBJECT_NAME
|
|
spec:
|
|
retentionPolicy: $RETENTION_POLICY
|
|
configuration:
|
|
destinationPath: s3://$GARAGE_BACKUP_BUCKET/
|
|
endpointURL: $GARAGE_S3_ENDPOINT
|
|
s3Credentials:
|
|
accessKeyId:
|
|
name: $GARAGE_BACKUP_SECRET_NAME
|
|
key: ACCESS_KEY_ID
|
|
secretAccessKey:
|
|
name: $GARAGE_BACKUP_SECRET_NAME
|
|
key: SECRET_ACCESS_KEY
|
|
region:
|
|
name: $GARAGE_BACKUP_SECRET_NAME
|
|
key: REGION
|
|
wal:
|
|
compression: gzip
|
|
data:
|
|
compression: gzip
|
|
OBJECTSTORE
|
|
}
|
|
|
|
ensure_barman_plugin_config() {
|
|
local plugin_names plugin_present
|
|
plugin_names=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.plugins[*].name}' 2>/dev/null || true)
|
|
plugin_present=$(printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -F "$BARMAN_PLUGIN_NAME" || true)
|
|
|
|
if [[ -n "$plugin_present" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local plugins_json
|
|
plugins_json=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.plugins}' 2>/dev/null || true)
|
|
|
|
echo "Configuring CNPG to use Barman Cloud plugin '$BARMAN_PLUGIN_NAME' ..."
|
|
if [[ -z "$plugins_json" || "$plugins_json" == "null" || "$plugins_json" == "[]" ]]; then
|
|
kubectl patch cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --type merge -p "{
|
|
\"spec\": {
|
|
\"plugins\": [
|
|
{
|
|
\"enabled\": true,
|
|
\"name\": \"$BARMAN_PLUGIN_NAME\",
|
|
\"isWALArchiver\": true,
|
|
\"parameters\": {\"barmanObjectName\": \"$BARMAN_OBJECT_NAME\"}
|
|
}
|
|
]
|
|
}
|
|
}"
|
|
else
|
|
kubectl patch cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --type json -p "[
|
|
{
|
|
\"op\": \"add\",
|
|
\"path\": \"/spec/plugins/-\",
|
|
\"value\": {
|
|
\"enabled\": true,
|
|
\"name\": \"$BARMAN_PLUGIN_NAME\",
|
|
\"isWALArchiver\": true,
|
|
\"parameters\": {\"barmanObjectName\": \"$BARMAN_OBJECT_NAME\"}
|
|
}
|
|
}
|
|
]"
|
|
fi
|
|
}
|
|
|
|
ensure_cluster_backup_config() {
|
|
# The Barman Cloud plugin is configured via:
|
|
# - `Cluster.spec.plugins[]` (for WAL archiving)
|
|
# - `Backup`/`ScheduledBackup` resources (method=plugin + pluginConfiguration)
|
|
# Retention is configured in `ObjectStore.spec.retentionPolicy`.
|
|
#
|
|
# Historical versions of this script used to patch `Cluster.spec.backup.*`.
|
|
# That configuration is not used by the plugin and can lead to warnings.
|
|
# Clean up only the legacy fields (if present) and otherwise leave `spec.backup` untouched.
|
|
|
|
local cur_retention cur_plugin cur_barman_object cur_barman_objectstore
|
|
cur_retention=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup.retentionPolicy}' 2>/dev/null || true)
|
|
cur_plugin=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup.pluginConfiguration.name}' 2>/dev/null || true)
|
|
cur_barman_object=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup.pluginConfiguration.parameters.barmanObjectName}' 2>/dev/null || true)
|
|
cur_barman_objectstore=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup.barmanObjectStore}' 2>/dev/null || true)
|
|
|
|
if [[ -z "$cur_retention" && -z "$cur_plugin" && -z "$cur_barman_object" && -z "$cur_barman_objectstore" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo "Cleaning up legacy Cluster.spec.backup fields (plugin uses ObjectStore + spec.plugins instead) ..."
|
|
local json_patch="[]"
|
|
if [[ -n "$cur_retention" ]]; then
|
|
json_patch='[{"op":"remove","path":"/spec/backup/retentionPolicy"}]'
|
|
fi
|
|
if [[ -n "$cur_plugin" || -n "$cur_barman_object" ]]; then
|
|
if [[ "$json_patch" == "[]" ]]; then
|
|
json_patch='[{"op":"remove","path":"/spec/backup/pluginConfiguration"}]'
|
|
else
|
|
json_patch='[{"op":"remove","path":"/spec/backup/retentionPolicy"},{"op":"remove","path":"/spec/backup/pluginConfiguration"}]'
|
|
fi
|
|
fi
|
|
if [[ -n "$cur_barman_objectstore" ]]; then
|
|
# Remove built-in barmanObjectStore config when using the plugin.
|
|
if [[ "$json_patch" == "[]" ]]; then
|
|
json_patch='[{"op":"remove","path":"/spec/backup/barmanObjectStore"}]'
|
|
else
|
|
json_patch=$(echo "$json_patch" | sed 's/]$/, {"op":"remove","path":"\/spec\/backup\/barmanObjectStore"}]/')
|
|
fi
|
|
fi
|
|
|
|
if [[ "$json_patch" != "[]" ]]; then
|
|
kubectl patch cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --type json -p "$json_patch" >/dev/null 2>&1 || true
|
|
fi
|
|
}
|
|
|
|
ensure_scheduled_backup() {
|
|
local incremental="${SCHEDULED_BACKUP_INCREMENTAL,,}"
|
|
echo "Ensuring ScheduledBackup '$SCHEDULED_BACKUP_NAME' ..."
|
|
kubectl_apply_retry "$NAMESPACE" <<SCHEDULEDBACKUP
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: ScheduledBackup
|
|
metadata:
|
|
name: $SCHEDULED_BACKUP_NAME
|
|
spec:
|
|
schedule: "$SCHEDULED_BACKUP_CRON"
|
|
immediate: $SCHEDULED_BACKUP_IMMEDIATE
|
|
backupOwnerReference: self
|
|
cluster:
|
|
name: $CNPG_CLUSTER_NAME
|
|
method: $SCHEDULED_BACKUP_METHOD
|
|
pluginConfiguration:
|
|
name: $BARMAN_PLUGIN_NAME
|
|
parameters:
|
|
barmanObjectName: $BARMAN_OBJECT_NAME
|
|
$(if [[ "$incremental" == "true" || "$incremental" == "1" || "$incremental" == "yes" ]]; then echo " backupType: incremental"; fi)
|
|
SCHEDULEDBACKUP
|
|
}
|
|
|
|
wait_for_plugin_ready() {
|
|
local start_time now plugin_names deployment_rows ns ready replicas
|
|
local plugin_deploy_ready pod socket_path
|
|
start_time=$(date +%s)
|
|
while true; do
|
|
plugin_deploy_ready=0
|
|
plugin_names=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.plugins[*].name}' 2>/dev/null || true)
|
|
if printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -Fxq "$BARMAN_PLUGIN_NAME"; then
|
|
deployment_rows=$(kubectl get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true)
|
|
if [[ -z "$deployment_rows" ]]; then
|
|
deployment_rows=$(kubectl get deployment -A -o jsonpath='{range .items[?(@.metadata.name=="barman-cloud")]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true)
|
|
fi
|
|
|
|
while IFS=$'\t' read -r ns ready replicas; do
|
|
[[ -z "$ns" ]] && continue
|
|
ready=${ready:-0}
|
|
replicas=${replicas:-0}
|
|
if (( ready >= 1 && replicas >= 1 )); then
|
|
plugin_deploy_ready=1
|
|
break
|
|
fi
|
|
done <<< "$deployment_rows"
|
|
|
|
if (( plugin_deploy_ready == 1 )); then
|
|
pod=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME,cnpg.io/instanceRole=primary" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
|
if [[ -z "$pod" ]]; then
|
|
pod=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
|
fi
|
|
socket_path="$PLUGIN_SOCKET_DIR/$BARMAN_PLUGIN_NAME"
|
|
if [[ -n "$pod" ]]; then
|
|
if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- test -S "$socket_path" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
if kubectl -n "$NAMESPACE" exec "$pod" -c manager -- test -S "$socket_path" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
now=$(date +%s)
|
|
if (( now - start_time >= PLUGIN_READY_TIMEOUT )); then
|
|
echo "ERROR: Timed out waiting for CNPG plugin '$BARMAN_PLUGIN_NAME' to become available." >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "Waiting for CNPG plugin '$BARMAN_PLUGIN_NAME' to become available ..."
|
|
sleep 5
|
|
done
|
|
}
|
|
|
|
has_successful_base_backup() {
|
|
local rows name phase method backup_type phase_lc method_lc backup_type_lc
|
|
rows=$(kubectl get backup -n "$NAMESPACE" -o jsonpath="{range .items[?(@.spec.cluster.name=='$CNPG_CLUSTER_NAME')]}{.metadata.name}{\"\t\"}{.status.phase}{\"\t\"}{.spec.method}{\"\t\"}{.spec.pluginConfiguration.parameters.backupType}{\"\n\"}{end}" 2>/dev/null || true)
|
|
|
|
if [[ -z "$rows" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
while IFS=$'\t' read -r name phase method backup_type; do
|
|
[[ -z "$name" ]] && continue
|
|
phase_lc=$(echo "${phase:-}" | tr '[:upper:]' '[:lower:]')
|
|
method_lc=$(echo "${method:-}" | tr '[:upper:]' '[:lower:]')
|
|
backup_type_lc=$(echo "${backup_type:-}" | tr '[:upper:]' '[:lower:]')
|
|
|
|
if [[ "$phase_lc" != "completed" && "$phase_lc" != "succeeded" ]]; then
|
|
continue
|
|
fi
|
|
|
|
if [[ "$method_lc" == "plugin" ]] && [[ "$backup_type_lc" == "incremental" || "$backup_type_lc" == "incr" ]]; then
|
|
continue
|
|
fi
|
|
|
|
return 0
|
|
done <<< "$rows"
|
|
|
|
return 1
|
|
}
|
|
|
|
wait_for_successful_base_backup() {
|
|
local start_time now elapsed
|
|
start_time=$(date +%s)
|
|
while true; do
|
|
if [[ -n "$LAST_BACKUP_NAME" ]]; then
|
|
local phase err
|
|
phase=$(kubectl -n "$NAMESPACE" get backup "$LAST_BACKUP_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
|
err=$(kubectl -n "$NAMESPACE" get backup "$LAST_BACKUP_NAME" -o jsonpath='{.status.error}' 2>/dev/null || true)
|
|
case "${phase:-}" in
|
|
Completed|Succeeded|completed|succeeded)
|
|
return 0
|
|
;;
|
|
Failed|failed)
|
|
echo "ERROR: Backup '$LAST_BACKUP_NAME' failed: ${err:-<no error provided>}" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if has_successful_base_backup; then
|
|
return 0
|
|
fi
|
|
|
|
now=$(date +%s)
|
|
elapsed=$((now - start_time))
|
|
if (( elapsed >= BACKUP_STATUS_TIMEOUT )); then
|
|
return 1
|
|
fi
|
|
|
|
echo "Waiting for a successful base backup ... (${elapsed}s/${BACKUP_STATUS_TIMEOUT}s)"
|
|
sleep "$BACKUP_STATUS_INTERVAL"
|
|
done
|
|
}
|
|
|
|
trigger_backup() {
|
|
local backup_type="${1:-full}"
|
|
local backup_name
|
|
backup_name="${CNPG_CLUSTER_NAME}-backup-$(date +%Y%m%d%H%M%S)"
|
|
LAST_BACKUP_NAME="$backup_name"
|
|
echo "Triggering ${backup_type} backup $backup_name ..."
|
|
if [[ "$backup_type" == "incremental" || "$backup_type" == "incr" ]]; then
|
|
kubectl_apply_retry "$NAMESPACE" <<BACKUP
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: Backup
|
|
metadata:
|
|
name: $backup_name
|
|
spec:
|
|
method: plugin
|
|
pluginConfiguration:
|
|
name: $BARMAN_PLUGIN_NAME
|
|
parameters:
|
|
barmanObjectName: $BARMAN_OBJECT_NAME
|
|
backupType: incremental
|
|
cluster:
|
|
name: $CNPG_CLUSTER_NAME
|
|
BACKUP
|
|
else
|
|
kubectl_apply_retry "$NAMESPACE" <<BACKUP
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: Backup
|
|
metadata:
|
|
name: $backup_name
|
|
spec:
|
|
method: plugin
|
|
pluginConfiguration:
|
|
name: $BARMAN_PLUGIN_NAME
|
|
parameters:
|
|
barmanObjectName: $BARMAN_OBJECT_NAME
|
|
cluster:
|
|
name: $CNPG_CLUSTER_NAME
|
|
BACKUP
|
|
fi
|
|
}
|
|
|
|
status() {
|
|
ensure_tools
|
|
echo "CNPG backup status for cluster '$CNPG_CLUSTER_NAME' (namespace: '$NAMESPACE')"
|
|
|
|
local backup_spec
|
|
backup_spec=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup}' 2>/dev/null || true)
|
|
if [[ -z "$backup_spec" || "$backup_spec" == "null" ]]; then
|
|
echo "Cluster spec.backup: MISSING"
|
|
else
|
|
local rp plugin obj
|
|
rp=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup.retentionPolicy}' 2>/dev/null || true)
|
|
plugin=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup.pluginConfiguration.name}' 2>/dev/null || true)
|
|
obj=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.backup.pluginConfiguration.parameters.barmanObjectName}' 2>/dev/null || true)
|
|
echo "Cluster spec.backup: present (retentionPolicy=${rp:-<unset>} plugin=${plugin:-<unset>} barmanObjectName=${obj:-<unset>})"
|
|
fi
|
|
|
|
local ca_line
|
|
ca_line=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{range .status.conditions[?(@.type=="ContinuousArchiving")]}{.status}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' 2>/dev/null || true)
|
|
if [[ -n "$ca_line" ]]; then
|
|
echo "ContinuousArchiving condition:"
|
|
printf '%s' "$ca_line" | awk -F$'\t' '{print "- status=" $1 (length($2)?" reason=" $2:"") (length($3)?" message=" $3:"")}'
|
|
else
|
|
echo "ContinuousArchiving condition: <not reported>"
|
|
fi
|
|
|
|
echo "ScheduledBackup resources:"
|
|
kubectl get scheduledbackup -n "$NAMESPACE" 2>/dev/null || true
|
|
|
|
echo "Backup resources:"
|
|
kubectl get backup -n "$NAMESPACE" 2>/dev/null | grep "$CNPG_CLUSTER_NAME" || true
|
|
|
|
if has_successful_base_backup; then
|
|
echo "Backup status: OK (at least one successful base backup is available)."
|
|
return 0
|
|
fi
|
|
|
|
echo "ERROR: No successful base backup found for cluster '$CNPG_CLUSTER_NAME'." >&2
|
|
return 1
|
|
}
|
|
|
|
case "$ACTION" in
|
|
start)
|
|
ensure_tools
|
|
ensure_namespace
|
|
wait_for_apiserver_ready 180
|
|
ensure_cluster
|
|
ensure_garage_bucket_and_key
|
|
apply_barman_object_store
|
|
ensure_barman_plugin_config
|
|
ensure_cluster_backup_config
|
|
wait_for_plugin_ready
|
|
ensure_scheduled_backup
|
|
if [[ "$RUN_FIRST_BACKUP" == "1" ]]; then
|
|
wait_for_cnpg_webhook 300
|
|
trigger_backup
|
|
if ! wait_for_successful_base_backup; then
|
|
echo "ERROR: Timed out waiting for a successful base backup." >&2
|
|
status || true
|
|
exit 1
|
|
fi
|
|
status
|
|
else
|
|
status || true
|
|
fi
|
|
;;
|
|
backup)
|
|
ensure_tools
|
|
ensure_namespace
|
|
wait_for_apiserver_ready 180
|
|
ensure_cluster
|
|
apply_barman_object_store
|
|
ensure_barman_plugin_config
|
|
ensure_cluster_backup_config
|
|
wait_for_plugin_ready
|
|
wait_for_cnpg_webhook 300
|
|
trigger_backup "${2:-full}"
|
|
;;
|
|
status)
|
|
wait_for_apiserver_ready 60 || true
|
|
status
|
|
;;
|
|
*)
|
|
usage
|
|
;;
|
|
esac
|