prole/etc/init_common_services.sh
chrisfu 5618b662dd Remove prole-db-manager; simplify deployment via prole-authority; fix pg18 downgrade & cluster name
Summary:
Removed the prole-db-manager microservice and simplified deployment to use
prole-authority as the internal management and authorization point. Fixed two
blocking bugs that prevented silent install from completing on knoe-dev-cluster.

Removed: prole-db-manager
- Deleted db-manager-deployment.yaml and db-manager-service.yaml from opentofu manifests
- Deleted src/db-manager/ (Dockerfile, server.js, package.json, tests)
- Removed prole-db-manager port-forward mapping from installer/core/env.py
- Removed init_db_manager.sh from Initialization Scripts (milestones.py, actions.py)
- Removed init_certmgr.sh and init_db_manager.sh tabs from services screen (services.py)
- Removed live k8s Deployment/Service from knoe-dev-cluster

Fixed: PostgreSQL version downgrade error (pg17 -> pg18)
- Created conf/postgresql/.version with value 18
- Updated k8s/prole/prole-db.yaml and prole-db-recovery.yaml.tpl imageName to prole-db:18-089
- Fixed _init_database_options_state() to restore saved version_type from prole.cfg
  so db_version_type defaults to v18 (pg18) instead of silently reverting to pg17
- Added database_options.* keys to _collect_input_snapshot() in cfg.py so
  distribution, version_type, and all extension toggles persist to prole.cfg

Fixed: Cluster name inconsistency
- Removed stale prole-dev-cluster references; all scripts now use knoe-dev-cluster
- Added knoe-dev-cluster to mode-detection case in etc/prole_cfg.sh

Config: conf/prole.cfg
- Set kerberos_config.enabled = False, KERBEROS_AUTO_ENABLED = False
- Added database_options.distribution = percona, version_type = v18
- Added all 13 extension flags set to True (postgis, pgvector, pgcrypto, pgaudit,
  pg_repack, pg_stat_statements, pg_buffercache, pg_freespacemap, pgrowlocks,
  postgres_fdw, dblink, pg_stat_monitor, pgbadger)

Verification:
./install.py -s -l -v -c conf/prole.cfg completed successfully.
CNPG deployed prole-db:18-089 to knoe-dev-cluster; all milestones passed.

Co-authored-by: Junie <junie@jetbrains.com>
2026-03-01 20:40:44 -08:00

588 lines
19 KiB
Bash
Executable File

#!/usr/bin/env bash
set -u
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"
# Inject default config if not provided
_has_config=0
for _arg in "$@"; do
[[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
done
if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then
set -- "-c" "$SCRIPT_DIR/../conf/prole.cfg" "$@"
fi
unset _has_config _arg
common_core_preparse_config "$@"
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"
usage() {
cat <<EOF
Usage: init_common_services.sh [-n|--namespace NS] [-k|--kerberos] <update|start|status|verify>
Deploys common infrastructure services (ArgoCD, OpenTofu, Garage, OpenBao, Kong, Cert-Manager)
into the given Kubernetes namespace. Use -k to include the Kerberos/KDC service.
EOF
}
NS=""
ACTION="update"
ENABLE_KERBEROS=0
while [ $# -gt 0 ]; do
case "$1" in
-c|--config)
shift
shift
;;
-m|--mode)
shift
prole_set_mode "${1:-}"
shift
;;
-m=*|--mode=*)
prole_set_mode "${1#*=}"
shift
;;
-n|--namespace)
shift
NS="${1:-}"
shift
;;
-n=*|--namespace=*)
NS="${1#*=}"
shift
;;
-k|--kerberos)
ENABLE_KERBEROS=1
shift
;;
update|start|status|verify|repair|reload|restart)
ACTION="$1"
[[ "$ACTION" == "repair" ]] && ACTION="update"
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage
exit 2
;;
esac
done
if [ -z "$NS" ]; then
NS="${SERVICE_NAMESPACE:-${NAMESPACE:-}}"
fi
if [ -z "$NS" ]; then
NS="default"
fi
ARGOCD_NS="${ARGOCD_NAMESPACE:-argocd}"
REGISTRY_NS="${REGISTRY_NAMESPACE:-default}"
prole_ensure_kubeconfig >/dev/null 2>&1 || true
# Check cluster reachability early to fail fast
if [[ "$ACTION" != "status" ]]; then
if ! kubectl cluster-info >/dev/null 2>&1; then
echo "ERROR: Cluster not reachable. Check KUBECONFIG and cluster status." >&2
exit 1
fi
fi
_verify_common_services() {
local status_script="$SCRIPT_DIR/status_common_services.sh"
if [[ ! -x "$status_script" ]]; then
echo "ERROR: status_common_services.sh not found at $status_script" >&2
return 2
fi
local args=(-n "$NS")
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
args+=(-k)
fi
echo "Verifying common services in namespace '$NS' ..."
"$status_script" "${args[@]}"
}
# Short-circuit verify before any deploy/reload/preload work
if [[ -n "${COMMON_SERVICES_INIT_LOG:-}" ]]; then
mkdir -p "$(dirname "$COMMON_SERVICES_INIT_LOG")"
exec > >(tee -a "$COMMON_SERVICES_INIT_LOG") 2>&1
fi
if [[ "$ACTION" == "verify" ]]; then
_verify_common_services
exit $?
fi
# ---------------------------------------------------------------------------
# Pre-import container images from $PROLE_DATA/docker-import/*.tar
# ---------------------------------------------------------------------------
# final_deployment.sh exports images as
# $PROLE_DATA/docker-import/<safe_name>.tar
# where safe_name = image ref with / and : replaced by _.
# Loading these tars into the local container runtime before the sub-scripts
# deploy pods avoids slow image pulls from remote registries.
# ---------------------------------------------------------------------------
_resolve_prole_data() {
local d="${PROLE_DATA:-}"
if [[ -z "$d" ]]; then
if [[ -f "$SCRIPT_DIR/../env.sh" ]]; then
# shellcheck disable=SC1090
source "$SCRIPT_DIR/../env.sh"
d="${PROLE_DATA:-}"
fi
fi
echo "${d:-}"
}
# Standard console execution element.
# Runs a command silently (output captured), displays a live elapsed-time footer
# on stderr when attached to a TTY, prints elapsed time on completion, and tails
# captured output for diagnostics on failure.
_exec_with_progress_footer() {
local label="$1"
shift
local start_time
start_time=$(date +%s)
local tmp_out
tmp_out=$(mktemp 2>/dev/null || echo "/tmp/_prole_exec_$$.out")
# Spawn live footer only when stderr is a real TTY
local spinner_pid=""
if [[ -t 2 ]]; then
(
while true; do
now=$(date +%s)
elapsed=$(( now - start_time ))
h=$(( elapsed / 3600 ))
m=$(( (elapsed % 3600) / 60 ))
s=$(( elapsed % 60 ))
printf "\r\033[2K [%02d:%02d:%02d elapsed] %s" "$h" "$m" "$s" "$label" >&2
sleep 1
done
) &
spinner_pid=$!
disown "$spinner_pid" 2>/dev/null || true
fi
"$@" >"$tmp_out" 2>&1
local rc=$?
if [[ -n "$spinner_pid" ]]; then
kill "$spinner_pid" 2>/dev/null || true
wait "$spinner_pid" 2>/dev/null || true
printf "\r\033[2K" >&2
fi
local end_time elapsed_total h m s
end_time=$(date +%s)
elapsed_total=$(( end_time - start_time ))
h=$(( elapsed_total / 3600 ))
m=$(( (elapsed_total % 3600) / 60 ))
s=$(( elapsed_total % 60 ))
printf " [%02dh:%02dm:%02ds elapsed] %s\n" "$h" "$m" "$s" "$label"
if [[ $rc -ne 0 && -s "$tmp_out" ]]; then
echo " [Last output]:" >&2
tail -n 10 "$tmp_out" >&2
fi
rm -f "$tmp_out"
return $rc
}
# Extract the primary image ref from a Docker tarball's manifest.json.
_image_name_from_tar() {
local tar_file="$1"
if command -v python3 >/dev/null 2>&1; then
python3 - "$tar_file" <<'PYEOF' 2>/dev/null
import json, sys, tarfile
try:
with tarfile.open(sys.argv[1]) as t:
tags = json.load(t.extractfile("manifest.json"))[0].get("RepoTags") or []
if tags:
print(tags[0])
except Exception:
pass
PYEOF
fi
}
_preload_docker_images() {
local prole_data
prole_data=$(_resolve_prole_data)
local import_dir="${DOCKER_IMPORT_DIR:-${prole_data:+${prole_data}/docker-import}}"
if [[ -z "$import_dir" || ! -d "$import_dir" ]]; then
echo "[SKIP] No docker-import directory found; images will be pulled from registries."
return 0
fi
local tar_files
tar_files=$(find "$import_dir" -maxdepth 1 -name '*.tar' -type f 2>/dev/null || true)
if [[ -z "$tar_files" ]]; then
echo "[SKIP] docker-import directory exists but contains no .tar files."
return 0
fi
local mode="${PROLE_MODE:-k3d}"
echo "Pre-loading container images from $import_dir (mode=$mode) ..."
case "$mode" in
k3d)
if ! command -v k3d >/dev/null 2>&1; then
echo "WARN: k3d not available; skipping docker-import pre-load." >&2
return 0
fi
local cluster_name="${K3D_CLUSTER_NAME:-knoe-dev-cluster}"
# ── Step 1: build indexed arrays of tar paths + resolved image names ──
local -a tar_paths=()
local -a tar_images=()
local _tf
while IFS= read -r _tf; do
[[ -z "$_tf" ]] && continue
tar_paths+=("$_tf")
tar_images+=("$(_image_name_from_tar "$_tf")")
done <<< "$tar_files"
local total_tars=${#tar_paths[@]}
if [[ $total_tars -eq 0 ]]; then
echo " [SKIP] No tar files to process."
return 0
fi
# ── Step 2: query images already in the k3d cluster ───────────────────
echo " Checking k3d cluster '$cluster_name' for existing images ..."
local existing_images=""
local node_name="k3d-${cluster_name}-server-0"
if docker inspect "$node_name" >/dev/null 2>&1; then
existing_images=$(docker exec "$node_name" \
ctr --address /run/k3s/containerd/containerd.sock \
--namespace k8s.io images ls -q 2>/dev/null || true)
else
echo " [WARN] k3d node '$node_name' not reachable; will import all images."
fi
# ── Step 3: delta — split into already-present vs needs-import ─────────
local -a to_import_tars=()
local -a to_import_images=()
local -a already_present=()
local _i
for (( _i=0; _i<total_tars; _i++ )); do
local _tar="${tar_paths[$_i]}"
local _img="${tar_images[$_i]}"
local _found=0
if [[ -n "$_img" && -n "$existing_images" ]]; then
# Strip digest suffix before matching
local _img_ref="${_img%%@*}"
if echo "$existing_images" | grep -qF "$_img_ref"; then
_found=1
fi
fi
if [[ $_found -eq 1 ]]; then
already_present+=("$(basename "$_tar")${_img:+ → $_img}")
else
to_import_tars+=("$_tar")
to_import_images+=("${_img:-}")
fi
done
# ── Step 4: report delta to user ───────────────────────────────────────
if [[ ${#already_present[@]} -gt 0 ]]; then
echo " Already in cluster — skipping ${#already_present[@]} image(s):"
local _entry
for _entry in "${already_present[@]}"; do
echo "$_entry"
done
fi
local import_count=${#to_import_tars[@]}
if [[ $import_count -eq 0 ]]; then
echo " [OK] All images already present in k3d cluster '$cluster_name'; nothing to import."
return 0
fi
echo " Bulk uploading $import_count image(s) into k3d cluster '$cluster_name':"
for (( _i=0; _i<import_count; _i++ )); do
local _lbl="${to_import_images[$_i]:-$(basename "${to_import_tars[$_i]}")}"
echo "$(basename "${to_import_tars[$_i]}")${_lbl:+ ($_lbl)}"
done
# ── Step 5: bulk import with live progress footer ──────────────────────
local overall_start elapsed_bulk
overall_start=$(date +%s)
if _exec_with_progress_footer \
"k3d image import — bulk ($import_count images) → $cluster_name" \
k3d image import "${to_import_tars[@]}" -c "$cluster_name"; then
elapsed_bulk=$(( $(date +%s) - overall_start ))
local bm=$(( elapsed_bulk / 60 )) bs=$(( elapsed_bulk % 60 ))
echo " [OK] Bulk import of $import_count image(s) complete (${bm}m${bs}s)."
else
echo " [WARN] Bulk import failed; falling back to sequential import." >&2
local _idx
for (( _i=0; _i<import_count; _i++ )); do
_idx=$(( _i + 1 ))
local _tar="${to_import_tars[$_i]}"
local _img="${to_import_images[$_i]:-$(basename "$_tar")}"
# ETA based on average time per completed image
local _now _elapsed_so_far _eta_str=""
_now=$(date +%s)
_elapsed_so_far=$(( _now - overall_start ))
if [[ $_i -gt 0 ]]; then
local _avg=$(( _elapsed_so_far / _i ))
local _remaining=$(( _avg * (import_count - _i) ))
local _rm=$(( _remaining / 60 )) _rs=$(( _remaining % 60 ))
_eta_str=" | ETA ~${_rm}m${_rs}s"
fi
echo " [$_idx/$import_count] $(basename "$_tar")${_img:+ ($_img)}${_eta_str}"
local _item_start
_item_start=$(date +%s)
if _exec_with_progress_footer \
"k3d image import [$_idx/$import_count] $(basename "$_tar")" \
k3d image import "$_tar" -c "$cluster_name"; then
local _item_elapsed=$(( $(date +%s) - _item_start ))
echo " [OK] $(basename "$_tar") (${_item_elapsed}s)"
else
echo " [WARN] Failed to import $(basename "$_tar"); will pull at deploy time." >&2
fi
done
local _total_elapsed=$(( $(date +%s) - overall_start ))
local _te_m=$(( _total_elapsed / 60 )) _te_s=$(( _total_elapsed % 60 ))
echo " [DONE] Sequential import complete. Total: ${_te_m}m${_te_s}s"
fi
;;
k3s)
# For k3s: copy tarballs directly to k3s nodes and import via
# `k3s ctr images import`. This avoids the slow docker-push-to-registry
# flow that can stall for minutes per image layer.
local playbook_dir="$SCRIPT_DIR/../infrastructure/playbooks"
local playbook="$playbook_dir/k3s_import_images.yml"
if [[ -f "$playbook" ]]; then
# Build JSON array of objects for Ansible extra-vars
local images_json="["
local first=1
local tar_file
while IFS= read -r tar_file; do
[[ -z "$tar_file" ]] && continue
# Extract image name from tarball manifest on host
local img_name=""
if command -v python3 >/dev/null 2>&1; then
img_name=$(python3 -c "import json, tarfile;
try:
with tarfile.open('$tar_file') as tar:
m = json.load(tar.extractfile('manifest.json'))
print(m[0]['RepoTags'][0])
except Exception:
pass" 2>/dev/null || true)
fi
# Fallback to safe_name derivation if python failed or not available
if [[ -z "$img_name" ]]; then
local base_name=$(basename "$tar_file" .tar)
img_name="${base_name//_//}"
img_name="${img_name%/*}:${img_name##*/}"
fi
[[ $first -eq 0 ]] && images_json+=","
images_json+="{\"path\": \"$tar_file\", \"image\": \"$img_name\"}"
first=0
echo " Queued: $(basename "$tar_file") ${img_name:+(image: $img_name)}"
done <<< "$tar_files"
images_json+="]"
echo " Importing images into k3s nodes via Ansible ..."
local ansible_cmd=(ansible-playbook)
local vault_pass="$SCRIPT_DIR/../.vault_pass"
if [[ -f "$vault_pass" ]]; then
ansible_cmd+=(--vault-password-file "$vault_pass")
elif [[ -n "${ANSIBLE_VAULT_PASSWORD_FILE:-}" ]]; then
ansible_cmd+=(--vault-password-file "$ANSIBLE_VAULT_PASSWORD_FILE")
fi
ansible_cmd+=(-e "{\"k3s_import_images\": $images_json}")
ansible_cmd+=("$playbook")
if "${ansible_cmd[@]}"; then
echo " [OK] All images checked/imported into k3s nodes."
else
echo " [WARN] Ansible import failed; images will be pulled at deploy time." >&2
fi
else
echo " [WARN] Ansible playbook not found ($playbook); images will be pulled at deploy time." >&2
fi
;;
*)
echo "WARN: Unknown mode '$mode'; skipping docker-import pre-load." >&2
;;
esac
echo "Pre-load complete."
}
case "$ACTION" in
start|update|reload|initialize|restart|repair)
_preload_docker_images
;;
esac
if [ -x "$SCRIPT_DIR/init_service_layer.sh" ]; then
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
"$SCRIPT_DIR/init_service_layer.sh" -n "$NS" -k "$ACTION"
else
"$SCRIPT_DIR/init_service_layer.sh" -n "$NS" "$ACTION"
fi
exit $?
fi
OPENTOFU_NAME=${OPENTOFU_NAME:-opentofu}
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
ARGOCD_SERVER_NAME=${ARGOCD_SERVER_NAME:-argocd-server}
GARAGE_NAME=${GARAGE_NAME:-garage}
OPENTOFU_CONFIGMAP=${OPENTOFU_CONFIGMAP:-opentofu-nginx}
OPENTOFU_SECRET=${OPENTOFU_SECRET:-opentofu-admin}
GARAGE_CONFIGMAP=${GARAGE_CONFIGMAP:-garage-config}
GARAGE_SECRET_NAME=${GARAGE_SECRET_NAME:-garage-secrets}
find_namespaces() {
local kind="$1"
local name="$2"
kubectl get "$kind" -A --no-headers 2>/dev/null | awk -v n="$name" '$2==n {print $1}' | sort -u
}
collect_other_namespaces() {
local target="$1"
shift
local name="$1"
shift
local kinds=("$@")
local found=""
local kind
for kind in "${kinds[@]}"; do
found+=$(find_namespaces "$kind" "$name")
found+=$'\n'
done
printf '%s\n' "$found" | awk -v target="$target" 'NF && $1 != target {print $1}' | sort -u
}
migrate_common_services() {
local old_ns
for old_ns in $(collect_other_namespaces "$NS" "$OPENTOFU_NAME" deployment service); do
echo "Found OpenTofu in namespace '$old_ns'; removing before deploy to '$NS' ..."
if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then
"$SCRIPT_DIR/init_opentofu.sh" -n "$old_ns" stop || true
else
kubectl delete -n "$old_ns" deploy "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true
fi
kubectl delete -n "$old_ns" configmap "$OPENTOFU_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" secret "$OPENTOFU_SECRET" --ignore-not-found >/dev/null 2>&1 || true
done
for old_ns in $(collect_other_namespaces "$ARGOCD_NS" "$ARGOCD_SERVER_NAME" deployment service); do
echo "Found ArgoCD in namespace '$old_ns'; removing before deploy to '$ARGOCD_NS' ..."
if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then
REGISTRY_NAMESPACE="$REGISTRY_NS" "$SCRIPT_DIR/init_registry.sh" -n "$old_ns" stop || true
else
kubectl delete -n "$old_ns" deploy "$ARGOCD_SERVER_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$ARGOCD_SERVER_NAME" --ignore-not-found >/dev/null 2>&1 || true
fi
done
for old_ns in $(collect_other_namespaces "$NS" "$GARAGE_NAME" statefulset service); do
echo "Found Garage in namespace '$old_ns'; removing before deploy to '$NS' ..."
kubectl delete -n "$old_ns" statefulset "$GARAGE_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$GARAGE_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" configmap "$GARAGE_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" secret "$GARAGE_SECRET_NAME" --ignore-not-found >/dev/null 2>&1 || true
done
for old_ns in $(collect_other_namespaces "$NS" "$OPENBAO_NAME" deployment service statefulset); do
echo "Found OpenBao in namespace '$old_ns'; removing before deploy to '$NS' ..."
if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then
"$SCRIPT_DIR/init_openbao.sh" -n "$old_ns" stop || true
else
kubectl delete -n "$old_ns" deploy "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" statefulset "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
fi
done
}
echo "Deploying common services (namespace=$NS, action=$ACTION)"
rc=0
case "$ACTION" in
start|update|reload|initialize|restart|status)
migrate_common_services
;;
esac
if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then
ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" "$SCRIPT_DIR/init_opentofu.sh" -n "$NS" "$ACTION" || rc=$?
else
echo "WARN: init_opentofu.sh not found; skipping OpenTofu."
fi
if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then
OPENBAO_NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" \
"$SCRIPT_DIR/init_openbao.sh" -n "$NS" "$ACTION" || rc=$?
else
echo "WARN: init_openbao.sh not found; skipping OpenBao."
fi
if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then
ARGOCD_NAMESPACE="$ARGOCD_NS" REGISTRY_NAMESPACE="$REGISTRY_NS" \
"$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NS" --registry-namespace "$REGISTRY_NS" "$ACTION" || rc=$?
else
echo "WARN: init_registry.sh not found; ArgoCD deploy skipped."
fi
if [ -x "$SCRIPT_DIR/init_garage_store.sh" ]; then
garage_action="$ACTION"
case "$garage_action" in
update|reload|initialize) garage_action="start" ;;
esac
NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" GARAGE_NAMESPACE="$NS" ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" \
"$SCRIPT_DIR/init_garage_store.sh" "$garage_action" || rc=$?
else
echo "WARN: init_garage_store.sh not found; garage deploy skipped."
fi
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
if [ -x "$SCRIPT_DIR/init_kdc.sh" ]; then
kdc_action="$ACTION"
case "$kdc_action" in
stop) kdc_action="cleanup" ;;
status) kdc_action="status" ;;
*) kdc_action="update" ;;
esac
SERVICE_NAMESPACE="$NS" PROLE_KDC_NAMESPACE="$NS" \
"$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$?
else
echo "WARN: init_kdc.sh not found; kerberos deploy skipped."
fi
fi
exit "$rc"