prole/mock_val/init_common_services.sh
chrisfu d397b3000e Fix installer ctx apply flow and registry namespace
- Add explicit Apply button for kubectl context switching to avoid half-applied changes

- Allow kubeconfig/context-based auth without requiring K3S_TOKEN when kubeconfig is valid

- Prompt cleanup/reset of previous Common Core services namespace to prevent resource collisions

- Remove hardcoded 'common-services' registry namespace; default registry deploy/check to SERVICE_NAMESPACE/REGISTRY_NAMESPACE

- Update mocks and add regression tests for namespace resolution and installer behavior
2026-03-07 10:35:27 -08:00

595 lines
20 KiB
Bash
Executable File

#!/usr/bin/env bash
set -u
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"
# Inject default config if not provided
_has_config=0
for _arg in "$@"; do
[[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
done
if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then
set -- "-c" "$SCRIPT_DIR/../conf/prole.cfg" "$@"
fi
unset _has_config _arg
common_core_preparse_config "$@"
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"
usage() {
cat <<EOF
Usage: init_common_services.sh [-n|--namespace NS] [-k|--kerberos] <update|start|status|verify>
Deploys common infrastructure services (ArgoCD, OpenTofu, Garage, OpenBao, Kong, Cert-Manager)
into the given Kubernetes namespace. Use -k to include the Kerberos/KDC service.
EOF
}
NS=""
ACTION="update"
ENABLE_KERBEROS=0
while [ $# -gt 0 ]; do
case "$1" in
-c|--config)
shift
shift
;;
-m|--mode)
shift
prole_set_mode "${1:-}"
shift
;;
-m=*|--mode=*)
prole_set_mode "${1#*=}"
shift
;;
-n|--namespace)
shift
NS="${1:-}"
shift
;;
-n=*|--namespace=*)
NS="${1#*=}"
shift
;;
-k|--kerberos)
ENABLE_KERBEROS=1
shift
;;
update|start|status|verify|repair|reload|restart)
ACTION="$1"
[[ "$ACTION" == "repair" ]] && ACTION="update"
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage
exit 2
;;
esac
done
if [ -z "$NS" ]; then
NS="${SERVICE_NAMESPACE:-${NAMESPACE:-}}"
fi
if [ -z "$NS" ]; then
NS="default"
fi
ARGOCD_NS="${ARGOCD_NAMESPACE:-argocd}"
REGISTRY_NS="${REGISTRY_NAMESPACE:-${NS}}"
prole_ensure_kubeconfig >/dev/null 2>&1 || true
# Check cluster reachability early to fail fast
if [[ "$ACTION" != "status" ]]; then
if ! kubectl cluster-info >/dev/null 2>&1; then
echo "ERROR: Cluster not reachable. Check KUBECONFIG and cluster status." >&2
exit 1
fi
fi
_verify_common_services() {
local status_script="$SCRIPT_DIR/status_common_services.sh"
if [[ ! -x "$status_script" ]]; then
echo "ERROR: status_common_services.sh not found at $status_script" >&2
return 2
fi
local args=(-n "$NS")
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
args+=(-k)
fi
echo "Verifying common services in namespace '$NS' ..."
"$status_script" "${args[@]}"
}
# Short-circuit verify before any deploy/reload/preload work
if [[ -n "${COMMON_SERVICES_INIT_LOG:-}" ]]; then
mkdir -p "$(dirname "$COMMON_SERVICES_INIT_LOG")"
exec > >(tee -a "$COMMON_SERVICES_INIT_LOG") 2>&1
fi
if [[ "$ACTION" == "verify" ]]; then
_verify_common_services
exit $?
fi
# ---------------------------------------------------------------------------
# Pre-import container images from $PROLE_DATA/docker-import/*.tar
# ---------------------------------------------------------------------------
# final_deployment.sh exports images as
# $PROLE_DATA/docker-import/<safe_name>.tar
# where safe_name = image ref with / and : replaced by _.
# Loading these tars into the local container runtime before the sub-scripts
# deploy pods avoids slow image pulls from remote registries.
# ---------------------------------------------------------------------------
_resolve_prole_data() {
local d="${PROLE_DATA:-}"
if [[ -z "$d" ]]; then
if [[ -f "$SCRIPT_DIR/../env.sh" ]]; then
# shellcheck disable=SC1090
source "$SCRIPT_DIR/../env.sh"
d="${PROLE_DATA:-}"
fi
fi
echo "${d:-}"
}
# Standard console execution element.
# Runs a command silently (output captured), displays a live elapsed-time footer
# on stderr when attached to a TTY, prints elapsed time on completion, and tails
# captured output for diagnostics on failure.
_exec_with_progress_footer() {
local label="$1"
shift
local start_time
start_time=$(date +%s)
local tmp_out
tmp_out=$(mktemp 2>/dev/null || echo "/tmp/_prole_exec_$$.out")
# Spawn live footer only when stderr is a real TTY
local spinner_pid=""
if [[ -t 2 ]]; then
(
while true; do
now=$(date +%s)
elapsed=$(( now - start_time ))
h=$(( elapsed / 3600 ))
m=$(( (elapsed % 3600) / 60 ))
s=$(( elapsed % 60 ))
printf "\r\033[2K [%02d:%02d:%02d elapsed] %s" "$h" "$m" "$s" "$label" >&2
sleep 1
done
) &
spinner_pid=$!
disown "$spinner_pid" 2>/dev/null || true
fi
"$@" >"$tmp_out" 2>&1
local rc=$?
if [[ -n "$spinner_pid" ]]; then
kill "$spinner_pid" 2>/dev/null || true
wait "$spinner_pid" 2>/dev/null || true
printf "\r\033[2K" >&2
fi
local end_time elapsed_total h m s
end_time=$(date +%s)
elapsed_total=$(( end_time - start_time ))
h=$(( elapsed_total / 3600 ))
m=$(( (elapsed_total % 3600) / 60 ))
s=$(( elapsed_total % 60 ))
printf " [%02dh:%02dm:%02ds elapsed] %s\n" "$h" "$m" "$s" "$label"
if [[ $rc -ne 0 && -s "$tmp_out" ]]; then
echo " [Last output]:" >&2
tail -n 10 "$tmp_out" >&2
fi
rm -f "$tmp_out"
return $rc
}
# Extract the primary image ref from a Docker tarball's manifest.json.
_image_name_from_tar() {
local tar_file="$1"
if command -v python3 >/dev/null 2>&1; then
python3 - "$tar_file" <<'PYEOF' 2>/dev/null
import json, sys, tarfile
try:
with tarfile.open(sys.argv[1]) as t:
tags = json.load(t.extractfile("manifest.json"))[0].get("RepoTags") or []
if tags:
print(tags[0])
except Exception:
pass
PYEOF
fi
}
_preload_docker_images() {
local prole_data
prole_data=$(_resolve_prole_data)
local import_dir="${DOCKER_IMPORT_DIR:-${prole_data:+${prole_data}/docker-import}}"
if [[ -z "$import_dir" || ! -d "$import_dir" ]]; then
echo "[SKIP] No docker-import directory found; images will be pulled from registries."
return 0
fi
local tar_files
tar_files=$(find "$import_dir" -maxdepth 1 -name '*.tar' -type f 2>/dev/null || true)
if [[ -z "$tar_files" ]]; then
echo "[SKIP] docker-import directory exists but contains no .tar files."
return 0
fi
local mode="${PROLE_MODE:-k3d}"
echo "Pre-loading container images from $import_dir (mode=$mode) ..."
case "$mode" in
k3d)
if ! command -v k3d >/dev/null 2>&1; then
echo "WARN: k3d not available; skipping docker-import pre-load." >&2
return 0
fi
local cluster_name="${K3D_CLUSTER_NAME:-prole-dev-cluster}"
# ── Step 1: build indexed arrays of tar paths + resolved image names ──
local -a tar_paths=()
local -a tar_images=()
local _tf
while IFS= read -r _tf; do
[[ -z "$_tf" ]] && continue
tar_paths+=("$_tf")
tar_images+=("$(_image_name_from_tar "$_tf")")
done <<< "$tar_files"
local total_tars=${#tar_paths[@]}
if [[ $total_tars -eq 0 ]]; then
echo " [SKIP] No tar files to process."
return 0
fi
# ── Step 2: query images already in the k3d cluster ───────────────────
echo " Checking k3d cluster '$cluster_name' for existing images ..."
local existing_images=""
local node_name="k3d-${cluster_name}-server-0"
if docker inspect "$node_name" >/dev/null 2>&1; then
existing_images=$(docker exec "$node_name" \
ctr --address /run/k3s/containerd/containerd.sock \
--namespace k8s.io images ls -q 2>/dev/null || true)
else
echo " [WARN] k3d node '$node_name' not reachable; will import all images."
fi
# ── Step 3: delta — split into already-present vs needs-import ─────────
local -a to_import_tars=()
local -a to_import_images=()
local -a already_present=()
local _i
for (( _i=0; _i<total_tars; _i++ )); do
local _tar="${tar_paths[$_i]}"
local _img="${tar_images[$_i]}"
local _found=0
if [[ -n "$_img" && -n "$existing_images" ]]; then
# Strip digest suffix before matching
local _img_ref="${_img%%@*}"
if echo "$existing_images" | grep -qF "$_img_ref"; then
_found=1
fi
fi
if [[ $_found -eq 1 ]]; then
already_present+=("$(basename "$_tar")${_img:+ → $_img}")
else
to_import_tars+=("$_tar")
to_import_images+=("${_img:-}")
fi
done
# ── Step 4: report delta to user ───────────────────────────────────────
if [[ ${#already_present[@]} -gt 0 ]]; then
echo " Already in cluster — skipping ${#already_present[@]} image(s):"
local _entry
for _entry in "${already_present[@]}"; do
echo "$_entry"
done
fi
local import_count=${#to_import_tars[@]}
if [[ $import_count -eq 0 ]]; then
echo " [OK] All images already present in k3d cluster '$cluster_name'; nothing to import."
return 0
fi
echo " Bulk uploading $import_count image(s) into k3d cluster '$cluster_name':"
for (( _i=0; _i<import_count; _i++ )); do
local _lbl="${to_import_images[$_i]:-$(basename "${to_import_tars[$_i]}")}"
echo "$(basename "${to_import_tars[$_i]}")${_lbl:+ ($_lbl)}"
done
# ── Step 5: bulk import with live progress footer ──────────────────────
local overall_start elapsed_bulk
overall_start=$(date +%s)
if _exec_with_progress_footer \
"k3d image import — bulk ($import_count images) → $cluster_name" \
k3d image import "${to_import_tars[@]}" -c "$cluster_name"; then
elapsed_bulk=$(( $(date +%s) - overall_start ))
local bm=$(( elapsed_bulk / 60 )) bs=$(( elapsed_bulk % 60 ))
echo " [OK] Bulk import of $import_count image(s) complete (${bm}m${bs}s)."
else
echo " [WARN] Bulk import failed; falling back to sequential import." >&2
local _idx
for (( _i=0; _i<import_count; _i++ )); do
_idx=$(( _i + 1 ))
local _tar="${to_import_tars[$_i]}"
local _img="${to_import_images[$_i]:-$(basename "$_tar")}"
# ETA based on average time per completed image
local _now _elapsed_so_far _eta_str=""
_now=$(date +%s)
_elapsed_so_far=$(( _now - overall_start ))
if [[ $_i -gt 0 ]]; then
local _avg=$(( _elapsed_so_far / _i ))
local _remaining=$(( _avg * (import_count - _i) ))
local _rm=$(( _remaining / 60 )) _rs=$(( _remaining % 60 ))
_eta_str=" | ETA ~${_rm}m${_rs}s"
fi
echo " [$_idx/$import_count] $(basename "$_tar")${_img:+ ($_img)}${_eta_str}"
local _item_start
_item_start=$(date +%s)
if _exec_with_progress_footer \
"k3d image import [$_idx/$import_count] $(basename "$_tar")" \
k3d image import "$_tar" -c "$cluster_name"; then
local _item_elapsed=$(( $(date +%s) - _item_start ))
echo " [OK] $(basename "$_tar") (${_item_elapsed}s)"
else
echo " [WARN] Failed to import $(basename "$_tar"); will pull at deploy time." >&2
fi
done
local _total_elapsed=$(( $(date +%s) - overall_start ))
local _te_m=$(( _total_elapsed / 60 )) _te_s=$(( _total_elapsed % 60 ))
echo " [DONE] Sequential import complete. Total: ${_te_m}m${_te_s}s"
fi
;;
k3s)
# For k3s: publish images to the in-cluster registry so nodes can pull them.
# Avoid heavy repeated I/O by skipping pushes when the tag already exists.
local registry="${LOCAL_REGISTRY:-}"
if [[ -z "$registry" ]]; then
local host
host="$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")"
if [[ -n "${host:-}" ]]; then
registry="${host}:5000"
fi
fi
if [[ -z "$registry" ]]; then
echo " [WARN] Could not resolve registry host (set LOCAL_REGISTRY or PROLE_K3S_SERVER)." >&2
echo " Images will be pulled at deploy time." >&2
;;
fi
_registry_ref_exists() {
local ref="$1"
local reg="${ref%%/*}"
local rest="${ref#*/}"
local repo="$rest" tag="latest"
case "$rest" in
*@*) repo="${rest%@*}"; tag="${rest#*@}" ;;
*:*) repo="${rest%:*}"; tag="${rest##*:}" ;;
esac
local url="http://${reg}/v2/${repo}/manifests/${tag}"
curl -fsS -I \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json' \
"$url" >/dev/null 2>&1
}
if ! curl -fsS "http://${registry}/v2/" >/dev/null 2>&1; then
echo " [WARN] Registry not reachable at http://${registry}/v2/." >&2
echo " Images will be pulled at deploy time." >&2
;;
fi
local tar_file
while IFS= read -r tar_file; do
[[ -z "$tar_file" ]] && continue
echo " Loading $(basename "$tar_file") ..."
local out
out=$(docker load -i "$tar_file" 2>&1 || true)
if [[ -z "$out" ]]; then
echo " [WARN] docker load produced no output for $(basename "$tar_file")." >&2
continue
fi
# `docker load` can load multiple tags; push each tag to the registry.
printf '%s\n' "$out" | sed -n 's/^Loaded image: //p' | while IFS= read -r img; do
[[ -z "$img" ]] && continue
local remote="${registry}/${img}"
if _registry_ref_exists "$remote"; then
echo " [SKIP] ${remote} already exists; skipping push."
continue
fi
docker tag "$img" "$remote" >/dev/null 2>&1 || true
echo " Pushing ${remote} ..."
if docker push "$remote" >/dev/null 2>&1; then
echo " [OK] Pushed ${remote}"
else
echo " [WARN] Failed to push ${remote}; will pull at deploy time." >&2
fi
done
done <<< "$tar_files"
;;
*)
echo "WARN: Unknown mode '$mode'; skipping docker-import pre-load." >&2
;;
esac
echo "Pre-load complete."
}
case "$ACTION" in
start|update|reload|initialize|restart|repair)
_preload_docker_images
;;
esac
if [ -x "$SCRIPT_DIR/init_service_layer.sh" ]; then
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
"$SCRIPT_DIR/init_service_layer.sh" -n "$NS" -k "$ACTION"
else
"$SCRIPT_DIR/init_service_layer.sh" -n "$NS" "$ACTION"
fi
exit $?
fi
OPENTOFU_NAME=${OPENTOFU_NAME:-opentofu}
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
ARGOCD_SERVER_NAME=${ARGOCD_SERVER_NAME:-argocd-server}
GARAGE_NAME=${GARAGE_NAME:-garage}
OPENTOFU_CONFIGMAP=${OPENTOFU_CONFIGMAP:-opentofu-nginx}
OPENTOFU_SECRET=${OPENTOFU_SECRET:-opentofu-admin}
GARAGE_CONFIGMAP=${GARAGE_CONFIGMAP:-garage-config}
GARAGE_SECRET_NAME=${GARAGE_SECRET_NAME:-garage-secrets}
find_namespaces() {
local kind="$1"
local name="$2"
kubectl get "$kind" -A --no-headers 2>/dev/null | awk -v n="$name" '$2==n {print $1}' | sort -u
}
collect_other_namespaces() {
local target="$1"
shift
local name="$1"
shift
local kinds=("$@")
local found=""
local kind
for kind in "${kinds[@]}"; do
found+=$(find_namespaces "$kind" "$name")
found+=$'\n'
done
printf '%s\n' "$found" | awk -v target="$target" 'NF && $1 != target {print $1}' | sort -u
}
migrate_common_services() {
local old_ns
for old_ns in $(collect_other_namespaces "$NS" "$OPENTOFU_NAME" deployment service); do
echo "Found OpenTofu in namespace '$old_ns'; removing before deploy to '$NS' ..."
if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then
"$SCRIPT_DIR/init_opentofu.sh" -n "$old_ns" stop || true
else
kubectl delete -n "$old_ns" deploy "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true
fi
kubectl delete -n "$old_ns" configmap "$OPENTOFU_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" secret "$OPENTOFU_SECRET" --ignore-not-found >/dev/null 2>&1 || true
done
for old_ns in $(collect_other_namespaces "$ARGOCD_NS" "$ARGOCD_SERVER_NAME" deployment service); do
echo "Found ArgoCD in namespace '$old_ns'; removing before deploy to '$ARGOCD_NS' ..."
if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then
REGISTRY_NAMESPACE="$REGISTRY_NS" "$SCRIPT_DIR/init_registry.sh" -n "$old_ns" stop || true
else
kubectl delete -n "$old_ns" deploy "$ARGOCD_SERVER_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$ARGOCD_SERVER_NAME" --ignore-not-found >/dev/null 2>&1 || true
fi
done
for old_ns in $(collect_other_namespaces "$NS" "$GARAGE_NAME" statefulset service); do
echo "Found Garage in namespace '$old_ns'; removing before deploy to '$NS' ..."
kubectl delete -n "$old_ns" statefulset "$GARAGE_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$GARAGE_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" configmap "$GARAGE_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" secret "$GARAGE_SECRET_NAME" --ignore-not-found >/dev/null 2>&1 || true
done
for old_ns in $(collect_other_namespaces "$NS" "$OPENBAO_NAME" deployment service statefulset); do
echo "Found OpenBao in namespace '$old_ns'; removing before deploy to '$NS' ..."
if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then
"$SCRIPT_DIR/init_openbao.sh" -n "$old_ns" stop || true
else
kubectl delete -n "$old_ns" deploy "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" statefulset "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
kubectl delete -n "$old_ns" svc "$OPENBAO_NAME" --ignore-not-found >/dev/null 2>&1 || true
fi
done
}
echo "Deploying common services (namespace=$NS, action=$ACTION)"
rc=0
case "$ACTION" in
start|update|reload|initialize|restart|status)
migrate_common_services
;;
esac
if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then
ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" "$SCRIPT_DIR/init_opentofu.sh" -n "$NS" "$ACTION" || rc=$?
else
echo "WARN: init_opentofu.sh not found; skipping OpenTofu."
fi
if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then
OPENBAO_NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" \
"$SCRIPT_DIR/init_openbao.sh" -n "$NS" "$ACTION" || rc=$?
else
echo "WARN: init_openbao.sh not found; skipping OpenBao."
fi
if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then
ARGOCD_NAMESPACE="$ARGOCD_NS" REGISTRY_NAMESPACE="$REGISTRY_NS" \
"$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NS" --registry-namespace "$REGISTRY_NS" "$ACTION" || rc=$?
else
echo "WARN: init_registry.sh not found; ArgoCD deploy skipped."
fi
if [ -x "$SCRIPT_DIR/init_garage_store.sh" ]; then
garage_action="$ACTION"
case "$garage_action" in
update|reload|initialize) garage_action="start" ;;
esac
NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" GARAGE_NAMESPACE="$NS" ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" \
"$SCRIPT_DIR/init_garage_store.sh" "$garage_action" || rc=$?
else
echo "WARN: init_garage_store.sh not found; garage deploy skipped."
fi
if [[ "$ENABLE_KERBEROS" == "1" ]]; then
if [ -x "$SCRIPT_DIR/init_kdc.sh" ]; then
kdc_action="$ACTION"
case "$kdc_action" in
stop) kdc_action="cleanup" ;;
status) kdc_action="status" ;;
*) kdc_action="update" ;;
esac
SERVICE_NAMESPACE="$NS" PROLE_KDC_NAMESPACE="$NS" \
"$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$?
else
echo "WARN: init_kdc.sh not found; kerberos deploy skipped."
fi
fi
exit "$rc"