prole/tests/etc/test_init_monitoring.sh
chrisfu 2c6df07933 Harden monitoring init; improve k3s reset cleanup
- Add diagnostics/reset/retry logic for stuck monitoring Helm installs\n- Add optional cluster-side node/lease deletion during k3s reset via delegated control-plane\n- Bump prole-db image tag/version to 18-123\n- Update monitoring init unit test for reset/diagnostics path
2026-03-15 16:02:15 -07:00

223 lines
4.9 KiB
Bash

#!/usr/bin/env bash
# Unit test for etc/init_monitoring.sh
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd)
ETC_DIR="$PROLE_HOME/etc"
SCRIPT_UNDER_TEST="$ETC_DIR/init_monitoring.sh"
TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR"' EXIT
MOCK_CALLS_LOG="$TMP_DIR/mock_calls.log"
HELM_STATUS_DIR="$TMP_DIR/helm_status"
NS_EXISTS_FILE="$TMP_DIR/ns_exists"
mkdir -p "$HELM_STATUS_DIR"
echo "pending-install" > "$HELM_STATUS_DIR/kps"
echo "1" > "$NS_EXISTS_FILE"
export MOCK_CALLS_LOG
export HELM_STATUS_DIR
export NS_EXISTS_FILE
write_mock() {
local name="$1"
cat > "$TMP_DIR/$name"
chmod +x "$TMP_DIR/$name"
}
write_mock kubectl <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
echo "Mocked kubectl called with $*" >> "$MOCK_CALLS_LOG"
# Handle optional namespace flag
if [[ "${1:-}" == "-n" ]]; then
shift 2
fi
if [[ "${1:-}" == "get" && "${2:-}" == "namespace" && "${3:-}" == "monitoring" ]]; then
if [[ "$(cat "$NS_EXISTS_FILE" 2>/dev/null || echo 0)" == "1" ]]; then
exit 0
fi
exit 1
fi
if [[ "${1:-}" == "create" && "${2:-}" == "namespace" && "${3:-}" == "monitoring" ]]; then
echo 1 > "$NS_EXISTS_FILE"
exit 0
fi
if [[ "${1:-}" == "delete" && "${2:-}" == "namespace" && "${3:-}" == "monitoring" ]]; then
echo 0 > "$NS_EXISTS_FILE"
exit 0
fi
if [[ "${1:-}" == "get" && "${2:-}" == "pvc" && "${3:-}" == "-o" && "${4:-}" == "json" ]]; then
echo '{"items":[]}'
exit 0
fi
exit 0
EOF
write_mock helm <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
echo "Mocked helm called with $*" >> "$MOCK_CALLS_LOG"
cmd="${1:-}"
shift || true
status_file_for_release() {
local release="$1"
echo "$HELM_STATUS_DIR/$release"
}
case "$cmd" in
status)
release="${1:-}"
st="$(cat "$(status_file_for_release "$release")" 2>/dev/null || true)"
if [[ -z "$st" ]]; then
echo '{}'
exit 0
fi
echo '{"info":{"status":"'"$st"'"}}'
;;
history)
echo "REVISION UPDATED STATUS CHART"
echo "1 2026-01-01 00:00:00 unknown kube-prometheus-stack"
;;
repo)
exit 0
;;
upgrade)
# Support: helm upgrade --install <release> <chart> ...
release=""
if [[ "${1:-}" == "--install" ]]; then
release="${2:-}"
else
release="${1:-}"
fi
if [[ -n "$release" ]]; then
echo "deployed" > "$(status_file_for_release "$release")"
fi
echo "Release upgraded"
exit 0
;;
uninstall)
release="${1:-}"
if [[ -n "$release" ]]; then
: > "$(status_file_for_release "$release")"
fi
echo "Release uninstalled"
exit 0
;;
*)
exit 0
;;
esac
EOF
write_mock jq <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
echo "Mocked jq called with $*" >> "$MOCK_CALLS_LOG"
query=""
for arg in "$@"; do
query="$arg"
done
input="$(cat)"
if [[ "$query" == *".info.status"* ]]; then
echo "$input" | sed -n 's/.*"status"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p'
exit 0
fi
exit 0
EOF
# Create mocks for other tools that may be on PATH in prole_cfg.sh
for tool in curl docker k3d ansible-playbook tofu terraform ollama; do
cat > "$TMP_DIR/$tool" <<EOF
#!/usr/bin/env bash
set -euo pipefail
echo "Mocked $tool called with \$*" >> "$MOCK_CALLS_LOG"
exit 0
EOF
chmod +x "$TMP_DIR/$tool"
done
# Mock prole.cfg
mkdir -p "$TMP_DIR/conf"
cat <<C_EOF > "$TMP_DIR/conf/prole.cfg"
[globals]
prole.home = $PROLE_HOME
prole.mode = k3d
[k3s]
server = https://localhost:6443
token = test-token
C_EOF
export PATH="$TMP_DIR:$PATH"
export PROLE_HOME="$PROLE_HOME"
export PROLE_CONF="$TMP_DIR/conf"
export PROLE_SERVICE="$PROLE_HOME"
export NAMESPACE="test-ns"
export PROLE_PASSWD="test-password"
# Force fast timeouts so the test completes quickly
export HELM_WAIT_TIMEOUT=1
export HELM_WAIT_INTERVAL=0.2
export MONITORING_RESET_TIMEOUT=2
export MONITORING_RESET_INTERVAL=0.2
export MONITORING_STABILIZE_TIMEOUT=1
export MONITORING_READY_TIMEOUT=1s
export MONITORING_RESET_RETRIES=1
export HELM_UPGRADE_RETRIES=2
export HELM_RETRY_DELAY=0
export MONITORING_HELM_TIMEOUT=1s
# Run the script (initialize)
set +e
bash "$SCRIPT_UNDER_TEST" initialize > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr"
RC=$?
set -e
if [[ $RC -ne 0 ]]; then
echo "FAILURE with RC $RC"
echo "--- stderr ---"
cat "$TMP_DIR/stderr"
echo "--- stdout ---"
cat "$TMP_DIR/stdout"
exit 1
fi
# Assert that we emitted diagnostics and performed a reset (pending-install timeout path)
if ! grep -q "Monitoring diagnostics" "$TMP_DIR/stdout"; then
echo "FAILURE: expected diagnostics section in stdout"
cat "$TMP_DIR/stdout"
exit 1
fi
if ! grep -q "Resetting monitoring" "$TMP_DIR/stdout"; then
echo "FAILURE: expected reset section in stdout"
cat "$TMP_DIR/stdout"
exit 1
fi
if ! grep -q "kubectl called with delete namespace monitoring" "$MOCK_CALLS_LOG"; then
echo "FAILURE: expected namespace deletion"
cat "$MOCK_CALLS_LOG"
exit 1
fi
echo "SUCCESS"
exit 0