prole/tests/etc/test_init_monitoring.sh

#!/usr/bin/env bash
# Unit test for etc/init_monitoring.sh

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
KNOE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd)
ETC_DIR="$KNOE_HOME/etc"
SCRIPT_UNDER_TEST="$ETC_DIR/init_monitoring.sh"

TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR"' EXIT

MOCK_CALLS_LOG="$TMP_DIR/mock_calls.log"
HELM_STATUS_DIR="$TMP_DIR/helm_status"
NS_EXISTS_FILE="$TMP_DIR/ns_exists"

mkdir -p "$HELM_STATUS_DIR"
echo "pending-install" > "$HELM_STATUS_DIR/kps"
echo "1" > "$NS_EXISTS_FILE"

export MOCK_CALLS_LOG
export HELM_STATUS_DIR
export NS_EXISTS_FILE
export TMP_DIR

write_mock() {
  local name="$1"
  cat > "$TMP_DIR/$name"
  chmod +x "$TMP_DIR/$name"
}

write_mock kubectl <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

echo "Mocked kubectl called with $*" >> "$MOCK_CALLS_LOG"

capture_apply_file() {
  # Capture `kubectl apply -f <file>` payloads for assertions.
  local f="$1"
  [[ -n "$f" && -f "$f" ]] || return 0
  local i=0
  while [[ -f "${TMP_DIR}/kubectl_apply_${i}.yaml" ]]; do
    i=$((i+1))
  done
  cp "$f" "${TMP_DIR}/kubectl_apply_${i}.yaml" 2>/dev/null || true
}

# Handle optional namespace flag
if [[ "${1:-}" == "-n" ]]; then
  shift 2
fi

if [[ "${1:-}" == "get" && "${2:-}" == "namespace" && "${3:-}" == "monitoring" ]]; then
  if [[ "$(cat "$NS_EXISTS_FILE" 2>/dev/null || echo 0)" == "1" ]]; then
    exit 0
  fi
  exit 1
fi

if [[ "${1:-}" == "create" && "${2:-}" == "namespace" && "${3:-}" == "monitoring" ]]; then
  echo 1 > "$NS_EXISTS_FILE"
  exit 0
fi

if [[ "${1:-}" == "delete" && "${2:-}" == "namespace" && "${3:-}" == "monitoring" ]]; then
  echo 0 > "$NS_EXISTS_FILE"
  exit 0
fi

if [[ "${1:-}" == "get" && "${2:-}" == "pvc" && "${3:-}" == "-o" && "${4:-}" == "json" ]]; then
  echo '{"items":[]}'
  exit 0
fi

if [[ "${1:-}" == "get" && "${2:-}" == "nodes" ]]; then
  # Used by monitoring_nodes_available(). Provide at least one node.
  echo "node/k3d-test"
  exit 0
fi

if [[ "${1:-}" == "apply" ]]; then
  # Capture the YAML applied when the script provisions Grafana datasources/dashboards.
  for ((i=1; i<=$#; i++)); do
    if [[ "${!i}" == "-f" ]]; then
      j=$((i+1))
      f="${!j:-}"
      capture_apply_file "$f"
    fi
  done
  exit 0
fi

exit 0
EOF

write_mock helm <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

echo "Mocked helm called with $*" >> "$MOCK_CALLS_LOG"

cmd="${1:-}"
shift || true

status_file_for_release() {
  local release="$1"
  echo "$HELM_STATUS_DIR/$release"
}

case "$cmd" in
  status)
    release="${1:-}"
    st="$(cat "$(status_file_for_release "$release")" 2>/dev/null || true)"
    if [[ -z "$st" ]]; then
      echo '{}'
      exit 0
    fi
    echo '{"info":{"status":"'"$st"'"}}'
    ;;
  history)
    echo "REVISION  UPDATED                   STATUS          CHART"
    echo "1         2026-01-01 00:00:00      unknown         kube-prometheus-stack"
    ;;
  repo)
    exit 0
    ;;
  upgrade)
    # Support: helm upgrade --install <release> <chart> ...
    release=""
    if [[ "${1:-}" == "--install" ]]; then
      release="${2:-}"
    else
      release="${1:-}"
    fi

    # Capture the values file(s) for assertions in tests.
    values_i=0
    for ((i=1; i<=$#; i++)); do
      if [[ "${!i}" == "-f" ]]; then
        j=$((i+1))
        vf="${!j:-}"
        if [[ -n "${vf}" && -f "${vf}" && -n "${release}" ]]; then
          values_i=$((values_i+1))
          cp "${vf}" "${TMP_DIR}/helm_values_${release}_${values_i}.yaml" 2>/dev/null || true
          cp "${vf}" "${TMP_DIR}/helm_values_${release}.yaml" 2>/dev/null || true
        fi
      fi
    done

    if [[ -n "$release" ]]; then
      echo "deployed" > "$(status_file_for_release "$release")"
    fi
    echo "Release upgraded"
    exit 0
    ;;
  uninstall)
    release="${1:-}"
    if [[ -n "$release" ]]; then
      : > "$(status_file_for_release "$release")"
    fi
    echo "Release uninstalled"
    exit 0
    ;;
  *)
    exit 0
    ;;
esac
EOF

write_mock jq <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

echo "Mocked jq called with $*" >> "$MOCK_CALLS_LOG"

query=""
for arg in "$@"; do
  query="$arg"
done

input="$(cat)"

if [[ "$query" == *".info.status"* ]]; then
  echo "$input" | sed -n 's/.*"status"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p'
  exit 0
fi

exit 0
EOF

# Create mocks for other tools that may be on PATH in knoe_cfg.sh
for tool in curl docker k3d ansible-playbook tofu terraform ollama; do
  cat > "$TMP_DIR/$tool" <<EOF
#!/usr/bin/env bash
set -euo pipefail
echo "Mocked $tool called with \$*" >> "$MOCK_CALLS_LOG"
exit 0
EOF
  chmod +x "$TMP_DIR/$tool"
done

# Mock knoe.cfg
mkdir -p "$TMP_DIR/conf"
cat <<C_EOF > "$TMP_DIR/conf/knoe.cfg"
[globals]
knoe.home = $KNOE_HOME
knoe.mode = k3d
[k3s]
server = https://localhost:6443
token = test-token
C_EOF

export PATH="$TMP_DIR:$PATH"
export KNOE_HOME="$KNOE_HOME"
export KNOE_CONF="$TMP_DIR/conf"
export KNOE_SERVICE="$KNOE_HOME"
export NAMESPACE="test-ns"
export PROLE_PASSWD="test-password"

# Force fast timeouts so the test completes quickly
export HELM_WAIT_TIMEOUT=1
export HELM_WAIT_INTERVAL=0.2
export MONITORING_RESET_TIMEOUT=2
export MONITORING_RESET_INTERVAL=0.2
export MONITORING_STABILIZE_TIMEOUT=1
export MONITORING_READY_TIMEOUT=1s
export MONITORING_RESET_RETRIES=1
export HELM_UPGRADE_RETRIES=2
export HELM_RETRY_DELAY=0
export MONITORING_HELM_TIMEOUT=1s

# Run the script (initialize)
set +e
bash "$SCRIPT_UNDER_TEST" initialize > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr"
RC=$?
set -e

if [[ $RC -ne 0 ]]; then
  echo "FAILURE with RC $RC"
  echo "--- stderr ---"
  cat "$TMP_DIR/stderr"
  echo "--- stdout ---"
  cat "$TMP_DIR/stdout"
  exit 1
fi

# Assert that we emitted diagnostics and performed a reset (pending-install timeout path)
if ! grep -q "Monitoring diagnostics" "$TMP_DIR/stdout"; then
  echo "FAILURE: expected diagnostics section in stdout"
  cat "$TMP_DIR/stdout"
  exit 1
fi

if ! grep -q "Resetting monitoring" "$TMP_DIR/stdout"; then
  echo "FAILURE: expected reset section in stdout"
  cat "$TMP_DIR/stdout"
  exit 1
fi

# In k3d mode, we should not pin monitoring to the merlin hostname.
if [[ -f "$TMP_DIR/helm_values_kps.yaml" ]]; then
  if grep -q "kubernetes.io/hostname: merlin.knoe.org" "$TMP_DIR/helm_values_kps.yaml"; then
    echo "FAILURE: did not expect merlin hostname pinning in k3d mode"
    sed -n '1,200p' "$TMP_DIR/helm_values_kps.yaml" || true
    exit 1
  fi

  # Grafana must have datasource provisioning enabled so dashboards can bind to a stable Prometheus UID.
  if ! grep -q "grafana_datasource" "$TMP_DIR/helm_values_kps.yaml"; then
    echo "FAILURE: expected grafana datasource sidecar label in Helm values"
    sed -n '1,220p' "$TMP_DIR/helm_values_kps.yaml" || true
    exit 1
  fi
fi

# Assert we applied a ConfigMap that provisions the Prometheus datasource.
if ! grep -q "name: knoe-grafana-datasource" "$TMP_DIR"/kubectl_apply_*.yaml 2>/dev/null; then
  echo "FAILURE: expected Grafana datasource ConfigMap to be applied"
  ls -la "$TMP_DIR" || true
  for f in "$TMP_DIR"/kubectl_apply_*.yaml; do
    [[ -f "$f" ]] || continue
    echo "--- $f ---"
    sed -n '1,200p' "$f" || true
  done
  exit 1
fi

if ! grep -q "kubectl called with delete namespace monitoring" "$MOCK_CALLS_LOG"; then
  echo "FAILURE: expected namespace deletion"
  cat "$MOCK_CALLS_LOG"
  exit 1
fi

echo "SUCCESS"
exit 0