prole/etc/init_monitoring.sh

#!/usr/bin/env bash

set -euo pipefail

# init_monitoring.sh
# Purpose:
# - Configure k3d environment for monitoring (Prometheus and Grafana)
# - Setup kube-prometheus-stack and CNPG prometheus rules

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
PROLE_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)

# Load environment and config via prole_cfg.sh
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
  prole_set_mode "${2:-}"
  shift 2
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
  prole_set_mode "${1#*=}"
  shift
fi

if [[ -z "${PROLE_SERVICE:-}" ]]; then
  echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2
  exit 1
fi

GRAFANA_RELEASE="grafana"
LEGACY_GRAFANA_RELEASE="grafana-prole"

ts() {
  date "+%Y-%m-%d %H:%M:%S"
}

log() {
  echo "==> [$(ts)] $*"
}

err() {
  echo "ERROR: [$(ts)] $*" >&2
}

section() {
  echo "==> [$(ts)] --- $* ---"
}

ensure_tools() {
  for t in helm kubectl curl jq; do
    command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; }
  done
}

ensure_namespace() {
  local ns="$1"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    log "Creating namespace '$ns' ..."
    kubectl create namespace "$ns" >/dev/null 2>&1 || true
  fi
}

storage_class_exists() {
  local sc="$1"
  [[ -n "$sc" ]] || return 1
  kubectl get storageclass "$sc" >/dev/null 2>&1
}

default_storage_class() {
  kubectl get storageclass -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -n1
}

choose_monitoring_storage_class() {
  if [[ -n "${MONITORING_STORAGE_CLASS:-}" ]] && storage_class_exists "$MONITORING_STORAGE_CLASS"; then
    echo "$MONITORING_STORAGE_CLASS"
    return 0
  fi
  if storage_class_exists "merlin-local-iscsi"; then
    echo "merlin-local-iscsi"
    return 0
  fi
  # Backward compatibility (older runs created prole-monitoring-<volumeId>)
  if storage_class_exists "prole-monitoring-d004"; then
    echo "prole-monitoring-d004"
    return 0
  fi
  local default_sc
  default_sc=$(default_storage_class)
  if [[ -n "$default_sc" ]]; then
    echo "$default_sc"
    return 0
  fi
  if storage_class_exists "local-path"; then
    echo "local-path"
    return 0
  fi
  echo ""
}

monitoring_nodes_available() {
  kubectl get nodes -l "prole.org/node-role=general" -o name 2>/dev/null | grep -q .
}

resolve_monitoring_primary_node() {
  local node
  if [[ -n "${MONITORING_PRIMARY_NODE:-}" ]]; then
    printf '%s' "${MONITORING_PRIMARY_NODE}"
    return 0
  fi
  if kubectl get node merlin.prole.org >/dev/null 2>&1; then
    printf '%s' "merlin.prole.org"
    return 0
  fi
  node=$(kubectl get nodes -l "prole.org/node-role=general" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -n "${node:-}" ]]; then
    printf '%s' "$node"
    return 0
  fi
  node=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  printf '%s' "$node"
}

prepare_k3s_monitoring_data_dirs() {
  local node="$1"
  # NOTE: Historically this came from prole.cfg. For now we pin it to the
  # monitoring deployment default and allow overrides via env var.
  local base="${PROLE_MONITORING_DATA_DIR:-/prole/d004}"
  base="${base%/}"
  if [[ -z "${node:-}" ]]; then
    err "Could not resolve a monitoring node to prepare storage on"
    return 1
  fi

  local parent base_name
  parent=$(dirname "$base")
  base_name=$(basename "$base")

  section "Preparing monitoring data directories on node '${node}'"
  log "Checking base directory exists on node: ${base}"

  local job_ns job_name timeout image
  job_ns="${MONITORING_DIRPREP_NAMESPACE:-kube-system}"
  job_name="${MONITORING_DIRPREP_JOB_NAME:-prole-monitoring-dirprep}"
  timeout="${MONITORING_DIRPREP_TIMEOUT:-60s}"
  image="${MONITORING_DIRPREP_IMAGE:-alpine:3.20}"

  kubectl -n "$job_ns" delete job "$job_name" --ignore-not-found >/dev/null 2>&1 || true

  cat <<EOF | kubectl apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
  name: ${job_name}
  namespace: ${job_ns}
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      nodeName: ${node}
      containers:
        - name: dirprep
          image: ${image}
          securityContext:
            runAsUser: 0
            runAsGroup: 0
          command: ["/bin/sh", "-ec"]
          args:
            - |
              BASE_DIR="/host-parent/${base_name}"
              echo "[dirprep] node=${node} base=${base}"
              if [ ! -d "\${BASE_DIR}" ]; then
                echo "[dirprep][ERROR] Base monitoring directory does not exist on node: ${base}" >&2
                echo "[dirprep][ERROR] Ensure the mount exists and is mounted on node '${node}'." >&2
                exit 2
              fi
              for d in prometheus alertmanager grafana; do
                mkdir -p "\${BASE_DIR}/\${d}"
                chmod 0777 "\${BASE_DIR}/\${d}" 2>/dev/null || true
              done
              echo "[dirprep][OK] Prepared ${base}/{prometheus,alertmanager,grafana}"
          volumeMounts:
            - name: host-parent
              mountPath: /host-parent
      volumes:
        - name: host-parent
          hostPath:
            path: ${parent}
            type: Directory
EOF

  if ! kubectl -n "$job_ns" wait --for=condition=complete "job/${job_name}" --timeout="$timeout" >/dev/null 2>&1; then
    err "Monitoring directory preparation job did not complete successfully (job=${job_name} ns=${job_ns})."
    set +e
    kubectl -n "$job_ns" logs "job/${job_name}" 2>/dev/null || true
    kubectl -n "$job_ns" describe "job/${job_name}" 2>/dev/null || true
    kubectl -n "$job_ns" get pod -l job-name="${job_name}" -o wide 2>/dev/null || true
    set -e
    return 1
  fi

  set +e
  kubectl -n "$job_ns" logs "job/${job_name}" 2>/dev/null || true
  kubectl -n "$job_ns" delete job "$job_name" --ignore-not-found >/dev/null 2>&1 || true
  set -e

  log "[OK] Monitoring storage paths prepared under ${base}"
}

apply_k3s_monitoring_local_pvs() {
  local mode="$1"
  [[ "$mode" == "k3s" ]] || return 0

  local node
  node=$(resolve_monitoring_primary_node)
  if [[ -z "${node:-}" ]]; then
    err "No Kubernetes node found; cannot prepare k3s monitoring local PV directories"
    return 1
  fi

  local base volume_id sc_name
  base="${PROLE_MONITORING_DATA_DIR:-/prole/d004}"
  base="${base%/}"
  volume_id=$(basename "$base")
  if [[ -z "${volume_id:-}" || "$volume_id" == "/" || "$volume_id" == "." ]]; then
    err "Invalid PROLE_MONITORING_DATA_DIR: '$base'"
    return 1
  fi
  sc_name="merlin-local-iscsi"

  prepare_k3s_monitoring_data_dirs "$node"

  section "Applying monitoring StorageClass and PersistentVolumes"
  log "Applying monitoring StorageClass '${sc_name}' ..."
  cat <<EOF | kubectl apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ${sc_name}
provisioner: kubernetes.io/no-provisioner
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
EOF

  log "Applying monitoring PersistentVolumes (local paths under ${base}) ..."
  cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolume
metadata:
  name: merlin-local-iscsi-${volume_id}-prometheus
  labels:
    prole.storage/role: prometheus
    prole.storage/volume: ${volume_id}
spec:
  capacity:
    storage: 30Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteOnce
  storageClassName: ${sc_name}
  persistentVolumeReclaimPolicy: Retain
  local:
    path: ${base}/prometheus
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - ${node}
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: merlin-local-iscsi-${volume_id}-alertmanager
  labels:
    prole.storage/role: alertmanager
    prole.storage/volume: ${volume_id}
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteOnce
  storageClassName: ${sc_name}
  persistentVolumeReclaimPolicy: Retain
  local:
    path: ${base}/alertmanager
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - ${node}
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: merlin-local-iscsi-${volume_id}-grafana
  labels:
    prole.storage/role: grafana
    prole.storage/volume: ${volume_id}
spec:
  capacity:
    storage: 10Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteOnce
  storageClassName: ${sc_name}
  persistentVolumeReclaimPolicy: Retain
  local:
    path: ${base}/grafana
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - ${node}
EOF

  # Default to the local-PV storageClass on k3s unless explicitly overridden.
  if [[ -z "${MONITORING_STORAGE_CLASS:-}" ]]; then
    MONITORING_STORAGE_CLASS="${sc_name}"
    export MONITORING_STORAGE_CLASS
  fi
}

render_node_selector() {
  local indent="$1"
  cat <<EOF
${indent}nodeSelector:
${indent}  prole.org/node-role: general
EOF
}

render_primary_node_selector() {
  local indent="$1"
  local mode
  mode="${PROLE_MODE:-${DEPLOYMENT_MODE:-}}"
  [[ "${mode}" == "k3s" ]] || { render_node_selector "$indent"; return 0; }

  local node
  node=$(resolve_monitoring_primary_node)
  if [[ -n "${node:-}" ]]; then
    cat <<EOF
${indent}nodeSelector:
${indent}  kubernetes.io/hostname: ${node}
EOF
  else
    render_node_selector "$indent"
  fi
}

render_tolerations() {
  # Legacy: tolerations used when monitoring nodes were tainted.
  # Monitoring now requires `prole.org/node-role=general` instead.
  return 0
}

render_node_affinity() {
  local indent="$1"
  cat <<EOF
${indent}affinity:
${indent}  nodeAffinity:
${indent}    requiredDuringSchedulingIgnoredDuringExecution:
${indent}      nodeSelectorTerms:
${indent}        - matchExpressions:
${indent}            - key: prole.org/node-role
${indent}              operator: In
${indent}              values:
${indent}                - general
EOF
}

render_primary_node_affinity() {
  local indent="$1"
  local mode
  mode="${PROLE_MODE:-${DEPLOYMENT_MODE:-}}"
  [[ "${mode}" == "k3s" ]] || { render_node_affinity "$indent"; return 0; }

  local node
  node=$(resolve_monitoring_primary_node)
  if [[ -n "${node:-}" ]]; then
    cat <<EOF
${indent}affinity:
${indent}  nodeAffinity:
${indent}    requiredDuringSchedulingIgnoredDuringExecution:
${indent}      nodeSelectorTerms:
${indent}        - matchExpressions:
${indent}            - key: kubernetes.io/hostname
${indent}              operator: In
${indent}              values:
${indent}                - ${node}
EOF
  else
    render_node_affinity "$indent"
  fi
}

render_prometheus_storage() {
  local indent="$1"
  local size="$2"
  if [[ -n "${MONITORING_STORAGE_CLASS_SELECTED:-}" ]]; then
    cat <<EOF
${indent}storageSpec:
${indent}  volumeClaimTemplate:
${indent}    spec:
${indent}      storageClassName: ${MONITORING_STORAGE_CLASS_SELECTED}
${indent}      accessModes: ["ReadWriteOnce"]
${indent}      resources:
${indent}        requests:
${indent}          storage: ${size}
EOF
  fi
}

render_alertmanager_storage() {
  local indent="$1"
  local size="$2"
  if [[ -n "${MONITORING_STORAGE_CLASS_SELECTED:-}" ]]; then
    cat <<EOF
${indent}storage:
${indent}  volumeClaimTemplate:
${indent}    spec:
${indent}      storageClassName: ${MONITORING_STORAGE_CLASS_SELECTED}
${indent}      accessModes: ["ReadWriteOnce"]
${indent}      resources:
${indent}        requests:
${indent}          storage: ${size}
EOF
  fi
}

render_grafana_persistence() {
  local indent="$1"
  local size="$2"
  if [[ -n "${MONITORING_STORAGE_CLASS_SELECTED:-}" ]]; then
    cat <<EOF
${indent}persistence:
${indent}  enabled: true
${indent}  storageClassName: ${MONITORING_STORAGE_CLASS_SELECTED}
${indent}  accessModes: ["ReadWriteOnce"]
${indent}  size: ${size}
EOF
  else
    cat <<EOF
${indent}persistence:
${indent}  enabled: false
EOF
  fi
}

cleanup_pending_pvcs() {
  local ns="$1"
  local expected_sc="$2"
  local pending
  pending=$(kubectl -n "$ns" get pvc -o json | jq -r '.items[] | select(.status.phase=="Pending") | "\(.metadata.name)|\(.spec.storageClassName // "")"' || true)
  while IFS='|' read -r pvc_name pvc_sc; do
    [[ -n "$pvc_name" ]] || continue
    if [[ -n "$pvc_sc" ]] && ! storage_class_exists "$pvc_sc"; then
      log "Deleting Pending PVC '$pvc_name' with missing storageClass '$pvc_sc' ..."
      kubectl -n "$ns" delete pvc "$pvc_name" >/dev/null 2>&1 || true
      continue
    fi
    if [[ -n "$expected_sc" && -n "$pvc_sc" && "$pvc_sc" != "$expected_sc" ]]; then
      log "Deleting Pending PVC '$pvc_name' with storageClass '$pvc_sc' (expected '$expected_sc') ..."
      kubectl -n "$ns" delete pvc "$pvc_name" >/dev/null 2>&1 || true
    fi
  done <<< "$pending"
}

apply_grafana_dashboard() {
  local ns="$1"
  local dashboard_path=""
  if [[ -n "${PROLE_GRAFANA_DASHBOARD_PATH:-}" ]]; then
    dashboard_path="$PROLE_GRAFANA_DASHBOARD_PATH"
  elif [[ -n "${PROLE_SERVICE:-}" && -f "$PROLE_SERVICE/prole-db/grafana-dashboard.json" ]]; then
    dashboard_path="$PROLE_SERVICE/prole-db/grafana-dashboard.json"
  elif [[ -f "$PROLE_ROOT/prole-db/grafana-dashboard.json" ]]; then
    dashboard_path="$PROLE_ROOT/prole-db/grafana-dashboard.json"
  fi

  if [[ -z "$dashboard_path" || ! -f "$dashboard_path" ]]; then
    log "Grafana dashboard not found; skipping."
    return 0
  fi

  local tmp
  tmp=$(mktemp)
  sed 's/\\${DS_PROMETHEUS}/prometheus/g' "$dashboard_path" > "$tmp"

  kubectl -n "$ns" delete configmap prole-db-grafana-dashboard >/dev/null 2>&1 || true
  kubectl -n "$ns" create configmap prole-db-grafana-dashboard --from-file=prole-db.json="$tmp" >/dev/null
  kubectl -n "$ns" label configmap prole-db-grafana-dashboard grafana_dashboard=1 --overwrite >/dev/null
  rm -f "$tmp"
}

helm_release_status() {
  local release="$1"
  local ns="$2"
  helm status "$release" -n "$ns" -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || true
}

wait_for_helm_release() {
  local release="$1"
  local ns="$2"
  local timeout="${HELM_WAIT_TIMEOUT:-${MONITORING_PENDING_TIMEOUT:-600}}"
  local interval="${HELM_WAIT_INTERVAL:-${MONITORING_PENDING_INTERVAL:-5}}"
  local start
  start=$(date +%s)

  while true; do
    local status
    status=$(helm_release_status "$release" "$ns")
    if [[ -z "$status" || "$status" == "null" ]]; then
      return 0
    fi
    case "$status" in
      pending-*)
        if (( $(date +%s) - start > timeout )); then
          err "Timed out waiting for Helm release '$release' in '$ns' (status=$status)."
          return 1
        fi
        log "Helm release '$release' is $status; waiting... (elapsed=$(( $(date +%s) - start ))s, timeout=${timeout}s)"
        sleep "$interval"
        ;;
      *)
        return 0
        ;;
    esac
  done
}

wait_for_monitoring_resources() {
  local ns="$1"
  local timeout="${MONITORING_READY_TIMEOUT:-600s}"

  section "Waiting for Prometheus stack resources to settle (timeout=$timeout)"
  set +e
  kubectl -n "$ns" rollout status deployment/kps-kube-prometheus-stack-operator --timeout="$timeout" >/dev/null 2>&1
  kubectl -n "$ns" rollout status deployment/kps-kube-state-metrics --timeout="$timeout" >/dev/null 2>&1
  kubectl -n "$ns" rollout status statefulset/prometheus-kps-kube-prometheus-stack-prometheus --timeout="$timeout" >/dev/null 2>&1
  kubectl -n "$ns" rollout status statefulset/alertmanager-kps-kube-prometheus-stack-alertmanager --timeout="$timeout" >/dev/null 2>&1
  set -e
}

collect_monitoring_diagnostics() {
  local ns="$1"
  local release="$2"

  section "Monitoring diagnostics (namespace=$ns release=$release)"
  set +e
  echo "[diag] helm status ${release} -n ${ns}" >&2
  helm status "$release" -n "$ns" >&2
  echo "[diag] helm history ${release} -n ${ns}" >&2
  helm history "$release" -n "$ns" >&2

  echo "[diag] kubectl -n ${ns} get all" >&2
  kubectl -n "$ns" get all -o wide >&2
  echo "[diag] kubectl -n ${ns} get pvc" >&2
  kubectl -n "$ns" get pvc -o wide >&2
  echo "[diag] kubectl -n ${ns} get events (last 100)" >&2
  kubectl -n "$ns" get events --sort-by=.lastTimestamp 2>/dev/null | tail -n 100 >&2

  echo "[diag] kubectl -n ${ns} get pods (not Running)" >&2
  kubectl -n "$ns" get pods --field-selector=status.phase!=Running -o wide >&2

  # Best-effort describe of non-Running pods (helps pinpoint scheduling/PVC/image issues)
  local pod_names
  pod_names=$(kubectl -n "$ns" get pods --no-headers 2>/dev/null | awk '$3 != "Running" {print $1}' || true)
  local p
  for p in $pod_names; do
    echo "[diag] kubectl -n ${ns} describe pod/${p}" >&2
    kubectl -n "$ns" describe pod "$p" >&2
  done

  local pending_pvcs
  pending_pvcs=$(kubectl -n "$ns" get pvc --no-headers 2>/dev/null | awk '$2 == "Pending" {print $1}' || true)
  local pvc
  for pvc in $pending_pvcs; do
    echo "[diag] kubectl -n ${ns} describe pvc/${pvc}" >&2
    kubectl -n "$ns" describe pvc "$pvc" >&2
  done
  set -e
}

wait_for_namespace_deleted() {
  local ns="$1"
  local timeout="${MONITORING_RESET_TIMEOUT:-300}"
  local interval="${MONITORING_RESET_INTERVAL:-2}"
  local start
  start=$(date +%s)

  while true; do
    if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
      return 0
    fi
    if (( $(date +%s) - start > timeout )); then
      err "Timed out waiting for namespace '$ns' deletion (timeout=${timeout}s)."
      return 1
    fi
    log "Waiting for namespace '$ns' to terminate... (elapsed=$(( $(date +%s) - start ))s, timeout=${timeout}s)"
    sleep "$interval"
  done
}

stabilize_cluster_after_reset() {
  local timeout="${MONITORING_STABILIZE_TIMEOUT:-120}"
  section "Waiting for cluster to stabilize (timeout=${timeout}s)"
  set +e
  kubectl wait --for=condition=Ready nodes --all --timeout="${timeout}s" >/dev/null 2>&1
  set -e
}

reset_monitoring_namespace() {
  local ns="$1"
  local release="$2"
  local helm_timeout="${MONITORING_HELM_TIMEOUT:-${HELM_TIMEOUT:-10m}}"

  section "Resetting monitoring (namespace=$ns release=$release)"
  set +e
  if helm status "$release" -n "$ns" >/dev/null 2>&1; then
    log "Uninstalling Helm release '$release' from namespace '$ns' (timeout=$helm_timeout)..."
    helm uninstall "$release" -n "$ns" --wait --timeout "$helm_timeout" >/dev/null 2>&1 || true
  fi
  log "Deleting namespace '$ns' ..."
  kubectl delete namespace "$ns" >/dev/null 2>&1 || true
  set -e

  wait_for_namespace_deleted "$ns" || true
  stabilize_cluster_after_reset || true
  ensure_namespace "$ns"
}

reset_monitoring_release() {
  local ns="$1"
  local release="$2"
  local helm_timeout="${MONITORING_HELM_TIMEOUT:-${HELM_TIMEOUT:-10m}}"

  section "Resetting Helm release only (namespace=$ns release=$release)"
  set +e
  if helm status "$release" -n "$ns" >/dev/null 2>&1; then
    log "Uninstalling Helm release '$release' from namespace '$ns' (timeout=$helm_timeout)..."
    helm uninstall "$release" -n "$ns" --wait --timeout "$helm_timeout" >/dev/null 2>&1 || true
  fi
  set -e
}

helm_upgrade_with_retry() {
  local release="$1"
  local ns="$2"
  local chart="$3"
  shift 3
  local attempts="${HELM_UPGRADE_RETRIES:-5}"
  local delay="${HELM_RETRY_DELAY:-5}"
  local helm_timeout="${MONITORING_HELM_TIMEOUT:-${HELM_TIMEOUT:-10m}}"
  local max_resets="${MONITORING_RESET_RETRIES:-1}"
  local reset_scope="${MONITORING_RESET_SCOPE:-namespace}"
  local reset_count=0
  local attempt out rc
  local delay_current
  delay_current="$delay"

  for ((attempt=1; attempt<=attempts; attempt++)); do
    if ! wait_for_helm_release "$release" "$ns"; then
      collect_monitoring_diagnostics "$ns" "$release" || true
      if (( reset_count < max_resets )); then
        reset_count=$(( reset_count + 1 ))
        if [[ "$reset_scope" == "release" ]]; then
          reset_monitoring_release "$ns" "$release" || true
        else
          reset_monitoring_namespace "$ns" "$release" || true
        fi
        continue
      fi
      err "Helm release '$release' remained in a pending state and reset retries are exhausted ($reset_count/$max_resets)."
      return 1
    fi
    set +e
    out=$(helm upgrade --install "$release" "$chart" --namespace "$ns" --wait --timeout "$helm_timeout" "$@" 2>&1)
    rc=$?
    set -e
    if [[ $rc -eq 0 ]]; then
      printf '%s\n' "$out"
      return 0
    fi
    if echo "$out" | grep -q "another operation (install/upgrade/rollback) is in progress"; then
      log "Helm release '$release' is busy; retrying ($attempt/$attempts)..."
      wait_for_helm_release "$release" "$ns" || true
      sleep "$delay_current"
      if (( delay_current < 60 )); then
        delay_current=$(( delay_current * 2 ))
        if (( delay_current > 60 )); then
          delay_current=60
        fi
      fi
      continue
    fi

    # Helm/k8s timeouts: diagnose + reset + retry.
    if echo "$out" | grep -Eq "timed out waiting for the condition|context deadline exceeded"; then
      err "Helm upgrade timed out (timeout=$helm_timeout)."
      echo "$out" >&2
      collect_monitoring_diagnostics "$ns" "$release" || true
      if (( reset_count < max_resets )); then
        reset_count=$(( reset_count + 1 ))
        if [[ "$reset_scope" == "release" ]]; then
          reset_monitoring_release "$ns" "$release" || true
        else
          reset_monitoring_namespace "$ns" "$release" || true
        fi
        continue
      fi
      return 1
    fi

    # Transient Kubernetes API/transport failures (common on small/loaded k3s clusters during CRD installs)
    if echo "$out" | grep -Eq "stream error: stream ID|INTERNAL_ERROR|the server was unable to return a response in the time allotted|i/o timeout|EOF"; then
      log "Transient error during Helm upgrade; retrying ($attempt/$attempts)..."
      echo "$out" >&2
      sleep "$delay_current"
      if (( delay_current < 60 )); then
        delay_current=$(( delay_current * 2 ))
        if (( delay_current > 60 )); then
          delay_current=60
        fi
      fi
      continue
    fi

    echo "$out" >&2
    return "$rc"
  done

  err "Helm upgrade failed after $attempts attempts for release '$release' in '$ns'."
  return 1
}

openbao_url() {
  if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
    echo "$PROLE_OPENBAO_URL"
    return 0
  fi
  if prole_is_in_cluster; then
    echo "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE:-default}}.svc.cluster.local:8200"
    return 0
  fi

  # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
  # (never via localhost or kubectl port-forward).
  if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
    if command -v _prole_host_from_url >/dev/null 2>&1; then
      local host
      host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
      if [[ -n "${host:-}" ]]; then
        echo "http://${host}:8200"
        return 0
      fi
    fi
    echo ""
    return 0
  fi

  if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
    echo "http://127.0.0.1:8200"
    return 0
  fi
  echo ""
  return 0
}

openbao_token() {
  if [[ -f "$PROLE_SERVICE/secrets/openbao-root-token" ]]; then
    cat "$PROLE_SERVICE/secrets/openbao-root-token"
  else
    echo "${OPENBAO_ROOT_TOKEN:-}"
  fi
}

fetch_openbao_secret() {
  local path="$1"
  local key="$2"
  local token url
  token=$(openbao_token)
  url=$(openbao_url)
  if [[ -z "$token" || -z "$url" ]]; then
    echo ""
    return 0
  fi
  curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
}

resolve_grafana_password() {
  if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then
    local fetched
    fetched=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/monitoring" "grafana_admin_password")
    if [[ -n "$fetched" && "$fetched" != "null" ]]; then
      GRAFANA_ADMIN_PASSWORD="$fetched"
    fi
  fi
  if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then
    local db_pw="${DB_PASSWORD:-}"
    if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
      local fetched_db
      fetched_db=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/db" "password")
      if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
        db_pw="$fetched_db"
      fi
    fi
    if [[ -n "$db_pw" ]]; then
      GRAFANA_ADMIN_PASSWORD="$db_pw"
    fi
  fi
  # Strip any trailing newlines/carriage-returns that may have crept in via
  # prole.cfg parsing, file reads, or shell substitution edge-cases.
  local _pw
  _pw="${GRAFANA_ADMIN_PASSWORD:-}"
  _pw="${_pw//$'\n'/}"
  _pw="${_pw//$'\r'/}"
  GRAFANA_ADMIN_PASSWORD="$_pw"
}

write_grafana_password_to_openbao() {
  local token url
  token=$(openbao_token)
  url=$(openbao_url)
  if [[ -z "$token" || -z "$url" || -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
    return 0
  fi
  local _clean_grafana_pw
  _clean_grafana_pw="${GRAFANA_ADMIN_PASSWORD//$'\n'/}"
  _clean_grafana_pw="${_clean_grafana_pw//$'\r'/}"
  curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
    -X POST "$url/v1/kv/data/prole/${NAMESPACE:-default}/monitoring" \
    -d "{\"data\":{\"grafana_admin_password\":\"$_clean_grafana_pw\"}}" >/dev/null || true
}

cleanup_grafana_rbac_conflicts() {
  local release="$GRAFANA_RELEASE"
  local ns="$NAMESPACE"
  local cr="${release}-clusterrole"
  local crb="${release}-clusterrolebinding"
  local rel_ns rel_name

  if kubectl get clusterrole "$cr" >/dev/null 2>&1; then
    rel_ns=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true)
    rel_name=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true)
    if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then
      log "Detected existing ClusterRole '$cr' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'."
      if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then
        log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..."
        helm uninstall "$rel_name" -n "$rel_ns" || true
      fi
      if kubectl get clusterrole "$cr" >/dev/null 2>&1; then
        log "Deleting orphaned ClusterRole '$cr' ..."
        kubectl delete clusterrole "$cr" || true
      fi
    fi
  fi

  if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then
    rel_ns=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true)
    rel_name=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true)
    if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then
      log "Detected existing ClusterRoleBinding '$crb' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'."
      if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then
        log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..."
        helm uninstall "$rel_name" -n "$rel_ns" || true
      fi
      if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then
        log "Deleting orphaned ClusterRoleBinding '$crb' ..."
        kubectl delete clusterrolebinding "$crb" || true
      fi
    fi
  fi
}

cleanup_legacy_grafana_release() {
  local ns="$NAMESPACE"
  if helm status "$LEGACY_GRAFANA_RELEASE" -n "$ns" >/dev/null 2>&1; then
    log "Uninstalling legacy Grafana release '$LEGACY_GRAFANA_RELEASE' from '$ns' ..."
    helm uninstall "$LEGACY_GRAFANA_RELEASE" -n "$ns" || true
  fi
}

install_monitoring() {
  ensure_tools
  local monitoring_ns="monitoring"
  ensure_namespace "$monitoring_ns"

  section "Pre-check monitoring state"
  log "Helm release status: kps=$(helm_release_status kps "$monitoring_ns"), grafana=$(helm_release_status "$GRAFANA_RELEASE" "$monitoring_ns")"

  local mode
  mode="${PROLE_MODE:-${DEPLOYMENT_MODE:-}}"

  # For k3s service clusters using local PVs, ensure the backing host directories
  # exist before applying StorageClass/PV manifests.
  apply_k3s_monitoring_local_pvs "$mode"

  if ! monitoring_nodes_available; then
    err "No nodes labeled prole.org/node-role=general; monitoring requires general nodes."
    return 2
  fi

  MONITORING_STORAGE_CLASS_SELECTED=$(choose_monitoring_storage_class)
  if [[ -n "$MONITORING_STORAGE_CLASS_SELECTED" ]]; then
    log "Using storageClass '$MONITORING_STORAGE_CLASS_SELECTED' for monitoring PVCs."
  else
    log "No storageClass detected; disabling persistence for Grafana and skipping Prometheus/Alertmanager storage."
  fi

  cleanup_pending_pvcs "$monitoring_ns" "$MONITORING_STORAGE_CLASS_SELECTED"

  section "Installing Prometheus stack (without Grafana)"
  helm repo add prometheus-community https://prometheus-community.github.io/helm-charts || true
  helm repo update prometheus-community || true

  local values_file
  values_file=$(mktemp)
  cat > "$values_file" <<EOF
prometheus:
  prometheusSpec:
$(render_primary_node_selector "    ")
$(render_primary_node_affinity "    ")
$(render_prometheus_storage "    " "30Gi")
    additionalScrapeConfigs:
      - job_name: 'kubernetes-pods'
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - action: keep
            source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
            regex: true
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
            action: replace
            target_label: __address__
            regex: (.*?):\d+;(\d+)
            replacement: \$1:\$2
      - job_name: 'cnpg-metrics'
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - action: keep
            source_labels: [__meta_kubernetes_pod_label_cnpg_io_cluster]
            regex: .+
          - action: keep
            source_labels: [__meta_kubernetes_pod_phase]
            regex: Running
          - action: replace
            source_labels: [__meta_kubernetes_pod_ip]
            target_label: __address__
            replacement: \$1:9187
          - action: replace
            source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - action: replace
            source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - action: replace
            source_labels: [__meta_kubernetes_pod_label_cnpg_io_cluster]
            target_label: cluster

grafana:
  enabled: false

alertmanager:
  alertmanagerSpec:
$(render_primary_node_selector "    ")
$(render_primary_node_affinity "    ")
$(render_alertmanager_storage "    " "5Gi")

prometheusOperator:
$(render_node_selector "  ")
$(render_node_affinity "  ")

kube-state-metrics:
$(render_node_selector "  ")
$(render_node_affinity "  ")
prometheus-node-exporter:
$(render_node_selector "  ")
$(render_node_affinity "  ")
EOF

  local monitoring_helm_server_side
  monitoring_helm_server_side="${MONITORING_HELM_SERVER_SIDE:-}"
  if [[ -z "$monitoring_helm_server_side" ]]; then
    if [[ "$mode" == "k3s" ]]; then
      monitoring_helm_server_side="false"
    else
      monitoring_helm_server_side="true"
    fi
  fi

  local helm_apply_flags=()
  case "$monitoring_helm_server_side" in
    1|true|TRUE|yes|YES|on|ON)
      helm_apply_flags=(--force-conflicts --server-side=true)
      ;;
  esac

  helm_upgrade_with_retry \
    "kps" \
    "$monitoring_ns" \
    "prometheus-community/kube-prometheus-stack" \
    "${helm_apply_flags[@]}" \
    -f "$values_file"

  rm -f "$values_file"

  wait_for_monitoring_resources "$monitoring_ns" || true

  section "Applying Prometheus-related additional resources"
  log "Applying myrddin-node-exporter resources in namespace '$monitoring_ns'..."
  cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: myrddin-node-exporter
  namespace: $monitoring_ns
  labels:
    app: myrddin-node-exporter
spec:
  ports:
    - name: metrics
      port: 9100
      targetPort: 9100
---
apiVersion: v1
kind: Endpoints
metadata:
  name: myrddin-node-exporter
  namespace: $monitoring_ns
subsets:
  - addresses:
      - ip: 10.0.0.203   # myrddin LAN IP
    ports:
      - name: metrics
        port: 9100
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: myrddin-node-exporter
  namespace: $monitoring_ns
spec:
  selector:
    matchLabels:
      app: myrddin-node-exporter
  endpoints:
    - port: metrics
      interval: 15s
EOF

  log "Applying CNPG prometheus rules in namespace '$monitoring_ns'..."
  kubectl apply --namespace "$monitoring_ns" -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/samples/monitoring/prometheusrule.yaml

  local separate_grafana
  separate_grafana="${MONITORING_SEPARATE_GRAFANA:-false}"
  case "$separate_grafana" in
    1|true|TRUE|yes|YES|on|ON)
      separate_grafana="true"
      ;;
    *)
      separate_grafana="false"
      ;;
  esac

  if [[ "$separate_grafana" == "true" ]]; then
    section "Installing Grafana (after Prometheus stack is ready)"
    cleanup_legacy_grafana_release
    cleanup_grafana_rbac_conflicts
    helm repo add grafana https://grafana.github.io/helm-charts || true
    helm repo update grafana || true

    resolve_grafana_password
    if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
      GRAFANA_ADMIN_PASSWORD="admin" # Fallback
    fi
    # Belt-and-suspenders: ensure no newline survives into the Helm values YAML.
    GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\n'/}"
    GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\r'/}"

    local grafana_values
    grafana_values=$(mktemp)
    cat > "$grafana_values" <<EOF
fullnameOverride: kps-grafana

adminPassword: "${GRAFANA_ADMIN_PASSWORD}"

service:
  port: 80
  targetPort: 3000

sidecar:
  dashboards:
    enabled: true
    label: grafana_dashboard
    labelValue: "1"

$(render_primary_node_selector "")
$(render_primary_node_affinity "")
$(render_tolerations "")
$(render_grafana_persistence "" "10Gi")
EOF

    MONITORING_RESET_SCOPE=release \
      helm_upgrade_with_retry \
        "$GRAFANA_RELEASE" \
        "$monitoring_ns" \
        "grafana/grafana" \
        -f "$grafana_values"

    rm -f "$grafana_values"
  else
    log "Skipping separate Grafana Helm release; Grafana is managed by kube-prometheus-stack (kps)."
  fi

  # Restart Grafana deployment so it picks up the (possibly updated) admin password.
  # Helm upgrade updates the secret but a running pod won't re-read it without a restart.
  log "Restarting Grafana deployment to apply admin credentials..."
  kubectl rollout restart deployment/kps-grafana -n "$monitoring_ns" 2>/dev/null || true
  kubectl rollout status deployment/kps-grafana -n "$monitoring_ns" --timeout=60s 2>/dev/null || true

  log "Applying Prole Grafana dashboard..."
  apply_grafana_dashboard "$monitoring_ns"

  # Register port forwards
  prole_register_port_forward "prometheus" "$monitoring_ns" "svc/kps-kube-prometheus-stack-prometheus" "9090" "9090" "127.0.0.1" "TCP" "Prometheus"
  prole_register_port_forward "grafana" "$monitoring_ns" "svc/kps-grafana" "3000" "80" "0.0.0.0" "TCP" "Grafana"
}

check_monitoring_status() {
  local monitoring_ns="monitoring"
  local rc=0

  # Check that the Helm releases exist and are deployed
  local helm_status
  helm_status=$(helm status kps -n "$monitoring_ns" -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || true)
  if [[ -z "$helm_status" || "$helm_status" == "null" ]]; then
    echo "[FAIL] kube-prometheus-stack Helm release not found in namespace '$monitoring_ns'"
    return 1
  fi
  if [[ "$helm_status" != "deployed" ]]; then
    echo "[FAIL] kube-prometheus-stack Helm release status: $helm_status"
    rc=1
  else
    echo "[OK] kube-prometheus-stack Helm release is deployed"
  fi

  local separate_grafana
  separate_grafana="${MONITORING_SEPARATE_GRAFANA:-false}"
  case "$separate_grafana" in
    1|true|TRUE|yes|YES|on|ON)
      separate_grafana="true"
      ;;
    *)
      separate_grafana="false"
      ;;
  esac

  if [[ "$separate_grafana" == "true" ]]; then
    local grafana_helm_status
    grafana_helm_status=$(helm status "$GRAFANA_RELEASE" -n "$monitoring_ns" -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || true)
    if [[ -z "$grafana_helm_status" || "$grafana_helm_status" == "null" ]]; then
      echo "[FAIL] Grafana Helm release '$GRAFANA_RELEASE' not found in namespace '$monitoring_ns'"
      rc=1
    elif [[ "$grafana_helm_status" != "deployed" ]]; then
      echo "[FAIL] Grafana Helm release '$GRAFANA_RELEASE' status: $grafana_helm_status"
      rc=1
    else
      echo "[OK] Grafana Helm release '$GRAFANA_RELEASE' is deployed"
    fi
  else
    echo "[OK] Separate Grafana Helm release is disabled (Grafana expected via kube-prometheus-stack)"
  fi

  # Check key deployments
  local deps=("kps-kube-prometheus-stack-operator" "kps-kube-state-metrics" "kps-grafana")
  for dep in "${deps[@]}"; do
    local avail
    avail=$(kubectl -n "$monitoring_ns" get deploy "$dep" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
    if [[ -z "$avail" || "$avail" == "0" ]]; then
      echo "[FAIL] deployment/$dep not available in namespace '$monitoring_ns'"
      rc=1
    else
      echo "[OK] deployment/$dep available (${avail} replicas)"
    fi
  done

  # Check Prometheus StatefulSet
  local prom_ready
  prom_ready=$(kubectl -n "$monitoring_ns" get statefulset prometheus-kps-kube-prometheus-stack-prometheus -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
  if [[ -z "$prom_ready" || "$prom_ready" == "0" ]]; then
    echo "[FAIL] statefulset/prometheus-kps-kube-prometheus-stack-prometheus not ready"
    rc=1
  else
    echo "[OK] Prometheus StatefulSet ready (${prom_ready} replicas)"
  fi

  # Check Alertmanager StatefulSet
  local am_ready
  am_ready=$(kubectl -n "$monitoring_ns" get statefulset alertmanager-kps-kube-prometheus-stack-alertmanager -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
  if [[ -z "$am_ready" || "$am_ready" == "0" ]]; then
    echo "[FAIL] statefulset/alertmanager not ready"
    rc=1
  else
    echo "[OK] Alertmanager StatefulSet ready (${am_ready} replicas)"
  fi

  return $rc
}

case "${1:-}" in
  initialize)
    install_monitoring
    ;;
  status)
    check_monitoring_status
    ;;
  *)
    echo "Usage: $0 {initialize|status}"
    exit 1
    ;;
esac