prole/etc/init_monitoring.sh

#!/usr/bin/env bash

set -euo pipefail

# init_monitoring.sh
# Purpose:
# - Configure k3d environment for monitoring (Prometheus and Grafana)
# - Setup kube-prometheus-stack and CNPG prometheus rules

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

# Load environment and config via prole_cfg.sh
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

if [[ -z "${PROLE_SERVICE:-}" ]]; then
  echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2
  exit 1
fi

log() {
  echo "==> $*"
}

err() {
  echo "ERROR: $*" >&2
}

ensure_tools() {
  for t in helm kubectl; do
    command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; }
  done
}

ensure_namespace() {
  local ns="$1"
  if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
    log "Creating namespace '$ns' ..."
    kubectl create namespace "$ns" >/dev/null 2>&1 || true
  fi
}

cleanup_grafana_rbac_conflicts() {
  local release="grafana-prole"
  local ns="$NAMESPACE"
  local cr="${release}-clusterrole"
  local crb="${release}-clusterrolebinding"
  local rel_ns rel_name

  if kubectl get clusterrole "$cr" >/dev/null 2>&1; then
    rel_ns=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true)
    rel_name=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true)
    if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then
      log "Detected existing ClusterRole '$cr' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'."
      if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then
        log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..."
        helm uninstall "$rel_name" -n "$rel_ns" || true
      fi
      if kubectl get clusterrole "$cr" >/dev/null 2>&1; then
        log "Deleting orphaned ClusterRole '$cr' ..."
        kubectl delete clusterrole "$cr" || true
      fi
    fi
  fi

  if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then
    rel_ns=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true)
    rel_name=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true)
    if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then
      log "Detected existing ClusterRoleBinding '$crb' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'."
      if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then
        log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..."
        helm uninstall "$rel_name" -n "$rel_ns" || true
      fi
      if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then
        log "Deleting orphaned ClusterRoleBinding '$crb' ..."
        kubectl delete clusterrolebinding "$crb" || true
      fi
    fi
  fi
}

install_monitoring() {
  ensure_tools
  local monitoring_ns="default"
  ensure_namespace "$monitoring_ns"
  ensure_namespace "$NAMESPACE"

  log "Installing kube-prometheus-stack in namespace '$monitoring_ns' ..."
  helm repo add prometheus-community https://prometheus-community.github.io/helm-charts || true
  helm repo update prometheus-community || true
  
  # Install Prometheus stack in default namespace, but disable Grafana there
  # Also set grafana.enabled=false explicitly to avoid conflicts if it was previously enabled.
  # Use --force-conflicts with Server-Side Apply (SSA) to handle webhook conflicts.
  # SSA is more robust for managing shared resources like webhooks.
  helm upgrade --install \
    --namespace "$monitoring_ns" \
    --set grafana.enabled=false \
    --force-conflicts \
    --server-side=true \
    -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/samples/monitoring/kube-stack-config.yaml \
    prometheus-community prometheus-community/kube-prometheus-stack

  log "Installing Grafana in namespace '$NAMESPACE' ..."
  # We use the same chart but only for Grafana, or we could use the standalone grafana chart.
  # Using the same chart ensures we can use the same config if needed, but we must avoid ClusterRole conflicts.
  # Actually, standalone grafana chart is cleaner if we only want Grafana.
  helm repo add grafana https://grafana.github.io/helm-charts || true
  helm repo update grafana || true

  local prometheus_svc="http://prometheus-community-kube-prometheus.$monitoring_ns.svc.cluster.local:9090"

  cleanup_grafana_rbac_conflicts
  
  # Note: kube-prometheus-stack may have already been installed with Grafana enabled in 'default'.
  # If we want to move Grafana to $NAMESPACE, we install it there.
  # We use the grafana/grafana chart for the per-namespace instance.
  # Use --force-conflicts with Server-Side Apply (SSA) to handle potential conflicts during upgrade.
  helm upgrade --install \
    --namespace "$NAMESPACE" \
    --force-conflicts \
    --server-side=true \
    --set "rbac.namespaced=true" \
    --set "datasources.datasources\.yaml.apiVersion=1" \
    --set "datasources.datasources\.yaml.datasources[0].name=Prometheus" \
    --set "datasources.datasources\.yaml.datasources[0].type=prometheus" \
    --set "datasources.datasources\.yaml.datasources[0].url=$prometheus_svc" \
    --set "datasources.datasources\.yaml.datasources[0].access=proxy" \
    --set "datasources.datasources\.yaml.datasources[0].isDefault=true" \
    --set "adminPassword=admin" \
    grafana-prole grafana/grafana

  log "Applying CNPG prometheus rules in namespace '$NAMESPACE'..."
  kubectl apply --namespace "$NAMESPACE" -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/samples/monitoring/prometheusrule.yaml

  log "Retrieving Grafana admin password from namespace '$NAMESPACE'..."
  # Secret name is 'grafana-prole' from the helm release name
  GRAFANA_PASSWORD=$(kubectl --namespace "$NAMESPACE" get secrets grafana-prole -o jsonpath="{.data.admin-password}" 2>/dev/null | base64 -d || true)
  
  if [[ -n "$GRAFANA_PASSWORD" ]]; then
    log "Grafana installation password: $GRAFANA_PASSWORD"
    # We will save this to prole.cfg via the installer, but also output it here for logs
    echo "GRAFANA_ADMIN_PASSWORD=$GRAFANA_PASSWORD"
  else
    err "Failed to retrieve Grafana admin password."
  fi
}

case "${1:-}" in
  initialize)
    install_monitoring
    ;;
  *)
    echo "Usage: $0 initialize"
    exit 1
    ;;
esac