#!/usr/bin/env bash set -euo pipefail # init_monitoring.sh # Purpose: # - Configure k3d environment for monitoring (Prometheus and Grafana) # - Setup kube-prometheus-stack and CNPG prometheus rules SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Load environment and config via prole_cfg.sh # shellcheck disable=SC1090 source "$SCRIPT_DIR/prole_cfg.sh" if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then prole_set_mode "${2:-}" shift 2 elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then prole_set_mode "${1#*=}" shift fi if [[ -z "${PROLE_SERVICE:-}" ]]; then echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2 exit 1 fi GRAFANA_RELEASE="grafana" LEGACY_GRAFANA_RELEASE="grafana-prole" log() { echo "==> $*" } err() { echo "ERROR: $*" >&2 } ensure_tools() { for t in helm kubectl curl jq; do command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; } done } ensure_namespace() { local ns="$1" if ! kubectl get namespace "$ns" >/dev/null 2>&1; then log "Creating namespace '$ns' ..." kubectl create namespace "$ns" >/dev/null 2>&1 || true fi } helm_release_status() { local release="$1" local ns="$2" helm status "$release" -n "$ns" -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || true } wait_for_helm_release() { local release="$1" local ns="$2" local timeout="${HELM_WAIT_TIMEOUT:-300}" local interval="${HELM_WAIT_INTERVAL:-5}" local start start=$(date +%s) while true; do local status status=$(helm_release_status "$release" "$ns") if [[ -z "$status" || "$status" == "null" ]]; then return 0 fi case "$status" in pending-*) if (( $(date +%s) - start > timeout )); then err "Timed out waiting for Helm release '$release' in '$ns' (status=$status)." return 1 fi log "Helm release '$release' is $status; waiting..." sleep "$interval" ;; *) return 0 ;; esac done } helm_upgrade_with_retry() { local release="$1" local ns="$2" local chart="$3" shift 3 local attempts="${HELM_UPGRADE_RETRIES:-5}" local delay="${HELM_RETRY_DELAY:-5}" local attempt out rc for ((attempt=1; attempt<=attempts; attempt++)); do wait_for_helm_release "$release" "$ns" || true set +e out=$(helm upgrade --install "$release" "$chart" --namespace "$ns" "$@" 2>&1) rc=$? set -e if [[ $rc -eq 0 ]]; then printf '%s\n' "$out" return 0 fi if echo "$out" | grep -q "another operation (install/upgrade/rollback) is in progress"; then log "Helm release '$release' is busy; retrying ($attempt/$attempts)..." wait_for_helm_release "$release" "$ns" || true sleep "$delay" continue fi echo "$out" >&2 return "$rc" done err "Helm upgrade failed after $attempts attempts for release '$release' in '$ns'." return 1 } openbao_url() { if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then echo "$PROLE_OPENBAO_URL" return 0 fi if prole_is_in_cluster; then echo "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE:-default}}.svc.cluster.local:8200" return 0 fi if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then echo "http://127.0.0.1:8200" return 0 elif curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then echo "http://127.0.0.1:18200" return 0 else echo "" return 0 fi } openbao_token() { if [[ -f "$PROLE_SERVICE/secrets/openbao-root-token" ]]; then cat "$PROLE_SERVICE/secrets/openbao-root-token" else echo "${OPENBAO_ROOT_TOKEN:-}" fi } fetch_openbao_secret() { local path="$1" local key="$2" local token url token=$(openbao_token) url=$(openbao_url) if [[ -z "$token" || -z "$url" ]]; then echo "" return 0 fi curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo "" } resolve_grafana_password() { if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then local fetched fetched=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/monitoring" "grafana_admin_password") if [[ -n "$fetched" && "$fetched" != "null" ]]; then GRAFANA_ADMIN_PASSWORD="$fetched" fi fi if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then local db_pw="${DB_PASSWORD:-}" if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then local fetched_db fetched_db=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/db" "password") if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then db_pw="$fetched_db" fi fi if [[ -n "$db_pw" ]]; then GRAFANA_ADMIN_PASSWORD="$db_pw" fi fi } write_grafana_password_to_openbao() { local token url token=$(openbao_token) url=$(openbao_url) if [[ -z "$token" || -z "$url" || -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then return 0 fi curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \ -X POST "$url/v1/kv/data/prole/${NAMESPACE:-default}/monitoring" \ -d "{\"data\":{\"grafana_admin_password\":\"$GRAFANA_ADMIN_PASSWORD\"}}" >/dev/null || true } cleanup_grafana_rbac_conflicts() { local release="$GRAFANA_RELEASE" local ns="$NAMESPACE" local cr="${release}-clusterrole" local crb="${release}-clusterrolebinding" local rel_ns rel_name if kubectl get clusterrole "$cr" >/dev/null 2>&1; then rel_ns=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true) rel_name=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true) if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then log "Detected existing ClusterRole '$cr' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'." if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..." helm uninstall "$rel_name" -n "$rel_ns" || true fi if kubectl get clusterrole "$cr" >/dev/null 2>&1; then log "Deleting orphaned ClusterRole '$cr' ..." kubectl delete clusterrole "$cr" || true fi fi fi if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then rel_ns=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true) rel_name=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true) if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then log "Detected existing ClusterRoleBinding '$crb' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'." if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..." helm uninstall "$rel_name" -n "$rel_ns" || true fi if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then log "Deleting orphaned ClusterRoleBinding '$crb' ..." kubectl delete clusterrolebinding "$crb" || true fi fi fi } cleanup_legacy_grafana_release() { local ns="$NAMESPACE" if helm status "$LEGACY_GRAFANA_RELEASE" -n "$ns" >/dev/null 2>&1; then log "Uninstalling legacy Grafana release '$LEGACY_GRAFANA_RELEASE' from '$ns' ..." helm uninstall "$LEGACY_GRAFANA_RELEASE" -n "$ns" || true fi } install_monitoring() { ensure_tools local monitoring_ns="default" ensure_namespace "$monitoring_ns" ensure_namespace "$NAMESPACE" log "Installing kube-prometheus-stack in namespace '$monitoring_ns' ..." helm repo add prometheus-community https://prometheus-community.github.io/helm-charts || true helm repo update prometheus-community || true # Install Prometheus stack in default namespace, but disable Grafana there # Also set grafana.enabled=false explicitly to avoid conflicts if it was previously enabled. # Use --force-conflicts with Server-Side Apply (SSA) to handle webhook conflicts. # SSA is more robust for managing shared resources like webhooks. helm_upgrade_with_retry \ "prometheus-community" \ "$monitoring_ns" \ "prometheus-community/kube-prometheus-stack" \ --set grafana.enabled=false \ --force-conflicts \ --server-side=true \ -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/samples/monitoring/kube-stack-config.yaml log "Installing Grafana in namespace '$NAMESPACE' ..." # We use the same chart but only for Grafana, or we could use the standalone grafana chart. # Using the same chart ensures we can use the same config if needed, but we must avoid ClusterRole conflicts. # Actually, standalone grafana chart is cleaner if we only want Grafana. helm repo add grafana https://grafana.github.io/helm-charts || true helm repo update grafana || true local prometheus_svc="http://prometheus-community-kube-prometheus.$monitoring_ns.svc.cluster.local:9090" cleanup_legacy_grafana_release # Note: kube-prometheus-stack may have already been installed with Grafana enabled in 'default'. # If we want to move Grafana to $NAMESPACE, we install it there. # We use the grafana/grafana chart for the per-namespace instance. if kubectl get deployment "$GRAFANA_RELEASE" -n "$NAMESPACE" >/dev/null 2>&1; then log "Grafana deployment '$GRAFANA_RELEASE' already exists in '$NAMESPACE'; skipping Helm install." else cleanup_grafana_rbac_conflicts resolve_grafana_password if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then err "GRAFANA_ADMIN_PASSWORD is empty. Set it or ensure DB_PASSWORD is available." exit 1 fi write_grafana_password_to_openbao # Use --force-conflicts with Server-Side Apply (SSA) to handle potential conflicts during upgrade. local grafana_persistence_enabled="true" local grafana_storage_class="prole-iscsi" if [[ "${PROLE_MODE:-}" == "k3d" ]]; then grafana_persistence_enabled="false" grafana_storage_class="" fi helm_upgrade_with_retry \ "$GRAFANA_RELEASE" \ "$NAMESPACE" \ "grafana/grafana" \ --force-conflicts \ --server-side=true \ --set "rbac.namespaced=true" \ --set "persistence.enabled=${grafana_persistence_enabled}" \ --set "persistence.size=5Gi" \ $(if [[ -n "$grafana_storage_class" ]]; then printf '%s ' --set "persistence.storageClassName=${grafana_storage_class}"; fi) \ --set "service.port=3000" \ --set "datasources.datasources\.yaml.apiVersion=1" \ --set "datasources.datasources\.yaml.datasources[0].name=Prometheus" \ --set "datasources.datasources\.yaml.datasources[0].type=prometheus" \ --set "datasources.datasources\.yaml.datasources[0].url=$prometheus_svc" \ --set "datasources.datasources\.yaml.datasources[0].access=proxy" \ --set "datasources.datasources\.yaml.datasources[0].isDefault=true" \ --set "adminPassword=$GRAFANA_ADMIN_PASSWORD" fi log "Applying CNPG prometheus rules in namespace '$NAMESPACE'..." kubectl apply --namespace "$NAMESPACE" -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/samples/monitoring/prometheusrule.yaml log "Retrieving Grafana admin password from namespace '$NAMESPACE'..." local grafana_secret="$GRAFANA_RELEASE" if ! kubectl --namespace "$NAMESPACE" get secret "$grafana_secret" >/dev/null 2>&1; then grafana_secret="grafana-admin" fi GRAFANA_PASSWORD=$(kubectl --namespace "$NAMESPACE" get secret "$grafana_secret" -o jsonpath="{.data.admin-password}" 2>/dev/null | base64 -d || true) if [[ -n "$GRAFANA_PASSWORD" ]]; then log "Grafana installation password: $GRAFANA_PASSWORD" # We will save this to prole.cfg via the installer, but also output it here for logs echo "GRAFANA_ADMIN_PASSWORD=$GRAFANA_PASSWORD" else err "Failed to retrieve Grafana admin password." fi # Register port forwards prole_register_port_forward "prometheus" "default" "svc/prometheus-community-kube-prometheus" "9090" "9090" "127.0.0.1" "TCP" "Prometheus" prole_register_port_forward "grafana" "${NAMESPACE:-default}" "svc/grafana" "3000" "3000" "0.0.0.0" "TCP" "Grafana" } case "${1:-}" in initialize) install_monitoring ;; *) echo "Usage: $0 initialize" exit 1 ;; esac