chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 12:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
86cf99cf76
prole/etc/init_monitoring.sh
chrisfu 86cf99cf76 milestone(k3s+cnpg): end-to-end deploy run via standalone installer 
We can now complete a full K3s deployment run and deliver a working CloudNativePG (CNPG) stack from the standalone installer.\n\nHighlights:\n- Installer scripts hardened for CNPG deploy + recovery templating + rollout control\n- OpenTofu/K3s manifests aligned with in-repo k8s templates for prole-db\n- Supporting init flows updated (OpenBao, monitoring, Kerberos, OpenTofu)

Co-authored-by: Junie <junie@jetbrains.com>
2026-03-03 20:03:15 -08:00

659 lines
20 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
# init_monitoring.sh
# Purpose:
# - Configure k3d environment for monitoring (Prometheus and Grafana)
# - Setup kube-prometheus-stack and CNPG prometheus rules
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
PROLE_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
# Load environment and config via prole_cfg.sh
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"
if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then
prole_set_mode "${2:-}"
shift 2
elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then
prole_set_mode "${1#*=}"
shift
fi
if [[ -z "${PROLE_SERVICE:-}" ]]; then
echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2
exit 1
fi
GRAFANA_RELEASE="grafana"
LEGACY_GRAFANA_RELEASE="grafana-prole"
log() {
echo "==> $*"
}
err() {
echo "ERROR: $*" >&2
}
ensure_tools() {
for t in helm kubectl curl jq; do
command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; }
done
}
ensure_namespace() {
local ns="$1"
if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
log "Creating namespace '$ns' ..."
kubectl create namespace "$ns" >/dev/null 2>&1 || true
fi
}
storage_class_exists() {
local sc="$1"
[[ -n "$sc" ]] || return 1
kubectl get storageclass "$sc" >/dev/null 2>&1
}
default_storage_class() {
kubectl get storageclass -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -n1
}
choose_monitoring_storage_class() {
if [[ -n "${MONITORING_STORAGE_CLASS:-}" ]] && storage_class_exists "$MONITORING_STORAGE_CLASS"; then
echo "$MONITORING_STORAGE_CLASS"
return 0
fi
if storage_class_exists "pi-local-iscsi"; then
echo "pi-local-iscsi"
return 0
fi
local default_sc
default_sc=$(default_storage_class)
if [[ -n "$default_sc" ]]; then
echo "$default_sc"
return 0
fi
if storage_class_exists "local-path"; then
echo "local-path"
return 0
fi
echo ""
}
monitoring_nodes_available() {
kubectl get nodes -l "prole.org/role=monitoring" -o name 2>/dev/null | grep -q .
}
render_node_selector() {
local indent="$1"
if [[ "${MONITORING_HAS_NODE_LABEL:-0}" == "1" ]]; then
cat <<EOF
${indent}nodeSelector:
${indent} prole.org/role: monitoring
EOF
fi
}
render_tolerations() {
local indent="$1"
if [[ "${MONITORING_HAS_NODE_LABEL:-0}" == "1" ]]; then
cat <<EOF
${indent}tolerations:
${indent} - key: "prole.org/monitoring"
${indent} operator: "Equal"
${indent} value: "true"
${indent} effect: "NoSchedule"
EOF
fi
}
render_prometheus_storage() {
local indent="$1"
local size="$2"
if [[ -n "${MONITORING_STORAGE_CLASS_SELECTED:-}" ]]; then
cat <<EOF
${indent}storageSpec:
${indent} volumeClaimTemplate:
${indent} spec:
${indent} storageClassName: ${MONITORING_STORAGE_CLASS_SELECTED}
${indent} accessModes: ["ReadWriteOnce"]
${indent} resources:
${indent} requests:
${indent} storage: ${size}
EOF
fi
}
render_alertmanager_storage() {
local indent="$1"
local size="$2"
if [[ -n "${MONITORING_STORAGE_CLASS_SELECTED:-}" ]]; then
cat <<EOF
${indent}storage:
${indent} volumeClaimTemplate:
${indent} spec:
${indent} storageClassName: ${MONITORING_STORAGE_CLASS_SELECTED}
${indent} accessModes: ["ReadWriteOnce"]
${indent} resources:
${indent} requests:
${indent} storage: ${size}
EOF
fi
}
render_grafana_persistence() {
local indent="$1"
local size="$2"
if [[ -n "${MONITORING_STORAGE_CLASS_SELECTED:-}" ]]; then
cat <<EOF
${indent}persistence:
${indent} enabled: true
${indent} storageClassName: ${MONITORING_STORAGE_CLASS_SELECTED}
${indent} accessModes: ["ReadWriteOnce"]
${indent} size: ${size}
EOF
else
cat <<EOF
${indent}persistence:
${indent} enabled: false
EOF
fi
}
cleanup_pending_pvcs() {
local ns="$1"
local expected_sc="$2"
local pending
pending=$(kubectl -n "$ns" get pvc -o json | jq -r '.items[] | select(.status.phase=="Pending") | "\(.metadata.name)|\(.spec.storageClassName // "")"' || true)
while IFS='|' read -r pvc_name pvc_sc; do
[[ -n "$pvc_name" ]] || continue
if [[ -n "$pvc_sc" ]] && ! storage_class_exists "$pvc_sc"; then
log "Deleting Pending PVC '$pvc_name' with missing storageClass '$pvc_sc' ..."
kubectl -n "$ns" delete pvc "$pvc_name" >/dev/null 2>&1 || true
continue
fi
if [[ -n "$expected_sc" && -n "$pvc_sc" && "$pvc_sc" != "$expected_sc" ]]; then
log "Deleting Pending PVC '$pvc_name' with storageClass '$pvc_sc' (expected '$expected_sc') ..."
kubectl -n "$ns" delete pvc "$pvc_name" >/dev/null 2>&1 || true
fi
done <<< "$pending"
}
apply_grafana_dashboard() {
local ns="$1"
local dashboard_path=""
if [[ -n "${PROLE_GRAFANA_DASHBOARD_PATH:-}" ]]; then
dashboard_path="$PROLE_GRAFANA_DASHBOARD_PATH"
elif [[ -n "${PROLE_SERVICE:-}" && -f "$PROLE_SERVICE/prole-db/grafana-dashboard.json" ]]; then
dashboard_path="$PROLE_SERVICE/prole-db/grafana-dashboard.json"
elif [[ -f "$PROLE_ROOT/prole-db/grafana-dashboard.json" ]]; then
dashboard_path="$PROLE_ROOT/prole-db/grafana-dashboard.json"
fi
if [[ -z "$dashboard_path" || ! -f "$dashboard_path" ]]; then
log "Grafana dashboard not found; skipping."
return 0
fi
local tmp
tmp=$(mktemp)
sed 's/\\${DS_PROMETHEUS}/prometheus/g' "$dashboard_path" > "$tmp"
kubectl -n "$ns" delete configmap prole-db-grafana-dashboard >/dev/null 2>&1 || true
kubectl -n "$ns" create configmap prole-db-grafana-dashboard --from-file=prole-db.json="$tmp" >/dev/null
kubectl -n "$ns" label configmap prole-db-grafana-dashboard grafana_dashboard=1 --overwrite >/dev/null
rm -f "$tmp"
}
helm_release_status() {
local release="$1"
local ns="$2"
helm status "$release" -n "$ns" -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || true
}
wait_for_helm_release() {
local release="$1"
local ns="$2"
local timeout="${HELM_WAIT_TIMEOUT:-300}"
local interval="${HELM_WAIT_INTERVAL:-5}"
local start
start=$(date +%s)
while true; do
local status
status=$(helm_release_status "$release" "$ns")
if [[ -z "$status" || "$status" == "null" ]]; then
return 0
fi
case "$status" in
pending-*)
if (( $(date +%s) - start > timeout )); then
err "Timed out waiting for Helm release '$release' in '$ns' (status=$status)."
return 1
fi
log "Helm release '$release' is $status; waiting..."
sleep "$interval"
;;
*)
return 0
;;
esac
done
}
helm_upgrade_with_retry() {
local release="$1"
local ns="$2"
local chart="$3"
shift 3
local attempts="${HELM_UPGRADE_RETRIES:-5}"
local delay="${HELM_RETRY_DELAY:-5}"
local attempt out rc
for ((attempt=1; attempt<=attempts; attempt++)); do
wait_for_helm_release "$release" "$ns" || true
set +e
out=$(helm upgrade --install "$release" "$chart" --namespace "$ns" "$@" 2>&1)
rc=$?
set -e
if [[ $rc -eq 0 ]]; then
printf '%s\n' "$out"
return 0
fi
if echo "$out" | grep -q "another operation (install/upgrade/rollback) is in progress"; then
log "Helm release '$release' is busy; retrying ($attempt/$attempts)..."
wait_for_helm_release "$release" "$ns" || true
sleep "$delay"
continue
fi
echo "$out" >&2
return "$rc"
done
err "Helm upgrade failed after $attempts attempts for release '$release' in '$ns'."
return 1
}
openbao_url() {
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
echo "$PROLE_OPENBAO_URL"
return 0
fi
if prole_is_in_cluster; then
echo "http://openbao.${SERVICE_NAMESPACE:-${NAMESPACE:-default}}.svc.cluster.local:8200"
return 0
fi
# In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host
# (never via localhost or kubectl port-forward).
if [[ "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then
if command -v _prole_host_from_url >/dev/null 2>&1; then
local host
host=$(_prole_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")
if [[ -n "${host:-}" ]]; then
echo "http://${host}:8200"
return 0
fi
fi
echo ""
return 0
fi
if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:8200"
return 0
fi
echo ""
return 0
}
openbao_token() {
if [[ -f "$PROLE_SERVICE/secrets/openbao-root-token" ]]; then
cat "$PROLE_SERVICE/secrets/openbao-root-token"
else
echo "${OPENBAO_ROOT_TOKEN:-}"
fi
}
fetch_openbao_secret() {
local path="$1"
local key="$2"
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -z "$token" || -z "$url" ]]; then
echo ""
return 0
fi
curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
}
resolve_grafana_password() {
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then
local fetched
fetched=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/monitoring" "grafana_admin_password")
if [[ -n "$fetched" && "$fetched" != "null" ]]; then
GRAFANA_ADMIN_PASSWORD="$fetched"
fi
fi
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then
local db_pw="${DB_PASSWORD:-}"
if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
local fetched_db
fetched_db=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/db" "password")
if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
db_pw="$fetched_db"
fi
fi
if [[ -n "$db_pw" ]]; then
GRAFANA_ADMIN_PASSWORD="$db_pw"
fi
fi
# Strip any trailing newlines/carriage-returns that may have crept in via
# prole.cfg parsing, file reads, or shell substitution edge-cases.
GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\n'/}"
GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\r'/}"
}
write_grafana_password_to_openbao() {
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -z "$token" || -z "$url" || -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
return 0
fi
local _clean_grafana_pw
_clean_grafana_pw="${GRAFANA_ADMIN_PASSWORD//$'\n'/}"
_clean_grafana_pw="${_clean_grafana_pw//$'\r'/}"
curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
-X POST "$url/v1/kv/data/prole/${NAMESPACE:-default}/monitoring" \
-d "{\"data\":{\"grafana_admin_password\":\"$_clean_grafana_pw\"}}" >/dev/null || true
}
cleanup_grafana_rbac_conflicts() {
local release="$GRAFANA_RELEASE"
local ns="$NAMESPACE"
local cr="${release}-clusterrole"
local crb="${release}-clusterrolebinding"
local rel_ns rel_name
if kubectl get clusterrole "$cr" >/dev/null 2>&1; then
rel_ns=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true)
rel_name=$(kubectl get clusterrole "$cr" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true)
if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then
log "Detected existing ClusterRole '$cr' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'."
if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then
log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..."
helm uninstall "$rel_name" -n "$rel_ns" || true
fi
if kubectl get clusterrole "$cr" >/dev/null 2>&1; then
log "Deleting orphaned ClusterRole '$cr' ..."
kubectl delete clusterrole "$cr" || true
fi
fi
fi
if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then
rel_ns=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || true)
rel_name=$(kubectl get clusterrolebinding "$crb" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true)
if [[ -n "$rel_ns" && "$rel_ns" != "$ns" ]]; then
log "Detected existing ClusterRoleBinding '$crb' owned by release '${rel_name:-unknown}' in namespace '$rel_ns'."
if [[ -n "$rel_name" ]] && helm status "$rel_name" -n "$rel_ns" >/dev/null 2>&1; then
log "Uninstalling old Grafana release '$rel_name' from '$rel_ns' ..."
helm uninstall "$rel_name" -n "$rel_ns" || true
fi
if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then
log "Deleting orphaned ClusterRoleBinding '$crb' ..."
kubectl delete clusterrolebinding "$crb" || true
fi
fi
fi
}
cleanup_legacy_grafana_release() {
local ns="$NAMESPACE"
if helm status "$LEGACY_GRAFANA_RELEASE" -n "$ns" >/dev/null 2>&1; then
log "Uninstalling legacy Grafana release '$LEGACY_GRAFANA_RELEASE' from '$ns' ..."
helm uninstall "$LEGACY_GRAFANA_RELEASE" -n "$ns" || true
fi
}
install_monitoring() {
ensure_tools
local monitoring_ns="monitoring"
ensure_namespace "$monitoring_ns"
MONITORING_HAS_NODE_LABEL=0
if monitoring_nodes_available; then
MONITORING_HAS_NODE_LABEL=1
else
log "No nodes labeled prole.org/role=monitoring; scheduling without nodeSelector/tolerations."
fi
MONITORING_STORAGE_CLASS_SELECTED=$(choose_monitoring_storage_class)
if [[ -n "$MONITORING_STORAGE_CLASS_SELECTED" ]]; then
log "Using storageClass '$MONITORING_STORAGE_CLASS_SELECTED' for monitoring PVCs."
else
log "No storageClass detected; disabling persistence for Grafana and skipping Prometheus/Alertmanager storage."
fi
cleanup_pending_pvcs "$monitoring_ns" "$MONITORING_STORAGE_CLASS_SELECTED"
log "Installing kube-prometheus-stack in namespace '$monitoring_ns' ..."
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts || true
helm repo update prometheus-community || true
resolve_grafana_password
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
GRAFANA_ADMIN_PASSWORD="admin" # Fallback
fi
# Belt-and-suspenders: ensure no newline survives into the Helm values YAML.
GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\n'/}"
GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\r'/}"
local values_file
values_file=$(mktemp)
cat > "$values_file" <<EOF
prometheus:
prometheusSpec:
$(render_node_selector " ")
$(render_tolerations " ")
$(render_prometheus_storage " " "30Gi")
additionalScrapeConfigs:
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: keep
source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: (.*?):\d+;(\d+)
replacement: \$1:\$2
- job_name: 'cnpg-metrics'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: keep
source_labels: [__meta_kubernetes_pod_label_cnpg_io_cluster]
regex: .+
- action: keep
source_labels: [__meta_kubernetes_pod_phase]
regex: Running
- action: replace
source_labels: [__meta_kubernetes_pod_ip]
target_label: __address__
replacement: \$1:9187
- action: replace
source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- action: replace
source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- action: replace
source_labels: [__meta_kubernetes_pod_label_cnpg_io_cluster]
target_label: cluster
grafana:
adminPassword: "${GRAFANA_ADMIN_PASSWORD}"
$(render_node_selector " ")
$(render_tolerations " ")
$(render_grafana_persistence " " "10Gi")
alertmanager:
alertmanagerSpec:
$(render_node_selector " ")
$(render_tolerations " ")
$(render_alertmanager_storage " " "5Gi")
kube-state-metrics:
nodeSelector: {}
prometheus-node-exporter:
nodeSelector: {}
EOF
helm_upgrade_with_retry \
"kps" \
"$monitoring_ns" \
"prometheus-community/kube-prometheus-stack" \
--force-conflicts \
--server-side=true \
-f "$values_file"
rm -f "$values_file"
# Restart Grafana deployment so it picks up the (possibly updated) admin password.
# Helm upgrade updates the secret but a running pod won't re-read it without a restart.
log "Restarting Grafana deployment to apply admin credentials..."
kubectl rollout restart deployment/kps-grafana -n "$monitoring_ns" 2>/dev/null || true
kubectl rollout status deployment/kps-grafana -n "$monitoring_ns" --timeout=60s 2>/dev/null || true
log "Applying myrddin-node-exporter resources in namespace '$monitoring_ns'..."
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: myrddin-node-exporter
namespace: $monitoring_ns
labels:
app: myrddin-node-exporter
spec:
ports:
- name: metrics
port: 9100
targetPort: 9100
---
apiVersion: v1
kind: Endpoints
metadata:
name: myrddin-node-exporter
namespace: $monitoring_ns
subsets:
- addresses:
- ip: 10.0.0.203 # myrddin LAN IP
ports:
- name: metrics
port: 9100
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: myrddin-node-exporter
namespace: $monitoring_ns
spec:
selector:
matchLabels:
app: myrddin-node-exporter
endpoints:
- port: metrics
interval: 15s
EOF
log "Applying CNPG prometheus rules in namespace '$monitoring_ns'..."
kubectl apply --namespace "$monitoring_ns" -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/samples/monitoring/prometheusrule.yaml
log "Applying Prole Grafana dashboard..."
apply_grafana_dashboard "$monitoring_ns"
# Register port forwards
prole_register_port_forward "prometheus" "$monitoring_ns" "svc/kps-kube-prometheus-stack-prometheus" "9090" "9090" "127.0.0.1" "TCP" "Prometheus"
prole_register_port_forward "grafana" "$monitoring_ns" "svc/kps-grafana" "3000" "3000" "0.0.0.0" "TCP" "Grafana"
}
check_monitoring_status() {
local monitoring_ns="monitoring"
local rc=0
# Check that the Helm release exists and is deployed
local helm_status
helm_status=$(helm status kps -n "$monitoring_ns" -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || true)
if [[ -z "$helm_status" || "$helm_status" == "null" ]]; then
echo "[FAIL] kube-prometheus-stack Helm release not found in namespace '$monitoring_ns'"
return 1
fi
if [[ "$helm_status" != "deployed" ]]; then
echo "[FAIL] kube-prometheus-stack Helm release status: $helm_status"
rc=1
else
echo "[OK] kube-prometheus-stack Helm release is deployed"
fi
# Check key deployments
local deps=("kps-grafana" "kps-kube-prometheus-stack-operator" "kps-kube-state-metrics")
for dep in "${deps[@]}"; do
local avail
avail=$(kubectl -n "$monitoring_ns" get deploy "$dep" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
if [[ -z "$avail" || "$avail" == "0" ]]; then
echo "[FAIL] deployment/$dep not available in namespace '$monitoring_ns'"
rc=1
else
echo "[OK] deployment/$dep available (${avail} replicas)"
fi
done
# Check Prometheus StatefulSet
local prom_ready
prom_ready=$(kubectl -n "$monitoring_ns" get statefulset prometheus-kps-kube-prometheus-stack-prometheus -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
if [[ -z "$prom_ready" || "$prom_ready" == "0" ]]; then
echo "[FAIL] statefulset/prometheus-kps-kube-prometheus-stack-prometheus not ready"
rc=1
else
echo "[OK] Prometheus StatefulSet ready (${prom_ready} replicas)"
fi
# Check Alertmanager StatefulSet
local am_ready
am_ready=$(kubectl -n "$monitoring_ns" get statefulset alertmanager-kps-kube-prometheus-stack-alertmanager -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)
if [[ -z "$am_ready" || "$am_ready" == "0" ]]; then
echo "[FAIL] statefulset/alertmanager not ready"
rc=1
else
echo "[OK] Alertmanager StatefulSet ready (${am_ready} replicas)"
fi
return $rc
}
case "${1:-}" in
initialize)
install_monitoring
;;
status)
check_monitoring_status
;;
*)
echo "Usage: $0 {initialize|status}"
exit 1
;;
esac
Reference in New Issue View Git Blame Copy Permalink