mirror of
https://github.com/dredx/prole.git
synced 2026-09-27 19:44:31 +00:00
5cd43c873c
- separate default app and db cluster contexts (knoe-dev-0 and knoe-cnpg-0) across actions, ops flows, and installer config\n- update CNPG placement and GKE manifests to keep PostgreSQL on the db cluster while app services stay on the app cluster\n- add GCS-backed Barman ObjectStore manifest and cross-cluster Garage patch workflow\n- refresh UI cluster/database/service screens and monitoring wiring for dual-cluster operation\n- add reset/patch scripts, db context selection test coverage, and architecture/network documentation updates Co-authored-by: Junie <junie@jetbrains.com>
273 lines
8.9 KiB
Python
273 lines
8.9 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
|
|
from ._services_common import _LogFn, _detect_mode, _helm, _kubectl, _log, _namespace, _to_bool
|
|
|
|
|
|
def _monitoring_namespace(namespace: str | None, env: dict | None) -> str:
|
|
if env:
|
|
explicit = str(env.get("MONITORING_NAMESPACE") or "").strip()
|
|
if explicit:
|
|
return explicit
|
|
return _namespace(namespace, env, default="monitoring")
|
|
|
|
|
|
def _release_name(env: dict | None) -> str:
|
|
return str((env or {}).get("MONITORING_RELEASE") or "prometheus")
|
|
|
|
|
|
def initialize(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
log: _LogFn | None = None,
|
|
mode: str | None = None,
|
|
) -> None:
|
|
update(namespace=namespace, env=env, log=log, mode=mode)
|
|
|
|
|
|
def start(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
log: _LogFn | None = None,
|
|
mode: str | None = None,
|
|
) -> None:
|
|
update(namespace=namespace, env=env, log=log, mode=mode)
|
|
|
|
|
|
def _values_yaml_k8s(grafana_password: str) -> str:
|
|
"""Helm values for GKE / cloud-managed Kubernetes (no local PV affinity)."""
|
|
return (
|
|
"prometheus:\n"
|
|
" prometheusSpec:\n"
|
|
" storageSpec:\n"
|
|
" volumeClaimTemplate:\n"
|
|
" spec:\n"
|
|
" storageClassName: standard\n"
|
|
" accessModes: [ReadWriteOnce]\n"
|
|
" resources:\n"
|
|
" requests:\n"
|
|
" storage: 10Gi\n"
|
|
"alertmanager:\n"
|
|
" alertmanagerSpec:\n"
|
|
" storage:\n"
|
|
" volumeClaimTemplate:\n"
|
|
" spec:\n"
|
|
" storageClassName: standard\n"
|
|
" accessModes: [ReadWriteOnce]\n"
|
|
" resources:\n"
|
|
" requests:\n"
|
|
" storage: 5Gi\n"
|
|
"grafana:\n"
|
|
" adminUser: admin\n"
|
|
f" adminPassword: {grafana_password}\n"
|
|
" persistence:\n"
|
|
" type: sts\n"
|
|
" enabled: true\n"
|
|
" storageClassName: standard\n"
|
|
" accessModes: [ReadWriteOnce]\n"
|
|
" size: 5Gi\n"
|
|
)
|
|
|
|
|
|
def _values_yaml_k3s(grafana_password: str, env: dict | None) -> str:
|
|
"""Helm values for k3s homelab (local iSCSI PVs pinned to merlin.prole.org)."""
|
|
data_dir = str((env or {}).get("PROLE_MONITORING_DATA_DIR") or "/synology/d004").rstrip("/")
|
|
volume_id = os.path.basename(data_dir) # e.g. "d004"
|
|
sc_prom = f"merlin-local-iscsi-{volume_id}-prometheus"
|
|
sc_alert = f"merlin-local-iscsi-{volume_id}-alertmanager"
|
|
sc_grafana = f"merlin-local-iscsi-{volume_id}-grafana"
|
|
|
|
monitoring_node = str((env or {}).get("MONITORING_PRIMARY_NODE") or "merlin.prole.org")
|
|
excluded_nodes = str((env or {}).get("MONITORING_NODE_EXPORTER_EXCLUDE_NODES") or "pi.prole.org")
|
|
excluded_list = [n.strip() for n in excluded_nodes.split(",") if n.strip()]
|
|
|
|
node_affinity_yaml = (
|
|
" affinity:\n"
|
|
" nodeAffinity:\n"
|
|
" requiredDuringSchedulingIgnoredDuringExecution:\n"
|
|
" nodeSelectorTerms:\n"
|
|
" - matchExpressions:\n"
|
|
" - key: kubernetes.io/hostname\n"
|
|
" operator: In\n"
|
|
" values:\n"
|
|
f" - {monitoring_node}\n"
|
|
)
|
|
|
|
values = "prometheus-node-exporter:\n"
|
|
if excluded_list:
|
|
values += (
|
|
" affinity:\n"
|
|
" nodeAffinity:\n"
|
|
" requiredDuringSchedulingIgnoredDuringExecution:\n"
|
|
" nodeSelectorTerms:\n"
|
|
" - matchExpressions:\n"
|
|
" - key: kubernetes.io/hostname\n"
|
|
" operator: NotIn\n"
|
|
" values:\n"
|
|
)
|
|
for node in excluded_list:
|
|
values += f" - {node}\n"
|
|
|
|
values += (
|
|
"prometheus:\n"
|
|
" prometheusSpec:\n"
|
|
+ node_affinity_yaml
|
|
+ " storageSpec:\n"
|
|
" volumeClaimTemplate:\n"
|
|
" spec:\n"
|
|
f" storageClassName: {sc_prom}\n"
|
|
" accessModes: [ReadWriteOnce]\n"
|
|
" resources:\n"
|
|
" requests:\n"
|
|
" storage: 30Gi\n"
|
|
)
|
|
values += (
|
|
"alertmanager:\n"
|
|
" alertmanagerSpec:\n"
|
|
+ node_affinity_yaml
|
|
+ " storage:\n"
|
|
" volumeClaimTemplate:\n"
|
|
" spec:\n"
|
|
f" storageClassName: {sc_alert}\n"
|
|
" accessModes: [ReadWriteOnce]\n"
|
|
" resources:\n"
|
|
" requests:\n"
|
|
" storage: 5Gi\n"
|
|
)
|
|
values += (
|
|
"grafana:\n"
|
|
" adminUser: admin\n"
|
|
f" adminPassword: {grafana_password}\n"
|
|
" persistence:\n"
|
|
" type: sts\n"
|
|
" enabled: true\n"
|
|
f" storageClassName: {sc_grafana}\n"
|
|
" accessModes: [ReadWriteOnce]\n"
|
|
" size: 10Gi\n"
|
|
)
|
|
return values
|
|
|
|
|
|
def update(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
log: _LogFn | None = None,
|
|
mode: str | None = None,
|
|
) -> None:
|
|
effective_mode = _detect_mode(mode, env)
|
|
ns = _monitoring_namespace(namespace, env)
|
|
release = _release_name(env)
|
|
chart = str((env or {}).get("MONITORING_CHART") or "prometheus-community/kube-prometheus-stack")
|
|
grafana_password = str((env or {}).get("GRAFANA_ADMIN_PASSWORD") or "prole")
|
|
|
|
_log(log, "[MONITORING] Ensuring helm repos")
|
|
_helm(["repo", "add", "prometheus-community", "https://prometheus-community.github.io/helm-charts"], env=env)
|
|
_helm(["repo", "update"], env=env)
|
|
|
|
_kubectl(["create", "namespace", ns], env=env, timeout=60)
|
|
|
|
if effective_mode == "k8s":
|
|
values_yaml = _values_yaml_k8s(grafana_password)
|
|
else:
|
|
# k3s / k3d: homelab local-PV setup
|
|
values_yaml = _values_yaml_k3s(grafana_password, env)
|
|
|
|
tmp_values = tempfile.NamedTemporaryFile(
|
|
mode="w", suffix=".yaml", prefix="monitoring-values-", delete=False
|
|
)
|
|
try:
|
|
tmp_values.write(values_yaml)
|
|
tmp_values.flush()
|
|
tmp_values.close()
|
|
|
|
_log(log, f"[MONITORING] Deploying {release} in namespace {ns}")
|
|
_helm(
|
|
[
|
|
"upgrade",
|
|
"--install",
|
|
release,
|
|
chart,
|
|
"--namespace",
|
|
ns,
|
|
"-f",
|
|
tmp_values.name,
|
|
"--wait",
|
|
],
|
|
env=env,
|
|
timeout=600,
|
|
check=True,
|
|
)
|
|
|
|
# Cross-cluster: install Prometheus Operator CRDs on the DB cluster so that
|
|
# CNPG (running on knoe-cnpg-0) can create PodMonitor resources without
|
|
# the reconciler looping on "PodMonitor CRD not present".
|
|
db_ctx = str((env or {}).get("DB_CLUSTER_KUBECONTEXT") or "").strip()
|
|
if db_ctx and effective_mode == "k8s":
|
|
_log(log, f"[MONITORING] Installing Prometheus Operator CRDs on DB cluster ({db_ctx})")
|
|
# helm show crds is client-side — clear KUBECONTEXT so no --kube-context flag
|
|
crd_env = {**(env or {}), "KUBECONTEXT": ""}
|
|
crd_res = _helm(["show", "crds", chart], env=crd_env, timeout=60)
|
|
if crd_res.returncode == 0 and crd_res.stdout.strip():
|
|
db_env = {**(env or {}), "KUBECONTEXT": db_ctx}
|
|
_kubectl(
|
|
["apply", "--server-side", "-f", "-"],
|
|
env=db_env,
|
|
input_text=crd_res.stdout,
|
|
timeout=120,
|
|
)
|
|
_log(log, "[MONITORING] Prometheus Operator CRDs installed on DB cluster.")
|
|
else:
|
|
_log(log, "[MONITORING] WARNING: could not fetch CRDs from chart — skipping DB cluster CRD install.")
|
|
finally:
|
|
os.unlink(tmp_values.name)
|
|
|
|
|
|
def restart(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
log: _LogFn | None = None,
|
|
) -> None:
|
|
ns = _monitoring_namespace(namespace, env)
|
|
_log(log, f"[MONITORING] Restarting grafana deployment in namespace {ns}")
|
|
_kubectl(["-n", ns, "rollout", "restart", "deployment/prometheus-grafana"], env=env, timeout=180)
|
|
|
|
|
|
def stop(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
log: _LogFn | None = None,
|
|
) -> None:
|
|
ns = _monitoring_namespace(namespace, env)
|
|
release = _release_name(env)
|
|
_log(log, f"[MONITORING] Uninstalling release {release} in namespace {ns}")
|
|
_helm(["uninstall", release, "--namespace", ns], env=env, timeout=180)
|
|
|
|
|
|
def status(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
) -> bool:
|
|
ns = _monitoring_namespace(namespace, env)
|
|
dep = _kubectl(
|
|
[
|
|
"-n",
|
|
ns,
|
|
"get",
|
|
"deployment",
|
|
"prometheus-grafana",
|
|
"-o",
|
|
"jsonpath={.status.readyReplicas}",
|
|
],
|
|
env=env,
|
|
timeout=20,
|
|
)
|
|
return _to_bool(dep.returncode == 0 and (dep.stdout or "0").strip() not in {"", "0"})
|