chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 11:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(monitoring): stop purge cycle on k3d retry

Browse Source 
Three issues caused the purge-and-reinstall loop:

1. _purge_broken_monitoring deleted ALL PVCs including Bound ones; grafana's
   working PVC was wiped on every retry. Now only delete PVCs in Pending state.

2. Broken-state detection keyed on Pending pods + unbound PVCs, which is true
   during any still-converging install (including ones that timed out but
   whose pods eventually came up). Gate on helm status=='failed' + unbound PVCs.

3. k3d install used --wait, which blocks on all kube-prometheus-stack components
   (prometheus, alertmanager, node-exporter). They converge async after the
   Prometheus operator starts; --wait always timed out. Drop --wait for k3d;
   the status_common_services.sh check verifies readiness independently.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
chrisfu 2026-05-03 12:55:11 -07:00
parent 489593dca2
commit 8dcec846cd
1 changed files with 56 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
knoe/core/ops/monitoring.py
View File

@ -335,15 +335,20 @@ def _values_yaml_k3s(grafana_password: str, env: dict | None) -> str:
def _purge_broken_monitoring(ns: str, release: str, env: dict | None, log: _LogFn | None) -> None:
"""Uninstall a broken monitoring release and delete its stuck PVCs."""
"""Uninstall a broken monitoring release and delete only its unbound (Pending) PVCs.
Bound PVCs are left intact so that any pod that came up after a prior timeout
can still access its data on the next install, and to avoid wiping working state.
"""
_log(log, f"[MONITORING] Purging broken {release} release in {ns}")
_helm(["uninstall", release, "-n", ns], env=env, timeout=300)
pvc_res = _kubectl(["-n", ns, "get", "pvc", "-o", "name", "--no-headers"], env=env, timeout=30)
for pvc in (pvc_res.stdout or "").splitlines():
pvc = pvc.strip()
if pvc:
_log(log, f"[MONITORING] Deleting stuck PVC {pvc}")
_kubectl(["-n", ns, "delete", pvc, "--ignore-not-found=true"], env=env, timeout=60)
pvc_res = _kubectl(["-n", ns, "get", "pvc", "--no-headers"], env=env, timeout=30)
for line in (pvc_res.stdout or "").splitlines():
parts = line.split()
if len(parts) >= 2 and parts[1] == "Pending":
pvc_name = parts[0].strip()
_log(log, f"[MONITORING] Deleting unbound PVC {pvc_name}")
_kubectl(["-n", ns, "delete", f"pvc/{pvc_name}", "--ignore-not-found=true"], env=env, timeout=60)
def update(
@ -373,16 +378,20 @@ def update(
# k3s: homelab local-PV setup pinned to merlin.knoe.org
values_yaml = _values_yaml_k3s(grafana_password, env)
# Detect broken state: stuck Pending pods + unbound PVCs mean a prior deploy
# used wrong storage classes (e.g. merlin-local-iscsi-* in k3d). Purge and
# reinstall so the correct values can take effect.
pods_res = _kubectl(["-n", ns, "get", "pods", "--no-headers"], env=env, timeout=30)
pvcs_res = _kubectl(["-n", ns, "get", "pvc", "--no-headers"], env=env, timeout=30)
has_stuck_pods = any("Pending" in line for line in (pods_res.stdout or "").splitlines() if line.strip())
has_unbound_pvcs = any("Pending" in line for line in (pvcs_res.stdout or "").splitlines() if line.strip())
if has_stuck_pods and has_unbound_pvcs:
_log(log, f"[MONITORING] Stuck Pending pods + unbound PVCs in {ns}; purging broken release before reinstall.")
_purge_broken_monitoring(ns, release, env, log)
# Detect broken state: Helm release is "failed" AND there are unbound PVCs.
# Gating on "failed" (not just Pending pods) avoids purging a still-converging
# install or a release that timed out but whose pods eventually came up.
helm_status = _helm_status_value(release, ns, env)
if helm_status == "failed":
pvcs_res = _kubectl(["-n", ns, "get", "pvc", "--no-headers"], env=env, timeout=30)
has_unbound_pvcs = any(
len(ln.split()) >= 2 and ln.split()[1] == "Pending"
for ln in (pvcs_res.stdout or "").splitlines()
if ln.strip()
)
if has_unbound_pvcs:
_log(log, f"[MONITORING] Helm release {release} is failed with unbound PVCs in {ns}; purging.")
_purge_broken_monitoring(ns, release, env, log)
tmp_values = tempfile.NamedTemporaryFile(
mode="w", suffix=".yaml", prefix="monitoring-values-", delete=False
@ -408,29 +417,37 @@ def update(
if recovered_stale_lock:
_log(log, f"[MONITORING] Retrying helm upgrade --install for {release} after stale-lock recovery.")
# k3d pulls images fresh and has no pre-warmed node caches; allow more time.
helm_timeout_flag = "20m" if effective_mode == "k3d" else "10m"
process_timeout = 1320 if effective_mode == "k3d" else 660 # 20m + 2m buffer
_log(log, f"[MONITORING] Deploying {release} in namespace {ns}")
_helm(
[
"upgrade",
"--install",
release,
chart,
"--namespace",
ns,
"-f",
tmp_values.name,
"--wait",
"--timeout",
helm_timeout_flag,
],
env=env,
timeout=process_timeout,
check=True,
)
if effective_mode == "k3d":
# k3d: the Prometheus operator converges StatefulSets asynchronously
# after Helm exits; --wait would block on all components (prometheus,
# alertmanager, node-exporter) and time out. Let Helm apply resources
# and return immediately; the status check in status_common_services.sh
# verifies readiness independently.
_helm(
[
"upgrade", "--install", release, chart,
"--namespace", ns,
"-f", tmp_values.name,
"--timeout", "5m",
],
env=env,
timeout=360,
check=True,
)
else:
_helm(
[
"upgrade", "--install", release, chart,
"--namespace", ns,
"-f", tmp_values.name,
"--wait",
"--timeout", "10m",
],
env=env,
timeout=660,
check=True,
)
# Cross-cluster: install Prometheus Operator CRDs on the DB cluster so that
# CNPG (running on knoe-cnpg-0) can create PodMonitor resources without