prole/knoe/core/ops/garage_store.py
chrisfu 15890187a3 Stabilize CNPG reset/update flow and finalize 3-node recovery
- add idempotent CNPG operator reconciliation to prevent duplicate active controller ReplicaSets
- restore and validate CNPG 3-instance convergence with PV claimRef recovery and storage path readiness
- wire canonical launcher aliases/reset behavior and improve namespace cleanup semantics
- harden backup/objectstore readiness handling and retry behavior
- enforce service namespace usage for common services and remove default-namespace drift
- enable Kerberos milestone auto-activation when realm/kdc are configured and apply CNPG GSS pg_hba rules
- keep final runtime healthy: knoe-db 3/3, operator stable, perfsnap captured

Co-authored-by: Junie <junie@jetbrains.com>
2026-03-27 21:54:53 -07:00

235 lines
7.1 KiB
Python

from __future__ import annotations
import json
import secrets
from pathlib import Path
from ._services_common import (
_LogFn,
_detect_mode,
_ensure_namespace,
_exists,
_kubectl,
_log,
_manifest_path,
_prune_named_workload_other_namespaces,
_to_bool,
_wait_rollout,
)
def _garage_namespace(namespace: str | None, env: dict | None) -> str:
service_ns = str((env or {}).get("SERVICE_NAMESPACE") or "").strip()
if env:
explicit = str(env.get("GARAGE_NAMESPACE") or "").strip()
if explicit:
return service_ns if explicit == "default" and service_ns else explicit
raw = (namespace or "").strip() or service_ns or str((env or {}).get("NAMESPACE") or "").strip() or "knoe-system"
if raw == "default":
return service_ns or "knoe-system"
return raw
def _manifest_files(project_root: str | Path) -> list[Path]:
root = Path(project_root)
return [
_manifest_path(root, "k8s", "prole", "storageclass-synology-iscsi.yaml"),
_manifest_path(root, "k8s", "prole", "iscsi-pvs.yaml"),
_manifest_path(root, "k8s", "prole", "garage-configmap.yaml"),
_manifest_path(root, "k8s", "prole", "garage-statefulset.yaml"),
_manifest_path(root, "k8s", "prole", "garage-service.yaml"),
]
def _repair_released_garage_pvs(*, env: dict | None = None, log: _LogFn | None = None) -> None:
listed = _kubectl(
["get", "pv", "-l", "synology.storage/role=garage", "-o", "json"],
env=env,
timeout=45,
)
if listed.returncode != 0 or not listed.stdout.strip():
return
try:
payload = json.loads(listed.stdout)
except Exception:
return
for item in payload.get("items", []):
meta = item.get("metadata") or {}
spec = item.get("spec") or {}
status = item.get("status") or {}
name = str(meta.get("name") or "").strip()
phase = str(status.get("phase") or "").strip()
if not name or phase != "Released":
continue
if not spec.get("claimRef"):
continue
_log(log, f"[GARAGE] Clearing stale claimRef on PV {name} (phase=Released)")
_kubectl(
[
"patch",
"pv",
name,
"--type=json",
"-p",
'[{"op":"remove","path":"/spec/claimRef"}]',
],
env=env,
timeout=60,
)
def _ensure_garage_secret(*, namespace: str, env: dict | None = None, log: _LogFn | None = None) -> None:
secret_name = str((env or {}).get("GARAGE_SECRET_NAME") or "garage-secrets").strip() or "garage-secrets"
force = _to_bool((env or {}).get("PROLE_GARAGE_FORCE_SECRET"), default=False)
if not force and _exists("secret", secret_name, namespace, env=env):
return
payload = (
"apiVersion: v1\n"
"kind: Secret\n"
"metadata:\n"
f" name: {secret_name}\n"
"type: Opaque\n"
"stringData:\n"
f" rpc_secret: {json.dumps(secrets.token_hex(32))}\n"
f" admin_token: {json.dumps(secrets.token_urlsafe(32))}\n"
f" metrics_token: {json.dumps(secrets.token_urlsafe(32))}\n"
)
_kubectl(
["-n", namespace, "apply", "-f", "-"],
env=env,
input_text=payload,
timeout=90,
check=True,
)
_log(log, f"[GARAGE] Ensured secret/{secret_name} in namespace {namespace}")
def initialize(
*,
namespace: str | None = None,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str | None = None,
) -> None:
update(namespace=namespace, env=env, project_root=project_root, log=log, mode=mode)
def start(
*,
namespace: str | None = None,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str | None = None,
) -> None:
update(namespace=namespace, env=env, project_root=project_root, log=log, mode=mode)
def update(
*,
namespace: str | None = None,
env: dict | None = None,
project_root: str | Path = ".",
log: _LogFn | None = None,
mode: str | None = None,
) -> None:
_detect_mode(mode, env) # mode kept for API parity
target_ns = _garage_namespace(namespace, env)
_ensure_namespace(target_ns, env)
_repair_released_garage_pvs(env=env, log=log)
_ensure_garage_secret(namespace=target_ns, env=env, log=log)
_prune_named_workload_other_namespaces(
kind="statefulset",
name="garage",
target_namespace=target_ns,
label_selector="app=garage",
env=env,
log=log,
)
_prune_named_workload_other_namespaces(
kind="service",
name="garage",
target_namespace=target_ns,
label_selector="app=garage",
env=env,
log=log,
)
_prune_named_workload_other_namespaces(
kind="configmap",
name="garage-config",
target_namespace=target_ns,
label_selector="app=garage",
env=env,
log=log,
)
for manifest in _manifest_files(project_root):
if not manifest.exists():
continue
_log(log, f"[GARAGE] Applying {manifest}")
name = manifest.name
if name in {"storageclass-synology-iscsi.yaml", "iscsi-pvs.yaml"}:
_kubectl(["apply", "-f", str(manifest)], env=env, timeout=240, check=True)
else:
_kubectl(["-n", target_ns, "apply", "-f", str(manifest)], env=env, timeout=240, check=True)
if _exists("statefulset", "garage", target_ns, env=env):
_wait_rollout("statefulset", "garage", target_ns, env=env)
def stop(
*,
namespace: str | None = None,
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
target_ns = _garage_namespace(namespace, env)
if _exists("statefulset", "garage", target_ns, env=env):
_log(log, f"[GARAGE] Scaling statefulset/garage to 0 in namespace {target_ns}")
_kubectl(
["-n", target_ns, "scale", "statefulset/garage", "--replicas=0"],
env=env,
timeout=90,
check=True,
)
def restart(
*,
namespace: str | None = None,
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
target_ns = _garage_namespace(namespace, env)
if _exists("statefulset", "garage", target_ns, env=env):
_log(log, f"[GARAGE] Restarting statefulset/garage in namespace {target_ns}")
_kubectl(
["-n", target_ns, "rollout", "restart", "statefulset/garage"],
env=env,
timeout=120,
check=True,
)
_wait_rollout("statefulset", "garage", target_ns, env=env)
def status(
*,
namespace: str | None = None,
env: dict | None = None,
) -> bool:
target_ns = _garage_namespace(namespace, env)
sts = _kubectl(
["-n", target_ns, "get", "statefulset", "garage", "-o", "jsonpath={.status.readyReplicas}"],
env=env,
timeout=20,
)
svc = _kubectl(["-n", target_ns, "get", "svc", "garage"], env=env, timeout=20)
return _to_bool(
sts.returncode == 0 and (sts.stdout or "0").strip() not in {"", "0"} and svc.returncode == 0
)