mirror of
https://github.com/dredx/prole.git
synced 2026-09-27 14:34:30 +00:00
- add idempotent CNPG operator reconciliation to prevent duplicate active controller ReplicaSets - restore and validate CNPG 3-instance convergence with PV claimRef recovery and storage path readiness - wire canonical launcher aliases/reset behavior and improve namespace cleanup semantics - harden backup/objectstore readiness handling and retry behavior - enforce service namespace usage for common services and remove default-namespace drift - enable Kerberos milestone auto-activation when realm/kdc are configured and apply CNPG GSS pg_hba rules - keep final runtime healthy: knoe-db 3/3, operator stable, perfsnap captured Co-authored-by: Junie <junie@jetbrains.com>
235 lines
7.1 KiB
Python
235 lines
7.1 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import secrets
|
|
from pathlib import Path
|
|
|
|
from ._services_common import (
|
|
_LogFn,
|
|
_detect_mode,
|
|
_ensure_namespace,
|
|
_exists,
|
|
_kubectl,
|
|
_log,
|
|
_manifest_path,
|
|
_prune_named_workload_other_namespaces,
|
|
_to_bool,
|
|
_wait_rollout,
|
|
)
|
|
|
|
|
|
def _garage_namespace(namespace: str | None, env: dict | None) -> str:
|
|
service_ns = str((env or {}).get("SERVICE_NAMESPACE") or "").strip()
|
|
if env:
|
|
explicit = str(env.get("GARAGE_NAMESPACE") or "").strip()
|
|
if explicit:
|
|
return service_ns if explicit == "default" and service_ns else explicit
|
|
|
|
raw = (namespace or "").strip() or service_ns or str((env or {}).get("NAMESPACE") or "").strip() or "knoe-system"
|
|
if raw == "default":
|
|
return service_ns or "knoe-system"
|
|
return raw
|
|
|
|
|
|
def _manifest_files(project_root: str | Path) -> list[Path]:
|
|
root = Path(project_root)
|
|
return [
|
|
_manifest_path(root, "k8s", "prole", "storageclass-synology-iscsi.yaml"),
|
|
_manifest_path(root, "k8s", "prole", "iscsi-pvs.yaml"),
|
|
_manifest_path(root, "k8s", "prole", "garage-configmap.yaml"),
|
|
_manifest_path(root, "k8s", "prole", "garage-statefulset.yaml"),
|
|
_manifest_path(root, "k8s", "prole", "garage-service.yaml"),
|
|
]
|
|
|
|
|
|
def _repair_released_garage_pvs(*, env: dict | None = None, log: _LogFn | None = None) -> None:
|
|
listed = _kubectl(
|
|
["get", "pv", "-l", "synology.storage/role=garage", "-o", "json"],
|
|
env=env,
|
|
timeout=45,
|
|
)
|
|
if listed.returncode != 0 or not listed.stdout.strip():
|
|
return
|
|
try:
|
|
payload = json.loads(listed.stdout)
|
|
except Exception:
|
|
return
|
|
|
|
for item in payload.get("items", []):
|
|
meta = item.get("metadata") or {}
|
|
spec = item.get("spec") or {}
|
|
status = item.get("status") or {}
|
|
name = str(meta.get("name") or "").strip()
|
|
phase = str(status.get("phase") or "").strip()
|
|
if not name or phase != "Released":
|
|
continue
|
|
if not spec.get("claimRef"):
|
|
continue
|
|
_log(log, f"[GARAGE] Clearing stale claimRef on PV {name} (phase=Released)")
|
|
_kubectl(
|
|
[
|
|
"patch",
|
|
"pv",
|
|
name,
|
|
"--type=json",
|
|
"-p",
|
|
'[{"op":"remove","path":"/spec/claimRef"}]',
|
|
],
|
|
env=env,
|
|
timeout=60,
|
|
)
|
|
|
|
|
|
def _ensure_garage_secret(*, namespace: str, env: dict | None = None, log: _LogFn | None = None) -> None:
|
|
secret_name = str((env or {}).get("GARAGE_SECRET_NAME") or "garage-secrets").strip() or "garage-secrets"
|
|
force = _to_bool((env or {}).get("PROLE_GARAGE_FORCE_SECRET"), default=False)
|
|
if not force and _exists("secret", secret_name, namespace, env=env):
|
|
return
|
|
|
|
payload = (
|
|
"apiVersion: v1\n"
|
|
"kind: Secret\n"
|
|
"metadata:\n"
|
|
f" name: {secret_name}\n"
|
|
"type: Opaque\n"
|
|
"stringData:\n"
|
|
f" rpc_secret: {json.dumps(secrets.token_hex(32))}\n"
|
|
f" admin_token: {json.dumps(secrets.token_urlsafe(32))}\n"
|
|
f" metrics_token: {json.dumps(secrets.token_urlsafe(32))}\n"
|
|
)
|
|
_kubectl(
|
|
["-n", namespace, "apply", "-f", "-"],
|
|
env=env,
|
|
input_text=payload,
|
|
timeout=90,
|
|
check=True,
|
|
)
|
|
_log(log, f"[GARAGE] Ensured secret/{secret_name} in namespace {namespace}")
|
|
|
|
|
|
def initialize(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
project_root: str | Path = ".",
|
|
log: _LogFn | None = None,
|
|
mode: str | None = None,
|
|
) -> None:
|
|
update(namespace=namespace, env=env, project_root=project_root, log=log, mode=mode)
|
|
|
|
|
|
def start(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
project_root: str | Path = ".",
|
|
log: _LogFn | None = None,
|
|
mode: str | None = None,
|
|
) -> None:
|
|
update(namespace=namespace, env=env, project_root=project_root, log=log, mode=mode)
|
|
|
|
|
|
def update(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
project_root: str | Path = ".",
|
|
log: _LogFn | None = None,
|
|
mode: str | None = None,
|
|
) -> None:
|
|
_detect_mode(mode, env) # mode kept for API parity
|
|
target_ns = _garage_namespace(namespace, env)
|
|
_ensure_namespace(target_ns, env)
|
|
_repair_released_garage_pvs(env=env, log=log)
|
|
_ensure_garage_secret(namespace=target_ns, env=env, log=log)
|
|
|
|
_prune_named_workload_other_namespaces(
|
|
kind="statefulset",
|
|
name="garage",
|
|
target_namespace=target_ns,
|
|
label_selector="app=garage",
|
|
env=env,
|
|
log=log,
|
|
)
|
|
_prune_named_workload_other_namespaces(
|
|
kind="service",
|
|
name="garage",
|
|
target_namespace=target_ns,
|
|
label_selector="app=garage",
|
|
env=env,
|
|
log=log,
|
|
)
|
|
_prune_named_workload_other_namespaces(
|
|
kind="configmap",
|
|
name="garage-config",
|
|
target_namespace=target_ns,
|
|
label_selector="app=garage",
|
|
env=env,
|
|
log=log,
|
|
)
|
|
|
|
for manifest in _manifest_files(project_root):
|
|
if not manifest.exists():
|
|
continue
|
|
_log(log, f"[GARAGE] Applying {manifest}")
|
|
name = manifest.name
|
|
if name in {"storageclass-synology-iscsi.yaml", "iscsi-pvs.yaml"}:
|
|
_kubectl(["apply", "-f", str(manifest)], env=env, timeout=240, check=True)
|
|
else:
|
|
_kubectl(["-n", target_ns, "apply", "-f", str(manifest)], env=env, timeout=240, check=True)
|
|
|
|
if _exists("statefulset", "garage", target_ns, env=env):
|
|
_wait_rollout("statefulset", "garage", target_ns, env=env)
|
|
|
|
|
|
def stop(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
log: _LogFn | None = None,
|
|
) -> None:
|
|
target_ns = _garage_namespace(namespace, env)
|
|
if _exists("statefulset", "garage", target_ns, env=env):
|
|
_log(log, f"[GARAGE] Scaling statefulset/garage to 0 in namespace {target_ns}")
|
|
_kubectl(
|
|
["-n", target_ns, "scale", "statefulset/garage", "--replicas=0"],
|
|
env=env,
|
|
timeout=90,
|
|
check=True,
|
|
)
|
|
|
|
|
|
def restart(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
log: _LogFn | None = None,
|
|
) -> None:
|
|
target_ns = _garage_namespace(namespace, env)
|
|
if _exists("statefulset", "garage", target_ns, env=env):
|
|
_log(log, f"[GARAGE] Restarting statefulset/garage in namespace {target_ns}")
|
|
_kubectl(
|
|
["-n", target_ns, "rollout", "restart", "statefulset/garage"],
|
|
env=env,
|
|
timeout=120,
|
|
check=True,
|
|
)
|
|
_wait_rollout("statefulset", "garage", target_ns, env=env)
|
|
|
|
|
|
def status(
|
|
*,
|
|
namespace: str | None = None,
|
|
env: dict | None = None,
|
|
) -> bool:
|
|
target_ns = _garage_namespace(namespace, env)
|
|
sts = _kubectl(
|
|
["-n", target_ns, "get", "statefulset", "garage", "-o", "jsonpath={.status.readyReplicas}"],
|
|
env=env,
|
|
timeout=20,
|
|
)
|
|
svc = _kubectl(["-n", target_ns, "get", "svc", "garage"], env=env, timeout=20)
|
|
return _to_bool(
|
|
sts.returncode == 0 and (sts.stdout or "0").strip() not in {"", "0"} and svc.returncode == 0
|
|
)
|