prole/knoe/core/ops/_garage_common.py
chrisfu b1755f4e7b Stabilize prod deployment flow and harden Garage/CNPG ops
Switch production config to k8s/GKE contexts and align service naming. Add immutable StatefulSet update fallback for Garage across k3d/k3s/k8s. Harden CNPG deploy and backup bootstrap paths, and update installer coverage for CNPG webhook and Garage common ops.

Co-authored-by: Junie <junie@jetbrains.com>
2026-04-11 20:19:36 -07:00

96 lines
3.0 KiB
Python

"""Shared helpers for Garage object-store ops across all deployment modes."""
from __future__ import annotations
import json
import secrets
from pathlib import Path
from ._services_common import (
_LogFn,
_exists,
_kubectl,
_log,
_to_bool,
)
_IMMUTABLE_STATEFULSET_MARKERS = (
"updates to statefulset spec for fields other than",
"field is immutable",
)
def _garage_namespace(namespace: str | None, env: dict | None) -> str:
service_ns = str((env or {}).get("SERVICE_NAMESPACE") or "").strip()
if env:
explicit = str(env.get("GARAGE_NAMESPACE") or "").strip()
if explicit:
return service_ns if explicit == "default" and service_ns else explicit
raw = (namespace or "").strip() or service_ns or str((env or {}).get("NAMESPACE") or "").strip() or "knoe-system"
if raw == "default":
return service_ns or "knoe-system"
return raw
def _ensure_garage_secret(*, namespace: str, env: dict | None = None, log: _LogFn | None = None) -> None:
secret_name = str((env or {}).get("GARAGE_SECRET_NAME") or "garage-secrets").strip() or "garage-secrets"
force = _to_bool((env or {}).get("PROLE_GARAGE_FORCE_SECRET"), default=False)
if not force and _exists("secret", secret_name, namespace, env=env):
return
payload = (
"apiVersion: v1\n"
"kind: Secret\n"
"metadata:\n"
f" name: {secret_name}\n"
"type: Opaque\n"
"stringData:\n"
f" rpc_secret: {json.dumps(secrets.token_hex(32))}\n"
f" admin_token: {json.dumps(secrets.token_urlsafe(32))}\n"
f" metrics_token: {json.dumps(secrets.token_urlsafe(32))}\n"
)
_kubectl(
["-n", namespace, "apply", "-f", "-"],
env=env,
input_text=payload,
timeout=90,
check=True,
)
_log(log, f"[GARAGE] Ensured secret/{secret_name} in namespace {namespace}")
def _apply_garage_statefulset_with_recreate(
*,
namespace: str,
manifest: str | Path,
env: dict | None = None,
log: _LogFn | None = None,
) -> None:
manifest_path = str(manifest)
apply_args = ["-n", namespace, "apply", "-f", manifest_path]
res = _kubectl(apply_args, env=env, timeout=240)
if res.returncode == 0:
return
stderr = (res.stderr or "").lower()
if any(marker in stderr for marker in _IMMUTABLE_STATEFULSET_MARKERS):
_log(
log,
f"[GARAGE] StatefulSet immutable spec conflict detected; recreating statefulset/garage in namespace {namespace}",
)
_kubectl(
["-n", namespace, "delete", "statefulset", "garage", "--ignore-not-found=true", "--wait=true"],
env=env,
timeout=300,
check=True,
)
_kubectl(apply_args, env=env, timeout=240, check=True)
return
raise RuntimeError(
f"Command failed ({res.returncode}): kubectl {' '.join(apply_args)}\n"
f"stdout: {res.stdout}\n"
f"stderr: {res.stderr}"
)