feat(mock_val): add diagnostics, utilities, and misc operational scripts

Diagnostics:
  diag_gitlab_boot.sh, diag_gitlab_webservice_oom.sh, diag_gke_storage.sh

Utilities:
  ensure_default_storage_class.sh — set/verify default StorageClass
  preflight_kubecontext.sh        — validate kubecontext before ops
  onboard_engineer.sh             — new engineer onboarding script
  gen_oidc_signing_key.sh         — generate OIDC signing key
  fetch_prole_secrets.sh          — pull secrets from vault
  set-k3s-token-1password.sh      — store k3s token in 1Password
  sync_cnpg_grafana_dashboard.py  — sync CNPG dashboard to Grafana

Config/certs:
  krb5.local.conf, knoe-db-ca.crt

Updated: build-a-bao.sh, hostprobe-*.yaml, hosts.txt, knoe-db-passwwd.sh,
         repair_pipeline.sh, status.sh, status_common_services.sh
Co-authored-by: Junie <junie@jetbrains.com>

mock_val/build-a-bao.sh

@@ -4,9 +4,9 @@ set -euo pipefail
 
 # build-a-bao.sh
 # Purpose:
-# - Decrypt temporary secrets stored in knoe.cfg
+# - Decrypt temporary secrets stored in the active config file
 # - Store them in OpenBao for the current namespace
-# - Replace knoe.cfg secrets with OpenBao placeholders
+# - Replace config-file secrets with OpenBao placeholders
 
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
@@ -15,16 +15,16 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 source "$SCRIPT_DIR/knoe_cfg.sh"
 
 cfg_path=""
-if [[ -n "${KNOE_CONF:-}" && -f "$KNOE_CONF/knoe.cfg" ]]; then
+if [[ -n "${KNOE_CONF:-}" ]]; then
-  cfg_path="$KNOE_CONF/knoe.cfg"
+  cfg_path="$(_knoe_cfg_select_cfg_file "$KNOE_CONF")"
-elif [[ -n "${KNOE_HOME:-}" && -f "$KNOE_HOME/conf/knoe.cfg" ]]; then
+elif [[ -n "${KNOE_HOME:-}" ]]; then
-  cfg_path="$KNOE_HOME/conf/knoe.cfg"
+  cfg_path="$(_knoe_cfg_select_cfg_file "$KNOE_HOME/conf")"
-elif [[ -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then
+else
-  cfg_path="$SCRIPT_DIR/../conf/knoe.cfg"
+  cfg_path="$(_knoe_cfg_select_cfg_file "$SCRIPT_DIR/../conf")"
 fi
 
 if [[ -z "$cfg_path" || ! -f "$cfg_path" ]]; then
-  echo "ERROR: knoe.cfg not found. Set KNOE_CONF or KNOE_HOME." >&2
+  echo "ERROR: config file not found. Set KNOE_CONF or KNOE_HOME." >&2
   exit 1
 fi
 

mock_val/diag_gitlab_boot.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# diag_gitlab_boot.sh — read-only snapshot of GitLab backend health.
+#
+# Investigates why the frontdoor returns "Waiting for GitLab to boot" even
+# after DNS + ingress are green. Focuses on:
+#   - webservice pod states + last-terminated reasons (OOM, probe failures)
+#   - webservice + workhorse log tail on the likely-serving pod
+#   - Gitaly readiness
+#   - recent Warning events
+#
+# Usage: ./etc/diag_gitlab_boot.sh [--namespace NS] [--kube-context CTX]
+#   --namespace defaults to 'gitlab'
+
+set -euo pipefail
+
+NS="gitlab"
+KUBE_CONTEXT=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --namespace) NS="$2"; shift 2 ;;
+    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@"
+  else
+    kubectl -n "$NS" "$@"
+  fi
+}
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+WEB_JSON="$TMPDIR/web.json"
+kctl get pods -l app=webservice -o json > "$WEB_JSON"
+
+echo "=== Namespace: $NS ==="
+echo
+echo "=== Webservice pods (restarts + last termination) ==="
+
+PY_SCRIPT="$TMPDIR/analyze.py"
+cat > "$PY_SCRIPT" <<'PY'
+import json, sys
+
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+
+pods = data.get("items") or []
+if not pods:
+    print("  (no webservice pods found)")
+
+for pod in pods:
+    name = (pod.get("metadata") or {}).get("name", "")
+    phase = (pod.get("status") or {}).get("phase", "")
+    print(f"\n-- {name}  phase={phase} --")
+    for cs in (pod.get("status") or {}).get("containerStatuses") or []:
+        cname = cs.get("name", "")
+        ready = cs.get("ready")
+        restart = cs.get("restartCount", 0)
+        state = cs.get("state") or {}
+        last = cs.get("lastState") or {}
+        state_key = next(iter(state.keys()), "unknown")
+        state_detail = state.get(state_key) or {}
+        last_term = (last.get("terminated") or {}) if last else {}
+        print(f"  container={cname:<22} ready={ready} restarts={restart} state={state_key}")
+        if state_detail.get("reason"):
+            print(f"    state.reason={state_detail.get('reason')} msg={(state_detail.get('message') or '')[:180]}")
+        if last_term:
+            print(f"    last terminated: exit={last_term.get('exitCode')} reason={last_term.get('reason')} "
+                  f"signal={last_term.get('signal')}")
+            print(f"    last start={last_term.get('startedAt')} finish={last_term.get('finishedAt')}")
+            if last_term.get("message"):
+                print(f"    last msg: {(last_term.get('message') or '')[:240]}")
+
+# Emit the best-candidate pod name to a sidecar file so the shell can pick it up
+def score(p):
+    cs = (p.get("status") or {}).get("containerStatuses") or []
+    ready = all(c.get("ready") for c in cs) if cs else False
+    restarts = max((c.get("restartCount", 0) for c in cs), default=0)
+    return (1 if ready else 0, -restarts)
+
+pods.sort(key=score, reverse=True)
+target = pods[0]["metadata"]["name"] if pods else ""
+with open(sys.argv[2], "w") as f:
+    f.write(target)
+PY
+
+TARGET_FILE="$TMPDIR/target.txt"
+python3 "$PY_SCRIPT" "$WEB_JSON" "$TARGET_FILE"
+POD="$(cat "$TARGET_FILE")"
+
+echo
+echo "=== Most-likely serving webservice pod: tail 60 lines (webservice container) ==="
+if [[ -n "$POD" ]]; then
+  echo "Pod: $POD"
+  kctl logs "$POD" -c webservice --tail=60 2>&1 | sed 's/^/  /' || true
+  echo
+  echo "=== ... gitlab-workhorse container (tail 30) ==="
+  kctl logs "$POD" -c gitlab-workhorse --tail=30 2>&1 | sed 's/^/  /' || true
+else
+  echo "  (no webservice pod found)"
+fi
+
+echo
+echo "=== Gitaly pods ==="
+kctl get pods -l app=gitaly -o wide || true
+
+echo
+echo "=== Recent Warning events (last 25) ==="
+kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 25 || true
+
+echo
+echo "=== GitLab CR status (if present) ==="
+GL_JSON="$TMPDIR/gl.json"
+if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then
+  python3 - "$GL_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for it in data.get("items") or []:
+    m = it.get("metadata") or {}
+    s = it.get("status") or {}
+    print(f"  {m.get('name')}: phase={s.get('phase','?')}")
+    for c in (s.get("conditions") or [])[:10]:
+        print(f"    {c.get('type')}={c.get('status')} reason={c.get('reason')} "
+              f"msg={(c.get('message') or '')[:140]}")
+PY
+else
+  echo "  (no GitLab CR or CRD not installed)"
+fi

mock_val/diag_gitlab_webservice_oom.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# diag_gitlab_webservice_oom.sh — memory ceilings + Puma tuning for webservice.
+#
+# Read-only. Prints:
+#   1. webservice container resources (requests/limits)
+#   2. Puma worker/thread env vars from pod spec
+#   3. node allocatable memory so we know upper bound
+#   4. prometheus-style /metrics snapshot from the running pod if available
+#
+# Usage: ./etc/diag_gitlab_webservice_oom.sh [--namespace NS] [--kube-context CTX]
+
+set -euo pipefail
+
+NS="gitlab"
+KUBE_CONTEXT=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --namespace) NS="$2"; shift 2 ;;
+    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@"
+  else
+    kubectl -n "$NS" "$@"
+  fi
+}
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+DEPLOY_JSON="$TMPDIR/deploy.json"
+POD_JSON="$TMPDIR/pod.json"
+NODE_JSON="$TMPDIR/node.json"
+
+kctl get deployment -l app=webservice -o json > "$DEPLOY_JSON"
+kctl get pods -l app=webservice -o json > "$POD_JSON"
+
+echo "=== webservice container resources (from deployment) ==="
+python3 - "$DEPLOY_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for d in data.get("items") or []:
+    name = (d.get("metadata") or {}).get("name", "")
+    print(f"\n-- deployment/{name} --")
+    spec = ((d.get("spec") or {}).get("template") or {}).get("spec") or {}
+    for c in spec.get("containers") or []:
+        cname = c.get("name", "")
+        res = c.get("resources") or {}
+        req = res.get("requests") or {}
+        lim = res.get("limits") or {}
+        print(f"  container={cname}")
+        print(f"    requests: cpu={req.get('cpu','?')}  mem={req.get('memory','?')}")
+        print(f"    limits:   cpu={lim.get('cpu','?')}  mem={lim.get('memory','?')}")
+        # Pick out Puma-relevant env vars
+        env = c.get("env") or []
+        interesting = {
+            "PUMA_WORKERS", "PUMA_THREADS_MIN", "PUMA_THREADS_MAX",
+            "WORKER_PROCESSES", "SIDEKIQ_CONCURRENCY",
+            "GITLAB_MEMORY_WATCHDOG_ENABLED", "GITLAB_MEMORY_WATCHDOG_PUMA_ENABLED",
+            "GITLAB_MEMORY_WATCHDOG_MAX_HEAP_FRAG_THRESHOLD_MB",
+            "GITLAB_MEMORY_WATCHDOG_MAX_STRIKES",
+            "MALLOC_ARENA_MAX",
+        }
+        seen = [e for e in env if e.get("name") in interesting]
+        if seen:
+            print("    puma/watchdog env:")
+            for e in seen:
+                val = e.get("value", "<valueFrom>") if "value" in e else "<valueFrom>"
+                print(f"      {e.get('name')}={val}")
+PY
+
+echo
+echo "=== Node memory allocatable ==="
+kctl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory,MEM_CAPACITY:.status.capacity.memory 2>/dev/null \
+  || kubectl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory
+
+echo
+echo "=== Running webservice pod memory usage (kubectl top if metrics-server present) ==="
+kctl top pod -l app=webservice --containers 2>&1 || echo "  (metrics-server not installed or not returning data)"
+
+echo
+echo "=== Pod QoS class + restart history ==="
+python3 - "$POD_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for p in data.get("items") or []:
+    m = p.get("metadata") or {}
+    s = p.get("status") or {}
+    cs = s.get("containerStatuses") or []
+    ws = next((c for c in cs if c.get("name") == "webservice"), None)
+    print(f"  {m.get('name'):<48}  qos={s.get('qosClass','?'):<10} "
+          f"phase={s.get('phase','?'):<10} restarts={ws.get('restartCount', '?') if ws else '?'}")
+PY
+
+echo
+echo "=== Last 40 Warning events (whole namespace, for noise diagnosis) ==="
+kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 40 || true
+
+echo
+echo "=== All PVCs in namespace (what the operator might be trying to reconcile) ==="
+kctl get pvc -o wide || true
+
+echo
+echo "=== GitLab operator CR spec.gitaly (replicas + storage) ==="
+GL_JSON="$TMPDIR/gl.json"
+if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then
+  python3 - "$GL_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for g in data.get("items") or []:
+    m = g.get("metadata") or {}
+    name = m.get("name", "")
+    chart = ((g.get("spec") or {}).get("chart") or {}).get("values") or {}
+    gitaly = ((chart.get("global") or {}).get("gitaly") or {})
+    wsvc = (chart.get("gitlab") or {}).get("webservice") or {}
+    print(f"  gitlab/{name}")
+    print(f"    global.gitaly: {json.dumps(gitaly, indent=6)[:800]}")
+    print(f"    gitlab.webservice.replicas: {wsvc.get('replicaCount', wsvc.get('minReplicas','?'))}")
+    print(f"    gitlab.webservice.resources: {json.dumps(wsvc.get('resources', {}), indent=6)}")
+    print(f"    gitlab.webservice.workerProcesses: {wsvc.get('workerProcesses','?')}")
+PY
+else
+  echo "  (no GitLab CR or not readable)"
+fi

mock_val/diag_gke_storage.sh

@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+# diag_gke_storage.sh — inspect StorageClass + SSD quota state on a GKE cluster.
+#
+# Safe, read-only. Prints:
+#   1. Every StorageClass, whether it's default, and its underlying pd type
+#   2. All PVs grouped by storage class, with capacity + bound claim
+#   3. Totals: how many GiB are pinned to each disk type (pd-ssd / pd-balanced
+#      count against SSD_TOTAL_GB; pd-standard is the separate HDD quota)
+#   4. Any PVC stuck in Pending with a QUOTA_EXCEEDED / ProvisioningFailed event
+#
+# Usage: ./etc/diag_gke_storage.sh [--kube-context CTX]
+
+set -euo pipefail
+
+KUBE_CONTEXT=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --kube-context) KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" "$@"
+  else
+    kubectl "$@"
+  fi
+}
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+SC_JSON="$TMPDIR/sc.json"
+PV_JSON="$TMPDIR/pv.json"
+PVC_JSON="$TMPDIR/pvc.json"
+PY_SCRIPT="$TMPDIR/diag.py"
+
+kctl get storageclass -o json > "$SC_JSON"
+kctl get pv -o json > "$PV_JSON"
+kctl get pvc -A -o json > "$PVC_JSON"
+
+# Single python script does all four sections — reads the three JSON files by path.
+cat > "$PY_SCRIPT" <<'PY'
+import json, re, sys
+from collections import defaultdict
+
+sc_path, pv_path, pvc_path = sys.argv[1], sys.argv[2], sys.argv[3]
+
+with open(sc_path) as f:
+    scs_raw = json.load(f).get("items") or []
+with open(pv_path) as f:
+    pvs = json.load(f).get("items") or []
+with open(pvc_path) as f:
+    pvcs = json.load(f).get("items") or []
+
+# --- Section 1: StorageClasses ------------------------------------------------
+print("=== StorageClasses ===")
+print(f"{'NAME':<24} {'PROVISIONER':<30} {'DISK_TYPE':<14} {'DEFAULT':<7} {'RECLAIM'}")
+sc_by_name = {}
+for sc in scs_raw:
+    m = sc.get("metadata") or {}
+    name = m.get("name", "")
+    ann = m.get("annotations") or {}
+    is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "")).lower() == "true"
+    prov = sc.get("provisioner", "") or ""
+    disk = ((sc.get("parameters") or {}).get("type") or "")
+    reclaim = sc.get("reclaimPolicy", "") or ""
+    sc_by_name[name] = {"disk": disk, "prov": prov, "default": is_default}
+    print(f"{name:<24} {prov:<30} {disk:<14} {'YES' if is_default else '-':<7} {reclaim}")
+
+# --- Section 2: PVs grouped by SC --------------------------------------------
+print()
+print("=== PVs by StorageClass ===")
+groups = defaultdict(list)
+for pv in pvs:
+    spec = pv.get("spec", {}) or {}
+    sc = spec.get("storageClassName", "") or "<none>"
+    cap = (spec.get("capacity") or {}).get("storage", "")
+    phase = (pv.get("status") or {}).get("phase", "")
+    cr = spec.get("claimRef") or {}
+    claim = f"{cr.get('namespace','')}/{cr.get('name','')}" if cr else ""
+    groups[sc].append((pv.get("metadata", {}).get("name", ""), cap, phase, claim))
+
+if not groups:
+    print("  (no PVs)")
+else:
+    for sc, rows in sorted(groups.items()):
+        print(f"-- {sc} --")
+        for name, cap, phase, claim in rows:
+            print(f"  {name:<50} {cap:<8} {phase:<10} {claim}")
+
+# --- Section 3: GiB per disk type --------------------------------------------
+def to_gib(cap):
+    if not cap:
+        return 0.0
+    m = re.match(r"^(\d+(?:\.\d+)?)\s*([KMGTP]i?)?$", cap)
+    if not m:
+        return 0.0
+    n = float(m.group(1))
+    u = m.group(2) or ""
+    unit = {
+        "": 1/1024**3, "Ki": 1/1024**2, "Mi": 1/1024, "Gi": 1, "Ti": 1024, "Pi": 1024**2,
+        "K": 1e3/1024**3, "M": 1e6/1024**3, "G": 1e9/1024**3, "T": 1e12/1024**3,
+    }.get(u, 0)
+    return n * unit
+
+print()
+print("=== GiB per disk type (what counts against which quota) ===")
+totals = defaultdict(float)
+for pv in pvs:
+    spec = pv.get("spec", {}) or {}
+    sc = spec.get("storageClassName", "") or "<none>"
+    disk = sc_by_name.get(sc, {}).get("disk", "<unknown-sc>")
+    cap = (spec.get("capacity") or {}).get("storage", "")
+    totals[disk] += to_gib(cap)
+
+ssd_quota = 0.0
+hdd_quota = 0.0
+if not totals:
+    print("  (no PVs)")
+else:
+    for disk, gib in sorted(totals.items()):
+        if disk in ("pd-balanced", "pd-ssd", "pd-extreme"):
+            bucket = "SSD_TOTAL_GB"
+            ssd_quota += gib
+        elif disk == "pd-standard":
+            bucket = "HDD (pd-standard)"
+            hdd_quota += gib
+        else:
+            bucket = "other"
+        print(f"  {disk:<16}  {gib:>8.1f} GiB  ({bucket})")
+    print()
+    print(f"  TOTAL counted against SSD_TOTAL_GB: {ssd_quota:.1f} GiB")
+    print(f"  TOTAL on HDD (pd-standard):         {hdd_quota:.1f} GiB")
+
+# --- Section 4: Pending PVCs --------------------------------------------------
+print()
+print("=== Pending PVCs ===")
+pending = []
+for pvc in pvcs:
+    phase = (pvc.get("status") or {}).get("phase", "")
+    if phase != "Pending":
+        continue
+    m = pvc.get("metadata") or {}
+    sc = (pvc.get("spec") or {}).get("storageClassName") or "<none>"
+    pending.append((m.get("namespace", ""), m.get("name", ""), sc))
+
+if not pending:
+    print("  (none)")
+else:
+    for ns, name, sc in pending:
+        print(f"  {ns}/{name}  (storageClass={sc})")
+
+# Print the namespace/name list on stderr so the shell can loop over it.
+import sys as _sys
+for ns, name, _sc in pending:
+    print(f"{ns}/{name}", file=_sys.stderr)
+PY
+
+# Run the python analysis; stderr carries the pending-PVC list for the shell
+# loop below.
+PENDING_LIST="$(python3 "$PY_SCRIPT" "$SC_JSON" "$PV_JSON" "$PVC_JSON" 2>"$TMPDIR/pending.txt")"
+printf '%s\n' "$PENDING_LIST"
+
+echo
+echo "=== Provisioning events for any Pending PVCs (tail) ==="
+if [[ ! -s "$TMPDIR/pending.txt" ]]; then
+  echo "  (no Pending PVCs)"
+else
+  while IFS='/' read -r ns pvc; do
+    [[ -n "$ns" && -n "$pvc" ]] || continue
+    echo "-- ${ns}/${pvc} --"
+    kctl -n "$ns" describe pvc "$pvc" \
+      | awk '/^Events:/{flag=1} flag' \
+      | head -n 20
+  done < "$TMPDIR/pending.txt"
+fi

mock_val/ensure_default_storage_class.sh

@@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+# ensure_default_storage_class.sh
+#
+# Preflight / remediator that asserts the cluster-wide default StorageClass is
+# pd-standard (HDD), not pd-balanced / pd-ssd. GKE ships `standard-rwo` as the
+# default, which provisions `pd-balanced` under the hood and draws from the
+# SSD_TOTAL_GB quota. That has repeatedly wedged provisioning on low-SSD-quota
+# projects (see init_cnpg_gke.sh step-down logic, which compensates on the
+# consumer side).
+#
+# This script fixes the root cause cluster-side: it un-defaults any built-in
+# SSD-backed class and promotes `standard-hdd` (pd-standard) as the default.
+# Workloads that actually need SSD latency (CNPG) opt in explicitly via
+# `storageClassName: premium-rwo` in their PVC template — they are unaffected
+# by the default change.
+#
+# Usage
+# -----
+#   ./etc/ensure_default_storage_class.sh [--check | --apply] [--kube-context CTX]
+#
+#   --check   (default) Report current state; exit 0 if compliant, 1 if drift
+#             detected. No cluster mutations.
+#   --apply             Remediate: create standard-hdd if absent, clear the
+#             default annotation from any SSD-backed class, set standard-hdd
+#             as the default. Idempotent.
+#   --kube-context CTX  Use the named kube-context (default: current).
+#
+# Exit codes
+# ----------
+#   0  Compliant (or successfully remediated)
+#   1  Drift detected in --check mode
+#   2  Remediation failed (in --apply mode)
+#   3  Cluster not a GKE cluster (no pd.csi.storage.gke.io provisioner seen)
+#   4  Invalid arguments / missing tooling
+
+set -euo pipefail
+
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+DEFAULT_SC_NAME="${PROLE_DEFAULT_SC_NAME:-standard-hdd}"
+DEFAULT_SC_DISK_TYPE="pd-standard"
+GKE_CSI_PROVISIONER="pd.csi.storage.gke.io"
+MODE="check"
+KUBE_CONTEXT=""
+
+log() { printf '[%s] %s\n' "$SCRIPT_NAME" "$*"; }
+warn() { printf '[%s] WARN: %s\n' "$SCRIPT_NAME" "$*" >&2; }
+die() { printf '[%s] ERROR: %s\n' "$SCRIPT_NAME" "$*" >&2; exit "${2:-4}"; }
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check)   MODE="check"; shift ;;
+    --apply)   MODE="apply"; shift ;;
+    --kube-context)
+      [[ -n "${2:-}" ]] || die "--kube-context requires an argument" 4
+      KUBE_CONTEXT="$2"; shift 2 ;;
+    -h|--help)
+      sed -n '1,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
+      exit 0 ;;
+    *) die "unknown argument: $1" 4 ;;
+  esac
+done
+
+command -v kubectl >/dev/null || die "kubectl not found in PATH" 4
+command -v python3 >/dev/null || die "python3 not found in PATH" 4
+
+kctl() {
+  if [[ -n "$KUBE_CONTEXT" ]]; then
+    kubectl --context "$KUBE_CONTEXT" "$@"
+  else
+    kubectl "$@"
+  fi
+}
+
+# ── 1. Confirm we're talking to a GKE cluster ────────────────────────────────
+sc_json="$(kctl get storageclass -o json 2>/dev/null || true)"
+if [[ -z "$sc_json" ]] || ! printf '%s' "$sc_json" | python3 -c '
+import json, sys
+data = json.load(sys.stdin)
+items = data.get("items") or []
+sys.exit(0 if any(it.get("provisioner") == "pd.csi.storage.gke.io" for it in items) else 1)
+'; then
+  die "no ${GKE_CSI_PROVISIONER} StorageClasses found; this does not look like a GKE cluster" 3
+fi
+
+# ── 2. Inspect current state ─────────────────────────────────────────────────
+# Emits: <name>|<provisioner>|<type>|<is_default>  for every StorageClass.
+sc_summary="$(printf '%s' "$sc_json" | python3 - <<'PY'
+import json, sys
+data = json.load(sys.stdin)
+for it in data.get("items") or []:
+    meta = it.get("metadata") or {}
+    name = meta.get("name", "")
+    ann = (meta.get("annotations") or {})
+    is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "") or "").lower() == "true"
+    prov = it.get("provisioner", "") or ""
+    disk_type = ((it.get("parameters") or {}).get("type", "")) or ""
+    print(f"{name}|{prov}|{disk_type}|{'true' if is_default else 'false'}")
+PY
+)"
+
+log "Current StorageClasses:"
+while IFS='|' read -r name prov disk_type is_default; do
+  [[ -n "$name" ]] || continue
+  if [[ "$is_default" == "true" ]]; then
+    log "  * ${name}  prov=${prov}  type=${disk_type}  (default)"
+  else
+    log "    ${name}  prov=${prov}  type=${disk_type}"
+  fi
+done <<< "$sc_summary"
+
+# Identify the current default + its disk type.
+current_default=""
+current_default_type=""
+ssd_backed_defaults=()
+while IFS='|' read -r name prov disk_type is_default; do
+  [[ -n "$name" ]] || continue
+  if [[ "$is_default" == "true" ]]; then
+    current_default="$name"
+    current_default_type="$disk_type"
+    if [[ "$disk_type" == "pd-balanced" || "$disk_type" == "pd-ssd" ]]; then
+      ssd_backed_defaults+=("$name")
+    fi
+  fi
+done <<< "$sc_summary"
+
+# Does standard-hdd (or PROLE_DEFAULT_SC_NAME override) exist with the right shape?
+target_exists="false"
+target_disk_type=""
+target_is_default="false"
+while IFS='|' read -r name prov disk_type is_default; do
+  if [[ "$name" == "$DEFAULT_SC_NAME" ]]; then
+    target_exists="true"
+    target_disk_type="$disk_type"
+    target_is_default="$is_default"
+  fi
+done <<< "$sc_summary"
+
+compliant="true"
+issues=()
+
+if [[ "$target_exists" != "true" ]]; then
+  compliant="false"
+  issues+=("target StorageClass '${DEFAULT_SC_NAME}' does not exist")
+elif [[ "$target_disk_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then
+  compliant="false"
+  issues+=("'${DEFAULT_SC_NAME}' has type='${target_disk_type}', want '${DEFAULT_SC_DISK_TYPE}'")
+fi
+
+if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then
+  compliant="false"
+  issues+=("SSD-backed class(es) still marked default: ${ssd_backed_defaults[*]}")
+fi
+
+if [[ "$target_exists" == "true" && "$target_is_default" != "true" ]]; then
+  compliant="false"
+  issues+=("'${DEFAULT_SC_NAME}' is not annotated is-default-class=true")
+fi
+
+if [[ "$compliant" == "true" ]]; then
+  log "OK — default StorageClass is '${current_default}' (type=${current_default_type}). No action needed."
+  exit 0
+fi
+
+log "Drift detected:"
+for issue in "${issues[@]}"; do
+  log "  - ${issue}"
+done
+
+if [[ "$MODE" == "check" ]]; then
+  log "Run with --apply to remediate."
+  exit 1
+fi
+
+# ── 3. Remediate ─────────────────────────────────────────────────────────────
+log "Applying remediation ..."
+
+# 3a. Create/update standard-hdd — but NOT yet marked default so we never have
+# a window with two defaults racing.
+cat <<EOF | kctl apply -f -
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${DEFAULT_SC_NAME}
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: ${GKE_CSI_PROVISIONER}
+parameters:
+  type: ${DEFAULT_SC_DISK_TYPE}
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+EOF
+
+# Verify the live class has the right disk type. If `parameters` drifted on an
+# existing class, the apply above would silently no-op (parameters are
+# immutable). Catch that and hard-fail with a clear message.
+live_type="$(kctl get storageclass "$DEFAULT_SC_NAME" -o jsonpath='{.parameters.type}' 2>/dev/null || true)"
+if [[ "$live_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then
+  warn "live '${DEFAULT_SC_NAME}' has type='${live_type}' (want '${DEFAULT_SC_DISK_TYPE}'). Recreating ..."
+  kctl delete storageclass "$DEFAULT_SC_NAME" --wait=true >/dev/null 2>&1 || die "failed to delete '${DEFAULT_SC_NAME}' for recreation" 2
+  cat <<EOF | kctl apply -f -
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${DEFAULT_SC_NAME}
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: ${GKE_CSI_PROVISIONER}
+parameters:
+  type: ${DEFAULT_SC_DISK_TYPE}
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+EOF
+  live_type="$(kctl get storageclass "$DEFAULT_SC_NAME" -o jsonpath='{.parameters.type}' 2>/dev/null || true)"
+  [[ "$live_type" == "$DEFAULT_SC_DISK_TYPE" ]] || die "after recreate, '${DEFAULT_SC_NAME}' still has type='${live_type}'" 2
+fi
+
+# 3b. Clear the default annotation from any SSD-backed class that currently
+# holds it. Patch to "false" (not remove) so we leave a clear audit trail.
+if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then
+  for sc in "${ssd_backed_defaults[@]}"; do
+    log "Un-defaulting SSD-backed class: ${sc}"
+    kctl annotate storageclass "$sc" \
+      "storageclass.kubernetes.io/is-default-class=false" --overwrite >/dev/null \
+      || die "failed to un-default '${sc}'" 2
+  done
+fi
+
+# 3c. Promote standard-hdd to default — atomic with respect to other defaults
+# because we cleared them in 3b first.
+kctl annotate storageclass "$DEFAULT_SC_NAME" \
+  "storageclass.kubernetes.io/is-default-class=true" --overwrite >/dev/null \
+  || die "failed to mark '${DEFAULT_SC_NAME}' as default" 2
+
+# ── 4. Re-verify ─────────────────────────────────────────────────────────────
+final_default="$(kctl get storageclass \
+  -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \
+  2>/dev/null | head -n1)"
+
+if [[ "$final_default" != "$DEFAULT_SC_NAME" ]]; then
+  die "post-remediation: default is '${final_default:-<none>}', expected '${DEFAULT_SC_NAME}'" 2
+fi
+
+# Count defaults — having two is a worse state than having zero.
+default_count="$(kctl get storageclass \
+  -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \
+  2>/dev/null | grep -cv '^$' || true)"
+if [[ "$default_count" != "1" ]]; then
+  die "post-remediation: ${default_count} default StorageClasses exist (want exactly 1)" 2
+fi
+
+log "Remediation OK. Default StorageClass is now '${DEFAULT_SC_NAME}' (${DEFAULT_SC_DISK_TYPE})."
+log "Re-run 'kubectl get storageclass' to confirm."
+exit 0

mock_val/fetch_prole_secrets.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# fetch_prole_secrets.sh
+#
+# Pulls prole.org Google OAuth credentials from 1Password into etc/secrets/
+# so the init scripts can consume them via the standard resolve_secret() path.
+#
+# Items read (Personal vault, category API_CREDENTIAL):
+#   svc-prole-org  →  grafana-google-oidc-client-{id,secret}-prole
+#   db-prole-org   →  oauth2-proxy-client-{id,secret}-prole
+#
+# Cookie secret:
+#   If already stored as field 'cookie_secret' in db-prole-org, uses that.
+#   Otherwise generates a 32-char base64 secret (openssl rand -base64 24 → 32 chars = 32 bytes, valid for AES), writes it to
+#   etc/secrets/oauth2-proxy-cookie-secret-prole, and saves it back to the
+#   db-prole-org item so it's durable in 1Password.
+#
+# Usage:
+#   ./etc/fetch_prole_secrets.sh
+#   # Then run the init scripts:
+#   ./etc/init_grafana_oauth_prole.sh
+#   ./etc/init_oauth2_proxy_prole.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+SECRETS_DIR="$REPO_ROOT/etc/secrets"
+
+log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
+die() { log "ERROR: $*" >&2; exit 1; }
+
+command -v op >/dev/null 2>&1 || die "'op' (1Password CLI) not found. Install: brew install 1password-cli"
+
+if ! op account list >/dev/null 2>&1 || ! op vault list >/dev/null 2>&1; then
+  log "Not signed in to 1Password — signing in..."
+  eval "$(op signin)"
+fi
+
+mkdir -p "$SECRETS_DIR"
+chmod 700 "$SECRETS_DIR"
+
+write_secret() {
+  local file="$SECRETS_DIR/$1" value="$2"
+  printf '%s' "$value" > "$file"
+  chmod 600 "$file"
+  log "  wrote $file"
+}
+
+# ── svc-prole-org → Grafana OAuth ────────────────────────────────────────────
+
+log "Fetching svc-prole-org (Grafana / svc.prole.org) ..."
+SVC_CLIENT_ID="$(op item get "svc-prole-org" --fields username 2>/dev/null)" \
+  || die "Could not read svc-prole-org from 1Password"
+SVC_CLIENT_SECRET="$(op item get "svc-prole-org" --fields credential --reveal 2>/dev/null)" \
+  || die "Could not read svc-prole-org credential from 1Password"
+
+write_secret "grafana-google-oidc-client-id-prole"     "$SVC_CLIENT_ID"
+write_secret "grafana-google-oidc-client-secret-prole" "$SVC_CLIENT_SECRET"
+
+# ── db-prole-org → oauth2-proxy (Studio / db.prole.org) ──────────────────────
+
+log "Fetching db-prole-org (oauth2-proxy / db.prole.org) ..."
+DB_CLIENT_ID="$(op item get "db-prole-org" --fields username 2>/dev/null)" \
+  || die "Could not read db-prole-org from 1Password"
+DB_CLIENT_SECRET="$(op item get "db-prole-org" --fields credential --reveal 2>/dev/null)" \
+  || die "Could not read db-prole-org credential from 1Password"
+
+write_secret "oauth2-proxy-client-id-prole"     "$DB_CLIENT_ID"
+write_secret "oauth2-proxy-client-secret-prole" "$DB_CLIENT_SECRET"
+
+# ── Cookie secret ─────────────────────────────────────────────────────────────
+
+log "Resolving oauth2-proxy cookie secret ..."
+COOKIE_SECRET=""
+
+# Check if already stored in 1Password (may not exist on first run)
+COOKIE_SECRET="$(op item get "db-prole-org" --fields label=cookie_secret --reveal 2>/dev/null || true)"
+
+if [[ -z "$COOKIE_SECRET" ]]; then
+  log "  No cookie_secret field in db-prole-org — generating new 32-byte secret ..."
+  COOKIE_SECRET="$(openssl rand -base64 24)"
+
+  log "  Saving cookie_secret back to db-prole-org in 1Password ..."
+  op item edit "db-prole-org" \
+    "cookie_secret[password]=${COOKIE_SECRET}" >/dev/null \
+    || die "Failed to save cookie_secret to 1Password"
+  log "  Saved."
+fi
+
+write_secret "oauth2-proxy-cookie-secret-prole" "$COOKIE_SECRET"
+
+# ── Summary ───────────────────────────────────────────────────────────────────
+
+log ""
+log "==> All prole.org secrets written to etc/secrets/:"
+ls -1 "$SECRETS_DIR/"*-prole 2>/dev/null | while read -r f; do log "    $f"; done
+log ""
+log "Next steps:"
+log "  ./etc/init_grafana_oauth_prole.sh    # applies grafana-google-oidc Secret"
+log "  ./etc/init_oauth2_proxy_prole.sh     # deploys oauth2-proxy for db.prole.org"

mock_val/gen_oidc_signing_key.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# gen_oidc_signing_key.sh
+#
+# Idempotent RS256 keypair generator for knoe-auth Phase 2 OIDC provider
+# in local k3d / TDD dev mode. Produces:
+#
+#   etc/secrets/knoe-auth-oidc-key.pem   PKCS#8 PEM of the private key
+#   etc/secrets/knoe-auth-oidc-key.b64   single-line base64 of the same key,
+#                                        suitable for KNOE_AUTH_OIDC_SIGNING_KEY
+#
+# Both files live under etc/secrets/ which is gitignored. If the .b64 file
+# already exists, the script is a no-op (so you keep stable token signatures
+# across restarts).
+#
+# Usage:
+#   bash etc/gen_oidc_signing_key.sh                 # generate if missing
+#   FORCE=1 bash etc/gen_oidc_signing_key.sh         # rotate
+#
+# Env consumption (in your shell or the spring-boot:run command):
+#   export KNOE_AUTH_OIDC_SIGNING_KEY=$(cat etc/secrets/knoe-auth-oidc-key.b64)
+#
+# Production (GKE) provides this via a K8s Secret + Workload Identity / OpenBao;
+# this script is for laptop dev only.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+SECRETS_DIR="${REPO_ROOT}/etc/secrets"
+PEM_PATH="${SECRETS_DIR}/knoe-auth-oidc-key.pem"
+B64_PATH="${SECRETS_DIR}/knoe-auth-oidc-key.b64"
+
+mkdir -p "${SECRETS_DIR}"
+chmod 700 "${SECRETS_DIR}" 2>/dev/null || true
+
+if [[ -f "${B64_PATH}" && "${FORCE:-0}" != "1" ]]; then
+  echo "[gen_oidc_signing_key] already present at ${B64_PATH} — keeping. Pass FORCE=1 to rotate."
+  exit 0
+fi
+
+if ! command -v openssl >/dev/null 2>&1; then
+  echo "[gen_oidc_signing_key] ERROR: openssl not found on PATH." >&2
+  exit 2
+fi
+
+echo "[gen_oidc_signing_key] generating RSA-2048 PKCS#8 keypair ..."
+# `openssl genpkey` writes a PEM-wrapped PKCS#8 private key. Spring's
+# OidcTokenService.init() reads KNOE_AUTH_OIDC_SIGNING_KEY as base64 of the
+# raw DER PKCS#8 bytes — i.e., the PEM with header/footer stripped and no
+# whitespace. We produce both forms.
+openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 \
+  -out "${PEM_PATH}" 2>/dev/null
+chmod 600 "${PEM_PATH}"
+
+# Strip PEM headers/footers and newlines to get the single-line base64 of the
+# DER PKCS#8 bytes that the Java `Base64.getDecoder().decode(signingKey)` call
+# expects (per OidcTokenService.java).
+grep -v -- '-----' "${PEM_PATH}" | tr -d '\n' > "${B64_PATH}"
+chmod 600 "${B64_PATH}"
+
+echo "[gen_oidc_signing_key] wrote ${PEM_PATH}"
+echo "[gen_oidc_signing_key] wrote ${B64_PATH}  ($(wc -c < "${B64_PATH}") bytes)"
+echo
+echo "Use it in your shell:"
+echo "  export KNOE_AUTH_OIDC_SIGNING_KEY=\$(cat ${B64_PATH})"

mock_val/hostprobe-myrddin.yaml

@@ -4,7 +4,7 @@ metadata:
   name: knoe-hostprobe-myrddin
   namespace: kube-system
 spec:
-  nodeName: myrddin.knoe.org
+  nodeName: myrddin.prole.org
   hostNetwork: true
   hostPID: true
   tolerations:

mock_val/hostprobe-pi.yaml

@@ -4,7 +4,7 @@ metadata:
   name: knoe-hostprobe-pi
   namespace: kube-system
 spec:
-  nodeName: pi.knoe.org
+  nodeName: pi.prole.org
   hostNetwork: true
   hostPID: true
   tolerations:

mock_val/hosts.txt

@@ -1,15 +1,15 @@
-myrddin.knoe.org       10.0.0.3
+myrddin.prole.org       10.0.0.3
-raspberry.knoe.org     10.0.0.4
+raspberry.prole.org     10.0.0.4
-pi.knoe.org            10.0.0.5
+pi.prole.org            10.0.0.5
-synology.knoe.org      10.0.0.203
+synology.prole.org      10.0.0.203
-morgoth.knoe.org       10.0.0.204
+morgoth.prole.org       10.0.0.204
-zinfandel.knoe.org     10.0.0.205
+zinfandel.prole.org     10.0.0.205
-aventage.knoe.org      10.0.0.206
+aventage.prole.org      10.0.0.206
-retropie.knoe.org      10.0.0.207
+retropie.prole.org      10.0.0.207
-fairyland.knoe.org     10.0.0.208
+fairyland.prole.org     10.0.0.208
-k8s.knoe.org           zinfandel.knoe.org
+k8s.prole.org           zinfandel.prole.org
-mc.knoe.org            73.15.20.166
+mc.prole.org            73.15.20.166
-morana.knoe.org        10.0.0.66
+morana.prole.org        10.0.0.66
-ollama.knoe.org        73.15.20.166
+ollama.prole.org        73.15.20.166
-svc.knoe.org           73.15.20.166
+svc.prole.org           73.15.20.166
-www.knoe.org           ghs.googlehosted.com
+www.prole.org           ghs.googlehosted.com

mock_val/knoe-db-ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBiTCCAS+gAwIBAgIUF+swDNndu9C+lNgcFIjTTHfF9VcwCgYIKoZIzj0EAwIw
+GjEYMBYGA1UEAwwPa25vZS1kYiBDTlBHIENBMB4XDTI2MDQxMDEzNDEwMVoXDTI3
+MDQxMDEzNDEwMVowGjEYMBYGA1UEAwwPa25vZS1kYiBDTlBHIENBMFkwEwYHKoZI
+zj0CAQYIKoZIzj0DAQcDQgAEzIAAV0RyJydcIsGGwEzgC/BIISmDowBray5pD7LG
+9bUwtGv6m5XaXHNGIv/shAW4E5D2ioHIWFBJtjwfS/pe4qNTMFEwHQYDVR0OBBYE
+FOzKPWn1lbqXOrCfo/fDTHV+aDSkMB8GA1UdIwQYMBaAFOzKPWn1lbqXOrCfo/fD
+THV+aDSkMA8GA1UdEwEB/wQFMAMBAf8wCgYIKoZIzj0EAwIDSAAwRQIhAPrg+W+T
+kRX5jGPDwFQZJHtaI/H1qhzEYUwWVsjnDFdoAiAB29iKZOFyIjaEPbYdixNRqU8p
+u8MAWBsjGNpS/SaLJA==
+-----END CERTIFICATE-----

mock_val/knoe-db-passwwd.sh

@@ -14,7 +14,7 @@ Usage: $0 [-k|--kdc] [-f|--force] [-c|--config <path>]
 Options:
   -k, --kdc     Authenticate current password against a running KDC
   -f, --force   Skip current password verification
-  -c, --config  Path to knoe.cfg (default: detected)
+  -c, --config  Path to config file (default: detected)
 USAGE
 }
 
@@ -49,17 +49,17 @@ while [[ $# -gt 0 ]]; do
 done
 
 if [[ -z "$CFG_PATH" ]]; then
-  if [[ -n "${KNOE_CONF:-}" && -f "$KNOE_CONF/knoe.cfg" ]]; then
+  if [[ -n "${KNOE_CONF:-}" ]]; then
-    CFG_PATH="$KNOE_CONF/knoe.cfg"
+    CFG_PATH="$(_knoe_cfg_select_cfg_file "$KNOE_CONF")"
-  elif [[ -n "${KNOE_HOME:-}" && -f "$KNOE_HOME/conf/knoe.cfg" ]]; then
+  elif [[ -n "${KNOE_HOME:-}" ]]; then
-    CFG_PATH="$KNOE_HOME/conf/knoe.cfg"
+    CFG_PATH="$(_knoe_cfg_select_cfg_file "$KNOE_HOME/conf")"
-  elif [[ -f "$ROOT_DIR/conf/knoe.cfg" ]]; then
+  else
-    CFG_PATH="$ROOT_DIR/conf/knoe.cfg"
+    CFG_PATH="$(_knoe_cfg_select_cfg_file "$ROOT_DIR/conf")"
   fi
 fi
 
 if [[ -z "$CFG_PATH" || ! -f "$CFG_PATH" ]]; then
-  echo "ERROR: knoe.cfg not found. Use -c to specify path." >&2
+  echo "ERROR: config file not found. Use -c to specify path." >&2
   exit 1
 fi
 

mock_val/krb5.local.conf

@@ -0,0 +1,16 @@
+[libdefaults]
+  default_realm = KNOE.LOCAL
+  rdns = false
+  forwardable = true
+  udp_preference_limit = 1   # force TCP — workaround for kubectl port-forward UDP flakiness on macOS
+
+[realms]
+  KNOE.LOCAL = {
+    kdc = localhost:88
+    admin_server = localhost:749
+    default_domain = local
+  }
+
+[domain_realm]
+  .local = KNOE.LOCAL
+  localhost = KNOE.LOCAL

mock_val/onboard_engineer.sh

@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+# onboard_engineer.sh — provision a per-engineer postgres role + 24h temp
+# password, then emit a one-time onboarding URL (and QR-code rendering of it)
+# that the new engineer redeems at https://db.0.knoe.dev/onboard.html.
+#
+# This is the Phase 1 bridge while Junie's Phase 2 (libpq OAUTHBEARER) is in
+# flight. When OAUTHBEARER lands, the temp-password mechanism goes away and
+# this script's role-creation step is replaced by an INSERT into
+# `knoe.oauth_role_map`. The engineer-facing URL stays the same (the page just
+# stops showing a password and instead displays the OAUTH connection string).
+#
+# Usage:
+#   ./etc/onboard_engineer.sh <username> <email>
+#   ./etc/onboard_engineer.sh --revoke <username>
+#
+# Env (resolved from knoe_cfg.sh / KUBECONTEXT etc):
+#   DB_CLUSTER_KUBECONTEXT  — the CNPG cluster context (knoe-dev-cnpg-0)
+#   DB_NAMESPACE            — defaults to knoe-db-0
+#   DB_PRIMARY_POD          — auto-discovered if unset
+#   ONBOARD_HOST            — defaults to db.0.knoe.dev
+#   ONBOARD_TTL_HOURS       — defaults to 24
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
+
+DB_CTX="${DB_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-gke_plenary-truck-485623-p7_us-west3_knoe-dev-cnpg-0}}"
+DB_NS="${DB_NAMESPACE:-knoe-db-0}"
+ONBOARD_HOST="${ONBOARD_HOST:-db.0.knoe.dev}"
+ONBOARD_TTL_HOURS="${ONBOARD_TTL_HOURS:-24}"
+
+log()  { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
+warn() { printf '\033[33m[!]\033[0m %s\n' "$*" >&2; }
+err()  { printf '\033[31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; }
+
+usage() {
+  cat <<EOF
+Usage:
+  $(basename "$0") <username> <email>     # provision new engineer
+  $(basename "$0") --revoke <username>    # drop role + cancel access
+  $(basename "$0") --rotate <username>    # generate a fresh 24h temp password
+
+Examples:
+  $(basename "$0") ron ron@knoey.com
+  $(basename "$0") --revoke test-user
+EOF
+  exit 1
+}
+
+primary_pod() {
+  if [[ -n "${DB_PRIMARY_POD:-}" ]]; then printf '%s' "$DB_PRIMARY_POD"; return; fi
+  kubectl --context="$DB_CTX" -n "$DB_NS" get pod \
+    -l cnpg.io/cluster=knoe-db,cnpg.io/instanceRole=primary \
+    -o jsonpath='{.items[0].metadata.name}'
+}
+
+run_psql() {
+  local sql="$1"
+  local pod
+  pod=$(primary_pod)
+  printf '%s' "$sql" | kubectl --context="$DB_CTX" -n "$DB_NS" exec -i "$pod" -c postgres -- \
+    psql -U postgres -d postgres -v ON_ERROR_STOP=1 -X --quiet 2>&1
+}
+
+ensure_dependencies() {
+  command -v kubectl   >/dev/null 2>&1 || err "kubectl not found"
+  command -v openssl   >/dev/null 2>&1 || err "openssl not found"
+  command -v base64    >/dev/null 2>&1 || err "base64 not found"
+  if ! command -v qrencode >/dev/null 2>&1; then
+    warn "qrencode not installed — output will be URL-only (brew install qrencode for QR rendering)"
+  fi
+}
+
+current_iso8601() {
+  date -u "+%Y-%m-%dT%H:%M:%SZ"
+}
+
+ttl_iso8601() {
+  # macOS date and GNU date have different flag syntax; both ISO 8601 a
+  # given offset-from-now expressed in hours. Use python as a portable fallback.
+  python3 - <<EOF
+from datetime import datetime, timedelta, timezone
+print((datetime.now(timezone.utc) + timedelta(hours=$ONBOARD_TTL_HOURS)).strftime("%Y-%m-%dT%H:%M:%SZ"))
+EOF
+}
+
+validate_username() {
+  local u="$1"
+  [[ "$u" =~ ^[a-z][a-z0-9_-]{1,30}$ ]] || err "username must match [a-z][a-z0-9_-]{1,30}: $u"
+}
+
+validate_email() {
+  local e="$1"
+  [[ "$e" =~ ^[^@[:space:]]+@knoey\.com$ ]] || err "email must be *@knoey.com (got: $e). To override, edit the script."
+}
+
+action_provision() {
+  local user="$1" email="$2"
+  validate_username "$user"
+  validate_email "$email"
+
+  local pw exp ttl_hint
+  pw=$(openssl rand -base64 24)
+  exp=$(ttl_iso8601)
+  ttl_hint="${ONBOARD_TTL_HOURS}h"
+
+  log "==> provisioning postgres role: $user (email: $email, valid for $ttl_hint)"
+
+  # SQL is idempotent: works for fresh CREATE and re-onboarding.
+  local sql
+  sql=$(cat <<SQL
+DO \$do\$ BEGIN
+  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='$user') THEN
+    CREATE ROLE "$user" LOGIN INHERIT VALID UNTIL '$exp';
+    RAISE NOTICE 'created role %', '$user';
+  ELSE
+    RAISE NOTICE 'role % exists; updating password and VALID UNTIL', '$user';
+  END IF;
+END \$do\$;
+
+ALTER ROLE "$user" PASSWORD '$pw' VALID UNTIL '$exp';
+GRANT knoe_developer TO "$user";
+
+\\echo '=== verification ==='
+SELECT rolname, rolcanlogin, rolvaliduntil
+FROM pg_roles WHERE rolname='$user';
+SELECT 'is_developer' AS check, pg_has_role('$user','knoe_developer','MEMBER') AS ok;
+SQL
+)
+
+  run_psql "$sql"
+
+  # URL-safe base64: replace + with -, / with _, drop = padding.
+  local pw_url
+  pw_url=$(printf '%s' "$pw" | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '=')
+  # Wait — pw is already base64 from openssl rand. Encode the *literal text* of
+  # the password in url-safe base64 so the page can decode and display it
+  # exactly as typed back into psql. atob() in the browser handles both
+  # standard and url-safe base64 (after replacement).
+
+  local url
+  url="https://${ONBOARD_HOST}/onboard.html#user=${user}&pw=${pw_url}&exp=${exp}"
+
+  printf '\n'
+  log "==> onboarding link"
+  printf '\n  %s\n\n' "$url"
+
+  if command -v qrencode >/dev/null 2>&1; then
+    log "==> QR code (scan with engineer's phone camera)"
+    printf '\n'
+    qrencode -t ANSI256 -m 1 "$url"
+    printf '\n'
+  fi
+
+  log "==> engineer's plaintext password (fallback if URL is unusable)"
+  printf '\n  %s\n\n' "$pw"
+
+  cat <<EOF
+==================================================================
+DELIVERY:
+  • Best:  Show the QR above on screenshare while $user scans with phone.
+           Phone opens URL → page shows password + connection string.
+  • Async: Compose Gmail to $email, paste the URL above, send.
+           Recipient clicks → page shows password + connection string.
+  • Last:  Hand the plaintext password over a secure channel (Signal).
+           Less ideal — the page bundles the connection-string UX.
+
+THE TEMP PASSWORD EXPIRES IN ${ttl_hint}.
+Engineer should rotate immediately on first connect via:  postgres=> \\password
+==================================================================
+
+EOF
+}
+
+action_revoke() {
+  local user="$1"
+  validate_username "$user"
+  log "==> revoking role: $user"
+
+  local sql
+  sql=$(cat <<SQL
+DO \$do\$ BEGIN
+  IF EXISTS (SELECT FROM pg_roles WHERE rolname='$user') THEN
+    REVOKE knoe_developer FROM "$user";
+    DROP ROLE "$user";
+    RAISE NOTICE 'dropped role %', '$user';
+  ELSE
+    RAISE NOTICE 'role % does not exist; nothing to do', '$user';
+  END IF;
+END \$do\$;
+SQL
+)
+  run_psql "$sql"
+  log "==> revocation complete"
+}
+
+action_rotate() {
+  local user="$1"
+  validate_username "$user"
+
+  local pw exp pw_url url
+  pw=$(openssl rand -base64 24)
+  exp=$(ttl_iso8601)
+
+  log "==> rotating temp password for $user (valid ${ONBOARD_TTL_HOURS}h)"
+  local sql
+  sql=$(cat <<SQL
+DO \$do\$ BEGIN
+  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='$user') THEN
+    RAISE EXCEPTION 'role % does not exist; use provision action instead', '$user';
+  END IF;
+END \$do\$;
+ALTER ROLE "$user" PASSWORD '$pw' VALID UNTIL '$exp';
+SQL
+)
+  run_psql "$sql"
+
+  pw_url=$(printf '%s' "$pw" | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '=')
+  url="https://${ONBOARD_HOST}/onboard.html#user=${user}&pw=${pw_url}&exp=${exp}"
+
+  printf '\n  %s\n\n' "$url"
+  if command -v qrencode >/dev/null 2>&1; then
+    qrencode -t ANSI256 -m 1 "$url"; printf '\n'
+  fi
+  printf '\n  plaintext: %s\n\n' "$pw"
+}
+
+# --- main ---
+ensure_dependencies
+
+case "${1:-}" in
+  ""|"-h"|"--help") usage ;;
+  "--revoke")       [[ $# -eq 2 ]] || usage; action_revoke "$2" ;;
+  "--rotate")       [[ $# -eq 2 ]] || usage; action_rotate "$2" ;;
+  *)                [[ $# -eq 2 ]] || usage; action_provision "$1" "$2" ;;
+esac

mock_val/preflight_kubecontext.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# preflight_kubecontext.sh
+#
+# Shared helper to detect the "wrong shell context for the install mode"
+# class of bug. Sourced by install.sh and deploy.sh.
+#
+# Exposes two functions:
+#
+#   verify_kubecontext_matches_config <config-path>
+#       Strict gate. Reads APP_CLUSTER_KUBECONTEXT from the config's [Global]
+#       section and refuses to proceed if `kubectl config current-context`
+#       doesn't match. Used by deploy.sh (unattended path: silent mismatches
+#       are dangerous).
+#
+#       Returns 0 on match (or when the config has no APP_CLUSTER_KUBECONTEXT,
+#       e.g. fresh k3d setup). Exits 1 with a clear error on mismatch.
+#       Set KNOE_SKIP_KUBECONTEXT_GUARD=true to bypass (escape hatch for
+#       deliberate cross-cluster maintenance).
+#
+#   print_kubecontext_notice
+#       Informational. Prints the current kubectl context (or "(none)") so
+#       the user sees what install.sh is about to inherit before the TUI
+#       launches. Never exits or fails. Used by install.sh.
+#
+# History: this guard was filed in response to the 2026-04-28 14:00 UTC
+# outage. An install.sh run in k3d mode with the shell pointed at GKE
+# overwrote the GKE-cluster ObjectStore + ScheduledBackup with k3d-mode
+# defaults, and CNPG backups silently failed for hours until the next
+# manual check. See CLAUDE.md §"Env-contamination warning" and
+# docs/TODO.md queue item #1 (drift R4).
+
+# Read APP_CLUSTER_KUBECONTEXT from the [Global] section of an INI-style
+# config. Empty string if absent. Quote-stripping is best-effort.
+_kubectx_from_config() {
+  local cfg="$1"
+  [[ -f "$cfg" ]] || { echo ""; return 0; }
+  # awk: print value when we're in [Global] and key matches.
+  # Strips surrounding whitespace and quotes.
+  awk '
+    /^\[/{section=$0; next}
+    section=="[Global]" && /^[[:space:]]*APP_CLUSTER_KUBECONTEXT[[:space:]]*=/ {
+      sub(/^[^=]*=[[:space:]]*/, "", $0)
+      sub(/^"/, "", $0); sub(/"$/, "", $0)
+      sub(/^'\''/, "", $0); sub(/'\''$/, "", $0)
+      print $0
+      exit
+    }
+  ' "$cfg"
+}
+
+# Read the live current-context, or empty if kubectl/config unavailable.
+_kubectx_current() {
+  command -v kubectl >/dev/null 2>&1 || { echo ""; return 0; }
+  kubectl config current-context 2>/dev/null || true
+}
+
+# verify_kubecontext_matches_config <config-path>
+#   Strict gate. Exits 1 on mismatch unless KNOE_SKIP_KUBECONTEXT_GUARD=true.
+verify_kubecontext_matches_config() {
+  local cfg_path="${1:-}"
+  if [[ -z "$cfg_path" ]]; then
+    echo "preflight_kubecontext: usage: verify_kubecontext_matches_config <config-path>" >&2
+    return 2
+  fi
+  if [[ "${KNOE_SKIP_KUBECONTEXT_GUARD:-false}" == "true" ]]; then
+    echo "preflight_kubecontext: KNOE_SKIP_KUBECONTEXT_GUARD=true — skipping check (escape hatch)." >&2
+    return 0
+  fi
+
+  local expected
+  expected="$(_kubectx_from_config "$cfg_path")"
+
+  # No baked context in the config (e.g. fresh k3d.cfg) → nothing to check.
+  if [[ -z "$expected" ]]; then
+    return 0
+  fi
+
+  local actual
+  actual="$(_kubectx_current)"
+
+  # No live current-context → user hasn't selected one; the config is
+  # authoritative and downstream code will pass --context explicitly.
+  if [[ -z "$actual" ]]; then
+    return 0
+  fi
+
+  if [[ "$expected" == "$actual" ]]; then
+    return 0
+  fi
+
+  cat >&2 <<EOF
+
+==============================================================================
+  ABORTING: kubectl context mismatch
+==============================================================================
+
+  Config:   $cfg_path
+  Expected: APP_CLUSTER_KUBECONTEXT = $expected
+  Actual:   kubectl config current-context = $actual
+
+  These don't match. The installer would dispatch kubectl operations
+  against the WRONG cluster, which has caused real outages in the past
+  (see CLAUDE.md §"Env-contamination warning"). Refusing to proceed.
+
+  To fix:
+    1) Switch context to the expected cluster:
+         kubectl config use-context $expected
+       Re-run this command afterward.
+
+    2) Or, if you really need to deploy against the active context, edit
+       \`$cfg_path\` and update [Global] APP_CLUSTER_KUBECONTEXT to match.
+
+    3) Override (only when you know what you're doing):
+         KNOE_SKIP_KUBECONTEXT_GUARD=true ./deploy.sh
+
+==============================================================================
+EOF
+  return 1
+}
+
+# print_kubecontext_notice
+#   Informational. Never fails; just shows the user what's about to be
+#   inherited so they can abort before the TUI launches if it looks wrong.
+print_kubecontext_notice() {
+  local actual
+  actual="$(_kubectx_current)"
+  if [[ -z "$actual" ]]; then
+    echo "==> kubectl current-context: (none set)"
+  else
+    echo "==> kubectl current-context: $actual"
+  fi
+  echo "    The mode you select must target this cluster, OR you must switch"
+  echo "    context (kubectl config use-context …) before proceeding."
+}

mock_val/repair_pipeline.sh

@@ -6,8 +6,8 @@ set -euo pipefail
 # Purpose:
 # - Thin wrapper around the Python-native cluster repair logic.
 #
-# The real orchestration now lives in `installer/core/actions.py` and is
+# The real orchestration now lives in `knoe/core/actions.py` and is
-# invoked through `installer/core/repair_pipeline_cli.py`.
+# invoked through `knoe/core/repair_pipeline_cli.py`.
 
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 ROOT_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
@@ -88,4 +88,4 @@ if [[ -n "$MODE" ]]; then
 fi
 args+=("--config" "$KNOE_CONF")
 
-exec python3 -m installer.core.repair_pipeline_cli "${args[@]}"
+exec python3 -m knoe.core.repair_pipeline_cli "${args[@]}"

mock_val/set-k3s-token-1password.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# set-k3s-token-1password.sh
+# Reads the k3s server join token and stores it in the 'knoey' 1Password vault.
+# Replaces etc/set-k3s-token-vault.sh (formerly used ansible-vault).
+#
+# Run on the k3s control-plane node (requires sudo to read the token file).
+
+set -euo pipefail
+
+TOKEN_FILE="${TOKEN_FILE:-/var/lib/rancher/k3s/server/node-token}"
+VAULT="knoey"
+ITEM="k3s-token"
+FIELD="credential"
+
+if [[ ! -r "$TOKEN_FILE" ]]; then
+  if command -v sudo >/dev/null 2>&1; then
+    TOKEN="$(sudo cat "$TOKEN_FILE" | tr -d '\r\n')"
+  else
+    echo "ERROR: Cannot read token file: $TOKEN_FILE" >&2
+    echo "Are you running this on a k3s server?" >&2
+    exit 1
+  fi
+else
+  TOKEN="$(cat "$TOKEN_FILE" | tr -d '\r\n')"
+fi
+
+if [[ -z "$TOKEN" ]]; then
+  echo "ERROR: Token read from $TOKEN_FILE is empty" >&2
+  exit 1
+fi
+
+if ! command -v op >/dev/null 2>&1; then
+  echo "ERROR: 1Password CLI (op) not found. Install: brew install 1password-cli" >&2
+  exit 1
+fi
+
+if ! op whoami >/dev/null 2>&1; then
+  op signin
+fi
+
+if op item get "$ITEM" --vault "$VAULT" >/dev/null 2>&1; then
+  echo "Updating existing item '$ITEM' in vault '$VAULT'..."
+  op item edit "$ITEM" --vault "$VAULT" "${FIELD}=${TOKEN}"
+else
+  echo "Creating item '$ITEM' in vault '$VAULT'..."
+  op item create \
+    --category login \
+    --title "$ITEM" \
+    --vault "$VAULT" \
+    "${FIELD}=${TOKEN}"
+fi
+
+echo "Done. k3s token stored in 1Password vault '$VAULT' as item '$ITEM'."
+echo
+echo "Verify with:"
+echo "  op item get $ITEM --vault $VAULT --fields $FIELD --reveal"

mock_val/status.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # etc/status.sh — Overall deployment health check for the Knoe k3d environment.
-# Usage: status.sh [-c conf/knoe.cfg] [-v|--verbose]
+# Usage: status.sh [-c conf/{k3d|k3s|gke}.cfg] [-v|--verbose]
 # Exit 0  = score of 100% (all components healthy)
 # Exit 1  = score < 100%
 set -u
@@ -8,6 +8,9 @@ set -u
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 
+# shellcheck disable=SC1090
+source "$SCRIPT_DIR/knoe_cfg.sh"
+
 # ── colours (disabled when stdout is not a tty) ─────────────────────────
 if [[ -t 1 ]] || [[ "${CLICOLOR_FORCE:-0}" == "1" ]]; then
   RED='\033[0;31m'
@@ -30,10 +33,10 @@ CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-knoe-db}"
 # ── helpers ──────────────────────────────────────────────────────────────
 usage() {
   cat <<EOF
-Usage: $(basename "$0") [-c <knoe.cfg>] [-v|--verbose] [-h|--help]
+Usage: $(basename "$0") [-c <config.cfg>] [-v|--verbose] [-h|--help]
 
 Overall deployment health check.
-  -c, --config <path>   Path to knoe.cfg (default: conf/knoe.cfg)
+  -c, --config <path>   Path to config file (default: conf/{k3d|k3s|gke}.cfg)
   -v, --verbose         Show full stdout/stderr from each status check
   -h, --help            Show this help
 EOF
@@ -81,17 +84,18 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-# ── load knoe.cfg (always) ─────────────────────────────────────────────
+# ── load config (always) ────────────────────────────────────────────────
 if [[ -z "$CFG_PATH" ]]; then
-  CFG_PATH="$PROJECT_ROOT/conf/knoe.cfg"
+  CFG_PATH="$(_knoe_cfg_select_cfg_file "$PROJECT_ROOT/conf")"
 fi
+[[ -z "$CFG_PATH" ]] && CFG_PATH="$PROJECT_ROOT/conf/k3d.cfg"
 if [[ ! -f "$CFG_PATH" ]]; then
-  echo "FATAL: knoe.cfg not found at $CFG_PATH" >&2
+  echo "FATAL: config file not found at $CFG_PATH" >&2
   exit 1
 fi
 
 # Minimal INI reader — pull NAMESPACE, KUBECONFIG, KERBEROS_ENABLED,
-# SUPABASE_ENABLED from knoe.cfg if not already set in the environment.
+# SUPABASE_ENABLED from config file if not already set in the environment.
 _read_cfg_value() {
   local key="$1"
   local val=""
@@ -257,7 +261,6 @@ STATUS_SCRIPTS=(
   "init_cloudnative_pg.sh"
   "init_openbao.sh"
   "init_kong.sh"
-  "init_db_manager.sh"
   "init_monitoring.sh"
   "init_cnpg_backup.sh"
   "init_port_forwards.sh"

mock_val/status_common_services.sh

@@ -11,10 +11,13 @@ _has_config=0
 for _arg in "$@"; do
   [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
 done
-if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then
+if [[ $_has_config -eq 0 ]]; then
-  set -- "-c" "$SCRIPT_DIR/../conf/knoe.cfg" "$@"
+  _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)"
+  if [[ -n "$_default_cfg" ]]; then
+    set -- "-c" "$_default_cfg" "$@"
   fi
-unset _has_config _arg
+fi
+unset _has_config _arg _default_cfg
 
 common_core_preparse_config "$@"
 
@@ -82,6 +85,10 @@ REGISTRY_NS="${REGISTRY_NAMESPACE:-${NS}}"
 KONG_NS="${KONG_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-}}}"
 CERTMGR_NS="${CERTMGR_NAMESPACE:-cert-manager}"
 KONG_NAME="${KONG_NAME:-knoe-svc-kong}"
+# In k8s (GKE/prod) mode Kong is deployed as knoe-svc-kong
+if [ "${KNOE_MODE:-}" = "k8s" ] && [ "$KONG_NAME" = "knoe-svc-kong" ]; then
+  KONG_NAME="knoe-svc-kong"
+fi
 
 if [ -z "$KONG_NS" ]; then
   KONG_NS="default"
@@ -108,6 +115,10 @@ REGISTRY_CHECK=0
 if [[ "$MODE" == "k3s" || "$MODE" == "k3d" ]]; then
   REGISTRY_CHECK=1
 fi
+OPENTOFU_CHECK=1
+if [[ "$MODE" == "k8s" ]]; then
+  OPENTOFU_CHECK=0
+fi
 
 timestamp=$(date "+%Y-%m-%d %H:%M:%S")
 ctx=$(kubectl config current-context 2>/dev/null || true)
@@ -139,7 +150,13 @@ if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
 elif [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" == "k3d" ]]; then
   run_cmd k3d registry list
 fi
+if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
   run_cmd kubectl -n "$NS" get svc opentofu garage openbao
+else
+  run_cmd kubectl -n "$NS" get svc garage openbao
+  echo "[INFO] k8s mode: skipping OpenTofu service check."
+  echo ""
+fi
 if [[ "$ENABLE_KERBEROS" == "1" ]]; then
   run_cmd kubectl -n "$NS" get svc auth
 fi
@@ -150,7 +167,12 @@ echo "== Workloads =="
 if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
   run_cmd kubectl -n "$REGISTRY_NS" get deploy registry
 fi
+if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
   run_cmd kubectl -n "$NS" get deploy opentofu
+else
+  echo "[INFO] k8s mode: skipping OpenTofu workload check."
+  echo ""
+fi
 if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then
   run_cmd kubectl -n "$NS" get statefulset openbao
 else
@@ -167,7 +189,11 @@ echo "== Pods =="
 if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then
   run_cmd kubectl -n "$REGISTRY_NS" get pods | grep -Ei "registry" || true
 fi
+if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
   run_cmd kubectl -n "$NS" get pods | grep -Ei "opentofu|garage|openbao" || true
+else
+  run_cmd kubectl -n "$NS" get pods | grep -Ei "garage|openbao" || true
+fi
 if [[ "$ENABLE_KERBEROS" == "1" ]]; then
   run_cmd kubectl -n "$NS" get pods | grep -Ei "auth" || true
 fi
@@ -267,7 +293,9 @@ check_deploy_image() {
 
 check_k3d_registry() {
   if command -v k3d >/dev/null 2>&1; then
-    if k3d registry list --no-headers 2>/dev/null | awk '{print $1}' | grep -qx 'knoe-registry'; then
+    # k3d prefixes registry container names with "k3d-"; match by suffix and
+    # require STATUS == running (not just "created").
+    if k3d registry list --no-headers 2>/dev/null | awk '$1 ~ /knoe-registry$/ && $NF == "running"' | grep -q .; then
       echo "[OK] registry/registry exists (k3d: knoe-registry)"
       return 0
     fi
@@ -292,7 +320,9 @@ if [[ "$REGISTRY_CHECK" -eq 1 ]]; then
   fi
 fi
 
+if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
   check_resource svc opentofu "$NS"
+fi
 check_resource svc garage "$NS"
 check_resource svc openbao "$NS"
 if [[ "$ENABLE_KERBEROS" == "1" ]]; then
@@ -302,7 +332,9 @@ check_resource svc "$KONG_NAME" "$KONG_NS"
 check_resource svc cert-manager "$CERTMGR_NS"
 check_resource svc cert-manager-webhook "$CERTMGR_NS"
 
+if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
   check_resource deploy opentofu "$NS"
+fi
 if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then
   check_resource statefulset openbao "$NS"
 else
@@ -317,9 +349,12 @@ check_resource deploy cert-manager "$CERTMGR_NS"
 check_resource deploy cert-manager-cainjector "$CERTMGR_NS"
 check_resource deploy cert-manager-webhook "$CERTMGR_NS"
 
-pod_filter="opentofu|garage|openbao|kong|cert-manager"
+pod_filter="garage|openbao|kong|cert-manager"
 if [[ "$ENABLE_KERBEROS" == "1" ]]; then
-  pod_filter="opentofu|garage|openbao|auth|kong|cert-manager"
+  pod_filter="garage|openbao|auth|kong|cert-manager"
+fi
+if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
+  pod_filter="opentofu|${pod_filter}"
 fi
 analyze_pods "$NS" "$pod_filter"
 analyze_pods "$KONG_NS" "kong"
@@ -387,7 +422,11 @@ _workloads_for_comp() {
         echo "deployment openbao $_ns"
       fi
       ;;
-    opentofu) echo "deployment opentofu $_ns" ;;
+    opentofu)
+      if [[ "$OPENTOFU_CHECK" -eq 1 ]]; then
+        echo "deployment opentofu $_ns"
+      fi
+      ;;
     garage)   echo "statefulset garage $_ns" ;;
     auth)     echo "deployment auth $_ns" ;;
     kong)     echo "deployment $KONG_NAME $KONG_NS" ;;
@@ -423,6 +462,7 @@ _blocked_comps() {
   for _bc in registry openbao garage opentofu auth kong certmgr; do
     [[ "$_bc" == "auth" && "$ENABLE_KERBEROS" != "1" ]] && continue
     [[ "$_bc" == "registry" && ( "$REGISTRY_CHECK" -ne 1 || "$MODE" == "k3d" ) ]] && continue
+    [[ "$_bc" == "opentofu" && "$OPENTOFU_CHECK" -ne 1 ]] && continue
     [[ ${BLOCKED_COUNT["$_bc"]:-0} -gt 0 ]] && echo "$_bc"
   done
 }

mock_val/sync_cnpg_grafana_dashboard.py

@@ -0,0 +1,418 @@
+#!/usr/bin/env python3
+"""
+Self-updating sync tool for the cloudnative-pg Grafana dashboard.
+
+Designed for unattended / background-agent runs: fetches the upstream
+dashboard JSON, applies the transformations declared in
+`monitoring/cnpg-dashboard-transforms.yaml`, writes the result to
+`knoe-db/grafana-dashboard.json` (the canonical source the
+`knoe-db-grafana-dashboard` ConfigMap is built from), and optionally
+applies the ConfigMap to a live cluster + reloads Grafana provisioning.
+
+Usage:
+
+    # Default: fetch upstream, apply transforms, write to disk if changed.
+    # Exit 0 = no change, 1 = updated, 2 = error.
+    ./etc/sync_cnpg_grafana_dashboard.py
+
+    # Dry run — print what would change, don't write anything.
+    ./etc/sync_cnpg_grafana_dashboard.py --dry-run
+
+    # Drift check — exit 0 if our committed copy matches the upstream-after-
+    # transforms, 1 if it has drifted (newer upstream waiting to be merged
+    # OR a local hand-edit that the transforms don't cover). Doesn't write.
+    ./etc/sync_cnpg_grafana_dashboard.py --check
+
+    # Apply to live cluster after writing (rolls Grafana provisioning).
+    ./etc/sync_cnpg_grafana_dashboard.py --apply --context $APP_CLUSTER_KUBECONTEXT
+
+Exit codes:
+    0  no change (output matches transformed upstream)
+    1  output updated (or would-be-updated in --dry-run / --check)
+    2  error (network, parse, transform mismatch, kubectl failure, etc.)
+
+Background-agent run pattern (cron / k8s CronJob):
+    1. Run with `--check` once. If exit 1, work to do.
+    2. Run with `--apply --context=<ctx>` to update + roll out.
+    3. Optionally `git commit -m 'sync(cnpg): ...'` to persist the file change.
+
+Transformation philosophy: edits live in
+`monitoring/cnpg-dashboard-transforms.yaml`, not in this tool. Adding a
+new transformation type means extending `Transformer.apply()` here AND
+documenting it in the transforms YAML's preamble. Today there's just one
+type, `regex_replace`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import difflib
+import hashlib
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    print(
+        "PyYAML required. Install via: pip install pyyaml  (or apt install python3-yaml)",
+        file=sys.stderr,
+    )
+    sys.exit(2)
+
+# --------------------------------------------------------------------------
+# Constants
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_TRANSFORMS = REPO_ROOT / "monitoring" / "cnpg-dashboard-transforms.yaml"
+DEFAULT_OUTPUT = REPO_ROOT / "knoe-db" / "grafana-dashboard.json"
+DEFAULT_CM_NAME = "knoe-db-grafana-dashboard"
+DEFAULT_GRAFANA_STS = "kps-grafana"
+DEFAULT_GRAFANA_SECRET = "kps-grafana"
+
+EXIT_NOCHANGE = 0
+EXIT_CHANGED = 1
+EXIT_ERROR = 2
+
+logger = logging.getLogger("cnpg-dashboard-sync")
+
+
+# --------------------------------------------------------------------------
+# Helpers
+
+def sha256_short(s: str) -> str:
+    return hashlib.sha256(s.encode("utf-8")).hexdigest()[:12]
+
+
+def fetch_upstream(url: str, timeout: int = 30) -> str:
+    """Download the upstream dashboard JSON. Raises urllib.error on HTTP failure."""
+    logger.info("fetching upstream: %s", url)
+    req = urllib.request.Request(url, headers={"User-Agent": "knoe-cnpg-dashboard-sync/1"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.read().decode("utf-8")
+
+
+def load_transforms(path: Path) -> dict:
+    if not path.exists():
+        raise FileNotFoundError(f"transforms file not found: {path}")
+    with path.open() as f:
+        data = yaml.safe_load(f)
+    if not isinstance(data, dict) or "source" not in data:
+        raise ValueError(f"transforms file missing required `source` block: {path}")
+    return data
+
+
+# --------------------------------------------------------------------------
+# Transformer
+
+class TransformError(Exception):
+    pass
+
+
+class Transformer:
+    """Applies an ordered list of transformations to a JSON string."""
+
+    def __init__(self, transforms: list[dict]):
+        self.transforms = transforms
+
+    def apply(self, content: str) -> tuple[str, list[dict]]:
+        """Apply all transforms; return (transformed_content, summaries).
+
+        Each summary is a dict {name, type, matches, ...} for logging / drift
+        detection.
+        """
+        summaries: list[dict] = []
+        for t in self.transforms:
+            name = t.get("name", "<unnamed>")
+            ttype = t.get("type")
+            try:
+                if ttype == "regex_replace":
+                    content, summary = self._apply_regex(content, t)
+                elif ttype == "set_template_variable_default":
+                    content, summary = self._apply_set_template_default(content, t)
+                else:
+                    raise TransformError(f"unknown transformation type: {ttype!r}")
+            except TransformError:
+                raise
+            except Exception as e:
+                raise TransformError(f"transform {name!r} failed: {e}") from e
+            summary["name"] = name
+            summary["type"] = ttype
+            summaries.append(summary)
+        return content, summaries
+
+    @staticmethod
+    def _apply_regex(content: str, t: dict) -> tuple[str, dict]:
+        pattern = t["pattern"]
+        replacement = t["replacement"]
+        compiled = re.compile(pattern)
+        new_content, n = compiled.subn(replacement, content)
+
+        emin = t.get("expected_min")
+        emax = t.get("expected_max")
+        warn = None
+        if emin is not None and n < emin:
+            warn = f"matched {n} < expected_min {emin} — upstream may have moved on"
+        elif emax is not None and n > emax:
+            warn = f"matched {n} > expected_max {emax} — upstream changed shape"
+
+        return new_content, {"matches": n, "warning": warn}
+
+    @staticmethod
+    def _apply_set_template_default(content: str, t: dict) -> tuple[str, dict]:
+        """Set the `current` value of a template variable.
+
+        Useful for pinning DS_PROMETHEUS (or any other variable) to a specific
+        value so the dashboard renders correctly on first load without the user
+        needing to override via URL parameters or the variable picker.
+
+        Spec:
+            type: set_template_variable_default
+            variable: DS_PROMETHEUS                 # required, name of variable
+            value: cnpg-prometheus                  # required, current.value
+            text: cnpg-prometheus                   # optional, defaults to value
+            selected: true                          # optional, defaults to true
+        """
+        var_name = t["variable"]
+        value = t["value"]
+        text = t.get("text", value)
+        selected = t.get("selected", True)
+
+        data = json.loads(content)
+        templating = data.get("templating", {}).get("list", [])
+        found = False
+        for v in templating:
+            if v.get("name") == var_name:
+                v["current"] = {"selected": selected, "text": text, "value": value}
+                found = True
+                break
+        if not found:
+            warn = f"variable {var_name!r} not found in dashboard.templating.list"
+            return content, {"matches": 0, "warning": warn}
+
+        # Re-serialize with the same format the upstream JSON uses (2-space indent,
+        # which is what `json.dumps(..., indent=2)` produces; matches what we get
+        # from the upstream so diffs stay clean).
+        return json.dumps(data, indent=2), {"matches": 1, "warning": None}
+
+
+# --------------------------------------------------------------------------
+# Cluster apply
+
+def kubectl(args: list[str], context: str, namespace: str | None = None,
+            stdin: bytes | None = None, capture: bool = False) -> bytes:
+    """Run kubectl with explicit context (no ambient context drift)."""
+    cmd = ["kubectl", "--context", context]
+    if namespace:
+        cmd += ["-n", namespace]
+    cmd += args
+    logger.debug("running: %s", " ".join(cmd))
+    if capture:
+        return subprocess.check_output(cmd, input=stdin)
+    subprocess.run(cmd, input=stdin, check=True)
+    return b""
+
+
+def apply_to_cluster(json_content: str, context: str, namespace: str = "monitoring",
+                     cm_name: str = DEFAULT_CM_NAME) -> None:
+    """Apply the dashboard JSON as a ConfigMap + reload Grafana provisioning.
+
+    Uses --server-side apply because the JSON is large enough to hit the 256KB
+    last-applied-configuration annotation limit on client-side apply.
+    """
+    if not context:
+        raise ValueError("--apply requires --context (or APP_CLUSTER_KUBECONTEXT env)")
+
+    logger.info("applying ConfigMap %s/%s in context %s", namespace, cm_name, context)
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+        f.write(json_content)
+        tmp = f.name
+    try:
+        # Render the cm yaml via `kubectl create --dry-run`
+        cm_yaml = kubectl(
+            ["create", "configmap", cm_name,
+             f"--from-file=knoe-db.json={tmp}",
+             "--dry-run=client", "-o", "yaml"],
+            context=context, namespace=namespace, capture=True,
+        )
+        # Apply server-side
+        kubectl(
+            ["apply", "--server-side", "--force-conflicts",
+             "--field-manager=cnpg-dashboard-sync", "-f", "-"],
+            context=context, namespace=namespace, stdin=cm_yaml,
+        )
+        # Sidecar discovers grafana_dashboard=1 labelled cms; ensure label is present
+        kubectl(
+            ["label", "configmap", cm_name, "grafana_dashboard=1", "--overwrite"],
+            context=context, namespace=namespace,
+        )
+    finally:
+        os.unlink(tmp)
+
+    # Reload Grafana provisioning so the new dashboard JSON is picked up
+    # without waiting for the sidecar's poll interval.
+    logger.info("reloading Grafana provisioning")
+    admin_pw_b64 = kubectl(
+        ["get", "secret", DEFAULT_GRAFANA_SECRET,
+         "-o", "jsonpath={.data.admin-password}"],
+        context=context, namespace=namespace, capture=True,
+    )
+    admin_pw = base64.b64decode(admin_pw_b64).decode()
+    kubectl(
+        ["exec", f"sts/{DEFAULT_GRAFANA_STS}", "-c", "grafana", "--",
+         "wget", "-qO-", "--post-data", "",
+         f"http://admin:{admin_pw}@localhost:3000/api/admin/provisioning/dashboards/reload"],
+        context=context, namespace=namespace, capture=True,
+    )
+    logger.info("grafana provisioning reload complete")
+
+
+# --------------------------------------------------------------------------
+# Main
+
+def render_diff(old: str, new: str, max_lines: int = 60) -> str:
+    diff = list(difflib.unified_diff(
+        old.splitlines(keepends=True),
+        new.splitlines(keepends=True),
+        fromfile="committed", tofile="upstream-transformed", n=2,
+    ))
+    if len(diff) > max_lines:
+        diff = diff[:max_lines] + [f"... ({len(diff) - max_lines} more lines elided)\n"]
+    return "".join(diff)
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--transforms", type=Path, default=DEFAULT_TRANSFORMS,
+                   help="Path to the transforms YAML (default: %(default)s)")
+    p.add_argument("--output", type=Path, default=DEFAULT_OUTPUT,
+                   help="Path to write the transformed JSON (default: %(default)s)")
+    mode = p.add_mutually_exclusive_group()
+    mode.add_argument("--dry-run", action="store_true",
+                      help="Show what would change; don't write or apply.")
+    mode.add_argument("--check", action="store_true",
+                      help="Exit 1 if output is drifted from upstream-transformed; don't write.")
+    p.add_argument("--apply", action="store_true",
+                   help="After writing, apply the ConfigMap to the live cluster and reload Grafana.")
+    p.add_argument("--context", default=os.environ.get("APP_CLUSTER_KUBECONTEXT", ""),
+                   help="kubectl context for --apply (default: $APP_CLUSTER_KUBECONTEXT)")
+    p.add_argument("--namespace", default="monitoring",
+                   help="ConfigMap namespace for --apply (default: %(default)s)")
+    p.add_argument("--show-diff", action="store_true",
+                   help="Print a unified diff of committed-vs-transformed.")
+    p.add_argument("-v", "--verbose", action="store_true",
+                   help="Verbose logging.")
+    args = p.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+    try:
+        config = load_transforms(args.transforms)
+    except (FileNotFoundError, ValueError) as e:
+        logger.error("failed to load transforms: %s", e)
+        return EXIT_ERROR
+
+    src = config["source"]
+    upstream_url = src["url"]
+    expected_uid = src.get("expected_uid")
+    transforms = config.get("transformations", [])
+
+    # 1. Fetch
+    try:
+        upstream = fetch_upstream(upstream_url)
+    except urllib.error.URLError as e:
+        logger.error("upstream fetch failed: %s", e)
+        return EXIT_ERROR
+    logger.info("upstream: sha256=%s, %d bytes", sha256_short(upstream), len(upstream))
+
+    # 2. Validate UID (catch silent renames)
+    if expected_uid:
+        try:
+            uid = json.loads(upstream).get("uid")
+        except json.JSONDecodeError as e:
+            logger.error("upstream is not valid JSON: %s", e)
+            return EXIT_ERROR
+        if uid != expected_uid:
+            logger.error("upstream uid %r != expected %r — refusing to proceed; "
+                         "dashboard URLs would silently break. Update transforms YAML if intentional.",
+                         uid, expected_uid)
+            return EXIT_ERROR
+
+    # 3. Transform
+    try:
+        transformed, summaries = Transformer(transforms).apply(upstream)
+    except TransformError as e:
+        logger.error("transform failed: %s", e)
+        return EXIT_ERROR
+
+    logger.info("applied %d transformation(s):", len(summaries))
+    saw_warning = False
+    for s in summaries:
+        msg = f"  - {s['name']} ({s['type']}): {s.get('matches', '?')} match(es)"
+        if s.get("warning"):
+            msg += f"  [WARNING: {s['warning']}]"
+            saw_warning = True
+        logger.info(msg)
+    logger.info("transformed: sha256=%s, %d bytes", sha256_short(transformed), len(transformed))
+
+    # 4. Compare to existing
+    existing = args.output.read_text() if args.output.exists() else ""
+    drifted = (existing != transformed)
+
+    if args.show_diff and drifted:
+        logger.info("DIFF (committed vs transformed-upstream):")
+        sys.stderr.write(render_diff(existing, transformed))
+
+    # 5. Act per mode
+    if args.check:
+        if drifted:
+            logger.warning("DRIFT: %s differs from upstream after transforms (run without --check to update)", args.output)
+            return EXIT_CHANGED
+        logger.info("OK: %s matches upstream after transforms", args.output)
+        return EXIT_ERROR if saw_warning else EXIT_NOCHANGE
+
+    if args.dry_run:
+        if drifted:
+            logger.info("WOULD UPDATE: %s (run without --dry-run to apply)", args.output)
+            return EXIT_CHANGED
+        logger.info("NO CHANGE: %s", args.output)
+        return EXIT_ERROR if saw_warning else EXIT_NOCHANGE
+
+    # Default mode: write if drifted, optionally apply
+    if drifted:
+        args.output.write_text(transformed)
+        logger.info("wrote %s", args.output)
+        rc = EXIT_CHANGED
+    else:
+        logger.info("no change to %s", args.output)
+        rc = EXIT_NOCHANGE
+
+    if args.apply:
+        try:
+            apply_to_cluster(transformed, context=args.context, namespace=args.namespace)
+        except (subprocess.CalledProcessError, ValueError) as e:
+            logger.error("cluster apply failed: %s", e)
+            return EXIT_ERROR
+
+    return EXIT_ERROR if saw_warning else rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())