prole/mock_val/init_garage_store.sh

#!/usr/bin/env bash

set -euo pipefail

# init_garage_store.sh
# Purpose:
# - Deploy Garage (S3-compatible object store) in Kubernetes
# - Initialize single-node layout for immediate use

# Initialize SCRIPT_DIR
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"

common_core_preparse_config "$@"

# shellcheck disable=SC1090
source "$SCRIPT_DIR/knoe_cfg.sh"

set -- "${COMMON_CORE_ARGS[@]}"
common_core_parse_args "$@"

if [[ "${COMMON_CORE_HELP:-0}" == 1 ]]; then
  common_core_usage "$0"
  exit 0
fi

if [[ -n "${COMMON_CORE_PARSE_ERROR:-}" ]]; then
  echo "ERROR: ${COMMON_CORE_PARSE_ERROR}" >&2
  common_core_usage "$0"
  exit 2
fi

ACTION="$COMMON_CORE_ACTION"

if [[ -n "${GARAGE_INIT_LOG:-}" ]]; then
  mkdir -p "$(dirname "$GARAGE_INIT_LOG")"
  exec > >(tee -a "$GARAGE_INIT_LOG") 2>&1
fi

RESOLVED_NAMESPACE="$(common_core_resolve_namespace "default")"
common_core_apply_namespace "$RESOLVED_NAMESPACE"

GARAGE_NAME=${GARAGE_NAME:-garage}
GARAGE_SECRET_NAME=${GARAGE_SECRET_NAME:-garage-secrets}
GARAGE_NODE_CAPACITY=${GARAGE_NODE_CAPACITY:-10GB}
GARAGE_ZONE=${GARAGE_ZONE:-local}
GARAGE_NAMESPACE=${GARAGE_NAMESPACE:-$RESOLVED_NAMESPACE}
NAMESPACE="$GARAGE_NAMESPACE"
APP_CLUSTER_KUBECONTEXT=${APP_CLUSTER_KUBECONTEXT:-${init_cluster_app_cluster_kubecontext:-}}
DB_CLUSTER_KUBECONTEXT=${DB_CLUSTER_KUBECONTEXT:-${init_cluster_db_cluster_kubecontext:-}}
GARAGE_AUTHORITY_ROLE=${GARAGE_AUTHORITY_ROLE:-db}
GARAGE_AUTHORITY_ROLE="${GARAGE_AUTHORITY_ROLE,,}"
ACTIVE_KUBE_CONTEXT=${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}

garage_context_role() {
  local ctx="${ACTIVE_KUBE_CONTEXT:-}"
  if [[ -z "$ctx" ]]; then
    ctx=$(kubectl config current-context 2>/dev/null || true)
  fi
  if [[ -n "$APP_CLUSTER_KUBECONTEXT" && "$ctx" == "$APP_CLUSTER_KUBECONTEXT" ]]; then
    printf '%s' "app"
    return 0
  fi
  if [[ -n "$DB_CLUSTER_KUBECONTEXT" && "$ctx" == "$DB_CLUSTER_KUBECONTEXT" ]]; then
    printf '%s' "db"
    return 0
  fi
  printf '%s' "unknown"
}

garage_split_cluster_contexts_enabled() {
  [[ -n "$APP_CLUSTER_KUBECONTEXT" && -n "$DB_CLUSTER_KUBECONTEXT" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]]
}

garage_is_authoritative_context() {
  local role
  if ! garage_split_cluster_contexts_enabled; then
    return 0
  fi
  role="$(garage_context_role)"
  [[ "$role" == "$GARAGE_AUTHORITY_ROLE" ]]
}

garage_ensure_absent_non_authoritative() {
  local role
  role="$(garage_context_role)"
  echo "[INFO] Garage authority role is '${GARAGE_AUTHORITY_ROLE}'; current role '${role}' is non-authoritative. Ensuring Garage is absent in namespace '$NAMESPACE'."
  delete_manifests || true
}

# Support both KNOE_HOME/k8s and sibling k8s directory
if [[ -d "$SCRIPT_DIR/../k8s/knoe" ]]; then
  GARAGE_MANIFEST_DIR="$SCRIPT_DIR/../k8s/knoe"
elif [[ -n "${KNOE_HOME:-}" && -d "$KNOE_HOME/k8s/knoe" ]]; then
  GARAGE_MANIFEST_DIR="$KNOE_HOME/k8s/knoe"
else
  GARAGE_MANIFEST_DIR="$SCRIPT_DIR/../k8s/knoe"
fi

GARAGE_FILES=(
  "$GARAGE_MANIFEST_DIR/storageclass-synology-iscsi.yaml"
  "$GARAGE_MANIFEST_DIR/iscsi-pvs.yaml"
  "$GARAGE_MANIFEST_DIR/garage-configmap.yaml"
  "$GARAGE_MANIFEST_DIR/garage-statefulset.yaml"
  "$GARAGE_MANIFEST_DIR/garage-service.yaml"
)
if [[ "${KNOE_MODE:-}" == "k3d" ]]; then
  # k3d: no local storage provisioner — skip synology StorageClass and static PVs.
  GARAGE_FILES=(
    "$GARAGE_MANIFEST_DIR/garage-configmap.yaml"
    "$GARAGE_MANIFEST_DIR/garage-statefulset.yaml"
    "$GARAGE_MANIFEST_DIR/garage-service.yaml"
  )
elif [[ "${KNOE_MODE:-}" == "k8s" ]]; then
  # GKE Autopilot: apply our custom garage-hdd StorageClass (pd-standard HDD, avoids SSD quota).
  # The skip-if-exists guard in apply_manifests handles idempotent re-runs safely.
  # Use GCP-specific statefulset (no synology selectors, explicit resource requests).
  GARAGE_FILES=(
    "$GARAGE_MANIFEST_DIR/storageclass-gcp-hdd.yaml"
    "$GARAGE_MANIFEST_DIR/garage-configmap.yaml"
    "$GARAGE_MANIFEST_DIR/garage-statefulset-gcp.yaml"
    "$GARAGE_MANIFEST_DIR/garage-service.yaml"
  )
fi

GARAGE_APPLY_CHANGED=0
GARAGE_CONFIG_CHANGED=0
GARAGE_STATEFULSET_CHANGED=0

ensure_tools() {
  for t in kubectl openssl; do
    command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
  done
}

ensure_namespace() {
  if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
    echo "Creating namespace '$NAMESPACE' ..."
    kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true
  fi
}

ensure_secrets() {
  if kubectl get secret "$GARAGE_SECRET_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
    return 0
  fi

  echo "Creating Garage secrets in namespace '$NAMESPACE' ..."
  # Wait for apiserver to be ready
  local count=0
  while ! kubectl get nodes >/dev/null 2>&1; do
      if [ $count -ge 30 ]; then
          echo "ERROR: apiserver not ready after 60s" >&2
          exit 1
      fi
      echo "Waiting for apiserver... ($count/30)"
      sleep 2
      count=$((count + 1))
  done

  local rpc_secret admin_token metrics_token
  rpc_secret=$(openssl rand -hex 32)
  admin_token=$(openssl rand -base64 32)
  metrics_token=$(openssl rand -base64 32)

  kubectl create secret generic "$GARAGE_SECRET_NAME" -n "$NAMESPACE" \
    --from-literal=rpc_secret="$rpc_secret" \
    --from-literal=admin_token="$admin_token" \
    --from-literal=metrics_token="$metrics_token"
}

k3d_cleanup_pending_pvc() {
  if [[ "${KNOE_MODE:-}" != "k3d" ]]; then
    return 0
  fi
  if ! kubectl get pvc data-garage-0 -n "$NAMESPACE" >/dev/null 2>&1; then
    return 0
  fi
  local phase selector
  phase=$(kubectl get pvc data-garage-0 -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || true)
  if [[ "$phase" == "Bound" ]]; then
    return 0
  fi
  selector=$(kubectl get pvc data-garage-0 -n "$NAMESPACE" -o jsonpath='{.spec.selector}' 2>/dev/null || true)
  if printf '%s' "$selector" | grep -q "synology.storage/"; then
    echo "Removing pending Garage PVC with selector for k3d local-path ..."
    kubectl delete statefulset "$GARAGE_NAME" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
    kubectl delete pvc data-garage-0 -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
  fi
}

apply_manifests() {
  GARAGE_APPLY_CHANGED=0
  GARAGE_CONFIG_CHANGED=0
  GARAGE_STATEFULSET_CHANGED=0
  for f in "${GARAGE_FILES[@]}"; do
    if [[ -f "$f" ]]; then
      local output=""
      local rendered=""
      local diff_out=""
      local diff_rc=0
      rendered=$(knoe_render_manifest "$f")
      if diff_out=$(printf '%s' "$rendered" | kubectl diff -n "$NAMESPACE" -f - 2>&1); then
        diff_rc=0
      else
        diff_rc=$?
      fi
      if [[ $diff_rc -eq 0 ]]; then
        continue
      fi
      if [[ $diff_rc -ne 1 ]]; then
        # For immutable StatefulSet VolumeClaimTemplates, kubectl diff itself returns an
        # error (diff_rc != 1 but contains the immutable spec error). Handle it here.
        if [[ "$(basename "$f")" == garage-statefulset*.yaml ]] \
          && echo "$diff_out" | grep -q "updates to statefulset spec"; then
          echo "WARN: Garage StatefulSet VolumeClaimTemplates changed (diff-stage); deleting and recreating ..."
          kubectl delete statefulset "$GARAGE_NAME" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
          local _pvc_sc_d
          _pvc_sc_d=$(kubectl get pvc "data-garage-0" -n "$NAMESPACE" \
            -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
          if [[ -n "$_pvc_sc_d" ]]; then
            echo "  Removing stale PVC 'data-garage-0' (storageClass: $_pvc_sc_d) ..."
            kubectl delete pvc "data-garage-0" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
          fi
          printf '%s' "$rendered" | kubectl apply --validate=false -n "$NAMESPACE" -f -
          GARAGE_APPLY_CHANGED=1
          GARAGE_STATEFULSET_CHANGED=1
          continue
        fi
        echo "$diff_out" >&2
        exit 1
      fi
      # StorageClass resources have immutable parameters/reclaimPolicy.
      # If the StorageClass already exists in the cluster, skip re-applying it.
      if echo "$rendered" | grep -q "^kind: StorageClass"; then
        local _sc_name
        _sc_name=$(echo "$rendered" | grep "^  name:" | head -1 | awk '{print $2}')
        if [[ -n "$_sc_name" ]] && kubectl get storageclass "$_sc_name" >/dev/null 2>&1; then
          echo "[SKIP] StorageClass '$_sc_name' already exists; skipping apply (immutable fields)."
          continue
        fi
      fi
      if output=$(printf '%s' "$rendered" | kubectl apply --validate=false -n "$NAMESPACE" -f - 2>&1); then
        printf '%s\n' "$output"
        GARAGE_APPLY_CHANGED=1
        case "$(basename "$f")" in
          garage-configmap.yaml) GARAGE_CONFIG_CHANGED=1 ;;
          garage-statefulset.yaml) GARAGE_STATEFULSET_CHANGED=1 ;;
        esac
      else
        if [[ "${KNOE_MODE:-}" == "k3d" && "$(basename "$f")" == "garage-statefulset.yaml" ]] \
          && echo "$output" | grep -q "updates to statefulset spec"; then
          echo "WARN: Garage StatefulSet immutable in k3d; skipping apply."
          continue
        elif [[ "$(basename "$f")" == garage-statefulset*.yaml ]] \
          && echo "$output" | grep -q "updates to statefulset spec"; then
          # VolumeClaimTemplates are immutable; delete the StatefulSet (PVCs are orphaned/preserved)
          # and recreate so the new storageClass name takes effect.
          echo "WARN: Garage StatefulSet VolumeClaimTemplates changed; deleting and recreating ..."
          kubectl delete statefulset "$GARAGE_NAME" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
          local _pvc_sc
          _pvc_sc=$(kubectl get pvc "data-garage-0" -n "$NAMESPACE" \
            -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)
          if [[ -n "$_pvc_sc" ]]; then
            echo "  Removing stale PVC 'data-garage-0' (storageClass: $_pvc_sc) ..."
            kubectl delete pvc "data-garage-0" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
          fi
          printf '%s' "$rendered" | kubectl apply --validate=false -n "$NAMESPACE" -f -
          GARAGE_APPLY_CHANGED=1
          GARAGE_STATEFULSET_CHANGED=1
          continue
        fi
        echo "$output" >&2
        exit 1
      fi
    else
      echo "ERROR: Missing manifest: $f" >&2
      exit 1
    fi
  done
}

ensure_container_command() {
  if ! kubectl get statefulset "$GARAGE_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
    return 0
  fi

  local cmd args
  cmd=$(kubectl get statefulset "$GARAGE_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.template.spec.containers[?(@.name=="garage")].command}' 2>/dev/null || true)
  args=$(kubectl get statefulset "$GARAGE_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.template.spec.containers[?(@.name=="garage")].args}' 2>/dev/null || true)

  if [[ -z "$cmd" || "$cmd" == "[]" || "$cmd" != *"/garage"* || -z "$args" || "$args" == "[]" || "$args" != *"server"* ]]; then
    echo "Patching Garage container command/args ..."
    kubectl patch statefulset "$GARAGE_NAME" -n "$NAMESPACE" --type merge -p '{
      "spec": {
        "template": {
          "spec": {
            "containers": [
              {
                "name": "garage",
                "command": ["/garage"],
                "args": ["server"]
              }
            ]
          }
        }
      }
    }' >/dev/null || true
  fi
}

pin_garage_node_k3d() {
  if [[ "${KNOE_MODE:-}" != "k3d" ]]; then
    return 0
  fi
  if [[ -z "${GARAGE_PIN_NODE:-}" ]]; then
    return 0
  fi
  if ! kubectl get statefulset "$GARAGE_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
    return 0
  fi
  local target=""
  target=$(kubectl get nodes -l node-role.kubernetes.io/master -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -z "$target" ]]; then
    target=$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  fi
  if [[ -z "$target" ]]; then
    return 0
  fi
  if [[ "$target" != k3d-* ]]; then
    return 0
  fi
  echo "Pinning Garage StatefulSet to node '$target' for k3d exec access ..."
  kubectl patch statefulset "$GARAGE_NAME" -n "$NAMESPACE" --type merge \
    -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\"$target\"}}}}}" >/dev/null || true
}

restart_statefulset() {
  if [[ "${GARAGE_CONFIG_CHANGED:-0}" -eq 0 ]]; then
    return 0
  fi
  if [[ "${GARAGE_STATEFULSET_CHANGED:-0}" -eq 1 ]]; then
    return 0
  fi
  if kubectl get statefulset "$GARAGE_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
    echo "Restarting Garage StatefulSet to pick up config changes ..."
    kubectl rollout restart statefulset/$GARAGE_NAME -n "$NAMESPACE" || true
  fi
}

delete_manifests() {
  for f in "${GARAGE_FILES[@]}"; do
    if [[ -f "$f" ]]; then
      kubectl delete -n "$NAMESPACE" -f "$f" --ignore-not-found
    fi
  done
}

dump_debug() {
  echo "---- Garage debug ----"
  kubectl get statefulset "$GARAGE_NAME" -n "$NAMESPACE" -o wide || true
  kubectl get pods -n "$NAMESPACE" -l "app=$GARAGE_NAME" -o wide || true
  kubectl get svc "$GARAGE_NAME" -n "$NAMESPACE" -o wide || true
  kubectl get events -n "$NAMESPACE" --sort-by=.metadata.creationTimestamp | tail -n 50 || true

  local pod
  pod=$(kubectl get pods -n "$NAMESPACE" -l "app=$GARAGE_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -n "$pod" ]]; then
    echo "--- pod: $pod (describe) ---"
    kubectl describe pod "$pod" -n "$NAMESPACE" || true
    echo "--- logs (current) ---"
    kubectl logs -n "$NAMESPACE" "$pod" --tail=200 || true
    echo "--- logs (previous) ---"
    kubectl logs -n "$NAMESPACE" "$pod" --previous --tail=200 || true
  fi
  echo "---- Garage debug end ----"
}

recycle_released_knoe_iscsi_pv_for_pvc() {
  # StorageClass `synology-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain
  # in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding.
  local ns="$1"
  local pvc="$2"

  if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then
    return 0
  fi

  local pvc_phase pvc_uid
  pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true)
  if [[ "$pvc_phase" != "Pending" ]]; then
    return 0
  fi
  pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)

  local pv sc rp phase claim_ns claim_name claim_uid
  while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do
    [[ "$sc" == "synology-iscsi" ]] || continue
    [[ "$rp" == "Retain" ]] || continue
    [[ "$phase" == "Released" ]] || continue
    [[ "$claim_ns" == "$ns" ]] || continue
    [[ "$claim_name" == "$pvc" ]] || continue

    # If the PV claimRef UID matches the current PVC UID, don't touch it.
    if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then
      continue
    fi

    echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..."
    kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
  done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true)
}

wait_ready() {
  echo "Waiting for Garage StatefulSet to become ready ..."

  local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}"
  if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout="$initial_timeout"; then
    if [[ "${KNOE_MODE:-}" == "k8s" ]]; then
      echo "WARN: Garage not ready after $initial_timeout; PVC provisioning may still be in progress (GKE CSI)." >&2
    else
      echo "WARN: Garage not ready after $initial_timeout; checking for Released synology-iscsi PVs with stale claimRefs ..." >&2
    fi
    recycle_released_knoe_iscsi_pv_for_pvc "$NAMESPACE" "data-garage-0" || true
  fi

  if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s}; then
    echo "ERROR: Garage StatefulSet did not become ready in time." >&2
    dump_debug
    return 1
  fi
}

init_layout() {
  local pod
  pod=$(kubectl get pods -n "$NAMESPACE" -l "app=$GARAGE_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
  if [[ -z "$pod" ]]; then
    echo "ERROR: Garage pod not found in namespace '$NAMESPACE'." >&2
    exit 1
  fi

  local node_id=""
  echo "Determining Garage node ID..."
  for i in {1..30}; do
    node_id=$(kubectl exec -n "$NAMESPACE" "$pod" -- /garage node id -q 2>/dev/null | awk '{print $1}' || true)
    if [[ -n "$node_id" ]]; then
      break
    fi
    echo "Waiting for Garage node ID to be available... ($i/30)"
    sleep 2
  done

  if [[ -z "$node_id" ]]; then
    echo "ERROR: Unable to determine Garage node ID after 30 attempts." >&2
    dump_debug
    exit 1
  fi

  # Extract the short ID for better matching
  local short_id
  short_id=$(echo "$node_id" | cut -d'@' -f1)
  echo "Garage node ID: $short_id"

  local layout
  layout=$(kubectl exec -n "$NAMESPACE" "$pod" -- /garage layout show 2>/dev/null || true)
  if ! echo "$layout" | grep -q "$short_id"; then
    echo "Assigning Garage node role (capacity: $GARAGE_NODE_CAPACITY, zone: $GARAGE_ZONE) ..."
    # Retry assigning role as it might fail if node is not yet fully ready in the cluster logic
    local max_assign_retries=10
    local assign_count=0
    while ! kubectl exec -n "$NAMESPACE" "$pod" -- /garage layout assign -z "$GARAGE_ZONE" -c "$GARAGE_NODE_CAPACITY" "$short_id"; do
      if [[ $assign_count -ge $max_assign_retries ]]; then
        echo "ERROR: Failed to assign Garage node role after $max_assign_retries retries." >&2
        exit 1
      fi
      echo "Retrying Garage layout assign... ($((assign_count + 1))/$max_assign_retries)"
      sleep 2
      assign_count=$((assign_count + 1))
    done
    
    # Re-fetch layout and apply staged changes using the NEXT layout version
    local layout_after version next_version apply_ok
    layout_after=$(kubectl exec -n "$NAMESPACE" "$pod" -- /garage layout show 2>/dev/null | grep -v "INFO" || true)
    # Extract "Current cluster layout version: X"
    version=$(echo "$layout_after" | awk -F: '/Current cluster layout version/ {gsub(/[[:space:]]/,"",$2); print $2; exit}' || true)
    
    if echo "$layout_after" | grep -qi "staged"; then
      if [[ -n "$version" ]]; then
        next_version=$((version + 1))
        echo "Applying Garage layout (current version $version, next version $next_version) ..."
        if kubectl exec -n "$NAMESPACE" "$pod" -- /garage layout apply --version "$next_version"; then
          apply_ok=1
        else
          apply_ok=0
        fi
      else
        apply_ok=0
      fi
      if [[ "${apply_ok:-0}" -eq 0 ]]; then
        echo "WARN: layout apply with explicit version failed; retrying without version..."
        kubectl exec -n "$NAMESPACE" "$pod" -- /garage layout apply || true
      fi
    else
      echo "No staged layout changes detected; skipping layout apply."
    fi
  else
    echo "Garage layout already assigned for node $short_id."
  fi
}

status() {
  ensure_tools
  echo "Garage status in namespace '$NAMESPACE':"
  kubectl get statefulset "$GARAGE_NAME" -n "$NAMESPACE" || true
  kubectl get pods -n "$NAMESPACE" -l "app=$GARAGE_NAME" || true
  kubectl get svc "$GARAGE_NAME" -n "$NAMESPACE" || true
}

case "$ACTION" in
  start|initialize|update|reload|restart)
    ensure_tools
    if [[ "$GARAGE_AUTHORITY_ROLE" != "db" && "$GARAGE_AUTHORITY_ROLE" != "app" ]]; then
      echo "ERROR: GARAGE_AUTHORITY_ROLE must be 'db' or 'app' (got '$GARAGE_AUTHORITY_ROLE')." >&2
      exit 1
    fi
    if garage_split_cluster_contexts_enabled; then
      current_role="$(garage_context_role)"
      if [[ "$current_role" == "unknown" ]]; then
        echo "ERROR: Unable to resolve Garage deployment cluster role from kubecontext '${ACTIVE_KUBE_CONTEXT:-<unset>}' (APP='${APP_CLUSTER_KUBECONTEXT:-<unset>}', DB='${DB_CLUSTER_KUBECONTEXT:-<unset>}')." >&2
        exit 1
      fi
      if ! garage_is_authoritative_context; then
        garage_ensure_absent_non_authoritative
        exit 0
      fi
    fi
    ensure_namespace
    ensure_secrets
    k3d_cleanup_pending_pvc
    echo "Applying Garage manifests in namespace '$NAMESPACE'..."
    apply_manifests
    ensure_container_command
    pin_garage_node_k3d
    restart_statefulset
    wait_ready
    init_layout
    knoe_register_port_forward "garage" "${NAMESPACE:-default}" "svc/garage" "3900" "3900" "0.0.0.0" "TCP" "Garage S3 API"
    ;;
  stop)
    ensure_tools
    echo "Deleting Garage manifests from namespace '$NAMESPACE'..."
    delete_manifests
    ;;
  status)
    status
    ;;
  *)
    common_core_usage "$0"
    exit 1
    ;;
esac