prole/etc/repair_pipeline.sh

#!/usr/bin/env bash

set -uo pipefail

# repair_pipeline.sh
# Purpose:
# - Validate cluster readiness and common service health
# - Repair or re-deploy safe-to-recreate services when anomalies are detected

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck disable=SC1090
source "$SCRIPT_DIR/prole_cfg.sh"

ACTION="repair"
NAMESPACE_OVERRIDE=""
DB_NAMESPACE_OVERRIDE=""

usage() {
  cat <<'USAGE'
Usage: repair_pipeline.sh [-n|--namespace NS] [--db-namespace NS] [-m|--mode MODE] [repair]

Runs a best-effort repair pass across common services and cluster add-ons.
USAGE
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    -m|--mode)
      shift
      prole_set_mode "${1:-}"
      ;;
    -m=*|--mode=*)
      prole_set_mode "${1#*=}"
      ;;
    -n|--namespace)
      shift
      NAMESPACE_OVERRIDE="${1:-}"
      ;;
    --db-namespace)
      shift
      DB_NAMESPACE_OVERRIDE="${1:-}"
      ;;
    --db-namespace=*)
      DB_NAMESPACE_OVERRIDE="${1#*=}"
      ;;
    -n=*|--namespace=*)
      NAMESPACE_OVERRIDE="${1#*=}"
      ;;
    repair)
      ACTION="repair"
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      usage
      exit 2
      ;;
  esac
  shift
done

BASE_NAMESPACE="${NAMESPACE:-}"
if [[ -n "$NAMESPACE_OVERRIDE" ]]; then
  SERVICE_NAMESPACE="$NAMESPACE_OVERRIDE"
elif [[ -n "${SERVICE_NAMESPACE:-}" ]]; then
  SERVICE_NAMESPACE="$SERVICE_NAMESPACE"
elif [[ -n "${NAMESPACE:-}" ]]; then
  SERVICE_NAMESPACE="$NAMESPACE"
else
  SERVICE_NAMESPACE="default"
fi

# In k3s mode, common core services must live in knoe-system by default.
_mode_resolved="${PROLE_MODE:-}"
if declare -F prole_normalize_mode >/dev/null 2>&1; then
  _mode_resolved="$(prole_normalize_mode "$_mode_resolved")"
fi
if [[ "$_mode_resolved" == "k3s" && ( -z "${SERVICE_NAMESPACE:-}" || "${SERVICE_NAMESPACE}" == "default" ) ]]; then
  SERVICE_NAMESPACE="knoe-system"
fi
unset _mode_resolved

if [[ -n "$DB_NAMESPACE_OVERRIDE" ]]; then
  DB_NAMESPACE="$DB_NAMESPACE_OVERRIDE"
elif [[ -n "${PROLE_DB_NAMESPACE:-}" ]]; then
  DB_NAMESPACE="$PROLE_DB_NAMESPACE"
elif [[ -n "$BASE_NAMESPACE" ]]; then
  DB_NAMESPACE="$BASE_NAMESPACE"
else
  DB_NAMESPACE="$SERVICE_NAMESPACE"
fi

ARGOCD_NAMESPACE="${ARGOCD_NAMESPACE:-argocd}"

export SERVICE_NAMESPACE
export DB_NAMESPACE
export NAMESPACE="$SERVICE_NAMESPACE"

log() { printf '%s\n' "$*"; }
warn() { printf 'WARN: %s\n' "$*" >&2; }
err() { printf 'ERROR: %s\n' "$*" >&2; }

have() { command -v "$1" >/dev/null 2>&1; }

run_with_timeout() {
  local timeout_s="$1"
  shift
  local cmd=( "$@" )
  "${cmd[@]}" &
  local pid=$!
  local start=$SECONDS
  while kill -0 "$pid" >/dev/null 2>&1; do
    if (( SECONDS - start > timeout_s )); then
      warn "Command timed out after ${timeout_s}s: ${cmd[*]}"
      kill "$pid" >/dev/null 2>&1 || true
      return 124
    fi
    sleep 2
  done
  wait "$pid"
  return $?
}

ensure_kubectl() {
  have kubectl || { err "kubectl not found"; exit 1; }
}

detect_mode() {
  local mode
  mode="${PROLE_MODE:-}"
  if have prole_normalize_mode; then
    mode="$(prole_normalize_mode "$mode")"
  fi
  printf '%s' "$mode"
}

ensure_kubeconfig_if_needed() {
  local mode
  mode="$(detect_mode)"
  if [[ "$mode" == "k3s" || -n "${PROLE_K3S_SERVER:-}" || -n "${K3S_SERVER_URL:-}" ]]; then
    prole_ensure_kubeconfig >/dev/null 2>&1 || true
  fi
}

cluster_ready() {
  ensure_kubectl
  if ! kubectl cluster-info >/dev/null 2>&1; then
    return 1
  fi
  return 0
}

k3d_cluster_ready() {
  local cluster="${K3D_CLUSTER:-knoe-dev-cluster}"
  if have k3d; then
    if k3d cluster list --no-headers 2>/dev/null | grep -q "^${cluster}[[:space:]]"; then
      return 0
    fi
    return 1
  fi
  return 0
}

resource_exists() {
  local kind="$1" name="$2" ns="$3"
  kubectl -n "$ns" get "$kind" "$name" >/dev/null 2>&1
}

resource_ready() {
  local kind="$1" name="$2" ns="$3"
  local desired ready
  if ! resource_exists "$kind" "$name" "$ns"; then
    return 2
  fi
  desired=$(kubectl -n "$ns" get "$kind" "$name" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")
  ready=$(kubectl -n "$ns" get "$kind" "$name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "")
  desired=${desired:-1}
  ready=${ready:-0}
  if [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]]; then
    return 0
  fi
  return 1
}

authority_context_exists() {
  local base
  if [[ -n "${PROLE_HOME:-}" ]]; then
    if [[ -d "$PROLE_HOME/authority" || -d "$PROLE_HOME/prole/authority" ]]; then
      return 0
    fi
  fi
  base="$SCRIPT_DIR/.."
  if [[ -d "$base/authority" || -d "$base/prole/authority" ]]; then
    return 0
  fi
  return 1
}

repair_dashboard() {
  local ns="kubernetes-dashboard"
  if ! kubectl get ns "$ns" >/dev/null 2>&1; then
    warn "Dashboard namespace missing; installing via Helm"
    if have helm; then
      helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/ >/dev/null 2>&1 || true
      if ! run_with_timeout 120 bash -c 'helm repo update >/dev/null 2>&1'; then
        warn "Helm repo update timed out; continuing with cached index"
      fi
      if ! run_with_timeout 300 helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard \
        --create-namespace --namespace "$ns" \
        --set kong.image.tag=3.8 --set kong.image.repository=docker.io/library/kong; then
        warn "Helm install timed out or failed"
      fi
    else
      warn "helm not found; cannot install kubernetes-dashboard"
      return 1
    fi
  fi

  local kong_pods
  kong_pods=$(kubectl -n "$ns" get pods --no-headers 2>/dev/null | awk '{print $1}' | grep -E 'kong' || true)
  if [[ -z "$kong_pods" ]]; then
    warn "Dashboard Kong pod missing; re-applying Helm release"
    if have helm; then
      if ! run_with_timeout 300 helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard \
        --create-namespace --namespace "$ns" \
        --set kong.image.tag=3.8 --set kong.image.repository=docker.io/library/kong; then
        warn "Helm install timed out or failed"
      fi
    else
      warn "helm not found; cannot reinstall kubernetes-dashboard"
      return 1
    fi
  else
    local unhealthy
    unhealthy=$(kubectl -n "$ns" get pods --no-headers 2>/dev/null | grep -E 'kong' | awk '$3 != "Running" {print $1}' || true)
    if [[ -n "$unhealthy" ]]; then
      warn "Dashboard Kong pod unhealthy; deleting pods for restart"
      kubectl -n "$ns" delete pod $unhealthy --wait=false >/dev/null 2>&1 || true
    fi
  fi
}

repair_argocd() {
  local ns="$1"
  local reg_ns
  reg_ns="${REGISTRY_NAMESPACE:-$SERVICE_NAMESPACE}"
  local need_fix=0
  local items=(
    "deploy argocd-server"
    "deploy argocd-repo-server"
    "deploy argocd-dex-server"
    "deploy argocd-applicationset-controller"
    "deploy argocd-notifications-controller"
    "deploy argocd-redis"
    "statefulset argocd-application-controller"
  )
  local item kind name status
  for item in "${items[@]}"; do
    kind="${item%% *}"
    name="${item##* }"
    resource_ready "$kind" "$name" "$ns"; status=$?
    if [[ "$status" -ne 0 ]]; then
      need_fix=1
      break
    fi
  done
  if [[ "$need_fix" -eq 1 ]]; then
    warn "ArgoCD not ready; re-deploying"
    REGISTRY_NAMESPACE="$reg_ns" \
      "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "$reg_ns" stop || true
    REGISTRY_NAMESPACE="$reg_ns" \
      "$SCRIPT_DIR/init_registry.sh" -n "$ns" --registry-namespace "$reg_ns" update || true
  else
    log "ArgoCD: OK"
  fi
}

delete_if_exists() {
  local ns="$1" kind="$2" name="$3"
  kubectl -n "$ns" delete "$kind" "$name" --ignore-not-found --wait=false >/dev/null 2>&1 || true
}

delete_pods_by_label() {
  local ns="$1" selector="$2"
  kubectl -n "$ns" delete pod -l "$selector" --ignore-not-found --wait=false >/dev/null 2>&1 || true
}

has_openbao() {
  local ns="$1"
  resource_exists deploy openbao "$ns" || resource_exists statefulset openbao "$ns"
}

has_opentofu() {
  local ns="$1"
  resource_exists deploy opentofu "$ns"
}

has_garage() {
  local ns="$1"
  resource_exists statefulset garage "$ns"
}

has_registry() {
  local ns="$1"
  resource_exists deploy registry "$ns"
}

dedupe_common_core_default_namespace() {
  # The only known/expected accidental duplicate location is `default`.
  local mode
  mode="$(detect_mode)"
  if [[ "$mode" != "k3s" ]]; then
    return 0
  fi

  local canonical="${SERVICE_NAMESPACE:-knoe-system}"
  if [[ "$canonical" == "default" ]]; then
    canonical="knoe-system"
  fi
  local legacy="default"
  if [[ "$canonical" == "$legacy" ]]; then
    return 0
  fi

  if ! kubectl get ns "$legacy" >/dev/null 2>&1; then
    return 0
  fi

  # OpenBao
  if has_openbao "$legacy"; then
    warn "Detected OpenBao in legacy namespace '$legacy'"
    if ! has_openbao "$canonical"; then
      warn "Canonical OpenBao missing in '$canonical'; deploying before cleanup"
      "$SCRIPT_DIR/init_openbao.sh" -n "$canonical" update || true
    fi
    delete_pods_by_label "$legacy" "app=openbao"
    delete_if_exists "$legacy" deploy openbao
    delete_if_exists "$legacy" statefulset openbao
    delete_if_exists "$legacy" svc openbao
    delete_if_exists "$legacy" pvc data-openbao-0
  fi

  # OpenTofu
  if has_opentofu "$legacy"; then
    warn "Detected OpenTofu in legacy namespace '$legacy'"
    if ! has_opentofu "$canonical"; then
      warn "Canonical OpenTofu missing in '$canonical'; deploying before cleanup"
      "$SCRIPT_DIR/init_opentofu.sh" -n "$canonical" update || true
    fi
    delete_if_exists "$legacy" deploy opentofu
    delete_if_exists "$legacy" svc opentofu
  fi

  # Garage
  if has_garage "$legacy"; then
    warn "Detected Garage in legacy namespace '$legacy'"
    if ! has_garage "$canonical"; then
      warn "Canonical Garage missing in '$canonical'; deploying before cleanup"
      "$SCRIPT_DIR/init_garage_store.sh" -n "$canonical" start || true
    fi
    delete_pods_by_label "$legacy" "app=garage"
    delete_if_exists "$legacy" statefulset garage
    delete_if_exists "$legacy" svc garage
  fi

  # Registry
  if has_registry "$legacy"; then
    warn "Detected registry in legacy namespace '$legacy'"
    if ! has_registry "$canonical"; then
      warn "Canonical registry missing in '$canonical'; deploying before cleanup"
      REGISTRY_NAMESPACE="$canonical" "$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NAMESPACE" --registry-namespace "$canonical" update || true
    fi
    delete_if_exists "$legacy" deploy registry
    delete_if_exists "$legacy" svc registry
  fi
}

repair_openbao() {
  local ns="$1"
  local status
  resource_ready deploy openbao "$ns"; status=$?
  if [[ "$status" -eq 2 ]]; then
    resource_ready statefulset openbao "$ns"; status=$?
  fi
  if [[ "$status" -ne 0 ]]; then
    warn "OpenBao not ready; restarting pods and re-applying"
    kubectl -n "$ns" delete pod -l app=openbao --wait=false >/dev/null 2>&1 || true
    "$SCRIPT_DIR/init_openbao.sh" -n "$ns" update || true
  else
    log "OpenBao: OK"
  fi
}

repair_opentofu() {
  local ns="$1"
  local status
  resource_ready deploy opentofu "$ns"; status=$?
  if [[ "$status" -ne 0 ]]; then
    warn "OpenTofu not ready; re-deploying"
    kubectl -n "$ns" delete deploy opentofu --ignore-not-found >/dev/null 2>&1 || true
    "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" update || true
  else
    log "OpenTofu: OK"
  fi
}

repair_garage() {
  local ns="$1"
  local status
  resource_ready statefulset garage "$ns"; status=$?
  if [[ "$status" -ne 0 ]]; then
    warn "Garage not ready; restarting pods (PVCs preserved)"
    kubectl -n "$ns" delete pod -l app=garage --wait=false >/dev/null 2>&1 || true
    "$SCRIPT_DIR/init_garage_store.sh" start || true
  else
    log "Garage: OK"
  fi
}

repair_cnpg() {
  local ns="$1"
  local cluster="${CNPG_CLUSTER_NAME:-prole-db}"
  if ! kubectl -n "$ns" get cluster "$cluster" >/dev/null 2>&1; then
    warn "CNPG cluster '$cluster' not found in namespace '$ns'"
    return 0
  fi

  local total ready bad
  total=$(kubectl -n "$ns" get pods -l "cnpg.io/cluster=$cluster" --no-headers 2>/dev/null | wc -l | tr -d ' ')
  if [[ "$total" -eq 0 ]]; then
    warn "CNPG cluster '$cluster' has no pods; skipping rollout"
    return 0
  fi
  ready=$(kubectl -n "$ns" get pods -l "cnpg.io/cluster=$cluster" --no-headers 2>/dev/null | awk '$2 ~ /^([0-9]+)\/\\1$/ {c++} END{print c+0}')
  bad=$(kubectl -n "$ns" get pods -l "cnpg.io/cluster=$cluster" --no-headers 2>/dev/null | awk '$3 ~ /CrashLoopBackOff|Error|ImagePullBackOff|ErrImagePull/ {c++} END{print c+0}')

  if [[ "$ready" -lt "$total" || "$bad" -gt 0 ]]; then
    warn "CNPG pods unhealthy ($ready/$total ready, $bad bad); running rollout"
    run_with_timeout 360 env NAMESPACE="$ns" "$SCRIPT_DIR/init_cloudnative_pg.sh" rollout || true
  else
    log "CNPG: OK"
  fi
}

repair_barman_plugin() {
  local ns="cnpg-system"
  if ! kubectl -n "$ns" get deploy barman-cloud >/dev/null 2>&1; then
    warn "Barman Cloud plugin missing; reinstalling"
    run_with_timeout 300 env NAMESPACE="$DB_NAMESPACE" "$SCRIPT_DIR/init_cloudnative_pg.sh" install-barman-plugin || true
    return 0
  fi

  local ready
  ready=$(kubectl -n "$ns" get deploy barman-cloud -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
  if [[ -z "$ready" || "$ready" == "0" ]]; then
    warn "Barman Cloud plugin not ready; reinstalling and restarting"
    run_with_timeout 300 env NAMESPACE="$DB_NAMESPACE" "$SCRIPT_DIR/init_cloudnative_pg.sh" install-barman-plugin || true
    kubectl -n "$ns" rollout restart deploy/barman-cloud >/dev/null 2>&1 || true
  else
    log "Barman Cloud plugin: OK"
  fi
}

repair_kdc() {
  local ns="$1"
  local status
  if ! authority_context_exists; then
    warn "Authority Docker context missing; skipping KDC rebuild"
    return 0
  fi
  resource_ready deploy auth "$ns"; status=$?
  if [[ "$status" -ne 0 ]]; then
    warn "KDC (auth) not ready; re-deploying"
    kubectl -n "$ns" delete deploy auth --ignore-not-found >/dev/null 2>&1 || true
    kubectl -n "$ns" delete svc auth --ignore-not-found >/dev/null 2>&1 || true
    "$SCRIPT_DIR/init_kdc.sh" update || true
  else
    log "KDC (auth): OK"
  fi
}

case "${ACTION}" in
  repair)
    ensure_kubectl
    ensure_kubeconfig_if_needed

    log "== Repair Pipeline =="
    log "Namespace: $NAMESPACE"

    if ! cluster_ready; then
      err "Cluster not reachable; aborting repair"
      exit 2
    fi

    if [[ "$(detect_mode)" == "k3d" ]]; then
      if ! k3d_cluster_ready; then
        err "k3d cluster not running"
        exit 2
      fi
    fi

    log "-- Dashboard (Kong) --"
    repair_dashboard || true

    log "-- Dedupe common core (k3s) --"
    dedupe_common_core_default_namespace || true

    log "-- ArgoCD --"
    repair_argocd "$ARGOCD_NAMESPACE"

    log "-- OpenBao --"
    repair_openbao "$SERVICE_NAMESPACE"

    log "-- OpenTofu --"
    repair_opentofu "$SERVICE_NAMESPACE"

    log "-- Garage --"
    repair_garage "$SERVICE_NAMESPACE"

    # One more pass after repairs in case the canonical deployments were missing
    # and we needed to deploy them first.
    dedupe_common_core_default_namespace || true

    if [[ "${KERBEROS_ENABLED:-}" == "0" || "${KERBEROS_ENABLED:-}" == "false" || "${KERBEROS_ENABLED:-}" == "False" ]]; then
      log "KDC (auth): skipped (Kerberos disabled)"
    elif [[ "${PROLE_KDC_ENABLED:-1}" != "0" ]]; then
      log "-- KDC (auth) --"
      repair_kdc "$SERVICE_NAMESPACE"
    else
      log "KDC (auth): skipped (disabled)"
    fi

    log "-- Barman Cloud Plugin --"
    repair_barman_plugin

    log "-- CNPG --"
    repair_cnpg "$DB_NAMESPACE"

    log "-- Common Services Status --"
    if [[ "${KERBEROS_ENABLED:-}" == "1" || "${KERBEROS_ENABLED:-}" == "true" || "${KERBEROS_ENABLED:-}" == "True" ]]; then
      "$SCRIPT_DIR/status_common_services.sh" -n "$SERVICE_NAMESPACE" -k || true
    else
      "$SCRIPT_DIR/status_common_services.sh" -n "$SERVICE_NAMESPACE" || true
    fi

    log "Repair pipeline complete."
    ;;
  *)
    usage
    exit 2
    ;;
esac