#!/usr/bin/env bash
# patch_clusters.sh — Migrate knoe-cnpg-0 boot disks from pd-ssd to pd-standard.
#
# The GCloud Console creates node pools with pd-ssd boot disks by default, which
# consumes the entire 300 GB pd-ssd quota before any PGDATA/WAL PVCs can be
# provisioned.  This script replaces the existing default-pool with a new
# cnpg-db-pool that uses pd-standard boot disks, freeing the quota for CNPG PVCs.
#
# What this does:
#   1. Detects the current node pool disk configuration.
#   2. Creates cnpg-db-pool (pd-standard boot, e2-standard-2, 3 nodes).
#   3. Cordons and drains all nodes in the old pool.
#   4. Deletes the old pool.
#   5. Verifies 3 CNPG pods return to Running.
#
# Usage:
#   ./scripts/patch_clusters.sh
#
# Environment overrides:
#   GCP_PROJECT      — GCP project ID (default: plenary-truck-485623-p7)
#   GCP_REGION       — GCP region     (default: us-west3)
#   CLUSTER_NAME     — GKE cluster    (default: knoe-cnpg-0)
#   OLD_POOL         — Pool to remove (default: default-pool)
#   NEW_POOL         — Pool to create (default: cnpg-db-pool)
#   MACHINE_TYPE     — Node machine type (default: e2-standard-2)
#   DISK_TYPE        — Boot disk type    (default: pd-standard)
#   DISK_SIZE_GB     — Boot disk size GB (default: 50)
#   NODE_COUNT       — Nodes per zone    (default: 1, regional = 3 zones)
#   CNPG_NAMESPACE   — Namespace for CNPG pods (default: knoe-db-0)
#   DRAIN_TIMEOUT    — kubectl drain timeout    (default: 300s)
#   DRY_RUN          — Set to "true" to print commands without running (default: false)

set -euo pipefail

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
GCP_REGION="${GCP_REGION:-us-west3}"
CLUSTER_NAME="${CLUSTER_NAME:-knoe-cnpg-0}"
OLD_POOL="${OLD_POOL:-default-pool}"
NEW_POOL="${NEW_POOL:-cnpg-db-pool}"
MACHINE_TYPE="${MACHINE_TYPE:-e2-standard-2}"
DISK_TYPE="${DISK_TYPE:-pd-standard}"
DISK_SIZE_GB="${DISK_SIZE_GB:-50}"
NODE_COUNT="${NODE_COUNT:-1}"        # 1 per zone; regional cluster spans 3 zones = 3 nodes total
CNPG_NAMESPACE="${CNPG_NAMESPACE:-knoe-db-0}"
DRAIN_TIMEOUT="${DRAIN_TIMEOUT:-300s}"
DRY_RUN="${DRY_RUN:-false}"

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

log() {
  printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"
}

require_tool() {
  command -v "$1" >/dev/null 2>&1 || { log "Error: required tool not found: $1" >&2; exit 1; }
}

run_cmd() {
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "[DRY-RUN] $*"
  else
    "$@"
  fi
}

pool_exists() {
  local pool="$1"
  gcloud container node-pools describe "${pool}" \
    --cluster="${CLUSTER_NAME}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --format="value(name)" \
    --quiet 2>/dev/null | grep -q "${pool}"
}

# ---------------------------------------------------------------------------
# Pre-flight
# ---------------------------------------------------------------------------

require_tool gcloud
require_tool kubectl

log "==> Patch: migrate ${CLUSTER_NAME}/${OLD_POOL} → ${NEW_POOL} (${DISK_TYPE}, ${DISK_SIZE_GB}GB)"
log "    Project : ${GCP_PROJECT}"
log "    Region  : ${GCP_REGION}"
log "    DRY_RUN : ${DRY_RUN}"
echo ""

# ---------------------------------------------------------------------------
# Phase 1: Detect current configuration
# ---------------------------------------------------------------------------

log "[phase 1] Detecting current node pool configuration ..."

if ! gcloud container clusters describe "${CLUSTER_NAME}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --format="value(name)" \
    --quiet 2>/dev/null | grep -q "${CLUSTER_NAME}"; then
  log "Error: cluster '${CLUSTER_NAME}' not found in project '${GCP_PROJECT}' region '${GCP_REGION}'." >&2
  exit 1
fi

if pool_exists "${OLD_POOL}"; then
  current_disk=$(gcloud container node-pools describe "${OLD_POOL}" \
    --cluster="${CLUSTER_NAME}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --format="value(config.diskType)" \
    --quiet 2>/dev/null || echo "unknown")
  current_size=$(gcloud container node-pools describe "${OLD_POOL}" \
    --cluster="${CLUSTER_NAME}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --format="value(config.diskSizeGb)" \
    --quiet 2>/dev/null || echo "unknown")
  log "  ${OLD_POOL}: disk_type=${current_disk}, disk_size_gb=${current_size}"
  if [[ "${current_disk}" == "pd-standard" ]]; then
    log "  NOTE: ${OLD_POOL} already uses pd-standard. Continuing to ensure pool is renamed to '${NEW_POOL}'."
  fi
else
  log "  ${OLD_POOL} not found — may have been removed already."
fi

# ---------------------------------------------------------------------------
# Phase 2: Create new node pool
# ---------------------------------------------------------------------------

log "[phase 2] Ensuring node pool '${NEW_POOL}' exists ..."

if pool_exists "${NEW_POOL}"; then
  log "  ${NEW_POOL} already exists — skipping creation."
else
  log "  Creating ${NEW_POOL} (machine=${MACHINE_TYPE}, disk=${DISK_TYPE}, size=${DISK_SIZE_GB}GB, num-nodes=${NODE_COUNT}/zone) ..."
  run_cmd gcloud container node-pools create "${NEW_POOL}" \
    --cluster="${CLUSTER_NAME}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --machine-type="${MACHINE_TYPE}" \
    --disk-type="${DISK_TYPE}" \
    --disk-size="${DISK_SIZE_GB}" \
    --num-nodes="${NODE_COUNT}" \
    --quiet
  log "  ${NEW_POOL} created."
fi

# ---------------------------------------------------------------------------
# Phase 3: Cordon and drain old pool
# ---------------------------------------------------------------------------

log "[phase 3] Cordoning and draining '${OLD_POOL}' ..."

if ! pool_exists "${OLD_POOL}"; then
  log "  ${OLD_POOL} not found — nothing to drain."
else
  old_nodes=$(kubectl get nodes \
    -l "cloud.google.com/gke-nodepool=${OLD_POOL}" \
    --no-headers \
    -o custom-columns=":metadata.name" 2>/dev/null || true)

  if [[ -z "${old_nodes}" ]]; then
    log "  No nodes found labelled with nodepool=${OLD_POOL} — already drained or pool empty."
  else
    for node in ${old_nodes}; do
      log "  Cordoning ${node} ..."
      run_cmd kubectl cordon "${node}"
    done

    for node in ${old_nodes}; do
      log "  Draining ${node} (timeout ${DRAIN_TIMEOUT}) ..."
      run_cmd kubectl drain "${node}" \
        --ignore-daemonsets \
        --delete-emptydir-data \
        --timeout="${DRAIN_TIMEOUT}" \
        --force
    done
    log "  All nodes in ${OLD_POOL} drained."
  fi
fi

# ---------------------------------------------------------------------------
# Phase 4: Delete old pool
# ---------------------------------------------------------------------------

log "[phase 4] Deleting old pool '${OLD_POOL}' ..."

if ! pool_exists "${OLD_POOL}"; then
  log "  ${OLD_POOL} does not exist — already removed."
else
  run_cmd gcloud container node-pools delete "${OLD_POOL}" \
    --cluster="${CLUSTER_NAME}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --quiet
  log "  ${OLD_POOL} deleted."
fi

# ---------------------------------------------------------------------------
# Phase 5: Verify CNPG pods
# ---------------------------------------------------------------------------

log "[phase 5] Verifying CNPG pods in namespace '${CNPG_NAMESPACE}' ..."

if [[ "${DRY_RUN}" == "true" ]]; then
  log "  [DRY-RUN] Would check: kubectl -n ${CNPG_NAMESPACE} get pods -l cnpg.io/cluster=knoe-db"
else
  max_attempts=20
  attempt=0
  while [[ ${attempt} -lt ${max_attempts} ]]; do
    running=$(kubectl -n "${CNPG_NAMESPACE}" get pods \
      -l "cnpg.io/cluster=knoe-db" \
      --no-headers 2>/dev/null \
      | grep -c "Running" || true)
    log "  Running CNPG pods: ${running} / 3 expected (attempt $((attempt + 1))/${max_attempts})"
    if [[ "${running}" -ge 3 ]]; then
      log "  All CNPG pods Running on ${NEW_POOL}."
      kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db"
      break
    fi
    attempt=$(( attempt + 1 ))
    if [[ ${attempt} -lt ${max_attempts} ]]; then
      sleep 30
    fi
  done

  if [[ "${running:-0}" -lt 3 ]]; then
    log "WARNING: Not all CNPG pods reached Running state within timeout." >&2
    kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db" || true
    log "Check pod events: kubectl -n ${CNPG_NAMESPACE} describe pods -l cnpg.io/cluster=knoe-db"
    exit 1
  fi
fi

# ---------------------------------------------------------------------------
# Done
# ---------------------------------------------------------------------------

echo ""
log "==> Patch complete."
log ""
log "Next steps:"
log "  1. Verify pd-ssd quota freed:"
log "     gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} --format='value(quotas[disksSSD].usage,quotas[disksSSD].limit)'"
log "  2. Apply updated CNPG manifest (premium-rwo PVCs, 50Gi PGDATA + WAL):"
log "     kubectl --context=<cnpg-context> apply -f deploy/gcp/gke/knoe-db.yaml"
log "  3. Watch PVC expansion and pod restart:"
log "     kubectl -n ${CNPG_NAMESPACE} get pvc -w"