#!/usr/bin/env bash
# reset_clusters.sh — Delete and recreate both GKE clusters with correct configuration.
#
# Problem: Both clusters were created via GCloud Console as Autopilot, which:
#   - Prevents manual node pool management (patch_clusters.sh cannot work)
#   - Uses pd-balanced boot disks (counts against SSD_TOTAL_GB quota, 300 GB at limit)
#
# Solution: Delete both, recreate with pd-standard boot disks:
#   knoe-dev-0   → Standard   (app workloads: GitLab, platform), e2-small × 3, VPA, pd-standard boot
#   knoe-cnpg-0  → Standard   (CNPG only, 3× e2-standard-2),     pd-standard boot
#
# SSD quota budget after reset:
#   Boot disks: pd-standard — does NOT count against SSD_TOTAL_GB
#   PGDATA PVCs (premium-rwo): 3 × 50 Gi = 150 Gi  \  Apply knoe-db.yaml AFTER
#   WAL PVCs   (premium-rwo): 3 × 50 Gi = 150 Gi   /  quota increase to 2 TB
#
# Usage:
#   CONFIRM=true ./scripts/reset_clusters.sh
#   CONFIRM=true DRY_RUN=true ./scripts/reset_clusters.sh   # inspect only
#
# Environment overrides:
#   GCP_PROJECT        (default: plenary-truck-485623-p7)
#   GCP_REGION         (default: us-west3)
#   APP_CLUSTER        (default: knoe-dev-0)
#   DB_CLUSTER         (default: knoe-cnpg-0)
#   DB_MACHINE_TYPE    (default: e2-standard-2)
#   DB_DISK_TYPE       (default: pd-standard)
#   DB_DISK_SIZE_GB    (default: 50)
#   DB_NODES_PER_ZONE  (default: 1 → 3 nodes across 3 zones)
#   APP_MACHINE_TYPE   (default: e2-small)
#   APP_DISK_TYPE      (default: pd-standard)
#   APP_DISK_SIZE_GB   (default: 50)
#   APP_NODES_PER_ZONE (default: 1 → 3 nodes across 3 zones)
#   CONFIRM            REQUIRED: must be "true" to allow destructive operations
#   DRY_RUN            (default: false)

set -euo pipefail

# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------

GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
GCP_REGION="${GCP_REGION:-us-west3}"
APP_CLUSTER="${APP_CLUSTER:-knoe-dev-0}"
DB_CLUSTER="${DB_CLUSTER:-knoe-cnpg-0}"
DB_MACHINE_TYPE="${DB_MACHINE_TYPE:-e2-standard-2}"
DB_DISK_TYPE="${DB_DISK_TYPE:-pd-standard}"
DB_DISK_SIZE_GB="${DB_DISK_SIZE_GB:-50}"
DB_NODES_PER_ZONE="${DB_NODES_PER_ZONE:-1}"   # regional cluster = 3 zones = 3 nodes total
APP_MACHINE_TYPE="${APP_MACHINE_TYPE:-e2-small}"
APP_DISK_TYPE="${APP_DISK_TYPE:-pd-standard}"
APP_DISK_SIZE_GB="${APP_DISK_SIZE_GB:-50}"
APP_NODES_PER_ZONE="${APP_NODES_PER_ZONE:-1}"   # regional cluster = 3 zones = 3 nodes total
CONFIRM="${CONFIRM:-false}"
DRY_RUN="${DRY_RUN:-false}"

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }

die() { log "ERROR: $*" >&2; exit 1; }

require_tool() {
  command -v "$1" >/dev/null 2>&1 || die "required tool not found: $1"
}

run_cmd() {
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "[DRY-RUN] $*"
  else
    "$@"
  fi
}

cluster_exists() {
  local name="$1"
  gcloud container clusters describe "${name}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --format="value(name)" \
    --quiet 2>/dev/null | grep -q "${name}"
}

wait_for_cluster_absent() {
  local name="$1"
  local timeout_s="${2:-900}"
  local poll_s="${3:-15}"
  local elapsed=0

  while cluster_exists "${name}"; do
    if [[ ${elapsed} -ge ${timeout_s} ]]; then
      return 1
    fi
    log "  ${name} still exists; waiting ${poll_s}s (${elapsed}s/${timeout_s}s) ..."
    sleep "${poll_s}"
    elapsed=$(( elapsed + poll_s ))
  done

  return 0
}

delete_cluster_until_absent() {
  local name="$1"
  local max_attempts="${2:-6}"
  local retry_sleep_s="${3:-20}"
  local attempt=1
  local output=""
  local status=0

  if ! cluster_exists "${name}"; then
    log "  ${name} not found — skipping."
    return 0
  fi

  while [[ ${attempt} -le ${max_attempts} ]]; do
    log "  Deleting ${name} (attempt ${attempt}/${max_attempts}) ..."

    set +e
    output=$(gcloud container clusters delete "${name}" \
      --project="${GCP_PROJECT}" \
      --region="${GCP_REGION}" \
      --quiet 2>&1)
    status=$?
    set -e

    if [[ ${status} -ne 0 ]]; then
      if grep -qi "incompatible operation" <<<"${output}"; then
        log "  ${name} has an incompatible operation in progress; retrying after ${retry_sleep_s}s."
      elif grep -qi "not found" <<<"${output}"; then
        log "  ${name} already absent."
        return 0
      else
        printf "%s\n" "${output}" >&2
        die "cluster delete failed for ${name}"
      fi
    fi

    if wait_for_cluster_absent "${name}" 900 15; then
      log "  ${name} deletion confirmed."
      return 0
    fi

    log "  ${name} still present after delete attempt ${attempt}; retrying."
    sleep "${retry_sleep_s}"
    attempt=$(( attempt + 1 ))
  done

  die "timed out deleting ${name} after ${max_attempts} attempts"
}

_ssd_quota_yaml() {
  # Emit the 3-line YAML block for SSD_TOTAL_GB quota entry, e.g.:
  #   - limit: 300.0
  #     metric: SSD_TOTAL_GB
  #     usage: 300.0
  gcloud compute regions describe "${GCP_REGION}" \
    --project="${GCP_PROJECT}" \
    --format=yaml \
    --quiet 2>/dev/null | grep -B 1 -A 1 "metric: SSD_TOTAL_GB" || true
}

ssd_usage_gb() {
  local block
  block=$(_ssd_quota_yaml)
  echo "${block}" | awk '/usage:/{print $2}' | head -1 || echo "unknown"
}

ssd_limit_gb() {
  local block
  block=$(_ssd_quota_yaml)
  echo "${block}" | awk '/limit:/{print $2}' | head -1 || echo "unknown"
}

# ---------------------------------------------------------------------------
# Pre-flight
# ---------------------------------------------------------------------------

require_tool gcloud
require_tool kubectl

if [[ "${CONFIRM}" != "true" ]]; then
  echo ""
  echo "  This script will DELETE and RECREATE both GKE clusters:"
  echo "    ${APP_CLUSTER} (Autopilot)  and  ${DB_CLUSTER} (Standard)"
  echo ""
  echo "  Set CONFIRM=true to proceed:"
  echo "    CONFIRM=true ./scripts/reset_clusters.sh"
  echo ""
  exit 1
fi

log "==> Cluster reset: ${APP_CLUSTER} (Standard) + ${DB_CLUSTER} (Standard)"
log "    Project  : ${GCP_PROJECT}"
log "    Region   : ${GCP_REGION}"
log "    DRY_RUN  : ${DRY_RUN}"
echo ""

# ---------------------------------------------------------------------------
# Phase 1: Show current SSD quota
# ---------------------------------------------------------------------------

log "[phase 1] Current SSD quota (SSD_TOTAL_GB) in ${GCP_REGION} ..."
if [[ "${DRY_RUN}" != "true" ]]; then
  usage=$(ssd_usage_gb)
  limit=$(ssd_limit_gb)
  log "  SSD usage: ${usage} GB / ${limit} GB limit"
  if [[ "${usage}" == "unknown" || "${limit}" == "unknown" ]]; then
    log "  WARNING: Could not read SSD quota — proceeding anyway."
  fi
else
  log "  [DRY-RUN] Would read SSD quota from ${GCP_REGION}"
fi

# ---------------------------------------------------------------------------
# Phase 2: Delete existing clusters
# ---------------------------------------------------------------------------

log "[phase 2] Deleting existing clusters ..."

for cluster in "${DB_CLUSTER}" "${APP_CLUSTER}"; do
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "[DRY-RUN] gcloud container clusters delete ${cluster} --project=${GCP_PROJECT} --region=${GCP_REGION} --quiet"
  else
    delete_cluster_until_absent "${cluster}"
  fi
done

# Confirm all target clusters are absent before proceeding
if [[ "${DRY_RUN}" != "true" ]]; then
  log "  Waiting for cluster deletions to complete ..."
  for cluster in "${DB_CLUSTER}" "${APP_CLUSTER}"; do
    wait_for_cluster_absent "${cluster}" 900 15 \
      || die "cluster ${cluster} still exists after deletion phase"
  done
  log "  All deletions complete."
fi

# ---------------------------------------------------------------------------
# Phase 2b: Delete orphaned pd-ssd disks left behind by cluster deletion
# ---------------------------------------------------------------------------
# GKE cluster deletion does not remove PersistentVolume-backed GCP disks.
# Orphaned pd-ssd disks hold SSD quota and prevent reset_clusters.sh from
# proceeding (Phase 3 waits for SSD usage < 50 GB).

log "[phase 2b] Deleting orphaned pd-ssd disks in ${GCP_REGION} ..."

if [[ "${DRY_RUN}" == "true" ]]; then
  log "  [DRY-RUN] Would list and delete orphaned pd-ssd disks in ${GCP_REGION}"
else
  # List all disks in the project, filter to pd-ssd in this region in bash
  mapfile -t ssd_disks < <(
    gcloud compute disks list \
      --project="${GCP_PROJECT}" \
      --format="csv[no-heading](name,zone,type)" \
      --quiet 2>/dev/null \
      | awk -F',' "/${GCP_REGION}/ && /pd-ssd/ {print \$1 \",\" \$2}" \
      || true
  )

  if [[ ${#ssd_disks[@]} -eq 0 ]]; then
    log "  No orphaned pd-ssd disks found."
  else
    for entry in "${ssd_disks[@]}"; do
      disk_name="${entry%%,*}"
      disk_zone="${entry##*,}"
      # Extract just the zone name (may be a full URL)
      disk_zone="${disk_zone##*/}"
      log "  Deleting pd-ssd disk: ${disk_name} (zone: ${disk_zone})"
      gcloud compute disks delete "${disk_name}" \
        --zone="${disk_zone}" \
        --project="${GCP_PROJECT}" \
        --quiet 2>/dev/null \
        && log "    Deleted ${disk_name}." \
        || log "    WARNING: Could not delete ${disk_name} — may already be gone."
    done
    log "  Orphaned pd-ssd disk cleanup complete."
  fi
fi

# ---------------------------------------------------------------------------
# Phase 3: Wait for SSD quota to be released
# ---------------------------------------------------------------------------

log "[phase 3] Waiting for SSD quota to be released ..."

if [[ "${DRY_RUN}" == "true" ]]; then
  log "  [DRY-RUN] Would poll SSD_TOTAL_GB until usage < 50 GB"
else
  max_wait_s=1200   # 20 min max
  poll_s=30
  elapsed=0
  while true; do
    usage=$(ssd_usage_gb)
    log "  SSD usage: ${usage} GB (${elapsed}s elapsed)"
    if [[ "${usage}" == "unknown" ]]; then
      log "  WARNING: Could not read quota — treating as released."
      break
    fi
    # Cast to int for comparison
    usage_int=${usage%.*}
    if [[ "${usage_int}" -lt 50 ]]; then
      log "  SSD quota released (${usage} GB remaining usage)."
      break
    fi
    if [[ ${elapsed} -ge ${max_wait_s} ]]; then
      log "  WARNING: SSD quota did not fully release within ${max_wait_s}s."
      log "  Current usage: ${usage} GB — proceeding with pd-standard (no SSD impact)."
      break
    fi
    sleep "${poll_s}"
    elapsed=$(( elapsed + poll_s ))
  done
fi

# ---------------------------------------------------------------------------
# Phase 4: Create knoe-cnpg-0 as Standard GKE cluster
# ---------------------------------------------------------------------------

log "[phase 4] Creating ${DB_CLUSTER} (Standard, ${DB_MACHINE_TYPE}, ${DB_DISK_TYPE}, ${DB_DISK_SIZE_GB}GB) ..."

run_cmd gcloud container clusters create "${DB_CLUSTER}" \
  --project="${GCP_PROJECT}" \
  --region="${GCP_REGION}" \
  --cluster-version=latest \
  --machine-type="${DB_MACHINE_TYPE}" \
  --node-labels="workload=db" \
  --disk-type="${DB_DISK_TYPE}" \
  --disk-size="${DB_DISK_SIZE_GB}" \
  --num-nodes="${DB_NODES_PER_ZONE}" \
  --enable-ip-alias \
  --workload-pool="${GCP_PROJECT}.svc.id.goog" \
  --quiet

log "  ${DB_CLUSTER} created."

# ---------------------------------------------------------------------------
# Phase 5: Create knoe-dev-0 as Standard cluster
# ---------------------------------------------------------------------------

log "[phase 5] Creating ${APP_CLUSTER} (Standard, ${APP_MACHINE_TYPE}, ${APP_DISK_TYPE}, ${APP_DISK_SIZE_GB}GB, VPA) ..."

run_cmd gcloud container clusters create "${APP_CLUSTER}" \
  --project="${GCP_PROJECT}" \
  --region="${GCP_REGION}" \
  --cluster-version=latest \
  --machine-type="${APP_MACHINE_TYPE}" \
  --disk-type="${APP_DISK_TYPE}" \
  --disk-size="${APP_DISK_SIZE_GB}" \
  --num-nodes="${APP_NODES_PER_ZONE}" \
  --enable-vertical-pod-autoscaling \
  --enable-ip-alias \
  --workload-pool="${GCP_PROJECT}.svc.id.goog" \
  --quiet
# Standard mode: pd-standard boot disks consume ZERO SSD quota (vs Autopilot's
# pd-balanced 100GB/node which exhausted the full 300GB SSD_TOTAL_GB quota).
# VPA handles dynamic resource adjustment for burstable e2-medium nodes.

log "  ${APP_CLUSTER} created."

# ---------------------------------------------------------------------------
# Phase 6: Fetch credentials and verify
# ---------------------------------------------------------------------------

log "[phase 6] Fetching kubeconfig credentials ..."

if [[ "${DRY_RUN}" != "true" ]]; then
  run_cmd gcloud container clusters get-credentials "${DB_CLUSTER}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --quiet
  db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
  log "  DB cluster context: ${db_ctx}"

  run_cmd gcloud container clusters get-credentials "${APP_CLUSTER}" \
    --project="${GCP_PROJECT}" \
    --region="${GCP_REGION}" \
    --quiet
  app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
  log "  App cluster context: ${app_ctx}"

  log "  Verifying cluster connectivity ..."
  kubectl --context="${db_ctx}" cluster-info --request-timeout=15s \
    && log "  ${DB_CLUSTER}: OK" \
    || log "  WARNING: ${DB_CLUSTER} not yet reachable — may need a moment."
  kubectl --context="${app_ctx}" cluster-info --request-timeout=15s \
    && log "  ${APP_CLUSTER}: OK" \
    || log "  WARNING: ${APP_CLUSTER} not yet reachable — may need a moment."
else
  db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
  app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
  log "  [DRY-RUN] Would fetch credentials for both clusters."
fi

# ---------------------------------------------------------------------------
# Phase 7: Register clusters in knoe-0 fleet + enable service mesh
# ---------------------------------------------------------------------------
# Cloud Service Mesh (CSM) is enabled per-cluster via fleet membership.
# This automates what was previously done via the GCP web console.

log "[phase 7] Registering clusters in fleet and enabling service mesh ..."

_register_and_mesh() {
  local cluster_name="$1"
  log "  Registering ${cluster_name} in fleet ..."
  if ! run_cmd gcloud container fleet memberships register "${cluster_name}" \
    --gke-cluster="${GCP_REGION}/${cluster_name}" \
    --enable-workload-identity \
    --project="${GCP_PROJECT}" \
    --quiet 2>&1; then
    log "  WARNING: Fleet registration for ${cluster_name} failed (may already be registered — continuing)."
  fi

  log "  Enabling automatic service mesh management for ${cluster_name} ..."
  if ! run_cmd gcloud container fleet mesh update \
    --management=automatic \
    --memberships="${cluster_name}" \
    --project="${GCP_PROJECT}" \
    --quiet 2>&1; then
    log "  WARNING: Service mesh update for ${cluster_name} failed — enable manually via GCP console."
  fi
}

if [[ "${DRY_RUN}" != "true" ]]; then
  _register_and_mesh "${DB_CLUSTER}"
  _register_and_mesh "${APP_CLUSTER}"
  log "  Fleet + mesh registration submitted. Mesh provisioning is async (~10 min)."
  log "  Check status: gcloud container fleet mesh describe --project=${GCP_PROJECT}"
else
  log "  [DRY-RUN] Would register ${DB_CLUSTER} and ${APP_CLUSTER} in fleet + enable service mesh."
fi

# ---------------------------------------------------------------------------
# Phase 8: Show final SSD quota
# ---------------------------------------------------------------------------

log "[phase 8] Final SSD quota ..."
if [[ "${DRY_RUN}" != "true" ]]; then
  usage=$(ssd_usage_gb)
  limit=$(ssd_limit_gb)
  log "  SSD usage: ${usage} GB / ${limit} GB (pd-standard boot disks use 0 SSD quota)"
fi

# ---------------------------------------------------------------------------
# Done
# ---------------------------------------------------------------------------

echo ""
log "==> Reset complete."
log ""
log "Contexts:"
log "  App  (Standard) : ${app_ctx}"
log "  DB   (Standard) : ${db_ctx}"
log ""
log "Next steps:"
log "  1. Run the installer to configure workloads:"
log "     ./install.sh"
log ""
log "  2. Once the SSD quota increase (300 GB → 2 TB) is approved, apply CNPG storage:"
log "     kubectl --context=${db_ctx} apply -f deploy/gcp/gke/knoe-db.yaml"
log "     # This provisions 3×50Gi PGDATA + 3×50Gi WAL = 300 Gi pd-ssd"
log ""
log "  3. Check your quota increase request:"
log "     gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} \\"
log "       --format='table(quotas.metric,quotas.usage,quotas.limit)' | grep SSD"
log ""
log "  4. Check service mesh provisioning status (~10 min after cluster creation):"
log "     gcloud container fleet mesh describe --project=${GCP_PROJECT}"