prole/demo/ci-migration/run_demo.sh

#!/usr/bin/env bash
# run_demo.sh — CI migration danger demo orchestration script.
#
# Prerequisites:
#   - kubectl configured with access to the knoe-db namespace
#   - psql and pgbench in PATH
#   - Port-forward to knoe-db-rw running on localhost:15432
#
# Usage:
#   ./demo/ci-migration/run_demo.sh [setup|broken|fixed|monitor|reset]
#
# Phases:
#   setup   — create schema, generate 5M rows, seed workload table
#   broken  — run the broken single-transaction migration (will fail on partitions)
#   bloat   — run the long-tx variant that actually inserts (shows MVCC bloat)
#   fixed   — run the chunked safe migration
#   monitor — start watch loop showing MVCC metrics
#   reset   — drop and recreate the target table, reset workload table

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# ── Connection settings ───────────────────────────────────────────────────────
DB_HOST="${DB_HOST:-127.0.0.1}"
DB_PORT="${DB_PORT:-15432}"
DB_USER="${DB_USER:-postgres}"
DB_NAME="${DB_NAME:-knoe-db}"

# Retrieve password from Kubernetes secret if not set
if [[ -z "${PGPASSWORD:-}" ]]; then
    PGPASSWORD="$(kubectl -n knoe-db get secret knoe-db-superuser \
        -o jsonpath='{.data.password}' | base64 -d)"
    export PGPASSWORD
fi

PSQL="psql -h $DB_HOST -p $DB_PORT -U $DB_USER $DB_NAME"
PGBENCH="pgbench -h $DB_HOST -p $DB_PORT -U $DB_USER $DB_NAME"

# ── Helpers ───────────────────────────────────────────────────────────────────
start_port_forward() {
    if ! lsof -ti tcp:15432 &>/dev/null; then
        echo "[+] Starting port-forward to knoe-db-rw on :15432..."
        # Find the current primary pod
        PRIMARY=$(kubectl -n knoe-db get pods \
            --field-selector=status.phase=Running \
            -o jsonpath='{range .items[?(@.status.containerStatuses[0].ready==true)]}{.metadata.name}{"\n"}{end}' \
            | grep '^knoe-db-' | head -1)
        echo "    Primary pod: $PRIMARY"
        kubectl -n knoe-db port-forward "pod/$PRIMARY" 15432:5432 &>/tmp/pf_knoe.log &
        sleep 3
    else
        echo "[✓] Port-forward already running on :15432"
    fi
}

run_sql() {
    local file="$1"
    echo "[+] Running $file..."
    $PSQL -f "$file"
}

# ── Commands ──────────────────────────────────────────────────────────────────
case "${1:-help}" in

setup)
    start_port_forward
    echo ""
    echo "=== PHASE 1: Schema setup ==="
    run_sql "$SCRIPT_DIR/01_schema_setup.sql"
    echo ""
    echo "=== PHASE 2: Data generation (5M rows — takes 3-6 min) ==="
    run_sql "$SCRIPT_DIR/02_generate_data.sql"
    echo ""
    echo "Setup complete. Run './run_demo.sh monitor' in another terminal,"
    echo "then './run_demo.sh broken' to start the demo."
    ;;

broken)
    start_port_forward
    echo ""
    echo "=== BROKEN MIGRATION (Issue 2 exposed — partition mismatch) ==="
    echo "Watch for: ERROR: no partition of relation found for row"
    echo ""
    run_sql "$SCRIPT_DIR/03_broken_migration.sql" || true
    echo ""
    echo "Expected failure demonstrated. Now run './run_demo.sh bloat'"
    echo "to see the long-transaction MVCC pileup."
    ;;

bloat)
    start_port_forward
    echo ""
    echo "=== MVCC BLOAT DEMO (partition bug fixed, long tx left in place) ==="
    echo "Start traffic in another terminal first:"
    echo "  ./run_demo.sh traffic"
    echo ""
    read -p "Press Enter when traffic is running to start the long migration..."
    echo ""
    echo "Recording WAL start position..."
    WAL_START=$($PSQL -tAc "SELECT pg_current_wal_lsn();")
    echo "WAL start: $WAL_START"
    echo ""
    run_sql "$SCRIPT_DIR/03b_broken_migration_long_tx.sql"
    echo ""
    WAL_END=$($PSQL -tAc "SELECT pg_current_wal_lsn();")
    echo "WAL end:   $WAL_END"
    WAL_DIFF=$($PSQL -tAc "SELECT pg_size_pretty(pg_wal_lsn_diff('$WAL_END'::pg_lsn, '$WAL_START'::pg_lsn));")
    echo "WAL generated: $WAL_DIFF"
    ;;

fixed)
    start_port_forward
    echo ""
    echo "=== FIXED MIGRATION (chunked, no long transaction) ==="
    echo "Start traffic in another terminal first:"
    echo "  ./run_demo.sh traffic"
    echo ""
    read -p "Press Enter when traffic is running to start the chunked migration..."
    echo ""
    echo "Recording WAL start position..."
    WAL_START=$($PSQL -tAc "SELECT pg_current_wal_lsn();")
    echo "WAL start: $WAL_START"
    echo ""
    run_sql "$SCRIPT_DIR/06_fixed_migration.sql"
    echo ""
    WAL_END=$($PSQL -tAc "SELECT pg_current_wal_lsn();")
    WAL_DIFF=$($PSQL -tAc "SELECT pg_size_pretty(pg_wal_lsn_diff('$WAL_END'::pg_lsn, '$WAL_START'::pg_lsn));")
    echo "WAL generated: $WAL_DIFF"
    echo "(Compare to the bloat run — same data, far less WAL spike)"
    ;;

traffic)
    start_port_forward
    echo ""
    echo "=== OLTP TRAFFIC SIMULATOR ==="
    echo "Running 10 concurrent workers for 600 seconds (10 min)."
    echo "Ctrl+C to stop early."
    echo ""
    $PGBENCH -c 10 -j 2 -T 600 \
        -f "$SCRIPT_DIR/04_pgbench_workload.sql" \
        --no-vacuum \
        -P 10
    ;;

monitor)
    start_port_forward
    echo ""
    echo "=== MONITORING (updates every 3 seconds) ==="
    echo "Ctrl+C to stop."
    echo ""
    watch -n 3 "$PSQL -f $SCRIPT_DIR/05_monitor.sql 2>&1"
    ;;

reset)
    start_port_forward
    echo ""
    echo "=== RESET: dropping ci_job_artifacts, truncating ci_build_status ==="
    $PSQL -c "DROP TABLE IF EXISTS demo.ci_job_artifacts CASCADE;"
    $PSQL -c "TRUNCATE demo.ci_build_status;"
    $PSQL -c "INSERT INTO demo.ci_build_status (job_id, status)
              SELECT (random()*999999+1)::BIGINT, 'running'
              FROM generate_series(1,100000);"
    echo "Reset complete. Run './run_demo.sh bloat' or './run_demo.sh fixed'."
    ;;

help|*)
    echo "Usage: $0 [setup|broken|bloat|fixed|traffic|monitor|reset]"
    echo ""
    echo "  setup    — create schema, generate 5M rows of test data"
    echo "  broken   — run the broken migration (partition error demo)"
    echo "  bloat    — run the long-tx backfill to demonstrate MVCC bloat"
    echo "  fixed    — run the chunked safe migration"
    echo "  traffic  — start pgbench OLTP workload (run in separate terminal)"
    echo "  monitor  — watch MVCC metrics in real time (run in separate terminal)"
    echo "  reset    — drop target table and reset workload for a clean re-run"
    echo ""
    echo "Typical demo flow:"
    echo "  Terminal 1: ./run_demo.sh setup"
    echo "  Terminal 2: ./run_demo.sh traffic"
    echo "  Terminal 3: ./run_demo.sh monitor"
    echo "  Terminal 1: ./run_demo.sh broken   (show Issue 2)"
    echo "  Terminal 1: ./run_demo.sh reset"
    echo "  Terminal 1: ./run_demo.sh bloat    (show MVCC pileup)"
    echo "  Terminal 1: ./run_demo.sh reset"
    echo "  Terminal 1: ./run_demo.sh fixed    (show safe approach)"
    ;;
esac