#!/usr/bin/env bash # run_demo.sh — CI migration danger demo orchestration script. # # Prerequisites: # - kubectl configured with access to the knoe-db namespace # - psql and pgbench in PATH # - Port-forward to knoe-db-rw running on localhost:15432 # # Usage: # ./demo/ci-migration/run_demo.sh [setup|broken|fixed|monitor|reset] # # Phases: # setup — create schema, generate 5M rows, seed workload table # broken — run the broken single-transaction migration (will fail on partitions) # bloat — run the long-tx variant that actually inserts (shows MVCC bloat) # fixed — run the chunked safe migration # monitor — start watch loop showing MVCC metrics # reset — drop and recreate the target table, reset workload table set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # ── Connection settings ─────────────────────────────────────────────────────── DB_HOST="${DB_HOST:-127.0.0.1}" DB_PORT="${DB_PORT:-15432}" DB_USER="${DB_USER:-postgres}" DB_NAME="${DB_NAME:-knoe-db}" # Retrieve password from Kubernetes secret if not set if [[ -z "${PGPASSWORD:-}" ]]; then PGPASSWORD="$(kubectl -n knoe-db get secret knoe-db-superuser \ -o jsonpath='{.data.password}' | base64 -d)" export PGPASSWORD fi PSQL="psql -h $DB_HOST -p $DB_PORT -U $DB_USER $DB_NAME" PGBENCH="pgbench -h $DB_HOST -p $DB_PORT -U $DB_USER $DB_NAME" # ── Helpers ─────────────────────────────────────────────────────────────────── start_port_forward() { if ! lsof -ti tcp:15432 &>/dev/null; then echo "[+] Starting port-forward to knoe-db-rw on :15432..." # Find the current primary pod PRIMARY=$(kubectl -n knoe-db get pods \ --field-selector=status.phase=Running \ -o jsonpath='{range .items[?(@.status.containerStatuses[0].ready==true)]}{.metadata.name}{"\n"}{end}' \ | grep '^knoe-db-' | head -1) echo " Primary pod: $PRIMARY" kubectl -n knoe-db port-forward "pod/$PRIMARY" 15432:5432 &>/tmp/pf_knoe.log & sleep 3 else echo "[✓] Port-forward already running on :15432" fi } run_sql() { local file="$1" echo "[+] Running $file..." $PSQL -f "$file" } # ── Commands ────────────────────────────────────────────────────────────────── case "${1:-help}" in setup) start_port_forward echo "" echo "=== PHASE 1: Schema setup ===" run_sql "$SCRIPT_DIR/01_schema_setup.sql" echo "" echo "=== PHASE 2: Data generation (5M rows — takes 3-6 min) ===" run_sql "$SCRIPT_DIR/02_generate_data.sql" echo "" echo "Setup complete. Run './run_demo.sh monitor' in another terminal," echo "then './run_demo.sh broken' to start the demo." ;; broken) start_port_forward echo "" echo "=== BROKEN MIGRATION (Issue 2 exposed — partition mismatch) ===" echo "Watch for: ERROR: no partition of relation found for row" echo "" run_sql "$SCRIPT_DIR/03_broken_migration.sql" || true echo "" echo "Expected failure demonstrated. Now run './run_demo.sh bloat'" echo "to see the long-transaction MVCC pileup." ;; bloat) start_port_forward echo "" echo "=== MVCC BLOAT DEMO (partition bug fixed, long tx left in place) ===" echo "Start traffic in another terminal first:" echo " ./run_demo.sh traffic" echo "" read -p "Press Enter when traffic is running to start the long migration..." echo "" echo "Recording WAL start position..." WAL_START=$($PSQL -tAc "SELECT pg_current_wal_lsn();") echo "WAL start: $WAL_START" echo "" run_sql "$SCRIPT_DIR/03b_broken_migration_long_tx.sql" echo "" WAL_END=$($PSQL -tAc "SELECT pg_current_wal_lsn();") echo "WAL end: $WAL_END" WAL_DIFF=$($PSQL -tAc "SELECT pg_size_pretty(pg_wal_lsn_diff('$WAL_END'::pg_lsn, '$WAL_START'::pg_lsn));") echo "WAL generated: $WAL_DIFF" ;; fixed) start_port_forward echo "" echo "=== FIXED MIGRATION (chunked, no long transaction) ===" echo "Start traffic in another terminal first:" echo " ./run_demo.sh traffic" echo "" read -p "Press Enter when traffic is running to start the chunked migration..." echo "" echo "Recording WAL start position..." WAL_START=$($PSQL -tAc "SELECT pg_current_wal_lsn();") echo "WAL start: $WAL_START" echo "" run_sql "$SCRIPT_DIR/06_fixed_migration.sql" echo "" WAL_END=$($PSQL -tAc "SELECT pg_current_wal_lsn();") WAL_DIFF=$($PSQL -tAc "SELECT pg_size_pretty(pg_wal_lsn_diff('$WAL_END'::pg_lsn, '$WAL_START'::pg_lsn));") echo "WAL generated: $WAL_DIFF" echo "(Compare to the bloat run — same data, far less WAL spike)" ;; traffic) start_port_forward echo "" echo "=== OLTP TRAFFIC SIMULATOR ===" echo "Running 10 concurrent workers for 600 seconds (10 min)." echo "Ctrl+C to stop early." echo "" $PGBENCH -c 10 -j 2 -T 600 \ -f "$SCRIPT_DIR/04_pgbench_workload.sql" \ --no-vacuum \ -P 10 ;; monitor) start_port_forward echo "" echo "=== MONITORING (updates every 3 seconds) ===" echo "Ctrl+C to stop." echo "" watch -n 3 "$PSQL -f $SCRIPT_DIR/05_monitor.sql 2>&1" ;; reset) start_port_forward echo "" echo "=== RESET: dropping ci_job_artifacts, truncating ci_build_status ===" $PSQL -c "DROP TABLE IF EXISTS demo.ci_job_artifacts CASCADE;" $PSQL -c "TRUNCATE demo.ci_build_status;" $PSQL -c "INSERT INTO demo.ci_build_status (job_id, status) SELECT (random()*999999+1)::BIGINT, 'running' FROM generate_series(1,100000);" echo "Reset complete. Run './run_demo.sh bloat' or './run_demo.sh fixed'." ;; help|*) echo "Usage: $0 [setup|broken|bloat|fixed|traffic|monitor|reset]" echo "" echo " setup — create schema, generate 5M rows of test data" echo " broken — run the broken migration (partition error demo)" echo " bloat — run the long-tx backfill to demonstrate MVCC bloat" echo " fixed — run the chunked safe migration" echo " traffic — start pgbench OLTP workload (run in separate terminal)" echo " monitor — watch MVCC metrics in real time (run in separate terminal)" echo " reset — drop target table and reset workload for a clean re-run" echo "" echo "Typical demo flow:" echo " Terminal 1: ./run_demo.sh setup" echo " Terminal 2: ./run_demo.sh traffic" echo " Terminal 3: ./run_demo.sh monitor" echo " Terminal 1: ./run_demo.sh broken (show Issue 2)" echo " Terminal 1: ./run_demo.sh reset" echo " Terminal 1: ./run_demo.sh bloat (show MVCC pileup)" echo " Terminal 1: ./run_demo.sh reset" echo " Terminal 1: ./run_demo.sh fixed (show safe approach)" ;; esac