diff --git a/conf/gke.cfg b/conf/gke.cfg index ddf933a..5999db0 100644 --- a/conf/gke.cfg +++ b/conf/gke.cfg @@ -137,7 +137,10 @@ GITLAB_WEBSERVICE_PUMA_THREADS_MIN = 2 GITLAB_WEBSERVICE_REQUESTS_CPU = 500m GITLAB_WEBSERVICE_REQUESTS_MEMORY = 2Gi GITLAB_WEBSERVICE_WORKER_PROCESSES = 1 +GRAFANA_GOOGLE_CLIENT_ID = secretref://google-oidc-client-id +GRAFANA_GOOGLE_CLIENT_SECRET = secretref://google-oidc-client-secret GRAFANA_HOSTNAME = svc.knoe.dev +GRAFANA_UPSTREAM_URL = http://prometheus-grafana.monitoring.svc.cluster.local:80 K3S_SERVER = K3S_TOKEN = KNOE_DB_USER = chrisfu @@ -164,6 +167,9 @@ SUPABASE_INGRESS_CLASS = gce SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME = supabase-studio SUPABASE_STUDIO_HOSTNAME = db.0.knoe.dev SVC_KNOE_GLOBAL_STATIC_IP_NAME = svc-knoe +DB_OIDC_CLIENT_ID = secretref://db-oidc-client-id +DB_OIDC_CLIENT_SECRET = secretref://db-oidc-client-secret +DB_OIDC_COOKIE_SECRET = secretref://db-oidc-cookie-secret [Welcome] ; No configuration values captured yet for this section. @@ -188,8 +194,8 @@ PORT_FORWARD_K3D_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao PORT_FORWARD_K3D_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu PORT_FORWARD_K3D_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard PORT_FORWARD_K3D_MAPPING_6 = id=postgres;namespace=${DATABASE_NAMESPACE};target=svc/knoe-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary) -PORT_FORWARD_K3D_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI -PORT_FORWARD_K3D_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI +PORT_FORWARD_K3D_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/prometheus-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI +PORT_FORWARD_K3D_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/prometheus-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI PORT_FORWARD_K3D_MAPPING_9 = id=supabase-kong;namespace=supabase;target=svc/kong;address=0.0.0.0;hostPort=8000;servicePort=8000;protocol=TCP;description=Supabase API (Kong) [System Environment] diff --git a/conf/k3s.cfg b/conf/k3s.cfg index eb7df8e..889afdc 100644 --- a/conf/k3s.cfg +++ b/conf/k3s.cfg @@ -139,8 +139,10 @@ DEPLOYMENT_TARGET = knoe-service-cluster DOCKER_IMPORT_DIR = DOCKER_PRELOAD = false GITEA_HOSTNAME = git-internal.prole.org +GITLAB_GITALY_STORAGE_CLASS = synology-iscsi GITLAB_PUBLIC_HOSTS = git.prole.org GITLAB_REPAIR_BLOCKED_AUTOCLEAN = 1 +GITLAB_STORAGE_NODE = merlin.prole.org K3S_SERVER = https://myrddin.prole.org:6443 K3S_TOKEN = ${KNOE_SECRET:v1:CWWf3RHFdbUrmfrY:It8a2G8QUUIsqVwMsm3LsI4UvSSChEc_uAdESwzYplZLOCiSsCbOKuT9FbPpIwQvEaG_gLz9ZAfkD0EQxJp81KAtpk_X3K_nxVUa0RPRlbt_wdeXXoMoFFpN5BqXXz2HZwKgh_gpK1hjVbsJQHKAbTqWfu8u_LTmYYg4ag==} KNOE_DB_USER = root diff --git a/demo/ci-migration/01_schema_setup.sql b/demo/ci-migration/01_schema_setup.sql new file mode 100644 index 0000000..884274a --- /dev/null +++ b/demo/ci-migration/01_schema_setup.sql @@ -0,0 +1,130 @@ +-- 01_schema_setup.sql +-- Creates the demo schema, legacy table, OLTP workload table, +-- and all monitoring views for the CI migration danger demo. +-- +-- Run as superuser (postgres) against knoe-db. +-- Safe to re-run: all objects use IF NOT EXISTS / CREATE OR REPLACE. + +CREATE SCHEMA IF NOT EXISTS demo; + +-- ── Legacy table (source of the backfill) ──────────────────────────────────── +-- Represents the old non-partitioned ci_job_artifacts table on a 10 TB GitLab +-- DB. In our scaled demo we'll fill this with ~5 million rows (~1 GB). +CREATE TABLE IF NOT EXISTS demo.ci_job_artifacts_legacy ( + id BIGSERIAL PRIMARY KEY, + job_id BIGINT NOT NULL, + project_id BIGINT NOT NULL, + file_type SMALLINT NOT NULL DEFAULT 0, + size_bytes BIGINT, + file_store SMALLINT NOT NULL DEFAULT 1, + checksum TEXT, + created_at TIMESTAMP NOT NULL, + expire_at TIMESTAMP, + locked BOOLEAN DEFAULT FALSE +); + +CREATE INDEX IF NOT EXISTS idx_legacy_created_at + ON demo.ci_job_artifacts_legacy (created_at); + +-- ── OLTP workload table (concurrent traffic during migration) ───────────────── +-- Simulates background CI pipeline activity during the migration window. +-- UPDATEs to this table generate dead tuples that autovacuum must clean. +-- When a long-running migration transaction holds an old snapshot, autovacuum +-- is blocked and dead tuples pile up — this is the MVCC bloat we'll observe. +CREATE TABLE IF NOT EXISTS demo.ci_build_status ( + id BIGSERIAL PRIMARY KEY, + job_id BIGINT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + updated_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +-- ── Monitoring views ────────────────────────────────────────────────────────── + +CREATE OR REPLACE VIEW demo.mvcc_bloat_monitor AS +SELECT + relname AS table_name, + n_live_tup AS live_rows, + n_dead_tup AS dead_rows, + ROUND(n_dead_tup::NUMERIC / NULLIF(n_live_tup, 0) * 100, 2) AS dead_pct, + last_autovacuum, + last_autoanalyze, + pg_size_pretty(pg_total_relation_size('demo.' || relname)) AS total_size, + pg_size_pretty(pg_relation_size('demo.' || relname)) AS table_size +FROM pg_stat_user_tables +WHERE schemaname = 'demo' +ORDER BY n_dead_tup DESC; + +-- Shows any transaction open longer than 5 seconds — the migration blocker. +CREATE OR REPLACE VIEW demo.long_running_tx AS +SELECT + pid, + now() - xact_start AS tx_duration, + now() - query_start AS query_duration, + state, + wait_event_type, + wait_event, + left(query, 200) AS query_snippet, + backend_type, + application_name +FROM pg_stat_activity +WHERE xact_start IS NOT NULL + AND now() - xact_start > INTERVAL '5 seconds' + AND pid <> pg_backend_pid() +ORDER BY tx_duration DESC; + +-- WAL generation tracker — shows how fast WAL is growing. +-- Run before and after to calculate the delta. +CREATE OR REPLACE VIEW demo.wal_progress AS +SELECT + pg_current_wal_lsn() AS current_lsn, + pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0'::pg_lsn)) AS total_wal, + (SELECT count(*) FROM pg_ls_waldir()) AS wal_segment_count, + pg_size_pretty((SELECT sum(size) FROM pg_ls_waldir())) AS wal_dir_size; + +-- Lock wait graph — shows who is blocking whom. +CREATE OR REPLACE VIEW demo.lock_waits AS +SELECT + blocked_a.pid AS blocked_pid, + blocking_a.pid AS blocking_pid, + now() - blocked_a.query_start AS wait_duration, + left(blocked_a.query, 120) AS blocked_query, + left(blocking_a.query, 120) AS blocking_query, + blocked_l.locktype, + blocked_l.relation::regclass AS locked_relation +FROM pg_catalog.pg_locks blocked_l +JOIN pg_catalog.pg_locks blocking_l + ON blocking_l.locktype = blocked_l.locktype + AND blocking_l.relation = blocked_l.relation + AND blocking_l.granted + AND NOT blocked_l.granted +JOIN pg_stat_activity blocked_a ON blocked_a.pid = blocked_l.pid +JOIN pg_stat_activity blocking_a ON blocking_a.pid = blocking_l.pid +ORDER BY wait_duration DESC; + +-- Autovacuum activity — shows if autovacuum is currently running or stuck. +CREATE OR REPLACE VIEW demo.autovacuum_status AS +SELECT + pid, + now() - xact_start AS running_for, + query AS vacuum_query, + wait_event_type, + wait_event +FROM pg_stat_activity +WHERE query LIKE 'autovacuum:%' +ORDER BY xact_start; + +-- Oldest transaction horizon — the horizon that blocks dead tuple cleanup. +-- When this is far in the past and dead tuples are climbing, we have MVCC bloat. +CREATE OR REPLACE VIEW demo.xmin_horizon AS +SELECT + pid, + backend_xmin AS xmin, + age(backend_xmin) AS xmin_age, + now() - xact_start AS tx_age, + state, + left(query, 120) AS query_snippet +FROM pg_stat_activity +WHERE backend_xmin IS NOT NULL +ORDER BY age(backend_xmin) DESC; + +\echo 'Schema and views created. Run 02_generate_data.sql next.' diff --git a/demo/ci-migration/02_generate_data.sql b/demo/ci-migration/02_generate_data.sql new file mode 100644 index 0000000..4c180cc --- /dev/null +++ b/demo/ci-migration/02_generate_data.sql @@ -0,0 +1,65 @@ +-- 02_generate_data.sql +-- Populates demo.ci_job_artifacts_legacy with ~5 million rows (~1 GB of data). +-- Also seeds demo.ci_build_status for the OLTP workload. +-- +-- Runtime estimate: 3–6 minutes on the CNPG cluster. +-- Scale: represents 1 GB out of the real 10 TB (1:10000 ratio). +-- +-- Data distribution: +-- - Rows span the last 180 days (6 months of history) +-- - 70% of expire_at values are in the past (already expired) +-- - 5% of rows are locked (should be skipped by cleanup) +-- - project_id values spread across 50,000 "projects" +-- - job_id values spread across 1,000,000 "jobs" + +\echo 'Generating 5,000,000 rows in demo.ci_job_artifacts_legacy...' +\echo 'This will take 3-6 minutes. Watch progress with: SELECT count(*) FROM demo.ci_job_artifacts_legacy;' + +INSERT INTO demo.ci_job_artifacts_legacy + (job_id, project_id, file_type, size_bytes, file_store, + checksum, created_at, expire_at, locked) +SELECT + (random() * 999999 + 1)::BIGINT AS job_id, + (random() * 49999 + 1)::BIGINT AS project_id, + (random() * 9)::SMALLINT AS file_type, + (random() * 104857600)::BIGINT AS size_bytes, -- up to 100MB per artifact + 1 AS file_store, + encode(sha256((random()::TEXT || i::TEXT)::BYTEA), 'hex') AS checksum, + NOW() - (random() * INTERVAL '180 days') AS created_at, + CASE + WHEN random() < 0.70 + THEN NOW() - (random() * INTERVAL '90 days') -- 70% already expired + ELSE NOW() + (random() * INTERVAL '30 days') -- 30% not yet expired + END AS expire_at, + random() < 0.05 AS locked -- 5% locked +FROM generate_series(1, 5000000) AS gs(i); + +\echo 'Legacy table populated. Running ANALYZE...' +ANALYZE demo.ci_job_artifacts_legacy; + +-- Seed the OLTP workload table +\echo 'Seeding OLTP workload table (100,000 rows)...' +INSERT INTO demo.ci_build_status (job_id, status) +SELECT + (random() * 999999 + 1)::BIGINT, + CASE (random() * 2)::INT + WHEN 0 THEN 'pending' + WHEN 1 THEN 'running' + ELSE 'created' + END +FROM generate_series(1, 100000); + +ANALYZE demo.ci_build_status; + +\echo '' +\echo 'Data generation complete. Verify with:' +\echo ' SELECT count(*), pg_size_pretty(sum(pg_column_size(t.*))) FROM demo.ci_job_artifacts_legacy t;' + +SELECT + count(*) AS row_count, + pg_size_pretty(pg_total_relation_size('demo.ci_job_artifacts_legacy')) AS table_size, + min(created_at)::DATE AS earliest_row, + max(created_at)::DATE AS latest_row, + count(*) FILTER (WHERE expire_at < NOW()) AS expired_rows, + count(*) FILTER (WHERE locked = TRUE) AS locked_rows +FROM demo.ci_job_artifacts_legacy; diff --git a/demo/ci-migration/03_broken_migration.sql b/demo/ci-migration/03_broken_migration.sql new file mode 100644 index 0000000..c418956 --- /dev/null +++ b/demo/ci-migration/03_broken_migration.sql @@ -0,0 +1,85 @@ +-- 03_broken_migration.sql +-- THE BROKEN MIGRATION — do not run this on production. +-- This is the verbatim migration from the change request, with annotations. +-- It is intentionally left broken to demonstrate the failure modes. +-- +-- Run in Session A. Watch Session B (monitoring) while this runs. +-- +-- ISSUES DEMONSTRATED: +-- 1. Single transaction wrapping the entire backfill → MVCC bloat +-- 2. Partition range only covers FUTURE months → immediate INSERT failure +-- 3. CREATE INDEX without CONCURRENTLY inside the transaction → ShareLock +-- 4. No chunking → all-or-nothing, no progress on failure +-- +-- EXPECTED OUTCOME: +-- The INSERT will fail with: +-- ERROR: no partition of relation "ci_job_artifacts" found for row +-- But before it fails, the open BEGIN will already be visible in: +-- demo.long_running_tx, demo.xmin_horizon +-- And dead tuples in demo.ci_build_status will be climbing because +-- autovacuum cannot advance past our snapshot. + +-- ── DROP target if it exists from a previous run ───────────────────────────── +DROP TABLE IF EXISTS demo.ci_job_artifacts CASCADE; + +BEGIN; + +-- ── CREATE partitioned table ────────────────────────────────────────────────── +CREATE TABLE demo.ci_job_artifacts ( + id BIGSERIAL, + job_id BIGINT NOT NULL, + project_id BIGINT NOT NULL, + file_type SMALLINT NOT NULL DEFAULT 0, + size_bytes BIGINT, + file_store SMALLINT NOT NULL DEFAULT 1, + checksum TEXT, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + expire_at TIMESTAMP, + locked BOOLEAN DEFAULT FALSE, + partition_key DATE NOT NULL DEFAULT CURRENT_DATE, + PRIMARY KEY (id, partition_key) +) PARTITION BY RANGE (partition_key); + +-- ISSUE 2: Creates partitions from CURRENT_DATE forward (future months only). +-- The backfill below selects rows from the PAST 6 months. +-- These rows have no matching partition → every INSERT row will error. +DO $$ +DECLARE + start_date DATE := DATE_TRUNC('month', CURRENT_DATE); + partition_name TEXT; + i INT; +BEGIN + FOR i IN 0..5 LOOP + partition_name := 'ci_job_artifacts_' || TO_CHAR(start_date + (i || ' months')::INTERVAL, 'YYYY_MM'); + EXECUTE FORMAT( + 'CREATE TABLE demo.%I PARTITION OF demo.ci_job_artifacts + FOR VALUES FROM (%L) TO (%L)', + partition_name, + start_date + (i || ' months')::INTERVAL, + start_date + ((i + 1) || ' months')::INTERVAL + ); + END LOOP; +END $$; + +-- ISSUE 3: Indexes created inside the transaction, without CONCURRENTLY. +-- These hold ShareLock for the entire transaction duration. +CREATE INDEX idx_ci_job_artifacts_job_id ON demo.ci_job_artifacts (job_id); +CREATE INDEX idx_ci_job_artifacts_project_id ON demo.ci_job_artifacts (project_id); +CREATE INDEX idx_ci_job_artifacts_expire_at ON demo.ci_job_artifacts (expire_at); +CREATE INDEX idx_ci_job_artifacts_checksum ON demo.ci_job_artifacts (checksum); + +-- ISSUE 1 + 2: Backfill inside a single transaction, targeting past 6 months. +-- Will fail because no partition exists for those months. +-- Even if we fix the partition range, running this as a single INSERT on +-- 5M rows (or 10TB at real scale) holds the transaction open for hours, +-- bloating MVCC dead tuples on every table touched by concurrent sessions. +INSERT INTO demo.ci_job_artifacts +SELECT + id, job_id, project_id, file_type, size_bytes, file_store, + checksum, created_at, expire_at, locked, + DATE_TRUNC('month', created_at)::DATE AS partition_key +FROM demo.ci_job_artifacts_legacy +WHERE created_at >= NOW() - INTERVAL '6 months'; + +-- This COMMIT will never be reached due to the partition error above. +COMMIT; diff --git a/demo/ci-migration/03b_broken_migration_long_tx.sql b/demo/ci-migration/03b_broken_migration_long_tx.sql new file mode 100644 index 0000000..37d8701 --- /dev/null +++ b/demo/ci-migration/03b_broken_migration_long_tx.sql @@ -0,0 +1,76 @@ +-- 03b_broken_migration_long_tx.sql +-- MVCC BLOAT DEMONSTRATION — the backfill that survives (partition bug fixed). +-- This version creates the correct historical partitions so the INSERT runs, +-- but keeps the fatal single-transaction pattern to demonstrate bloat. +-- +-- Run in Session A. While it runs, hammer Session B (04_traffic_sim.sh), +-- and watch Session C (05_monitor.sql) for dead tuple accumulation. +-- +-- The ~5M row INSERT will take several minutes. +-- Every UPDATE to demo.ci_build_status during that window creates a dead tuple +-- that autovacuum cannot collect because our open snapshot holds the xmin horizon. +-- +-- Watch for: +-- demo.mvcc_bloat_monitor → dead_rows climbing on ci_build_status +-- demo.long_running_tx → this session visible for the duration +-- demo.xmin_horizon → our xmin locking autovacuum out +-- demo.wal_progress → WAL size growing at alarming rate + +DROP TABLE IF EXISTS demo.ci_job_artifacts CASCADE; + +BEGIN; + +CREATE TABLE demo.ci_job_artifacts ( + id BIGSERIAL, + job_id BIGINT NOT NULL, + project_id BIGINT NOT NULL, + file_type SMALLINT NOT NULL DEFAULT 0, + size_bytes BIGINT, + file_store SMALLINT NOT NULL DEFAULT 1, + checksum TEXT, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + expire_at TIMESTAMP, + locked BOOLEAN DEFAULT FALSE, + partition_key DATE NOT NULL DEFAULT CURRENT_DATE, + PRIMARY KEY (id, partition_key) +) PARTITION BY RANGE (partition_key); + +-- Partitions covering past 6 months AND next 6 months (12 total). +DO $$ +DECLARE + start_date DATE := DATE_TRUNC('month', CURRENT_DATE - INTERVAL '6 months'); + partition_name TEXT; + i INT; +BEGIN + FOR i IN 0..11 LOOP + partition_name := 'ci_job_artifacts_' || TO_CHAR(start_date + (i || ' months')::INTERVAL, 'YYYY_MM'); + EXECUTE FORMAT( + 'CREATE TABLE demo.%I PARTITION OF demo.ci_job_artifacts + FOR VALUES FROM (%L) TO (%L)', + partition_name, + start_date + (i || ' months')::INTERVAL, + start_date + ((i + 1) || ' months')::INTERVAL + ); + END LOOP; +END $$; + +-- Indexes in the same transaction (ShareLock held while INSERT runs below). +CREATE INDEX idx_cia_job_id ON demo.ci_job_artifacts (job_id); +CREATE INDEX idx_cia_project_id ON demo.ci_job_artifacts (project_id); +CREATE INDEX idx_cia_expire_at ON demo.ci_job_artifacts (expire_at); +CREATE INDEX idx_cia_checksum ON demo.ci_job_artifacts (checksum); + +-- THE PROBLEM: 5 million rows in one statement, one transaction. +-- At 10 TB real scale this runs for hours. +-- Autovacuum cannot advance past our xmin for the entire duration. +INSERT INTO demo.ci_job_artifacts +SELECT + id, job_id, project_id, file_type, size_bytes, file_store, + checksum, created_at, expire_at, locked, + DATE_TRUNC('month', created_at)::DATE AS partition_key +FROM demo.ci_job_artifacts_legacy +WHERE created_at >= NOW() - INTERVAL '6 months'; + +COMMIT; + +\echo 'Migration committed. Check demo.mvcc_bloat_monitor for bloat residue.' diff --git a/demo/ci-migration/04_pgbench_workload.sql b/demo/ci-migration/04_pgbench_workload.sql new file mode 100644 index 0000000..19f0f9d --- /dev/null +++ b/demo/ci-migration/04_pgbench_workload.sql @@ -0,0 +1,28 @@ +-- 04_pgbench_workload.sql +-- pgbench custom script for OLTP traffic simulation. +-- Run with: +-- pgbench -h 127.0.0.1 -p 15432 -U postgres knoe-db \ +-- -c 10 -j 2 -T 600 \ +-- -f demo/ci-migration/04_pgbench_workload.sql +-- +-- Each worker randomly UPDATEs and SELECTs ci_build_status rows. +-- This creates a steady stream of dead tuples that autovacuum must collect. +-- When a long migration transaction holds an old xmin, these dead tuples +-- accumulate unboundedly — the core of the MVCC bloat problem. + +\set job_id random(1, 999999) +\set row_id random(1, 100000) + +UPDATE demo.ci_build_status +SET status = CASE (:job_id % 3) + WHEN 0 THEN 'running' + WHEN 1 THEN 'completed' + ELSE 'failed' + END, + updated_at = NOW() +WHERE id = :row_id; + +SELECT count(*) +FROM demo.ci_build_status +WHERE status = 'running' + AND updated_at > NOW() - INTERVAL '1 minute'; diff --git a/demo/ci-migration/05_monitor.sql b/demo/ci-migration/05_monitor.sql new file mode 100644 index 0000000..1eb177f --- /dev/null +++ b/demo/ci-migration/05_monitor.sql @@ -0,0 +1,54 @@ +-- 05_monitor.sql +-- Run this in a SEPARATE psql session while the migration is executing. +-- Poll every few seconds to watch MVCC bloat build up in real time. +-- +-- Usage (run continuously): +-- watch -n 3 'PGPASSWORD=... psql -h 127.0.0.1 -p 15432 -U postgres knoe-db -f demo/ci-migration/05_monitor.sql' +-- +-- Or in interactive psql with \watch: +-- \i demo/ci-migration/05_monitor.sql +-- \watch 3 + +\echo '═══════════════════════════════════════════════════════════' +\echo ' MVCC BLOAT MONITOR — updated every \watch cycle' +\echo '═══════════════════════════════════════════════════════════' + +\echo '' +\echo '── Long-running transactions (xmin holders) ─────────────' +SELECT pid, + tx_duration, + state, + wait_event_type || '/' || COALESCE(wait_event,'') AS wait, + left(query_snippet, 80) AS query +FROM demo.long_running_tx +LIMIT 5; + +\echo '' +\echo '── xmin horizon (blocks autovacuum cleanup) ─────────────' +SELECT pid, xmin_age, tx_age, state, left(query_snippet, 60) AS query +FROM demo.xmin_horizon +LIMIT 5; + +\echo '' +\echo '── Dead tuple accumulation ──────────────────────────────' +SELECT table_name, live_rows, dead_rows, dead_pct, total_size +FROM demo.mvcc_bloat_monitor +LIMIT 10; + +\echo '' +\echo '── Active lock waits ────────────────────────────────────' +SELECT blocked_pid, blocking_pid, wait_duration, + left(blocked_query, 60) AS blocked_q, + left(blocking_query, 60) AS blocking_q +FROM demo.lock_waits +LIMIT 5; + +\echo '' +\echo '── WAL generation progress ──────────────────────────────' +SELECT current_lsn, total_wal, wal_segment_count, wal_dir_size +FROM demo.wal_progress; + +\echo '' +\echo '── Autovacuum activity ──────────────────────────────────' +SELECT pid, running_for, left(vacuum_query, 80) AS query +FROM demo.autovacuum_status; diff --git a/demo/ci-migration/06_fixed_migration.sql b/demo/ci-migration/06_fixed_migration.sql new file mode 100644 index 0000000..4e01fb8 --- /dev/null +++ b/demo/ci-migration/06_fixed_migration.sql @@ -0,0 +1,144 @@ +-- 06_fixed_migration.sql +-- THE FIXED MIGRATION — safe to run on a live database. +-- Demonstrates the pg_chunker approach: small autonomous transactions, +-- each committing independently so autovacuum stays unblocked throughout. +-- +-- Key differences from the broken version: +-- ✓ No wrapping BEGIN/COMMIT — each chunk is its own transaction +-- ✓ Partitions cover BOTH historical and future months +-- ✓ Indexes created with CONCURRENTLY after data load (no table lock) +-- ✓ pg_sleep(0.05) breathing room between chunks for autovacuum +-- ✓ Progress reporting at each chunk boundary +-- ✓ Idempotent: safe to re-run if interrupted (ON CONFLICT DO NOTHING) +-- +-- Reference: https://github.com/eyupmiduck/pg_chunker +-- (The DO $$ chunker loop below implements the same chunked-keyset pattern.) + +-- ── Step 1: Create the partitioned table (outside any transaction) ──────────── +DROP TABLE IF EXISTS demo.ci_job_artifacts CASCADE; + +CREATE TABLE demo.ci_job_artifacts ( + id BIGINT NOT NULL, + job_id BIGINT NOT NULL, + project_id BIGINT NOT NULL, + file_type SMALLINT NOT NULL DEFAULT 0, + size_bytes BIGINT, + file_store SMALLINT NOT NULL DEFAULT 1, + checksum TEXT, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + expire_at TIMESTAMP, + locked BOOLEAN DEFAULT FALSE, + partition_key DATE NOT NULL, + PRIMARY KEY (id, partition_key) +) PARTITION BY RANGE (partition_key); + +-- ── Step 2: Create partitions covering past 6 months AND next 6 months ──────── +DO $$ +DECLARE + start_date DATE := DATE_TRUNC('month', CURRENT_DATE - INTERVAL '6 months'); + partition_name TEXT; + i INT; +BEGIN + FOR i IN 0..11 LOOP + partition_name := 'ci_job_artifacts_' + || TO_CHAR(start_date + (i || ' months')::INTERVAL, 'YYYY_MM'); + EXECUTE FORMAT( + 'CREATE TABLE IF NOT EXISTS demo.%I + PARTITION OF demo.ci_job_artifacts + FOR VALUES FROM (%L) TO (%L)', + partition_name, + (start_date + (i || ' months')::INTERVAL)::DATE, + (start_date + ((i+1) || ' months')::INTERVAL)::DATE + ); + END LOOP; + RAISE NOTICE 'Created 12 partitions from % to %', + start_date, + start_date + INTERVAL '12 months'; +END $$; + +-- ── Step 3: Chunked backfill (pg_chunker keyset pattern) ───────────────────── +-- Each iteration is a small autonomous transaction (~50K rows). +-- Autovacuum runs freely between chunks because no long snapshot is held. +-- On a 10 TB table at real scale: 200M chunks × 50K rows each = fine. +DO $$ +DECLARE + v_min_id BIGINT; + v_max_id BIGINT; + v_cursor BIGINT := 0; + v_chunk INT := 50000; + v_inserted INT := 0; + v_total INT := 0; + v_chunk_num INT := 0; +BEGIN + SELECT COALESCE(MIN(id), 0), COALESCE(MAX(id), 0) + INTO v_min_id, v_max_id + FROM demo.ci_job_artifacts_legacy + WHERE created_at >= NOW() - INTERVAL '6 months'; + + RAISE NOTICE 'Backfill range: id % to % (eligible rows for past 6 months)', + v_min_id, v_max_id; + + v_cursor := v_min_id - 1; + + WHILE v_cursor < v_max_id LOOP + -- Each INSERT is its own implicit transaction (no explicit BEGIN here). + -- In a real migration script this would be called from a shell loop or + -- pg_chunker, which gives each chunk its own connection/transaction. + INSERT INTO demo.ci_job_artifacts + (id, job_id, project_id, file_type, size_bytes, file_store, + checksum, created_at, expire_at, locked, partition_key) + SELECT + id, job_id, project_id, file_type, size_bytes, file_store, + checksum, created_at, expire_at, locked, + DATE_TRUNC('month', created_at)::DATE AS partition_key + FROM demo.ci_job_artifacts_legacy + WHERE id > v_cursor + AND id <= v_cursor + v_chunk + AND created_at >= NOW() - INTERVAL '6 months' + ON CONFLICT (id, partition_key) DO NOTHING; -- idempotent re-run + + GET DIAGNOSTICS v_inserted = ROW_COUNT; + v_total := v_total + v_inserted; + v_chunk_num := v_chunk_num + 1; + v_cursor := v_cursor + v_chunk; + + IF v_chunk_num % 20 = 0 THEN + RAISE NOTICE 'Chunk %: inserted % rows this chunk, % total, cursor at %', + v_chunk_num, v_inserted, v_total, v_cursor; + END IF; + + -- Yield briefly so autovacuum can run between chunks. + -- This is what keeps dead_pct from exploding during the backfill. + PERFORM pg_sleep(0.05); + END LOOP; + + RAISE NOTICE 'Backfill complete: % total rows inserted in % chunks', + v_total, v_chunk_num; +END $$; + +-- ── Step 4: Indexes CONCURRENTLY — no table lock, runs alongside traffic ────── +-- Must be run OUTSIDE any transaction block (cannot use CONCURRENTLY in a tx). +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_cia_job_id + ON demo.ci_job_artifacts (job_id); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_cia_project_id + ON demo.ci_job_artifacts (project_id); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_cia_expire_at + ON demo.ci_job_artifacts (expire_at); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_cia_checksum + ON demo.ci_job_artifacts (checksum); + +\echo '' +\echo 'Fixed migration complete. Check demo.mvcc_bloat_monitor:' +\echo ' dead_rows on ci_build_status should be near zero throughout.' + +SELECT + relname AS table_name, + n_live_tup AS live_rows, + n_dead_tup AS dead_rows, + ROUND(n_dead_tup::NUMERIC / NULLIF(n_live_tup,0) * 100, 2) AS dead_pct +FROM pg_stat_user_tables +WHERE schemaname = 'demo' +ORDER BY n_dead_tup DESC; diff --git a/demo/ci-migration/07_corrected_functions.sql b/demo/ci-migration/07_corrected_functions.sql new file mode 100644 index 0000000..86c118f --- /dev/null +++ b/demo/ci-migration/07_corrected_functions.sql @@ -0,0 +1,151 @@ +-- 07_corrected_functions.sql +-- Corrected versions of all four utility functions from the change request. +-- Each fix is annotated with the issue number it addresses (see REVIEW.md). + +-- ── Fix for cleanup_expired_artifacts ──────────────────────────────────────── +-- Issue 4: Added partition_key to WHERE and IN-list for partition pruning. +-- Issue 4: Replaced FOR UPDATE with FOR UPDATE SKIP LOCKED to prevent +-- stacking behind a concurrent cleanup job. +-- Bonus: RETURNING clause allows callers to log what was actually deleted. +CREATE OR REPLACE FUNCTION demo.cleanup_expired_artifacts( + p_batch_size INT DEFAULT 10000 +) +RETURNS INT AS $$ +DECLARE + v_deleted INT; +BEGIN + DELETE FROM demo.ci_job_artifacts + WHERE (id, partition_key) IN ( + SELECT id, partition_key + FROM demo.ci_job_artifacts + WHERE expire_at < NOW() + AND locked = FALSE + AND partition_key < DATE_TRUNC('month', NOW())::DATE -- prune to old partitions only + ORDER BY expire_at ASC + LIMIT p_batch_size + FOR UPDATE SKIP LOCKED + ); + + GET DIAGNOSTICS v_deleted = ROW_COUNT; + RAISE NOTICE 'Cleaned up % expired artifacts', v_deleted; + RETURN v_deleted; +END; +$$ LANGUAGE plpgsql; + + +-- ── Fix for terminate_idle_sessions ────────────────────────────────────────── +-- Issue 5: Added filter to exclude PgBouncer pooler connections. +-- In transaction-mode PgBouncer the backend appears idle between +-- client transactions. Terminating it breaks the pool and causes +-- "SSL connection has been closed unexpectedly" for active clients. +-- Bonus: filter also excludes replication and background workers. +CREATE OR REPLACE FUNCTION demo.terminate_idle_sessions( + p_idle_threshold INTERVAL DEFAULT '10 minutes', + p_max_terminated INT DEFAULT 50, + p_pgbouncer_app TEXT DEFAULT 'pgbouncer' -- set to your app_name +) +RETURNS TABLE(pid INT, username TEXT, idle_duration INTERVAL, query TEXT) AS $$ +BEGIN + RETURN QUERY + WITH idle_sessions AS ( + SELECT + a.pid, + a.usename::TEXT, + NOW() - a.state_change AS duration, + a.query + FROM pg_stat_activity a + WHERE a.state = 'idle' + AND NOW() - a.state_change > p_idle_threshold + AND a.pid <> pg_backend_pid() + AND a.backend_type = 'client backend' + AND a.application_name NOT ILIKE '%' || p_pgbouncer_app || '%' + AND a.client_addr IS NOT NULL -- exclude unix socket (local tools) + ORDER BY duration DESC + LIMIT p_max_terminated + ) + SELECT s.pid, s.usename, s.duration, s.query + FROM idle_sessions s + WHERE pg_terminate_backend(s.pid); +END; +$$ LANGUAGE plpgsql; + + +-- ── Fix for calculate_namespace_storage ────────────────────────────────────── +-- Issue 6: Cache race condition fixed with FOR UPDATE SKIP LOCKED. +-- Callers that can't acquire the cache row lock return the stale +-- cached value rather than all recomputing simultaneously. +-- Issue 7: Recursive CTE depth limit enforced (max 20 levels). +-- Requires tables: namespaces(id, parent_id), project_storages(namespace_id, storage_bytes), +-- namespace_storage_cache(namespace_id, total_bytes, calculated_at) +-- (These don't exist in our demo schema; this is illustrative code only.) +CREATE OR REPLACE FUNCTION demo.calculate_namespace_storage( + p_namespace_id BIGINT +) +RETURNS BIGINT AS $$ +DECLARE + v_total_bytes BIGINT := 0; + v_cached_bytes BIGINT; + v_cached_at TIMESTAMP; +BEGIN + -- Attempt to lock the cache row exclusively. + -- SKIP LOCKED means concurrent callers return the stale value immediately + -- instead of piling up and all recomputing the expensive CTE. + SELECT total_bytes, calculated_at + INTO v_cached_bytes, v_cached_at + FROM namespace_storage_cache + WHERE namespace_id = p_namespace_id + FOR UPDATE SKIP LOCKED; + + -- Cache hit: return without recomputing + IF v_cached_at IS NOT NULL AND v_cached_at > NOW() - INTERVAL '1 hour' THEN + RETURN v_cached_bytes; + END IF; + + -- Cache miss or expired: recompute with depth-limited recursive CTE. + -- Issue 7 fix: depth column added to prevent infinite recursion on cycles. + WITH RECURSIVE ns_tree AS ( + SELECT id, 0 AS depth + FROM namespaces + WHERE id = p_namespace_id + + UNION ALL + + SELECT n.id, t.depth + 1 + FROM namespaces n + JOIN ns_tree t ON n.parent_id = t.id + WHERE t.depth < 20 -- hard cap; cyclic namespaces stop here + ) + SELECT COALESCE(SUM(ps.storage_bytes), 0) + INTO v_total_bytes + FROM ns_tree nt + JOIN project_storages ps ON ps.namespace_id = nt.id; + + INSERT INTO namespace_storage_cache (namespace_id, total_bytes, calculated_at) + VALUES (p_namespace_id, v_total_bytes, NOW()) + ON CONFLICT (namespace_id) DO UPDATE + SET total_bytes = EXCLUDED.total_bytes, + calculated_at = EXCLUDED.calculated_at; + + RETURN v_total_bytes; +END; +$$ LANGUAGE plpgsql; + + +-- ── Fix for database_health view ───────────────────────────────────────────── +-- Issue 8: NULLIF prevents division by zero if n_live_tup is ever 0. +-- The original WHERE n_live_tup > 1000 makes the crash unlikely in practice, +-- but this is fragile to future refactoring that removes the filter. +CREATE OR REPLACE VIEW demo.database_health AS +SELECT + schemaname, + relname AS table_name, + n_live_tup AS live_rows, + n_dead_tup AS dead_rows, + ROUND(n_dead_tup::NUMERIC / NULLIF(n_live_tup, 0) * 100, 2) AS dead_row_pct, + last_vacuum, + last_autovacuum, + last_analyze, + pg_size_pretty(pg_total_relation_size(schemaname || '.' || relname)) AS total_size +FROM pg_stat_user_tables +WHERE n_live_tup > 1000 +ORDER BY n_dead_tup DESC; diff --git a/demo/ci-migration/run_demo.sh b/demo/ci-migration/run_demo.sh new file mode 100755 index 0000000..4a72546 --- /dev/null +++ b/demo/ci-migration/run_demo.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# run_demo.sh — CI migration danger demo orchestration script. +# +# Prerequisites: +# - kubectl configured with access to the knoe-db namespace +# - psql and pgbench in PATH +# - Port-forward to knoe-db-rw running on localhost:15432 +# +# Usage: +# ./demo/ci-migration/run_demo.sh [setup|broken|fixed|monitor|reset] +# +# Phases: +# setup — create schema, generate 5M rows, seed workload table +# broken — run the broken single-transaction migration (will fail on partitions) +# bloat — run the long-tx variant that actually inserts (shows MVCC bloat) +# fixed — run the chunked safe migration +# monitor — start watch loop showing MVCC metrics +# reset — drop and recreate the target table, reset workload table + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ── Connection settings ─────────────────────────────────────────────────────── +DB_HOST="${DB_HOST:-127.0.0.1}" +DB_PORT="${DB_PORT:-15432}" +DB_USER="${DB_USER:-postgres}" +DB_NAME="${DB_NAME:-knoe-db}" + +# Retrieve password from Kubernetes secret if not set +if [[ -z "${PGPASSWORD:-}" ]]; then + PGPASSWORD="$(kubectl -n knoe-db get secret knoe-db-superuser \ + -o jsonpath='{.data.password}' | base64 -d)" + export PGPASSWORD +fi + +PSQL="psql -h $DB_HOST -p $DB_PORT -U $DB_USER $DB_NAME" +PGBENCH="pgbench -h $DB_HOST -p $DB_PORT -U $DB_USER $DB_NAME" + +# ── Helpers ─────────────────────────────────────────────────────────────────── +start_port_forward() { + if ! lsof -ti tcp:15432 &>/dev/null; then + echo "[+] Starting port-forward to knoe-db-rw on :15432..." + # Find the current primary pod + PRIMARY=$(kubectl -n knoe-db get pods \ + --field-selector=status.phase=Running \ + -o jsonpath='{range .items[?(@.status.containerStatuses[0].ready==true)]}{.metadata.name}{"\n"}{end}' \ + | grep '^knoe-db-' | head -1) + echo " Primary pod: $PRIMARY" + kubectl -n knoe-db port-forward "pod/$PRIMARY" 15432:5432 &>/tmp/pf_knoe.log & + sleep 3 + else + echo "[✓] Port-forward already running on :15432" + fi +} + +run_sql() { + local file="$1" + echo "[+] Running $file..." + $PSQL -f "$file" +} + +# ── Commands ────────────────────────────────────────────────────────────────── +case "${1:-help}" in + +setup) + start_port_forward + echo "" + echo "=== PHASE 1: Schema setup ===" + run_sql "$SCRIPT_DIR/01_schema_setup.sql" + echo "" + echo "=== PHASE 2: Data generation (5M rows — takes 3-6 min) ===" + run_sql "$SCRIPT_DIR/02_generate_data.sql" + echo "" + echo "Setup complete. Run './run_demo.sh monitor' in another terminal," + echo "then './run_demo.sh broken' to start the demo." + ;; + +broken) + start_port_forward + echo "" + echo "=== BROKEN MIGRATION (Issue 2 exposed — partition mismatch) ===" + echo "Watch for: ERROR: no partition of relation found for row" + echo "" + run_sql "$SCRIPT_DIR/03_broken_migration.sql" || true + echo "" + echo "Expected failure demonstrated. Now run './run_demo.sh bloat'" + echo "to see the long-transaction MVCC pileup." + ;; + +bloat) + start_port_forward + echo "" + echo "=== MVCC BLOAT DEMO (partition bug fixed, long tx left in place) ===" + echo "Start traffic in another terminal first:" + echo " ./run_demo.sh traffic" + echo "" + read -p "Press Enter when traffic is running to start the long migration..." + echo "" + echo "Recording WAL start position..." + WAL_START=$($PSQL -tAc "SELECT pg_current_wal_lsn();") + echo "WAL start: $WAL_START" + echo "" + run_sql "$SCRIPT_DIR/03b_broken_migration_long_tx.sql" + echo "" + WAL_END=$($PSQL -tAc "SELECT pg_current_wal_lsn();") + echo "WAL end: $WAL_END" + WAL_DIFF=$($PSQL -tAc "SELECT pg_size_pretty(pg_wal_lsn_diff('$WAL_END'::pg_lsn, '$WAL_START'::pg_lsn));") + echo "WAL generated: $WAL_DIFF" + ;; + +fixed) + start_port_forward + echo "" + echo "=== FIXED MIGRATION (chunked, no long transaction) ===" + echo "Start traffic in another terminal first:" + echo " ./run_demo.sh traffic" + echo "" + read -p "Press Enter when traffic is running to start the chunked migration..." + echo "" + echo "Recording WAL start position..." + WAL_START=$($PSQL -tAc "SELECT pg_current_wal_lsn();") + echo "WAL start: $WAL_START" + echo "" + run_sql "$SCRIPT_DIR/06_fixed_migration.sql" + echo "" + WAL_END=$($PSQL -tAc "SELECT pg_current_wal_lsn();") + WAL_DIFF=$($PSQL -tAc "SELECT pg_size_pretty(pg_wal_lsn_diff('$WAL_END'::pg_lsn, '$WAL_START'::pg_lsn));") + echo "WAL generated: $WAL_DIFF" + echo "(Compare to the bloat run — same data, far less WAL spike)" + ;; + +traffic) + start_port_forward + echo "" + echo "=== OLTP TRAFFIC SIMULATOR ===" + echo "Running 10 concurrent workers for 600 seconds (10 min)." + echo "Ctrl+C to stop early." + echo "" + $PGBENCH -c 10 -j 2 -T 600 \ + -f "$SCRIPT_DIR/04_pgbench_workload.sql" \ + --no-vacuum \ + -P 10 + ;; + +monitor) + start_port_forward + echo "" + echo "=== MONITORING (updates every 3 seconds) ===" + echo "Ctrl+C to stop." + echo "" + watch -n 3 "$PSQL -f $SCRIPT_DIR/05_monitor.sql 2>&1" + ;; + +reset) + start_port_forward + echo "" + echo "=== RESET: dropping ci_job_artifacts, truncating ci_build_status ===" + $PSQL -c "DROP TABLE IF EXISTS demo.ci_job_artifacts CASCADE;" + $PSQL -c "TRUNCATE demo.ci_build_status;" + $PSQL -c "INSERT INTO demo.ci_build_status (job_id, status) + SELECT (random()*999999+1)::BIGINT, 'running' + FROM generate_series(1,100000);" + echo "Reset complete. Run './run_demo.sh bloat' or './run_demo.sh fixed'." + ;; + +help|*) + echo "Usage: $0 [setup|broken|bloat|fixed|traffic|monitor|reset]" + echo "" + echo " setup — create schema, generate 5M rows of test data" + echo " broken — run the broken migration (partition error demo)" + echo " bloat — run the long-tx backfill to demonstrate MVCC bloat" + echo " fixed — run the chunked safe migration" + echo " traffic — start pgbench OLTP workload (run in separate terminal)" + echo " monitor — watch MVCC metrics in real time (run in separate terminal)" + echo " reset — drop target table and reset workload for a clean re-run" + echo "" + echo "Typical demo flow:" + echo " Terminal 1: ./run_demo.sh setup" + echo " Terminal 2: ./run_demo.sh traffic" + echo " Terminal 3: ./run_demo.sh monitor" + echo " Terminal 1: ./run_demo.sh broken (show Issue 2)" + echo " Terminal 1: ./run_demo.sh reset" + echo " Terminal 1: ./run_demo.sh bloat (show MVCC pileup)" + echo " Terminal 1: ./run_demo.sh reset" + echo " Terminal 1: ./run_demo.sh fixed (show safe approach)" + ;; +esac diff --git a/deploy/opentofu/k3s/manifests/knoe/iscsi-pvs.yaml b/deploy/opentofu/k3s/manifests/knoe/iscsi-pvs.yaml index b8fe785..5329f46 100644 --- a/deploy/opentofu/k3s/manifests/knoe/iscsi-pvs.yaml +++ b/deploy/opentofu/k3s/manifests/knoe/iscsi-pvs.yaml @@ -22,7 +22,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - myrddin.knoe.org + - myrddin.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -48,7 +48,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - myrddin.knoe.org + - myrddin.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -74,7 +74,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - myrddin.knoe.org + - myrddin.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -100,7 +100,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - merlin.knoe.org + - merlin.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -126,7 +126,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - merlin.knoe.org + - merlin.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -152,7 +152,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - pi.knoe.org + - pi.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -178,7 +178,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - pi.knoe.org + - pi.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -204,7 +204,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - pi.knoe.org + - pi.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -230,7 +230,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - merlin.knoe.org + - merlin.prole.org --- apiVersion: v1 kind: PersistentVolume @@ -256,4 +256,56 @@ spec: - key: kubernetes.io/hostname operator: In values: - - myrddin.knoe.org + - myrddin.prole.org +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: synology-iscsi-d004-data + labels: + synology.storage/role: data + synology.storage/volume: d004 +spec: + capacity: + storage: 29Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + storageClassName: synology-iscsi + persistentVolumeReclaimPolicy: Retain + local: + path: /synology/d004/data + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - gandalf.prole.org +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: synology-iscsi-d004-wal + labels: + synology.storage/role: wal + synology.storage/volume: d004 +spec: + capacity: + storage: 29Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + storageClassName: synology-iscsi + persistentVolumeReclaimPolicy: Retain + local: + path: /synology/d004/wal + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - gandalf.prole.org diff --git a/deploy/opentofu/k3s/manifests/knoe/knoe-db-barman-objectstore.yaml b/deploy/opentofu/k3s/manifests/knoe/knoe-db-barman-objectstore.yaml index fa9d56c..032e5e2 100644 --- a/deploy/opentofu/k3s/manifests/knoe/knoe-db-barman-objectstore.yaml +++ b/deploy/opentofu/k3s/manifests/knoe/knoe-db-barman-objectstore.yaml @@ -2,11 +2,12 @@ apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: name: knoe-db-barman-objectstore + namespace: knoe-db spec: retentionPolicy: 30d configuration: destinationPath: s3://knoe-db-backups/ - endpointURL: http://garage:3900 + endpointURL: http://garage.knoe-system.svc.cluster.local:3900 s3Credentials: accessKeyId: name: knoe-db-barman-s3 diff --git a/deploy/opentofu/k3s/manifests/knoe/knoe-db-postgres-tcp-ingress.yaml b/deploy/opentofu/k3s/manifests/knoe/knoe-db-postgres-tcp-ingress.yaml new file mode 100644 index 0000000..dbfd9d5 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/knoe/knoe-db-postgres-tcp-ingress.yaml @@ -0,0 +1,25 @@ +# knoe-db-postgres-tcp-ingress.yaml +# Traefik IngressRouteTCP for direct PostgreSQL access on db.internal.prole.org:5432. +# Routes to knoe-db-rw (CNPG primary read-write endpoint) in the knoe-db namespace. +# +# Prerequisites: +# - traefik HelmChart must have ports.postgres.port=5432 configured (exposedPort 5432) +# - DNS: db.internal.prole.org → 10.0.0.3 (myrddin, traefik LB VIP) +# - Access is local-network only (no port 5432 forwarded through NAT router) +# +# NOTE: HostSNI("*") is required for plain TCP (non-TLS) passthrough. +# All TCP connections on the 'postgres' entrypoint go to knoe-db-rw:5432. +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: knoe-db-rw-postgres + namespace: knoe-db +spec: + entryPoints: + - postgres + routes: + - match: HostSNI(`*`) + services: + - name: knoe-db-rw + port: 5432 diff --git a/deploy/opentofu/k3s/manifests/knoe/kong-deployment.yaml b/deploy/opentofu/k3s/manifests/knoe/kong-deployment.yaml index 5bd2451..51149b0 100644 --- a/deploy/opentofu/k3s/manifests/knoe/kong-deployment.yaml +++ b/deploy/opentofu/k3s/manifests/knoe/kong-deployment.yaml @@ -1,31 +1,31 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: knoe-svc-kong + name: prole-svc-kong annotations: argocd.argoproj.io/sync-wave: "1" labels: - app: knoe-svc-kong + app: prole-svc-kong spec: replicas: 1 selector: matchLabels: - app: knoe-svc-kong + app: prole-svc-kong template: metadata: labels: - app: knoe-svc-kong + app: prole-svc-kong spec: affinity: nodeAffinity: - # Never schedule on pi.knoe.org — pihole-FTL owns ports 80/443 there + # Never schedule on pi.prole.org — pihole-FTL owns ports 80/443 there requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: NotIn values: - - pi.knoe.org + - pi.prole.org preferredDuringSchedulingIgnoredDuringExecution: - weight: 80 preference: @@ -33,7 +33,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - gandalf.knoe.org + - gandalf.prole.org containers: - name: kong image: kong:3.9 @@ -85,4 +85,4 @@ spec: volumes: - name: kong-config configMap: - name: knoe-svc-kong-config + name: prole-svc-kong-config diff --git a/deploy/opentofu/k3s/manifests/knoe/kong-service.yaml b/deploy/opentofu/k3s/manifests/knoe/kong-service.yaml index beb822e..7b93088 100644 --- a/deploy/opentofu/k3s/manifests/knoe/kong-service.yaml +++ b/deploy/opentofu/k3s/manifests/knoe/kong-service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: knoe-svc-kong + name: prole-svc-kong annotations: argocd.argoproj.io/sync-wave: "1" labels: - app: knoe-svc-kong + app: prole-svc-kong spec: selector: - app: knoe-svc-kong + app: prole-svc-kong ports: - name: proxy port: 8000 diff --git a/deploy/opentofu/k3s/manifests/knoe/kustomization.yaml b/deploy/opentofu/k3s/manifests/knoe/kustomization.yaml index 88ef779..979da4e 100644 --- a/deploy/opentofu/k3s/manifests/knoe/kustomization.yaml +++ b/deploy/opentofu/k3s/manifests/knoe/kustomization.yaml @@ -18,3 +18,11 @@ resources: - openbao-statefulset.yaml - openbao-service.yaml - ingress.yaml + # prole.org migration — Kong (prole-svc-kong) replaces knoe-svc-kong + - prole-svc-kong-configmap.yaml + - kong-deployment.yaml + - kong-service.yaml + # prole.org ingress (*.prole.org → prole-svc-kong) + - prole-svc-ingress.yaml + # CNPG TCP ingress for direct PostgreSQL on db.internal.prole.org:5432 + - knoe-db-postgres-tcp-ingress.yaml diff --git a/deploy/opentofu/k3s/manifests/knoe/prole-svc-ingress.yaml b/deploy/opentofu/k3s/manifests/knoe/prole-svc-ingress.yaml new file mode 100644 index 0000000..d3287d4 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/knoe/prole-svc-ingress.yaml @@ -0,0 +1,82 @@ +# prole-svc-ingress.yaml +# Traefik ingress for *.prole.org — routes all prole.org API/UI domains through +# prole-svc-kong (declarative Kong in knoe-system). cert-manager issues a +# single multi-SAN Let's Encrypt cert (svc-prole-org-tls) for all hosts. +# +# Routing map (handled by Kong declarative config in prole-svc-kong-configmap.yaml): +# api.prole.org → supabase-kong.supabase:8000 (PostgREST / Auth / Storage / Functions) +# db.prole.org → supabase-kong.supabase:8000 (Supabase Studio + all APIs) +# supabase.prole.org → supabase-kong.supabase:8000 (alias for db.prole.org) +# svc.prole.org → prometheus-grafana.monitoring:80 (Grafana) +# git.prole.org → gitlab-webservice-default.gitlab:8080 (GitLab HTTPS) +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: svc-prole-ingress + namespace: knoe-system + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.entrypoints: web,websecure + traefik.ingress.kubernetes.io/router.priority: "10" +spec: + ingressClassName: traefik + rules: + - host: api.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-svc-kong + port: + number: 8000 + - host: db.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-svc-kong + port: + number: 8000 + - host: supabase.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-svc-kong + port: + number: 8000 + - host: svc.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-svc-kong + port: + number: 8000 + - host: git.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-svc-kong + port: + number: 8000 + tls: + - hosts: + - api.prole.org + - db.prole.org + - supabase.prole.org + - svc.prole.org + - git.prole.org + secretName: svc-prole-org-tls diff --git a/deploy/opentofu/k3s/manifests/knoe/prole-svc-kong-configmap.yaml b/deploy/opentofu/k3s/manifests/knoe/prole-svc-kong-configmap.yaml new file mode 100644 index 0000000..736f8cd --- /dev/null +++ b/deploy/opentofu/k3s/manifests/knoe/prole-svc-kong-configmap.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prole-svc-kong-config + namespace: knoe-system + annotations: + argocd.argoproj.io/sync-wave: "0" +data: + kong.yml: | + _format_version: "3.0" + _transform: true + + services: + # ── svc.prole.org — internal tools ─────────────────────────────────────── + - name: prole-service + url: http://prole-svc.knoe-system.svc.cluster.local:8080 + routes: + - name: prole-k3s-kubeconfig + hosts: + - svc.prole.org + paths: + - /k3s/kube_config.sh + strip_path: false + + - name: db-manager + url: http://knoe-db-manager.knoe-db.svc.cluster.local:80 + routes: + - name: backup-route + hosts: + - svc.prole.org + paths: + - /backup + strip_path: false + + - name: grafana + url: http://kps-grafana.monitoring.svc.cluster.local:80 + routes: + - name: grafana-root + hosts: + - svc.prole.org + paths: + - / + strip_path: false + + # ── api.prole.org / db.prole.org / supabase.prole.org → Supabase Kong ── + # Supabase Kong handles internal routing: /auth/, /rest/, /storage/, + # /realtime/, /functions/, and / (Studio UI). + - name: supabase-kong + url: http://supabase-kong.supabase.svc.cluster.local:8000 + routes: + - name: supabase-api + hosts: + - api.prole.org + paths: + - / + strip_path: false + preserve_host: true + - name: supabase-db + hosts: + - db.prole.org + paths: + - / + strip_path: false + preserve_host: true + - name: supabase-studio + hosts: + - supabase.prole.org + paths: + - / + strip_path: false + preserve_host: true + + # ── git.prole.org → GitLab (Workhorse + Puma on :8080) ────────────────── + # Routes HTTP/HTTPS git traffic through prole-svc-kong to the GitLab + # webservice. GitLab's own nginx ingress is blocked by hostPort conflicts + # with traefik svclb; this bypasses it cleanly. + # SSH git access (port 22) is handled separately by gitlab-gitlab-shell. + - name: gitlab-web + url: http://gitlab-webservice-default.gitlab.svc.cluster.local:8080 + routes: + - name: gitlab-root + hosts: + - git.prole.org + paths: + - / + strip_path: false + preserve_host: true diff --git a/deploy/opentofu/k3s/manifests/knoe/supabase-studio.yaml b/deploy/opentofu/k3s/manifests/knoe/supabase-studio.yaml new file mode 100644 index 0000000..fe469e4 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/knoe/supabase-studio.yaml @@ -0,0 +1,151 @@ +# supabase-studio.yaml +# Supabase Studio Deployment + Service for k3s (prole homelab). +# +# The upstream knoe-supabase Helm chart includes studio but with: +# - nodeSelector: knoe.dev/node-role=general (GKE-only label) +# - POSTGRES_HOST: GKE pod IP (wrong for k3s) +# - supabase-snippets PVC reference (not provisioned on k3s) +# +# This manifest is the k3s-compatible version: +# - No nodeSelector / affinity (schedules freely on gandalf/merlin) +# - POSTGRES_HOST: knoe-db-rw.knoe-db.svc.cluster.local +# - Only supabase-functions PVC (supabase-snippets excluded) +# - SUPABASE_PUBLIC_URL: https://db.prole.org +# +# Supabase Kong routes / → http://supabase-studio:3000/ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-studio + namespace: supabase + labels: + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase +--- +apiVersion: v1 +kind: Service +metadata: + name: supabase-studio + namespace: supabase + labels: + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase +spec: + type: ClusterIP + ports: + - port: 3000 + targetPort: 3000 + protocol: TCP + name: http + selector: + app.kubernetes.io/name: knoe-supabase-studio + app.kubernetes.io/instance: supabase +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: supabase-studio + namespace: supabase + labels: + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: knoe-supabase-studio + app.kubernetes.io/instance: supabase + template: + metadata: + labels: + app.kubernetes.io/name: knoe-supabase-studio + app.kubernetes.io/instance: supabase + spec: + serviceAccountName: supabase-studio + containers: + - name: knoe-supabase-studio + image: supabase/studio:2026.02.16-sha-26c615c + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 3000 + env: + - name: DEFAULT_ORGANIZATION_NAME + value: "Default Organization" + - name: DEFAULT_PROJECT_NAME + value: "Default Project" + - name: HOSTNAME + value: "::" + - name: NEXT_ANALYTICS_BACKEND_PROVIDER + value: postgres + - name: NEXT_PUBLIC_ENABLE_LOGS + value: "true" + - name: STUDIO_PORT + value: "3000" + - name: SUPABASE_PUBLIC_URL + value: "https://db.prole.org" + - name: SUPABASE_URL + value: "http://supabase-kong:8000" + - name: STUDIO_PG_META_URL + value: "http://supabase-meta:8080" + - name: POSTGRES_HOST + value: "knoe-db-rw.knoe-db.svc.cluster.local" + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: supabase-db + key: database + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: supabase-db + key: password + - name: PG_META_CRYPTO_KEY + valueFrom: + secretKeyRef: + name: supabase-meta + key: cryptoKey + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: supabase-dashboard + key: openAiApiKey + - name: SUPABASE_ANON_KEY + valueFrom: + secretKeyRef: + name: supabase-jwt + key: anonKey + - name: SUPABASE_SERVICE_KEY + valueFrom: + secretKeyRef: + name: supabase-jwt + key: serviceKey + - name: AUTH_JWT_SECRET + valueFrom: + secretKeyRef: + name: supabase-jwt + key: secret + - name: LOGFLARE_URL + value: "http://supabase-analytics:4000" + - name: LOGFLARE_PUBLIC_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: supabase-analytics + key: publicAccessToken + - name: LOGFLARE_PRIVATE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: supabase-analytics + key: privateAccessToken + - name: EDGE_FUNCTIONS_MANAGEMENT_FOLDER + value: "/home/deno/functions" + volumeMounts: + - name: functions-storage + mountPath: /home/deno/functions + volumes: + - name: functions-storage + persistentVolumeClaim: + claimName: supabase-functions diff --git a/deploy/opentofu/k8s/manifests/knoe/kong-deployment.yaml b/deploy/opentofu/k8s/manifests/knoe/kong-deployment.yaml new file mode 100644 index 0000000..f1cd5f9 --- /dev/null +++ b/deploy/opentofu/k8s/manifests/knoe/kong-deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: knoe-svc-kong + labels: + app: knoe-svc-kong +spec: + replicas: 1 + selector: + matchLabels: + app: knoe-svc-kong + template: + metadata: + labels: + app: knoe-svc-kong + spec: + nodeSelector: + prole.org/node-role: general + containers: + - name: kong + image: kong:3.9 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + name: proxy + - containerPort: 8001 + name: admin + - containerPort: 3022 + name: ssh-proxy + env: + - name: KONG_DATABASE + value: "off" + - name: KONG_PROXY_LISTEN + value: "0.0.0.0:8000" + - name: KONG_ADMIN_LISTEN + value: "0.0.0.0:8001" + - name: KONG_STREAM_LISTEN + value: "0.0.0.0:3022 reuseport backlog=16384" + - name: KONG_DECLARATIVE_CONFIG + value: "/etc/kong/declarative/kong.yml" + - name: KONG_NGINX_WORKER_PROCESSES + value: "1" + - name: KONG_MEM_CACHE_SIZE + value: "64m" + volumeMounts: + - name: kong-config + mountPath: /etc/kong/declarative + readinessProbe: + httpGet: + path: /status + port: admin + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /status + port: admin + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + volumes: + - name: kong-config + configMap: + name: knoe-svc-kong-config diff --git a/deploy/opentofu/k8s/manifests/knoe/kong-service.yaml b/deploy/opentofu/k8s/manifests/knoe/kong-service.yaml new file mode 100644 index 0000000..9a20c6e --- /dev/null +++ b/deploy/opentofu/k8s/manifests/knoe/kong-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: knoe-svc-kong + labels: + app: knoe-svc-kong +spec: + selector: + app: knoe-svc-kong + ports: + - name: proxy + port: 8000 + targetPort: proxy + - name: admin + port: 8001 + targetPort: admin + - name: ssh-proxy + port: 3022 + targetPort: ssh-proxy + type: ClusterIP diff --git a/etc/init_monitoring.sh b/etc/init_monitoring.sh index f80de7e..4dab40c 100755 --- a/etc/init_monitoring.sh +++ b/etc/init_monitoring.sh @@ -634,13 +634,32 @@ EOF ;; esac + local google_client_id="${GRAFANA_GOOGLE_CLIENT_ID:-}" + local google_client_secret="${GRAFANA_GOOGLE_CLIENT_SECRET:-}" + local google_block="" + if [[ -n "$google_client_id" && -n "$google_client_secret" ]]; then + google_block=$(cat < http://{{ include "supabase.oauth2proxy.fullname" . }}:{{ .Values.deployment.oauth2proxy.port }}/*' + url: http://{{ include "supabase.oauth2proxy.fullname" . }}:{{ .Values.deployment.oauth2proxy.port }}/ + routes: + - name: dashboard-all + strip_path: true + paths: + - / + plugins: + - name: cors + {{- else }} - name: dashboard _comment: 'Studio: /* -> http://{{ include "supabase.studio.fullname" . }}:{{ .Values.service.studio.port }}/*' url: http://{{ include "supabase.studio.fullname" . }}:{{ .Values.service.studio.port }}/ @@ -281,11 +293,12 @@ data: strip_path: true paths: - / - {{- if .Values.secret.dashboard }} + {{- if .Values.secret.dashboard }} plugins: - name: cors - name: basic-auth config: hide_credentials: true + {{- end }} {{- end }} {{- end }} diff --git a/supabase/helm/knoe-supabase/templates/secrets/_helpers.tpl b/supabase/helm/knoe-supabase/templates/secrets/_helpers.tpl index a6debd4..228d14d 100644 --- a/supabase/helm/knoe-supabase/templates/secrets/_helpers.tpl +++ b/supabase/helm/knoe-supabase/templates/secrets/_helpers.tpl @@ -60,3 +60,10 @@ Expand the name of the minio secret. {{- define "supabase.secret.minio" -}} {{- printf "%s-minio" (include "supabase.fullname" .) }} {{- end -}} + +{{/* +Expand the name of the oauth2proxy secret. +*/}} +{{- define "supabase.secret.oauth2proxy" -}} +{{- printf "%s-oauth2proxy" (include "supabase.fullname" .) }} +{{- end -}} diff --git a/supabase/helm/knoe-supabase/templates/secrets/oauth2proxy.yaml b/supabase/helm/knoe-supabase/templates/secrets/oauth2proxy.yaml new file mode 100644 index 0000000..6e88991 --- /dev/null +++ b/supabase/helm/knoe-supabase/templates/secrets/oauth2proxy.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.deployment.oauth2proxy.enabled (not .Values.secret.oauth2proxy.secretRef) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "supabase.secret.oauth2proxy" . }} + labels: + {{- include "supabase.labels" . | nindent 4 }} +data: + clientId: {{ .Values.secret.oauth2proxy.clientId | b64enc | quote }} + clientSecret: {{ .Values.secret.oauth2proxy.clientSecret | b64enc | quote }} + cookieSecret: {{ .Values.secret.oauth2proxy.cookieSecret | b64enc | quote }} +{{- end }} diff --git a/supabase/helm/knoe-supabase/templates/studio/_helpers.tpl b/supabase/helm/knoe-supabase/templates/studio/_helpers.tpl index 695a9cc..d8dcbc9 100644 --- a/supabase/helm/knoe-supabase/templates/studio/_helpers.tpl +++ b/supabase/helm/knoe-supabase/templates/studio/_helpers.tpl @@ -41,3 +41,28 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.studio.name }} {{- end }} {{- end }} + +{{/* +oauth2-proxy component helpers +*/}} +{{- define "supabase.oauth2proxy.name" -}} +{{- default (print .Chart.Name "-oauth2proxy") .Values.deployment.oauth2proxy.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "supabase.oauth2proxy.fullname" -}} +{{- if .Values.deployment.oauth2proxy.fullnameOverride }} +{{- .Values.deployment.oauth2proxy.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default (print .Chart.Name "-oauth2proxy") .Values.deployment.oauth2proxy.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "supabase.oauth2proxy.selectorLabels" -}} +app.kubernetes.io/name: {{ include "supabase.oauth2proxy.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/supabase/helm/knoe-supabase/templates/studio/oauth2proxy-deployment.yaml b/supabase/helm/knoe-supabase/templates/studio/oauth2proxy-deployment.yaml new file mode 100644 index 0000000..7211439 --- /dev/null +++ b/supabase/helm/knoe-supabase/templates/studio/oauth2proxy-deployment.yaml @@ -0,0 +1,88 @@ +{{- if .Values.deployment.oauth2proxy.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "supabase.oauth2proxy.fullname" . }} + labels: + {{- include "supabase.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "supabase.oauth2proxy.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "supabase.oauth2proxy.selectorLabels" . | nindent 8 }} + spec: + containers: + - name: oauth2-proxy + image: "{{ .Values.deployment.oauth2proxy.image.repository }}:{{ .Values.deployment.oauth2proxy.image.tag }}" + imagePullPolicy: IfNotPresent + args: + - --provider=oidc + - --oidc-issuer-url=https://accounts.google.com + - --email-domain={{ .Values.deployment.oauth2proxy.emailDomain | default "knoey.com" }} + - --upstream=http://{{ include "supabase.studio.fullname" . }}:{{ .Values.service.studio.port }} + - --http-address=0.0.0.0:{{ .Values.deployment.oauth2proxy.port }} + - --redirect-url={{ .Values.deployment.oauth2proxy.redirectUrl }} + - --cookie-secure=true + - --skip-provider-button=false + - --silence-ping-logging=true + env: + - name: OAUTH2_PROXY_CLIENT_ID + valueFrom: + secretKeyRef: + {{- if .Values.secret.oauth2proxy.secretRef }} + name: {{ .Values.secret.oauth2proxy.secretRef }} + key: {{ .Values.secret.oauth2proxy.secretRefKey.clientId | default "clientId" }} + {{- else }} + name: {{ include "supabase.secret.oauth2proxy" . }} + key: clientId + {{- end }} + - name: OAUTH2_PROXY_CLIENT_SECRET + valueFrom: + secretKeyRef: + {{- if .Values.secret.oauth2proxy.secretRef }} + name: {{ .Values.secret.oauth2proxy.secretRef }} + key: {{ .Values.secret.oauth2proxy.secretRefKey.clientSecret | default "clientSecret" }} + {{- else }} + name: {{ include "supabase.secret.oauth2proxy" . }} + key: clientSecret + {{- end }} + - name: OAUTH2_PROXY_COOKIE_SECRET + valueFrom: + secretKeyRef: + {{- if .Values.secret.oauth2proxy.secretRef }} + name: {{ .Values.secret.oauth2proxy.secretRef }} + key: {{ .Values.secret.oauth2proxy.secretRefKey.cookieSecret | default "cookieSecret" }} + {{- else }} + name: {{ include "supabase.secret.oauth2proxy" . }} + key: cookieSecret + {{- end }} + ports: + - name: http + containerPort: {{ .Values.deployment.oauth2proxy.port }} + protocol: TCP + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + {{- include "supabase.enforcedGeneralNodeSelector" (dict "nodeSelector" dict "enforceGeneralNodeRole" .Values.scheduling.enforceGeneralNodeRole) | nindent 6 }} + {{- include "supabase.enforcedGeneralAffinity" (dict "affinity" dict "enforceGeneralNodeRole" .Values.scheduling.enforceGeneralNodeRole) | nindent 6 }} +{{- end }} diff --git a/supabase/helm/knoe-supabase/templates/studio/oauth2proxy-service.yaml b/supabase/helm/knoe-supabase/templates/studio/oauth2proxy-service.yaml new file mode 100644 index 0000000..c856ed8 --- /dev/null +++ b/supabase/helm/knoe-supabase/templates/studio/oauth2proxy-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.deployment.oauth2proxy.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "supabase.oauth2proxy.fullname" . }} + labels: + {{- include "supabase.labels" . | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "supabase.oauth2proxy.selectorLabels" . | nindent 4 }} + ports: + - name: http + port: {{ .Values.deployment.oauth2proxy.port }} + targetPort: http + protocol: TCP +{{- end }} diff --git a/supabase/helm/knoe-supabase/values-gke.yaml b/supabase/helm/knoe-supabase/values-gke.yaml index cb27663..181989d 100644 --- a/supabase/helm/knoe-supabase/values-gke.yaml +++ b/supabase/helm/knoe-supabase/values-gke.yaml @@ -40,6 +40,31 @@ environment: # Auto-confirm email (Google accounts are pre-verified) GOTRUE_MAILER_AUTOCONFIRM: "true" +# ── Studio: oauth2-proxy for Google Workspace OIDC login ──────────────────── +# Replaces the basic-auth popup with a proper Google login page. +# Credentials are injected at deploy time via secretRef (see conf/gke.cfg: +# DB_OIDC_CLIENT_ID = secretref://db-oidc-client-id +# DB_OIDC_CLIENT_SECRET = secretref://db-oidc-client-secret +# DB_OIDC_COOKIE_SECRET = secretref://db-oidc-cookie-secret) +# +# To create the Google OAuth client: +# GCP Console → APIs & Services → Credentials → Create OAuth Client +# Type: Web application, name: "knoe.dev Supabase Studio" +# Redirect URI: https://db.0.knoe.dev/oauth2/callback +deployment: + oauth2proxy: + enabled: true + emailDomain: "knoey.com" + redirectUrl: "https://db.0.knoe.dev/oauth2/callback" + +secret: + dashboard: ~ # disables basic-auth Kong consumer when oauth2proxy is active + oauth2proxy: + secretRef: "" # set to a pre-existing secret name, or leave blank to use + clientId: "" # DB_OIDC_CLIENT_ID (injected at deploy time via secretref) + clientSecret: "" # DB_OIDC_CLIENT_SECRET + cookieSecret: "" # DB_OIDC_COOKIE_SECRET + # ── Ingress: use GKE-managed ingress class ────────────────────────────────── ingress: studio: diff --git a/supabase/helm/knoe-supabase/values.yaml b/supabase/helm/knoe-supabase/values.yaml index b192c7d..ec7b6a6 100644 --- a/supabase/helm/knoe-supabase/values.yaml +++ b/supabase/helm/knoe-supabase/values.yaml @@ -174,6 +174,25 @@ secret: # user: user # password: password + ## oauth2-proxy credentials for Supabase Studio OIDC login. + ## Used when deployment.oauth2proxy.enabled=true. + ## In production, point secretRef at a pre-existing K8s secret instead of + ## storing plaintext values here. + ## + oauth2proxy: + clientId: "" + clientSecret: "" + cookieSecret: "" + + ## Reference to existing secret (skips creating the Secret resource above) + # secretRef: "" + + ## Map to actual keys inside secretRef if they differ + # secretRefKey: + # clientId: clientId + # clientSecret: clientSecret + # cookieSecret: cookieSecret + scheduling: enforceGeneralNodeRole: true @@ -378,6 +397,25 @@ deployment: volumeMounts: {} volumes: {} resources: {} + + ## oauth2-proxy sits in front of Studio and handles Google Workspace OIDC. + ## When enabled, the Kong dashboard route proxies through oauth2proxy instead + ## of directly to Studio. Disabled by default; enable in values-gke.yaml. + ## + oauth2proxy: + enabled: false + nameOverride: "" + fullnameOverride: "" + image: + repository: quay.io/oauth2-proxy/oauth2-proxy + tag: "v7.7.1" + port: 4180 + ## Google Workspace domain restriction (--email-domain) + emailDomain: "knoey.com" + ## Full redirect URL registered in GCP Console + ## e.g. https://db.0.knoe.dev/oauth2/callback + redirectUrl: "" + vector: enabled: true replicaCount: 1