From 30df7d1c12bbd29218f3899108f1482b8437641a Mon Sep 17 00:00:00 2001 From: chrisfu Date: Sun, 21 Jun 2026 14:42:27 -0700 Subject: [PATCH] preserve: prole-specific OpenBao K8s-auth + Kong secret-bridge + phase2-3 canary runbook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captured for the knoe-db convergence (Phase 2): configure_kubernetes_auth() (OpenBao K8s auth → scoped knoe-jobs tokens vs root), the Kong knoe-secret route (secure-dropbox bridge /secret/* → knoe-jobs:8081), and the prole.org canary-deploy runbook. None of this is in canonical knoe-db — it is prole-staging-specific, not stale. Co-Authored-By: Claude Opus 4.8 --- docs/plans/phase2-3-runbook.md | 317 ++++++++++++++++++ etc/init_openbao.sh | 58 ++++ .../knoe-supabase/templates/kong/config.yaml | 12 + 3 files changed, 387 insertions(+) create mode 100644 docs/plans/phase2-3-runbook.md diff --git a/docs/plans/phase2-3-runbook.md b/docs/plans/phase2-3-runbook.md new file mode 100644 index 0000000..df06abe --- /dev/null +++ b/docs/plans/phase2-3-runbook.md @@ -0,0 +1,317 @@ +# Phase 2–3 runbook — prole.org canary deploy + vector store + +**Run this after Junie Batch 1 completes and tests are green.** +Reference: `~/dev/knoe.dev/knoe-db/docs/plans/release-2026-05.md` + +--- + +## Phase 2 — k3s maintenance window + +**Estimated downtime:** 45–90 min (all services on merlin/gandalf go dark). +**myrddin** (Samba AD DC) survives — Kerberos and DNS stay up during the wipe. +**Schedule:** off-peak. Announce in Slack before starting. + +--- + +### Pre-wipe checklist (run from this machine) + +```bash +# 1. Confirm a recent backup exists +kubectl --context=prole-service-cluster -n knoe-db \ + get scheduledbackup knoe-db-scheduled-backup \ + -o jsonpath='{.status.lastBackupTime}{"\n"}' +# Expect: timestamp within 12h. If older, trigger manually (step 2). + +# 2. Force a fresh backup NOW +kubectl --context=prole-service-cluster -n knoe-db \ + annotate cluster knoe-db \ + backup.cnpg.io/force-backup="$(date -u +%Y%m%dT%H%M%SZ)" --overwrite +# Wait ~5 min, then confirm: +kubectl --context=prole-service-cluster -n knoe-db get backup --sort-by=.metadata.creationTimestamp | tail -3 + +# 3. Capture current DB state (safe-to-lose but useful for diff) +psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \ + sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \ + -c "SELECT rolname FROM pg_roles WHERE rolname NOT LIKE 'pg_%' ORDER BY 1;" + +# 4. Note the iSCSI volume status (data survives wipe if external iSCSI) +kubectl --context=prole-service-cluster -n knoe-db get pvc +kubectl --context=prole-service-cluster get pv | grep knoe-db + +# 5. Confirm git is clean in both repos +cd ~/dev/prole && git status --short +cd ~/dev/knoe.dev/knoe-db && git status --short +# Expected: only mock_val unstaged in knoe-db (that's intentional — Junie handles it) +``` + +--- + +### Wipe and redeploy + +Follow `docs/plans/junie/kdc-trust-reset-repeatable.md` exactly. + +Key steps (summary — brief is authoritative): +```bash +cd ~/dev/prole + +# 1. Capture pre-reset state +./mock_val/status.sh > /tmp/prole-pre-reset-status.txt 2>&1 + +# 2. Apply the prole ekosystem CNPG ConfigMap BEFORE the wipe +# so it's available in the manifest when CNPG bootstraps +kubectl --context=prole-service-cluster apply \ + -f k8s/knoe/knoe-ekosystem-sql.yaml + +# 3. Reset k3s (wipes merlin + gandalf; myrddin survives) +./install.sh --mode k3s --reset +# Watch for: KDC realm = KNOE.LOCAL (not PROLE.LOCAL) +# Watch for: "Creating outbound trust principal krbtgt/PROLE.ORG@KNOE.LOCAL" +# Watch for: "Creating inbound trust principal krbtgt/KNOE.LOCAL@PROLE.ORG" + +# 4. Samba-side trust (on myrddin) +ssh myrddin.prole.org "sudo ansible-playbook /etc/ansible/knoe-kdc-trust.yml" + +# 5. Verify Kerberos cross-realm (from myrddin) +ssh myrddin.prole.org "kinit chrisfu && kvno krbtgt/KNOE.LOCAL@PROLE.ORG" +# Expect: positive kvno, no "Server not found" errors +``` + +--- + +### Post-wipe gate checks + +```bash +# DB reachable +psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \ + sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \ + -c "SELECT 1;" + +# Ekosystem functions present (landed via postInitApplicationSQLRefs) +psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \ + sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \ + -c "SELECT routine_name FROM information_schema.routines + WHERE routine_schema = 'knoe' AND routine_name LIKE 'ekosystem%' + ORDER BY 1;" +# Expect: ekosystem_decode, ekosystem_id, ekosystem_kind, ekosystem_object, +# ekosystem_tenant, grant_object, my_grants, project_prefix, register_tenant + +# Tenant 0 seeded +psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \ + sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \ + -c "SELECT * FROM knoe.tenants;" +# Expect: (0, NULL, 'knoe-db', ) + +# Studio loads +curl -sk https://db.prole.org | grep -c "Supabase" +``` + +**Do not proceed to Phase 3 until all gate checks pass.** + +--- + +## Phase 3 — Vector store on prole.org + +Run these statements in sequence via psql. Copy the UUID output from Step 1 +and substitute `{PROLE_UUID}` in all subsequent steps. + +```bash +PSQL="psql host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \ + sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" +``` + +--- + +### Step 3.1 — Register prole as tenant 1 + +```sql +SELECT * FROM knoe.register_tenant('prole'); +``` + +**Record the output** — you need the `uuid` value for all subsequent steps. + +``` + tenant_id | uuid | name +-----------+------------------+------- + 1 | {PROLE_UUID} | prole +``` + +Export it for the rest of this session: +```bash +PROLE_UUID=$(psql "host=pg.prole.org ..." -tAc \ + "SELECT uuid FROM knoe.tenants WHERE name='prole';") +echo "PROLE_UUID=$PROLE_UUID" +``` + +--- + +### Step 3.2 — Create the PG role for the prole project + +```sql +-- Service account for the prole vector project +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'u_' || :'PROLE_UUID') THEN + EXECUTE format('CREATE ROLE %I LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT', + 'u_' || :'PROLE_UUID'); + END IF; +END $$; +``` + +Or with psql variable substitution: +```bash +psql "host=pg.prole.org ..." \ + -v PROLE_UUID="$PROLE_UUID" \ + -c "DO \$\$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'u_' || :'PROLE_UUID') THEN + EXECUTE format('CREATE ROLE %I LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT', + 'u_' || :'PROLE_UUID'); + END IF; + END \$\$;" +``` + +--- + +### Step 3.3 — Create the vector schema + +```sql +-- Schema: p_{PROLE_UUID} +-- Embedding model: mxbai-embed-large-v1 (dim=1024) +-- Follows the knoeledge tenant-onboarding pattern. + +\set SCHEMA 'p_' :'PROLE_UUID' + +CREATE SCHEMA IF NOT EXISTS :SCHEMA AUTHORIZATION knoe; +GRANT USAGE ON SCHEMA :SCHEMA TO "u_:PROLE_UUID"; + +-- Chunks: raw text from indexed repositories +CREATE TABLE IF NOT EXISTS :SCHEMA.chunks ( + ekosystem_uuid text NOT NULL DEFAULT :'PROLE_UUID', + id text NOT NULL, + repo text NOT NULL, + path text NOT NULL, + lang text, + line_start integer, + line_end integer, + content text NOT NULL, + header text NOT NULL DEFAULT '', + contributor_id text NOT NULL DEFAULT 'system', + indexed_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (id) +); +CREATE INDEX IF NOT EXISTS idx_chunks_repo + ON :SCHEMA.chunks (repo, path); +CREATE INDEX IF NOT EXISTS idx_chunks_contributor + ON :SCHEMA.chunks (contributor_id); + +-- Per-contributor embeddings +CREATE TABLE IF NOT EXISTS :SCHEMA.embeddings ( + ekosystem_uuid text NOT NULL DEFAULT :'PROLE_UUID', + id bigserial PRIMARY KEY, + chunk_id text NOT NULL REFERENCES :SCHEMA.chunks(id) ON DELETE CASCADE, + contributor_id text NOT NULL DEFAULT 'system', + embedding vector(1024) NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- Merged canonical embeddings — this is what MCP searches +CREATE TABLE IF NOT EXISTS :SCHEMA.embeddings_merged ( + ekosystem_uuid text NOT NULL DEFAULT :'PROLE_UUID', + id bigserial PRIMARY KEY, + chunk_id text NOT NULL REFERENCES :SCHEMA.chunks(id) ON DELETE CASCADE, + embedding vector(1024) NOT NULL, + source_contributor_id text NOT NULL DEFAULT 'system', + merged_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_merged_hnsw + ON :SCHEMA.embeddings_merged + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 64); + +GRANT SELECT, INSERT, UPDATE, DELETE + ON ALL TABLES IN SCHEMA :SCHEMA TO knoe; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA :SCHEMA TO knoe; +``` + +--- + +### Step 3.4 — Register in knoeledge catalog (if knoeledge schema exists) + +*If the `knoeledge` schema is present on this DB (it lives in knoeserver migrations — +check with `\dn`). If not present, skip this step.* + +```sql +INSERT INTO knoeledge.projects + (ekosystem_uuid, uuid_tenant_id, short_id, + pg_schema, pg_role, display_name, description, embed_model, embed_dim) +VALUES ( + :'PROLE_UUID', + 1, -- tenant_id = 1 (prole) + '01prole', -- short_id placeholder + 'p_' || :'PROLE_UUID', + 'u_' || :'PROLE_UUID', + 'prole.org knoeledge', + 'Canary ekosystem — vector store for prole.org repository index', + 'mxbai-embed-large-v1', + 1024 +) +ON CONFLICT DO NOTHING; +``` + +--- + +### Step 3.5 — Grant prole embedding store to tenant 0 (knoe.dev) + +This is the cross-ekosystem share that lets mcp.0.knoe.dev search it. + +```sql +SELECT knoe.grant_object( + :'PROLE_UUID', -- the embedding store UUID + 'embedding', -- kind + 0::smallint, -- grantee = knoe.dev (tenant 0) + '{SELECT}'::text[], + NULL -- permanent until revoked +); + +-- Verify +SELECT direction, object_uuid, object_kind, privileges, granted_at +FROM knoe.my_grants(1::smallint); +-- Expect: 1 row, direction='issued', grantee=0 +``` + +--- + +### Step 3.6 — Index a repo (run knoe-loader) + +Pick the target repo. Suggested: the knoe-db repo on git.knoe.dev. +```bash +# From a machine with access to the knoe-loader image: +SCHEMA="p_${PROLE_UUID}" +REPO_URL="https://git.knoe.dev/knoe.dev/knoe-db.git" + +knoe-loader \ + --db-url "postgresql://knoe@pg.prole.org:5432/knoe-db?sslmode=verify-full&sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \ + --schema "$SCHEMA" \ + --repo "$REPO_URL" \ + --model mxbai-embed-large-v1 + +# Verify rows loaded +psql "host=pg.prole.org ..." \ + -c "SELECT count(*) FROM ${SCHEMA}.embeddings_merged;" +# Expect: > 0 +``` + +--- + +### Phase 3 gate + +```bash +psql "host=pg.prole.org ..." -c " + SELECT + (SELECT count(*) FROM p_${PROLE_UUID}.embeddings_merged) AS merged_count, + (SELECT can_access FROM knoe.can_access('${PROLE_UUID}', 'embedding', 0::smallint)) AS knoedev_can_access, + (SELECT tenant_id FROM knoe.tenants WHERE name = 'prole') AS prole_tenant_id; +" +-- Expect: merged_count > 0, knoedev_can_access = true, prole_tenant_id = 1 +``` + +**Capture `PROLE_UUID` for Junie Batch 2** — it is needed when wiring +`mcp.0.knoe.dev` to include the prole cross-grant in its search. diff --git a/etc/init_openbao.sh b/etc/init_openbao.sh index e1ece01..bf7a5dc 100755 --- a/etc/init_openbao.sh +++ b/etc/init_openbao.sh @@ -822,6 +822,61 @@ init_openbao_kv_and_store_admin_key() { fi } +configure_kubernetes_auth() { + # Enable + configure the OpenBao Kubernetes auth method so in-cluster workloads + # (knoe-jobs, the OpenBao→secure-dropbox bridge) can exchange their + # ServiceAccount JWT for a short-lived, scoped token — instead of using the dev + # root token. Runs via `kubectl exec` inside the OpenBao pod, where the `bao` + # CLI, the cluster CA, and a SA token are all already mounted. + local ns="$OPENBAO_RESOURCE_NAMESPACE" + local svc_ns="${SERVICE_NAMESPACE:-knoe-system}" + local role="${OPENBAO_K8S_ROLE:-knoe-jobs}" + local sa="${OPENBAO_K8S_SA:-knoe-jobs}" + local token + token=$(cat "$root_token_file" 2>/dev/null || echo "root") + + local pod + pod=$(kubectl -n "$ns" get pod -l app="$OPENBAO_NAME" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$pod" ]]; then + echo "WARN: OpenBao pod not found in '$ns'; skipping kubernetes auth setup." >&2 + return 0 + fi + + echo "Configuring OpenBao Kubernetes auth (role: $role, SA: $svc_ns/$sa) ..." + # Note: \${KUBERNETES_SERVICE_*} are escaped so they expand *inside* the pod; + # $token/$role/$sa/$svc_ns expand here in the installer shell. + if ! kubectl -n "$ns" exec -i "$pod" -- sh -s </dev/null | grep -q '^kubernetes/' || bao auth enable kubernetes + +# Trust this cluster, using the pod's own SA token + CA as the reviewer. +bao write auth/kubernetes/config \ + kubernetes_host="https://\${KUBERNETES_SERVICE_HOST}:\${KUBERNETES_SERVICE_PORT}" \ + kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ + token_reviewer_jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token + +# Least-privilege policy: read-only on the knoe KV tree. +echo 'path "kv/data/knoe/*" { capabilities = ["read"] }' | bao policy write knoe-jobs - + +# Bind the knoe-jobs ServiceAccount in the service namespace to that policy. +bao write auth/kubernetes/role/${role} \ + bound_service_account_names="${sa}" \ + bound_service_account_namespaces="${svc_ns}" \ + policies="knoe-jobs" \ + ttl="1h" +EOF + then + echo "WARN: OpenBao kubernetes auth setup failed (non-fatal)." >&2 + return 0 + fi + echo "OpenBao Kubernetes auth configured (role '$role' → policy 'knoe-jobs')." +} + # Removed: prompt_admin_password_and_apply_secret (no AD admin secret required) # status command implementation wrapped in a function to avoid top-level 'local' @@ -1034,6 +1089,9 @@ case "$ACTION" in wait_for_openbao_http 20 "$local_url" init_openbao_kv_and_store_admin_key "$local_url" fi + # Enable Kubernetes auth so in-cluster workloads (knoe-jobs secure-dropbox + # bridge) get scoped, short-lived tokens instead of the dev root token. + configure_kubernetes_auth pf_ns="${OPENBAO_NAMESPACE:-default}" if should_skip_deployment; then pf_ns="$PRIMARY_SERVICE_NAMESPACE" diff --git a/supabase/helm/knoe-supabase/templates/kong/config.yaml b/supabase/helm/knoe-supabase/templates/kong/config.yaml index fb0609e..283cea5 100644 --- a/supabase/helm/knoe-supabase/templates/kong/config.yaml +++ b/supabase/helm/knoe-supabase/templates/kong/config.yaml @@ -102,6 +102,18 @@ data: plugins: - name: cors {{- end }} + {{- if .Values.knoeJobs.enabled }} + - name: knoe-secret + _comment: "Secure-dropbox bridge: /secret/* -> knoe-jobs.knoe-system:8081. No key-auth/acl: knoe-jobs validates the knoe-auth Bearer JWT itself; strip_path:false so /secret/retrieve reaches the handler." + url: http://knoe-jobs.knoe-system.svc.cluster.local:8081 + routes: + - name: knoe-secret-all + strip_path: false + paths: + - /secret/ + plugins: + - name: cors + {{- end }} {{- if .Values.deployment.auth.enabled }} - name: auth-v1-open url: http://{{ include "supabase.auth.fullname" . }}:{{ .Values.service.auth.port }}/verify