prole/docs/plans/phase2-3-runbook.md

Phase 2–3 runbook — prole.org canary deploy + vector store

Run this after Junie Batch 1 completes and tests are green. Reference: ~/dev/knoe.dev/knoe-db/docs/plans/release-2026-05.md

---

Phase 2 — k3s maintenance window

Estimated downtime: 45–90 min (all services on merlin/gandalf go dark). myrddin (Samba AD DC) survives — Kerberos and DNS stay up during the wipe. Schedule: off-peak. Announce in Slack before starting.

---

Pre-wipe checklist (run from this machine)

# 1. Confirm a recent backup exists
kubectl --context=prole-service-cluster -n knoe-db \
  get scheduledbackup knoe-db-scheduled-backup \
  -o jsonpath='{.status.lastBackupTime}{"\n"}'
# Expect: timestamp within 12h. If older, trigger manually (step 2).

# 2. Force a fresh backup NOW
kubectl --context=prole-service-cluster -n knoe-db \
  annotate cluster knoe-db \
  backup.cnpg.io/force-backup="$(date -u +%Y%m%dT%H%M%SZ)" --overwrite
# Wait ~5 min, then confirm:
kubectl --context=prole-service-cluster -n knoe-db get backup --sort-by=.metadata.creationTimestamp | tail -3

# 3. Capture current DB state (safe-to-lose but useful for diff)
psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \
  sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \
  -c "SELECT rolname FROM pg_roles WHERE rolname NOT LIKE 'pg_%' ORDER BY 1;"

# 4. Note the iSCSI volume status (data survives wipe if external iSCSI)
kubectl --context=prole-service-cluster -n knoe-db get pvc
kubectl --context=prole-service-cluster get pv | grep knoe-db

# 5. Confirm git is clean in both repos
cd ~/dev/prole       && git status --short
cd ~/dev/knoe.dev/knoe-db && git status --short
# Expected: only mock_val unstaged in knoe-db (that's intentional — Junie handles it)

---

Wipe and redeploy

Follow docs/plans/junie/kdc-trust-reset-repeatable.md exactly.

Key steps (summary — brief is authoritative):

cd ~/dev/prole

# 1. Capture pre-reset state
./mock_val/status.sh > /tmp/prole-pre-reset-status.txt 2>&1

# 2. Apply the prole ekosystem CNPG ConfigMap BEFORE the wipe
#    so it's available in the manifest when CNPG bootstraps
kubectl --context=prole-service-cluster apply \
  -f k8s/knoe/knoe-ekosystem-sql.yaml

# 3. Reset k3s (wipes merlin + gandalf; myrddin survives)
./install.sh --mode k3s --reset
# Watch for: KDC realm = KNOE.LOCAL (not PROLE.LOCAL)
# Watch for: "Creating outbound trust principal krbtgt/PROLE.ORG@KNOE.LOCAL"
# Watch for: "Creating inbound trust principal krbtgt/KNOE.LOCAL@PROLE.ORG"

# 4. Samba-side trust (on myrddin)
ssh myrddin.prole.org "sudo ansible-playbook /etc/ansible/knoe-kdc-trust.yml"

# 5. Verify Kerberos cross-realm (from myrddin)
ssh myrddin.prole.org "kinit chrisfu && kvno krbtgt/KNOE.LOCAL@PROLE.ORG"
# Expect: positive kvno, no "Server not found" errors

---

Post-wipe gate checks

# DB reachable
psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \
  sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \
  -c "SELECT 1;"

# Ekosystem functions present (landed via postInitApplicationSQLRefs)
psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \
  sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \
  -c "SELECT routine_name FROM information_schema.routines
      WHERE routine_schema = 'knoe' AND routine_name LIKE 'ekosystem%'
      ORDER BY 1;"
# Expect: ekosystem_decode, ekosystem_id, ekosystem_kind, ekosystem_object,
#         ekosystem_tenant, grant_object, my_grants, project_prefix, register_tenant

# Tenant 0 seeded
psql "host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \
  sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \
  -c "SELECT * FROM knoe.tenants;"
# Expect: (0, NULL, 'knoe-db', <timestamp>)

# Studio loads
curl -sk https://db.prole.org | grep -c "Supabase"

Do not proceed to Phase 3 until all gate checks pass.

---

Phase 3 — Vector store on prole.org

Run these statements in sequence via psql. Copy the UUID output from Step 1 and substitute {PROLE_UUID} in all subsequent steps.

PSQL="psql host=pg.prole.org port=5432 user=chrisfu dbname=knoe-db \
  sslmode=verify-full sslrootcert=$HOME/.knoe/knoe-db-ca.crt"

---

Step 3.1 — Register prole as tenant 1

SELECT * FROM knoe.register_tenant('prole');

Record the output — you need the uuid value for all subsequent steps.

 tenant_id |       uuid       |  name
-----------+------------------+-------
         1 | {PROLE_UUID}     | prole

Export it for the rest of this session:

PROLE_UUID=$(psql "host=pg.prole.org ..." -tAc \
  "SELECT uuid FROM knoe.tenants WHERE name='prole';")
echo "PROLE_UUID=$PROLE_UUID"

---

Step 3.2 — Create the PG role for the prole project

-- Service account for the prole vector project
DO $$ BEGIN
  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'u_' || :'PROLE_UUID') THEN
    EXECUTE format('CREATE ROLE %I LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT',
                   'u_' || :'PROLE_UUID');
  END IF;
END $$;

Or with psql variable substitution:

psql "host=pg.prole.org ..." \
  -v PROLE_UUID="$PROLE_UUID" \
  -c "DO \$\$ BEGIN
        IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'u_' || :'PROLE_UUID') THEN
          EXECUTE format('CREATE ROLE %I LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT',
                         'u_' || :'PROLE_UUID');
        END IF;
      END \$\$;"

---

Step 3.3 — Create the vector schema

-- Schema: p_{PROLE_UUID}
-- Embedding model: mxbai-embed-large-v1 (dim=1024)
-- Follows the knoeledge tenant-onboarding pattern.

\set SCHEMA 'p_' :'PROLE_UUID'

CREATE SCHEMA IF NOT EXISTS :SCHEMA AUTHORIZATION knoe;
GRANT USAGE ON SCHEMA :SCHEMA TO "u_:PROLE_UUID";

-- Chunks: raw text from indexed repositories
CREATE TABLE IF NOT EXISTS :SCHEMA.chunks (
    ekosystem_uuid  text        NOT NULL DEFAULT :'PROLE_UUID',
    id              text        NOT NULL,
    repo            text        NOT NULL,
    path            text        NOT NULL,
    lang            text,
    line_start      integer,
    line_end        integer,
    content         text        NOT NULL,
    header          text        NOT NULL DEFAULT '',
    contributor_id  text        NOT NULL DEFAULT 'system',
    indexed_at      timestamptz NOT NULL DEFAULT now(),
    PRIMARY KEY (id)
);
CREATE INDEX IF NOT EXISTS idx_chunks_repo
    ON :SCHEMA.chunks (repo, path);
CREATE INDEX IF NOT EXISTS idx_chunks_contributor
    ON :SCHEMA.chunks (contributor_id);

-- Per-contributor embeddings
CREATE TABLE IF NOT EXISTS :SCHEMA.embeddings (
    ekosystem_uuid  text        NOT NULL DEFAULT :'PROLE_UUID',
    id              bigserial   PRIMARY KEY,
    chunk_id        text        NOT NULL REFERENCES :SCHEMA.chunks(id) ON DELETE CASCADE,
    contributor_id  text        NOT NULL DEFAULT 'system',
    embedding       vector(1024) NOT NULL,
    created_at      timestamptz NOT NULL DEFAULT now()
);

-- Merged canonical embeddings — this is what MCP searches
CREATE TABLE IF NOT EXISTS :SCHEMA.embeddings_merged (
    ekosystem_uuid        text        NOT NULL DEFAULT :'PROLE_UUID',
    id                    bigserial   PRIMARY KEY,
    chunk_id              text        NOT NULL REFERENCES :SCHEMA.chunks(id) ON DELETE CASCADE,
    embedding             vector(1024) NOT NULL,
    source_contributor_id text        NOT NULL DEFAULT 'system',
    merged_at             timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_merged_hnsw
    ON :SCHEMA.embeddings_merged
    USING hnsw (embedding vector_cosine_ops)
    WITH (m = 16, ef_construction = 64);

GRANT SELECT, INSERT, UPDATE, DELETE
    ON ALL TABLES IN SCHEMA :SCHEMA TO knoe;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA :SCHEMA TO knoe;

---

Step 3.4 — Register in knoeledge catalog (if knoeledge schema exists)

If the knoeledge schema is present on this DB (it lives in knoeserver migrations — check with \dn). If not present, skip this step.

INSERT INTO knoeledge.projects
    (ekosystem_uuid, uuid_tenant_id, short_id,
     pg_schema, pg_role, display_name, description, embed_model, embed_dim)
VALUES (
    :'PROLE_UUID',
    1,                                   -- tenant_id = 1 (prole)
    '01prole',                           -- short_id placeholder
    'p_' || :'PROLE_UUID',
    'u_' || :'PROLE_UUID',
    'prole.org knoeledge',
    'Canary ekosystem — vector store for prole.org repository index',
    'mxbai-embed-large-v1',
    1024
)
ON CONFLICT DO NOTHING;

---

Step 3.5 — Grant prole embedding store to tenant 0 (knoe.dev)

This is the cross-ekosystem share that lets mcp.0.knoe.dev search it.

SELECT knoe.grant_object(
    :'PROLE_UUID',        -- the embedding store UUID
    'embedding',          -- kind
    0::smallint,          -- grantee = knoe.dev (tenant 0)
    '{SELECT}'::text[],
    NULL                  -- permanent until revoked
);

-- Verify
SELECT direction, object_uuid, object_kind, privileges, granted_at
FROM knoe.my_grants(1::smallint);
-- Expect: 1 row, direction='issued', grantee=0

---

Step 3.6 — Index a repo (run knoe-loader)

Pick the target repo. Suggested: the knoe-db repo on git.knoe.dev.

# From a machine with access to the knoe-loader image:
SCHEMA="p_${PROLE_UUID}"
REPO_URL="https://git.knoe.dev/knoe.dev/knoe-db.git"

knoe-loader \
  --db-url "postgresql://knoe@pg.prole.org:5432/knoe-db?sslmode=verify-full&sslrootcert=$HOME/.knoe/knoe-db-ca.crt" \
  --schema  "$SCHEMA" \
  --repo    "$REPO_URL" \
  --model   mxbai-embed-large-v1

# Verify rows loaded
psql "host=pg.prole.org ..." \
  -c "SELECT count(*) FROM ${SCHEMA}.embeddings_merged;"
# Expect: > 0

---

Phase 3 gate

psql "host=pg.prole.org ..." -c "
  SELECT
    (SELECT count(*) FROM p_${PROLE_UUID}.embeddings_merged) AS merged_count,
    (SELECT can_access FROM knoe.can_access('${PROLE_UUID}', 'embedding', 0::smallint)) AS knoedev_can_access,
    (SELECT tenant_id   FROM knoe.tenants WHERE name = 'prole') AS prole_tenant_id;
"
-- Expect: merged_count > 0, knoedev_can_access = true, prole_tenant_id = 1

Capture PROLE_UUID for Junie Batch 2 — it is needed when wiring mcp.0.knoe.dev to include the prole cross-grant in its search.