prole/k8s/knoe/knoe-ekosystem-sql.yaml
chrisfu d114801758 feat: wire ekosystem UUID system into CNPG manifests (Tasks 1, 3, 4)
Task 1 — ConfigMap + CNPG wiring:
- Add k8s/knoe/knoe-ekosystem-sql.yaml: ConfigMap embedding ekosystem.sql
  and ekosystem_objects.sql for CNPG postInitApplicationSQLRefs
- Add scripts/gen-ekosystem-configmap.py: generation script to keep the
  ConfigMap in sync with knoe-db/schema/ekosystem*.sql source files
- Add Makefile target: make k8s/knoe/knoe-ekosystem-sql.yaml
- Wire postInitApplicationSQLRefs into all three CNPG cluster manifests:
    k8s/knoe/knoe-db.yaml (k3s / prole-service-context production)
    deploy/gcp/gke/knoe-db.yaml (GKE)
    deploy/opentofu/k3s/manifests/knoe/knoe-db.yaml (OpenTofu k3s)
- Add knoe-ekosystem-sql.yaml to k8s/knoe/kustomization.yaml

Task 3 — Python counterpart utility:
- Add knoe/ekosystem.py: thread-safe EkosystemID generator matching the
  PostgreSQL bit layout [49:ts_ms|12:tenant|10:shard|11:seq], with
  decode() and can_access() helpers
- Add tests/test_ekosystem.py: 23 tests covering base36 encoding,
  round-trips, thread safety, can_access, and the spec round-trip assertion

Task 4 — knoe.user ekosystem_uuid column:
- Add ALTER TABLE knoe.user ADD COLUMN IF NOT EXISTS ekosystem_uuid text UNIQUE
  to postInitSQL in all three CNPG manifests

Task 2 (register prole tenant) requires a live DB connection — manual step.
Task 5 (LDAP/AD reconciler) is design-only per spec.

Co-authored-by: Junie <junie@jetbrains.com>
2026-05-30 00:26:48 -07:00

501 lines
22 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

apiVersion: v1
kind: ConfigMap
metadata:
name: knoe-ekosystem-sql
namespace: knoe-db
data:
ekosystem.sql: |
-- =============================================================================
-- knoe.ekosystem — Stable routable base36 UUID system for federated clusters
--
-- Bit layout (82 bits → 16 base36 chars):
-- [49: timestamp_ms][12: tenant_id][10: shard_id][11: seq]
--
-- Epoch: 2026-01-01 00:00:00 UTC
-- Range: ~17,839 years (to ~19,865 AD)
-- Tenants: 4,096 (tenant_id 0 = knoe-db root authority)
-- Shards: 1,024 per tenant
-- Seq: 2,048 IDs/ms/shard (~2M IDs/sec/shard)
-- Alphabet: 0-9 a-z (base36 lowercase — safe for DNS labels and PG identifiers)
--
-- Object naming pattern: p_{16-char-uuid}_{table_name}
-- Prefix is 19 chars. PostgreSQL limit 63 bytes → table_name budget: 44 chars.
-- SQLite and ClickHouse have no meaningful limit.
--
-- LDAP integration point: knoe.tenants is the authority for access control.
-- Routing: knoe.ekosystem_tenant(table_name) extracts tenant_id from any
-- project-namespaced object name without a registry lookup.
-- =============================================================================
-- ---------------------------------------------------------------------------
-- Tenant registry — root of trust for all ekosystems
-- knoe-db is tenant 0 and the sole issuer of tenant UUIDs.
-- ---------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS knoe.tenants (
tenant_id smallint PRIMARY KEY CHECK (tenant_id BETWEEN 0 AND 4095),
uuid text UNIQUE, -- 16-char ekosystem UUID; NULL for root
name text NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT now()
);
INSERT INTO knoe.tenants (tenant_id, uuid, name)
VALUES (0, NULL, 'knoe-db')
ON CONFLICT DO NOTHING;
-- Monotonic counter for the seq field; nextval() is O(1) with no row locking.
-- Wraps modulo 2048 (11 bits) inside the function.
CREATE SEQUENCE IF NOT EXISTS knoe.ekosystem_seq
AS bigint MINVALUE 0 NO CYCLE;
-- ---------------------------------------------------------------------------
-- knoe.ekosystem_id(tenant_id, shard_id) → 16-char base36
--
-- Primary generator. numeric arithmetic handles the 82-bit value safely
-- since PostgreSQL bigint is capped at 63 bits.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.ekosystem_id(
p_tenant_id integer DEFAULT 0,
p_shard_id integer DEFAULT 0
) RETURNS text
LANGUAGE plpgsql
AS $$
DECLARE
c_epoch_ms constant bigint := 1767225600000; -- 2026-01-01 00:00:00 UTC
c_alphabet constant text := '0123456789abcdefghijklmnopqrstuvwxyz';
v_ts_ms bigint;
v_seq integer;
v_packed numeric;
v_result text := '';
v_rem integer;
i integer;
BEGIN
IF p_tenant_id NOT BETWEEN 0 AND 4095 THEN
RAISE EXCEPTION 'tenant_id must be 04095, got %', p_tenant_id;
END IF;
IF p_shard_id NOT BETWEEN 0 AND 1023 THEN
RAISE EXCEPTION 'shard_id must be 01023, got %', p_shard_id;
END IF;
v_ts_ms := (extract(epoch from clock_timestamp()) * 1000)::bigint - c_epoch_ms;
v_seq := nextval('knoe.ekosystem_seq')::integer % 2048;
-- [49: ts_ms | 12: tenant | 10: shard | 11: seq] packed as numeric
-- Power-of-2 multipliers avoid floating-point error in numeric division.
v_packed := v_ts_ms::numeric * 8589934592::numeric -- 2^33
+ p_tenant_id::numeric * 2097152::numeric -- 2^21
+ p_shard_id::numeric * 2048::numeric -- 2^11
+ v_seq::numeric;
FOR i IN 1..16 LOOP
v_rem := mod(v_packed, 36)::integer;
v_result := substr(c_alphabet, v_rem + 1, 1) || v_result;
v_packed := floor(v_packed / 36);
END LOOP;
RETURN v_result;
END;
$$;
-- ---------------------------------------------------------------------------
-- knoe.ekosystem_decode(id) → (tenant_id, shard_id, ts_ms, seq, issued_at)
--
-- Unpack any UUID back to its constituent fields. IMMUTABLE — safe to index.
-- Primary use: routing queries to the correct ekosystem shard without a
-- registry lookup, and LDAP principal resolution.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.ekosystem_decode(p_id text)
RETURNS TABLE(
tenant_id integer,
shard_id integer,
ts_ms bigint,
seq integer,
issued_at timestamptz
)
LANGUAGE plpgsql IMMUTABLE STRICT
AS $$
DECLARE
c_epoch_ms constant bigint := 1767225600000;
c_alphabet constant text := '0123456789abcdefghijklmnopqrstuvwxyz';
v_n numeric := 0;
v_pos integer;
i integer;
BEGIN
IF length(p_id) != 16 THEN
RAISE EXCEPTION 'ekosystem UUID must be exactly 16 chars, got %', length(p_id);
END IF;
FOR i IN 1..16 LOOP
v_pos := position(substr(p_id, i, 1) IN c_alphabet) - 1;
IF v_pos < 0 THEN
RAISE EXCEPTION 'Invalid base36 character at position %: ''%''',
i, substr(p_id, i, 1);
END IF;
v_n := v_n * 36 + v_pos;
END LOOP;
seq := mod(v_n, 2048)::integer; v_n := floor(v_n / 2048);
shard_id := mod(v_n, 1024)::integer; v_n := floor(v_n / 1024);
tenant_id := mod(v_n, 4096)::integer; v_n := floor(v_n / 4096);
ts_ms := v_n::bigint;
issued_at := to_timestamp((ts_ms + c_epoch_ms) / 1000.0);
RETURN NEXT;
END;
$$;
-- ---------------------------------------------------------------------------
-- knoe.ekosystem_tenant(table_name) → tenant_id
--
-- Extract the owning tenant from a project-namespaced object name without any
-- registry join. Extracts the UUID from position 3 (past 'p_') for 16 chars.
--
-- Usage in triggers: SELECT knoe.ekosystem_tenant(TG_TABLE_NAME)
-- Usage in routing: SELECT knoe.ekosystem_tenant('p_3k9mxqt2f8vn0r7b_accounts')
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.ekosystem_tenant(p_object_name text)
RETURNS integer
LANGUAGE sql IMMUTABLE STRICT
AS $$
SELECT tenant_id
FROM knoe.ekosystem_decode(substring(p_object_name FROM 3 FOR 16));
$$;
-- ---------------------------------------------------------------------------
-- knoe.register_tenant(name) → (tenant_id, uuid, name)
--
-- Register a new ekosystem tenant. The UUID is minted by the root authority
-- (tenant 0, shard 0) and becomes the tenant's stable external identifier —
-- used in DNS (db.{uuid}.prole.org) and service-account prefixes.
-- Subsequent IDs generated by the tenant embed their assigned tenant_id.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.register_tenant(p_name text)
RETURNS TABLE(tenant_id integer, uuid text, name text)
LANGUAGE plpgsql
AS $$
DECLARE
v_tenant_id integer;
v_uuid text;
BEGIN
SELECT COALESCE(MAX(t.tenant_id), 0) + 1
INTO v_tenant_id
FROM knoe.tenants t;
IF v_tenant_id > 4095 THEN
RAISE EXCEPTION 'Tenant capacity exhausted (max 4096 ekosystems)';
END IF;
v_uuid := knoe.ekosystem_id(0, 0); -- root authority mints the UUID
INSERT INTO knoe.tenants (tenant_id, uuid, name)
VALUES (v_tenant_id, v_uuid, p_name)
RETURNING tenants.tenant_id, tenants.uuid, tenants.name;
RETURN NEXT;
END;
$$;
-- ---------------------------------------------------------------------------
-- knoe.project_prefix(tenant_id, shard_id) → 'p_{uuid}_'
--
-- Convenience: generate the full 19-char prefix for a new project namespace.
-- Caller appends the table name: prefix || 'accounts'
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.project_prefix(
p_tenant_id integer DEFAULT 0,
p_shard_id integer DEFAULT 0
) RETURNS text
LANGUAGE sql
AS $$
SELECT 'p_' || knoe.ekosystem_id(p_tenant_id, p_shard_id) || '_';
$$;
-- ---------------------------------------------------------------------------
-- Grants (mirrors existing knoe schema pattern)
-- ---------------------------------------------------------------------------
GRANT EXECUTE ON FUNCTION knoe.ekosystem_id(integer, integer) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.ekosystem_decode(text) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.ekosystem_tenant(text) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.register_tenant(text) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.project_prefix(integer, integer) TO knoe;
GRANT SELECT, INSERT ON TABLE knoe.tenants TO knoe;
GRANT USAGE ON SEQUENCE knoe.ekosystem_seq TO knoe;
ekosystem_objects.sql: |
-- =============================================================================
-- knoe.ekosystem_objects — Modular object kind registry, naming, and
-- cross-ekosystem sharing / grant model
--
-- Depends on: ekosystem.sql (knoe.tenants, knoe.ekosystem_decode)
--
-- Object naming convention:
-- {prefix}_{uuid} — standalone objects (project, user, embedding)
-- {prefix}_{uuid}_{name} — compound objects (table, index, view)
--
-- All prefixes are {letter}_ so ekosystem_tenant() works uniformly:
-- substring(object_name FROM 3 FOR 16) always extracts the UUID.
--
-- Cross-ekosystem sharing:
-- UUIDs embed the originating tenant_id in bits — ownership is always
-- decodable without a registry join, so imported objects cannot collide
-- even when mixed across ekosystems. knoe.cross_grants records the
-- explicit sharing approvals that LDAP/Samba AD will eventually enforce.
-- =============================================================================
-- ---------------------------------------------------------------------------
-- knoe.object_kinds — Modular kind registry
-- Add new kinds here; ekosystem_object() picks them up automatically.
-- ---------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS knoe.object_kinds (
kind text PRIMARY KEY,
prefix char(1) NOT NULL UNIQUE CHECK (prefix ~ '^[a-z]$'),
description text NOT NULL
);
INSERT INTO knoe.object_kinds (kind, prefix, description) VALUES
('project', 'p', 'Project namespace — schema or resource root'),
('user', 'u', 'Service account / PostgreSQL role'),
('embedding', 'e', 'Vector embedding store'),
('table', 't', 'Data table'),
('index', 'x', 'Index or search structure'),
('view', 'v', 'Logical view or materialised view'),
('sequence', 's', 'Auto-increment sequence'),
('function', 'f', 'Stored function or procedure')
ON CONFLICT DO NOTHING;
GRANT SELECT ON TABLE knoe.object_kinds TO knoe;
-- ---------------------------------------------------------------------------
-- knoe.ekosystem_object(uuid, kind) → '{prefix}_{uuid}'
--
-- Returns the canonical identifier for any typed object.
-- Unknown kind raises — register it in knoe.object_kinds first.
--
-- Compound names (tables, indexes):
-- knoe.ekosystem_object(uuid, 'table') || '_accounts'
-- → 't_3k9mxqt2f8vn0r7b_accounts'
--
-- The 2-char prefix is the same width for every kind, so
-- ekosystem_tenant() always finds the UUID at position 3.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.ekosystem_object(p_uuid text, p_kind text)
RETURNS text
LANGUAGE plpgsql STABLE
AS $$
DECLARE
v_prefix char(1);
BEGIN
SELECT prefix INTO v_prefix FROM knoe.object_kinds WHERE kind = p_kind;
IF NOT FOUND THEN
RAISE EXCEPTION
'Unknown object kind: %. Register it in knoe.object_kinds first.', p_kind;
END IF;
RETURN v_prefix || '_' || p_uuid;
END;
$$;
-- ---------------------------------------------------------------------------
-- knoe.ekosystem_kind(object_name) → kind
--
-- Reverse lookup: given any prefixed object name, return its kind.
-- Returns NULL if the prefix is unregistered (not an error — enables routing
-- filters that skip non-ekosystem objects).
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.ekosystem_kind(p_object_name text)
RETURNS text
LANGUAGE sql STABLE
AS $$
SELECT kind
FROM knoe.object_kinds
WHERE prefix = substring(p_object_name FROM 1 FOR 1);
$$;
-- ---------------------------------------------------------------------------
-- knoe.cross_grants — Cross-ekosystem sharing policy
--
-- Records that grantor_tenant has shared an object with grantee_tenant.
-- This table is the policy source of truth; actual PG GRANT statements
-- and future LDAP/Samba AD group memberships are derived from it.
--
-- Ownership is always decodable from the UUID itself via ekosystem_decode(),
-- so this table only needs to record the *exceptions* (sharing across
-- tenant boundaries). Within a single ekosystem, use standard PG GRANT.
--
-- privileges: PostgreSQL privilege names, e.g. '{SELECT}', '{SELECT,INSERT}'
-- expires_at: NULL = permanent until explicitly revoked
-- ---------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS knoe.cross_grants (
id text PRIMARY KEY
DEFAULT knoe.ekosystem_id(0, 0),
object_uuid text NOT NULL,
object_kind text NOT NULL REFERENCES knoe.object_kinds(kind),
grantor_tenant smallint NOT NULL REFERENCES knoe.tenants(tenant_id),
grantee_tenant smallint NOT NULL REFERENCES knoe.tenants(tenant_id),
privileges text[] NOT NULL CHECK (array_length(privileges, 1) > 0),
granted_at timestamptz NOT NULL DEFAULT now(),
expires_at timestamptz,
CONSTRAINT cross_grants_no_self
CHECK (grantor_tenant != grantee_tenant),
CONSTRAINT cross_grants_unique
UNIQUE (object_uuid, object_kind, grantee_tenant)
);
CREATE INDEX IF NOT EXISTS cross_grants_grantee_idx
ON knoe.cross_grants (grantee_tenant, object_kind);
CREATE INDEX IF NOT EXISTS cross_grants_object_idx
ON knoe.cross_grants (object_uuid);
GRANT SELECT, INSERT, DELETE ON TABLE knoe.cross_grants TO knoe;
-- ---------------------------------------------------------------------------
-- knoe.grant_object(object_uuid, kind, grantee_tenant, privileges, expires_at)
--
-- Record that the owning tenant shares an object with another ekosystem.
-- Validates that the caller's tenant actually owns the object (via UUID bits).
-- On conflict: refreshes privileges and expiry (upsert semantics).
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.grant_object(
p_object_uuid text,
p_kind text,
p_grantee_tenant smallint,
p_privileges text[] DEFAULT '{SELECT}',
p_expires_at timestamptz DEFAULT NULL
) RETURNS knoe.cross_grants
LANGUAGE plpgsql
AS $$
DECLARE
v_grantor smallint;
v_row knoe.cross_grants;
BEGIN
-- Derive owner from UUID bits — no registry join needed
SELECT tenant_id INTO v_grantor FROM knoe.ekosystem_decode(p_object_uuid);
IF NOT EXISTS (SELECT 1 FROM knoe.tenants WHERE tenant_id = p_grantee_tenant) THEN
RAISE EXCEPTION 'Grantee tenant % is not a registered ekosystem', p_grantee_tenant;
END IF;
IF v_grantor = p_grantee_tenant THEN
RAISE EXCEPTION
'Grant target is the object owner (tenant %). Use standard PG GRANT within an ekosystem.',
v_grantor;
END IF;
INSERT INTO knoe.cross_grants
(object_uuid, object_kind, grantor_tenant, grantee_tenant, privileges, expires_at)
VALUES
(p_object_uuid, p_kind, v_grantor, p_grantee_tenant, p_privileges, p_expires_at)
ON CONFLICT (object_uuid, object_kind, grantee_tenant) DO UPDATE
SET privileges = EXCLUDED.privileges,
expires_at = EXCLUDED.expires_at,
granted_at = now()
RETURNING * INTO v_row;
RETURN v_row;
END;
$$;
-- ---------------------------------------------------------------------------
-- knoe.revoke_object(object_uuid, kind, grantee_tenant)
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.revoke_object(
p_object_uuid text,
p_kind text,
p_grantee_tenant smallint
) RETURNS boolean
LANGUAGE plpgsql
AS $$
BEGIN
DELETE FROM knoe.cross_grants
WHERE object_uuid = p_object_uuid
AND object_kind = p_kind
AND grantee_tenant = p_grantee_tenant;
RETURN FOUND;
END;
$$;
-- ---------------------------------------------------------------------------
-- knoe.can_access(object_uuid, kind, tenant_id) → boolean
--
-- True when:
-- (a) the tenant owns the object (tenant_id embedded in UUID), OR
-- (b) a non-expired cross-grant exists, OR
-- (c) tenant 0 (knoe-db root) — root has universal read access
--
-- STABLE: safe in WHERE clauses and on replicas.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.can_access(
p_object_uuid text,
p_kind text,
p_tenant_id smallint
) RETURNS boolean
LANGUAGE sql STABLE
AS $$
SELECT
p_tenant_id = 0 -- root authority
OR
(SELECT tenant_id
FROM knoe.ekosystem_decode(p_object_uuid)) = p_tenant_id -- owner
OR
EXISTS (
SELECT 1 FROM knoe.cross_grants
WHERE object_uuid = p_object_uuid
AND object_kind = p_kind
AND grantee_tenant = p_tenant_id
AND (expires_at IS NULL OR expires_at > now())
);
$$;
-- ---------------------------------------------------------------------------
-- knoe.my_grants(tenant_id) → active cross-grants visible to this ekosystem
-- Includes both grants received and grants issued by this tenant.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION knoe.my_grants(p_tenant_id smallint)
RETURNS TABLE(
direction text, -- 'received' | 'issued'
object_uuid text,
object_kind text,
object_name text, -- prefixed identifier
other_tenant smallint,
privileges text[],
granted_at timestamptz,
expires_at timestamptz
)
LANGUAGE sql STABLE
AS $$
SELECT 'received'::text,
g.object_uuid,
g.object_kind,
knoe.ekosystem_object(g.object_uuid, g.object_kind),
g.grantor_tenant,
g.privileges,
g.granted_at,
g.expires_at
FROM knoe.cross_grants g
WHERE g.grantee_tenant = p_tenant_id
AND (g.expires_at IS NULL OR g.expires_at > now())
UNION ALL
SELECT 'issued'::text,
g.object_uuid,
g.object_kind,
knoe.ekosystem_object(g.object_uuid, g.object_kind),
g.grantee_tenant,
g.privileges,
g.granted_at,
g.expires_at
FROM knoe.cross_grants g
WHERE g.grantor_tenant = p_tenant_id
AND (g.expires_at IS NULL OR g.expires_at > now());
$$;
-- ---------------------------------------------------------------------------
-- Grants (mirrors existing knoe schema pattern)
-- ---------------------------------------------------------------------------
GRANT EXECUTE ON FUNCTION knoe.ekosystem_object(text, text) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.ekosystem_kind(text) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.grant_object(text, text, smallint, text[], timestamptz)
TO knoe;
GRANT EXECUTE ON FUNCTION knoe.revoke_object(text, text, smallint) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.can_access(text, text, smallint) TO knoe;
GRANT EXECUTE ON FUNCTION knoe.my_grants(smallint) TO knoe;