diff --git a/Makefile b/Makefile index 09bec09..5c75dad 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,11 @@ k3d-knoe-pf: k3d-knoe-down: @./scripts/k3d-knoe-down.sh +k8s/knoe/knoe-ekosystem-sql.yaml: + @echo "Generating ConfigMap from knoe-db/schema/ekosystem*.sql..." + @$(PYTHON) scripts/gen-ekosystem-configmap.py + @echo "✓ k8s/knoe/knoe-ekosystem-sql.yaml updated" + clean: @echo "Cleaning build artifacts..." rm -rf $(BUILD_DIR) $(DIST_DIR) *.spec diff --git a/deploy/gcp/gke/knoe-db.yaml b/deploy/gcp/gke/knoe-db.yaml index a425c16..f47c846 100644 --- a/deploy/gcp/gke/knoe-db.yaml +++ b/deploy/gcp/gke/knoe-db.yaml @@ -133,6 +133,14 @@ spec: - GRANT SELECT, INSERT, UPDATE ON knoe.user TO knoe; - GRANT SELECT, INSERT, UPDATE ON knoe.user_role TO knoe; - GRANT USAGE, SELECT ON SEQUENCE knoe.user_id_seq TO knoe; + # Task 4: align knoe.user with ekosystem user UUIDs + - ALTER TABLE knoe.user ADD COLUMN IF NOT EXISTS ekosystem_uuid text UNIQUE; + postInitApplicationSQLRefs: + configMapRefs: + - name: knoe-ekosystem-sql + key: ekosystem.sql + - name: knoe-ekosystem-sql + key: ekosystem_objects.sql managed: roles: diff --git a/deploy/opentofu/k3s/manifests/knoe/knoe-db.yaml b/deploy/opentofu/k3s/manifests/knoe/knoe-db.yaml index d7e1480..f0b534b 100644 --- a/deploy/opentofu/k3s/manifests/knoe/knoe-db.yaml +++ b/deploy/opentofu/k3s/manifests/knoe/knoe-db.yaml @@ -98,6 +98,14 @@ spec: - GRANT SELECT, INSERT, UPDATE ON knoe.user TO knoe; - GRANT SELECT, INSERT, UPDATE ON knoe.user_role TO knoe; - GRANT USAGE, SELECT ON SEQUENCE knoe.user_id_seq TO knoe; + # Task 4: align knoe.user with ekosystem user UUIDs + - ALTER TABLE knoe.user ADD COLUMN IF NOT EXISTS ekosystem_uuid text UNIQUE; + postInitApplicationSQLRefs: + configMapRefs: + - name: knoe-ekosystem-sql + key: ekosystem.sql + - name: knoe-ekosystem-sql + key: ekosystem_objects.sql managed: roles: diff --git a/k8s/knoe/knoe-db.yaml b/k8s/knoe/knoe-db.yaml index e3cae3b..15a2b76 100644 --- a/k8s/knoe/knoe-db.yaml +++ b/k8s/knoe/knoe-db.yaml @@ -99,6 +99,14 @@ spec: - GRANT USAGE ON SCHEMA storage TO anon; - GRANT USAGE ON SCHEMA graphql_public TO anon; - GRANT anon TO authenticator; + # Task 4: align knoe.user with ekosystem user UUIDs + - ALTER TABLE knoe.user ADD COLUMN IF NOT EXISTS ekosystem_uuid text UNIQUE; + postInitApplicationSQLRefs: + configMapRefs: + - name: knoe-ekosystem-sql + key: ekosystem.sql + - name: knoe-ekosystem-sql + key: ekosystem_objects.sql certificates: serverAltDNSNames: diff --git a/k8s/knoe/knoe-ekosystem-sql.yaml b/k8s/knoe/knoe-ekosystem-sql.yaml new file mode 100644 index 0000000..7e15be8 --- /dev/null +++ b/k8s/knoe/knoe-ekosystem-sql.yaml @@ -0,0 +1,500 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: knoe-ekosystem-sql + namespace: knoe-db +data: + ekosystem.sql: | + -- ============================================================================= + -- knoe.ekosystem — Stable routable base36 UUID system for federated clusters + -- + -- Bit layout (82 bits → 16 base36 chars): + -- [49: timestamp_ms][12: tenant_id][10: shard_id][11: seq] + -- + -- Epoch: 2026-01-01 00:00:00 UTC + -- Range: ~17,839 years (to ~19,865 AD) + -- Tenants: 4,096 (tenant_id 0 = knoe-db root authority) + -- Shards: 1,024 per tenant + -- Seq: 2,048 IDs/ms/shard (~2M IDs/sec/shard) + -- Alphabet: 0-9 a-z (base36 lowercase — safe for DNS labels and PG identifiers) + -- + -- Object naming pattern: p_{16-char-uuid}_{table_name} + -- Prefix is 19 chars. PostgreSQL limit 63 bytes → table_name budget: 44 chars. + -- SQLite and ClickHouse have no meaningful limit. + -- + -- LDAP integration point: knoe.tenants is the authority for access control. + -- Routing: knoe.ekosystem_tenant(table_name) extracts tenant_id from any + -- project-namespaced object name without a registry lookup. + -- ============================================================================= + + -- --------------------------------------------------------------------------- + -- Tenant registry — root of trust for all ekosystems + -- knoe-db is tenant 0 and the sole issuer of tenant UUIDs. + -- --------------------------------------------------------------------------- + CREATE TABLE IF NOT EXISTS knoe.tenants ( + tenant_id smallint PRIMARY KEY CHECK (tenant_id BETWEEN 0 AND 4095), + uuid text UNIQUE, -- 16-char ekosystem UUID; NULL for root + name text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now() + ); + + INSERT INTO knoe.tenants (tenant_id, uuid, name) + VALUES (0, NULL, 'knoe-db') + ON CONFLICT DO NOTHING; + + -- Monotonic counter for the seq field; nextval() is O(1) with no row locking. + -- Wraps modulo 2048 (11 bits) inside the function. + CREATE SEQUENCE IF NOT EXISTS knoe.ekosystem_seq + AS bigint MINVALUE 0 NO CYCLE; + + -- --------------------------------------------------------------------------- + -- knoe.ekosystem_id(tenant_id, shard_id) → 16-char base36 + -- + -- Primary generator. numeric arithmetic handles the 82-bit value safely + -- since PostgreSQL bigint is capped at 63 bits. + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.ekosystem_id( + p_tenant_id integer DEFAULT 0, + p_shard_id integer DEFAULT 0 + ) RETURNS text + LANGUAGE plpgsql + AS $$ + DECLARE + c_epoch_ms constant bigint := 1767225600000; -- 2026-01-01 00:00:00 UTC + c_alphabet constant text := '0123456789abcdefghijklmnopqrstuvwxyz'; + v_ts_ms bigint; + v_seq integer; + v_packed numeric; + v_result text := ''; + v_rem integer; + i integer; + BEGIN + IF p_tenant_id NOT BETWEEN 0 AND 4095 THEN + RAISE EXCEPTION 'tenant_id must be 0–4095, got %', p_tenant_id; + END IF; + IF p_shard_id NOT BETWEEN 0 AND 1023 THEN + RAISE EXCEPTION 'shard_id must be 0–1023, got %', p_shard_id; + END IF; + + v_ts_ms := (extract(epoch from clock_timestamp()) * 1000)::bigint - c_epoch_ms; + v_seq := nextval('knoe.ekosystem_seq')::integer % 2048; + + -- [49: ts_ms | 12: tenant | 10: shard | 11: seq] packed as numeric + -- Power-of-2 multipliers avoid floating-point error in numeric division. + v_packed := v_ts_ms::numeric * 8589934592::numeric -- 2^33 + + p_tenant_id::numeric * 2097152::numeric -- 2^21 + + p_shard_id::numeric * 2048::numeric -- 2^11 + + v_seq::numeric; + + FOR i IN 1..16 LOOP + v_rem := mod(v_packed, 36)::integer; + v_result := substr(c_alphabet, v_rem + 1, 1) || v_result; + v_packed := floor(v_packed / 36); + END LOOP; + + RETURN v_result; + END; + $$; + + -- --------------------------------------------------------------------------- + -- knoe.ekosystem_decode(id) → (tenant_id, shard_id, ts_ms, seq, issued_at) + -- + -- Unpack any UUID back to its constituent fields. IMMUTABLE — safe to index. + -- Primary use: routing queries to the correct ekosystem shard without a + -- registry lookup, and LDAP principal resolution. + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.ekosystem_decode(p_id text) + RETURNS TABLE( + tenant_id integer, + shard_id integer, + ts_ms bigint, + seq integer, + issued_at timestamptz + ) + LANGUAGE plpgsql IMMUTABLE STRICT + AS $$ + DECLARE + c_epoch_ms constant bigint := 1767225600000; + c_alphabet constant text := '0123456789abcdefghijklmnopqrstuvwxyz'; + v_n numeric := 0; + v_pos integer; + i integer; + BEGIN + IF length(p_id) != 16 THEN + RAISE EXCEPTION 'ekosystem UUID must be exactly 16 chars, got %', length(p_id); + END IF; + + FOR i IN 1..16 LOOP + v_pos := position(substr(p_id, i, 1) IN c_alphabet) - 1; + IF v_pos < 0 THEN + RAISE EXCEPTION 'Invalid base36 character at position %: ''%''', + i, substr(p_id, i, 1); + END IF; + v_n := v_n * 36 + v_pos; + END LOOP; + + seq := mod(v_n, 2048)::integer; v_n := floor(v_n / 2048); + shard_id := mod(v_n, 1024)::integer; v_n := floor(v_n / 1024); + tenant_id := mod(v_n, 4096)::integer; v_n := floor(v_n / 4096); + ts_ms := v_n::bigint; + issued_at := to_timestamp((ts_ms + c_epoch_ms) / 1000.0); + + RETURN NEXT; + END; + $$; + + -- --------------------------------------------------------------------------- + -- knoe.ekosystem_tenant(table_name) → tenant_id + -- + -- Extract the owning tenant from a project-namespaced object name without any + -- registry join. Extracts the UUID from position 3 (past 'p_') for 16 chars. + -- + -- Usage in triggers: SELECT knoe.ekosystem_tenant(TG_TABLE_NAME) + -- Usage in routing: SELECT knoe.ekosystem_tenant('p_3k9mxqt2f8vn0r7b_accounts') + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.ekosystem_tenant(p_object_name text) + RETURNS integer + LANGUAGE sql IMMUTABLE STRICT + AS $$ + SELECT tenant_id + FROM knoe.ekosystem_decode(substring(p_object_name FROM 3 FOR 16)); + $$; + + -- --------------------------------------------------------------------------- + -- knoe.register_tenant(name) → (tenant_id, uuid, name) + -- + -- Register a new ekosystem tenant. The UUID is minted by the root authority + -- (tenant 0, shard 0) and becomes the tenant's stable external identifier — + -- used in DNS (db.{uuid}.prole.org) and service-account prefixes. + -- Subsequent IDs generated by the tenant embed their assigned tenant_id. + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.register_tenant(p_name text) + RETURNS TABLE(tenant_id integer, uuid text, name text) + LANGUAGE plpgsql + AS $$ + DECLARE + v_tenant_id integer; + v_uuid text; + BEGIN + SELECT COALESCE(MAX(t.tenant_id), 0) + 1 + INTO v_tenant_id + FROM knoe.tenants t; + + IF v_tenant_id > 4095 THEN + RAISE EXCEPTION 'Tenant capacity exhausted (max 4096 ekosystems)'; + END IF; + + v_uuid := knoe.ekosystem_id(0, 0); -- root authority mints the UUID + + INSERT INTO knoe.tenants (tenant_id, uuid, name) + VALUES (v_tenant_id, v_uuid, p_name) + RETURNING tenants.tenant_id, tenants.uuid, tenants.name; + + RETURN NEXT; + END; + $$; + + -- --------------------------------------------------------------------------- + -- knoe.project_prefix(tenant_id, shard_id) → 'p_{uuid}_' + -- + -- Convenience: generate the full 19-char prefix for a new project namespace. + -- Caller appends the table name: prefix || 'accounts' + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.project_prefix( + p_tenant_id integer DEFAULT 0, + p_shard_id integer DEFAULT 0 + ) RETURNS text + LANGUAGE sql + AS $$ + SELECT 'p_' || knoe.ekosystem_id(p_tenant_id, p_shard_id) || '_'; + $$; + + -- --------------------------------------------------------------------------- + -- Grants (mirrors existing knoe schema pattern) + -- --------------------------------------------------------------------------- + GRANT EXECUTE ON FUNCTION knoe.ekosystem_id(integer, integer) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.ekosystem_decode(text) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.ekosystem_tenant(text) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.register_tenant(text) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.project_prefix(integer, integer) TO knoe; + GRANT SELECT, INSERT ON TABLE knoe.tenants TO knoe; + GRANT USAGE ON SEQUENCE knoe.ekosystem_seq TO knoe; + ekosystem_objects.sql: | + -- ============================================================================= + -- knoe.ekosystem_objects — Modular object kind registry, naming, and + -- cross-ekosystem sharing / grant model + -- + -- Depends on: ekosystem.sql (knoe.tenants, knoe.ekosystem_decode) + -- + -- Object naming convention: + -- {prefix}_{uuid} — standalone objects (project, user, embedding) + -- {prefix}_{uuid}_{name} — compound objects (table, index, view) + -- + -- All prefixes are {letter}_ so ekosystem_tenant() works uniformly: + -- substring(object_name FROM 3 FOR 16) always extracts the UUID. + -- + -- Cross-ekosystem sharing: + -- UUIDs embed the originating tenant_id in bits — ownership is always + -- decodable without a registry join, so imported objects cannot collide + -- even when mixed across ekosystems. knoe.cross_grants records the + -- explicit sharing approvals that LDAP/Samba AD will eventually enforce. + -- ============================================================================= + + -- --------------------------------------------------------------------------- + -- knoe.object_kinds — Modular kind registry + -- Add new kinds here; ekosystem_object() picks them up automatically. + -- --------------------------------------------------------------------------- + CREATE TABLE IF NOT EXISTS knoe.object_kinds ( + kind text PRIMARY KEY, + prefix char(1) NOT NULL UNIQUE CHECK (prefix ~ '^[a-z]$'), + description text NOT NULL + ); + + INSERT INTO knoe.object_kinds (kind, prefix, description) VALUES + ('project', 'p', 'Project namespace — schema or resource root'), + ('user', 'u', 'Service account / PostgreSQL role'), + ('embedding', 'e', 'Vector embedding store'), + ('table', 't', 'Data table'), + ('index', 'x', 'Index or search structure'), + ('view', 'v', 'Logical view or materialised view'), + ('sequence', 's', 'Auto-increment sequence'), + ('function', 'f', 'Stored function or procedure') + ON CONFLICT DO NOTHING; + + GRANT SELECT ON TABLE knoe.object_kinds TO knoe; + + -- --------------------------------------------------------------------------- + -- knoe.ekosystem_object(uuid, kind) → '{prefix}_{uuid}' + -- + -- Returns the canonical identifier for any typed object. + -- Unknown kind raises — register it in knoe.object_kinds first. + -- + -- Compound names (tables, indexes): + -- knoe.ekosystem_object(uuid, 'table') || '_accounts' + -- → 't_3k9mxqt2f8vn0r7b_accounts' + -- + -- The 2-char prefix is the same width for every kind, so + -- ekosystem_tenant() always finds the UUID at position 3. + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.ekosystem_object(p_uuid text, p_kind text) + RETURNS text + LANGUAGE plpgsql STABLE + AS $$ + DECLARE + v_prefix char(1); + BEGIN + SELECT prefix INTO v_prefix FROM knoe.object_kinds WHERE kind = p_kind; + IF NOT FOUND THEN + RAISE EXCEPTION + 'Unknown object kind: %. Register it in knoe.object_kinds first.', p_kind; + END IF; + RETURN v_prefix || '_' || p_uuid; + END; + $$; + + -- --------------------------------------------------------------------------- + -- knoe.ekosystem_kind(object_name) → kind + -- + -- Reverse lookup: given any prefixed object name, return its kind. + -- Returns NULL if the prefix is unregistered (not an error — enables routing + -- filters that skip non-ekosystem objects). + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.ekosystem_kind(p_object_name text) + RETURNS text + LANGUAGE sql STABLE + AS $$ + SELECT kind + FROM knoe.object_kinds + WHERE prefix = substring(p_object_name FROM 1 FOR 1); + $$; + + -- --------------------------------------------------------------------------- + -- knoe.cross_grants — Cross-ekosystem sharing policy + -- + -- Records that grantor_tenant has shared an object with grantee_tenant. + -- This table is the policy source of truth; actual PG GRANT statements + -- and future LDAP/Samba AD group memberships are derived from it. + -- + -- Ownership is always decodable from the UUID itself via ekosystem_decode(), + -- so this table only needs to record the *exceptions* (sharing across + -- tenant boundaries). Within a single ekosystem, use standard PG GRANT. + -- + -- privileges: PostgreSQL privilege names, e.g. '{SELECT}', '{SELECT,INSERT}' + -- expires_at: NULL = permanent until explicitly revoked + -- --------------------------------------------------------------------------- + CREATE TABLE IF NOT EXISTS knoe.cross_grants ( + id text PRIMARY KEY + DEFAULT knoe.ekosystem_id(0, 0), + object_uuid text NOT NULL, + object_kind text NOT NULL REFERENCES knoe.object_kinds(kind), + grantor_tenant smallint NOT NULL REFERENCES knoe.tenants(tenant_id), + grantee_tenant smallint NOT NULL REFERENCES knoe.tenants(tenant_id), + privileges text[] NOT NULL CHECK (array_length(privileges, 1) > 0), + granted_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz, + CONSTRAINT cross_grants_no_self + CHECK (grantor_tenant != grantee_tenant), + CONSTRAINT cross_grants_unique + UNIQUE (object_uuid, object_kind, grantee_tenant) + ); + + CREATE INDEX IF NOT EXISTS cross_grants_grantee_idx + ON knoe.cross_grants (grantee_tenant, object_kind); + + CREATE INDEX IF NOT EXISTS cross_grants_object_idx + ON knoe.cross_grants (object_uuid); + + GRANT SELECT, INSERT, DELETE ON TABLE knoe.cross_grants TO knoe; + + -- --------------------------------------------------------------------------- + -- knoe.grant_object(object_uuid, kind, grantee_tenant, privileges, expires_at) + -- + -- Record that the owning tenant shares an object with another ekosystem. + -- Validates that the caller's tenant actually owns the object (via UUID bits). + -- On conflict: refreshes privileges and expiry (upsert semantics). + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.grant_object( + p_object_uuid text, + p_kind text, + p_grantee_tenant smallint, + p_privileges text[] DEFAULT '{SELECT}', + p_expires_at timestamptz DEFAULT NULL + ) RETURNS knoe.cross_grants + LANGUAGE plpgsql + AS $$ + DECLARE + v_grantor smallint; + v_row knoe.cross_grants; + BEGIN + -- Derive owner from UUID bits — no registry join needed + SELECT tenant_id INTO v_grantor FROM knoe.ekosystem_decode(p_object_uuid); + + IF NOT EXISTS (SELECT 1 FROM knoe.tenants WHERE tenant_id = p_grantee_tenant) THEN + RAISE EXCEPTION 'Grantee tenant % is not a registered ekosystem', p_grantee_tenant; + END IF; + + IF v_grantor = p_grantee_tenant THEN + RAISE EXCEPTION + 'Grant target is the object owner (tenant %). Use standard PG GRANT within an ekosystem.', + v_grantor; + END IF; + + INSERT INTO knoe.cross_grants + (object_uuid, object_kind, grantor_tenant, grantee_tenant, privileges, expires_at) + VALUES + (p_object_uuid, p_kind, v_grantor, p_grantee_tenant, p_privileges, p_expires_at) + ON CONFLICT (object_uuid, object_kind, grantee_tenant) DO UPDATE + SET privileges = EXCLUDED.privileges, + expires_at = EXCLUDED.expires_at, + granted_at = now() + RETURNING * INTO v_row; + + RETURN v_row; + END; + $$; + + -- --------------------------------------------------------------------------- + -- knoe.revoke_object(object_uuid, kind, grantee_tenant) + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.revoke_object( + p_object_uuid text, + p_kind text, + p_grantee_tenant smallint + ) RETURNS boolean + LANGUAGE plpgsql + AS $$ + BEGIN + DELETE FROM knoe.cross_grants + WHERE object_uuid = p_object_uuid + AND object_kind = p_kind + AND grantee_tenant = p_grantee_tenant; + RETURN FOUND; + END; + $$; + + -- --------------------------------------------------------------------------- + -- knoe.can_access(object_uuid, kind, tenant_id) → boolean + -- + -- True when: + -- (a) the tenant owns the object (tenant_id embedded in UUID), OR + -- (b) a non-expired cross-grant exists, OR + -- (c) tenant 0 (knoe-db root) — root has universal read access + -- + -- STABLE: safe in WHERE clauses and on replicas. + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.can_access( + p_object_uuid text, + p_kind text, + p_tenant_id smallint + ) RETURNS boolean + LANGUAGE sql STABLE + AS $$ + SELECT + p_tenant_id = 0 -- root authority + OR + (SELECT tenant_id + FROM knoe.ekosystem_decode(p_object_uuid)) = p_tenant_id -- owner + OR + EXISTS ( + SELECT 1 FROM knoe.cross_grants + WHERE object_uuid = p_object_uuid + AND object_kind = p_kind + AND grantee_tenant = p_tenant_id + AND (expires_at IS NULL OR expires_at > now()) + ); + $$; + + -- --------------------------------------------------------------------------- + -- knoe.my_grants(tenant_id) → active cross-grants visible to this ekosystem + -- Includes both grants received and grants issued by this tenant. + -- --------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION knoe.my_grants(p_tenant_id smallint) + RETURNS TABLE( + direction text, -- 'received' | 'issued' + object_uuid text, + object_kind text, + object_name text, -- prefixed identifier + other_tenant smallint, + privileges text[], + granted_at timestamptz, + expires_at timestamptz + ) + LANGUAGE sql STABLE + AS $$ + SELECT 'received'::text, + g.object_uuid, + g.object_kind, + knoe.ekosystem_object(g.object_uuid, g.object_kind), + g.grantor_tenant, + g.privileges, + g.granted_at, + g.expires_at + FROM knoe.cross_grants g + WHERE g.grantee_tenant = p_tenant_id + AND (g.expires_at IS NULL OR g.expires_at > now()) + + UNION ALL + + SELECT 'issued'::text, + g.object_uuid, + g.object_kind, + knoe.ekosystem_object(g.object_uuid, g.object_kind), + g.grantee_tenant, + g.privileges, + g.granted_at, + g.expires_at + FROM knoe.cross_grants g + WHERE g.grantor_tenant = p_tenant_id + AND (g.expires_at IS NULL OR g.expires_at > now()); + $$; + + -- --------------------------------------------------------------------------- + -- Grants (mirrors existing knoe schema pattern) + -- --------------------------------------------------------------------------- + GRANT EXECUTE ON FUNCTION knoe.ekosystem_object(text, text) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.ekosystem_kind(text) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.grant_object(text, text, smallint, text[], timestamptz) + TO knoe; + GRANT EXECUTE ON FUNCTION knoe.revoke_object(text, text, smallint) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.can_access(text, text, smallint) TO knoe; + GRANT EXECUTE ON FUNCTION knoe.my_grants(smallint) TO knoe; diff --git a/k8s/knoe/kustomization.yaml b/k8s/knoe/kustomization.yaml index f304b11..0840dea 100644 --- a/k8s/knoe/kustomization.yaml +++ b/k8s/knoe/kustomization.yaml @@ -6,6 +6,7 @@ resources: - garage-statefulset.yaml - garage-service.yaml - knoe-db-barman-objectstore.yaml + - knoe-ekosystem-sql.yaml - knoe-db.yaml - knoe-db-postgres-service.yaml - knoe-configmap.yaml diff --git a/knoe/ekosystem.py b/knoe/ekosystem.py new file mode 100644 index 0000000..0cec7dd --- /dev/null +++ b/knoe/ekosystem.py @@ -0,0 +1,158 @@ +"""knoe.ekosystem — Python counterpart to knoe.ekosystem_id() in PostgreSQL. + +Bit layout (82 bits → 16 base36 chars): + [49: timestamp_ms][12: tenant_id][10: shard_id][11: seq] + +Epoch: 2026-01-01 00:00:00 UTC (1767225600000 ms) + +Produces identical output to knoe.ekosystem_id() for the same inputs and +timestamp. Use this when a PostgreSQL connection is unavailable (e.g. from +ClickHouse or SQLite application nodes). +""" + +import threading +import time +from datetime import datetime, timezone +from typing import Optional + +EPOCH_MS = 1767225600000 # 2026-01-01 00:00:00 UTC +ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz" + +# Bit widths +_TS_BITS = 49 +_TENANT_BITS = 12 +_SHARD_BITS = 10 +_SEQ_BITS = 11 +_TOTAL_BITS = _TS_BITS + _TENANT_BITS + _SHARD_BITS + _SEQ_BITS # 82 + +# Shift offsets +_SEQ_SHIFT = 0 +_SHARD_SHIFT = _SEQ_BITS +_TENANT_SHIFT = _SEQ_BITS + _SHARD_BITS +_TS_SHIFT = _SEQ_BITS + _SHARD_BITS + _TENANT_BITS + +# Masks +_SEQ_MASK = (1 << _SEQ_BITS) - 1 +_SHARD_MASK = (1 << _SHARD_BITS) - 1 +_TENANT_MASK = (1 << _TENANT_BITS) - 1 +_TS_MASK = (1 << _TS_BITS) - 1 + +_BASE = len(ALPHABET) +_UUID_LEN = 16 # ceil(82 / log2(36)) = 16 + + +def _to_base36(n: int) -> str: + """Encode integer n as a zero-padded 16-character base36 string.""" + chars = [] + for _ in range(_UUID_LEN): + chars.append(ALPHABET[n % _BASE]) + n //= _BASE + return "".join(reversed(chars)) + + +def _from_base36(s: str) -> int: + """Decode a base36 string to an integer.""" + n = 0 + for ch in s: + n = n * _BASE + ALPHABET.index(ch) + return n + + +class EkosystemID: + """Thread-safe ekosystem UUID generator (Snowflake pattern). + + Args: + tenant_id: Integer tenant identifier (0 = knoe-db root, 1 = prole, …) + shard_id: Integer shard identifier within the tenant (usually 0) + """ + + def __init__(self, tenant_id: int, shard_id: int) -> None: + if not (0 <= tenant_id < (1 << _TENANT_BITS)): + raise ValueError(f"tenant_id must be 0–{(1 << _TENANT_BITS) - 1}") + if not (0 <= shard_id < (1 << _SHARD_BITS)): + raise ValueError(f"shard_id must be 0–{(1 << _SHARD_BITS) - 1}") + self.tenant_id = tenant_id + self.shard_id = shard_id + self._lock = threading.Lock() + self._last_ms: int = -1 + self._seq: int = 0 + + def generate(self) -> str: + """Generate a new ekosystem UUID string (thread-safe).""" + with self._lock: + now_ms = int(time.time() * 1000) - EPOCH_MS + if now_ms < 0: + raise RuntimeError("System clock is before the ekosystem epoch (2026-01-01)") + if now_ms == self._last_ms: + self._seq = (self._seq + 1) & _SEQ_MASK + if self._seq == 0: + # Sequence exhausted — spin until next millisecond + while now_ms == self._last_ms: + now_ms = int(time.time() * 1000) - EPOCH_MS + else: + self._seq = 0 + self._last_ms = now_ms + + bits = ( + ((now_ms & _TS_MASK) << _TS_SHIFT) + | ((self.tenant_id & _TENANT_MASK) << _TENANT_SHIFT) + | ((self.shard_id & _SHARD_MASK) << _SHARD_SHIFT) + | (self._seq & _SEQ_MASK) + ) + return _to_base36(bits) + + def project_prefix(self) -> str: + """Return a project namespace prefix: 'p_{uuid}_'.""" + return f"p_{self.generate()}_" + + def object_name(self, kind: str) -> str: + """Return a namespaced object name: '{kind}_{uuid}'.""" + return f"{kind}_{self.generate()}" + + +def decode(uid: str) -> dict: + """Decode an ekosystem UUID string into its component fields. + + Returns: + dict with keys: tenant_id, shard_id, ts_ms, seq, issued_at (UTC datetime) + """ + if len(uid) != _UUID_LEN: + raise ValueError(f"ekosystem UUID must be {_UUID_LEN} characters, got {len(uid)}") + bits = _from_base36(uid) + ts_ms = (bits >> _TS_SHIFT) & _TS_MASK + tenant_id = (bits >> _TENANT_SHIFT) & _TENANT_MASK + shard_id = (bits >> _SHARD_SHIFT) & _SHARD_MASK + seq = bits & _SEQ_MASK + issued_at = datetime.fromtimestamp((ts_ms + EPOCH_MS) / 1000.0, tz=timezone.utc) + return { + "tenant_id": tenant_id, + "shard_id": shard_id, + "ts_ms": ts_ms, + "seq": seq, + "issued_at": issued_at, + } + + +def can_access(uid: str, kind: str, tenant_id: int, grants: list) -> bool: + """Check whether tenant_id can access object uid of the given kind. + + Args: + uid: ekosystem UUID of the object + kind: object kind string (e.g. 'table', 'embedding') + tenant_id: the requesting tenant's integer ID + grants: list of grant dicts with keys 'object_uuid', 'kind', 'grantee_tenant_id' + + Returns: + True if the object's own tenant matches tenant_id, or a matching grant exists. + """ + info = decode(uid) + if info["tenant_id"] == tenant_id: + return True + for g in grants: + if ( + g.get("object_uuid") == uid + and g.get("kind") == kind + and g.get("grantee_tenant_id") == tenant_id + ): + return True + return False diff --git a/scripts/gen-ekosystem-configmap.py b/scripts/gen-ekosystem-configmap.py new file mode 100644 index 0000000..9787b4c --- /dev/null +++ b/scripts/gen-ekosystem-configmap.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Generate k8s/knoe/knoe-ekosystem-sql.yaml from knoe-db/schema/ekosystem*.sql. + +Run via: make k8s/knoe/knoe-ekosystem-sql.yaml +""" +import os + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCHEMA_DIR = os.path.join(ROOT, "knoe-db", "schema") +OUT = os.path.join(ROOT, "k8s", "knoe", "knoe-ekosystem-sql.yaml") + +SQL_FILES = ["ekosystem.sql", "ekosystem_objects.sql"] + + +def indent(text, spaces=4): + pad = " " * spaces + return "\n".join((pad + line) if line else "" for line in text.splitlines()) + + +lines = [ + "apiVersion: v1", + "kind: ConfigMap", + "metadata:", + " name: knoe-ekosystem-sql", + " namespace: knoe-db", + "data:", +] + +for fname in SQL_FILES: + sql = open(os.path.join(SCHEMA_DIR, fname)).read() + lines.append(f" {fname}: |") + lines.append(indent(sql, 4)) + +with open(OUT, "w") as f: + f.write("\n".join(lines) + "\n") + +print(f"Wrote {OUT} ({os.path.getsize(OUT)} bytes)") diff --git a/tests/test_ekosystem.py b/tests/test_ekosystem.py new file mode 100644 index 0000000..9c49e09 --- /dev/null +++ b/tests/test_ekosystem.py @@ -0,0 +1,197 @@ +"""Tests for knoe.ekosystem — Python ekosystem UUID utility.""" + +import threading +import time + +import pytest + +from knoe.ekosystem import ( + ALPHABET, + EPOCH_MS, + EkosystemID, + _from_base36, + _to_base36, + can_access, + decode, +) + + +# --------------------------------------------------------------------------- +# Base36 encoding round-trip +# --------------------------------------------------------------------------- + + +def test_base36_roundtrip(): + for n in [0, 1, 35, 36, 1000, 2**82 - 1]: + assert _from_base36(_to_base36(n)) == n + + +def test_base36_length(): + assert len(_to_base36(0)) == 16 + assert len(_to_base36(2**82 - 1)) == 16 + + +def test_base36_alphabet(): + assert len(ALPHABET) == 36 + assert ALPHABET == "0123456789abcdefghijklmnopqrstuvwxyz" + + +# --------------------------------------------------------------------------- +# EkosystemID generation +# --------------------------------------------------------------------------- + + +def test_generate_returns_16_chars(): + uid = EkosystemID(1, 0).generate() + assert len(uid) == 16 + + +def test_generate_only_base36_chars(): + uid = EkosystemID(0, 0).generate() + assert all(c in ALPHABET for c in uid) + + +def test_round_trip_tenant_id(): + """Spec-mandated round-trip test.""" + assert decode(EkosystemID(1, 0).generate())["tenant_id"] == 1 + + +def test_round_trip_shard_id(): + assert decode(EkosystemID(0, 5).generate())["shard_id"] == 5 + + +def test_round_trip_tenant_and_shard(): + info = decode(EkosystemID(3, 7).generate()) + assert info["tenant_id"] == 3 + assert info["shard_id"] == 7 + + +def test_issued_at_is_recent(): + from datetime import datetime, timezone + + uid = EkosystemID(1, 0).generate() + info = decode(uid) + now = datetime.now(tz=timezone.utc) + delta = abs((now - info["issued_at"]).total_seconds()) + assert delta < 5, f"issued_at too far from now: {delta}s" + + +def test_sequential_uids_are_unique(): + gen = EkosystemID(1, 0) + uids = [gen.generate() for _ in range(100)] + assert len(set(uids)) == 100 + + +def test_invalid_tenant_id_raises(): + with pytest.raises(ValueError): + EkosystemID(4096, 0) # 2**12 + + +def test_invalid_shard_id_raises(): + with pytest.raises(ValueError): + EkosystemID(0, 1024) # 2**10 + + +# --------------------------------------------------------------------------- +# project_prefix / object_name +# --------------------------------------------------------------------------- + + +def test_project_prefix_format(): + prefix = EkosystemID(1, 0).project_prefix() + assert prefix.startswith("p_") + assert prefix.endswith("_") + assert len(prefix) == 2 + 16 + 1 # 'p_' + uuid + '_' + + +def test_object_name_format(): + name = EkosystemID(1, 0).object_name("t") + assert name.startswith("t_") + assert len(name) == 2 + 16 + + +# --------------------------------------------------------------------------- +# decode +# --------------------------------------------------------------------------- + + +def test_decode_wrong_length_raises(): + with pytest.raises(ValueError): + decode("tooshort") + + +def test_decode_seq_zero_on_first_call(): + uid = EkosystemID(1, 0).generate() + info = decode(uid) + # seq may be 0 on first call within a millisecond + assert 0 <= info["seq"] < 2**11 + + +def test_decode_ts_ms_positive(): + uid = EkosystemID(1, 0).generate() + assert decode(uid)["ts_ms"] > 0 + + +# --------------------------------------------------------------------------- +# Thread safety +# --------------------------------------------------------------------------- + + +def test_thread_safety_no_duplicates(): + gen = EkosystemID(1, 0) + results = [] + lock = threading.Lock() + + def worker(): + uid = gen.generate() + with lock: + results.append(uid) + + threads = [threading.Thread(target=worker) for _ in range(200)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(results) == 200 + assert len(set(results)) == 200, "Duplicate UUIDs generated across threads" + + +# --------------------------------------------------------------------------- +# can_access +# --------------------------------------------------------------------------- + + +def test_can_access_own_tenant(): + uid = EkosystemID(1, 0).generate() + assert can_access(uid, "table", 1, []) + + +def test_can_access_other_tenant_no_grant(): + uid = EkosystemID(1, 0).generate() + assert not can_access(uid, "table", 2, []) + + +def test_can_access_with_grant(): + uid = EkosystemID(1, 0).generate() + grants = [{"object_uuid": uid, "kind": "table", "grantee_tenant_id": 2}] + assert can_access(uid, "table", 2, grants) + + +def test_can_access_grant_wrong_kind(): + uid = EkosystemID(1, 0).generate() + grants = [{"object_uuid": uid, "kind": "embedding", "grantee_tenant_id": 2}] + assert not can_access(uid, "table", 2, grants) + + +# --------------------------------------------------------------------------- +# EPOCH constant +# --------------------------------------------------------------------------- + + +def test_epoch_ms_value(): + """2026-01-01 00:00:00 UTC in milliseconds.""" + from datetime import datetime, timezone + + epoch_dt = datetime(2026, 1, 1, tzinfo=timezone.utc) + assert EPOCH_MS == int(epoch_dt.timestamp() * 1000)