mirror of
https://github.com/dredx/prole.git
synced 2026-09-28 10:54:30 +00:00
Itemized changes:
1. knoe-auth: New cluster-internal KDC and SSO gateway service
- Created etc/init_knoe_auth.sh based on init_kdc.sh with knoe-auth naming
- Namespace defaults to SERVICE_NAMESPACE (knoe-system)
- ConfigMap: knoe-auth-kdc-config, Secret: knoe-auth-secrets
- Legacy cleanup removes old auth/dog/authority deployments
2. Orchestration: knoe-auth initializes before CloudNativePG
- Updated prole.sh to insert init_knoe_auth.sh as step 2 (before CNPG)
- Renumbered all subsequent initialization steps
3. Kong routing: Updated init_kong.sh to route to knoe-auth in SERVICE_NAMESPACE
4. Comment/reference updates for knoe-auth
- Updated init_common_services.sh, init_service_layer.sh, init_kerberos.sh
5. prole-db renamed to knoe-db across the entire codebase
- Renamed prole-db/ directory to knoe-db/
- Renamed all prole-db Kubernetes manifests (deploy/opentofu, k8s/)
- Renamed scripts: docker-root-knoe-db.sh, docker-run-knoe-db.sh, test-cnpg-knoe-db.sh
- Renamed etc/init_prole-db-reset.sh to etc/init_knoe-db-reset.sh
- Renamed etc/prole-db-passwwd.sh to etc/knoe-db-passwwd.sh
- Renamed mock_val counterparts accordingly
- Renamed tests/etc/test_init_prole-db-reset.sh to test_init_knoe-db-reset.sh
- Renamed docs/prole-db-documentation-mcp-architecture.md to knoe-db variant
- Renamed modes/k3d/prole-db/ to modes/k3d/knoe-db/
- Renamed prole-db.iml to knoe-db.iml
6. Configuration updates
- Updated conf/dev, conf/prod, conf/test, conf/service prole.cfg files
- Updated conf/port-mapping.cfg
- Updated etc/prole_cfg.sh and mock_val/prole_cfg.sh
- Updated service/prole.cfg
7. Kubernetes manifests and deploy configuration
- Updated deploy/opentofu/k3s ArgoCD application YAMLs
- Updated kong-configmap.yaml and kustomization.yaml
- Updated k3s/kong-config.yml and prole-resources.yaml
- Updated prole-mssql-db deployment YAMLs
- Updated supabase helm render and deploy scripts
8. Infrastructure and GCP Terraform
- Updated deploy/gcp/terraform: folders, groups, IAM, service-projects
9. Python/installer code updates
- Updated knoe/core: actions, build_context, controller, env, milestones
- Updated knoe/milestone.py
- Updated knoe/ui/screens: cfg, database, database_options, deploy, docker,
navigation, security, services, validate
- Updated knoe.spec, status.py
10. Shell script updates
- Updated etc/: build_db, init_cloudnative_pg, init_cnpg_backup,
init_db_manager, init_forgejo, init_gitlab, init_monitoring, init_openbao,
init_port_forwards, init_postgrest, init_supabase_ports, status
- Updated mock_val/ counterparts for all above scripts
- Updated prole-net/init-prole-dns.sh
- Updated bin/prole-kpf.sh, gitea/deploy.sh, supabase/deploy.sh
11. Test updates
- Updated tests/etc/: test_init_cloudnative_pg*, test_init_cnpg_backup*,
test_init_kdc*, test_init_kerberos*, test_init_kong*, test_prole_cfg*
- Updated tests/installer/: test_actions_helpers, test_cfg_save_kubecontext,
test_controller, test_core_classes, test_milestones, test_milestones_extended,
test_namespace_propagation
- Updated tests/: test_database_options, test_navigation,
test_render_supabase_hostname, test_docker_build_fix,
test_all_prole_home_fixes, silent_install_test, final_test
12. Documentation updates
- Updated docs/: DOCKER-BUILD-FIX, PROLE-CFG-SECRETS, PROLE-HOME-DIRECTORY,
build-system, patent
- Updated scan/network_description.txt
- Updated pom.xml
13. Miscellaneous script updates
- Updated root-level: _adopt_replica_pvcs, _fix_replica_merlin, _import_pi,
_patch_cluster, _prebind_pvcs, _rebind_d002, _rebind_d002b, test_resolve
- Updated scripts/generate_spec.py
Co-authored-by: Junie <junie@jetbrains.com>
350 lines
11 KiB
Python
Executable File
350 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Render a personalized Supabase Helm values file and pre-template manifests
|
|
using the local prole.cfg. Defaults are tailored for Prole's CNPG cluster
|
|
(knoe-db-rw Service) and Traefik ingress.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import configparser
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
import secrets
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
from typing import Any, Dict
|
|
|
|
# Allow imports from the repo root
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
|
|
try:
|
|
from knoe.config import _resolve_secret_value # type: ignore
|
|
except Exception: # pragma: no cover - fallback for minimal environments
|
|
def _resolve_secret_value(value: str | None) -> str:
|
|
return value or ""
|
|
|
|
|
|
def _read_cfg(path: Path) -> configparser.ConfigParser:
|
|
parser = configparser.ConfigParser(interpolation=None)
|
|
try:
|
|
from knoe import prole_conf as prole_conf_mgr # type: ignore
|
|
|
|
files = [p for p in prole_conf_mgr.layered_cfg_files(path) if p.exists()]
|
|
if files:
|
|
parser.read([str(p) for p in files])
|
|
else:
|
|
parser.read(path)
|
|
except Exception:
|
|
parser.read(path)
|
|
return parser
|
|
|
|
|
|
def _cfg_get(cfg: configparser.ConfigParser, section: str, key: str, default: str = "") -> str:
|
|
try:
|
|
raw = cfg.get(section, key, fallback=default)
|
|
except Exception:
|
|
return default
|
|
return _resolve_secret_value((raw or "").strip())
|
|
|
|
|
|
def _first(*values: str, default: str = "") -> str:
|
|
for v in values:
|
|
if v:
|
|
return v
|
|
return default
|
|
|
|
|
|
def _is_placeholder(value: str) -> bool:
|
|
v = (value or "").strip()
|
|
return v.startswith("${") and v.endswith("}")
|
|
|
|
|
|
def _discover_k8s_service_namespace(service_name: str) -> str:
|
|
try:
|
|
raw = subprocess.check_output(["kubectl", "get", "svc", "-A", "-o", "json"], text=True)
|
|
data = json.loads(raw)
|
|
for item in data.get("items", []) or []:
|
|
md = item.get("metadata", {}) or {}
|
|
if md.get("name") == service_name and md.get("namespace"):
|
|
return str(md.get("namespace"))
|
|
except Exception:
|
|
return ""
|
|
return ""
|
|
|
|
|
|
def _ensure_dir(path: Path) -> Path:
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
return path
|
|
|
|
|
|
def _load_or_create_secret(path: Path, length: int = 32) -> str:
|
|
if path.exists():
|
|
return path.read_text().strip()
|
|
val = secrets.token_hex(length)
|
|
path.write_text(val)
|
|
return val
|
|
|
|
|
|
def _b64url(data: bytes) -> str:
|
|
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8")
|
|
|
|
|
|
def _jwt(payload: Dict[str, Any], secret: str) -> str:
|
|
header = {"alg": "HS256", "typ": "JWT"}
|
|
head = _b64url(json.dumps(header, separators=(",", ":")).encode())
|
|
body = _b64url(json.dumps(payload, separators=(",", ":")).encode())
|
|
signing_input = f"{head}.{body}".encode()
|
|
sig = hmac.new(secret.encode(), signing_input, hashlib.sha256).digest()
|
|
return f"{head}.{body}.{_b64url(sig)}"
|
|
|
|
|
|
def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) -> tuple[dict, dict]:
|
|
db_service = os.environ.get("KNOE_DB_SERVICE", "knoe-db-rw")
|
|
|
|
db_ns = _first(
|
|
os.environ.get("KNOE_DB_NAMESPACE", ""),
|
|
_cfg_get(cfg, "Global", "NAMESPACE"),
|
|
os.environ.get("NAMESPACE", ""),
|
|
default="",
|
|
)
|
|
if _is_placeholder(db_ns):
|
|
db_ns = ""
|
|
if not db_ns:
|
|
db_ns = _discover_k8s_service_namespace(db_service) or ""
|
|
if not db_ns:
|
|
db_ns = "default"
|
|
|
|
supabase_ns = _first(
|
|
_cfg_get(cfg, "Supabase", "NAMESPACE"),
|
|
os.environ.get("SUPABASE_NAMESPACE", ""),
|
|
"supabase",
|
|
)
|
|
db_host = f"{db_service}.{db_ns}.svc.cluster.local"
|
|
db_port = _first(
|
|
_cfg_get(cfg, "Global", "DB_HOST_PORT"),
|
|
_cfg_get(cfg, "Inputs", "init_password.db_host_port"),
|
|
"5432",
|
|
)
|
|
db_name = _first(
|
|
_cfg_get(cfg, "Database Creation", "DB_NAME"),
|
|
_cfg_get(cfg, "Global", "DB_NAME"),
|
|
"postgres",
|
|
)
|
|
db_password = _first(
|
|
_cfg_get(cfg, "Inputs", "init_password.db_password"),
|
|
_cfg_get(cfg, "Global", "DB_PASSWORD"),
|
|
os.environ.get("DB_PASSWORD", ""),
|
|
)
|
|
if not db_password:
|
|
raise SystemExit("Database password is required (init_password.db_password or DB_PASSWORD).")
|
|
|
|
gen_dir = _ensure_dir(Path(args.output_dir))
|
|
secrets_dir = _ensure_dir(gen_dir / "secrets")
|
|
|
|
jwt_secret = _first(os.environ.get("SUPABASE_JWT_SECRET", "")) or _load_or_create_secret(
|
|
secrets_dir / "jwt.secret", length=32
|
|
)
|
|
now = int(time.time())
|
|
exp = now + 10 * 365 * 24 * 3600
|
|
anon_payload = {"role": "anon", "iss": "prole-supabase", "iat": now, "exp": exp}
|
|
service_payload = {"role": "service_role", "iss": "prole-supabase", "iat": now, "exp": exp}
|
|
anon_key = _jwt(anon_payload, jwt_secret)
|
|
service_key = _jwt(service_payload, jwt_secret)
|
|
|
|
supabase_hostname = _first(
|
|
_cfg_get(cfg, "User", "supabase_hostname"),
|
|
_cfg_get(cfg, "Global", "supabase_hostname"),
|
|
_cfg_get(cfg, "User", "SUPABASE_HOSTNAME"),
|
|
_cfg_get(cfg, "Global", "SUPABASE_HOSTNAME"),
|
|
os.environ.get("SUPABASE_HOSTNAME", ""),
|
|
os.environ.get("SUPABASE_HOST", ""),
|
|
default="db.prole.org",
|
|
).strip()
|
|
if _is_placeholder(supabase_hostname):
|
|
supabase_hostname = ""
|
|
if not supabase_hostname:
|
|
supabase_hostname = "db.prole.org"
|
|
|
|
# Canonical public URL for Supabase (used for auth callbacks + Studio links).
|
|
public_url = ""
|
|
ingress_host = supabase_hostname
|
|
if "://" in supabase_hostname:
|
|
parsed = urllib.parse.urlparse(supabase_hostname)
|
|
ingress_host = (parsed.hostname or "").strip() or ingress_host
|
|
scheme = (parsed.scheme or "https").strip()
|
|
if parsed.netloc:
|
|
public_url = f"{scheme}://{parsed.netloc}"
|
|
else:
|
|
public_url = f"{scheme}://{ingress_host}"
|
|
else:
|
|
public_url = f"https://{ingress_host}"
|
|
|
|
# Placement: in k3s we run Supabase on the monitoring/service node (merlin) and keep it
|
|
# off the control-plane/db node.
|
|
mode = _first(
|
|
os.environ.get("PROLE_MODE", ""),
|
|
_cfg_get(cfg, "Global", "DEPLOYMENT_MODE"),
|
|
_cfg_get(cfg, "globals", "prole.mode"),
|
|
default="",
|
|
).strip().lower()
|
|
supabase_primary_node = _first(
|
|
os.environ.get("SUPABASE_PRIMARY_NODE", ""),
|
|
_cfg_get(cfg, "Supabase", "PRIMARY_NODE"),
|
|
default="merlin.prole.org",
|
|
).strip()
|
|
if _is_placeholder(supabase_primary_node):
|
|
supabase_primary_node = ""
|
|
supabase_node_selector = (
|
|
{"kubernetes.io/hostname": supabase_primary_node} if supabase_primary_node else {}
|
|
)
|
|
|
|
overlay: dict[str, Any] = {
|
|
"nameOverride": "supabase",
|
|
"fullnameOverride": "supabase",
|
|
"deployment": {
|
|
"db": {"enabled": False},
|
|
"functions": {"enabled": False},
|
|
"vector": {"enabled": False},
|
|
},
|
|
"externalDatabase": {
|
|
"enabled": True,
|
|
"host": db_host,
|
|
"port": int(str(db_port)),
|
|
"database": db_name,
|
|
"ssl": "disable",
|
|
"createAliasService": True,
|
|
"aliasServiceName": "db",
|
|
},
|
|
"environment": {
|
|
"auth": {
|
|
"DB_HOST": db_host,
|
|
"DB_PORT": str(db_port),
|
|
"DB_SSL": "disable",
|
|
"API_EXTERNAL_URL": public_url,
|
|
"GOTRUE_SITE_URL": public_url,
|
|
},
|
|
"analytics": {"DB_HOST": db_host, "DB_PORT": str(db_port)},
|
|
"meta": {"DB_HOST": db_host, "DB_PORT": str(db_port)},
|
|
"studio": {"SUPABASE_PUBLIC_URL": public_url},
|
|
},
|
|
"secret": {
|
|
"db": {"password": db_password, "database": db_name},
|
|
"jwt": {"secret": jwt_secret, "anonKey": anon_key, "serviceKey": service_key},
|
|
},
|
|
"ingress": {
|
|
"enabled": True,
|
|
"className": "traefik",
|
|
"hosts": [
|
|
{"host": ingress_host, "paths": [{"path": "/", "pathType": "Prefix"}]}
|
|
],
|
|
},
|
|
}
|
|
|
|
if mode == "k3s" and supabase_node_selector:
|
|
deployment = overlay.setdefault("deployment", {})
|
|
for component in (
|
|
"analytics",
|
|
"auth",
|
|
"db",
|
|
"functions",
|
|
"imgproxy",
|
|
"kong",
|
|
"meta",
|
|
"minio",
|
|
"realtime",
|
|
"rest",
|
|
"storage",
|
|
"studio",
|
|
"vector",
|
|
):
|
|
component_cfg = deployment.setdefault(component, {})
|
|
component_cfg.setdefault("nodeSelector", {})
|
|
component_cfg["nodeSelector"].update(supabase_node_selector)
|
|
|
|
meta = {
|
|
"supabase_namespace": supabase_ns,
|
|
"db_namespace": db_ns,
|
|
"db_host": db_host,
|
|
}
|
|
return overlay, meta
|
|
|
|
|
|
def render(args: argparse.Namespace) -> None:
|
|
cfg_path = Path(args.config)
|
|
if cfg_path.is_dir():
|
|
cfg_path = cfg_path / "prole.cfg"
|
|
if not cfg_path.exists():
|
|
raise SystemExit(f"Config not found: {cfg_path}")
|
|
|
|
cfg = _read_cfg(cfg_path)
|
|
overlay, meta = _build_overlay(cfg, args)
|
|
|
|
gen_dir = _ensure_dir(Path(args.output_dir))
|
|
values_path = gen_dir / "values.generated.json"
|
|
values_path.write_text(json.dumps(overlay, indent=2))
|
|
|
|
# Write Namespace manifest for Supabase
|
|
k8s_dir = _ensure_dir(Path(args.manifests_dir))
|
|
namespace_manifest = {
|
|
"apiVersion": "v1",
|
|
"kind": "Namespace",
|
|
"metadata": {"name": meta["supabase_namespace"]},
|
|
}
|
|
(k8s_dir / "namespace.yaml").write_text(json.dumps(namespace_manifest))
|
|
|
|
# Render chart to a single manifest file (Helm accepts JSON values)
|
|
rendered_path = k8s_dir / "supabase-helm.yaml"
|
|
cmd = [
|
|
"helm",
|
|
"template",
|
|
"prole-supabase",
|
|
str(REPO_ROOT / "supabase" / "helm" / "prole-supabase"),
|
|
"-n",
|
|
meta["supabase_namespace"],
|
|
"-f",
|
|
str(values_path),
|
|
]
|
|
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
if proc.returncode != 0:
|
|
sys.stderr.write(proc.stderr)
|
|
raise SystemExit(f"Helm template failed (code {proc.returncode})")
|
|
rendered_path.write_text(proc.stdout)
|
|
|
|
summary = {
|
|
"values": str(values_path),
|
|
"manifests": str(rendered_path),
|
|
"db_host": meta["db_host"],
|
|
"supabase_namespace": meta["supabase_namespace"],
|
|
}
|
|
(gen_dir / "manifest-summary.json").write_text(json.dumps(summary, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="Render Supabase Helm chart from prole.cfg")
|
|
parser.add_argument(
|
|
"--config",
|
|
"-c",
|
|
default=os.environ.get("PROLE_CONF", str(REPO_ROOT / "conf")),
|
|
help="Path to prole.cfg or its directory",
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
default=str(REPO_ROOT / "supabase" / "helm" / "generated"),
|
|
help="Directory to write generated values and metadata",
|
|
)
|
|
parser.add_argument(
|
|
"--manifests-dir",
|
|
default=str(REPO_ROOT / "supabase" / "k8s"),
|
|
help="Directory to write rendered Kubernetes manifests",
|
|
)
|
|
args = parser.parse_args()
|
|
render(args)
|