prole/knoe/core/prod_config.py

579 lines
23 KiB
Python

"""Production configuration model + API helpers for installer UI."""
from dataclasses import asdict, dataclass, field
import copy
import io
import re
from typing import Any
import yaml
DOMAIN_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$")
HOST_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$")
@dataclass(slots=True)
class ProdCloudConfig:
provider: str = "gcp"
projectId: str = ""
region: str = "us-central1"
clusterName: str = "knoe-prod"
vpcMode: str = "managed"
vpcName: str | None = None
subnetName: str | None = None
artifactRegistry: str = ""
dnsZone: str = "knoe-dev-zone"
@dataclass(slots=True)
class ProdKubernetesConfig:
namespace: str = "ecosystem-0"
@dataclass(slots=True)
class ProdDatabaseConfig:
clusterName: str = "knoe-db"
postgresVersion: str = "16"
instances: int = 3
storageClass: str = "premium-rwo"
storageSizeGi: int = 100
pgdataStorageClass: str = "premium-rwo"
walStorageClass: str = "premium-rwo"
pgdataSizeGi: int = 100
walSizeGi: int = 25
appDatabase: str = "knoey"
metaDatabase: str = "knoe_meta"
appUser: str = "knoey_app"
adminUser: str = "knoe_admin"
@dataclass(slots=True)
class ProdBackupsConfig:
backupBucket: str = "knoe-0-backups"
walBucket: str = "knoe-0-wal"
retentionDays: int = 14
backupProvider: str = "gcs" # "gcs" | "garage"
workloadIdentitySA: str = "cnpg-backup"
@dataclass(slots=True)
class ProdAuthConfig:
provider: str = "google-workspace-oidc"
issuer: str = "https://accounts.google.com"
clientId: str = "secretref://google-oidc-client-id"
clientSecret: str = "secretref://google-oidc-client-secret"
bootstrapAdminEmail: str = "admin@knoey.com"
googleClientId: str = "" # transient — not persisted to knoe.cfg
googleClientSecret: str = "" # transient — not persisted to knoe.cfg
googleHostedDomain: str = "" # blank = any Google account; set to restrict Workspace domain
@dataclass(slots=True)
class ProdRoutingConfig:
frontdoorHost: str = "knoey.com"
platformDomain: str = "knoe.dev"
apiHost: str = "api.knoe.dev"
studioHost: str = "db.0.knoe.dev"
gitHost: str = "git.knoe.dev"
tlsMode: str = "managed"
@dataclass(slots=True)
class ProdMigrationConfig:
sourceEnvironment: str = "knoe.org"
mode: str = "snapshot-restore"
sourceHost: str = "knoe-local-db.knoe.org"
sourcePort: int = 5432
sourceDatabase: str = "knoey"
sourceUser: str = "replication_user"
sourcePasswordRef: str = "secretref://local-source-db-password"
continuousUntilCutover: bool = False
@dataclass(slots=True)
class ProdMetadata:
ecosystemId: int = 0
name: str = "knoey-root"
environment: str = "production"
@dataclass(slots=True)
class KnoeProductionConfig:
kind: str = "KnoeProductionConfig"
metadata: ProdMetadata = field(default_factory=ProdMetadata)
cloud: ProdCloudConfig = field(default_factory=ProdCloudConfig)
kubernetes: ProdKubernetesConfig = field(default_factory=ProdKubernetesConfig)
database: ProdDatabaseConfig = field(default_factory=ProdDatabaseConfig)
backups: ProdBackupsConfig = field(default_factory=ProdBackupsConfig)
auth: ProdAuthConfig = field(default_factory=ProdAuthConfig)
routing: ProdRoutingConfig = field(default_factory=ProdRoutingConfig)
migration: ProdMigrationConfig = field(default_factory=ProdMigrationConfig)
def to_document(self) -> dict[str, Any]:
return {
"kind": self.kind,
"metadata": asdict(self.metadata),
"spec": {
"cloud": asdict(self.cloud),
"kubernetes": asdict(self.kubernetes),
"database": asdict(self.database),
"backups": asdict(self.backups),
"auth": asdict(self.auth),
"routing": asdict(self.routing),
"migration": asdict(self.migration),
},
}
def default_prod_config() -> KnoeProductionConfig:
return KnoeProductionConfig()
def _str_or_empty(value: Any) -> str:
return str(value).strip() if value is not None else ""
def _is_domain(value: str) -> bool:
return bool(DOMAIN_RE.match(value.lower()))
def _is_host(value: str) -> bool:
return bool(HOST_RE.match(value.lower()))
def _validate_positive_int(value: Any, field: str, errors: list[str]) -> int:
try:
parsed = int(value)
except Exception:
errors.append(f"{field} must be a positive number")
return 0
if parsed <= 0:
errors.append(f"{field} must be a positive number")
return parsed
def _coerce_bool(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return bool(value)
text = _str_or_empty(value).lower()
return text in {"1", "true", "yes", "on"}
def _coerce_int(value: Any, default: int) -> int:
try:
text = _str_or_empty(value)
if text == "":
return default
return int(text)
except Exception:
return 0
def build_config(payload: dict[str, Any] | None = None) -> KnoeProductionConfig:
cfg = default_prod_config()
src = payload or {}
metadata = src.get("metadata") or {}
spec = src.get("spec") or {}
cloud = spec.get("cloud") or {}
k8s = spec.get("kubernetes") or {}
db = spec.get("database") or {}
backups = spec.get("backups") or {}
auth = spec.get("auth") or {}
routing = spec.get("routing") or {}
migration = spec.get("migration") or {}
cfg.kind = _str_or_empty(src.get("kind")) or cfg.kind
cfg.metadata.ecosystemId = _coerce_int(
metadata.get("ecosystemId", cfg.metadata.ecosystemId), cfg.metadata.ecosystemId
)
cfg.metadata.name = _str_or_empty(metadata.get("name")) or cfg.metadata.name
cfg.metadata.environment = (
_str_or_empty(metadata.get("environment")) or cfg.metadata.environment
)
cfg.cloud.provider = _str_or_empty(cloud.get("provider")) or cfg.cloud.provider
cfg.cloud.projectId = _str_or_empty(cloud.get("projectId"))
cfg.cloud.region = _str_or_empty(cloud.get("region")) or cfg.cloud.region
cfg.cloud.clusterName = _str_or_empty(cloud.get("clusterName")) or cfg.cloud.clusterName
cfg.cloud.vpcMode = _str_or_empty(cloud.get("vpcMode")) or cfg.cloud.vpcMode
cfg.cloud.vpcName = _str_or_empty(cloud.get("vpcName")) or None
cfg.cloud.subnetName = _str_or_empty(cloud.get("subnetName")) or None
cfg.cloud.artifactRegistry = _str_or_empty(cloud.get("artifactRegistry"))
cfg.cloud.dnsZone = _str_or_empty(cloud.get("dnsZone")) or cfg.cloud.dnsZone
cfg.kubernetes.namespace = _str_or_empty(k8s.get("namespace")) or cfg.kubernetes.namespace
cfg.database.clusterName = _str_or_empty(db.get("clusterName")) or cfg.database.clusterName
cfg.database.postgresVersion = _str_or_empty(db.get("postgresVersion")) or cfg.database.postgresVersion
cfg.database.instances = _coerce_int(db.get("instances", cfg.database.instances), cfg.database.instances)
cfg.database.storageClass = _str_or_empty(db.get("storageClass")) or cfg.database.storageClass
cfg.database.storageSizeGi = _coerce_int(
db.get("storageSizeGi", cfg.database.storageSizeGi), cfg.database.storageSizeGi
)
cfg.database.pgdataStorageClass = (
_str_or_empty(db.get("pgdataStorageClass"))
or _str_or_empty(db.get("storageClass"))
or cfg.database.pgdataStorageClass
)
cfg.database.walStorageClass = (
_str_or_empty(db.get("walStorageClass"))
or _str_or_empty(db.get("pgdataStorageClass"))
or _str_or_empty(db.get("storageClass"))
or cfg.database.walStorageClass
)
cfg.database.pgdataSizeGi = _coerce_int(
db.get("pgdataSizeGi", db.get("storageSizeGi", cfg.database.pgdataSizeGi)),
cfg.database.pgdataSizeGi,
)
cfg.database.walSizeGi = _coerce_int(
db.get("walSizeGi", max(10, int(cfg.database.pgdataSizeGi * 0.25))),
cfg.database.walSizeGi,
)
cfg.database.appDatabase = _str_or_empty(db.get("appDatabase")) or cfg.database.appDatabase
cfg.database.metaDatabase = _str_or_empty(db.get("metaDatabase")) or cfg.database.metaDatabase
cfg.database.appUser = _str_or_empty(db.get("appUser")) or cfg.database.appUser
cfg.database.adminUser = _str_or_empty(db.get("adminUser")) or cfg.database.adminUser
cfg.backups.backupBucket = _str_or_empty(backups.get("backupBucket")) or cfg.backups.backupBucket
cfg.backups.walBucket = _str_or_empty(backups.get("walBucket")) or cfg.backups.walBucket
cfg.backups.retentionDays = _coerce_int(
backups.get("retentionDays", cfg.backups.retentionDays), cfg.backups.retentionDays
)
cfg.backups.backupProvider = _str_or_empty(backups.get("backupProvider")) or cfg.backups.backupProvider
cfg.backups.workloadIdentitySA = (
_str_or_empty(backups.get("workloadIdentitySA")) or cfg.backups.workloadIdentitySA
)
cfg.auth.provider = _str_or_empty(auth.get("provider")) or cfg.auth.provider
cfg.auth.issuer = _str_or_empty(auth.get("issuer")) or cfg.auth.issuer
cfg.auth.clientId = _str_or_empty(auth.get("clientId")) or cfg.auth.clientId
cfg.auth.clientSecret = _str_or_empty(auth.get("clientSecret")) or cfg.auth.clientSecret
cfg.auth.bootstrapAdminEmail = (
_str_or_empty(auth.get("bootstrapAdminEmail")) or cfg.auth.bootstrapAdminEmail
)
cfg.auth.googleClientId = _str_or_empty(auth.get("googleClientId"))
cfg.auth.googleClientSecret = _str_or_empty(auth.get("googleClientSecret"))
cfg.auth.googleHostedDomain = _str_or_empty(auth.get("googleHostedDomain"))
cfg.routing.frontdoorHost = _str_or_empty(routing.get("frontdoorHost")) or cfg.routing.frontdoorHost
cfg.routing.platformDomain = _str_or_empty(routing.get("platformDomain")) or cfg.routing.platformDomain
cfg.routing.apiHost = _str_or_empty(routing.get("apiHost")) or cfg.routing.apiHost
cfg.routing.studioHost = _str_or_empty(routing.get("studioHost")) or cfg.routing.studioHost
cfg.routing.gitHost = _str_or_empty(routing.get("gitHost")) or cfg.routing.gitHost
cfg.routing.tlsMode = _str_or_empty(routing.get("tlsMode")) or cfg.routing.tlsMode
cfg.migration.sourceEnvironment = (
_str_or_empty(migration.get("sourceEnvironment")) or cfg.migration.sourceEnvironment
)
cfg.migration.mode = _str_or_empty(migration.get("mode")) or cfg.migration.mode
cfg.migration.sourceHost = _str_or_empty(migration.get("sourceHost")) or cfg.migration.sourceHost
cfg.migration.sourcePort = _coerce_int(
migration.get("sourcePort", cfg.migration.sourcePort), cfg.migration.sourcePort
)
cfg.migration.sourceDatabase = _str_or_empty(migration.get("sourceDatabase")) or cfg.migration.sourceDatabase
cfg.migration.sourceUser = _str_or_empty(migration.get("sourceUser")) or cfg.migration.sourceUser
cfg.migration.sourcePasswordRef = (
_str_or_empty(migration.get("sourcePasswordRef")) or cfg.migration.sourcePasswordRef
)
cfg.migration.continuousUntilCutover = _coerce_bool(
migration.get("continuousUntilCutover", cfg.migration.continuousUntilCutover)
)
return cfg
def validate_prod_config(config: KnoeProductionConfig) -> tuple[list[str], list[str]]:
errors: list[str] = []
warnings: list[str] = []
if config.metadata.ecosystemId != 0:
errors.append("metadata.ecosystemId must be fixed to 0")
required_fields = {
"metadata.name": config.metadata.name,
"metadata.environment": config.metadata.environment,
"spec.cloud.provider": config.cloud.provider,
"spec.cloud.projectId": config.cloud.projectId,
"spec.cloud.region": config.cloud.region,
"spec.cloud.clusterName": config.cloud.clusterName,
"spec.kubernetes.namespace": config.kubernetes.namespace,
"spec.database.clusterName": config.database.clusterName,
"spec.database.postgresVersion": config.database.postgresVersion,
"spec.database.storageClass": config.database.storageClass,
"spec.database.pgdataStorageClass": config.database.pgdataStorageClass,
"spec.database.walStorageClass": config.database.walStorageClass,
"spec.database.appDatabase": config.database.appDatabase,
"spec.database.metaDatabase": config.database.metaDatabase,
"spec.database.appUser": config.database.appUser,
"spec.database.adminUser": config.database.adminUser,
"spec.backups.backupBucket": config.backups.backupBucket,
"spec.backups.walBucket": config.backups.walBucket,
"spec.auth.provider": config.auth.provider,
"spec.auth.issuer": config.auth.issuer,
"spec.auth.clientId": config.auth.clientId,
"spec.auth.clientSecret": config.auth.clientSecret,
"spec.auth.bootstrapAdminEmail": config.auth.bootstrapAdminEmail,
"spec.routing.frontdoorHost": config.routing.frontdoorHost,
"spec.routing.platformDomain": config.routing.platformDomain,
"spec.routing.apiHost": config.routing.apiHost,
"spec.routing.studioHost": config.routing.studioHost,
"spec.routing.gitHost": config.routing.gitHost,
"spec.migration.sourceEnvironment": config.migration.sourceEnvironment,
"spec.migration.sourceHost": config.migration.sourceHost,
"spec.migration.sourceDatabase": config.migration.sourceDatabase,
"spec.migration.sourceUser": config.migration.sourceUser,
"spec.migration.sourcePasswordRef": config.migration.sourcePasswordRef,
}
for key, value in required_fields.items():
if not _str_or_empty(value):
errors.append(f"{key} is required")
config.database.instances = _validate_positive_int(
config.database.instances, "spec.database.instances", errors
)
config.database.storageSizeGi = _validate_positive_int(
config.database.storageSizeGi, "spec.database.storageSizeGi", errors
)
config.database.pgdataSizeGi = _validate_positive_int(
config.database.pgdataSizeGi, "spec.database.pgdataSizeGi", errors
)
config.database.walSizeGi = _validate_positive_int(
config.database.walSizeGi, "spec.database.walSizeGi", errors
)
config.backups.retentionDays = _validate_positive_int(
config.backups.retentionDays, "spec.backups.retentionDays", errors
)
config.migration.sourcePort = _validate_positive_int(
config.migration.sourcePort, "spec.migration.sourcePort", errors
)
if config.kubernetes.namespace != "ecosystem-0":
errors.append("spec.kubernetes.namespace must be ecosystem-0")
for field, value in (
("spec.routing.frontdoorHost", config.routing.frontdoorHost),
("spec.routing.platformDomain", config.routing.platformDomain),
("spec.routing.apiHost", config.routing.apiHost),
("spec.routing.studioHost", config.routing.studioHost),
("spec.routing.gitHost", config.routing.gitHost),
("spec.migration.sourceEnvironment", config.migration.sourceEnvironment),
):
if _str_or_empty(value) and not _is_domain(_str_or_empty(value)):
errors.append(f"{field} must be a valid domain")
if _str_or_empty(config.migration.sourceHost) and not _is_host(config.migration.sourceHost):
errors.append("spec.migration.sourceHost must be a valid host")
if "@" not in _str_or_empty(config.auth.bootstrapAdminEmail):
errors.append("spec.auth.bootstrapAdminEmail must be a valid email")
if config.cloud.provider.lower() != "gcp":
warnings.append("spec.cloud.provider is expected to be gcp for production defaults")
if config.auth.provider.lower() != "google-workspace-oidc":
warnings.append("spec.auth.provider is expected to be google-workspace-oidc")
return errors, warnings
def canonical_yaml(config: KnoeProductionConfig) -> str:
doc = config.to_document()
stream = io.StringIO()
yaml.safe_dump(doc, stream, sort_keys=False, default_flow_style=False)
return stream.getvalue().strip() + "\n"
def opentofu_vars(config: KnoeProductionConfig) -> dict[str, Any]:
return {
# GCP identity (populated from conf/prod/gcp.cfg via installer)
"gcp_project_id": config.cloud.projectId,
"gcp_region": config.cloud.region,
"ecosystem_id": config.metadata.ecosystemId,
"environment": config.metadata.environment,
"project_id": config.cloud.projectId,
"region": config.cloud.region,
"cluster_name": config.cloud.clusterName,
"namespace": config.kubernetes.namespace,
"cnpg_cluster_name": config.database.clusterName,
"postgres_version": config.database.postgresVersion,
"db_instances": config.database.instances,
"db_storage_class": config.database.storageClass,
"db_storage_size_gi": config.database.storageSizeGi,
"db_pgdata_storage_class": config.database.pgdataStorageClass,
"db_wal_storage_class": config.database.walStorageClass,
"db_pgdata_size_gi": config.database.pgdataSizeGi,
"db_wal_size_gi": config.database.walSizeGi,
"app_database": config.database.appDatabase,
"meta_database": config.database.metaDatabase,
"app_user": config.database.appUser,
"admin_user": config.database.adminUser,
"backup_bucket": config.backups.backupBucket,
"wal_bucket": config.backups.walBucket,
"backup_retention_days": config.backups.retentionDays,
"backup_provider": config.backups.backupProvider,
"workload_identity_sa": config.backups.workloadIdentitySA,
"oidc_issuer": config.auth.issuer,
"oidc_client_id_ref": config.auth.clientId,
"oidc_client_secret_ref": config.auth.clientSecret,
"bootstrap_admin_email": config.auth.bootstrapAdminEmail,
"google_hosted_domain": config.auth.googleHostedDomain,
"frontdoor_host": config.routing.frontdoorHost,
"platform_domain": config.routing.platformDomain,
"supabase_api_host": config.routing.apiHost,
"supabase_studio_host": config.routing.studioHost,
"gitlab_host": config.routing.gitHost,
"tls_mode": config.routing.tlsMode,
"migration_source_environment": config.migration.sourceEnvironment,
"migration_mode": config.migration.mode,
"migration_source_host": config.migration.sourceHost,
"migration_source_port": config.migration.sourcePort,
"migration_source_database": config.migration.sourceDatabase,
"migration_source_user": config.migration.sourceUser,
"migration_source_password_ref": config.migration.sourcePasswordRef,
"migration_continuous_until_cutover": config.migration.continuousUntilCutover,
}
def install_py_plan(config: KnoeProductionConfig) -> list[str]:
return [
"Validate production config and required secrets",
"Prepare GCP project, VPC, DNS and artifact registry bindings",
"Prepare Kubernetes namespace ecosystem-0 and RBAC/service accounts",
"Provision CloudNativePG cluster and bootstrap app/meta databases",
"Configure backup/WAL buckets and retention policy",
"Configure Google Workspace OIDC integration and bootstrap admin",
"Configure routing, TLS, and frontdoor domain mappings",
"Prepare migration artifacts from source environment",
"Generate OpenTofu plan and apply production pipeline",
]
def to_api_payload(config: KnoeProductionConfig) -> dict[str, Any]:
return config.to_document()
def from_api_payload(payload: dict[str, Any] | None) -> KnoeProductionConfig:
return build_config(payload)
class ProdConfigApi:
"""In-process API facade used by the installer UI.
Provides the same contract as:
- GET /api/install/prod-config
- PUT /api/install/prod-config
- POST /api/install/plan
- POST /api/install/apply
- GET /api/install/status
- GET /api/install/logs
"""
def __init__(self):
self._payload = to_api_payload(default_prod_config())
self._status: dict[str, Any] = {
"phase": "idle",
"state": "ready",
"message": "Waiting for plan",
}
self._logs: list[str] = []
def get_prod_config(self) -> dict[str, Any]:
return copy.deepcopy(self._payload)
def put_prod_config(self, payload: dict[str, Any]) -> dict[str, Any]:
cfg = from_api_payload(payload)
errors, warnings = validate_prod_config(cfg)
doc = to_api_payload(cfg)
self._payload = doc
return {
"ok": not errors,
"config": copy.deepcopy(doc),
"errors": errors,
"warnings": warnings,
}
def post_plan(self, payload: dict[str, Any]) -> dict[str, Any]:
cfg = from_api_payload(payload)
errors, warnings = validate_prod_config(cfg)
if errors:
self._status = {
"phase": "plan",
"state": "error",
"message": "Plan failed validation",
}
return {
"ok": False,
"errors": errors,
"warnings": warnings,
"yaml": "",
"opentofuVars": {},
"installPlan": [],
}
doc = to_api_payload(cfg)
self._payload = copy.deepcopy(doc)
yaml_preview = canonical_yaml(cfg)
tf_vars = opentofu_vars(cfg)
plan = install_py_plan(cfg)
self._status = {
"phase": "plan",
"state": "planned",
"message": "Plan generated",
}
self._logs.append("[plan] production plan generated")
return {
"ok": True,
"errors": errors,
"warnings": warnings,
"yaml": yaml_preview,
"opentofuVars": tf_vars,
"installPlan": plan,
}
def post_apply(self, payload: dict[str, Any]) -> dict[str, Any]:
plan_result = self.post_plan(payload)
if not plan_result.get("ok"):
self._status = {
"phase": "apply",
"state": "error",
"message": "Apply blocked by validation errors",
}
return {
"ok": False,
"errors": plan_result.get("errors", []),
"warnings": plan_result.get("warnings", []),
"yaml": plan_result.get("yaml", ""),
"opentofuVars": plan_result.get("opentofuVars", {}),
"installPlan": plan_result.get("installPlan", []),
}
self._status = {
"phase": "apply",
"state": "applied",
"message": "Apply finished",
}
self._logs.extend(
[
"[apply] validating production config",
"[apply] generating OpenTofu variables",
"[apply] running OpenTofu plan",
"[apply] running OpenTofu apply",
"[apply] completed",
]
)
return {
"ok": True,
"errors": [],
"warnings": plan_result.get("warnings", []),
"yaml": plan_result.get("yaml", ""),
"opentofuVars": plan_result.get("opentofuVars", {}),
"installPlan": plan_result.get("installPlan", []),
}
def get_status(self) -> dict[str, Any]:
return copy.deepcopy(self._status)
def get_logs(self) -> dict[str, Any]:
return {"logs": list(self._logs)}