mirror of
https://github.com/dredx/prole.git
synced 2026-09-24 19:54:32 +00:00
fb7993de1d
- Introduced `knoe.core.prod_config` with production configuration schema and helper methods. - Added validation logic to enforce required fields and expected formats. - Implemented in-process API (`ProdConfigApi`) for managing production configs. - Added test coverage for default config behavior, validation, YAML generation, and API workflow. - Updated cluster UI layout and tests to integrate production config with new navigation flows. - Added host inventory and updated service flow to reflect production setup changes.
511 lines
20 KiB
Python
511 lines
20 KiB
Python
"""Production configuration model + API helpers for installer UI."""
|
|
|
|
from dataclasses import asdict, dataclass, field
|
|
import copy
|
|
import io
|
|
import re
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
|
|
DOMAIN_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$")
|
|
HOST_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$")
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdCloudConfig:
|
|
provider: str = "gcp"
|
|
projectId: str = ""
|
|
region: str = "us-central1"
|
|
clusterName: str = "knoe-prod"
|
|
vpcMode: str = "managed"
|
|
vpcName: str | None = None
|
|
subnetName: str | None = None
|
|
artifactRegistry: str = ""
|
|
dnsZone: str = "knoe-dev-zone"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdKubernetesConfig:
|
|
namespace: str = "ecosystem-0"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdDatabaseConfig:
|
|
clusterName: str = "knoe-db"
|
|
postgresVersion: str = "16"
|
|
instances: int = 3
|
|
storageClass: str = "premium-rwo"
|
|
storageSizeGi: int = 100
|
|
appDatabase: str = "knoey"
|
|
metaDatabase: str = "knoe_meta"
|
|
appUser: str = "knoey_app"
|
|
adminUser: str = "knoe_admin"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdBackupsConfig:
|
|
backupBucket: str = "knoe-0-backups"
|
|
walBucket: str = "knoe-0-wal"
|
|
retentionDays: int = 14
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdAuthConfig:
|
|
provider: str = "google-workspace-oidc"
|
|
issuer: str = "https://accounts.google.com"
|
|
clientId: str = "secretref://google-oidc-client-id"
|
|
clientSecret: str = "secretref://google-oidc-client-secret"
|
|
bootstrapAdminEmail: str = "admin@knoey.com"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdRoutingConfig:
|
|
frontdoorHost: str = "knoey.com"
|
|
platformDomain: str = "knoe.dev"
|
|
tlsMode: str = "managed"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdMigrationConfig:
|
|
sourceEnvironment: str = "prole.org"
|
|
mode: str = "snapshot-restore"
|
|
sourceHost: str = "knoe-local-db.prole.org"
|
|
sourcePort: int = 5432
|
|
sourceDatabase: str = "knoey"
|
|
sourceUser: str = "replication_user"
|
|
sourcePasswordRef: str = "secretref://local-source-db-password"
|
|
continuousUntilCutover: bool = False
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProdMetadata:
|
|
ecosystemId: int = 0
|
|
name: str = "knoey-root"
|
|
environment: str = "production"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class KnoeProductionConfig:
|
|
kind: str = "KnoeProductionConfig"
|
|
metadata: ProdMetadata = field(default_factory=ProdMetadata)
|
|
cloud: ProdCloudConfig = field(default_factory=ProdCloudConfig)
|
|
kubernetes: ProdKubernetesConfig = field(default_factory=ProdKubernetesConfig)
|
|
database: ProdDatabaseConfig = field(default_factory=ProdDatabaseConfig)
|
|
backups: ProdBackupsConfig = field(default_factory=ProdBackupsConfig)
|
|
auth: ProdAuthConfig = field(default_factory=ProdAuthConfig)
|
|
routing: ProdRoutingConfig = field(default_factory=ProdRoutingConfig)
|
|
migration: ProdMigrationConfig = field(default_factory=ProdMigrationConfig)
|
|
|
|
def to_document(self) -> dict[str, Any]:
|
|
return {
|
|
"kind": self.kind,
|
|
"metadata": asdict(self.metadata),
|
|
"spec": {
|
|
"cloud": asdict(self.cloud),
|
|
"kubernetes": asdict(self.kubernetes),
|
|
"database": asdict(self.database),
|
|
"backups": asdict(self.backups),
|
|
"auth": asdict(self.auth),
|
|
"routing": asdict(self.routing),
|
|
"migration": asdict(self.migration),
|
|
},
|
|
}
|
|
|
|
|
|
def default_prod_config() -> KnoeProductionConfig:
|
|
return KnoeProductionConfig()
|
|
|
|
|
|
def _str_or_empty(value: Any) -> str:
|
|
return str(value).strip() if value is not None else ""
|
|
|
|
|
|
def _is_domain(value: str) -> bool:
|
|
return bool(DOMAIN_RE.match(value.lower()))
|
|
|
|
|
|
def _is_host(value: str) -> bool:
|
|
return bool(HOST_RE.match(value.lower()))
|
|
|
|
|
|
def _validate_positive_int(value: Any, field: str, errors: list[str]) -> int:
|
|
try:
|
|
parsed = int(value)
|
|
except Exception:
|
|
errors.append(f"{field} must be a positive number")
|
|
return 0
|
|
if parsed <= 0:
|
|
errors.append(f"{field} must be a positive number")
|
|
return parsed
|
|
|
|
|
|
def _coerce_bool(value: Any) -> bool:
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, (int, float)):
|
|
return bool(value)
|
|
text = _str_or_empty(value).lower()
|
|
return text in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _coerce_int(value: Any, default: int) -> int:
|
|
try:
|
|
text = _str_or_empty(value)
|
|
if text == "":
|
|
return default
|
|
return int(text)
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def build_config(payload: dict[str, Any] | None = None) -> KnoeProductionConfig:
|
|
cfg = default_prod_config()
|
|
src = payload or {}
|
|
metadata = src.get("metadata") or {}
|
|
spec = src.get("spec") or {}
|
|
cloud = spec.get("cloud") or {}
|
|
k8s = spec.get("kubernetes") or {}
|
|
db = spec.get("database") or {}
|
|
backups = spec.get("backups") or {}
|
|
auth = spec.get("auth") or {}
|
|
routing = spec.get("routing") or {}
|
|
migration = spec.get("migration") or {}
|
|
|
|
cfg.kind = _str_or_empty(src.get("kind")) or cfg.kind
|
|
|
|
cfg.metadata.ecosystemId = _coerce_int(
|
|
metadata.get("ecosystemId", cfg.metadata.ecosystemId), cfg.metadata.ecosystemId
|
|
)
|
|
cfg.metadata.name = _str_or_empty(metadata.get("name")) or cfg.metadata.name
|
|
cfg.metadata.environment = (
|
|
_str_or_empty(metadata.get("environment")) or cfg.metadata.environment
|
|
)
|
|
|
|
cfg.cloud.provider = _str_or_empty(cloud.get("provider")) or cfg.cloud.provider
|
|
cfg.cloud.projectId = _str_or_empty(cloud.get("projectId"))
|
|
cfg.cloud.region = _str_or_empty(cloud.get("region")) or cfg.cloud.region
|
|
cfg.cloud.clusterName = _str_or_empty(cloud.get("clusterName")) or cfg.cloud.clusterName
|
|
cfg.cloud.vpcMode = _str_or_empty(cloud.get("vpcMode")) or cfg.cloud.vpcMode
|
|
cfg.cloud.vpcName = _str_or_empty(cloud.get("vpcName")) or None
|
|
cfg.cloud.subnetName = _str_or_empty(cloud.get("subnetName")) or None
|
|
cfg.cloud.artifactRegistry = _str_or_empty(cloud.get("artifactRegistry"))
|
|
cfg.cloud.dnsZone = _str_or_empty(cloud.get("dnsZone")) or cfg.cloud.dnsZone
|
|
|
|
cfg.kubernetes.namespace = _str_or_empty(k8s.get("namespace")) or cfg.kubernetes.namespace
|
|
|
|
cfg.database.clusterName = _str_or_empty(db.get("clusterName")) or cfg.database.clusterName
|
|
cfg.database.postgresVersion = _str_or_empty(db.get("postgresVersion")) or cfg.database.postgresVersion
|
|
cfg.database.instances = _coerce_int(db.get("instances", cfg.database.instances), cfg.database.instances)
|
|
cfg.database.storageClass = _str_or_empty(db.get("storageClass")) or cfg.database.storageClass
|
|
cfg.database.storageSizeGi = _coerce_int(
|
|
db.get("storageSizeGi", cfg.database.storageSizeGi), cfg.database.storageSizeGi
|
|
)
|
|
cfg.database.appDatabase = _str_or_empty(db.get("appDatabase")) or cfg.database.appDatabase
|
|
cfg.database.metaDatabase = _str_or_empty(db.get("metaDatabase")) or cfg.database.metaDatabase
|
|
cfg.database.appUser = _str_or_empty(db.get("appUser")) or cfg.database.appUser
|
|
cfg.database.adminUser = _str_or_empty(db.get("adminUser")) or cfg.database.adminUser
|
|
|
|
cfg.backups.backupBucket = _str_or_empty(backups.get("backupBucket")) or cfg.backups.backupBucket
|
|
cfg.backups.walBucket = _str_or_empty(backups.get("walBucket")) or cfg.backups.walBucket
|
|
cfg.backups.retentionDays = _coerce_int(
|
|
backups.get("retentionDays", cfg.backups.retentionDays), cfg.backups.retentionDays
|
|
)
|
|
|
|
cfg.auth.provider = _str_or_empty(auth.get("provider")) or cfg.auth.provider
|
|
cfg.auth.issuer = _str_or_empty(auth.get("issuer")) or cfg.auth.issuer
|
|
cfg.auth.clientId = _str_or_empty(auth.get("clientId")) or cfg.auth.clientId
|
|
cfg.auth.clientSecret = _str_or_empty(auth.get("clientSecret")) or cfg.auth.clientSecret
|
|
cfg.auth.bootstrapAdminEmail = (
|
|
_str_or_empty(auth.get("bootstrapAdminEmail")) or cfg.auth.bootstrapAdminEmail
|
|
)
|
|
|
|
cfg.routing.frontdoorHost = _str_or_empty(routing.get("frontdoorHost")) or cfg.routing.frontdoorHost
|
|
cfg.routing.platformDomain = _str_or_empty(routing.get("platformDomain")) or cfg.routing.platformDomain
|
|
cfg.routing.tlsMode = _str_or_empty(routing.get("tlsMode")) or cfg.routing.tlsMode
|
|
|
|
cfg.migration.sourceEnvironment = (
|
|
_str_or_empty(migration.get("sourceEnvironment")) or cfg.migration.sourceEnvironment
|
|
)
|
|
cfg.migration.mode = _str_or_empty(migration.get("mode")) or cfg.migration.mode
|
|
cfg.migration.sourceHost = _str_or_empty(migration.get("sourceHost")) or cfg.migration.sourceHost
|
|
cfg.migration.sourcePort = _coerce_int(
|
|
migration.get("sourcePort", cfg.migration.sourcePort), cfg.migration.sourcePort
|
|
)
|
|
cfg.migration.sourceDatabase = _str_or_empty(migration.get("sourceDatabase")) or cfg.migration.sourceDatabase
|
|
cfg.migration.sourceUser = _str_or_empty(migration.get("sourceUser")) or cfg.migration.sourceUser
|
|
cfg.migration.sourcePasswordRef = (
|
|
_str_or_empty(migration.get("sourcePasswordRef")) or cfg.migration.sourcePasswordRef
|
|
)
|
|
cfg.migration.continuousUntilCutover = _coerce_bool(
|
|
migration.get("continuousUntilCutover", cfg.migration.continuousUntilCutover)
|
|
)
|
|
|
|
return cfg
|
|
|
|
|
|
def validate_prod_config(config: KnoeProductionConfig) -> tuple[list[str], list[str]]:
|
|
errors: list[str] = []
|
|
warnings: list[str] = []
|
|
|
|
if config.metadata.ecosystemId != 0:
|
|
errors.append("metadata.ecosystemId must be fixed to 0")
|
|
|
|
required_fields = {
|
|
"metadata.name": config.metadata.name,
|
|
"metadata.environment": config.metadata.environment,
|
|
"spec.cloud.provider": config.cloud.provider,
|
|
"spec.cloud.projectId": config.cloud.projectId,
|
|
"spec.cloud.region": config.cloud.region,
|
|
"spec.cloud.clusterName": config.cloud.clusterName,
|
|
"spec.kubernetes.namespace": config.kubernetes.namespace,
|
|
"spec.database.clusterName": config.database.clusterName,
|
|
"spec.database.postgresVersion": config.database.postgresVersion,
|
|
"spec.database.storageClass": config.database.storageClass,
|
|
"spec.database.appDatabase": config.database.appDatabase,
|
|
"spec.database.metaDatabase": config.database.metaDatabase,
|
|
"spec.database.appUser": config.database.appUser,
|
|
"spec.database.adminUser": config.database.adminUser,
|
|
"spec.backups.backupBucket": config.backups.backupBucket,
|
|
"spec.backups.walBucket": config.backups.walBucket,
|
|
"spec.auth.provider": config.auth.provider,
|
|
"spec.auth.issuer": config.auth.issuer,
|
|
"spec.auth.clientId": config.auth.clientId,
|
|
"spec.auth.clientSecret": config.auth.clientSecret,
|
|
"spec.auth.bootstrapAdminEmail": config.auth.bootstrapAdminEmail,
|
|
"spec.routing.frontdoorHost": config.routing.frontdoorHost,
|
|
"spec.routing.platformDomain": config.routing.platformDomain,
|
|
"spec.migration.sourceEnvironment": config.migration.sourceEnvironment,
|
|
"spec.migration.sourceHost": config.migration.sourceHost,
|
|
"spec.migration.sourceDatabase": config.migration.sourceDatabase,
|
|
"spec.migration.sourceUser": config.migration.sourceUser,
|
|
"spec.migration.sourcePasswordRef": config.migration.sourcePasswordRef,
|
|
}
|
|
for key, value in required_fields.items():
|
|
if not _str_or_empty(value):
|
|
errors.append(f"{key} is required")
|
|
|
|
config.database.instances = _validate_positive_int(
|
|
config.database.instances, "spec.database.instances", errors
|
|
)
|
|
config.database.storageSizeGi = _validate_positive_int(
|
|
config.database.storageSizeGi, "spec.database.storageSizeGi", errors
|
|
)
|
|
config.backups.retentionDays = _validate_positive_int(
|
|
config.backups.retentionDays, "spec.backups.retentionDays", errors
|
|
)
|
|
config.migration.sourcePort = _validate_positive_int(
|
|
config.migration.sourcePort, "spec.migration.sourcePort", errors
|
|
)
|
|
|
|
if config.kubernetes.namespace != "ecosystem-0":
|
|
errors.append("spec.kubernetes.namespace must be ecosystem-0")
|
|
|
|
for field, value in (
|
|
("spec.routing.frontdoorHost", config.routing.frontdoorHost),
|
|
("spec.routing.platformDomain", config.routing.platformDomain),
|
|
("spec.migration.sourceEnvironment", config.migration.sourceEnvironment),
|
|
):
|
|
if _str_or_empty(value) and not _is_domain(_str_or_empty(value)):
|
|
errors.append(f"{field} must be a valid domain")
|
|
|
|
if _str_or_empty(config.migration.sourceHost) and not _is_host(config.migration.sourceHost):
|
|
errors.append("spec.migration.sourceHost must be a valid host")
|
|
|
|
if "@" not in _str_or_empty(config.auth.bootstrapAdminEmail):
|
|
errors.append("spec.auth.bootstrapAdminEmail must be a valid email")
|
|
|
|
if config.cloud.provider.lower() != "gcp":
|
|
warnings.append("spec.cloud.provider is expected to be gcp for production defaults")
|
|
if config.auth.provider.lower() != "google-workspace-oidc":
|
|
warnings.append("spec.auth.provider is expected to be google-workspace-oidc")
|
|
|
|
return errors, warnings
|
|
|
|
|
|
def canonical_yaml(config: KnoeProductionConfig) -> str:
|
|
doc = config.to_document()
|
|
stream = io.StringIO()
|
|
yaml.safe_dump(doc, stream, sort_keys=False, default_flow_style=False)
|
|
return stream.getvalue().strip() + "\n"
|
|
|
|
|
|
def opentofu_vars(config: KnoeProductionConfig) -> dict[str, Any]:
|
|
return {
|
|
"ecosystem_id": config.metadata.ecosystemId,
|
|
"environment": config.metadata.environment,
|
|
"project_id": config.cloud.projectId,
|
|
"region": config.cloud.region,
|
|
"cluster_name": config.cloud.clusterName,
|
|
"namespace": config.kubernetes.namespace,
|
|
"cnpg_cluster_name": config.database.clusterName,
|
|
"postgres_version": config.database.postgresVersion,
|
|
"db_instances": config.database.instances,
|
|
"db_storage_class": config.database.storageClass,
|
|
"db_storage_size_gi": config.database.storageSizeGi,
|
|
"app_database": config.database.appDatabase,
|
|
"meta_database": config.database.metaDatabase,
|
|
"app_user": config.database.appUser,
|
|
"admin_user": config.database.adminUser,
|
|
"backup_bucket": config.backups.backupBucket,
|
|
"wal_bucket": config.backups.walBucket,
|
|
"backup_retention_days": config.backups.retentionDays,
|
|
"oidc_issuer": config.auth.issuer,
|
|
"oidc_client_id_ref": config.auth.clientId,
|
|
"oidc_client_secret_ref": config.auth.clientSecret,
|
|
"bootstrap_admin_email": config.auth.bootstrapAdminEmail,
|
|
"frontdoor_host": config.routing.frontdoorHost,
|
|
"platform_domain": config.routing.platformDomain,
|
|
"tls_mode": config.routing.tlsMode,
|
|
"migration_source_environment": config.migration.sourceEnvironment,
|
|
"migration_mode": config.migration.mode,
|
|
"migration_source_host": config.migration.sourceHost,
|
|
"migration_source_port": config.migration.sourcePort,
|
|
"migration_source_database": config.migration.sourceDatabase,
|
|
"migration_source_user": config.migration.sourceUser,
|
|
"migration_source_password_ref": config.migration.sourcePasswordRef,
|
|
"migration_continuous_until_cutover": config.migration.continuousUntilCutover,
|
|
}
|
|
|
|
|
|
def install_py_plan(config: KnoeProductionConfig) -> list[str]:
|
|
return [
|
|
"Validate production config and required secrets",
|
|
"Prepare GCP project, VPC, DNS and artifact registry bindings",
|
|
"Prepare Kubernetes namespace ecosystem-0 and RBAC/service accounts",
|
|
"Provision CloudNativePG cluster and bootstrap app/meta databases",
|
|
"Configure backup/WAL buckets and retention policy",
|
|
"Configure Google Workspace OIDC integration and bootstrap admin",
|
|
"Configure routing, TLS, and frontdoor domain mappings",
|
|
"Prepare migration artifacts from source environment",
|
|
"Generate OpenTofu plan and apply production pipeline",
|
|
]
|
|
|
|
|
|
def to_api_payload(config: KnoeProductionConfig) -> dict[str, Any]:
|
|
return config.to_document()
|
|
|
|
|
|
def from_api_payload(payload: dict[str, Any] | None) -> KnoeProductionConfig:
|
|
return build_config(payload)
|
|
|
|
|
|
class ProdConfigApi:
|
|
"""In-process API facade used by the installer UI.
|
|
|
|
Provides the same contract as:
|
|
- GET /api/install/prod-config
|
|
- PUT /api/install/prod-config
|
|
- POST /api/install/plan
|
|
- POST /api/install/apply
|
|
- GET /api/install/status
|
|
- GET /api/install/logs
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._payload = to_api_payload(default_prod_config())
|
|
self._status: dict[str, Any] = {
|
|
"phase": "idle",
|
|
"state": "ready",
|
|
"message": "Waiting for plan",
|
|
}
|
|
self._logs: list[str] = []
|
|
|
|
def get_prod_config(self) -> dict[str, Any]:
|
|
return copy.deepcopy(self._payload)
|
|
|
|
def put_prod_config(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
cfg = from_api_payload(payload)
|
|
errors, warnings = validate_prod_config(cfg)
|
|
doc = to_api_payload(cfg)
|
|
self._payload = doc
|
|
return {
|
|
"ok": not errors,
|
|
"config": copy.deepcopy(doc),
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
}
|
|
|
|
def post_plan(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
cfg = from_api_payload(payload)
|
|
errors, warnings = validate_prod_config(cfg)
|
|
if errors:
|
|
self._status = {
|
|
"phase": "plan",
|
|
"state": "error",
|
|
"message": "Plan failed validation",
|
|
}
|
|
return {
|
|
"ok": False,
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
"yaml": "",
|
|
"opentofuVars": {},
|
|
"installPlan": [],
|
|
}
|
|
doc = to_api_payload(cfg)
|
|
self._payload = copy.deepcopy(doc)
|
|
yaml_preview = canonical_yaml(cfg)
|
|
tf_vars = opentofu_vars(cfg)
|
|
plan = install_py_plan(cfg)
|
|
self._status = {
|
|
"phase": "plan",
|
|
"state": "planned",
|
|
"message": "Plan generated",
|
|
}
|
|
self._logs.append("[plan] production plan generated")
|
|
return {
|
|
"ok": True,
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
"yaml": yaml_preview,
|
|
"opentofuVars": tf_vars,
|
|
"installPlan": plan,
|
|
}
|
|
|
|
def post_apply(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
plan_result = self.post_plan(payload)
|
|
if not plan_result.get("ok"):
|
|
self._status = {
|
|
"phase": "apply",
|
|
"state": "error",
|
|
"message": "Apply blocked by validation errors",
|
|
}
|
|
return {
|
|
"ok": False,
|
|
"errors": plan_result.get("errors", []),
|
|
"warnings": plan_result.get("warnings", []),
|
|
"yaml": plan_result.get("yaml", ""),
|
|
"opentofuVars": plan_result.get("opentofuVars", {}),
|
|
"installPlan": plan_result.get("installPlan", []),
|
|
}
|
|
self._status = {
|
|
"phase": "apply",
|
|
"state": "applied",
|
|
"message": "Apply finished",
|
|
}
|
|
self._logs.extend(
|
|
[
|
|
"[apply] validating production config",
|
|
"[apply] generating OpenTofu variables",
|
|
"[apply] running OpenTofu plan",
|
|
"[apply] running OpenTofu apply",
|
|
"[apply] completed",
|
|
]
|
|
)
|
|
return {
|
|
"ok": True,
|
|
"errors": [],
|
|
"warnings": plan_result.get("warnings", []),
|
|
"yaml": plan_result.get("yaml", ""),
|
|
"opentofuVars": plan_result.get("opentofuVars", {}),
|
|
"installPlan": plan_result.get("installPlan", []),
|
|
}
|
|
|
|
def get_status(self) -> dict[str, Any]:
|
|
return copy.deepcopy(self._status)
|
|
|
|
def get_logs(self) -> dict[str, Any]:
|
|
return {"logs": list(self._logs)}
|