diff --git a/conf/prod/prole.cfg b/conf/prod/prole.cfg index b04006e..a1ee2d4 100644 --- a/conf/prod/prole.cfg +++ b/conf/prod/prole.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-09 17:30:49 +; Generated by install.py on 2026-04-11 16:02:49 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -8,26 +8,8 @@ [Inputs] ; Screen-scoped inputs used for unattended replays (-S) -argocd.node_selector = build.deploy_env = Dev build.run_build = false -database_options.distribution = percona -database_options.ext.dblink = true -database_options.ext.pg_buffercache = true -database_options.ext.pg_cron = true -database_options.ext.pg_freespacemap = true -database_options.ext.pg_repack = true -database_options.ext.pg_stat_monitor = true -database_options.ext.pg_stat_statements = true -database_options.ext.pg_tde = true -database_options.ext.pgaudit = true -database_options.ext.pgbadger = true -database_options.ext.pgcrypto = true -database_options.ext.pgrowlocks = true -database_options.ext.pgvector = true -database_options.ext.postgis = true -database_options.ext.postgres_fdw = true -database_options.version_type = v18 dependencies.ansible.install = true dependencies.auto_install_missing = true dependencies.brew.install = true @@ -39,86 +21,95 @@ dependencies.opentofu.install = true dependencies.python.install = true dependencies.verify_all = false disk_selection.disk_type = local -disk_selection.local_path = $HOME/dev/prole/prole-tools-app/dist +disk_selection.local_path = /Users/chrisfu disk_selection.removable_mount = +env_setup.APP_CLUSTER_KUBECONTEXT = +env_setup.APP_CLUSTER_MODE = standard +env_setup.APP_CLUSTER_NAME = knoe-dev-0 env_setup.CLUSTER_NAME = ${CLUSTER_NAME} env_setup.DATABASE_NAMESPACE = ${DATABASE_NAMESPACE} -env_setup.PROLE_CONF = $HOME/dev/prole/conf -env_setup.PROLE_DATA = $HOME/dev/prole/data -env_setup.PROLE_HOME = $HOME/dev/prole -env_setup.PROLE_LOGS = $HOME/dev/prole/logs -env_setup.PROLE_SERVICE = $HOME/dev/prole/etc -gitops.git_provider = GitLab -gitops.node_selector = -init_cluster.argocd_enabled = false +env_setup.DB_CLUSTER_KUBECONTEXT = +env_setup.DB_CLUSTER_MODE = standard +env_setup.DB_CLUSTER_NAME = knoe-cnpg-0 +env_setup.PROLE_CONF = /Users/chrisfu/dev/prole/conf +env_setup.PROLE_DATA = /Users/chrisfu/dev/prole/data +env_setup.PROLE_HOME = /Users/chrisfu/dev/prole +env_setup.PROLE_LOGS = /Users/chrisfu/dev/prole/logs +env_setup.PROLE_SERVICE = /Users/chrisfu/dev/prole/etc +init_cluster.app_cluster_kubecontext = +init_cluster.app_cluster_machine_type = e2-small +init_cluster.app_cluster_mode = standard +init_cluster.app_cluster_name = knoe-dev-0 +init_cluster.app_cluster_node_count = 3 init_cluster.at_rest_encryption_enabled = true -init_cluster.cluster_env = prod -init_cluster.deployment_target = prole-prod-cluster -init_cluster.gitops_enabled = true -init_cluster.k3s_server_url = https://myrddin.prole.org:6443 -init_cluster.k3s_token = ${PROLE_SECRET:v1:vxIxsLYZSClD_vzj:aSQOAaWbKOIT26PQPCqqIwenTzN4YKcaCa8OihlkDA9cdhbHrPeHtaeT3Pu140a67xKPV45ExSEiJ7jnWSW3lOH0KSO56qgzKxMXzrynKFfQVyJp5OzzQeCl-jrz15AdknLhPeh3eTmOEfLuhNY2Yy9fsBQ-l0gYikw37Q==} -init_cluster.kerberos_enabled = true -init_cluster.mode = k8s +init_cluster.cluster_env = dev +init_cluster.db_boot_disk_size_gb = 50 +init_cluster.db_boot_disk_type = pd-standard +init_cluster.db_cluster_kubecontext = +init_cluster.db_cluster_machine_type = e2-standard-2 +init_cluster.db_cluster_mode = standard +init_cluster.db_cluster_name = knoe-cnpg-0 +init_cluster.db_cluster_node_count = 3 +init_cluster.db_cluster_region = +init_cluster.db_cluster_zones = +init_cluster.gitops_enabled = false +init_cluster.k3s_server_url = +init_cluster.k3s_token = +init_cluster.kerberos_enabled = false init_cluster.start_cluster = true -init_cluster.supabase_enabled = true +init_cluster.supabase_analytics_enabled = true +init_cluster.supabase_auth_enabled = true +init_cluster.supabase_enabled = false +init_cluster.supabase_meta_enabled = true +init_cluster.supabase_realtime_enabled = true +init_cluster.supabase_studio_enabled = false +init_cluster.supabase_studio_url = db.0.knoe.dev init_cnpg_deploy.force_rollout = false init_cnpg_deploy.run_deploy = true init_db_build.run_build = true +init_password.app_cluster_name = knoe-dev-0 init_password.cluster_name = ${CLUSTER_NAME} +init_password.db_cluster_name = knoe-cnpg-0 init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password= vzx.wC4Akd4x-Dj6Wguh -init_password.db_password_confirm= vzx.wC4Akd4x-Dj6Wguh -init_password.db_username = root +init_password.db_password = +init_password.db_password_confirm = +init_password.db_username = chrisfu init_password.generate_ssh_key = true init_scripts.run_scripts = true -kerberos_config.enabled = true -kerberos_config.kdc = 10.0.0.3 +kerberos_config.enabled = false +kerberos_config.init_authority = false +kerberos_config.kdc = kerberos_config.password = -kerberos_config.realm = PROLE.ORG +kerberos_config.realm = kerberos_config.test_connection = false kerberos_config.user = administrator network_scan.run = true ollama_config.model = ollama_config.server_host = ollama_config.server_port = 11434 -supabase_config.pv_base_dir = /synology/d005 -supabase_config.pv_node = gandalf.prole.org [Global] ; Variables used by name in more than one place or assumed global scope -APP_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-0 +ARGOCD_NAMESPACE = argocd ARTIFACT_REGISTRY = us-west3-docker.pkg.dev/plenary-truck-485623-p7/knoe-system -ARTIFACT_REGISTRY_AVAILABLE = true -CLUSTER_ENV = prod +CLUSTER_ENV = dev CLUSTER_NAME = knoe-db -CNPG_ELIGIBLE_NODES = gke-knoe-cnpg-0-default-pool-56407930-wlg9,gke-knoe-cnpg-0-default-pool-b142a290-bmr3,gke-knoe-cnpg-0-default-pool-d8e3a5ef-lmpl -CNPG_PLACEMENT_PLAN_FILE = $HOME/dev/prole/conf/cnpg-placement/knoe-system-knoe-db.json -CNPG_PLACEMENT_PLAN_HASH = a44825aeb5ed4e13 -CNPG_PLACEMENT_PLAN_ID = cnpg-placement-a44825aeb5ed4e13 -CNPG_STAGE1_NODE = gke-knoe-cnpg-0-default-pool-56407930-wlg9 -DATABASE_NAMESPACE = knoe-db-0 -DB_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-cnpg-0 +DATABASE_NAMESPACE = knoe-db DB_HOST_PORT = 5432 -DB_PASSWORD= vzx.wC4Akd4x-Dj6Wguh -DEPLOYMENT_MODE = k8s -DEPLOYMENT_TARGET = prole-prod-cluster -GITLAB_DOMAIN = git.knoe.dev -GITLAB_INGRESS_CLASS = gce +DB_PASSWORD = +DEPLOYMENT_MODE = k3d +DEPLOYMENT_TARGET = prole-dev-cluster +DOCKER_IMPORT_DIR = DOCKER_PRELOAD = false -KNOE_DB_USER = root -KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-0 +K3S_SERVER = +K3S_TOKEN = +KNOE_DB_USER = chrisfu +OPENTOFU_URL = http://127.0.0.1:8080 OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2 -PROLE_K3S_SERVER = https://myrddin.prole.org:6443 -PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:wEwOvfE5WeOdbVjW:yjYu4AJqgYo_EnwvYx4fgeSA1coQb8uv8FxE4_NNSIkQadVroCjaEpOyHXcStehLxGl7Bk-elU43yKzYQT_85BkLXp4-9k-LVh_kAfJp2cXlw7q8PczBr3-_MJXeWv1Cw7mCiMuWEfVpKjpCzZ5-URsBsIH3iYz20_6qwg==} -SUPABASE_API_HOSTNAME = api.knoe.dev -SUPABASE_HOSTNAME = db.0.knoe.dev -SUPABASE_INGRESS_CLASS = gce -SUPABASE_STUDIO_HOSTNAME = db.0.knoe.dev +PROLE_HOME = /Users/chrisfu/dev/prole +REGISTRY_NAMESPACE = knoe-system SERVICE_NAMESPACE = knoe-system -SUPABASE_PV_BASE = /synology/d005 -SUPABASE_PV_BASE_DIR = /synology/d005 -SUPABASE_PV_NODE = gandalf.prole.org [Welcome] ; No configuration values captured yet for this section. @@ -127,26 +118,20 @@ SUPABASE_PV_NODE = gandalf.prole.org ; No configuration values captured yet for this section. [Network] -AD_DC_HOST = myrddin.prole.org -AD_DC_IP = 10.0.0.3 -ANSIBLE_DOMAIN = prole.org -ANSIBLE_INFRASTRUCTURE = $HOME/dev/prole/infrastructure -ANSIBLE_INVENTORY = $HOME/dev/prole/infrastructure/inventory -ANSIBLE_REALM = PROLE.ORG -ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org","gandalf.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["merlin.prole.org","gandalf.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org","gandalf.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"],"k3s_hosts":["gandalf.prole.org"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["gandalf.prole.org","k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]} -KDC_ANSIBLE_DETECTED = 10.0.0.3 -KDC_AUTO_DETECTED = 10.0.0.3 -KERBEROS_AUTO_ENABLED = True - -[Port Forwards] ; No configuration values captured yet for this section. +[Port Forwards] +PORT_FORWARD_K3D_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD +PORT_FORWARD_K3D_MAPPING_2 = id=garage;namespace=knoe-system;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3 +PORT_FORWARD_K3D_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao;address=127.0.0.1;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao +PORT_FORWARD_K3D_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu +PORT_FORWARD_K3D_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard +PORT_FORWARD_K3D_MAPPING_6 = id=postgres;namespace=${DATABASE_NAMESPACE};target=svc/knoe-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary) +PORT_FORWARD_K3D_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI +PORT_FORWARD_K3D_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI + [System Environment] -PROLE_CONF = $HOME/dev/prole/conf -PROLE_DATA = $HOME/dev/prole/data -PROLE_HOME = $HOME/dev/prole -PROLE_LOGS = $HOME/dev/prole/logs -PROLE_SERVICE = $HOME/dev/prole/etc +; No configuration values captured yet for this section. [Monitoring] ; No configuration values captured yet for this section. @@ -155,130 +140,45 @@ PROLE_SERVICE = $HOME/dev/prole/etc ; No configuration values captured yet for this section. [Ollama] -OLLAMA_HOST = http://fairyland.prole.org:11434 -OLLAMA_SERVER_HOST = fairyland.prole.org -OLLAMA_SERVER_PORT = 11434 +; No configuration values captured yet for this section. [Optional Features] -AT_REST_ENCRYPTION_ENABLED = True -GITOPS_ENABLED = True -KERBEROS_ENABLED = True -SUPABASE_ENABLED = True +AT_REST_ENCRYPTION_ENABLED = true +GITOPS_ENABLED = false +KERBEROS_ENABLED = false +SUPABASE_ENABLED = false [GitOps] -; No configuration values captured yet for this section. [Database Creation] ; No configuration values captured yet for this section. [Initialize Cluster] -DB_BOOT_DISK_SIZE_GB = 50 -DB_BOOT_DISK_TYPE = pd-standard -ENVIRONMENT = prod -K3S_SERVER_URL = https://myrddin.prole.org:6443 -K3S_TOKEN = ${PROLE_SECRET:v1:4gEuXvZk46ZZEE84:n0hMI_eOPmwOWax62B7G1n5M-aEA39gMkEoTl4m8d_zvn-fBhoSsyjebENabZc8cDbGroo1Cunge3tLH9oZ_KHIz_tLQBGvGgG9pLg9X_raTNoH-ezMgbllWWfUH3BHznawYDxun3hDDgqTuOdhBPctVg3nrW8kSa7gJqQ==} +; No configuration values captured yet for this section. [Dev Cluster (k3d)] -CLUSTER_ENV = dev +CLUSTER_ENV = k3d-knoe-dev-cluster DISPLAY_NAME = knoe-dev-cluster -KUBECTL_CONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-cnpg-0 +KUBECTL_CONTEXT = dev MODE = k3d [Service Cluster (k3s)] CLUSTER_ENV = prole-service-cluster DISPLAY_NAME = prole-service-cluster -K3S_SERVER_URL = https://myrddin.prole.org:6443 -K3S_TOKEN = ${PROLE_SECRET:v1:6-xBvCegq8BIAUbO:li8GVpawfjJUs4YhZE1ho5o2S3p2BAE6IH1cMX0SftuGkHOHWrwv_rYmufzVzCVt7RV0PtJevGSGOS_hoaPKA5EPtSE520HcQX0f2UJfAMfpvFzsKziVa0UqIkKYGics2c3JXsydP6eVerWagtJ9ciaESbCoy0YnvVgLKg==} +K3S_SERVER_URL = +K3S_TOKEN = MODE = k3s +PIPELINE_URL = http://127.0.0.1:8080 [GCP] -BILLING_ACCOUNT = 01193C-25783B-3211AD -BILLING_PROJECT = plenary-truck-485623-p7 -ORG_ID = 584001916389 -PROJECT_ID = plenary-truck-485623-p7 -billing_account = 01193C-25783B-3211AD -billing_project = plenary-truck-485623-p7 -org_id = 584001916389 -project_id = plenary-truck-485623-p7 +; No configuration values captured yet for this section. [Prod Cluster (k8s)] -ARTIFACTS_DIR = $HOME/dev/prole/data/staging +ARTIFACTS_DIR = CLUSTER_ENV = prole-prod-cluster DISPLAY_NAME = prole-prod-cluster -INSTALL_PLAN = Validate production config and required secrets -Prepare GCP project, VPC, DNS and artifact registry bindings -Prepare Kubernetes namespace ecosystem-0 and RBAC/service accounts -Provision CloudNativePG cluster and bootstrap app/meta databases -Configure backup/WAL buckets and retention policy -Configure Google Workspace OIDC integration and bootstrap admin -Configure routing, TLS, and frontdoor domain mappings -Prepare migration artifacts from source environment -Generate OpenTofu plan and apply production pipeline MODE = k8s -OPENTOFU_VARS = {"admin_user":"knoe_admin","app_database":"knoey","app_user":"knoey_app","backup_bucket":"knoe-0-backups","backup_provider":"gcs","backup_retention_days":14,"bootstrap_admin_email":"admin@knoey.com","cluster_name":"knoe-prod","cnpg_cluster_name":"knoe-db","db_instances":3,"db_pgdata_size_gi":100,"db_pgdata_storage_class":"premium-rwo","db_storage_class":"premium-rwo","db_storage_size_gi":100,"db_wal_size_gi":25,"db_wal_storage_class":"premium-rwo","ecosystem_id":0,"environment":"production","frontdoor_host":"knoey.com","gcp_project_id":"plenary-truck-485623-p7","gcp_region":"us-west3","gitlab_host":"git.knoe.dev","google_hosted_domain":"","meta_database":"knoe_meta","migration_continuous_until_cutover":false,"migration_mode":"snapshot-restore","migration_source_database":"knoey","migration_source_environment":"prole.org","migration_source_host":"knoe-local-db.prole.org","migration_source_password_ref":"secretref://local-source-db-password","migration_source_port":5432,"migration_source_user":"replication_user","namespace":"ecosystem-0","oidc_client_id_ref":"secretref://google-oidc-client-id","oidc_client_secret_ref":"secretref://google-oidc-client-secret","oidc_issuer":"https://accounts.google.com","platform_domain":"knoe.dev","postgres_version":"16","project_id":"plenary-truck-485623-p7","region":"us-west3","supabase_api_host":"api.knoe.dev","supabase_studio_host":"db.0.knoe.dev","tls_mode":"managed","wal_bucket":"knoe-0-wal","workload_identity_sa":"cnpg-backup"} -PRODUCTION_CONFIG_YAML = kind: KnoeProductionConfig -metadata: - ecosystemId: 0 - name: knoey-root - environment: production -spec: - cloud: - provider: gcp - projectId: plenary-truck-485623-p7 - region: us-west3 - clusterName: knoe-prod - vpcMode: managed - vpcName: null - subnetName: null - artifactRegistry: '' - dnsZone: knoe-dev-zone - kubernetes: - namespace: ecosystem-0 - database: - clusterName: knoe-db - postgresVersion: '16' - instances: 3 - storageClass: premium-rwo - storageSizeGi: 100 - pgdataStorageClass: premium-rwo - walStorageClass: premium-rwo - pgdataSizeGi: 100 - walSizeGi: 25 - appDatabase: knoey - metaDatabase: knoe_meta - appUser: knoey_app - adminUser: knoe_admin - backups: - backupBucket: knoe-0-backups - walBucket: knoe-0-wal - retentionDays: 14 - backupProvider: gcs - workloadIdentitySA: cnpg-backup - auth: - provider: google-workspace-oidc - issuer: https://accounts.google.com - clientId: secretref://google-oidc-client-id - clientSecret: secretref://google-oidc-client-secret - bootstrapAdminEmail: admin@knoey.com - googleClientId: '' - googleClientSecret: '' - googleHostedDomain: '' - routing: - frontdoorHost: knoey.com - platformDomain: knoe.dev - apiHost: api.knoe.dev - studioHost: db.0.knoe.dev - gitHost: git.knoe.dev - tlsMode: managed - migration: - sourceEnvironment: prole.org - mode: snapshot-restore - sourceHost: knoe-local-db.prole.org - sourcePort: 5432 - sourceDatabase: knoey - sourceUser: replication_user - sourcePasswordRef: secretref://local-source-db-password - continuousUntilCutover: false +PIPELINE_URL = http://127.0.0.1:8080 [Docker Build] ; No configuration values captured yet for this section. @@ -287,8 +187,8 @@ spec: ; No configuration values captured yet for this section. [Deployment] -MODE = k8s -TARGET = prole-prod-cluster +MODE = k3d +TARGET = prole-dev-cluster [Install] ; No configuration values captured yet for this section. diff --git a/deploy.sh b/deploy.sh old mode 100644 new mode 100755 diff --git a/knoe/prole_conf.py b/knoe/prole_conf.py index 900f1c9..6b3e016 100644 --- a/knoe/prole_conf.py +++ b/knoe/prole_conf.py @@ -163,12 +163,16 @@ def layered_cfg_files(cfg_path: Path) -> list[Path]: if env and (env_dir / "prole.cfg").exists(): files.append(env_dir / "prole.cfg") try: + skip_names = { + "prod.cfg", # standalone deploy config, not an overlay for prole.cfg + "gcp.cfg", # generated metadata file (key=value, no INI sections) + } overrides = [ p for p in env_dir.iterdir() if p.is_file() and p.name != "prole.cfg" - and p.name != "prod.cfg" # prod.cfg is a standalone GKE deploy config, not a prole.cfg overlay + and p.name not in skip_names and p.suffix == ".cfg" and not p.name.startswith(".") ] diff --git a/tests/installer/test_prole_conf.py b/tests/installer/test_prole_conf.py index beead21..39600ba 100644 --- a/tests/installer/test_prole_conf.py +++ b/tests/installer/test_prole_conf.py @@ -30,6 +30,8 @@ def test_layered_cfg_files_order_and_merge(tmp_path: Path): ) (env_dir / "10-first.cfg").write_text("[Global]\nNAMESPACE = first\n") (env_dir / "20-second.cfg").write_text("[Global]\nNAMESPACE = second\n") + (env_dir / "gcp.cfg").write_text('project_id = "demo"\n') + (env_dir / "prod.cfg").write_text("[Global]\nNAMESPACE = ignored\n") entry = conf_dir / "prole.cfg" os.symlink(str(env_dir / "prole.cfg"), str(entry))