From 64e1cef16c08a3220f817d65da4d176e4c59d13b Mon Sep 17 00:00:00 2001 From: chrisfu Date: Tue, 7 Apr 2026 21:11:25 -0700 Subject: [PATCH] checkpoint: stabilize k8s cluster setup and shelve cluster-storage UI - make cluster-storage milestone opt-in and remove installer cluster-storage step from UI navigation\n- add cluster storage browser and GKE cluster ops helpers with CLI coverage\n- update k8s/CNPG config and install flow files for corrected cluster setup\n- add/refresh tests for storage browser, GKE ops, prod config, and service-layer navigation Co-authored-by: Junie --- conf/cnpg-placement/ecosystem-0-knoe-db.json | 20 +- conf/cnpg-placement/knoe-system-knoe-db.json | 14 +- conf/dev/prole.cfg | 8 +- conf/prod/prole.cfg | 37 +- deploy/gcp/gke/knoe-db.yaml | 18 +- install.sh | 37 ++ knoe/core/actions.py | 376 ++++++++++++++-- knoe/core/cluster_storage_browser.py | 416 ++++++++++++++++++ knoe/core/milestones.py | 23 + knoe/core/ops/gke_clusters.py | 181 ++++++++ knoe/core/prod_config.py | 35 ++ knoe/ops/cli.py | 63 +++ knoe/ui/screens/cluster.py | 54 ++- knoe/ui/screens/services.py | 46 +- modes/k8s/knoe-db/.version | 2 +- scan/network_description.txt | 94 ++-- scripts/patch_clusters.sh | 251 +++++++++++ scripts/reset_clusters.sh | 317 +++++++++++++ .../installer/test_cluster_storage_browser.py | 90 ++++ tests/installer/test_gke_clusters.py | 83 ++++ .../installer/test_ops_cli_storage_browser.py | 97 ++++ tests/installer/test_prod_config.py | 12 + tests/test_service_layer_navigation.py | 7 + 23 files changed, 2126 insertions(+), 155 deletions(-) create mode 100644 knoe/core/cluster_storage_browser.py create mode 100644 knoe/core/ops/gke_clusters.py create mode 100755 scripts/patch_clusters.sh create mode 100755 scripts/reset_clusters.sh create mode 100644 tests/installer/test_cluster_storage_browser.py create mode 100644 tests/installer/test_gke_clusters.py create mode 100644 tests/installer/test_ops_cli_storage_browser.py diff --git a/conf/cnpg-placement/ecosystem-0-knoe-db.json b/conf/cnpg-placement/ecosystem-0-knoe-db.json index d343c65..a2430da 100644 --- a/conf/cnpg-placement/ecosystem-0-knoe-db.json +++ b/conf/cnpg-placement/ecosystem-0-knoe-db.json @@ -1,23 +1,21 @@ { "assignments": { - "0": "gk3-knoe-dev-0-pool-1-11c87d92-8qr5", - "1": "gk3-knoe-dev-0-pool-1-88b7392b-62c4", - "2": "gk3-knoe-dev-0-pool-1-88b7392b-d6lb" + "0": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z", + "1": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z", + "2": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z" }, "cluster_name": "knoe-db", "desired_instances": 3, "eligible_nodes": [ - "gk3-knoe-dev-0-pool-1-11c87d92-8qr5", - "gk3-knoe-dev-0-pool-1-88b7392b-62c4", - "gk3-knoe-dev-0-pool-1-88b7392b-d6lb" + "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z" ], "metadata": { "prior_plan_present": true, - "reason": "reused", - "regenerated": false, - "reused": true + "reason": "assigned_node_no_longer_eligible", + "regenerated": true, + "reused": false }, - "plan_hash": "334860873593911a", - "plan_id": "cnpg-placement-334860873593911a", + "plan_hash": "c1213b43098112f3", + "plan_id": "cnpg-placement-c1213b43098112f3", "schema_version": "v1" } diff --git a/conf/cnpg-placement/knoe-system-knoe-db.json b/conf/cnpg-placement/knoe-system-knoe-db.json index d343c65..be1b741 100644 --- a/conf/cnpg-placement/knoe-system-knoe-db.json +++ b/conf/cnpg-placement/knoe-system-knoe-db.json @@ -1,15 +1,13 @@ { "assignments": { - "0": "gk3-knoe-dev-0-pool-1-11c87d92-8qr5", - "1": "gk3-knoe-dev-0-pool-1-88b7392b-62c4", - "2": "gk3-knoe-dev-0-pool-1-88b7392b-d6lb" + "0": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z", + "1": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z", + "2": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z" }, "cluster_name": "knoe-db", "desired_instances": 3, "eligible_nodes": [ - "gk3-knoe-dev-0-pool-1-11c87d92-8qr5", - "gk3-knoe-dev-0-pool-1-88b7392b-62c4", - "gk3-knoe-dev-0-pool-1-88b7392b-d6lb" + "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z" ], "metadata": { "prior_plan_present": true, @@ -17,7 +15,7 @@ "regenerated": false, "reused": true }, - "plan_hash": "334860873593911a", - "plan_id": "cnpg-placement-334860873593911a", + "plan_hash": "c1213b43098112f3", + "plan_id": "cnpg-placement-c1213b43098112f3", "schema_version": "v1" } diff --git a/conf/dev/prole.cfg b/conf/dev/prole.cfg index 37011d4..fe82047 100644 --- a/conf/dev/prole.cfg +++ b/conf/dev/prole.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-06 05:37:48 +; Generated by install.py on 2026-04-07 09:23:28 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -56,7 +56,7 @@ init_cluster.cluster_env = dev init_cluster.deployment_target = prole-dev-cluster init_cluster.gitops_enabled = false init_cluster.k3s_server_url = https://myrddin.prole.org:6443 -init_cluster.k3s_token = ${PROLE_SECRET:v1:UQd8eq6blSjrABFq:npCjmA_RyXc2352u-CgKeHZSuEmOmjygNrDHQH9N6B_JgqYFbbnw0e3lKX_VSHTOQvfBi9OwJ2Nh3kPXoFGgbOnxcp3UzIwG0yPXcAXAu3KHeaYFe1eVnONz63oz_qYSaXOwh7a7NS-05cmTUqB5g2YJRK_yVXmn1LJf3g==} +init_cluster.k3s_token = ${PROLE_SECRET:v1:9Vez82_9Cxpex2oY:oWLW5wEBqIzPEIOt3EZnB2JLj4IAORTKMxMxtNwDC58Q266OzM83h2l-PV30e_V4QblEmDmGpQyAg28FkDNBpxlYa6HsNk8HnkCLQIP8MK_j51e3ft6UUnMISGSKjkoSl2wMdLy5k9QhCdoQeOGhxN2BpiEscZBUgpDIQw==} init_cluster.kerberos_enabled = true init_cluster.mode = k3d init_cluster.start_cluster = true @@ -100,7 +100,7 @@ DOCKER_PRELOAD = false KNOE_DB_USER = root OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2 PROLE_K3S_SERVER = https://myrddin.prole.org:6443 -PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:ViP2nnYnrHPJoSVL:NBMjuyphR6SayWvXTLzQVPZRis5CobeAbEP7vdBaHHSNO7fseQMkDTJLt3ssAHEVpJiQcAwqc84f5-x9AaniXgIgXo9Ka1cl560C6uMPrVvQ_YzlWQiUoD87PboGyBcXms794JvTJ7LLAiCDwHrxm-f6-oaPOgnAzMvwZA==} +PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:J-6LlQf6BDV1Szx0:b1D60bQkckSeXNHuMobcF5LXy2-ui6oarFisXzjOjTPX2dvXua_v3O8kBicxjvStyTUX5N1EDnunnm4eGWYElfXtgdoHsF8kc_7KkRiNt37ad7-KVShwKMHPDhNsJA8TQ_L3PjhxpQbrwyCo2PWkXoNlCtjn57qUHeqW1Q==} PROLE_OPENTOFU_URL = http://127.0.0.1:8080 SERVICE_NAMESPACE = knoe-system SUPABASE_PV_BASE = /synology/d005 @@ -173,7 +173,7 @@ MODE = k3d CLUSTER_ENV = prole-service-cluster DISPLAY_NAME = prole-service-cluster K3S_SERVER_URL = https://myrddin.prole.org:6443 -K3S_TOKEN = ${PROLE_SECRET:v1:ZYtbrUl6yqVscqkt:mMI_jblCsK2_fhlGoUUeXgQPD3yJ0bdN9OYTpO_Bv25BoGdp5C-nJK7EmdPtYC_dlw4UzxDZIYtz7NssNeT7PlYOKgK7Vq2rpKElBxzfLiE8QYYQmIB66IM_Pmu1FwleLDf5GT71A3UuWkzqHfh0EAe0dsX181loromFpQ==} +K3S_TOKEN = ${PROLE_SECRET:v1:sOSNr3SQq1As2lxH:RM-5UCAufudeBWFaEST-pC4y17Gs0Fu106yX_0VsseCRcjrLQUT4f4uFmUxY8ICYE9zArCyJEtgZNNGoRcybQfBUoTTAAAuGrK1nMuakuRd8j_aPEnH3XHSLgraIJIlizCSpA8miq1U2UJRv6KmJjxzoMmfKpExKqKiWtw==} MODE = k3s PIPELINE_URL = http://127.0.0.1:8080 diff --git a/conf/prod/prole.cfg b/conf/prod/prole.cfg index 530cb45..d3e5668 100644 --- a/conf/prod/prole.cfg +++ b/conf/prod/prole.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-06 05:49:04 +; Generated by install.py on 2026-04-07 09:24:02 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -56,7 +56,7 @@ init_cluster.cluster_env = prod init_cluster.deployment_target = prole-prod-cluster init_cluster.gitops_enabled = false init_cluster.k3s_server_url = https://myrddin.prole.org:6443 -init_cluster.k3s_token = ${PROLE_SECRET:v1:f6Ey7pFow7vqJCXh:L0bJyYn5OfamHRyoOHLCd0WpwnukFKUnSW72rN3v28sPNsTX16bDOjnuXOES4SVZ6beRH159KOK7UhopGu6g7CWiOn52-CF2fE8jnl3N_inZDUwyH8IDVt2b_xDbagVcyjoqbZddABUWJMAQ56wXcYaxh1o3y8awfB5SMA==} +init_cluster.k3s_token = ${PROLE_SECRET:v1:8jsa5B6uohhtyjxS:pfwgP6OkFpEcR-f4_3_DkCYwRuWlGIWkdg--1vesoya0OpGR9V6nFFW5DsEs5ARzBFkxkIxbgiNDOXC-t7oa9u578IDg2fO0FX4BXguvhqJi8lFjltEbjP4EL8LaHmkBpEZ3ko9egaNqVc03mFn5Fl5Ndc_uj9a-Ehe1jg==} init_cluster.kerberos_enabled = true init_cluster.mode = k8s init_cluster.start_cluster = true @@ -67,8 +67,8 @@ init_db_build.run_build = true init_password.cluster_name = ${CLUSTER_NAME} init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password = ${PROLE_SECRET:v1:cVWA3Hry0N50ExMf:VKkhmpeYpd9xB_v0zPzb7Hv7cvq7LXl7scOU1wHGS0NSc3Fa} -init_password.db_password_confirm = ${PROLE_SECRET:v1:cVWA3Hry0N50ExMf:VKkhmpeYpd9xB_v0zPzb7Hv7cvq7LXl7scOU1wHGS0NSc3Fa} +init_password.db_password = +init_password.db_password_confirm = init_password.db_username = root init_password.generate_ssh_key = true init_scripts.run_scripts = true @@ -91,14 +91,13 @@ ARTIFACT_REGISTRY = us-west3-docker.pkg.dev/plenary-truck-485623-p7/knoe-system ARTIFACT_REGISTRY_AVAILABLE = true CLUSTER_ENV = prod CLUSTER_NAME = knoe-db -CNPG_ELIGIBLE_NODES = gk3-knoe-dev-0-pool-1-11c87d92-8qr5,gk3-knoe-dev-0-pool-1-88b7392b-62c4,gk3-knoe-dev-0-pool-1-88b7392b-d6lb +CNPG_ELIGIBLE_NODES = gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z CNPG_PLACEMENT_PLAN_FILE = /Users/chrisfu/dev/prole/conf/cnpg-placement/knoe-system-knoe-db.json -CNPG_PLACEMENT_PLAN_HASH = 334860873593911a -CNPG_PLACEMENT_PLAN_ID = cnpg-placement-334860873593911a -CNPG_STAGE1_NODE = gk3-knoe-dev-0-pool-1-11c87d92-8qr5 -DATABASE_NAMESPACE = knoe-db-0 +CNPG_PLACEMENT_PLAN_HASH = c1213b43098112f3 +CNPG_PLACEMENT_PLAN_ID = cnpg-placement-c1213b43098112f3 +CNPG_STAGE1_NODE = gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z +DATABASE_NAMESPACE = knoe-db-18-008-18-009-18-013 DB_HOST_PORT = 5432 -DB_PASSWORD = ${PROLE_SECRET:v1:iPFMI8pPl5-KJ9Fp:rMxZd-E-FjuTvzcjxlR5ox9wuJ2lEOwduEjZ72QOsSgt153s} DEPLOYMENT_MODE = k8s DEPLOYMENT_TARGET = prole-prod-cluster DOCKER_PRELOAD = false @@ -107,7 +106,7 @@ KUBECONFIG = /Users/chrisfu/.kube/config KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-0 OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2 PROLE_K3S_SERVER = https://myrddin.prole.org:6443 -PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:Dq3A_1cu9f2v_AbU:OJyqpneBdSuXI9ej-nVVh8SIJx-7hmY9Wg4FCu313zk76rNw7STN6ymVudSnJZtUUSKP4IxLoA-bcUkndm4CZWNWCNMnQ-WpvXBIG4GIBeExmD15FcOIJy1VaeW5To0hrKBEEgtwov3EiWr8aVEvpgaUlUfir2m_SbLwpQ==} +PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:10g1WjSwQq3MiUNh:XkobEkCjr8C5VHxPEFeXEOTA6CA5olncCUpaRgUNZuwrsWBKR_p0jeo5xAKcsRkYT-s1mrmV4ZaqgP93JfBGnCybgC5kHJe-ch_UVf4rkJKV75kVnHzrSlIRYZdng-o1_rwfLEARxJ2nLHvmNLnLV6n-TZUmSy3PZYfOeg==} SERVICE_NAMESPACE = knoe-system SUPABASE_PV_BASE = /synology/d005 SUPABASE_PV_BASE_DIR = /synology/d005 @@ -167,7 +166,7 @@ SUPABASE_ENABLED = False [Initialize Cluster] ENVIRONMENT = prod K3S_SERVER_URL = https://myrddin.prole.org:6443 -K3S_TOKEN = ${PROLE_SECRET:v1:ebAZ1lr6dlSQnleP:PUuIW7MUrt-HM_wwAW5BJep9wY5iwf9QuItmnsi627esMDWpZdfcYlBOCa5FrGR8Zhz8y9lB5GOIHseFz31GQMq6Ogbisurue7TWL2kSJS0oFGYfz1FH0zuvtIWwwFwArAubY2FDld9bjAZkr5w2hGzg2A9UnmPkuCslAg==} +K3S_TOKEN = ${PROLE_SECRET:v1:ILhG782khL7eyT7y:b3s76jimM0Q7hYH7hnSYM2XOfBXknl9mXvjzfLI4jTz-c4dHLCJXG8WlOiFf8FM0HfoIK-xu7NxSdym2UXBzCkuENFIJ2Y1GwNr9Nt2ForgfuwKaYVfoGnDc0ERp_S6qaKP01uLDeR95hUS56Wtuh3TNPN0gp14qvBjX4g==} [Dev Cluster (k3d)] CLUSTER_ENV = dev @@ -179,7 +178,7 @@ MODE = k3d CLUSTER_ENV = prole-service-cluster DISPLAY_NAME = prole-service-cluster K3S_SERVER_URL = https://myrddin.prole.org:6443 -K3S_TOKEN = ${PROLE_SECRET:v1:PRJRFNuCskXkEtvt:oh5MP9bB-iEjgP_E7FlqgNpi_5VV-W7tDt1sjz-Fsigy8nrGuliliPyUEz2sqPouMnq_qlfqKTi_vC7G3bIYGQpbf55xX2GXebQBvjUJIfrujGq7Yk-8IQRWBBhfWKcQdTy0EcT99ynwHmjTlI0o56JV5pP1GiCP59TmDw==} +K3S_TOKEN = ${PROLE_SECRET:v1:wxpiSOY6t_5_gaiT:vfyzLLUJUYc3SmQ48yDYrTpeWQ2LfVoiEWvzRwDPzm1Mx6a6aW0QjJLK52Rbwe5hVtwwDjcglakhM8eBjFV-ceFaLIl7Tsr7CwARLuPD6NBJQrc1qnHuKTh5DOSMGF_emnP_fWT06-M242VZImmVE_2Q82Cr45Gd91h93g==} MODE = k3s [GCP] @@ -187,10 +186,6 @@ BILLING_ACCOUNT = 01193C-25783B-3211AD BILLING_PROJECT = plenary-truck-485623-p7 ORG_ID = 584001916389 PROJECT_ID = plenary-truck-485623-p7 -billing_account = 01193C-25783B-3211AD -billing_project = plenary-truck-485623-p7 -org_id = 584001916389 -project_id = plenary-truck-485623-p7 [Prod Cluster (k8s)] ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging @@ -206,7 +201,7 @@ Configure routing, TLS, and frontdoor domain mappings Prepare migration artifacts from source environment Generate OpenTofu plan and apply production pipeline MODE = k8s -OPENTOFU_VARS = {"admin_user":"knoe_admin","app_database":"knoey","app_user":"knoey_app","backup_bucket":"knoe-0-backups","backup_provider":"gcs","backup_retention_days":14,"bootstrap_admin_email":"admin@knoey.com","cluster_name":"knoe-dev-0","cnpg_cluster_name":"knoe-db","db_instances":3,"db_storage_class":"premium-rwo","db_storage_size_gi":100,"ecosystem_id":0,"environment":"production","frontdoor_host":"knoey.com","gcp_project_id":"plenary-truck-485623-p7","gcp_region":"\u2713 us-west3","google_hosted_domain":"","meta_database":"knoe_meta","migration_continuous_until_cutover":false,"migration_mode":"snapshot-restore","migration_source_database":"knoey","migration_source_environment":"prole.org","migration_source_host":"knoe-local-db.prole.org","migration_source_password_ref":"secretref://local-source-db-password","migration_source_port":5432,"migration_source_user":"replication_user","namespace":"ecosystem-0","oidc_client_id_ref":"secretref://google-oidc-client-id","oidc_client_secret_ref":"secretref://google-oidc-client-secret","oidc_issuer":"https://accounts.google.com","platform_domain":"knoe.dev","postgres_version":"16","project_id":"plenary-truck-485623-p7","region":"\u2713 us-west3","tls_mode":"managed","wal_bucket":"knoe-0-wal","workload_identity_sa":"cnpg-backup"} +OPENTOFU_VARS = {"admin_user":"knoe_admin","app_database":"knoey","app_user":"knoey_app","backup_bucket":"knoe-0-backups","backup_provider":"gcs","backup_retention_days":14,"bootstrap_admin_email":"admin@knoey.com","cluster_name":"knoe-dev-0","cnpg_cluster_name":"knoe-db","db_instances":3,"db_pgdata_size_gi":100,"db_pgdata_storage_class":"premium-rwo","db_storage_class":"premium-rwo","db_storage_size_gi":100,"db_wal_size_gi":25,"db_wal_storage_class":"premium-rwo","ecosystem_id":0,"environment":"production","frontdoor_host":"knoey.com","gcp_project_id":"plenary-truck-485623-p7","gcp_region":"\u2713 us-west3","google_hosted_domain":"","meta_database":"knoe_meta","migration_continuous_until_cutover":false,"migration_mode":"snapshot-restore","migration_source_database":"knoey","migration_source_environment":"prole.org","migration_source_host":"knoe-local-db.prole.org","migration_source_password_ref":"secretref://local-source-db-password","migration_source_port":5432,"migration_source_user":"replication_user","namespace":"ecosystem-0","oidc_client_id_ref":"secretref://google-oidc-client-id","oidc_client_secret_ref":"secretref://google-oidc-client-secret","oidc_issuer":"https://accounts.google.com","platform_domain":"knoe.dev","postgres_version":"16","project_id":"plenary-truck-485623-p7","region":"\u2713 us-west3","tls_mode":"managed","wal_bucket":"knoe-0-wal","workload_identity_sa":"cnpg-backup"} PRODUCTION_CONFIG_YAML = kind: KnoeProductionConfig metadata: ecosystemId: 0 @@ -231,6 +226,10 @@ spec: instances: 3 storageClass: premium-rwo storageSizeGi: 100 + pgdataStorageClass: premium-rwo + walStorageClass: premium-rwo + pgdataSizeGi: 100 + walSizeGi: 25 appDatabase: knoey metaDatabase: knoe_meta appUser: knoey_app @@ -268,7 +267,7 @@ spec: ; No configuration values captured yet for this section. [Initialization Scripts] -STATUS = Attempted +; No configuration values captured yet for this section. [Deployment] MODE = k3d diff --git a/deploy/gcp/gke/knoe-db.yaml b/deploy/gcp/gke/knoe-db.yaml index d473a1e..f764de9 100644 --- a/deploy/gcp/gke/knoe-db.yaml +++ b/deploy/gcp/gke/knoe-db.yaml @@ -14,12 +14,12 @@ spec: affinity: enablePodAntiAffinity: true - # Keep spread as a preference during baseline Autopilot bring-up so 3 pods can still schedule - # when the cluster is temporarily constrained; strict topology can be enforced in later rollout. + # Keep spread as a preference for small dedicated Standard DB clusters so 3 pods can still + # schedule while nodes reconcile; strict topology can be enforced in later rollout. podAntiAffinityType: preferred topologyKey: kubernetes.io/hostname # physical node boundary (not zone) tolerations: - # Allow scheduling on GKE Spot nodes (common in knoe-dev-0 to reduce cost). + # Allow scheduling on GKE Spot nodes when explicitly enabled for this DB cluster. # Without this toleration the cluster-autoscaler predicate simulation fails # for any MIG whose nodes carry the spot taint, blocking scale-up entirely. - key: "cloud.google.com/gke-spot" @@ -117,24 +117,24 @@ spec: enableSuperuserAccess: true storage: - size: 20Gi + size: 50Gi pvcTemplate: accessModes: - ReadWriteOnce resources: requests: - storage: 20Gi - storageClassName: standard-rwo # pd-balanced: faster dynamic provisioning in GKE Autopilot than pd-ssd + storage: 50Gi + storageClassName: premium-rwo # pd-ssd; 3×50Gi PGDATA + 3×50Gi WAL = 300Gi total (fits 300GB quota) walStorage: - size: 5Gi + size: 50Gi pvcTemplate: accessModes: - ReadWriteOnce resources: requests: - storage: 5Gi - storageClassName: standard-rwo + storage: 50Gi + storageClassName: premium-rwo monitoring: enablePodMonitor: true # GKE managed Prometheus diff --git a/install.sh b/install.sh index 313df10..d4a6579 100755 --- a/install.sh +++ b/install.sh @@ -6,6 +6,23 @@ # # Bootstrap via curl (first-time install): # curl -fsSL https://knoey.com/install.sh | bash +# +# --------------------------------------------------------------------------- +# Dual-cluster GKE architecture (prod / k8s mode) +# --------------------------------------------------------------------------- +# knoe-dev-0 — Autopilot cluster: app workloads (GitLab, platform), service mesh knoe-0 +# knoe-cnpg-0 — Standard GKE cluster: CloudNative-PG only (rw/ro/r pod separation) +# 3 nodes × e2-standard-2, node pool: cnpg-db-pool +# +# GCP quota requirements (project: plenary-truck-485623-p7, region: us-west3): +# pd-standard : node boot disks (3 × 50 GB = 150 GB — does NOT count against pd-ssd quota) +# pd-ssd : PGDATA PVCs (3 × 50 Gi = 150 Gi) +# WAL PVCs (3 × 50 Gi = 150 Gi) +# Total pd-ssd = 300 Gi (requires 300 GB pd-ssd regional quota) +# +# To patch an existing knoe-cnpg-0 cluster with wrong boot disk type: +# ./scripts/patch_clusters.sh # dry-run first: DRY_RUN=true ./scripts/patch_clusters.sh +# --------------------------------------------------------------------------- set -euo pipefail @@ -14,6 +31,25 @@ PROLE_REPO_URL="${PROLE_REPO_URL:-https://gitlab.knoey.com/prole/prole.git}" # Default install directory for bootstrap installs. _INSTALL_DIR="${PROLE_HOME:-$HOME/prole}" +_check_gcp_tools() { + local missing=0 + if ! command -v gcloud >/dev/null 2>&1; then + echo "Error: gcloud CLI is required for GKE cluster management." >&2 + echo " Install: https://cloud.google.com/sdk/docs/install" >&2 + missing=1 + fi + if ! command -v kubectl >/dev/null 2>&1; then + echo "Error: kubectl is required for Kubernetes cluster operations." >&2 + echo " Install: https://kubernetes.io/docs/tasks/tools/" >&2 + missing=1 + fi + if [[ "$missing" -ne 0 ]]; then + echo "" >&2 + echo "Install missing tools and re-run install.sh." >&2 + exit 1 + fi +} + _bootstrap() { echo "==> Bootstrapping Prole installer..." command -v git >/dev/null 2>&1 || { echo "Error: git is required." >&2; exit 1; } @@ -40,6 +76,7 @@ if [[ -n "${_script_dir}" && -d "${_script_dir}/knoe" ]]; then # Prefer the venv Python at PROLE_HOME (or the repo root) when available; # it carries all prole_requirements.txt dependencies. _VENV_PYTHON="${PROLE_HOME:-${_script_dir}}/bin/python3" + _check_gcp_tools if [[ -x "${_VENV_PYTHON}" ]]; then exec "${_VENV_PYTHON}" -m knoe.ui.screens "$@" fi diff --git a/knoe/core/actions.py b/knoe/core/actions.py index d11213e..a8d0e8f 100644 --- a/knoe/core/actions.py +++ b/knoe/core/actions.py @@ -88,6 +88,13 @@ from knoe.core.ops import monitoring as monitoring_ops from knoe.core.ops import openbao as openbao_ops from knoe.core.ops import opentofu as opentofu_ops from knoe.core.ops import registry as registry_ops +from knoe.core.ops.gke_clusters import ( + GkeClusterSpec, + build_kubectl_env_for_cluster, + ensure_app_cluster, + ensure_db_cluster, + get_cluster_credentials, +) from knoe.core.ops.cloudnative_pg import ( initialize as cnpg_initialize, deploy as cnpg_deploy, @@ -108,6 +115,16 @@ from knoe.core.stream_exec import run_streaming_cmd from typing import Callable, Sequence +DEFAULT_APP_CLUSTER_NAME = "knoe-dev-0" +DEFAULT_APP_CLUSTER_MODE = "autopilot" +DEFAULT_DB_CLUSTER_NAME = "knoe-cnpg-0" +DEFAULT_DB_CLUSTER_MODE = "standard" +DEFAULT_DB_CLUSTER_NODE_COUNT = 3 +DEFAULT_DB_CLUSTER_MACHINE_TYPE = "e2-standard-2" +DEFAULT_DB_BOOT_DISK_TYPE = "pd-standard" +DEFAULT_DB_BOOT_DISK_SIZE_GB = 50 + + def _configure_unbuffered_io(): os.environ.setdefault("PYTHONUNBUFFERED", "1") for stream in (sys.stdout, sys.stderr): @@ -1480,14 +1497,22 @@ class KnoeInstaller: self.prole_cfg_data["Port Forwards"] = pf_section # --------------------------------------------- script env - def _script_env_for_namespace(self, namespace: str) -> dict: + def _script_env_for_namespace(self, namespace: str, cluster_role: str = "db") -> dict: namespace = _safe_str(namespace) + cluster_role = "app" if str(cluster_role).strip().lower() == "app" else "db" env = os.environ.copy() root = getattr(self, "project_root", None) or PROJECT_ROOT + cluster_name = ( + self._app_cluster_name() if cluster_role == "app" else self._cnpg_cluster_name() + ) env["PROLE_HOME"] = str(root) env["PROLE_SERVICE"] = str(root) env["DATABASE_NAMESPACE"] = namespace - env["CLUSTER_NAME"] = self._cnpg_cluster_name() + env["CLUSTER_NAME"] = cluster_name + env["APP_CLUSTER_NAME"] = self._app_cluster_name() + env["DB_CLUSTER_NAME"] = self._cnpg_cluster_name() + env["APP_CLUSTER_MODE"] = self._app_cluster_mode() + env["DB_CLUSTER_MODE"] = self._db_cluster_mode() # Transitional compatibility for scripts still reading NAMESPACE. env["NAMESPACE"] = namespace service_ns = self._service_namespace() @@ -1597,7 +1622,9 @@ class KnoeInstaller: # For GKE/prod clusters, gcloud get-credentials writes to ~/.kube/config. # Read the context that was saved after cluster selection. gke_context = ( - (self.prole_cfg_data.get("Global", {}) or {}).get("KUBECONTEXT", "") or "" + self._cluster_kubecontext(cluster_role) + or (self.prole_cfg_data.get("Global", {}) or {}).get("KUBECONTEXT", "") + or "" ).strip() default_kube = str(Path.home() / ".kube" / "config") if Path(default_kube).exists(): @@ -1716,19 +1743,73 @@ class KnoeInstaller: ) except Exception as e: self.err(f"[WARN] CNPG placement planning failed; using fallback topology: {e}") - - return env + return build_kubectl_env_for_cluster( + base_env=env, + kubecontext=self._cluster_kubecontext(cluster_role), + cluster_name=cluster_name, + cluster_role=cluster_role, + ) def _cnpg_cluster_name(self) -> str: glob = self.prole_cfg_data.get("Global", {}) or {} cluster_name = ( os.environ.get("CLUSTER_NAME") or os.environ.get("CNPG_CLUSTER_NAME") + or self._get_input("init_password.db_cluster_name", "") + or self._get_input("env_setup.DB_CLUSTER_NAME", "") or str(glob.get("CLUSTER_NAME") or "") or str(glob.get("CNPG_CLUSTER_NAME") or "") ).strip() return cluster_name or "knoe-db" + def _app_cluster_name(self) -> str: + glob = self.prole_cfg_data.get("Global", {}) or {} + name = ( + self._get_input("init_password.app_cluster_name", "") + or self._get_input("env_setup.APP_CLUSTER_NAME", "") + or str(glob.get("APP_CLUSTER_NAME") or "") + or DEFAULT_APP_CLUSTER_NAME + ) + return str(name).strip() or DEFAULT_APP_CLUSTER_NAME + + def _app_cluster_mode(self) -> str: + glob = self.prole_cfg_data.get("Global", {}) or {} + mode = ( + self._get_input("init_cluster.app_cluster_mode", "") + or self._get_input("env_setup.APP_CLUSTER_MODE", "") + or str(glob.get("APP_CLUSTER_MODE") or "") + or DEFAULT_APP_CLUSTER_MODE + ) + return str(mode).strip() or DEFAULT_APP_CLUSTER_MODE + + def _db_cluster_mode(self) -> str: + glob = self.prole_cfg_data.get("Global", {}) or {} + mode = ( + self._get_input("init_cluster.db_cluster_mode", "") + or self._get_input("env_setup.DB_CLUSTER_MODE", "") + or str(glob.get("DB_CLUSTER_MODE") or "") + or DEFAULT_DB_CLUSTER_MODE + ) + return str(mode).strip() or DEFAULT_DB_CLUSTER_MODE + + def _cluster_kubecontext(self, role: str) -> str: + key = ( + "init_cluster.app_cluster_kubecontext" + if role == "app" + else "init_cluster.db_cluster_kubecontext" + ) + fallback_key = ( + "env_setup.APP_CLUSTER_KUBECONTEXT" + if role == "app" + else "env_setup.DB_CLUSTER_KUBECONTEXT" + ) + context = (self._get_input(key, "") or self._get_input(fallback_key, "")).strip() + if context: + return context + if role == "app": + return (self._get_input("init_cluster.selected_kubectx", "") or "").strip() + return "" + def _cnpg_desired_instances(self) -> int: glob = self.prole_cfg_data.get("Global", {}) or {} raw = (os.environ.get("CNPG_INSTANCES") or str(glob.get("CNPG_INSTANCES") or "")).strip() @@ -1893,6 +1974,12 @@ class KnoeInstaller: f"data_path={provisioned.data_path} wal_path={provisioned.wal_path}" ) + def _cluster_storage_milestone_enabled(self) -> bool: + raw = str( + os.environ.get("PROLE_ENABLE_CLUSTER_STORAGE_MILESTONE", "") + ).strip().lower() + return raw in {"1", "true", "yes", "on"} + # --------------------------------------------- authority / repair def _authority_context_missing(self) -> bool: enabled = self._get_input_bool( @@ -3394,12 +3481,74 @@ class KnoeConsoleInstaller(KnoeInstaller): cluster_name = _expand_cfg_value(sec.get("CLUSTER_NAME", ""), cfg_vars) legacy["init_password.cluster_name"] = cluster_name legacy["env_setup.CLUSTER_NAME"] = cluster_name + legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name) + legacy.setdefault("init_password.db_cluster_name", cluster_name) if "CNPG_CLUSTER_NAME" in sec and "init_password.cluster_name" not in legacy: cluster_name = _expand_cfg_value( sec.get("CNPG_CLUSTER_NAME", ""), cfg_vars ) legacy["init_password.cluster_name"] = cluster_name legacy["env_setup.CLUSTER_NAME"] = cluster_name + legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name) + legacy.setdefault("init_password.db_cluster_name", cluster_name) + if "APP_CLUSTER_NAME" in sec: + app_cluster_name = _expand_cfg_value( + sec.get("APP_CLUSTER_NAME", ""), cfg_vars + ) + if app_cluster_name: + legacy["env_setup.APP_CLUSTER_NAME"] = app_cluster_name + legacy["init_password.app_cluster_name"] = app_cluster_name + if "APP_CLUSTER_MODE" in sec: + legacy["env_setup.APP_CLUSTER_MODE"] = _expand_cfg_value( + sec.get("APP_CLUSTER_MODE", ""), cfg_vars + ) + legacy["init_cluster.app_cluster_mode"] = legacy[ + "env_setup.APP_CLUSTER_MODE" + ] + if "DB_CLUSTER_NAME" in sec: + db_cluster_name = _expand_cfg_value( + sec.get("DB_CLUSTER_NAME", ""), cfg_vars + ) + if db_cluster_name: + legacy["env_setup.DB_CLUSTER_NAME"] = db_cluster_name + legacy["init_password.db_cluster_name"] = db_cluster_name + if "DB_CLUSTER_MODE" in sec: + legacy["env_setup.DB_CLUSTER_MODE"] = _expand_cfg_value( + sec.get("DB_CLUSTER_MODE", ""), cfg_vars + ) + legacy["init_cluster.db_cluster_mode"] = legacy["env_setup.DB_CLUSTER_MODE"] + if "DB_CLUSTER_NODE_COUNT" in sec: + legacy["init_cluster.db_cluster_node_count"] = _expand_cfg_value( + sec.get("DB_CLUSTER_NODE_COUNT", ""), cfg_vars + ) + if "DB_CLUSTER_MACHINE_TYPE" in sec: + legacy["init_cluster.db_cluster_machine_type"] = _expand_cfg_value( + sec.get("DB_CLUSTER_MACHINE_TYPE", ""), cfg_vars + ) + if "DB_BOOT_DISK_TYPE" in sec: + legacy["init_cluster.db_boot_disk_type"] = _expand_cfg_value( + sec.get("DB_BOOT_DISK_TYPE", ""), cfg_vars + ) + if "DB_BOOT_DISK_SIZE_GB" in sec: + legacy["init_cluster.db_boot_disk_size_gb"] = _expand_cfg_value( + sec.get("DB_BOOT_DISK_SIZE_GB", ""), cfg_vars + ) + if "DB_CLUSTER_REGION" in sec: + legacy["init_cluster.db_cluster_region"] = _expand_cfg_value( + sec.get("DB_CLUSTER_REGION", ""), cfg_vars + ) + if "APP_CLUSTER_KUBECONTEXT" in sec: + kubecontext = _expand_cfg_value( + sec.get("APP_CLUSTER_KUBECONTEXT", ""), cfg_vars + ) + legacy["env_setup.APP_CLUSTER_KUBECONTEXT"] = kubecontext + legacy["init_cluster.app_cluster_kubecontext"] = kubecontext + if "DB_CLUSTER_KUBECONTEXT" in sec: + kubecontext = _expand_cfg_value( + sec.get("DB_CLUSTER_KUBECONTEXT", ""), cfg_vars + ) + legacy["env_setup.DB_CLUSTER_KUBECONTEXT"] = kubecontext + legacy["init_cluster.db_cluster_kubecontext"] = kubecontext if "KNOE_DB_USER" in sec: legacy["init_password.db_username"] = _expand_cfg_value( sec.get("KNOE_DB_USER", ""), cfg_vars @@ -3605,10 +3754,18 @@ class KnoeConsoleInstaller(KnoeInstaller): inputs[f"env_setup.{k}"] = env_vals.get(k, "") inputs["env_setup.DATABASE_NAMESPACE"] = namespace inputs["env_setup.CLUSTER_NAME"] = cluster_name + inputs["env_setup.APP_CLUSTER_NAME"] = DEFAULT_APP_CLUSTER_NAME + inputs["env_setup.APP_CLUSTER_MODE"] = DEFAULT_APP_CLUSTER_MODE + inputs["env_setup.DB_CLUSTER_NAME"] = DEFAULT_DB_CLUSTER_NAME + inputs["env_setup.DB_CLUSTER_MODE"] = DEFAULT_DB_CLUSTER_MODE + inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = "" + inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = "" # Database creation inputs["init_password.db_namespace"] = namespace inputs["init_password.cluster_name"] = cluster_name + inputs["init_password.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME + inputs["init_password.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME inputs["init_password.db_username"] = owner inputs["init_password.db_password"] = "" inputs["init_password.db_password_confirm"] = "" @@ -3628,7 +3785,19 @@ class KnoeConsoleInstaller(KnoeInstaller): inputs["init_cluster.k3s_token"] = "" inputs["init_cluster.supabase_enabled"] = _bool_str(False) inputs["init_cluster.supabase_studio_enabled"] = _bool_str(False) - inputs["init_cluster.supabase_studio_url"] = "db.prole.org" + inputs["init_cluster.supabase_studio_url"] = "db.knoe.org" + inputs["init_cluster.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME + inputs["init_cluster.app_cluster_mode"] = DEFAULT_APP_CLUSTER_MODE + inputs["init_cluster.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME + inputs["init_cluster.db_cluster_mode"] = DEFAULT_DB_CLUSTER_MODE + inputs["init_cluster.db_cluster_node_count"] = str(DEFAULT_DB_CLUSTER_NODE_COUNT) + inputs["init_cluster.db_cluster_machine_type"] = DEFAULT_DB_CLUSTER_MACHINE_TYPE + inputs["init_cluster.db_boot_disk_type"] = DEFAULT_DB_BOOT_DISK_TYPE + inputs["init_cluster.db_boot_disk_size_gb"] = str(DEFAULT_DB_BOOT_DISK_SIZE_GB) + inputs["init_cluster.db_cluster_region"] = "" + inputs["init_cluster.db_cluster_zones"] = "" + inputs["init_cluster.app_cluster_kubecontext"] = "" + inputs["init_cluster.db_cluster_kubecontext"] = "" inputs["init_cluster.supabase_auth_enabled"] = _bool_str(True) inputs["init_cluster.supabase_realtime_enabled"] = _bool_str(True) inputs["init_cluster.supabase_meta_enabled"] = _bool_str(True) @@ -5064,6 +5233,10 @@ class KnoeConsoleInstaller(KnoeInstaller): ) except Exception: pass + self.inputs["init_cluster.app_cluster_kubecontext"] = f"k3d-{cluster_name}" + self.inputs["init_cluster.db_cluster_kubecontext"] = f"k3d-{cluster_name}" + self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}" + self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}" else: # Ensure we don't pin KUBECONFIG to a stale local file when another # kubeconfig (e.g. Ansible-fetched client-cert auth) is available. @@ -5099,6 +5272,100 @@ class KnoeConsoleInstaller(KnoeInstaller): os.environ["KUBECONFIG"] = str(fetched) self.log(f"Using fetched kubeconfig: {fetched}") + if mode == "k8s": + gcp_cfg = self.gcp_cfg if isinstance(self.gcp_cfg, dict) else {} + project_id = str( + gcp_cfg.get("project_id") + or gcp_cfg.get("PROJECT_ID") + or self._get_input("init_cluster.project_id", "") + or "" + ).strip() + if not project_id: + raise Exception("GCP project_id is required for dual-cluster GKE setup.") + + app_cluster_name = self._app_cluster_name() + app_cluster_mode = self._app_cluster_mode() + db_cluster_name = self._cnpg_cluster_name() + db_cluster_mode = self._db_cluster_mode() + db_node_count = int( + (self._get_input("init_cluster.db_cluster_node_count", "") or "3").strip() or "3" + ) + db_machine_type = ( + self._get_input("init_cluster.db_cluster_machine_type", "") + or DEFAULT_DB_CLUSTER_MACHINE_TYPE + ).strip() or DEFAULT_DB_CLUSTER_MACHINE_TYPE + db_boot_disk_type = ( + self._get_input("init_cluster.db_boot_disk_type", "") + or DEFAULT_DB_BOOT_DISK_TYPE + ).strip() or DEFAULT_DB_BOOT_DISK_TYPE + db_boot_disk_size_gb = int( + ( + self._get_input("init_cluster.db_boot_disk_size_gb", "") + or str(DEFAULT_DB_BOOT_DISK_SIZE_GB) + ).strip() + or str(DEFAULT_DB_BOOT_DISK_SIZE_GB) + ) + db_location = ( + self._get_input("init_cluster.db_cluster_region", "") + or gcp_cfg.get("region") + or gcp_cfg.get("REGION") + or gcp_cfg.get("zone") + or gcp_cfg.get("ZONE") + or "us-central1" + ) + app_location = ( + self._get_input("init_cluster.app_cluster_region", "") + or gcp_cfg.get("region") + or gcp_cfg.get("REGION") + or db_location + ) + + app_spec = GkeClusterSpec( + name=app_cluster_name, + mode=app_cluster_mode, + location=str(app_location).strip(), + ) + db_spec = GkeClusterSpec( + name=db_cluster_name, + mode=db_cluster_mode, + location=str(db_location).strip(), + machine_type=db_machine_type, + node_count=max(3, db_node_count), + node_pool_name="cnpg-db-pool", + boot_disk_type=db_boot_disk_type, + boot_disk_size_gb=db_boot_disk_size_gb, + ) + + self.log(f"[app-cluster {app_cluster_name}] validating cluster") + ensure_app_cluster(project_id=project_id, spec=app_spec, log=self.log) + self.log(f"[db-cluster {db_cluster_name}] ensuring Standard GKE cluster") + ensure_db_cluster(project_id=project_id, spec=db_spec, log=self.log) + + app_ctx = get_cluster_credentials( + project_id=project_id, + cluster_name=app_cluster_name, + location=app_spec.location, + log=self.log, + ) + db_ctx = get_cluster_credentials( + project_id=project_id, + cluster_name=db_cluster_name, + location=db_spec.location, + log=self.log, + ) + self.inputs["init_cluster.app_cluster_kubecontext"] = app_ctx + self.inputs["init_cluster.db_cluster_kubecontext"] = db_ctx + self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = app_ctx + self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = db_ctx + self.prole_cfg_data.setdefault("Global", {})["APP_CLUSTER_NAME"] = app_cluster_name + self.prole_cfg_data.setdefault("Global", {})["APP_CLUSTER_MODE"] = app_cluster_mode + self.prole_cfg_data.setdefault("Global", {})["APP_CLUSTER_KUBECONTEXT"] = app_ctx + self.prole_cfg_data.setdefault("Global", {})["DB_CLUSTER_NAME"] = db_cluster_name + self.prole_cfg_data.setdefault("Global", {})["DB_CLUSTER_MODE"] = db_cluster_mode + self.prole_cfg_data.setdefault("Global", {})["DB_CLUSTER_KUBECONTEXT"] = db_ctx + # Back-compat default context points to app cluster for platform services. + self.prole_cfg_data.setdefault("Global", {})["KUBECONTEXT"] = app_ctx + kubectl = subprocess.run(["which", "kubectl"], capture_output=True) if kubectl.returncode != 0: raise Exception( @@ -5582,52 +5849,51 @@ class KnoeConsoleInstaller(KnoeInstaller): ns = ( self._get_input("init_password.db_namespace", "") or "" ).strip() or "default" - env = self._script_env_for_namespace(ns) + db_env = self._script_env_for_namespace(ns, cluster_role="db") + app_env = self._script_env_for_namespace(ns, cluster_role="app") - opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(env) + opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(app_env) password = self._get_input("init_password.db_password", "") kerberos_enabled = self._get_input_bool("kerberos_config.enabled", False) - steps = [ + pre_cnpg_steps = [ ("init_certmgr.sh", ["initialize"], False), ] if kerberos_enabled: - steps.append(("init_kerberos.sh", ["initialize"], False)) - steps.extend( - [ - ("init_cnpg_backup.sh", ["start"], False), - ("init_kong.sh", ["start"], False), - ] - ) + pre_cnpg_steps.append(("init_kerberos.sh", ["initialize"], False)) + post_cnpg_steps: list[tuple[str, list[str], bool, str]] = [ + ("init_cnpg_backup.sh", ["start"], False, "db"), + ("init_kong.sh", ["start"], False, "app"), + ] if not opt_allowed: self.log(f"[SKIP] Monitoring disabled by policy: {opt_reason}") self.prole_cfg_data.setdefault("Monitoring", {})["STATUS"] = "Skipped" mode = self._deployment_mode() if mode != "k3d": - steps.append(("init_nginx_ingress.sh", ["initialize"], False)) + post_cnpg_steps.append(("init_nginx_ingress.sh", ["initialize"], False, "app")) overall_success = True # Common services (Python owners) — replaces init_common_services.sh - self.log("--> common services (python owners)") + self.log(f"[app-cluster {self._app_cluster_name()}] common services (python owners)") try: - registry_ns = str(env.get("REGISTRY_NAMESPACE") or ns).strip() or ns - registry_ops.update(namespace=registry_ns, env=env, mode=mode, log=self.log) - openbao_ops.update(namespace=ns, env=env, mode=mode, log=self.log) - garage_store_ops.update(namespace=ns, env=env, mode=mode, log=self.log) - opentofu_ops.update(namespace=ns, env=env, mode=mode, log=self.log) + registry_ns = str(app_env.get("REGISTRY_NAMESPACE") or ns).strip() or ns + registry_ops.update(namespace=registry_ns, env=app_env, mode=mode, log=self.log) + openbao_ops.update(namespace=ns, env=app_env, mode=mode, log=self.log) + garage_store_ops.update(namespace=ns, env=app_env, mode=mode, log=self.log) + opentofu_ops.update(namespace=ns, env=app_env, mode=mode, log=self.log) except Exception as e: self.err(f"[ERROR] common services (python owners) failed: {e}") overall_success = False # Run pre-CNPG shell scripts (cert-manager, optional kerberos) - for script, args, needs_password in steps[:1] if not kerberos_enabled else steps[:2]: + for script, args, needs_password in pre_cnpg_steps: self.log(f"--> {script} {' '.join(args)}") stdin_text = f"{password}\n" if needs_password else None rc = self._run_script( script, args=args, - env=env, + env=db_env, stdin_text=stdin_text, on_line=self._process_script_output_line, ) @@ -5637,22 +5903,28 @@ class KnoeConsoleInstaller(KnoeInstaller): # CNPG initialization (Python) — replaces init_cloudnative_pg.sh initialize if overall_success: - self.log("--> cnpg_initialize (Python)") + self.log(f"[db-cluster {self._cnpg_cluster_name()}] cnpg_initialize (Python)") try: self.ensure_db_k8s_secrets(ns, password, log_fn=self.log) except Exception as e: self.err(f"[ERROR] Failed to ensure DB secrets before CNPG init: {e}") overall_success = False if overall_success: - try: - self._ensure_cnpg_storage_provisioned(ns, env) - except Exception as e: - self.err(f"[WARN] Failed to provision CNPG storage before init: {e}") + if self._cluster_storage_milestone_enabled(): + try: + self._ensure_cnpg_storage_provisioned(ns, db_env) + except Exception as e: + self.err(f"[WARN] Failed to provision CNPG storage before init: {e}") + else: + self.log( + "[SKIP] Cluster storage milestone shelved; " + "retaining utility workflows only." + ) try: cnpg_initialize( namespace=ns, - cluster_name=str(env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") or "knoe-db").strip(), - env=env, + cluster_name=str(db_env.get("CLUSTER_NAME") or db_env.get("CNPG_CLUSTER_NAME") or "knoe-db").strip(), + env=db_env, project_root=self.project_root, log=self.log, mode=mode, @@ -5662,15 +5934,15 @@ class KnoeConsoleInstaller(KnoeInstaller): overall_success = False # Run post-CNPG shell scripts (backup, kong, ingress) - post_cnpg_start = 1 if not kerberos_enabled else 2 if overall_success: - for script, args, needs_password in steps[post_cnpg_start:]: + for script, args, needs_password, role in post_cnpg_steps: self.log(f"--> {script} {' '.join(args)}") stdin_text = f"{password}\n" if needs_password else None + target_env = db_env if role == "db" else app_env rc = self._run_script( script, args=args, - env=env, + env=target_env, stdin_text=stdin_text, on_line=self._process_script_output_line, ) @@ -5679,12 +5951,12 @@ class KnoeConsoleInstaller(KnoeInstaller): overall_success = False if overall_success and opt_allowed: - self.log("--> monitoring (python owner) initialize") + self.log(f"[app-cluster {self._app_cluster_name()}] monitoring (python owner) initialize") try: - monitoring_ns = str(env.get("MONITORING_NAMESPACE") or "monitoring").strip() or "monitoring" + monitoring_ns = str(app_env.get("MONITORING_NAMESPACE") or "monitoring").strip() or "monitoring" monitoring_ops.initialize( namespace=monitoring_ns, - env=env, + env=app_env, mode=mode, log=self.log, ) @@ -5722,13 +5994,14 @@ class KnoeConsoleInstaller(KnoeInstaller): self.log("[SKIP] CnPG deploy disabled.") self.prole_cfg_data["Deployment"]["STATUS"] = "Skipped" return - self.log("==> Deploy CloudNative-PG") + db_cluster = self._cnpg_cluster_name() + self.log(f"==> [db-cluster {db_cluster}] Deploy CloudNativePG") ns = (self._get_input("init_password.db_namespace", "") or "").strip() if not ns: ns = (self._get_input("env_setup.DATABASE_NAMESPACE", "") or "").strip() if not ns: ns = "default" - env = self._script_env_for_namespace(ns) + env = self._script_env_for_namespace(ns, cluster_role="db") cluster_name = str( env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") @@ -5738,15 +6011,22 @@ class KnoeConsoleInstaller(KnoeInstaller): cluster_name = "knoe-db" mode = self._deployment_mode() - try: - self._ensure_cnpg_storage_provisioned(ns, env) - except Exception as exc: - self._cnpg_success = False - self.prole_cfg_data["Deployment"]["STATUS"] = "Attempted" - self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}") - return + if self._cluster_storage_milestone_enabled(): + try: + self._ensure_cnpg_storage_provisioned(ns, env) + except Exception as exc: + self._cnpg_success = False + self.prole_cfg_data["Deployment"]["STATUS"] = "Attempted" + self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}") + return + else: + self.log( + "[SKIP] Cluster storage milestone shelved; " + "retaining utility workflows only." + ) try: + self.log(f"[db-cluster {db_cluster}] applying knoe-db cluster") cnpg_deploy( namespace=ns, cluster_name=cluster_name, @@ -5777,7 +6057,7 @@ class KnoeConsoleInstaller(KnoeInstaller): ) has_pods = bool((rc_check.stdout or "").strip()) if has_pods: - self.log("==> Force rollout") + self.log(f"==> [db-cluster {db_cluster}] Force rollout") try: cnpg_rollout( namespace=ns, diff --git a/knoe/core/cluster_storage_browser.py b/knoe/core/cluster_storage_browser.py new file mode 100644 index 0000000..8f2cc50 --- /dev/null +++ b/knoe/core/cluster_storage_browser.py @@ -0,0 +1,416 @@ +from __future__ import annotations + +import json +import re +import subprocess +import time +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from typing import Protocol + + +PD_SSD_QUOTA_METRIC = "SSD_TOTAL_GB" +PD_BALANCED_QUOTA_METRIC = "DISKS_TOTAL_GB" + +STORAGE_CLASS_PREMIUM = "premium-rwo" +STORAGE_CLASS_STANDARD = "standard-rwo" + +_REGION_TOKEN_RE = re.compile(r"^[a-z]+-[a-z]+\d+$") + + +@dataclass(slots=True, frozen=True) +class RegionCatalogEntry: + region: str + timezone_group: str + country: str + + +@dataclass(slots=True, frozen=True) +class QuotaMetricHeadroom: + metric: str + limit_gb: float | None + usage_gb: float | None + + @property + def headroom_gb(self) -> float | None: + if self.limit_gb is None or self.usage_gb is None: + return None + return self.limit_gb - self.usage_gb + + +@dataclass(slots=True, frozen=True) +class RegionQuotaSnapshot: + region: str + premium: QuotaMetricHeadroom + standard: QuotaMetricHeadroom + + +@dataclass(slots=True, frozen=True) +class RegionLatencyResult: + region: str + latency_ms: float | None + ok: bool + error: str = "" + + +@dataclass(slots=True, frozen=True) +class ClusterStorageRequest: + pgdata_gb: float + wal_gb: float + + @property + def total_gb(self) -> float: + return max(0.0, self.pgdata_gb) + max(0.0, self.wal_gb) + + +@dataclass(slots=True, frozen=True) +class RegionFeasibility: + region: str + timezone_group: str + premium_headroom_gb: float | None + standard_headroom_gb: float | None + can_premium: bool + can_standard: bool + default_pgdata_class: str + default_wal_class: str + feasibility_rank: int + notes: tuple[str, ...] + + +@dataclass(slots=True, frozen=True) +class RegionCandidate: + feasibility: RegionFeasibility + latency_ms: float | None + + +@dataclass(slots=True, frozen=True) +class ClusterStorageBrowserResult: + generated_at: str + project_id: str + timezone_group: str + request: ClusterStorageRequest + candidates: tuple[RegionCandidate, ...] + quota_link: str + quota_console_link: str + + def to_dict(self) -> dict: + return { + "generatedAt": self.generated_at, + "projectId": self.project_id, + "timezoneGroup": self.timezone_group, + "request": asdict(self.request), + "candidates": [ + { + "region": c.feasibility.region, + "timezoneGroup": c.feasibility.timezone_group, + "feasibilityRank": c.feasibility.feasibility_rank, + "premiumHeadroomGb": c.feasibility.premium_headroom_gb, + "standardHeadroomGb": c.feasibility.standard_headroom_gb, + "canPremium": c.feasibility.can_premium, + "canStandard": c.feasibility.can_standard, + "defaultPgdataClass": c.feasibility.default_pgdata_class, + "defaultWalClass": c.feasibility.default_wal_class, + "latencyMs": c.latency_ms, + "notes": list(c.feasibility.notes), + } + for c in self.candidates + ], + "quotaLink": self.quota_link, + "quotaConsoleLink": self.quota_console_link, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":"), indent=2) + + +class RegionQuotaProvider(Protocol): + def fetch(self, *, project_id: str, regions: list[str]) -> dict[str, RegionQuotaSnapshot]: ... + + +class RegionLatencyProvider(Protocol): + def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]: ... + + +def default_region_catalog() -> tuple[RegionCatalogEntry, ...]: + return ( + RegionCatalogEntry("us-central1", "americas", "us"), + RegionCatalogEntry("us-east1", "americas", "us"), + RegionCatalogEntry("us-east4", "americas", "us"), + RegionCatalogEntry("us-west1", "americas", "us"), + RegionCatalogEntry("us-west2", "americas", "us"), + RegionCatalogEntry("us-west3", "americas", "us"), + RegionCatalogEntry("us-west4", "americas", "us"), + RegionCatalogEntry("northamerica-northeast1", "americas", "ca"), + RegionCatalogEntry("southamerica-east1", "americas", "br"), + RegionCatalogEntry("europe-west1", "emea", "be"), + RegionCatalogEntry("europe-west2", "emea", "uk"), + RegionCatalogEntry("europe-west3", "emea", "de"), + RegionCatalogEntry("europe-west4", "emea", "nl"), + RegionCatalogEntry("europe-west6", "emea", "ch"), + RegionCatalogEntry("europe-west8", "emea", "it"), + RegionCatalogEntry("europe-west9", "emea", "fr"), + RegionCatalogEntry("europe-west10", "emea", "de"), + RegionCatalogEntry("europe-central2", "emea", "pl"), + RegionCatalogEntry("asia-east1", "apac", "tw"), + RegionCatalogEntry("asia-east2", "apac", "hk"), + RegionCatalogEntry("asia-northeast1", "apac", "jp"), + RegionCatalogEntry("asia-northeast2", "apac", "jp"), + RegionCatalogEntry("asia-northeast3", "apac", "kr"), + RegionCatalogEntry("asia-south1", "apac", "in"), + RegionCatalogEntry("asia-south2", "apac", "in"), + RegionCatalogEntry("asia-southeast1", "apac", "sg"), + RegionCatalogEntry("asia-southeast2", "apac", "id"), + RegionCatalogEntry("australia-southeast1", "apac", "au"), + RegionCatalogEntry("australia-southeast2", "apac", "au"), + ) + + +def detect_timezone_group() -> str: + now_local = datetime.now().astimezone() + offset_hours = float(now_local.utcoffset().total_seconds()) / 3600.0 if now_local.utcoffset() else 0.0 + if -10.0 <= offset_hours <= -2.0: + return "americas" + if -1.0 <= offset_hours <= 4.0: + return "emea" + return "apac" + + +def filter_regions_by_timezone( + regions: list[str], + timezone_group: str, + catalog: tuple[RegionCatalogEntry, ...] | None = None, +) -> list[RegionCatalogEntry]: + entries = catalog or default_region_catalog() + allowed = {r for r in regions if _REGION_TOKEN_RE.match(r)} + return [entry for entry in entries if entry.region in allowed and entry.timezone_group == timezone_group] + + +def _bool_feasible(headroom_gb: float | None, requested_gb: float) -> bool: + if headroom_gb is None: + return False + return headroom_gb >= max(0.0, requested_gb) + + +def _choose_default_class(premium_ok: bool, standard_ok: bool) -> str: + if premium_ok: + return STORAGE_CLASS_PREMIUM + if standard_ok: + return STORAGE_CLASS_STANDARD + return STORAGE_CLASS_PREMIUM + + +def _feasibility_rank(can_premium: bool, can_standard: bool) -> int: + if can_premium: + return 0 + if can_standard: + return 1 + return 2 + + +def evaluate_region_feasibility( + region: str, + timezone_group: str, + quota: RegionQuotaSnapshot | None, + requested: ClusterStorageRequest, +) -> RegionFeasibility: + premium_headroom = quota.premium.headroom_gb if quota is not None else None + standard_headroom = quota.standard.headroom_gb if quota is not None else None + + can_premium = _bool_feasible(premium_headroom, requested.total_gb) + can_standard = _bool_feasible(standard_headroom, requested.total_gb) + + notes: list[str] = [] + if quota is None: + notes.append("quota-unavailable") + if not can_premium and not can_standard: + notes.append("insufficient-quota") + + default_class = _choose_default_class(can_premium, can_standard) + return RegionFeasibility( + region=region, + timezone_group=timezone_group, + premium_headroom_gb=premium_headroom, + standard_headroom_gb=standard_headroom, + can_premium=can_premium, + can_standard=can_standard, + default_pgdata_class=default_class, + default_wal_class=default_class, + feasibility_rank=_feasibility_rank(can_premium, can_standard), + notes=tuple(notes), + ) + + +def _quota_links(project_id: str) -> tuple[str, str]: + project = project_id.strip() + if not project: + return ( + "https://cloud.google.com/compute/quotas", + "https://console.cloud.google.com/iam-admin/quotas", + ) + return ( + "https://cloud.google.com/compute/quotas", + f"https://console.cloud.google.com/iam-admin/quotas?project={project}", + ) + + +def _sort_candidates(candidates: list[RegionCandidate]) -> list[RegionCandidate]: + def _key(candidate: RegionCandidate): + latency = candidate.latency_ms if candidate.latency_ms is not None else float("inf") + return (candidate.feasibility.feasibility_rank, latency, candidate.feasibility.region) + + return sorted(candidates, key=_key) + + +def build_cluster_storage_browser_result( + *, + project_id: str, + requested: ClusterStorageRequest, + available_regions: list[str], + timezone_group: str | None = None, + catalog: tuple[RegionCatalogEntry, ...] | None = None, + quota_provider: RegionQuotaProvider, + latency_provider: RegionLatencyProvider, +) -> ClusterStorageBrowserResult: + tz_group = (timezone_group or detect_timezone_group()).strip().lower() or detect_timezone_group() + candidates = filter_regions_by_timezone(available_regions, tz_group, catalog) + candidate_regions = [entry.region for entry in candidates] + + quota_by_region = quota_provider.fetch(project_id=project_id, regions=candidate_regions) + feasibility: list[RegionFeasibility] = [ + evaluate_region_feasibility( + region=entry.region, + timezone_group=entry.timezone_group, + quota=quota_by_region.get(entry.region), + requested=requested, + ) + for entry in candidates + ] + + probe_regions = [f.region for f in feasibility if f.can_premium or f.can_standard] + latency_by_region = latency_provider.probe(probe_regions) + + combined: list[RegionCandidate] = [] + for f in feasibility: + latency = latency_by_region.get(f.region) + combined.append( + RegionCandidate( + feasibility=f, + latency_ms=latency.latency_ms if latency and latency.ok else None, + ) + ) + + link, console_link = _quota_links(project_id) + return ClusterStorageBrowserResult( + generated_at=datetime.now(timezone.utc).isoformat(), + project_id=project_id, + timezone_group=tz_group, + request=requested, + candidates=tuple(_sort_candidates(combined)), + quota_link=link, + quota_console_link=console_link, + ) + + +def _safe_float(value: str | int | float | None) -> float | None: + if value is None: + return None + try: + return float(value) + except Exception: + return None + + +def _parse_limit_usage(raw: str) -> tuple[float | None, float | None]: + # gcloud output format from --format=value(limit,usage) + parts = [p.strip() for p in raw.split() if p.strip()] + if len(parts) >= 2: + return _safe_float(parts[0]), _safe_float(parts[1]) + if len(parts) == 1: + return _safe_float(parts[0]), None + return None, None + + +class GcloudRegionQuotaProvider: + def __init__(self, timeout_sec: int = 20): + self.timeout_sec = timeout_sec + + def _metric_headroom(self, project_id: str, region: str, metric: str) -> QuotaMetricHeadroom: + cmd = [ + "gcloud", + "compute", + "regions", + "describe", + region, + "--project", + project_id, + "--format=value(quotas[metric=%s].limit,quotas[metric=%s].usage)" % (metric, metric), + ] + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=self.timeout_sec, + ) + if result.returncode != 0: + return QuotaMetricHeadroom(metric=metric, limit_gb=None, usage_gb=None) + limit, usage = _parse_limit_usage(result.stdout.strip()) + return QuotaMetricHeadroom(metric=metric, limit_gb=limit, usage_gb=usage) + except Exception: + return QuotaMetricHeadroom(metric=metric, limit_gb=None, usage_gb=None) + + def fetch(self, *, project_id: str, regions: list[str]) -> dict[str, RegionQuotaSnapshot]: + project = project_id.strip() + if not project: + return {} + out: dict[str, RegionQuotaSnapshot] = {} + for region in regions: + premium = self._metric_headroom(project, region, PD_SSD_QUOTA_METRIC) + standard = self._metric_headroom(project, region, PD_BALANCED_QUOTA_METRIC) + out[region] = RegionQuotaSnapshot(region=region, premium=premium, standard=standard) + return out + + +class PingLatencyProvider: + def __init__(self, timeout_sec: int = 2, count: int = 2): + self.timeout_sec = timeout_sec + self.count = count + + def _probe_one(self, region: str) -> RegionLatencyResult: + # Best-effort: use region DNS style endpoint as latency anchor. + host = f"{region}.gcping.com" + cmd = ["ping", "-c", str(self.count), "-t", str(self.timeout_sec), host] + started = time.monotonic() + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=max(2, self.timeout_sec + 2), + ) + elapsed = (time.monotonic() - started) * 1000.0 + if result.returncode == 0: + return RegionLatencyResult(region=region, latency_ms=round(elapsed, 2), ok=True) + return RegionLatencyResult(region=region, latency_ms=None, ok=False, error="ping-failed") + except Exception as exc: + return RegionLatencyResult(region=region, latency_ms=None, ok=False, error=str(exc)) + + def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]: + return {region: self._probe_one(region) for region in regions} + + +class StaticQuotaProvider: + def __init__(self, values: dict[str, RegionQuotaSnapshot]): + self.values = dict(values) + + def fetch(self, *, project_id: str, regions: list[str]) -> dict[str, RegionQuotaSnapshot]: + _ = project_id + return {r: self.values[r] for r in regions if r in self.values} + + +class StaticLatencyProvider: + def __init__(self, values: dict[str, RegionLatencyResult]): + self.values = dict(values) + + def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]: + return {r: self.values[r] for r in regions if r in self.values} diff --git a/knoe/core/milestones.py b/knoe/core/milestones.py index 3e6e3b9..75244ac 100644 --- a/knoe/core/milestones.py +++ b/knoe/core/milestones.py @@ -300,6 +300,17 @@ class DatabaseCreationMilestone(Milestone): if not cluster_name: cluster_name = "knoe-db" + app_cluster_name = ( + state.inputs.get("init_password.app_cluster_name", "") + or state.inputs.get("env_setup.APP_CLUSTER_NAME", "") + or "knoe-dev-0" + ).strip() or "knoe-dev-0" + db_cluster_name = ( + state.inputs.get("init_password.db_cluster_name", "") + or state.inputs.get("env_setup.DB_CLUSTER_NAME", "") + or "knoe-cnpg-0" + ).strip() or "knoe-cnpg-0" + user = (state.inputs.get("init_password.db_username", "") or "").strip() p1 = state.inputs.get("init_password.db_password", "") @@ -356,12 +367,24 @@ class DatabaseCreationMilestone(Milestone): state.inputs["env_setup.DATABASE_NAMESPACE"] = ns state.inputs["init_password.cluster_name"] = cluster_name state.inputs["env_setup.CLUSTER_NAME"] = cluster_name + state.inputs["init_password.app_cluster_name"] = app_cluster_name + state.inputs["env_setup.APP_CLUSTER_NAME"] = app_cluster_name + state.inputs["init_password.db_cluster_name"] = db_cluster_name + state.inputs["env_setup.DB_CLUSTER_NAME"] = db_cluster_name + + self.logger.info(f"Application Cluster: {app_cluster_name} (Autopilot)") + self.logger.info(f"Database Cluster: {db_cluster_name} (Standard)") + self.logger.info( + f"CloudNativePG targets dedicated DB cluster '{db_cluster_name}' with CNPG cluster name '{cluster_name}'." + ) if "Database Creation" not in state.config_data: state.config_data["Database Creation"] = {} state.config_data["Database Creation"]["DB_USER"] = user state.config_data["Database Creation"]["DATABASE_NAMESPACE"] = ns state.config_data["Database Creation"]["CLUSTER_NAME"] = cluster_name + state.config_data["Database Creation"]["APP_CLUSTER_NAME"] = app_cluster_name + state.config_data["Database Creation"]["DB_CLUSTER_NAME"] = db_cluster_name state.config_data["Database Creation"].pop("DB_NAME", None) state.config_data["Database Creation"].pop("NAMESPACE", None) diff --git a/knoe/core/ops/gke_clusters.py b/knoe/core/ops/gke_clusters.py new file mode 100644 index 0000000..8c6f09b --- /dev/null +++ b/knoe/core/ops/gke_clusters.py @@ -0,0 +1,181 @@ +"""Helpers for explicit app/db GKE cluster targeting in installer flows.""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from typing import Callable + + +_LogFn = Callable[[str], None] + + +@dataclass(frozen=True) +class GkeClusterSpec: + name: str + mode: str + location: str + machine_type: str = "e2-small" + node_count: int = 3 + node_pool_name: str = "default-pool" + boot_disk_type: str = "pd-standard" # pd-standard keeps boot disks off pd-ssd quota + boot_disk_size_gb: int = 50 + + +def _run(cmd: list[str], log: _LogFn | None = None) -> subprocess.CompletedProcess: + if log: + log("$ " + " ".join(cmd)) + return subprocess.run(cmd, capture_output=True, text=True) + + +def build_kubectl_env_for_cluster( + base_env: dict | None, + kubecontext: str, + cluster_name: str, + cluster_role: str, +) -> dict: + env = dict(base_env or os.environ) + if kubecontext: + env["KUBECTL_CONTEXT"] = kubecontext + env["CLUSTER_NAME"] = cluster_name + env["KNOE_CLUSTER_ROLE"] = cluster_role + if cluster_role == "app": + env["KNOE_APP_CLUSTER_NAME"] = cluster_name + elif cluster_role == "db": + env["KNOE_DB_CLUSTER_NAME"] = cluster_name + return env + + +def get_cluster_credentials( + *, + project_id: str, + cluster_name: str, + location: str, + log: _LogFn | None = None, +) -> str: + cmd = [ + "gcloud", + "container", + "clusters", + "get-credentials", + cluster_name, + "--project", + project_id, + "--region", + location, + "--quiet", + ] + result = _run(cmd, log=log) + if result.returncode != 0: + stderr = (result.stderr or "").strip() + raise RuntimeError( + f"Failed to get credentials for cluster '{cluster_name}' ({location}): {stderr}" + ) + return f"gke_{project_id}_{location}_{cluster_name}" + + +def ensure_app_cluster( + *, + project_id: str, + spec: GkeClusterSpec, + log: _LogFn | None = None, +) -> None: + describe_cmd = [ + "gcloud", + "container", + "clusters", + "describe", + spec.name, + "--project", + project_id, + "--region", + spec.location, + "--format=value(name)", + "--quiet", + ] + result = _run(describe_cmd, log=log) + if result.returncode == 0 and (result.stdout or "").strip() == spec.name: + return + + # Cluster not found — create it as Autopilot. + # NOTE: gcloud container clusters create-auto does NOT support --disk-type + # or --disk-size. Autopilot manages all node infrastructure automatically. + # Autopilot only provisions physical nodes when pods are scheduled, so SSD + # quota is not consumed while the cluster is idle. + create_cmd = [ + "gcloud", + "container", + "clusters", + "create-auto", + spec.name, + "--project", + project_id, + "--region", + spec.location, + "--workload-policies=allow-net-admin", + "--quiet", + ] + created = _run(create_cmd, log=log) + if created.returncode != 0: + stderr = (created.stderr or "").strip() + raise RuntimeError( + f"Failed to create app cluster '{spec.name}' in '{spec.location}': {stderr}" + ) + + +def ensure_db_cluster( + *, + project_id: str, + spec: GkeClusterSpec, + log: _LogFn | None = None, +) -> None: + describe_cmd = [ + "gcloud", + "container", + "clusters", + "describe", + spec.name, + "--project", + project_id, + "--region", + spec.location, + "--format=value(name)", + "--quiet", + ] + describe = _run(describe_cmd, log=log) + if describe.returncode == 0 and (describe.stdout or "").strip() == spec.name: + return + + if spec.mode.lower() != "standard": + raise RuntimeError( + f"Database cluster '{spec.name}' must run in Standard mode, got '{spec.mode}'." + ) + + create_cmd = [ + "gcloud", + "container", + "clusters", + "create", + spec.name, + "--project", + project_id, + "--region", + spec.location, + "--num-nodes", + str(max(3, int(spec.node_count))), + "--machine-type", + spec.machine_type, + "--disk-type", + spec.boot_disk_type, + "--disk-size", + str(spec.boot_disk_size_gb), + "--enable-ip-alias", + "--workload-pool", + f"{project_id}.svc.id.goog", + "--quiet", + ] + created = _run(create_cmd, log=log) + if created.returncode != 0: + stderr = (created.stderr or "").strip() + raise RuntimeError(f"Failed to create DB cluster '{spec.name}': {stderr}") diff --git a/knoe/core/prod_config.py b/knoe/core/prod_config.py index 51ed251..5889348 100644 --- a/knoe/core/prod_config.py +++ b/knoe/core/prod_config.py @@ -38,6 +38,10 @@ class ProdDatabaseConfig: instances: int = 3 storageClass: str = "premium-rwo" storageSizeGi: int = 100 + pgdataStorageClass: str = "premium-rwo" + walStorageClass: str = "premium-rwo" + pgdataSizeGi: int = 100 + walSizeGi: int = 25 appDatabase: str = "knoey" metaDatabase: str = "knoe_meta" appUser: str = "knoey_app" @@ -207,6 +211,25 @@ def build_config(payload: dict[str, Any] | None = None) -> KnoeProductionConfig: cfg.database.storageSizeGi = _coerce_int( db.get("storageSizeGi", cfg.database.storageSizeGi), cfg.database.storageSizeGi ) + cfg.database.pgdataStorageClass = ( + _str_or_empty(db.get("pgdataStorageClass")) + or _str_or_empty(db.get("storageClass")) + or cfg.database.pgdataStorageClass + ) + cfg.database.walStorageClass = ( + _str_or_empty(db.get("walStorageClass")) + or _str_or_empty(db.get("pgdataStorageClass")) + or _str_or_empty(db.get("storageClass")) + or cfg.database.walStorageClass + ) + cfg.database.pgdataSizeGi = _coerce_int( + db.get("pgdataSizeGi", db.get("storageSizeGi", cfg.database.pgdataSizeGi)), + cfg.database.pgdataSizeGi, + ) + cfg.database.walSizeGi = _coerce_int( + db.get("walSizeGi", max(10, int(cfg.database.pgdataSizeGi * 0.25))), + cfg.database.walSizeGi, + ) cfg.database.appDatabase = _str_or_empty(db.get("appDatabase")) or cfg.database.appDatabase cfg.database.metaDatabase = _str_or_empty(db.get("metaDatabase")) or cfg.database.metaDatabase cfg.database.appUser = _str_or_empty(db.get("appUser")) or cfg.database.appUser @@ -275,6 +298,8 @@ def validate_prod_config(config: KnoeProductionConfig) -> tuple[list[str], list[ "spec.database.clusterName": config.database.clusterName, "spec.database.postgresVersion": config.database.postgresVersion, "spec.database.storageClass": config.database.storageClass, + "spec.database.pgdataStorageClass": config.database.pgdataStorageClass, + "spec.database.walStorageClass": config.database.walStorageClass, "spec.database.appDatabase": config.database.appDatabase, "spec.database.metaDatabase": config.database.metaDatabase, "spec.database.appUser": config.database.appUser, @@ -304,6 +329,12 @@ def validate_prod_config(config: KnoeProductionConfig) -> tuple[list[str], list[ config.database.storageSizeGi = _validate_positive_int( config.database.storageSizeGi, "spec.database.storageSizeGi", errors ) + config.database.pgdataSizeGi = _validate_positive_int( + config.database.pgdataSizeGi, "spec.database.pgdataSizeGi", errors + ) + config.database.walSizeGi = _validate_positive_int( + config.database.walSizeGi, "spec.database.walSizeGi", errors + ) config.backups.retentionDays = _validate_positive_int( config.backups.retentionDays, "spec.backups.retentionDays", errors ) @@ -359,6 +390,10 @@ def opentofu_vars(config: KnoeProductionConfig) -> dict[str, Any]: "db_instances": config.database.instances, "db_storage_class": config.database.storageClass, "db_storage_size_gi": config.database.storageSizeGi, + "db_pgdata_storage_class": config.database.pgdataStorageClass, + "db_wal_storage_class": config.database.walStorageClass, + "db_pgdata_size_gi": config.database.pgdataSizeGi, + "db_wal_size_gi": config.database.walSizeGi, "app_database": config.database.appDatabase, "meta_database": config.database.metaDatabase, "app_user": config.database.appUser, diff --git a/knoe/ops/cli.py b/knoe/ops/cli.py index a45d945..5d8edbc 100644 --- a/knoe/ops/cli.py +++ b/knoe/ops/cli.py @@ -6,6 +6,13 @@ import os import sys from pathlib import Path +from knoe.core.cluster_storage_browser import ( + ClusterStorageRequest, + GcloudRegionQuotaProvider, + PingLatencyProvider, + build_cluster_storage_browser_result, +) + from .context import KnoeContext @@ -15,6 +22,62 @@ def _project_root() -> Path: def main(argv: list[str] | None = None) -> int: + argv = list(argv) if argv is not None else list(sys.argv[1:]) + + if argv and argv[0] == "storage-browser": + storage = argparse.ArgumentParser(prog="knoe-ops storage-browser", add_help=True) + storage.add_argument("--project", required=True, help="GCP project ID") + storage.add_argument("--timezone", default="", help="Timezone group: americas|emea|apac") + storage.add_argument("--pgdata-gb", type=float, default=100.0) + storage.add_argument("--wal-gb", type=float, default=25.0) + storage.add_argument( + "--regions", + default="", + help="Comma-separated region list. If omitted, fetch via gcloud compute regions list.", + ) + storage.add_argument("--output", choices=["table", "json"], default="table") + args = storage.parse_args(argv[1:]) + + regions = [r.strip() for r in (args.regions or "").split(",") if r.strip()] + if not regions: + import subprocess + + cmd = ["gcloud", "compute", "regions", "list", "--project", args.project, "--format=value(name)"] + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=20) + if result.returncode == 0: + regions = [r.strip() for r in result.stdout.splitlines() if r.strip()] + except Exception: + regions = [] + + result = build_cluster_storage_browser_result( + project_id=args.project, + requested=ClusterStorageRequest(pgdata_gb=args.pgdata_gb, wal_gb=args.wal_gb), + available_regions=regions, + timezone_group=(args.timezone or "").strip().lower() or None, + quota_provider=GcloudRegionQuotaProvider(), + latency_provider=PingLatencyProvider(), + ) + if args.output == "json": + print(result.to_json()) + return 0 + + print(f"Project: {result.project_id} Timezone: {result.timezone_group}") + print(f"Quota docs: {result.quota_link}") + print(f"Cloud Hub: {result.quota_console_link}") + print("region\tfeasible\tdefault(pg/wal)\tpremium_headroom\tstandard_headroom\tlatency_ms") + for c in result.candidates: + f = c.feasibility + feasible = "yes" if (f.can_premium or f.can_standard) else "no" + premium = "-" if f.premium_headroom_gb is None else f"{f.premium_headroom_gb:.1f}" + standard = "-" if f.standard_headroom_gb is None else f"{f.standard_headroom_gb:.1f}" + latency = "-" if c.latency_ms is None else f"{c.latency_ms:.2f}" + print( + f"{f.region}\t{feasible}\t{f.default_pgdata_class}/{f.default_wal_class}\t" + f"{premium}\t{standard}\t{latency}" + ) + return 0 + p = argparse.ArgumentParser(prog="knoe-ops", add_help=True) p.add_argument("component", help="component name (e.g. common_core, openbao)") p.add_argument( diff --git a/knoe/ui/screens/cluster.py b/knoe/ui/screens/cluster.py index de6991e..5652f21 100644 --- a/knoe/ui/screens/cluster.py +++ b/knoe/ui/screens/cluster.py @@ -586,6 +586,8 @@ class ClusterScreenMixin: db_row = _add_row(db_tab, db_row, "CNPG Cluster", "database.clusterName") db_row = _add_row(db_tab, db_row, "Postgres Version", "database.postgresVersion") db_row = _add_row(db_tab, db_row, "Instances", "database.instances") + db_row = _add_row(db_tab, db_row, "Node Boot Disk Type", "init_cluster.db_boot_disk_type") + db_row = _add_row(db_tab, db_row, "Node Boot Disk Size (GB)", "init_cluster.db_boot_disk_size_gb") db_row = _add_row(db_tab, db_row, "Storage Class", "database.storageClass") db_row = _add_row(db_tab, db_row, "Storage Size (Gi)", "database.storageSizeGi") db_row = _add_row(db_tab, db_row, "App Database", "database.appDatabase") @@ -2065,7 +2067,16 @@ class ClusterScreenMixin: "migration.sourcePasswordRef": tk.StringVar( value="secretref://local-source-db-password" ), + # GKE node pool configuration (persisted to [Initialize Cluster] in prole.cfg) + "init_cluster.db_boot_disk_type": tk.StringVar(value="pd-standard"), + "init_cluster.db_boot_disk_size_gb": tk.StringVar(value="50"), } + # Pre-populate GKE node pool vars from existing prole.cfg if present + _init_clus = (getattr(self, "prole_cfg_data", None) or {}).get("Initialize Cluster") or {} + if _init_clus.get("DB_BOOT_DISK_TYPE"): + self.prod_form_vars["init_cluster.db_boot_disk_type"].set(_init_clus["DB_BOOT_DISK_TYPE"]) + if _init_clus.get("DB_BOOT_DISK_SIZE_GB"): + self.prod_form_vars["init_cluster.db_boot_disk_size_gb"].set(_init_clus["DB_BOOT_DISK_SIZE_GB"]) self.prod_bool_vars = { "migration.continuousUntilCutover": tk.BooleanVar(value=False) } @@ -2797,6 +2808,31 @@ class ClusterScreenMixin: except Exception: return 0 + def _to_int_text(value, default: int) -> int: + try: + text = (value or "").strip() + if not text: + return int(default) + return int(text) + except Exception: + return int(default) + + region_form = self.prod_form_vars["cloud.region"].get().strip() + selected_region = region_form + + db_storage_class = self.prod_form_vars["database.storageClass"].get().strip() or "premium-rwo" + pgdata_storage_class = db_storage_class + wal_storage_class = pgdata_storage_class + if pgdata_storage_class not in {"premium-rwo", "standard-rwo"}: + pgdata_storage_class = "premium-rwo" + if wal_storage_class not in {"premium-rwo", "standard-rwo"}: + wal_storage_class = pgdata_storage_class + + default_pgdata_size = max(1, _to_int("database.storageSizeGi")) + pgdata_size_gi = _to_int_text(str(default_pgdata_size), default_pgdata_size) + default_wal = max(10, int(pgdata_size_gi * 0.25)) + wal_size_gi = _to_int_text(str(default_wal), default_wal) + doc = { "kind": "KnoeProductionConfig", "metadata": { @@ -2808,7 +2844,7 @@ class ClusterScreenMixin: "cloud": { "provider": self.prod_form_vars["cloud.provider"].get().strip(), "projectId": self.prod_form_vars["cloud.projectId"].get().strip(), - "region": self.prod_form_vars["cloud.region"].get().strip(), + "region": selected_region, "clusterName": self.prod_form_vars["cloud.clusterName"].get().strip(), "vpcMode": self.prod_form_vars["cloud.vpcMode"].get().strip(), "vpcName": self.prod_form_vars["cloud.vpcName"].get().strip() or None, @@ -2827,8 +2863,12 @@ class ClusterScreenMixin: "database.postgresVersion" ].get().strip(), "instances": _to_int("database.instances"), - "storageClass": self.prod_form_vars["database.storageClass"].get().strip(), - "storageSizeGi": _to_int("database.storageSizeGi"), + "storageClass": pgdata_storage_class, + "storageSizeGi": pgdata_size_gi, + "pgdataStorageClass": pgdata_storage_class, + "walStorageClass": wal_storage_class, + "pgdataSizeGi": pgdata_size_gi, + "walSizeGi": wal_size_gi, "appDatabase": self.prod_form_vars["database.appDatabase"].get().strip(), "metaDatabase": self.prod_form_vars["database.metaDatabase"].get().strip(), "appUser": self.prod_form_vars["database.appUser"].get().strip(), @@ -2944,6 +2984,14 @@ class ClusterScreenMixin: def _prod_put_config(self, show_dialog: bool = False) -> dict: self._ensure_prod_config_state() result = self.prod_config_api.put_prod_config(self._prod_payload_from_vars()) + # Persist GKE node pool config to [Initialize Cluster] section of prole.cfg + _gke_sec = self.prole_cfg_data.setdefault("Initialize Cluster", {}) + _disk_type = self.prod_form_vars["init_cluster.db_boot_disk_type"].get().strip() + _disk_size = self.prod_form_vars["init_cluster.db_boot_disk_size_gb"].get().strip() + if _disk_type: + _gke_sec["DB_BOOT_DISK_TYPE"] = _disk_type + if _disk_size: + _gke_sec["DB_BOOT_DISK_SIZE_GB"] = _disk_size cfg_doc = result.get("config") or {} if cfg_doc: self._prod_apply_payload_to_vars(cfg_doc) diff --git a/knoe/ui/screens/services.py b/knoe/ui/screens/services.py index ea53406..a76363b 100644 --- a/knoe/ui/screens/services.py +++ b/knoe/ui/screens/services.py @@ -114,6 +114,16 @@ class ServicesScreenMixin: def _render_init_scripts_page(self): content_width = self.bg_canvas.winfo_width() or 975 right_margin = content_width - 48 + app_cluster = ( + self._get_input("init_password.app_cluster_name", "") + or self._get_input("env_setup.APP_CLUSTER_NAME", "") + or "knoe-dev-0" + ).strip() or "knoe-dev-0" + db_cluster = ( + self._get_input("init_password.db_cluster_name", "") + or self._get_input("env_setup.DB_CLUSTER_NAME", "") + or "knoe-cnpg-0" + ).strip() or "knoe-cnpg-0" ui.canvas_text( self, right_margin, 40, "knoe.dev", @@ -125,8 +135,36 @@ class ServicesScreenMixin: ) self._render_title("Initialization Scripts", y=150) + ui.canvas_text( + self, + 48, + 178, + f"Application Cluster: {app_cluster} (Autopilot)", + fill="#1d1d1f", + font=("SF Pro Text", 11), + anchor="nw", + ) + ui.canvas_text( + self, + 48, + 196, + f"Database Cluster: {db_cluster} (Standard)", + fill="#1d1d1f", + font=("SF Pro Text", 11), + anchor="nw", + ) + ui.canvas_text( + self, + 48, + 214, + "CloudNativePG runs on the dedicated Standard GKE cluster; platform services remain on the Autopilot cluster.", + fill="#6e6e73", + font=("SF Pro Text", 10), + anchor="nw", + ) + x_label = 48 - gy = 198 # top of traffic-light section + gy = 242 # top of traffic-light section # --- Traffic light section --- # Column x positions @@ -1570,7 +1608,7 @@ class ServicesScreenMixin: def _inline_worker(): try: namespace = (self.db_namespace.get() or "").strip() or "default" - env = self._script_env_for_namespace(namespace) + env = self._script_env_for_namespace(namespace, cluster_role="app") env["SERVICE_NAMESPACE"] = self._get_service_namespace() rc = self.controller.run_script( "init_service_layer.sh", @@ -1652,7 +1690,7 @@ class ServicesScreenMixin: ) ) namespace = (self.db_namespace.get() or "").strip() or "default" - env = self._script_env_for_namespace(namespace) + env = self._script_env_for_namespace(namespace, cluster_role="app") env["SERVICE_NAMESPACE"] = self._get_service_namespace() try: log_fp = log_path.open("a", encoding="utf-8") @@ -1989,7 +2027,7 @@ class ServicesScreenMixin: else None ) ) - env = self._script_env_for_namespace(namespace) + env = self._script_env_for_namespace(namespace, cluster_role="app") env["KNOE_DB_USER"] = self.db_username.get().strip() env["DB_PASSWORD"] = password env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str( diff --git a/modes/k8s/knoe-db/.version b/modes/k8s/knoe-db/.version index dce6588..86ee83a 100644 --- a/modes/k8s/knoe-db/.version +++ b/modes/k8s/knoe-db/.version @@ -1 +1 @@ -36 \ No newline at end of file +40 \ No newline at end of file diff --git a/scan/network_description.txt b/scan/network_description.txt index 2743a8a..f788bbc 100644 --- a/scan/network_description.txt +++ b/scan/network_description.txt @@ -1,57 +1,55 @@ Network Discovery Summary: -Primary Router: 10.0.0.1 (eero_5d:50:f2) +Primary Router: 10.0.0.1 (Unknown) DNS Servers: 10.0.0.5, 10.0.0.4, 100.100.100.100 Detected Devices: -- 10.0.0.35 [7e:19:9a:fe:9f:ed] (7e:19:9a:fe:9f:ed): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related'] -- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)'] -- 10.0.0.1 [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: [] -- 10.0.0.3 [2c:cf:67:8c:2b:47] (RaspberryPi_8c:2b:47): Ports [22, 53, 80, 443, 2049, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'Active Directory Related'] -- 10.0.0.6 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server'] -- 10.0.0.7 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH'] -- 10.0.0.5 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC'] -- 10.0.0.4 [b8:27:eb:b3:6f:2b] (RaspberryPiF_b3:6f:2b): Ports [22, 53, 2049], Services: ['DNS', 'SSH', 'NFS'] -- 10.0.0.2 [dc:a4:ca:ea:1a:2f] (Apple_ea:1a:2f): Ports [], Services: [] -- 10.0.0.26 [c8:db:26:08:52:f8] (Logitech_08:52:f8): Ports [], Services: [] -- 10.0.0.22 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server'] -- 10.0.0.205 [a8:20:66:28:12:e7] (Apple_28:12:e7): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] -- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server'] -- 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server'] -- 10.0.0.45 [f8:b4:6a:30:a4:1f] (HewlettPacka_30:a4:1f): Ports [80, 443], Services: ['Web Server'] -- 10.0.0.48 [0c:fe:45:53:f4:3c] (SonyInteract_53:f4:3c): Ports [], Services: [] -- 10.0.0.37 [54:07:7d:22:c0:b2] (Netgear_22:c0:b2): Ports [80], Services: ['Web Server'] -- 10.0.0.54 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: [] -- 10.0.0.52 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH'] -- 10.0.0.46 [d4:f7:d5:40:ab:17] (SonyInteract_40:ab:17): Ports [], Services: [] -- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: [] -- 10.0.0.58 [7e:46:74:2a:b0:b2] (7e:46:74:2a:b0:b2): Ports [], Services: [] -- 10.0.0.207 [e4:5f:01:90:35:0b] (RaspberryPiT_90:35:0b): Ports [22, 5900], Services: ['SSH', 'VNC'] -- 10.0.0.95 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server'] -- 10.0.0.106 [48:a6:b8:a7:50:60] (Sonos_a7:50:60): Ports [], Services: [] -- 10.0.0.99 [20:28:bc:f1:dc:ba] (Visionscape_f1:dc:ba): Ports [], Services: [] -- 10.0.0.113 [60:81:10:92:02:4e] (Apple_92:02:4e): Ports [], Services: [] -- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: [] -- 10.0.0.112 [f8:33:31:40:58:c0] (TexasInstrum_40:58:c0): Ports [443], Services: ['Web Server'] -- 10.0.0.124 [9c:8e:cd:02:73:c4] (AmcrestTechn_02:73:c4): Ports [80], Services: ['Web Server'] -- 10.0.0.123 [3c:ef:8c:96:e3:3c] (ZhejiangDahu_96:e3:3c): Ports [80], Services: ['Web Server'] -- 10.0.0.125 [a0:60:32:04:73:ba] (AmcrestTechn_04:73:ba): Ports [80], Services: ['Web Server'] -- 10.0.0.117 [40:f5:20:1e:5e:91] (Espressif_1e:5e:91): Ports [80], Services: ['Web Server'] +- 10.0.0.27 [3a:b4:ce:79:b0:55] (3a:b4:ce:79:b0:55): Ports [], Services: [] +- 10.0.0.63 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: [] - 10.0.0.111 [ca:0b:6c:5d:0a:e5] (ca:0b:6c:5d:0a:e5): Ports [], Services: [] -- 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH'] -- 10.0.0.1\ [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: [] -- 10.0.0.130 [f8:bb:bf:a3:84:eb] (eero_a3:84:eb): Ports [53], Services: ['DNS'] -- 10.0.0.128 [20:c9:d0:94:30:dd] (Apple_94:30:dd): Ports [5900], Services: ['VNC'] -- 10.0.0.143 [60:5f:8d:7a:aa:32] (eero_7a:aa:32): Ports [53], Services: ['DNS'] -- 10.0.0.208 [a0:ad:9f:30:84:5f] (ASUSTekCOMPU_30:84:5f): Ports [], Services: [] -- 10.0.0.145 [7c:a6:b0:04:db:1c] (7c:a6:b0:04:db:1c): Ports [], Services: [] -- 10.0.0.155 [62:17:97:b3:06:a1] (62:17:97:b3:06:a1): Ports [], Services: [] -- 10.0.0.170 [b0:ee:7b:ca:73:99] (Roku_ca:73:99): Ports [], Services: [] -- 10.0.0.175 [28:80:88:e4:10:1a] (Netgear_e4:10:1a): Ports [80], Services: ['Web Server'] -- 10.0.0.188 [00:16:6c:c5:1f:54] (SamsungElect_c5:1f:54): Ports [80, 443], Services: ['Web Server'] +- 10.0.0.49 [00:e0:4c:08:0c:75] (RealtekSemic_08:0c:75): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related'] +- 10.0.0.31 [2e:c1:c1:e9:87:b4] (2e:c1:c1:e9:87:b4): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related'] - 10.0.0.189 [00:17:88:a3:2f:cc] (PhilipsLight_a3:2f:cc): Ports [80, 443], Services: ['Web Server'] -- 10.0.0.196 [60:5f:8d:88:08:f2] (eero_88:08:f2): Ports [53], Services: ['DNS'] +- 10.0.0.31\ [2e:c1:c1:e9:87:b4] (2e:c1:c1:e9:87:b4): Ports [], Services: [] +- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: [] +- 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server'] +- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: [] - 10.0.0.204 [50:eb:f6:56:f3:26] (ASUSTekCOMPU_56:f3:26): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama'] -- 10.0.0.199 [7c:a6:b0:01:a7:33] (7c:a6:b0:01:a7:33): Ports [], Services: [] -Ollama Instances found at: 10.0.0.35, 10.0.0.204 +- 10.0.0.1 [Unknown] (Unknown): Ports [53], Services: ['DNS'] +- 10.0.0.2 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.3 [Unknown] (Unknown): Ports [22, 53, 80, 443, 2049, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] +- 10.0.0.4 [Unknown] (Unknown): Ports [22, 53, 443, 2049], Services: ['DNS', 'SSH', 'NFS', 'Web Server'] +- 10.0.0.5 [Unknown] (Unknown): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC'] +- 10.0.0.6 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server'] +- 10.0.0.7 [Unknown] (Unknown): Ports [22], Services: ['SSH'] +- 10.0.0.22 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server'] +- 10.0.0.26 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.32 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.37 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.45 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server'] +- 10.0.0.46 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.48 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.52 [Unknown] (Unknown): Ports [22], Services: ['SSH'] +- 10.0.0.62 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.95 [Unknown] (Unknown): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC'] +- 10.0.0.99 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.106 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.112 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server'] +- 10.0.0.117 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.123 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.124 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.127 [Unknown] (Unknown): Ports [22], Services: ['SSH'] +- 10.0.0.128 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] +- 10.0.0.130 [Unknown] (Unknown): Ports [53], Services: ['DNS'] +- 10.0.0.143 [Unknown] (Unknown): Ports [53], Services: ['DNS'] +- 10.0.0.145 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.155 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.170 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.175 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.188 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server'] +- 10.0.0.196 [Unknown] (Unknown): Ports [53], Services: ['DNS'] +- 10.0.0.203 [Unknown] (Unknown): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)'] +- 10.0.0.205 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] +- 10.0.0.206 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +Ollama Instances found at: 10.0.0.49, 10.0.0.31, 10.0.0.204 Ansible Inventory Summary: Inventory directory: /Users/chrisfu/dev/prole/infrastructure diff --git a/scripts/patch_clusters.sh b/scripts/patch_clusters.sh new file mode 100755 index 0000000..a86b14b --- /dev/null +++ b/scripts/patch_clusters.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# patch_clusters.sh — Migrate knoe-cnpg-0 boot disks from pd-ssd to pd-standard. +# +# The GCloud Console creates node pools with pd-ssd boot disks by default, which +# consumes the entire 300 GB pd-ssd quota before any PGDATA/WAL PVCs can be +# provisioned. This script replaces the existing default-pool with a new +# cnpg-db-pool that uses pd-standard boot disks, freeing the quota for CNPG PVCs. +# +# What this does: +# 1. Detects the current node pool disk configuration. +# 2. Creates cnpg-db-pool (pd-standard boot, e2-standard-2, 3 nodes). +# 3. Cordons and drains all nodes in the old pool. +# 4. Deletes the old pool. +# 5. Verifies 3 CNPG pods return to Running. +# +# Usage: +# ./scripts/patch_clusters.sh +# +# Environment overrides: +# GCP_PROJECT — GCP project ID (default: plenary-truck-485623-p7) +# GCP_REGION — GCP region (default: us-west3) +# CLUSTER_NAME — GKE cluster (default: knoe-cnpg-0) +# OLD_POOL — Pool to remove (default: default-pool) +# NEW_POOL — Pool to create (default: cnpg-db-pool) +# MACHINE_TYPE — Node machine type (default: e2-standard-2) +# DISK_TYPE — Boot disk type (default: pd-standard) +# DISK_SIZE_GB — Boot disk size GB (default: 50) +# NODE_COUNT — Nodes per zone (default: 1, regional = 3 zones) +# CNPG_NAMESPACE — Namespace for CNPG pods (default: knoe-db-0) +# DRAIN_TIMEOUT — kubectl drain timeout (default: 300s) +# DRY_RUN — Set to "true" to print commands without running (default: false) + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}" +GCP_REGION="${GCP_REGION:-us-west3}" +CLUSTER_NAME="${CLUSTER_NAME:-knoe-cnpg-0}" +OLD_POOL="${OLD_POOL:-default-pool}" +NEW_POOL="${NEW_POOL:-cnpg-db-pool}" +MACHINE_TYPE="${MACHINE_TYPE:-e2-standard-2}" +DISK_TYPE="${DISK_TYPE:-pd-standard}" +DISK_SIZE_GB="${DISK_SIZE_GB:-50}" +NODE_COUNT="${NODE_COUNT:-1}" # 1 per zone; regional cluster spans 3 zones = 3 nodes total +CNPG_NAMESPACE="${CNPG_NAMESPACE:-knoe-db-0}" +DRAIN_TIMEOUT="${DRAIN_TIMEOUT:-300s}" +DRY_RUN="${DRY_RUN:-false}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +log() { + printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*" +} + +require_tool() { + command -v "$1" >/dev/null 2>&1 || { log "Error: required tool not found: $1" >&2; exit 1; } +} + +run_cmd() { + if [[ "${DRY_RUN}" == "true" ]]; then + echo "[DRY-RUN] $*" + else + "$@" + fi +} + +pool_exists() { + local pool="$1" + gcloud container node-pools describe "${pool}" \ + --cluster="${CLUSTER_NAME}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --format="value(name)" \ + --quiet 2>/dev/null | grep -q "${pool}" +} + +# --------------------------------------------------------------------------- +# Pre-flight +# --------------------------------------------------------------------------- + +require_tool gcloud +require_tool kubectl + +log "==> Patch: migrate ${CLUSTER_NAME}/${OLD_POOL} → ${NEW_POOL} (${DISK_TYPE}, ${DISK_SIZE_GB}GB)" +log " Project : ${GCP_PROJECT}" +log " Region : ${GCP_REGION}" +log " DRY_RUN : ${DRY_RUN}" +echo "" + +# --------------------------------------------------------------------------- +# Phase 1: Detect current configuration +# --------------------------------------------------------------------------- + +log "[phase 1] Detecting current node pool configuration ..." + +if ! gcloud container clusters describe "${CLUSTER_NAME}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --format="value(name)" \ + --quiet 2>/dev/null | grep -q "${CLUSTER_NAME}"; then + log "Error: cluster '${CLUSTER_NAME}' not found in project '${GCP_PROJECT}' region '${GCP_REGION}'." >&2 + exit 1 +fi + +if pool_exists "${OLD_POOL}"; then + current_disk=$(gcloud container node-pools describe "${OLD_POOL}" \ + --cluster="${CLUSTER_NAME}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --format="value(config.diskType)" \ + --quiet 2>/dev/null || echo "unknown") + current_size=$(gcloud container node-pools describe "${OLD_POOL}" \ + --cluster="${CLUSTER_NAME}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --format="value(config.diskSizeGb)" \ + --quiet 2>/dev/null || echo "unknown") + log " ${OLD_POOL}: disk_type=${current_disk}, disk_size_gb=${current_size}" + if [[ "${current_disk}" == "pd-standard" ]]; then + log " NOTE: ${OLD_POOL} already uses pd-standard. Continuing to ensure pool is renamed to '${NEW_POOL}'." + fi +else + log " ${OLD_POOL} not found — may have been removed already." +fi + +# --------------------------------------------------------------------------- +# Phase 2: Create new node pool +# --------------------------------------------------------------------------- + +log "[phase 2] Ensuring node pool '${NEW_POOL}' exists ..." + +if pool_exists "${NEW_POOL}"; then + log " ${NEW_POOL} already exists — skipping creation." +else + log " Creating ${NEW_POOL} (machine=${MACHINE_TYPE}, disk=${DISK_TYPE}, size=${DISK_SIZE_GB}GB, num-nodes=${NODE_COUNT}/zone) ..." + run_cmd gcloud container node-pools create "${NEW_POOL}" \ + --cluster="${CLUSTER_NAME}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --machine-type="${MACHINE_TYPE}" \ + --disk-type="${DISK_TYPE}" \ + --disk-size="${DISK_SIZE_GB}" \ + --num-nodes="${NODE_COUNT}" \ + --quiet + log " ${NEW_POOL} created." +fi + +# --------------------------------------------------------------------------- +# Phase 3: Cordon and drain old pool +# --------------------------------------------------------------------------- + +log "[phase 3] Cordoning and draining '${OLD_POOL}' ..." + +if ! pool_exists "${OLD_POOL}"; then + log " ${OLD_POOL} not found — nothing to drain." +else + old_nodes=$(kubectl get nodes \ + -l "cloud.google.com/gke-nodepool=${OLD_POOL}" \ + --no-headers \ + -o custom-columns=":metadata.name" 2>/dev/null || true) + + if [[ -z "${old_nodes}" ]]; then + log " No nodes found labelled with nodepool=${OLD_POOL} — already drained or pool empty." + else + for node in ${old_nodes}; do + log " Cordoning ${node} ..." + run_cmd kubectl cordon "${node}" + done + + for node in ${old_nodes}; do + log " Draining ${node} (timeout ${DRAIN_TIMEOUT}) ..." + run_cmd kubectl drain "${node}" \ + --ignore-daemonsets \ + --delete-emptydir-data \ + --timeout="${DRAIN_TIMEOUT}" \ + --force + done + log " All nodes in ${OLD_POOL} drained." + fi +fi + +# --------------------------------------------------------------------------- +# Phase 4: Delete old pool +# --------------------------------------------------------------------------- + +log "[phase 4] Deleting old pool '${OLD_POOL}' ..." + +if ! pool_exists "${OLD_POOL}"; then + log " ${OLD_POOL} does not exist — already removed." +else + run_cmd gcloud container node-pools delete "${OLD_POOL}" \ + --cluster="${CLUSTER_NAME}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --quiet + log " ${OLD_POOL} deleted." +fi + +# --------------------------------------------------------------------------- +# Phase 5: Verify CNPG pods +# --------------------------------------------------------------------------- + +log "[phase 5] Verifying CNPG pods in namespace '${CNPG_NAMESPACE}' ..." + +if [[ "${DRY_RUN}" == "true" ]]; then + log " [DRY-RUN] Would check: kubectl -n ${CNPG_NAMESPACE} get pods -l cnpg.io/cluster=knoe-db" +else + max_attempts=20 + attempt=0 + while [[ ${attempt} -lt ${max_attempts} ]]; do + running=$(kubectl -n "${CNPG_NAMESPACE}" get pods \ + -l "cnpg.io/cluster=knoe-db" \ + --no-headers 2>/dev/null \ + | grep -c "Running" || true) + log " Running CNPG pods: ${running} / 3 expected (attempt $((attempt + 1))/${max_attempts})" + if [[ "${running}" -ge 3 ]]; then + log " All CNPG pods Running on ${NEW_POOL}." + kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db" + break + fi + attempt=$(( attempt + 1 )) + if [[ ${attempt} -lt ${max_attempts} ]]; then + sleep 30 + fi + done + + if [[ "${running:-0}" -lt 3 ]]; then + log "WARNING: Not all CNPG pods reached Running state within timeout." >&2 + kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db" || true + log "Check pod events: kubectl -n ${CNPG_NAMESPACE} describe pods -l cnpg.io/cluster=knoe-db" + exit 1 + fi +fi + +# --------------------------------------------------------------------------- +# Done +# --------------------------------------------------------------------------- + +echo "" +log "==> Patch complete." +log "" +log "Next steps:" +log " 1. Verify pd-ssd quota freed:" +log " gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} --format='value(quotas[disksSSD].usage,quotas[disksSSD].limit)'" +log " 2. Apply updated CNPG manifest (premium-rwo PVCs, 50Gi PGDATA + WAL):" +log " kubectl --context= apply -f deploy/gcp/gke/knoe-db.yaml" +log " 3. Watch PVC expansion and pod restart:" +log " kubectl -n ${CNPG_NAMESPACE} get pvc -w" diff --git a/scripts/reset_clusters.sh b/scripts/reset_clusters.sh new file mode 100755 index 0000000..4ab5238 --- /dev/null +++ b/scripts/reset_clusters.sh @@ -0,0 +1,317 @@ +#!/usr/bin/env bash +# reset_clusters.sh — Delete and recreate both GKE clusters with correct configuration. +# +# Problem: Both clusters were created via GCloud Console as Autopilot, which: +# - Prevents manual node pool management (patch_clusters.sh cannot work) +# - Uses pd-balanced boot disks (counts against SSD_TOTAL_GB quota, 300 GB at limit) +# +# Solution: Delete both, recreate with pd-standard boot disks: +# knoe-dev-0 → Autopilot (app workloads: GitLab, platform), pd-standard boot +# knoe-cnpg-0 → Standard (CNPG only, 3× e2-standard-2), pd-standard boot +# +# SSD quota budget after reset: +# Boot disks: pd-standard — does NOT count against SSD_TOTAL_GB +# PGDATA PVCs (premium-rwo): 3 × 50 Gi = 150 Gi \ Apply knoe-db.yaml AFTER +# WAL PVCs (premium-rwo): 3 × 50 Gi = 150 Gi / quota increase to 2 TB +# +# Usage: +# CONFIRM=true ./scripts/reset_clusters.sh +# CONFIRM=true DRY_RUN=true ./scripts/reset_clusters.sh # inspect only +# +# Environment overrides: +# GCP_PROJECT (default: plenary-truck-485623-p7) +# GCP_REGION (default: us-west3) +# APP_CLUSTER (default: knoe-dev-0) +# DB_CLUSTER (default: knoe-cnpg-0) +# DB_MACHINE_TYPE (default: e2-standard-2) +# DB_DISK_TYPE (default: pd-standard) +# DB_DISK_SIZE_GB (default: 50) +# DB_NODES_PER_ZONE (default: 1 → 3 nodes across 3 zones) +# APP_DISK_TYPE (default: pd-standard) +# APP_DISK_SIZE_GB (default: 50) +# CONFIRM REQUIRED: must be "true" to allow destructive operations +# DRY_RUN (default: false) + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}" +GCP_REGION="${GCP_REGION:-us-west3}" +APP_CLUSTER="${APP_CLUSTER:-knoe-dev-0}" +DB_CLUSTER="${DB_CLUSTER:-knoe-cnpg-0}" +DB_MACHINE_TYPE="${DB_MACHINE_TYPE:-e2-standard-2}" +DB_DISK_TYPE="${DB_DISK_TYPE:-pd-standard}" +DB_DISK_SIZE_GB="${DB_DISK_SIZE_GB:-50}" +DB_NODES_PER_ZONE="${DB_NODES_PER_ZONE:-1}" # regional cluster = 3 zones = 3 nodes total +APP_DISK_TYPE="${APP_DISK_TYPE:-pd-standard}" +APP_DISK_SIZE_GB="${APP_DISK_SIZE_GB:-50}" +CONFIRM="${CONFIRM:-false}" +DRY_RUN="${DRY_RUN:-false}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } + +die() { log "ERROR: $*" >&2; exit 1; } + +require_tool() { + command -v "$1" >/dev/null 2>&1 || die "required tool not found: $1" +} + +run_cmd() { + if [[ "${DRY_RUN}" == "true" ]]; then + echo "[DRY-RUN] $*" + else + "$@" + fi +} + +cluster_exists() { + local name="$1" + gcloud container clusters describe "${name}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --format="value(name)" \ + --quiet 2>/dev/null | grep -q "${name}" +} + +_ssd_quota_yaml() { + # Emit the 3-line YAML block for SSD_TOTAL_GB quota entry, e.g.: + # - limit: 300.0 + # metric: SSD_TOTAL_GB + # usage: 300.0 + gcloud compute regions describe "${GCP_REGION}" \ + --project="${GCP_PROJECT}" \ + --format=yaml \ + --quiet 2>/dev/null | grep -B 1 -A 1 "metric: SSD_TOTAL_GB" || true +} + +ssd_usage_gb() { + local block + block=$(_ssd_quota_yaml) + echo "${block}" | awk '/usage:/{print $2}' | head -1 || echo "unknown" +} + +ssd_limit_gb() { + local block + block=$(_ssd_quota_yaml) + echo "${block}" | awk '/limit:/{print $2}' | head -1 || echo "unknown" +} + +# --------------------------------------------------------------------------- +# Pre-flight +# --------------------------------------------------------------------------- + +require_tool gcloud +require_tool kubectl + +if [[ "${CONFIRM}" != "true" ]]; then + echo "" + echo " This script will DELETE and RECREATE both GKE clusters:" + echo " ${APP_CLUSTER} (Autopilot) and ${DB_CLUSTER} (Standard)" + echo "" + echo " Set CONFIRM=true to proceed:" + echo " CONFIRM=true ./scripts/reset_clusters.sh" + echo "" + exit 1 +fi + +log "==> Cluster reset: ${APP_CLUSTER} (Autopilot) + ${DB_CLUSTER} (Standard)" +log " Project : ${GCP_PROJECT}" +log " Region : ${GCP_REGION}" +log " DRY_RUN : ${DRY_RUN}" +echo "" + +# --------------------------------------------------------------------------- +# Phase 1: Show current SSD quota +# --------------------------------------------------------------------------- + +log "[phase 1] Current SSD quota (SSD_TOTAL_GB) in ${GCP_REGION} ..." +if [[ "${DRY_RUN}" != "true" ]]; then + usage=$(ssd_usage_gb) + limit=$(ssd_limit_gb) + log " SSD usage: ${usage} GB / ${limit} GB limit" + if [[ "${usage}" == "unknown" || "${limit}" == "unknown" ]]; then + log " WARNING: Could not read SSD quota — proceeding anyway." + fi +else + log " [DRY-RUN] Would read SSD quota from ${GCP_REGION}" +fi + +# --------------------------------------------------------------------------- +# Phase 2: Delete existing clusters +# --------------------------------------------------------------------------- + +log "[phase 2] Deleting existing clusters ..." + +for cluster in "${DB_CLUSTER}" "${APP_CLUSTER}"; do + if [[ "${DRY_RUN}" == "true" ]]; then + echo "[DRY-RUN] gcloud container clusters delete ${cluster} --project=${GCP_PROJECT} --region=${GCP_REGION} --quiet" + else + if cluster_exists "${cluster}"; then + log " Deleting ${cluster} (this takes ~5-10 min) ..." + gcloud container clusters delete "${cluster}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --quiet & + log " ${cluster} deletion running in background (PID $!)." + else + log " ${cluster} not found — skipping." + fi + fi +done + +# Wait for all background deletions to finish +if [[ "${DRY_RUN}" != "true" ]]; then + log " Waiting for cluster deletions to complete ..." + wait + log " All deletions complete." +fi + +# --------------------------------------------------------------------------- +# Phase 3: Wait for SSD quota to be released +# --------------------------------------------------------------------------- + +log "[phase 3] Waiting for SSD quota to be released ..." + +if [[ "${DRY_RUN}" == "true" ]]; then + log " [DRY-RUN] Would poll SSD_TOTAL_GB until usage < 50 GB" +else + max_wait_s=1200 # 20 min max + poll_s=30 + elapsed=0 + while true; do + usage=$(ssd_usage_gb) + log " SSD usage: ${usage} GB (${elapsed}s elapsed)" + if [[ "${usage}" == "unknown" ]]; then + log " WARNING: Could not read quota — treating as released." + break + fi + # Cast to int for comparison + usage_int=${usage%.*} + if [[ "${usage_int}" -lt 50 ]]; then + log " SSD quota released (${usage} GB remaining usage)." + break + fi + if [[ ${elapsed} -ge ${max_wait_s} ]]; then + log " WARNING: SSD quota did not fully release within ${max_wait_s}s." + log " Current usage: ${usage} GB — proceeding with pd-standard (no SSD impact)." + break + fi + sleep "${poll_s}" + elapsed=$(( elapsed + poll_s )) + done +fi + +# --------------------------------------------------------------------------- +# Phase 4: Create knoe-cnpg-0 as Standard GKE cluster +# --------------------------------------------------------------------------- + +log "[phase 4] Creating ${DB_CLUSTER} (Standard, ${DB_MACHINE_TYPE}, ${DB_DISK_TYPE}, ${DB_DISK_SIZE_GB}GB) ..." + +run_cmd gcloud container clusters create "${DB_CLUSTER}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --cluster-version=latest \ + --machine-type="${DB_MACHINE_TYPE}" \ + --disk-type="${DB_DISK_TYPE}" \ + --disk-size="${DB_DISK_SIZE_GB}" \ + --num-nodes="${DB_NODES_PER_ZONE}" \ + --enable-ip-alias \ + --workload-pool="${GCP_PROJECT}.svc.id.goog" \ + --quiet + +log " ${DB_CLUSTER} created." + +# --------------------------------------------------------------------------- +# Phase 5: Create knoe-dev-0 as Autopilot cluster +# --------------------------------------------------------------------------- + +log "[phase 5] Creating ${APP_CLUSTER} (Autopilot, ${APP_DISK_TYPE}, ${APP_DISK_SIZE_GB}GB) ..." + +run_cmd gcloud container clusters create-auto "${APP_CLUSTER}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --cluster-version=latest \ + --workload-policies=allow-net-admin \ + --quiet +# NOTE: Autopilot manages node infrastructure (disk type, size) automatically. +# Boot disk type cannot be set at Autopilot cluster creation time. +# Autopilot only provisions nodes when pods are scheduled — no quota impact +# while the cluster is idle. Deploy workloads after the SSD quota increase +# (300 GB → 2 TB) is approved to ensure headroom for GitLab node boot disks. + +log " ${APP_CLUSTER} created." + +# --------------------------------------------------------------------------- +# Phase 6: Fetch credentials and verify +# --------------------------------------------------------------------------- + +log "[phase 6] Fetching kubeconfig credentials ..." + +if [[ "${DRY_RUN}" != "true" ]]; then + run_cmd gcloud container clusters get-credentials "${DB_CLUSTER}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --quiet + db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}" + log " DB cluster context: ${db_ctx}" + + run_cmd gcloud container clusters get-credentials "${APP_CLUSTER}" \ + --project="${GCP_PROJECT}" \ + --region="${GCP_REGION}" \ + --quiet + app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}" + log " App cluster context: ${app_ctx}" + + log " Verifying cluster connectivity ..." + kubectl --context="${db_ctx}" cluster-info --request-timeout=15s \ + && log " ${DB_CLUSTER}: OK" \ + || log " WARNING: ${DB_CLUSTER} not yet reachable — may need a moment." + kubectl --context="${app_ctx}" cluster-info --request-timeout=15s \ + && log " ${APP_CLUSTER}: OK" \ + || log " WARNING: ${APP_CLUSTER} not yet reachable — may need a moment." +else + db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}" + app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}" + log " [DRY-RUN] Would fetch credentials for both clusters." +fi + +# --------------------------------------------------------------------------- +# Phase 7: Show final SSD quota +# --------------------------------------------------------------------------- + +log "[phase 7] Final SSD quota ..." +if [[ "${DRY_RUN}" != "true" ]]; then + usage=$(ssd_usage_gb) + limit=$(ssd_limit_gb) + log " SSD usage: ${usage} GB / ${limit} GB (pd-standard boot disks use 0 SSD quota)" +fi + +# --------------------------------------------------------------------------- +# Done +# --------------------------------------------------------------------------- + +echo "" +log "==> Reset complete." +log "" +log "Contexts:" +log " App (Autopilot): ${app_ctx}" +log " DB (Standard) : ${db_ctx}" +log "" +log "Next steps:" +log " 1. Run the installer to configure workloads:" +log " ./install.sh" +log "" +log " 2. Once the SSD quota increase (300 GB → 2 TB) is approved, apply CNPG storage:" +log " kubectl --context=${db_ctx} apply -f deploy/gcp/gke/knoe-db.yaml" +log " # This provisions 3×50Gi PGDATA + 3×50Gi WAL = 300 Gi pd-ssd" +log "" +log " 3. Check your quota increase request:" +log " gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} \\" +log " --format='table(quotas.metric,quotas.usage,quotas.limit)' | grep SSD" diff --git a/tests/installer/test_cluster_storage_browser.py b/tests/installer/test_cluster_storage_browser.py new file mode 100644 index 0000000..fbd213a --- /dev/null +++ b/tests/installer/test_cluster_storage_browser.py @@ -0,0 +1,90 @@ +from pathlib import Path +import sys + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from knoe.core.cluster_storage_browser import ( + ClusterStorageRequest, + QuotaMetricHeadroom, + RegionLatencyResult, + RegionQuotaSnapshot, + StaticQuotaProvider, + build_cluster_storage_browser_result, + evaluate_region_feasibility, +) + + +class RecordingLatencyProvider: + def __init__(self, values: dict[str, RegionLatencyResult]): + self.values = dict(values) + self.calls: list[list[str]] = [] + + def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]: + self.calls.append(list(regions)) + return {r: self.values[r] for r in regions if r in self.values} + + +def _quota(region: str, premium_headroom: float, standard_headroom: float) -> RegionQuotaSnapshot: + return RegionQuotaSnapshot( + region=region, + premium=QuotaMetricHeadroom( + metric="SSD_TOTAL_GB", + limit_gb=premium_headroom + 100.0, + usage_gb=100.0, + ), + standard=QuotaMetricHeadroom( + metric="DISKS_TOTAL_GB", + limit_gb=standard_headroom + 100.0, + usage_gb=100.0, + ), + ) + + +def test_browser_filters_timezone_probes_only_viable_and_ranks_by_feasibility_then_latency(): + quotas = { + "us-central1": _quota("us-central1", premium_headroom=200.0, standard_headroom=20.0), + "us-east1": _quota("us-east1", premium_headroom=10.0, standard_headroom=180.0), + "southamerica-east1": _quota("southamerica-east1", premium_headroom=5.0, standard_headroom=5.0), + } + quota_provider = StaticQuotaProvider(quotas) + latency_provider = RecordingLatencyProvider( + { + "us-central1": RegionLatencyResult(region="us-central1", latency_ms=40.0, ok=True), + "us-east1": RegionLatencyResult(region="us-east1", latency_ms=15.0, ok=True), + } + ) + + result = build_cluster_storage_browser_result( + project_id="p-1", + requested=ClusterStorageRequest(pgdata_gb=100.0, wal_gb=25.0), + available_regions=["us-central1", "us-east1", "southamerica-east1", "europe-west1"], + timezone_group="americas", + quota_provider=quota_provider, + latency_provider=latency_provider, + ) + + assert latency_provider.calls == [["us-central1", "us-east1"]] + assert [c.feasibility.region for c in result.candidates] == [ + "us-central1", + "us-east1", + "southamerica-east1", + ] + assert result.candidates[0].feasibility.default_pgdata_class == "premium-rwo" + assert result.candidates[1].feasibility.default_pgdata_class == "standard-rwo" + assert result.candidates[2].latency_ms is None + + +def test_feasibility_defaults_to_premium_when_quota_is_missing(): + f = evaluate_region_feasibility( + region="us-west1", + timezone_group="americas", + quota=None, + requested=ClusterStorageRequest(pgdata_gb=80.0, wal_gb=20.0), + ) + assert f.can_premium is False + assert f.can_standard is False + assert f.default_pgdata_class == "premium-rwo" + assert "quota-unavailable" in f.notes diff --git a/tests/installer/test_gke_clusters.py b/tests/installer/test_gke_clusters.py new file mode 100644 index 0000000..961025c --- /dev/null +++ b/tests/installer/test_gke_clusters.py @@ -0,0 +1,83 @@ +from pathlib import Path +import sys + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from subprocess import CompletedProcess + +from knoe.core.ops import gke_clusters + + +def test_build_kubectl_env_for_cluster_sets_explicit_role_and_context(): + env = gke_clusters.build_kubectl_env_for_cluster( + base_env={"EXISTING": "1"}, + kubecontext="ctx-app", + cluster_name="knoe-dev-0", + cluster_role="app", + ) + + assert env["EXISTING"] == "1" + assert env["KUBECTL_CONTEXT"] == "ctx-app" + assert env["CLUSTER_NAME"] == "knoe-dev-0" + assert env["KNOE_CLUSTER_ROLE"] == "app" + assert env["KNOE_APP_CLUSTER_NAME"] == "knoe-dev-0" + + +def test_get_cluster_credentials_builds_expected_gcloud_command(monkeypatch): + called = {} + + def fake_run(cmd, capture_output, text): + called["cmd"] = cmd + return CompletedProcess(cmd, 0, stdout="", stderr="") + + monkeypatch.setattr(gke_clusters.subprocess, "run", fake_run) + + context = gke_clusters.get_cluster_credentials( + project_id="proj-1", + cluster_name="knoe-cnpg-0", + location="us-central1", + ) + + assert context == "gke_proj-1_us-central1_knoe-cnpg-0" + assert called["cmd"] == [ + "gcloud", + "container", + "clusters", + "get-credentials", + "knoe-cnpg-0", + "--project", + "proj-1", + "--region", + "us-central1", + "--quiet", + ] + + +def test_ensure_db_cluster_creates_standard_cluster_with_minimum_nodes(monkeypatch): + commands = [] + + def fake_run(cmd, log=None): + commands.append(cmd) + if "describe" in cmd: + return CompletedProcess(cmd, 1, stdout="", stderr="not found") + return CompletedProcess(cmd, 0, stdout="created", stderr="") + + monkeypatch.setattr(gke_clusters, "_run", fake_run) + + spec = gke_clusters.GkeClusterSpec( + name="knoe-cnpg-0", + mode="standard", + location="us-central1", + machine_type="e2-small", + node_count=1, + node_pool_name="cnpg-db-pool", + ) + + gke_clusters.ensure_db_cluster(project_id="proj-1", spec=spec) + + create_cmd = commands[-1] + assert create_cmd[:5] == ["gcloud", "container", "clusters", "create", "knoe-cnpg-0"] + assert "--num-nodes" in create_cmd + assert create_cmd[create_cmd.index("--num-nodes") + 1] == "3" diff --git a/tests/installer/test_ops_cli_storage_browser.py b/tests/installer/test_ops_cli_storage_browser.py new file mode 100644 index 0000000..e909bc3 --- /dev/null +++ b/tests/installer/test_ops_cli_storage_browser.py @@ -0,0 +1,97 @@ +from pathlib import Path +import json +import sys +from types import SimpleNamespace + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from knoe.core.cluster_storage_browser import ( + ClusterStorageBrowserResult, + ClusterStorageRequest, + RegionCandidate, + RegionFeasibility, +) +from knoe.ops import cli as ops_cli + + +def _fake_browser_result() -> ClusterStorageBrowserResult: + req = ClusterStorageRequest(pgdata_gb=100.0, wal_gb=25.0) + feasibility = RegionFeasibility( + region="us-central1", + timezone_group="americas", + premium_headroom_gb=250.0, + standard_headroom_gb=180.0, + can_premium=True, + can_standard=True, + default_pgdata_class="premium-rwo", + default_wal_class="premium-rwo", + feasibility_rank=0, + notes=(), + ) + return ClusterStorageBrowserResult( + generated_at="2026-01-01T00:00:00+00:00", + project_id="p-1", + timezone_group="americas", + request=req, + candidates=(RegionCandidate(feasibility=feasibility, latency_ms=12.3),), + quota_link="https://cloud.google.com/compute/quotas", + quota_console_link="https://console.cloud.google.com/iam-admin/quotas?project=p-1", + ) + + +def test_storage_browser_json_output(monkeypatch, capsys): + monkeypatch.setattr(ops_cli, "build_cluster_storage_browser_result", lambda **_kwargs: _fake_browser_result()) + + rc = ops_cli.main( + [ + "storage-browser", + "--project", + "p-1", + "--regions", + "us-central1", + "--output", + "json", + ] + ) + out = capsys.readouterr().out + assert rc == 0 + doc = json.loads(out) + assert doc["projectId"] == "p-1" + assert doc["candidates"][0]["defaultPgdataClass"] == "premium-rwo" + + +def test_storage_browser_table_output(monkeypatch, capsys): + monkeypatch.setattr(ops_cli, "build_cluster_storage_browser_result", lambda **_kwargs: _fake_browser_result()) + + rc = ops_cli.main( + [ + "storage-browser", + "--project", + "p-1", + "--regions", + "us-central1", + "--output", + "table", + ] + ) + out = capsys.readouterr().out + assert rc == 0 + assert "region\tfeasible\tdefault(pg/wal)" in out + assert "us-central1" in out + assert "premium-rwo/premium-rwo" in out + + +def test_legacy_component_action_dispatch_remains_supported(monkeypatch): + called = {} + + def _status(_ctx): + called["ok"] = True + return 7 + + monkeypatch.setattr(ops_cli.importlib, "import_module", lambda _name: SimpleNamespace(status=_status)) + rc = ops_cli.main(["common_core", "status", "--mode", "prod"]) + assert called["ok"] is True + assert rc == 7 diff --git a/tests/installer/test_prod_config.py b/tests/installer/test_prod_config.py index c5f6a8b..27dc0e5 100644 --- a/tests/installer/test_prod_config.py +++ b/tests/installer/test_prod_config.py @@ -24,6 +24,8 @@ def test_default_prod_config_uses_knoey_knoe_conventions(): assert cfg.kubernetes.namespace == "ecosystem-0" assert cfg.cloud.clusterName == "knoe-prod" assert cfg.database.clusterName == "knoe-db" + assert cfg.database.pgdataStorageClass == "premium-rwo" + assert cfg.database.walStorageClass == "premium-rwo" assert cfg.database.appDatabase == "knoey" assert cfg.database.metaDatabase == "knoe_meta" assert cfg.backups.backupBucket == "knoe-0-backups" @@ -57,6 +59,10 @@ def test_validation_requires_project_and_enforces_ecosystem_and_positive_numbers def test_yaml_and_opentofu_preview_contains_expected_values(): cfg = default_prod_config() cfg.cloud.projectId = "my-gcp-project" + cfg.database.pgdataStorageClass = "standard-rwo" + cfg.database.walStorageClass = "premium-rwo" + cfg.database.pgdataSizeGi = 120 + cfg.database.walSizeGi = 32 yaml_preview = canonical_yaml(cfg) tf_vars = opentofu_vars(cfg) @@ -64,10 +70,16 @@ def test_yaml_and_opentofu_preview_contains_expected_values(): assert "kind: KnoeProductionConfig" in yaml_preview assert "ecosystemId: 0" in yaml_preview assert "projectId: my-gcp-project" in yaml_preview + assert "pgdataStorageClass: standard-rwo" in yaml_preview + assert "walStorageClass: premium-rwo" in yaml_preview assert "frontdoorHost: knoey.com" in yaml_preview assert tf_vars["project_id"] == "my-gcp-project" assert tf_vars["namespace"] == "ecosystem-0" assert tf_vars["cnpg_cluster_name"] == "knoe-db" + assert tf_vars["db_pgdata_storage_class"] == "standard-rwo" + assert tf_vars["db_wal_storage_class"] == "premium-rwo" + assert tf_vars["db_pgdata_size_gi"] == 120 + assert tf_vars["db_wal_size_gi"] == 32 def test_prod_config_api_plan_apply_status_and_logs_flow(): diff --git a/tests/test_service_layer_navigation.py b/tests/test_service_layer_navigation.py index 02b897f..eb64d34 100644 --- a/tests/test_service_layer_navigation.py +++ b/tests/test_service_layer_navigation.py @@ -105,3 +105,10 @@ def test_cluster_nodes_prev_goes_back_to_cluster_environment(mock_installer): mock_installer.show_page("cluster_nodes") mock_installer.on_prev() assert mock_installer.pages[mock_installer.page_index][0] == "init_cluster" + + +def test_common_services_prev_goes_back_to_cluster_nodes(mock_installer): + mock_installer.show_page("common_services") + with patch.object(mock_installer, "_should_show_cluster_nodes_screen", return_value=True): + mock_installer.on_prev() + assert mock_installer.pages[mock_installer.page_index][0] == "cluster_nodes"