mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 10:13:58 +00:00
checkpoint: stabilize k8s cluster setup and shelve cluster-storage UI
- make cluster-storage milestone opt-in and remove installer cluster-storage step from UI navigation\n- add cluster storage browser and GKE cluster ops helpers with CLI coverage\n- update k8s/CNPG config and install flow files for corrected cluster setup\n- add/refresh tests for storage browser, GKE ops, prod config, and service-layer navigation Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
chrisfu
parent
df6de9138e
commit
64e1cef16c
@ -1,23 +1,21 @@
|
||||
{
|
||||
"assignments": {
|
||||
"0": "gk3-knoe-dev-0-pool-1-11c87d92-8qr5",
|
||||
"1": "gk3-knoe-dev-0-pool-1-88b7392b-62c4",
|
||||
"2": "gk3-knoe-dev-0-pool-1-88b7392b-d6lb"
|
||||
"0": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z",
|
||||
"1": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z",
|
||||
"2": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z"
|
||||
},
|
||||
"cluster_name": "knoe-db",
|
||||
"desired_instances": 3,
|
||||
"eligible_nodes": [
|
||||
"gk3-knoe-dev-0-pool-1-11c87d92-8qr5",
|
||||
"gk3-knoe-dev-0-pool-1-88b7392b-62c4",
|
||||
"gk3-knoe-dev-0-pool-1-88b7392b-d6lb"
|
||||
"gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z"
|
||||
],
|
||||
"metadata": {
|
||||
"prior_plan_present": true,
|
||||
"reason": "reused",
|
||||
"regenerated": false,
|
||||
"reused": true
|
||||
"reason": "assigned_node_no_longer_eligible",
|
||||
"regenerated": true,
|
||||
"reused": false
|
||||
},
|
||||
"plan_hash": "334860873593911a",
|
||||
"plan_id": "cnpg-placement-334860873593911a",
|
||||
"plan_hash": "c1213b43098112f3",
|
||||
"plan_id": "cnpg-placement-c1213b43098112f3",
|
||||
"schema_version": "v1"
|
||||
}
|
||||
|
||||
@ -1,15 +1,13 @@
|
||||
{
|
||||
"assignments": {
|
||||
"0": "gk3-knoe-dev-0-pool-1-11c87d92-8qr5",
|
||||
"1": "gk3-knoe-dev-0-pool-1-88b7392b-62c4",
|
||||
"2": "gk3-knoe-dev-0-pool-1-88b7392b-d6lb"
|
||||
"0": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z",
|
||||
"1": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z",
|
||||
"2": "gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z"
|
||||
},
|
||||
"cluster_name": "knoe-db",
|
||||
"desired_instances": 3,
|
||||
"eligible_nodes": [
|
||||
"gk3-knoe-dev-0-pool-1-11c87d92-8qr5",
|
||||
"gk3-knoe-dev-0-pool-1-88b7392b-62c4",
|
||||
"gk3-knoe-dev-0-pool-1-88b7392b-d6lb"
|
||||
"gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z"
|
||||
],
|
||||
"metadata": {
|
||||
"prior_plan_present": true,
|
||||
@ -17,7 +15,7 @@
|
||||
"regenerated": false,
|
||||
"reused": true
|
||||
},
|
||||
"plan_hash": "334860873593911a",
|
||||
"plan_id": "cnpg-placement-334860873593911a",
|
||||
"plan_hash": "c1213b43098112f3",
|
||||
"plan_id": "cnpg-placement-c1213b43098112f3",
|
||||
"schema_version": "v1"
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
; Prole Master Configuration File
|
||||
; Generated by install.py on 2026-04-06 05:37:48
|
||||
; Generated by install.py on 2026-04-07 09:23:28
|
||||
; This file is used as input for Ansible deployment and k8s cluster creation.
|
||||
|
||||
[User]
|
||||
@ -56,7 +56,7 @@ init_cluster.cluster_env = dev
|
||||
init_cluster.deployment_target = prole-dev-cluster
|
||||
init_cluster.gitops_enabled = false
|
||||
init_cluster.k3s_server_url = https://myrddin.prole.org:6443
|
||||
init_cluster.k3s_token = ${PROLE_SECRET:v1:UQd8eq6blSjrABFq:npCjmA_RyXc2352u-CgKeHZSuEmOmjygNrDHQH9N6B_JgqYFbbnw0e3lKX_VSHTOQvfBi9OwJ2Nh3kPXoFGgbOnxcp3UzIwG0yPXcAXAu3KHeaYFe1eVnONz63oz_qYSaXOwh7a7NS-05cmTUqB5g2YJRK_yVXmn1LJf3g==}
|
||||
init_cluster.k3s_token = ${PROLE_SECRET:v1:9Vez82_9Cxpex2oY:oWLW5wEBqIzPEIOt3EZnB2JLj4IAORTKMxMxtNwDC58Q266OzM83h2l-PV30e_V4QblEmDmGpQyAg28FkDNBpxlYa6HsNk8HnkCLQIP8MK_j51e3ft6UUnMISGSKjkoSl2wMdLy5k9QhCdoQeOGhxN2BpiEscZBUgpDIQw==}
|
||||
init_cluster.kerberos_enabled = true
|
||||
init_cluster.mode = k3d
|
||||
init_cluster.start_cluster = true
|
||||
@ -100,7 +100,7 @@ DOCKER_PRELOAD = false
|
||||
KNOE_DB_USER = root
|
||||
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
|
||||
PROLE_K3S_SERVER = https://myrddin.prole.org:6443
|
||||
PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:ViP2nnYnrHPJoSVL:NBMjuyphR6SayWvXTLzQVPZRis5CobeAbEP7vdBaHHSNO7fseQMkDTJLt3ssAHEVpJiQcAwqc84f5-x9AaniXgIgXo9Ka1cl560C6uMPrVvQ_YzlWQiUoD87PboGyBcXms794JvTJ7LLAiCDwHrxm-f6-oaPOgnAzMvwZA==}
|
||||
PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:J-6LlQf6BDV1Szx0:b1D60bQkckSeXNHuMobcF5LXy2-ui6oarFisXzjOjTPX2dvXua_v3O8kBicxjvStyTUX5N1EDnunnm4eGWYElfXtgdoHsF8kc_7KkRiNt37ad7-KVShwKMHPDhNsJA8TQ_L3PjhxpQbrwyCo2PWkXoNlCtjn57qUHeqW1Q==}
|
||||
PROLE_OPENTOFU_URL = http://127.0.0.1:8080
|
||||
SERVICE_NAMESPACE = knoe-system
|
||||
SUPABASE_PV_BASE = /synology/d005
|
||||
@ -173,7 +173,7 @@ MODE = k3d
|
||||
CLUSTER_ENV = prole-service-cluster
|
||||
DISPLAY_NAME = prole-service-cluster
|
||||
K3S_SERVER_URL = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:ZYtbrUl6yqVscqkt:mMI_jblCsK2_fhlGoUUeXgQPD3yJ0bdN9OYTpO_Bv25BoGdp5C-nJK7EmdPtYC_dlw4UzxDZIYtz7NssNeT7PlYOKgK7Vq2rpKElBxzfLiE8QYYQmIB66IM_Pmu1FwleLDf5GT71A3UuWkzqHfh0EAe0dsX181loromFpQ==}
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:sOSNr3SQq1As2lxH:RM-5UCAufudeBWFaEST-pC4y17Gs0Fu106yX_0VsseCRcjrLQUT4f4uFmUxY8ICYE9zArCyJEtgZNNGoRcybQfBUoTTAAAuGrK1nMuakuRd8j_aPEnH3XHSLgraIJIlizCSpA8miq1U2UJRv6KmJjxzoMmfKpExKqKiWtw==}
|
||||
MODE = k3s
|
||||
PIPELINE_URL = http://127.0.0.1:8080
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
; Prole Master Configuration File
|
||||
; Generated by install.py on 2026-04-06 05:49:04
|
||||
; Generated by install.py on 2026-04-07 09:24:02
|
||||
; This file is used as input for Ansible deployment and k8s cluster creation.
|
||||
|
||||
[User]
|
||||
@ -56,7 +56,7 @@ init_cluster.cluster_env = prod
|
||||
init_cluster.deployment_target = prole-prod-cluster
|
||||
init_cluster.gitops_enabled = false
|
||||
init_cluster.k3s_server_url = https://myrddin.prole.org:6443
|
||||
init_cluster.k3s_token = ${PROLE_SECRET:v1:f6Ey7pFow7vqJCXh:L0bJyYn5OfamHRyoOHLCd0WpwnukFKUnSW72rN3v28sPNsTX16bDOjnuXOES4SVZ6beRH159KOK7UhopGu6g7CWiOn52-CF2fE8jnl3N_inZDUwyH8IDVt2b_xDbagVcyjoqbZddABUWJMAQ56wXcYaxh1o3y8awfB5SMA==}
|
||||
init_cluster.k3s_token = ${PROLE_SECRET:v1:8jsa5B6uohhtyjxS:pfwgP6OkFpEcR-f4_3_DkCYwRuWlGIWkdg--1vesoya0OpGR9V6nFFW5DsEs5ARzBFkxkIxbgiNDOXC-t7oa9u578IDg2fO0FX4BXguvhqJi8lFjltEbjP4EL8LaHmkBpEZ3ko9egaNqVc03mFn5Fl5Ndc_uj9a-Ehe1jg==}
|
||||
init_cluster.kerberos_enabled = true
|
||||
init_cluster.mode = k8s
|
||||
init_cluster.start_cluster = true
|
||||
@ -67,8 +67,8 @@ init_db_build.run_build = true
|
||||
init_password.cluster_name = ${CLUSTER_NAME}
|
||||
init_password.db_host_port = 5432
|
||||
init_password.db_namespace = ${DATABASE_NAMESPACE}
|
||||
init_password.db_password = ${PROLE_SECRET:v1:cVWA3Hry0N50ExMf:VKkhmpeYpd9xB_v0zPzb7Hv7cvq7LXl7scOU1wHGS0NSc3Fa}
|
||||
init_password.db_password_confirm = ${PROLE_SECRET:v1:cVWA3Hry0N50ExMf:VKkhmpeYpd9xB_v0zPzb7Hv7cvq7LXl7scOU1wHGS0NSc3Fa}
|
||||
init_password.db_password =
|
||||
init_password.db_password_confirm =
|
||||
init_password.db_username = root
|
||||
init_password.generate_ssh_key = true
|
||||
init_scripts.run_scripts = true
|
||||
@ -91,14 +91,13 @@ ARTIFACT_REGISTRY = us-west3-docker.pkg.dev/plenary-truck-485623-p7/knoe-system
|
||||
ARTIFACT_REGISTRY_AVAILABLE = true
|
||||
CLUSTER_ENV = prod
|
||||
CLUSTER_NAME = knoe-db
|
||||
CNPG_ELIGIBLE_NODES = gk3-knoe-dev-0-pool-1-11c87d92-8qr5,gk3-knoe-dev-0-pool-1-88b7392b-62c4,gk3-knoe-dev-0-pool-1-88b7392b-d6lb
|
||||
CNPG_ELIGIBLE_NODES = gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z
|
||||
CNPG_PLACEMENT_PLAN_FILE = /Users/chrisfu/dev/prole/conf/cnpg-placement/knoe-system-knoe-db.json
|
||||
CNPG_PLACEMENT_PLAN_HASH = 334860873593911a
|
||||
CNPG_PLACEMENT_PLAN_ID = cnpg-placement-334860873593911a
|
||||
CNPG_STAGE1_NODE = gk3-knoe-dev-0-pool-1-11c87d92-8qr5
|
||||
DATABASE_NAMESPACE = knoe-db-0
|
||||
CNPG_PLACEMENT_PLAN_HASH = c1213b43098112f3
|
||||
CNPG_PLACEMENT_PLAN_ID = cnpg-placement-c1213b43098112f3
|
||||
CNPG_STAGE1_NODE = gk3-knoe-dev-0-nap-4jvx077g-1a599f8c-pv5z
|
||||
DATABASE_NAMESPACE = knoe-db-18-008-18-009-18-013
|
||||
DB_HOST_PORT = 5432
|
||||
DB_PASSWORD = ${PROLE_SECRET:v1:iPFMI8pPl5-KJ9Fp:rMxZd-E-FjuTvzcjxlR5ox9wuJ2lEOwduEjZ72QOsSgt153s}
|
||||
DEPLOYMENT_MODE = k8s
|
||||
DEPLOYMENT_TARGET = prole-prod-cluster
|
||||
DOCKER_PRELOAD = false
|
||||
@ -107,7 +106,7 @@ KUBECONFIG = /Users/chrisfu/.kube/config
|
||||
KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-0
|
||||
OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2
|
||||
PROLE_K3S_SERVER = https://myrddin.prole.org:6443
|
||||
PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:Dq3A_1cu9f2v_AbU:OJyqpneBdSuXI9ej-nVVh8SIJx-7hmY9Wg4FCu313zk76rNw7STN6ymVudSnJZtUUSKP4IxLoA-bcUkndm4CZWNWCNMnQ-WpvXBIG4GIBeExmD15FcOIJy1VaeW5To0hrKBEEgtwov3EiWr8aVEvpgaUlUfir2m_SbLwpQ==}
|
||||
PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:10g1WjSwQq3MiUNh:XkobEkCjr8C5VHxPEFeXEOTA6CA5olncCUpaRgUNZuwrsWBKR_p0jeo5xAKcsRkYT-s1mrmV4ZaqgP93JfBGnCybgC5kHJe-ch_UVf4rkJKV75kVnHzrSlIRYZdng-o1_rwfLEARxJ2nLHvmNLnLV6n-TZUmSy3PZYfOeg==}
|
||||
SERVICE_NAMESPACE = knoe-system
|
||||
SUPABASE_PV_BASE = /synology/d005
|
||||
SUPABASE_PV_BASE_DIR = /synology/d005
|
||||
@ -167,7 +166,7 @@ SUPABASE_ENABLED = False
|
||||
[Initialize Cluster]
|
||||
ENVIRONMENT = prod
|
||||
K3S_SERVER_URL = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:ebAZ1lr6dlSQnleP:PUuIW7MUrt-HM_wwAW5BJep9wY5iwf9QuItmnsi627esMDWpZdfcYlBOCa5FrGR8Zhz8y9lB5GOIHseFz31GQMq6Ogbisurue7TWL2kSJS0oFGYfz1FH0zuvtIWwwFwArAubY2FDld9bjAZkr5w2hGzg2A9UnmPkuCslAg==}
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:ILhG782khL7eyT7y:b3s76jimM0Q7hYH7hnSYM2XOfBXknl9mXvjzfLI4jTz-c4dHLCJXG8WlOiFf8FM0HfoIK-xu7NxSdym2UXBzCkuENFIJ2Y1GwNr9Nt2ForgfuwKaYVfoGnDc0ERp_S6qaKP01uLDeR95hUS56Wtuh3TNPN0gp14qvBjX4g==}
|
||||
|
||||
[Dev Cluster (k3d)]
|
||||
CLUSTER_ENV = dev
|
||||
@ -179,7 +178,7 @@ MODE = k3d
|
||||
CLUSTER_ENV = prole-service-cluster
|
||||
DISPLAY_NAME = prole-service-cluster
|
||||
K3S_SERVER_URL = https://myrddin.prole.org:6443
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:PRJRFNuCskXkEtvt:oh5MP9bB-iEjgP_E7FlqgNpi_5VV-W7tDt1sjz-Fsigy8nrGuliliPyUEz2sqPouMnq_qlfqKTi_vC7G3bIYGQpbf55xX2GXebQBvjUJIfrujGq7Yk-8IQRWBBhfWKcQdTy0EcT99ynwHmjTlI0o56JV5pP1GiCP59TmDw==}
|
||||
K3S_TOKEN = ${PROLE_SECRET:v1:wxpiSOY6t_5_gaiT:vfyzLLUJUYc3SmQ48yDYrTpeWQ2LfVoiEWvzRwDPzm1Mx6a6aW0QjJLK52Rbwe5hVtwwDjcglakhM8eBjFV-ceFaLIl7Tsr7CwARLuPD6NBJQrc1qnHuKTh5DOSMGF_emnP_fWT06-M242VZImmVE_2Q82Cr45Gd91h93g==}
|
||||
MODE = k3s
|
||||
|
||||
[GCP]
|
||||
@ -187,10 +186,6 @@ BILLING_ACCOUNT = 01193C-25783B-3211AD
|
||||
BILLING_PROJECT = plenary-truck-485623-p7
|
||||
ORG_ID = 584001916389
|
||||
PROJECT_ID = plenary-truck-485623-p7
|
||||
billing_account = 01193C-25783B-3211AD
|
||||
billing_project = plenary-truck-485623-p7
|
||||
org_id = 584001916389
|
||||
project_id = plenary-truck-485623-p7
|
||||
|
||||
[Prod Cluster (k8s)]
|
||||
ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging
|
||||
@ -206,7 +201,7 @@ Configure routing, TLS, and frontdoor domain mappings
|
||||
Prepare migration artifacts from source environment
|
||||
Generate OpenTofu plan and apply production pipeline
|
||||
MODE = k8s
|
||||
OPENTOFU_VARS = {"admin_user":"knoe_admin","app_database":"knoey","app_user":"knoey_app","backup_bucket":"knoe-0-backups","backup_provider":"gcs","backup_retention_days":14,"bootstrap_admin_email":"admin@knoey.com","cluster_name":"knoe-dev-0","cnpg_cluster_name":"knoe-db","db_instances":3,"db_storage_class":"premium-rwo","db_storage_size_gi":100,"ecosystem_id":0,"environment":"production","frontdoor_host":"knoey.com","gcp_project_id":"plenary-truck-485623-p7","gcp_region":"\u2713 us-west3","google_hosted_domain":"","meta_database":"knoe_meta","migration_continuous_until_cutover":false,"migration_mode":"snapshot-restore","migration_source_database":"knoey","migration_source_environment":"prole.org","migration_source_host":"knoe-local-db.prole.org","migration_source_password_ref":"secretref://local-source-db-password","migration_source_port":5432,"migration_source_user":"replication_user","namespace":"ecosystem-0","oidc_client_id_ref":"secretref://google-oidc-client-id","oidc_client_secret_ref":"secretref://google-oidc-client-secret","oidc_issuer":"https://accounts.google.com","platform_domain":"knoe.dev","postgres_version":"16","project_id":"plenary-truck-485623-p7","region":"\u2713 us-west3","tls_mode":"managed","wal_bucket":"knoe-0-wal","workload_identity_sa":"cnpg-backup"}
|
||||
OPENTOFU_VARS = {"admin_user":"knoe_admin","app_database":"knoey","app_user":"knoey_app","backup_bucket":"knoe-0-backups","backup_provider":"gcs","backup_retention_days":14,"bootstrap_admin_email":"admin@knoey.com","cluster_name":"knoe-dev-0","cnpg_cluster_name":"knoe-db","db_instances":3,"db_pgdata_size_gi":100,"db_pgdata_storage_class":"premium-rwo","db_storage_class":"premium-rwo","db_storage_size_gi":100,"db_wal_size_gi":25,"db_wal_storage_class":"premium-rwo","ecosystem_id":0,"environment":"production","frontdoor_host":"knoey.com","gcp_project_id":"plenary-truck-485623-p7","gcp_region":"\u2713 us-west3","google_hosted_domain":"","meta_database":"knoe_meta","migration_continuous_until_cutover":false,"migration_mode":"snapshot-restore","migration_source_database":"knoey","migration_source_environment":"prole.org","migration_source_host":"knoe-local-db.prole.org","migration_source_password_ref":"secretref://local-source-db-password","migration_source_port":5432,"migration_source_user":"replication_user","namespace":"ecosystem-0","oidc_client_id_ref":"secretref://google-oidc-client-id","oidc_client_secret_ref":"secretref://google-oidc-client-secret","oidc_issuer":"https://accounts.google.com","platform_domain":"knoe.dev","postgres_version":"16","project_id":"plenary-truck-485623-p7","region":"\u2713 us-west3","tls_mode":"managed","wal_bucket":"knoe-0-wal","workload_identity_sa":"cnpg-backup"}
|
||||
PRODUCTION_CONFIG_YAML = kind: KnoeProductionConfig
|
||||
metadata:
|
||||
ecosystemId: 0
|
||||
@ -231,6 +226,10 @@ spec:
|
||||
instances: 3
|
||||
storageClass: premium-rwo
|
||||
storageSizeGi: 100
|
||||
pgdataStorageClass: premium-rwo
|
||||
walStorageClass: premium-rwo
|
||||
pgdataSizeGi: 100
|
||||
walSizeGi: 25
|
||||
appDatabase: knoey
|
||||
metaDatabase: knoe_meta
|
||||
appUser: knoey_app
|
||||
@ -268,7 +267,7 @@ spec:
|
||||
; No configuration values captured yet for this section.
|
||||
|
||||
[Initialization Scripts]
|
||||
STATUS = Attempted
|
||||
; No configuration values captured yet for this section.
|
||||
|
||||
[Deployment]
|
||||
MODE = k3d
|
||||
|
||||
@ -14,12 +14,12 @@ spec:
|
||||
|
||||
affinity:
|
||||
enablePodAntiAffinity: true
|
||||
# Keep spread as a preference during baseline Autopilot bring-up so 3 pods can still schedule
|
||||
# when the cluster is temporarily constrained; strict topology can be enforced in later rollout.
|
||||
# Keep spread as a preference for small dedicated Standard DB clusters so 3 pods can still
|
||||
# schedule while nodes reconcile; strict topology can be enforced in later rollout.
|
||||
podAntiAffinityType: preferred
|
||||
topologyKey: kubernetes.io/hostname # physical node boundary (not zone)
|
||||
tolerations:
|
||||
# Allow scheduling on GKE Spot nodes (common in knoe-dev-0 to reduce cost).
|
||||
# Allow scheduling on GKE Spot nodes when explicitly enabled for this DB cluster.
|
||||
# Without this toleration the cluster-autoscaler predicate simulation fails
|
||||
# for any MIG whose nodes carry the spot taint, blocking scale-up entirely.
|
||||
- key: "cloud.google.com/gke-spot"
|
||||
@ -117,24 +117,24 @@ spec:
|
||||
enableSuperuserAccess: true
|
||||
|
||||
storage:
|
||||
size: 20Gi
|
||||
size: 50Gi
|
||||
pvcTemplate:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
storageClassName: standard-rwo # pd-balanced: faster dynamic provisioning in GKE Autopilot than pd-ssd
|
||||
storage: 50Gi
|
||||
storageClassName: premium-rwo # pd-ssd; 3×50Gi PGDATA + 3×50Gi WAL = 300Gi total (fits 300GB quota)
|
||||
|
||||
walStorage:
|
||||
size: 5Gi
|
||||
size: 50Gi
|
||||
pvcTemplate:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
storageClassName: standard-rwo
|
||||
storage: 50Gi
|
||||
storageClassName: premium-rwo
|
||||
|
||||
monitoring:
|
||||
enablePodMonitor: true # GKE managed Prometheus
|
||||
|
||||
37
install.sh
37
install.sh
@ -6,6 +6,23 @@
|
||||
#
|
||||
# Bootstrap via curl (first-time install):
|
||||
# curl -fsSL https://knoey.com/install.sh | bash
|
||||
#
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dual-cluster GKE architecture (prod / k8s mode)
|
||||
# ---------------------------------------------------------------------------
|
||||
# knoe-dev-0 — Autopilot cluster: app workloads (GitLab, platform), service mesh knoe-0
|
||||
# knoe-cnpg-0 — Standard GKE cluster: CloudNative-PG only (rw/ro/r pod separation)
|
||||
# 3 nodes × e2-standard-2, node pool: cnpg-db-pool
|
||||
#
|
||||
# GCP quota requirements (project: plenary-truck-485623-p7, region: us-west3):
|
||||
# pd-standard : node boot disks (3 × 50 GB = 150 GB — does NOT count against pd-ssd quota)
|
||||
# pd-ssd : PGDATA PVCs (3 × 50 Gi = 150 Gi)
|
||||
# WAL PVCs (3 × 50 Gi = 150 Gi)
|
||||
# Total pd-ssd = 300 Gi (requires 300 GB pd-ssd regional quota)
|
||||
#
|
||||
# To patch an existing knoe-cnpg-0 cluster with wrong boot disk type:
|
||||
# ./scripts/patch_clusters.sh # dry-run first: DRY_RUN=true ./scripts/patch_clusters.sh
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@ -14,6 +31,25 @@ PROLE_REPO_URL="${PROLE_REPO_URL:-https://gitlab.knoey.com/prole/prole.git}"
|
||||
# Default install directory for bootstrap installs.
|
||||
_INSTALL_DIR="${PROLE_HOME:-$HOME/prole}"
|
||||
|
||||
_check_gcp_tools() {
|
||||
local missing=0
|
||||
if ! command -v gcloud >/dev/null 2>&1; then
|
||||
echo "Error: gcloud CLI is required for GKE cluster management." >&2
|
||||
echo " Install: https://cloud.google.com/sdk/docs/install" >&2
|
||||
missing=1
|
||||
fi
|
||||
if ! command -v kubectl >/dev/null 2>&1; then
|
||||
echo "Error: kubectl is required for Kubernetes cluster operations." >&2
|
||||
echo " Install: https://kubernetes.io/docs/tasks/tools/" >&2
|
||||
missing=1
|
||||
fi
|
||||
if [[ "$missing" -ne 0 ]]; then
|
||||
echo "" >&2
|
||||
echo "Install missing tools and re-run install.sh." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
_bootstrap() {
|
||||
echo "==> Bootstrapping Prole installer..."
|
||||
command -v git >/dev/null 2>&1 || { echo "Error: git is required." >&2; exit 1; }
|
||||
@ -40,6 +76,7 @@ if [[ -n "${_script_dir}" && -d "${_script_dir}/knoe" ]]; then
|
||||
# Prefer the venv Python at PROLE_HOME (or the repo root) when available;
|
||||
# it carries all prole_requirements.txt dependencies.
|
||||
_VENV_PYTHON="${PROLE_HOME:-${_script_dir}}/bin/python3"
|
||||
_check_gcp_tools
|
||||
if [[ -x "${_VENV_PYTHON}" ]]; then
|
||||
exec "${_VENV_PYTHON}" -m knoe.ui.screens "$@"
|
||||
fi
|
||||
|
||||
@ -88,6 +88,13 @@ from knoe.core.ops import monitoring as monitoring_ops
|
||||
from knoe.core.ops import openbao as openbao_ops
|
||||
from knoe.core.ops import opentofu as opentofu_ops
|
||||
from knoe.core.ops import registry as registry_ops
|
||||
from knoe.core.ops.gke_clusters import (
|
||||
GkeClusterSpec,
|
||||
build_kubectl_env_for_cluster,
|
||||
ensure_app_cluster,
|
||||
ensure_db_cluster,
|
||||
get_cluster_credentials,
|
||||
)
|
||||
from knoe.core.ops.cloudnative_pg import (
|
||||
initialize as cnpg_initialize,
|
||||
deploy as cnpg_deploy,
|
||||
@ -108,6 +115,16 @@ from knoe.core.stream_exec import run_streaming_cmd
|
||||
from typing import Callable, Sequence
|
||||
|
||||
|
||||
DEFAULT_APP_CLUSTER_NAME = "knoe-dev-0"
|
||||
DEFAULT_APP_CLUSTER_MODE = "autopilot"
|
||||
DEFAULT_DB_CLUSTER_NAME = "knoe-cnpg-0"
|
||||
DEFAULT_DB_CLUSTER_MODE = "standard"
|
||||
DEFAULT_DB_CLUSTER_NODE_COUNT = 3
|
||||
DEFAULT_DB_CLUSTER_MACHINE_TYPE = "e2-standard-2"
|
||||
DEFAULT_DB_BOOT_DISK_TYPE = "pd-standard"
|
||||
DEFAULT_DB_BOOT_DISK_SIZE_GB = 50
|
||||
|
||||
|
||||
def _configure_unbuffered_io():
|
||||
os.environ.setdefault("PYTHONUNBUFFERED", "1")
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
@ -1480,14 +1497,22 @@ class KnoeInstaller:
|
||||
self.prole_cfg_data["Port Forwards"] = pf_section
|
||||
|
||||
# --------------------------------------------- script env
|
||||
def _script_env_for_namespace(self, namespace: str) -> dict:
|
||||
def _script_env_for_namespace(self, namespace: str, cluster_role: str = "db") -> dict:
|
||||
namespace = _safe_str(namespace)
|
||||
cluster_role = "app" if str(cluster_role).strip().lower() == "app" else "db"
|
||||
env = os.environ.copy()
|
||||
root = getattr(self, "project_root", None) or PROJECT_ROOT
|
||||
cluster_name = (
|
||||
self._app_cluster_name() if cluster_role == "app" else self._cnpg_cluster_name()
|
||||
)
|
||||
env["PROLE_HOME"] = str(root)
|
||||
env["PROLE_SERVICE"] = str(root)
|
||||
env["DATABASE_NAMESPACE"] = namespace
|
||||
env["CLUSTER_NAME"] = self._cnpg_cluster_name()
|
||||
env["CLUSTER_NAME"] = cluster_name
|
||||
env["APP_CLUSTER_NAME"] = self._app_cluster_name()
|
||||
env["DB_CLUSTER_NAME"] = self._cnpg_cluster_name()
|
||||
env["APP_CLUSTER_MODE"] = self._app_cluster_mode()
|
||||
env["DB_CLUSTER_MODE"] = self._db_cluster_mode()
|
||||
# Transitional compatibility for scripts still reading NAMESPACE.
|
||||
env["NAMESPACE"] = namespace
|
||||
service_ns = self._service_namespace()
|
||||
@ -1597,7 +1622,9 @@ class KnoeInstaller:
|
||||
# For GKE/prod clusters, gcloud get-credentials writes to ~/.kube/config.
|
||||
# Read the context that was saved after cluster selection.
|
||||
gke_context = (
|
||||
(self.prole_cfg_data.get("Global", {}) or {}).get("KUBECONTEXT", "") or ""
|
||||
self._cluster_kubecontext(cluster_role)
|
||||
or (self.prole_cfg_data.get("Global", {}) or {}).get("KUBECONTEXT", "")
|
||||
or ""
|
||||
).strip()
|
||||
default_kube = str(Path.home() / ".kube" / "config")
|
||||
if Path(default_kube).exists():
|
||||
@ -1716,19 +1743,73 @@ class KnoeInstaller:
|
||||
)
|
||||
except Exception as e:
|
||||
self.err(f"[WARN] CNPG placement planning failed; using fallback topology: {e}")
|
||||
|
||||
return env
|
||||
return build_kubectl_env_for_cluster(
|
||||
base_env=env,
|
||||
kubecontext=self._cluster_kubecontext(cluster_role),
|
||||
cluster_name=cluster_name,
|
||||
cluster_role=cluster_role,
|
||||
)
|
||||
|
||||
def _cnpg_cluster_name(self) -> str:
|
||||
glob = self.prole_cfg_data.get("Global", {}) or {}
|
||||
cluster_name = (
|
||||
os.environ.get("CLUSTER_NAME")
|
||||
or os.environ.get("CNPG_CLUSTER_NAME")
|
||||
or self._get_input("init_password.db_cluster_name", "")
|
||||
or self._get_input("env_setup.DB_CLUSTER_NAME", "")
|
||||
or str(glob.get("CLUSTER_NAME") or "")
|
||||
or str(glob.get("CNPG_CLUSTER_NAME") or "")
|
||||
).strip()
|
||||
return cluster_name or "knoe-db"
|
||||
|
||||
def _app_cluster_name(self) -> str:
|
||||
glob = self.prole_cfg_data.get("Global", {}) or {}
|
||||
name = (
|
||||
self._get_input("init_password.app_cluster_name", "")
|
||||
or self._get_input("env_setup.APP_CLUSTER_NAME", "")
|
||||
or str(glob.get("APP_CLUSTER_NAME") or "")
|
||||
or DEFAULT_APP_CLUSTER_NAME
|
||||
)
|
||||
return str(name).strip() or DEFAULT_APP_CLUSTER_NAME
|
||||
|
||||
def _app_cluster_mode(self) -> str:
|
||||
glob = self.prole_cfg_data.get("Global", {}) or {}
|
||||
mode = (
|
||||
self._get_input("init_cluster.app_cluster_mode", "")
|
||||
or self._get_input("env_setup.APP_CLUSTER_MODE", "")
|
||||
or str(glob.get("APP_CLUSTER_MODE") or "")
|
||||
or DEFAULT_APP_CLUSTER_MODE
|
||||
)
|
||||
return str(mode).strip() or DEFAULT_APP_CLUSTER_MODE
|
||||
|
||||
def _db_cluster_mode(self) -> str:
|
||||
glob = self.prole_cfg_data.get("Global", {}) or {}
|
||||
mode = (
|
||||
self._get_input("init_cluster.db_cluster_mode", "")
|
||||
or self._get_input("env_setup.DB_CLUSTER_MODE", "")
|
||||
or str(glob.get("DB_CLUSTER_MODE") or "")
|
||||
or DEFAULT_DB_CLUSTER_MODE
|
||||
)
|
||||
return str(mode).strip() or DEFAULT_DB_CLUSTER_MODE
|
||||
|
||||
def _cluster_kubecontext(self, role: str) -> str:
|
||||
key = (
|
||||
"init_cluster.app_cluster_kubecontext"
|
||||
if role == "app"
|
||||
else "init_cluster.db_cluster_kubecontext"
|
||||
)
|
||||
fallback_key = (
|
||||
"env_setup.APP_CLUSTER_KUBECONTEXT"
|
||||
if role == "app"
|
||||
else "env_setup.DB_CLUSTER_KUBECONTEXT"
|
||||
)
|
||||
context = (self._get_input(key, "") or self._get_input(fallback_key, "")).strip()
|
||||
if context:
|
||||
return context
|
||||
if role == "app":
|
||||
return (self._get_input("init_cluster.selected_kubectx", "") or "").strip()
|
||||
return ""
|
||||
|
||||
def _cnpg_desired_instances(self) -> int:
|
||||
glob = self.prole_cfg_data.get("Global", {}) or {}
|
||||
raw = (os.environ.get("CNPG_INSTANCES") or str(glob.get("CNPG_INSTANCES") or "")).strip()
|
||||
@ -1893,6 +1974,12 @@ class KnoeInstaller:
|
||||
f"data_path={provisioned.data_path} wal_path={provisioned.wal_path}"
|
||||
)
|
||||
|
||||
def _cluster_storage_milestone_enabled(self) -> bool:
|
||||
raw = str(
|
||||
os.environ.get("PROLE_ENABLE_CLUSTER_STORAGE_MILESTONE", "")
|
||||
).strip().lower()
|
||||
return raw in {"1", "true", "yes", "on"}
|
||||
|
||||
# --------------------------------------------- authority / repair
|
||||
def _authority_context_missing(self) -> bool:
|
||||
enabled = self._get_input_bool(
|
||||
@ -3394,12 +3481,74 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
cluster_name = _expand_cfg_value(sec.get("CLUSTER_NAME", ""), cfg_vars)
|
||||
legacy["init_password.cluster_name"] = cluster_name
|
||||
legacy["env_setup.CLUSTER_NAME"] = cluster_name
|
||||
legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name)
|
||||
legacy.setdefault("init_password.db_cluster_name", cluster_name)
|
||||
if "CNPG_CLUSTER_NAME" in sec and "init_password.cluster_name" not in legacy:
|
||||
cluster_name = _expand_cfg_value(
|
||||
sec.get("CNPG_CLUSTER_NAME", ""), cfg_vars
|
||||
)
|
||||
legacy["init_password.cluster_name"] = cluster_name
|
||||
legacy["env_setup.CLUSTER_NAME"] = cluster_name
|
||||
legacy.setdefault("env_setup.DB_CLUSTER_NAME", cluster_name)
|
||||
legacy.setdefault("init_password.db_cluster_name", cluster_name)
|
||||
if "APP_CLUSTER_NAME" in sec:
|
||||
app_cluster_name = _expand_cfg_value(
|
||||
sec.get("APP_CLUSTER_NAME", ""), cfg_vars
|
||||
)
|
||||
if app_cluster_name:
|
||||
legacy["env_setup.APP_CLUSTER_NAME"] = app_cluster_name
|
||||
legacy["init_password.app_cluster_name"] = app_cluster_name
|
||||
if "APP_CLUSTER_MODE" in sec:
|
||||
legacy["env_setup.APP_CLUSTER_MODE"] = _expand_cfg_value(
|
||||
sec.get("APP_CLUSTER_MODE", ""), cfg_vars
|
||||
)
|
||||
legacy["init_cluster.app_cluster_mode"] = legacy[
|
||||
"env_setup.APP_CLUSTER_MODE"
|
||||
]
|
||||
if "DB_CLUSTER_NAME" in sec:
|
||||
db_cluster_name = _expand_cfg_value(
|
||||
sec.get("DB_CLUSTER_NAME", ""), cfg_vars
|
||||
)
|
||||
if db_cluster_name:
|
||||
legacy["env_setup.DB_CLUSTER_NAME"] = db_cluster_name
|
||||
legacy["init_password.db_cluster_name"] = db_cluster_name
|
||||
if "DB_CLUSTER_MODE" in sec:
|
||||
legacy["env_setup.DB_CLUSTER_MODE"] = _expand_cfg_value(
|
||||
sec.get("DB_CLUSTER_MODE", ""), cfg_vars
|
||||
)
|
||||
legacy["init_cluster.db_cluster_mode"] = legacy["env_setup.DB_CLUSTER_MODE"]
|
||||
if "DB_CLUSTER_NODE_COUNT" in sec:
|
||||
legacy["init_cluster.db_cluster_node_count"] = _expand_cfg_value(
|
||||
sec.get("DB_CLUSTER_NODE_COUNT", ""), cfg_vars
|
||||
)
|
||||
if "DB_CLUSTER_MACHINE_TYPE" in sec:
|
||||
legacy["init_cluster.db_cluster_machine_type"] = _expand_cfg_value(
|
||||
sec.get("DB_CLUSTER_MACHINE_TYPE", ""), cfg_vars
|
||||
)
|
||||
if "DB_BOOT_DISK_TYPE" in sec:
|
||||
legacy["init_cluster.db_boot_disk_type"] = _expand_cfg_value(
|
||||
sec.get("DB_BOOT_DISK_TYPE", ""), cfg_vars
|
||||
)
|
||||
if "DB_BOOT_DISK_SIZE_GB" in sec:
|
||||
legacy["init_cluster.db_boot_disk_size_gb"] = _expand_cfg_value(
|
||||
sec.get("DB_BOOT_DISK_SIZE_GB", ""), cfg_vars
|
||||
)
|
||||
if "DB_CLUSTER_REGION" in sec:
|
||||
legacy["init_cluster.db_cluster_region"] = _expand_cfg_value(
|
||||
sec.get("DB_CLUSTER_REGION", ""), cfg_vars
|
||||
)
|
||||
if "APP_CLUSTER_KUBECONTEXT" in sec:
|
||||
kubecontext = _expand_cfg_value(
|
||||
sec.get("APP_CLUSTER_KUBECONTEXT", ""), cfg_vars
|
||||
)
|
||||
legacy["env_setup.APP_CLUSTER_KUBECONTEXT"] = kubecontext
|
||||
legacy["init_cluster.app_cluster_kubecontext"] = kubecontext
|
||||
if "DB_CLUSTER_KUBECONTEXT" in sec:
|
||||
kubecontext = _expand_cfg_value(
|
||||
sec.get("DB_CLUSTER_KUBECONTEXT", ""), cfg_vars
|
||||
)
|
||||
legacy["env_setup.DB_CLUSTER_KUBECONTEXT"] = kubecontext
|
||||
legacy["init_cluster.db_cluster_kubecontext"] = kubecontext
|
||||
if "KNOE_DB_USER" in sec:
|
||||
legacy["init_password.db_username"] = _expand_cfg_value(
|
||||
sec.get("KNOE_DB_USER", ""), cfg_vars
|
||||
@ -3605,10 +3754,18 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
inputs[f"env_setup.{k}"] = env_vals.get(k, "")
|
||||
inputs["env_setup.DATABASE_NAMESPACE"] = namespace
|
||||
inputs["env_setup.CLUSTER_NAME"] = cluster_name
|
||||
inputs["env_setup.APP_CLUSTER_NAME"] = DEFAULT_APP_CLUSTER_NAME
|
||||
inputs["env_setup.APP_CLUSTER_MODE"] = DEFAULT_APP_CLUSTER_MODE
|
||||
inputs["env_setup.DB_CLUSTER_NAME"] = DEFAULT_DB_CLUSTER_NAME
|
||||
inputs["env_setup.DB_CLUSTER_MODE"] = DEFAULT_DB_CLUSTER_MODE
|
||||
inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = ""
|
||||
inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = ""
|
||||
|
||||
# Database creation
|
||||
inputs["init_password.db_namespace"] = namespace
|
||||
inputs["init_password.cluster_name"] = cluster_name
|
||||
inputs["init_password.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME
|
||||
inputs["init_password.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME
|
||||
inputs["init_password.db_username"] = owner
|
||||
inputs["init_password.db_password"] = ""
|
||||
inputs["init_password.db_password_confirm"] = ""
|
||||
@ -3628,7 +3785,19 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
inputs["init_cluster.k3s_token"] = ""
|
||||
inputs["init_cluster.supabase_enabled"] = _bool_str(False)
|
||||
inputs["init_cluster.supabase_studio_enabled"] = _bool_str(False)
|
||||
inputs["init_cluster.supabase_studio_url"] = "db.prole.org"
|
||||
inputs["init_cluster.supabase_studio_url"] = "db.knoe.org"
|
||||
inputs["init_cluster.app_cluster_name"] = DEFAULT_APP_CLUSTER_NAME
|
||||
inputs["init_cluster.app_cluster_mode"] = DEFAULT_APP_CLUSTER_MODE
|
||||
inputs["init_cluster.db_cluster_name"] = DEFAULT_DB_CLUSTER_NAME
|
||||
inputs["init_cluster.db_cluster_mode"] = DEFAULT_DB_CLUSTER_MODE
|
||||
inputs["init_cluster.db_cluster_node_count"] = str(DEFAULT_DB_CLUSTER_NODE_COUNT)
|
||||
inputs["init_cluster.db_cluster_machine_type"] = DEFAULT_DB_CLUSTER_MACHINE_TYPE
|
||||
inputs["init_cluster.db_boot_disk_type"] = DEFAULT_DB_BOOT_DISK_TYPE
|
||||
inputs["init_cluster.db_boot_disk_size_gb"] = str(DEFAULT_DB_BOOT_DISK_SIZE_GB)
|
||||
inputs["init_cluster.db_cluster_region"] = ""
|
||||
inputs["init_cluster.db_cluster_zones"] = ""
|
||||
inputs["init_cluster.app_cluster_kubecontext"] = ""
|
||||
inputs["init_cluster.db_cluster_kubecontext"] = ""
|
||||
inputs["init_cluster.supabase_auth_enabled"] = _bool_str(True)
|
||||
inputs["init_cluster.supabase_realtime_enabled"] = _bool_str(True)
|
||||
inputs["init_cluster.supabase_meta_enabled"] = _bool_str(True)
|
||||
@ -5064,6 +5233,10 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self.inputs["init_cluster.app_cluster_kubecontext"] = f"k3d-{cluster_name}"
|
||||
self.inputs["init_cluster.db_cluster_kubecontext"] = f"k3d-{cluster_name}"
|
||||
self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}"
|
||||
self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = f"k3d-{cluster_name}"
|
||||
else:
|
||||
# Ensure we don't pin KUBECONFIG to a stale local file when another
|
||||
# kubeconfig (e.g. Ansible-fetched client-cert auth) is available.
|
||||
@ -5099,6 +5272,100 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
os.environ["KUBECONFIG"] = str(fetched)
|
||||
self.log(f"Using fetched kubeconfig: {fetched}")
|
||||
|
||||
if mode == "k8s":
|
||||
gcp_cfg = self.gcp_cfg if isinstance(self.gcp_cfg, dict) else {}
|
||||
project_id = str(
|
||||
gcp_cfg.get("project_id")
|
||||
or gcp_cfg.get("PROJECT_ID")
|
||||
or self._get_input("init_cluster.project_id", "")
|
||||
or ""
|
||||
).strip()
|
||||
if not project_id:
|
||||
raise Exception("GCP project_id is required for dual-cluster GKE setup.")
|
||||
|
||||
app_cluster_name = self._app_cluster_name()
|
||||
app_cluster_mode = self._app_cluster_mode()
|
||||
db_cluster_name = self._cnpg_cluster_name()
|
||||
db_cluster_mode = self._db_cluster_mode()
|
||||
db_node_count = int(
|
||||
(self._get_input("init_cluster.db_cluster_node_count", "") or "3").strip() or "3"
|
||||
)
|
||||
db_machine_type = (
|
||||
self._get_input("init_cluster.db_cluster_machine_type", "")
|
||||
or DEFAULT_DB_CLUSTER_MACHINE_TYPE
|
||||
).strip() or DEFAULT_DB_CLUSTER_MACHINE_TYPE
|
||||
db_boot_disk_type = (
|
||||
self._get_input("init_cluster.db_boot_disk_type", "")
|
||||
or DEFAULT_DB_BOOT_DISK_TYPE
|
||||
).strip() or DEFAULT_DB_BOOT_DISK_TYPE
|
||||
db_boot_disk_size_gb = int(
|
||||
(
|
||||
self._get_input("init_cluster.db_boot_disk_size_gb", "")
|
||||
or str(DEFAULT_DB_BOOT_DISK_SIZE_GB)
|
||||
).strip()
|
||||
or str(DEFAULT_DB_BOOT_DISK_SIZE_GB)
|
||||
)
|
||||
db_location = (
|
||||
self._get_input("init_cluster.db_cluster_region", "")
|
||||
or gcp_cfg.get("region")
|
||||
or gcp_cfg.get("REGION")
|
||||
or gcp_cfg.get("zone")
|
||||
or gcp_cfg.get("ZONE")
|
||||
or "us-central1"
|
||||
)
|
||||
app_location = (
|
||||
self._get_input("init_cluster.app_cluster_region", "")
|
||||
or gcp_cfg.get("region")
|
||||
or gcp_cfg.get("REGION")
|
||||
or db_location
|
||||
)
|
||||
|
||||
app_spec = GkeClusterSpec(
|
||||
name=app_cluster_name,
|
||||
mode=app_cluster_mode,
|
||||
location=str(app_location).strip(),
|
||||
)
|
||||
db_spec = GkeClusterSpec(
|
||||
name=db_cluster_name,
|
||||
mode=db_cluster_mode,
|
||||
location=str(db_location).strip(),
|
||||
machine_type=db_machine_type,
|
||||
node_count=max(3, db_node_count),
|
||||
node_pool_name="cnpg-db-pool",
|
||||
boot_disk_type=db_boot_disk_type,
|
||||
boot_disk_size_gb=db_boot_disk_size_gb,
|
||||
)
|
||||
|
||||
self.log(f"[app-cluster {app_cluster_name}] validating cluster")
|
||||
ensure_app_cluster(project_id=project_id, spec=app_spec, log=self.log)
|
||||
self.log(f"[db-cluster {db_cluster_name}] ensuring Standard GKE cluster")
|
||||
ensure_db_cluster(project_id=project_id, spec=db_spec, log=self.log)
|
||||
|
||||
app_ctx = get_cluster_credentials(
|
||||
project_id=project_id,
|
||||
cluster_name=app_cluster_name,
|
||||
location=app_spec.location,
|
||||
log=self.log,
|
||||
)
|
||||
db_ctx = get_cluster_credentials(
|
||||
project_id=project_id,
|
||||
cluster_name=db_cluster_name,
|
||||
location=db_spec.location,
|
||||
log=self.log,
|
||||
)
|
||||
self.inputs["init_cluster.app_cluster_kubecontext"] = app_ctx
|
||||
self.inputs["init_cluster.db_cluster_kubecontext"] = db_ctx
|
||||
self.inputs["env_setup.APP_CLUSTER_KUBECONTEXT"] = app_ctx
|
||||
self.inputs["env_setup.DB_CLUSTER_KUBECONTEXT"] = db_ctx
|
||||
self.prole_cfg_data.setdefault("Global", {})["APP_CLUSTER_NAME"] = app_cluster_name
|
||||
self.prole_cfg_data.setdefault("Global", {})["APP_CLUSTER_MODE"] = app_cluster_mode
|
||||
self.prole_cfg_data.setdefault("Global", {})["APP_CLUSTER_KUBECONTEXT"] = app_ctx
|
||||
self.prole_cfg_data.setdefault("Global", {})["DB_CLUSTER_NAME"] = db_cluster_name
|
||||
self.prole_cfg_data.setdefault("Global", {})["DB_CLUSTER_MODE"] = db_cluster_mode
|
||||
self.prole_cfg_data.setdefault("Global", {})["DB_CLUSTER_KUBECONTEXT"] = db_ctx
|
||||
# Back-compat default context points to app cluster for platform services.
|
||||
self.prole_cfg_data.setdefault("Global", {})["KUBECONTEXT"] = app_ctx
|
||||
|
||||
kubectl = subprocess.run(["which", "kubectl"], capture_output=True)
|
||||
if kubectl.returncode != 0:
|
||||
raise Exception(
|
||||
@ -5582,52 +5849,51 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
ns = (
|
||||
self._get_input("init_password.db_namespace", "") or ""
|
||||
).strip() or "default"
|
||||
env = self._script_env_for_namespace(ns)
|
||||
db_env = self._script_env_for_namespace(ns, cluster_role="db")
|
||||
app_env = self._script_env_for_namespace(ns, cluster_role="app")
|
||||
|
||||
opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(env)
|
||||
opt_allowed, _opt_count, opt_reason = self._optional_workloads_policy(app_env)
|
||||
|
||||
password = self._get_input("init_password.db_password", "")
|
||||
kerberos_enabled = self._get_input_bool("kerberos_config.enabled", False)
|
||||
steps = [
|
||||
pre_cnpg_steps = [
|
||||
("init_certmgr.sh", ["initialize"], False),
|
||||
]
|
||||
if kerberos_enabled:
|
||||
steps.append(("init_kerberos.sh", ["initialize"], False))
|
||||
steps.extend(
|
||||
[
|
||||
("init_cnpg_backup.sh", ["start"], False),
|
||||
("init_kong.sh", ["start"], False),
|
||||
]
|
||||
)
|
||||
pre_cnpg_steps.append(("init_kerberos.sh", ["initialize"], False))
|
||||
post_cnpg_steps: list[tuple[str, list[str], bool, str]] = [
|
||||
("init_cnpg_backup.sh", ["start"], False, "db"),
|
||||
("init_kong.sh", ["start"], False, "app"),
|
||||
]
|
||||
if not opt_allowed:
|
||||
self.log(f"[SKIP] Monitoring disabled by policy: {opt_reason}")
|
||||
self.prole_cfg_data.setdefault("Monitoring", {})["STATUS"] = "Skipped"
|
||||
mode = self._deployment_mode()
|
||||
if mode != "k3d":
|
||||
steps.append(("init_nginx_ingress.sh", ["initialize"], False))
|
||||
post_cnpg_steps.append(("init_nginx_ingress.sh", ["initialize"], False, "app"))
|
||||
|
||||
overall_success = True
|
||||
|
||||
# Common services (Python owners) — replaces init_common_services.sh
|
||||
self.log("--> common services (python owners)")
|
||||
self.log(f"[app-cluster {self._app_cluster_name()}] common services (python owners)")
|
||||
try:
|
||||
registry_ns = str(env.get("REGISTRY_NAMESPACE") or ns).strip() or ns
|
||||
registry_ops.update(namespace=registry_ns, env=env, mode=mode, log=self.log)
|
||||
openbao_ops.update(namespace=ns, env=env, mode=mode, log=self.log)
|
||||
garage_store_ops.update(namespace=ns, env=env, mode=mode, log=self.log)
|
||||
opentofu_ops.update(namespace=ns, env=env, mode=mode, log=self.log)
|
||||
registry_ns = str(app_env.get("REGISTRY_NAMESPACE") or ns).strip() or ns
|
||||
registry_ops.update(namespace=registry_ns, env=app_env, mode=mode, log=self.log)
|
||||
openbao_ops.update(namespace=ns, env=app_env, mode=mode, log=self.log)
|
||||
garage_store_ops.update(namespace=ns, env=app_env, mode=mode, log=self.log)
|
||||
opentofu_ops.update(namespace=ns, env=app_env, mode=mode, log=self.log)
|
||||
except Exception as e:
|
||||
self.err(f"[ERROR] common services (python owners) failed: {e}")
|
||||
overall_success = False
|
||||
|
||||
# Run pre-CNPG shell scripts (cert-manager, optional kerberos)
|
||||
for script, args, needs_password in steps[:1] if not kerberos_enabled else steps[:2]:
|
||||
for script, args, needs_password in pre_cnpg_steps:
|
||||
self.log(f"--> {script} {' '.join(args)}")
|
||||
stdin_text = f"{password}\n" if needs_password else None
|
||||
rc = self._run_script(
|
||||
script,
|
||||
args=args,
|
||||
env=env,
|
||||
env=db_env,
|
||||
stdin_text=stdin_text,
|
||||
on_line=self._process_script_output_line,
|
||||
)
|
||||
@ -5637,22 +5903,28 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
|
||||
# CNPG initialization (Python) — replaces init_cloudnative_pg.sh initialize
|
||||
if overall_success:
|
||||
self.log("--> cnpg_initialize (Python)")
|
||||
self.log(f"[db-cluster {self._cnpg_cluster_name()}] cnpg_initialize (Python)")
|
||||
try:
|
||||
self.ensure_db_k8s_secrets(ns, password, log_fn=self.log)
|
||||
except Exception as e:
|
||||
self.err(f"[ERROR] Failed to ensure DB secrets before CNPG init: {e}")
|
||||
overall_success = False
|
||||
if overall_success:
|
||||
try:
|
||||
self._ensure_cnpg_storage_provisioned(ns, env)
|
||||
except Exception as e:
|
||||
self.err(f"[WARN] Failed to provision CNPG storage before init: {e}")
|
||||
if self._cluster_storage_milestone_enabled():
|
||||
try:
|
||||
self._ensure_cnpg_storage_provisioned(ns, db_env)
|
||||
except Exception as e:
|
||||
self.err(f"[WARN] Failed to provision CNPG storage before init: {e}")
|
||||
else:
|
||||
self.log(
|
||||
"[SKIP] Cluster storage milestone shelved; "
|
||||
"retaining utility workflows only."
|
||||
)
|
||||
try:
|
||||
cnpg_initialize(
|
||||
namespace=ns,
|
||||
cluster_name=str(env.get("CLUSTER_NAME") or env.get("CNPG_CLUSTER_NAME") or "knoe-db").strip(),
|
||||
env=env,
|
||||
cluster_name=str(db_env.get("CLUSTER_NAME") or db_env.get("CNPG_CLUSTER_NAME") or "knoe-db").strip(),
|
||||
env=db_env,
|
||||
project_root=self.project_root,
|
||||
log=self.log,
|
||||
mode=mode,
|
||||
@ -5662,15 +5934,15 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
overall_success = False
|
||||
|
||||
# Run post-CNPG shell scripts (backup, kong, ingress)
|
||||
post_cnpg_start = 1 if not kerberos_enabled else 2
|
||||
if overall_success:
|
||||
for script, args, needs_password in steps[post_cnpg_start:]:
|
||||
for script, args, needs_password, role in post_cnpg_steps:
|
||||
self.log(f"--> {script} {' '.join(args)}")
|
||||
stdin_text = f"{password}\n" if needs_password else None
|
||||
target_env = db_env if role == "db" else app_env
|
||||
rc = self._run_script(
|
||||
script,
|
||||
args=args,
|
||||
env=env,
|
||||
env=target_env,
|
||||
stdin_text=stdin_text,
|
||||
on_line=self._process_script_output_line,
|
||||
)
|
||||
@ -5679,12 +5951,12 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
overall_success = False
|
||||
|
||||
if overall_success and opt_allowed:
|
||||
self.log("--> monitoring (python owner) initialize")
|
||||
self.log(f"[app-cluster {self._app_cluster_name()}] monitoring (python owner) initialize")
|
||||
try:
|
||||
monitoring_ns = str(env.get("MONITORING_NAMESPACE") or "monitoring").strip() or "monitoring"
|
||||
monitoring_ns = str(app_env.get("MONITORING_NAMESPACE") or "monitoring").strip() or "monitoring"
|
||||
monitoring_ops.initialize(
|
||||
namespace=monitoring_ns,
|
||||
env=env,
|
||||
env=app_env,
|
||||
mode=mode,
|
||||
log=self.log,
|
||||
)
|
||||
@ -5722,13 +5994,14 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
self.log("[SKIP] CnPG deploy disabled.")
|
||||
self.prole_cfg_data["Deployment"]["STATUS"] = "Skipped"
|
||||
return
|
||||
self.log("==> Deploy CloudNative-PG")
|
||||
db_cluster = self._cnpg_cluster_name()
|
||||
self.log(f"==> [db-cluster {db_cluster}] Deploy CloudNativePG")
|
||||
ns = (self._get_input("init_password.db_namespace", "") or "").strip()
|
||||
if not ns:
|
||||
ns = (self._get_input("env_setup.DATABASE_NAMESPACE", "") or "").strip()
|
||||
if not ns:
|
||||
ns = "default"
|
||||
env = self._script_env_for_namespace(ns)
|
||||
env = self._script_env_for_namespace(ns, cluster_role="db")
|
||||
cluster_name = str(
|
||||
env.get("CLUSTER_NAME")
|
||||
or env.get("CNPG_CLUSTER_NAME")
|
||||
@ -5738,15 +6011,22 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
cluster_name = "knoe-db"
|
||||
mode = self._deployment_mode()
|
||||
|
||||
try:
|
||||
self._ensure_cnpg_storage_provisioned(ns, env)
|
||||
except Exception as exc:
|
||||
self._cnpg_success = False
|
||||
self.prole_cfg_data["Deployment"]["STATUS"] = "Attempted"
|
||||
self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}")
|
||||
return
|
||||
if self._cluster_storage_milestone_enabled():
|
||||
try:
|
||||
self._ensure_cnpg_storage_provisioned(ns, env)
|
||||
except Exception as exc:
|
||||
self._cnpg_success = False
|
||||
self.prole_cfg_data["Deployment"]["STATUS"] = "Attempted"
|
||||
self.err(f"[ERROR] CNPG pre-provisioning failed: {exc}")
|
||||
return
|
||||
else:
|
||||
self.log(
|
||||
"[SKIP] Cluster storage milestone shelved; "
|
||||
"retaining utility workflows only."
|
||||
)
|
||||
|
||||
try:
|
||||
self.log(f"[db-cluster {db_cluster}] applying knoe-db cluster")
|
||||
cnpg_deploy(
|
||||
namespace=ns,
|
||||
cluster_name=cluster_name,
|
||||
@ -5777,7 +6057,7 @@ class KnoeConsoleInstaller(KnoeInstaller):
|
||||
)
|
||||
has_pods = bool((rc_check.stdout or "").strip())
|
||||
if has_pods:
|
||||
self.log("==> Force rollout")
|
||||
self.log(f"==> [db-cluster {db_cluster}] Force rollout")
|
||||
try:
|
||||
cnpg_rollout(
|
||||
namespace=ns,
|
||||
|
||||
416
knoe/core/cluster_storage_browser.py
Normal file
416
knoe/core/cluster_storage_browser.py
Normal file
@ -0,0 +1,416 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
PD_SSD_QUOTA_METRIC = "SSD_TOTAL_GB"
|
||||
PD_BALANCED_QUOTA_METRIC = "DISKS_TOTAL_GB"
|
||||
|
||||
STORAGE_CLASS_PREMIUM = "premium-rwo"
|
||||
STORAGE_CLASS_STANDARD = "standard-rwo"
|
||||
|
||||
_REGION_TOKEN_RE = re.compile(r"^[a-z]+-[a-z]+\d+$")
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class RegionCatalogEntry:
|
||||
region: str
|
||||
timezone_group: str
|
||||
country: str
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class QuotaMetricHeadroom:
|
||||
metric: str
|
||||
limit_gb: float | None
|
||||
usage_gb: float | None
|
||||
|
||||
@property
|
||||
def headroom_gb(self) -> float | None:
|
||||
if self.limit_gb is None or self.usage_gb is None:
|
||||
return None
|
||||
return self.limit_gb - self.usage_gb
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class RegionQuotaSnapshot:
|
||||
region: str
|
||||
premium: QuotaMetricHeadroom
|
||||
standard: QuotaMetricHeadroom
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class RegionLatencyResult:
|
||||
region: str
|
||||
latency_ms: float | None
|
||||
ok: bool
|
||||
error: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class ClusterStorageRequest:
|
||||
pgdata_gb: float
|
||||
wal_gb: float
|
||||
|
||||
@property
|
||||
def total_gb(self) -> float:
|
||||
return max(0.0, self.pgdata_gb) + max(0.0, self.wal_gb)
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class RegionFeasibility:
|
||||
region: str
|
||||
timezone_group: str
|
||||
premium_headroom_gb: float | None
|
||||
standard_headroom_gb: float | None
|
||||
can_premium: bool
|
||||
can_standard: bool
|
||||
default_pgdata_class: str
|
||||
default_wal_class: str
|
||||
feasibility_rank: int
|
||||
notes: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class RegionCandidate:
|
||||
feasibility: RegionFeasibility
|
||||
latency_ms: float | None
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class ClusterStorageBrowserResult:
|
||||
generated_at: str
|
||||
project_id: str
|
||||
timezone_group: str
|
||||
request: ClusterStorageRequest
|
||||
candidates: tuple[RegionCandidate, ...]
|
||||
quota_link: str
|
||||
quota_console_link: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"generatedAt": self.generated_at,
|
||||
"projectId": self.project_id,
|
||||
"timezoneGroup": self.timezone_group,
|
||||
"request": asdict(self.request),
|
||||
"candidates": [
|
||||
{
|
||||
"region": c.feasibility.region,
|
||||
"timezoneGroup": c.feasibility.timezone_group,
|
||||
"feasibilityRank": c.feasibility.feasibility_rank,
|
||||
"premiumHeadroomGb": c.feasibility.premium_headroom_gb,
|
||||
"standardHeadroomGb": c.feasibility.standard_headroom_gb,
|
||||
"canPremium": c.feasibility.can_premium,
|
||||
"canStandard": c.feasibility.can_standard,
|
||||
"defaultPgdataClass": c.feasibility.default_pgdata_class,
|
||||
"defaultWalClass": c.feasibility.default_wal_class,
|
||||
"latencyMs": c.latency_ms,
|
||||
"notes": list(c.feasibility.notes),
|
||||
}
|
||||
for c in self.candidates
|
||||
],
|
||||
"quotaLink": self.quota_link,
|
||||
"quotaConsoleLink": self.quota_console_link,
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":"), indent=2)
|
||||
|
||||
|
||||
class RegionQuotaProvider(Protocol):
|
||||
def fetch(self, *, project_id: str, regions: list[str]) -> dict[str, RegionQuotaSnapshot]: ...
|
||||
|
||||
|
||||
class RegionLatencyProvider(Protocol):
|
||||
def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]: ...
|
||||
|
||||
|
||||
def default_region_catalog() -> tuple[RegionCatalogEntry, ...]:
|
||||
return (
|
||||
RegionCatalogEntry("us-central1", "americas", "us"),
|
||||
RegionCatalogEntry("us-east1", "americas", "us"),
|
||||
RegionCatalogEntry("us-east4", "americas", "us"),
|
||||
RegionCatalogEntry("us-west1", "americas", "us"),
|
||||
RegionCatalogEntry("us-west2", "americas", "us"),
|
||||
RegionCatalogEntry("us-west3", "americas", "us"),
|
||||
RegionCatalogEntry("us-west4", "americas", "us"),
|
||||
RegionCatalogEntry("northamerica-northeast1", "americas", "ca"),
|
||||
RegionCatalogEntry("southamerica-east1", "americas", "br"),
|
||||
RegionCatalogEntry("europe-west1", "emea", "be"),
|
||||
RegionCatalogEntry("europe-west2", "emea", "uk"),
|
||||
RegionCatalogEntry("europe-west3", "emea", "de"),
|
||||
RegionCatalogEntry("europe-west4", "emea", "nl"),
|
||||
RegionCatalogEntry("europe-west6", "emea", "ch"),
|
||||
RegionCatalogEntry("europe-west8", "emea", "it"),
|
||||
RegionCatalogEntry("europe-west9", "emea", "fr"),
|
||||
RegionCatalogEntry("europe-west10", "emea", "de"),
|
||||
RegionCatalogEntry("europe-central2", "emea", "pl"),
|
||||
RegionCatalogEntry("asia-east1", "apac", "tw"),
|
||||
RegionCatalogEntry("asia-east2", "apac", "hk"),
|
||||
RegionCatalogEntry("asia-northeast1", "apac", "jp"),
|
||||
RegionCatalogEntry("asia-northeast2", "apac", "jp"),
|
||||
RegionCatalogEntry("asia-northeast3", "apac", "kr"),
|
||||
RegionCatalogEntry("asia-south1", "apac", "in"),
|
||||
RegionCatalogEntry("asia-south2", "apac", "in"),
|
||||
RegionCatalogEntry("asia-southeast1", "apac", "sg"),
|
||||
RegionCatalogEntry("asia-southeast2", "apac", "id"),
|
||||
RegionCatalogEntry("australia-southeast1", "apac", "au"),
|
||||
RegionCatalogEntry("australia-southeast2", "apac", "au"),
|
||||
)
|
||||
|
||||
|
||||
def detect_timezone_group() -> str:
|
||||
now_local = datetime.now().astimezone()
|
||||
offset_hours = float(now_local.utcoffset().total_seconds()) / 3600.0 if now_local.utcoffset() else 0.0
|
||||
if -10.0 <= offset_hours <= -2.0:
|
||||
return "americas"
|
||||
if -1.0 <= offset_hours <= 4.0:
|
||||
return "emea"
|
||||
return "apac"
|
||||
|
||||
|
||||
def filter_regions_by_timezone(
|
||||
regions: list[str],
|
||||
timezone_group: str,
|
||||
catalog: tuple[RegionCatalogEntry, ...] | None = None,
|
||||
) -> list[RegionCatalogEntry]:
|
||||
entries = catalog or default_region_catalog()
|
||||
allowed = {r for r in regions if _REGION_TOKEN_RE.match(r)}
|
||||
return [entry for entry in entries if entry.region in allowed and entry.timezone_group == timezone_group]
|
||||
|
||||
|
||||
def _bool_feasible(headroom_gb: float | None, requested_gb: float) -> bool:
|
||||
if headroom_gb is None:
|
||||
return False
|
||||
return headroom_gb >= max(0.0, requested_gb)
|
||||
|
||||
|
||||
def _choose_default_class(premium_ok: bool, standard_ok: bool) -> str:
|
||||
if premium_ok:
|
||||
return STORAGE_CLASS_PREMIUM
|
||||
if standard_ok:
|
||||
return STORAGE_CLASS_STANDARD
|
||||
return STORAGE_CLASS_PREMIUM
|
||||
|
||||
|
||||
def _feasibility_rank(can_premium: bool, can_standard: bool) -> int:
|
||||
if can_premium:
|
||||
return 0
|
||||
if can_standard:
|
||||
return 1
|
||||
return 2
|
||||
|
||||
|
||||
def evaluate_region_feasibility(
|
||||
region: str,
|
||||
timezone_group: str,
|
||||
quota: RegionQuotaSnapshot | None,
|
||||
requested: ClusterStorageRequest,
|
||||
) -> RegionFeasibility:
|
||||
premium_headroom = quota.premium.headroom_gb if quota is not None else None
|
||||
standard_headroom = quota.standard.headroom_gb if quota is not None else None
|
||||
|
||||
can_premium = _bool_feasible(premium_headroom, requested.total_gb)
|
||||
can_standard = _bool_feasible(standard_headroom, requested.total_gb)
|
||||
|
||||
notes: list[str] = []
|
||||
if quota is None:
|
||||
notes.append("quota-unavailable")
|
||||
if not can_premium and not can_standard:
|
||||
notes.append("insufficient-quota")
|
||||
|
||||
default_class = _choose_default_class(can_premium, can_standard)
|
||||
return RegionFeasibility(
|
||||
region=region,
|
||||
timezone_group=timezone_group,
|
||||
premium_headroom_gb=premium_headroom,
|
||||
standard_headroom_gb=standard_headroom,
|
||||
can_premium=can_premium,
|
||||
can_standard=can_standard,
|
||||
default_pgdata_class=default_class,
|
||||
default_wal_class=default_class,
|
||||
feasibility_rank=_feasibility_rank(can_premium, can_standard),
|
||||
notes=tuple(notes),
|
||||
)
|
||||
|
||||
|
||||
def _quota_links(project_id: str) -> tuple[str, str]:
|
||||
project = project_id.strip()
|
||||
if not project:
|
||||
return (
|
||||
"https://cloud.google.com/compute/quotas",
|
||||
"https://console.cloud.google.com/iam-admin/quotas",
|
||||
)
|
||||
return (
|
||||
"https://cloud.google.com/compute/quotas",
|
||||
f"https://console.cloud.google.com/iam-admin/quotas?project={project}",
|
||||
)
|
||||
|
||||
|
||||
def _sort_candidates(candidates: list[RegionCandidate]) -> list[RegionCandidate]:
|
||||
def _key(candidate: RegionCandidate):
|
||||
latency = candidate.latency_ms if candidate.latency_ms is not None else float("inf")
|
||||
return (candidate.feasibility.feasibility_rank, latency, candidate.feasibility.region)
|
||||
|
||||
return sorted(candidates, key=_key)
|
||||
|
||||
|
||||
def build_cluster_storage_browser_result(
|
||||
*,
|
||||
project_id: str,
|
||||
requested: ClusterStorageRequest,
|
||||
available_regions: list[str],
|
||||
timezone_group: str | None = None,
|
||||
catalog: tuple[RegionCatalogEntry, ...] | None = None,
|
||||
quota_provider: RegionQuotaProvider,
|
||||
latency_provider: RegionLatencyProvider,
|
||||
) -> ClusterStorageBrowserResult:
|
||||
tz_group = (timezone_group or detect_timezone_group()).strip().lower() or detect_timezone_group()
|
||||
candidates = filter_regions_by_timezone(available_regions, tz_group, catalog)
|
||||
candidate_regions = [entry.region for entry in candidates]
|
||||
|
||||
quota_by_region = quota_provider.fetch(project_id=project_id, regions=candidate_regions)
|
||||
feasibility: list[RegionFeasibility] = [
|
||||
evaluate_region_feasibility(
|
||||
region=entry.region,
|
||||
timezone_group=entry.timezone_group,
|
||||
quota=quota_by_region.get(entry.region),
|
||||
requested=requested,
|
||||
)
|
||||
for entry in candidates
|
||||
]
|
||||
|
||||
probe_regions = [f.region for f in feasibility if f.can_premium or f.can_standard]
|
||||
latency_by_region = latency_provider.probe(probe_regions)
|
||||
|
||||
combined: list[RegionCandidate] = []
|
||||
for f in feasibility:
|
||||
latency = latency_by_region.get(f.region)
|
||||
combined.append(
|
||||
RegionCandidate(
|
||||
feasibility=f,
|
||||
latency_ms=latency.latency_ms if latency and latency.ok else None,
|
||||
)
|
||||
)
|
||||
|
||||
link, console_link = _quota_links(project_id)
|
||||
return ClusterStorageBrowserResult(
|
||||
generated_at=datetime.now(timezone.utc).isoformat(),
|
||||
project_id=project_id,
|
||||
timezone_group=tz_group,
|
||||
request=requested,
|
||||
candidates=tuple(_sort_candidates(combined)),
|
||||
quota_link=link,
|
||||
quota_console_link=console_link,
|
||||
)
|
||||
|
||||
|
||||
def _safe_float(value: str | int | float | None) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_limit_usage(raw: str) -> tuple[float | None, float | None]:
|
||||
# gcloud output format from --format=value(limit,usage)
|
||||
parts = [p.strip() for p in raw.split() if p.strip()]
|
||||
if len(parts) >= 2:
|
||||
return _safe_float(parts[0]), _safe_float(parts[1])
|
||||
if len(parts) == 1:
|
||||
return _safe_float(parts[0]), None
|
||||
return None, None
|
||||
|
||||
|
||||
class GcloudRegionQuotaProvider:
|
||||
def __init__(self, timeout_sec: int = 20):
|
||||
self.timeout_sec = timeout_sec
|
||||
|
||||
def _metric_headroom(self, project_id: str, region: str, metric: str) -> QuotaMetricHeadroom:
|
||||
cmd = [
|
||||
"gcloud",
|
||||
"compute",
|
||||
"regions",
|
||||
"describe",
|
||||
region,
|
||||
"--project",
|
||||
project_id,
|
||||
"--format=value(quotas[metric=%s].limit,quotas[metric=%s].usage)" % (metric, metric),
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout_sec,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return QuotaMetricHeadroom(metric=metric, limit_gb=None, usage_gb=None)
|
||||
limit, usage = _parse_limit_usage(result.stdout.strip())
|
||||
return QuotaMetricHeadroom(metric=metric, limit_gb=limit, usage_gb=usage)
|
||||
except Exception:
|
||||
return QuotaMetricHeadroom(metric=metric, limit_gb=None, usage_gb=None)
|
||||
|
||||
def fetch(self, *, project_id: str, regions: list[str]) -> dict[str, RegionQuotaSnapshot]:
|
||||
project = project_id.strip()
|
||||
if not project:
|
||||
return {}
|
||||
out: dict[str, RegionQuotaSnapshot] = {}
|
||||
for region in regions:
|
||||
premium = self._metric_headroom(project, region, PD_SSD_QUOTA_METRIC)
|
||||
standard = self._metric_headroom(project, region, PD_BALANCED_QUOTA_METRIC)
|
||||
out[region] = RegionQuotaSnapshot(region=region, premium=premium, standard=standard)
|
||||
return out
|
||||
|
||||
|
||||
class PingLatencyProvider:
|
||||
def __init__(self, timeout_sec: int = 2, count: int = 2):
|
||||
self.timeout_sec = timeout_sec
|
||||
self.count = count
|
||||
|
||||
def _probe_one(self, region: str) -> RegionLatencyResult:
|
||||
# Best-effort: use region DNS style endpoint as latency anchor.
|
||||
host = f"{region}.gcping.com"
|
||||
cmd = ["ping", "-c", str(self.count), "-t", str(self.timeout_sec), host]
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=max(2, self.timeout_sec + 2),
|
||||
)
|
||||
elapsed = (time.monotonic() - started) * 1000.0
|
||||
if result.returncode == 0:
|
||||
return RegionLatencyResult(region=region, latency_ms=round(elapsed, 2), ok=True)
|
||||
return RegionLatencyResult(region=region, latency_ms=None, ok=False, error="ping-failed")
|
||||
except Exception as exc:
|
||||
return RegionLatencyResult(region=region, latency_ms=None, ok=False, error=str(exc))
|
||||
|
||||
def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]:
|
||||
return {region: self._probe_one(region) for region in regions}
|
||||
|
||||
|
||||
class StaticQuotaProvider:
|
||||
def __init__(self, values: dict[str, RegionQuotaSnapshot]):
|
||||
self.values = dict(values)
|
||||
|
||||
def fetch(self, *, project_id: str, regions: list[str]) -> dict[str, RegionQuotaSnapshot]:
|
||||
_ = project_id
|
||||
return {r: self.values[r] for r in regions if r in self.values}
|
||||
|
||||
|
||||
class StaticLatencyProvider:
|
||||
def __init__(self, values: dict[str, RegionLatencyResult]):
|
||||
self.values = dict(values)
|
||||
|
||||
def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]:
|
||||
return {r: self.values[r] for r in regions if r in self.values}
|
||||
@ -300,6 +300,17 @@ class DatabaseCreationMilestone(Milestone):
|
||||
if not cluster_name:
|
||||
cluster_name = "knoe-db"
|
||||
|
||||
app_cluster_name = (
|
||||
state.inputs.get("init_password.app_cluster_name", "")
|
||||
or state.inputs.get("env_setup.APP_CLUSTER_NAME", "")
|
||||
or "knoe-dev-0"
|
||||
).strip() or "knoe-dev-0"
|
||||
db_cluster_name = (
|
||||
state.inputs.get("init_password.db_cluster_name", "")
|
||||
or state.inputs.get("env_setup.DB_CLUSTER_NAME", "")
|
||||
or "knoe-cnpg-0"
|
||||
).strip() or "knoe-cnpg-0"
|
||||
|
||||
user = (state.inputs.get("init_password.db_username", "") or "").strip()
|
||||
p1 = state.inputs.get("init_password.db_password", "")
|
||||
|
||||
@ -356,12 +367,24 @@ class DatabaseCreationMilestone(Milestone):
|
||||
state.inputs["env_setup.DATABASE_NAMESPACE"] = ns
|
||||
state.inputs["init_password.cluster_name"] = cluster_name
|
||||
state.inputs["env_setup.CLUSTER_NAME"] = cluster_name
|
||||
state.inputs["init_password.app_cluster_name"] = app_cluster_name
|
||||
state.inputs["env_setup.APP_CLUSTER_NAME"] = app_cluster_name
|
||||
state.inputs["init_password.db_cluster_name"] = db_cluster_name
|
||||
state.inputs["env_setup.DB_CLUSTER_NAME"] = db_cluster_name
|
||||
|
||||
self.logger.info(f"Application Cluster: {app_cluster_name} (Autopilot)")
|
||||
self.logger.info(f"Database Cluster: {db_cluster_name} (Standard)")
|
||||
self.logger.info(
|
||||
f"CloudNativePG targets dedicated DB cluster '{db_cluster_name}' with CNPG cluster name '{cluster_name}'."
|
||||
)
|
||||
|
||||
if "Database Creation" not in state.config_data:
|
||||
state.config_data["Database Creation"] = {}
|
||||
state.config_data["Database Creation"]["DB_USER"] = user
|
||||
state.config_data["Database Creation"]["DATABASE_NAMESPACE"] = ns
|
||||
state.config_data["Database Creation"]["CLUSTER_NAME"] = cluster_name
|
||||
state.config_data["Database Creation"]["APP_CLUSTER_NAME"] = app_cluster_name
|
||||
state.config_data["Database Creation"]["DB_CLUSTER_NAME"] = db_cluster_name
|
||||
state.config_data["Database Creation"].pop("DB_NAME", None)
|
||||
state.config_data["Database Creation"].pop("NAMESPACE", None)
|
||||
|
||||
|
||||
181
knoe/core/ops/gke_clusters.py
Normal file
181
knoe/core/ops/gke_clusters.py
Normal file
@ -0,0 +1,181 @@
|
||||
"""Helpers for explicit app/db GKE cluster targeting in installer flows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
|
||||
_LogFn = Callable[[str], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GkeClusterSpec:
|
||||
name: str
|
||||
mode: str
|
||||
location: str
|
||||
machine_type: str = "e2-small"
|
||||
node_count: int = 3
|
||||
node_pool_name: str = "default-pool"
|
||||
boot_disk_type: str = "pd-standard" # pd-standard keeps boot disks off pd-ssd quota
|
||||
boot_disk_size_gb: int = 50
|
||||
|
||||
|
||||
def _run(cmd: list[str], log: _LogFn | None = None) -> subprocess.CompletedProcess:
|
||||
if log:
|
||||
log("$ " + " ".join(cmd))
|
||||
return subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
|
||||
def build_kubectl_env_for_cluster(
|
||||
base_env: dict | None,
|
||||
kubecontext: str,
|
||||
cluster_name: str,
|
||||
cluster_role: str,
|
||||
) -> dict:
|
||||
env = dict(base_env or os.environ)
|
||||
if kubecontext:
|
||||
env["KUBECTL_CONTEXT"] = kubecontext
|
||||
env["CLUSTER_NAME"] = cluster_name
|
||||
env["KNOE_CLUSTER_ROLE"] = cluster_role
|
||||
if cluster_role == "app":
|
||||
env["KNOE_APP_CLUSTER_NAME"] = cluster_name
|
||||
elif cluster_role == "db":
|
||||
env["KNOE_DB_CLUSTER_NAME"] = cluster_name
|
||||
return env
|
||||
|
||||
|
||||
def get_cluster_credentials(
|
||||
*,
|
||||
project_id: str,
|
||||
cluster_name: str,
|
||||
location: str,
|
||||
log: _LogFn | None = None,
|
||||
) -> str:
|
||||
cmd = [
|
||||
"gcloud",
|
||||
"container",
|
||||
"clusters",
|
||||
"get-credentials",
|
||||
cluster_name,
|
||||
"--project",
|
||||
project_id,
|
||||
"--region",
|
||||
location,
|
||||
"--quiet",
|
||||
]
|
||||
result = _run(cmd, log=log)
|
||||
if result.returncode != 0:
|
||||
stderr = (result.stderr or "").strip()
|
||||
raise RuntimeError(
|
||||
f"Failed to get credentials for cluster '{cluster_name}' ({location}): {stderr}"
|
||||
)
|
||||
return f"gke_{project_id}_{location}_{cluster_name}"
|
||||
|
||||
|
||||
def ensure_app_cluster(
|
||||
*,
|
||||
project_id: str,
|
||||
spec: GkeClusterSpec,
|
||||
log: _LogFn | None = None,
|
||||
) -> None:
|
||||
describe_cmd = [
|
||||
"gcloud",
|
||||
"container",
|
||||
"clusters",
|
||||
"describe",
|
||||
spec.name,
|
||||
"--project",
|
||||
project_id,
|
||||
"--region",
|
||||
spec.location,
|
||||
"--format=value(name)",
|
||||
"--quiet",
|
||||
]
|
||||
result = _run(describe_cmd, log=log)
|
||||
if result.returncode == 0 and (result.stdout or "").strip() == spec.name:
|
||||
return
|
||||
|
||||
# Cluster not found — create it as Autopilot.
|
||||
# NOTE: gcloud container clusters create-auto does NOT support --disk-type
|
||||
# or --disk-size. Autopilot manages all node infrastructure automatically.
|
||||
# Autopilot only provisions physical nodes when pods are scheduled, so SSD
|
||||
# quota is not consumed while the cluster is idle.
|
||||
create_cmd = [
|
||||
"gcloud",
|
||||
"container",
|
||||
"clusters",
|
||||
"create-auto",
|
||||
spec.name,
|
||||
"--project",
|
||||
project_id,
|
||||
"--region",
|
||||
spec.location,
|
||||
"--workload-policies=allow-net-admin",
|
||||
"--quiet",
|
||||
]
|
||||
created = _run(create_cmd, log=log)
|
||||
if created.returncode != 0:
|
||||
stderr = (created.stderr or "").strip()
|
||||
raise RuntimeError(
|
||||
f"Failed to create app cluster '{spec.name}' in '{spec.location}': {stderr}"
|
||||
)
|
||||
|
||||
|
||||
def ensure_db_cluster(
|
||||
*,
|
||||
project_id: str,
|
||||
spec: GkeClusterSpec,
|
||||
log: _LogFn | None = None,
|
||||
) -> None:
|
||||
describe_cmd = [
|
||||
"gcloud",
|
||||
"container",
|
||||
"clusters",
|
||||
"describe",
|
||||
spec.name,
|
||||
"--project",
|
||||
project_id,
|
||||
"--region",
|
||||
spec.location,
|
||||
"--format=value(name)",
|
||||
"--quiet",
|
||||
]
|
||||
describe = _run(describe_cmd, log=log)
|
||||
if describe.returncode == 0 and (describe.stdout or "").strip() == spec.name:
|
||||
return
|
||||
|
||||
if spec.mode.lower() != "standard":
|
||||
raise RuntimeError(
|
||||
f"Database cluster '{spec.name}' must run in Standard mode, got '{spec.mode}'."
|
||||
)
|
||||
|
||||
create_cmd = [
|
||||
"gcloud",
|
||||
"container",
|
||||
"clusters",
|
||||
"create",
|
||||
spec.name,
|
||||
"--project",
|
||||
project_id,
|
||||
"--region",
|
||||
spec.location,
|
||||
"--num-nodes",
|
||||
str(max(3, int(spec.node_count))),
|
||||
"--machine-type",
|
||||
spec.machine_type,
|
||||
"--disk-type",
|
||||
spec.boot_disk_type,
|
||||
"--disk-size",
|
||||
str(spec.boot_disk_size_gb),
|
||||
"--enable-ip-alias",
|
||||
"--workload-pool",
|
||||
f"{project_id}.svc.id.goog",
|
||||
"--quiet",
|
||||
]
|
||||
created = _run(create_cmd, log=log)
|
||||
if created.returncode != 0:
|
||||
stderr = (created.stderr or "").strip()
|
||||
raise RuntimeError(f"Failed to create DB cluster '{spec.name}': {stderr}")
|
||||
@ -38,6 +38,10 @@ class ProdDatabaseConfig:
|
||||
instances: int = 3
|
||||
storageClass: str = "premium-rwo"
|
||||
storageSizeGi: int = 100
|
||||
pgdataStorageClass: str = "premium-rwo"
|
||||
walStorageClass: str = "premium-rwo"
|
||||
pgdataSizeGi: int = 100
|
||||
walSizeGi: int = 25
|
||||
appDatabase: str = "knoey"
|
||||
metaDatabase: str = "knoe_meta"
|
||||
appUser: str = "knoey_app"
|
||||
@ -207,6 +211,25 @@ def build_config(payload: dict[str, Any] | None = None) -> KnoeProductionConfig:
|
||||
cfg.database.storageSizeGi = _coerce_int(
|
||||
db.get("storageSizeGi", cfg.database.storageSizeGi), cfg.database.storageSizeGi
|
||||
)
|
||||
cfg.database.pgdataStorageClass = (
|
||||
_str_or_empty(db.get("pgdataStorageClass"))
|
||||
or _str_or_empty(db.get("storageClass"))
|
||||
or cfg.database.pgdataStorageClass
|
||||
)
|
||||
cfg.database.walStorageClass = (
|
||||
_str_or_empty(db.get("walStorageClass"))
|
||||
or _str_or_empty(db.get("pgdataStorageClass"))
|
||||
or _str_or_empty(db.get("storageClass"))
|
||||
or cfg.database.walStorageClass
|
||||
)
|
||||
cfg.database.pgdataSizeGi = _coerce_int(
|
||||
db.get("pgdataSizeGi", db.get("storageSizeGi", cfg.database.pgdataSizeGi)),
|
||||
cfg.database.pgdataSizeGi,
|
||||
)
|
||||
cfg.database.walSizeGi = _coerce_int(
|
||||
db.get("walSizeGi", max(10, int(cfg.database.pgdataSizeGi * 0.25))),
|
||||
cfg.database.walSizeGi,
|
||||
)
|
||||
cfg.database.appDatabase = _str_or_empty(db.get("appDatabase")) or cfg.database.appDatabase
|
||||
cfg.database.metaDatabase = _str_or_empty(db.get("metaDatabase")) or cfg.database.metaDatabase
|
||||
cfg.database.appUser = _str_or_empty(db.get("appUser")) or cfg.database.appUser
|
||||
@ -275,6 +298,8 @@ def validate_prod_config(config: KnoeProductionConfig) -> tuple[list[str], list[
|
||||
"spec.database.clusterName": config.database.clusterName,
|
||||
"spec.database.postgresVersion": config.database.postgresVersion,
|
||||
"spec.database.storageClass": config.database.storageClass,
|
||||
"spec.database.pgdataStorageClass": config.database.pgdataStorageClass,
|
||||
"spec.database.walStorageClass": config.database.walStorageClass,
|
||||
"spec.database.appDatabase": config.database.appDatabase,
|
||||
"spec.database.metaDatabase": config.database.metaDatabase,
|
||||
"spec.database.appUser": config.database.appUser,
|
||||
@ -304,6 +329,12 @@ def validate_prod_config(config: KnoeProductionConfig) -> tuple[list[str], list[
|
||||
config.database.storageSizeGi = _validate_positive_int(
|
||||
config.database.storageSizeGi, "spec.database.storageSizeGi", errors
|
||||
)
|
||||
config.database.pgdataSizeGi = _validate_positive_int(
|
||||
config.database.pgdataSizeGi, "spec.database.pgdataSizeGi", errors
|
||||
)
|
||||
config.database.walSizeGi = _validate_positive_int(
|
||||
config.database.walSizeGi, "spec.database.walSizeGi", errors
|
||||
)
|
||||
config.backups.retentionDays = _validate_positive_int(
|
||||
config.backups.retentionDays, "spec.backups.retentionDays", errors
|
||||
)
|
||||
@ -359,6 +390,10 @@ def opentofu_vars(config: KnoeProductionConfig) -> dict[str, Any]:
|
||||
"db_instances": config.database.instances,
|
||||
"db_storage_class": config.database.storageClass,
|
||||
"db_storage_size_gi": config.database.storageSizeGi,
|
||||
"db_pgdata_storage_class": config.database.pgdataStorageClass,
|
||||
"db_wal_storage_class": config.database.walStorageClass,
|
||||
"db_pgdata_size_gi": config.database.pgdataSizeGi,
|
||||
"db_wal_size_gi": config.database.walSizeGi,
|
||||
"app_database": config.database.appDatabase,
|
||||
"meta_database": config.database.metaDatabase,
|
||||
"app_user": config.database.appUser,
|
||||
|
||||
@ -6,6 +6,13 @@ import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from knoe.core.cluster_storage_browser import (
|
||||
ClusterStorageRequest,
|
||||
GcloudRegionQuotaProvider,
|
||||
PingLatencyProvider,
|
||||
build_cluster_storage_browser_result,
|
||||
)
|
||||
|
||||
from .context import KnoeContext
|
||||
|
||||
|
||||
@ -15,6 +22,62 @@ def _project_root() -> Path:
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
argv = list(argv) if argv is not None else list(sys.argv[1:])
|
||||
|
||||
if argv and argv[0] == "storage-browser":
|
||||
storage = argparse.ArgumentParser(prog="knoe-ops storage-browser", add_help=True)
|
||||
storage.add_argument("--project", required=True, help="GCP project ID")
|
||||
storage.add_argument("--timezone", default="", help="Timezone group: americas|emea|apac")
|
||||
storage.add_argument("--pgdata-gb", type=float, default=100.0)
|
||||
storage.add_argument("--wal-gb", type=float, default=25.0)
|
||||
storage.add_argument(
|
||||
"--regions",
|
||||
default="",
|
||||
help="Comma-separated region list. If omitted, fetch via gcloud compute regions list.",
|
||||
)
|
||||
storage.add_argument("--output", choices=["table", "json"], default="table")
|
||||
args = storage.parse_args(argv[1:])
|
||||
|
||||
regions = [r.strip() for r in (args.regions or "").split(",") if r.strip()]
|
||||
if not regions:
|
||||
import subprocess
|
||||
|
||||
cmd = ["gcloud", "compute", "regions", "list", "--project", args.project, "--format=value(name)"]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
|
||||
if result.returncode == 0:
|
||||
regions = [r.strip() for r in result.stdout.splitlines() if r.strip()]
|
||||
except Exception:
|
||||
regions = []
|
||||
|
||||
result = build_cluster_storage_browser_result(
|
||||
project_id=args.project,
|
||||
requested=ClusterStorageRequest(pgdata_gb=args.pgdata_gb, wal_gb=args.wal_gb),
|
||||
available_regions=regions,
|
||||
timezone_group=(args.timezone or "").strip().lower() or None,
|
||||
quota_provider=GcloudRegionQuotaProvider(),
|
||||
latency_provider=PingLatencyProvider(),
|
||||
)
|
||||
if args.output == "json":
|
||||
print(result.to_json())
|
||||
return 0
|
||||
|
||||
print(f"Project: {result.project_id} Timezone: {result.timezone_group}")
|
||||
print(f"Quota docs: {result.quota_link}")
|
||||
print(f"Cloud Hub: {result.quota_console_link}")
|
||||
print("region\tfeasible\tdefault(pg/wal)\tpremium_headroom\tstandard_headroom\tlatency_ms")
|
||||
for c in result.candidates:
|
||||
f = c.feasibility
|
||||
feasible = "yes" if (f.can_premium or f.can_standard) else "no"
|
||||
premium = "-" if f.premium_headroom_gb is None else f"{f.premium_headroom_gb:.1f}"
|
||||
standard = "-" if f.standard_headroom_gb is None else f"{f.standard_headroom_gb:.1f}"
|
||||
latency = "-" if c.latency_ms is None else f"{c.latency_ms:.2f}"
|
||||
print(
|
||||
f"{f.region}\t{feasible}\t{f.default_pgdata_class}/{f.default_wal_class}\t"
|
||||
f"{premium}\t{standard}\t{latency}"
|
||||
)
|
||||
return 0
|
||||
|
||||
p = argparse.ArgumentParser(prog="knoe-ops", add_help=True)
|
||||
p.add_argument("component", help="component name (e.g. common_core, openbao)")
|
||||
p.add_argument(
|
||||
|
||||
@ -586,6 +586,8 @@ class ClusterScreenMixin:
|
||||
db_row = _add_row(db_tab, db_row, "CNPG Cluster", "database.clusterName")
|
||||
db_row = _add_row(db_tab, db_row, "Postgres Version", "database.postgresVersion")
|
||||
db_row = _add_row(db_tab, db_row, "Instances", "database.instances")
|
||||
db_row = _add_row(db_tab, db_row, "Node Boot Disk Type", "init_cluster.db_boot_disk_type")
|
||||
db_row = _add_row(db_tab, db_row, "Node Boot Disk Size (GB)", "init_cluster.db_boot_disk_size_gb")
|
||||
db_row = _add_row(db_tab, db_row, "Storage Class", "database.storageClass")
|
||||
db_row = _add_row(db_tab, db_row, "Storage Size (Gi)", "database.storageSizeGi")
|
||||
db_row = _add_row(db_tab, db_row, "App Database", "database.appDatabase")
|
||||
@ -2065,7 +2067,16 @@ class ClusterScreenMixin:
|
||||
"migration.sourcePasswordRef": tk.StringVar(
|
||||
value="secretref://local-source-db-password"
|
||||
),
|
||||
# GKE node pool configuration (persisted to [Initialize Cluster] in prole.cfg)
|
||||
"init_cluster.db_boot_disk_type": tk.StringVar(value="pd-standard"),
|
||||
"init_cluster.db_boot_disk_size_gb": tk.StringVar(value="50"),
|
||||
}
|
||||
# Pre-populate GKE node pool vars from existing prole.cfg if present
|
||||
_init_clus = (getattr(self, "prole_cfg_data", None) or {}).get("Initialize Cluster") or {}
|
||||
if _init_clus.get("DB_BOOT_DISK_TYPE"):
|
||||
self.prod_form_vars["init_cluster.db_boot_disk_type"].set(_init_clus["DB_BOOT_DISK_TYPE"])
|
||||
if _init_clus.get("DB_BOOT_DISK_SIZE_GB"):
|
||||
self.prod_form_vars["init_cluster.db_boot_disk_size_gb"].set(_init_clus["DB_BOOT_DISK_SIZE_GB"])
|
||||
self.prod_bool_vars = {
|
||||
"migration.continuousUntilCutover": tk.BooleanVar(value=False)
|
||||
}
|
||||
@ -2797,6 +2808,31 @@ class ClusterScreenMixin:
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def _to_int_text(value, default: int) -> int:
|
||||
try:
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return int(default)
|
||||
return int(text)
|
||||
except Exception:
|
||||
return int(default)
|
||||
|
||||
region_form = self.prod_form_vars["cloud.region"].get().strip()
|
||||
selected_region = region_form
|
||||
|
||||
db_storage_class = self.prod_form_vars["database.storageClass"].get().strip() or "premium-rwo"
|
||||
pgdata_storage_class = db_storage_class
|
||||
wal_storage_class = pgdata_storage_class
|
||||
if pgdata_storage_class not in {"premium-rwo", "standard-rwo"}:
|
||||
pgdata_storage_class = "premium-rwo"
|
||||
if wal_storage_class not in {"premium-rwo", "standard-rwo"}:
|
||||
wal_storage_class = pgdata_storage_class
|
||||
|
||||
default_pgdata_size = max(1, _to_int("database.storageSizeGi"))
|
||||
pgdata_size_gi = _to_int_text(str(default_pgdata_size), default_pgdata_size)
|
||||
default_wal = max(10, int(pgdata_size_gi * 0.25))
|
||||
wal_size_gi = _to_int_text(str(default_wal), default_wal)
|
||||
|
||||
doc = {
|
||||
"kind": "KnoeProductionConfig",
|
||||
"metadata": {
|
||||
@ -2808,7 +2844,7 @@ class ClusterScreenMixin:
|
||||
"cloud": {
|
||||
"provider": self.prod_form_vars["cloud.provider"].get().strip(),
|
||||
"projectId": self.prod_form_vars["cloud.projectId"].get().strip(),
|
||||
"region": self.prod_form_vars["cloud.region"].get().strip(),
|
||||
"region": selected_region,
|
||||
"clusterName": self.prod_form_vars["cloud.clusterName"].get().strip(),
|
||||
"vpcMode": self.prod_form_vars["cloud.vpcMode"].get().strip(),
|
||||
"vpcName": self.prod_form_vars["cloud.vpcName"].get().strip() or None,
|
||||
@ -2827,8 +2863,12 @@ class ClusterScreenMixin:
|
||||
"database.postgresVersion"
|
||||
].get().strip(),
|
||||
"instances": _to_int("database.instances"),
|
||||
"storageClass": self.prod_form_vars["database.storageClass"].get().strip(),
|
||||
"storageSizeGi": _to_int("database.storageSizeGi"),
|
||||
"storageClass": pgdata_storage_class,
|
||||
"storageSizeGi": pgdata_size_gi,
|
||||
"pgdataStorageClass": pgdata_storage_class,
|
||||
"walStorageClass": wal_storage_class,
|
||||
"pgdataSizeGi": pgdata_size_gi,
|
||||
"walSizeGi": wal_size_gi,
|
||||
"appDatabase": self.prod_form_vars["database.appDatabase"].get().strip(),
|
||||
"metaDatabase": self.prod_form_vars["database.metaDatabase"].get().strip(),
|
||||
"appUser": self.prod_form_vars["database.appUser"].get().strip(),
|
||||
@ -2944,6 +2984,14 @@ class ClusterScreenMixin:
|
||||
def _prod_put_config(self, show_dialog: bool = False) -> dict:
|
||||
self._ensure_prod_config_state()
|
||||
result = self.prod_config_api.put_prod_config(self._prod_payload_from_vars())
|
||||
# Persist GKE node pool config to [Initialize Cluster] section of prole.cfg
|
||||
_gke_sec = self.prole_cfg_data.setdefault("Initialize Cluster", {})
|
||||
_disk_type = self.prod_form_vars["init_cluster.db_boot_disk_type"].get().strip()
|
||||
_disk_size = self.prod_form_vars["init_cluster.db_boot_disk_size_gb"].get().strip()
|
||||
if _disk_type:
|
||||
_gke_sec["DB_BOOT_DISK_TYPE"] = _disk_type
|
||||
if _disk_size:
|
||||
_gke_sec["DB_BOOT_DISK_SIZE_GB"] = _disk_size
|
||||
cfg_doc = result.get("config") or {}
|
||||
if cfg_doc:
|
||||
self._prod_apply_payload_to_vars(cfg_doc)
|
||||
|
||||
@ -114,6 +114,16 @@ class ServicesScreenMixin:
|
||||
def _render_init_scripts_page(self):
|
||||
content_width = self.bg_canvas.winfo_width() or 975
|
||||
right_margin = content_width - 48
|
||||
app_cluster = (
|
||||
self._get_input("init_password.app_cluster_name", "")
|
||||
or self._get_input("env_setup.APP_CLUSTER_NAME", "")
|
||||
or "knoe-dev-0"
|
||||
).strip() or "knoe-dev-0"
|
||||
db_cluster = (
|
||||
self._get_input("init_password.db_cluster_name", "")
|
||||
or self._get_input("env_setup.DB_CLUSTER_NAME", "")
|
||||
or "knoe-cnpg-0"
|
||||
).strip() or "knoe-cnpg-0"
|
||||
|
||||
ui.canvas_text(
|
||||
self, right_margin, 40, "knoe.dev",
|
||||
@ -125,8 +135,36 @@ class ServicesScreenMixin:
|
||||
)
|
||||
self._render_title("Initialization Scripts", y=150)
|
||||
|
||||
ui.canvas_text(
|
||||
self,
|
||||
48,
|
||||
178,
|
||||
f"Application Cluster: {app_cluster} (Autopilot)",
|
||||
fill="#1d1d1f",
|
||||
font=("SF Pro Text", 11),
|
||||
anchor="nw",
|
||||
)
|
||||
ui.canvas_text(
|
||||
self,
|
||||
48,
|
||||
196,
|
||||
f"Database Cluster: {db_cluster} (Standard)",
|
||||
fill="#1d1d1f",
|
||||
font=("SF Pro Text", 11),
|
||||
anchor="nw",
|
||||
)
|
||||
ui.canvas_text(
|
||||
self,
|
||||
48,
|
||||
214,
|
||||
"CloudNativePG runs on the dedicated Standard GKE cluster; platform services remain on the Autopilot cluster.",
|
||||
fill="#6e6e73",
|
||||
font=("SF Pro Text", 10),
|
||||
anchor="nw",
|
||||
)
|
||||
|
||||
x_label = 48
|
||||
gy = 198 # top of traffic-light section
|
||||
gy = 242 # top of traffic-light section
|
||||
|
||||
# --- Traffic light section ---
|
||||
# Column x positions
|
||||
@ -1570,7 +1608,7 @@ class ServicesScreenMixin:
|
||||
def _inline_worker():
|
||||
try:
|
||||
namespace = (self.db_namespace.get() or "").strip() or "default"
|
||||
env = self._script_env_for_namespace(namespace)
|
||||
env = self._script_env_for_namespace(namespace, cluster_role="app")
|
||||
env["SERVICE_NAMESPACE"] = self._get_service_namespace()
|
||||
rc = self.controller.run_script(
|
||||
"init_service_layer.sh",
|
||||
@ -1652,7 +1690,7 @@ class ServicesScreenMixin:
|
||||
)
|
||||
)
|
||||
namespace = (self.db_namespace.get() or "").strip() or "default"
|
||||
env = self._script_env_for_namespace(namespace)
|
||||
env = self._script_env_for_namespace(namespace, cluster_role="app")
|
||||
env["SERVICE_NAMESPACE"] = self._get_service_namespace()
|
||||
try:
|
||||
log_fp = log_path.open("a", encoding="utf-8")
|
||||
@ -1989,7 +2027,7 @@ class ServicesScreenMixin:
|
||||
else None
|
||||
)
|
||||
)
|
||||
env = self._script_env_for_namespace(namespace)
|
||||
env = self._script_env_for_namespace(namespace, cluster_role="app")
|
||||
env["KNOE_DB_USER"] = self.db_username.get().strip()
|
||||
env["DB_PASSWORD"] = password
|
||||
env["AT_REST_ENCRYPTION_ENABLED"] = _bool_str(
|
||||
|
||||
@ -1 +1 @@
|
||||
36
|
||||
40
|
||||
@ -1,57 +1,55 @@
|
||||
Network Discovery Summary:
|
||||
Primary Router: 10.0.0.1 (eero_5d:50:f2)
|
||||
Primary Router: 10.0.0.1 (Unknown)
|
||||
DNS Servers: 10.0.0.5, 10.0.0.4, 100.100.100.100
|
||||
Detected Devices:
|
||||
- 10.0.0.35 [7e:19:9a:fe:9f:ed] (7e:19:9a:fe:9f:ed): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
|
||||
- 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
|
||||
- 10.0.0.1 [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: []
|
||||
- 10.0.0.3 [2c:cf:67:8c:2b:47] (RaspberryPi_8c:2b:47): Ports [22, 53, 80, 443, 2049, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'Active Directory Related']
|
||||
- 10.0.0.6 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.7 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.5 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC']
|
||||
- 10.0.0.4 [b8:27:eb:b3:6f:2b] (RaspberryPiF_b3:6f:2b): Ports [22, 53, 2049], Services: ['DNS', 'SSH', 'NFS']
|
||||
- 10.0.0.2 [dc:a4:ca:ea:1a:2f] (Apple_ea:1a:2f): Ports [], Services: []
|
||||
- 10.0.0.26 [c8:db:26:08:52:f8] (Logitech_08:52:f8): Ports [], Services: []
|
||||
- 10.0.0.22 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.205 [a8:20:66:28:12:e7] (Apple_28:12:e7): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.45 [f8:b4:6a:30:a4:1f] (HewlettPacka_30:a4:1f): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.48 [0c:fe:45:53:f4:3c] (SonyInteract_53:f4:3c): Ports [], Services: []
|
||||
- 10.0.0.37 [54:07:7d:22:c0:b2] (Netgear_22:c0:b2): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.54 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: []
|
||||
- 10.0.0.52 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.46 [d4:f7:d5:40:ab:17] (SonyInteract_40:ab:17): Ports [], Services: []
|
||||
- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: []
|
||||
- 10.0.0.58 [7e:46:74:2a:b0:b2] (7e:46:74:2a:b0:b2): Ports [], Services: []
|
||||
- 10.0.0.207 [e4:5f:01:90:35:0b] (RaspberryPiT_90:35:0b): Ports [22, 5900], Services: ['SSH', 'VNC']
|
||||
- 10.0.0.95 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 53, 443, 2049, 5900], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server']
|
||||
- 10.0.0.106 [48:a6:b8:a7:50:60] (Sonos_a7:50:60): Ports [], Services: []
|
||||
- 10.0.0.99 [20:28:bc:f1:dc:ba] (Visionscape_f1:dc:ba): Ports [], Services: []
|
||||
- 10.0.0.113 [60:81:10:92:02:4e] (Apple_92:02:4e): Ports [], Services: []
|
||||
- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
|
||||
- 10.0.0.112 [f8:33:31:40:58:c0] (TexasInstrum_40:58:c0): Ports [443], Services: ['Web Server']
|
||||
- 10.0.0.124 [9c:8e:cd:02:73:c4] (AmcrestTechn_02:73:c4): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.123 [3c:ef:8c:96:e3:3c] (ZhejiangDahu_96:e3:3c): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.125 [a0:60:32:04:73:ba] (AmcrestTechn_04:73:ba): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.117 [40:f5:20:1e:5e:91] (Espressif_1e:5e:91): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.27 [3a:b4:ce:79:b0:55] (3a:b4:ce:79:b0:55): Ports [], Services: []
|
||||
- 10.0.0.63 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: []
|
||||
- 10.0.0.111 [ca:0b:6c:5d:0a:e5] (ca:0b:6c:5d:0a:e5): Ports [], Services: []
|
||||
- 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.1\ [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [], Services: []
|
||||
- 10.0.0.130 [f8:bb:bf:a3:84:eb] (eero_a3:84:eb): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.128 [20:c9:d0:94:30:dd] (Apple_94:30:dd): Ports [5900], Services: ['VNC']
|
||||
- 10.0.0.143 [60:5f:8d:7a:aa:32] (eero_7a:aa:32): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.208 [a0:ad:9f:30:84:5f] (ASUSTekCOMPU_30:84:5f): Ports [], Services: []
|
||||
- 10.0.0.145 [7c:a6:b0:04:db:1c] (7c:a6:b0:04:db:1c): Ports [], Services: []
|
||||
- 10.0.0.155 [62:17:97:b3:06:a1] (62:17:97:b3:06:a1): Ports [], Services: []
|
||||
- 10.0.0.170 [b0:ee:7b:ca:73:99] (Roku_ca:73:99): Ports [], Services: []
|
||||
- 10.0.0.175 [28:80:88:e4:10:1a] (Netgear_e4:10:1a): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.188 [00:16:6c:c5:1f:54] (SamsungElect_c5:1f:54): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.49 [00:e0:4c:08:0c:75] (RealtekSemic_08:0c:75): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
|
||||
- 10.0.0.31 [2e:c1:c1:e9:87:b4] (2e:c1:c1:e9:87:b4): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related']
|
||||
- 10.0.0.189 [00:17:88:a3:2f:cc] (PhilipsLight_a3:2f:cc): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.196 [60:5f:8d:88:08:f2] (eero_88:08:f2): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.31\ [2e:c1:c1:e9:87:b4] (2e:c1:c1:e9:87:b4): Ports [], Services: []
|
||||
- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: []
|
||||
- 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: []
|
||||
- 10.0.0.204 [50:eb:f6:56:f3:26] (ASUSTekCOMPU_56:f3:26): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama']
|
||||
- 10.0.0.199 [7c:a6:b0:01:a7:33] (7c:a6:b0:01:a7:33): Ports [], Services: []
|
||||
Ollama Instances found at: 10.0.0.35, 10.0.0.204
|
||||
- 10.0.0.1 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.2 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.3 [Unknown] (Unknown): Ports [22, 53, 80, 443, 2049, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
- 10.0.0.4 [Unknown] (Unknown): Ports [22, 53, 443, 2049], Services: ['DNS', 'SSH', 'NFS', 'Web Server']
|
||||
- 10.0.0.5 [Unknown] (Unknown): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC']
|
||||
- 10.0.0.6 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.7 [Unknown] (Unknown): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.22 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server']
|
||||
- 10.0.0.26 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.32 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.37 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.45 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.46 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.48 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.52 [Unknown] (Unknown): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.62 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.95 [Unknown] (Unknown): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC']
|
||||
- 10.0.0.99 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.106 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.112 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.117 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.123 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.124 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.127 [Unknown] (Unknown): Ports [22], Services: ['SSH']
|
||||
- 10.0.0.128 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
- 10.0.0.130 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.143 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.145 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.155 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.170 [Unknown] (Unknown): Ports [], Services: []
|
||||
- 10.0.0.175 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
- 10.0.0.188 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server']
|
||||
- 10.0.0.196 [Unknown] (Unknown): Ports [53], Services: ['DNS']
|
||||
- 10.0.0.203 [Unknown] (Unknown): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)']
|
||||
- 10.0.0.205 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related']
|
||||
- 10.0.0.206 [Unknown] (Unknown): Ports [80], Services: ['Web Server']
|
||||
Ollama Instances found at: 10.0.0.49, 10.0.0.31, 10.0.0.204
|
||||
|
||||
Ansible Inventory Summary:
|
||||
Inventory directory: /Users/chrisfu/dev/prole/infrastructure
|
||||
|
||||
251
scripts/patch_clusters.sh
Executable file
251
scripts/patch_clusters.sh
Executable file
@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
# patch_clusters.sh — Migrate knoe-cnpg-0 boot disks from pd-ssd to pd-standard.
|
||||
#
|
||||
# The GCloud Console creates node pools with pd-ssd boot disks by default, which
|
||||
# consumes the entire 300 GB pd-ssd quota before any PGDATA/WAL PVCs can be
|
||||
# provisioned. This script replaces the existing default-pool with a new
|
||||
# cnpg-db-pool that uses pd-standard boot disks, freeing the quota for CNPG PVCs.
|
||||
#
|
||||
# What this does:
|
||||
# 1. Detects the current node pool disk configuration.
|
||||
# 2. Creates cnpg-db-pool (pd-standard boot, e2-standard-2, 3 nodes).
|
||||
# 3. Cordons and drains all nodes in the old pool.
|
||||
# 4. Deletes the old pool.
|
||||
# 5. Verifies 3 CNPG pods return to Running.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/patch_clusters.sh
|
||||
#
|
||||
# Environment overrides:
|
||||
# GCP_PROJECT — GCP project ID (default: plenary-truck-485623-p7)
|
||||
# GCP_REGION — GCP region (default: us-west3)
|
||||
# CLUSTER_NAME — GKE cluster (default: knoe-cnpg-0)
|
||||
# OLD_POOL — Pool to remove (default: default-pool)
|
||||
# NEW_POOL — Pool to create (default: cnpg-db-pool)
|
||||
# MACHINE_TYPE — Node machine type (default: e2-standard-2)
|
||||
# DISK_TYPE — Boot disk type (default: pd-standard)
|
||||
# DISK_SIZE_GB — Boot disk size GB (default: 50)
|
||||
# NODE_COUNT — Nodes per zone (default: 1, regional = 3 zones)
|
||||
# CNPG_NAMESPACE — Namespace for CNPG pods (default: knoe-db-0)
|
||||
# DRAIN_TIMEOUT — kubectl drain timeout (default: 300s)
|
||||
# DRY_RUN — Set to "true" to print commands without running (default: false)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
|
||||
GCP_REGION="${GCP_REGION:-us-west3}"
|
||||
CLUSTER_NAME="${CLUSTER_NAME:-knoe-cnpg-0}"
|
||||
OLD_POOL="${OLD_POOL:-default-pool}"
|
||||
NEW_POOL="${NEW_POOL:-cnpg-db-pool}"
|
||||
MACHINE_TYPE="${MACHINE_TYPE:-e2-standard-2}"
|
||||
DISK_TYPE="${DISK_TYPE:-pd-standard}"
|
||||
DISK_SIZE_GB="${DISK_SIZE_GB:-50}"
|
||||
NODE_COUNT="${NODE_COUNT:-1}" # 1 per zone; regional cluster spans 3 zones = 3 nodes total
|
||||
CNPG_NAMESPACE="${CNPG_NAMESPACE:-knoe-db-0}"
|
||||
DRAIN_TIMEOUT="${DRAIN_TIMEOUT:-300s}"
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log() {
|
||||
printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"
|
||||
}
|
||||
|
||||
require_tool() {
|
||||
command -v "$1" >/dev/null 2>&1 || { log "Error: required tool not found: $1" >&2; exit 1; }
|
||||
}
|
||||
|
||||
run_cmd() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "[DRY-RUN] $*"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
pool_exists() {
|
||||
local pool="$1"
|
||||
gcloud container node-pools describe "${pool}" \
|
||||
--cluster="${CLUSTER_NAME}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--format="value(name)" \
|
||||
--quiet 2>/dev/null | grep -q "${pool}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-flight
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
require_tool gcloud
|
||||
require_tool kubectl
|
||||
|
||||
log "==> Patch: migrate ${CLUSTER_NAME}/${OLD_POOL} → ${NEW_POOL} (${DISK_TYPE}, ${DISK_SIZE_GB}GB)"
|
||||
log " Project : ${GCP_PROJECT}"
|
||||
log " Region : ${GCP_REGION}"
|
||||
log " DRY_RUN : ${DRY_RUN}"
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Detect current configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 1] Detecting current node pool configuration ..."
|
||||
|
||||
if ! gcloud container clusters describe "${CLUSTER_NAME}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--format="value(name)" \
|
||||
--quiet 2>/dev/null | grep -q "${CLUSTER_NAME}"; then
|
||||
log "Error: cluster '${CLUSTER_NAME}' not found in project '${GCP_PROJECT}' region '${GCP_REGION}'." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if pool_exists "${OLD_POOL}"; then
|
||||
current_disk=$(gcloud container node-pools describe "${OLD_POOL}" \
|
||||
--cluster="${CLUSTER_NAME}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--format="value(config.diskType)" \
|
||||
--quiet 2>/dev/null || echo "unknown")
|
||||
current_size=$(gcloud container node-pools describe "${OLD_POOL}" \
|
||||
--cluster="${CLUSTER_NAME}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--format="value(config.diskSizeGb)" \
|
||||
--quiet 2>/dev/null || echo "unknown")
|
||||
log " ${OLD_POOL}: disk_type=${current_disk}, disk_size_gb=${current_size}"
|
||||
if [[ "${current_disk}" == "pd-standard" ]]; then
|
||||
log " NOTE: ${OLD_POOL} already uses pd-standard. Continuing to ensure pool is renamed to '${NEW_POOL}'."
|
||||
fi
|
||||
else
|
||||
log " ${OLD_POOL} not found — may have been removed already."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Create new node pool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 2] Ensuring node pool '${NEW_POOL}' exists ..."
|
||||
|
||||
if pool_exists "${NEW_POOL}"; then
|
||||
log " ${NEW_POOL} already exists — skipping creation."
|
||||
else
|
||||
log " Creating ${NEW_POOL} (machine=${MACHINE_TYPE}, disk=${DISK_TYPE}, size=${DISK_SIZE_GB}GB, num-nodes=${NODE_COUNT}/zone) ..."
|
||||
run_cmd gcloud container node-pools create "${NEW_POOL}" \
|
||||
--cluster="${CLUSTER_NAME}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--machine-type="${MACHINE_TYPE}" \
|
||||
--disk-type="${DISK_TYPE}" \
|
||||
--disk-size="${DISK_SIZE_GB}" \
|
||||
--num-nodes="${NODE_COUNT}" \
|
||||
--quiet
|
||||
log " ${NEW_POOL} created."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Cordon and drain old pool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 3] Cordoning and draining '${OLD_POOL}' ..."
|
||||
|
||||
if ! pool_exists "${OLD_POOL}"; then
|
||||
log " ${OLD_POOL} not found — nothing to drain."
|
||||
else
|
||||
old_nodes=$(kubectl get nodes \
|
||||
-l "cloud.google.com/gke-nodepool=${OLD_POOL}" \
|
||||
--no-headers \
|
||||
-o custom-columns=":metadata.name" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "${old_nodes}" ]]; then
|
||||
log " No nodes found labelled with nodepool=${OLD_POOL} — already drained or pool empty."
|
||||
else
|
||||
for node in ${old_nodes}; do
|
||||
log " Cordoning ${node} ..."
|
||||
run_cmd kubectl cordon "${node}"
|
||||
done
|
||||
|
||||
for node in ${old_nodes}; do
|
||||
log " Draining ${node} (timeout ${DRAIN_TIMEOUT}) ..."
|
||||
run_cmd kubectl drain "${node}" \
|
||||
--ignore-daemonsets \
|
||||
--delete-emptydir-data \
|
||||
--timeout="${DRAIN_TIMEOUT}" \
|
||||
--force
|
||||
done
|
||||
log " All nodes in ${OLD_POOL} drained."
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Delete old pool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 4] Deleting old pool '${OLD_POOL}' ..."
|
||||
|
||||
if ! pool_exists "${OLD_POOL}"; then
|
||||
log " ${OLD_POOL} does not exist — already removed."
|
||||
else
|
||||
run_cmd gcloud container node-pools delete "${OLD_POOL}" \
|
||||
--cluster="${CLUSTER_NAME}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--quiet
|
||||
log " ${OLD_POOL} deleted."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Verify CNPG pods
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 5] Verifying CNPG pods in namespace '${CNPG_NAMESPACE}' ..."
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
log " [DRY-RUN] Would check: kubectl -n ${CNPG_NAMESPACE} get pods -l cnpg.io/cluster=knoe-db"
|
||||
else
|
||||
max_attempts=20
|
||||
attempt=0
|
||||
while [[ ${attempt} -lt ${max_attempts} ]]; do
|
||||
running=$(kubectl -n "${CNPG_NAMESPACE}" get pods \
|
||||
-l "cnpg.io/cluster=knoe-db" \
|
||||
--no-headers 2>/dev/null \
|
||||
| grep -c "Running" || true)
|
||||
log " Running CNPG pods: ${running} / 3 expected (attempt $((attempt + 1))/${max_attempts})"
|
||||
if [[ "${running}" -ge 3 ]]; then
|
||||
log " All CNPG pods Running on ${NEW_POOL}."
|
||||
kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db"
|
||||
break
|
||||
fi
|
||||
attempt=$(( attempt + 1 ))
|
||||
if [[ ${attempt} -lt ${max_attempts} ]]; then
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${running:-0}" -lt 3 ]]; then
|
||||
log "WARNING: Not all CNPG pods reached Running state within timeout." >&2
|
||||
kubectl -n "${CNPG_NAMESPACE}" get pods -l "cnpg.io/cluster=knoe-db" || true
|
||||
log "Check pod events: kubectl -n ${CNPG_NAMESPACE} describe pods -l cnpg.io/cluster=knoe-db"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Done
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
echo ""
|
||||
log "==> Patch complete."
|
||||
log ""
|
||||
log "Next steps:"
|
||||
log " 1. Verify pd-ssd quota freed:"
|
||||
log " gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} --format='value(quotas[disksSSD].usage,quotas[disksSSD].limit)'"
|
||||
log " 2. Apply updated CNPG manifest (premium-rwo PVCs, 50Gi PGDATA + WAL):"
|
||||
log " kubectl --context=<cnpg-context> apply -f deploy/gcp/gke/knoe-db.yaml"
|
||||
log " 3. Watch PVC expansion and pod restart:"
|
||||
log " kubectl -n ${CNPG_NAMESPACE} get pvc -w"
|
||||
317
scripts/reset_clusters.sh
Executable file
317
scripts/reset_clusters.sh
Executable file
@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env bash
|
||||
# reset_clusters.sh — Delete and recreate both GKE clusters with correct configuration.
|
||||
#
|
||||
# Problem: Both clusters were created via GCloud Console as Autopilot, which:
|
||||
# - Prevents manual node pool management (patch_clusters.sh cannot work)
|
||||
# - Uses pd-balanced boot disks (counts against SSD_TOTAL_GB quota, 300 GB at limit)
|
||||
#
|
||||
# Solution: Delete both, recreate with pd-standard boot disks:
|
||||
# knoe-dev-0 → Autopilot (app workloads: GitLab, platform), pd-standard boot
|
||||
# knoe-cnpg-0 → Standard (CNPG only, 3× e2-standard-2), pd-standard boot
|
||||
#
|
||||
# SSD quota budget after reset:
|
||||
# Boot disks: pd-standard — does NOT count against SSD_TOTAL_GB
|
||||
# PGDATA PVCs (premium-rwo): 3 × 50 Gi = 150 Gi \ Apply knoe-db.yaml AFTER
|
||||
# WAL PVCs (premium-rwo): 3 × 50 Gi = 150 Gi / quota increase to 2 TB
|
||||
#
|
||||
# Usage:
|
||||
# CONFIRM=true ./scripts/reset_clusters.sh
|
||||
# CONFIRM=true DRY_RUN=true ./scripts/reset_clusters.sh # inspect only
|
||||
#
|
||||
# Environment overrides:
|
||||
# GCP_PROJECT (default: plenary-truck-485623-p7)
|
||||
# GCP_REGION (default: us-west3)
|
||||
# APP_CLUSTER (default: knoe-dev-0)
|
||||
# DB_CLUSTER (default: knoe-cnpg-0)
|
||||
# DB_MACHINE_TYPE (default: e2-standard-2)
|
||||
# DB_DISK_TYPE (default: pd-standard)
|
||||
# DB_DISK_SIZE_GB (default: 50)
|
||||
# DB_NODES_PER_ZONE (default: 1 → 3 nodes across 3 zones)
|
||||
# APP_DISK_TYPE (default: pd-standard)
|
||||
# APP_DISK_SIZE_GB (default: 50)
|
||||
# CONFIRM REQUIRED: must be "true" to allow destructive operations
|
||||
# DRY_RUN (default: false)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
GCP_PROJECT="${GCP_PROJECT:-plenary-truck-485623-p7}"
|
||||
GCP_REGION="${GCP_REGION:-us-west3}"
|
||||
APP_CLUSTER="${APP_CLUSTER:-knoe-dev-0}"
|
||||
DB_CLUSTER="${DB_CLUSTER:-knoe-cnpg-0}"
|
||||
DB_MACHINE_TYPE="${DB_MACHINE_TYPE:-e2-standard-2}"
|
||||
DB_DISK_TYPE="${DB_DISK_TYPE:-pd-standard}"
|
||||
DB_DISK_SIZE_GB="${DB_DISK_SIZE_GB:-50}"
|
||||
DB_NODES_PER_ZONE="${DB_NODES_PER_ZONE:-1}" # regional cluster = 3 zones = 3 nodes total
|
||||
APP_DISK_TYPE="${APP_DISK_TYPE:-pd-standard}"
|
||||
APP_DISK_SIZE_GB="${APP_DISK_SIZE_GB:-50}"
|
||||
CONFIRM="${CONFIRM:-false}"
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
|
||||
|
||||
die() { log "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
require_tool() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "required tool not found: $1"
|
||||
}
|
||||
|
||||
run_cmd() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "[DRY-RUN] $*"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
cluster_exists() {
|
||||
local name="$1"
|
||||
gcloud container clusters describe "${name}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--format="value(name)" \
|
||||
--quiet 2>/dev/null | grep -q "${name}"
|
||||
}
|
||||
|
||||
_ssd_quota_yaml() {
|
||||
# Emit the 3-line YAML block for SSD_TOTAL_GB quota entry, e.g.:
|
||||
# - limit: 300.0
|
||||
# metric: SSD_TOTAL_GB
|
||||
# usage: 300.0
|
||||
gcloud compute regions describe "${GCP_REGION}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--format=yaml \
|
||||
--quiet 2>/dev/null | grep -B 1 -A 1 "metric: SSD_TOTAL_GB" || true
|
||||
}
|
||||
|
||||
ssd_usage_gb() {
|
||||
local block
|
||||
block=$(_ssd_quota_yaml)
|
||||
echo "${block}" | awk '/usage:/{print $2}' | head -1 || echo "unknown"
|
||||
}
|
||||
|
||||
ssd_limit_gb() {
|
||||
local block
|
||||
block=$(_ssd_quota_yaml)
|
||||
echo "${block}" | awk '/limit:/{print $2}' | head -1 || echo "unknown"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-flight
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
require_tool gcloud
|
||||
require_tool kubectl
|
||||
|
||||
if [[ "${CONFIRM}" != "true" ]]; then
|
||||
echo ""
|
||||
echo " This script will DELETE and RECREATE both GKE clusters:"
|
||||
echo " ${APP_CLUSTER} (Autopilot) and ${DB_CLUSTER} (Standard)"
|
||||
echo ""
|
||||
echo " Set CONFIRM=true to proceed:"
|
||||
echo " CONFIRM=true ./scripts/reset_clusters.sh"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "==> Cluster reset: ${APP_CLUSTER} (Autopilot) + ${DB_CLUSTER} (Standard)"
|
||||
log " Project : ${GCP_PROJECT}"
|
||||
log " Region : ${GCP_REGION}"
|
||||
log " DRY_RUN : ${DRY_RUN}"
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Show current SSD quota
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 1] Current SSD quota (SSD_TOTAL_GB) in ${GCP_REGION} ..."
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
usage=$(ssd_usage_gb)
|
||||
limit=$(ssd_limit_gb)
|
||||
log " SSD usage: ${usage} GB / ${limit} GB limit"
|
||||
if [[ "${usage}" == "unknown" || "${limit}" == "unknown" ]]; then
|
||||
log " WARNING: Could not read SSD quota — proceeding anyway."
|
||||
fi
|
||||
else
|
||||
log " [DRY-RUN] Would read SSD quota from ${GCP_REGION}"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Delete existing clusters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 2] Deleting existing clusters ..."
|
||||
|
||||
for cluster in "${DB_CLUSTER}" "${APP_CLUSTER}"; do
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "[DRY-RUN] gcloud container clusters delete ${cluster} --project=${GCP_PROJECT} --region=${GCP_REGION} --quiet"
|
||||
else
|
||||
if cluster_exists "${cluster}"; then
|
||||
log " Deleting ${cluster} (this takes ~5-10 min) ..."
|
||||
gcloud container clusters delete "${cluster}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--quiet &
|
||||
log " ${cluster} deletion running in background (PID $!)."
|
||||
else
|
||||
log " ${cluster} not found — skipping."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for all background deletions to finish
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
log " Waiting for cluster deletions to complete ..."
|
||||
wait
|
||||
log " All deletions complete."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Wait for SSD quota to be released
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 3] Waiting for SSD quota to be released ..."
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
log " [DRY-RUN] Would poll SSD_TOTAL_GB until usage < 50 GB"
|
||||
else
|
||||
max_wait_s=1200 # 20 min max
|
||||
poll_s=30
|
||||
elapsed=0
|
||||
while true; do
|
||||
usage=$(ssd_usage_gb)
|
||||
log " SSD usage: ${usage} GB (${elapsed}s elapsed)"
|
||||
if [[ "${usage}" == "unknown" ]]; then
|
||||
log " WARNING: Could not read quota — treating as released."
|
||||
break
|
||||
fi
|
||||
# Cast to int for comparison
|
||||
usage_int=${usage%.*}
|
||||
if [[ "${usage_int}" -lt 50 ]]; then
|
||||
log " SSD quota released (${usage} GB remaining usage)."
|
||||
break
|
||||
fi
|
||||
if [[ ${elapsed} -ge ${max_wait_s} ]]; then
|
||||
log " WARNING: SSD quota did not fully release within ${max_wait_s}s."
|
||||
log " Current usage: ${usage} GB — proceeding with pd-standard (no SSD impact)."
|
||||
break
|
||||
fi
|
||||
sleep "${poll_s}"
|
||||
elapsed=$(( elapsed + poll_s ))
|
||||
done
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Create knoe-cnpg-0 as Standard GKE cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 4] Creating ${DB_CLUSTER} (Standard, ${DB_MACHINE_TYPE}, ${DB_DISK_TYPE}, ${DB_DISK_SIZE_GB}GB) ..."
|
||||
|
||||
run_cmd gcloud container clusters create "${DB_CLUSTER}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--cluster-version=latest \
|
||||
--machine-type="${DB_MACHINE_TYPE}" \
|
||||
--disk-type="${DB_DISK_TYPE}" \
|
||||
--disk-size="${DB_DISK_SIZE_GB}" \
|
||||
--num-nodes="${DB_NODES_PER_ZONE}" \
|
||||
--enable-ip-alias \
|
||||
--workload-pool="${GCP_PROJECT}.svc.id.goog" \
|
||||
--quiet
|
||||
|
||||
log " ${DB_CLUSTER} created."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Create knoe-dev-0 as Autopilot cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 5] Creating ${APP_CLUSTER} (Autopilot, ${APP_DISK_TYPE}, ${APP_DISK_SIZE_GB}GB) ..."
|
||||
|
||||
run_cmd gcloud container clusters create-auto "${APP_CLUSTER}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--cluster-version=latest \
|
||||
--workload-policies=allow-net-admin \
|
||||
--quiet
|
||||
# NOTE: Autopilot manages node infrastructure (disk type, size) automatically.
|
||||
# Boot disk type cannot be set at Autopilot cluster creation time.
|
||||
# Autopilot only provisions nodes when pods are scheduled — no quota impact
|
||||
# while the cluster is idle. Deploy workloads after the SSD quota increase
|
||||
# (300 GB → 2 TB) is approved to ensure headroom for GitLab node boot disks.
|
||||
|
||||
log " ${APP_CLUSTER} created."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 6: Fetch credentials and verify
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 6] Fetching kubeconfig credentials ..."
|
||||
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
run_cmd gcloud container clusters get-credentials "${DB_CLUSTER}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--quiet
|
||||
db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
|
||||
log " DB cluster context: ${db_ctx}"
|
||||
|
||||
run_cmd gcloud container clusters get-credentials "${APP_CLUSTER}" \
|
||||
--project="${GCP_PROJECT}" \
|
||||
--region="${GCP_REGION}" \
|
||||
--quiet
|
||||
app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
|
||||
log " App cluster context: ${app_ctx}"
|
||||
|
||||
log " Verifying cluster connectivity ..."
|
||||
kubectl --context="${db_ctx}" cluster-info --request-timeout=15s \
|
||||
&& log " ${DB_CLUSTER}: OK" \
|
||||
|| log " WARNING: ${DB_CLUSTER} not yet reachable — may need a moment."
|
||||
kubectl --context="${app_ctx}" cluster-info --request-timeout=15s \
|
||||
&& log " ${APP_CLUSTER}: OK" \
|
||||
|| log " WARNING: ${APP_CLUSTER} not yet reachable — may need a moment."
|
||||
else
|
||||
db_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${DB_CLUSTER}"
|
||||
app_ctx="gke_${GCP_PROJECT}_${GCP_REGION}_${APP_CLUSTER}"
|
||||
log " [DRY-RUN] Would fetch credentials for both clusters."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 7: Show final SSD quota
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log "[phase 7] Final SSD quota ..."
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
usage=$(ssd_usage_gb)
|
||||
limit=$(ssd_limit_gb)
|
||||
log " SSD usage: ${usage} GB / ${limit} GB (pd-standard boot disks use 0 SSD quota)"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Done
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
echo ""
|
||||
log "==> Reset complete."
|
||||
log ""
|
||||
log "Contexts:"
|
||||
log " App (Autopilot): ${app_ctx}"
|
||||
log " DB (Standard) : ${db_ctx}"
|
||||
log ""
|
||||
log "Next steps:"
|
||||
log " 1. Run the installer to configure workloads:"
|
||||
log " ./install.sh"
|
||||
log ""
|
||||
log " 2. Once the SSD quota increase (300 GB → 2 TB) is approved, apply CNPG storage:"
|
||||
log " kubectl --context=${db_ctx} apply -f deploy/gcp/gke/knoe-db.yaml"
|
||||
log " # This provisions 3×50Gi PGDATA + 3×50Gi WAL = 300 Gi pd-ssd"
|
||||
log ""
|
||||
log " 3. Check your quota increase request:"
|
||||
log " gcloud compute regions describe ${GCP_REGION} --project=${GCP_PROJECT} \\"
|
||||
log " --format='table(quotas.metric,quotas.usage,quotas.limit)' | grep SSD"
|
||||
90
tests/installer/test_cluster_storage_browser.py
Normal file
90
tests/installer/test_cluster_storage_browser.py
Normal file
@ -0,0 +1,90 @@
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from knoe.core.cluster_storage_browser import (
|
||||
ClusterStorageRequest,
|
||||
QuotaMetricHeadroom,
|
||||
RegionLatencyResult,
|
||||
RegionQuotaSnapshot,
|
||||
StaticQuotaProvider,
|
||||
build_cluster_storage_browser_result,
|
||||
evaluate_region_feasibility,
|
||||
)
|
||||
|
||||
|
||||
class RecordingLatencyProvider:
|
||||
def __init__(self, values: dict[str, RegionLatencyResult]):
|
||||
self.values = dict(values)
|
||||
self.calls: list[list[str]] = []
|
||||
|
||||
def probe(self, regions: list[str]) -> dict[str, RegionLatencyResult]:
|
||||
self.calls.append(list(regions))
|
||||
return {r: self.values[r] for r in regions if r in self.values}
|
||||
|
||||
|
||||
def _quota(region: str, premium_headroom: float, standard_headroom: float) -> RegionQuotaSnapshot:
|
||||
return RegionQuotaSnapshot(
|
||||
region=region,
|
||||
premium=QuotaMetricHeadroom(
|
||||
metric="SSD_TOTAL_GB",
|
||||
limit_gb=premium_headroom + 100.0,
|
||||
usage_gb=100.0,
|
||||
),
|
||||
standard=QuotaMetricHeadroom(
|
||||
metric="DISKS_TOTAL_GB",
|
||||
limit_gb=standard_headroom + 100.0,
|
||||
usage_gb=100.0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_browser_filters_timezone_probes_only_viable_and_ranks_by_feasibility_then_latency():
|
||||
quotas = {
|
||||
"us-central1": _quota("us-central1", premium_headroom=200.0, standard_headroom=20.0),
|
||||
"us-east1": _quota("us-east1", premium_headroom=10.0, standard_headroom=180.0),
|
||||
"southamerica-east1": _quota("southamerica-east1", premium_headroom=5.0, standard_headroom=5.0),
|
||||
}
|
||||
quota_provider = StaticQuotaProvider(quotas)
|
||||
latency_provider = RecordingLatencyProvider(
|
||||
{
|
||||
"us-central1": RegionLatencyResult(region="us-central1", latency_ms=40.0, ok=True),
|
||||
"us-east1": RegionLatencyResult(region="us-east1", latency_ms=15.0, ok=True),
|
||||
}
|
||||
)
|
||||
|
||||
result = build_cluster_storage_browser_result(
|
||||
project_id="p-1",
|
||||
requested=ClusterStorageRequest(pgdata_gb=100.0, wal_gb=25.0),
|
||||
available_regions=["us-central1", "us-east1", "southamerica-east1", "europe-west1"],
|
||||
timezone_group="americas",
|
||||
quota_provider=quota_provider,
|
||||
latency_provider=latency_provider,
|
||||
)
|
||||
|
||||
assert latency_provider.calls == [["us-central1", "us-east1"]]
|
||||
assert [c.feasibility.region for c in result.candidates] == [
|
||||
"us-central1",
|
||||
"us-east1",
|
||||
"southamerica-east1",
|
||||
]
|
||||
assert result.candidates[0].feasibility.default_pgdata_class == "premium-rwo"
|
||||
assert result.candidates[1].feasibility.default_pgdata_class == "standard-rwo"
|
||||
assert result.candidates[2].latency_ms is None
|
||||
|
||||
|
||||
def test_feasibility_defaults_to_premium_when_quota_is_missing():
|
||||
f = evaluate_region_feasibility(
|
||||
region="us-west1",
|
||||
timezone_group="americas",
|
||||
quota=None,
|
||||
requested=ClusterStorageRequest(pgdata_gb=80.0, wal_gb=20.0),
|
||||
)
|
||||
assert f.can_premium is False
|
||||
assert f.can_standard is False
|
||||
assert f.default_pgdata_class == "premium-rwo"
|
||||
assert "quota-unavailable" in f.notes
|
||||
83
tests/installer/test_gke_clusters.py
Normal file
83
tests/installer/test_gke_clusters.py
Normal file
@ -0,0 +1,83 @@
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from subprocess import CompletedProcess
|
||||
|
||||
from knoe.core.ops import gke_clusters
|
||||
|
||||
|
||||
def test_build_kubectl_env_for_cluster_sets_explicit_role_and_context():
|
||||
env = gke_clusters.build_kubectl_env_for_cluster(
|
||||
base_env={"EXISTING": "1"},
|
||||
kubecontext="ctx-app",
|
||||
cluster_name="knoe-dev-0",
|
||||
cluster_role="app",
|
||||
)
|
||||
|
||||
assert env["EXISTING"] == "1"
|
||||
assert env["KUBECTL_CONTEXT"] == "ctx-app"
|
||||
assert env["CLUSTER_NAME"] == "knoe-dev-0"
|
||||
assert env["KNOE_CLUSTER_ROLE"] == "app"
|
||||
assert env["KNOE_APP_CLUSTER_NAME"] == "knoe-dev-0"
|
||||
|
||||
|
||||
def test_get_cluster_credentials_builds_expected_gcloud_command(monkeypatch):
|
||||
called = {}
|
||||
|
||||
def fake_run(cmd, capture_output, text):
|
||||
called["cmd"] = cmd
|
||||
return CompletedProcess(cmd, 0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(gke_clusters.subprocess, "run", fake_run)
|
||||
|
||||
context = gke_clusters.get_cluster_credentials(
|
||||
project_id="proj-1",
|
||||
cluster_name="knoe-cnpg-0",
|
||||
location="us-central1",
|
||||
)
|
||||
|
||||
assert context == "gke_proj-1_us-central1_knoe-cnpg-0"
|
||||
assert called["cmd"] == [
|
||||
"gcloud",
|
||||
"container",
|
||||
"clusters",
|
||||
"get-credentials",
|
||||
"knoe-cnpg-0",
|
||||
"--project",
|
||||
"proj-1",
|
||||
"--region",
|
||||
"us-central1",
|
||||
"--quiet",
|
||||
]
|
||||
|
||||
|
||||
def test_ensure_db_cluster_creates_standard_cluster_with_minimum_nodes(monkeypatch):
|
||||
commands = []
|
||||
|
||||
def fake_run(cmd, log=None):
|
||||
commands.append(cmd)
|
||||
if "describe" in cmd:
|
||||
return CompletedProcess(cmd, 1, stdout="", stderr="not found")
|
||||
return CompletedProcess(cmd, 0, stdout="created", stderr="")
|
||||
|
||||
monkeypatch.setattr(gke_clusters, "_run", fake_run)
|
||||
|
||||
spec = gke_clusters.GkeClusterSpec(
|
||||
name="knoe-cnpg-0",
|
||||
mode="standard",
|
||||
location="us-central1",
|
||||
machine_type="e2-small",
|
||||
node_count=1,
|
||||
node_pool_name="cnpg-db-pool",
|
||||
)
|
||||
|
||||
gke_clusters.ensure_db_cluster(project_id="proj-1", spec=spec)
|
||||
|
||||
create_cmd = commands[-1]
|
||||
assert create_cmd[:5] == ["gcloud", "container", "clusters", "create", "knoe-cnpg-0"]
|
||||
assert "--num-nodes" in create_cmd
|
||||
assert create_cmd[create_cmd.index("--num-nodes") + 1] == "3"
|
||||
97
tests/installer/test_ops_cli_storage_browser.py
Normal file
97
tests/installer/test_ops_cli_storage_browser.py
Normal file
@ -0,0 +1,97 @@
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from knoe.core.cluster_storage_browser import (
|
||||
ClusterStorageBrowserResult,
|
||||
ClusterStorageRequest,
|
||||
RegionCandidate,
|
||||
RegionFeasibility,
|
||||
)
|
||||
from knoe.ops import cli as ops_cli
|
||||
|
||||
|
||||
def _fake_browser_result() -> ClusterStorageBrowserResult:
|
||||
req = ClusterStorageRequest(pgdata_gb=100.0, wal_gb=25.0)
|
||||
feasibility = RegionFeasibility(
|
||||
region="us-central1",
|
||||
timezone_group="americas",
|
||||
premium_headroom_gb=250.0,
|
||||
standard_headroom_gb=180.0,
|
||||
can_premium=True,
|
||||
can_standard=True,
|
||||
default_pgdata_class="premium-rwo",
|
||||
default_wal_class="premium-rwo",
|
||||
feasibility_rank=0,
|
||||
notes=(),
|
||||
)
|
||||
return ClusterStorageBrowserResult(
|
||||
generated_at="2026-01-01T00:00:00+00:00",
|
||||
project_id="p-1",
|
||||
timezone_group="americas",
|
||||
request=req,
|
||||
candidates=(RegionCandidate(feasibility=feasibility, latency_ms=12.3),),
|
||||
quota_link="https://cloud.google.com/compute/quotas",
|
||||
quota_console_link="https://console.cloud.google.com/iam-admin/quotas?project=p-1",
|
||||
)
|
||||
|
||||
|
||||
def test_storage_browser_json_output(monkeypatch, capsys):
|
||||
monkeypatch.setattr(ops_cli, "build_cluster_storage_browser_result", lambda **_kwargs: _fake_browser_result())
|
||||
|
||||
rc = ops_cli.main(
|
||||
[
|
||||
"storage-browser",
|
||||
"--project",
|
||||
"p-1",
|
||||
"--regions",
|
||||
"us-central1",
|
||||
"--output",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
doc = json.loads(out)
|
||||
assert doc["projectId"] == "p-1"
|
||||
assert doc["candidates"][0]["defaultPgdataClass"] == "premium-rwo"
|
||||
|
||||
|
||||
def test_storage_browser_table_output(monkeypatch, capsys):
|
||||
monkeypatch.setattr(ops_cli, "build_cluster_storage_browser_result", lambda **_kwargs: _fake_browser_result())
|
||||
|
||||
rc = ops_cli.main(
|
||||
[
|
||||
"storage-browser",
|
||||
"--project",
|
||||
"p-1",
|
||||
"--regions",
|
||||
"us-central1",
|
||||
"--output",
|
||||
"table",
|
||||
]
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "region\tfeasible\tdefault(pg/wal)" in out
|
||||
assert "us-central1" in out
|
||||
assert "premium-rwo/premium-rwo" in out
|
||||
|
||||
|
||||
def test_legacy_component_action_dispatch_remains_supported(monkeypatch):
|
||||
called = {}
|
||||
|
||||
def _status(_ctx):
|
||||
called["ok"] = True
|
||||
return 7
|
||||
|
||||
monkeypatch.setattr(ops_cli.importlib, "import_module", lambda _name: SimpleNamespace(status=_status))
|
||||
rc = ops_cli.main(["common_core", "status", "--mode", "prod"])
|
||||
assert called["ok"] is True
|
||||
assert rc == 7
|
||||
@ -24,6 +24,8 @@ def test_default_prod_config_uses_knoey_knoe_conventions():
|
||||
assert cfg.kubernetes.namespace == "ecosystem-0"
|
||||
assert cfg.cloud.clusterName == "knoe-prod"
|
||||
assert cfg.database.clusterName == "knoe-db"
|
||||
assert cfg.database.pgdataStorageClass == "premium-rwo"
|
||||
assert cfg.database.walStorageClass == "premium-rwo"
|
||||
assert cfg.database.appDatabase == "knoey"
|
||||
assert cfg.database.metaDatabase == "knoe_meta"
|
||||
assert cfg.backups.backupBucket == "knoe-0-backups"
|
||||
@ -57,6 +59,10 @@ def test_validation_requires_project_and_enforces_ecosystem_and_positive_numbers
|
||||
def test_yaml_and_opentofu_preview_contains_expected_values():
|
||||
cfg = default_prod_config()
|
||||
cfg.cloud.projectId = "my-gcp-project"
|
||||
cfg.database.pgdataStorageClass = "standard-rwo"
|
||||
cfg.database.walStorageClass = "premium-rwo"
|
||||
cfg.database.pgdataSizeGi = 120
|
||||
cfg.database.walSizeGi = 32
|
||||
|
||||
yaml_preview = canonical_yaml(cfg)
|
||||
tf_vars = opentofu_vars(cfg)
|
||||
@ -64,10 +70,16 @@ def test_yaml_and_opentofu_preview_contains_expected_values():
|
||||
assert "kind: KnoeProductionConfig" in yaml_preview
|
||||
assert "ecosystemId: 0" in yaml_preview
|
||||
assert "projectId: my-gcp-project" in yaml_preview
|
||||
assert "pgdataStorageClass: standard-rwo" in yaml_preview
|
||||
assert "walStorageClass: premium-rwo" in yaml_preview
|
||||
assert "frontdoorHost: knoey.com" in yaml_preview
|
||||
assert tf_vars["project_id"] == "my-gcp-project"
|
||||
assert tf_vars["namespace"] == "ecosystem-0"
|
||||
assert tf_vars["cnpg_cluster_name"] == "knoe-db"
|
||||
assert tf_vars["db_pgdata_storage_class"] == "standard-rwo"
|
||||
assert tf_vars["db_wal_storage_class"] == "premium-rwo"
|
||||
assert tf_vars["db_pgdata_size_gi"] == 120
|
||||
assert tf_vars["db_wal_size_gi"] == 32
|
||||
|
||||
|
||||
def test_prod_config_api_plan_apply_status_and_logs_flow():
|
||||
|
||||
@ -105,3 +105,10 @@ def test_cluster_nodes_prev_goes_back_to_cluster_environment(mock_installer):
|
||||
mock_installer.show_page("cluster_nodes")
|
||||
mock_installer.on_prev()
|
||||
assert mock_installer.pages[mock_installer.page_index][0] == "init_cluster"
|
||||
|
||||
|
||||
def test_common_services_prev_goes_back_to_cluster_nodes(mock_installer):
|
||||
mock_installer.show_page("common_services")
|
||||
with patch.object(mock_installer, "_should_show_cluster_nodes_screen", return_value=True):
|
||||
mock_installer.on_prev()
|
||||
assert mock_installer.pages[mock_installer.page_index][0] == "cluster_nodes"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user