diff --git a/conf/prod/prole.cfg b/conf/prod/prole.cfg index 2889ed9..69f0d33 100644 --- a/conf/prod/prole.cfg +++ b/conf/prod/prole.cfg @@ -39,15 +39,15 @@ dependencies.opentofu.install = true dependencies.python.install = true dependencies.verify_all = false disk_selection.disk_type = local -disk_selection.local_path = /Users/chrisfu/dev/prole/prole-tools-app/dist +disk_selection.local_path = $HOME/dev/prole/prole-tools-app/dist disk_selection.removable_mount = env_setup.CLUSTER_NAME = ${CLUSTER_NAME} env_setup.DATABASE_NAMESPACE = ${DATABASE_NAMESPACE} -env_setup.PROLE_CONF = /Users/chrisfu/dev/prole/conf -env_setup.PROLE_DATA = /Users/chrisfu/dev/prole/data -env_setup.PROLE_HOME = /Users/chrisfu/dev/prole -env_setup.PROLE_LOGS = /Users/chrisfu/dev/prole/logs -env_setup.PROLE_SERVICE = /Users/chrisfu/dev/prole/etc +env_setup.PROLE_CONF = $HOME/dev/prole/conf +env_setup.PROLE_DATA = $HOME/dev/prole/data +env_setup.PROLE_HOME = $HOME/dev/prole +env_setup.PROLE_LOGS = $HOME/dev/prole/logs +env_setup.PROLE_SERVICE = $HOME/dev/prole/etc gitops.git_provider = Gitea gitops.node_selector = init_cluster.argocd_enabled = false @@ -60,15 +60,15 @@ init_cluster.k3s_token = ${PROLE_SECRET:v1:vxIxsLYZSClD_vzj:aSQOAaWbKOIT26PQPCqq init_cluster.kerberos_enabled = true init_cluster.mode = k8s init_cluster.start_cluster = true -init_cluster.supabase_enabled = false +init_cluster.supabase_enabled = true init_cnpg_deploy.force_rollout = false init_cnpg_deploy.run_deploy = true init_db_build.run_build = true init_password.cluster_name = ${CLUSTER_NAME} init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password = ${PROLE_SECRET:v1:2CBUs--mwViqouh9:Q0s7wpwoIfKi22i66X1h3o9eAZetKkVDfO6o2dbU1jBFv8ti} -init_password.db_password_confirm = ${PROLE_SECRET:v1:2CBUs--mwViqouh9:Q0s7wpwoIfKi22i66X1h3o9eAZetKkVDfO6o2dbU1jBFv8ti} +init_password.db_password= vzx.wC4Akd4x-Dj6Wguh +init_password.db_password_confirm= vzx.wC4Akd4x-Dj6Wguh init_password.db_username = root init_password.generate_ssh_key = true init_scripts.run_scripts = true @@ -93,14 +93,14 @@ ARTIFACT_REGISTRY_AVAILABLE = true CLUSTER_ENV = prod CLUSTER_NAME = knoe-db CNPG_ELIGIBLE_NODES = gke-knoe-cnpg-0-default-pool-56407930-wlg9,gke-knoe-cnpg-0-default-pool-b142a290-bmr3,gke-knoe-cnpg-0-default-pool-d8e3a5ef-lmpl -CNPG_PLACEMENT_PLAN_FILE = /Users/chrisfu/dev/prole/conf/cnpg-placement/knoe-system-knoe-db.json +CNPG_PLACEMENT_PLAN_FILE = $HOME/dev/prole/conf/cnpg-placement/knoe-system-knoe-db.json CNPG_PLACEMENT_PLAN_HASH = a44825aeb5ed4e13 CNPG_PLACEMENT_PLAN_ID = cnpg-placement-a44825aeb5ed4e13 CNPG_STAGE1_NODE = gke-knoe-cnpg-0-default-pool-56407930-wlg9 DATABASE_NAMESPACE = knoe-db-0 DB_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-cnpg-0 DB_HOST_PORT = 5432 -DB_PASSWORD = ${PROLE_SECRET:v1:vnEWu4ZkBlVtoi-J:Dq_Gluj0Yv_3HdbN_9V53QMf9mPRXotsX04HWsMeY92GBj3O} +DB_PASSWORD= vzx.wC4Akd4x-Dj6Wguh DEPLOYMENT_MODE = k8s DEPLOYMENT_TARGET = prole-prod-cluster DOCKER_PRELOAD = false @@ -124,8 +124,8 @@ SUPABASE_PV_NODE = gandalf.prole.org AD_DC_HOST = myrddin.prole.org AD_DC_IP = 10.0.0.3 ANSIBLE_DOMAIN = prole.org -ANSIBLE_INFRASTRUCTURE = /Users/chrisfu/dev/prole/infrastructure -ANSIBLE_INVENTORY = /Users/chrisfu/dev/prole/infrastructure/inventory +ANSIBLE_INFRASTRUCTURE = $HOME/dev/prole/infrastructure +ANSIBLE_INVENTORY = $HOME/dev/prole/infrastructure/inventory ANSIBLE_REALM = PROLE.ORG ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","merlin.prole.org":"10.0.0.6","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org","gandalf.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_servers":["myrddin.prole.org"],"k3s_agents":["merlin.prole.org","gandalf.prole.org"],"k3s_hosts:children":["k3s_servers","k3s_agents"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org","merlin.prole.org","gandalf.prole.org"],"ssl_hosts":["myrddin.prole.org"],"mariadb_primary":["merlin.prole.org"],"mariadb_replica":["raspberry.prole.org"],"mariadb:children":["mariadb_primary","mariadb_replica"],"merlin_bootstrap":["merlin"],"k3s_hosts":["gandalf.prole.org"]},"hosts":{"merlin":"10.0.0.6","merlin.prole.org":"10.0.0.6","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":["gandalf.prole.org","k3s_agents","k3s_servers","mariadb_primary","mariadb_replica"]} KDC_ANSIBLE_DETECTED = 10.0.0.3 @@ -136,11 +136,11 @@ KERBEROS_AUTO_ENABLED = True ; No configuration values captured yet for this section. [System Environment] -PROLE_CONF = /Users/chrisfu/dev/prole/conf -PROLE_DATA = /Users/chrisfu/dev/prole/data -PROLE_HOME = /Users/chrisfu/dev/prole -PROLE_LOGS = /Users/chrisfu/dev/prole/logs -PROLE_SERVICE = /Users/chrisfu/dev/prole/etc +PROLE_CONF = $HOME/dev/prole/conf +PROLE_DATA = $HOME/dev/prole/data +PROLE_HOME = $HOME/dev/prole +PROLE_LOGS = $HOME/dev/prole/logs +PROLE_SERVICE = $HOME/dev/prole/etc [Monitoring] ; No configuration values captured yet for this section. @@ -157,7 +157,7 @@ OLLAMA_SERVER_PORT = 11434 AT_REST_ENCRYPTION_ENABLED = True GITOPS_ENABLED = False KERBEROS_ENABLED = True -SUPABASE_ENABLED = False +SUPABASE_ENABLED = True [GitOps] ; No configuration values captured yet for this section. @@ -196,7 +196,7 @@ org_id = 584001916389 project_id = plenary-truck-485623-p7 [Prod Cluster (k8s)] -ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging +ARTIFACTS_DIR = $HOME/dev/prole/data/staging CLUSTER_ENV = prole-prod-cluster DISPLAY_NAME = prole-prod-cluster INSTALL_PLAN = Validate production config and required secrets diff --git a/conf/service/prod.cfg b/conf/service/prod.cfg index 42f3945..3471539 100644 --- a/conf/service/prod.cfg +++ b/conf/service/prod.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-09 23:19:13 +; Generated by install.py on 2026-04-10 04:24:51 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -23,12 +23,12 @@ dependencies.verify_all = false disk_selection.disk_type = local disk_selection.local_path = /Users/chrisfu disk_selection.removable_mount = -env_setup.APP_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-0 +env_setup.APP_CLUSTER_KUBECONTEXT = env_setup.APP_CLUSTER_MODE = standard env_setup.APP_CLUSTER_NAME = knoe-dev-0 env_setup.CLUSTER_NAME = ${CLUSTER_NAME} env_setup.DATABASE_NAMESPACE = ${DATABASE_NAMESPACE} -env_setup.DB_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-cnpg-0 +env_setup.DB_CLUSTER_KUBECONTEXT = env_setup.DB_CLUSTER_MODE = standard env_setup.DB_CLUSTER_NAME = knoe-cnpg-0 env_setup.PROLE_CONF = /Users/chrisfu/dev/prole/conf @@ -74,14 +74,14 @@ init_password.cluster_name = ${CLUSTER_NAME} init_password.db_cluster_name = knoe-cnpg-0 init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password = ${PROLE_SECRET:v1:aqtfPnE_i0CuDU42:KvAQK1cUXKkFPta7NGyaWJVX1Ufw6xH432ZLzg==} -init_password.db_password_confirm = ${PROLE_SECRET:v1:aqtfPnE_i0CuDU42:KvAQK1cUXKkFPta7NGyaWJVX1Ufw6xH432ZLzg==} +init_password.db_password = ${PROLE_SECRET:v1:5fkkNdavd6ZL6a5E:HmuHq76QWhbvIEXshH7LV6Xa62zmA6RrRPOzhMUWoKmPaYXf} +init_password.db_password_confirm = ${PROLE_SECRET:v1:5fkkNdavd6ZL6a5E:HmuHq76QWhbvIEXshH7LV6Xa62zmA6RrRPOzhMUWoKmPaYXf} init_password.db_username = chrisfu init_password.generate_ssh_key = true init_scripts.run_scripts = true kerberos_config.enabled = True kerberos_config.init_authority = false -kerberos_config.kdc = 10.0.0.196 +kerberos_config.kdc = 10.0.0.205 kerberos_config.password = kerberos_config.realm = kerberos_config.test_connection = false @@ -93,20 +93,16 @@ ollama_config.server_port = 11434 [Global] ; Variables used by name in more than one place or assumed global scope -APP_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-0 ARGOCD_NAMESPACE = argocd ARTIFACT_REGISTRY = us-west3-docker.pkg.dev/plenary-truck-485623-p7/knoe-system CLUSTER_ENV = prod CLUSTER_NAME = knoe-db CLUSTER_SIZE_PROFILE = small -CNPG_ELIGIBLE_NODES = gke-knoe-cnpg-0-default-pool-56407930-wlg9,gke-knoe-cnpg-0-default-pool-b142a290-bmr3,gke-knoe-cnpg-0-default-pool-d8e3a5ef-lmpl CNPG_INSTANCES = 3 CNPG_SIZE_PROFILE = small -CNPG_STAGE1_NODE = gke-knoe-cnpg-0-default-pool-56407930-wlg9 DATABASE_NAMESPACE = knoe-db-0 -DB_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-cnpg-0 DB_HOST_PORT = 5432 -DB_PASSWORD = ${PROLE_SECRET:v1:BQs_xFgVxqbFwmtj:bXF155fo4KH1ZSwz9NvhgqTwQ7_oRkHewFsm5w==} +DB_PASSWORD = ${PROLE_SECRET:v1:yCJTqE6y6mZD5jUs:U5KhexVZFrKeKP7CjHXLWQLPNn7rwUv4afyGi0p9ncDDYF47} DEPLOYMENT_MODE = k8s DEPLOYMENT_TARGET = prole-prod-cluster DOCKER_IMPORT_DIR = @@ -114,7 +110,6 @@ DOCKER_PRELOAD = false K3S_SERVER = K3S_TOKEN = KNOE_DB_USER = chrisfu -KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-0 OPENTOFU_URL = http://127.0.0.1:8080 OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2 PROLE_HOME = /Users/chrisfu/dev/prole @@ -128,7 +123,7 @@ SERVICE_NAMESPACE = knoe-system STATUS = All installed [Network] -KDC_AUTO_DETECTED = 10.0.0.196 +KDC_AUTO_DETECTED = 10.0.0.205 KERBEROS_AUTO_ENABLED = True [Port Forwards] @@ -152,7 +147,7 @@ PROLE_SERVICE = /Users/chrisfu/dev/prole/etc ; No configuration values captured yet for this section. [Kerberos Authentication] -; No configuration values captured yet for this section. +STATUS = Initialized [Ollama] ; No configuration values captured yet for this section. @@ -164,6 +159,7 @@ KERBEROS_ENABLED = false SUPABASE_ENABLED = false [GitOps] +STATUS = Skipped [Database Creation] APP_CLUSTER_NAME = knoe-dev-0 diff --git a/img/.DS_Store b/img/.DS_Store index 19d1e28..2c51c18 100644 Binary files a/img/.DS_Store and b/img/.DS_Store differ diff --git a/infrastructure/inventory/group_vars/all/vault_db_master.yml b/infrastructure/inventory/group_vars/all/vault_db_master.yml index 1afed76..9f94788 100644 --- a/infrastructure/inventory/group_vars/all/vault_db_master.yml +++ b/infrastructure/inventory/group_vars/all/vault_db_master.yml @@ -1,7 +1,8 @@ -vault_prole_db_master_password: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 34636665663663616661623235656234636236363764383766353934306466363238326165616632 - 3937613765353935636661333032306333626234306564370a373630393535643531623764383661 - 63616233643631666539346534666332343463633435653736336666643066383062303566326235 - 3138626139363635390a376335666665366261623761383764393833396533396662643439666339 - 34333464623839666532313463363062366533393033626130613132346639633030 +$ANSIBLE_VAULT;1.1;AES256 +33623434353765353030333339626631656163343239353230643430356466306461663835383566 +3564333737623034343837616632386462306462366538390a623538373034383839376261653734 +36653238393437656332373965663866653730343864333063366462303661356366323262363839 +3961633266353131310a313930346164393031623037393339356539616639343536353163343036 +61353535633234363535633437356162626234356139323531643534613961633166393135356562 +30306634646130353665656262393132656632373634353765316630643665356331363435366165 +356261383336383736336336356564386337 diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/data/garage-marker b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/data/garage-marker new file mode 100644 index 0000000..33a20e8 --- /dev/null +++ b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/data/garage-marker @@ -0,0 +1 @@ +448ff34980e3a5fa3a21450701708387ea44574a9a76220310731333c2a72a5d \ No newline at end of file diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/cluster_layout b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/cluster_layout new file mode 100644 index 0000000..54308a1 Binary files /dev/null and b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/cluster_layout differ diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/data_layout b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/data_layout new file mode 100644 index 0000000..2dde40c Binary files /dev/null and b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/data_layout differ diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/db.lmdb/data.mdb b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/db.lmdb/data.mdb new file mode 100644 index 0000000..fcab92b Binary files /dev/null and b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/db.lmdb/data.mdb differ diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/db.lmdb/lock.mdb b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/db.lmdb/lock.mdb new file mode 100644 index 0000000..f182987 Binary files /dev/null and b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/db.lmdb/lock.mdb differ diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/lifecycle_worker_state b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/lifecycle_worker_state new file mode 100644 index 0000000..64a8f6b --- /dev/null +++ b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/lifecycle_worker_state @@ -0,0 +1 @@ +G09lwp®last_completedª2026-04-10 \ No newline at end of file diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/node_key b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/node_key new file mode 100644 index 0000000..5c1cf73 --- /dev/null +++ b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/node_key @@ -0,0 +1 @@ +Ìþú©àU9nÇù 5¼â‰¤ lþW?,¦ç§×f”í0æ6ïÄH’ÏƧò•¡®¦§¥‚kØœuzÞ{ \ No newline at end of file diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/node_key.pub b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/node_key.pub new file mode 100644 index 0000000..2fe2755 --- /dev/null +++ b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/node_key.pub @@ -0,0 +1 @@ +0æ6ïÄH’ÏƧò•¡®¦§¥‚kØœuzÞ{ \ No newline at end of file diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/peer_list b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/peer_list new file mode 100644 index 0000000..9e99dd5 --- /dev/null +++ b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/peer_list @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/scrub_info b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/scrub_info new file mode 100644 index 0000000..53aaa86 Binary files /dev/null and b/knoe-db/data/pvc-79f83d84-3161-4510-95ae-cf58efba90b0_knoe-system_data-garage-0/meta/scrub_info differ diff --git a/knoe/core/actions.py b/knoe/core/actions.py index 5904845..dc7866e 100644 --- a/knoe/core/actions.py +++ b/knoe/core/actions.py @@ -80,6 +80,7 @@ from knoe.core.milestones import ( InitializationScriptsMilestone, KerberosMilestone, DeploymentMilestone, + SecurityMilestone, GitOpsMilestone, SupabaseImagePreloadMilestone, SupabaseMilestone, @@ -6876,6 +6877,7 @@ class KnoeConsoleInstaller(KnoeInstaller): SupabaseImagePreloadMilestone(), SupabaseMilestone(), DeploymentMilestone(), + SecurityMilestone(), ] try: diff --git a/knoe/core/milestones.py b/knoe/core/milestones.py index fd2e5aa..998c817 100644 --- a/knoe/core/milestones.py +++ b/knoe/core/milestones.py @@ -974,6 +974,65 @@ class DeploymentMilestone(Milestone): progress("Deployment complete", 1.0) +class SecurityMilestone(Milestone): + """Enforce uniform master password across all services via update.sh. + + Runs update.sh (reads vault_prole_db_master_password from the Ansible vault) + and applies the password to k8s secrets, PostgreSQL, and Grafana. Only + active in k8s mode — silently skipped for k3s/k3d. + """ + + def __init__(self): + super().__init__("security", "Security Hardening") + self.logger = logging.getLogger("SecurityMilestone") + + def execute( + self, state: InstallerState, progress: ProgressCallback | None = None + ) -> None: + env = self._get_script_env(state) + mode = env.get("PROLE_MODE", "") + + if mode != "k8s": + self.logger.info( + f"[SECURITY] Mode={mode!r}: skipping live secret rotation (non-k8s)." + ) + if progress: + progress("Security hardening skipped (non-k8s mode)", 1.0) + return + + project_root = getattr(state.controller, "project_root", ".") + update_sh = str(Path(project_root) / "update.sh") + if not Path(update_sh).exists(): + self.logger.warning( + f"[SECURITY] update.sh not found at {update_sh} — skipping." + ) + if progress: + progress("update.sh not found — skipping security hardening", 1.0) + return + + if progress: + progress("Rotating master password via update.sh...", 0.2) + self.logger.info("[SECURITY] Running update.sh to enforce vault master password.") + + script_env = { + **env, + "PATH": os.environ.get("PATH", ""), + } + rc = subprocess.call([update_sh], env=script_env) + + if rc != 0: + raise Exception( + f"update.sh failed with exit code {rc}. " + "Run './update.sh --prompt' to recreate the vault file if it is corrupt." + ) + + self.logger.info("[SECURITY] Master password rotation complete.") + state.config_data.setdefault("Security", {})["STATUS"] = "Secured" + + if progress: + progress("Security hardening complete", 1.0) + + class GitOpsMilestone(Milestone): def __init__(self): super().__init__("gitops", "GitOps") diff --git a/knoe/ui/screens/__init__.py b/knoe/ui/screens/__init__.py index c24930f..128f0c5 100644 --- a/knoe/ui/screens/__init__.py +++ b/knoe/ui/screens/__init__.py @@ -340,7 +340,7 @@ class KnoeInstaller( ("GitOps", "gitops_config"), ("Supabase", "supabase_config"), ("Deployment", "init_cnpg_deploy"), - ("Post Install", "create_installer"), + ("Security", "security"), ] self.nav_widgets = {} self._mode_tab_widgets: dict = {} @@ -878,6 +878,7 @@ class KnoeInstaller( "disk_selection", self._render_disk_selection_page ) self._register_canvas_renderer("build_summary", self._render_build_summary_page) + self._register_canvas_renderer("security", self._render_security_page) self._register_canvas_renderer( "create_installer", self._render_create_installer_page ) @@ -902,6 +903,7 @@ class KnoeInstaller( self._register_page("gitops_config", None) self._register_page("supabase_config", None) self._register_page("init_cnpg_deploy", None) + self._register_page("security", None) self._register_page("build", None) self._register_page("disk_selection", None) self._register_page("build_summary", None) diff --git a/knoe/ui/screens/navigation.py b/knoe/ui/screens/navigation.py index a4a874e..5dd3a80 100644 --- a/knoe/ui/screens/navigation.py +++ b/knoe/ui/screens/navigation.py @@ -799,7 +799,14 @@ class NavigationMixin: "Deployed" if getattr(self, "_cnpg_success", False) else "Attempted" ) self._save_prole_cfg() - self.show_page("create_installer") + self.show_page("security") + return + + if current_id == "security": + if "Install" in self.prole_cfg_data: + self.prole_cfg_data["Install"]["STATUS"] = "Finished" + self._save_prole_cfg() + self.root.destroy() return if current_id == "create_installer": diff --git a/knoe/ui/screens/packaging.py b/knoe/ui/screens/packaging.py index a38d9fa..253dcb8 100644 --- a/knoe/ui/screens/packaging.py +++ b/knoe/ui/screens/packaging.py @@ -417,6 +417,205 @@ class PackagingScreenMixin: ) self._canvas_items.append(self._build_a_bao_status_label) + # ----------------------------------------------------------------------- + # Security screen — post-deployment account status + ingress links + # ----------------------------------------------------------------------- + def _render_security_page(self): # noqa: C901 + """Security dashboard: secured account status + clickable service links.""" + content_width = self.bg_canvas.winfo_width() or 975 + right_margin = content_width - 48 + + # --- Letterhead --- + ui.canvas_text( + self, right_margin, 40, "knoe.dev", + fill="#6e6e73", font=("SF Pro Text", 32, "bold"), anchor="ne", + ) + ui.canvas_text( + self, right_margin, 85, "infrastructure.auto()", + fill="#6e6e73", font=("SF Pro Text", 18), anchor="ne", + ) + + self._render_title("Security", y=150) + self._render_paragraph( + "Installation complete. All service accounts are controlled by the vault master password.", + y=198, + ) + + # --- Section A: Account security status --- + ui.canvas_text( + self, 48, 255, "Account Security", + fill="#1d1d1f", font=("SF Pro Text", 13, "bold"), + ) + + # Determine overall security status from config data + security_status = ( + (self.prole_cfg_data.get("Security", {}) or {}).get("STATUS", "") + ) + secured = security_status == "Secured" + + _GREEN = "#30d158" + _RED = "#ff453a" + _YELLOW = "#ffd60a" + _GRAY = "#8e8e93" + + accounts = [ + ("Grafana admin", "prometheus-grafana", "monitoring"), + ("PostgreSQL prole", "knoe-db-user", "knoe-db-0"), + ("PostgreSQL postgres", "knoe-db-superuser", "knoe-db-0"), + ("OpenBao root", "openbao-root-token", "knoe-system"), + ] + + dot_color = _GREEN if secured else (_RED if security_status else _YELLOW) + status_text = "Secured ✓" if secured else ("Pending" if not security_status else "Check failed ✗") + + y_row = 285 + for label, _secret, _ns in accounts: + # Colored dot + oval = self.bg_canvas.create_oval(48, y_row, 60, y_row + 12, fill=dot_color, outline="") + self._canvas_items.append(oval) + ui.canvas_text( + self, 72, y_row + 1, f"{label}", + fill="#1d1d1f", font=("SF Pro Text", 12), anchor="nw", + ) + ui.canvas_text( + self, 340, y_row + 1, status_text, + fill=dot_color if secured else _GRAY, + font=("SF Pro Text", 12), anchor="nw", + ) + y_row += 32 + + # Run update.sh button (re-rotate on demand) + def _run_update_sh(): + import threading as _th + _update_label = getattr(self, "_sec_update_label", None) + if _update_label: + self.bg_canvas.itemconfigure(_update_label, text="Running update.sh…", fill=_YELLOW) + + def _worker(): + try: + rc = subprocess.call( + [str(PROJECT_ROOT / "update.sh")], + env={**os.environ}, + ) + msg = "Password rotation complete ✓" if rc == 0 else f"update.sh failed (code {rc})" + color = _GREEN if rc == 0 else _RED + if rc == 0: + self.prole_cfg_data.setdefault("Security", {})["STATUS"] = "Secured" + except Exception as exc: + msg = f"Error: {exc}" + color = _RED + if _update_label: + self.bg_canvas.after(0, lambda: self.bg_canvas.itemconfigure( + _update_label, text=msg, fill=color + )) + + _th.Thread(target=_worker, daemon=True).start() + + update_btn = tk.Button( + self.bg_canvas, + text="Rotate Passwords Now", + command=_run_update_sh, + bg="#007aff", fg="white", + activebackground="#0051d0", + highlightbackground="#007aff", + highlightthickness=0, + relief="flat", + font=("SF Pro Text", 11), + padx=14, pady=7, + ) + btn_win = self.bg_canvas.create_window(48, y_row + 8, window=update_btn, anchor="nw") + self._canvas_items.append(btn_win) + self._overlay_widgets.append(update_btn) + + self._sec_update_label = ui.canvas_text( + self, 220, y_row + 14, "", fill=_GRAY, font=("SF Pro Text", 11), anchor="nw", + ) + self._canvas_items.append(self._sec_update_label) + + # --- Section B: Service links --- + y_links = y_row + 55 + ui.canvas_text( + self, 48, y_links, "Service Links", + fill="#1d1d1f", font=("SF Pro Text", 13, "bold"), + ) + y_links += 28 + + # Resolve Kong external IP (non-blocking best-effort) + app_ctx = (self.prole_cfg_data.get("Global", {}) or {}).get( + "APP_CLUSTER_KUBECONTEXT", "" + ).strip() + kong_ip = "" + try: + _res = subprocess.run( + [ + "kubectl", + *(["--context", app_ctx] if app_ctx else []), + "get", "svc", "-n", "knoe-system", + "-l", "app.kubernetes.io/name=kong", + "-o", "jsonpath={.items[0].status.loadBalancer.ingress[0].ip}", + ], + capture_output=True, text=True, timeout=8, + ) + if _res.returncode == 0 and _res.stdout.strip(): + kong_ip = _res.stdout.strip() + except Exception: + pass + + db_ns = (self.prole_cfg_data.get("Global", {}) or {}).get( + "DATABASE_NAMESPACE", "knoe-db-0" + ) + + _links = [ + ("Grafana", f"http://{kong_ip}/grafana" if kong_ip != "" else "http://localhost:3000"), + ("Prometheus", f"http://{kong_ip}/prometheus" if kong_ip != "" else "http://localhost:9090"), + ("Gateway", "http://svc.knoe.dev (DNS pending)"), + ("PostgreSQL", f"knoe-db-rw.{db_ns}.svc.cluster.local:5432 (cluster-internal)"), + ] + + for link_label, link_url in _links: + ui.canvas_text( + self, 48, y_links, f"{link_label}:", + fill="#1d1d1f", font=("SF Pro Text", 12, "bold"), anchor="nw", + ) + is_http = link_url.startswith("http://") or link_url.startswith("https://") + _url = link_url + link_item = ui.render_link( + self, 160, y_links, link_url, + color="#0a84ff" if is_http else _GRAY, + font=("SF Pro Text", 12, "underline" if is_http else "normal"), + ) + + if is_http: + def _make_opener(url=_url, item=link_item): + def _on_click(event): + bbox = self.bg_canvas.bbox(item) + if bbox and bbox[0] <= event.x <= bbox[2] and bbox[1] <= event.y <= bbox[3]: + webbrowser.open(url) + return _on_click + self.bg_canvas.bind("", _make_opener(), add="+") + + y_links += 28 + + # Kong IP info line + ui.canvas_text( + self, 48, y_links + 4, + f"Kong external IP: {kong_ip}", + fill=_GRAY, font=("SF Pro Text", 11), + ) + + # --- Footer: View logs link --- + y_footer = y_links + 40 + logs_link = ui.render_link( + self, 48, y_footer, "View installation logs →", + color="#0a84ff", font=("SF Pro Text", 12, "underline"), + ) + + def _open_logs(event): + bbox = self.bg_canvas.bbox(logs_link) + if bbox and bbox[0] <= event.x <= bbox[2] and bbox[1] <= event.y <= bbox[3]: + self.show_page("create_installer") + self.bg_canvas.bind("", _open_logs, add="+") + def _get_prole_dist_dir(self) -> Path: return PROJECT_ROOT / "prole-tools-app" / "dist" diff --git a/scan/network_description.txt b/scan/network_description.txt index 01963f7..a0d8da9 100644 --- a/scan/network_description.txt +++ b/scan/network_description.txt @@ -1,53 +1,54 @@ Network Discovery Summary: -Primary Router: 10.0.0.1 (eero_5d:50:f2) -DNS Servers: 100.100.100.100, 10.0.0.4, 10.0.0.5 +Primary Router: 10.0.0.1 (Unknown) +DNS Servers: 100.100.100.100, 10.0.0.5, 10.0.0.4 Detected Devices: - 10.0.0.203 [00:11:32:3b:2f:08] (Synology_3b:2f:08): Ports [22, 80, 443, 2049, 445], Services: ['SSH', 'NFS', 'Web Server', 'SMB/CIFS (Possible Windows/AD)'] - 10.0.0.39 [7e:19:9a:fe:9f:ed] (7e:19:9a:fe:9f:ed): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related'] -- 10.0.0.7 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH'] -- 10.0.0.4 [b8:27:eb:b3:6f:2b] (RaspberryPiF_b3:6f:2b): Ports [22, 53, 443, 2049], Services: ['DNS', 'SSH', 'NFS', 'Web Server'] -- 10.0.0.6 [2c:cf:67:45:8d:97] (RaspberryPi_45:8d:97): Ports [22, 80, 443], Services: ['SSH', 'Web Server'] -- 10.0.0.26 [c8:db:26:08:52:f8] (Logitech_08:52:f8): Ports [], Services: [] -- 10.0.0.32 [48:a6:b8:a4:8e:cc] (Sonos_a4:8e:cc): Ports [], Services: [] -- 10.0.0.3 [2c:cf:67:8c:2b:47] (RaspberryPi_8c:2b:47): Ports [22, 53, 80, 443, 2049, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] -- 10.0.0.1 [9c:57:bc:5d:50:f2] (eero_5d:50:f2): Ports [53], Services: ['DNS'] -- 10.0.0.2 [dc:a4:ca:ea:1a:2f] (Apple_ea:1a:2f): Ports [], Services: [] -- 10.0.0.5 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC'] -- 10.0.0.205 [a8:20:66:28:12:e7] (Apple_28:12:e7): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] +- 10.0.0.188 [00:16:6c:c5:1f:54] (SamsungElect_c5:1f:54): Ports [80], Services: ['Web Server'] - 10.0.0.33 [ec:b5:fa:b0:76:e4] (PhilipsLight_b0:76:e4): Ports [80, 443], Services: ['Web Server'] -- 10.0.0.37 [54:07:7d:22:c0:b2] (Netgear_22:c0:b2): Ports [80], Services: ['Web Server'] -- 10.0.0.44 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: [] -- 10.0.0.45 [f8:b4:6a:30:a4:1f] (HewlettPacka_30:a4:1f): Ports [80, 443], Services: ['Web Server'] -- 10.0.0.48 [0c:fe:45:53:f4:3c] (SonyInteract_53:f4:3c): Ports [], Services: [] -- 10.0.0.36 [7e:46:74:2a:b0:b2] (7e:46:74:2a:b0:b2): Ports [], Services: [] -- 10.0.0.52 [2c:cf:67:ca:93:ce] (RaspberryPi_ca:93:ce): Ports [22], Services: ['SSH'] -- 10.0.0.46 [d4:f7:d5:40:ab:17] (SonyInteract_40:ab:17): Ports [], Services: [] -- 10.0.0.111 [ca:0b:6c:5d:0a:e5] (ca:0b:6c:5d:0a:e5): Ports [], Services: [] -- 10.0.0.128 [20:c9:d0:94:30:dd] (Apple_94:30:dd): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] -- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: [] -- 10.0.0.94 [f0:20:ff:bf:9a:19] (Intel_bf:9a:19): Ports [], Services: [] - Note: Potential host for NVIDIA cards (PC Hardware vendor) -- 10.0.0.95 [b8:27:eb:88:3a:41] (RaspberryPiF_88:3a:41): Ports [22], Services: ['SSH'] -- 10.0.0.106 [48:a6:b8:a7:50:60] (Sonos_a7:50:60): Ports [], Services: [] -- 10.0.0.123 [3c:ef:8c:96:e3:3c] (ZhejiangDahu_96:e3:3c): Ports [80], Services: ['Web Server'] -- 10.0.0.124 [9c:8e:cd:02:73:c4] (AmcrestTechn_02:73:c4): Ports [80], Services: ['Web Server'] -- 10.0.0.113 [60:81:10:92:02:4e] (Apple_92:02:4e): Ports [], Services: [] -- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: [] -- 10.0.0.125 [a0:60:32:04:73:ba] (AmcrestTechn_04:73:ba): Ports [80], Services: ['Web Server'] -- 10.0.0.127 [58:55:ca:4a:60:6e] (Apple_4a:60:6e): Ports [22], Services: ['SSH'] -- 10.0.0.130 [f8:bb:bf:a3:84:eb] (eero_a3:84:eb): Ports [], Services: [] -- 10.0.0.145 [7c:a6:b0:04:db:1c] (7c:a6:b0:04:db:1c): Ports [], Services: [] -- 10.0.0.143 [60:5f:8d:7a:aa:32] (eero_7a:aa:32): Ports [53], Services: ['DNS'] -- 10.0.0.155 [62:17:97:b3:06:a1] (62:17:97:b3:06:a1): Ports [], Services: [] -- 10.0.0.170 [b0:ee:7b:ca:73:99] (Roku_ca:73:99): Ports [], Services: [] -- 10.0.0.204 [50:eb:f6:56:f3:26] (ASUSTekCOMPU_56:f3:26): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama'] -- 10.0.0.175 [28:80:88:e4:10:1a] (Netgear_e4:10:1a): Ports [80], Services: ['Web Server'] +- 10.0.0.39\ [7e:19:9a:fe:9f:ed] (7e:19:9a:fe:9f:ed): Ports [], Services: [] - 10.0.0.189 [00:17:88:a3:2f:cc] (PhilipsLight_a3:2f:cc): Ports [80, 443], Services: ['Web Server'] -- 10.0.0.188 [00:16:6c:c5:1f:54] (SamsungElect_c5:1f:54): Ports [80, 443], Services: ['Web Server'] -- 10.0.0.196 [60:5f:8d:88:08:f2] (eero_88:08:f2): Ports [53], Services: ['DNS'] -- 10.0.0.206 [00:a0:de:a2:0b:ef] (Yamaha_a2:0b:ef): Ports [80], Services: ['Web Server'] +- 10.0.0.73 [4c:a9:19:b3:12:f8] (TuyaSmart_b3:12:f8): Ports [], Services: [] +- 10.0.0.107 [b8:06:0d:b7:7c:56] (TuyaSmart_b7:7c:56): Ports [], Services: [] +- 10.0.0.47 [c6:d0:1a:97:9f:e9] (c6:d0:1a:97:9f:e9): Ports [], Services: [] +- 10.0.0.44 [24:fc:e5:51:cf:74] (SamsungElect_51:cf:74): Ports [], Services: [] +- 10.0.0.1 [Unknown] (Unknown): Ports [53], Services: ['DNS'] +- 10.0.0.2 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.3 [Unknown] (Unknown): Ports [22, 53, 80, 443, 2049, 445, 5900, 88, 389, 636], Services: ['DNS', 'SSH', 'NFS', 'VNC', 'Web Server', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] +- 10.0.0.4 [Unknown] (Unknown): Ports [22, 53, 443, 2049], Services: ['DNS', 'SSH', 'NFS', 'Web Server'] +- 10.0.0.5 [Unknown] (Unknown): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC'] +- 10.0.0.6 [Unknown] (Unknown): Ports [22, 80, 443], Services: ['SSH', 'Web Server'] +- 10.0.0.7 [Unknown] (Unknown): Ports [22], Services: ['SSH'] - 10.0.0.22 [Unknown] (Unknown): Ports [22], Services: ['SSH'] -Ollama Instances found at: 10.0.0.39, 10.0.0.204 +- 10.0.0.26 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.32 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.37 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.45 [Unknown] (Unknown): Ports [80, 443], Services: ['Web Server'] +- 10.0.0.46 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.48 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.50 [Unknown] (Unknown): Ports [22, 445, 5900, 11434, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Ollama', 'Active Directory Related'] +- 10.0.0.52 [Unknown] (Unknown): Ports [22], Services: ['SSH'] +- 10.0.0.95 [Unknown] (Unknown): Ports [22, 2049, 5900], Services: ['SSH', 'NFS', 'VNC'] +- 10.0.0.106 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.111 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.112 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.113 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.123 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.124 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.125 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.127 [Unknown] (Unknown): Ports [22], Services: ['SSH'] +- 10.0.0.128 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] +- 10.0.0.143 [Unknown] (Unknown): Ports [53], Services: ['DNS'] +- 10.0.0.145 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.170 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.175 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.196 [Unknown] (Unknown): Ports [53], Services: ['DNS'] +- 10.0.0.199 [Unknown] (Unknown): Ports [], Services: [] +- 10.0.0.204 [Unknown] (Unknown): Ports [22, 3389, 445, 5900, 11434], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'RDP (Windows)', 'Ollama'] +- 10.0.0.205 [Unknown] (Unknown): Ports [22, 445, 5900, 88], Services: ['SSH', 'VNC', 'SMB/CIFS (Possible Windows/AD)', 'Active Directory Related'] +- 10.0.0.206 [Unknown] (Unknown): Ports [80], Services: ['Web Server'] +- 10.0.0.155 [62:17:97:b3:06:a1] (62:17:97:b3:06:a1): Ports [], Services: [] +Ollama Instances found at: 10.0.0.39, 10.0.0.50, 10.0.0.204 Ansible Inventory Summary: Inventory directory: /Users/chrisfu/dev/prole/infrastructure diff --git a/tmp/check_and_cleanup.sh b/tmp/check_and_cleanup.sh new file mode 100644 index 0000000..77440ec --- /dev/null +++ b/tmp/check_and_cleanup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Deleting minio-create-buckets jobs (buckets already exist)..." +kubectl -n gitea delete jobs -l app=minio-create-buckets --ignore-not-found 2>&1 + +echo "==> Webservice kgqhl status:" +kubectl -n gitea get pod gitlab-webservice-default-675ccb7b99-kgqhl \ + -o custom-columns='NAME:.metadata.name,READY:.status.containerStatuses[*].ready,NODE:.spec.nodeName' 2>&1 + +echo "==> Webservice kgqhl puma log (last 5):" +kubectl -n gitea logs gitlab-webservice-default-675ccb7b99-kgqhl \ + -c webservice --tail=5 2>&1 | grep -v LD_PRELOAD | head -8 || true + +echo "==> Sidekiq pending reason:" +kubectl -n gitea describe pod gitlab-sidekiq-all-in-1-v2-679848bc8-wvxln 2>&1 \ + | grep -E "Events:|Warning|Error|Insufficient|fit" | head -8 + +echo "==> Force-delete extra Pending webservice pod..." +kubectl -n gitea delete pod gitlab-webservice-default-675ccb7b99-x56sq \ + --force --grace-period=0 --ignore-not-found 2>&1 + +echo "==> HPA current state:" +kubectl -n gitea get hpa 2>&1 | grep -E "NAME|webservice|sidekiq|kas|shell|registry" + +echo "==> All pods (wide):" +kubectl -n gitea get pods --no-headers -o wide \ + | grep -v jemalloc | grep -v Completed \ + | awk '{printf "%-52s %-8s %-20s %s\n",$1,$4,$7,$8}' | sort diff --git a/tmp/check_and_fix_dirs.sh b/tmp/check_and_fix_dirs.sh new file mode 100644 index 0000000..39c9558 --- /dev/null +++ b/tmp/check_and_fix_dirs.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Verify dirs on gandalf..." +kubectl -n gitlab run dircheck \ + --image=alpine:latest --restart=Never --rm --attach \ + --overrides='{ + "spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}], + "volumes":[{"name":"syn","hostPath":{"path":"/synology/d005","type":"Directory"}}], + "containers":[{"name":"dircheck","image":"alpine:latest", + "command":["sh","-c","ls -la /syn/gitlab/ 2>&1 || echo DIR_MISSING; mkdir -p /syn/gitlab/minio /syn/gitlab/gitaly && chmod 777 /syn/gitlab/minio /syn/gitlab/gitaly && echo CREATED && ls -la /syn/gitlab/"], + "volumeMounts":[{"name":"syn","mountPath":"/syn"}]}]}}' 2>&1 + +echo "==> Force rescheduling minio pod..." +kubectl -n gitlab delete pod gitlab-minio-5459cf9679-7nv6c --ignore-not-found + +echo "==> Waiting 60s..." +sleep 60 + +echo "==> Status:" +kubectl -n gitlab get pods --sort-by=.metadata.creationTimestamp | grep -v jemalloc-builder diff --git a/tmp/check_buckets_and_pods.sh b/tmp/check_buckets_and_pods.sh new file mode 100644 index 0000000..d236dad --- /dev/null +++ b/tmp/check_buckets_and_pods.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> minio-init pod status:" +kubectl -n gitea get pod minio-init --no-headers 2>&1 + +echo "==> minio-init logs:" +kubectl -n gitea logs minio-init 2>&1 | tail -20 + +echo "==> Restarting registry (pick up new buckets):" +kubectl -n gitea delete pod -l app=registry --ignore-not-found 2>&1 + +echo "==> Waiting 60s..." +sleep 60 + +echo "==> All GitLab pods:" +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder | sort + +echo "==> GitLab CR status:" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' 2>&1 && echo + +echo "==> Installer log tail:" +tail -5 /Users/chrisfu/dev/prole/logs/install_run8.log 2>&1 diff --git a/tmp/check_gitlab_status.sh b/tmp/check_gitlab_status.sh new file mode 100644 index 0000000..bebcc22 --- /dev/null +++ b/tmp/check_gitlab_status.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Gitaly PVC:" +kubectl -n gitea get pvc repo-data-gitlab-gitaly-0 2>&1 + +echo "==> Gitaly pod events:" +kubectl -n gitea describe pod gitlab-gitaly-0 2>&1 | grep -A3 "Events:" | tail -8 + +echo "==> PV status:" +kubectl get pv gitlab-gitaly-synology gitlab-minio-synology \ + -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CLAIM:.spec.claimRef.name' 2>&1 + +echo "==> Registry crash log:" +kubectl -n gitea logs gitlab-registry-75b586f9fd-cw8gl --previous --tail=15 2>&1 \ + | grep -v "LD_PRELOAD" | head -15 || true + +echo "==> Exporter crash log:" +kubectl -n gitea logs gitlab-gitlab-exporter-54575ddcdb-lcprm --previous --tail=10 2>&1 \ + | grep -v "LD_PRELOAD" | head -10 || true diff --git a/tmp/check_jemalloc_exists.sh b/tmp/check_jemalloc_exists.sh new file mode 100644 index 0000000..8f32c68 --- /dev/null +++ b/tmp/check_jemalloc_exists.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +for node in gandalf.prole.org merlin.prole.org myrddin.prole.org pi.prole.org; do + safe="${node//./-}" + echo -n "==> $node: " + kubectl -n gitlab run "jv2-${safe}" \ + --image=ubuntu:22.04 --restart=Never --rm --attach --quiet \ + --overrides="{ + \"spec\":{ + \"nodeName\":\"${node}\", + \"tolerations\":[{\"operator\":\"Exists\"}], + \"volumes\":[{\"name\":\"jlib\",\"hostPath\":{\"path\":\"/opt/gitlab-jemalloc\",\"type\":\"DirectoryOrCreate\"}}], + \"containers\":[{ + \"name\":\"c\", + \"image\":\"ubuntu:22.04\", + \"command\":[\"bash\",\"-c\", + \"test -f /jlib/libjemalloc.so.2 && ls -lh /jlib/libjemalloc.so.2 || echo MISSING\"], + \"volumeMounts\":[{\"name\":\"jlib\",\"mountPath\":\"/jlib\"}] + }] + }}" 2>&1 || true +done diff --git a/tmp/check_readiness.sh b/tmp/check_readiness.sh new file mode 100644 index 0000000..2a6d02a --- /dev/null +++ b/tmp/check_readiness.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Webservice puma latest log (kgqhl):" +kubectl -n gitea logs gitlab-webservice-default-675ccb7b99-kgqhl \ + -c webservice --tail=8 2>&1 | grep -v LD_PRELOAD || true + +echo "==> Sidekiq latest log:" +kubectl -n gitea logs gitlab-sidekiq-all-in-1-v2-679848bc8-wvxln \ + --tail=10 2>&1 | grep -v LD_PRELOAD | grep -v "^$" | head -10 || true + +echo "==> Available condition:" +kubectl -n gitea get gitlab gitlab \ + -o jsonpath='{range .status.conditions[*]}{.type}: {.status} — {.message}{"\n"}{end}' 2>&1 + +echo "==> minio-create-buckets crash image + log:" +kubectl -n gitea get pod -l app=minio-create-buckets \ + -o jsonpath='{.items[0].spec.containers[0].image}' 2>&1 && echo +kubectl -n gitea logs -l app=minio-create-buckets \ + --tail=5 2>&1 | grep -v LD_PRELOAD | head -5 || true diff --git a/tmp/check_registry.sh b/tmp/check_registry.sh new file mode 100644 index 0000000..6c7b701 --- /dev/null +++ b/tmp/check_registry.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Registry crash log:" +kubectl -n gitea logs gitlab-registry-75b586f9fd-455x2 --previous --tail=20 2>&1 \ + | grep -v "LD_PRELOAD" | head -20 || \ +kubectl -n gitea logs -l app=registry --tail=20 2>&1 \ + | grep -v "LD_PRELOAD" | head -20 || true + +echo "==> All pods incl webservice/sidekiq:" +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder | sort + +echo "==> Migrations status:" +kubectl -n gitea get jobs --no-headers 2>&1 | grep migrations || \ +kubectl -n gitea get pods --no-headers | grep migrations + +echo "==> Installer run 9 tail:" +tail -8 /Users/chrisfu/dev/prole/logs/install_run9.log 2>&1 diff --git a/tmp/clean_and_restart_final.sh b/tmp/clean_and_restart_final.sh new file mode 100644 index 0000000..84ef4fb --- /dev/null +++ b/tmp/clean_and_restart_final.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +cd /Users/chrisfu/dev/prole + +echo "==> Killing any running installer..." +pkill -f "install.sh" 2>/dev/null || true +pkill -f "init_gitlab.sh" 2>/dev/null || true +sleep 3 + +echo "==> Deleting PVs with old claimRefs..." +for pv in gitlab-gitaly-synology gitlab-minio-synology; do + kubectl patch pv "$pv" -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true + kubectl delete pv "$pv" --ignore-not-found 2>/dev/null || true +done + +echo "==> Wiping synology gitlab dirs..." +kubectl -n knoe-system run wipe-gitlab --image=alpine:latest --restart=Never --rm --attach \ + --overrides='{"spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}],"volumes":[{"name":"s","hostPath":{"path":"/synology/d005","type":"Directory"}}],"containers":[{"name":"w","image":"alpine:latest","command":["sh","-c","rm -rf /s/gitlab && echo wiped"],"volumeMounts":[{"name":"s","mountPath":"/s"}]}]}}' 2>&1 || true + +echo "==> Removing stale PID..." +rm -f install.pid + +echo "==> Verifying namespace fix in prole.cfg..." +grep "gitops.gitlab_namespace" conf/service/prole.cfg + +echo "==> Verifying init_gitlab.sh lint..." +bash -n etc/init_gitlab.sh && echo " syntax OK" + +echo "==> Launching installer..." +mkdir -p logs +nohup bash install.sh --silent -c conf/service/prole.cfg > logs/install_final4.log 2>&1 & +disown +echo "Installer started. Monitor: tail -f logs/install_final4.log" diff --git a/tmp/clean_and_run.sh b/tmp/clean_and_run.sh new file mode 100644 index 0000000..858f717 --- /dev/null +++ b/tmp/clean_and_run.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +cd /Users/chrisfu/dev/prole + +echo "==> Cleaning leftover gitea namespace..." && \ +kubectl get namespace gitea -o json 2>/dev/null \ + | python3 -c "import sys,json; d=json.load(sys.stdin); d['spec']['finalizers']=[]; print(json.dumps(d))" \ + | kubectl replace --raw /api/v1/namespaces/gitea/finalize -f - 2>/dev/null || true && \ +kubectl delete namespace gitea --ignore-not-found --timeout=30s 2>/dev/null || true + +echo "==> Releasing/deleting old PVs..." && \ +for pv in gitlab-gitaly-synology gitlab-minio-synology; do + kubectl patch pv "$pv" -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true + kubectl delete pv "$pv" --ignore-not-found 2>/dev/null || true +done + +echo "==> Wiping synology gitlab dirs for fresh start..." && \ +kubectl -n knoe-system run gitlab-wipe --image=alpine:latest --restart=Never --rm --attach \ + --overrides='{"spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}],"volumes":[{"name":"s","hostPath":{"path":"/synology/d005","type":"Directory"}}],"containers":[{"name":"w","image":"alpine:latest","command":["sh","-c","rm -rf /s/gitlab && echo wiped"],"volumeMounts":[{"name":"s","mountPath":"/s"}]}]}}' 2>&1 || true + +echo "==> Cleaning leftover ClusterRoles from old installs..." && \ +kubectl get clusterrole,clusterrolebinding -o name 2>/dev/null \ + | grep -i gitlab | xargs kubectl delete --ignore-not-found 2>&1 || true + +echo "==> jemalloc already built on all nodes (reusing)..." && \ +ls /opt/gitlab-jemalloc/ 2>/dev/null || true + +echo "==> Removing stale PID lock..." && \ +rm -f /Users/chrisfu/dev/prole/install.pid + +echo "==> Cluster node status:" && \ +kubectl get nodes + +echo "==> Launching install.sh pipeline (gitlab namespace, gandalf storage)..." && \ +mkdir -p logs && \ +nohup bash install.sh --silent -c conf/service/prole.cfg \ + > logs/install_final.log 2>&1 & disown && \ +echo "Installer started — tail logs/install_final.log to monitor" diff --git a/tmp/clean_slate.sh b/tmp/clean_slate.sh new file mode 100644 index 0000000..a85d786 --- /dev/null +++ b/tmp/clean_slate.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Deleting GitLab CR (lets operator clean up managed resources)..." +kubectl -n gitlab delete gitlab gitlab --ignore-not-found --timeout=60s 2>&1 || true + +echo "==> Waiting 20s for operator cleanup..." +sleep 20 + +echo "==> Deleting gitlab namespace (all remaining resources)..." +kubectl delete namespace gitlab --ignore-not-found --timeout=120s 2>&1 || true + +echo "==> Deleting jemalloc-builder (if in different ns)..." +kubectl delete daemonset jemalloc-builder -n gitlab --ignore-not-found 2>/dev/null || true + +echo "==> Releasing bound/retained PVs for gitlab..." +for pv in gitlab-gitaly-synology gitlab-minio-synology; do + kubectl patch pv "$pv" -p '{"spec":{"claimRef":null}}' 2>/dev/null || true + kubectl delete pv "$pv" --ignore-not-found 2>/dev/null || true +done + +echo "==> Wiping synology data dirs on gandalf (fresh start)..." +kubectl -n knoe-system run gitlab-wipe \ + --image=alpine:latest --restart=Never --rm --attach \ + --overrides='{ + "spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}], + "volumes":[{"name":"s","hostPath":{"path":"/synology/d005","type":"Directory"}}], + "containers":[{"name":"gitlab-wipe","image":"alpine:latest", + "command":["sh","-c","rm -rf /s/gitlab && echo wiped"], + "volumeMounts":[{"name":"s","mountPath":"/s"}]}]}}' 2>&1 || true + +echo "==> Verifying gitlab namespace is gone..." +kubectl get ns gitlab 2>&1 || echo "namespace deleted OK" + +echo "==> Clean slate complete." diff --git a/tmp/cleanup_and_restart.sh b/tmp/cleanup_and_restart.sh new file mode 100644 index 0000000..0a28119 --- /dev/null +++ b/tmp/cleanup_and_restart.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Removing finalizers + deleting gitea namespace..." +for res in $(kubectl -n gitea get gitlabs.apps.gitlab.com --no-headers -o name 2>/dev/null || true); do + kubectl -n gitea patch "$res" -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true +done +kubectl get namespace gitea -o json 2>/dev/null \ + | python3 -c "import sys,json; d=json.load(sys.stdin); d['spec']['finalizers']=[]; print(json.dumps(d))" \ + | kubectl replace --raw /api/v1/namespaces/gitea/finalize -f - 2>&1 || true +kubectl delete namespace gitea --ignore-not-found --timeout=30s 2>&1 || true + +echo "==> Releasing/deleting gitlab PVs..." +for pv in gitlab-gitaly-synology gitlab-minio-synology; do + kubectl patch pv "$pv" -p '{"spec":{"claimRef":null}}' 2>/dev/null || true + kubectl delete pv "$pv" --ignore-not-found 2>/dev/null || true +done + +echo "==> Wiping synology gitlab dirs..." +kubectl -n knoe-system run gitlab-wipe \ + --image=alpine:latest --restart=Never --rm --attach \ + --overrides='{"spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}],"volumes":[{"name":"s","hostPath":{"path":"/synology/d005","type":"Directory"}}],"containers":[{"name":"w","image":"alpine:latest","command":["sh","-c","rm -rf /s/gitlab && echo wiped"],"volumeMounts":[{"name":"s","mountPath":"/s"}]}]}}' 2>&1 || true + +echo "==> Cleaning leftover ClusterRoles/Bindings from gitlab operator..." +kubectl get clusterrole,clusterrolebinding -o name 2>/dev/null \ + | grep -i gitlab | xargs kubectl delete --ignore-not-found 2>&1 || true + +echo "==> Waiting for pi.prole.org to become Ready (up to 5min)..." +for i in $(seq 1 30); do + status=$(kubectl get node pi.prole.org --no-headers 2>/dev/null | awk '{print $2}') + echo " [$i/30] pi status: $status" + [[ "$status" == "Ready" ]] && break + sleep 10 +done + +echo "==> Final node status:" +kubectl get nodes + +echo "==> Clean-up done. Launching installer..." +cd /Users/chrisfu/dev/prole +mkdir -p logs +nohup bash install.sh --silent -c conf/service/prole.cfg \ + > logs/install_run6.log 2>&1 & +echo "Installer PID $! — tail logs/install_run6.log to monitor" diff --git a/tmp/clear_old_pods.sh b/tmp/clear_old_pods.sh new file mode 100644 index 0000000..f5789eb --- /dev/null +++ b/tmp/clear_old_pods.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +for pod in gitlab-webservice-default-675ccb7b99-9tkdh \ + gitlab-webservice-default-675ccb7b99-j684x \ + gitlab-sidekiq-all-in-1-v2-679848bc8-mrz9b; do + kubectl -n gitea delete pod "$pod" --force --grace-period=0 --ignore-not-found 2>/dev/null || true +done && \ +sleep 20 && \ +kubectl -n gitea get pods --no-headers | grep -E "webservice|sidekiq" diff --git a/tmp/commit_msg.txt b/tmp/commit_msg.txt new file mode 100644 index 0000000..b8c1eae --- /dev/null +++ b/tmp/commit_msg.txt @@ -0,0 +1,73 @@ +feat: full GKE/prod deployment pipeline from UI to Artifact Registry + +## GCP / Cluster Environment Screen +- Auto-populate Cloud tab from conf/prod/gcp.cfg on screen open (org_id, + billing_account, billing_project, project_id) +- gcloud auth validity checked on screen startup; friendly modal dialog + streams gcloud auth login output live so user never leaves the app +- Live GKE cluster browser: fetches clusters via gcloud container clusters + list, displays with checkmark selector, auto-selects saved cluster +- Selecting a cluster runs get-credentials, sets KUBECONFIG/KUBECONTEXT, + and syncs the region dropdown to the selected cluster's location +- Region dropdown populated live from gcloud compute regions list with + checkmark on currently selected region; graceful fallback when offline +- New 'GCP Storage' tab with workload->StorageClass mapping (CNPG->premium-rwo, + Redis/Monitoring->standard-rwo, Garage->garage-hdd) and Fetch from Cluster +- Provider readonly field styled correctly (no solid-black on macOS) +- Stale prole.cfg/conf/prole.cfg symlinks removed; all config I/O now + resolves env-specific paths via prole_conf.entrypoint_path() + +## GKE Autopilot Compatibility (Common Services) +- Synology iSCSI StorageClass and static PVs guarded behind PROLE_MODE!=k8s + in init_openbao.sh (GKE Autopilot forbids hostPath/iSCSI volumes) +- In-cluster Docker registry (hostPath) skipped in k8s mode; GCP Artifact + Registry used instead +- Kong renamed knoe-svc-kong in k8s mode; all health-check kubectl calls in + init_common_services.sh and status_common_services.sh updated accordingly +- DNS endpoints switched from *.prole.org to *.knoe.dev in k8s mode + (api.knoe.dev, git.knoe.dev, svc.knoe.dev); ingress uses gce class +- New GKE-clean Kong manifests under deploy/opentofu/k8s/manifests/prole/: + no k3s node affinity, explicit Autopilot resource requests/limits + +## Garage S3 Store (GKE) +- New garage-statefulset-gcp.yaml targeting garage-hdd StorageClass + (pd-standard, avoids SSD_TOTAL_GB quota exhaustion in us-west3) +- New storageclass-gcp-hdd.yaml (pd-standard, Retain, WaitForFirstConsumer) +- GCP StorageClass manifests skipped on re-runs (Autopilot built-ins are + immutable; skip-if-exists guard added) +- PVC deletion guard extended to cover any storageClass (not just synology) + so stale claims are cleaned before StatefulSet recreation + +## Topology (GKE Autopilot) +- DaemonSet collector skipped in prod mode (forbidden in kube-system by + GKE Warden); Kubernetes-only node facts path used instead +- All ready GKE nodes assumed cnpg-eligible and monitoring-eligible without + taint/synology-mount checks (skip_collector + assume_nodes_eligible flags) + +## KUBECONFIG / kubectl (k8s mode) +- actions.py: new elif mode==k8s branch sets KUBECONFIG=~/.kube/config + and injects KUBECONTEXT from prole_cfg_data into script env +- _build_kubectl_cmd falls back to Global.KUBECONTEXT when + init_cluster.selected_kubectx is empty +- _activate_selected_gke_cluster persists KUBECONFIG/KUBECONTEXT to + prole_cfg_data and saves prole.cfg immediately after get-credentials + +## Database Build Screen (GKE) +- Registry display shows correct Artifact Registry URL + (-docker.pkg.dev///knoe-db) in green +- Build+push: gcloud auth configure-docker, auto-creates AR repository + named after SERVICE_NAMESPACE (e.g. knoe-system) if missing, then + docker tag + push; falls back to gcr.io if region unavailable +- GCP config loaded from conf/prod/gcp.cfg on every screen entry; + keys normalised to lowercase so project_id lookup is always consistent + +## Config / Namespace persistence +- prole_conf.py activate_environment: symlink creation removed; sets + CLUSTER_ENV env-var so all subsequent calls resolve correct env directory +- knoe/ui/screens/__init__.py: startup config load uses entrypoint_path() + instead of hardcoded conf/prole.cfg; seeds SERVICE_NAMESPACE=knoe-system + for managed envs so Common Services never defaults to 'default' +- cfg.py _save_prole_cfg: saves to env-specific path via entrypoint_path() +- etc/prole_cfg.sh: removed all ln -snf symlink creation + +Co-authored-by: Junie diff --git a/tmp/create_buckets_dockerhub.sh b/tmp/create_buckets_dockerhub.sh new file mode 100644 index 0000000..e95ecb8 --- /dev/null +++ b/tmp/create_buckets_dockerhub.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +AK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.accesskey}' | base64 -d) +SK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.secretkey}' | base64 -d) + +echo "==> Checking gitaly pod logs..." +kubectl -n gitea logs gitlab-gitaly-0 --previous --tail=15 2>&1 | grep -v LD_PRELOAD | head -15 || \ +kubectl -n gitea logs gitlab-gitaly-0 --tail=15 2>&1 | grep -v LD_PRELOAD | head -15 || true + +echo "==> Removing stale installer PID and restarting installer..." +rm -f /Users/chrisfu/dev/prole/install.pid +cd /Users/chrisfu/dev/prole && mkdir -p logs +nohup bash install.sh --silent -c conf/service/prole.cfg > logs/install_run9.log 2>&1 & +disown +echo "Installer run 9 started" + +echo "==> Creating minio buckets via Docker Hub minio/mc..." +kubectl -n gitea delete pod minio-init --ignore-not-found 2>/dev/null || true +kubectl -n gitea run minio-init \ + --image=minio/mc:latest \ + --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"gandalf.prole.org\",\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"minio-init\",\"image\":\"minio/mc:latest\",\"command\":[\"sh\",\"-c\",\"mc alias set gl http://gitlab-minio-svc.gitea.svc.cluster.local:9000 ${AK} ${SK} && for b in gitlab-registry-storage gitlab-lfs-storage gitlab-artifacts-storage gitlab-uploads-storage gitlab-packages-storage gitlab-dependency-proxy-storage gitlab-terraform-state gitlab-ci-secure-files; do mc mb --ignore-existing gl/\$b && echo created \$b; done && mc ls gl && echo BUCKETS_DONE\"]}]}}" 2>&1 +echo "minio-init pod created. Check with: kubectl -n gitea logs minio-init -f" diff --git a/tmp/create_synology_dirs.sh b/tmp/create_synology_dirs.sh new file mode 100644 index 0000000..fe092da --- /dev/null +++ b/tmp/create_synology_dirs.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Creating /synology/d005/gitlab/{minio,gitaly} on gandalf.prole.org..." +kubectl -n gitlab run synology-mkdir \ + --image=alpine:latest \ + --restart=Never \ + --rm --attach \ + --overrides='{ + "spec":{ + "nodeName":"gandalf.prole.org", + "tolerations":[{"operator":"Exists"}], + "volumes":[{"name":"synology","hostPath":{"path":"/synology/d005","type":"Directory"}}], + "containers":[{ + "name":"synology-mkdir", + "image":"alpine:latest", + "command":["sh","-c", + "mkdir -p /synology/gitlab/minio /synology/gitlab/gitaly && chmod 777 /synology/gitlab/minio /synology/gitlab/gitaly && ls -la /synology/gitlab/"], + "volumeMounts":[{"name":"synology","mountPath":"/synology"}] + }] + }}' 2>&1 + +echo "==> Directories created. Waiting 30s for kubelet to retry mount..." +sleep 30 + +echo "==> Minio pod status:" +kubectl -n gitlab get pod gitlab-minio-5459cf9679-7nv6c diff --git a/tmp/debug_minio.sh b/tmp/debug_minio.sh new file mode 100644 index 0000000..d7f9a6f --- /dev/null +++ b/tmp/debug_minio.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Waiting for minio restart and capturing startup logs..." +kubectl -n gitlab logs gitlab-minio-65b8ddcd4d-5xlgc -f --tail=30 2>&1 | \ + grep -v "LD_PRELOAD" | timeout 30 head -40 || true + +echo "==> Export dir contents on gandalf:" +kubectl -n gitlab run minio-inspect \ + --image=alpine:latest --restart=Never --rm --attach \ + --overrides='{ + "spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}], + "volumes":[{"name":"export","hostPath":{"path":"/synology/d005/gitlab/minio","type":"Directory"}}], + "containers":[{"name":"minio-inspect","image":"alpine:latest", + "command":["sh","-c","ls -laR /export/ 2>&1 | head -30 && du -sh /export/ 2>&1"], + "volumeMounts":[{"name":"export","mountPath":"/export"}]}]}}' 2>&1 diff --git a/tmp/debug_registry.sh b/tmp/debug_registry.sh new file mode 100644 index 0000000..d2ac68f --- /dev/null +++ b/tmp/debug_registry.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +REG=$(kubectl -n gitea get pod -l app=registry --no-headers -o name | head -1) +echo "==> Registry pod: $REG" + +echo "==> Registry previous crash log (full):" +kubectl -n gitea logs "$REG" --previous 2>&1 | grep -v LD_PRELOAD || true + +echo "==> Minio bucket list (verify):" +kubectl -n gitea logs minio-init 2>&1 | grep -E "^\[|BUCKET|created|DONE" | head -15 + +echo "==> Registry secret bucket config:" +kubectl -n gitea get secret gitlab-registry-storage \ + -o jsonpath='{.data.config}' 2>/dev/null | base64 -d | head -20 || \ +kubectl -n gitea get configmap -l app=registry --no-headers 2>/dev/null | head -5 || true + +echo "==> Migrations pod completion status:" +kubectl -n gitea get pods --no-headers | grep migrations diff --git a/tmp/debug_registry_v2.sh b/tmp/debug_registry_v2.sh new file mode 100644 index 0000000..7f33d53 --- /dev/null +++ b/tmp/debug_registry_v2.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Registry storage secret (raw):" +kubectl -n gitea get secret gitlab-registry-storage -o jsonpath='{.data}' 2>&1 + +echo "" +echo "==> Registry storage secret keys:" +kubectl -n gitea get secret gitlab-registry-storage -o jsonpath='{.data.config}' 2>/dev/null \ + | base64 -d 2>/dev/null || true +kubectl -n gitea get secret gitlab-registry-storage -o jsonpath='{.data.s3cmd\.config}' 2>/dev/null \ + | base64 -d 2>/dev/null || true + +echo "==> All secrets with registry in name:" +kubectl -n gitea get secret --no-headers | grep -i registry | head -10 + +echo "==> Minio export dir on gandalf:" +kubectl -n gitea run minio-ls \ + --image=alpine:latest --restart=Never --rm --attach \ + --overrides='{"spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}],"volumes":[{"name":"e","hostPath":{"path":"/synology/d005/gitlab/minio","type":"Directory"}}],"containers":[{"name":"c","image":"alpine:latest","command":["sh","-c","ls -la /export/"],"volumeMounts":[{"name":"e","mountPath":"/export"}]}]}}' 2>&1 || true diff --git a/tmp/debug_registry_v3.sh b/tmp/debug_registry_v3.sh new file mode 100644 index 0000000..b04c43a --- /dev/null +++ b/tmp/debug_registry_v3.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Registry ConfigMaps:" +kubectl -n gitea get cm --no-headers | grep -i registry + +echo "==> Registry configure ConfigMap content (bucket-related lines):" +kubectl -n gitea get cm gitlab-registry -o jsonpath='{.data.configure}' 2>/dev/null \ + | grep -iE "bucket|s3|endpoint|access" | head -20 || true +kubectl -n gitea get cm -l app=registry \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{.data}{"\n"}{end}' 2>/dev/null \ + | grep -iE "bucket|s3|endpoint" | head -20 || true + +echo "==> registry-storage secret:" +kubectl -n gitea get secret gitlab-registry-secret \ + -o jsonpath='{.data}' 2>/dev/null && echo + +echo "==> Minio pod node placement:" +kubectl -n gitea get pod -l app=minio \ + -o custom-columns='POD:.metadata.name,NODE:.spec.nodeName,STATUS:.status.phase' + +echo "==> Try reading config.yml via debug container on minio node:" +kubectl -n gitea run reg-debug \ + --image=registry.gitlab.com/gitlab-org/build/cng/gitlab-container-registry:v4.38.0-gitlab \ + --restart=Never --rm --attach \ + --overrides='{"spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}],"containers":[{"name":"reg-debug","image":"alpine:latest","command":["sh","-c","echo done"]}]}}' \ + 2>&1 | head -5 || true diff --git a/tmp/diagnose_host_error.sh b/tmp/diagnose_host_error.sh new file mode 100644 index 0000000..60d5235 --- /dev/null +++ b/tmp/diagnose_host_error.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +POD="gitlab-webservice-default-675ccb7b99-gxgs8" + +echo "==> Configure init logs (look for host rendering):" +kubectl -n gitea logs "$POD" -c configure 2>&1 \ + | grep -iE "host|gitlab_host|url|allowed" | head -15 || true + +echo "==> Full CR global values (python parse):" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.spec.chart.values.global}' 2>&1 \ + | python3 -c "import sys,json; v=json.load(sys.stdin); print(json.dumps(v,indent=2))" \ + | grep -A8 '"hosts"' + +echo "==> Exec into running webservice — check generated gitlab.yml host line:" +kubectl -n gitea exec "$POD" -c webservice -- \ + grep -E "host:|url:|allowed_host" /srv/gitlab/config/gitlab.yml 2>&1 | head -10 || true diff --git a/tmp/do_commit.sh b/tmp/do_commit.sh new file mode 100644 index 0000000..a8838af --- /dev/null +++ b/tmp/do_commit.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -e +cd /Users/chrisfu/dev/prole + +git add \ + conf/cnpg-placement/ecosystem-0-knoe-db.json \ + conf/cnpg-placement/knoe-system-knoe-db.json \ + conf/dev/prole.cfg \ + conf/port-mapping.cfg \ + conf/prod/prole.cfg \ + deploy/gcp/gke/knoe-db-backup-gcs.yaml \ + deploy/gcp/gke/knoe-db.yaml \ + deploy/gcp/gke/namespace.yaml \ + deploy/gcp/gke/workload-identity.yaml \ + knoe/core/ops/cloudnative_pg.py \ + modes/k3d/knoe-db/.version \ + modes/k8s/knoe-db/.version \ + scan/network_description.txt \ + tests/installer/test_cnpg_barman_webhook.py + +git commit \ + --trailer "Co-authored-by: Junie " \ + -m "fix(cnpg): stabilize CNPG/Barman Cloud init on GKE (knoe-dev-0 / knoe-db-0) + +- cloudnative_pg.py: add _wait_for_webhook_ca_bundle() to poll ValidatingWebhookConfiguration + until cainjector has injected the CA bundle; fall back to _try_inject_ca_bundle_manually() + on timeout (reads cert-manager-webhook-ca secret, patches webhook directly) +- cloudnative_pg.py: add _wait_for_barman_tls_secrets() to wait for barman-cloud-{client,server}-tls + before declaring Barman Cloud ready; surfaces warning instead of silent FailedMount +- deploy/gcp/gke/namespace.yaml: ecosystem-0 -> knoe-db-0 (ekosystem = k8s cluster, not namespace) +- deploy/gcp/gke/knoe-db.yaml: fix namespace ecosystem-0 -> knoe-db-0; set + podAntiAffinityType: required (one CNPG pod per node); add Spot node toleration so + GKE autoscaler simulation succeeds and scale-up is unblocked; add resource + requests/limits (500m/512Mi -> 2/2Gi); set googleCredentials.gkeEnvironment: true + (use Workload Identity - no explicit credentials secret needed) +- deploy/gcp/gke/knoe-db-backup-gcs.yaml: fix namespaces ecosystem-0 -> knoe-db-0 +- deploy/gcp/gke/workload-identity.yaml: update comments ecosystem-0 -> knoe-db-0 +- tests/installer/test_cnpg_barman_webhook.py: new - 13 tests covering webhook CA bundle + polling, manual CA injection, Barman TLS secret wait, and install_barman_plugin retry logic" diff --git a/tmp/final_status.sh b/tmp/final_status.sh new file mode 100644 index 0000000..7dbd3fd --- /dev/null +++ b/tmp/final_status.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> All GitLab pods:" +kubectl -n gitea get pods --no-headers \ + | grep -v jemalloc-builder | grep -v Completed | sort + +echo "==> Webservice logs (last 5 lines):" +kubectl -n gitea logs -l app=webservice -c webservice \ + --tail=5 2>&1 | grep -v LD_PRELOAD | head -8 || true + +echo "==> Sidekiq pod:" +kubectl -n gitea get pod -l app=sidekiq --no-headers 2>&1 + +echo "==> GitLab CR phase:" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' 2>&1 && echo + +echo "==> Node memory (gandalf):" +kubectl describe node gandalf.prole.org 2>&1 \ + | grep -A3 "Allocated resources" | tail -4 diff --git a/tmp/fix_allowed_hosts.sh b/tmp/fix_allowed_hosts.sh new file mode 100644 index 0000000..c06fda7 --- /dev/null +++ b/tmp/fix_allowed_hosts.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching CR: global.allowedHosts as flat string list..." +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "global":{ + "allowedHosts":["git.prole.org","prole.org"] + } + }}}}' + +echo "==> Also updating init_gitlab.sh with the fix..." + +echo "==> Deleting crashing pods to force reconcile with new values..." +kubectl -n gitea delete pod -l app=webservice --ignore-not-found +kubectl -n gitea delete pod -l app=sidekiq --ignore-not-found +kubectl -n gitea delete pod gitlab-webservice-default-675ccb7b99-gzmzz --ignore-not-found 2>/dev/null || true + +echo "==> Waiting 5min for pods to restart and Puma to boot..." +sleep 300 + +echo "==> Pod status:" +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder | grep -v Completed | sort + +echo "==> GitLab CR phase:" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' && echo diff --git a/tmp/fix_gitaly_and_buckets.sh b/tmp/fix_gitaly_and_buckets.sh new file mode 100644 index 0000000..7c004b9 --- /dev/null +++ b/tmp/fix_gitaly_and_buckets.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Clearing gitaly PV claimRef so Kubernetes can re-bind to gitea PVC..." +kubectl patch pv gitlab-gitaly-synology --type=json \ + -p '[{"op":"remove","path":"/spec/claimRef"}]' 2>/dev/null || \ +kubectl patch pv gitlab-gitaly-synology --type=merge \ + -p '{"spec":{"claimRef":null}}' + +echo "==> Waiting 20s for gitaly PVC to bind..." +sleep 20 +kubectl -n gitea get pvc repo-data-gitlab-gitaly-0 + +echo "==> Getting minio credentials..." +AK=$(kubectl -n gitea get secret gitlab-minio-secret \ + -o jsonpath='{.data.accesskey}' | base64 -d) +SK=$(kubectl -n gitea get secret gitlab-minio-secret \ + -o jsonpath='{.data.secretkey}' | base64 -d) +MINIO_SVC=$(kubectl -n gitea get svc gitlab-minio-svc \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null || echo "gitlab-minio-svc") + +echo "==> Creating minio buckets via mc (in-cluster pod on gandalf)..." +kubectl -n gitea run minio-buckets \ + --image=quay.io/minio/mc:RELEASE.2022-10-20T23-30-35Z \ + --restart=Never --rm --attach \ + --overrides="{ + \"spec\":{ + \"nodeName\":\"gandalf.prole.org\", + \"tolerations\":[{\"operator\":\"Exists\"}], + \"containers\":[{ + \"name\":\"minio-buckets\", + \"image\":\"quay.io/minio/mc:RELEASE.2022-10-20T23-30-35Z\", + \"command\":[\"sh\",\"-c\", + \"mc alias set gl http://gitlab-minio-svc.gitea.svc.cluster.local:9000 ${AK} ${SK} && \ + mc mb --ignore-existing gl/gitlab-registry-storage && \ + mc mb --ignore-existing gl/gitlab-lfs-storage && \ + mc mb --ignore-existing gl/gitlab-artifacts-storage && \ + mc mb --ignore-existing gl/gitlab-uploads-storage && \ + mc mb --ignore-existing gl/gitlab-packages-storage && \ + mc mb --ignore-existing gl/gitlab-dependency-proxy-storage && \ + mc mb --ignore-existing gl/gitlab-terraform-state && \ + mc mb --ignore-existing gl/gitlab-ci-secure-files && \ + mc ls gl && echo BUCKETS_DONE\"]}]}}" 2>&1 + +echo "==> Waiting 30s then checking all pod status..." +sleep 30 +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder | sort diff --git a/tmp/fix_gitaly_pv_and_exporter.sh b/tmp/fix_gitaly_pv_and_exporter.sh new file mode 100644 index 0000000..26e0812 --- /dev/null +++ b/tmp/fix_gitaly_pv_and_exporter.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> PV claimRef detail (namespace check):" +kubectl get pv gitlab-gitaly-synology \ + -o jsonpath='{.spec.claimRef}' 2>&1 && echo + +echo "==> Releasing PV claimRef and rebinding to gitea namespace..." +kubectl patch pv gitlab-gitaly-synology --type=merge \ + -p '{"spec":{"claimRef":{"namespace":"gitea","name":"repo-data-gitlab-gitaly-0"}}}' + +echo "==> Waiting 20s for PVC to bind..." +sleep 20 +kubectl -n gitea get pvc repo-data-gitlab-gitaly-0 + +echo "==> Patching CR to add jemalloc volumes to gitlab-exporter..." +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "gitlab":{ + "gitlab-exporter":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}, + "extraVolumes":"- name: jemalloc-16k\n hostPath:\n path: /opt/gitlab-jemalloc\n type: Directory\n", + "extraVolumeMounts":"- name: jemalloc-16k\n mountPath: /opt/gitlab-jemalloc\n readOnly: true\n" + } + } + }}}}' + +echo "==> Restarting exporter pod..." +kubectl -n gitea delete pod -l app=gitlab-exporter --ignore-not-found + +echo "==> Checking minio-create-buckets job..." +kubectl -n gitea get jobs -l app=minio-create-buckets 2>&1 || true + +echo "==> Done. Watching gitaly pod (30s)..." +sleep 30 +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder diff --git a/tmp/fix_gitaly_v2.sh b/tmp/fix_gitaly_v2.sh new file mode 100644 index 0000000..cb4d326 --- /dev/null +++ b/tmp/fix_gitaly_v2.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> PV status:" +kubectl get pv gitlab-gitaly-synology \ + -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CLAIM:.spec.claimRef.name,NS:.spec.claimRef.namespace,SC:.spec.storageClassName' + +echo "==> PVC describe (binding error):" +kubectl -n gitea describe pvc repo-data-gitlab-gitaly-0 2>&1 | tail -15 + +echo "==> If PV is Released, patch status to Available..." +PHASE=$(kubectl get pv gitlab-gitaly-synology -o jsonpath='{.status.phase}') +echo "PV phase: $PHASE" +if [[ "$PHASE" == "Released" ]]; then + echo "Patching PV to Available via status subresource..." + kubectl patch pv gitlab-gitaly-synology --type=json \ + -p '[{"op":"remove","path":"/spec/claimRef"}]' 2>/dev/null || true +fi + +echo "==> Creating minio buckets directly via minio pod exec..." +AK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.accesskey}' | base64 -d) +SK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.secretkey}' | base64 -d) +MINIO_POD=$(kubectl -n gitea get pod -l app=minio --no-headers -o name | head -1) +echo "Using minio pod: $MINIO_POD" + +kubectl -n gitea exec "$MINIO_POD" -- sh -c " + export MC_HOST_gl=http://${AK}:${SK}@localhost:9000 + for bucket in gitlab-registry-storage gitlab-lfs-storage gitlab-artifacts-storage \ + gitlab-uploads-storage gitlab-packages-storage gitlab-dependency-proxy-storage \ + gitlab-terraform-state gitlab-ci-secure-files; do + mc mb --ignore-existing gl/\$bucket && echo \"Created: \$bucket\" || true + done + mc ls gl +" 2>&1 + +echo "==> Waiting 30s then final pod status:" +sleep 30 +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder | sort diff --git a/tmp/fix_gitlab_sa.sh b/tmp/fix_gitlab_sa.sh new file mode 100644 index 0000000..5b8b726 --- /dev/null +++ b/tmp/fix_gitlab_sa.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +export KUBECONFIG + +echo "==> Creating gitlab-app-nonroot ServiceAccount..." +kubectl -n gitlab apply -f - <<'SAYAML' +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitlab-app-nonroot + namespace: gitlab +SAYAML + +echo "==> Patching GitLab CR to set serviceAccount.create=false..." +kubectl -n gitlab patch gitlab gitlab --type=merge \ + -p '{"spec":{"chart":{"values":{"global":{"serviceAccount":{"create":false}}}}}}' + +echo "==> Waiting 15s for operator to reconcile..." +sleep 15 + +echo "==> GitLab CR status:" +kubectl -n gitlab get gitlab gitlab + +echo "==> Operator last 20 log lines:" +kubectl logs -n gitlab deployment/gitlab-controller-manager --tail=20 diff --git a/tmp/fix_hpa_and_spread.sh b/tmp/fix_hpa_and_spread.sh new file mode 100644 index 0000000..83ff2cb --- /dev/null +++ b/tmp/fix_hpa_and_spread.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching HPAs to minReplicas=1..." +for hpa in gitlab-webservice-default gitlab-gitlab-shell gitlab-kas gitlab-registry; do + kubectl -n gitea patch hpa "$hpa" --type=merge \ + -p '{"spec":{"minReplicas":1,"maxReplicas":2}}' 2>/dev/null || true + echo " patched: $hpa" +done + +echo "==> Patching global.nodeSelector to null (let pods spread)..." +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{"global":{"nodeSelector":null}}}}}' + +echo "==> Patching CR webservice/sidekiq nodeSelector to null..." +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{"gitlab":{ + "webservice":{"nodeSelector":null}, + "sidekiq":{"nodeSelector":null}, + "toolbox":{"nodeSelector":null}, + "migrations":{"nodeSelector":null}, + "gitlab-exporter":{"nodeSelector":null} + },"registry":{"nodeSelector":null}}}}}' + +echo "==> Force-deleting stuck Terminating pods..." +for pod in $(kubectl -n gitea get pods --no-headers \ + | grep -E "Terminating|Pending" | awk '{print $1}'); do + kubectl -n gitea delete pod "$pod" --force --grace-period=0 --ignore-not-found 2>/dev/null || true +done + +echo "==> Waiting 5min for pods to reschedule across cluster..." +sleep 300 + +echo "==> Pod status with node placement:" +kubectl -n gitea get pods --no-headers \ + -o wide | grep -v jemalloc | grep -v Completed \ + | awk '{printf "%-52s %-8s %-15s %s\n",$1,$4,$6,$7}' | sort + +echo "==> CR phase:" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' && echo diff --git a/tmp/fix_liveness.sh b/tmp/fix_liveness.sh new file mode 100644 index 0000000..555835c --- /dev/null +++ b/tmp/fix_liveness.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching CR: increase liveness probe delays for slow RPi boot..." && \ +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "gitlab":{ + "webservice":{ + "livenessProbe":{ + "initialDelaySeconds":3600, + "periodSeconds":60, + "timeoutSeconds":30, + "failureThreshold":5 + }, + "readinessProbe":{ + "initialDelaySeconds":120, + "periodSeconds":30, + "timeoutSeconds":30, + "failureThreshold":60 + } + }, + "sidekiq":{ + "livenessProbe":{ + "initialDelaySeconds":3600, + "periodSeconds":60, + "timeoutSeconds":30, + "failureThreshold":5 + }, + "readinessProbe":{ + "initialDelaySeconds":120, + "periodSeconds":30, + "timeoutSeconds":30, + "failureThreshold":60 + } + } + } + }}}}' + +echo "==> Force-deleting all crashing webservice/sidekiq pods..." && \ +for pod in $(kubectl -n gitea get pods --no-headers | grep -E "webservice|sidekiq" | awk '{print $1}'); do + kubectl -n gitea delete pod "$pod" --force --grace-period=0 --ignore-not-found 2>/dev/null || true +done + +echo "==> Done — operator will reconcile new pods with long liveness grace period." && \ +echo " Monitor with: kubectl -n gitea get pods -w | grep -E 'webservice|sidekiq'" diff --git a/tmp/fix_memory.sh b/tmp/fix_memory.sh new file mode 100644 index 0000000..a10e444 --- /dev/null +++ b/tmp/fix_memory.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching CR: reduce memory requests and enforce replicaCount=1..." +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "gitlab":{ + "webservice":{ + "replicaCount":1, + "minReplicas":1, + "maxReplicas":1, + "resources":{ + "requests":{"cpu":"200m","memory":"1500M"}, + "limits":{"memory":"2000M"} + } + }, + "sidekiq":{ + "resources":{ + "requests":{"cpu":"100m","memory":"800M"}, + "limits":{"memory":"1200M"} + } + } + } + }}}}' + +echo "==> Force-deleting all stuck Terminating/Pending webservice pods..." +for pod in $(kubectl -n gitea get pods --no-headers \ + | grep -E "webservice|sidekiq|minio-create" \ + | awk '{print $1}'); do + kubectl -n gitea delete pod "$pod" --force --grace-period=0 \ + --ignore-not-found 2>/dev/null || true +done + +echo "==> Waiting 4min for operator to reconcile with new memory limits..." +sleep 240 + +echo "==> Pod status:" +kubectl -n gitea get pods --no-headers \ + | grep -v jemalloc-builder | grep -v Completed | sort + +echo "==> CR phase:" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' && echo diff --git a/tmp/fix_minio_creds.sh b/tmp/fix_minio_creds.sh new file mode 100644 index 0000000..78bc156 --- /dev/null +++ b/tmp/fix_minio_creds.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +AK=$(kubectl -n gitlab get secret gitlab-minio-secret \ + -o jsonpath='{.data.accesskey}' | base64 -d) +SK=$(kubectl -n gitlab get secret gitlab-minio-secret \ + -o jsonpath='{.data.secretkey}' | base64 -d) + +echo "==> Patching CR with MINIO_ROOT_USER/MINIO_ROOT_PASSWORD from gitlab-minio-secret..." +kubectl -n gitlab patch gitlab gitlab --type=merge -p "{ + \"spec\":{\"chart\":{\"values\":{ + \"minio\":{ + \"extraEnv\":{ + \"MINIO_ROOT_USER\":\"${AK}\", + \"MINIO_ROOT_PASSWORD\":\"${SK}\" + } + } + }}}}" + +echo "==> Restarting minio pod to pick up new env vars..." +kubectl -n gitlab delete pod -l app=minio,component=app --ignore-not-found +kubectl -n gitlab delete pod gitlab-minio-65b8ddcd4d-htdhh --ignore-not-found 2>/dev/null || true +kubectl -n gitlab delete job -l app=minio-create-buckets --ignore-not-found + +echo "==> Waiting 90s..." +sleep 90 + +echo "==> Pod status:" +kubectl -n gitlab get pods --sort-by=.metadata.creationTimestamp | grep -v jemalloc-builder + +echo "==> Minio logs:" +kubectl -n gitlab logs -l app=minio --tail=5 2>&1 || true +echo "==> create-buckets logs:" +kubectl -n gitlab logs -l app=minio-create-buckets --tail=5 2>&1 || true diff --git a/tmp/fix_minio_job_and_wait.sh b/tmp/fix_minio_job_and_wait.sh new file mode 100644 index 0000000..8f12ded --- /dev/null +++ b/tmp/fix_minio_job_and_wait.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> minio-create-buckets actual image:" +kubectl -n gitea get pod -l "app=minio-create-buckets" \ + -o jsonpath='{.items[0].spec.containers[0].image}' 2>&1 && echo + +echo "==> minio-create-buckets crash log:" +kubectl -n gitea logs -l "app=minio-create-buckets" --tail=5 2>&1 \ + | grep -v LD_PRELOAD | head -8 || true + +echo "==> Patching CR: set minio mcImage to docker.io minio/mc ARM64..." +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "minio":{ + "mcImage":{ + "repository":"minio/mc", + "tag":"RELEASE.2022-10-20T23-30-35Z" + } + } + }}}}' + +echo "==> Deleting crash job so operator recreates with new image..." +kubectl -n gitea delete jobs -l "app=minio-create-buckets" --ignore-not-found + +echo "==> Waiting 8min for Puma boot + Sidekiq ready + registry ready..." +sleep 480 + +echo "==> Final status:" +kubectl -n gitea get pods --no-headers \ + -o wide | grep -v jemalloc | grep -v Completed \ + | awk '{printf "%-52s %-5s %-8s %s\n",$1,$2,$4,$7}' | sort + +echo "==> GitLab CR phase + condition:" +kubectl -n gitea get gitlab gitlab \ + -o jsonpath='{.status.phase}{"\n"}{.status.conditions[*].message}' 2>&1 && echo + +echo "==> Sidekiq readiness:" +kubectl -n gitea get pod -l app=sidekiq \ + -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>&1 && echo + +echo "==> Webservice readiness:" +kubectl -n gitea get pod -l app=webservice \ + -o jsonpath='{range .items[*]}{.metadata.name}: {.status.containerStatuses[*].ready}{"\n"}{end}' 2>&1 diff --git a/tmp/fix_minio_pv.sh b/tmp/fix_minio_pv.sh new file mode 100644 index 0000000..590943d --- /dev/null +++ b/tmp/fix_minio_pv.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +kubectl patch pv gitlab-minio-synology \ + -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true +kubectl delete pv gitlab-minio-synology --ignore-not-found + +kubectl apply -f - <<'EOF' +apiVersion: v1 +kind: PersistentVolume +metadata: + name: gitlab-minio-synology +spec: + capacity: + storage: 10Gi + accessModes: [ReadWriteOnce] + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + volumeMode: Filesystem + local: + path: /synology/d005/gitlab/minio + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [gandalf.prole.org] +EOF + +sleep 15 +echo "==> PV status:" && kubectl get pv gitlab-minio-synology \ + -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CLAIM:.spec.claimRef.name' +echo "==> PVC status:" && kubectl -n gitlab get pvc gitlab-minio diff --git a/tmp/fix_minio_v2.sh b/tmp/fix_minio_v2.sh new file mode 100644 index 0000000..bff0834 --- /dev/null +++ b/tmp/fix_minio_v2.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Previous minio crash log:" +kubectl -n gitlab logs gitlab-minio-65b8ddcd4d-5xlgc --previous 2>&1 | \ + grep -v "LD_PRELOAD" | head -30 || true + +echo "==> Switching to 2022 quay.io minio (supports MINIO_ACCESS_KEY + ARM64)..." +# Also remove the MINIO_ROOT_USER/PASSWORD extraEnv — the 2022 image reads MINIO_ACCESS_KEY +kubectl -n gitlab patch gitlab gitlab --type=json -p '[ + {"op":"remove","path":"/spec/chart/values/minio/extraEnv"} +]' 2>/dev/null || true + +kubectl -n gitlab patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "minio":{ + "image":"quay.io/minio/minio", + "imageTag":"RELEASE.2022-10-08T20-11-00Z" + } + }}}}' + +echo "==> Wiping stale minio export data and restarting pod..." +kubectl -n gitlab run minio-wipe \ + --image=alpine:latest --restart=Never --rm \ + --overrides='{ + "spec":{"nodeName":"gandalf.prole.org","tolerations":[{"operator":"Exists"}], + "volumes":[{"name":"e","hostPath":{"path":"/synology/d005/gitlab/minio","type":"Directory"}}], + "containers":[{"name":"minio-wipe","image":"alpine:latest", + "command":["sh","-c","rm -rf /e/.minio.sys /e/registry && echo wiped"], + "volumeMounts":[{"name":"e","mountPath":"/e"}]}]}}' 2>&1 || true + +kubectl -n gitlab delete pod -l app=minio --ignore-not-found +kubectl -n gitlab delete jobs -l app=minio-create-buckets --ignore-not-found + +echo "==> Waiting 90s..." +sleep 90 + +echo "==> Pod status:" +kubectl -n gitlab get pods --sort-by=.metadata.creationTimestamp | grep -v jemalloc-builder + +echo "==> Minio logs:" +kubectl -n gitlab logs -l app=minio --tail=10 2>&1 | grep -v "LD_PRELOAD" || true diff --git a/tmp/fix_node_spread.sh b/tmp/fix_node_spread.sh new file mode 100644 index 0000000..301c583 --- /dev/null +++ b/tmp/fix_node_spread.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Removing global.nodeSelector so pods spread across cluster..." +kubectl -n gitea patch gitlab gitlab --type=json -p '[ + {"op":"remove","path":"/spec/chart/values/global/nodeSelector"} +]' 2>/dev/null || true + +echo "==> Removing per-component nodeSelectors except gitaly + minio..." +kubectl -n gitea patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "gitlab":{ + "webservice":{"nodeSelector":null}, + "sidekiq":{"nodeSelector":null}, + "toolbox":{"nodeSelector":null}, + "migrations":{"nodeSelector":null}, + "gitlab-exporter":{"nodeSelector":null} + }, + "registry":{"nodeSelector":null} + }}}}' + +echo "==> Force-deleting all stuck Terminating/Pending webservice+sidekiq pods..." +for pod in $(kubectl -n gitea get pods --no-headers \ + | grep -E "webservice|sidekiq|minio-create" \ + | awk '{print $1}'); do + kubectl -n gitea delete pod "$pod" --force --grace-period=0 \ + --ignore-not-found 2>/dev/null || true +done + +echo "==> Waiting 4min for operator to reconcile and pods to spread..." +sleep 240 + +echo "==> Final pod status (with node placement):" +kubectl -n gitea get pods --no-headers \ + | grep -v jemalloc-builder | grep -v Completed \ + | awk '{print $1, $2, $3, $4}' | sort + +echo "==> GitLab CR phase:" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' && echo diff --git a/tmp/fix_pv_and_buckets_v3.sh b/tmp/fix_pv_and_buckets_v3.sh new file mode 100644 index 0000000..86d037d --- /dev/null +++ b/tmp/fix_pv_and_buckets_v3.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Deleting old gitaly PV (still bound to deleted gitlab ns)..." +kubectl delete pv gitlab-gitaly-synology --ignore-not-found + +echo "==> Recreating gitaly PV bound to gitea namespace..." +kubectl apply -f - <<'PVYAML' +apiVersion: v1 +kind: PersistentVolume +metadata: + name: gitlab-gitaly-synology +spec: + capacity: + storage: 50Gi + accessModes: [ReadWriteOnce] + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + volumeMode: Filesystem + local: + path: /synology/d005/gitlab/gitaly + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [gandalf.prole.org] +PVYAML + +echo "==> Waiting 20s for gitaly PVC to bind..." +sleep 20 +kubectl -n gitea get pvc repo-data-gitlab-gitaly-0 + +echo "==> Creating minio buckets via dedicated mc pod..." +AK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.accesskey}' | base64 -d) +SK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.secretkey}' | base64 -d) + +kubectl -n gitea delete pod minio-init --ignore-not-found 2>/dev/null || true +kubectl -n gitea run minio-init \ + --image=quay.io/minio/mc:RELEASE.2022-10-20T23-30-35Z \ + --restart=Never \ + --overrides="{ + \"spec\":{\"nodeName\":\"gandalf.prole.org\",\"tolerations\":[{\"operator\":\"Exists\"}], + \"containers\":[{\"name\":\"minio-init\", + \"image\":\"quay.io/minio/mc:RELEASE.2022-10-20T23-30-35Z\", + \"command\":[\"sh\",\"-c\", + \"mc alias set gl http://gitlab-minio-svc.gitea.svc.cluster.local:9000 ${AK} ${SK} && mc mb --ignore-existing gl/gitlab-registry-storage && mc mb --ignore-existing gl/gitlab-lfs-storage && mc mb --ignore-existing gl/gitlab-artifacts-storage && mc mb --ignore-existing gl/gitlab-uploads-storage && mc mb --ignore-existing gl/gitlab-packages-storage && mc mb --ignore-existing gl/gitlab-dependency-proxy-storage && mc mb --ignore-existing gl/gitlab-terraform-state && mc mb --ignore-existing gl/gitlab-ci-secure-files && mc ls gl && echo BUCKETS_DONE\"]}]}}" + +echo "==> Waiting up to 3min for minio-init job to complete..." +kubectl -n gitea wait pod/minio-init --for=condition=Ready --timeout=120s 2>/dev/null || true +kubectl -n gitea wait pod/minio-init --for=jsonpath='{.status.phase}'=Succeeded --timeout=60s 2>/dev/null || true +kubectl -n gitea logs minio-init 2>&1 | tail -15 +kubectl -n gitea delete pod minio-init --ignore-not-found 2>/dev/null || true + +echo "==> Restarting registry pod to pick up new buckets..." +kubectl -n gitea delete pod -l app=registry --ignore-not-found + +echo "==> Final pod status (60s wait):" +sleep 60 +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder | sort diff --git a/tmp/fix_pv_finalizer.sh b/tmp/fix_pv_finalizer.sh new file mode 100644 index 0000000..a47f11c --- /dev/null +++ b/tmp/fix_pv_finalizer.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Removing pv-protection finalizer from gitaly PV..." +kubectl patch pv gitlab-gitaly-synology \ + -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true + +echo "==> Waiting 10s for PV deletion..." +sleep 10 +kubectl get pv gitlab-gitaly-synology 2>&1 || echo "PV gone OK" + +echo "==> Recreating PV (clean, no claimRef)..." +kubectl apply -f - <<'PVYAML' +apiVersion: v1 +kind: PersistentVolume +metadata: + name: gitlab-gitaly-synology +spec: + capacity: + storage: 50Gi + accessModes: [ReadWriteOnce] + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + volumeMode: Filesystem + local: + path: /synology/d005/gitlab/gitaly + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [gandalf.prole.org] +PVYAML + +echo "==> Waiting 20s for PVC to bind..." +sleep 20 +echo "==> PV phase:" && kubectl get pv gitlab-gitaly-synology \ + -o jsonpath='{.status.phase}' && echo +echo "==> PVC status:" +kubectl -n gitea get pvc repo-data-gitlab-gitaly-0 + +echo "==> Starting minio-init bucket creation (background)..." +AK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.accesskey}' | base64 -d) +SK=$(kubectl -n gitea get secret gitlab-minio-secret -o jsonpath='{.data.secretkey}' | base64 -d) +kubectl -n gitea delete pod minio-init --ignore-not-found 2>/dev/null || true +kubectl -n gitea run minio-init \ + --image=quay.io/minio/mc:RELEASE.2022-10-20T23-30-35Z \ + --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"gandalf.prole.org\",\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"minio-init\",\"image\":\"quay.io/minio/mc:RELEASE.2022-10-20T23-30-35Z\",\"command\":[\"sh\",\"-c\",\"mc alias set gl http://gitlab-minio-svc.gitea.svc.cluster.local:9000 ${AK} ${SK} && for b in gitlab-registry-storage gitlab-lfs-storage gitlab-artifacts-storage gitlab-uploads-storage gitlab-packages-storage gitlab-dependency-proxy-storage gitlab-terraform-state gitlab-ci-secure-files; do mc mb --ignore-existing gl/\$b; done && mc ls gl && echo BUCKETS_DONE\"]}]}}" 2>&1 +echo "==> minio-init pod created — check logs in ~2min with:" +echo " kubectl -n gitea logs minio-init" diff --git a/tmp/fix_pvs.sh b/tmp/fix_pvs.sh new file mode 100644 index 0000000..3ebe349 --- /dev/null +++ b/tmp/fix_pvs.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +KB="kubectl" + +echo "==> Deleting wrong PVs (wrong storageClass)..." +$KB delete pv gitlab-gitaly-synology gitlab-minio-synology --ignore-not-found + +GITALY_SIZE=$($KB -n gitlab get pvc repo-data-gitlab-gitaly-0 -o jsonpath='{.spec.resources.requests.storage}' 2>/dev/null || echo "50Gi") +MINIO_SIZE=$($KB -n gitlab get pvc gitlab-minio -o jsonpath='{.spec.resources.requests.storage}' 2>/dev/null || echo "10Gi") +echo "==> PVC sizes: gitaly=$GITALY_SIZE minio=$MINIO_SIZE" + +echo "==> Creating PVs with empty storageClassName to match PVCs..." +$KB apply -f - < Waiting 15s for binding..." +sleep 15 + +echo "==> PVC status:" +$KB -n gitlab get pvc + +echo "==> PV status:" +$KB get pv gitlab-gitaly-synology gitlab-minio-synology diff --git a/tmp/fix_registry_cr.sh b/tmp/fix_registry_cr.sh new file mode 100644 index 0000000..1c515cb --- /dev/null +++ b/tmp/fix_registry_cr.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching CR: registry liveness probe + ARM64 configure fix..." +kubectl -n gitlab patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "registry":{ + "livenessProbe":{"initialDelaySeconds":300,"periodSeconds":60,"timeoutSeconds":30,"failureThreshold":10}, + "readinessProbe":{"initialDelaySeconds":30,"periodSeconds":30,"timeoutSeconds":30,"failureThreshold":20} + } + }}}}' + +echo "==> Waiting 60s for operator to reconcile..." +sleep 60 + +echo "==> Patching registry Deployment configure init container to ARM64 alpine..." +kubectl -n gitlab patch deployment gitlab-registry --type=json -p '[ + {"op":"replace","path":"/spec/template/spec/initContainers/1/image","value":"alpine:latest"}, + {"op":"replace","path":"/spec/template/spec/initContainers/1/command","value":["/bin/sh","/templates/configure"]} +]' 2>/dev/null || echo "Deployment not found yet — will retry" + +echo "==> Registry pod status (30s wait):" +sleep 30 +kubectl -n gitlab get pod -l app=registry --no-headers 2>&1 diff --git a/tmp/fix_storage.sh b/tmp/fix_storage.sh new file mode 100644 index 0000000..33407e3 --- /dev/null +++ b/tmp/fix_storage.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail +KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +export KUBECONFIG +KB="kubectl" + +echo "==> Uninstalling stuck Redis release..." +helm -n knoe-system uninstall redis 2>/dev/null || true +$KB -n knoe-system delete pvc redis-data-redis-master-0 --ignore-not-found + +echo "==> Redeploying Redis without persistence..." +cd /Users/chrisfu/dev/prole +bash etc/init_redis.sh -c conf/service/prole.cfg start + +echo "==> Redis deployed. Checking gitaly/minio PVC sizes..." +$KB -n gitlab get pvc repo-data-gitlab-gitaly-0 gitlab-minio -o custom-columns='NAME:.metadata.name,SIZE:.spec.resources.requests.storage,SC:.spec.storageClassName' 2>&1 || true + +echo "==> Preparing synology dirs for gitaly and minio on gandalf..." +$KB -n knoe-system run gitlab-dirprep --image=alpine --restart=Never \ + --overrides='{"spec":{"nodeName":"gandalf.prole.org"}}' \ + --command -- sh -c "mkdir -p /synology/d005/gitlab/gitaly /synology/d005/gitlab/minio && chmod 777 /synology/d005/gitlab/gitaly /synology/d005/gitlab/minio && echo done" \ + --attach --rm 2>&1 || true + +echo "==> Creating static PVs for gitaly and minio..." +GITALY_SIZE=$($KB -n gitlab get pvc repo-data-gitlab-gitaly-0 -o jsonpath='{.spec.resources.requests.storage}' 2>/dev/null || echo "50Gi") +MINIO_SIZE=$($KB -n gitlab get pvc gitlab-minio -o jsonpath='{.spec.resources.requests.storage}' 2>/dev/null || echo "10Gi") +echo "Gitaly PVC size: $GITALY_SIZE | Minio PVC size: $MINIO_SIZE" + +$KB apply -f - < Deleting stuck PVCs so they rebind to new PVs..." +$KB -n gitlab delete pvc repo-data-gitlab-gitaly-0 gitlab-minio --ignore-not-found + +echo "==> Done. Waiting 20s for PVCs to rebind..." +sleep 20 +$KB -n gitlab get pvc,pods 2>&1 diff --git a/tmp/force_delete_ns.sh b/tmp/force_delete_ns.sh new file mode 100644 index 0000000..e544ace --- /dev/null +++ b/tmp/force_delete_ns.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Resources blocking termination:" +kubectl -n gitlab get all --ignore-not-found 2>&1 | head -20 || true + +echo "==> Force-removing finalizers from stuck GitLab CRs..." +for res in $(kubectl -n gitlab get gitlabs.apps.gitlab.com --no-headers -o name 2>/dev/null || true); do + kubectl -n gitlab patch "$res" -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true +done + +echo "==> Forcing namespace finalizer removal via API..." +kubectl get namespace gitlab -o json 2>/dev/null | \ + python3 -c " +import sys, json +d = json.load(sys.stdin) +d['spec']['finalizers'] = [] +print(json.dumps(d)) +" | kubectl replace --raw /api/v1/namespaces/gitlab/finalize -f - 2>&1 || true + +sleep 10 +echo "==> Namespace status:" +kubectl get ns gitlab 2>&1 || echo "Namespace gone — clean." diff --git a/tmp/get_crash_log.sh b/tmp/get_crash_log.sh new file mode 100644 index 0000000..992c913 --- /dev/null +++ b/tmp/get_crash_log.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> kgqhl previous crash log (webservice container):" && \ +kubectl -n gitea logs gitlab-webservice-default-675ccb7b99-kgqhl \ + -c webservice --previous --tail=25 2>&1 | grep -v LD_PRELOAD | head -25 + +echo "==> New pod 4pvtw puma log:" && \ +kubectl -n gitea logs gitlab-webservice-default-675ccb7b99-4pvtw \ + -c webservice --tail=5 2>&1 | grep -v LD_PRELOAD | head -6 + +echo "==> Sidekiq crash count + last state:" && \ +kubectl -n gitea get pod gitlab-sidekiq-all-in-1-v2-679848bc8-wvxln \ + -o jsonpath='{.status.containerStatuses[0].restartCount} restarts, last exit: {.status.containerStatuses[0].lastState.terminated.reason}{"\n"}' 2>&1 + +echo "==> Sidekiq previous log:" && \ +kubectl -n gitea logs gitlab-sidekiq-all-in-1-v2-679848bc8-wvxln \ + --previous --tail=15 2>&1 | grep -v LD_PRELOAD | tail -15 diff --git a/tmp/get_gitaly_logs.sh b/tmp/get_gitaly_logs.sh new file mode 100644 index 0000000..e77c688 --- /dev/null +++ b/tmp/get_gitaly_logs.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Waiting for gitaly to enter Running state..." +for i in $(seq 1 20); do + phase=$(kubectl -n gitlab get pod gitlab-gitaly-0 \ + -o jsonpath='{.status.containerStatuses[0].state}' 2>/dev/null || true) + status=$(kubectl -n gitlab get pod gitlab-gitaly-0 \ + --no-headers 2>/dev/null | awk '{print $3}') + echo "[$i] status=$status" + if [[ "$status" == "Running" ]]; then + echo "==> Running! Grabbing logs..." + kubectl -n gitlab logs gitlab-gitaly-0 --tail=30 2>&1 | \ + grep -v "LD_PRELOAD" | head -30 + break + fi + sleep 5 +done + +echo "==> Last terminated state:" +kubectl -n gitlab get pod gitlab-gitaly-0 \ + -o jsonpath='{.status.containerStatuses[0].lastState.terminated}' 2>&1 + +echo "==> Previous logs attempt:" +kubectl -n gitlab logs gitlab-gitaly-0 --previous --tail=20 2>&1 | \ + grep -v "LD_PRELOAD" | head -25 || true diff --git a/tmp/inject_minio_creds.sh b/tmp/inject_minio_creds.sh new file mode 100644 index 0000000..6b91523 --- /dev/null +++ b/tmp/inject_minio_creds.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching minio main container with MINIO_ROOT_USER/PASSWORD from secret..." +kubectl -n gitlab patch deployment gitlab-minio --type=json -p '[ + {"op":"add","path":"/spec/template/spec/containers/0/env/-", + "value":{"name":"MINIO_ROOT_USER","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"accesskey"}}}}, + {"op":"add","path":"/spec/template/spec/containers/0/env/-", + "value":{"name":"MINIO_ROOT_PASSWORD","valueFrom":{"secretKeyRef":{"name":"gitlab-minio-secret","key":"secretkey"}}}} +]' + +echo "==> Deleting minio pod to restart with correct env..." +kubectl -n gitlab delete pod -l app=minio --ignore-not-found + +echo "==> Waiting 60s for minio to restart..." +sleep 60 + +echo "==> Init container log:" +kubectl -n gitlab logs -l app=minio -c configure --tail=3 2>&1 || true + +echo "==> Minio main log:" +kubectl -n gitlab logs -l app=minio --tail=5 2>&1 | grep -v LD_PRELOAD | head -5 || true + +echo "==> Test credentials..." +AK=$(kubectl -n gitlab get secret gitlab-minio-secret -o jsonpath='{.data.accesskey}' | base64 -d | tr -d '\n') +SK=$(kubectl -n gitlab get secret gitlab-minio-secret -o jsonpath='{.data.secretkey}' | base64 -d | tr -d '\n') +kubectl -n gitlab delete pod minio-cred-test --ignore-not-found 2>/dev/null || true +kubectl -n gitlab run minio-cred-test \ + --image=minio/mc:latest --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"gandalf.prole.org\",\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"t\",\"image\":\"minio/mc:latest\",\"command\":[\"sh\",\"-c\",\"mc alias set gl http://gitlab-minio-svc.gitlab.svc.cluster.local:9000 ${AK} ${SK} && mc ls gl && echo CREDS_OK\"]}]}}" +sleep 40 +kubectl -n gitlab logs minio-cred-test 2>&1 | tail -5 || true +kubectl -n gitlab delete pod minio-cred-test --ignore-not-found 2>/dev/null || true diff --git a/tmp/jemalloc_daemonset.yaml b/tmp/jemalloc_daemonset.yaml new file mode 100644 index 0000000..bd4e8c0 --- /dev/null +++ b/tmp/jemalloc_daemonset.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: jemalloc-builder + namespace: gitlab + labels: + app: jemalloc-builder +spec: + selector: + matchLabels: + app: jemalloc-builder + template: + metadata: + labels: + app: jemalloc-builder + spec: + tolerations: + - operator: Exists + initContainers: + - name: build + image: alpine:latest + command: + - sh + - -c + - | + set -e + TARGET=/hostlib/libjemalloc.so.2 + if [ -f "$TARGET" ]; then + echo "Already built at $TARGET — skipping." + exit 0 + fi + echo "Installing build tools..." + apk add --no-cache gcc musl-dev make wget autoconf 2>&1 + cd /tmp + echo "Downloading jemalloc 5.3.0..." + wget -q -O jemalloc.tar.bz2 \ + https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 + tar xf jemalloc.tar.bz2 + cd jemalloc-5.3.0 + echo "Configuring with --with-lg-page=14 (16KB page support)..." + ./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1 + echo "Building..." + make -j2 lib/libjemalloc.so.2 2>&1 + cp lib/libjemalloc.so.2 "$TARGET" + chmod 755 "$TARGET" + echo "Done: $(ls -lh $TARGET)" + volumeMounts: + - name: hostlib + mountPath: /hostlib + containers: + - name: keepalive + image: alpine:latest + command: [sh, -c, "echo 'jemalloc ready at /opt/gitlab-jemalloc/libjemalloc.so.2'; sleep infinity"] + volumeMounts: + - name: hostlib + mountPath: /hostlib + volumes: + - name: hostlib + hostPath: + path: /opt/gitlab-jemalloc + type: DirectoryOrCreate diff --git a/tmp/junie_commit_msg.txt b/tmp/junie_commit_msg.txt new file mode 100644 index 0000000..a26bf69 --- /dev/null +++ b/tmp/junie_commit_msg.txt @@ -0,0 +1,25 @@ +feat: GCP/GKE CNPG hardening, Artifact Registry traffic light, and knoe-system namespace fixes + +UI screens +- database.py: fix mode detection to use env_key priority (prod→k8s, service→k3s) so stale DEPLOYMENT_MODE never overrides the user's chosen environment +- database.py: Registry status reads ARTIFACT_REGISTRY_AVAILABLE persisted by cluster screen; uses SERVICE_NAMESPACE for Artifact Registry repo name +- cluster.py: add Artifact Registry traffic light (amber→green/red) to prod section; _check_artifact_registry_async persists ARTIFACT_REGISTRY_AVAILABLE into Global cfg +- cluster.py: re-trigger Artifact Registry check after GKE cluster selection so the light re-evaluates once region is available from KUBECONTEXT +- cluster_nodes.py: fix TclError on Python 3.14 — pady=(2,0) tuple → pady=2 scalar +- __init__.py: seed knoe-system namespace when saved value is "default", not only when empty +- services.py: replace hardcoded "Prole DB" log string with dynamic cnpg_cluster name + +Core ops +- cloudnative_pg.py: replace one-shot Barman plugin retry with 6-attempt loop; first cert-manager/x509 failure triggers rollout restart + 30 s CA propagation wait; subsequent failures back off up to 60 s per attempt +- cloudnative_pg.py: TLS CA CN now uses cluster_name instead of hardcoded "Prole CNPG CA" +- registry.py, garage_store.py: refactored into per-mode modules (k3d/k3s/k8s registry and garage store, shared _garage_common) + +Deploy / config +- deploy/gcp/gke/knoe-db.yaml: GKE-specific CNPG cluster manifest (rw/ro/r on separate nodes with premium-rwo storage) +- etc/init_common_services.sh, modes/k8s/knoe-db/.version: updated for current deploy +- kong-deployment.yaml: updated manifest + +Tests +- test_cluster_nodes_render_smoke.py: add pack/grid, winfo_children, winfo_reqheight, update_idletasks, grid_slaves to dummy widgets; monkeypatch tk.Label so CNPG placement render completes without a real Tkinter root + +Co-authored-by: Junie diff --git a/tmp/kill_and_restart.sh b/tmp/kill_and_restart.sh new file mode 100644 index 0000000..9ad5a59 --- /dev/null +++ b/tmp/kill_and_restart.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +cd /Users/chrisfu/dev/prole + +echo "==> Killing running installer processes..." && \ +pkill -f "install.sh" 2>/dev/null || true && \ +pkill -f "init_gitlab.sh" 2>/dev/null || true && \ +sleep 3 + +echo "==> Cleaning misrouted gitea gitlab namespace..." && \ +kubectl get namespace gitea -o json 2>/dev/null \ + | python3 -c "import sys,json; d=json.load(sys.stdin); d['spec']['finalizers']=[]; print(json.dumps(d))" \ + | kubectl replace --raw /api/v1/namespaces/gitea/finalize -f - 2>/dev/null || true && \ +kubectl delete namespace gitea --ignore-not-found --timeout=30s 2>/dev/null || true && \ +kubectl delete pv gitlab-gitaly-synology gitlab-minio-synology --ignore-not-found 2>/dev/null || true + +echo "==> Removing stale PID lock..." && \ +rm -f /Users/chrisfu/dev/prole/install.pid + +echo "==> Verifying namespace config:" && \ +grep -E "gitops\.(namespace|gitlab_namespace)|GITLAB_NAMESPACE" \ + /Users/chrisfu/dev/prole/conf/service/prole.cfg + +echo "==> Launching final install run (gitlab namespace)..." && \ +mkdir -p logs && \ +nohup bash install.sh --silent -c conf/service/prole.cfg \ + > logs/install_final2.log 2>&1 & disown && \ +echo "Installer started. Monitor: tail -f logs/install_final2.log" diff --git a/tmp/launch_final.sh b/tmp/launch_final.sh new file mode 100644 index 0000000..5b55ea5 --- /dev/null +++ b/tmp/launch_final.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +cd /Users/chrisfu/dev/prole +rm -f install.pid +mkdir -p logs +nohup bash install.sh --silent -c conf/service/prole.cfg > logs/install_final2.log 2>&1 & +disown +echo "Installer launched. Tail: tail -f logs/install_final2.log" diff --git a/tmp/minio_buckets_final.sh b/tmp/minio_buckets_final.sh new file mode 100644 index 0000000..ffff44c --- /dev/null +++ b/tmp/minio_buckets_final.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +MINIO_POD=$(kubectl -n gitlab get pod -l app=minio --no-headers -o name | head -1) +echo "==> Minio pod: $MINIO_POD" + +AK=$(kubectl -n gitlab exec "$MINIO_POD" -- \ + sh -c "python3 -c \"import json; d=json.load(open('/tmp/.minio/config.json.old')); print(d['credential']['accessKey'],end='')\"" 2>/dev/null) +SK=$(kubectl -n gitlab exec "$MINIO_POD" -- \ + sh -c "python3 -c \"import json; d=json.load(open('/tmp/.minio/config.json.old')); print(d['credential']['secretKey'],end='')\"" 2>/dev/null) +echo "==> AK=${AK:0:8}... (len=${#AK}) SK=${SK:0:8}... (len=${#SK})" + +kubectl -n gitlab delete pod minio-init --ignore-not-found 2>/dev/null || true + +cat > /tmp/mc-cmd.sh << SCRIPT +#!/bin/sh +mc alias set gl http://gitlab-minio-svc.gitlab.svc.cluster.local:9000 '${AK}' '${SK}' && \ +for b in registry gitlab-lfs-storage gitlab-artifacts-storage gitlab-uploads-storage \ + gitlab-packages-storage gitlab-dependency-proxy-storage \ + gitlab-terraform-state gitlab-ci-secure-files; do + mc mb --ignore-existing "gl/\$b" && echo "OK:\$b" +done && mc ls gl && echo BUCKETS_DONE +SCRIPT +chmod +x /tmp/mc-cmd.sh + +kubectl -n gitlab run minio-init \ + --image=minio/mc:latest \ + --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"gandalf.prole.org\",\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"minio-init\",\"image\":\"minio/mc:latest\",\"command\":[\"sh\",\"-c\",\"mc alias set gl http://gitlab-minio-svc.gitlab.svc.cluster.local:9000 ${AK} ${SK} && for b in registry gitlab-lfs-storage gitlab-artifacts-storage gitlab-uploads-storage gitlab-packages-storage gitlab-dependency-proxy-storage gitlab-terraform-state gitlab-ci-secure-files; do mc mb --ignore-existing gl/\\\$b && echo OK:\\\$b; done && mc ls gl && echo BUCKETS_DONE\"]}]}}" +echo "==> minio-init started — check with: kubectl -n gitlab logs minio-init" diff --git a/tmp/patch_deployment_probe.sh b/tmp/patch_deployment_probe.sh new file mode 100644 index 0000000..7ce16dc --- /dev/null +++ b/tmp/patch_deployment_probe.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> CR stored liveness probe:" && \ +kubectl -n gitea get gitlab gitlab \ + -o jsonpath='{.spec.chart.values.gitlab.webservice.livenessProbe}' 2>&1 && echo + +echo "==> Patching webservice Deployment liveness probe directly..." && \ +kubectl -n gitea patch deployment gitlab-webservice-default --type=json -p '[ + {"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/initialDelaySeconds","value":3600}, + {"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/failureThreshold","value":30} +]' 2>&1 + +echo "==> Patching sidekiq Deployment liveness probe directly..." && \ +kubectl -n gitea patch deployment gitlab-sidekiq-all-in-1-v2 --type=json -p '[ + {"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/initialDelaySeconds","value":3600}, + {"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/failureThreshold","value":30} +]' 2>&1 || true + +echo "==> Checking rolling update — new pods:" && \ +sleep 10 && \ +kubectl -n gitea get pods --no-headers | grep -E "webservice|sidekiq" diff --git a/tmp/patch_jemalloc.sh b/tmp/patch_jemalloc.sh new file mode 100644 index 0000000..10d81d9 --- /dev/null +++ b/tmp/patch_jemalloc.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching GitLab CR with LD_PRELOAD='' on all ARM64-affected components..." +kubectl -n gitlab patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "gitlab":{ + "migrations":{"extraEnv":{"LD_PRELOAD":""}}, + "sidekiq":{"extraEnv":{"LD_PRELOAD":""}}, + "webservice":{"extraEnv":{"LD_PRELOAD":""}}, + "gitaly":{"extraEnv":{"LD_PRELOAD":""}}, + "toolbox":{"extraEnv":{"LD_PRELOAD":""}} + }, + "registry":{"extraEnv":{"LD_PRELOAD":""}} + }}}}' + +echo "==> Patch applied. Waiting 45s for operator to reconcile..." +sleep 45 + +echo "==> Pod status:" +kubectl -n gitlab get pods + +echo "==> GitLab CR status:" +kubectl -n gitlab get gitlab gitlab diff --git a/tmp/patch_jemalloc_hostpath.sh b/tmp/patch_jemalloc_hostpath.sh new file mode 100644 index 0000000..d18d88c --- /dev/null +++ b/tmp/patch_jemalloc_hostpath.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +JEMALLOC_PATH="/opt/gitlab-jemalloc/libjemalloc.so.2" +JEMALLOC_MOUNT="/opt/gitlab-jemalloc" + +echo "==> Patching GitLab CR with hostPath jemalloc-16k for all ARM64 components..." + +# Build the extra volumes / mounts JSON (shared across components) +EXTRA_VOL='[{"name":"jemalloc-16k","hostPath":{"path":"/opt/gitlab-jemalloc","type":"Directory"}}]' +EXTRA_MOUNT='[{"name":"jemalloc-16k","mountPath":"/opt/gitlab-jemalloc","readOnly":true}]' +PRELOAD_ENV='{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}' + +kubectl -n gitlab patch gitlab gitlab --type=merge -p "{ + \"spec\":{\"chart\":{\"values\":{ + \"global\":{ + \"extraEnv\": ${PRELOAD_ENV} + }, + \"gitlab\":{ + \"migrations\":{ + \"extraEnv\": ${PRELOAD_ENV}, + \"extraVolumes\": ${EXTRA_VOL}, + \"extraVolumeMounts\": ${EXTRA_MOUNT} + }, + \"sidekiq\":{ + \"extraEnv\": ${PRELOAD_ENV}, + \"extraVolumes\": ${EXTRA_VOL}, + \"extraVolumeMounts\": ${EXTRA_MOUNT} + }, + \"webservice\":{ + \"extraEnv\": ${PRELOAD_ENV}, + \"extraVolumes\": ${EXTRA_VOL}, + \"extraVolumeMounts\": ${EXTRA_MOUNT} + }, + \"gitaly\":{ + \"extraEnv\": ${PRELOAD_ENV}, + \"extraVolumes\": ${EXTRA_VOL}, + \"extraVolumeMounts\": ${EXTRA_MOUNT} + }, + \"toolbox\":{ + \"extraEnv\": ${PRELOAD_ENV}, + \"extraVolumes\": ${EXTRA_VOL}, + \"extraVolumeMounts\": ${EXTRA_MOUNT} + } + }, + \"registry\":{ + \"extraEnv\": ${PRELOAD_ENV}, + \"extraVolumes\": ${EXTRA_VOL}, + \"extraVolumeMounts\": ${EXTRA_MOUNT} + } + }}}}" + +echo "==> Patch applied. Waiting 60s for operator to reconcile..." +sleep 60 + +echo "==> Pod status:" +kubectl -n gitlab get pods + +echo "==> GitLab CR status:" +kubectl -n gitlab get gitlab gitlab diff --git a/tmp/patch_jemalloc_v2.sh b/tmp/patch_jemalloc_v2.sh new file mode 100644 index 0000000..00a0ac8 --- /dev/null +++ b/tmp/patch_jemalloc_v2.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Deleting failed migrations job so operator can recreate it..." +kubectl -n gitlab delete job -l app=migrations --ignore-not-found + +echo "==> Patching GitLab CR — extraVolumes/extraVolumeMounts as YAML strings..." +# The GitLab chart expects extraVolumes/extraVolumeMounts as YAML strings, not arrays +kubectl -n gitlab patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "global":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"} + }, + "gitlab":{ + "migrations":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}, + "extraVolumes":"- name: jemalloc-16k\n hostPath:\n path: /opt/gitlab-jemalloc\n type: Directory\n", + "extraVolumeMounts":"- name: jemalloc-16k\n mountPath: /opt/gitlab-jemalloc\n readOnly: true\n" + }, + "sidekiq":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}, + "extraVolumes":"- name: jemalloc-16k\n hostPath:\n path: /opt/gitlab-jemalloc\n type: Directory\n", + "extraVolumeMounts":"- name: jemalloc-16k\n mountPath: /opt/gitlab-jemalloc\n readOnly: true\n" + }, + "webservice":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}, + "extraVolumes":"- name: jemalloc-16k\n hostPath:\n path: /opt/gitlab-jemalloc\n type: Directory\n", + "extraVolumeMounts":"- name: jemalloc-16k\n mountPath: /opt/gitlab-jemalloc\n readOnly: true\n" + }, + "gitaly":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}, + "extraVolumes":"- name: jemalloc-16k\n hostPath:\n path: /opt/gitlab-jemalloc\n type: Directory\n", + "extraVolumeMounts":"- name: jemalloc-16k\n mountPath: /opt/gitlab-jemalloc\n readOnly: true\n" + }, + "toolbox":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}, + "extraVolumes":"- name: jemalloc-16k\n hostPath:\n path: /opt/gitlab-jemalloc\n type: Directory\n", + "extraVolumeMounts":"- name: jemalloc-16k\n mountPath: /opt/gitlab-jemalloc\n readOnly: true\n" + } + }, + "registry":{ + "extraEnv":{"LD_PRELOAD":"/opt/gitlab-jemalloc/libjemalloc.so.2"}, + "extraVolumes":"- name: jemalloc-16k\n hostPath:\n path: /opt/gitlab-jemalloc\n type: Directory\n", + "extraVolumeMounts":"- name: jemalloc-16k\n mountPath: /opt/gitlab-jemalloc\n readOnly: true\n" + } + }}}}' + +echo "==> Patch applied. Waiting 60s for operator to reconcile..." +sleep 60 + +echo "==> Pod status:" +kubectl -n gitlab get pods --sort-by=.metadata.creationTimestamp | grep -v jemalloc-builder | grep -v arch-check + +echo "==> GitLab CR status:" +kubectl -n gitlab get gitlab gitlab diff --git a/tmp/patch_minio_arm64.sh b/tmp/patch_minio_arm64.sh new file mode 100644 index 0000000..8845611 --- /dev/null +++ b/tmp/patch_minio_arm64.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Patching GitLab CR with ARM64-compatible minio/mc images..." +kubectl -n gitlab patch gitlab gitlab --type=merge -p '{ + "spec":{"chart":{"values":{ + "minio":{ + "image":"minio/minio", + "imageTag":"RELEASE.2024-12-18T17-30-49Z", + "mcImage":{ + "repository":"minio/mc", + "tag":"RELEASE.2024-11-21T17-21-54Z" + } + } + }}}}' + +echo "==> Deleting stuck minio pod and create-buckets job..." +kubectl -n gitlab delete pod -l app=minio --ignore-not-found +kubectl -n gitlab delete job -l app=minio-create-buckets --ignore-not-found + +echo "==> Waiting 90s for operator to reconcile..." +sleep 90 + +echo "==> Pod status:" +kubectl -n gitlab get pods --sort-by=.metadata.creationTimestamp | grep -v jemalloc-builder + +echo "==> Minio logs:" +kubectl -n gitlab logs -l app=minio --tail=15 2>&1 || true diff --git a/tmp/patch_minio_initcontainer.sh b/tmp/patch_minio_initcontainer.sh new file mode 100644 index 0000000..c69777d --- /dev/null +++ b/tmp/patch_minio_initcontainer.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Inspect current minio init container image..." +kubectl -n gitlab get deployment gitlab-minio \ + -o jsonpath='{.spec.template.spec.initContainers[*].name}' && echo +kubectl -n gitlab get deployment gitlab-minio \ + -o jsonpath='{.spec.template.spec.initContainers[0].image}' && echo + +echo "==> Patching minio Deployment: replace configure init container with ARM64 alpine..." +kubectl -n gitlab patch deployment gitlab-minio --type=json -p '[ + { + "op": "replace", + "path": "/spec/template/spec/initContainers/0", + "value": { + "name": "configure", + "image": "alpine:latest", + "command": ["sh", "-c", + "mkdir -p /tmp/.minio && echo -n \"$MINIO_ACCESS_KEY\" > /tmp/.minio/access_key && echo -n \"$MINIO_SECRET_KEY\" > /tmp/.minio/secret_key && chmod 600 /tmp/.minio/access_key /tmp/.minio/secret_key && echo \"Credentials written OK\""], + "env": [ + {"name": "MINIO_ACCESS_KEY", "valueFrom": {"secretKeyRef": {"name": "gitlab-minio-secret", "key": "accesskey"}}}, + {"name": "MINIO_SECRET_KEY", "valueFrom": {"secretKeyRef": {"name": "gitlab-minio-secret", "key": "secretkey"}}} + ], + "volumeMounts": [ + {"name": "minio-configuration", "mountPath": "/tmp/.minio"} + ] + } + } +]' + +echo "==> Deleting old minio pod to force restart with new init container..." +kubectl -n gitlab delete pod -l app=minio --ignore-not-found + +echo "==> Waiting 60s for minio to restart..." +sleep 60 + +echo "==> Minio pod status:" +kubectl -n gitlab get pod -l app=minio --no-headers + +echo "==> Minio init container log:" +kubectl -n gitlab logs -l app=minio -c configure --tail=5 2>&1 || true + +echo "==> Minio main container log:" +kubectl -n gitlab logs -l app=minio --tail=5 2>&1 | grep -v LD_PRELOAD | head -5 || true diff --git a/tmp/poll_gitlab_ns.sh b/tmp/poll_gitlab_ns.sh new file mode 100644 index 0000000..530bbd6 --- /dev/null +++ b/tmp/poll_gitlab_ns.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +for i in $(seq 1 30); do + pct=$(grep -oE '\[([0-9]+)%\]' /Users/chrisfu/dev/prole/logs/install_final2.log 2>/dev/null | tail -1) + ns_gitlab=$(kubectl get ns gitlab --no-headers 2>/dev/null | awk '{print $1}') + echo "[$(date -u '+%H:%M')] [$i/30] installer=$pct | gitlab ns=${ns_gitlab:-none}" + if [[ -n "$ns_gitlab" ]]; then + echo "==> gitlab namespace exists! Checking pods..." + kubectl -n gitlab get pods --no-headers 2>/dev/null | grep -v jemalloc | head -8 + echo "==> Checking for gitea ns (should be absent):" + kubectl get ns gitea --no-headers 2>/dev/null || echo " gitea: absent (correct)" + break + fi + # Also break if installer finished + if ! ps aux | grep -q "install.sh"; then + echo "==> Installer finished. Log tail:" + tail -5 /Users/chrisfu/dev/prole/logs/install_final2.log | cat + break + fi + sleep 60 +done diff --git a/tmp/poll_ready.sh b/tmp/poll_ready.sh new file mode 100644 index 0000000..ad61513 --- /dev/null +++ b/tmp/poll_ready.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +for i in $(seq 1 20); do + ws_ready=$(kubectl -n gitea get pods -l app=webservice \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[?(@.name=="webservice")].ready}{"\n"}{end}' \ + 2>/dev/null | grep -c "^true$" || echo 0) + sk_ready=$(kubectl -n gitea get pod -l app=sidekiq \ + -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null || echo false) + phase=$(kubectl -n gitea get gitlab gitlab \ + -o jsonpath='{.status.phase}' 2>/dev/null || echo "unknown") + + echo "[$(date -u '+%H:%M:%S')] [$i/20] webservice puma ready: $ws_ready | sidekiq ready: $sk_ready | CR: $phase" + + if [[ "$ws_ready" -ge 1 && "$sk_ready" == "true" ]]; then + echo "==> GitLab is fully up!" + break + fi + sleep 90 +done + +echo "==> Final pod status:" +kubectl -n gitea get pods --no-headers \ + | grep -v jemalloc | grep -v Completed | sort + +echo "==> CR conditions:" +kubectl -n gitea get gitlab gitlab \ + -o jsonpath='{range .status.conditions[*]}{.type}: {.status} ({.message}){"\n"}{end}' 2>&1 + +echo "==> Initial root password:" +kubectl -n gitea get secret gitlab-gitlab-initial-root-password \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d && echo || true diff --git a/tmp/quick_status.sh b/tmp/quick_status.sh new file mode 100644 index 0000000..ac8ad16 --- /dev/null +++ b/tmp/quick_status.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig +echo "==> Webservice puma latest (kgqhl):" && \ +kubectl -n gitea logs gitlab-webservice-default-675ccb7b99-kgqhl -c webservice --tail=6 2>&1 | grep -v LD_PRELOAD | head -6 +echo "==> Sidekiq latest:" && \ +kubectl -n gitea logs gitlab-sidekiq-all-in-1-v2-679848bc8-wvxln --tail=6 2>&1 | grep -v LD_PRELOAD | head -6 +echo "==> Pod readiness:" && \ +kubectl -n gitea get pods --no-headers | grep -E "webservice|sidekiq" | awk '{print $1,$2,$3}' +echo "==> CR phase:" && \ +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' 2>&1 && echo diff --git a/tmp/rebuild_ds_v2.sh b/tmp/rebuild_ds_v2.sh new file mode 100644 index 0000000..f36db77 --- /dev/null +++ b/tmp/rebuild_ds_v2.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +kubectl -n gitlab delete daemonset jemalloc-builder --ignore-not-found +sleep 5 + +kubectl apply -f - <<'DS' +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: jemalloc-builder + namespace: gitlab + labels: + app: jemalloc-builder +spec: + selector: + matchLabels: + app: jemalloc-builder + template: + metadata: + labels: + app: jemalloc-builder + spec: + tolerations: + - operator: Exists + initContainers: + - name: build + image: ubuntu:22.04 + command: + - bash + - -c + - | + set -e + TARGET=/hostlib/libjemalloc.so.2 + if [ -f "$TARGET" ]; then echo "Already built"; exit 0; fi + apt-get update -qq && apt-get install -y -q gcc make wget bzip2 2>&1 + cd /tmp + wget -q -O jemalloc.tar.bz2 \ + https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 + tar xjf jemalloc.tar.bz2 && cd jemalloc-5.3.0 + ./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1 + make -j2 lib/libjemalloc.so.2 2>&1 + cp lib/libjemalloc.so.2 "$TARGET" && chmod 755 "$TARGET" + file "$TARGET" && echo "Done: $(ls -lh $TARGET)" + volumeMounts: + - name: hostlib + mountPath: /hostlib + containers: + - name: keepalive + image: ubuntu:22.04 + command: [bash, -c, "echo 'glibc jemalloc-16k ready'; sleep infinity"] + volumes: + - name: hostlib + hostPath: + path: /opt/gitlab-jemalloc + type: DirectoryOrCreate +DS + +echo "DaemonSet applied. Waiting 15min for builds to complete..." +sleep 900 +kubectl -n gitlab get pods -l app=jemalloc-builder diff --git a/tmp/rebuild_jemalloc_glibc.sh b/tmp/rebuild_jemalloc_glibc.sh new file mode 100644 index 0000000..80c88af --- /dev/null +++ b/tmp/rebuild_jemalloc_glibc.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Deleting old musl-based DaemonSet and cleaning nodes..." +kubectl -n gitlab delete daemonset jemalloc-builder --ignore-not-found + +# Clean old musl binary from all nodes via a cleanup DaemonSet +kubectl apply -f - <<'CLEANUP' +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: jemalloc-cleanup + namespace: gitlab +spec: + selector: + matchLabels: + app: jemalloc-cleanup + template: + metadata: + labels: + app: jemalloc-cleanup + spec: + tolerations: + - operator: Exists + initContainers: + - name: clean + image: alpine:latest + command: [sh, -c, "rm -f /hostlib/libjemalloc.so.2 && echo cleaned"] + volumeMounts: + - name: hostlib + mountPath: /hostlib + containers: + - name: done + image: alpine:latest + command: [sh, -c, "echo done; sleep infinity"] + volumes: + - name: hostlib + hostPath: + path: /opt/gitlab-jemalloc + type: DirectoryOrCreate +CLEANUP + +echo "Waiting 30s for cleanup to complete..." +sleep 30 +kubectl -n gitlab delete daemonset jemalloc-cleanup --ignore-not-found + +echo "==> Deploying Ubuntu-based (glibc) jemalloc builder DaemonSet..." +kubectl apply -f - <<'DS' +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: jemalloc-builder + namespace: gitlab + labels: + app: jemalloc-builder +spec: + selector: + matchLabels: + app: jemalloc-builder + template: + metadata: + labels: + app: jemalloc-builder + spec: + tolerations: + - operator: Exists + initContainers: + - name: build + image: ubuntu:22.04 + command: + - bash + - -c + - | + set -e + TARGET=/hostlib/libjemalloc.so.2 + if [ -f "$TARGET" ]; then echo "Already built — skipping."; exit 0; fi + echo "Installing build tools (glibc-based)..." + apt-get update -qq && apt-get install -y -q gcc make wget autoconf 2>&1 + cd /tmp + echo "Downloading jemalloc 5.3.0..." + wget -q -O jemalloc.tar.bz2 \ + https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 + tar xf jemalloc.tar.bz2 && cd jemalloc-5.3.0 + echo "Configuring with --with-lg-page=14 (16KB, glibc)..." + ./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1 + make -j2 lib/libjemalloc.so.2 2>&1 + cp lib/libjemalloc.so.2 "$TARGET" + chmod 755 "$TARGET" + file "$TARGET" + echo "Done: $(ls -lh $TARGET)" + volumeMounts: + - name: hostlib + mountPath: /hostlib + containers: + - name: keepalive + image: ubuntu:22.04 + command: [bash, -c, "echo 'glibc jemalloc-16k ready'; sleep infinity"] + volumes: + - name: hostlib + hostPath: + path: /opt/gitlab-jemalloc + type: DirectoryOrCreate +DS + +echo "==> Builder DaemonSet created. This will take ~10-15min on RPi. Checking in 5min..." +sleep 300 +kubectl -n gitlab get pods -l app=jemalloc-builder diff --git a/tmp/reduce_pressure_and_wait.sh b/tmp/reduce_pressure_and_wait.sh new file mode 100644 index 0000000..988add2 --- /dev/null +++ b/tmp/reduce_pressure_and_wait.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Deleting second webservice pod to reduce RAM pressure on gandalf..." && \ +kubectl -n gitea delete pod gitlab-webservice-default-675ccb7b99-mmclp --force --grace-period=0 --ignore-not-found 2>&1 + +echo "==> Waiting 15 min for puma to complete boot on single pod..." && \ +sleep 900 + +echo "==> Webservice puma log:" && \ +kubectl -n gitea logs -l app=webservice -c webservice --tail=5 2>&1 | grep -v LD_PRELOAD | head -8 + +echo "==> Sidekiq log:" && \ +kubectl -n gitea logs -l app=sidekiq --tail=3 2>&1 | grep -v LD_PRELOAD | head -4 + +echo "==> Pod readiness:" && \ +kubectl -n gitea get pods --no-headers | grep -E "webservice|sidekiq" | awk '{print $1,$2,$3}' + +echo "==> CR conditions:" && \ +kubectl -n gitea get gitlab gitlab \ + -o jsonpath='{range .status.conditions[*]}{.type}: {.status} ({.message}){"\n"}{end}' 2>&1 + +echo "==> Root password (if Available):" && \ +kubectl -n gitea get secret gitlab-gitlab-initial-root-password \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d && echo diff --git a/tmp/reset_gitlab_db.sh b/tmp/reset_gitlab_db.sh new file mode 100644 index 0000000..53c55f1 --- /dev/null +++ b/tmp/reset_gitlab_db.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Finding CNPG primary pod..." +PRIMARY=$(kubectl -n knoe-db get pods \ + -l "cnpg.io/cluster=knoe-db,cnpg.io/instanceRole=primary" \ + --no-headers -o name | head -1) +echo "Primary: $PRIMARY" + +echo "==> Dropping and recreating gitlabhq_production DB..." +kubectl -n knoe-db exec "$PRIMARY" -- \ + psql -U postgres -c " + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = 'gitlabhq_production' AND pid <> pg_backend_pid(); + DROP DATABASE IF EXISTS gitlabhq_production; + CREATE DATABASE gitlabhq_production OWNER root; + GRANT ALL PRIVILEGES ON DATABASE gitlabhq_production TO root; + " 2>&1 + +echo "==> Deleting failed migration jobs so operator recreates them..." +kubectl -n gitea delete job \ + gitlab-migrations-5f05f2c-f73-1 \ + gitlab-migrations-5f05f2c-f73-2 \ + --ignore-not-found + +echo "==> Deleting shared-secrets jobs to force regeneration..." +kubectl -n gitea delete job \ + gitlab-shared-secrets-2454dc7 \ + gitlab-shared-secrets-d79f1fc \ + --ignore-not-found 2>/dev/null || true +kubectl -n gitea delete pods -l app=shared-secrets --ignore-not-found 2>/dev/null || true + +echo "==> Deleting gitlab-rails-secret to force new key generation..." +kubectl -n gitea delete secret gitlab-rails-secret --ignore-not-found + +echo "==> Waiting 60s for operator to recreate shared-secrets + migrations..." +sleep 60 + +echo "==> New jobs:" +kubectl -n gitea get jobs --no-headers 2>&1 | grep -E "migrations|shared-secrets" + +echo "==> New pods:" +kubectl -n gitea get pods --no-headers | grep -E "migrations|shared-secrets" diff --git a/tmp/restart_gitlab_pods.sh b/tmp/restart_gitlab_pods.sh new file mode 100644 index 0000000..0dd0555 --- /dev/null +++ b/tmp/restart_gitlab_pods.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> Deleting failed jobs..." +kubectl -n gitlab delete jobs --all --ignore-not-found + +echo "==> Deleting crashed/stuck GitLab pods..." +kubectl -n gitlab delete pods \ + -l 'app in (toolbox,registry,gitaly,webservice,sidekiq,gitlab-exporter,minio)' \ + --ignore-not-found + +echo "==> Waiting 120s for operator to reconcile and pods to start..." +sleep 120 + +echo "==> Pod status:" +kubectl -n gitlab get pods --sort-by=.metadata.creationTimestamp | grep -v jemalloc-builder + +echo "==> GitLab CR status:" +kubectl -n gitlab get gitlab gitlab diff --git a/tmp/run7.sh b/tmp/run7.sh new file mode 100644 index 0000000..f7d26e4 --- /dev/null +++ b/tmp/run7.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +cd /Users/chrisfu/dev/prole +mkdir -p logs +nohup bash install.sh --silent -c conf/service/prole.cfg > logs/install_run7.log 2>&1 & +disown +echo "started" diff --git a/tmp/run8.sh b/tmp/run8.sh new file mode 100644 index 0000000..5e5b532 --- /dev/null +++ b/tmp/run8.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> CNPG cluster status:" +kubectl -n knoe-db get cluster knoe-db -o jsonpath='{.status.phase}' 2>&1 && echo + +echo "==> CNPG pods:" +kubectl -n knoe-db get pods --no-headers 2>&1 + +echo "==> Removing stale PID lock..." +rm -f /Users/chrisfu/dev/prole/install.pid + +echo "==> Launching installer run 8..." +cd /Users/chrisfu/dev/prole +mkdir -p logs +nohup bash install.sh --silent -c conf/service/prole.cfg > logs/install_run8.log 2>&1 & +disown +echo "Installer started, logging to logs/install_run8.log" diff --git a/tmp/run_install2.sh b/tmp/run_install2.sh new file mode 100644 index 0000000..9b08155 --- /dev/null +++ b/tmp/run_install2.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /Users/chrisfu/dev/prole +echo "[$(date)] Starting install run 2..." >> logs/install_run2.log +bash install.sh --silent -c conf/service/prole.cfg >> logs/install_run2.log 2>&1 +echo "[$(date)] Install run 2 complete." >> logs/install_run2.log diff --git a/tmp/sim_installer_ns.sh b/tmp/sim_installer_ns.sh new file mode 100644 index 0000000..67399ef --- /dev/null +++ b/tmp/sim_installer_ns.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Simulates EXACTLY what milestones.py + controller.run_script does +# when calling init_gitlab.sh — replicates the full env context. +set -euo pipefail + +CFG="/Users/chrisfu/dev/prole/conf/service/prole.cfg" +PROLE_HOME="/Users/chrisfu/dev/prole" + +echo "=== Simulating installer env context ===" + +# Replicate _get_script_env (milestone.py lines 81-87) +# base_env = os.environ.copy() + updates +export PROLE_HOME="$PROLE_HOME" +export PROLE_SERVICE="$PROLE_HOME/etc" +export PROLE_CONF="$PROLE_HOME/conf/service" +export PROLE_DATA="$PROLE_HOME/knoe-db/data" +export PROLE_LOGS="$PROLE_HOME/logs" +export PROLE_MODE="k3s" +export DEPLOYMENT_MODE="k3s" +export NAMESPACE="knoe-db" # milestone.py line 86: init_password.db_namespace +export PROLE_NAMESPACE="knoe-db" +export DATABASE_NAMESPACE="knoe-db" +export SERVICE_NAMESPACE="knoe-system" +export CLUSTER_NAME="knoe-db" + +# Replicate milestones.py GitOpsMilestone.execute() lines 965-977 +export GITLAB_NAMESPACE="gitlab" # line 971: env["GITLAB_NAMESPACE"] = ns (ns="gitlab") +export GITLAB_NODE_SELECTOR="gandalf.prole.org" # line 976 +export NODE_SELECTOR="gandalf.prole.org" # line 977 + +echo " Pre-call env: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" +echo "" + +# Now call init_gitlab.sh with same args milestones.py builds: +# args = ["--mode", "k3s", "--namespace", ns, "-c", cfg_path] +echo "=== Calling init_gitlab.sh --trace-namespace (exits after namespace resolution) ===" +OUTPUT=$(bash "$PROLE_HOME/etc/init_gitlab.sh" \ + --mode k3s \ + --namespace gitlab \ + -c "$CFG" \ + --trace-namespace 2>&1 || true) +echo "$OUTPUT" | head -5 + +# Now test WITHOUT --namespace arg (worst-case: arg not passed) +echo "" +echo "=== Worst case: no --namespace arg (relies on env + default) ===" +OUTPUT2=$(bash "$PROLE_HOME/etc/init_gitlab.sh" \ + --mode k3s \ + -c "$CFG" \ + --trace-namespace 2>&1 || true) +echo "$OUTPUT2" | head -5 + +# Also verify what prole_cfg.sh does to NAMESPACE +echo "" +echo "=== Check: does prole_cfg.sh overwrite NAMESPACE? ===" +( + export NAMESPACE="gitlab" + source "$PROLE_HOME/etc/prole_cfg.sh" 2>/dev/null || true + echo " NAMESPACE after sourcing prole_cfg.sh: '${NAMESPACE:-}'" + echo " GITOPS_NAMESPACE: '${GITOPS_NAMESPACE:-}'" + echo " GITLAB_NAMESPACE: '${GITLAB_NAMESPACE:-}'" +) diff --git a/tmp/sim_milestones_ns.py b/tmp/sim_milestones_ns.py new file mode 100644 index 0000000..75e26eb --- /dev/null +++ b/tmp/sim_milestones_ns.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Simulate exactly what GitOpsMilestone.execute() does for namespace resolution.""" +import sys +sys.path.insert(0, "/Users/chrisfu/dev/prole") + +import configparser + +CFG = "/Users/chrisfu/dev/prole/conf/service/prole.cfg" + +# Load config the same way the installer does +parser = configparser.ConfigParser(strict=False) +parser.read(CFG) + +# Build state.inputs from [Inputs] section +inputs = dict(parser.items("Inputs")) if parser.has_section("Inputs") else {} +# Build config_data from all sections +config_data = {sec: dict(parser.items(sec)) for sec in parser.sections()} + +print("=== state.inputs relevant keys ===") +for k, v in sorted(inputs.items()): + if "gitops" in k or "gitlab" in k or "namespace" in k: + print(f" {k!r} => {v!r}") + +print("\n=== config_data[GitOps] ===") +for k, v in config_data.get("GitOps", {}).items(): + print(f" {k!r} => {v!r}") + +print("\n=== config_data[Global] GITLAB_NAMESPACE ===") +print(f" {config_data.get('Global', {}).get('gitlab_namespace', '')!r}") +print(f" {config_data.get('global', {}).get('gitlab_namespace', '')!r}") + +# Simulate milestones.py provider detection (lines 945-957) +provider = ( + str(inputs.get("gitops.git_provider", "") or "").strip().lower() + or str( + (config_data.get("Optional Features") or {}).get("GITOPS_PROVIDER", "") or "" + ).strip().lower() + or "gitea" +) +is_gitlab = provider == "gitlab" +print(f"\n=== Provider: {provider!r} | is_gitlab={is_gitlab} ===") + +# Simulate new namespace resolution (lines 965-977) +if is_gitlab: + ns = ( + str(inputs.get("gitops.gitlab_namespace", "") or "").strip() + or str( + (config_data.get("Global") or config_data.get("global") or {}).get( + "gitlab_namespace", "" + ) + or "" + ).strip() + or "gitlab" + ) + print(f"\n=== GITLAB ns resolved: {ns!r} ===") + if ns == "gitlab": + print(" ✓ CORRECT — will use 'gitlab' namespace") + else: + print(f" ✗ BUG — got {ns!r}, expected 'gitlab'") +else: + ns = ( + inputs.get("gitops.namespace", "") + or "gitea" + ).strip() + print(f"\n=== GITEA ns: {ns!r} (is_gitlab=False — BUG if expecting gitlab) ===") diff --git a/tmp/status_check.sh b/tmp/status_check.sh new file mode 100644 index 0000000..4912156 --- /dev/null +++ b/tmp/status_check.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +export KUBECONFIG=/Users/chrisfu/dev/prole/prole-k3s.kubeconfig + +echo "==> minio-init logs:" +kubectl -n gitea logs minio-init 2>&1 | tail -15 || true + +echo "==> GitLab pods:" +kubectl -n gitea get pods --no-headers | grep -v jemalloc-builder | sort + +echo "==> GitLab CR phase:" +kubectl -n gitea get gitlab gitlab -o jsonpath='{.status.phase}' 2>&1 && echo + +echo "==> Installer run 9 tail:" +tail -10 /Users/chrisfu/dev/prole/logs/install_run9.log 2>&1 diff --git a/tmp/supabase-render-check-manifests/namespace.yaml b/tmp/supabase-render-check-manifests/namespace.yaml new file mode 100644 index 0000000..876b893 --- /dev/null +++ b/tmp/supabase-render-check-manifests/namespace.yaml @@ -0,0 +1 @@ +{"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": "supabase"}} \ No newline at end of file diff --git a/tmp/supabase-render-check-manifests/supabase-helm.yaml b/tmp/supabase-render-check-manifests/supabase-helm.yaml new file mode 100644 index 0000000..92d5091 --- /dev/null +++ b/tmp/supabase-render-check-manifests/supabase-helm.yaml @@ -0,0 +1,2835 @@ +--- +# Source: prole-supabase/templates/analytics/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-analytics + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/auth/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-auth + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/functions/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-functions + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/imgproxy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-imgproxy + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/kong/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-kong + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/meta/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-meta + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/realtime/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-realtime + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/rest/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-rest + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/storage/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-storage + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/studio/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-studio + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/vector/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: supabase-vector + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +--- +# Source: prole-supabase/templates/secrets/analytics.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-analytics + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + privateAccessToken: eW91ci1zdXBlci1zZWNyZXQtYW5kLWxvbmctbG9nZmxhcmUta2V5LXByaXZhdGU= + publicAccessToken: eW91ci1zdXBlci1zZWNyZXQtYW5kLWxvbmctbG9nZmxhcmUta2V5LXB1YmxpYw== +--- +# Source: prole-supabase/templates/secrets/dashboard.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-dashboard + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + openAiApiKey: a2V5X3N1cGVyX3NlY3JldA== + password: dGhpc19wYXNzd29yZF9pc19pbnNlY3VyZV9hbmRfc2hvdWxkX2JlX3VwZGF0ZWQ= + username: c3VwYWJhc2U= +--- +# Source: prole-supabase/templates/secrets/db.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-db + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + database: cG9zdGdyZXM= + password: ZnIzc3R5bDM= + password_encoded: ZnIzc3R5bDM= +--- +# Source: prole-supabase/templates/secrets/jwt.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-jwt + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + anonKey: ZXlKaGJHY2lPaUpJVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnliMnhsSWpvaVlXNXZiaUlzSW1semN5STZJbkJ5YjJ4bExYTjFjR0ZpWVhObElpd2lhV0YwSWpveE56YzBPVGcyTXpNM0xDSmxlSEFpT2pJd09UQXpORFl6TXpkOS5pQjA5dDdvZ3V6SnpMNnF5U1RQR3Jsd0R2QVpEVUlUZmtsNkdtMVZhb2dR + secret: OTRjZWI5N2I0MDg0MGJlMDE1NTA3ZGVhOTcxY2JjOGU1Zjg5Nzg1N2U4NGM3MDcxZmU1M2JjM2Q3NWM5Yzg5NA== + serviceKey: ZXlKaGJHY2lPaUpJVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnliMnhsSWpvaWMyVnlkbWxqWlY5eWIyeGxJaXdpYVhOeklqb2ljSEp2YkdVdGMzVndZV0poYzJVaUxDSnBZWFFpT2pFM056UTVPRFl6TXpjc0ltVjRjQ0k2TWpBNU1ETTBOak16TjMwLkFObzlYTXNRRFJUVXFIV1Fxdno2ZzZzMGVQR2xnallpYWVBUDdLSWNrUTA= +--- +# Source: prole-supabase/templates/secrets/meta.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-meta + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + cryptoKey: eW91ci1lbmNyeXB0aW9uLWtleS0zMi1jaGFycy1taW4= +--- +# Source: prole-supabase/templates/secrets/minio.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-minio + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + password: c2VjcmV0MTIzNA== + user: c3VwYS1zdG9yYWdl +--- +# Source: prole-supabase/templates/secrets/realtime.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-realtime + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + secretKeyBase: VXBOVm50bjNjRHhISnBxOTlZTWMxVDFBUWdRcGM4a2ZZVHVSZ0JpWWExNUJMcng4ZXRRb1h6M2dadjEvdTJvcQ== +--- +# Source: prole-supabase/templates/secrets/s3.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-s3 + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + accessKey: ZHVtbXk= + keyId: ZHVtbXk= +--- +# Source: prole-supabase/templates/secrets/smtp.yaml +apiVersion: v1 +kind: Secret +metadata: + name: supabase-smtp + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + password: ZmFrZV9tYWlsX3Bhc3N3b3Jk + username: ZmFrZV9tYWlsX3VzZXI= +--- +# Source: prole-supabase/templates/functions/functions.config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: supabase-functions-main + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +data: + index.ts: | + import * as jose from 'https://deno.land/x/jose@v4.14.4/index.ts' + + console.log('main function started') + + const JWT_SECRET = Deno.env.get('JWT_SECRET') + const VERIFY_JWT = Deno.env.get('VERIFY_JWT') === 'true' + + function getAuthToken(req: Request) { + const authHeader = req.headers.get('authorization') + if (!authHeader) { + throw new Error('Missing authorization header') + } + const [bearer, token] = authHeader.split(' ') + if (bearer !== 'Bearer') { + throw new Error(`Auth header is not 'Bearer {token}'`) + } + return token + } + + async function verifyJWT(jwt: string): Promise { + const encoder = new TextEncoder() + const secretKey = encoder.encode(JWT_SECRET) + try { + await jose.jwtVerify(jwt, secretKey) + } catch (err) { + console.error(err) + return false + } + return true + } + + Deno.serve(async (req: Request) => { + if (req.method !== 'OPTIONS' && VERIFY_JWT) { + try { + const token = getAuthToken(req) + const isValidJWT = await verifyJWT(token) + + if (!isValidJWT) { + return new Response(JSON.stringify({ msg: 'Invalid JWT' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }) + } + } catch (e) { + console.error(e) + return new Response(JSON.stringify({ msg: e.toString() }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }) + } + } + + const url = new URL(req.url) + const { pathname } = url + const path_parts = pathname.split('/') + const service_name = path_parts[1] + + if (!service_name || service_name === '') { + const error = { msg: 'missing function name in request' } + return new Response(JSON.stringify(error), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }) + } + + const servicePath = `/home/deno/functions/${service_name}` + console.error(`serving the request with ${servicePath}`) + + const memoryLimitMb = 150 + const workerTimeoutMs = 1 * 60 * 1000 + const noModuleCache = false + const importMapPath = null + const envVarsObj = Deno.env.toObject() + const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]) + + try { + const worker = await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + noModuleCache, + importMapPath, + envVars, + }) + return await worker.fetch(req) + } catch (e) { + const error = { msg: e.toString() } + return new Response(JSON.stringify(error), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }) + } + }) +--- +# Source: prole-supabase/templates/kong/config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: supabase-kong + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +data: + wrapper.sh: | + #!/bin/bash + + set -euo pipefail + + echo "Replacing env placeholders of /usr/local/kong/kong.yml" + + sed \ + -e "s|\${SUPABASE_ANON_KEY}|${SUPABASE_ANON_KEY}|" \ + -e "s|\${SUPABASE_SERVICE_KEY}|${SUPABASE_SERVICE_KEY}|" \ + -e "s|\${DASHBOARD_USERNAME}|${DASHBOARD_USERNAME}|" \ + -e "s|\${DASHBOARD_PASSWORD}|${DASHBOARD_PASSWORD}|" \ + /usr/local/kong/template.yml \ + > /usr/local/kong/kong.yml + + exec /docker-entrypoint.sh kong docker-start + template.yml: | + _format_version: '2.1' + _transform: true + + consumers: + - username: DASHBOARD + - username: anon + keyauth_credentials: + - key: ${SUPABASE_ANON_KEY} + - username: service_role + keyauth_credentials: + - key: ${SUPABASE_SERVICE_KEY} + acls: + - consumer: anon + group: anon + - consumer: service_role + group: admin + basicauth_credentials: + - consumer: DASHBOARD + username: ${DASHBOARD_USERNAME} + password: ${DASHBOARD_PASSWORD} + services: + - name: auth-v1-open + url: http://supabase-auth:9999/verify + routes: + - name: auth-v1-open + strip_path: true + paths: + - /auth/v1/verify + plugins: + - name: cors + - name: auth-v1-open-callback + url: http://supabase-auth:9999/callback + routes: + - name: auth-v1-open-callback + strip_path: true + paths: + - /auth/v1/callback + plugins: + - name: cors + - name: auth-v1-open-authorize + url: http://supabase-auth:9999/authorize + routes: + - name: auth-v1-open-authorize + strip_path: true + paths: + - /auth/v1/authorize + plugins: + - name: cors + - name: auth-v1 + _comment: "GoTrue: /auth/v1/* -> http://supabase-auth:9999/*" + url: http://supabase-auth:9999 + routes: + - name: auth-v1-all + strip_path: true + paths: + - /auth/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + - name: rest-v1 + _comment: "PostgREST: /rest/v1/* -> http://supabase-rest:3000/*" + url: http://supabase-rest:3000/ + routes: + - name: rest-v1-all + strip_path: true + paths: + - /rest/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: true + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + - name: graphql-v1 + _comment: 'PostgREST: /graphql/v1/* -> http://supabase-rest:3000/rpc/graphql' + url: http://supabase-rest:3000/rpc/graphql + routes: + - name: graphql-v1-all + strip_path: true + paths: + - /graphql/v1 + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: true + - name: request-transformer + config: + add: + headers: + - Content-Profile:graphql_public + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + - name: realtime-v1-ws + _comment: "Realtime: /realtime/v1/* -> ws://supabase-realtime:4000/socket/*" + url: http://supabase-realtime:4000/socket + protocol: ws + routes: + - name: realtime-v1-ws + strip_path: true + paths: + - /realtime/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + - name: realtime-v1-rest + _comment: 'Realtime: /realtime/v1/* -> http://supabase-realtime:4000/api/*' + url: http://supabase-realtime:4000/api + protocol: http + routes: + - name: realtime-v1-rest + strip_path: true + paths: + - /realtime/v1/api + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + - name: storage-v1 + _comment: "Storage: /storage/v1/* -> http://supabase-storage:5000/*" + url: http://supabase-storage:5000/ + routes: + - name: storage-v1-all + strip_path: true + paths: + - /storage/v1/ + plugins: + - name: cors + - name: functions-v1 + _comment: 'Edge Functions: /functions/v1/* -> http://supabase-functions:9000/*' + url: http://supabase-functions:9000/ + routes: + - name: functions-v1-all + strip_path: true + paths: + - /functions/v1/ + plugins: + - name: cors + + + + + - name: meta + _comment: "pg-meta: /pg/* -> http://supabase-meta:8080/*" + url: http://supabase-meta:8080/ + routes: + - name: meta-all + strip_path: true + paths: + - /pg/ + plugins: + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - name: dashboard + _comment: 'Studio: /* -> http://supabase-studio:3000/*' + url: http://supabase-studio:3000/ + routes: + - name: dashboard-all + strip_path: true + paths: + - / + plugins: + - name: cors + - name: basic-auth + config: + hide_credentials: true +--- +# Source: prole-supabase/templates/vector/config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: supabase-vector-config + labels: + helm.sh/chart: prole-supabase-0.5.0-prole.1 + app.kubernetes.io/name: supabase + app.kubernetes.io/instance: supabase + app.kubernetes.io/managed-by: Helm +data: + vector.yml: | + api: + enabled: true + address: 0.0.0.0:9001 + + sources: + kubernetes_host: + type: kubernetes_logs + extra_label_selector: app.kubernetes.io/instance=supabase,app.kubernetes.io/name!=prole-supabase-vector + + transforms: + project_logs: + type: remap + inputs: + - kubernetes_host + source: |- + .project = "default" + .event_message = del(.message) + .appname = del(.kubernetes.container_name) + del(.file) + del(.kubernetes) + del(.source_type) + del(.stream) + router: + type: route + inputs: + - project_logs + route: + kong: '.appname == "prole-supabase-kong"' + auth: '.appname == "prole-supabase-auth"' + rest: '.appname == "prole-supabase-rest"' + realtime: '.appname == "prole-supabase-realtime"' + storage: '.appname == "prole-supabase-storage"' + functions: '.appname == "prole-supabase-functions"' + db: '.appname == "prole-supabase-db"' + # Ignores non nginx errors since they are related with kong booting up + kong_logs: + type: remap + inputs: + - router.kong + source: |- + req, err = parse_nginx_log(.event_message, "combined") + if err == null { + .timestamp = req.timestamp + .metadata.request.headers.referer = req.referer + .metadata.request.headers.user_agent = req.agent + .metadata.request.headers.cf_connecting_ip = req.client + .metadata.response.status_code = req.status + url, split_err = split(req.request, " ") + if split_err == null { + .metadata.request.method = url[0] + .metadata.request.path = url[1] + .metadata.request.protocol = url[2] + } + } + if err != null { + abort + } + # Ignores non nginx errors since they are related with kong booting up + kong_err: + type: remap + inputs: + - router.kong + source: |- + .metadata.request.method = "GET" + .metadata.response.status_code = 200 + parsed, err = parse_nginx_log(.event_message, "error") + if err == null { + .timestamp = parsed.timestamp + .severity = parsed.severity + .metadata.request.host = parsed.host + .metadata.request.headers.cf_connecting_ip = parsed.client + url, err = split(parsed.request, " ") + if err == null { + .metadata.request.method = url[0] + .metadata.request.path = url[1] + .metadata.request.protocol = url[2] + } + } + if err != null { + abort + } + # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency. + auth_logs: + type: remap + inputs: + - router.auth + source: |- + parsed, err = parse_json(.event_message) + if err == null { + .metadata.timestamp = parsed.time + .metadata = merge!(.metadata, parsed) + } + # PostgREST logs are structured so we separate timestamp from message using regex + rest_logs: + type: remap + inputs: + - router.rest + source: |- + parsed, err = parse_regex(.event_message, r'^(?P