From 5715add3662d371ac68abfa17b77ec2c128f6bc6 Mon Sep 17 00:00:00 2001 From: chrisfu Date: Wed, 1 Apr 2026 15:26:39 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20GitLab=20deployment=20pipeline=20?= =?UTF-8?q?=E2=80=94=20operator=20fix,=20CI=20config,=20and=20namespace=20?= =?UTF-8?q?isolation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - etc/init_gitlab.sh: Add global.redis block (host/port/auth) to the GitLab CR so the chart does not fail NOTES.txt validation when redis.install: false. Drop the GITOPS_NAMESPACE config fallback in namespace resolution to prevent the Gitea namespace from bleeding into GitLab deployments; GITLAB_NAMESPACE is now the sole source of truth with a hard default of "gitlab". - knoe/core/milestones.py: Fix GitOpsMilestone to route to init_gitlab.sh when gitops.git_provider = GitLab (was hardcoded to init_gitea.sh). Namespace resolution now prefers gitops.gitlab_namespace input key, then gitops.namespace, then "gitlab" — never picks up a stale GITLAB_NAMESPACE from the OS environment. - conf/service/prole.cfg: Switch gitops.git_provider / GITOPS_PROVIDER to GitLab. Update accumulated runtime state from install runs. - install.sh: Prefer the repo-local venv Python (PROLE_HOME/bin/python3) so that PyYAML and other prole_requirements.txt deps are always available. - .gitlab-ci.yml: New CI pipeline — on every push to main, run the silent install (./install.sh -S -c conf/service/prole.cfg) to deploy a fresh CNPG ecosystem. Co-Authored-By: Claude Sonnet 4.6 --- .gitlab-ci.yml | 11 ++++++++ conf/service/prole.cfg | 57 ++++++++++++++++++++++++++--------------- etc/init_gitlab.sh | 14 +++++++--- install.sh | 6 +++++ knoe/core/milestones.py | 56 ++++++++++++++++++++++++++++++++-------- 5 files changed, 110 insertions(+), 34 deletions(-) create mode 100644 .gitlab-ci.yml mode change 100644 => 100755 etc/init_gitlab.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..af82dfc --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,11 @@ +stages: + - deploy + +silent-install: + stage: deploy + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + script: + - ./install.sh -S -c conf/service/prole.cfg + environment: + name: service diff --git a/conf/service/prole.cfg b/conf/service/prole.cfg index 82830d6..aa2fcfa 100644 --- a/conf/service/prole.cfg +++ b/conf/service/prole.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-01 00:50:45 +; Generated by install.py on 2026-04-01 13:35:58 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -48,7 +48,7 @@ env_setup.PROLE_DATA = /Users/chrisfu/dev/prole/data env_setup.PROLE_HOME = /Users/chrisfu/dev/prole env_setup.PROLE_LOGS = /Users/chrisfu/dev/prole/logs env_setup.PROLE_SERVICE = /Users/chrisfu/dev/prole/etc -gitops.git_provider = Gitea +gitops.git_provider = GitLab gitops.node_selector = gandalf.prole.org init_cluster.argocd_enabled = false init_cluster.at_rest_encryption_enabled = true @@ -60,19 +60,26 @@ init_cluster.k3s_token = ${PROLE_SECRET:v1:-5rud_XHzHRKoZIU:Dxv2nKbY883uDrJlmjRI init_cluster.kerberos_enabled = true init_cluster.mode = k3s init_cluster.start_cluster = true +init_cluster.supabase_analytics_enabled = true +init_cluster.supabase_auth_enabled = true init_cluster.supabase_enabled = false +init_cluster.supabase_meta_enabled = true +init_cluster.supabase_realtime_enabled = true +init_cluster.supabase_studio_enabled = false +init_cluster.supabase_studio_url = db.prole.org init_cnpg_deploy.force_rollout = false init_cnpg_deploy.run_deploy = true init_db_build.run_build = true init_password.cluster_name = ${CLUSTER_NAME} init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password = ${PROLE_SECRET:v1:iC68SlzGpoPl9nma:iQEsd-2BzbTXdxTnSldMjHmrFrij_hWJ} -init_password.db_password_confirm = ${PROLE_SECRET:v1:iC68SlzGpoPl9nma:iQEsd-2BzbTXdxTnSldMjHmrFrij_hWJ} +init_password.db_password = ${PROLE_SECRET:v1:SF8jY2pqJ4t-1rfJ:NEjRPksckcDf26QZwxQ8lf6ckH4PGk39} +init_password.db_password_confirm = ${PROLE_SECRET:v1:SF8jY2pqJ4t-1rfJ:NEjRPksckcDf26QZwxQ8lf6ckH4PGk39} init_password.db_username = root init_password.generate_ssh_key = true init_scripts.run_scripts = true kerberos_config.enabled = true +kerberos_config.init_authority = false kerberos_config.kdc = 10.0.0.3 kerberos_config.password = ${PROLE_SECRET:v1:U-EPMl7qv4heEB1k:BEcXGbI_LT4lCXwBuvFEGgPbLFa9MpVzECObdH0pbLtClDHf} kerberos_config.realm = PROLE.ORG @@ -101,14 +108,16 @@ DB_HOST_PORT = 5432 DB_PASSWORD = ${PROLE_SECRET:v1:t7NgRHfXTXH-mHcx:CRbKP9b6ccuxrTMMeJ742Q48_vegoUY4} DEPLOYMENT_MODE = k3s DEPLOYMENT_TARGET = prole-service-cluster +DOCKER_IMPORT_DIR = DOCKER_PRELOAD = false GITEA_NODE_SELECTOR = gandalf.prole.org K3S_SERVER = https://myrddin.prole.org:6443 K3S_TOKEN = ${PROLE_SECRET:v1:gnQjvO_dhM7qkU_g:kFb7E6IQ40RM64JfsrJWBPzkJsMaLXNpNPxfho6t8w5KAeWCAyrw2bMsQtJ8n7cPc_SLWh_zX53Sa1pvEOclmd79e-c_Qff0gZleba1PzzfkjsYRfVlpzQ34gdAHrNHQG2mKnpkcyrNFZ0j6J-w4c3m_m6bTN1cV1KQvMQ==} KNOE_DB_USER = root KUBECONTEXT = knoe.dev.prole.org -OPENTOFU_URL = http://myrddin.prole.org:8080 +OPENTOFU_URL = http://127.0.0.1:8080 OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES = 2 +PROLE_HOME = /Users/chrisfu/dev/prole PROLE_K3S_SERVER = https://myrddin.prole.org:6443 PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:-5rud_XHzHRKoZIU:Dxv2nKbY883uDrJlmjRIqUKTtF42KPwXmzONy1s0-cIsJ2TLpTkaSWjfYx0tXNnz65lPnwIR16kXVtICYHhOxhfSIaz-lQiNGwZ_hm1tGMjp9BvpANt_l5Ie6swbOHVw8QfpGLtiXuIv7ur2ocSX4xAYSdCKtYWWV58U0Q==} REGISTRY_NAMESPACE = knoe-system @@ -137,7 +146,16 @@ KDC_AUTO_DETECTED = 10.0.0.3 KERBEROS_AUTO_ENABLED = True [Port Forwards] -; No configuration values captured yet for this section. +PORT_FORWARD_K3S_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD +PORT_FORWARD_K3S_MAPPING_10 = id=gitea-ssh;namespace=gitea;target=svc/gitea-ssh;address=0.0.0.0;hostPort=22;servicePort=22;protocol=TCP;description=Gitea SSH +PORT_FORWARD_K3S_MAPPING_2 = id=garage;namespace=knoe-system;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3 +PORT_FORWARD_K3S_MAPPING_3 = id=openbao;namespace=knoe-system;target=svc/openbao;address=0.0.0.0;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao +PORT_FORWARD_K3S_MAPPING_4 = id=opentofu;namespace=knoe-system;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu +PORT_FORWARD_K3S_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard +PORT_FORWARD_K3S_MAPPING_6 = id=postgres;namespace=${DATABASE_NAMESPACE};target=svc/knoe-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary) +PORT_FORWARD_K3S_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI +PORT_FORWARD_K3S_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI +PORT_FORWARD_K3S_MAPPING_9 = id=gitea-http;namespace=gitea;target=svc/gitea-http;address=0.0.0.0;hostPort=13000;servicePort=3000;protocol=TCP;description=Gitea Web [System Environment] PROLE_CONF = /Users/chrisfu/dev/prole/conf @@ -171,14 +189,14 @@ OLLAMA_SERVER_PORT = 11434 [Optional Features] ARGOCD_ENABLED = False -AT_REST_ENCRYPTION_ENABLED = True -GITOPS_ENABLED = True -GITOPS_PROVIDER = gitea -KERBEROS_ENABLED = True -SUPABASE_ENABLED = False +AT_REST_ENCRYPTION_ENABLED = true +GITOPS_ENABLED = true +GITOPS_PROVIDER = gitlab +KERBEROS_ENABLED = true +SUPABASE_ENABLED = false [GitOps] -; No configuration values captured yet for this section. +STATUS = Attempted [Database Creation] DB_USER = root @@ -189,9 +207,9 @@ K3S_SERVER_URL = https://myrddin.prole.org:6443 K3S_TOKEN = ${PROLE_SECRET:v1:-5rud_XHzHRKoZIU:Dxv2nKbY883uDrJlmjRIqUKTtF42KPwXmzONy1s0-cIsJ2TLpTkaSWjfYx0tXNnz65lPnwIR16kXVtICYHhOxhfSIaz-lQiNGwZ_hm1tGMjp9BvpANt_l5Ie6swbOHVw8QfpGLtiXuIv7ur2ocSX4xAYSdCKtYWWV58U0Q==} [Dev Cluster (k3d)] -CLUSTER_ENV = dev +CLUSTER_ENV = k3d-knoe-dev-cluster DISPLAY_NAME = knoe-dev-cluster -KUBECTL_CONTEXT = knoe.dev.prole.org +KUBECTL_CONTEXT = service MODE = k3d [Service Cluster (k3s)] @@ -200,18 +218,17 @@ DISPLAY_NAME = prole-service-cluster K3S_SERVER_URL = https://myrddin.prole.org:6443 K3S_TOKEN = ${PROLE_SECRET:v1:-5rud_XHzHRKoZIU:Dxv2nKbY883uDrJlmjRIqUKTtF42KPwXmzONy1s0-cIsJ2TLpTkaSWjfYx0tXNnz65lPnwIR16kXVtICYHhOxhfSIaz-lQiNGwZ_hm1tGMjp9BvpANt_l5Ie6swbOHVw8QfpGLtiXuIv7ur2ocSX4xAYSdCKtYWWV58U0Q==} MODE = k3s +PIPELINE_URL = http://127.0.0.1:8080 [GCP] -BILLING_ACCOUNT = 01193C-25783B-3211AD -BILLING_PROJECT = plenary-truck-485623-p7 -ORG_ID = 584001916389 -PROJECT_ID = plenary-truck-485623-p7 +; No configuration values captured yet for this section. [Prod Cluster (k8s)] -ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging +ARTIFACTS_DIR = CLUSTER_ENV = prole-prod-cluster DISPLAY_NAME = prole-prod-cluster MODE = k8s +PIPELINE_URL = http://127.0.0.1:8080 [Docker Build] ; No configuration values captured yet for this section. @@ -224,4 +241,4 @@ MODE = k3s TARGET = prole-service-cluster [Install] -STATUS = Finished +STATUS = Failed diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh old mode 100644 new mode 100755 index 41269b5..b2244d3 --- a/etc/init_gitlab.sh +++ b/etc/init_gitlab.sh @@ -67,9 +67,7 @@ elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then fi if [[ -z "$NAMESPACE" && -n "$CFG_PATH" ]]; then - maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")" - [[ -z "$maybe_ns" ]] && maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITOPS_NAMESPACE")" - NAMESPACE="$maybe_ns" + NAMESPACE="$(_prole_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")" fi NAMESPACE="${NAMESPACE:-gitlab}" export GITLAB_NAMESPACE="$NAMESPACE" @@ -101,6 +99,10 @@ GITLAB_DOMAIN="${GITLAB_DOMAIN:-git.prole.org}" FRONTDOOR_HOST="${FRONTDOOR_HOST:-}" GITLAB_RELEASE="gitlab" +# External Redis — reuse the shared instance in knoe-system +REDIS_HOST="${REDIS_HOST:-redis-master.knoe-system.svc.cluster.local}" +REDIS_PORT="${REDIS_PORT:-6379}" + # --------------------------------------------------------------------------- # Operator chart coordinates # --------------------------------------------------------------------------- @@ -371,6 +373,12 @@ spec: password: secret: gitlab-db-password key: password + # External Redis — must be set when redis.install: false + redis: + host: ${REDIS_HOST} + port: ${REDIS_PORT} + auth: + enabled: false # Reuse knoe-system namespace conventions; the operator deploys into # the gitlab namespace but can reference services in knoe-db. serviceAccount: diff --git a/install.sh b/install.sh index 7aa57ad..313df10 100755 --- a/install.sh +++ b/install.sh @@ -37,6 +37,12 @@ _script_dir="$(cd "$(dirname "${_src:-/}")" 2>/dev/null && pwd || echo "")" if [[ -n "${_script_dir}" && -d "${_script_dir}/knoe" ]]; then # Local checkout: delegate to the Python installer module. cd "${_script_dir}" + # Prefer the venv Python at PROLE_HOME (or the repo root) when available; + # it carries all prole_requirements.txt dependencies. + _VENV_PYTHON="${PROLE_HOME:-${_script_dir}}/bin/python3" + if [[ -x "${_VENV_PYTHON}" ]]; then + exec "${_VENV_PYTHON}" -m knoe.ui.screens "$@" + fi exec python3 -m knoe.ui.screens "$@" else # No local checkout found — bootstrap from the remote repo. diff --git a/knoe/core/milestones.py b/knoe/core/milestones.py index c97e7d9..00359dc 100644 --- a/knoe/core/milestones.py +++ b/knoe/core/milestones.py @@ -940,16 +940,50 @@ class GitOpsMilestone(Milestone): state.config_data.setdefault("GitOps", {})["STATUS"] = "Skipped" return + # Resolve git provider: [Inputs] gitops.git_provider takes precedence, + # then [Optional Features] GITOPS_PROVIDER, then default to gitea. + provider = ( + str(state.inputs.get("gitops.git_provider", "") or "").strip().lower() + or str( + (state.config_data.get("Optional Features") or {}).get( + "GITOPS_PROVIDER", "" + ) + or "" + ) + .strip() + .lower() + or "gitea" + ) + is_gitlab = provider == "gitlab" + label = "GitLab" if is_gitlab else "Gitea" + if progress: - progress("Deploying Gitea...", 0.4) + progress(f"Deploying {label}...", 0.4) env = self._get_script_env(state) - ns = ( - state.inputs.get("gitops.namespace", "") - or env.get("GITEA_NAMESPACE") - or "gitea" - ).strip() - env["GITEA_NAMESPACE"] = ns + + if is_gitlab: + ns = ( + str(state.inputs.get("gitops.gitlab_namespace", "") or "").strip() + or str(state.inputs.get("gitops.namespace", "") or "").strip() + or "gitlab" + ) + env["GITLAB_NAMESPACE"] = ns + node_selector = ( + str(state.inputs.get("gitops.node_selector", "") or "").strip() + ) + if node_selector: + env["GITLAB_NODE_SELECTOR"] = node_selector + env["NODE_SELECTOR"] = node_selector + script = "init_gitlab.sh" + else: + ns = ( + state.inputs.get("gitops.namespace", "") + or env.get("GITEA_NAMESPACE") + or "gitea" + ).strip() + env["GITEA_NAMESPACE"] = ns + script = "init_gitea.sh" mode = env.get("PROLE_MODE", "") args: list[str] = [] @@ -962,21 +996,21 @@ class GitOpsMilestone(Milestone): if cfg_path.exists(): args.extend(["-c", str(cfg_path)]) - self.logger.info(f"init_gitea.sh {' '.join(args)}") + self.logger.info(f"{script} {' '.join(args)}") rc = state.controller.run_script( - "init_gitea.sh", args=args, env=env, on_line=_stream_line + script, args=args, env=env, on_line=_stream_line ) if rc == 0: gitops_sec = state.config_data.setdefault("GitOps", {}) gitops_sec["STATUS"] = "Deployed" gitops_sec["GITOPS_NAMESPACE"] = ns if progress: - progress("Gitea deployed", 1.0) + progress(f"{label} deployed", 1.0) else: gitops_sec = state.config_data.setdefault("GitOps", {}) gitops_sec["STATUS"] = "Attempted" gitops_sec["GITOPS_NAMESPACE"] = ns - msg = f"Gitea deploy failed (code {rc})" + msg = f"{label} deploy failed (code {rc})" self.logger.error(msg) raise Exception(msg)