From b03efa8f693a3e3d61a6ca3c4b01ec93c62ce336 Mon Sep 17 00:00:00 2001 From: chrisfu Date: Sun, 22 Feb 2026 00:57:49 -0800 Subject: [PATCH] Kong API gateway, docker-import preload, OpenTofu graceful fallback, milestone fix Kong API Gateway (replacing prole nginx): - Add etc/init_kong.sh provisioning script (DB-less mode, prole-db namespace) - Add kong-deployment.yaml and kong-service.yaml manifests - Rewire ingress rules (svc/git/api.prole.org) to prole-db-kong:8000 - Update kustomization.yaml to reference kong manifests PostgREST & DB Manager in prole-db namespace: - Add etc/init_postgrest.sh and etc/init_db_manager.sh scripts - Add postgrest/db-manager deployment and service manifests - Add src/db-manager/ Node.js REST endpoint for backup triggers - Default NAMESPACE changed to prole-db in both scripts Docker image pre-load from PROLE_DATA/docker-import: - Add _preload_docker_images() to init_common_services.sh - Scan for .tar files exported by final_deployment.sh - Import via k3d image import (k3d) or ctr (k3s) before deployments - Increase rollout timeouts to 300s (configurable via ROLLOUT_TIMEOUT) in init_openbao.sh, init_opentofu.sh, init_garage_store.sh, init_registry.sh OpenTofu password resolution fix: - Add Kubernetes secret fallback in resolve_admin_password() - Change hard exit 1 to graceful return 1 with warning - Wrap call in if-guard so set -e doesn't abort the script chain Milestone fix (init scripts not running): - Add init_kong.sh, init_postgrest.sh, init_db_manager.sh to InitializationScriptsMilestone.execute() script list and arg branches - Previously only actions.py had these; milestones.py was missing them Installer integration: - Add Kong/PostgREST/DB Manager to silent installer _step_init_scripts - Add corresponding tabs and execution blocks in UI services.py --- conf/port-forwards.cfg | 12 + conf/port-mapping.cfg | 6 +- conf/prole.cfg | 92 +- .../k3s/manifests/openbao/deployment.yaml | 2 +- .../k3s/manifests/opentofu/deployment.yaml | 2 +- .../manifests/prole/dashboard-ingress.yaml | 19 + .../prole/db-manager-deployment.yaml | 89 ++ .../manifests/prole/db-manager-service.yaml | 14 + .../opentofu/k3s/manifests/prole/ingress.yaml | 88 ++ .../k3s/manifests/prole/kong-deployment.yaml | 60 ++ .../k3s/manifests/prole/kong-service.yaml | 17 + .../k3s/manifests/prole/kustomization.yaml | 9 +- .../manifests/prole/openbao-statefulset.yaml | 2 +- .../manifests/prole/postgrest-deployment.yaml | 70 ++ .../manifests/prole/postgrest-service.yaml | 14 + .../prole/prole-db-postgres-service.yaml | 2 +- .../k3s/manifests/prole/prole-db.yaml | 8 +- .../k3s/manifests/prole/supabase-ingress.yaml | 19 + deploy/opentofu/k3s/opentofu.auto.tfvars | 4 +- etc/final_deployment.sh | 136 ++- etc/init_common_services.sh | 97 +- etc/init_db_manager.sh | 201 +++++ etc/init_garage_store.sh | 2 +- etc/init_k8s.sh | 17 +- etc/init_kong.sh | 190 ++++ etc/init_openbao.sh | 4 +- etc/init_opentofu.sh | 21 +- etc/init_postgrest.sh | 211 +++++ etc/init_prole-db-backup.sh | 31 +- etc/init_registry.sh | 46 +- etc/init_supabase_ports.sh | 18 +- installer/config.py | 10 + installer/core/actions.py | 145 ++- installer/core/env.py | 220 ++++- installer/core/milestones.py | 10 +- installer/core/monitor.py | 261 ++++++ installer/ui/screens/__init__.py | 30 +- installer/ui/screens/deploy.py | 60 +- installer/ui/screens/packaging.py | 8 + installer/ui/screens/services.py | 141 +++ installer/ui/screens/supabase.py | 2 +- prole.sh | 135 ++- src/db-manager/.gitignore | 1 + src/db-manager/Dockerfile | 19 + src/db-manager/package-lock.json | 830 ++++++++++++++++++ src/db-manager/package.json | 16 + src/db-manager/server.js | 161 ++++ src/db-manager/test/server.test.js | 143 +++ supabase/deploy.sh | 181 +++- tests/etc/test_init_db_manager.sh | 72 ++ 50 files changed, 3764 insertions(+), 184 deletions(-) create mode 100644 conf/port-forwards.cfg create mode 100644 deploy/opentofu/k3s/manifests/prole/dashboard-ingress.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/db-manager-deployment.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/db-manager-service.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/ingress.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/kong-service.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/postgrest-deployment.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/postgrest-service.yaml create mode 100644 deploy/opentofu/k3s/manifests/prole/supabase-ingress.yaml create mode 100755 etc/init_db_manager.sh create mode 100755 etc/init_kong.sh create mode 100755 etc/init_postgrest.sh create mode 100644 installer/core/monitor.py create mode 100644 src/db-manager/.gitignore create mode 100644 src/db-manager/Dockerfile create mode 100644 src/db-manager/package-lock.json create mode 100644 src/db-manager/package.json create mode 100644 src/db-manager/server.js create mode 100644 src/db-manager/test/server.test.js create mode 100755 tests/etc/test_init_db_manager.sh diff --git a/conf/port-forwards.cfg b/conf/port-forwards.cfg new file mode 100644 index 0000000..011e71a --- /dev/null +++ b/conf/port-forwards.cfg @@ -0,0 +1,12 @@ +# Port forward configuration for Prole services (generated). +# Format: id: local= remote= ns= svc= address= + +argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0 +garage: local=3900 remote=3900 ns=default svc=garage address=0.0.0.0 +openbao: local=8200 remote=8200 ns=default svc=openbao address=127.0.0.1 +opentofu: local=8080 remote=8080 ns=default svc=opentofu address=0.0.0.0 +dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1 +postgres: local=5432 remote=5432 ns=prole-db-a0001 svc=prole-db-rw address=0.0.0.0 +prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1 +grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0 +prole-db-manager: local=80 remote=80 ns=prole-db-a0001 svc=prole-db-manager address=0.0.0.0 diff --git a/conf/port-mapping.cfg b/conf/port-mapping.cfg index 4c9b0ba..9ddfbca 100644 --- a/conf/port-mapping.cfg +++ b/conf/port-mapping.cfg @@ -2,10 +2,6 @@ # Format: key: local=... remote=... ns=... svc=... address=... argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0 -supabase-studio: local=8088 remote=3000 ns=supabase svc=studio address=0.0.0.0 -supabase-auth: local=9999 remote=9999 ns=supabase svc=auth address=0.0.0.0 -supabase-rest: local=3001 remote=3000 ns=supabase svc=rest address=0.0.0.0 -supabase-realtime: local=4000 remote=4000 ns=supabase svc=realtime address=0.0.0.0 garage: local=3900 remote=3900 ns=default svc=garage address=0.0.0.0 openbao: local=8200 remote=8200 ns=default svc=openbao address=127.0.0.1 opentofu: local=8080 remote=8080 ns=default svc=opentofu address=0.0.0.0 @@ -13,4 +9,4 @@ dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboar postgres: local=5432 remote=5432 ns=prole-db-a0001 svc=prole-db-rw address=0.0.0.0 prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1 grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0 -supabase-kong: local=8000 remote=8000 ns=supabase svc=kong address=0.0.0.0 +prole-db-manager: local=80 remote=80 ns=prole-db-a0001 svc=prole-db-manager address=0.0.0.0 diff --git a/conf/prole.cfg b/conf/prole.cfg index 628c797..e18cd52 100644 --- a/conf/prole.cfg +++ b/conf/prole.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-02-20 23:07:16 +; Generated by install.py on 2026-02-22 00:56:01 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -22,11 +22,13 @@ dependencies.brew.install = true dependencies.docker.install = true dependencies.k3d.install = true dependencies.kubectl.install = true +dependencies.opentofu.install = true dependencies.python.install = true dependencies.verify_all = false disk_selection.disk_type = local disk_selection.local_path = ${PROLE_HOME}/prole-tools-app/dist disk_selection.removable_mount = +env_setup.NAMESPACE = ${NAMESPACE} env_setup.PROLE_CONF = ${PROLE_CONF} env_setup.PROLE_DATA = ${PROLE_DATA} env_setup.PROLE_HOME = ${PROLE_HOME} @@ -36,52 +38,56 @@ init_cluster.at_rest_encryption_enabled = true init_cluster.cluster_env = dev init_cluster.deployment_target = prole-dev-cluster init_cluster.k3s_server_url = https://myrddin.prole.org:6443 -init_cluster.k3s_token = ${OPENBAO:kv/prole/prole-db-a0001/k3s#token} +init_cluster.k3s_token = ${PROLE_SECRET:v1:pbLtur2o2-NpzrUk:DwwxyF6oS4y9DxjRplLditr1rVZJXcXuqlJ__0G7k5UXGBdwJfmDT8i2CvJ4Qpu7PVOexpOyak9duHm4vq0D5wOkMOKJAf2DHOjpT8biN9U5L8kpTUrtWaxvWheFzN0vVZypff9P7tm8ZFafTqqkVEJD_rC5mQgUI_6yBw==} init_cluster.kerberos_enabled = true init_cluster.mode = k3d init_cluster.start_cluster = true -init_cluster.supabase_enabled = true +init_cluster.supabase_enabled = false init_cnpg_deploy.force_rollout = false init_cnpg_deploy.run_deploy = true init_db_build.run_build = true init_password.db_host_port = 5432 -init_password.db_password = ${PROLE_SECRET:v1:8EhTsXIY1HsgEKKM:yWkrV7taKG98uXx3g6hZlUIhsAFalYTc} -init_password.db_password_confirm = ${PROLE_SECRET:v1:8EhTsXIY1HsgEKKM:yWkrV7taKG98uXx3g6hZlUIhsAFalYTc} -init_password.db_username = chrisfu +init_password.db_namespace = ${NAMESPACE} +init_password.db_password = ${PROLE_SECRET:v1:2C5bW0rIcL86kBU3:QwS410Y3kmpqycsic8S_YJwJnDh8HEJpJ6Sw8hRrLiV_MonBWHodsw==} +init_password.db_password_confirm = ${PROLE_SECRET:v1:2C5bW0rIcL86kBU3:QwS410Y3kmpqycsic8S_YJwJnDh8HEJpJ6Sw8hRrLiV_MonBWHodsw==} +init_password.db_username = root init_password.generate_ssh_key = true init_scripts.run_scripts = true kerberos_config.enabled = true +kerberos_config.init_authority = false kerberos_config.kdc = 10.0.0.3 -kerberos_config.password = ${PROLE_SECRET:v1:m25Ip6n_vkrWCJbs:JctekuYhBjVDE_2Ef40Jk7l_lAzIHuzJ} +kerberos_config.password = kerberos_config.realm = PROLE.ORG -kerberos_config.test_connection = true -kerberos_config.user = chrisfu +kerberos_config.test_connection = false +kerberos_config.user = network_scan.run = true -ollama_config.model = qwen3-coder:latest -ollama_config.server_host = k3d.localhost +ollama_config.model = +ollama_config.server_host = ollama_config.server_port = 11434 [Global] ; Variables used by name in more than one place or assumed global scope +ARGOCD_NAMESPACE = argocd CLUSTER_ENV = dev DB_HOST_PORT = 5432 -DB_PASSWORD = ${PROLE_SECRET:v1:FdjQbe3QGzS4toDe:qVW_5s6qiThibRw8tpKk-_36GgFqlL-5} +DB_PASSWORD = ${PROLE_SECRET:v1:k0OoPBKzyOZvR_lJ:IOLBvf_qtNoOzTMZup0oo5LK1CuNyhI_uU-shfydqwvriMBfdeTRUA==} DEPLOYMENT_MODE = k3d DEPLOYMENT_TARGET = prole-dev-cluster DOCKER_IMPORT_DIR = NAMESPACE = ${NAMESPACE} -PROLE_DB_USER = chrisfu +PROLE_DB_USER = root PROLE_HOME = ${PROLE_HOME} PROLE_K3S_SERVER = https://myrddin.prole.org:6443 -PROLE_K3S_TOKEN = ${OPENBAO:kv/prole/prole-db-a0001/k3s#token} +PROLE_K3S_TOKEN = ${PROLE_SECRET:v1:pbLtur2o2-NpzrUk:DwwxyF6oS4y9DxjRplLditr1rVZJXcXuqlJ__0G7k5UXGBdwJfmDT8i2CvJ4Qpu7PVOexpOyak9duHm4vq0D5wOkMOKJAf2DHOjpT8biN9U5L8kpTUrtWaxvWheFzN0vVZypff9P7tm8ZFafTqqkVEJD_rC5mQgUI_6yBw==} PROLE_OPENTOFU_URL = http://127.0.0.1:8080 +REGISTRY_NAMESPACE = default SERVICE_NAMESPACE = default [Welcome] ; No configuration values captured yet for this section. [Dependencies] -; No configuration values captured yet for this section. +STATUS = All installed [Network] AD_DC_HOST = myrddin.prole.org @@ -96,9 +102,19 @@ KDC_AUTO_DETECTED = 10.0.0.3 KERBEROS_AUTO_ENABLED = True [Port Forwards] -; No configuration values captured yet for this section. +PORT_FORWARD_K3D_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD +PORT_FORWARD_K3D_MAPPING_2 = id=garage;namespace=default;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3 +PORT_FORWARD_K3D_MAPPING_3 = id=openbao;namespace=default;target=svc/openbao;address=127.0.0.1;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao +PORT_FORWARD_K3D_MAPPING_4 = id=opentofu;namespace=default;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu +PORT_FORWARD_K3D_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard +PORT_FORWARD_K3D_MAPPING_6 = id=postgres;namespace=${NAMESPACE};target=svc/prole-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary) +PORT_FORWARD_K3D_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI +PORT_FORWARD_K3D_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI +PORT_FORWARD_K3D_MAPPING_9 = id=prole-db-manager;namespace=${NAMESPACE};target=svc/prole-db-manager;address=0.0.0.0;hostPort=80;servicePort=80;protocol=TCP;description=DB Manager UI +STATUS = Failed [System Environment] +NAMESPACE = prole-db-a0001 PROLE_CONF = ${PROLE_CONF} PROLE_DATA = ${PROLE_DATA} PROLE_HOME = ${PROLE_HOME} @@ -109,68 +125,52 @@ PROLE_SERVICE = ${PROLE_SERVICE} ; No configuration values captured yet for this section. [Kerberos Authentication] -AD_PORT_FORWARD = 1 -AD_PROXY_HOST_NETWORK = 1 -AD_PROXY_IMAGE = alpine/socat -AD_PROXY_SERVICE = prole-kerberos-ad-dc -AD_TCP_PORTS = 88 389 445 464 636 -AD_UDP_PORTS = 88 464 -ENABLED = True -KDC = 10.0.0.3 -PASSWORD = ${PROLE_SECRET:v1:9F1gGFfO11MF4Tui:VGB5mrP6JE2YrrY7wiOJxFcn3C2kE28p} -REALM = PROLE.ORG -SERVER = 10.0.0.3 -USER = chrisfu +; No configuration values captured yet for this section. [Ollama] -OLLAMA_HOST = http://k3d.localhost:11434 -OLLAMA_MODEL = qwen3-coder:latest -OLLAMA_SERVER_HOST = k3d.localhost -OLLAMA_SERVER_PORT = 11434 +; No configuration values captured yet for this section. [Optional Features] -AT_REST_ENCRYPTION_ENABLED = True -KERBEROS_ENABLED = True -SUPABASE_ENABLED = False +; No configuration values captured yet for this section. [Database Creation] +DB_NAME = prole-db-a0001 +DB_USER = root +NAMESPACE = ${NAMESPACE} [Initialize Cluster] -ENVIRONMENT = prole-dev-cluster -K3S_SERVER_URL = https://myrddin.prole.org:6443 -K3S_TOKEN = ${OPENBAO:kv/prole/prole-db-a0001/k3s#token} +; No configuration values captured yet for this section. [Dev Cluster (k3d)] -CLUSTER_ENV = dev +CLUSTER_ENV = k3d-prole-dev-cluster DISPLAY_NAME = prole-dev-cluster -KUBECTL_CONTEXT = +KUBECTL_CONTEXT = dev MODE = k3d [Service Cluster (k3s)] CLUSTER_ENV = prole-service-cluster DISPLAY_NAME = prole-service-cluster K3S_SERVER_URL = https://myrddin.prole.org:6443 -K3S_TOKEN = ${OPENBAO:kv/prole/prole-db-a0001/k3s#token} +K3S_TOKEN = ${PROLE_SECRET:v1:pbLtur2o2-NpzrUk:DwwxyF6oS4y9DxjRplLditr1rVZJXcXuqlJ__0G7k5UXGBdwJfmDT8i2CvJ4Qpu7PVOexpOyak9duHm4vq0D5wOkMOKJAf2DHOjpT8biN9U5L8kpTUrtWaxvWheFzN0vVZypff9P7tm8ZFafTqqkVEJD_rC5mQgUI_6yBw==} MODE = k3s PIPELINE_URL = http://127.0.0.1:8080 [Prod Cluster (k8s)] -ARTIFACTS_DIR = /Users/chrisfu/dev/prole/data/staging +ARTIFACTS_DIR = ${PROLE_DATA}/staging CLUSTER_ENV = prole-prod-cluster DISPLAY_NAME = prole-prod-cluster MODE = k8s PIPELINE_URL = http://127.0.0.1:8080 [Docker Build] -LOCAL_REGISTRY = localhost:5000 -LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000 +; No configuration values captured yet for this section. [Initialization Scripts] -STATUS = Completed +; No configuration values captured yet for this section. [Deployment] MODE = k3d TARGET = prole-dev-cluster [Install] -; No configuration values captured yet for this section. +STATUS = Failed diff --git a/deploy/opentofu/k3s/manifests/openbao/deployment.yaml b/deploy/opentofu/k3s/manifests/openbao/deployment.yaml index 0545e47..47190cf 100644 --- a/deploy/opentofu/k3s/manifests/openbao/deployment.yaml +++ b/deploy/opentofu/k3s/manifests/openbao/deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: openbao - image: ghcr.io/openbao/openbao:latest + image: ghcr.io/openbao/openbao:2.0.0 imagePullPolicy: IfNotPresent args: - "server" diff --git a/deploy/opentofu/k3s/manifests/opentofu/deployment.yaml b/deploy/opentofu/k3s/manifests/opentofu/deployment.yaml index f5ee5d8..3a1e849 100644 --- a/deploy/opentofu/k3s/manifests/opentofu/deployment.yaml +++ b/deploy/opentofu/k3s/manifests/opentofu/deployment.yaml @@ -72,7 +72,7 @@ spec: mountPath: /etc/nginx/auth readOnly: true - name: opentofu-runner - image: ghcr.io/opentofu/opentofu:latest + image: ghcr.io/opentofu/opentofu:1.8.2 command: ["sh", "-c", "tofu version && tail -f /dev/null"] env: - name: OPENTOFU_ADMIN_PASSWORD diff --git a/deploy/opentofu/k3s/manifests/prole/dashboard-ingress.yaml b/deploy/opentofu/k3s/manifests/prole/dashboard-ingress.yaml new file mode 100644 index 0000000..6ad8c8a --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/dashboard-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: kong-dashboard-ingress + namespace: kubernetes-dashboard + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: dashboard.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kubernetes-dashboard-kong-proxy + port: + number: 443 diff --git a/deploy/opentofu/k3s/manifests/prole/db-manager-deployment.yaml b/deploy/opentofu/k3s/manifests/prole/db-manager-deployment.yaml new file mode 100644 index 0000000..6a27e13 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/db-manager-deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prole-db-manager + labels: + app: prole-db-manager +spec: + replicas: 1 + selector: + matchLabels: + app: prole-db-manager + template: + metadata: + labels: + app: prole-db-manager + spec: + serviceAccountName: prole-db-manager + containers: + - name: db-manager + image: prole-db-manager:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 + name: http + env: + - name: PORT + value: "80" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: BARMAN_PLUGIN_NAME + value: "barman-cloud.cloudnative-pg.io" + - name: BARMAN_OBJECT_NAME + value: "prole-db-barman-objectstore" + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 30 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prole-db-manager + labels: + app: prole-db-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prole-db-manager + labels: + app: prole-db-manager +rules: + - apiGroups: ["postgresql.cnpg.io"] + resources: ["backups"] + verbs: ["create", "get", "list", "watch"] + - apiGroups: ["postgresql.cnpg.io"] + resources: ["clusters"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prole-db-manager + labels: + app: prole-db-manager +subjects: + - kind: ServiceAccount + name: prole-db-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prole-db-manager diff --git a/deploy/opentofu/k3s/manifests/prole/db-manager-service.yaml b/deploy/opentofu/k3s/manifests/prole/db-manager-service.yaml new file mode 100644 index 0000000..48dd7c4 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/db-manager-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: prole-db-manager + labels: + app: prole-db-manager +spec: + selector: + app: prole-db-manager + ports: + - name: http + port: 80 + targetPort: http + type: ClusterIP diff --git a/deploy/opentofu/k3s/manifests/prole/ingress.yaml b/deploy/opentofu/k3s/manifests/prole/ingress.yaml new file mode 100644 index 0000000..0a18d52 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/ingress.yaml @@ -0,0 +1,88 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prole-ingress + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: grafana.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 3000 + - host: svc.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-db-kong + port: + number: 8000 + - host: git.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-db-kong + port: + number: 8000 + - host: api.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prole-db-kong + port: + number: 8000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: kong-dashboard-ingress + namespace: kubernetes-dashboard + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: dashboard.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kubernetes-dashboard-kong-proxy + port: + number: 443 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: supabase-ingress + namespace: supabase + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: supabase.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kong + port: + number: 8000 diff --git a/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml b/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml new file mode 100644 index 0000000..fb94494 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prole-db-kong + labels: + app: prole-db-kong +spec: + replicas: 1 + selector: + matchLabels: + app: prole-db-kong + template: + metadata: + labels: + app: prole-db-kong + spec: + containers: + - name: kong + image: kong:3.9 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + name: proxy + - containerPort: 8001 + name: admin + env: + - name: KONG_DATABASE + value: "off" + - name: KONG_PROXY_LISTEN + value: "0.0.0.0:8000" + - name: KONG_ADMIN_LISTEN + value: "0.0.0.0:8001" + - name: KONG_DECLARATIVE_CONFIG + value: "/etc/kong/kong.yml" + volumeMounts: + - name: kong-config + mountPath: /etc/kong + readinessProbe: + httpGet: + path: /status + port: admin + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /status + port: admin + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumes: + - name: kong-config + configMap: + name: prole-db-kong-config diff --git a/deploy/opentofu/k3s/manifests/prole/kong-service.yaml b/deploy/opentofu/k3s/manifests/prole/kong-service.yaml new file mode 100644 index 0000000..92f9735 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/kong-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: prole-db-kong + labels: + app: prole-db-kong +spec: + selector: + app: prole-db-kong + ports: + - name: proxy + port: 8000 + targetPort: proxy + - name: admin + port: 8001 + targetPort: admin + type: ClusterIP diff --git a/deploy/opentofu/k3s/manifests/prole/kustomization.yaml b/deploy/opentofu/k3s/manifests/prole/kustomization.yaml index fcf6e64..81a7a73 100644 --- a/deploy/opentofu/k3s/manifests/prole/kustomization.yaml +++ b/deploy/opentofu/k3s/manifests/prole/kustomization.yaml @@ -9,7 +9,12 @@ resources: - prole-db.yaml - prole-db-postgres-service.yaml - prole-configmap.yaml - - prole-deployment.yaml - - prole-service.yaml + - kong-deployment.yaml + - kong-service.yaml - openbao-statefulset.yaml - openbao-service.yaml + - db-manager-deployment.yaml + - db-manager-service.yaml + - postgrest-deployment.yaml + - postgrest-service.yaml + - ingress.yaml diff --git a/deploy/opentofu/k3s/manifests/prole/openbao-statefulset.yaml b/deploy/opentofu/k3s/manifests/prole/openbao-statefulset.yaml index 3ecdfaa..01916e5 100644 --- a/deploy/opentofu/k3s/manifests/prole/openbao-statefulset.yaml +++ b/deploy/opentofu/k3s/manifests/prole/openbao-statefulset.yaml @@ -19,7 +19,7 @@ spec: kubernetes.io/hostname: myrddin.prole.org containers: - name: openbao - image: ghcr.io/openbao/openbao:latest + image: ghcr.io/openbao/openbao:2.0.0 imagePullPolicy: IfNotPresent args: - server diff --git a/deploy/opentofu/k3s/manifests/prole/postgrest-deployment.yaml b/deploy/opentofu/k3s/manifests/prole/postgrest-deployment.yaml new file mode 100644 index 0000000..db6db28 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/postgrest-deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prole-db-postgrest + labels: + app: prole-db-postgrest +spec: + replicas: 1 + selector: + matchLabels: + app: prole-db-postgrest + template: + metadata: + labels: + app: prole-db-postgrest + spec: + containers: + - name: postgrest + image: postgrest/postgrest:v14.5 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: prole-db-postgrest-secrets + key: postgres-password + - name: PGRST_DB_URI + value: "postgres://authenticator:$(POSTGRES_PASSWORD)@prole-db-rw:5432/postgres" + - name: PGRST_DB_SCHEMAS + value: "public,storage,graphql_public" + - name: PGRST_DB_ANON_ROLE + value: "anon" + - name: PGRST_JWT_SECRET + valueFrom: + secretKeyRef: + name: prole-db-postgrest-secrets + key: jwt-secret + - name: PGRST_DB_USE_LEGACY_GUCS + value: "false" + - name: PGRST_APP_SETTINGS_JWT_SECRET + valueFrom: + secretKeyRef: + name: prole-db-postgrest-secrets + key: jwt-secret + - name: PGRST_APP_SETTINGS_JWT_EXP + value: "3600" + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + command: + - postgrest diff --git a/deploy/opentofu/k3s/manifests/prole/postgrest-service.yaml b/deploy/opentofu/k3s/manifests/prole/postgrest-service.yaml new file mode 100644 index 0000000..88afebd --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/postgrest-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: prole-db-postgrest + labels: + app: prole-db-postgrest +spec: + selector: + app: prole-db-postgrest + ports: + - name: http + port: 3000 + targetPort: http + type: ClusterIP diff --git a/deploy/opentofu/k3s/manifests/prole/prole-db-postgres-service.yaml b/deploy/opentofu/k3s/manifests/prole/prole-db-postgres-service.yaml index 7aa8013..2b886b9 100644 --- a/deploy/opentofu/k3s/manifests/prole/prole-db-postgres-service.yaml +++ b/deploy/opentofu/k3s/manifests/prole/prole-db-postgres-service.yaml @@ -12,4 +12,4 @@ spec: - name: postgres port: 5432 targetPort: 5432 - type: ClusterIP + type: LoadBalancer diff --git a/deploy/opentofu/k3s/manifests/prole/prole-db.yaml b/deploy/opentofu/k3s/manifests/prole/prole-db.yaml index 3b5d173..bba175a 100644 --- a/deploy/opentofu/k3s/manifests/prole/prole-db.yaml +++ b/deploy/opentofu/k3s/manifests/prole/prole-db.yaml @@ -4,7 +4,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: registry.default.svc.cluster.local:5000/prole-db:17.7-059 + imageName: k3d-prole-registry:5000/prole-db:17.7-059 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 @@ -88,12 +88,6 @@ spec: selector: matchLabels: prole.storage/role: wal - plugins: - - name: barman-cloud.cloudnative-pg.io - isWALArchiver: true - parameters: - barmanObjectName: prole-db-barman-objectstore - monitoring: enablePodMonitor: false diff --git a/deploy/opentofu/k3s/manifests/prole/supabase-ingress.yaml b/deploy/opentofu/k3s/manifests/prole/supabase-ingress.yaml new file mode 100644 index 0000000..68f9a78 --- /dev/null +++ b/deploy/opentofu/k3s/manifests/prole/supabase-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: supabase-ingress + namespace: supabase + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: supabase.prole.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kong + port: + number: 8000 diff --git a/deploy/opentofu/k3s/opentofu.auto.tfvars b/deploy/opentofu/k3s/opentofu.auto.tfvars index 1054b6a..0c43d10 100644 --- a/deploy/opentofu/k3s/opentofu.auto.tfvars +++ b/deploy/opentofu/k3s/opentofu.auto.tfvars @@ -1,3 +1,3 @@ k3s_server_url = "https://myrddin.prole.org:6443" -k3s_token = "${OPENBAO:kv/prole/prole-db-101/k3s#token}" -namespace = "prole-db-101" +k3s_token = "K107c8c6000488eca4a067d8a73119bbae2f07b4ea1bac7d8d3dc9c500cbb8acb18::server:04572345810eae2f9619a6ed4239702b" +namespace = "prole-db-a0001" diff --git a/etc/final_deployment.sh b/etc/final_deployment.sh index 4fcbc51..62a6ceb 100755 --- a/etc/final_deployment.sh +++ b/etc/final_deployment.sh @@ -1,6 +1,6 @@ #!/bin/bash -# usage: etc/final_deployment.sh [-e|--docker-export] [-c|--helm-chart] [-k|--kustomize] [-o|--output-dir DIR] +# usage: etc/final_deployment.sh [-e|--docker-export] [-c|--helm-chart] [-k|--kustomize] [-p|--prepare-k3s-pipeline] [-o|--output-dir DIR] # # This script handles post-installation deployment tasks for Prole. # 1st use case: -e|--docker-export @@ -12,21 +12,26 @@ # 3rd use case: -k|--kustomize # Creates kustomize deployments (one per component). # Artifacts are written to KUSTOMIZE_DIR. +# 4th use case: -p|--prepare-k3s-pipeline +# Stages all artifacts for k3s delivery via OpenTofu + ArgoCD. +# Copies manifests, generates tfvars, and creates ArgoCD Application manifests. usage() { echo "Usage: $0 [options]" echo "" echo "Options:" - echo " -e, --docker-export Export required docker images to DOCKER_IMPORT_DIR" - echo " -c, --helm-chart Create Helm chart from k8s manifests" - echo " -k, --kustomize Create kustomize deployments from k8s manifests" - echo " -o, --output-dir DIR Base directory for docker-import/, helm-chart/, and kustomize/" + echo " -e, --docker-export Export required docker images to DOCKER_IMPORT_DIR" + echo " -c, --helm-chart Create Helm chart from k8s manifests" + echo " -k, --kustomize Create kustomize deployments from k8s manifests" + echo " -p, --prepare-k3s-pipeline Stage artifacts for k3s delivery (OpenTofu + ArgoCD)" + echo " -o, --output-dir DIR Base directory for docker-import/, helm-chart/, and kustomize/" echo "" } do_docker_export=0 do_helm_chart=0 do_kustomize=0 +do_k3s_pipeline=0 export_dir="" if [[ $# -eq 0 ]]; then @@ -48,6 +53,10 @@ while [[ $# -gt 0 ]]; do do_kustomize=1 shift ;; + -p|--prepare-k3s-pipeline) + do_k3s_pipeline=1 + shift + ;; -o|--output-dir) if [[ -z "${2:-}" ]]; then echo "Missing value for $1" @@ -69,7 +78,7 @@ while [[ $# -gt 0 ]]; do esac done -if [[ $do_docker_export -eq 0 && $do_helm_chart -eq 0 && $do_kustomize -eq 0 ]]; then +if [[ $do_docker_export -eq 0 && $do_helm_chart -eq 0 && $do_kustomize -eq 0 && $do_k3s_pipeline -eq 0 ]]; then usage exit 1 fi @@ -265,3 +274,118 @@ fi if [[ $do_kustomize -eq 1 ]]; then create_kustomize_deployments fi + +prepare_k3s_pipeline() { + echo "========================================" + echo "Preparing k3s pipeline (OpenTofu + ArgoCD)" + echo "========================================" + echo "" + + local pipeline_dir="${PROJECT_ROOT}/deploy/opentofu/k3s" + local manifest_root="${pipeline_dir}/manifests" + local argocd_dir="${pipeline_dir}/argocd" + mkdir -p "${manifest_root}" "${argocd_dir}" + + echo "==> Staging k8s manifests to ${manifest_root}" + local total=0 + for component in prole openbao opentofu; do + local src="${PROJECT_ROOT}/k8s/${component}" + if [[ ! -d "$src" ]]; then + echo " [SKIP] ${component}/ (not found)" + continue + fi + local dst="${manifest_root}/${component}" + mkdir -p "$dst" + local count=0 + for f in "${src}"/*.yaml; do + [[ -f "$f" ]] || continue + cp "$f" "$dst/" + count=$((count + 1)) + done + total=$((total + count)) + echo " [OK] ${component}/ -> manifests/${component}/ (${count} files)" + done + echo " Total manifests staged: ${total}" + + # Stage Supabase manifests if present + local supabase_k8s="${PROJECT_ROOT}/supabase/k8s" + if [[ -d "$supabase_k8s" ]]; then + local dst="${manifest_root}/supabase" + mkdir -p "$dst" + local count=0 + for f in "${supabase_k8s}"/*.yaml; do + [[ -f "$f" ]] || continue + cp "$f" "$dst/" + count=$((count + 1)) + done + if [[ $count -gt 0 ]]; then + echo " [OK] supabase/k8s/ -> manifests/supabase/ (${count} files)" + fi + fi + + # Write tfvars + local k3s_server="${K3S_SERVER_URL:-}" + local k3s_token="${K3S_TOKEN:-}" + local namespace="${NAMESPACE:-default}" + + echo "" + echo "==> Writing opentofu.auto.tfvars" + echo " k3s_server_url = ${k3s_server:-(not set)}" + echo " namespace = ${namespace}" + + cat > "${pipeline_dir}/opentofu.auto.tfvars" < Generating ArgoCD Application manifests in argocd/" + + local repo_url="${PROLE_GIT_REPO:-}" + if [[ -z "$repo_url" ]]; then + repo_url=$(cd "${PROJECT_ROOT}" && git remote get-url origin 2>/dev/null || true) + fi + repo_url="${repo_url:-https://github.com/prole-dev/prole.git}" + local target_revision="${PROLE_GIT_BRANCH:-main}" + + for component in prole openbao opentofu; do + cat > "${argocd_dir}/application-${component}.yaml" < k3s pipeline preparation complete." + echo " Pipeline dir: ${pipeline_dir}" + echo " ArgoCD apps: ${argocd_dir}" + echo "" + echo "Next steps:" + echo " cd ${pipeline_dir} && tofu init && tofu plan && tofu apply" + echo " kubectl apply -f ${argocd_dir}/" +} + +if [[ $do_k3s_pipeline -eq 1 ]]; then + prepare_k3s_pipeline +fi diff --git a/etc/init_common_services.sh b/etc/init_common_services.sh index a713ee9..22ee37a 100755 --- a/etc/init_common_services.sh +++ b/etc/init_common_services.sh @@ -69,6 +69,97 @@ REGISTRY_NS="${REGISTRY_NAMESPACE:-default}" prole_ensure_kubeconfig >/dev/null 2>&1 || true +# --------------------------------------------------------------------------- +# Pre-import container images from $PROLE_DATA/docker-import/*.tar +# --------------------------------------------------------------------------- +# final_deployment.sh exports images as +# $PROLE_DATA/docker-import/.tar +# where safe_name = image ref with / and : replaced by _. +# Loading these tars into the local container runtime before the sub-scripts +# deploy pods avoids slow image pulls from remote registries. +# --------------------------------------------------------------------------- + +_resolve_prole_data() { + local d="${PROLE_DATA:-}" + if [[ -z "$d" ]]; then + if [[ -f "$SCRIPT_DIR/../env.sh" ]]; then + # shellcheck disable=SC1090 + source "$SCRIPT_DIR/../env.sh" + d="${PROLE_DATA:-}" + fi + fi + echo "${d:-}" +} + +_preload_docker_images() { + local prole_data + prole_data=$(_resolve_prole_data) + local import_dir="${DOCKER_IMPORT_DIR:-${prole_data:+${prole_data}/docker-import}}" + + if [[ -z "$import_dir" || ! -d "$import_dir" ]]; then + echo "[SKIP] No docker-import directory found; images will be pulled from registries." + return 0 + fi + + local tar_files + tar_files=$(find "$import_dir" -maxdepth 1 -name '*.tar' -type f 2>/dev/null || true) + if [[ -z "$tar_files" ]]; then + echo "[SKIP] docker-import directory exists but contains no .tar files." + return 0 + fi + + local mode="${PROLE_MODE:-k3d}" + echo "Pre-loading container images from $import_dir (mode=$mode) ..." + + case "$mode" in + k3d) + if ! command -v k3d >/dev/null 2>&1; then + echo "WARN: k3d not available; skipping docker-import pre-load." >&2 + return 0 + fi + local cluster_name="${K3D_CLUSTER_NAME:-prole-dev-cluster}" + local tar_file + while IFS= read -r tar_file; do + echo " Importing $(basename "$tar_file") into k3d cluster '$cluster_name' ..." + if k3d image import "$tar_file" -c "$cluster_name" >/dev/null 2>&1; then + echo " [OK] $(basename "$tar_file")" + else + echo " [WARN] Failed to import $(basename "$tar_file"); image will be pulled at deploy time." >&2 + fi + done <<< "$tar_files" + ;; + k3s) + # For k3s we import directly via ctr on the local node (assumes this + # script runs on a k3s server node or has access to the containerd socket). + local ctr_sock="/run/k3s/containerd/containerd.sock" + if [[ ! -S "$ctr_sock" ]]; then + echo "WARN: k3s containerd socket not found at $ctr_sock; skipping docker-import pre-load." >&2 + return 0 + fi + local tar_file + while IFS= read -r tar_file; do + echo " Importing $(basename "$tar_file") into k3s containerd ..." + if ctr -n k8s.io -a "$ctr_sock" images import --no-unpack "$tar_file" >/dev/null 2>&1; then + echo " [OK] $(basename "$tar_file")" + else + echo " [WARN] Failed to import $(basename "$tar_file"); image will be pulled at deploy time." >&2 + fi + done <<< "$tar_files" + ;; + *) + echo "WARN: Unknown mode '$mode'; skipping docker-import pre-load." >&2 + ;; + esac + + echo "Pre-load complete." +} + +case "$ACTION" in + start|update|reload|initialize|restart) + _preload_docker_images + ;; +esac + if [ -x "$SCRIPT_DIR/init_service_layer.sh" ]; then if [[ "$ENABLE_KERBEROS" == "1" ]]; then "$SCRIPT_DIR/init_service_layer.sh" -n "$NS" -k "$ACTION" @@ -167,13 +258,13 @@ case "$ACTION" in esac if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then - "$SCRIPT_DIR/init_openbao.sh" -n "$NS" "$ACTION" || rc=$? + ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" "$SCRIPT_DIR/init_openbao.sh" -n "$NS" "$ACTION" || rc=$? else echo "WARN: init_openbao.sh not found; skipping OpenBao." fi if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then - "$SCRIPT_DIR/init_opentofu.sh" -n "$NS" "$ACTION" || rc=$? + ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" "$SCRIPT_DIR/init_opentofu.sh" -n "$NS" "$ACTION" || rc=$? else echo "WARN: init_opentofu.sh not found; skipping OpenTofu." fi @@ -190,7 +281,7 @@ if [ -x "$SCRIPT_DIR/init_garage_store.sh" ]; then case "$garage_action" in update|reload|initialize) garage_action="start" ;; esac - NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" GARAGE_NAMESPACE="$NS" \ + NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" GARAGE_NAMESPACE="$NS" ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" \ "$SCRIPT_DIR/init_garage_store.sh" "$garage_action" || rc=$? else echo "WARN: init_garage_store.sh not found; garage deploy skipped." diff --git a/etc/init_db_manager.sh b/etc/init_db_manager.sh new file mode 100755 index 0000000..b0f6928 --- /dev/null +++ b/etc/init_db_manager.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -euo pipefail +# init_db_manager.sh +# Purpose: +# - Build and deploy the prole-db-manager Node.js REST endpoint +# - Provides /backup//full to trigger barman full backups into Garage +# - Deploys to the prole-db namespace; accessible via Kong endpoint /backup +# - Deploys to the current cluster (k3d or k3s) based on PROLE_MODE from conf/prole.cfg +# +# Usage: +# ./init_db_manager.sh [--mode MODE] + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/prole_cfg.sh" + +if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then + prole_set_mode "${2:-}" + shift 2 +elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then + prole_set_mode "${1#*=}" + shift +fi + +ACTION=${1:-start} +NAMESPACE=${NAMESPACE:-prole-db} +PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} +DB_MANAGER_IMAGE=${DB_MANAGER_IMAGE:-prole-db-manager:latest} +DB_MANAGER_NAME=${DB_MANAGER_NAME:-prole-db-manager} +DB_MANAGER_PORT=${DB_MANAGER_PORT:-80} +DB_MANAGER_SRC=${DB_MANAGER_SRC:-$PROLE_HOME/src/db-manager} +REGISTRY=${REGISTRY:-} + +usage() { + cat < + +Actions: + start Build image and deploy db-manager to current cluster (k3d/k3s) + stop Remove db-manager deployment + status Show db-manager pod/service status + restart Restart db-manager pods + build Build the container image only +USAGE + exit 1 +} + +ensure_tools() { + for t in kubectl docker; do + command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; } + done +} + +ensure_namespace() { + if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + echo "Creating namespace '$NAMESPACE' ..." + kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true + fi +} + +build_image() { + echo "Building $DB_MANAGER_IMAGE from $DB_MANAGER_SRC ..." + docker build -t "$DB_MANAGER_IMAGE" "$DB_MANAGER_SRC" + + if [[ -n "$REGISTRY" ]]; then + local remote_tag="${REGISTRY}/${DB_MANAGER_IMAGE}" + echo "Pushing $remote_tag ..." + docker tag "$DB_MANAGER_IMAGE" "$remote_tag" + docker push "$remote_tag" + fi +} + +import_image_k3d() { + if ! command -v k3d >/dev/null 2>&1; then + echo "WARN: k3d not available; skipping k3d image import." >&2 + return 0 + fi + + local cluster_name="${K3D_CLUSTER_NAME:-prole-dev-cluster}" + echo "Importing $DB_MANAGER_IMAGE into k3d cluster '$cluster_name' ..." + k3d image import "$DB_MANAGER_IMAGE" -c "$cluster_name" >/dev/null 2>&1 || true + echo "Image imported to k3d cluster successfully." +} + +import_image_k3s() { + if ! command -v ansible >/dev/null 2>&1; then + echo "WARN: ansible not available; skipping k3s image import (image must already exist on nodes)." >&2 + return 0 + fi + + local export_dir="${DOCKER_IMPORT_DIR:-/tmp}" + local safe_name + safe_name=$(echo "$DB_MANAGER_IMAGE" | sed 's/\//_/g; s/:/_/g') + local tar_path="${export_dir%/}/${safe_name}.tar" + local ansible_cfg="$PROLE_HOME/ansible.cfg" + local vault_file="${ANSIBLE_VAULT_PASSWORD_FILE:-}" + local dest="/tmp/$(basename "$tar_path")" + local ansible_args=("ansible" "k3s_hosts") + + mkdir -p "$export_dir" + echo "Exporting $DB_MANAGER_IMAGE to $tar_path ..." + docker save -o "$tar_path" "$DB_MANAGER_IMAGE" + + if [[ -z "$vault_file" && -f "$PROLE_HOME/.vault_pass" ]]; then + vault_file="$PROLE_HOME/.vault_pass" + fi + if [[ -n "$vault_file" ]]; then + ansible_args+=("--vault-password-file" "$vault_file") + fi + + echo "Importing $DB_MANAGER_IMAGE into k3s nodes ..." + if [[ -f "$ansible_cfg" ]]; then + ANSIBLE_CONFIG="$ansible_cfg" "${ansible_args[@]}" -m copy -a "src=$tar_path dest=$dest mode=0644" || { echo "ERROR: Failed to copy image to k3s nodes." >&2; return 1; } + ANSIBLE_CONFIG="$ansible_cfg" "${ansible_args[@]}" -m shell -a "ctr -n k8s.io -a /run/k3s/containerd/containerd.sock images import --no-unpack $dest" || { echo "ERROR: Failed to import image into k3s nodes." >&2; return 1; } + else + "${ansible_args[@]}" -m copy -a "src=$tar_path dest=$dest mode=0644" || { echo "ERROR: Failed to copy image to k3s nodes." >&2; return 1; } + "${ansible_args[@]}" -m shell -a "ctr -n k8s.io -a /run/k3s/containerd/containerd.sock images import --no-unpack $dest" || { echo "ERROR: Failed to import image into k3s nodes." >&2; return 1; } + fi + echo "Image imported to k3s nodes successfully." +} + +deploy() { + echo "Deploying $DB_MANAGER_NAME to namespace '$NAMESPACE' ..." + + local manifests_dir="$PROLE_HOME/deploy/opentofu/k3s/manifests/prole" + + # Only deploy db-manager resources — the full kustomization includes services + # (garage, openbao, prole, etc.) that belong in the service namespace and would + # create duplicates if applied here. + kubectl apply -f "$manifests_dir/db-manager-deployment.yaml" -n "$NAMESPACE" + kubectl apply -f "$manifests_dir/db-manager-service.yaml" -n "$NAMESPACE" + + # Apply cross-namespace ingresses separately (they have their own namespace metadata) + # Only apply if the target namespace already exists to avoid errors + if [[ -f "$manifests_dir/dashboard-ingress.yaml" ]] && kubectl get namespace kubernetes-dashboard >/dev/null 2>&1; then + kubectl apply -f "$manifests_dir/dashboard-ingress.yaml" || true + fi + if [[ -f "$manifests_dir/supabase-ingress.yaml" ]] && kubectl get namespace supabase >/dev/null 2>&1; then + kubectl apply -f "$manifests_dir/supabase-ingress.yaml" || true + fi + + echo "Waiting for $DB_MANAGER_NAME rollout ..." + kubectl rollout status deployment/"$DB_MANAGER_NAME" -n "$NAMESPACE" --timeout=120s + echo "$DB_MANAGER_NAME deployed successfully." +} + +stop() { + echo "Removing $DB_MANAGER_NAME from namespace '$NAMESPACE' ..." + kubectl delete deployment "$DB_MANAGER_NAME" -n "$NAMESPACE" --ignore-not-found=true + kubectl delete service "$DB_MANAGER_NAME" -n "$NAMESPACE" --ignore-not-found=true + echo "$DB_MANAGER_NAME removed." +} + +status() { + echo "=== $DB_MANAGER_NAME pods ===" + kubectl get pods -n "$NAMESPACE" -l app="$DB_MANAGER_NAME" 2>/dev/null || echo "No pods found" + echo "" + echo "=== $DB_MANAGER_NAME service ===" + kubectl get svc "$DB_MANAGER_NAME" -n "$NAMESPACE" 2>/dev/null || echo "No service found" +} + +restart() { + echo "Restarting $DB_MANAGER_NAME ..." + kubectl rollout restart deployment/"$DB_MANAGER_NAME" -n "$NAMESPACE" + kubectl rollout status deployment/"$DB_MANAGER_NAME" -n "$NAMESPACE" --timeout=120s + echo "$DB_MANAGER_NAME restarted." +} + +case "$ACTION" in + start) + ensure_tools + ensure_namespace + build_image + case "${PROLE_MODE:-k3d}" in + k3d) import_image_k3d ;; + k3s) import_image_k3s ;; + *) echo "WARN: Unknown mode '${PROLE_MODE:-}'; skipping image import." >&2 ;; + esac + deploy + ;; + stop) + ensure_tools + stop + ;; + status) + ensure_tools + status + ;; + restart) + ensure_tools + restart + ;; + build) + ensure_tools + build_image + ;; + *) + usage + ;; +esac diff --git a/etc/init_garage_store.sh b/etc/init_garage_store.sh index e6b3c47..fee4a70 100755 --- a/etc/init_garage_store.sh +++ b/etc/init_garage_store.sh @@ -291,7 +291,7 @@ dump_debug() { wait_ready() { echo "Waiting for Garage StatefulSet to become ready ..." - if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout=180s; then + if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s}; then echo "ERROR: Garage StatefulSet did not become ready in time." >&2 dump_debug return 1 diff --git a/etc/init_k8s.sh b/etc/init_k8s.sh index 05616fc..2df81e6 100755 --- a/etc/init_k8s.sh +++ b/etc/init_k8s.sh @@ -154,12 +154,27 @@ initialize() { if [[ -n "$data_dir" ]]; then volume_args=(--volume "${data_dir}:/var/lib/rancher/k3s/storage@all") fi + # Generate a registries.yaml so k3d nodes can pull from the local + # registry over plain HTTP (insecure). The file is written to a + # temp location and passed via --registry-config at creation time. + registry_config_args=() + local reg_cfg + reg_cfg=$(mktemp "${TMPDIR:-/tmp}/k3d-registries-XXXXXX.yaml") + cat > "$reg_cfg" <<'REGEOF' +mirrors: + "k3d-prole-registry:5000": + endpoint: + - "http://k3d-prole-registry:5000" +REGEOF + registry_config_args=(--registry-config "$reg_cfg") log "Creating k3d cluster '$CLUSTER_NAME'..." k3d cluster create "$CLUSTER_NAME" \ --api-port "0.0.0.0:${CLUSTER_PORT_DEFAULT}" \ --agents 2 \ -p "0.0.0.0:${CLUSTER_PORT_DEFAULT}:6443@server:0" \ - "${volume_args[@]}" + "${volume_args[@]}" \ + "${registry_config_args[@]}" + rm -f "$reg_cfg" 2>/dev/null || true fi else if [[ -n "$HOST" ]]; then diff --git a/etc/init_kong.sh b/etc/init_kong.sh new file mode 100755 index 0000000..e3b29ac --- /dev/null +++ b/etc/init_kong.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +set -euo pipefail +# init_kong.sh +# Purpose: +# - Deploy Kong API Gateway (DB-less) into the prole-db namespace +# - Replaces the prole nginx deployment as the API endpoint +# - Routes /backup/* to prole-db-manager and /rest/* to prole-db-postgrest +# - Creates the kong declarative config as a ConfigMap +# - Applies the kong deployment and service manifests +# - Provides start/stop/status/restart actions +# +# Usage: +# ./init_kong.sh [--mode MODE] [-n NAMESPACE] + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/prole_cfg.sh" + +if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then + prole_set_mode "${2:-}" + shift 2 +elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then + prole_set_mode "${1#*=}" + shift +fi + +NAMESPACE="${NAMESPACE:-default}" + +while [[ $# -gt 0 ]]; do + case "$1" in + -n|--namespace) + shift + if [[ -z "${1:-}" ]]; then + echo "ERROR: -n/--namespace requires a value" >&2 + exit 2 + fi + NAMESPACE="$1" + ;; + -n=*|--namespace=*) + NAMESPACE="${1#*=}" + ;; + start|stop|status|restart) + ACTION="$1" + ;; + -h|--help) + usage + ;; + *) + echo "ERROR: Unknown argument: $1" >&2 + exit 2 + ;; + esac + shift +done + +ACTION="${ACTION:-start}" +PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} +KONG_IMAGE="${KONG_IMAGE:-kong:3.9}" +KONG_NAME="${KONG_NAME:-prole-db-kong}" +KONG_PROXY_PORT="${KONG_PROXY_PORT:-8000}" +KONG_ADMIN_PORT="${KONG_ADMIN_PORT:-8001}" +KONG_CONFIG_NAME="${KONG_CONFIG_NAME:-prole-db-kong-config}" + +# Upstream service defaults +POSTGREST_SERVICE="${POSTGREST_SERVICE:-prole-db-postgrest}" +POSTGREST_PORT="${POSTGREST_PORT:-3000}" +DB_MANAGER_SERVICE="${DB_MANAGER_SERVICE:-prole-db-manager}" +DB_MANAGER_PORT="${DB_MANAGER_PORT:-80}" + +usage() { + cat < + +Actions: + start Create ConfigMap and deploy Kong to the prole-db namespace + stop Remove Kong deployment, service, and ConfigMap + status Show Kong pod/service status + restart Restart Kong pods +USAGE + exit 1 +} + +ensure_tools() { + for t in kubectl; do + command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; } + done +} + +ensure_namespace() { + if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + echo "Creating namespace '$NAMESPACE' ..." + kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true + fi +} + +create_kong_config() { + echo "Creating/updating Kong declarative config '$KONG_CONFIG_NAME' in namespace '$NAMESPACE' ..." + + local kong_yml + kong_yml=$(cat </dev/null || echo "No pods found" + echo "" + echo "=== $KONG_NAME service ===" + kubectl get svc "$KONG_NAME" -n "$NAMESPACE" 2>/dev/null || echo "No service found" +} + +restart() { + echo "Restarting $KONG_NAME ..." + kubectl rollout restart deployment/"$KONG_NAME" -n "$NAMESPACE" + kubectl rollout status deployment/"$KONG_NAME" -n "$NAMESPACE" --timeout=120s + echo "$KONG_NAME restarted." +} + +case "$ACTION" in + start) + ensure_tools + ensure_namespace + create_kong_config + deploy + ;; + stop) + ensure_tools + stop + ;; + status) + ensure_tools + status + ;; + restart) + ensure_tools + restart + ;; + *) + usage + ;; +esac diff --git a/etc/init_openbao.sh b/etc/init_openbao.sh index 6465c29..ac72d48 100755 --- a/etc/init_openbao.sh +++ b/etc/init_openbao.sh @@ -436,9 +436,9 @@ apply_openbao_host_network_patch() { wait_for_openbao() { echo "Waiting for OpenBao to become ready ..." if kubectl get statefulset/$OPENBAO_NAME -n "$OPENBAO_RESOURCE_NAMESPACE" >/dev/null 2>&1; then - kubectl rollout status statefulset/$OPENBAO_NAME -n "$OPENBAO_RESOURCE_NAMESPACE" --timeout=120s + kubectl rollout status statefulset/$OPENBAO_NAME -n "$OPENBAO_RESOURCE_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} else - kubectl rollout status deploy/$OPENBAO_NAME -n "$OPENBAO_RESOURCE_NAMESPACE" --timeout=120s + kubectl rollout status deploy/$OPENBAO_NAME -n "$OPENBAO_RESOURCE_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} fi } diff --git a/etc/init_opentofu.sh b/etc/init_opentofu.sh index b9e6200..e7b4619 100755 --- a/etc/init_opentofu.sh +++ b/etc/init_opentofu.sh @@ -160,9 +160,19 @@ resolve_admin_password() { fi fi + # Fallback: read password from existing Kubernetes secret if [[ -z "${OPENTOFU_ADMIN_PASSWORD:-}" ]]; then - echo "ERROR: OPENTOFU_ADMIN_PASSWORD/DB_PASSWORD is missing." >&2 - exit 1 + local k8s_pw + k8s_pw=$(kubectl get secret opentofu-admin -n "$OPENTOFU_NAMESPACE" \ + -o jsonpath='{.data.admin_password}' 2>/dev/null | base64 -d 2>/dev/null || true) + if [[ -n "$k8s_pw" ]]; then + OPENTOFU_ADMIN_PASSWORD="$k8s_pw" + fi + fi + + if [[ -z "${OPENTOFU_ADMIN_PASSWORD:-}" ]]; then + echo "WARN: OPENTOFU_ADMIN_PASSWORD/DB_PASSWORD is not available; skipping OpenTofu secret creation." >&2 + return 1 fi } @@ -185,7 +195,7 @@ ensure_opentofu_secret() { apply_k8s() { echo "Applying OpenTofu manifest to namespace '$OPENTOFU_NAMESPACE' ..." prole_render_manifest "$OPENTOFU_MANIFEST_DIR/deployment.yaml" | kubectl apply -n "$OPENTOFU_NAMESPACE" -f - - kubectl rollout status deploy/$OPENTOFU_NAME -n "$OPENTOFU_NAMESPACE" --timeout=120s || true + kubectl rollout status deploy/$OPENTOFU_NAME -n "$OPENTOFU_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true } delete_k8s() { @@ -203,8 +213,9 @@ case "${ACTION:-}" in start|initialize|update|reload|restart) ensure_tools ensure_namespace - resolve_admin_password - ensure_opentofu_secret + if resolve_admin_password; then + ensure_opentofu_secret + fi apply_k8s prole_register_port_forward "opentofu" "${OPENTOFU_NAMESPACE:-default}" "svc/opentofu" "8080" "8080" "0.0.0.0" "TCP" "OpenTofu" ;; diff --git a/etc/init_postgrest.sh b/etc/init_postgrest.sh new file mode 100755 index 0000000..f3a714c --- /dev/null +++ b/etc/init_postgrest.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +set -euo pipefail +# init_postgrest.sh +# Purpose: +# - Deploy PostgREST (postgrest/postgrest) into the prole-db namespace +# - Creates the prole-db-postgrest-secrets k8s secret (postgres password + JWT secret) +# - Applies the postgrest deployment and service manifests +# - Provides start/stop/status/restart actions +# +# Usage: +# ./init_postgrest.sh [--mode MODE] [-n NAMESPACE] +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/prole_cfg.sh" + +if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then + prole_set_mode "${2:-}" + shift 2 +elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then + prole_set_mode "${1#*=}" + shift +fi + +NAMESPACE="${NAMESPACE:-prole-db}" + +while [[ $# -gt 0 ]]; do + case "$1" in + -n|--namespace) + shift + if [[ -z "${1:-}" ]]; then + echo "ERROR: -n/--namespace requires a value" >&2 + exit 2 + fi + NAMESPACE="$1" + ;; + -n=*|--namespace=*) + NAMESPACE="${1#*=}" + ;; + start|stop|status|restart) + ACTION="$1" + ;; + -h|--help) + usage + ;; + *) + echo "ERROR: Unknown argument: $1" >&2 + exit 2 + ;; + esac + shift +done + +ACTION="${ACTION:-start}" +PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} +POSTGREST_IMAGE="${POSTGREST_IMAGE:-postgrest/postgrest:v14.5}" +POSTGREST_NAME="${POSTGREST_NAME:-prole-db-postgrest}" +POSTGREST_PORT="${POSTGREST_PORT:-3000}" +PROLE_DB_SERVICE="${PROLE_DB_SERVICE:-prole-db-rw}" +POSTGREST_SECRET_NAME="${POSTGREST_SECRET_NAME:-prole-db-postgrest-secrets}" + +usage() { + cat < + +Actions: + start Create secrets and deploy PostgREST to the prole-db namespace + stop Remove PostgREST deployment and secrets + status Show PostgREST pod/service status + restart Restart PostgREST pods +USAGE + exit 1 +} + +ensure_tools() { + for t in kubectl; do + command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; } + done +} + +ensure_namespace() { + if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + echo "Creating namespace '$NAMESPACE' ..." + kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true + fi +} + +resolve_postgres_password() { + # Try to read the postgres password from the CloudNativePG superuser secret + local cnpg_secret="prole-db-superuser" + local pw="" + + if kubectl get secret "$cnpg_secret" -n "$NAMESPACE" >/dev/null 2>&1; then + pw=$(kubectl get secret "$cnpg_secret" -n "$NAMESPACE" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) + fi + + if [[ -z "$pw" ]]; then + # Fallback: check for POSTGRES_PASSWORD env var or prole.cfg + pw="${POSTGRES_PASSWORD:-}" + fi + + if [[ -z "$pw" ]]; then + echo "ERROR: Cannot resolve postgres password. Ensure prole-db-superuser secret exists in namespace '$NAMESPACE' or set POSTGRES_PASSWORD." >&2 + exit 1 + fi + + echo "$pw" +} + +resolve_jwt_secret() { + # Try to read JWT secret from supabase namespace secrets + local jwt="" + + if kubectl get secret supabase-jwt -n supabase >/dev/null 2>&1; then + jwt=$(kubectl get secret supabase-jwt -n supabase -o jsonpath='{.data.jwt-secret}' 2>/dev/null | base64 -d 2>/dev/null || true) + fi + + if [[ -z "$jwt" ]]; then + # Fallback: try reading from supabase .env file + local env_file="${DEV_HOME:-$HOME/dev}/supabase/docker/.env" + if [[ -f "$env_file" ]]; then + jwt=$(grep -E '^JWT_SECRET=' "$env_file" | head -1 | cut -d= -f2- | tr -d "'\"" || true) + fi + fi + + if [[ -z "$jwt" ]]; then + jwt="${JWT_SECRET:-}" + fi + + if [[ -z "$jwt" ]]; then + echo "ERROR: Cannot resolve JWT secret. Set JWT_SECRET or ensure supabase-jwt secret exists." >&2 + exit 1 + fi + + echo "$jwt" +} + +create_secrets() { + echo "Resolving secrets for PostgREST ..." + local pg_password + local jwt_secret + pg_password=$(resolve_postgres_password) + jwt_secret=$(resolve_jwt_secret) + + echo "Creating/updating secret '$POSTGREST_SECRET_NAME' in namespace '$NAMESPACE' ..." + kubectl create secret generic "$POSTGREST_SECRET_NAME" \ + --namespace="$NAMESPACE" \ + --from-literal=postgres-password="$pg_password" \ + --from-literal=jwt-secret="$jwt_secret" \ + --dry-run=client -o yaml | kubectl apply -f - + + echo "Secret '$POSTGREST_SECRET_NAME' ready." +} + +deploy() { + echo "Deploying $POSTGREST_NAME to namespace '$NAMESPACE' ..." + local manifests_dir="$PROLE_HOME/deploy/opentofu/k3s/manifests/prole" + + kubectl apply -f "$manifests_dir/postgrest-deployment.yaml" -n "$NAMESPACE" + kubectl apply -f "$manifests_dir/postgrest-service.yaml" -n "$NAMESPACE" + + echo "Waiting for $POSTGREST_NAME rollout ..." + kubectl rollout status deployment/"$POSTGREST_NAME" -n "$NAMESPACE" --timeout=120s + echo "$POSTGREST_NAME deployed successfully." +} + +stop() { + echo "Removing $POSTGREST_NAME from namespace '$NAMESPACE' ..." + kubectl delete deployment "$POSTGREST_NAME" -n "$NAMESPACE" --ignore-not-found=true + kubectl delete service "$POSTGREST_NAME" -n "$NAMESPACE" --ignore-not-found=true + kubectl delete secret "$POSTGREST_SECRET_NAME" -n "$NAMESPACE" --ignore-not-found=true + echo "$POSTGREST_NAME removed." +} + +status() { + echo "=== $POSTGREST_NAME pods ===" + kubectl get pods -n "$NAMESPACE" -l app="$POSTGREST_NAME" 2>/dev/null || echo "No pods found" + echo "" + echo "=== $POSTGREST_NAME service ===" + kubectl get svc "$POSTGREST_NAME" -n "$NAMESPACE" 2>/dev/null || echo "No service found" +} + +restart() { + echo "Restarting $POSTGREST_NAME ..." + kubectl rollout restart deployment/"$POSTGREST_NAME" -n "$NAMESPACE" + kubectl rollout status deployment/"$POSTGREST_NAME" -n "$NAMESPACE" --timeout=120s + echo "$POSTGREST_NAME restarted." +} + +case "$ACTION" in + start) + ensure_tools + ensure_namespace + create_secrets + deploy + ;; + stop) + ensure_tools + stop + ;; + status) + ensure_tools + status + ;; + restart) + ensure_tools + restart + ;; + *) + usage + ;; +esac diff --git a/etc/init_prole-db-backup.sh b/etc/init_prole-db-backup.sh index a56ac23..531a1a6 100755 --- a/etc/init_prole-db-backup.sh +++ b/etc/init_prole-db-backup.sh @@ -50,9 +50,9 @@ usage() { Usage: $0 [start|backup|status] Actions: - start Configure Garage-backed backups and run initial backup - backup Trigger a new backup now - status Show backup resources + start Configure Garage-backed backups and run initial backup + backup [full|incr] Trigger a new backup now (default: full) + status Show backup resources USAGE exit 1 } @@ -239,10 +239,28 @@ remove_native_barman_config() { } trigger_backup() { + local backup_type="${1:-full}" local backup_name backup_name="${CNPG_CLUSTER_NAME}-backup-$(date +%Y%m%d%H%M%S)" - echo "Triggering backup $backup_name ..." - kubectl apply -n "$NAMESPACE" -f - < /etc/rancher/k3s/registries.yaml <<'EOF' +${registries_yaml} +EOF" || { echo "WARN: failed to write registries.yaml to $node" >&2; continue; } + # Restart k3s inside the node so it picks up the new mirror config. + docker restart "$node" >/dev/null 2>&1 || true + echo " configured $node" + done + # Wait briefly for nodes to come back up + sleep 5 + echo "Waiting for k3d nodes to become Ready ..." + kubectl wait --for=condition=Ready nodes --all --timeout=${ROLLOUT_TIMEOUT:-300s} 2>/dev/null || true } apply_registry() { @@ -252,6 +280,8 @@ apply_registry() { fi fi fi + # Configure k3d nodes to use the registry over plain HTTP (insecure). + configure_k3d_insecure_registry "$cluster_name" # Ensure no k8s registry resources remain in k3d clusters. echo "Removing k8s registry resources (k3d uses k3d-prole-registry only) ..." cleanup_registry_k8s @@ -281,7 +311,7 @@ apply_registry() { echo "Applying registry manifest to namespace '$REGISTRY_NAMESPACE' ..." render_registry_manifest | kubectl apply --server-side --force-conflicts --field-manager=prole-installer --validate=false -n "$REGISTRY_NAMESPACE" -f - - kubectl rollout status deploy/registry -n "$REGISTRY_NAMESPACE" --timeout=120s || true + kubectl rollout status deploy/registry -n "$REGISTRY_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true } delete_k8s() { diff --git a/etc/init_supabase_ports.sh b/etc/init_supabase_ports.sh index 5b52495..f97dd7c 100755 --- a/etc/init_supabase_ports.sh +++ b/etc/init_supabase_ports.sh @@ -212,18 +212,14 @@ if kubectl -n "$SUPABASE_NAMESPACE" get svc "$SUPABASE_POSTGRES_SERVICE" >/dev/n supabase_pg_type=$(kubectl -n "$SUPABASE_NAMESPACE" get svc "$SUPABASE_POSTGRES_SERVICE" -o jsonpath='{.spec.type}' 2>/dev/null || true) fi -if [[ "$supabase_pg_exists" -eq 1 && "$supabase_pg_type" != "ClusterIP" ]]; then - log "Replacing service $SUPABASE_NAMESPACE/$SUPABASE_POSTGRES_SERVICE with ClusterIP on ${SUPABASE_POSTGRES_PORT}" +if [[ "$supabase_pg_exists" -eq 1 && "$supabase_pg_type" != "ExternalName" ]]; then + log "Replacing service $SUPABASE_NAMESPACE/$SUPABASE_POSTGRES_SERVICE with ExternalName on ${SUPABASE_POSTGRES_PORT}" kubectl -n "$SUPABASE_NAMESPACE" delete svc "$SUPABASE_POSTGRES_SERVICE" --ignore-not-found >/dev/null 2>&1 || true supabase_pg_exists=0 fi -selector_yaml="$(selector_pairs_to_yaml "$DB_SELECTOR")" -if [[ -z "$selector_yaml" ]]; then - warn "Unable to determine selector for $SUPABASE_POSTGRES_SERVICE; skipping creation/update." -else - log "Ensuring $SUPABASE_NAMESPACE/$SUPABASE_POSTGRES_SERVICE uses port ${SUPABASE_POSTGRES_PORT}" - cat < int: +def _prepare_k3s_pipeline(controller: ProleController, log_fn=None, skip_validation: bool = False) -> int: + """Prepare OpenTofu k3s pipeline: stage manifests, tfvars, ArgoCD apps. + + When *skip_validation* is True the silent-install-test / repair loop is + skipped entirely — the caller (e.g. a successful silent install) already + validated the environment. The main prole.cfg is **not** overwritten in + this case so it keeps whatever cluster_env the install used. + """ + def _log(msg): + if log_fn: + try: log_fn(msg) + except: print(msg, end='', flush=True) + else: + print(msg, end='', flush=True) + project_root = controller.project_root cfg_path = project_root / "conf" / "prole.cfg" + _log("==> Preparing k3s pipeline\n") + _log(f" Project root: {project_root}\n") + _log(f" Config: {cfg_path}\n\n") + + _log("==> Detecting Ansible topology...\n") info = _detect_ansible_topology(project_root) k3s_server = (info.get('k3s_server_url') or '').strip() if info else '' k3s_token = (info.get('k3s_token') or '').strip() if info else '' if k3s_server and not k3s_server.startswith('http'): k3s_server = f"https://{k3s_server}" + _log(f" k3s_server: {k3s_server or '(not detected)'}\n") + _log(f" k3s_token: {'detected' if k3s_token else '(not detected)'}\n\n") + # Load config to resolve namespace and secrets for pipeline tfvars + _log("==> Loading installer config...\n") installer = ProleSilentInstaller(controller, str(cfg_path)) try: existing_inputs = installer._load_inputs_from_cfg() @@ -2627,64 +2655,93 @@ def _prepare_k3s_pipeline(controller: ProleController) -> int: existing_inputs = {} installer.inputs = {**installer._default_inputs(), **existing_inputs} - # Override for k3s pipeline defaults - installer.inputs['init_cluster.cluster_env'] = 'prole-service-cluster' - installer.inputs['init_cluster.supabase_enabled'] = _bool_str(False) - installer.inputs['init_cluster.kerberos_enabled'] = _bool_str(False) - installer.inputs['init_cluster.at_rest_encryption_enabled'] = _bool_str(True) - installer.inputs['kerberos_config.enabled'] = _bool_str(False) - installer.inputs['kerberos_config.test_connection'] = _bool_str(False) if k3s_server and not installer.inputs.get('init_cluster.k3s_server_url'): installer.inputs['init_cluster.k3s_server_url'] = k3s_server if k3s_token and not installer.inputs.get('init_cluster.k3s_token'): installer.inputs['init_cluster.k3s_token'] = k3s_token - installer.inputs['env_setup.PROLE_HOME'] = str(project_root) - installer.inputs['env_setup.PROLE_CONF'] = str(project_root / "conf") - installer.inputs['env_setup.PROLE_DATA'] = ( - installer._resolve_env_value('PROLE_DATA', str(Path.home() / '.prole' / 'data')) - or str(Path.home() / '.prole' / 'data') - ) - installer.inputs['env_setup.PROLE_LOGS'] = ( - installer._resolve_env_value('PROLE_LOGS', str(Path.home() / '.prole' / 'logs')) - or str(Path.home() / '.prole' / 'logs') - ) - installer.inputs['env_setup.PROLE_SERVICE'] = str(project_root / "etc") - - installer._write_cfg() - namespace = (installer._get_input('init_password.db_namespace', '') or '').strip() if not namespace: namespace = (installer._get_input('env_setup.NAMESPACE', '') or '').strip() or 'default' + _log(f" Target namespace: {namespace}\n") + _log(" [OK] Config loaded\n\n") - db_password = installer._get_input('init_password.db_password', '').strip() + if not skip_validation: + # Standalone mode: write a pipeline-specific cfg and validate + installer.inputs['init_cluster.cluster_env'] = 'prole-service-cluster' + installer.inputs['init_cluster.supabase_enabled'] = _bool_str(False) + installer.inputs['init_cluster.kerberos_enabled'] = _bool_str(False) + installer.inputs['init_cluster.at_rest_encryption_enabled'] = _bool_str(True) + installer.inputs['kerberos_config.enabled'] = _bool_str(False) + installer.inputs['kerberos_config.test_connection'] = _bool_str(False) - attempt = 1 - total_attempts = 0 - max_attempts_env = os.environ.get("PROLE_SILENT_TEST_MAX_ATTEMPTS", "").strip() - max_attempts = int(max_attempts_env) if max_attempts_env.isdigit() else 0 + installer.inputs['env_setup.PROLE_HOME'] = str(project_root) + installer.inputs['env_setup.PROLE_CONF'] = str(project_root / "conf") + installer.inputs['env_setup.PROLE_DATA'] = ( + installer._resolve_env_value('PROLE_DATA', str(Path.home() / '.prole' / 'data')) + or str(Path.home() / '.prole' / 'data') + ) + installer.inputs['env_setup.PROLE_LOGS'] = ( + installer._resolve_env_value('PROLE_LOGS', str(Path.home() / '.prole' / 'logs')) + or str(Path.home() / '.prole' / 'logs') + ) + installer.inputs['env_setup.PROLE_SERVICE'] = str(project_root / "etc") - while True: - total_attempts += 1 - rc = _run_silent_install_test(project_root, cfg_path) - if rc == 0: - break + # Ensure required keys that the silent install test validates + if not installer.inputs.get('env_setup.NAMESPACE'): + installer.inputs['env_setup.NAMESPACE'] = installer.inputs.get('init_password.db_namespace', '') or 'default' + if not installer.inputs.get('init_password.db_namespace'): + installer.inputs['init_password.db_namespace'] = installer.inputs.get('env_setup.NAMESPACE', '') or 'default' + if not installer.inputs.get('kerberos_config.init_authority'): + installer.inputs['kerberos_config.init_authority'] = _bool_str(False) + if not installer.inputs.get('init_password.db_host_port'): + installer.inputs['init_password.db_host_port'] = '5432' + _log(" [OK] Config overrides applied for k3s pipeline\n") - cur_server = installer.inputs.get('init_cluster.k3s_server_url', k3s_server) - cur_token = installer.inputs.get('init_cluster.k3s_token', k3s_token) + _log("==> Writing prole.cfg...\n") + installer._write_cfg() + _log(f" [OK] {cfg_path}\n\n") - if attempt == 1: - _attempt_k3s_repair(controller, namespace, cur_server, cur_token, db_password) - elif attempt == 2: - _reset_k3s_namespace(project_root, namespace, cur_server, cur_token) - else: - attempt = 0 + db_password = installer._get_input('init_password.db_password', '').strip() + _log(f"==> Target namespace: {namespace}\n\n") - attempt += 1 - if max_attempts and total_attempts >= max_attempts: - return rc + attempt = 1 + total_attempts = 0 + max_attempts_env = os.environ.get("PROLE_SILENT_TEST_MAX_ATTEMPTS", "").strip() + max_attempts = int(max_attempts_env) if max_attempts_env.isdigit() else 0 + + _log("==> Running silent install validation...\n") + while True: + total_attempts += 1 + _log(f" Attempt {total_attempts}...\n") + rc = _run_silent_install_test(project_root, cfg_path) + if rc == 0: + _log(f" [OK] Silent install test passed\n\n") + break + + _log(f" [WARN] Silent install test failed (code {rc})\n") + cur_server = installer.inputs.get('init_cluster.k3s_server_url', k3s_server) + cur_token = installer.inputs.get('init_cluster.k3s_token', k3s_token) + + if attempt == 1: + _log(" Attempting k3s repair...\n") + _attempt_k3s_repair(controller, namespace, cur_server, cur_token, db_password) + elif attempt == 2: + _log(" Resetting k3s namespace...\n") + _reset_k3s_namespace(project_root, namespace, cur_server, cur_token) + else: + attempt = 0 + + attempt += 1 + if max_attempts and total_attempts >= max_attempts: + _log(f" [ERROR] Max attempts ({max_attempts}) reached. Aborting.\n") + return rc + else: + _log("==> Skipping validation (silent install already succeeded)\n\n") cur_server = installer.inputs.get('init_cluster.k3s_server_url', k3s_server) cur_token = installer.inputs.get('init_cluster.k3s_token', k3s_token) - _sync_opentofu_pipeline(project_root, namespace, cur_server, cur_token) + _sync_opentofu_pipeline(project_root, namespace, cur_server, cur_token, log_fn=_log) + + _log("\n==> k3s pipeline preparation complete.\n") return 0 diff --git a/installer/core/env.py b/installer/core/env.py index 0983580..5a1c3d0 100644 --- a/installer/core/env.py +++ b/installer/core/env.py @@ -1076,23 +1076,62 @@ def _write_k3s_kubeconfig(server_url: str, token: str) -> Path: return path -def _sync_opentofu_pipeline(project_root: Path, namespace: str, k3s_server_url: str, k3s_token: str) -> Path: +def _sync_opentofu_pipeline(project_root: Path, namespace: str, k3s_server_url: str, k3s_token: str, + log_fn=None) -> Path: + def _log(msg): + if log_fn: + try: log_fn(msg) + except: print(msg, end='', flush=True) + else: + print(msg, end='', flush=True) + pipeline_dir = project_root / 'deploy' / 'opentofu' / 'k3s' manifest_root = pipeline_dir / 'manifests' manifest_root.mkdir(parents=True, exist_ok=True) + argocd_dir = pipeline_dir / 'argocd' + argocd_dir.mkdir(parents=True, exist_ok=True) + _log(f"==> Syncing OpenTofu k3s pipeline to {pipeline_dir}\n") + + # --- Stage 1: Copy k8s manifests --- sources = ( project_root / 'k8s' / 'prole', project_root / 'k8s' / 'openbao', project_root / 'k8s' / 'opentofu', ) + total_copied = 0 for src in sources: if not src.exists(): + _log(f" [SKIP] {src.name}/ (not found)\n") continue dst = manifest_root / src.name dst.mkdir(parents=True, exist_ok=True) + count = 0 for path in src.glob('*.yaml'): shutil.copy2(path, dst / path.name) + count += 1 + total_copied += count + _log(f" [OK] {src.name}/ -> manifests/{src.name}/ ({count} files)\n") + _log(f" Total manifests staged: {total_copied}\n") + + # --- Stage 2: Copy Supabase k8s manifests if present --- + supabase_k8s = project_root / 'supabase' / 'k8s' + if supabase_k8s.exists(): + dst = manifest_root / 'supabase' + dst.mkdir(parents=True, exist_ok=True) + count = 0 + for path in supabase_k8s.glob('*.yaml'): + shutil.copy2(path, dst / path.name) + count += 1 + if count: + _log(f" [OK] supabase/k8s/ -> manifests/supabase/ ({count} files)\n") + + # --- Stage 3: Write tfvars --- + token_display = k3s_token[:8] + '...' if k3s_token and len(k3s_token) > 8 else '(not set)' + _log(f"\n==> Writing opentofu.auto.tfvars\n") + _log(f" k3s_server_url = {k3s_server_url or '(not set)'}\n") + _log(f" k3s_token = {token_display}\n") + _log(f" namespace = {namespace or 'default'}\n") tfvars = [ f'k3s_server_url = "{k3s_server_url}"', @@ -1101,6 +1140,175 @@ def _sync_opentofu_pipeline(project_root: Path, namespace: str, k3s_server_url: '' ] (pipeline_dir / 'opentofu.auto.tfvars').write_text("\n".join(tfvars)) + + # --- Stage 4: Generate ArgoCD Application manifests --- + _log(f"\n==> Generating ArgoCD Application manifests in argocd/\n") + + repo_url = (os.environ.get('PROLE_GIT_REPO') or '').strip() + if not repo_url: + # Attempt to detect from git remote + try: + res = subprocess.run( + ['git', 'remote', 'get-url', 'origin'], + capture_output=True, text=True, cwd=str(project_root) + ) + if res.returncode == 0 and res.stdout.strip(): + repo_url = res.stdout.strip() + except Exception: + pass + if not repo_url: + repo_url = 'https://github.com/prole-dev/prole.git' + + target_revision = (os.environ.get('PROLE_GIT_BRANCH') or '').strip() or 'main' + + components = ['prole', 'openbao', 'opentofu'] + for component in components: + app_manifest = ( + "apiVersion: argoproj.io/v1alpha1\n" + "kind: Application\n" + "metadata:\n" + f" name: prole-{component}\n" + " namespace: argocd\n" + "spec:\n" + " project: default\n" + " source:\n" + f" repoURL: {repo_url}\n" + f" targetRevision: {target_revision}\n" + f" path: deploy/opentofu/k3s/manifests/{component}\n" + " destination:\n" + f" server: {k3s_server_url or 'https://kubernetes.default.svc'}\n" + f" namespace: {namespace or 'default'}\n" + " syncPolicy:\n" + " automated:\n" + " prune: true\n" + " selfHeal: true\n" + " syncOptions:\n" + " - CreateNamespace=true\n" + ) + app_path = argocd_dir / f"application-{component}.yaml" + app_path.write_text(app_manifest, encoding='utf-8') + _log(f" [OK] application-{component}.yaml\n") + + # --- Stage 5: Generate k3s-to-prod pipeline scaffold --- + prod_pipeline_dir = project_root / 'deploy' / 'opentofu' / 'prod' + if not prod_pipeline_dir.exists(): + prod_pipeline_dir.mkdir(parents=True, exist_ok=True) + _log(f"\n==> Scaffolding prod pipeline at {prod_pipeline_dir}\n") + + prod_main_tf = ( + 'terraform {\n' + ' required_version = ">= 1.6.0"\n' + ' required_providers {\n' + ' kubernetes = {\n' + ' source = "hashicorp/kubernetes"\n' + ' version = "~> 2.30"\n' + ' }\n' + ' }\n' + '}\n' + '\n' + '# Configure for AWS EKS or GCloud GKE via variables\n' + 'provider "kubernetes" {\n' + ' host = var.cluster_endpoint\n' + ' token = var.cluster_token\n' + ' cluster_ca_certificate = var.cluster_ca_cert != "" ? base64decode(var.cluster_ca_cert) : null\n' + ' insecure = var.cluster_ca_cert == ""\n' + '}\n' + '\n' + 'locals {\n' + ' manifest_dir = "${path.module}/manifests"\n' + ' manifest_files = fileset(local.manifest_dir, "**/*.yaml")\n' + ' raw_documents = flatten([\n' + ' for f in local.manifest_files : [\n' + ' for doc in split("\\n---", trimspace(file("${local.manifest_dir}/${f}"))) :\n' + ' trimspace(doc)\n' + ' if trimspace(doc) != ""\n' + ' ]\n' + ' ])\n' + ' decoded_documents = [\n' + ' for doc in local.raw_documents : yamldecode(doc)\n' + ' if try(yamldecode(doc).kind, "") != ""\n' + ' ]\n' + ' cluster_scoped_kinds = toset([\n' + ' "Namespace", "CustomResourceDefinition", "ClusterRole",\n' + ' "ClusterRoleBinding", "PersistentVolume", "StorageClass",\n' + ' ])\n' + ' namespaced_documents = [\n' + ' for m in local.decoded_documents :\n' + ' contains(local.cluster_scoped_kinds, m.kind) ? m : merge(\n' + ' m, { metadata = merge(lookup(m, "metadata", {}), { namespace = var.namespace }) }\n' + ' )\n' + ' ]\n' + '}\n' + '\n' + 'resource "kubernetes_manifest" "namespace" {\n' + ' manifest = {\n' + ' apiVersion = "v1"\n' + ' kind = "Namespace"\n' + ' metadata = { name = var.namespace }\n' + ' }\n' + '}\n' + '\n' + 'resource "kubernetes_manifest" "resources" {\n' + ' for_each = { for idx, m in local.namespaced_documents : tostring(idx) => m }\n' + ' manifest = each.value\n' + ' depends_on = [kubernetes_manifest.namespace]\n' + '}\n' + ) + (prod_pipeline_dir / 'main.tf').write_text(prod_main_tf, encoding='utf-8') + + prod_variables_tf = ( + 'variable "cluster_endpoint" {\n' + ' type = string\n' + ' description = "Production cluster API endpoint (EKS/GKE)"\n' + '}\n' + '\n' + 'variable "cluster_token" {\n' + ' type = string\n' + ' description = "Production cluster auth token"\n' + ' sensitive = true\n' + '}\n' + '\n' + 'variable "cluster_ca_cert" {\n' + ' type = string\n' + ' description = "Base64-encoded CA certificate (leave empty for insecure)"\n' + ' default = ""\n' + '}\n' + '\n' + 'variable "namespace" {\n' + ' type = string\n' + ' description = "Target namespace for Prole resources"\n' + ' default = "default"\n' + '}\n' + ) + (prod_pipeline_dir / 'variables.tf').write_text(prod_variables_tf, encoding='utf-8') + (prod_pipeline_dir / 'manifests').mkdir(parents=True, exist_ok=True) + (prod_pipeline_dir / 'manifests' / '.gitkeep').write_text('', encoding='utf-8') + + prod_readme = ( + '# OpenTofu Production Pipeline\n' + '\n' + 'This pipeline deploys Prole into a production cluster (AWS EKS / GCloud GKE).\n' + '\n' + '## Usage\n' + '1. Copy manifests from the k3s pipeline into `manifests/`.\n' + '2. Set `cluster_endpoint`, `cluster_token`, and optionally `cluster_ca_cert`.\n' + '3. Run:\n' + '```bash\n' + 'tofu init\n' + 'tofu plan\n' + 'tofu apply\n' + '```\n' + ) + (prod_pipeline_dir / 'README.md').write_text(prod_readme, encoding='utf-8') + _log(f" [OK] main.tf, variables.tf, README.md\n") + else: + _log(f"\n==> Prod pipeline already exists at {prod_pipeline_dir} (skipped)\n") + + _log(f"\n==> OpenTofu pipeline sync complete.\n") + _log(f" k3s pipeline: {pipeline_dir}\n") + _log(f" ArgoCD apps: {argocd_dir}\n") + _log(f" Prod scaffold: {prod_pipeline_dir}\n") + return pipeline_dir @@ -1317,14 +1525,15 @@ def _render_prole_cfg(inputs: dict, globals_to_save: dict, sections: dict, gener content.append('; Generated by install.py on ' + (generated_at or time.strftime('%Y-%m-%d %H:%M:%S'))) content.append('; This file is used as input for Ansible deployment and k8s cluster creation.') content.append('') - allowed_namespace_keys = {'NAMESPACE', 'SERVICE_NAMESPACE'} + allowed_namespace_keys = {'NAMESPACE', 'SERVICE_NAMESPACE', + 'env_setup.NAMESPACE', 'init_password.db_namespace'} def is_namespace_key(key: str) -> bool: return 'namespace' in key.lower() def emit_kv(data: dict, allow_namespace: bool = False) -> None: for k in sorted(data.keys()): - if is_namespace_key(k) and (not allow_namespace or k not in allowed_namespace_keys): + if is_namespace_key(k) and not allow_namespace and k not in allowed_namespace_keys: continue content.append(f'{k} = {data[k]}') @@ -1463,13 +1672,16 @@ def _build_required_port_forwards( "PostgreSQL (primary)"), _pf_mapping_str("prometheus", "monitoring", "svc/kps-kube-prometheus-stack-prometheus", "9090", "9090", addr_local, "TCP", "Prometheus UI"), _pf_mapping_str("grafana", "monitoring", "svc/kps-grafana", "3000", "80", addr_all, "TCP", "Grafana UI"), + _pf_mapping_str("prole-db-manager", db_ns, "svc/prole-db-manager", "80", "80", addr_all, "TCP", "DB Manager UI"), ] if supabase_enabled: used_ports = {part.split('hostPort=', 1)[1].split(';', 1)[0] for part in mappings if 'hostPort=' in part} supabase_port = "8080" if supabase_port in used_ports: - supabase_port = "8088" + supabase_port = "18080" + if supabase_port in used_ports: + supabase_port = "28080" # Core Supabase services (minimal exposure for client + studio) mappings.append( diff --git a/installer/core/milestones.py b/installer/core/milestones.py index af469bd..853e5f8 100644 --- a/installer/core/milestones.py +++ b/installer/core/milestones.py @@ -510,7 +510,13 @@ class InitializationScriptsMilestone(Milestone): # Optional scripts if self._parse_bool(state.inputs.get('kerberos_config.enabled', 'False')): scripts.append("init_kerberos.sh") - scripts.extend(["init_prole-db-backup.sh", "init_monitoring.sh"]) + scripts.extend([ + "init_prole-db-backup.sh", + "init_kong.sh", + "init_postgrest.sh", + "init_db_manager.sh", + "init_monitoring.sh", + ]) if mode != "k3d": scripts.append("init_nginx_ingress.sh") @@ -531,6 +537,8 @@ class InitializationScriptsMilestone(Milestone): args = mode_args + ["initialize"] elif script == "init_prole-db-backup.sh": args = mode_args + ["start"] + elif script in ("init_kong.sh", "init_postgrest.sh", "init_db_manager.sh"): + args = mode_args + ["start"] elif script in ("init_monitoring.sh", "init_nginx_ingress.sh"): args = mode_args + ["initialize"] diff --git a/installer/core/monitor.py b/installer/core/monitor.py new file mode 100644 index 0000000..6e1ee34 --- /dev/null +++ b/installer/core/monitor.py @@ -0,0 +1,261 @@ +"""Monitor subcommand — read port-forwards.cfg and manage kubectl port-forward processes.""" +from __future__ import annotations + +import configparser +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +from installer.config import _expand_path, _collect_cfg_vars, _expand_cfg_value +from installer.core.env import ( + _build_required_port_forwards, _pf_extract_id, _pf_mapping_str, _pf_upsert_mapping, +) + + +# --------------------------------------------------------------------------- +# Port-forwards.cfg I/O +# --------------------------------------------------------------------------- + +def _parse_pf_cfg(path: Path) -> list[dict]: + """Parse a port-forwards.cfg file and return a list of mapping dicts.""" + mappings: list[dict] = [] + if not path.exists(): + return mappings + for raw_line in path.read_text().splitlines(): + line = raw_line.strip() + if not line or line.startswith('#'): + continue + # Format: id: local=... remote=... ns=... svc=... address=... + if ':' not in line: + continue + m_id, _, rest = line.partition(':') + m_id = m_id.strip() + parts: dict[str, str] = {'id': m_id} + for token in rest.split(): + if '=' in token: + k, v = token.split('=', 1) + parts[k.strip()] = v.strip() + mappings.append(parts) + return mappings + + +def write_port_forwards_cfg(path: Path, mappings: list[str]) -> None: + """Write a port-forwards.cfg from internal mapping strings.""" + lines = [ + "# Port forward configuration for Prole services (generated).", + "# Format: id: local= remote= ns= svc= address=", + "", + ] + for m in mappings: + parts: dict[str, str] = {} + for token in m.split(';'): + if '=' in token: + k, v = token.split('=', 1) + parts[k.strip()] = v.strip() + m_id = parts.get('id', '') + if not m_id: + continue + local = parts.get('hostPort', '') + remote = parts.get('servicePort', '') + ns = parts.get('namespace', '') + target = parts.get('target', '') + addr = parts.get('address', '0.0.0.0') + svc = target[4:] if target.startswith('svc/') else target + lines.append(f"{m_id}: local={local} remote={remote} ns={ns} svc={svc} address={addr}") + path.write_text('\n'.join(lines) + '\n') + + +# --------------------------------------------------------------------------- +# Conflict detection +# --------------------------------------------------------------------------- + +def check_port_conflicts(mappings: list[dict]) -> list[str]: + """Return list of conflict descriptions (empty == no conflicts).""" + seen: dict[str, str] = {} + conflicts: list[str] = [] + for m in mappings: + port = m.get('local', '') + m_id = m.get('id', '?') + if port in seen: + conflicts.append(f"Port {port} conflict: {seen[port]} and {m_id}") + else: + seen[port] = m_id + return conflicts + + +# --------------------------------------------------------------------------- +# kubectl port-forward launcher +# --------------------------------------------------------------------------- + +def _build_kubectl_cmd(m: dict) -> list[str]: + """Build a kubectl port-forward command from a mapping dict.""" + ns = m.get('ns', 'default') + svc = m.get('svc', '') + local = m.get('local', '') + remote = m.get('remote', '') + addr = m.get('address', '0.0.0.0') + + if not svc.startswith('svc/'): + svc = f"svc/{svc}" + + cmd = ['kubectl', 'port-forward', '-n', ns, svc, f'{local}:{remote}'] + if addr and addr != '127.0.0.1': + cmd.extend(['--address', addr]) + return cmd + + +def _start_port_forwards(mappings: list[dict], verbose: bool = False) -> list[tuple[dict, subprocess.Popen | None]]: + """Start kubectl port-forward for each mapping; return (mapping, proc) pairs.""" + procs: list[tuple[dict, subprocess.Popen | None]] = [] + for m in mappings: + cmd = _build_kubectl_cmd(m) + m_id = m.get('id', '?') + if verbose: + print(f"[PORT-FWD] Starting: {' '.join(cmd)}", flush=True) + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + procs.append((m, proc)) + print(f" ✓ {m_id}: localhost:{m.get('local','')} → {m.get('ns','')}/{m.get('svc','')}:{m.get('remote','')}", flush=True) + except Exception as e: + print(f" ✗ {m_id}: failed to start — {e}", flush=True) + procs.append((m, None)) + return procs + + +# --------------------------------------------------------------------------- +# Monitor loop +# --------------------------------------------------------------------------- + +def run_monitor(controller, cfg_path: str | None, pf_cfg_path: str | None, verbose: bool = False) -> int: + """Main entry point for the 'monitor' subcommand.""" + project_root = controller.project_root + + # Resolve prole.cfg for context + if cfg_path: + prole_cfg = Path(_expand_path(cfg_path)) + else: + prole_cfg = project_root / 'conf' / 'prole.cfg' + + # Read prole.cfg to extract deployment parameters + cfg = configparser.ConfigParser(interpolation=None) + cfg.optionxform = str + if prole_cfg.exists(): + cfg.read(prole_cfg) + # Build vars: start with User section (concrete values), then overlay Global/System + cfg_vars: dict[str, str] = {} + if cfg.has_section('User'): + cfg_vars.update(dict(cfg.items('User'))) + overlay = _collect_cfg_vars(cfg) + # Only overlay non-self-referencing values from Global/System + for k, v in overlay.items(): + if '${' not in v: + cfg_vars[k] = v + # Resolve remaining references iteratively + for _ in range(5): + changed = False + for k, v in list(cfg_vars.items()): + expanded = _expand_cfg_value(v, cfg_vars) + if expanded != v: + cfg_vars[k] = expanded + changed = True + if not changed: + break + + mode = _expand_cfg_value(cfg.get('Global', 'DEPLOYMENT_MODE', fallback='k3d'), cfg_vars).strip() or 'k3d' + service_ns = _expand_cfg_value(cfg.get('Global', 'SERVICE_NAMESPACE', fallback='default'), cfg_vars).strip() or 'default' + db_ns = _expand_cfg_value(cfg.get('Global', 'NAMESPACE', fallback='default'), cfg_vars).strip() or 'default' + db_host_port = _expand_cfg_value(cfg.get('Global', 'DB_HOST_PORT', fallback='5432'), cfg_vars).strip() or '5432' + supabase_enabled = _expand_cfg_value(cfg.get('Inputs', 'init_cluster.supabase_enabled', fallback='false'), cfg_vars).strip().lower() in ('true', '1', 'yes') + supabase_ns = 'supabase' + + # Resolve port-forwards.cfg path + if pf_cfg_path: + pf_path = Path(pf_cfg_path) + if not pf_path.is_absolute(): + pf_path = project_root / pf_path + else: + pf_path = project_root / 'conf' / 'port-forwards.cfg' + + # Generate canonical mappings + raw_mappings = _build_required_port_forwards( + mode=mode, + service_ns=service_ns, + argocd_ns='argocd', + db_ns=db_ns, + db_host_port=db_host_port, + supabase_enabled=supabase_enabled, + supabase_namespace=supabase_ns, + ) + + # Write / refresh the port-forwards.cfg + write_port_forwards_cfg(pf_path, raw_mappings) + print(f"[CONFIG] Port-forwards config written to {pf_path}", flush=True) + + # Parse the written file back for monitoring + mappings = _parse_pf_cfg(pf_path) + + # Check for conflicts + conflicts = check_port_conflicts(mappings) + if conflicts: + print("\n[CONFLICT] Port conflicts detected:", flush=True) + for c in conflicts: + print(f" ⚠ {c}", flush=True) + return 1 + + print(f"\n[OK] {len(mappings)} port-forward mappings — no conflicts.\n", flush=True) + + # Print summary table + print(f"{'Service':<25} {'Local':>6} → {'Remote':>6} {'Namespace':<25} {'Target':<40} {'Address'}", flush=True) + print("-" * 130, flush=True) + for m in mappings: + print(f"{m.get('id',''):<25} {m.get('local',''):>6} → {m.get('remote',''):>6} {m.get('ns',''):<25} svc/{m.get('svc',''):<36} {m.get('address','')}", flush=True) + print(flush=True) + + # Start port-forwards + procs = _start_port_forwards(mappings, verbose=verbose) + + failed = [m_id for (m, p) in procs if p is None for m_id in [m.get('id', '?')]] + started = [(m, p) for (m, p) in procs if p is not None] + + if failed: + print(f"\n[WARN] Failed to start: {', '.join(failed)}", flush=True) + + if not started: + print("[ERROR] No port-forwards could be started.", flush=True) + return 1 + + # Brief monitoring: wait a few seconds and check which processes are still alive + print(f"\n[MONITOR] Checking port-forward health (3s)...", flush=True) + time.sleep(3) + + alive = 0 + dead_services: list[str] = [] + for m, p in started: + m_id = m.get('id', '?') + if p.poll() is None: + alive += 1 + else: + stderr_out = '' + try: + stderr_out = p.stderr.read().decode(errors='replace').strip() + except Exception: + pass + dead_services.append(m_id) + print(f" ✗ {m_id}: exited (rc={p.returncode}) {stderr_out[:200]}", flush=True) + + print(f"\n[RESULT] {alive}/{len(started)} port-forwards running.", flush=True) + if dead_services: + print(f"[WARN] Services that did not come up: {', '.join(dead_services)}", flush=True) + + # Cleanup + for m, p in started: + if p.poll() is None: + try: + p.terminate() + except Exception: + pass + + return 0 if alive > 0 else 1 diff --git a/installer/ui/screens/__init__.py b/installer/ui/screens/__init__.py index b7786af..2b27540 100644 --- a/installer/ui/screens/__init__.py +++ b/installer/ui/screens/__init__.py @@ -570,8 +570,10 @@ def main(): parser = argparse.ArgumentParser(description='Prole Database Installer') parser.add_argument('-c', '--config', default=None, help='Path to prole.cfg for saving or silent replay') - parser.add_argument('-S', '--silent', action='store_true', + parser.add_argument('-s', '-S', '--silent', action='store_true', help='Run unattended install in console mode using prole.cfg') + parser.add_argument('-l', '--load', default=None, + help='Path to prole.cfg to load (alias for -c)') parser.add_argument('--prepare-k3s-pipeline', action='store_true', help='Prepare OpenTofu k3s pipeline and run silent install test') parser.add_argument('--reset', action='store_true', @@ -584,21 +586,39 @@ def main(): help='Enable verbose output') parser.add_argument('-d', '--debug', action='store_true', help='Enable debug logging') + parser.add_argument('command', nargs='?', default=None, + help='Subcommand to run (e.g. monitor)') + parser.add_argument('command_args', nargs='*', + help='Arguments for the subcommand') args = parser.parse_args() + # -l is an alias for -c + if args.load and not args.config: + args.config = args.load + from installer.config import setup_logging setup_logging(verbose=args.verbose, debug=args.debug) controller = ProleController(PROJECT_ROOT, verbose=args.verbose) - if args.prepare_k3s_pipeline: - rc = _prepare_k3s_pipeline(controller) - sys.exit(rc) + if args.command == 'monitor': + from installer.core.monitor import run_monitor + pf_cfg = args.command_args[0] if args.command_args else None + sys.exit(run_monitor(controller, args.config, pf_cfg, verbose=args.verbose)) if args.silent: _ensure_ansible_vault_credentials(prompt_ui=False) installer = ProleSilentInstaller(controller, args.config, reset_cluster=args.reset) - sys.exit(installer.run()) + rc = installer.run() + if rc != 0: + sys.exit(rc) + if args.prepare_k3s_pipeline: + rc = _prepare_k3s_pipeline(controller, skip_validation=True) + sys.exit(rc) + + if args.prepare_k3s_pipeline: + rc = _prepare_k3s_pipeline(controller) + sys.exit(rc) use_gui = False if args.gui: diff --git a/installer/ui/screens/deploy.py b/installer/ui/screens/deploy.py index e3bc53a..aa157e7 100644 --- a/installer/ui/screens/deploy.py +++ b/installer/ui/screens/deploy.py @@ -116,7 +116,13 @@ class DeployScreenMixin: self.safe_after(lambda: self._save_deployment_button.configure(state='disabled') if hasattr(self, '_save_deployment_button') and self._save_deployment_button.winfo_exists() else None) self.safe_after(lambda: self.bg_canvas.itemconfig(self._save_deployment_status_label, text="Saving deployment...", fill='blue') if hasattr(self, '_save_deployment_status_label') and self.bg_canvas.winfo_exists() else None) + # Use the deploy console on the services screen for visible output + console = getattr(self, '_cnpg_deploy_console', None) + + # Fall back to install_consoles tab if available script_name = "final_deployment.sh" + if not console: + console = getattr(self, 'install_consoles', {}).get(script_name) if hasattr(self, 'install_consoles') and script_name in self.install_consoles: self.safe_after(lambda: self.install_tabs.select(self.install_consoles[script_name].master) if hasattr(self, 'install_tabs') and self.install_tabs.winfo_exists() else None) @@ -126,19 +132,24 @@ class DeployScreenMixin: env["HELM_CHART_DIR"] = helm_export_dir env["KUSTOMIZE_DIR"] = kustomize_export_dir - console = getattr(self, 'install_consoles', {}).get(script_name) if console: console.clear() - console.write("Saving deployment artifacts...\n") - console.write(f"Export base: {export_base}\n") - console.write(f"Docker images: {docker_export_dir}\n") - console.write(f"Helm chart: {helm_export_dir}\n") - console.write(f"Kustomize: {kustomize_export_dir}\n\n") + console.write("========================================\n") + console.write("Save Deployment\n") + console.write("========================================\n\n") + console.write(f"Export base: {export_base}\n") + console.write(f"Docker images: {docker_export_dir}\n") + console.write(f"Helm chart: {helm_export_dir}\n") + console.write(f"Kustomize: {kustomize_export_dir}\n\n") def _on_line(line): if console: console.write(line) + # Stage 1: Docker export, Helm chart, Kustomize + if console: + console.write("==> Stage 1: Exporting deployment artifacts...\n\n") + rc = self.controller.run_script( script_name, args=['--docker-export', '--helm-chart', '--kustomize', '--output-dir', export_base], @@ -146,8 +157,43 @@ class DeployScreenMixin: on_line=_on_line ) - if rc == 0: + if rc != 0: + if console: + console.write(f"\n[ERROR] Artifact export failed with code {rc}\n") + self.safe_after(lambda: self.bg_canvas.itemconfig(self._save_deployment_status_label, text=f"Failed with code {rc}", fill='#ff3b30') if hasattr(self, '_save_deployment_status_label') and self.bg_canvas.winfo_exists() else None) + self.safe_after(lambda: self._save_deployment_button.configure(state='normal') if hasattr(self, '_save_deployment_button') and self._save_deployment_button.winfo_exists() else None) + return + + # Stage 2: Prepare k3s pipeline (OpenTofu + ArgoCD) + if console: + console.write("\n==> Stage 2: Preparing k3s pipeline (OpenTofu + ArgoCD)...\n\n") + + self.safe_after(lambda: self.bg_canvas.itemconfig(self._save_deployment_status_label, text="Preparing k3s pipeline...", fill='blue') if hasattr(self, '_save_deployment_status_label') and self.bg_canvas.winfo_exists() else None) + + pipeline_rc = self.controller.run_script( + script_name, + args=['--prepare-k3s-pipeline'], + env=env, + on_line=_on_line + ) + + if pipeline_rc != 0: + if console: + console.write(f"\n[WARN] k3s pipeline preparation failed with code {pipeline_rc}\n") + console.write("Deployment artifacts were saved, but pipeline staging had issues.\n") + + if console: + console.write("\n========================================\n") + console.write("Save Deployment Complete\n") + console.write("========================================\n") + console.write(f"\nArtifacts: {export_base}\n") + console.write(f"Pipeline: {PROJECT_ROOT / 'deploy' / 'opentofu' / 'k3s'}\n") + console.write(f"ArgoCD: {PROJECT_ROOT / 'deploy' / 'opentofu' / 'k3s' / 'argocd'}\n") + + if rc == 0 and pipeline_rc == 0: self.safe_after(lambda: self.bg_canvas.itemconfig(self._save_deployment_status_label, text="Deployment saved successfully.", fill='#34c759') if hasattr(self, '_save_deployment_status_label') and self.bg_canvas.winfo_exists() else None) + elif rc == 0: + self.safe_after(lambda: self.bg_canvas.itemconfig(self._save_deployment_status_label, text="Saved (pipeline warnings)", fill='#ff9500') if hasattr(self, '_save_deployment_status_label') and self.bg_canvas.winfo_exists() else None) else: self.safe_after(lambda: self.bg_canvas.itemconfig(self._save_deployment_status_label, text=f"Failed with code {rc}", fill='#ff3b30') if hasattr(self, '_save_deployment_status_label') and self.bg_canvas.winfo_exists() else None) diff --git a/installer/ui/screens/packaging.py b/installer/ui/screens/packaging.py index d2ce0d9..619ccd9 100644 --- a/installer/ui/screens/packaging.py +++ b/installer/ui/screens/packaging.py @@ -665,6 +665,13 @@ exec "$DIR/ProleTools.bin" "$@" 'url': 'https://cert-manager.io', 'install_cmd': 'brew install cmctl', 'check_cmd': 'cmctl version' + }, + { + 'name': 'OpenTofu', + 'description': 'Open-source infrastructure as code tool', + 'url': 'https://opentofu.org', + 'install_cmd': 'brew install opentofu', + 'check_cmd': 'tofu --version' } ] @@ -775,6 +782,7 @@ exec "$DIR/ProleTools.bin" "$@" 'install_prole_helm.sh', 'install_prole_krew.sh', 'install_prole_cmctl.sh', + 'install_prole_opentofu.sh', 'install_prole_kubectl_plugins.sh', ] diff --git a/installer/ui/screens/services.py b/installer/ui/screens/services.py index 53755be..5501780 100644 --- a/installer/ui/screens/services.py +++ b/installer/ui/screens/services.py @@ -105,6 +105,9 @@ class ServicesScreenMixin: if self.kerberos_enabled.get(): scripts.append(('Kerberos Realm', 'init_kerberos.sh')) scripts.append(('Prole DB Backup', 'init_prole-db-backup.sh')) + scripts.append(('Kong API Gateway', 'init_kong.sh')) + scripts.append(('PostgREST', 'init_postgrest.sh')) + scripts.append(('DB Manager', 'init_db_manager.sh')) scripts.append(('Monitoring', 'init_monitoring.sh')) mode = self._deployment_mode() if mode != "k3d": @@ -372,6 +375,144 @@ class ServicesScreenMixin: else: self.script_consoles["init_prole-db-backup.sh"].write("Skipping Prole DB backup because previous steps failed.\n") + # 7. init_kong.sh start + if overall_success: + script = "init_kong.sh" + if script in self.script_consoles: + _select_tab(script) + self.script_consoles[script].clear() + self.script_consoles[script].write(f"Running {script} start...\n") + + log_path = _log_path_for(script) + self._record_install_log(log_path) + try: + log_fp = log_path.open('a', encoding='utf-8') + except Exception: + log_fp = None + + def _kong_line(line): + self.script_consoles[script].write(line) + self._process_script_output_line(line) + if log_fp: + try: + log_fp.write(line) + log_fp.flush() + except Exception: + pass + + rc_kong = self.controller.run_script( + script, + args=mode_args + ['start'], + env=env, + on_line=_kong_line + ) + if log_fp: + try: + log_fp.close() + except Exception: + pass + + if rc_kong != 0: + self.script_consoles[script].write(f"\nERROR: {script} start failed with code {rc_kong}\n") + overall_success = False + else: + self.script_consoles[script].write(f"\n{script} completed successfully.\n") + else: + if "init_kong.sh" in self.script_consoles: + self.script_consoles["init_kong.sh"].write("Skipping Kong because previous steps failed.\n") + + # 7b. init_postgrest.sh start + if overall_success: + script = "init_postgrest.sh" + if script in self.script_consoles: + _select_tab(script) + self.script_consoles[script].clear() + self.script_consoles[script].write(f"Running {script} start...\n") + + log_path = _log_path_for(script) + self._record_install_log(log_path) + try: + log_fp = log_path.open('a', encoding='utf-8') + except Exception: + log_fp = None + + def _postgrest_line(line): + self.script_consoles[script].write(line) + self._process_script_output_line(line) + if log_fp: + try: + log_fp.write(line) + log_fp.flush() + except Exception: + pass + + rc_postgrest = self.controller.run_script( + script, + args=mode_args + ['start'], + env=env, + on_line=_postgrest_line + ) + if log_fp: + try: + log_fp.close() + except Exception: + pass + + if rc_postgrest != 0: + self.script_consoles[script].write(f"\nERROR: {script} start failed with code {rc_postgrest}\n") + overall_success = False + else: + self.script_consoles[script].write(f"\n{script} completed successfully.\n") + else: + if "init_postgrest.sh" in self.script_consoles: + self.script_consoles["init_postgrest.sh"].write("Skipping PostgREST because previous steps failed.\n") + + # 7c. init_db_manager.sh start + if overall_success: + script = "init_db_manager.sh" + if script in self.script_consoles: + _select_tab(script) + self.script_consoles[script].clear() + self.script_consoles[script].write(f"Running {script} start...\n") + + log_path = _log_path_for(script) + self._record_install_log(log_path) + try: + log_fp = log_path.open('a', encoding='utf-8') + except Exception: + log_fp = None + + def _dbmgr_line(line): + self.script_consoles[script].write(line) + self._process_script_output_line(line) + if log_fp: + try: + log_fp.write(line) + log_fp.flush() + except Exception: + pass + + rc_dbmgr = self.controller.run_script( + script, + args=mode_args + ['start'], + env=env, + on_line=_dbmgr_line + ) + if log_fp: + try: + log_fp.close() + except Exception: + pass + + if rc_dbmgr != 0: + self.script_consoles[script].write(f"\nERROR: {script} start failed with code {rc_dbmgr}\n") + overall_success = False + else: + self.script_consoles[script].write(f"\n{script} completed successfully.\n") + else: + if "init_db_manager.sh" in self.script_consoles: + self.script_consoles["init_db_manager.sh"].write("Skipping DB Manager because previous steps failed.\n") + # 8. init_monitoring.sh initialize if overall_success: script = "init_monitoring.sh" diff --git a/installer/ui/screens/supabase.py b/installer/ui/screens/supabase.py index 9e2821f..d5fd780 100644 --- a/installer/ui/screens/supabase.py +++ b/installer/ui/screens/supabase.py @@ -20,7 +20,7 @@ class SupabaseScreenMixin: ui.canvas_text(self, right_margin, 85, "Infrastructure Automated.", fill='#6e6e73', font=('SF Pro Text', 18), anchor='ne') - self._render_title('Prole::Supabase', y=150) + self._render_title('Supabase', y=150) description = ( "Launches the Supabase open-source stack via supabase/deploy.sh.\n" diff --git a/prole.sh b/prole.sh index dadf7b8..73ee28b 100755 --- a/prole.sh +++ b/prole.sh @@ -49,6 +49,7 @@ usage() { echo " deploy Full end-to-end: site run, and silent install (reset optional)" echo " start Verify/clean environment, ensure cluster availability, run silent install" echo " stop Validate environment, run full backup+archive, stop services (keep cluster idle)" + echo " backup Run a database backup (-f/full for full, default incremental)" echo " passwd Change the Prole master password and propagate secrets" echo echo "Any other arguments are passed directly to ansible.sh" @@ -64,7 +65,7 @@ while [[ $# -gt 0 ]]; do -s|--silent) SILENT=1; shift ;; -r|--reset) RESET=1; shift ;; -h|--help) usage; exit 0 ;; - install|ansible|site|reset|deploy|start|stop|passwd) break ;; + install|ansible|site|reset|deploy|start|stop|backup|passwd) break ;; *) EXTRA_ARGS+=("$1"); shift ;; esac done @@ -297,7 +298,7 @@ filter_install_args() { local arg for arg in "$@"; do case "$arg" in - -S|--silent|-c|--config|--no-gui|--gui) ;; + -s|-S|--silent|-c|--config|--no-gui|--gui) ;; *) out+=("$arg") ;; esac done @@ -506,7 +507,55 @@ fi case "${CMD}" in install) LOGFILE="${ROOT_DIR}/logs/install-${TS}.log" - + + # Re-parse install-specific flags that may appear after the command word + _INSTALL_PASSTHRU=() + _skip_next=0 + _next_is_config=0 + for _arg in "${CMD_ARGS[@]}"; do + if [[ $_skip_next -eq 1 ]]; then + _skip_next=0 + continue + fi + if [[ $_next_is_config -eq 1 ]]; then + _next_is_config=0 + # Pass -c through to install.py + _INSTALL_PASSTHRU+=("-c" "$_arg") + continue + fi + case "$_arg" in + -s|--silent|-S) + SILENT=1 + ;; + -v|--verbose) + VERBOSE=1 + export PROLE_VERBOSE=1 VERBOSE=1 CLICOLOR_FORCE=1 PY_COLORS=1 + ;; + -c|--config) + _next_is_config=1 + ;; + -r|--reset) + RESET=1 + ;; + *) + _INSTALL_PASSTHRU+=("$_arg") + ;; + esac + done + # Rebuild INSTALL_ONLY_ARGS from (possibly updated) flags + if [[ "${SILENT}" -eq 1 ]]; then + # Ensure -S is present exactly once + _has_S=0 + for _a in "${INSTALL_ONLY_ARGS[@]}"; do [[ "$_a" == "-S" ]] && _has_S=1; done + [[ $_has_S -eq 0 ]] && INSTALL_ONLY_ARGS+=("-S") + fi + if [[ "${RESET}" -eq 1 ]]; then + _has_reset=0 + for _a in "${INSTALL_ONLY_ARGS[@]}"; do [[ "$_a" == "--reset" ]] && _has_reset=1; done + [[ $_has_reset -eq 0 ]] && INSTALL_ONLY_ARGS+=("--reset") + fi + CMD_ARGS=("${_INSTALL_PASSTHRU[@]}") + PY_CMD=("python3") if [[ "${COVERAGE}" -eq 1 ]]; then PY_CMD=("coverage" "run" "--source=installer") @@ -761,6 +810,86 @@ case "${CMD}" in fi fi ;; + backup) + if [[ ! -f "${PROLE_CFG_DEFAULT}" ]]; then + err_msg "Missing ${PROLE_CFG_DEFAULT}" + exit 1 + fi + load_prole_cfg "${PROLE_CFG_DEFAULT}" + + local_mode=$(resolve_prole_mode) + if [[ -z "${local_mode:-}" ]]; then + local_mode="k3d" + fi + + # Determine backup type: -f / --full / full → full, otherwise incremental + backup_type="incremental" + for arg in "${CMD_ARGS[@]}"; do + case "$arg" in + -f|--full|full) backup_type="full" ;; + esac + done + + log_msg "Validating environment (mode=${local_mode})..." + case "$local_mode" in + k3d) + ensure_tools "k3d" + cluster_name=$(resolve_k3d_cluster_name) + if ! kubectl --context "${KUBECTL_CONTEXT:-k3d-${cluster_name}}" get nodes >/dev/null 2>&1; then + err_msg "k3d cluster '${cluster_name}' not reachable" + exit 2 + fi + ;; + k3s|k8s) + ensure_k3s_cluster_ready + ;; + *) + err_msg "Unsupported mode '${local_mode}'" + exit 2 + ;; + esac + + # Ensure backup infrastructure is ready + if [[ -x "${ROOT_DIR}/etc/init_garage_store.sh" ]]; then + "${ROOT_DIR}/etc/init_garage_store.sh" start + fi + if [[ -x "${ROOT_DIR}/etc/init_cloudnative_pg.sh" ]]; then + "${ROOT_DIR}/etc/init_cloudnative_pg.sh" install-barman-plugin + fi + if [[ -x "${ROOT_DIR}/etc/init_prole-db-backup.sh" ]]; then + RUN_FIRST_BACKUP=0 "${ROOT_DIR}/etc/init_prole-db-backup.sh" start + fi + + log_msg "Running ${backup_type} backup..." + backup_before=$(kubectl -n "${NAMESPACE:-default}" get backup \ + --sort-by=.metadata.creationTimestamp \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true) + + if [[ "$backup_type" == "full" ]]; then + "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup full + else + "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup incr + fi + + backup_name="" + for i in {1..30}; do + backup_name=$(kubectl -n "${NAMESPACE:-default}" get backup \ + --sort-by=.metadata.creationTimestamp \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true) + if [[ -n "$backup_name" && "$backup_name" != "$backup_before" ]]; then + break + fi + sleep 2 + done + + if [[ -z "$backup_name" || "$backup_name" == "$backup_before" ]]; then + err_msg "Unable to detect newly created backup" + exit 1 + fi + + wait_for_backup "${NAMESPACE:-default}" "$backup_name" "${BACKUP_WAIT_TIMEOUT:-1800}" + log_msg "Backup ${backup_name} (${backup_type}) completed successfully." + ;; passwd) if [[ -x "${ROOT_DIR}/etc/prole-db-passwwd.sh" ]]; then "${ROOT_DIR}/etc/prole-db-passwwd.sh" "${CMD_ARGS[@]}" diff --git a/src/db-manager/.gitignore b/src/db-manager/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/src/db-manager/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/src/db-manager/Dockerfile b/src/db-manager/Dockerfile new file mode 100644 index 0000000..b374f63 --- /dev/null +++ b/src/db-manager/Dockerfile @@ -0,0 +1,19 @@ +FROM node:20-alpine + +RUN apk add --no-cache curl \ + && curl -LO "https://dl.k8s.io/release/$(curl -Ls https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/ + +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev + +COPY server.js ./ + +EXPOSE 80 + +USER node + +CMD ["node", "server.js"] diff --git a/src/db-manager/package-lock.json b/src/db-manager/package-lock.json new file mode 100644 index 0000000..c9607de --- /dev/null +++ b/src/db-manager/package-lock.json @@ -0,0 +1,830 @@ +{ + "name": "prole-db-manager", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "prole-db-manager", + "version": "1.0.0", + "dependencies": { + "express": "^4.21.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/src/db-manager/package.json b/src/db-manager/package.json new file mode 100644 index 0000000..df3d823 --- /dev/null +++ b/src/db-manager/package.json @@ -0,0 +1,16 @@ +{ + "name": "prole-db-manager", + "version": "1.0.0", + "description": "Prole database management REST endpoint for backup operations", + "main": "server.js", + "scripts": { + "start": "node server.js", + "test": "node --test test/" + }, + "dependencies": { + "express": "^4.21.0" + }, + "engines": { + "node": ">=20.0.0" + } +} diff --git a/src/db-manager/server.js b/src/db-manager/server.js new file mode 100644 index 0000000..f3b8ccb --- /dev/null +++ b/src/db-manager/server.js @@ -0,0 +1,161 @@ +#!/usr/bin/env node +'use strict'; + +const express = require('express'); +const { execFile } = require('child_process'); +const { promisify } = require('util'); + +const execFileAsync = promisify(execFile); + +const app = express(); +const PORT = process.env.PORT || 80; +const NAMESPACE = process.env.NAMESPACE || 'default'; +const BARMAN_PLUGIN_NAME = process.env.BARMAN_PLUGIN_NAME || 'barman-cloud.cloudnative-pg.io'; +const BARMAN_OBJECT_NAME = process.env.BARMAN_OBJECT_NAME || 'prole-db-barman-objectstore'; + +function buildBackupManifest(clusterName, backupName) { + return JSON.stringify({ + apiVersion: 'postgresql.cnpg.io/v1', + kind: 'Backup', + metadata: { + name: backupName, + namespace: NAMESPACE + }, + spec: { + method: 'plugin', + pluginConfiguration: { + name: BARMAN_PLUGIN_NAME, + parameters: { + barmanObjectName: BARMAN_OBJECT_NAME + } + }, + cluster: { + name: clusterName + } + } + }); +} + +function generateBackupName(clusterName) { + const ts = new Date().toISOString().replace(/[-:T]/g, '').replace(/\..+/, ''); + return `${clusterName}-backup-${ts}`; +} + +async function triggerBackup(clusterName) { + const backupName = generateBackupName(clusterName); + const manifest = buildBackupManifest(clusterName, backupName); + + const { stdout, stderr } = await execFileAsync('kubectl', [ + 'apply', '-n', NAMESPACE, '-f', '-' + ], { input: manifest, timeout: 30000 }); + + return { backupName, stdout: stdout.trim(), stderr: stderr.trim() }; +} + +async function getBackupStatus(clusterName) { + const { stdout } = await execFileAsync('kubectl', [ + 'get', 'backup', '-n', NAMESPACE, + '-l', `cnpg.io/cluster=${clusterName}`, + '-o', 'json' + ], { timeout: 15000 }); + + return JSON.parse(stdout); +} + +// Health check +app.get('/health', (_req, res) => { + res.json({ status: 'ok', service: 'prole-db-manager' }); +}); + +// Trigger full backup +app.post('/backup/:clusterId/full', async (req, res) => { + const { clusterId } = req.params; + + if (!clusterId || !/^[a-z0-9][a-z0-9-]*[a-z0-9]$/.test(clusterId)) { + return res.status(400).json({ + error: 'Invalid cluster ID', + message: 'Cluster ID must be a valid Kubernetes resource name' + }); + } + + try { + const result = await triggerBackup(clusterId); + res.status(201).json({ + status: 'backup_triggered', + cluster: clusterId, + backupName: result.backupName, + namespace: NAMESPACE, + message: result.stdout + }); + } catch (err) { + const statusCode = err.code === 'ENOENT' ? 503 : 500; + res.status(statusCode).json({ + error: 'Backup trigger failed', + cluster: clusterId, + message: err.stderr || err.message + }); + } +}); + +// GET endpoint for convenience (e.g., curl from browser/grafana) +app.get('/backup/:clusterId/full', async (req, res) => { + const { clusterId } = req.params; + + if (!clusterId || !/^[a-z0-9][a-z0-9-]*[a-z0-9]$/.test(clusterId)) { + return res.status(400).json({ + error: 'Invalid cluster ID', + message: 'Cluster ID must be a valid Kubernetes resource name' + }); + } + + try { + const result = await triggerBackup(clusterId); + res.status(201).json({ + status: 'backup_triggered', + cluster: clusterId, + backupName: result.backupName, + namespace: NAMESPACE, + message: result.stdout + }); + } catch (err) { + const statusCode = err.code === 'ENOENT' ? 503 : 500; + res.status(statusCode).json({ + error: 'Backup trigger failed', + cluster: clusterId, + message: err.stderr || err.message + }); + } +}); + +// Backup status +app.get('/backup/:clusterId/status', async (req, res) => { + const { clusterId } = req.params; + + try { + const backups = await getBackupStatus(clusterId); + res.json({ + cluster: clusterId, + namespace: NAMESPACE, + backups: backups + }); + } catch (err) { + res.status(500).json({ + error: 'Failed to get backup status', + cluster: clusterId, + message: err.stderr || err.message + }); + } +}); + +// Module exports for testing +module.exports = { app, buildBackupManifest, generateBackupName, triggerBackup, getBackupStatus }; + +// Start server only when run directly +if (require.main === module) { + app.listen(PORT, () => { + console.log(`prole-db-manager listening on port ${PORT}`); + console.log(` Namespace: ${NAMESPACE}`); + console.log(` Barman plugin: ${BARMAN_PLUGIN_NAME}`); + console.log(` Barman object: ${BARMAN_OBJECT_NAME}`); + }); +} diff --git a/src/db-manager/test/server.test.js b/src/db-manager/test/server.test.js new file mode 100644 index 0000000..be5badd --- /dev/null +++ b/src/db-manager/test/server.test.js @@ -0,0 +1,143 @@ +#!/usr/bin/env node +'use strict'; + +const { describe, it, before, after, mock } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('http'); + +// Set test environment before requiring server +process.env.NAMESPACE = 'test-ns'; +process.env.BARMAN_PLUGIN_NAME = 'barman-cloud.cloudnative-pg.io'; +process.env.BARMAN_OBJECT_NAME = 'prole-db-barman-objectstore'; + +const { app, buildBackupManifest, generateBackupName } = require('../server'); + +function request(server, method, path) { + return new Promise((resolve, reject) => { + const url = new URL(path, `http://localhost:${server.address().port}`); + const req = http.request(url, { method }, (res) => { + let body = ''; + res.on('data', (chunk) => { body += chunk; }); + res.on('end', () => { + try { + resolve({ status: res.statusCode, body: JSON.parse(body) }); + } catch { + resolve({ status: res.statusCode, body }); + } + }); + }); + req.on('error', reject); + req.end(); + }); +} + +describe('buildBackupManifest', () => { + it('should generate valid CNPG Backup manifest JSON', () => { + const manifest = buildBackupManifest('prole-db-a0001', 'test-backup-001'); + const parsed = JSON.parse(manifest); + + assert.equal(parsed.apiVersion, 'postgresql.cnpg.io/v1'); + assert.equal(parsed.kind, 'Backup'); + assert.equal(parsed.metadata.name, 'test-backup-001'); + assert.equal(parsed.metadata.namespace, 'test-ns'); + assert.equal(parsed.spec.method, 'plugin'); + assert.equal(parsed.spec.pluginConfiguration.name, 'barman-cloud.cloudnative-pg.io'); + assert.equal(parsed.spec.pluginConfiguration.parameters.barmanObjectName, 'prole-db-barman-objectstore'); + assert.equal(parsed.spec.cluster.name, 'prole-db-a0001'); + }); + + it('should use configured namespace from environment', () => { + const manifest = buildBackupManifest('my-cluster', 'bk-1'); + const parsed = JSON.parse(manifest); + assert.equal(parsed.metadata.namespace, 'test-ns'); + }); +}); + +describe('generateBackupName', () => { + it('should include cluster name as prefix', () => { + const name = generateBackupName('prole-db-a0001'); + assert.ok(name.startsWith('prole-db-a0001-backup-')); + }); + + it('should include a timestamp suffix', () => { + const name = generateBackupName('test-cluster'); + const suffix = name.replace('test-cluster-backup-', ''); + assert.ok(/^\d{14}$/.test(suffix), `Expected 14-digit timestamp, got: ${suffix}`); + }); + + it('should generate unique names on successive calls', () => { + const a = generateBackupName('c'); + const b = generateBackupName('c'); + // Within same second they may match, but structure should be valid + assert.ok(a.startsWith('c-backup-')); + assert.ok(b.startsWith('c-backup-')); + }); +}); + +describe('HTTP endpoints', () => { + let server; + + before((_, done) => { + server = app.listen(0, done); + }); + + after((_, done) => { + server.close(done); + }); + + describe('GET /health', () => { + it('should return ok status', async () => { + const res = await request(server, 'GET', '/health'); + assert.equal(res.status, 200); + assert.equal(res.body.status, 'ok'); + assert.equal(res.body.service, 'prole-db-manager'); + }); + }); + + describe('POST /backup/:clusterId/full', () => { + it('should reject invalid cluster IDs', async () => { + const res = await request(server, 'POST', '/backup/INVALID_NAME!/full'); + assert.equal(res.status, 400); + assert.equal(res.body.error, 'Invalid cluster ID'); + }); + + it('should reject single-char cluster ID that fails regex', async () => { + const res = await request(server, 'POST', '/backup/-/full'); + assert.equal(res.status, 400); + }); + + it('should attempt backup for valid cluster ID (will fail without kubectl)', async () => { + const res = await request(server, 'POST', '/backup/prole-db-a0001/full'); + // Without kubectl available, expect 500 or 503 + assert.ok([500, 503].includes(res.status)); + assert.equal(res.body.error, 'Backup trigger failed'); + assert.equal(res.body.cluster, 'prole-db-a0001'); + }); + }); + + describe('GET /backup/:clusterId/full', () => { + it('should reject invalid cluster IDs', async () => { + const res = await request(server, 'GET', '/backup/INVALID!/full'); + assert.equal(res.status, 400); + }); + + it('should attempt backup for valid cluster ID', async () => { + const res = await request(server, 'GET', '/backup/prole-db-a0001/full'); + assert.ok([500, 503].includes(res.status)); + assert.equal(res.body.cluster, 'prole-db-a0001'); + }); + }); + + describe('GET /backup/:clusterId/status', () => { + it('should return a response for status request', async () => { + const res = await request(server, 'GET', '/backup/prole-db-a0001/status'); + assert.ok([200, 500].includes(res.status)); + if (res.status === 200) { + assert.equal(res.body.cluster, 'prole-db-a0001'); + assert.equal(res.body.namespace, 'test-ns'); + } else { + assert.equal(res.body.error, 'Failed to get backup status'); + } + }); + }); +}); diff --git a/supabase/deploy.sh b/supabase/deploy.sh index ad3ab84..852b031 100755 --- a/supabase/deploy.sh +++ b/supabase/deploy.sh @@ -553,13 +553,12 @@ metadata: labels: app: supabase-postgres spec: - selector: - io.kompose.service: db + type: ExternalName + externalName: ${PROLE_DB_SERVICE}.${PROLE_DB_NAMESPACE}.svc.cluster.local ports: - name: postgres port: ${SUPABASE_POSTGRES_PORT} targetPort: 5432 - type: ClusterIP EOF } @@ -584,6 +583,161 @@ spec: EOF } +write_rest_alias_service() { + local file="$SUPABASE_K8S_DIR/rest-service.yaml" + + cat > "$file" < emptyDir (postgres needs writable data dir) + local db_deploy="$SUPABASE_K8S_DIR/db-deployment.yaml" + if [[ -f "$db_deploy" ]]; then + python3 - "$db_deploy" <<'PYFIX' +import sys, re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +# Replace the db-cm4 configMap volume definition with emptyDir +# Matches: - configMap:\n name: db-cm4\n name: db-cm4 +content = re.sub( + r'(\s+)-\s+configMap:\s*\n\s+name:\s*db-cm4\s*\n(\s+name:\s*db-cm4)', + r'\1- emptyDir: {}\n\2', + content +) + +# Also handle the alternate format kompose might produce +content = re.sub( + r'(\s+)-\s+configMap:\s*\n\s+defaultMode:\s*\d+\s*\n\s+name:\s*db-cm4\s*\n(\s+name:\s*db-cm4)', + r'\1- emptyDir: {}\n\2', + content +) + +with open(path, 'w') as f: + f.write(content) +print(f" [OK] db data volume -> emptyDir") +PYFIX + fi + + # 2. Remove db-cm4 configmap (no longer needed; postgres inits its own data dir) + local db_cm4="$SUPABASE_K8S_DIR/db-cm4-configmap.yaml" + if [[ -f "$db_cm4" ]]; then + rm -f "$db_cm4" + log " Removed db-cm4-configmap.yaml (data dir handled by emptyDir)" + fi + + # 3. Scale vector to 0 replicas (docker_logs source needs docker.sock, unavailable in k8s) + local vector_deploy="$SUPABASE_K8S_DIR/vector-deployment.yaml" + if [[ -f "$vector_deploy" ]]; then + python3 - "$vector_deploy" <<'PYFIX' +import sys, re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +content = re.sub(r'(spec:\s*\n\s+replicas:)\s*\d+', r'\1 0', content, count=1) + +with open(path, 'w') as f: + f.write(content) +print(" [OK] vector replicas -> 0 (docker_logs incompatible with k8s)") +PYFIX + fi + + # 4. Scale functions to 0 replicas (edge-runtime needs function files not present in k8s) + local functions_deploy="$SUPABASE_K8S_DIR/functions-deployment.yaml" + if [[ -f "$functions_deploy" ]]; then + python3 - "$functions_deploy" <<'PYFIX' +import sys, re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +content = re.sub(r'(spec:\s*\n\s+replicas:)\s*\d+', r'\1 0', content, count=1) + +with open(path, 'w') as f: + f.write(content) +print(" [OK] functions replicas -> 0 (edge-runtime entrypoint not available in k8s)") +PYFIX + fi + + # 5. Scale rest (PostgREST) to 0 replicas (using prole-db postgrest instead) + local rest_deploy="$SUPABASE_K8S_DIR/rest-deployment.yaml" + if [[ -f "$rest_deploy" ]]; then + python3 - "$rest_deploy" <<'PYFIX' +import sys, re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +content = re.sub(r'(spec:\s*\n\s+replicas:)\s*\d+', r'\1 0', content, count=1) + +with open(path, 'w') as f: + f.write(content) +print(" [OK] rest replicas -> 0 (using prole-db postgrest)") +PYFIX + fi + + # 6. Scale db deployment to 0 replicas (using prole-db instead of supabase standalone postgres) + if [[ -f "$db_deploy" ]]; then + python3 - "$db_deploy" <<'PYFIX' +import sys, re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +content = re.sub(r'(spec:\s*\n\s+replicas:)\s*\d+', r'\1 0', content, count=1) + +with open(path, 'w') as f: + f.write(content) +print(" [OK] db replicas -> 0 (using prole-db as database backend)") +PYFIX + fi + + # 7. Fix liveness probes that reference Docker Compose service hostnames + # In Docker Compose, service names resolve via internal DNS, but in k8s + # there is no matching Service for every container. Rewrite probes to + # use localhost (the probe checks the container's own health endpoint). + local storage_deploy="$SUPABASE_K8S_DIR/storage-deployment.yaml" + if [[ -f "$storage_deploy" ]]; then + python3 - "$storage_deploy" <<'PYFIX' +import sys + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +content = content.replace('http://storage:5000/status', 'http://localhost:5000/status') + +with open(path, 'w') as f: + f.write(content) +print(" [OK] storage liveness probe -> localhost:5000 (no k8s Service needed)") +PYFIX + fi +} + generate_k8s_manifests() { ensure_kompose ensure_env_file @@ -597,13 +751,12 @@ generate_k8s_manifests() { log "Generating Kubernetes manifests from docker-compose.yml" (cd "$DOCKER_DIR" && kompose "${files[@]}" -n supabase -o "$SUPABASE_K8S_DIR" --volumes=configMap --suppress-warnings convert) + fix_k8s_manifests resolve_prole_db_namespace - if ! kubectl get svc "$PROLE_DB_SERVICE" -n "$PROLE_DB_NAMESPACE" >/dev/null 2>&1; then - warn "Prole DB service not found: ${PROLE_DB_SERVICE} in namespace ${PROLE_DB_NAMESPACE}" - fi + write_db_alias_service + write_rest_alias_service patch_db_deployment_port write_supabase_postgres_service - write_db_alias_service sanitize_container_names } @@ -648,8 +801,18 @@ ensure_supabase_namespace() { } force_reset_supabase_namespace() { + if ! kubectl get namespace supabase >/dev/null 2>&1; then + ensure_supabase_namespace + return + fi + log "Resetting 'supabase' namespace..." - kubectl delete namespace supabase --ignore-not-found >/dev/null 2>&1 || true + + # Delete all workloads first to avoid finalizer stalls + kubectl delete all --all -n supabase --timeout=60s >/dev/null 2>&1 || true + + # Request namespace deletion without blocking + kubectl delete namespace supabase --ignore-not-found --wait=false >/dev/null 2>&1 || true if kubectl get namespace supabase >/dev/null 2>&1; then if ! kubectl wait --for=delete namespace/supabase --timeout=120s >/dev/null 2>&1; then @@ -670,7 +833,7 @@ try: except Exception: sys.exit(0) PY - kubectl wait --for=delete namespace/supabase --timeout=120s >/dev/null 2>&1 || true + kubectl wait --for=delete namespace/supabase --timeout=60s >/dev/null 2>&1 || true fi fi diff --git a/tests/etc/test_init_db_manager.sh b/tests/etc/test_init_db_manager.sh new file mode 100755 index 0000000..d073483 --- /dev/null +++ b/tests/etc/test_init_db_manager.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Unit test for etc/init_db_manager.sh +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) +ETC_DIR="$PROLE_HOME/etc" +SCRIPT_UNDER_TEST="$ETC_DIR/init_db_manager.sh" +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT +# Mock tools +mock_tool() { + cat < "$TMP_DIR/$1" +#!/usr/bin/env bash +echo "Mocked $1 called with \$@" >> "$TMP_DIR/mock_calls.log" +exit 0 +M_EOF + chmod +x "$TMP_DIR/$1" +} +# Create mocks for common tools +mock_tool kubectl +mock_tool curl +mock_tool docker +mock_tool k3d +mock_tool ansible-playbook +mock_tool tofu +mock_tool terraform +mock_tool ollama +mock_tool jq +# Mock prole.cfg +mkdir -p "$TMP_DIR/conf" +cat < "$TMP_DIR/conf/prole.cfg" +[globals] +prole.home = $PROLE_HOME +prole.mode = k3d +[k3s] +server = https://localhost:6443 +token = test-token +C_EOF +export PATH="$TMP_DIR:$PATH" +export PROLE_HOME="$PROLE_HOME" +export PROLE_CONF="$TMP_DIR/conf" +export NAMESPACE="test-ns" +export PROLE_PASSWD="test-password" +# Run the script +if [[ "init_db_manager.sh" == *.py ]]; then + if [[ "init_db_manager.sh" == "render_manifest.py" ]]; then + echo "apiVersion: v1" > "$TMP_DIR/test.yaml" + python3 "$SCRIPT_UNDER_TEST" "$TMP_DIR/test.yaml" > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr" + else + python3 "$SCRIPT_UNDER_TEST" --help > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr" + fi + RC=$? +else + # Check for usage() or if it's a script that likely needs arguments + if grep -q "usage()" "$SCRIPT_UNDER_TEST"; then + bash "$SCRIPT_UNDER_TEST" --help > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr" + RC=$? + else + # Try a few common non-destructive actions + bash "$SCRIPT_UNDER_TEST" status > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr" || \ + bash "$SCRIPT_UNDER_TEST" --help > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr" || \ + bash "$SCRIPT_UNDER_TEST" > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr" + RC=$? + fi +fi +if [[ $RC -eq 0 || $RC -eq 1 || $RC -eq 2 ]]; then + echo "SUCCESS (RC=$RC)" + exit 0 +else + echo "FAILURE with RC $RC" + cat "$TMP_DIR/stderr" + exit 1 +fi