diff --git a/conf/port-mapping.cfg b/conf/port-mapping.cfg index ec67e3a..02dd462 100644 --- a/conf/port-mapping.cfg +++ b/conf/port-mapping.cfg @@ -2,6 +2,10 @@ # Format: key: local=... remote=... ns=... svc=... address=... argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0 +supabase-studio: local=18080 remote=3000 ns=supabase svc=studio address=0.0.0.0 +supabase-auth: local=9999 remote=9999 ns=supabase svc=auth address=127.0.0.1 +supabase-rest: local=3001 remote=3000 ns=supabase svc=rest address=0.0.0.0 +supabase-realtime: local=4000 remote=4000 ns=supabase svc=realtime address=0.0.0.0 garage: local=3900 remote=3900 ns=default svc=garage address=0.0.0.0 openbao: local=8200 remote=8200 ns=default svc=openbao address=127.0.0.1 opentofu: local=8080 remote=8080 ns=default svc=opentofu address=0.0.0.0 @@ -9,3 +13,4 @@ dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboar postgres: local=5432 remote=5432 ns=prole-db001 svc=prole-db-rw address=0.0.0.0 prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1 grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0 +supabase-kong: local=8000 remote=8000 ns=supabase svc=kong address=0.0.0.0 diff --git a/conf/prole.cfg b/conf/prole.cfg index 40c2c2a..9e43bbf 100644 --- a/conf/prole.cfg +++ b/conf/prole.cfg @@ -1,15 +1,15 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-03-01 19:22:14 +; Generated by install.py on 2026-03-02 00:06:12 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] ; User-editable values; derived values below reference these by default. NAMESPACE = prole-db001 -PROLE_CONF = /Users/chrisfu/dev/prole/conf +PROLE_CONF = ${PROLE_HOME}/conf PROLE_DATA = /Users/chrisfu/.prole/data -PROLE_HOME = mock_val +PROLE_HOME = /Users/chrisfu/dev/prole PROLE_LOGS = /opt/prole/logs/chrisfu -PROLE_SERVICE = /Users/chrisfu/dev/prole/etc +PROLE_SERVICE = ${PROLE_HOME}/etc SERVICE_NAMESPACE = default [Inputs] @@ -17,19 +17,19 @@ SERVICE_NAMESPACE = default build.deploy_env = Dev build.run_build = false database_options.distribution = percona -database_options.ext.dblink = True -database_options.ext.pg_buffercache = True -database_options.ext.pg_freespacemap = True -database_options.ext.pg_repack = True -database_options.ext.pg_stat_monitor = True -database_options.ext.pg_stat_statements = True -database_options.ext.pgaudit = True -database_options.ext.pgbadger = True -database_options.ext.pgcrypto = True -database_options.ext.pgrowlocks = True -database_options.ext.pgvector = True -database_options.ext.postgis = True -database_options.ext.postgres_fdw = True +database_options.ext.dblink = true +database_options.ext.pg_buffercache = true +database_options.ext.pg_freespacemap = true +database_options.ext.pg_repack = true +database_options.ext.pg_stat_monitor = true +database_options.ext.pg_stat_statements = true +database_options.ext.pgaudit = true +database_options.ext.pgbadger = true +database_options.ext.pgcrypto = true +database_options.ext.pgrowlocks = true +database_options.ext.pgvector = true +database_options.ext.postgis = true +database_options.ext.postgres_fdw = true database_options.version_type = v18 dependencies.ansible.install = true dependencies.auto_install_missing = true @@ -42,7 +42,7 @@ dependencies.opentofu.install = true dependencies.python.install = true dependencies.verify_all = false disk_selection.disk_type = local -disk_selection.local_path = /Users/chrisfu +disk_selection.local_path = ${PROLE_HOME}/prole-tools-app/dist disk_selection.removable_mount = env_setup.NAMESPACE = ${NAMESPACE} env_setup.PROLE_CONF = ${PROLE_CONF} @@ -52,14 +52,14 @@ env_setup.PROLE_LOGS = ${PROLE_LOGS} env_setup.PROLE_SERVICE = ${PROLE_SERVICE} init_cluster.at_rest_encryption_enabled = true init_cluster.cluster_env = dev -init_cluster.deployment_target = prole-dev-cluster +init_cluster.deployment_target = knoe-dev-cluster init_cluster.gitops_enabled = false -init_cluster.k3s_server_url = -init_cluster.k3s_token = -init_cluster.kerberos_enabled = false +init_cluster.k3s_server_url = https://myrddin.prole.org:6443 +init_cluster.k3s_token = ${OPENBAO:kv/prole/prole-db001/k3s#token} +init_cluster.kerberos_enabled = true init_cluster.mode = k3d init_cluster.start_cluster = true -init_cluster.supabase_enabled = false +init_cluster.supabase_enabled = true init_cnpg_deploy.force_rollout = false init_cnpg_deploy.run_deploy = true init_db_build.run_build = true @@ -67,15 +67,15 @@ init_password.db_host_port = 5432 init_password.db_namespace = ${NAMESPACE} init_password.db_password = ${OPENBAO:kv/prole/prole-db001/db#password} init_password.db_password_confirm = ${OPENBAO:kv/prole/prole-db001/db#password} -init_password.db_username = root +init_password.db_username = chrisfu init_password.generate_ssh_key = true init_scripts.run_scripts = true -kerberos_config.enabled = False +kerberos_config.enabled = true kerberos_config.init_authority = false -kerberos_config.kdc = 10.0.0.205 +kerberos_config.kdc = 10.0.0.3 kerberos_config.password = ${OPENBAO:kv/prole/prole-db001/kerberos#password} -kerberos_config.realm = -kerberos_config.test_connection = false +kerberos_config.realm = PROLE.ORG +kerberos_config.test_connection = true kerberos_config.user = administrator network_scan.run = true ollama_config.model = @@ -92,9 +92,9 @@ DEPLOYMENT_MODE = k3d DEPLOYMENT_TARGET = knoe-dev-cluster DOCKER_IMPORT_DIR = NAMESPACE = ${NAMESPACE} -PROLE_DB_USER = root +PROLE_DB_USER = chrisfu PROLE_HOME = ${PROLE_HOME} -PROLE_K3S_SERVER = +PROLE_K3S_SERVER = https://myrddin.prole.org:6443 PROLE_K3S_TOKEN = ${OPENBAO:kv/prole/prole-db001/k3s#token} PROLE_OPENTOFU_URL = http://127.0.0.1:8080 REGISTRY_NAMESPACE = default @@ -104,22 +104,26 @@ SERVICE_NAMESPACE = default ; No configuration values captured yet for this section. [Dependencies] -STATUS = All installed +; No configuration values captured yet for this section. [Network] AD_DC_HOST = myrddin.prole.org AD_DC_IP = 10.0.0.3 ANSIBLE_DOMAIN = prole.org -ANSIBLE_INFRASTRUCTURE = /Users/chrisfu/dev/prole/infrastructure -ANSIBLE_INVENTORY = /Users/chrisfu/dev/prole/infrastructure/inventory +ANSIBLE_INFRASTRUCTURE = ${PROLE_HOME}/infrastructure +ANSIBLE_INVENTORY = ${PROLE_HOME}/infrastructure/inventory ANSIBLE_REALM = PROLE.ORG ANSIBLE_TOPOLOGY = {"domain":"prole.org","realm":"PROLE.ORG","internal_records":{"aventage.prole.org":"10.0.0.206","fairyland.prole.org":"10.0.0.208","loghost.prole.org":"10.0.0.3","morana.prole.org":"10.0.0.66","morgoth.prole.org":"10.0.0.204","myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207","synology.prole.org":"10.0.0.203","zinfandel.prole.org":"10.0.0.205"},"ad_dc":{"host":"myrddin.prole.org","ip":"10.0.0.3"},"k3s":{"server_url":"https://myrddin.prole.org:6443","server_host":"myrddin.prole.org","token_present":true},"groups":{"iscsi":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"],"pihole":["pi.prole.org","raspberry.prole.org"],"ad_dc":["myrddin.prole.org"],"k3s_hosts":["myrddin.prole.org","pi.prole.org","retropie.prole.org"],"linux_hosts":["pi.prole.org","raspberry.prole.org","myrddin.prole.org","retropie.prole.org"]},"hosts":{"myrddin.prole.org":"10.0.0.3","pi.prole.org":"10.0.0.5","raspberry.prole.org":"10.0.0.4","retropie.prole.org":"10.0.0.207"},"unmapped_hosts":[]} KDC_ANSIBLE_DETECTED = 10.0.0.3 -KDC_AUTO_DETECTED = 10.0.0.205 -KERBEROS_AUTO_ENABLED = False +KDC_AUTO_DETECTED = 10.0.0.3 +KERBEROS_AUTO_ENABLED = True [Port Forwards] PORT_FORWARD_K3D_MAPPING_1 = id=argocd;namespace=argocd;target=svc/argocd-server;address=0.0.0.0;hostPort=8081;servicePort=80;protocol=TCP;description=ArgoCD +PORT_FORWARD_K3D_MAPPING_10 = id=supabase-studio;namespace=supabase;target=svc/studio;address=0.0.0.0;hostPort=18080;servicePort=3000;protocol=TCP;description=Supabase Studio +PORT_FORWARD_K3D_MAPPING_11 = id=supabase-auth;namespace=supabase;target=svc/auth;address=127.0.0.1;hostPort=9999;servicePort=9999;protocol=TCP;description=Supabase Auth (GoTrue) +PORT_FORWARD_K3D_MAPPING_12 = id=supabase-rest;namespace=supabase;target=svc/rest;address=0.0.0.0;hostPort=3001;servicePort=3000;protocol=TCP;description=Supabase REST (PostgREST) +PORT_FORWARD_K3D_MAPPING_13 = id=supabase-realtime;namespace=supabase;target=svc/realtime;address=0.0.0.0;hostPort=4000;servicePort=4000;protocol=TCP;description=Supabase Realtime PORT_FORWARD_K3D_MAPPING_2 = id=garage;namespace=default;target=svc/garage;address=0.0.0.0;hostPort=3900;servicePort=3900;protocol=TCP;description=Garage S3 PORT_FORWARD_K3D_MAPPING_3 = id=openbao;namespace=default;target=svc/openbao;address=127.0.0.1;hostPort=8200;servicePort=8200;protocol=TCP;description=OpenBao PORT_FORWARD_K3D_MAPPING_4 = id=opentofu;namespace=default;target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu @@ -127,10 +131,9 @@ PORT_FORWARD_K3D_MAPPING_5 = id=dashboard;namespace=kubernetes-dashboard;target= PORT_FORWARD_K3D_MAPPING_6 = id=postgres;namespace=${NAMESPACE};target=svc/prole-db-rw;address=0.0.0.0;hostPort=5432;servicePort=5432;protocol=TCP;description=PostgreSQL (primary) PORT_FORWARD_K3D_MAPPING_7 = id=prometheus;namespace=monitoring;target=svc/kps-kube-prometheus-stack-prometheus;address=127.0.0.1;hostPort=9090;servicePort=9090;protocol=TCP;description=Prometheus UI PORT_FORWARD_K3D_MAPPING_8 = id=grafana;namespace=monitoring;target=svc/kps-grafana;address=0.0.0.0;hostPort=3000;servicePort=80;protocol=TCP;description=Grafana UI -STATUS = Running +PORT_FORWARD_K3D_MAPPING_9 = id=supabase-kong;namespace=supabase;target=svc/kong;address=0.0.0.0;hostPort=8000;servicePort=8000;protocol=TCP;description=Supabase API (Kong) [System Environment] -NAMESPACE = prole-db001 PROLE_CONF = ${PROLE_CONF} PROLE_DATA = ${PROLE_DATA} PROLE_HOME = ${PROLE_HOME} @@ -141,8 +144,18 @@ PROLE_SERVICE = ${PROLE_SERVICE} ; No configuration values captured yet for this section. [Kerberos Authentication] +AD_PORT_FORWARD = 1 +AD_PROXY_HOST_NETWORK = 1 +AD_PROXY_IMAGE = alpine/socat +AD_PROXY_SERVICE = prole-kerberos-ad-dc +AD_TCP_PORTS = 88 389 445 464 636 +AD_UDP_PORTS = 88 464 +ENABLED = True +KDC = 10.0.0.3 PASSWORD = ${OPENBAO:kv/prole/prole-db001/kerberos#password} -STATUS = Skipped +REALM = PROLE.ORG +SERVER = 10.0.0.3 +USER = administrator [Ollama] ; No configuration values captured yet for this section. @@ -150,19 +163,18 @@ STATUS = Skipped [Optional Features] AT_REST_ENCRYPTION_ENABLED = true GITOPS_ENABLED = false -KERBEROS_ENABLED = false -SUPABASE_ENABLED = false +KERBEROS_ENABLED = true +SUPABASE_ENABLED = true [GitOps] -STATUS = Skipped [Database Creation] -DB_NAME = prole-db001 -DB_USER = root NAMESPACE = ${NAMESPACE} [Initialize Cluster] -; No configuration values captured yet for this section. +ENVIRONMENT = knoe-dev-cluster +K3S_SERVER_URL = https://myrddin.prole.org:6443 +K3S_TOKEN = ${OPENBAO:kv/prole/prole-db001/k3s#token} [Dev Cluster (k3d)] CLUSTER_ENV = k3d-knoe-dev-cluster @@ -173,7 +185,7 @@ MODE = k3d [Service Cluster (k3s)] CLUSTER_ENV = prole-service-cluster DISPLAY_NAME = prole-service-cluster -K3S_SERVER_URL = +K3S_SERVER_URL = https://myrddin.prole.org:6443 K3S_TOKEN = ${OPENBAO:kv/prole/prole-db001/k3s#token} MODE = k3s PIPELINE_URL = http://127.0.0.1:8080 @@ -186,14 +198,15 @@ MODE = k8s PIPELINE_URL = http://127.0.0.1:8080 [Docker Build] -; No configuration values captured yet for this section. +LOCAL_REGISTRY = localhost:5000 +LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000 [Initialization Scripts] -; No configuration values captured yet for this section. +STATUS = Completed [Deployment] MODE = k3d -TARGET = prole-dev-cluster +TARGET = knoe-dev-cluster [Install] -STATUS = Finished +; No configuration values captured yet for this section. diff --git a/etc/init_cloudnative_pg.sh b/etc/init_cloudnative_pg.sh index 914cd09..417a9cb 100755 --- a/etc/init_cloudnative_pg.sh +++ b/etc/init_cloudnative_pg.sh @@ -1046,13 +1046,13 @@ wait_for_backup() { } run_garage_backup() { - if [[ ! -x "$SCRIPT_DIR/init_prole-db-backup.sh" ]]; then - echo "WARN: init_prole-db-backup.sh not found; skipping Garage backup." >&2 + if [[ ! -x "$SCRIPT_DIR/init_cnpg_backup.sh" ]]; then + echo "WARN: init_cnpg_backup.sh not found; skipping Garage backup." >&2 return 1 fi - echo "Running Garage backup via init_prole-db-backup.sh ..." - if ! "$SCRIPT_DIR/init_prole-db-backup.sh" start; then + echo "Running Garage backup via init_cnpg_backup.sh ..." + if ! "$SCRIPT_DIR/init_cnpg_backup.sh" start; then echo "Garage backup script failed." >&2 return 1 fi diff --git a/etc/init_prole-db-backup.sh b/etc/init_cnpg_backup.sh similarity index 72% rename from etc/init_prole-db-backup.sh rename to etc/init_cnpg_backup.sh index 531a1a6..5bc3d07 100755 --- a/etc/init_prole-db-backup.sh +++ b/etc/init_cnpg_backup.sh @@ -2,7 +2,7 @@ set -euo pipefail -# init_prole-db-backup.sh +# init_cnpg_backup.sh # Purpose: # - Configure CloudNative-PG to backup to Garage (S3-compatible) via Barman Cloud Plugin # - Create initial backup @@ -44,6 +44,9 @@ RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1} RETENTION_POLICY=${RETENTION_POLICY:-30d} BARMAN_PLUGIN_NAME=${BARMAN_PLUGIN_NAME:-barman-cloud.cloudnative-pg.io} BARMAN_OBJECT_NAME=${BARMAN_OBJECT_NAME:-prole-db-barman-objectstore} +BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600} +BACKUP_STATUS_INTERVAL=${BACKUP_STATUS_INTERVAL:-10} +PLUGIN_READY_TIMEOUT=${PLUGIN_READY_TIMEOUT:-180} usage() { cat </dev/null 2>&1 || true } +wait_for_plugin_ready() { + local start_time now plugin_names deployment_rows ns ready replicas + start_time=$(date +%s) + while true; do + plugin_names=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.plugins[*].name}' 2>/dev/null || true) + if printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -Fxq "$BARMAN_PLUGIN_NAME"; then + deployment_rows=$(kubectl get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) + if [[ -z "$deployment_rows" ]]; then + deployment_rows=$(kubectl get deployment -A -o jsonpath='{range .items[?(@.metadata.name=="barman-cloud")]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) + fi + + while IFS=$'\t' read -r ns ready replicas; do + [[ -z "$ns" ]] && continue + ready=${ready:-0} + replicas=${replicas:-0} + if (( ready >= 1 && replicas >= 1 )); then + return 0 + fi + done <<< "$deployment_rows" + fi + + now=$(date +%s) + if (( now - start_time >= PLUGIN_READY_TIMEOUT )); then + echo "ERROR: Timed out waiting for CNPG plugin '$BARMAN_PLUGIN_NAME' to become available." >&2 + return 1 + fi + + echo "Waiting for CNPG plugin '$BARMAN_PLUGIN_NAME' to become available ..." + sleep 5 + done +} + +has_successful_base_backup() { + local rows name phase method backup_type phase_lc method_lc backup_type_lc + rows=$(kubectl get backup -n "$NAMESPACE" -o jsonpath="{range .items[?(@.spec.cluster.name=='$CNPG_CLUSTER_NAME')]}{.metadata.name}{\"\t\"}{.status.phase}{\"\t\"}{.spec.method}{\"\t\"}{.spec.pluginConfiguration.parameters.backupType}{\"\n\"}{end}" 2>/dev/null || true) + + if [[ -z "$rows" ]]; then + return 1 + fi + + while IFS=$'\t' read -r name phase method backup_type; do + [[ -z "$name" ]] && continue + phase_lc=$(echo "${phase:-}" | tr '[:upper:]' '[:lower:]') + method_lc=$(echo "${method:-}" | tr '[:upper:]' '[:lower:]') + backup_type_lc=$(echo "${backup_type:-}" | tr '[:upper:]' '[:lower:]') + + if [[ "$phase_lc" != "completed" && "$phase_lc" != "succeeded" ]]; then + continue + fi + + if [[ "$method_lc" == "plugin" ]] && [[ "$backup_type_lc" == "incremental" || "$backup_type_lc" == "incr" ]]; then + continue + fi + + return 0 + done <<< "$rows" + + return 1 +} + +wait_for_successful_base_backup() { + local start_time now elapsed + start_time=$(date +%s) + while true; do + if has_successful_base_backup; then + return 0 + fi + + now=$(date +%s) + elapsed=$((now - start_time)) + if (( elapsed >= BACKUP_STATUS_TIMEOUT )); then + return 1 + fi + + echo "Waiting for a successful base backup ... (${elapsed}s/${BACKUP_STATUS_TIMEOUT}s)" + sleep "$BACKUP_STATUS_INTERVAL" + done +} + trigger_backup() { local backup_type="${1:-full}" local backup_name @@ -281,6 +363,14 @@ status() { ensure_tools echo "Backups for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE':" kubectl get backup -n "$NAMESPACE" | grep "$CNPG_CLUSTER_NAME" || true + + if has_successful_base_backup; then + echo "Backup status: OK (at least one successful base backup is available)." + return 0 + fi + + echo "ERROR: No successful base backup found for cluster '$CNPG_CLUSTER_NAME'." >&2 + return 1 } case "$ACTION" in @@ -291,10 +381,17 @@ case "$ACTION" in ensure_garage_bucket_and_key apply_barman_object_store ensure_barman_plugin_config + wait_for_plugin_ready remove_native_barman_config if [[ "$RUN_FIRST_BACKUP" == "1" ]]; then trigger_backup fi + if ! wait_for_successful_base_backup; then + echo "ERROR: Timed out waiting for a successful base backup." >&2 + status || true + exit 1 + fi + status ;; backup) ensure_tools diff --git a/etc/init_common_services.sh b/etc/init_common_services.sh index b6129db..12457b9 100755 --- a/etc/init_common_services.sh +++ b/etc/init_common_services.sh @@ -89,6 +89,7 @@ ARGOCD_NS="${ARGOCD_NAMESPACE:-argocd}" REGISTRY_NS="${REGISTRY_NAMESPACE:-default}" prole_ensure_kubeconfig >/dev/null 2>&1 || true +prole_ensure_kube_context || exit 1 # Check cluster reachability early to fail fast if [[ "$ACTION" != "status" ]]; then diff --git a/etc/init_gitea.sh b/etc/init_gitea.sh deleted file mode 100755 index 861c6e5..0000000 --- a/etc/init_gitea.sh +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -PROG="init_gitea" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# shellcheck disable=SC1090 -source "$SCRIPT_DIR/prole_cfg.sh" - -MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")" -NAMESPACE="${GITEA_NAMESPACE:-}" -CFG_PATH="" -FORCE=0 - -usage() { - cat < Deployment mode (default: ${MODE:-k3d}) - -n, --namespace Target namespace (default: gitea) - -c, --config Path to prole.cfg (defaults to detected) - --force Delete existing release before deploy - --help Show this help - -Behavior: - - Prefers Helm install/upgrade (gitea-charts/gitea). - - Falls back to applying a generated Deployment/Service/Ingress manifest. - - If a local registry is enabled, the image is mirrored there first. -EOF -} - -die() { echo "[ERROR] $*" >&2; exit 2; } -log() { echo "[INFO] $*"; } -warn() { echo "[WARN] $*" >&2; } - -while [[ $# -gt 0 ]]; do - case "$1" in - --mode) MODE="$(prole_normalize_mode "${2:-}")"; shift 2 ;; - --mode=*) MODE="$(prole_normalize_mode "${1#*=}")"; shift 1 ;; - -n|--namespace) NAMESPACE="${2:-}"; shift 2 ;; - --namespace=*) NAMESPACE="${1#*=}"; shift 1 ;; - -c|--config) CFG_PATH="${2:-}"; shift 2 ;; - --config=*) CFG_PATH="${1#*=}"; shift 1 ;; - --force) FORCE=1; shift ;; - -h|--help) usage; exit 0 ;; - *) break ;; - esac -done - -# Resolve config path and namespace defaults from prole.cfg when present -if [[ -z "$CFG_PATH" && -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then - CFG_PATH="${PROLE_CONF}/prole.cfg" -elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then - CFG_PATH="$SCRIPT_DIR/../conf/prole.cfg" -fi - -if [[ -z "$NAMESPACE" && -n "$CFG_PATH" ]]; then - maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITOPS_NAMESPACE")" - [[ -z "$maybe_ns" ]] && maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITEA_NAMESPACE")" - NAMESPACE="$maybe_ns" -fi -NAMESPACE="${NAMESPACE:-gitea}" -export GITEA_NAMESPACE="$NAMESPACE" - -case "$MODE" in - k3d|k3s|k8s|local) ;; - *) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;; -esac -export PROLE_MODE="$MODE" - -command -v kubectl >/dev/null || die "kubectl not found" -command -v helm >/dev/null || warn "helm not found — will use manifest fallback" - -kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 - -CHART_REPO="https://dl.gitea.com/charts" -CHART_NAME="gitea-charts/gitea" -RELEASE_NAME="gitea" -IMAGE_REPO_DEFAULT="gitea/gitea" -IMAGE_TAG="${GITEA_IMAGE_TAG:-1.22.3}" -IMAGE_REPO="$IMAGE_REPO_DEFAULT" - -# Optional reset -if [[ "$FORCE" -eq 1 ]]; then - warn "--force specified; removing existing Gitea resources in namespace $NAMESPACE" - if command -v helm >/dev/null 2>&1; then - helm uninstall "$RELEASE_NAME" -n "$NAMESPACE" >/dev/null 2>&1 || true - fi - kubectl -n "$NAMESPACE" delete deploy/gitea svc/gitea-http svc/gitea-ssh ingress/gitea >/dev/null 2>&1 || true -fi - -# Mirror to local registry if enabled -if _prole_local_registry_enabled && command -v docker >/dev/null; then - HOST_REG="${LOCAL_REGISTRY:-${PROLE_LOCAL_REGISTRY:-}}" - INTERNAL_REG="${LOCAL_REGISTRY_INTERNAL:-${PROLE_LOCAL_REGISTRY_INTERNAL:-}}" - if [[ -n "$HOST_REG" ]]; then - HOST_IMG="${HOST_REG}/${IMAGE_REPO_DEFAULT}" - log "Pulling ${IMAGE_REPO_DEFAULT}:${IMAGE_TAG}" - docker pull "${IMAGE_REPO_DEFAULT}:${IMAGE_TAG}" >/dev/null - log "Tagging ${HOST_IMG}:${IMAGE_TAG}" - docker tag "${IMAGE_REPO_DEFAULT}:${IMAGE_TAG}" "${HOST_IMG}:${IMAGE_TAG}" - log "Pushing ${HOST_IMG}:${IMAGE_TAG}" - docker push "${HOST_IMG}:${IMAGE_TAG}" >/dev/null || warn "Push to local registry failed; continuing with upstream image" - if [[ -n "$INTERNAL_REG" ]]; then - IMAGE_REPO="${INTERNAL_REG}/gitea/gitea" - else - IMAGE_REPO="${HOST_REG}/gitea/gitea" - fi - fi -fi - -VALUES_FILE="$(mktemp /tmp/gitea-values.XXXXXX.yaml)" -cat >"$VALUES_FILE" </dev/null; then - if ! helm repo list 2>/dev/null | grep -q "gitea-charts"; then - helm repo add gitea-charts "$CHART_REPO" >/dev/null 2>&1 || warn "helm repo add failed" - fi - helm repo update >/dev/null 2>&1 || true - - log "Deploying Gitea via Helm to namespace ${NAMESPACE} (mode=${MODE})" - set +e - helm upgrade --install "$RELEASE_NAME" "$CHART_NAME" \ - --namespace "$NAMESPACE" \ - --create-namespace \ - -f "$VALUES_FILE" - rc=$? - set -e - if [[ $rc -eq 0 ]]; then - helm_deployed=1 - log "Helm deployment successful." - else - warn "Helm deployment failed (code $rc); falling back to manifests." - fi -fi - -if [[ $helm_deployed -eq 0 ]]; then - MANIFEST_FILE="$(mktemp /tmp/gitea-manifest.XXXXXX.yaml)" - cat >"$MANIFEST_FILE" <>"$MANIFEST_FILE" </dev/null 2>&1 || true +prole_ensure_kube_context || exit 1 ACTION=${1:-initialize} diff --git a/etc/init_monitoring.sh b/etc/init_monitoring.sh index 14c068f..b660bce 100755 --- a/etc/init_monitoring.sh +++ b/etc/init_monitoring.sh @@ -328,7 +328,6 @@ resolve_grafana_password() { GRAFANA_ADMIN_PASSWORD="$fetched" fi fi - if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then local db_pw="${DB_PASSWORD:-}" if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then @@ -342,6 +341,10 @@ resolve_grafana_password() { GRAFANA_ADMIN_PASSWORD="$db_pw" fi fi + # Strip any trailing newlines/carriage-returns that may have crept in via + # prole.cfg parsing, file reads, or shell substitution edge-cases. + GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\n'/}" + GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\r'/}" } write_grafana_password_to_openbao() { @@ -351,9 +354,12 @@ write_grafana_password_to_openbao() { if [[ -z "$token" || -z "$url" || -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then return 0 fi + local _clean_grafana_pw + _clean_grafana_pw="${GRAFANA_ADMIN_PASSWORD//$'\n'/}" + _clean_grafana_pw="${_clean_grafana_pw//$'\r'/}" curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \ -X POST "$url/v1/kv/data/prole/${NAMESPACE:-default}/monitoring" \ - -d "{\"data\":{\"grafana_admin_password\":\"$GRAFANA_ADMIN_PASSWORD\"}}" >/dev/null || true + -d "{\"data\":{\"grafana_admin_password\":\"$_clean_grafana_pw\"}}" >/dev/null || true } cleanup_grafana_rbac_conflicts() { @@ -433,6 +439,9 @@ install_monitoring() { if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then GRAFANA_ADMIN_PASSWORD="admin" # Fallback fi + # Belt-and-suspenders: ensure no newline survives into the Helm values YAML. + GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\n'/}" + GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD//$'\r'/}" local values_file values_file=$(mktemp) diff --git a/etc/init_nginx_ingress.sh b/etc/init_nginx_ingress.sh index 19a31e7..ef3b1f1 100755 --- a/etc/init_nginx_ingress.sh +++ b/etc/init_nginx_ingress.sh @@ -61,6 +61,7 @@ if [[ -n "$MODE" && "$MODE" != "k3d" ]]; then fi prole_ensure_kubeconfig >/dev/null 2>&1 || true +prole_ensure_kube_context || exit 1 PORT_MAPPING_FILE_PATH="${PORT_MAPPING_FILE:-}" if [[ -z "$PORT_MAPPING_FILE_PATH" ]]; then diff --git a/etc/init_openbao.sh b/etc/init_openbao.sh index 5359f75..afef4e3 100755 --- a/etc/init_openbao.sh +++ b/etc/init_openbao.sh @@ -652,9 +652,12 @@ init_openbao_kv_and_store_admin_key() { if [[ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then echo "Writing Grafana admin password to kv/$BAO_PATH_MONITORING ..." + local _clean_grafana_pw + _clean_grafana_pw="${GRAFANA_ADMIN_PASSWORD//$'\n'/}" + _clean_grafana_pw="${_clean_grafana_pw//$'\r'/}" curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \ -X POST "$svc/v1/kv/data/$BAO_PATH_MONITORING" \ - -d "{\"data\":{\"grafana_admin_password\":\"$GRAFANA_ADMIN_PASSWORD\"}}" >/dev/null + -d "{\"data\":{\"grafana_admin_password\":\"$_clean_grafana_pw\"}}" >/dev/null echo "Stored Grafana admin password in OpenBao kv/$BAO_PATH_MONITORING." fi } diff --git a/etc/init_prole-db-reset.sh b/etc/init_prole-db-reset.sh index 8a514d7..b4093c4 100755 --- a/etc/init_prole-db-reset.sh +++ b/etc/init_prole-db-reset.sh @@ -162,15 +162,15 @@ run_barman_backup() { return 1 fi - if [[ -x "$SCRIPT_DIR/init_prole-db-backup.sh" ]]; then + if [[ -x "$SCRIPT_DIR/init_cnpg_backup.sh" ]]; then log "Running barman backup (Garage bucket: $GARAGE_BACKUP_BUCKET) ..." NAMESPACE="$NAMESPACE" \ GARAGE_BACKUP_BUCKET="$GARAGE_BACKUP_BUCKET" \ GARAGE_BACKUP_SECRET_NAME="$GARAGE_BACKUP_SECRET_NAME" \ GARAGE_S3_ENDPOINT="$GARAGE_S3_ENDPOINT" \ - RUN_FIRST_BACKUP=1 "$SCRIPT_DIR/init_prole-db-backup.sh" start + RUN_FIRST_BACKUP=1 "$SCRIPT_DIR/init_cnpg_backup.sh" start else - err "init_prole-db-backup.sh not found; cannot run backup." + err "init_cnpg_backup.sh not found; cannot run backup." return 1 fi diff --git a/etc/init_service_layer.sh b/etc/init_service_layer.sh index 75ff209..2e17874 100755 --- a/etc/init_service_layer.sh +++ b/etc/init_service_layer.sh @@ -29,6 +29,7 @@ common_core_preparse_config "$@" source "$SCRIPT_DIR/prole_cfg.sh" prole_ensure_kubeconfig >/dev/null 2>&1 || true +prole_ensure_kube_context || exit 1 ACTION="" SERVICE_NAMESPACE_OVERRIDE="" diff --git a/etc/prole_cfg.sh b/etc/prole_cfg.sh index 86ca5d6..d86c5fa 100644 --- a/etc/prole_cfg.sh +++ b/etc/prole_cfg.sh @@ -645,6 +645,64 @@ prole_ensure_kubeconfig() { _prole_kubeconfig_maybe_insecure } +# Validate and switch the active kubectl context to match PROLE_MODE set in prole.cfg. +# prole.cfg always overrides the local environment. +# Call this after prole_ensure_kubeconfig in every init_*.sh script. +# Usage: prole_ensure_kube_context +prole_ensure_kube_context() { + local mode + mode="${PROLE_MODE:-}" + if [[ -z "$mode" ]]; then + return 0 + fi + + local current_context + current_context=$(kubectl config current-context 2>/dev/null || true) + + case "$mode" in + k3d) + # k3d contexts are always named k3d- + if [[ "$current_context" == k3d-* ]]; then + return 0 + fi + local cluster_name target_context + cluster_name="${K3D_CLUSTER_NAME:-knoe-dev-cluster}" + target_context="k3d-${cluster_name}" + echo "WARN: prole.cfg mode is 'k3d' but current kubectx is '${current_context:-}'" >&2 + echo "INFO: Switching kubectl context to '${target_context}' (prole.cfg overrides local env)" >&2 + if kubectl config use-context "$target_context" >/dev/null 2>&1; then + echo "INFO: kubectl context set to '${target_context}'" >&2 + else + echo "ERROR: Could not switch to context '${target_context}'. Available contexts:" >&2 + kubectl config get-contexts --no-headers 2>/dev/null | awk '{print " "$2}' >&2 || true + return 1 + fi + ;; + k3s|k8s) + # k3s/k8s contexts must NOT be a k3d context + if [[ "$current_context" != k3d-* ]]; then + return 0 + fi + echo "WARN: prole.cfg mode is '${mode}' but current kubectx is '${current_context}' (a k3d context)" >&2 + echo "INFO: Switching kubectl context to match KUBECONFIG='${KUBECONFIG:-}' (prole.cfg overrides local env)" >&2 + local target_context + target_context=$(kubectl config get-contexts --no-headers 2>/dev/null \ + | awk '{print $2}' | grep -v '^k3d-' | head -1 || true) + if [[ -z "$target_context" ]]; then + echo "ERROR: No non-k3d context found in KUBECONFIG='${KUBECONFIG:-}'. Available contexts:" >&2 + kubectl config get-contexts --no-headers 2>/dev/null | awk '{print " "$2}' >&2 || true + return 1 + fi + if kubectl config use-context "$target_context" >/dev/null 2>&1; then + echo "INFO: kubectl context set to '${target_context}'" >&2 + else + echo "ERROR: Could not switch to context '${target_context}'" >&2 + return 1 + fi + ;; + esac +} + if _prole_is_secret_ref "${PROLE_K3S_TOKEN:-}"; then if _prole_is_openbao_ref "${PROLE_K3S_TOKEN:-}"; then _decoded=$(_prole_resolve_openbao_ref "${PROLE_K3S_TOKEN:-}") diff --git a/etc/status.sh b/etc/status.sh index 62aa2e7..c70f640 100755 --- a/etc/status.sh +++ b/etc/status.sh @@ -253,7 +253,7 @@ STATUS_SCRIPTS=( "init_kong.sh" "init_db_manager.sh" "init_monitoring.sh" - "init_prole-db-backup.sh" + "init_cnpg_backup.sh" "init_port_forwards.sh" ) diff --git a/gitea/deploy.sh b/gitea/deploy.sh new file mode 100755 index 0000000..172776c --- /dev/null +++ b/gitea/deploy.sh @@ -0,0 +1,1142 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +if [[ -f "$PROJECT_ROOT/env.sh" ]]; then + # shellcheck disable=SC1090 + source "$PROJECT_ROOT/env.sh" +fi + +MODE="k3d" +HELM_RELEASE="gitea" +USE_DEV_HELPERS="false" +FOREGROUND="false" +FORCE="true" +PROLE_CFG_PATH="" +HELM_TEMPLATE_ONLY="false" + +usage() { + cat <<'USAGE' +gitea/deploy.sh + +Launches the full Gitea open-source stack locally, following the +README/DEVELOPERS guidance (Docker Compose). This is a multi-service deployment +(Postgres, Auth, Storage, Realtime, Kong, Studio, etc.) + +Usage: + ./deploy.sh [options] + +Options: + --mode Deployment mode ('local' for Docker Compose, 'k3d' for local k3d, 'k8s' for generic k8s/k3s; default: k3d) + --helm-template-only Render Helm manifests to gitea/k8s without applying (for pipelines) + --with-dev-helpers Include docker/dev/docker-compose.dev.yml + --foreground Run docker compose in the foreground (default: detached, local mode only) + -f, --force Reset the gitea namespace before applying manifests (k3d only; default) + --no-force Skip namespace reset (k3d only) + -c, --config Path to prole.cfg (loads environment defaults) + -h, --help Show help + +Notes: + - This script syncs the gitea repo into $DEV_HOME/gitea. + - For k3d mode, manifests are applied from $GITEA_K8S_DIR. + - Docker image artifacts are cached in $DOCKER_IMPORT_DIR. + - steps: + 1) use docker/docker-compose.yml + 2) copy docker/.env.example -> docker/.env (if missing) + 3) run docker compose up +USAGE +} + +die() { + echo "Error: $*" >&2 + exit 1 +} + +warn() { + echo "Warning: $*" >&2 +} + +log() { + echo "==> $*" +} + +apply_defaults() { + DEV_HOME="${DEV_HOME:-$HOME/dev}" + DEV_HOME="${DEV_HOME/#\~/$HOME}" + GITEA_DIR="$DEV_HOME/gitea" + DOCKER_DIR="$GITEA_DIR/docker" + COMPOSE_FILE="$DOCKER_DIR/docker-compose.yml" + DEV_COMPOSE_FILE="$DOCKER_DIR/dev/docker-compose.dev.yml" + ENV_EXAMPLE="$DOCKER_DIR/.env.example" + ENV_FILE="$DOCKER_DIR/.env" + GITEA_K8S_DIR="${GITEA_K8S_DIR:-$SCRIPT_DIR/k8s}" + local default_owner + default_owner="$(id -un 2>/dev/null || echo prole)" + local data_root="${PROLE_DATA:-/opt/prole/data/${default_owner}}" + data_root="${data_root%/}" + DOCKER_IMPORT_DIR="${DOCKER_IMPORT_DIR:-$data_root/docker-import}" + GITEA_IMAGE_PLATFORM="${GITEA_IMAGE_PLATFORM:-}" + GITEA_IMAGE_PLATFORMS="${GITEA_IMAGE_PLATFORMS:-linux/amd64 linux/arm64}" + GITEA_IMAGE_TAG="${GITEA_IMAGE_TAG:-1.22.3}" + GITEA_DOMAIN="${GITEA_DOMAIN:-git.prole.org}" + GITEA_SSH_DOMAIN="${GITEA_SSH_DOMAIN:-git.prole.org}" + GITEA_NAMESPACE="${GITEA_NAMESPACE:-gitea}" + PROLE_DB_SERVICE="${PROLE_DB_SERVICE:-prole-db-rw}" + PROLE_DB_NAMESPACE="${PROLE_DB_NAMESPACE:-}" + AUTHORITY_NAMESPACE="${AUTHORITY_NAMESPACE:-authority}" + AUTHORITY_SERVICE="${AUTHORITY_SERVICE:-authority-ldap}" + AUTHORITY_LDAP_PORT="${AUTHORITY_LDAP_PORT:-389}" + AUTHORITY_BASE_DN="${AUTHORITY_BASE_DN:-dc=prole,dc=org}" + AUTHORITY_BIND_DN="${AUTHORITY_BIND_DN:-cn=gitea,ou=services,dc=prole,dc=org}" + AUTHORITY_BIND_PASSWORD="${AUTHORITY_BIND_PASSWORD:-}" +} + +load_prole_cfg() { + local cfg_loader="$PROJECT_ROOT/etc/prole_cfg.sh" + local cfg_path="${PROLE_CFG_PATH:-}" + + if [[ -z "$cfg_path" ]]; then + if [[ -n "${PROLE_CONF:-}" ]]; then + cfg_path="${PROLE_CONF%/}/prole.cfg" + elif [[ -f "$PROJECT_ROOT/conf/prole.cfg" ]]; then + cfg_path="$PROJECT_ROOT/conf/prole.cfg" + fi + fi + + if [[ -z "$cfg_path" ]]; then + return 0 + fi + + if [[ -n "$cfg_path" ]]; then + if [[ -d "$cfg_path" ]]; then + cfg_path="$cfg_path/prole.cfg" + fi + [[ -f "$cfg_path" ]] || die "Config file not found: $cfg_path" + PROLE_CONF="$(cd "$(dirname "$cfg_path")" && pwd)" + export PROLE_CONF + PROLE_CFG_PATH="$cfg_path" + fi + + if [[ -f "$cfg_loader" ]]; then + # shellcheck disable=SC1090 + source "$cfg_loader" + else + warn "prole_cfg.sh not found; config defaults may be incomplete." + fi +} + +register_gitea_ports() { + if command -v prole_register_port_forward >/dev/null 2>&1; then + prole_register_port_forward "gitea-http" "$GITEA_NAMESPACE" "svc/gitea-http" "3000" "3000" "0.0.0.0" "TCP" "Gitea Web UI" + prole_register_port_forward "gitea-ssh" "$GITEA_NAMESPACE" "svc/gitea-ssh" "2222" "22" "0.0.0.0" "TCP" "Gitea SSH" + fi +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +detect_platform() { + case "$(uname -m 2>/dev/null || true)" in + arm64|aarch64) echo "linux/arm64" ;; + x86_64|amd64) echo "linux/amd64" ;; + *) echo "linux/amd64" ;; + esac +} + +normalize_platform() { + case "${1:-}" in + linux/*) echo "$1" ;; + arm64|aarch64) echo "linux/arm64" ;; + x86_64|amd64) echo "linux/amd64" ;; + "") detect_platform ;; + *) detect_platform ;; + esac +} + +compose_cmd() { + if docker compose version >/dev/null 2>&1; then + echo "docker compose" + return 0 + fi + if command -v docker-compose >/dev/null 2>&1; then + echo "docker-compose" + return 0 + fi + die "Docker Compose not found (expected 'docker compose' or 'docker-compose')" +} + +ensure_k3d() { + require_cmd k3d +} + +ensure_helm() { + require_cmd helm +} + +list_k3d_clusters() { + local json="" + if json="$(k3d cluster list -o json 2>/dev/null)"; then + if [[ -n "$json" ]]; then + if printf '%s' "$json" | python3 - <<'PY' +import json +import sys + +raw = sys.stdin.read() +if not raw.strip(): + sys.exit(1) +try: + data = json.loads(raw) +except Exception: + sys.exit(1) + +items = data.get("items") if isinstance(data, dict) else data +if not items: + sys.exit(0) +for item in items: + name = item.get("name") if isinstance(item, dict) else None + if name: + print(name) +PY + then + return 0 + fi + fi + fi + + k3d cluster list 2>/dev/null | awk 'NR>1 {print $1}' +} + +cluster_exists() { + local name="$1" + list_k3d_clusters | awk -v target="$name" '$0 == target {found=1} END {exit found ? 0 : 1}' +} + +ensure_k3d_cluster() { + local cluster="${K3D_CLUSTER_NAME:-}" + local -a clusters + local picked="" + + mapfile -t clusters < <(list_k3d_clusters || true) + + if [[ -n "$cluster" ]]; then + if ! cluster_exists "$cluster"; then + log "Creating k3d cluster '$cluster'..." + k3d cluster create "$cluster" >/dev/null + fi + picked="$cluster" + elif [[ ${#clusters[@]} -gt 0 ]]; then + for name in "${clusters[@]}"; do + if [[ "$name" == "k3s-default" ]]; then + picked="$name" + break + fi + done + if [[ -z "$picked" ]]; then + picked="${clusters[0]}" + fi + else + picked="k3s-default" + log "Creating k3d cluster '$picked'..." + k3d cluster create "$picked" >/dev/null + fi + + export K3D_CLUSTER_NAME="$picked" + log "Using k3d cluster: $K3D_CLUSTER_NAME" + k3d cluster start "$K3D_CLUSTER_NAME" >/dev/null 2>&1 || true + k3d kubeconfig merge "$K3D_CLUSTER_NAME" --switch-context >/dev/null 2>&1 || true +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --mode|-m) + MODE="${2:-}" + shift 2 + ;; + -c|--config) + PROLE_CFG_PATH="${2:-}" + [[ -n "$PROLE_CFG_PATH" ]] || die "Missing value for $1" + shift 2 + ;; + --config=*) + PROLE_CFG_PATH="${1#*=}" + [[ -n "$PROLE_CFG_PATH" ]] || die "Missing value for $1" + shift + ;; + --with-dev-helpers) + USE_DEV_HELPERS="true" + shift + ;; + --helm-template-only) + HELM_TEMPLATE_ONLY="true" + shift + ;; + --foreground) + FOREGROUND="true" + shift + ;; + -f|--force) + FORCE="true" + shift + ;; + --no-force) + FORCE="false" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "Unknown option: $1" + ;; + esac + done +} + +ensure_env_file() { + if [[ -f "$ENV_FILE" ]]; then + return 0 + fi + [[ -f "$ENV_EXAMPLE" ]] || die "Missing env example file: $ENV_EXAMPLE" + log "Creating docker/.env from docker/.env.example" + cp "$ENV_EXAMPLE" "$ENV_FILE" + warn "docker/.env contains default secrets. Update them before any production use." +} + +compose_files() { + local -a files=("-f" "docker-compose.yml") + if [[ "$USE_DEV_HELPERS" == "true" ]]; then + [[ -f "$DEV_COMPOSE_FILE" ]] || die "Missing dev helpers compose file: $DEV_COMPOSE_FILE" + files+=("-f" "dev/docker-compose.dev.yml") + fi + echo "${files[@]}" +} + +ensure_repo() { + require_cmd git + mkdir -p "$DEV_HOME" + if [[ -d "$GITEA_DIR/.git" ]]; then + log "Updating Gitea repo in $GITEA_DIR" + (cd "$GITEA_DIR" && git pull) + return 0 + fi + if [[ -e "$GITEA_DIR" ]]; then + die "Path exists but is not a git repo: $GITEA_DIR" + fi + log "Cloning Gitea repo into $GITEA_DIR" + git clone https://github.com/go-gitea/gitea.git "$GITEA_DIR" +} + +run_local() { + ensure_repo + + [[ -f "$COMPOSE_FILE" ]] || die "Missing compose file: $COMPOSE_FILE" + + require_cmd docker + docker info >/dev/null 2>&1 || die "Docker daemon is not running" + + local compose + compose="$(compose_cmd)" + + log "Prole::Gitea local deploy (Docker Compose)" + log "Repo: $GITEA_DIR" + log "Namespace: gitea (Compose project name)" + + ensure_env_file + + local compose_args=("-f" "docker-compose.yml") + if [[ "$USE_DEV_HELPERS" == "true" ]]; then + [[ -f "$DEV_COMPOSE_FILE" ]] || die "Missing dev helpers compose file: $DEV_COMPOSE_FILE" + compose_args+=("-f" "dev/docker-compose.dev.yml") + fi + + local up_args=("up") + if [[ "$FOREGROUND" != "true" ]]; then + up_args+=("-d") + fi + + log "Starting Gitea services..." + (cd "$DOCKER_DIR" && COMPOSE_PROJECT_NAME="gitea" $compose "${compose_args[@]}" "${up_args[@]}") + + log "Deployment complete." + log "Access Gitea at http://localhost:3000" +} + +image_safe_name() { + echo "$1" | sed 's/[\/:@]/_/g' +} + +image_platform_tag() { + local img="$1" + local platform="$2" + local suffix="${platform//\//-}" + local name tag + + if [[ "$img" == *@* ]]; then + name="${img%@*}" + tag="digest-${suffix}" + elif [[ "$img" == *:* ]]; then + name="${img%:*}" + tag="${img##*:}-${suffix}" + else + name="$img" + tag="latest-${suffix}" + fi + + echo "${name}:${tag}" +} + +ensure_image_artifact() { + local img="$1" + local platform="$2" + local safe_name + local tar_path + local platform_tag + + safe_name="$(image_safe_name "$img")" + tar_path="${DOCKER_IMPORT_DIR}/${safe_name}.${platform//\//-}.tar" + platform_tag="$(image_platform_tag "$img" "$platform")" + + if [[ -f "$tar_path" ]]; then + return 0 + fi + + mkdir -p "$DOCKER_IMPORT_DIR" + + if docker buildx version >/dev/null 2>&1; then + log "Fetching image $img for platform $platform (buildx export)" + if ! printf 'FROM %s\n' "$img" \ + | docker buildx build --pull --platform "$platform" -t "$platform_tag" \ + --output "type=docker,dest=$tar_path" - >/dev/null 2>&1; then + die "Failed to export image $img ($platform) via buildx" + fi + return 0 + fi + + log "Fetching image $img for platform $platform" + if ! docker pull --platform "$platform" "$img" >/dev/null 2>&1; then + if docker image inspect "$img" >/dev/null 2>&1; then + warn "Using local image for $img (pull failed for $platform)" + else + die "Failed to pull image for $img ($platform)" + fi + fi + + docker tag "$img" "$platform_tag" >/dev/null 2>&1 || true + log "Saving $img ($platform) to $tar_path" + if ! docker save -o "$tar_path" "$platform_tag" >/dev/null 2>&1; then + die "Failed to save image $img ($platform) to $tar_path" + fi +} + +load_image_for_platform() { + local img="$1" + local platform="$2" + local safe_name + local tar_path + local platform_tag + + safe_name="$(image_safe_name "$img")" + tar_path="${DOCKER_IMPORT_DIR}/${safe_name}.${platform//\//-}.tar" + platform_tag="$(image_platform_tag "$img" "$platform")" + + if [[ ! -f "$tar_path" ]]; then + ensure_image_artifact "$img" "$platform" + fi + + log "Loading $img ($platform) from $tar_path" + docker load -i "$tar_path" 2>&1 | sed 's/^/ /' + docker tag "$platform_tag" "$img" +} + +# Returns 0 if the image is already present in k3d's containerd store, 1 otherwise. +image_in_k3d() { + local img="$1" + local cluster="${K3D_CLUSTER_NAME:-}" + local node="k3d-${cluster}-server-0" + + if ! docker inspect "$node" >/dev/null 2>&1; then + return 1 + fi + + docker exec "$node" crictl images --no-trunc -o json 2>/dev/null \ + | python3 - "$img" <<'PY' +import json, sys +target = sys.argv[1] +try: + data = json.load(sys.stdin) +except Exception: + sys.exit(1) +for entry in data.get('images', []): + for tag in entry.get('repoTags', []): + if tag == target: + sys.exit(0) +sys.exit(1) +PY +} + +# Build the list of images already present in k3d's containerd store. +list_k3d_images() { + local cluster="${K3D_CLUSTER_NAME:-}" + local node="k3d-${cluster}-server-0" + + if ! docker inspect "$node" >/dev/null 2>&1; then + return 0 + fi + + docker exec "$node" crictl images --no-trunc -o json 2>/dev/null \ + | python3 - <<'PY' +import json, sys +try: + data = json.load(sys.stdin) +except Exception: + sys.exit(0) +for entry in data.get('images', []): + for tag in entry.get('repoTags', []): + print(tag) +PY +} + +write_artifact_manifest() { + local images="$1" + local list_path="${DOCKER_IMPORT_DIR}/gitea-images.txt" + + mkdir -p "$DOCKER_IMPORT_DIR" + printf "%s\n" $images > "$list_path" + + local platform + for platform in $GITEA_IMAGE_PLATFORMS; do + local platform_list="${DOCKER_IMPORT_DIR}/gitea-images-${platform//\//-}.txt" + : > "$platform_list" + for img in $images; do + local safe_name + safe_name="$(image_safe_name "$img")" + echo "${DOCKER_IMPORT_DIR}/${safe_name}.${platform//\//-}.tar" >> "$platform_list" + done + done +} + +prefetch_k3d_images() { + require_cmd docker + docker info >/dev/null 2>&1 || die "Docker daemon is not running" + + # Gitea core images (from the helm chart / compose) + local images + images="gitea/gitea:${GITEA_IMAGE_TAG}" + + # If a docker-compose.yml exists in the gitea repo, prefer its image list + if [[ -f "$COMPOSE_FILE" ]]; then + local compose + compose="$(compose_cmd)" + ensure_env_file + local files + files=($(compose_files)) + local discovered + discovered=$(cd "$DOCKER_DIR" && $compose "${files[@]}" config --images 2>/dev/null | awk 'NF' | sort -u || true) + if [[ -n "$discovered" ]]; then + images="$discovered" + fi + fi + + log "Gitea images discovered:" + printf " - %s\n" $images + + write_artifact_manifest "$images" + log "Image artifact list written to $DOCKER_IMPORT_DIR/gitea-images.txt" + + local platform + for platform in $GITEA_IMAGE_PLATFORMS; do + log "Ensuring artifacts for platform '$platform' in $DOCKER_IMPORT_DIR" + for img in $images; do + ensure_image_artifact "$img" "$platform" + done + done + + local deploy_platform + deploy_platform="$(normalize_platform "${GITEA_IMAGE_PLATFORM:-}")" + log "Loading images for platform '$deploy_platform'" + for img in $images; do + load_image_for_platform "$img" "$deploy_platform" + done + + # Build a list of images already present in k3d to avoid re-importing them. + log "Checking k3d registry for already-imported images..." + local k3d_existing + k3d_existing="$(list_k3d_images || true)" + + local -a import_delta=() + local -a skipped=() + for img in $images; do + if printf '%s\n' "$k3d_existing" | grep -qxF "$img"; then + skipped+=("$img") + else + import_delta+=("$img") + fi + done + + if [[ ${#skipped[@]} -gt 0 ]]; then + log "Already in k3d registry (skipping ${#skipped[@]} image(s)):" + printf " [skip] %s\n" "${skipped[@]}" + fi + + if [[ ${#import_delta[@]} -eq 0 ]]; then + log "All images already present in k3d registry; nothing to import." + return 0 + fi + + log "Importing ${#import_delta[@]} image(s) into k3d cluster '${K3D_CLUSTER_NAME:-}' (delta only)..." + local idx=0 + for img in "${import_delta[@]}"; do + idx=$(( idx + 1 )) + log " [${idx}/${#import_delta[@]}] Importing $img into k3d..." + if [[ -n "${K3D_CLUSTER_NAME:-}" ]]; then + k3d image import "$img" -c "$K3D_CLUSTER_NAME" + else + k3d image import "$img" + fi + log " [${idx}/${#import_delta[@]}] Done: $img" + done + log "k3d image import complete (${#import_delta[@]} imported, ${#skipped[@]} already present)." +} + +resolve_prole_db_namespace() { + if [[ -n "${PROLE_DB_NAMESPACE:-}" ]]; then + return 0 + fi + + if [[ -n "${NAMESPACE:-}" ]]; then + PROLE_DB_NAMESPACE="$NAMESPACE" + return 0 + fi + + if kubectl get namespace prole >/dev/null 2>&1; then + PROLE_DB_NAMESPACE="prole" + else + PROLE_DB_NAMESPACE="default" + fi +} + +setup_prole_db_for_gitea() { + log "Ensuring Gitea database and role exist in prole-db..." + + local ns="${PROLE_DB_NAMESPACE:-prole-db-a0001}" + + local primary + primary=$(kubectl -n "$ns" get pods \ + -l "cnpg.io/cluster=prole-db,cnpg.io/instanceRole=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$primary" ]]; then + primary=$(kubectl -n "$ns" get pods -l "cnpg.io/cluster=prole-db" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + fi + if [[ -z "$primary" ]]; then + warn "No prole-db pod found in namespace '$ns'; skipping Gitea DB setup." + return 0 + fi + + local pg_password="" + if kubectl get secret prole-db-superuser -n "$ns" >/dev/null 2>&1; then + pg_password=$(kubectl get secret prole-db-superuser -n "$ns" \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) + fi + if [[ -z "$pg_password" ]]; then + pg_password="${POSTGRES_PASSWORD:-}" + fi + if [[ -z "$pg_password" ]]; then + warn "Cannot resolve postgres password; skipping Gitea DB role setup." + return 0 + fi + + local gitea_db_password="${GITEA_DB_PASSWORD:-${pg_password}}" + + kubectl -n "$ns" exec "$primary" -c postgres -- psql -U postgres -d postgres -c " + DO \$\$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='gitea') THEN + CREATE ROLE gitea LOGIN PASSWORD '${gitea_db_password}'; + END IF; + END \$\$; + " 2>&1 || warn "Could not create gitea role." + + local db_exists + db_exists=$(kubectl -n "$ns" exec "$primary" -c postgres -- psql -U postgres -d postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname = 'gitea';" 2>/dev/null || true) + if [[ "$db_exists" != "1" ]]; then + kubectl -n "$ns" exec "$primary" -c postgres -- psql -U postgres -d postgres -c \ + "CREATE DATABASE gitea OWNER gitea;" 2>&1 || warn "Could not create gitea database." + fi + + log " Gitea database and role ready in prole-db." +} + +resolve_authority_ldap() { + # Resolve the LDAP bind password from the authority pod/secret if not already set + if [[ -n "${AUTHORITY_BIND_PASSWORD:-}" ]]; then + return 0 + fi + + local ns="${AUTHORITY_NAMESPACE}" + if kubectl get secret authority-ldap-bind -n "$ns" >/dev/null 2>&1; then + AUTHORITY_BIND_PASSWORD=$(kubectl get secret authority-ldap-bind -n "$ns" \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) + fi + + if [[ -z "${AUTHORITY_BIND_PASSWORD:-}" ]]; then + warn "Could not resolve LDAP bind password from authority-ldap-bind secret; LDAP config may be incomplete." + fi +} + +# Build the Helm values for Gitea, incorporating cloudnative-pg and authority LDAP. +build_helm_values() { + local values_file="$1" + local pg_password="${2:-}" + local gitea_db_password="${GITEA_DB_PASSWORD:-${pg_password}}" + local db_ns="${PROLE_DB_NAMESPACE:-prole-db-a0001}" + + resolve_authority_ldap + + local ldap_host + ldap_host="${AUTHORITY_SERVICE}.${AUTHORITY_NAMESPACE}.svc.cluster.local" + + local ingress_enabled="false" + local ingress_class="${GITEA_INGRESS_CLASS:-kong}" + if [[ "$MODE" == "k3s" || "$MODE" == "k8s" ]]; then + ingress_enabled="true" + fi + + cat > "$values_file" </dev/null 2>&1; then + log "Creating '$GITEA_NAMESPACE' namespace..." + kubectl create namespace "$GITEA_NAMESPACE" + fi +} + +force_reset_gitea_namespace() { + if ! kubectl get namespace "$GITEA_NAMESPACE" >/dev/null 2>&1; then + ensure_gitea_namespace + return + fi + + log "Resetting '$GITEA_NAMESPACE' namespace..." + + kubectl delete all --all -n "$GITEA_NAMESPACE" --timeout=60s >/dev/null 2>&1 || true + kubectl delete namespace "$GITEA_NAMESPACE" --ignore-not-found --wait=false >/dev/null 2>&1 || true + + if kubectl get namespace "$GITEA_NAMESPACE" >/dev/null 2>&1; then + if ! kubectl wait --for=delete "namespace/$GITEA_NAMESPACE" --timeout=120s >/dev/null 2>&1; then + warn "Namespace deletion stalled; clearing finalizers" + python3 - </dev/null 2>&1 || true +import json +import subprocess +import sys + +try: + raw = subprocess.check_output(["kubectl", "get", "namespace", "${GITEA_NAMESPACE}", "-o", "json"]) + if not raw: + sys.exit(0) + data = json.loads(raw) + if "spec" in data: + data["spec"]["finalizers"] = [] + print(json.dumps(data)) +except Exception: + sys.exit(0) +PY + kubectl wait --for=delete "namespace/$GITEA_NAMESPACE" --timeout=60s >/dev/null 2>&1 || true + fi + fi + + ensure_gitea_namespace +} + +write_kong_ingress_k3s() { + local out_dir="$1" + cat > "$out_dir/gitea-kong-ingress.yaml" </dev/null 2>&1 || return 1 + + if [[ "$FORCE" == "true" ]]; then + kubectl delete namespace "$GITEA_NAMESPACE" --ignore-not-found --wait=false >/dev/null 2>&1 || true + fi + + resolve_prole_db_namespace + setup_prole_db_for_gitea + + # Resolve postgres password for DB values + local pg_password="" + local db_ns="${PROLE_DB_NAMESPACE:-prole-db-a0001}" + if kubectl get secret prole-db-superuser -n "$db_ns" >/dev/null 2>&1; then + pg_password=$(kubectl get secret prole-db-superuser -n "$db_ns" \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) + fi + if [[ -z "$pg_password" ]]; then + pg_password="${POSTGRES_PASSWORD:-}" + fi + + local values_file + values_file="$(mktemp /tmp/gitea-values.XXXXXX.yaml)" + build_helm_values "$values_file" "$pg_password" + + # Ensure the helm-gitea chart repo is registered + if ! helm repo list 2>/dev/null | grep -q "gitea-charts"; then + log "Adding gitea-charts Helm repo..." + helm repo add gitea-charts https://dl.gitea.com/charts/ >/dev/null 2>&1 || warn "helm repo add failed" + fi + helm repo update >/dev/null 2>&1 || true + + local k8s_dir="$SCRIPT_DIR/k8s" + mkdir -p "$k8s_dir" + + if [[ "$HELM_TEMPLATE_ONLY" == "true" ]]; then + log "Rendering Gitea Helm manifests to $k8s_dir (template only)..." + helm template "$HELM_RELEASE" gitea-charts/gitea \ + --namespace "$GITEA_NAMESPACE" \ + -f "$values_file" \ + --output-dir "$k8s_dir" >/dev/null + if [[ "$MODE" == "k3s" ]]; then + write_kong_ingress_k3s "$k8s_dir" + fi + log "Helm template only; manifests ready in gitea/k8s" + rm -f "$values_file" + return 0 + fi + + log "Installing Gitea via Helm into namespace '$GITEA_NAMESPACE'..." + if helm upgrade --install "$HELM_RELEASE" gitea-charts/gitea \ + -n "$GITEA_NAMESPACE" --create-namespace -f "$values_file"; then + rm -f "$values_file" + # Apply Kong ingress separately for k3s context + if [[ "$MODE" == "k3s" ]]; then + write_kong_ingress_k3s "$k8s_dir" + log "Applying Kong ingress for git.prole.org..." + kubectl apply -f "$k8s_dir/gitea-kong-ingress.yaml" + fi + register_gitea_ports + log "Deployment complete via Helm." + return 0 + else + rm -f "$values_file" + warn "Helm install failed; falling back to manifest deploy." + return 1 + fi +} + +run_manifest_fallback() { + local manifest_file + manifest_file="$(mktemp /tmp/gitea-manifest.XXXXXX.yaml)" + + cat > "$manifest_file" <> "$manifest_file" </dev/null 2>&1 || die "Kubernetes cluster not reachable" + + if [[ "$FORCE" == "true" ]]; then + force_reset_gitea_namespace + else + ensure_gitea_namespace + fi + + GITEA_IMAGE_PLATFORM="$(normalize_platform "${GITEA_IMAGE_PLATFORM:-}")" + + prefetch_k3d_images + resolve_prole_db_namespace + setup_prole_db_for_gitea + run_manifest_fallback + register_gitea_ports + + log "Deployment complete." + log "Access Gitea via Kong proxy (check ingress/service in '$GITEA_NAMESPACE' namespace)." +} + +run_k8s() { + # Helm-first path (preferred) — for k3s use context prole_k3s + if [[ "$MODE" == "k3s" ]]; then + local ctx="${PROLE_K3S_CONTEXT:-prole_k3s}" + if kubectl config get-contexts "$ctx" >/dev/null 2>&1; then + log "Switching to k3s context: $ctx" + kubectl config use-context "$ctx" >/dev/null 2>&1 || true + fi + fi + + if run_helm; then + return 0 + fi + + log "Prole::Gitea k8s/k3s deploy (Kubernetes - generic)" + log "Repo: $GITEA_DIR" + + require_cmd kubectl + kubectl cluster-info >/dev/null 2>&1 || die "Kubernetes cluster not reachable" + + if [[ "$FORCE" == "true" ]]; then + force_reset_gitea_namespace + else + ensure_gitea_namespace + fi + + resolve_prole_db_namespace + setup_prole_db_for_gitea + run_manifest_fallback + register_gitea_ports + + log "Deployment complete." + log "Access Gitea at http://${GITEA_DOMAIN}/ (via Kong ingress)" +} + +main() { + parse_args "$@" + load_prole_cfg + apply_defaults + + if [[ -z "$MODE" ]]; then + usage + exit 1 + fi + + log "Starting Gitea deployment..." + log "Mode: $MODE" + + case "$MODE" in + local) + run_local + ;; + k3d) + run_k3d + ;; + k8s|k3s) + run_k8s + ;; + *) + die "Unsupported mode: $MODE" + ;; + esac +} + +main "$@" diff --git a/installer/core/actions.py b/installer/core/actions.py index 63872d8..09c5f24 100644 --- a/installer/core/actions.py +++ b/installer/core/actions.py @@ -70,6 +70,7 @@ from installer.core.milestones import ( KerberosMilestone, DeploymentMilestone, GitOpsMilestone, + SupabaseImagePreloadMilestone, SupabaseMilestone, ) from installer.core.stream_exec import run_streaming_cmd @@ -3545,7 +3546,7 @@ class ProleSilentInstaller(ProleInstallerBase): steps.append(("init_kerberos.sh", ["initialize"], False)) steps.extend( [ - ("init_prole-db-backup.sh", ["start"], False), + ("init_cnpg_backup.sh", ["start"], False), ("init_kong.sh", ["start"], False), ("init_monitoring.sh", ["initialize"], False), ] @@ -3897,6 +3898,7 @@ class ProleSilentInstaller(ProleInstallerBase): InitializationScriptsMilestone(), KerberosMilestone(), GitOpsMilestone(), + SupabaseImagePreloadMilestone(), SupabaseMilestone(), DeploymentMilestone(), ] diff --git a/installer/core/milestones.py b/installer/core/milestones.py index 200dead..379d210 100644 --- a/installer/core/milestones.py +++ b/installer/core/milestones.py @@ -663,7 +663,7 @@ class InitializationScriptsMilestone(Milestone): scripts.extend( [ - "init_prole-db-backup.sh", + "init_cnpg_backup.sh", "init_kong.sh", "init_monitoring.sh", ] @@ -689,7 +689,7 @@ class InitializationScriptsMilestone(Milestone): args.append("update") elif script == "init_cloudnative_pg.sh": args = mode_args + ["initialize"] - elif script == "init_prole-db-backup.sh": + elif script == "init_cnpg_backup.sh": args = mode_args + ["start"] elif script == "init_kong.sh": args = mode_args + ["start"] @@ -902,6 +902,86 @@ class KerberosMilestone(Milestone): raise Exception(msg) +def _resolve_supabase_deploy_mode(state: InstallerState) -> str: + from installer.core.env import _normalize_cluster_env + + deploy_mode = ( + os.environ.get("SUPABASE_DEPLOY_MODE") or os.environ.get("SUPABASE_MODE") or "" + ).strip() + if deploy_mode: + return deploy_mode + + cluster_env = _normalize_cluster_env(state.inputs.get("init_cluster.cluster_env", "dev")) + if cluster_env in ("service", "prod"): + return "k8s" + return "k3d" + + +class SupabaseImagePreloadMilestone(Milestone): + def __init__(self): + super().__init__("supabase_images_preload", "Supabase Image Preload") + self.logger = logging.getLogger("SupabaseImagePreloadMilestone") + + def execute( + self, state: InstallerState, progress: ProgressCallback | None = None + ) -> None: + enabled = self._parse_bool( + state.inputs.get("init_cluster.supabase_enabled", "False"), + default=False, + ) + supa_sec = state.config_data.setdefault("Supabase", {}) + if not enabled: + supa_sec["IMAGES_STATUS"] = "Skipped" + if progress: + progress("Supabase disabled; skipping image preload", 1.0) + return + + from installer.core.env import _parse_bool + + env = self._get_script_env(state) + script_path = state.controller.project_root / "supabase" / "deploy.sh" + if not script_path.exists(): + self.logger.error(f"Supabase deploy script not found: {script_path}") + supa_sec["IMAGES_STATUS"] = "Attempted" + return + + deploy_mode = _resolve_supabase_deploy_mode(state) + if deploy_mode != "k3d": + supa_sec["IMAGES_STATUS"] = "Skipped" + if progress: + progress( + f"Supabase image preload not required for mode '{deploy_mode}'", + 1.0, + ) + return + + args = ["--mode", deploy_mode, "--prefetch-images-only"] + cfg_path = state.controller.project_root / "conf" / "prole.cfg" + if cfg_path.exists(): + args.extend(["-c", str(cfg_path)]) + if _parse_bool(os.environ.get("SUPABASE_USE_DEV_COMPOSE"), False): + args.append("--with-dev-helpers") + + if progress: + progress("Preloading Supabase images...", 0.4) + + self.logger.info(f"supabase/deploy.sh {' '.join(args)}") + rc = subprocess.run( + ["bash", str(script_path)] + args, + env=env, + ).returncode + if rc == 0: + supa_sec["IMAGES_STATUS"] = "Prepared" + state.inputs["supabase.images_preloaded"] = "True" + if progress: + progress("Supabase images preloaded", 1.0) + else: + supa_sec["IMAGES_STATUS"] = "Attempted" + msg = f"Supabase image preload failed (code {rc})" + self.logger.error(msg) + raise Exception(msg) + + class SupabaseMilestone(Milestone): def __init__(self): super().__init__("supabase", "Supabase Deployment") @@ -921,7 +1001,7 @@ class SupabaseMilestone(Milestone): progress("Supabase disabled; skipping", 1.0) return - from installer.core.env import _normalize_cluster_env, _parse_bool + from installer.core.env import _parse_bool env = self._get_script_env(state) script_path = state.controller.project_root / "supabase" / "deploy.sh" @@ -930,19 +1010,7 @@ class SupabaseMilestone(Milestone): supa_sec["STATUS"] = "Attempted" return - deploy_mode = ( - os.environ.get("SUPABASE_DEPLOY_MODE") - or os.environ.get("SUPABASE_MODE") - or "" - ).strip() - if not deploy_mode: - cluster_env = _normalize_cluster_env( - state.inputs.get("init_cluster.cluster_env", "dev") - ) - if cluster_env in ("service", "prod"): - deploy_mode = "k8s" - else: - deploy_mode = "k3d" + deploy_mode = _resolve_supabase_deploy_mode(state) args = ["--mode", deploy_mode] cfg_path = state.controller.project_root / "conf" / "prole.cfg" @@ -952,6 +1020,8 @@ class SupabaseMilestone(Milestone): args.append("--with-dev-helpers") if _parse_bool(os.environ.get("SUPABASE_FOREGROUND"), False): args.append("--foreground") + if _parse_bool(state.inputs.get("supabase.images_preloaded", "False"), False): + args.append("--skip-prefetch") self.logger.info(f"supabase/deploy.sh {' '.join(args)}") rc = subprocess.run( diff --git a/installer/ui/screens/base.py b/installer/ui/screens/base.py index c458ee9..ff91704 100644 --- a/installer/ui/screens/base.py +++ b/installer/ui/screens/base.py @@ -18,11 +18,11 @@ class ScreenBaseMixin: def _render_paragraph(self, text, y, wrap=600): ui.render_paragraph(self, text, y, wrap) - def _create_console_output(self, y=280, title="Scan Output", width=880, height=450): + def _create_console_output(self, y=280, title="Scan Output", width=880, height=450, x=48): """Create a standardized console output area with a label and scrollable text.""" # Section title/label for the console ui.canvas_text( - self, 48, y, title, fill="#1d1d1f", font=("SF Pro Text", 12, "bold") + self, x, y, title, fill="#1d1d1f", font=("SF Pro Text", 12, "bold") ) y_console = y + 30 @@ -32,7 +32,7 @@ class ScreenBaseMixin: console.pack(fill="both", expand=True, padx=1, pady=1) console_window = self.bg_canvas.create_window( - 48, y_console, window=console_bg, anchor="nw", width=width, height=height + x, y_console, window=console_bg, anchor="nw", width=width, height=height ) self._canvas_items.append(console_window) self._overlay_widgets.append(console_bg) diff --git a/installer/ui/screens/security.py b/installer/ui/screens/security.py index 024d2b4..d6b65d3 100644 --- a/installer/ui/screens/security.py +++ b/installer/ui/screens/security.py @@ -80,7 +80,7 @@ class SecurityScreenMixin: status_item = ui.canvas_text( self, - 340, + 250, y + 6, f"Status: {initial_status}", fill="black", @@ -97,10 +97,14 @@ class SecurityScreenMixin: self._kerberos_status_var.trace_add("write", _update_status_text) - x_field = 200 - field_w = 400 + left_col = content_width // 3 + right_col_x = left_col + 16 + right_col_w = content_width - right_col_x - 48 + x_field = 160 + field_w = left_col - x_field - 8 y += 60 + console_y = y ui.canvas_text( self, 48, y, "Kerberos Realm:", fill="black", font=("SF Pro Text", 12) ) @@ -201,9 +205,9 @@ class SecurityScreenMixin: self._canvas_items.append(btn_window) self._overlay_widgets.append(test_btn) - y += 60 self._kerberos_console = self._create_console_output( - y=y, title="Kerberos Output", width=900, height=260 + x=right_col_x, y=console_y, title="Kerberos Output", + width=right_col_w, height=350 ) def test_kerberos_connection(self): @@ -304,7 +308,10 @@ class SecurityScreenMixin: if rc2 == 0: write_line("\nKerberos test completed successfully.\n") self.safe_after( - lambda: self._kerberos_status_var.set("Test Passed") + lambda: ( + self._kerberos_status_var.set("Test Passed"), + self.next_button.configure(state="normal"), + ) ) else: write_line(f"\nKerberos test failed with code {rc2}\n") diff --git a/installer/ui/screens/services.py b/installer/ui/screens/services.py index 7af8184..0863ae8 100644 --- a/installer/ui/screens/services.py +++ b/installer/ui/screens/services.py @@ -158,7 +158,7 @@ class ServicesScreenMixin: ("Common Services Log", "common_services_log"), ("CloudNative-PG", "init_cloudnative_pg.sh"), ] - scripts.append(("Prole DB Backup", "init_prole-db-backup.sh")) + scripts.append(("Prole DB Backup", "init_cnpg_backup.sh")) scripts.append(("Kong API Gateway", "init_kong.sh")) scripts.append(("Monitoring", "init_monitoring.sh")) mode = self._deployment_mode() @@ -369,9 +369,9 @@ class ServicesScreenMixin: "Skipping CloudNative-PG initialization because previous steps failed.\n" ) - # 4. init_prole-db-backup.sh start + # 4. init_cnpg_backup.sh start if overall_success: - script = "init_prole-db-backup.sh" + script = "init_cnpg_backup.sh" _select_tab(script) self.script_consoles[script].clear() self.script_consoles[script].write(f"Running {script} start...\n") @@ -412,7 +412,7 @@ class ServicesScreenMixin: f"\n{script} completed successfully.\n" ) else: - self.script_consoles["init_prole-db-backup.sh"].write( + self.script_consoles["init_cnpg_backup.sh"].write( "Skipping Prole DB backup because previous steps failed.\n" ) diff --git a/k8s/prole/prole-db-recovery.yaml.tpl b/k8s/prole/prole-db-recovery.yaml.tpl index 95ae8f4..320b0e0 100644 --- a/k8s/prole/prole-db-recovery.yaml.tpl +++ b/k8s/prole/prole-db-recovery.yaml.tpl @@ -5,7 +5,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: k3d-prole-registry:5000/prole-db:18-089 + imageName: k3d-prole-registry:5000/prole-db:18-092 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 diff --git a/k8s/prole/prole-db.yaml b/k8s/prole/prole-db.yaml index 918e791..e21d83e 100644 --- a/k8s/prole/prole-db.yaml +++ b/k8s/prole/prole-db.yaml @@ -4,7 +4,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: k3d-prole-registry:5000/prole-db:18-089 + imageName: k3d-prole-registry:5000/prole-db:18-092 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 diff --git a/mock_val/init_cloudnative_pg.sh b/mock_val/init_cloudnative_pg.sh index db3148c..a0fb9de 100755 --- a/mock_val/init_cloudnative_pg.sh +++ b/mock_val/init_cloudnative_pg.sh @@ -1046,13 +1046,13 @@ wait_for_backup() { } run_garage_backup() { - if [[ ! -x "$SCRIPT_DIR/init_prole-db-backup.sh" ]]; then - echo "WARN: init_prole-db-backup.sh not found; skipping Garage backup." >&2 + if [[ ! -x "$SCRIPT_DIR/init_cnpg_backup.sh" ]]; then + echo "WARN: init_cnpg_backup.sh not found; skipping Garage backup." >&2 return 1 fi - echo "Running Garage backup via init_prole-db-backup.sh ..." - if ! "$SCRIPT_DIR/init_prole-db-backup.sh" start; then + echo "Running Garage backup via init_cnpg_backup.sh ..." + if ! "$SCRIPT_DIR/init_cnpg_backup.sh" start; then echo "Garage backup script failed." >&2 return 1 fi diff --git a/mock_val/init_prole-db-backup.sh b/mock_val/init_cnpg_backup.sh similarity index 72% rename from mock_val/init_prole-db-backup.sh rename to mock_val/init_cnpg_backup.sh index 531a1a6..5bc3d07 100755 --- a/mock_val/init_prole-db-backup.sh +++ b/mock_val/init_cnpg_backup.sh @@ -2,7 +2,7 @@ set -euo pipefail -# init_prole-db-backup.sh +# init_cnpg_backup.sh # Purpose: # - Configure CloudNative-PG to backup to Garage (S3-compatible) via Barman Cloud Plugin # - Create initial backup @@ -44,6 +44,9 @@ RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1} RETENTION_POLICY=${RETENTION_POLICY:-30d} BARMAN_PLUGIN_NAME=${BARMAN_PLUGIN_NAME:-barman-cloud.cloudnative-pg.io} BARMAN_OBJECT_NAME=${BARMAN_OBJECT_NAME:-prole-db-barman-objectstore} +BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600} +BACKUP_STATUS_INTERVAL=${BACKUP_STATUS_INTERVAL:-10} +PLUGIN_READY_TIMEOUT=${PLUGIN_READY_TIMEOUT:-180} usage() { cat </dev/null 2>&1 || true } +wait_for_plugin_ready() { + local start_time now plugin_names deployment_rows ns ready replicas + start_time=$(date +%s) + while true; do + plugin_names=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.plugins[*].name}' 2>/dev/null || true) + if printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -Fxq "$BARMAN_PLUGIN_NAME"; then + deployment_rows=$(kubectl get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) + if [[ -z "$deployment_rows" ]]; then + deployment_rows=$(kubectl get deployment -A -o jsonpath='{range .items[?(@.metadata.name=="barman-cloud")]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) + fi + + while IFS=$'\t' read -r ns ready replicas; do + [[ -z "$ns" ]] && continue + ready=${ready:-0} + replicas=${replicas:-0} + if (( ready >= 1 && replicas >= 1 )); then + return 0 + fi + done <<< "$deployment_rows" + fi + + now=$(date +%s) + if (( now - start_time >= PLUGIN_READY_TIMEOUT )); then + echo "ERROR: Timed out waiting for CNPG plugin '$BARMAN_PLUGIN_NAME' to become available." >&2 + return 1 + fi + + echo "Waiting for CNPG plugin '$BARMAN_PLUGIN_NAME' to become available ..." + sleep 5 + done +} + +has_successful_base_backup() { + local rows name phase method backup_type phase_lc method_lc backup_type_lc + rows=$(kubectl get backup -n "$NAMESPACE" -o jsonpath="{range .items[?(@.spec.cluster.name=='$CNPG_CLUSTER_NAME')]}{.metadata.name}{\"\t\"}{.status.phase}{\"\t\"}{.spec.method}{\"\t\"}{.spec.pluginConfiguration.parameters.backupType}{\"\n\"}{end}" 2>/dev/null || true) + + if [[ -z "$rows" ]]; then + return 1 + fi + + while IFS=$'\t' read -r name phase method backup_type; do + [[ -z "$name" ]] && continue + phase_lc=$(echo "${phase:-}" | tr '[:upper:]' '[:lower:]') + method_lc=$(echo "${method:-}" | tr '[:upper:]' '[:lower:]') + backup_type_lc=$(echo "${backup_type:-}" | tr '[:upper:]' '[:lower:]') + + if [[ "$phase_lc" != "completed" && "$phase_lc" != "succeeded" ]]; then + continue + fi + + if [[ "$method_lc" == "plugin" ]] && [[ "$backup_type_lc" == "incremental" || "$backup_type_lc" == "incr" ]]; then + continue + fi + + return 0 + done <<< "$rows" + + return 1 +} + +wait_for_successful_base_backup() { + local start_time now elapsed + start_time=$(date +%s) + while true; do + if has_successful_base_backup; then + return 0 + fi + + now=$(date +%s) + elapsed=$((now - start_time)) + if (( elapsed >= BACKUP_STATUS_TIMEOUT )); then + return 1 + fi + + echo "Waiting for a successful base backup ... (${elapsed}s/${BACKUP_STATUS_TIMEOUT}s)" + sleep "$BACKUP_STATUS_INTERVAL" + done +} + trigger_backup() { local backup_type="${1:-full}" local backup_name @@ -281,6 +363,14 @@ status() { ensure_tools echo "Backups for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE':" kubectl get backup -n "$NAMESPACE" | grep "$CNPG_CLUSTER_NAME" || true + + if has_successful_base_backup; then + echo "Backup status: OK (at least one successful base backup is available)." + return 0 + fi + + echo "ERROR: No successful base backup found for cluster '$CNPG_CLUSTER_NAME'." >&2 + return 1 } case "$ACTION" in @@ -291,10 +381,17 @@ case "$ACTION" in ensure_garage_bucket_and_key apply_barman_object_store ensure_barman_plugin_config + wait_for_plugin_ready remove_native_barman_config if [[ "$RUN_FIRST_BACKUP" == "1" ]]; then trigger_backup fi + if ! wait_for_successful_base_backup; then + echo "ERROR: Timed out waiting for a successful base backup." >&2 + status || true + exit 1 + fi + status ;; backup) ensure_tools diff --git a/mock_val/init_prole-db-reset.sh b/mock_val/init_prole-db-reset.sh index 8a514d7..b4093c4 100755 --- a/mock_val/init_prole-db-reset.sh +++ b/mock_val/init_prole-db-reset.sh @@ -162,15 +162,15 @@ run_barman_backup() { return 1 fi - if [[ -x "$SCRIPT_DIR/init_prole-db-backup.sh" ]]; then + if [[ -x "$SCRIPT_DIR/init_cnpg_backup.sh" ]]; then log "Running barman backup (Garage bucket: $GARAGE_BACKUP_BUCKET) ..." NAMESPACE="$NAMESPACE" \ GARAGE_BACKUP_BUCKET="$GARAGE_BACKUP_BUCKET" \ GARAGE_BACKUP_SECRET_NAME="$GARAGE_BACKUP_SECRET_NAME" \ GARAGE_S3_ENDPOINT="$GARAGE_S3_ENDPOINT" \ - RUN_FIRST_BACKUP=1 "$SCRIPT_DIR/init_prole-db-backup.sh" start + RUN_FIRST_BACKUP=1 "$SCRIPT_DIR/init_cnpg_backup.sh" start else - err "init_prole-db-backup.sh not found; cannot run backup." + err "init_cnpg_backup.sh not found; cannot run backup." return 1 fi diff --git a/mock_val/status.sh b/mock_val/status.sh index 62aa2e7..c70f640 100755 --- a/mock_val/status.sh +++ b/mock_val/status.sh @@ -253,7 +253,7 @@ STATUS_SCRIPTS=( "init_kong.sh" "init_db_manager.sh" "init_monitoring.sh" - "init_prole-db-backup.sh" + "init_cnpg_backup.sh" "init_port_forwards.sh" ) diff --git a/prole-db/.version b/prole-db/.version index 7730ef7..69226f7 100644 --- a/prole-db/.version +++ b/prole-db/.version @@ -1 +1 @@ -89 \ No newline at end of file +92 \ No newline at end of file diff --git a/prole.sh b/prole.sh index 7f5f2ad..e222d9a 100755 --- a/prole.sh +++ b/prole.sh @@ -800,15 +800,15 @@ case "${CMD}" in if [[ -x "${ROOT_DIR}/etc/init_cloudnative_pg.sh" ]]; then "${ROOT_DIR}/etc/init_cloudnative_pg.sh" install-barman-plugin fi - if [[ -x "${ROOT_DIR}/etc/init_prole-db-backup.sh" ]]; then - RUN_FIRST_BACKUP=0 "${ROOT_DIR}/etc/init_prole-db-backup.sh" start + if [[ -x "${ROOT_DIR}/etc/init_cnpg_backup.sh" ]]; then + RUN_FIRST_BACKUP=0 "${ROOT_DIR}/etc/init_cnpg_backup.sh" start fi log_msg "Running full backup..." backup_before=$(kubectl -n "${NAMESPACE:-default}" get backup \ --sort-by=.metadata.creationTimestamp \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true) - "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup + "${ROOT_DIR}/etc/init_cnpg_backup.sh" backup backup_name="" for i in {1..30}; do @@ -893,8 +893,8 @@ case "${CMD}" in if [[ -x "${ROOT_DIR}/etc/init_cloudnative_pg.sh" ]]; then "${ROOT_DIR}/etc/init_cloudnative_pg.sh" install-barman-plugin fi - if [[ -x "${ROOT_DIR}/etc/init_prole-db-backup.sh" ]]; then - RUN_FIRST_BACKUP=0 "${ROOT_DIR}/etc/init_prole-db-backup.sh" start + if [[ -x "${ROOT_DIR}/etc/init_cnpg_backup.sh" ]]; then + RUN_FIRST_BACKUP=0 "${ROOT_DIR}/etc/init_cnpg_backup.sh" start fi log_msg "Running ${backup_type} backup..." @@ -903,9 +903,9 @@ case "${CMD}" in -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 || true) if [[ "$backup_type" == "full" ]]; then - "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup full + "${ROOT_DIR}/etc/init_cnpg_backup.sh" backup full else - "${ROOT_DIR}/etc/init_prole-db-backup.sh" backup incr + "${ROOT_DIR}/etc/init_cnpg_backup.sh" backup incr fi backup_name="" @@ -1008,7 +1008,7 @@ case "${CMD}" in fi # 4. Database backup infrastructure - run_init_script init_prole-db-backup.sh start + run_init_script init_cnpg_backup.sh start # 5. Kong API Gateway run_init_script init_kong.sh -n "${NAMESPACE:-prole-db}" start diff --git a/status.py b/status.py index 6b4da3c..688aed7 100755 --- a/status.py +++ b/status.py @@ -109,7 +109,7 @@ _STATUS_SCRIPTS: list[str] = [ "init_postgrest.sh", "init_db_manager.sh", "init_monitoring.sh", - "init_prole-db-backup.sh", + "init_cnpg_backup.sh", "init_port_forwards.sh", ] diff --git a/supabase/deploy.sh b/supabase/deploy.sh index 8a99ec1..1bad01f 100755 --- a/supabase/deploy.sh +++ b/supabase/deploy.sh @@ -15,10 +15,12 @@ FOREGROUND="false" FORCE="true" PROLE_CFG_PATH="" HELM_TEMPLATE_ONLY="false" +PREFETCH_IMAGES_ONLY="false" +SKIP_PREFETCH="false" usage() { cat <<'USAGE' -Prole::Supabase +supabase/deploy.sh Launches the full Supabase open-source stack locally, following the README/DEVELOPERS guidance (Docker Compose). This is a multi-service deployment @@ -30,6 +32,8 @@ Usage: Options: --mode Deployment mode ('local' for Docker Compose, 'k3d' for local k3d, 'k8s' for generic k8s/k3s; default: k3d) --helm-template-only Render Helm manifests to supabase/k8s without applying (for pipelines) + --prefetch-images-only Discover/prepare/load/import Supabase images only, then exit (k3d mode) + --skip-prefetch Skip image prefetch during deploy (for preloaded flows) --with-dev-helpers Include docker/dev/docker-compose.dev.yml --foreground Run docker compose in the foreground (default: detached, local mode only) -f, --force Reset the Supabase namespace before applying manifests (k3d only; default) @@ -270,6 +274,14 @@ parse_args() { HELM_TEMPLATE_ONLY="true" shift ;; + --prefetch-images-only) + PREFETCH_IMAGES_ONLY="true" + shift + ;; + --skip-prefetch) + SKIP_PREFETCH="true" + shift + ;; --foreground) FOREGROUND="true" shift @@ -448,10 +460,58 @@ load_image_for_platform() { fi log "Loading $img ($platform) from $tar_path" - docker load -i "$tar_path" >/dev/null + docker load -i "$tar_path" 2>&1 | sed 's/^/ /' docker tag "$platform_tag" "$img" } +# Returns 0 if the image is already present in k3d's containerd store, 1 otherwise. +image_in_k3d() { + local img="$1" + local cluster="${K3D_CLUSTER_NAME:-}" + local node="k3d-${cluster}-server-0" + + if ! docker inspect "$node" >/dev/null 2>&1; then + return 1 + fi + + docker exec "$node" crictl images --no-trunc -o json 2>/dev/null \ + | python3 - "$img" <<'PY' +import json, sys +target = sys.argv[1] +try: + data = json.load(sys.stdin) +except Exception: + sys.exit(1) +for entry in data.get('images', []): + for tag in entry.get('repoTags', []): + if tag == target: + sys.exit(0) +sys.exit(1) +PY +} + +# Build the list of images already present in k3d's containerd store. +list_k3d_images() { + local cluster="${K3D_CLUSTER_NAME:-}" + local node="k3d-${cluster}-server-0" + + if ! docker inspect "$node" >/dev/null 2>&1; then + return 0 + fi + + docker exec "$node" crictl images --no-trunc -o json 2>/dev/null \ + | python3 - <<'PY' +import json, sys +try: + data = json.load(sys.stdin) +except Exception: + sys.exit(0) +for entry in data.get('images', []): + for tag in entry.get('repoTags', []): + print(tag) +PY +} + write_artifact_manifest() { local images="$1" local list_path="${DOCKER_IMPORT_DIR}/supabase-images.txt" @@ -510,12 +570,44 @@ prefetch_k3d_images() { load_image_for_platform "$img" "$deploy_platform" done - # shellcheck disable=SC2086 - if [[ -n "${K3D_CLUSTER_NAME:-}" ]]; then - k3d image import $images -c "$K3D_CLUSTER_NAME" - else - k3d image import $images + # Build a list of images already present in k3d to avoid re-importing them. + log "Checking k3d registry for already-imported images..." + local k3d_existing + k3d_existing="$(list_k3d_images || true)" + + local -a import_delta=() + local -a skipped=() + for img in $images; do + if printf '%s\n' "$k3d_existing" | grep -qxF "$img"; then + skipped+=("$img") + else + import_delta+=("$img") + fi + done + + if [[ ${#skipped[@]} -gt 0 ]]; then + log "Already in k3d registry (skipping ${#skipped[@]} image(s)):" + printf " [skip] %s\n" "${skipped[@]}" fi + + if [[ ${#import_delta[@]} -eq 0 ]]; then + log "All images already present in k3d registry; nothing to import." + return 0 + fi + + log "Importing ${#import_delta[@]} image(s) into k3d cluster '${K3D_CLUSTER_NAME:-}' (delta only)..." + local idx=0 + for img in "${import_delta[@]}"; do + idx=$(( idx + 1 )) + log " [${idx}/${#import_delta[@]}] Importing $img into k3d..." + if [[ -n "${K3D_CLUSTER_NAME:-}" ]]; then + k3d image import "$img" -c "$K3D_CLUSTER_NAME" + else + k3d image import "$img" + fi + log " [${idx}/${#import_delta[@]}] Done: $img" + done + log "k3d image import complete (${#import_delta[@]} imported, ${#skipped[@]} already present)." } resolve_prole_db_namespace() { @@ -1123,7 +1215,11 @@ run_k3d() { export SUPABASE_IMAGE_PLATFORM SUPABASE_IMAGE_PLATFORM="$(normalize_platform "${SUPABASE_IMAGE_PLATFORM:-}")" - prefetch_k3d_images + if [[ "$SKIP_PREFETCH" != "true" ]]; then + prefetch_k3d_images + else + log "Skipping Supabase image prefetch (requested)." + fi generate_k8s_manifests setup_prole_db_for_supabase @@ -1139,6 +1235,32 @@ run_k3d() { log "Access Studio via Kong proxy (check ingress/service in 'supabase' namespace)." } +run_prefetch_images_only() { + ensure_repo + + case "$MODE" in + k3d) + require_cmd kubectl + ensure_k3d + ensure_k3d_cluster + kubectl cluster-info >/dev/null 2>&1 || die "Kubernetes cluster not reachable" + + export SUPABASE_HOME="$SUPABASE_DIR" + if [[ "$USE_DEV_HELPERS" == "true" ]]; then + export SUPABASE_USE_DEV_COMPOSE=1 + fi + export SUPABASE_IMAGE_PLATFORM + SUPABASE_IMAGE_PLATFORM="$(normalize_platform "${SUPABASE_IMAGE_PLATFORM:-}")" + + prefetch_k3d_images + log "Supabase image prefetch complete." + ;; + *) + die "--prefetch-images-only currently supports only --mode k3d" + ;; + esac +} + run_k8s() { # Helm-first path (preferred) if run_helm; then @@ -1191,6 +1313,11 @@ main() { exit 1 fi + if [[ "$PREFETCH_IMAGES_ONLY" == "true" ]]; then + run_prefetch_images_only + return 0 + fi + case "$MODE" in local) run_local diff --git a/supabase/helm/generated/values.generated.json b/supabase/helm/generated/values.generated.json index 8e18006..721c979 100644 --- a/supabase/helm/generated/values.generated.json +++ b/supabase/helm/generated/values.generated.json @@ -29,13 +29,13 @@ }, "secret": { "db": { - "password": "${OPENBAO:kv/prole/${NAMESPACE}/db#password}", - "database": "${NAMESPACE}" + "password": "${OPENBAO:kv/prole/prole-db001/db#password}", + "database": "postgres" }, "jwt": { "secret": "4e53e1ecb98eb32ee83b890bea79e678e952c6ae237edf3f65f37bf6dd2262eb", - "anonKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiIsImlzcyI6InByb2xlLXN1cGFiYXNlIiwiaWF0IjoxNzcyMDk5NDM2LCJleHAiOjIwODc0NTk0MzZ9.NM4QHNzdWWSyvvBm-xiHdFJm8T12cROyGhvrx2eMaPk", - "serviceKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaXNzIjoicHJvbGUtc3VwYWJhc2UiLCJpYXQiOjE3NzIwOTk0MzYsImV4cCI6MjA4NzQ1OTQzNn0.53NMhvhhEmCQRJQ_UW5Cc6TRMI3X0Zrf4gmiMNjU2As" + "anonKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiIsImlzcyI6InByb2xlLXN1cGFiYXNlIiwiaWF0IjoxNzcyNDM0NjM4LCJleHAiOjIwODc3OTQ2Mzh9.Wg05U6ioVntunXskmJs3DXMc1i9-O7UBts2cnBwPCoI", + "serviceKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaXNzIjoicHJvbGUtc3VwYWJhc2UiLCJpYXQiOjE3NzI0MzQ2MzgsImV4cCI6MjA4Nzc5NDYzOH0.xZ179fB412UFdgDxc8XUv2EQt-L1nGyhocll09u2u38" } }, "ingress": { diff --git a/tests/etc/test_init_prole-db-backup.sh b/tests/etc/test_init_cnpg_backup.sh similarity index 90% rename from tests/etc/test_init_prole-db-backup.sh rename to tests/etc/test_init_cnpg_backup.sh index 9142089..8cb6043 100644 --- a/tests/etc/test_init_prole-db-backup.sh +++ b/tests/etc/test_init_cnpg_backup.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -# Unit test for etc/init_prole-db-backup.sh +# Unit test for etc/init_cnpg_backup.sh SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) ETC_DIR="$PROLE_HOME/etc" -SCRIPT_UNDER_TEST="$ETC_DIR/init_prole-db-backup.sh" +SCRIPT_UNDER_TEST="$ETC_DIR/init_cnpg_backup.sh" TMP_DIR=$(mktemp -d) trap 'rm -rf "$TMP_DIR"' EXIT @@ -48,8 +48,8 @@ export NAMESPACE="test-ns" export PROLE_PASSWD="test-password" # Run the script -if [[ "init_prole-db-backup.sh" == *.py ]]; then - if [[ "init_prole-db-backup.sh" == "render_manifest.py" ]]; then +if [[ "init_cnpg_backup.sh" == *.py ]]; then + if [[ "init_cnpg_backup.sh" == "render_manifest.py" ]]; then echo "apiVersion: v1" > "$TMP_DIR/test.yaml" python3 "$SCRIPT_UNDER_TEST" "$TMP_DIR/test.yaml" > "$TMP_DIR/stdout" 2> "$TMP_DIR/stderr" else diff --git a/tests/installer/test_core_classes.py b/tests/installer/test_core_classes.py index 1c18ace..869d5ef 100644 --- a/tests/installer/test_core_classes.py +++ b/tests/installer/test_core_classes.py @@ -324,6 +324,27 @@ class TestProleController: # Will fail since the script doesn't exist assert rc != 0 or rc is None + def test_silent_installer_runs_supabase_preload_before_deploy(self, tmp_path): + c = ProleController(tmp_path) + installer = ProleSilentInstaller(c) + + with mock.patch.object(installer, "_load_inputs_from_cfg", return_value={}), \ + mock.patch.object(installer, "_write_cfg"), \ + mock.patch.object(installer, "_perform_cluster_reset"), \ + mock.patch.object(installer, "_close_log_file"): + captured = {} + + def fake_run_milestones(milestones, progress_callback=None): + captured["ids"] = [m.id for m in milestones] + + c.run_milestones = fake_run_milestones + rc = installer.run() + + assert rc == 0 + assert "supabase_images_preload" in captured["ids"] + assert "supabase" in captured["ids"] + assert captured["ids"].index("supabase_images_preload") < captured["ids"].index("supabase") + # ===== Milestone concrete subclasses (from core/milestones.py) ===== diff --git a/tests/installer/test_milestones_extended.py b/tests/installer/test_milestones_extended.py index 6ecc8f8..4fa3683 100644 --- a/tests/installer/test_milestones_extended.py +++ b/tests/installer/test_milestones_extended.py @@ -18,6 +18,7 @@ from installer.core.milestones import ( DockerBuildMilestone, GitOpsMilestone, KerberosMilestone, + SupabaseImagePreloadMilestone, SupabaseMilestone, DependenciesMilestone, InitializationScriptsMilestone, @@ -350,9 +351,89 @@ def test_kerberos_enabled_no_mode(): # --------------------------------------------------------------------------- -# SupabaseMilestone +# Supabase milestones # --------------------------------------------------------------------------- + +def test_supabase_image_preload_disabled(): + state = _make_state(**{"init_cluster.supabase_enabled": "False"}) + milestone = SupabaseImagePreloadMilestone() + milestone.execute(state) + assert state.config_data["Supabase"]["IMAGES_STATUS"] == "Skipped" + + +def test_supabase_image_preload_success(tmp_path): + state = _make_state(**{ + "init_cluster.supabase_enabled": "True", + "init_cluster.cluster_env": "dev", + }) + state.controller.project_root = tmp_path + supa_dir = tmp_path / "supabase" + supa_dir.mkdir() + (supa_dir / "deploy.sh").write_text("#!/bin/bash\nexit 0\n") + mock_result = MagicMock() + mock_result.returncode = 0 + captured_cmd = [] + + def capture_run(cmd, **kwargs): + captured_cmd.extend(cmd) + return mock_result + + milestone = SupabaseImagePreloadMilestone() + with patch.object(milestone, "_get_script_env", return_value={}), \ + patch("installer.core.milestones.subprocess.run", side_effect=capture_run), \ + patch("installer.core.env._normalize_cluster_env", return_value="dev"), \ + patch("installer.core.env._parse_bool", return_value=False): + milestone.execute(state) + + assert state.config_data["Supabase"]["IMAGES_STATUS"] == "Prepared" + assert state.inputs["supabase.images_preloaded"] == "True" + assert "--prefetch-images-only" in captured_cmd + assert "k3d" in captured_cmd + + +def test_supabase_image_preload_failure_raises(tmp_path): + state = _make_state(**{ + "init_cluster.supabase_enabled": "True", + "init_cluster.cluster_env": "dev", + }) + state.controller.project_root = tmp_path + supa_dir = tmp_path / "supabase" + supa_dir.mkdir() + (supa_dir / "deploy.sh").write_text("#!/bin/bash\nexit 1\n") + mock_result = MagicMock() + mock_result.returncode = 1 + milestone = SupabaseImagePreloadMilestone() + + with patch.object(milestone, "_get_script_env", return_value={}), \ + patch("installer.core.milestones.subprocess.run", return_value=mock_result), \ + patch("installer.core.env._normalize_cluster_env", return_value="dev"), \ + patch("installer.core.env._parse_bool", return_value=False): + with pytest.raises(Exception, match="[Pp]reload"): + milestone.execute(state) + + assert state.config_data["Supabase"]["IMAGES_STATUS"] == "Attempted" + + +def test_supabase_image_preload_skips_for_k8s_mode(tmp_path): + state = _make_state(**{ + "init_cluster.supabase_enabled": "True", + "init_cluster.cluster_env": "prod", + }) + state.controller.project_root = tmp_path + supa_dir = tmp_path / "supabase" + supa_dir.mkdir() + (supa_dir / "deploy.sh").write_text("#!/bin/bash\nexit 0\n") + + milestone = SupabaseImagePreloadMilestone() + with patch.object(milestone, "_get_script_env", return_value={}), \ + patch("installer.core.milestones.subprocess.run") as mock_run, \ + patch("installer.core.env._normalize_cluster_env", return_value="prod"): + milestone.execute(state) + + assert state.config_data["Supabase"]["IMAGES_STATUS"] == "Skipped" + mock_run.assert_not_called() + def test_supabase_disabled(): state = _make_state(**{"init_cluster.supabase_enabled": "False"}) milestone = SupabaseMilestone() @@ -447,6 +528,33 @@ def test_supabase_enabled_with_progress(tmp_path): progress.assert_called() +def test_supabase_deploy_skips_prefetch_when_preloaded(tmp_path): + state = _make_state(**{ + "init_cluster.supabase_enabled": "True", + "init_cluster.cluster_env": "dev", + "supabase.images_preloaded": "True", + }) + state.controller.project_root = tmp_path + supa_dir = tmp_path / "supabase" + supa_dir.mkdir() + (supa_dir / "deploy.sh").write_text("#!/bin/bash\nexit 0\n") + mock_result = MagicMock() + mock_result.returncode = 0 + captured_cmd = [] + + def capture_run(cmd, **kwargs): + captured_cmd.extend(cmd) + return mock_result + + milestone = SupabaseMilestone() + with patch.object(milestone, "_get_script_env", return_value={}), \ + patch("installer.core.milestones.subprocess.run", side_effect=capture_run), \ + patch("installer.core.env._normalize_cluster_env", return_value="dev"): + milestone.execute(state) + + assert "--skip-prefetch" in captured_cmd + + # --------------------------------------------------------------------------- # DependenciesMilestone — error paths not covered by existing tests # ---------------------------------------------------------------------------