From ce5ff83eb4165ceb186c6113070cc9b2b20a0995 Mon Sep 17 00:00:00 2001 From: chrisfu Date: Sat, 14 Mar 2026 20:07:54 -0700 Subject: [PATCH] checkpoint: improve init scripts, installer flows, and kube context handling - Fix kube context switching for k3s single-context kubeconfigs and k3d shorthand prefixes - Update common init/status scripts (registry, kerberos, cnpg backup, service layer, common services) - Add Gitea init script and installer ArgoCD screen - Add Supabase realtime probe patching plus regression tests - Extend installer core/UI test coverage --- conf/port-mapping.cfg | 13 + etc/init_cloudnative_pg.sh | 90 +- etc/init_cnpg_backup.sh | 32 +- etc/init_common_services.sh | 21 +- etc/init_gitea.sh | 207 +++ etc/init_kerberos.sh | 36 +- etc/init_registry.sh | 124 +- etc/init_service_layer.sh | 21 +- etc/prole_cfg.sh | 63 +- etc/status.sh | 16 +- etc/status_common_services.sh | 111 +- installer/core/actions.py | 587 ++++---- installer/core/env.py | 99 +- installer/core/repair_pipeline_cli.py | 4 +- installer/ui/screens/__init__.py | 53 +- installer/ui/screens/argocd.py | 276 ++++ installer/ui/screens/base.py | 1 + installer/ui/screens/cfg.py | 100 +- installer/ui/screens/cluster.py | 207 ++- installer/ui/screens/database.py | 393 ++++- installer/ui/screens/docker.py | 152 +- installer/ui/screens/navigation.py | 38 +- k8s/prole/prole-db-recovery.yaml.tpl | 2 +- k8s/prole/prole-db.yaml | 2 +- mock_val/init_cloudnative_pg.sh | 1334 +---------------- mock_val/init_cnpg_backup.sh | 32 +- prole-db/.version | 2 +- prole/cli.py | 4 +- prole/deployment.py | 4 +- run_with_coverage.sh | 17 +- scripts/validation/check_kerberos.sh | 35 + supabase/deploy.sh | 58 +- supabase/helm/generated/values.generated.json | 6 +- supabase/patch_realtime_probe.py | 79 + ...nit_cloudnative_pg_bootstrap_admin_keys.sh | 169 +++ ...pg_backup_start_waits_for_plugin_socket.sh | 202 +++ .../test_prole_cfg_kube_context_k3d_prefix.sh | 76 + ...fg_kube_context_single_context_fallback.sh | 71 + ...st_status_k3d_common_services_namespace.sh | 112 ++ .../etc/test_supabase_patch_realtime_probe.sh | 60 + tests/installer/test_actions_helpers.py | 23 +- .../installer/test_argocd_canvas_only_page.py | 48 + ...test_cluster_save_triggers_status_check.py | 92 ++ tests/installer/test_core_classes.py | 8 +- .../test_gitea_init_script_present.py | 7 + tests/installer/test_namespace_propagation.py | 88 ++ .../test_push_docker_image_ca_fallback.py | 74 + .../installer/test_registry_k3s_selection.py | 71 + .../test_registry_rescan_after_init.py | 30 + tests/test_cluster_screen_observe_only.py | 141 ++ ...st_init_kerberos_ad_forwarder_readiness.py | 19 + ...st_k3s_common_core_namespace_and_repair.py | 4 +- tests/test_reconcile_blocked_cluster_state.py | 6 +- .../test_repair_stale_released_pv_claimref.py | 6 +- 54 files changed, 3433 insertions(+), 2093 deletions(-) create mode 100644 etc/init_gitea.sh create mode 100644 installer/ui/screens/argocd.py create mode 100644 supabase/patch_realtime_probe.py create mode 100644 tests/etc/test_init_cloudnative_pg_bootstrap_admin_keys.sh create mode 100644 tests/etc/test_init_cnpg_backup_start_waits_for_plugin_socket.sh create mode 100644 tests/etc/test_prole_cfg_kube_context_k3d_prefix.sh create mode 100644 tests/etc/test_prole_cfg_kube_context_single_context_fallback.sh create mode 100644 tests/etc/test_status_k3d_common_services_namespace.sh create mode 100644 tests/etc/test_supabase_patch_realtime_probe.sh create mode 100644 tests/installer/test_argocd_canvas_only_page.py create mode 100644 tests/installer/test_cluster_save_triggers_status_check.py create mode 100644 tests/installer/test_gitea_init_script_present.py create mode 100644 tests/installer/test_namespace_propagation.py create mode 100644 tests/installer/test_push_docker_image_ca_fallback.py create mode 100644 tests/installer/test_registry_k3s_selection.py create mode 100644 tests/installer/test_registry_rescan_after_init.py create mode 100644 tests/test_cluster_screen_observe_only.py create mode 100644 tests/test_init_kerberos_ad_forwarder_readiness.py diff --git a/conf/port-mapping.cfg b/conf/port-mapping.cfg index 986985d..e30ec6e 100644 --- a/conf/port-mapping.cfg +++ b/conf/port-mapping.cfg @@ -1,3 +1,16 @@ # Port mappings for Prole Tools (generated). # Format: key: local=... remote=... ns=... svc=... address=... +argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0 +supabase-studio: local=18080 remote=3000 ns=supabase svc=studio address=0.0.0.0 +supabase-auth: local=9999 remote=9999 ns=supabase svc=auth address=127.0.0.1 +supabase-rest: local=3001 remote=3000 ns=supabase svc=rest address=0.0.0.0 +supabase-realtime: local=4000 remote=4000 ns=supabase svc=realtime address=0.0.0.0 +garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0 +openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0 +opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0 +dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1 +postgres: local=5432 remote=5432 ns=knoe-db svc=prole-db-rw address=0.0.0.0 +prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1 +grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0 +supabase-kong: local=8000 remote=8000 ns=supabase svc=kong address=0.0.0.0 diff --git a/etc/init_cloudnative_pg.sh b/etc/init_cloudnative_pg.sh index 2475a0a..89e7737 100755 --- a/etc/init_cloudnative_pg.sh +++ b/etc/init_cloudnative_pg.sh @@ -78,8 +78,16 @@ fi CNPG_MANIFEST="$K8S_PROLE_DIR/prole-db.yaml" BARMAN_OBJECTSTORE_MANIFEST="$K8S_PROLE_DIR/prole-db-barman-objectstore.yaml" -SECRETS_DIR="$PROLE_SERVICE/secrets" -# Resolving CNPG admin keys. +# Secrets and token locations +# Use PROLE_SERVICE if writable locally, otherwise fallback to ~/.prole +if [[ -n "${PROLE_SERVICE:-}" ]] && _prole_usable_dir "${PROLE_SERVICE}/secrets" >/dev/null; then + SECRETS_DIR="$PROLE_SERVICE/secrets" +else + SECRETS_DIR="$HOME/.prole/etc/secrets" +fi +mkdir -p "$SECRETS_DIR" 2>/dev/null || true + +# Resolving CNPG admin keys. # We prefer names without algorithm suffixes to be more generic, matching install.py fallback strategy. ADMIN_PRIV_ED25519="$SECRETS_DIR/admin_ed25519.key" ADMIN_PUB_ED25519="$SECRETS_DIR/admin_ed25519.pub" @@ -101,6 +109,40 @@ ensure_tools() { done } +ensure_local_admin_keypair() { + mkdir -p "$SECRETS_DIR" 2>/dev/null || true + + # If generic already exists, prefer it. + if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then + # Ensure legacy mirror exists for compatibility. + cp "$ADMIN_PRIV_GENERIC" "$ADMIN_PRIV_ED25519" 2>/dev/null || true + cp "$ADMIN_PUB_GENERIC" "$ADMIN_PUB_ED25519" 2>/dev/null || true + return 0 + fi + + # If legacy exists, normalize it into generic. + if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then + cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC" 2>/dev/null || true + cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC" 2>/dev/null || true + chmod 0600 "$ADMIN_PRIV_GENERIC" 2>/dev/null || true + return 0 + fi + + echo "Generating CNPG admin keypair in $SECRETS_DIR ..." + # Attempt ed25519, fallback to rsa if not available + if ! openssl genpkey -algorithm ED25519 -out "$ADMIN_PRIV_GENERIC" 2>/dev/null; then + echo "WARN: ED25519 not supported by openssl, falling back to RSA 4096..." >&2 + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out "$ADMIN_PRIV_GENERIC" + fi + openssl pkey -in "$ADMIN_PRIV_GENERIC" -pubout -out "$ADMIN_PUB_GENERIC" + chmod 0600 "$ADMIN_PRIV_GENERIC" + + # Mirror to legacy filenames for compatibility with other scripts. + cp "$ADMIN_PRIV_GENERIC" "$ADMIN_PRIV_ED25519" 2>/dev/null || true + cp "$ADMIN_PUB_GENERIC" "$ADMIN_PUB_ED25519" 2>/dev/null || true + return 0 +} + _kubectl_is_transient_error() { local msg="${1:-}" # Errors we frequently see on k3s under load / during restart. @@ -1298,18 +1340,29 @@ wait_for_cnpg_pods() { if [[ "$ready_pods" -ge "$target_pods" ]]; then echo "All $ready_pods/$target_pods pods are Ready." - # Verify DB connectivity if kubectl-cnpg is available + # By default, pod readiness is sufficient. Connectivity checks can be enabled explicitly. + # This avoids unnecessary timeouts/resets during initial bootstrap when CNPG may still be + # initializing even though pods are already Ready. if kubectl cnpg version >/dev/null 2>&1; then - echo "Verifying database connectivity via 'kubectl cnpg psql'..." - if kubectl cnpg -n "$NAMESPACE" psql "$CNPG_CLUSTER_NAME" -- -c "SELECT 1" >/dev/null 2>&1; then - echo "Database connection verified." - return 0 - else - echo " [${elapsed}s/${timeout}s] Pods are Ready but database connection failed — waiting for database to accept connections..." - fi + local require_psql="${CNPG_REQUIRE_PSQL:-false}" + if [[ "$require_psql" == "1" || "$require_psql" == "true" || "$require_psql" == "True" || "$require_psql" == "yes" || "$require_psql" == "YES" ]]; then + echo "Verifying database connectivity via 'kubectl cnpg psql'..." + if kubectl cnpg -n "$NAMESPACE" psql "$CNPG_CLUSTER_NAME" -- -c "SELECT 1" >/dev/null 2>&1; then + echo "Database connection verified." + return 0 + else + echo " [${elapsed}s/${timeout}s] Pods are Ready but database connection failed — waiting for database to accept connections..." + fi + else + # Best-effort connectivity check (warn-only) + if ! kubectl cnpg -n "$NAMESPACE" psql "$CNPG_CLUSTER_NAME" -- -c "SELECT 1" >/dev/null 2>&1; then + echo "WARN: CNPG pods are Ready but 'kubectl cnpg psql' is not yet available; continuing." >&2 + fi + return 0 + fi else - # Fallback if no cnpg plugin: just return 0 if pods are ready - return 0 + # Fallback if no cnpg plugin: just return 0 if pods are ready + return 0 fi fi @@ -1793,19 +1846,14 @@ fetch_admin_keys_and_db_pass_from_bao_or_local() { return 1 fi + ensure_local_admin_keypair if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then echo "Using local admin key pair at $SECRETS_DIR" return 0 fi - if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then - echo "Using local legacy admin key pair at $SECRETS_DIR" - # Link or copy to generic for consistent use below - cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC" - cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC" - return 0 - fi - echo "ERROR: Could not obtain admin key pair from OpenBao and no local files found." >&2 - exit 1 + + echo "ERROR: Could not obtain admin key pair from OpenBao and could not generate local keys." >&2 + return 1 } apply_cnpg_admin_secret() { diff --git a/etc/init_cnpg_backup.sh b/etc/init_cnpg_backup.sh index 85184d7..ae217f9 100755 --- a/etc/init_cnpg_backup.sh +++ b/etc/init_cnpg_backup.sh @@ -49,6 +49,9 @@ BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600} BACKUP_STATUS_INTERVAL=${BACKUP_STATUS_INTERVAL:-10} PLUGIN_READY_TIMEOUT=${PLUGIN_READY_TIMEOUT:-180} +PLUGIN_SOCKET_DIR=${PLUGIN_SOCKET_DIR:-/plugins} +LAST_BACKUP_NAME="" + usage() { cat </dev/null || true) if printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -Fxq "$BARMAN_PLUGIN_NAME"; then deployment_rows=$(kubectl get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) @@ -343,9 +348,18 @@ wait_for_plugin_ready() { ready=${ready:-0} replicas=${replicas:-0} if (( ready >= 1 && replicas >= 1 )); then - return 0 + plugin_deploy_ready=1 + break fi done <<< "$deployment_rows" + + if (( plugin_deploy_ready == 1 )); then + pod=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + socket_path="$PLUGIN_SOCKET_DIR/$BARMAN_PLUGIN_NAME" + if [[ -n "$pod" ]] && kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- test -S "$socket_path" >/dev/null 2>&1; then + return 0 + fi + fi fi now=$(date +%s) @@ -391,6 +405,21 @@ wait_for_successful_base_backup() { local start_time now elapsed start_time=$(date +%s) while true; do + if [[ -n "$LAST_BACKUP_NAME" ]]; then + local phase err + phase=$(kubectl -n "$NAMESPACE" get backup "$LAST_BACKUP_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true) + err=$(kubectl -n "$NAMESPACE" get backup "$LAST_BACKUP_NAME" -o jsonpath='{.status.error}' 2>/dev/null || true) + case "${phase:-}" in + Completed|Succeeded|completed|succeeded) + return 0 + ;; + Failed|failed) + echo "ERROR: Backup '$LAST_BACKUP_NAME' failed: ${err:-}" >&2 + return 1 + ;; + esac + fi + if has_successful_base_backup; then return 0 fi @@ -410,6 +439,7 @@ trigger_backup() { local backup_type="${1:-full}" local backup_name backup_name="${CNPG_CLUSTER_NAME}-backup-$(date +%Y%m%d%H%M%S)" + LAST_BACKUP_NAME="$backup_name" echo "Triggering ${backup_type} backup $backup_name ..." if [[ "$backup_type" == "incremental" || "$backup_type" == "incr" ]]; then kubectl_apply_retry "$NAMESPACE" < -Deploys common infrastructure services (ArgoCD, OpenTofu, Garage, OpenBao, Kong, Cert-Manager) +Deploys common infrastructure services (Registry, OpenTofu, Garage, OpenBao, Kong, Cert-Manager) into the given Kubernetes namespace. Use -k to include the Kerberos/KDC service. EOF } @@ -86,7 +86,6 @@ if [ -z "$NS" ]; then NS="default" fi -ARGOCD_NS="${ARGOCD_NAMESPACE:-argocd}" # Registry should live in the common-core/service namespace unless explicitly overridden. REGISTRY_NS="${REGISTRY_NAMESPACE:-${NS}}" @@ -500,7 +499,7 @@ fi OPENTOFU_NAME=${OPENTOFU_NAME:-opentofu} OPENBAO_NAME=${OPENBAO_NAME:-openbao} -ARGOCD_SERVER_NAME=${ARGOCD_SERVER_NAME:-argocd-server} +REGISTRY_NAME=${REGISTRY_NAME:-registry} GARAGE_NAME=${GARAGE_NAME:-garage} KONG_NAME=${KONG_NAME:-prole-svc-kong} OPENTOFU_CONFIGMAP=${OPENTOFU_CONFIGMAP:-opentofu-nginx} @@ -545,13 +544,13 @@ migrate_common_services() { kubectl delete -n "$old_ns" secret "$OPENTOFU_SECRET" --ignore-not-found >/dev/null 2>&1 || true done - for old_ns in $(collect_other_namespaces "$ARGOCD_NS" "$ARGOCD_SERVER_NAME" deployment service); do - echo "Found ArgoCD in namespace '$old_ns'; removing before deploy to '$ARGOCD_NS' ..." + for old_ns in $(collect_other_namespaces "$REGISTRY_NS" "$REGISTRY_NAME" deployment service); do + echo "Found Registry ($REGISTRY_NAME) in namespace '$old_ns'; removing before deploy to '$REGISTRY_NS' ..." if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then - REGISTRY_NAMESPACE="$REGISTRY_NS" "$SCRIPT_DIR/init_registry.sh" -n "$old_ns" stop || true + "$SCRIPT_DIR/init_registry.sh" -n "$old_ns" stop || true else - kubectl delete -n "$old_ns" deploy "$ARGOCD_SERVER_NAME" --ignore-not-found >/dev/null 2>&1 || true - kubectl delete -n "$old_ns" svc "$ARGOCD_SERVER_NAME" --ignore-not-found >/dev/null 2>&1 || true + kubectl delete -n "$old_ns" deploy "$REGISTRY_NAME" --ignore-not-found >/dev/null 2>&1 || true + kubectl delete -n "$old_ns" svc "$REGISTRY_NAME" --ignore-not-found >/dev/null 2>&1 || true fi done @@ -611,10 +610,10 @@ else fi if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then - ARGOCD_NAMESPACE="$ARGOCD_NS" REGISTRY_NAMESPACE="$REGISTRY_NS" \ - "$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NS" --registry-namespace "$REGISTRY_NS" "$ACTION" || rc=$? + REGISTRY_NAMESPACE="$REGISTRY_NS" SERVICE_NAMESPACE="$NS" \ + "$SCRIPT_DIR/init_registry.sh" -n "$REGISTRY_NS" "$ACTION" || rc=$? else - echo "WARN: init_registry.sh not found; ArgoCD deploy skipped." + echo "WARN: init_registry.sh not found; registry deploy skipped." fi if [ -x "$SCRIPT_DIR/init_garage_store.sh" ]; then diff --git a/etc/init_gitea.sh b/etc/init_gitea.sh new file mode 100644 index 0000000..6177c5f --- /dev/null +++ b/etc/init_gitea.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROG="init_gitea" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/prole_cfg.sh" + +MODE="$(prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-k3d}}")" +NAMESPACE="${GITEA_NAMESPACE:-}" +CFG_PATH="" +FORCE=0 + +usage() { + cat < Deployment mode (default: ${MODE:-k3d}) + -n, --namespace Target namespace (default: gitea) + -c, --config Path to prole.cfg (defaults to detected) + --force Delete existing release before deploy + --help Show this help + +Behavior: + - Prefers Helm install/upgrade (gitea-charts/gitea). + - Falls back to applying a generated Deployment/Service/Ingress manifest. + - If a local registry is enabled, the image is mirrored there first. +EOF +} + +die() { echo "[ERROR] $*" >&2; exit 2; } +log() { echo "[INFO] $*"; } +warn() { echo "[WARN] $*" >&2; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --mode) MODE="$(prole_normalize_mode "${2:-}")"; shift 2 ;; + --mode=*) MODE="$(prole_normalize_mode "${1#*=}")"; shift 1 ;; + -n|--namespace) NAMESPACE="${2:-}"; shift 2 ;; + --namespace=*) NAMESPACE="${1#*=}"; shift 1 ;; + -c|--config) CFG_PATH="${2:-}"; shift 2 ;; + --config=*) CFG_PATH="${1#*=}"; shift 1 ;; + --force) FORCE=1; shift ;; + -h|--help) usage; exit 0 ;; + *) break ;; + esac +done + +# Resolve config path and namespace defaults from prole.cfg when present +if [[ -z "$CFG_PATH" && -n "${PROLE_CONF:-}" && -f "${PROLE_CONF}/prole.cfg" ]]; then + CFG_PATH="${PROLE_CONF}/prole.cfg" +elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/prole.cfg" ]]; then + CFG_PATH="$SCRIPT_DIR/../conf/prole.cfg" +fi + +if [[ -z "$NAMESPACE" && -n "$CFG_PATH" ]]; then + maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITOPS_NAMESPACE")" + [[ -z "$maybe_ns" ]] && maybe_ns="$(_prole_cfg_extract_key "$CFG_PATH" "GITEA_NAMESPACE")" + NAMESPACE="$maybe_ns" +fi +NAMESPACE="${NAMESPACE:-gitea}" +export GITEA_NAMESPACE="$NAMESPACE" + +case "$MODE" in + k3d|k3s|k8s|local) ;; + *) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;; +esac +export PROLE_MODE="$MODE" + +command -v kubectl >/dev/null || die "kubectl not found" +command -v helm >/dev/null || warn "helm not found — will use manifest fallback" + +kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 + +CHART_REPO="https://dl.gitea.com/charts" +CHART_NAME="gitea-charts/gitea" +RELEASE_NAME="gitea" +IMAGE_REPO_DEFAULT="gitea/gitea" +IMAGE_TAG="${GITEA_IMAGE_TAG:-1.22.3}" +IMAGE_REPO="$IMAGE_REPO_DEFAULT" + +# Optional reset +if [[ "$FORCE" -eq 1 ]]; then + warn "--force specified; removing existing Gitea resources in namespace $NAMESPACE" + if command -v helm >/dev/null 2>&1; then + helm uninstall "$RELEASE_NAME" -n "$NAMESPACE" >/dev/null 2>&1 || true + fi + kubectl -n "$NAMESPACE" delete deploy/gitea svc/gitea-http svc/gitea-ssh ingress/gitea >/dev/null 2>&1 || true +fi + +# Mirror to local registry if enabled +if _prole_local_registry_enabled && command -v docker >/dev/null; then + HOST_REG="${LOCAL_REGISTRY:-${PROLE_LOCAL_REGISTRY:-}}" + INTERNAL_REG="${LOCAL_REGISTRY_INTERNAL:-${PROLE_LOCAL_REGISTRY_INTERNAL:-}}" + if [[ -n "$HOST_REG" ]]; then + HOST_IMG="${HOST_REG}/${IMAGE_REPO_DEFAULT}" + log "Pulling ${IMAGE_REPO_DEFAULT}:${IMAGE_TAG}" + docker pull "${IMAGE_REPO_DEFAULT}:${IMAGE_TAG}" >/dev/null + log "Tagging ${HOST_IMG}:${IMAGE_TAG}" + docker tag "${IMAGE_REPO_DEFAULT}:${IMAGE_TAG}" "${HOST_IMG}:${IMAGE_TAG}" + log "Pushing ${HOST_IMG}:${IMAGE_TAG}" + docker push "${HOST_IMG}:${IMAGE_TAG}" >/dev/null || warn "Push to local registry failed; continuing with upstream image" + if [[ -n "$INTERNAL_REG" ]]; then + IMAGE_REPO="${INTERNAL_REG}/gitea/gitea" + else + IMAGE_REPO="${HOST_REG}/gitea/gitea" + fi + fi +fi + +deploy_with_helm() { + command -v helm >/dev/null 2>&1 || return 1 + + helm repo add gitea-charts "$CHART_REPO" >/dev/null 2>&1 || true + helm repo update >/dev/null 2>&1 || true + + # Minimal config: NodePort or ClusterIP + Ingress depending on environment. + # Avoid heavy persistence defaults; users can override via values later. + helm upgrade --install "$RELEASE_NAME" "$CHART_NAME" \ + -n "$NAMESPACE" \ + --set image.repository="$IMAGE_REPO" \ + --set image.tag="$IMAGE_TAG" \ + --set service.http.type=ClusterIP \ + --set service.ssh.type=ClusterIP \ + --set gitea.admin.username="${GITEA_ADMIN_USER:-gitea_admin}" \ + --set gitea.admin.password="${GITEA_ADMIN_PASSWORD:-gitea_admin}" \ + --set gitea.admin.email="${GITEA_ADMIN_EMAIL:-gitea_admin@example.local}" \ + --wait --timeout 10m +} + +apply_manifest_fallback() { + warn "Helm unavailable; applying fallback manifest" + cat <&2; } +wait_for_ad_forwarder_ready() { + local ns="$1" + local deploy_name="$2" + local service_name="$3" + local timeout_s="${4:-120}" + + log "Waiting for AD forwarder deployment '${deploy_name}' to become available in namespace '${ns}' ..." + if ! kubectl -n "$ns" rollout status deploy/"$deploy_name" --timeout="${timeout_s}s"; then + err "ERROR: AD forwarder deployment '${deploy_name}' did not become ready in namespace '${ns}'." + kubectl -n "$ns" get pods -l "app=${deploy_name}" -o wide 1>&2 || true + kubectl -n "$ns" describe deploy "${deploy_name}" 1>&2 || true + kubectl -n "$ns" logs -l "app=${deploy_name}" --tail=200 1>&2 || true + exit 1 + fi + + log "Waiting for AD forwarder service '${service_name}' endpoints in namespace '${ns}' ..." + local deadline=$((SECONDS + timeout_s)) + while [[ $SECONDS -lt $deadline ]]; do + local ip + ip=$(kubectl -n "$ns" get endpoints "$service_name" -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null | tr ' ' '\n' | head -n 1 || true) + if [[ -n "$ip" ]]; then + log "[OK] AD forwarder service '${service_name}' has endpoints (e.g. ${ip})." + return 0 + fi + sleep 2 + done + + err "ERROR: AD forwarder service '${service_name}' has no ready endpoints in namespace '${ns}' after ${timeout_s}s." + kubectl -n "$ns" get svc "$service_name" -o wide 1>&2 || true + kubectl -n "$ns" get endpoints "$service_name" -o yaml 1>&2 || true + kubectl -n "$ns" get pods -l "app=${deploy_name}" -o wide 1>&2 || true + exit 1 +} + ensure_tools() { for t in kubectl curl jq; do command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; } @@ -817,7 +851,7 @@ $( ) EOF - kubectl -n "$KRB5_AD_NAMESPACE" rollout status deploy/${KRB5_AD_PROXY_NAME} --timeout=120s || true + wait_for_ad_forwarder_ready "$KRB5_AD_NAMESPACE" "$KRB5_AD_PROXY_NAME" "$KRB5_AD_SERVICE_NAME" 120 } cleanup_ad_forwarder() { diff --git a/etc/init_registry.sh b/etc/init_registry.sh index db02493..1bb3536 100755 --- a/etc/init_registry.sh +++ b/etc/init_registry.sh @@ -4,8 +4,9 @@ set -euo pipefail # init_registry.sh # Purpose: -# - Deploy ArgoCD into Kubernetes -# - Deploy registry:2 into k3s clusters (service namespace) when enabled +# - Deploy registry support for local clusters +# - k3s: deploy in-cluster `registry:2` into the service namespace +# - k3d: create/connect a `k3d` registry on port 5000 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -42,16 +43,6 @@ ACTION="" NAMESPACE_OVERRIDE="" REGISTRY_NAMESPACE_OVERRIDE="" -if [[ -d "$SCRIPT_DIR/../k8s/argocd" ]]; then - ARGOCD_MANIFEST_DIR="$SCRIPT_DIR/../k8s/argocd" -elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/argocd" ]]; then - ARGOCD_MANIFEST_DIR="$PROLE_HOME/k8s/argocd" -else - ARGOCD_MANIFEST_DIR="$SCRIPT_DIR/../k8s/argocd" -fi - -ARGOCD_MANIFEST_FILE=${ARGOCD_MANIFEST_FILE:-"$ARGOCD_MANIFEST_DIR/install.yaml"} - if [[ -d "$SCRIPT_DIR/../k8s/registry" ]]; then REGISTRY_MANIFEST_DIR="$SCRIPT_DIR/../k8s/registry" elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/registry" ]]; then @@ -66,9 +57,12 @@ usage() { cat < -Deploys ArgoCD into the target namespace and, for k3s, deploys registry:2 -into the registry namespace (default: SERVICE_NAMESPACE). -Note: The local Docker registry (port 5000) is managed separately for k3d. +Deploys registry support for local clusters. + +- In k3s mode, deploys `registry:2` into the service namespace (default: SERVICE_NAMESPACE). +- In k3d mode, manages the `k3d` registry (port 5000) and removes any in-cluster registry resources. + +NOTE: `-r/--registry-namespace` is kept for backwards compatibility and is treated as an alias for `-n/--namespace`. EOF } @@ -123,16 +117,10 @@ while [[ $# -gt 0 ]]; do shift done -if [[ -n "$NAMESPACE_OVERRIDE" ]]; then - ARGOCD_NAMESPACE="$NAMESPACE_OVERRIDE" -elif [[ -n "${ARGOCD_NAMESPACE:-}" ]]; then - ARGOCD_NAMESPACE="$ARGOCD_NAMESPACE" -else - ARGOCD_NAMESPACE="argocd" -fi - if [[ -n "$REGISTRY_NAMESPACE_OVERRIDE" ]]; then REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE_OVERRIDE" +elif [[ -n "$NAMESPACE_OVERRIDE" ]]; then + REGISTRY_NAMESPACE="$NAMESPACE_OVERRIDE" elif [[ -n "${REGISTRY_NAMESPACE:-}" ]]; then REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE" else @@ -151,15 +139,7 @@ if [[ "$_mode_resolved" == "k3s" && ( -z "${REGISTRY_NAMESPACE:-}" || "${REGISTR fi unset _mode_resolved -export ARGOCD_NAMESPACE export REGISTRY_NAMESPACE -export NAMESPACE="$ARGOCD_NAMESPACE" - -ARGOCD_LABEL_SELECTOR=${ARGOCD_LABEL_SELECTOR:-"app.kubernetes.io/part-of=argocd"} -ARGOCD_SERVER_SERVICE=${ARGOCD_SERVER_SERVICE:-argocd-server} -ARGOCD_PORT_FORWARD_LOCAL=${ARGOCD_PORT_FORWARD_LOCAL:-8081} -ARGOCD_PORT_FORWARD_REMOTE=${ARGOCD_PORT_FORWARD_REMOTE:-80} -ARGOCD_NODE_SELECTOR=${ARGOCD_NODE_SELECTOR:-} REGISTRY_NODE_SELECTOR=${REGISTRY_NODE_SELECTOR:-} current_mode() { @@ -176,11 +156,7 @@ registry_enabled() { [[ "$mode" == "k3s" || "$mode" == "k3d" ]] } -resolve_argocd_node_selector() { - if [[ -n "$ARGOCD_NODE_SELECTOR" ]]; then - printf '%s' "$ARGOCD_NODE_SELECTOR" - return 0 - fi +resolve_control_plane_node_selector() { local node="" node=$(kubectl get nodes -l 'node-role.kubernetes.io/control-plane' -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "$node" ]]; then @@ -195,7 +171,7 @@ resolve_registry_node_selector() { fi # Default: pin to the control-plane node so hostPort:5000 is reachable via # the k3s server host (e.g. myrddin.prole.org:5000). - resolve_argocd_node_selector + resolve_control_plane_node_selector } apply_registry_node_selector() { @@ -215,32 +191,6 @@ apply_registry_node_selector() { -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"$key\":\"$val\"}}}}}" >/dev/null 2>&1 || true } -apply_argocd_node_selector() { - local selector - selector="$(resolve_argocd_node_selector)" - if [[ -z "$selector" ]]; then - return 0 - fi - local key="${selector%%=*}" - local val="${selector#*=}" - if [[ -z "$key" || -z "$val" ]]; then - echo "WARN: invalid ARGOCD_NODE_SELECTOR '$selector' (expected key=value); skipping." >&2 - return 0 - fi - for target in \ - deployment/argocd-applicationset-controller \ - deployment/argocd-dex-server \ - deployment/argocd-notifications-controller \ - deployment/argocd-repo-server \ - deployment/argocd-server \ - deployment/argocd-redis \ - statefulset/argocd-application-controller; do - kubectl -n "$ARGOCD_NAMESPACE" patch "$target" \ - --type merge \ - -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"$key\":\"$val\"}}}}}" >/dev/null 2>&1 || true - done -} - ensure_tools() { command -v kubectl >/dev/null || { echo "Missing required tool: kubectl" >&2; exit 1; } } @@ -256,14 +206,6 @@ ensure_namespace() { fi } -render_manifest() { - if [[ ! -f "$ARGOCD_MANIFEST_FILE" ]]; then - echo "ERROR: ArgoCD manifest not found: $ARGOCD_MANIFEST_FILE" >&2 - exit 1 - fi - sed "s|\\${ARGOCD_NAMESPACE}|$ARGOCD_NAMESPACE|g" "$ARGOCD_MANIFEST_FILE" -} - render_registry_manifest() { if [[ ! -f "$REGISTRY_MANIFEST_FILE" ]]; then echo "ERROR: Registry manifest not found: $REGISTRY_MANIFEST_FILE" >&2 @@ -276,19 +218,6 @@ cleanup_registry_k8s() { render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found >/dev/null 2>&1 || true } -apply_k8s() { - echo "Applying ArgoCD manifest to namespace '$ARGOCD_NAMESPACE' ..." - render_manifest | kubectl apply --server-side --force-conflicts --field-manager=prole-installer --validate=false -n "$ARGOCD_NAMESPACE" -f - - apply_argocd_node_selector - kubectl rollout status deploy/argocd-server -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true - kubectl rollout status deploy/argocd-repo-server -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true - kubectl rollout status deploy/argocd-dex-server -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true - kubectl rollout status deploy/argocd-applicationset-controller -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true - kubectl rollout status deploy/argocd-notifications-controller -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true - kubectl rollout status deploy/argocd-redis -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true - kubectl rollout status statefulset/argocd-application-controller -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true -} - configure_k3d_insecure_registry() { local cluster_name="${1:-knoe-dev-cluster}" local reg_host="k3d-prole-registry" @@ -454,11 +383,6 @@ apply_registry() { kubectl rollout status deploy/registry -n "$REGISTRY_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true } -delete_k8s() { - echo "Removing ArgoCD resources from namespace '$ARGOCD_NAMESPACE' ..." - render_manifest | kubectl delete -n "$ARGOCD_NAMESPACE" -f - --ignore-not-found -} - delete_registry() { local mode mode=$(current_mode) @@ -475,12 +399,18 @@ delete_registry() { render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found } -status_k8s() { - kubectl -n "$ARGOCD_NAMESPACE" get deploy,statefulset,svc -l "$ARGOCD_LABEL_SELECTOR" 2>/dev/null || true - kubectl -n "$ARGOCD_NAMESPACE" get pods -l "$ARGOCD_LABEL_SELECTOR" 2>/dev/null || true -} - status_registry() { + local mode + mode=$(current_mode) + if [[ "$mode" == "k3d" ]]; then + if command -v k3d >/dev/null 2>&1; then + k3d registry list 2>/dev/null || true + fi + if command -v docker >/dev/null 2>&1; then + docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}' 2>/dev/null | grep -E 'k3d-prole-registry|prole-registry' || true + fi + return 0 + fi kubectl -n "$REGISTRY_NAMESPACE" get deploy,svc -l "app=registry" 2>/dev/null || true kubectl -n "$REGISTRY_NAMESPACE" get pods -l "app=registry" 2>/dev/null || true } @@ -488,27 +418,21 @@ status_registry() { case "${ACTION:-}" in start|initialize|update|reload|restart) ensure_tools - ensure_namespace "$ARGOCD_NAMESPACE" - apply_k8s if registry_enabled; then ensure_namespace "$REGISTRY_NAMESPACE" apply_registry else echo "Skipping registry deploy (mode=$(current_mode))" fi - prole_register_port_forward "argocd" "$ARGOCD_NAMESPACE" "svc/${ARGOCD_SERVER_SERVICE}" \ - "$ARGOCD_PORT_FORWARD_LOCAL" "$ARGOCD_PORT_FORWARD_REMOTE" "0.0.0.0" "TCP" "ArgoCD" ;; stop) ensure_tools - delete_k8s if registry_enabled; then delete_registry fi ;; status) ensure_tools - status_k8s if registry_enabled; then status_registry fi diff --git a/etc/init_service_layer.sh b/etc/init_service_layer.sh index 6e8f380..7438c77 100755 --- a/etc/init_service_layer.sh +++ b/etc/init_service_layer.sh @@ -4,7 +4,7 @@ set -euo pipefail # init_service_layer.sh # Purpose: -# - Deploy the Prole service layer (ArgoCD, OpenTofu, Garage, OpenBao; Kerberos optional) +# - Deploy the Prole service layer (OpenTofu, Garage, OpenBao, Kong; Kerberos optional) # - Keep service-layer resources grouped in SERVICE_NAMESPACE # - Migrate service layer to a new namespace @@ -36,10 +36,11 @@ SERVICE_NAMESPACE_OVERRIDE="" FROM_NAMESPACE="" TO_NAMESPACE="" ENABLE_KERBEROS=0 +ENABLE_ARGOCD=0 usage() { cat < +Usage: init_service_layer.sh [-n|--namespace NS] [-k|--kerberos] [-a|--argocd] [--from OLD_NS] [--to NEW_NS] EOF } @@ -71,6 +72,10 @@ while [[ $# -gt 0 ]]; do ENABLE_KERBEROS=1 shift ;; + -a|--argocd) + ENABLE_ARGOCD=1 + shift + ;; --from) shift FROM_NAMESPACE="${1:-}" @@ -215,8 +220,10 @@ deploy_service_layer() { *) kdc_action="update" ;; esac - ARGOCD_NAMESPACE="$ARGOCD_NS" REGISTRY_NAMESPACE="$REGISTRY_NS" SERVICE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NS" --registry-namespace "$REGISTRY_NS" "$argocd_action" || rc=$? + if [[ "$ENABLE_ARGOCD" == "1" ]]; then + ARGOCD_NAMESPACE="$ARGOCD_NS" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_argocd.sh" -n "$ARGOCD_NS" "$argocd_action" || rc=$? + fi OPENTOFU_NAMESPACE="$ns" OPENTOFU_SECRET_NAMESPACE="${NAMESPACE:-$ns}" OPENTOFU_OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" "$opentofu_action" || rc=$? @@ -244,8 +251,10 @@ deploy_service_layer() { cleanup_old_namespace() { local ns="$1" log "Cleaning up service layer in old namespace '$ns' ..." - ARGOCD_NAMESPACE="$ARGOCD_NS" REGISTRY_NAMESPACE="$REGISTRY_NS" SERVICE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_registry.sh" -n "$ARGOCD_NS" --registry-namespace "$REGISTRY_NS" stop || true + if [[ "$ENABLE_ARGOCD" == "1" ]]; then + ARGOCD_NAMESPACE="$ARGOCD_NS" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_argocd.sh" -n "$ARGOCD_NS" stop || true + fi OPENTOFU_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" stop || true OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ diff --git a/etc/prole_cfg.sh b/etc/prole_cfg.sh index f5f5b73..17c3740 100644 --- a/etc/prole_cfg.sh +++ b/etc/prole_cfg.sh @@ -925,15 +925,60 @@ prole_ensure_kube_context() { current_context=$(kubectl config current-context 2>/dev/null || true) if [[ -n "${KUBECONTEXT:-}" && "$current_context" != "$KUBECONTEXT" ]]; then - echo "DEBUG: prole_ensure_kube_context: Switching context to '${KUBECONTEXT}'" >&2 - echo "INFO: Switching kubectl context to '${KUBECONTEXT}' (prole.cfg overrides local env)" >&2 - if kubectl config use-context "$KUBECONTEXT" >/dev/null 2>&1; then - echo "INFO: kubectl context set to '${KUBECONTEXT}'" >&2 - return 0 - else - echo "ERROR: Could not switch to context '${KUBECONTEXT}'. Available contexts:" >&2 - kubectl config get-contexts --no-headers 2>/dev/null | awk '{print " "$2}' >&2 || true - return 1 + local desired_context + desired_context="$KUBECONTEXT" + + local context_names + context_names=$(kubectl config get-contexts -o name 2>/dev/null || true) + + local do_switch + do_switch=1 + + # If the requested context isn't present, try to map common shorthand to real contexts. + # For k3d, contexts are typically named `k3d-`. + if ! grep -Fxq "$desired_context" <<<"$context_names"; then + if [[ "$mode" == "k3d" && "$desired_context" != k3d-* ]]; then + local prefixed + prefixed="k3d-${desired_context}" + if grep -Fxq "$prefixed" <<<"$context_names"; then + desired_context="$prefixed" + fi + fi + fi + + # k3s-generated kubeconfigs often contain a single context called `default`. + # If the config only has one context, use it even when the configured KUBECONTEXT name + # doesn't match. + if ! grep -Fxq "$desired_context" <<<"$context_names"; then + local only_context="" + local context_count=0 + while IFS= read -r _ctx; do + [[ -z "${_ctx:-}" ]] && continue + context_count=$((context_count + 1)) + only_context="$_ctx" + done <<<"$context_names" + unset _ctx + + if [[ $context_count -eq 1 && -n "${only_context:-}" ]]; then + echo "WARN: Requested kubectl context '${desired_context}' not found in active KUBECONFIG; using only available context '${only_context}'" >&2 + desired_context="$only_context" + else + echo "WARN: Requested kubectl context '${desired_context}' not found in active KUBECONFIG; skipping explicit context switch and falling back to mode-based selection" >&2 + do_switch=0 + fi + fi + + if [[ $do_switch -eq 1 ]]; then + echo "DEBUG: prole_ensure_kube_context: Switching context to '${desired_context}'" >&2 + echo "INFO: Switching kubectl context to '${desired_context}' (prole.cfg overrides local env)" >&2 + if kubectl config use-context "$desired_context" >/dev/null 2>&1; then + echo "INFO: kubectl context set to '${desired_context}'" >&2 + return 0 + else + echo "ERROR: Could not switch to context '${desired_context}'. Available contexts:" >&2 + kubectl config get-contexts --no-headers 2>/dev/null | awk '{print " "$2}' >&2 || true + return 1 + fi fi else echo "DEBUG: prole_ensure_kube_context: current_context='${current_context}', KUBECONTEXT='${KUBECONTEXT:-}'" >&2 diff --git a/etc/status.sh b/etc/status.sh index 2b5e634..a2d60f9 100755 --- a/etc/status.sh +++ b/etc/status.sh @@ -118,6 +118,8 @@ if [[ -z "$PROLE_NAMESPACE" ]]; then PROLE_NAMESPACE="default" fi +_deployment_mode="$(_read_cfg_value 'DEPLOYMENT_MODE')" + if [[ -z "$KUBECONFIG" ]]; then _kc="$(_read_cfg_value 'KUBECONFIG')" if [[ -n "$_kc" && -f "$_kc" ]]; then @@ -126,8 +128,12 @@ if [[ -z "$KUBECONFIG" ]]; then fi # Fallback: project-local kubeconfig (only if it can reach the cluster) if [[ -z "${KUBECONFIG:-}" && -f "$PROJECT_ROOT/prole-k3s.kubeconfig" ]]; then - if KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig" kubectl cluster-info >/dev/null 2>&1; then - export KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig" + # Only auto-select the project-local k3s kubeconfig when explicitly running in k3s mode. + # In k3d mode, this kubeconfig may be reachable but is the wrong cluster. + if [[ "${_deployment_mode,,}" == "k3s" ]]; then + if KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig" kubectl cluster-info >/dev/null 2>&1; then + export KUBECONFIG="$PROJECT_ROOT/prole-k3s.kubeconfig" + fi fi fi @@ -360,7 +366,11 @@ if [[ -f "$SCRIPT_DIR/status_common_services.sh" ]]; then echo "" printf "${BOLD}Common Services${NC}\n" echo "───────────────────────────────────────────" - cs_args=("-n" "$NAMESPACE") + _service_ns="$(_read_cfg_value 'SERVICE_NAMESPACE')" + if [[ -z "${_service_ns:-}" ]]; then + _service_ns="$PROLE_NAMESPACE" + fi + cs_args=("-n" "${_service_ns:-default}") if _is_true "${_kerberos_enabled:-false}"; then cs_args+=("-k") fi diff --git a/etc/status_common_services.sh b/etc/status_common_services.sh index e9a093c..0e7e136 100755 --- a/etc/status_common_services.sh +++ b/etc/status_common_services.sh @@ -25,7 +25,7 @@ usage() { cat </dev/null 2>&1; then exit 1 fi +current_mode() { + if declare -F prole_normalize_mode >/dev/null 2>&1; then + prole_normalize_mode "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" + return 0 + fi + printf '%s' "${PROLE_MODE:-${DEPLOYMENT_MODE:-}}" +} + +MODE="$(current_mode)" +REGISTRY_CHECK=0 +if [[ "$MODE" == "k3s" || "$MODE" == "k3d" ]]; then + REGISTRY_CHECK=1 +fi + timestamp=$(date "+%Y-%m-%d %H:%M:%S") ctx=$(kubectl config current-context 2>/dev/null || true) server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true) @@ -103,11 +117,12 @@ echo "Common service status" echo "Time: $timestamp" echo "Context: ${ctx:-}" echo "Server: ${server:-}" +echo "Mode: ${MODE:-}" if [[ -n "${KUBECONFIG:-}" ]]; then echo "Kubeconfig: $KUBECONFIG" fi echo "Namespace: $NS" -echo "ArgoCD Namespace: $ARGOCD_NS" +echo "Registry Namespace: $REGISTRY_NS" echo "Kong Namespace: $KONG_NS" echo "Cert-Manager Namespace: $CERTMGR_NS" echo "" @@ -119,7 +134,11 @@ run_cmd() { } echo "== Services ==" -run_cmd kubectl -n "$ARGOCD_NS" get svc argocd-server +if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then + run_cmd kubectl -n "$REGISTRY_NS" get svc registry +elif [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" == "k3d" ]]; then + run_cmd k3d registry list +fi run_cmd kubectl -n "$NS" get svc opentofu garage openbao if [[ "$ENABLE_KERBEROS" == "1" ]]; then run_cmd kubectl -n "$NS" get svc auth @@ -128,8 +147,9 @@ run_cmd kubectl -n "$KONG_NS" get svc "$KONG_NAME" run_cmd kubectl -n "$CERTMGR_NS" get svc cert-manager cert-manager-webhook echo "== Workloads ==" -run_cmd kubectl -n "$ARGOCD_NS" get deploy argocd-server argocd-repo-server argocd-dex-server argocd-applicationset-controller argocd-notifications-controller argocd-redis -run_cmd kubectl -n "$ARGOCD_NS" get statefulset argocd-application-controller +if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then + run_cmd kubectl -n "$REGISTRY_NS" get deploy registry +fi run_cmd kubectl -n "$NS" get deploy opentofu if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then run_cmd kubectl -n "$NS" get statefulset openbao @@ -144,7 +164,9 @@ run_cmd kubectl -n "$KONG_NS" get deploy "$KONG_NAME" run_cmd kubectl -n "$CERTMGR_NS" get deploy cert-manager cert-manager-cainjector cert-manager-webhook echo "== Pods ==" -run_cmd kubectl -n "$ARGOCD_NS" get pods | grep -Ei "argocd" || true +if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then + run_cmd kubectl -n "$REGISTRY_NS" get pods | grep -Ei "registry" || true +fi run_cmd kubectl -n "$NS" get pods | grep -Ei "opentofu|garage|openbao" || true if [[ "$ENABLE_KERBEROS" == "1" ]]; then run_cmd kubectl -n "$NS" get pods | grep -Ei "auth" || true @@ -224,7 +246,52 @@ check_resource() { fi } -check_resource svc argocd-server "$ARGOCD_NS" +check_deploy_image() { + local name="$1" + local ns="$2" + local expected="$3" + local actual + actual=$(kubectl -n "$ns" get deploy "$name" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + if [[ -z "${actual:-}" ]]; then + echo "[FAIL] image/$name is missing (ns=$ns)" + missing=$((missing + 1)) + return 0 + fi + if [[ "$actual" != "$expected" ]]; then + echo "[FAIL] image/$name expected '$expected' got '$actual' (ns=$ns)" + missing=$((missing + 1)) + return 0 + fi + echo "[OK] image/$name is '$expected' (ns=$ns)" +} + +check_k3d_registry() { + if command -v k3d >/dev/null 2>&1; then + if k3d registry list --no-headers 2>/dev/null | awk '{print $1}' | grep -qx 'prole-registry'; then + echo "[OK] registry/registry exists (k3d: prole-registry)" + return 0 + fi + fi + if command -v docker >/dev/null 2>&1; then + if docker ps --format '{{.Names}}' 2>/dev/null | grep -Eq '^(k3d-prole-registry|prole-registry)$'; then + echo "[OK] registry/registry exists (docker)" + return 0 + fi + fi + echo "[FAIL] registry/registry is missing (k3d/docker)" + missing=$((missing + 1)) +} + +if [[ "$REGISTRY_CHECK" -eq 1 ]]; then + if [[ "$MODE" == "k3d" ]]; then + check_k3d_registry + else + check_resource svc registry "$REGISTRY_NS" + check_resource deploy registry "$REGISTRY_NS" + check_deploy_image registry "$REGISTRY_NS" 'registry:2' + fi +fi + check_resource svc opentofu "$NS" check_resource svc garage "$NS" check_resource svc openbao "$NS" @@ -235,12 +302,6 @@ check_resource svc "$KONG_NAME" "$KONG_NS" check_resource svc cert-manager "$CERTMGR_NS" check_resource svc cert-manager-webhook "$CERTMGR_NS" -check_resource deploy argocd-server "$ARGOCD_NS" -check_resource deploy argocd-repo-server "$ARGOCD_NS" -check_resource deploy argocd-dex-server "$ARGOCD_NS" -check_resource deploy argocd-applicationset-controller "$ARGOCD_NS" -check_resource deploy argocd-notifications-controller "$ARGOCD_NS" -check_resource deploy argocd-redis "$ARGOCD_NS" check_resource deploy opentofu "$NS" if kubectl -n "$NS" get statefulset openbao >/dev/null 2>&1; then check_resource statefulset openbao "$NS" @@ -250,7 +311,6 @@ fi if [[ "$ENABLE_KERBEROS" == "1" ]]; then check_resource deploy auth "$NS" fi -check_resource statefulset argocd-application-controller "$ARGOCD_NS" check_resource statefulset garage "$NS" check_resource deploy "$KONG_NAME" "$KONG_NS" check_resource deploy cert-manager "$CERTMGR_NS" @@ -261,10 +321,12 @@ pod_filter="opentofu|garage|openbao|kong|cert-manager" if [[ "$ENABLE_KERBEROS" == "1" ]]; then pod_filter="opentofu|garage|openbao|auth|kong|cert-manager" fi -analyze_pods "$ARGOCD_NS" "argocd" analyze_pods "$NS" "$pod_filter" analyze_pods "$KONG_NS" "kong" analyze_pods "$CERTMGR_NS" "cert-manager" +if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then + analyze_pods "$REGISTRY_NS" "registry" +fi if [[ -n "$blocked_lines" ]]; then echo "" @@ -313,12 +375,10 @@ _workloads_for_comp() { local _comp="$1" local _ns="${COMP_NAMESPACE[$_comp]:-$NS}" case "$_comp" in - argocd) - echo "statefulset argocd-application-controller $ARGOCD_NS" - for _d in argocd-server argocd-repo-server argocd-dex-server \ - argocd-applicationset-controller argocd-notifications-controller argocd-redis; do - echo "deployment $_d $ARGOCD_NS" - done + registry) + if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then + echo "deployment registry $REGISTRY_NS" + fi ;; openbao) if kubectl -n "$_ns" get statefulset openbao >/dev/null 2>&1; then @@ -347,10 +407,12 @@ _recheck_pods() { COMP_NAMESPACE=() blocked_lines="" blocked=0 - analyze_pods "$ARGOCD_NS" "argocd" analyze_pods "$NS" "$pod_filter" analyze_pods "$KONG_NS" "kong" analyze_pods "$CERTMGR_NS" "cert-manager" + if [[ "$REGISTRY_CHECK" -eq 1 && "$MODE" != "k3d" ]]; then + analyze_pods "$REGISTRY_NS" "registry" + fi if [[ -n "$blocked_lines" ]]; then blocked=1 fi @@ -358,8 +420,9 @@ _recheck_pods() { # Print the name of each component that currently has blocked pods. _blocked_comps() { - for _bc in argocd openbao opentofu garage auth kong certmgr; do + for _bc in registry openbao opentofu garage auth kong certmgr; do [[ "$_bc" == "auth" && "$ENABLE_KERBEROS" != "1" ]] && continue + [[ "$_bc" == "registry" && ( "$REGISTRY_CHECK" -ne 1 || "$MODE" == "k3d" ) ]] && continue [[ ${BLOCKED_COUNT["$_bc"]:-0} -gt 0 ]] && echo "$_bc" done } diff --git a/installer/core/actions.py b/installer/core/actions.py index ea2dd03..33aae61 100644 --- a/installer/core/actions.py +++ b/installer/core/actions.py @@ -92,7 +92,7 @@ def _configure_unbuffered_io(): pass -class ProleInstallerBase: +class ProleInstaller: """Shared business logic for both silent and interactive installer modes. Subclasses must provide a :meth:`_get_input` implementation that bridges @@ -1509,6 +1509,169 @@ class ProleInstallerBase: phase = ((ns.get("status") or {}).get("phase") or "").strip() return phase == "Terminating" + # ------------------------------------------ blocked cluster reconciliation + def _classify_blocked_pod(self, env: dict, pod: dict) -> dict: + """Classify common pod blockers into stable, testable categories. + + Returns a dict with: + - `type`: primary classification (storage-first) + - `types`: list of all detected classification tags + """ + + _ = env # reserved for future use + meta = pod.get("metadata") or {} + status = pod.get("status") or {} + + name = (meta.get("name") or "").strip() + ns = (meta.get("namespace") or "").strip() + phase = (status.get("phase") or "").strip() + + types: list[str] = [] + + # ContainerCreating / ImagePullBackOff etc. + for cs in status.get("containerStatuses") or []: + waiting = ((cs.get("state") or {}).get("waiting") or {}) + reason = (waiting.get("reason") or "").strip() + if reason == "ContainerCreating": + types.append("container_creating") + elif reason: + types.append(f"container_waiting_{reason.lower()}") + + # Unschedulable reasons. + for cond in status.get("conditions") or []: + try: + if (cond.get("type") or "").strip() != "PodScheduled": + continue + if (cond.get("status") or "").strip() != "False": + continue + if (cond.get("reason") or "").strip() != "Unschedulable": + continue + + msg = ((cond.get("message") or "").strip() or "").lower() + if "persistent volumes" in msg and "bind" in msg: + types.append("unschedulable_due_to_pv_binding") + if "didn't match pod's node affinity" in msg or "node affinity/selector" in msg: + types.append("unschedulable_due_to_node_affinity") + if "had untolerated taint" in msg: + types.append("unschedulable_due_to_untolerated_taint") + except Exception: + continue + + # Primary type: prefer storage/root-cause categories. + primary = "ok" + for preferred in ( + "unschedulable_due_to_pv_binding", + "unschedulable_due_to_node_affinity", + "unschedulable_due_to_untolerated_taint", + "container_creating", + ): + if preferred in types: + primary = preferred + break + if primary == "ok" and phase in ("Pending", "Unknown") and types: + primary = types[0] + + # Dedupe while keeping order. + seen: set[str] = set() + deduped: list[str] = [] + for t in types: + if t not in seen: + seen.add(t) + deduped.append(t) + + return { + "pod": f"{ns}/{name}" if ns and name else (name or ns or ""), + "namespace": ns, + "name": name, + "phase": phase, + "type": primary, + "types": deduped, + } + + def _reconcile_blocked_cluster_state(self, env: dict, service_ns: str) -> None: + """Detect hard cluster blockers and raise before running deployment scripts. + + This is intentionally conservative: it attempts safe PV claimRef repair + and then surfaces clear diagnostics for scheduling/storage blockers. + """ + + svc_ns = (service_ns or "").strip() or self._service_namespace() + + # Always attempt safe stale Released PV claimRef repair first. + try: + self._repair_stale_released_pvs(env, reset=False) + except Exception as e: + self.err(f"[WARN] Stale PV claimRef repair failed: {e}") + + pods_data = self._kubectl_get_json(env, ["get", "pod", "-A", "-o", "json"]) or {} + pods = pods_data.get("items") or [] + classified = [self._classify_blocked_pod(env, p) for p in pods] + blocked = [c for c in classified if (c.get("type") or "") != "ok"] + + # If we can't positively identify blocked pods, do not block deployment. + # This keeps behavior conservative when `kubectl` is unavailable. + if not blocked: + return + + # Downstream missingness: surface but treat separately. + missing: list[str] = [] + for kind in ("svc", "deploy"): + try: + res = subprocess.run( + ["kubectl", "-n", svc_ns, "get", kind, "prole-svc-kong"], + env=env, + capture_output=True, + text=True, + timeout=10, + ) + if res.returncode != 0: + missing.append(f"{kind}/prole-svc-kong") + except FileNotFoundError: + # If kubectl isn't available, don't guess missing resources. + pass + except Exception: + pass + + all_types: set[str] = set() + for b in blocked: + for t in b.get("types") or []: + all_types.add(t) + + # Group summary for easier scanning. + storage_types = [t for t in sorted(all_types) if "pv_binding" in t] + sched_types = [ + t + for t in sorted(all_types) + if "node_affinity" in t or "untolerated_taint" in t + ] + container_types = [t for t in sorted(all_types) if t.startswith("container_")] + + lines: list[str] = ["Cluster prerequisites are blocked"] + if storage_types or any("pv_binding" in (b.get("type") or "") for b in blocked): + lines.append("\nStorage blockers:") + for t in storage_types or ["unschedulable_due_to_pv_binding"]: + lines.append(f"- {t}") + if sched_types: + lines.append("\nScheduling blockers:") + for t in sched_types: + lines.append(f"- {t}") + if container_types: + lines.append("\nContainer blockers:") + for t in container_types: + lines.append(f"- {t}") + if blocked: + lines.append("\nBlocked pods:") + for b in blocked: + lines.append( + f"- {b.get('pod')} type={b.get('type')} types={','.join(b.get('types') or [])}" + ) + if missing: + lines.append("\nDownstream missing resources:") + for m in sorted(set(missing)): + lines.append(f"- {m}") + + raise Exception("\n".join(lines)) + def _ensure_namespace_ready(self, env: dict, name: str, timeout_s: int = 90) -> None: """Ensure a namespace is usable for deployments. @@ -2251,16 +2414,36 @@ class ProleInstallerBase: images = {img for img in images if "supabase" not in img} if include_kerberos_proxy: - images.add("ghcr.io/bsharp-tech/prole-kerberos-proxy:latest") + krb_img = os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat") + if krb_img: + images.add(krb_img) return sorted(images) + def _get_docker_import_dir(self) -> str: + """Best-effort accessor for a docker-import directory. + + GUI uses a Tk variable with `.get()`, while console flows tend to store a + plain string. This helper normalizes both. + """ + + val = getattr(self, "docker_import_dir", None) + if val is None: + return "" + try: + getter = getattr(val, "get", None) + if callable(getter): + val = getter() + except Exception: + pass + return (str(val) if val is not None else "").strip() + def _prepull_images_to_registry( self, include_supabase: bool, include_kerberos_proxy: bool, log: Callable[[str], None] | None = None, - ) -> None: + ) -> bool: def _log(msg: str): if log: try: @@ -2269,48 +2452,120 @@ class ProleInstallerBase: pass self.log(msg) + cluster_env = "" + try: + cluster_env = self._get_input("init_cluster.cluster_env", "") + except Exception: + cluster_env = "" + + if cluster_env and not _local_registry_enabled(cluster_env): + _log("[SKIP] Local registry disabled; skipping image pre-pull.") + return True + + mode = _deployment_mode_from_env(cluster_env) + if mode == "k3s": + # Keep behavior consistent across installers: k3s relies on in-cluster + # image pulls/import paths rather than a local docker registry. + _log("[SKIP] Image pre-pull is not supported in k3s mode.") + return True + + ensure_fn = getattr(self, "ensure_local_registry_available", None) + if not callable(ensure_fn): + ensure_fn = getattr(self, "_ensure_local_registry_available", None) + info = ensure_fn() if callable(ensure_fn) else None + if not info: + _log("[SKIP] Local registry unavailable; skipping image pre-pull.") + return False + + registry, _cluster_registry = info + images = self._collect_dependent_images( include_supabase, include_kerberos_proxy ) if not images: - _log("[SKIP] No dependent images to prepull.") - return - reg = getattr(self, "local_registry_url", None) - if not reg: - reg = self.prole_cfg_data.get("Docker Build", {}).get("LOCAL_REGISTRY", "") - if not reg: - _log("[SKIP] No local registry configured; skipping prepull.") - return - host, _, port_str = reg.partition(":") - try: - port = int(port_str) - except (ValueError, TypeError): - port = 5000 - if not _http_ping_registry(host, port): - _log(f"[SKIP] Registry {reg} not reachable; skipping prepull.") - return + _log("[SKIP] No dependent images found to pre-pull.") + return True + + overall_ok = True + import_dir = self._get_docker_import_dir() + for image in images: - _log(f"[INFO] Pulling {image} ...") - try: - rc = subprocess.run( - ["docker", "pull", image], - capture_output=True, - text=True, - timeout=300, - ) - if rc.returncode != 0: - self.err( - f"[WARN] docker pull failed for {image}: {(rc.stderr or '').strip()}" - ) - continue - except Exception as e: - self.err(f"[WARN] docker pull exception for {image}: {e}") + local_tag = image + if not image.startswith(f"{registry}/"): + local_tag = f"{registry}/{image}" + + # 1. Check if already in local registry + _log(f"[INFO] Checking if {image} exists in local registry...") + check_reg = subprocess.run( + ["docker", "pull", local_tag], capture_output=True, text=True + ) + if check_reg.returncode == 0: + _log(f"[OK] {image} already exists in local registry as {local_tag}") continue - ok = _push_docker_image(image, reg) - if ok: - _log(f"[OK] Pushed {image} → {reg}") - else: - self.err(f"[WARN] Push failed for {image} → {reg}") + + # 2. Check if we already have it in local docker daemon + check_local = subprocess.run( + ["docker", "image", "inspect", image], + capture_output=True, + text=True, + ) + found = check_local.returncode == 0 + + if not found and import_dir and os.path.isdir(import_dir): + # 3. Check import directory + safe_name = image.replace("/", "_").replace(":", "_") + tar_path = Path(import_dir) / f"{safe_name}.tar" + if tar_path.exists(): + _log(f"[INFO] Found {tar_path} in import directory, loading...") + load = subprocess.run( + ["docker", "load", "-i", str(tar_path)], + capture_output=True, + text=True, + ) + if load.returncode == 0: + found = True + else: + overall_ok = False + self.err(f"[WARN] Failed to load {tar_path}: {load.stderr}") + + if not found: + # 4. Pull from upstream + _log(f"[INFO] Pulling {image} from upstream...") + pull = subprocess.run( + ["docker", "pull", image], capture_output=True, text=True + ) + if pull.returncode != 0: + overall_ok = False + self.err(pull.stdout or "") + self.err(pull.stderr or "") + self.err(f"[ERROR] docker pull failed for {image}") + continue + found = True + + # If we have the image locally, tag and push to local registry + if found: + if local_tag != image: + tag = subprocess.run( + ["docker", "tag", image, local_tag], + capture_output=True, + text=True, + ) + if tag.returncode != 0: + overall_ok = False + self.err(tag.stdout or "") + self.err(tag.stderr or "") + self.err( + f"[ERROR] docker tag failed for {image} to {local_tag}" + ) + continue + + _log(f"[INFO] Pushing {local_tag} to local registry...") + if not _push_docker_image(local_tag, log_fn=self.log): + overall_ok = False + continue + _log(f"[OK] Stored {image} in local registry as {local_tag}") + + return overall_ok class _TeeWriter: @@ -2351,7 +2606,7 @@ class _TeeWriter: return getattr(self._original, name) -class ProleSilentInstaller(ProleInstallerBase): +class ProleConsoleInstaller(ProleInstaller): """Console-based unattended installer driven by prole.cfg inputs.""" silent = True @@ -3128,237 +3383,6 @@ class ProleSilentInstaller(ProleInstallerBase): self.prole_cfg_data["Docker Build"] = db - def _collect_dependent_images( - self, include_supabase: bool, include_kerberos_proxy: bool - ) -> list[str]: - images = set() - for rel_dir in ("k8s/prole", "k8s/openbao"): - base_dir = self.project_root / rel_dir - if not base_dir.exists(): - continue - images.update(_collect_images_from_files(list(base_dir.glob("*.yaml")))) - - if include_supabase: - supa_home = _resolve_supabase_home(self.project_root) - if supa_home: - docker_dir = supa_home / "docker" - compose_files = [docker_dir / "docker-compose.yml"] - if os.environ.get("SUPABASE_USE_DEV_COMPOSE") == "1": - compose_files.append(docker_dir / "dev" / "docker-compose.dev.yml") - images.update(_collect_images_from_files(compose_files)) - - if not include_supabase: - images = {img for img in images if "supabase" not in img} - - if include_kerberos_proxy: - krb_img = os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat") - if krb_img: - images.add(krb_img) - - return sorted(images) - - def _prepull_images_to_registry( - self, - include_supabase: bool, - include_kerberos_proxy: bool, - log: Callable[[str], None] | None = None, - ) -> bool: - def _log(msg: str): - if log: - try: - log(msg) - except Exception: - pass - self.log(msg) - - if not _local_registry_enabled(self._get_input("init_cluster.cluster_env", "")): - _log("[SKIP] Local registry disabled; skipping image pre-pull.") - return False - mode = _deployment_mode_from_env( - self._get_input("init_cluster.cluster_env", "") - ) - - if mode == "k3s": - return self._prepull_images_k3s(include_supabase, include_kerberos_proxy) - - if mode != "k3d": - self.log("[SKIP] Local registry pre-pull only supported for k3d/k3s.") - return False - info = self._ensure_local_registry_available() - if not info: - self.err("[WARN] Local registry unavailable; skipping image pre-pull.") - return False - - registry, _cluster_registry = info - images = self._collect_dependent_images( - include_supabase, include_kerberos_proxy - ) - if not images: - self.log("[INFO] No dependent images found to pre-pull.") - return True - - overall_ok = True - import_dir = self.docker_import_dir - - for image in images: - local_tag = image - if not image.startswith(f"{registry}/"): - local_tag = f"{registry}/{image}" - - # 1. Check if already in local registry - self.log(f"[INFO] Checking if {image} exists in local registry...") - check_reg = subprocess.run( - ["docker", "pull", local_tag], capture_output=True, text=True - ) - if check_reg.returncode == 0: - self.log( - f"[OK] {image} already exists in local registry as {local_tag}" - ) - continue - - # 2. Check if we already have it in local docker daemon - check_local = subprocess.run( - ["docker", "image", "inspect", image], capture_output=True, text=True - ) - found = check_local.returncode == 0 - - if not found and import_dir and os.path.isdir(import_dir): - # 3. Check import directory - safe_name = image.replace("/", "_").replace(":", "_") - tar_path = Path(import_dir) / f"{safe_name}.tar" - if tar_path.exists(): - self.log(f"[INFO] Found {tar_path} in import directory, loading...") - load = subprocess.run( - ["docker", "load", "-i", str(tar_path)], - capture_output=True, - text=True, - ) - if load.returncode == 0: - found = True - else: - self.err(f"[WARN] Failed to load {tar_path}: {load.stderr}") - - if not found: - # 4. Pull from Docker Hub - self.log(f"[INFO] Pulling {image} from Docker Hub...") - pull = subprocess.run( - ["docker", "pull", image], capture_output=True, text=True - ) - if pull.returncode != 0: - overall_ok = False - self.err(pull.stdout or "") - self.err(pull.stderr or "") - self.err(f"[ERROR] docker pull failed for {image}") - continue - found = True - - # If we have the image locally, tag and push to local registry - if found: - if local_tag != image: - tag = subprocess.run( - ["docker", "tag", image, local_tag], - capture_output=True, - text=True, - ) - if tag.returncode != 0: - overall_ok = False - self.err(tag.stdout or "") - self.err(tag.stderr or "") - self.err( - f"[ERROR] docker tag failed for {image} to {local_tag}" - ) - continue - - self.log(f"[INFO] Pushing {local_tag} to local registry...") - if not _push_docker_image(local_tag, log_fn=self.log): - overall_ok = False - continue - self.log(f"[OK] Stored {image} in local registry as {local_tag}") - - return overall_ok - - def _prepull_images_k3s( - self, include_supabase: bool, include_kerberos_proxy: bool - ) -> bool: - """Stage images for k3s by pushing to the in-cluster `registry:2`.""" - db_cfg = self.prole_cfg_data.get("Docker Build", {}) or {} - registry = (db_cfg.get("LOCAL_REGISTRY") or "").strip() - if not registry: - self.log( - "[SKIP] No LOCAL_REGISTRY configured for k3s; skipping image staging." - ) - return False - - images = self._collect_dependent_images( - include_supabase, include_kerberos_proxy - ) - if not images: - self.log("[INFO] No dependent images found to stage for k3s.") - return True - - overall_ok = True - import_dir = self.docker_import_dir - - for image in images: - remote_tag = ( - f"{registry}/{image}" if not image.startswith(f"{registry}/") else image - ) - - # 1. Check local docker daemon - check_local = subprocess.run( - ["docker", "image", "inspect", image], capture_output=True, text=True - ) - found = check_local.returncode == 0 - - # 2. Try docker-import directory - if not found and import_dir and os.path.isdir(import_dir): - safe_name = image.replace("/", "_").replace(":", "_") - tar_path = Path(import_dir) / f"{safe_name}.tar" - if tar_path.exists(): - self.log(f"[INFO] Loading {tar_path} from import directory...") - load = subprocess.run( - ["docker", "load", "-i", str(tar_path)], - capture_output=True, - text=True, - ) - if load.returncode == 0: - found = True - else: - self.err(f"[WARN] Failed to load {tar_path}: {load.stderr}") - - # 3. Pull from upstream - if not found: - self.log(f"[INFO] Pulling {image} ...") - pull = subprocess.run( - ["docker", "pull", image], capture_output=True, text=True - ) - if pull.returncode != 0: - self.err(f"[WARN] docker pull failed for {image}") - else: - found = True - - if not found: - continue - - # 4. Tag and push to k3s registry (skip if already present) - if remote_tag != image: - subprocess.run( - ["docker", "tag", image, remote_tag], capture_output=True, text=True - ) - - if _registry_image_ref_exists(remote_tag): - self.log(f"[SKIP] {remote_tag} already exists in registry; skipping push.") - continue - - self.log(f"[INFO] Pushing {remote_tag} to {registry} ...") - if _push_docker_image(remote_tag, log_fn=self.log): - self.log(f"[OK] Pushed {image} to {registry}") - else: - overall_ok = False - self.err(f"[ERROR] Failed to push {remote_tag} to {registry}") - - return overall_ok - def _fetch_k3s_kubeconfig(self) -> Path | None: """Fetch a fresh kubeconfig from k3s via the Ansible playbook.""" kubeconfig = self.project_root / "prole-k3s.kubeconfig" @@ -4408,25 +4432,17 @@ class ProleSilentInstaller(ProleInstallerBase): if env_key == "service": try: - argocd_ns = self._argocd_namespace() registry_ns = self._registry_namespace() env = self._script_env_for_namespace(self._service_namespace()) - env["ARGOCD_NAMESPACE"] = argocd_ns env["REGISTRY_NAMESPACE"] = registry_ns - self.log("==> Registry/ArgoCD preflight") + self.log("==> Registry preflight") self._run_script( "init_registry.sh", - args=[ - "-n", - argocd_ns, - "--registry-namespace", - registry_ns, - "update", - ], + args=["-n", registry_ns, "update"], env=env, ) except Exception as e: - self.err(f"[WARN] Registry/ArgoCD preflight failed: {e}") + self.err(f"[WARN] Registry preflight failed: {e}") # If this run was launched with a reset request, attempt to reclaim any # strongly-matched stale Released PVs early, before later deploy/init @@ -4510,6 +4526,11 @@ class ProleSilentInstaller(ProleInstallerBase): # Avoid deploying into a namespace that is currently being deleted. self._ensure_namespace_ready(env, service_ns) + # Refuse to proceed when the cluster has hard prerequisites blocked + # (e.g. PV binding / scheduling issues). This prevents blind retries + # and surfaces actionable diagnostics. + self._reconcile_blocked_cluster_state(env=env, service_ns=service_ns) + # Check status before update status_args = ["-n", service_ns] if kerberos_enabled: @@ -5297,7 +5318,7 @@ def _prepare_k3s_pipeline( # Load config to resolve namespace and secrets for pipeline tfvars _log("==> Loading installer config...\n") - installer = ProleSilentInstaller(controller, str(cfg_path)) + installer = ProleConsoleInstaller(controller, str(cfg_path)) try: existing_inputs = installer._load_inputs_from_cfg() except Exception: diff --git a/installer/core/env.py b/installer/core/env.py index f7e3226..7514126 100644 --- a/installer/core/env.py +++ b/installer/core/env.py @@ -1311,7 +1311,12 @@ def _default_opentofu_pipeline_url() -> str: def _push_docker_image(image_tag: str, log_fn=None) -> bool: - """Push Docker image with skopeo fallback for insecure registries.""" + """Push Docker image with skopeo fallback. + + If Docker push fails due to TLS/CA issues, this will try `skopeo copy`. + When a registry CA is configured (via `PROLE_REGISTRY_CA_CERT` or + `PROLE_REGISTRY_CERT_DIR`), it will prefer a secure skopeo push. + """ def _log(msg): if log_fn: @@ -1333,11 +1338,93 @@ def _push_docker_image(image_tag: str, log_fn=None) -> bool: f"[WARN] Docker push failed: {res.stderr.strip() if res.stderr else 'unknown error'}\n" ) - # 2. Try skopeo fallback for insecure registry + # 2. Try skopeo fallback skopeo = shutil.which("skopeo") if skopeo: + cert_dir = (os.environ.get("PROLE_REGISTRY_CERT_DIR") or "").strip() + cert_file = ( + os.environ.get("PROLE_REGISTRY_CA_CERT") + or os.environ.get("REGISTRY_CA_CERT") + or "" + ).strip() + + # Convenience auto-detection (matches how Ansible ships certs from this repo). + # If a registry host is `myrddin.prole.org:5000`, look for + # `ssl/prole/myrddin-registry.crt` when no env override is provided. + if not cert_dir and not cert_file: + try: + ref = (image_tag or "").strip() + if "/" in ref: + registry = ref.split("/", 1)[0] + host_only = registry.split(":", 1)[0] + short = host_only.split(".", 1)[0] + candidates = [ + PROJECT_ROOT / "ssl" / "prole" / f"{short}-registry.crt", + PROJECT_ROOT / "ssl" / "prole" / f"{host_only}-registry.crt", + ] + for c in candidates: + if c.is_file(): + cert_file = str(c) + break + except Exception: + pass + + # Prefer a secure skopeo push if we have a CA configured. + try: + if cert_dir: + p = Path(cert_dir).expanduser() + if p.is_dir(): + _log("Retrying with skopeo (TLS verify, custom cert dir) ...\n") + cmd = [ + skopeo, + "copy", + "--dest-tls-verify=true", + "--dest-cert-dir", + str(p), + f"docker-daemon:{image_tag}", + f"docker://{image_tag}", + ] + res2 = subprocess.run(cmd, capture_output=True, text=True) + if res2.returncode == 0: + _log(f"[OK] Pushed {image_tag} using skopeo (TLS verified)\n") + return True + _log( + f"[WARN] Skopeo TLS-verified push failed: {res2.stderr.strip() if res2.stderr else 'unknown error'}\n" + ) + except Exception as e: + _log(f"[WARN] Unable to use PROLE_REGISTRY_CERT_DIR: {e}\n") + + try: + if cert_file: + p = Path(cert_file).expanduser() + if p.is_file(): + _log("Retrying with skopeo (TLS verify, custom CA cert) ...\n") + with tempfile.TemporaryDirectory(prefix="prole-registry-cert-") as td: + ca_dest = Path(td) / "ca.crt" + shutil.copyfile(str(p), str(ca_dest)) + cmd = [ + skopeo, + "copy", + "--dest-tls-verify=true", + "--dest-cert-dir", + td, + f"docker-daemon:{image_tag}", + f"docker://{image_tag}", + ] + res2 = subprocess.run(cmd, capture_output=True, text=True) + if res2.returncode == 0: + _log( + f"[OK] Pushed {image_tag} using skopeo (TLS verified)\n" + ) + return True + _log( + f"[WARN] Skopeo TLS-verified push failed: {res2.stderr.strip() if res2.stderr else 'unknown error'}\n" + ) + except Exception as e: + _log(f"[WARN] Unable to use PROLE_REGISTRY_CA_CERT: {e}\n") + + # Backward-compatible fallback (explicitly insecure). _log("Retrying with skopeo (insecure registry) ...\n") - # skopeo copy --dest-tls-verify=false docker-daemon:TAG docker://TAG cmd = [ skopeo, "copy", @@ -1345,12 +1432,12 @@ def _push_docker_image(image_tag: str, log_fn=None) -> bool: f"docker-daemon:{image_tag}", f"docker://{image_tag}", ] - res2 = subprocess.run(cmd, capture_output=True, text=True) - if res2.returncode == 0: + res3 = subprocess.run(cmd, capture_output=True, text=True) + if res3.returncode == 0: _log(f"[OK] Pushed {image_tag} using skopeo\n") return True _log( - f"[ERROR] Skopeo push failed: {res2.stderr.strip() if res2.stderr else 'unknown error'}\n" + f"[ERROR] Skopeo push failed: {res3.stderr.strip() if res3.stderr else 'unknown error'}\n" ) else: _log("[ERROR] skopeo not found; cannot retry push.\n") diff --git a/installer/core/repair_pipeline_cli.py b/installer/core/repair_pipeline_cli.py index d18bdfd..fcb1f15 100644 --- a/installer/core/repair_pipeline_cli.py +++ b/installer/core/repair_pipeline_cli.py @@ -7,7 +7,7 @@ from pathlib import Path from installer import prole_conf from installer.core.controller import ProleController -from installer.core.actions import ProleSilentInstaller +from installer.core.actions import ProleConsoleInstaller def _project_root() -> Path: @@ -81,7 +81,7 @@ def main(argv: list[str] | None = None) -> int: cfg = str(prole_conf.entrypoint_path(prole_conf.resolve_prole_conf_dir(root))) controller = ProleController(project_root=root, verbose=bool(args.verbose), cfg_path=Path(cfg)) - installer = ProleSilentInstaller(controller, cfg_path=cfg) + installer = ProleConsoleInstaller(controller, cfg_path=cfg) try: existing_inputs = installer._load_inputs_from_cfg() diff --git a/installer/ui/screens/__init__.py b/installer/ui/screens/__init__.py index ce5de72..67d3da4 100644 --- a/installer/ui/screens/__init__.py +++ b/installer/ui/screens/__init__.py @@ -72,8 +72,8 @@ if platform.system() == "Darwin": from installer.core.env import * # noqa: F401,F403 from installer.core.actions import ( - ProleInstallerBase, - ProleSilentInstaller, + ProleInstaller, + ProleConsoleInstaller, _prepare_k3s_pipeline, _reset_k3s_namespace, ) @@ -98,6 +98,7 @@ from installer.ui.screens.security import SecurityScreenMixin from installer.ui.screens.ollama import OllamaScreenMixin from installer.ui.screens.supabase import SupabaseScreenMixin from installer.ui.screens.gitops import GitOpsScreenMixin +from installer.ui.screens.argocd import ArgoCDScreenMixin from installer.ui.screens.docker import DockerScreenMixin from installer.ui.screens.build import BuildScreenMixin from installer.ui.screens.packaging import PackagingScreenMixin @@ -110,7 +111,7 @@ from installer.ui.screens.cfg import ConfigMixin # Composed installer class # --------------------------------------------------------------------------- class ProleInstaller( - ProleInstallerBase, + ProleInstaller, ScreenBaseMixin, NavigationMixin, WelcomeScreenMixin, @@ -125,6 +126,7 @@ class ProleInstaller( OllamaScreenMixin, SupabaseScreenMixin, GitOpsScreenMixin, + ArgoCDScreenMixin, DockerScreenMixin, BuildScreenMixin, PackagingScreenMixin, @@ -152,6 +154,7 @@ class ProleInstaller( "init_cluster.kerberos_enabled": "kerberos_enabled", "init_cluster.supabase_enabled": "supabase_enabled", "init_cluster.gitops_enabled": "gitops_enabled", + "init_cluster.argocd_enabled": "argocd_enabled", "init_cluster.at_rest_encryption_enabled": "at_rest_encryption_enabled", "init_cluster.selected_kubectx": "selected_kubectx", "init_cluster.k3s_server_url": "k3s_server_url", @@ -160,6 +163,7 @@ class ProleInstaller( "ollama_config.server_port": "ollama_server_port", "ollama_config.model": "ollama_model", "gitops.namespace": "gitops_namespace", + "argocd.namespace": "argocd_namespace", } def _get_input(self, key: str, default: str | None = None) -> str: @@ -353,6 +357,7 @@ class ProleInstaller( ("Initialization Scripts", "init_scripts"), ("Kerberos Authentication", "kerberos_config"), ("GitOps", "gitops_config"), + ("ArgoCD", "argocd_config"), ("Supabase", "supabase_config"), ("Ollama", "ollama_config"), ("Deployment", "init_cnpg_deploy"), @@ -430,6 +435,10 @@ class ProleInstaller( # Must exist before reading prole.cfg so saved values are applied (avoid silent fallbacks). self.cluster_env = tk.StringVar(value="dev") self.service_namespace = tk.StringVar(value="default") + self.gitops_enabled = tk.BooleanVar(value=False) + self.gitops_namespace = tk.StringVar(value="gitea") + self.argocd_enabled = tk.BooleanVar(value=False) + self.argocd_namespace = tk.StringVar(value="argocd") # Load existing docker_import_dir from prole.cfg try: @@ -460,6 +469,9 @@ class ProleInstaller( saved_service_ns = _expand_cfg_value( cfg["Global"].get("SERVICE_NAMESPACE", ""), cfg_vars ).strip() + saved_argocd_ns = _expand_cfg_value( + cfg["Global"].get("ARGOCD_NAMESPACE", ""), cfg_vars + ).strip() saved_gitops_ns = ( _expand_cfg_value( cfg.get("GitOps", "GITOPS_NAMESPACE", fallback=""), @@ -468,6 +480,18 @@ class ProleInstaller( if cfg.has_section("GitOps") else "" ) + saved_argocd_enabled = ( + _expand_cfg_value( + cfg.get( + "Optional Features", + "ARGOCD_ENABLED", + fallback="", + ), + cfg_vars, + ).strip() + if cfg.has_section("Optional Features") + else "" + ) saved_gitops_enabled = ( _expand_cfg_value( cfg.get( @@ -510,6 +534,14 @@ class ProleInstaller( self.prole_cfg_data["Global"][ "SERVICE_NAMESPACE" ] = saved_service_ns + if saved_argocd_ns: + try: + self.argocd_namespace.set(saved_argocd_ns) + except Exception: + pass + self.prole_cfg_data.setdefault("Global", {})[ + "ARGOCD_NAMESPACE" + ] = saved_argocd_ns if saved_gitops_ns: try: self.gitops_namespace.set(saved_gitops_ns) @@ -518,6 +550,15 @@ class ProleInstaller( self.prole_cfg_data.setdefault("GitOps", {})[ "NAMESPACE" ] = saved_gitops_ns + if saved_argocd_enabled: + val = saved_argocd_enabled.lower() in ("true", "1", "yes") + try: + self.argocd_enabled.set(val) + except Exception: + pass + self.prole_cfg_data.setdefault("Optional Features", {})[ + "ARGOCD_ENABLED" + ] = str(val) if saved_gitops_enabled: val = saved_gitops_enabled.lower() in ("true", "1", "yes") try: @@ -623,9 +664,7 @@ class ProleInstaller( self.kerberos_password = tk.StringVar() self.kerberos_kdc = tk.StringVar() self.supabase_enabled = tk.BooleanVar(value=False) - self.gitops_enabled = tk.BooleanVar(value=False) self.at_rest_encryption_enabled = tk.BooleanVar(value=True) - self.gitops_namespace = tk.StringVar(value="gitea") self.ollama_server_host = tk.StringVar() self.ollama_server_port = tk.StringVar(value=DEFAULT_OLLAMA_PORT) self.ollama_model = tk.StringVar() @@ -676,6 +715,7 @@ class ProleInstaller( self._register_canvas_renderer( "supabase_config", self._render_supabase_config_page ) + self._register_canvas_renderer("argocd_config", self._render_argocd_config_page) self._register_canvas_renderer("gitops_config", self._render_gitops_config_page) self._register_canvas_renderer("init_cluster", self._render_init_cluster_page) self._register_canvas_renderer("init_db_build", self._render_init_db_build_page) @@ -707,6 +747,7 @@ class ProleInstaller( self._register_page("init_db_build", None) self._register_page("init_scripts", None) self._register_page("kerberos_config", None) + self._register_page("argocd_config", None) self._register_page("gitops_config", None) self._register_page("supabase_config", None) self._register_page("ollama_config", None) @@ -825,7 +866,7 @@ def main(): if args.silent: _ensure_ansible_vault_credentials(prompt_ui=False) - installer = ProleSilentInstaller( + installer = ProleConsoleInstaller( controller, args.config, reset_cluster=args.reset ) if log_file_path: diff --git a/installer/ui/screens/argocd.py b/installer/ui/screens/argocd.py new file mode 100644 index 0000000..e860918 --- /dev/null +++ b/installer/ui/screens/argocd.py @@ -0,0 +1,276 @@ +"""Optional ArgoCD deployment screen. + +This screen intentionally mirrors the layout and behavior of the GitOps (Gitea) +optional feature screen. +""" + +from __future__ import annotations + +import os +import subprocess +import threading +import tkinter as tk + +from installer import screen as ui +from installer.core.env import PROJECT_ROOT, _normalize_cluster_env + + +class ArgoCDScreenMixin: + def _render_argocd_config_page(self): + """Optional feature screen: ArgoCD.""" + self.clear_overlay_widgets() + self._clear_canvas_items() + + ui.create_section_header( + self, "Optional Feature: ArgoCD", "Deploy ArgoCD into your cluster" + ) + + # Enable toggle + enabled_check = tk.Checkbutton( + self.bg_canvas, + text="Enable ArgoCD", + variable=self.argocd_enabled, + bg="white", + fg="black", + activebackground="white", + activeforeground="black", + selectcolor="white", + command=self._on_argocd_toggle, + font=("SF Pro Text", 12, "bold"), + ) + enabled_window = self.bg_canvas.create_window( + 48, 180, window=enabled_check, anchor="nw" + ) + self._canvas_items.append(enabled_window) + self._overlay_widgets.append(enabled_check) + + # Namespace entry + ui.canvas_text( + self, + 48, + 290, + "Kubernetes Namespace", + fill="black", + font=("SF Pro Text", 12, "bold"), + ) + ns_entry = tk.Entry( + self.bg_canvas, + textvariable=self.argocd_namespace, + bg="white", + fg="black", + insertbackground="black", + highlightbackground="#CCCCCC", + highlightthickness=1, + relief="flat", + font=("SF Pro Text", 11), + ) + ns_window = self.bg_canvas.create_window( + 48, 315, window=ns_entry, anchor="nw", width=220, height=30 + ) + self._canvas_items.append(ns_window) + self._overlay_widgets.append(ns_entry) + + y = 360 + initial_status = "Ready" if self.argocd_enabled.get() else "Disabled" + self._argocd_status_var = tk.StringVar(value=initial_status) + status_item = ui.canvas_text( + self, + 48, + y, + f"Status: {initial_status}", + fill="black", + font=("SF Pro Text", 12), + ) + + def update_status_text(*_): + try: + self.bg_canvas.itemconfig( + status_item, text=f"Status: {self._argocd_status_var.get()}" + ) + except Exception: + pass + + self._argocd_status_var.trace_add("write", update_status_text) + + # Console output + self._argocd_console = self._create_console_output( + y=400, title="Deployment Output", width=880, height=330 + ) + + # Deploy button + self._argocd_deploy_button = tk.Button( + self.bg_canvas, + text="Deploy ArgoCD", + command=self.run_argocd_deploy, + bg="#F5F5DC", + fg="black", + activebackground="#E5E5D5", + activeforeground="black", + highlightbackground="#F5F5DC", + highlightcolor="#F5F5DC", + highlightthickness=0, + relief="flat", + bd=0, + cursor="hand2", + disabledforeground="#8B8B7A", + font=("SF Pro Text", 11), + padx=20, + pady=10, + ) + btn_window = self.bg_canvas.create_window( + 48, 760, window=self._argocd_deploy_button, anchor="nw" + ) + self._overlay_widgets.append(self._argocd_deploy_button) + self._canvas_items.append(btn_window) + + def run_argocd_deploy(self): + if getattr(self, "_argocd_deploying", False): + return + if not self.argocd_enabled.get(): + self._argocd_status_var.set("Disabled") + self.update_footer() + return + + self._argocd_deploying = True + self._argocd_deploy_button.configure(state="disabled") + self._argocd_status_var.set("Deploying...") + self._argocd_success = False + self._argocd_console.clear() + self.update_footer() + + def worker(): + namespace = ( + (self.argocd_namespace.get() or "").strip() + or os.environ.get("ARGOCD_NAMESPACE") + or "argocd" + ) + env = os.environ.copy() + env["PROLE_HOME"] = str(PROJECT_ROOT) + env["PROLE_SERVICE"] = str(PROJECT_ROOT) + env["NAMESPACE"] = namespace + env["ARGOCD_NAMESPACE"] = namespace + + self._argocd_console.write("Starting ArgoCD deployment...\n") + + script_path = PROJECT_ROOT / "etc" / "init_argocd.sh" + if not script_path.exists(): + self._argocd_console.write(f"Error: {script_path} not found.\n") + self.safe_after( + lambda: self._argocd_status_var.set("Failed (Script not found)") + ) + self.safe_after( + lambda: self._argocd_deploy_button.configure(state="normal") + ) + self._argocd_deploying = False + return + + mode = (os.environ.get("PROLE_MODE") or "").strip() + if not mode: + cluster_env = _normalize_cluster_env(self.cluster_env.get()) + if cluster_env == "dev": + mode = "k3d" + elif cluster_env in ("service", "prod"): + mode = "k3s" if cluster_env == "service" else "k8s" + else: + mode = "k3d" + args = ["--mode", mode, "--namespace", namespace, "update"] + + cfg_path = None + try: + if self._cfg_path_override and self._cfg_path_override.exists(): + cfg_path = self._cfg_path_override + else: + conf_dir = self._resolve_prole_conf_dir() + candidate = conf_dir / "prole.cfg" + if candidate.exists(): + cfg_path = candidate + if not cfg_path: + from installer import prole_conf + + candidate = prole_conf.entrypoint_path( + prole_conf.resolve_prole_conf_dir(PROJECT_ROOT) + ) + if candidate.exists(): + cfg_path = candidate + except Exception: + cfg_path = None + if cfg_path: + args.extend(["-c", str(cfg_path)]) + + self._argocd_console.write(f"Mode: {mode}\n") + cmd = ["bash", str(script_path)] + args + + try: + proc = subprocess.Popen( + cmd, + cwd=str(PROJECT_ROOT), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + except Exception as e: + self._argocd_console.write(f"Failed to start deploy: {e}\n") + self.safe_after( + lambda: self._argocd_status_var.set("Failed (Launch error)") + ) + self.safe_after( + lambda: self._argocd_deploy_button.configure(state="normal") + ) + self._argocd_deploying = False + return + + if proc.stdout: + for line in iter(proc.stdout.readline, ""): + if line: + self._argocd_console.write(line) + proc.stdout.close() + rc = proc.wait() + + if rc == 0: + self._argocd_success = True + self.safe_after( + lambda: self._argocd_status_var.set("Deployed Successfully") + ) + self._argocd_console.write( + "\nArgoCD deployment completed successfully.\n" + ) + else: + self._argocd_success = False + self.safe_after( + lambda: self._argocd_status_var.set(f"Failed (Code {rc})") + ) + self._argocd_console.write( + f"\nArgoCD deployment failed with exit code {rc}.\n" + ) + + self._argocd_deploying = False + self.safe_after( + lambda: self._argocd_deploy_button.configure(state="normal") + ) + self.safe_after(self.update_footer) + + threading.Thread(target=worker, daemon=True).start() + + def _on_argocd_toggle(self): + enabled = self.argocd_enabled.get() + try: + self.prole_cfg_data["Optional Features"]["ARGOCD_ENABLED"] = str(enabled) + # Keep compatibility with other parts of the installer that expect Global.ARGOCD_NAMESPACE. + self.prole_cfg_data.setdefault("Global", {})["ARGOCD_NAMESPACE"] = ( + (self.argocd_namespace.get() or "").strip() or "argocd" + ) + self._save_prole_cfg() + except Exception: + pass + + if not enabled: + self._argocd_success = False + if hasattr(self, "_argocd_status_var"): + self._argocd_status_var.set("Disabled") + try: + self._argocd_console.write("ArgoCD disabled. Deployment skipped.\n") + except Exception: + pass + self.update_footer() diff --git a/installer/ui/screens/base.py b/installer/ui/screens/base.py index ff91704..c363634 100644 --- a/installer/ui/screens/base.py +++ b/installer/ui/screens/base.py @@ -159,6 +159,7 @@ class ScreenBaseMixin: "supabase_config", "ollama_config", "gitops_config", + "argocd_config", "database_options", "create_installer", ) diff --git a/installer/ui/screens/cfg.py b/installer/ui/screens/cfg.py index 53ed3cb..1d5e399 100644 --- a/installer/ui/screens/cfg.py +++ b/installer/ui/screens/cfg.py @@ -34,6 +34,10 @@ from installer.config import _collect_cfg_vars, _encrypt_cfg_secret, _expand_cfg class ConfigMixin: """prole.cfg persistence, input snapshot collection and port-forward management.""" + # Namespace typing can trigger many trace events; debounce propagation to avoid + # incremental substring rewrites across unrelated fields. + NAMESPACE_PROPAGATE_DEBOUNCE_MS = 250 + def _save_prole_cfg(self): """Generates prole.cfg; master configuration file containing all values used in install process.""" try: @@ -330,6 +334,9 @@ class ConfigMixin: "init_cluster.supabase_enabled", _get_var(self.supabase_enabled, False) ) _set_bool("init_cluster.gitops_enabled", _get_var(self.gitops_enabled, False)) + _set_bool( + "init_cluster.argocd_enabled", _get_var(self.argocd_enabled, False) + ) _set_bool( "init_cluster.kerberos_enabled", _get_var(self.kerberos_enabled, False) ) @@ -399,6 +406,9 @@ class ConfigMixin: # GitOps _set("gitops.namespace", _get_var(self.gitops_namespace, "gitea")) + # ArgoCD + _set("argocd.namespace", _get_var(self.argocd_namespace, "argocd")) + # Disk selection (installer packaging) _set("disk_selection.disk_type", _get_var(self.selected_disk_type, "local")) _set( @@ -915,21 +925,91 @@ class ConfigMixin: self.k3s_token.set(token_val) def _propagate_namespace_change(self, *args): - """Reacts to namespace change by updating all dependent inputs and config files.""" + """Reacts to namespace change by updating dependent inputs and config files. + + This is triggered by a `tk.StringVar` trace while the user types. We debounce + the propagation so we only apply a single stable transition from the last + synced namespace to the final value (instead of rewriting on each keystroke). + """ + try: new_ns = (self.db_namespace.get() or "").strip() except Exception: return - if not new_ns or new_ns == getattr(self, "_last_synced_ns", None): + + if not new_ns: return - old_ns = getattr(self, "_last_synced_ns", "") - self._last_synced_ns = new_ns + # If there is no Tk root (e.g. some test harnesses), fall back to immediate. + root = getattr(self, "root", None) + if root is None: + self._apply_pending_namespace_change(new_ns) + return + + # Debounce: cancel any in-flight scheduled propagation and reschedule. + after_id = getattr(self, "_namespace_propagate_after_id", None) + if after_id: + try: + root.after_cancel(after_id) + except Exception: + pass + + self._pending_namespace_value = new_ns + delay_ms = getattr(self, "_namespace_debounce_ms", None) + if delay_ms is None: + delay_ms = getattr(self, "NAMESPACE_PROPAGATE_DEBOUNCE_MS", 250) + + try: + self._namespace_propagate_after_id = root.after( + int(delay_ms), self._apply_pending_namespace_change + ) + except Exception: + # If scheduling fails, apply immediately. + self._apply_pending_namespace_change(new_ns) + + def _apply_pending_namespace_change(self, namespace: str | None = None): + """Apply the pending namespace propagation using a stable old namespace.""" + + # Clear the timer id if we were invoked from `after`. + try: + self._namespace_propagate_after_id = None + except Exception: + pass + + if namespace is None: + try: + namespace = (getattr(self, "_pending_namespace_value", "") or "").strip() + except Exception: + namespace = "" + + new_ns = (namespace or "").strip() + if not new_ns: + return + + old_ns = (getattr(self, "_last_synced_ns", "") or "").strip() + if not old_ns: + # Initialize sync baseline and persist. + try: + self._last_synced_ns = new_ns + except Exception: + pass + if new_ns == old_ns: + return # Propagate to inputs and StringVars (e.g. OpenBao paths) if old_ns: + # Do not rewrite independent namespaces. + exclude_var_names = { + "db_namespace", + "service_namespace", + "gitops_namespace", + "argocd_namespace", + } + # 1. Sweep StringVars for attr_name in dir(self): + if attr_name in exclude_var_names: + continue try: attr = getattr(self, attr_name) if isinstance(attr, tk.StringVar) and attr != self.db_namespace: @@ -942,10 +1022,18 @@ class ConfigMixin: # 2. Sweep inputs if hasattr(self, "inputs"): for k, v in list(self.inputs.items()): - if isinstance(v, str) and old_ns in v: - self.inputs[k] = v.replace(old_ns, new_ns) + if not isinstance(v, str) or old_ns not in v: + continue + lk = (k or "").lower() + if "service_namespace" in lk or lk.endswith("service_namespace"): + continue + self.inputs[k] = v.replace(old_ns, new_ns) # Update environment and save prole.cfg + try: + self._last_synced_ns = new_ns + except Exception: + pass try: self._update_env_namespace(new_ns) # Re-save prole.cfg so it's always in sync diff --git a/installer/ui/screens/cluster.py b/installer/ui/screens/cluster.py index 5a70e8a..e084b6d 100644 --- a/installer/ui/screens/cluster.py +++ b/installer/ui/screens/cluster.py @@ -388,7 +388,7 @@ class ClusterScreenMixin: # Traffic light status indicators for each service component self._service_traffic_lights = {} components = [ - "Argo", + "Registry", "CertMgr", "Garage", "Kong", @@ -538,7 +538,13 @@ class ClusterScreenMixin: return key def _get_service_namespace(self) -> str: - """Resolve service namespace with UI/env/config fallbacks.""" + """Resolve service namespace with UI/env/config fallbacks. + + Safety rule: avoid defaulting to the Kubernetes `default` namespace unless we truly + cannot infer a better option. + """ + + # 1) Explicit user/config/env choice wins. try: ns = (self.service_namespace.get() or "").strip() except Exception: @@ -554,17 +560,126 @@ class ClusterScreenMixin: ns = "" if not ns: ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip() + if ns: + return ns - default_ns = "default" + # 2) If the cluster is reachable, infer the common services namespace. + inferred = "" + try: + inferred = (self._infer_common_services_namespace() or "").strip() + except Exception: + inferred = "" + if inferred: + return inferred + + # 3) Stable defaults for known environments. try: - # Common services should never fall back to the Kubernetes "default" namespace. # For local dev (k3d) and service (k3s), keep knoe-system stable across restarts. if self._cluster_env_key() in ("dev", "service"): - default_ns = "knoe-system" + return "knoe-system" except Exception: pass - return ns or default_ns + # 4) Absolute last resort. + return "default" + + def _infer_common_services_namespace(self) -> str: + """Infer the namespace hosting common services. + + If the cluster is reachable, try to locate an existing Docker Registry workload + (image containing `registry:2`). If found, treat that namespace as the common + services namespace. + + This method is read-only. + """ + + # Cache to avoid repeated `kubectl -A` probes during periodic refresh. + cache_ttl_s = 60 + now = time.time() + cached_ns = getattr(self, "_common_services_ns_cache", "") + cached_at = getattr(self, "_common_services_ns_cache_at", 0.0) + if cached_ns and (now - float(cached_at or 0.0)) < cache_ttl_s: + return cached_ns + + # Ensure kubeconfig is available when possible (read-only action). + try: + self._ensure_k3s_kubeconfig_merged() + except Exception: + pass + + # Build a kubectl command suitable for the active mode. + try: + mode = self._deployment_mode() + except Exception: + mode = None + try: + base_cmd = self._kubectl_base_cmd(mode) + except Exception: + base_cmd = ["kubectl"] + + # Respect an existing kubeconfig if present. + env = os.environ.copy() + try: + kc = _find_kubeconfig_file(env) + except Exception: + kc = _find_kubeconfig_file() + if kc: + env["KUBECONFIG"] = kc + + def _probe(kind: str) -> list[str]: + cmd = base_cmd + ["get", kind, "-A", "-o", "json"] + res = subprocess.run( + cmd, + capture_output=True, + text=True, + env=env, + timeout=5, + ) + if res.returncode != 0: + return [] + try: + data = json.loads(res.stdout or "{}") + except Exception: + return [] + out = [] + for item in (data.get("items") or []): + try: + ns = (item.get("metadata") or {}).get("namespace") or "" + containers = ( + (((item.get("spec") or {}).get("template") or {}).get("spec") or {}).get( + "containers" + ) + or [] + ) + for c in containers: + img = (c.get("image") or "").strip() + if "registry:2" in img: + if ns: + out.append(ns) + break + except Exception: + continue + return out + + namespaces = [] + # Deployments are the most common shape; fall back to StatefulSets. + namespaces.extend(_probe("deploy")) + if not namespaces: + namespaces.extend(_probe("sts")) + + # Prefer knoe-system if present, otherwise prefer a non-default namespace. + ns_choice = "" + if namespaces: + if "knoe-system" in namespaces: + ns_choice = "knoe-system" + else: + non_default = [n for n in namespaces if n and n != "default"] + ns_choice = (non_default[0] if non_default else namespaces[0]) or "" + + if ns_choice: + setattr(self, "_common_services_ns_cache", ns_choice) + setattr(self, "_common_services_ns_cache_at", now) + return ns_choice def _on_cluster_env_change(self, *args): # Keep configuration environment selection explicit and stable. @@ -1478,6 +1593,13 @@ class ClusterScreenMixin: threading.Thread(target=_cleanup_worker, daemon=True).start() except Exception: pass + + # On save, immediately run a services status refresh so the UI reflects the + # newly persisted namespace/cluster settings. + try: + self._verify_k3s_services() + except Exception: + pass return True def _cluster_status_snapshot(self) -> dict: @@ -1544,54 +1666,27 @@ class ClusterScreenMixin: kong_status = None if status.get("cluster_ok"): kong_status = self._dashboard_kong_status(status.get("env")) - self._set_cluster_env_message("Assess Actions", "#34c759") + self._set_cluster_env_message("Observe", "#34c759") - # Repair step: reset Dashboard (Kong) pods if unhealthy, with cooldown - if kong_status and kong_status.get("reset_pods"): - now = time.time() - if now - self._kong_last_repair_at >= self._kong_repair_cooldown_s: - self._set_cluster_env_message( - "Execute Actions (REPAIR): reset Dashboard (Kong) pod(s)", - "#34c759", - ) - pods = kong_status.get("reset_pods") or [] - cmd = ( - self._kubectl_base_cmd(status.get("env")) - + ["delete", "pod", "-n", "kubernetes-dashboard"] - + pods - ) - rc, _out = self._run_cmd_capture(cmd, timeout=10) - if rc != 0: - self._set_cluster_env_message( - "Notice: Dashboard (Kong) repair failed", - "#ff3b30", - clear_after_ms=6000, - ) - notice_active = True - else: - self._kong_last_repair_at = now - # Give the cluster a moment to recreate pods before re-check - time.sleep(2) - self._set_cluster_env_message("Collect Status", "#34c759") - status = self._cluster_status_snapshot() - else: - self._set_cluster_env_message( - "Execute Actions (REPAIR): skipped (recently attempted)", - "#34c759", - clear_after_ms=1500, - ) - else: - self._set_cluster_env_message( - "Execute Actions (REPAIR): none needed", - "#34c759", - clear_after_ms=1500, - ) - - # Run the repair pipeline if anomalies are detected on a ready cluster + # Observe-only: do not mutate the cluster automatically. + anomalies = [] try: - self._maybe_run_repair_pipeline(status, kong_status) + if kong_status and not kong_status.get("ok", True): + anomalies.append("dashboard") except Exception: pass + try: + if self._authority_context_missing(): + anomalies.append("authority") + except Exception: + pass + if anomalies: + self._set_cluster_env_message( + "Notice: issues detected. Click Repair to reconcile.", + "#ff9f0a", + clear_after_ms=6000, + ) + notice_active = True def update_ui(): if hasattr(self, "k3d_status_label"): @@ -1870,18 +1965,12 @@ class ClusterScreenMixin: def _parse_service_status_lines(self, lines: list[str]) -> dict[str, bool]: """Parse [OK]/[FAIL] lines from status_common_services.sh output. - Returns a dict mapping component keys (argo, certmgr, + Returns a dict mapping component keys (registry, certmgr, garage, kong, openbao, opentofu) to True (healthy) or False (unhealthy). """ # Map resource names to traffic light component keys name_map = { - "argocd-server": "argo", - "argocd-repo-server": "argo", - "argocd-dex-server": "argo", - "argocd-applicationset-controller": "argo", - "argocd-notifications-controller": "argo", - "argocd-redis": "argo", - "argocd-application-controller": "argo", + "registry": "registry", "opentofu": "opentofu", "garage": "garage", "openbao": "openbao", @@ -1910,7 +1999,7 @@ class ClusterScreenMixin: return result def _deploy_k3s_services(self): - """Deploy common services (ArgoCD/OpenTofu/Garage) to remote k3s cluster.""" + """Deploy common services (Registry/OpenTofu/Garage) to remote k3s cluster.""" def _deploy(): console = self._k3s_service_deploy_console diff --git a/installer/ui/screens/database.py b/installer/ui/screens/database.py index 9ee984e..8c4b248 100644 --- a/installer/ui/screens/database.py +++ b/installer/ui/screens/database.py @@ -8,13 +8,14 @@ import subprocess import threading from pathlib import Path import tkinter as tk -from tkinter import ttk, messagebox, filedialog +from tkinter import ttk, messagebox, filedialog, simpledialog from installer import screen as ui from installer.config import get_docker_build_platform_args from installer.core.env import ( PROJECT_ROOT, _bool_str, _deployment_mode_from_env, + _http_ping_registry, _safe_str, _push_docker_image, get_resource_path, @@ -942,17 +943,43 @@ class DatabaseScreenMixin: "Building the prole-db Postgres image. This may take a few minutes.", y=200 ) - # Local registry status (checked async) + # Registry status (checked async) self._db_registry_status_label = ui.canvas_text( self, 48, 230, - "Local Registry: Checking...", + "Registry: Checking...", fill="#6e6e73", font=("SF Pro Text", 11), ) self._canvas_items.append(self._db_registry_status_label) + # k3s registry initialization (shown only when needed) + self._db_init_registry_button = tk.Button( + self.bg_canvas, + text="Initialize Registry", + command=self._db_init_registry, + bg="#F5F5DC", + fg="black", + activebackground="#E5E5D5", + highlightbackground="#F5F5DC", + highlightthickness=0, + relief="flat", + font=("SF Pro Text", 11), + padx=16, + pady=6, + ) + init_btn_window = self.bg_canvas.create_window( + 48, 224, window=self._db_init_registry_button, anchor="nw", width=200 + ) + self._db_init_registry_button_canvas_window = init_btn_window + self._canvas_items.append(init_btn_window) + self._overlay_widgets.append(self._db_init_registry_button) + try: + self.bg_canvas.itemconfigure(init_btn_window, state="hidden") + except Exception: + pass + # Output Console self._db_build_console = self._create_console_output( y=260, title="Build Output", width=900, height=520 @@ -991,43 +1018,112 @@ class DatabaseScreenMixin: def _ensure_db_build_registry_async(self): def worker(): try: - if not self.check_docker_running(): + env_key = "dev" + try: + env_key = self._cluster_env_key() + except Exception: + env_key = "dev" + + # If the user selected Service, treat it as k3s even when Global/DEPLOYMENT_MODE remains k3d. + mode = _deployment_mode_from_env( + ((self.prole_cfg_data.get("Global", {}) or {}).get("DEPLOYMENT_MODE") or "").strip() + or env_key + ) + + def _show_init_button(show: bool): + try: + if not hasattr(self, "_db_init_registry_button_canvas_window"): + return + state = "normal" if show else "hidden" + self.bg_canvas.itemconfigure( + self._db_init_registry_button_canvas_window, state=state + ) + except Exception: + return + + # k3s: never warn about localhost registry. Instead offer init + show reachable status. + if mode == "k3s": + registry_url = "" + try: + registry_url = (self.ensure_registry_available(env_key) or "").strip() + except Exception: + registry_url = "" + + reachable = False + if registry_url and ":" in registry_url: + host, port_s = registry_url.rsplit(":", 1) + try: + reachable = _http_ping_registry(host.strip(), int(port_s.strip())) + except Exception: + reachable = False + + if registry_url and reachable: + self.safe_after( + lambda: ( + _show_init_button(False), + self.bg_canvas.itemconfig( + self._db_registry_status_label, + text=f"Registry: {registry_url}", + fill="#34c759", + ), + ) + ) + return + + # Not reachable / not configured -> prompt action + msg = ( + f"Registry: {registry_url} (not initialized)" + if registry_url + else "Registry: not initialized" + ) self.safe_after( lambda: ( + _show_init_button(True), self.bg_canvas.itemconfig( self._db_registry_status_label, - text="Local Registry: Docker not running", - fill="#ff3b30", - ) - if self.bg_canvas.winfo_exists() - else None + text=msg, + fill="#ff9500", + ), ) ) return + + # Non-k3s: keep local registry checks (requires Docker) + if not self.check_docker_running(): + self.safe_after( + lambda: ( + _show_init_button(False), + self.bg_canvas.itemconfig( + self._db_registry_status_label, + text="Registry: Docker not running", + fill="#ff3b30", + ), + ) + ) + return + info = self.ensure_local_registry_available() if info: host_registry, _cluster_registry = info self.safe_after( lambda: ( + _show_init_button(False), self.bg_canvas.itemconfig( self._db_registry_status_label, - text=f"Local Registry: {host_registry}", + text=f"Registry: {host_registry}", fill="#34c759", - ) - if self.bg_canvas.winfo_exists() - else None + ), ) ) else: self.safe_after( lambda: ( + _show_init_button(False), self.bg_canvas.itemconfig( self._db_registry_status_label, - text="Local Registry: unavailable", + text="Registry: unavailable", fill="#ff3b30", - ) - if self.bg_canvas.winfo_exists() - else None + ), ) ) log_path = getattr(self, "_last_registry_log_path", None) @@ -1038,7 +1134,7 @@ class DatabaseScreenMixin: ) ) except Exception as e: - msg = f"Local Registry: error ({e})" + msg = f"Registry: error ({e})" self.safe_after( lambda: ( self.bg_canvas.itemconfig( @@ -1058,6 +1154,181 @@ class DatabaseScreenMixin: threading.Thread(target=worker, daemon=True).start() + def _db_rescan_registry_status_after_init(self): + """Re-scan registry availability after init. + + Registry pods/services may take a few seconds to become reachable after + `etc/init_registry.sh` finishes. Schedule a few checks so the UI status + flips to green once ready. + """ + + try: + if getattr(self, "_db_registry_status_label", None) is not None: + self.safe_after( + lambda: self.bg_canvas.itemconfig( + self._db_registry_status_label, + text="Registry: Checking...", + fill="#6e6e73", + ) + ) + except Exception: + pass + + # Progressive delays to allow the in-cluster registry to become ready. + for d in (0, 1000, 2000, 4000, 8000, 15000): + self.safe_after(lambda: self._ensure_db_build_registry_async(), delay=d) + + def _db_init_registry(self): + """Initialize the k3s in-cluster registry (registry:2) via etc/init_registry.sh.""" + + def _resolve_argocd_namespace() -> str: + ns = ( + (self.prole_cfg_data.get("Global", {}) or {}) + .get("ARGOCD_NAMESPACE", "") + .strip() + ) + if not ns: + ns = (os.environ.get("ARGOCD_NAMESPACE") or "").strip() + return ns or "argocd" + + def _resolve_service_namespace_interactive() -> str: + # Prefer explicit UI value if available + ns = "" + try: + ns = (self.service_namespace.get() or "").strip() + except Exception: + ns = "" + if not ns: + ns = ( + (self.prole_cfg_data.get("Global", {}) or {}) + .get("SERVICE_NAMESPACE", "") + .strip() + ) + if not ns: + ns = (os.environ.get("SERVICE_NAMESPACE") or "").strip() + + if ns: + return ns + + # UI is interactive: ask the user. + try: + ns2 = simpledialog.askstring( + "Service Namespace", + "Enter the service namespace for the k3s registry:", + parent=self.root, + ) + except Exception: + ns2 = None + return (ns2 or "").strip() + + def worker(): + env_key = "service" + try: + env_key = self._cluster_env_key() + except Exception: + env_key = "service" + + mode = _deployment_mode_from_env(env_key) + if mode != "k3s": + try: + messagebox.showinfo( + "Registry", + "Registry initialization is only required for k3s (Service) mode.", + ) + except Exception: + pass + return + + service_ns = _resolve_service_namespace_interactive() + if not service_ns: + # In UI mode we must not guess; prompt is available above. + # Only fall back to Kubernetes 'default' when no input is available. + try: + if getattr(self, "root", None) is not None: + self.safe_after( + lambda: messagebox.showerror( + "Registry", + "Service namespace is required to initialize the registry.", + ) + ) + return + except Exception: + pass + service_ns = "default" + + argocd_ns = _resolve_argocd_namespace() + + script = PROJECT_ROOT / "etc" / "init_registry.sh" + cfg_path = getattr(self.controller, "cfg_path", None) + cmd = [str(script)] + if cfg_path: + cmd += ["--config", str(cfg_path)] + cmd += [ + "--mode", + "k3s", + "-n", + argocd_ns, + "--registry-namespace", + service_ns, + "update", + ] + + self.safe_after( + lambda: ( + self._db_init_registry_button.configure(state="disabled") + if self._db_init_registry_button.winfo_exists() + else None + ) + ) + if self._db_build_console: + self.safe_after( + lambda: self._db_build_console.write( + f"Running registry init: {' '.join(cmd)}\n\n" + ) + ) + + try: + proc = subprocess.Popen( + cmd, + cwd=str(PROJECT_ROOT), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=os.environ.copy(), + ) + if proc.stdout: + for line in iter(proc.stdout.readline, ""): + if not line: + break + if self._db_build_console: + self._db_build_console.write(line) + proc.wait() + rc = proc.returncode + except Exception as e: + rc = 1 + if self._db_build_console: + self._db_build_console.write(f"Registry init failed: {e}\n") + + if self._db_build_console: + self._db_build_console.write( + f"\nRegistry init {'completed' if rc == 0 else 'failed'} (exit {rc}).\n" + ) + + self.safe_after( + lambda: ( + self._db_init_registry_button.configure(state="normal") + if self._db_init_registry_button.winfo_exists() + else None + ) + ) + if rc == 0: + self._db_rescan_registry_status_after_init() + else: + self._ensure_db_build_registry_async() + + threading.Thread(target=worker, daemon=True).start() + def run_db_build(self): self._action_flags["init_db_build.run_build"] = True @@ -1289,14 +1560,96 @@ class DatabaseScreenMixin: ) ) else: + # Service/prod clusters must pull from a registry; do not skip push. + registry_url = "" + try: + registry_url = (self.ensure_registry_available(env_key) or "").strip() + except Exception: + registry_url = "" + + if not registry_url: + self._db_build_console.write( + "No registry configured/resolved for this environment. Initialize the registry first.\n" + ) + self.safe_after( + lambda: ( + self.bg_canvas.itemconfig( + self._db_build_status_label, + text="Build ok, registry not configured.", + fill="#ff3b30", + ) + if self.bg_canvas.winfo_exists() + and self._db_build_status_label in self.bg_canvas.find_all() + else None + ) + ) + return + + remote_tag = f"{registry_url}/prole-db:{tag}" self._db_build_console.write( - "Import skipped for non-dev clusters.\n" + f"Tagging image for registry: {remote_tag}\n" ) + tag_res = subprocess.run( + ["docker", "tag", image_name, remote_tag], + capture_output=True, + text=True, + ) + if tag_res.returncode != 0: + self._db_build_console.write(tag_res.stdout or "") + self._db_build_console.write(tag_res.stderr or "") + self._db_build_console.write("Tag failed; cannot push.\n") + self.safe_after( + lambda: ( + self.bg_canvas.itemconfig( + self._db_build_status_label, + text="Build ok, tag failed.", + fill="#ff3b30", + ) + if self.bg_canvas.winfo_exists() + and self._db_build_status_label in self.bg_canvas.find_all() + else None + ) + ) + return + self.safe_after( lambda: ( self.bg_canvas.itemconfig( self._db_build_status_label, - text="Build complete. Import skipped.", + text="Pushing to registry...", + fill="#34c759", + ) + if self.bg_canvas.winfo_exists() + and self._db_build_status_label in self.bg_canvas.find_all() + else None + ) + ) + if not _push_docker_image( + remote_tag, log_fn=self._db_build_console.write + ): + self._db_build_console.write( + "Push failed. Ensure the registry is initialized and reachable.\n" + ) + self.safe_after( + lambda: ( + self.bg_canvas.itemconfig( + self._db_build_status_label, + text="Build ok, push failed.", + fill="#ff3b30", + ) + if self.bg_canvas.winfo_exists() + and self._db_build_status_label in self.bg_canvas.find_all() + else None + ) + ) + return + + self._db_build_console.write(f"Push complete: {remote_tag}\n") + self.safe_after( + lambda: ( + self.bg_canvas.itemconfig( + self._db_build_status_label, + text="Build, tag and push complete.", fill="#34c759", ) if self.bg_canvas.winfo_exists() diff --git a/installer/ui/screens/docker.py b/installer/ui/screens/docker.py index 6f4ab7e..586c0b2 100644 --- a/installer/ui/screens/docker.py +++ b/installer/ui/screens/docker.py @@ -176,9 +176,27 @@ class DockerScreenMixin: registry_container = f"k3d-{reg_name}" host_registry = "localhost:5000" - mode = _deployment_mode_from_env( - self.prole_cfg_data.get("Global", {}).get("DEPLOYMENT_MODE") - ) + # Prefer the currently selected environment (UI) over the global config. + # This avoids incorrectly treating Service (k3s) as Dev (k3d) when + # Global/DEPLOYMENT_MODE is still set to k3d. + mode_source = "" + try: + mode_source = (self._cluster_env_key() or "").strip() + except Exception: + mode_source = "" + if not mode_source: + try: + mode_source = ( + (self.prole_cfg_data.get("Global", {}) or {}) + .get("DEPLOYMENT_MODE", "") + .strip() + ) + except Exception: + mode_source = "" + if not mode_source: + mode_source = (mode_hint or "").strip() + + mode = _deployment_mode_from_env(mode_source) if mode == "k3s": k3s_registry = self._k3s_registry_hostport() if not k3s_registry: @@ -344,134 +362,6 @@ class DockerScreenMixin: log_fp.close() return None - def _collect_dependent_images( - self, include_supabase: bool, include_kerberos_proxy: bool - ) -> list[str]: - images = set() - for rel_dir in ("k8s/prole", "k8s/openbao"): - base_dir = PROJECT_ROOT / rel_dir - if not base_dir.exists(): - continue - images.update(_collect_images_from_files(list(base_dir.glob("*.yaml")))) - - if include_supabase: - supa_home = _resolve_supabase_home(PROJECT_ROOT) - if supa_home: - docker_dir = supa_home / "docker" - compose_files = [docker_dir / "docker-compose.yml"] - if os.environ.get("SUPABASE_USE_DEV_COMPOSE") == "1": - compose_files.append(docker_dir / "dev" / "docker-compose.dev.yml") - images.update(_collect_images_from_files(compose_files)) - - if not include_supabase: - images = {img for img in images if "supabase" not in img} - - if include_kerberos_proxy: - krb_img = os.environ.get("KRB5_AD_PROXY_IMAGE", "alpine/socat") - if krb_img: - images.add(krb_img) - - return sorted(images) - - def _prepull_images_to_registry( - self, include_supabase: bool, include_kerberos_proxy: bool, log=None - ) -> bool: - def _log(msg: str): - if log: - try: - log(msg) - except Exception: - pass - - info = self.ensure_local_registry_available() - if not info: - _log("Local registry unavailable; skipping image pre-pull.\n") - return False - - registry, _cluster_registry = info - images = self._collect_dependent_images( - include_supabase, include_kerberos_proxy - ) - if not images: - _log("No dependent images found to pre-pull.\n") - return True - - overall_ok = True - import_dir = self.docker_import_dir.get().strip() - - for image in images: - local_tag = image - if not image.startswith(f"{registry}/"): - local_tag = f"{registry}/{image}" - - # 1. Check if already in local registry - _log(f"Checking if {image} exists in local registry...\n") - check_reg = subprocess.run( - ["docker", "pull", local_tag], capture_output=True, text=True - ) - if check_reg.returncode == 0: - _log(f"[OK] {image} already exists in local registry as {local_tag}\n") - continue - - # 2. Check if we already have it in local docker daemon - check_local = subprocess.run( - ["docker", "image", "inspect", image], capture_output=True, text=True - ) - found = check_local.returncode == 0 - - if not found and import_dir and os.path.isdir(import_dir): - # 3. Check import directory - safe_name = image.replace("/", "_").replace(":", "_") - tar_path = Path(import_dir) / f"{safe_name}.tar" - if tar_path.exists(): - _log(f"Found {tar_path} in import directory, loading...\n") - load = subprocess.run( - ["docker", "load", "-i", str(tar_path)], - capture_output=True, - text=True, - ) - if load.returncode == 0: - found = True - else: - _log(f"[WARN] Failed to load {tar_path}: {load.stderr}\n") - - if not found: - # 4. Pull from Docker Hub - _log(f"Pulling {image} from Docker Hub...\n") - pull = subprocess.run( - ["docker", "pull", image], capture_output=True, text=True - ) - if pull.returncode != 0: - overall_ok = False - _log(pull.stdout or "") - _log(pull.stderr or "") - _log(f"[ERROR] docker pull failed for {image}\n") - continue - found = True - - # If we have the image locally, tag and push to local registry - if found: - if local_tag != image: - tag = subprocess.run( - ["docker", "tag", image, local_tag], - capture_output=True, - text=True, - ) - if tag.returncode != 0: - overall_ok = False - _log(tag.stdout or "") - _log(tag.stderr or "") - _log(f"[ERROR] docker tag failed for {image} to {local_tag}\n") - continue - - _log(f"Pushing {local_tag} to local registry...\n") - if not _push_docker_image(local_tag, log_fn=log): - overall_ok = False - continue - _log(f"[OK] Stored {image} in local registry as {local_tag}\n") - - return overall_ok - def build_docker_image(self): """Build prole-db Docker image""" # If Kerberos is enabled, update pg_hba.conf in conf/postgresql before copying diff --git a/installer/ui/screens/navigation.py b/installer/ui/screens/navigation.py index bf627e8..e64d87e 100644 --- a/installer/ui/screens/navigation.py +++ b/installer/ui/screens/navigation.py @@ -339,9 +339,12 @@ class NavigationMixin: if current_id == "kerberos_config": self.show_page("init_scripts") return - if current_id == "gitops_config": + if current_id == "argocd_config": self.show_page("kerberos_config") return + if current_id == "gitops_config": + self.show_page("argocd_config") + return if current_id == "supabase_config": self.show_page("gitops_config") return @@ -527,7 +530,7 @@ class NavigationMixin: if self.kerberos_enabled.get(): self.show_page("kerberos_config") else: - self.show_page("gitops_config") + self.show_page("argocd_config") return if current_id == "kerberos_config": @@ -569,6 +572,29 @@ class NavigationMixin: os.environ.get("KRB5_AD_SERVICE_NAME", "prole-kerberos-ad-dc") ) self._save_prole_cfg() + self.show_page("argocd_config") + return + + if current_id == "argocd_config": + self.prole_cfg_data["Optional Features"]["ARGOCD_ENABLED"] = str( + self.argocd_enabled.get() + ) + argocd_status = ( + "Skipped" + if not self.argocd_enabled.get() + else ( + "Deployed" + if getattr(self, "_argocd_success", False) + else "Attempted" + ) + ) + argocd_ns = (self.argocd_namespace.get() or "argocd").strip() or "argocd" + self.prole_cfg_data.setdefault("Global", {})["ARGOCD_NAMESPACE"] = argocd_ns + self.prole_cfg_data["ArgoCD"] = { + "STATUS": argocd_status, + "ARGOCD_NAMESPACE": argocd_ns, + } + self._save_prole_cfg() self.show_page("gitops_config") return @@ -682,6 +708,14 @@ class NavigationMixin: self.next_button.configure(state="disabled") else: self.next_button.configure(state="normal") + elif pid == "argocd_config": + if self.argocd_enabled.get(): + if getattr(self, "_argocd_success", False): + self.next_button.configure(state="normal") + else: + self.next_button.configure(state="disabled") + else: + self.next_button.configure(state="normal") elif pid == "init_scripts": # Initialization Scripts: Next is disabled until success if getattr(self, "_scripts_success", False): diff --git a/k8s/prole/prole-db-recovery.yaml.tpl b/k8s/prole/prole-db-recovery.yaml.tpl index ceac291..56ac626 100644 --- a/k8s/prole/prole-db-recovery.yaml.tpl +++ b/k8s/prole/prole-db-recovery.yaml.tpl @@ -5,7 +5,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: myrddin.prole.org:5000/prole-db:17.7-106 + imageName: myrddin.prole.org:5000/prole-db:18-118 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 diff --git a/k8s/prole/prole-db.yaml b/k8s/prole/prole-db.yaml index 2834486..d356250 100644 --- a/k8s/prole/prole-db.yaml +++ b/k8s/prole/prole-db.yaml @@ -4,7 +4,7 @@ metadata: name: prole-db spec: instances: 3 - imageName: myrddin.prole.org:5000/prole-db:17.7-106 + imageName: myrddin.prole.org:5000/prole-db:18-118 postgresUID: 100 postgresGID: 101 maxSyncReplicas: 1 diff --git a/mock_val/init_cloudnative_pg.sh b/mock_val/init_cloudnative_pg.sh index eefe677..f5c400d 100755 --- a/mock_val/init_cloudnative_pg.sh +++ b/mock_val/init_cloudnative_pg.sh @@ -1,1324 +1,3 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# init_cloudnative_pg.sh -# Purpose: -# - Distribute administrator ed25519 key pair to CloudNative‑PG as a Kubernetes Secret for cert auth -# - Patch CNPG cluster to enable TLS where possible -# -# Usage: -# ./init_cloudnative_pg.sh start|stop|status|restart -# ./init_cloudnative_pg.sh initialize # install CNPG, create cluster, configure secrets -# ./init_cloudnative_pg.sh update|reload # re-apply/patch -# ./init_cloudnative_pg.sh deploy [version] # apply CNPG manifest and update image -# ./init_cloudnative_pg.sh rollout # rolling restart of CNPG pods -# -# Requirements: -# - init_openbao.sh has been run (OpenBao running as a local container) -# - $PROLE_HOME/env.sh or $HOME/.prole/env.sh defining PROLE_SERVICE -# - Optional: CNPG_MANIFEST_OVERRIDE to apply a recovery manifest instead of kustomize - -# Initialize SCRIPT_DIR -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) - -# Load environment and config via prole_cfg.sh -# shellcheck disable=SC1090 -source "$SCRIPT_DIR/prole_cfg.sh" - -if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then - prole_set_mode "${2:-}" - shift 2 -elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then - prole_set_mode "${1#*=}" - shift -fi - -if [[ -z "${PROLE_SERVICE:-}" ]]; then - echo "ERROR: PROLE_SERVICE is not defined. Provide PROLE_HOME/env.sh or ~/.prole/env.sh" >&2 - exit 1 -fi - -ACTION=${1:-} -CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-prole-db} -VERSION=${2:-latest} -OPENBAO_NAME=${OPENBAO_NAME:-openbao} -REALM=${REALM:-PROLE.ORG} -DOMAIN=${DOMAIN:-prole.org} -CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-} -PROLE_HOME=${PROLE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} -BACKUP_DIR=${BACKUP_DIR:-$PROLE_HOME/prole/backup} -BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800} -RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/prole/prole-db-recovery.yaml.tpl" -BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-} -BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0} -CERT_MANAGER_MANIFEST_URL=${CERT_MANAGER_MANIFEST_URL:-} -CERT_MANAGER_FALLBACK_VERSION=${CERT_MANAGER_FALLBACK_VERSION:-1.19.3} - -if [[ "$ACTION" != "deploy" && "$ACTION" != "rollout" && "$ACTION" != "force-rollout" ]]; then - if [[ -n "${2:-}" ]]; then - CNPG_CLUSTER_NAME="${2}" - fi - VERSION=${3:-latest} -fi - -# Support both PROLE_HOME/k8s and sibling k8s directory -if [[ -d "$SCRIPT_DIR/../k8s/prole" ]]; then - K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole" -elif [[ -n "${PROLE_HOME:-}" && -d "$PROLE_HOME/k8s/prole" ]]; then - K8S_PROLE_DIR="$PROLE_HOME/k8s/prole" -else - K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/prole" -fi -CNPG_MANIFEST="$K8S_PROLE_DIR/prole-db.yaml" -BARMAN_OBJECTSTORE_MANIFEST="$K8S_PROLE_DIR/prole-db-barman-objectstore.yaml" - -SECRETS_DIR="$PROLE_SERVICE/secrets" -# Resolving CNPG admin keys. -# We prefer names without algorithm suffixes to be more generic, matching install.py fallback strategy. -ADMIN_PRIV_ED25519="$SECRETS_DIR/admin_ed25519.key" -ADMIN_PUB_ED25519="$SECRETS_DIR/admin_ed25519.pub" -ADMIN_PRIV_GENERIC="$SECRETS_DIR/admin.key" -ADMIN_PUB_GENERIC="$SECRETS_DIR/admin.pub" -OPENBAO_TOKEN_FILE="$SECRETS_DIR/openbao-root-token" -# NAMESPACE is always derived from prole.cfg via PROLE_NAMESPACE — never from the environment -NAMESPACE="${PROLE_NAMESPACE}" -BAO_NAMESPACE="${PROLE_NAMESPACE}" -OPENBAO_NAMESPACE="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${PROLE_NAMESPACE}}}" -BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}" -BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin" -BAO_PATH_DB="${BAO_PATH_PREFIX}/db" -BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring" - -ensure_tools() { - for t in kubectl curl openssl base64 jq; do - command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; } - done -} - -ensure_namespace() { - if [[ -z "${PROLE_NAMESPACE:-}" ]]; then - echo "ERROR: PROLE_NAMESPACE is empty. Set NAMESPACE in conf/prole.cfg." >&2 - exit 1 - fi - if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then - echo "Creating namespace '$NAMESPACE' ..." - kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 || true - fi -} - -get_latest_image() { - local pg_version_file release_file pg_version release - if [[ -f "$SCRIPT_DIR/../conf/postgresql/.version" ]]; then - pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" - elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/conf/postgresql/.version" ]]; then - pg_version_file="$PROLE_HOME/conf/postgresql/.version" - else - pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" - fi - - if [[ -f "$SCRIPT_DIR/../prole-db/.version" ]]; then - release_file="$SCRIPT_DIR/../prole-db/.version" - elif [[ -n "${PROLE_HOME:-}" && -f "$PROLE_HOME/prole-db/.version" ]]; then - release_file="$PROLE_HOME/prole-db/.version" - else - release_file="$SCRIPT_DIR/../prole-db/.version" - fi - - if [[ -f "$pg_version_file" ]]; then - pg_version=$(tr -d '[:space:]' < "$pg_version_file") - else - pg_version="17.7" - fi - - if [[ -f "$release_file" ]]; then - release=$(tr -d '[:space:]' < "$release_file") - else - release="43" - fi - - if [[ "$release" =~ ^[0-9]+$ ]]; then - release=$(printf "%03d" "$release") - fi - - echo "prole-db:${pg_version}-${release}" -} - -resolve_cnpg_image() { - local image="$1" - if _prole_local_registry_enabled; then - # Prefer the in-cluster/internal registry for k3d/k3s nodes pulling images. - # Using LOCAL_REGISTRY (e.g., localhost:5000) breaks inside cluster and may - # also prefer IPv6 ::1, leading to connection refused. Avoid it. - local registry="" - if [[ -n "${LOCAL_REGISTRY_INTERNAL:-}" ]]; then - registry="${LOCAL_REGISTRY_INTERNAL}" - elif [[ -n "${LOCAL_REGISTRY:-}" ]]; then - # Only fall back to LOCAL_REGISTRY when INTERNAL is not available - # and it's not pointing at localhost (which is invalid for cluster pulls). - if [[ "${LOCAL_REGISTRY}" != "localhost:5000" && "${LOCAL_REGISTRY}" != "127.0.0.1:5000" ]]; then - registry="${LOCAL_REGISTRY}" - fi - fi - - if [[ -n "$registry" ]]; then - local first="${image%%/*}" - # If the image is unqualified (no registry), prefix it with the chosen registry - if [[ "$image" != */* ]]; then - image="${registry}/${image}" - # If the first path segment has no dot/colon, it's still unqualified (e.g., prole-db:TAG) - elif [[ "$first" != *"."* && "$first" != *":"* ]]; then - image="${registry}/${image}" - fi - fi - fi - printf '%s' "$image" -} - -sync_manifest_image() { - local image="$1" - local files=() - if [[ -n "$CNPG_MANIFEST" ]]; then - files+=("$CNPG_MANIFEST") - fi - if [[ -f "$RECOVERY_TEMPLATE" ]]; then - files+=("$RECOVERY_TEMPLATE") - fi - local f tmp - for f in "${files[@]}"; do - if [[ -f "$f" ]] && grep -qE '^[[:space:]]*imageName:' "$f"; then - tmp=$(mktemp) - sed -E "s|^([[:space:]]*imageName:).*|\\1 ${image}|" "$f" > "$tmp" - mv "$tmp" "$f" - fi - done -} - -openbao_url() { - if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then - echo "$PROLE_OPENBAO_URL" - return 0 - fi - if prole_is_in_cluster; then - echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200" - return 0 - fi - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - elif curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - else - echo "" - return 0 - fi -} - -openbao_token() { - if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then - cat "$OPENBAO_TOKEN_FILE" - else - echo "${OPENBAO_ROOT_TOKEN:-}" - fi -} - -fetch_openbao_secret() { - local path="$1" - local key="$2" - local token url - token=$(openbao_token) - url=$(openbao_url) - if [[ -z "$token" || -z "$url" ]]; then - echo "" - return 0 - fi - curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo "" -} - -resolve_db_password() { - local db_pw="${DB_PASSWORD:-}" - if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then - local fetched_db - fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password") - if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then - db_pw="$fetched_db" - fi - fi - printf '%s' "$db_pw" -} - - -ensure_cnpg_operator() { - if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then - kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true - wait_for_cnpg_webhook 180 || true - return 0 - fi - - local latest_version minor_version yaml_url - latest_version=$(get_latest_cnpg_version) - minor_version=$(echo "$latest_version" | cut -d. -f1,2) - yaml_url="https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${minor_version}/releases/cnpg-${latest_version}.yaml" - - echo "Installing CloudNative-PG operator version ${latest_version} ..." - kubectl apply --server-side -f "$yaml_url" - - if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then - kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true - wait_for_cnpg_webhook 180 || true - fi -} - -get_latest_barman_plugin_version() { - local version tag - tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "") - if [[ -z "$tag" || "$tag" == "null" ]]; then - echo "v${BARMAN_PLUGIN_FALLBACK_VERSION}" - return 0 - fi - echo "$tag" -} - -resolve_barman_plugin_manifest_url() { - if [[ -n "$BARMAN_PLUGIN_MANIFEST_URL" ]]; then - echo "$BARMAN_PLUGIN_MANIFEST_URL" - return 0 - fi - local tag - tag=$(get_latest_barman_plugin_version) - echo "https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/${tag}/manifest.yaml" -} - -cert_manager_ready() { - if ! kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then - return 1 - fi - if ! kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then - return 1 - fi - return 0 -} - -resolve_control_plane_selector() { - local node="" - node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - if [[ -n "$node" ]]; then - printf 'kubernetes.io/hostname=%s' "$node" - fi -} - -pin_cert_manager() { - local selector="${CERT_MANAGER_NODE_SELECTOR:-}" - if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then - selector="$(resolve_control_plane_selector)" - fi - if [[ -z "$selector" ]]; then - return 0 - fi - local key value - key="${selector%%=*}" - value="${selector#*=}" - if [[ -z "$key" || -z "$value" ]]; then - echo "WARN: CERT_MANAGER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 - return 0 - fi - if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then - echo "WARN: No nodes match CERT_MANAGER_NODE_SELECTOR=${selector}. Skipping pin." >&2 - return 0 - fi - echo "Pinning cert-manager deployments to nodes with ${selector} ..." - for dep in cert-manager cert-manager-webhook cert-manager-cainjector; do - kubectl -n cert-manager patch deployment "$dep" --type merge \ - -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true - done - kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true -} - -pin_barman_cloud() { - local selector="${BARMAN_NODE_SELECTOR:-}" - if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then - selector="$(resolve_control_plane_selector)" - fi - if [[ -z "$selector" ]]; then - return 0 - fi - local key value - key="${selector%%=*}" - value="${selector#*=}" - if [[ -z "$key" || -z "$value" ]]; then - echo "WARN: BARMAN_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 - return 0 - fi - if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then - echo "WARN: No nodes match BARMAN_NODE_SELECTOR=${selector}. Skipping pin." >&2 - return 0 - fi - echo "Pinning barman-cloud deployment to nodes with ${selector} ..." - kubectl -n cnpg-system patch deployment barman-cloud --type merge \ - -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true - kubectl -n cnpg-system rollout restart deploy/barman-cloud >/dev/null 2>&1 || true -} - -get_latest_cert_manager_version() { - local tag - tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cert-manager/cert-manager/releases/latest" | jq -r '.tag_name' || echo "") - if [[ -z "$tag" || "$tag" == "null" ]]; then - echo "v${CERT_MANAGER_FALLBACK_VERSION}" - return 0 - fi - echo "$tag" -} - -resolve_cert_manager_manifest_url() { - if [[ -n "$CERT_MANAGER_MANIFEST_URL" ]]; then - echo "$CERT_MANAGER_MANIFEST_URL" - return 0 - fi - local tag - tag=$(get_latest_cert_manager_version) - echo "https://github.com/cert-manager/cert-manager/releases/download/${tag}/cert-manager.yaml" -} - -ensure_cert_manager() { - if cert_manager_ready; then - pin_cert_manager - if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then - kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true - fi - return 0 - fi - - local cm_url - cm_url=$(resolve_cert_manager_manifest_url) - echo "Installing cert-manager from ${cm_url} ..." - kubectl apply -f "$cm_url" - pin_cert_manager - - if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then - kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true - fi -} - -wait_for_barman_crd() { - local timeout=${1:-120} - local start_time - start_time=$(date +%s) - echo "Waiting for Barman Cloud CRD (timeout: ${timeout}s)..." - while true; do - if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then - echo "Barman Cloud CRD is available." - return 0 - fi - local elapsed=$(( $(date +%s) - start_time )) - if (( elapsed > timeout )); then - echo "Barman Cloud CRD not ready after ${elapsed}s." >&2 - return 1 - fi - if (( elapsed % 30 < 6 && elapsed > 5 )); then - echo " [${elapsed}s/${timeout}s] Still waiting for Barman Cloud CRD..." - fi - sleep 5 - done -} - -wait_for_barman_tls_secrets() { - local timeout=${1:-180} - local start_time - start_time=$(date +%s) - echo "Waiting for Barman Cloud TLS secrets (timeout: ${timeout}s)..." - while true; do - if kubectl -n cnpg-system get secret barman-cloud-client-tls >/dev/null 2>&1 \ - && kubectl -n cnpg-system get secret barman-cloud-server-tls >/dev/null 2>&1; then - echo "Barman Cloud TLS secrets are available." - return 0 - fi - local elapsed=$(( $(date +%s) - start_time )) - if (( elapsed > timeout )); then - echo "Barman Cloud TLS secrets not ready after ${elapsed}s." >&2 - return 1 - fi - if (( elapsed % 30 < 6 && elapsed > 5 )); then - echo " [${elapsed}s/${timeout}s] Still waiting for Barman Cloud TLS secrets..." - fi - sleep 5 - done -} - -ensure_barman_plugin() { - ensure_cert_manager - - local plugin_url - plugin_url=$(resolve_barman_plugin_manifest_url) - echo "Installing Barman Cloud plugin from ${plugin_url} ..." - local apply_out="" - if ! apply_out=$(kubectl apply -f "$plugin_url" 2>&1); then - echo "$apply_out" >&2 - if echo "$apply_out" | grep -qi "webhook.cert-manager.io"; then - echo "WARN: cert-manager webhook error detected; restarting cert-manager components and retrying..." >&2 - kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true - kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s >/dev/null 2>&1 || true - kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s >/dev/null 2>&1 || true - kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s >/dev/null 2>&1 || true - kubectl apply -f "$plugin_url" || true - fi - else - printf '%s\n' "$apply_out" - fi - - if ! wait_for_barman_crd 120; then - echo "WARN: Barman Cloud ObjectStore CRD not ready after install." >&2 - fi - - if ! wait_for_barman_tls_secrets 180; then - echo "WARN: Barman Cloud TLS secrets not ready after install." >&2 - fi - - pin_barman_cloud - - if kubectl -n cnpg-system get deploy barman-cloud >/dev/null 2>&1; then - kubectl -n cnpg-system rollout status deploy/barman-cloud --timeout=180s || true - fi -} - -pin_cnpg_controller() { - local selector="${CNPG_CONTROLLER_NODE_SELECTOR:-}" - if [[ -z "$selector" && "${PROLE_MODE:-}" == "k3s" ]]; then - if kubectl get nodes -l "storage=primary" >/dev/null 2>&1; then - selector="storage=primary" - else - local cp_node="" - cp_node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - if [[ -n "$cp_node" ]]; then - selector="kubernetes.io/hostname=${cp_node}" - fi - fi - fi - if [[ -z "$selector" ]]; then - return 0 - fi - local key value - key="${selector%%=*}" - value="${selector#*=}" - if [[ -z "$key" || -z "$value" ]]; then - echo "WARN: CNPG_CONTROLLER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 - return 0 - fi - if ! kubectl get nodes -l "${key}=${value}" >/dev/null 2>&1; then - echo "WARN: No nodes match CNPG_CONTROLLER_NODE_SELECTOR=${selector}. Skipping pin." >&2 - return 0 - fi - echo "Pinning cnpg-controller-manager to nodes with ${selector} ..." - kubectl -n cnpg-system patch deployment cnpg-controller-manager --type merge \ - -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true -} - -wait_for_cnpg_webhook() { - local timeout=${1:-120} - local start_time - start_time=$(date +%s) - - echo "Waiting for CNPG webhook service endpoints to be ready (timeout: ${timeout}s)..." - while true; do - local endpoints - endpoints=$(kubectl -n cnpg-system get endpoints cnpg-webhook-service -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true) - if [[ -n "$endpoints" ]]; then - echo "CNPG webhook service has endpoints." - return 0 - fi - local elapsed=$(( $(date +%s) - start_time )) - if (( elapsed > timeout )); then - echo "WARN: CNPG webhook endpoints not ready after ${elapsed}s." >&2 - kubectl -n cnpg-system get pods 2>/dev/null >&2 || true - return 1 - fi - if (( elapsed % 30 < 6 && elapsed > 5 )); then - echo " [${elapsed}s/${timeout}s] Waiting for CNPG webhook endpoints..." - kubectl -n cnpg-system get pods --no-headers 2>/dev/null | sed 's/^/ /' || true - fi - sleep 5 - done -} - -apply_barman_objectstore_if_present() { - if [[ ! -f "$BARMAN_OBJECTSTORE_MANIFEST" ]]; then - return 0 - fi - if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - local endpoint - endpoint="http://garage.${SERVICE_NAMESPACE}.svc.cluster.local:3900" - prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" \ - | sed -E "s|^[[:space:]]*endpointURL:.*| endpointURL: ${endpoint}|" \ - | kubectl apply -n "$NAMESPACE" -f - - else - prole_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" | kubectl apply -n "$NAMESPACE" -f - - fi - else - echo "WARN: Barman Cloud ObjectStore CRD not found; skipping $BARMAN_OBJECTSTORE_MANIFEST." - fi -} - -apply_prole_manifest_file() { - local file="$1" - local output="" - if output=$(prole_render_manifest "$file" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then - printf '%s\n' "$output" - return 0 - fi - if [[ "${PROLE_MODE:-}" == "k3d" && "$(basename "$file")" == "garage-statefulset.yaml" ]] \ - && echo "$output" | grep -q "updates to statefulset spec"; then - echo "WARN: Garage StatefulSet immutable in k3d; skipping apply." - return 0 - fi - echo "$output" >&2 - return 1 -} - -# Push image to local registry (via LOCAL_REGISTRY host address) with k3d import fallback. -_push_to_k3d_registry() { - local image="$1" - local cluster_name="$2" - local push_host="${LOCAL_REGISTRY:-localhost:5000}" - local plain_image="${image##*/}" # strip registry prefix, e.g. prole-db:18-088 - - if [[ -n "$push_host" ]]; then - local push_ref="${push_host}/${plain_image}" - docker tag "$image" "$push_ref" 2>/dev/null || true - if docker push "$push_ref" 2>/dev/null; then - echo " ✓ Image pushed to registry at '${push_host}'." - return 0 - fi - echo " WARN: push to '${push_host}' failed; falling back to k3d image import ..." >&2 - fi - - if k3d image import "$image" -c "$cluster_name" 2>/dev/null; then - echo " ✓ Image '$image' imported directly into k3d cluster '$cluster_name'." - return 0 - fi - echo "ERROR: Failed to push or import image '$image'." >&2 - return 1 -} - -# Pre-flight: ensure the prole-db image is available in the k3d cluster before -# the CNPG operator ever tries to pull it, avoiding ErrImagePull backoff loops. -# Steps: containerd cache → Docker daemon (registry tag) → Docker daemon (plain tag) -# → tar import → docker build + push/import. -_ensure_prole_db_image() { - [[ "${PROLE_MODE:-}" != "k3d" ]] && return 0 - - local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}" - local image="" - if [[ -n "$image_override" ]]; then - image="$image_override" - else - if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then - image=$(get_latest_image) - else - image="prole-db:$VERSION" - fi - fi - image=$(resolve_cnpg_image "$image") - - # Auto-detect active k3d cluster name - local cluster_name="${K3D_CLUSTER_NAME:-}" - if [[ -z "$cluster_name" ]]; then - cluster_name=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | head -1) - fi - cluster_name="${cluster_name:-prole-dev-cluster}" - - local prole_db_dir="${PROLE_HOME:-$SCRIPT_DIR/..}/prole-db" - local plain_image="${image##*/}" # e.g. prole-db:18-088 - - echo "Pre-flight: verifying image '$image' is available in k3d cluster '$cluster_name' ..." - - # Step 1: check if already present in k3d containerd with matching digest - local containerd_digest local_digest containerd_sha - containerd_digest=$(docker exec "k3d-${cluster_name}-server-0" \ - ctr images ls -q 2>/dev/null | grep -F "$image" | head -1 || true) - if [[ -n "$containerd_digest" ]]; then - local_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$image" 2>/dev/null \ - | awk -F@ '{print $2}' || true) - containerd_sha=$(docker exec "k3d-${cluster_name}-server-0" \ - ctr images ls 2>/dev/null | grep -F "$image" | awk '{print $3}' | head -1 || true) - if [[ -z "$local_digest" || "$containerd_sha" == "$local_digest" ]]; then - echo " ✓ Image '$image' already in k3d containerd (digest match); no import needed." - return 0 - fi - echo " Image '$image' in containerd but digest mismatch (local: ${local_digest:-unknown}, containerd: ${containerd_sha:-unknown}); re-importing ..." - docker exec "k3d-${cluster_name}-server-0" ctr images rm "$image" 2>/dev/null || true - fi - - # Step 2a: registry-tagged image in local Docker daemon → push + import - if docker image inspect "$image" >/dev/null 2>&1; then - echo " Image '$image' found in Docker daemon; pushing to registry ..." - _push_to_k3d_registry "$image" "$cluster_name" - return $? - fi - - # Step 2b: plain-tagged image in local Docker daemon → tag + push + import - if docker image inspect "$plain_image" >/dev/null 2>&1; then - echo " Plain image '$plain_image' found in Docker daemon; tagging as '$image' and pushing ..." - docker tag "$plain_image" "$image" - _push_to_k3d_registry "$image" "$cluster_name" - return $? - fi - - # Step 3: look for a matching tar in the docker-import directory - local prole_data="${PROLE_DATA:-$HOME/.prole/data}" - local docker_import_dir="${DOCKER_IMPORT_DIR:-${prole_data}/docker-import}" - local name_part="${plain_image%%:*}" # e.g. prole-db - local tag_part="${plain_image##*:}" # e.g. 18-088 - local found_tar="" - if [[ -d "$docker_import_dir" ]]; then - for _t in "$docker_import_dir"/*.tar; do - [[ -f "$_t" ]] || continue - local _bn - _bn=$(basename "$_t" .tar) - if [[ "$_bn" == *"$name_part"* && "$_bn" == *"$tag_part"* ]]; then - found_tar="$_t" - break - fi - done - fi - if [[ -n "$found_tar" ]]; then - echo " Loading tar '$(basename "$found_tar")' into Docker daemon ..." - docker load -i "$found_tar" - docker tag "$plain_image" "$image" 2>/dev/null || true - _push_to_k3d_registry "$image" "$cluster_name" - return $? - fi - - # Step 4: image not found anywhere — build from source then push + import - if [[ ! -f "$prole_db_dir/Dockerfile" ]]; then - echo "ERROR: Dockerfile not found in '$prole_db_dir'; cannot build prole-db image." >&2 - return 1 - fi - echo " Image '$image' not found in k3d, Docker daemon, or docker-import dir." - echo " Building prole-db image from '$prole_db_dir' ..." - if ! docker build -t "$plain_image" "$prole_db_dir"; then - echo "ERROR: docker build failed for image '$plain_image'." >&2 - return 1 - fi - docker tag "$plain_image" "$image" - echo " Build complete. Pushing '$image' to registry ..." - _push_to_k3d_registry "$image" "$cluster_name" - return $? -} - -ensure_prole_stack_resources() { - echo "Applying CloudNative-PG cluster and related resources ..." - local image_override="${CNPG_IMAGE:-${PROLE_DB_IMAGE:-}}" - local image="" - if [[ -n "$image_override" ]]; then - image="$image_override" - else - if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then - image=$(get_latest_image) - else - image="prole-db:$VERSION" - fi - fi - image=$(resolve_cnpg_image "$image") - sync_manifest_image "$image" - if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then - if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then - echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2 - return 1 - fi - local dir file - dir="$K8S_PROLE_DIR" - for file in "$dir"/*.yaml; do - case "$(basename "$file")" in - prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml|ingress.yaml) - continue - ;; - openbao-statefulset.yaml|openbao-service.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - garage-*.yaml|grafana-*.yaml|prometheus-*.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - esac - apply_prole_manifest_file "$file" - done - # Apply ingress.yaml without -n flag so each document targets its own namespace - if [[ -f "$dir/ingress.yaml" ]]; then - # Ensure referenced namespaces exist before applying multi-namespace ingress - for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do - if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then - echo "Creating namespace '$_ing_ns' for ingress resource ..." - kubectl create namespace "$_ing_ns" 2>/dev/null || true - fi - done - prole_render_manifest "$dir/ingress.yaml" | kubectl apply -f - 2>&1 || true - fi - apply_barman_objectstore_if_present - apply_cnpg_cluster_manifest "$CNPG_MANIFEST_OVERRIDE" - else - local dir file - dir="$K8S_PROLE_DIR" - for file in "$dir"/*.yaml; do - case "$(basename "$file")" in - prole-db.yaml|kustomization.yaml|supabase-*.yaml|prole-db-barman-objectstore.yaml|ingress.yaml) - continue - ;; - openbao-statefulset.yaml|openbao-service.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - garage-*.yaml|grafana-*.yaml|prometheus-*.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - esac - apply_prole_manifest_file "$file" - done - # Apply ingress.yaml without -n flag so each document targets its own namespace - if [[ -f "$dir/ingress.yaml" ]]; then - # Ensure referenced namespaces exist before applying multi-namespace ingress - for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do - if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then - echo "Creating namespace '$_ing_ns' for ingress resource ..." - kubectl create namespace "$_ing_ns" 2>/dev/null || true - fi - done - prole_render_manifest "$dir/ingress.yaml" | kubectl apply -f - 2>&1 || true - fi - apply_barman_objectstore_if_present - apply_cnpg_cluster_manifest "$CNPG_MANIFEST" - fi - - # Ensure prole-index-html exists for prole deployment readiness probe - if ! kubectl get configmap prole-index-html -n "$NAMESPACE" >/dev/null 2>&1; then - echo "Creating prole-index-html configmap..." - printf "

Prole

" > /tmp/index.html - kubectl create configmap prole-index-html --from-file=/tmp/index.html -n "$NAMESPACE" - rm /tmp/index.html - fi - - # Ensure prole-nginx-tls exists (self-signed for dev) - if ! kubectl get secret prole-nginx-tls -n "$NAMESPACE" >/dev/null 2>&1; then - echo "Generating self-signed prole-nginx-tls for development..." - openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -keyout /tmp/nginx-tls.key -out /tmp/nginx-tls.crt \ - -subj "/CN=prole.org" >/dev/null 2>&1 - kubectl create secret tls prole-nginx-tls --key /tmp/nginx-tls.key --cert /tmp/nginx-tls.crt -n "$NAMESPACE" - rm /tmp/nginx-tls.key /tmp/nginx-tls.crt - fi -} - -apply_cnpg_cluster_manifest() { - local manifest="$1" - local attempts=${CNPG_APPLY_RETRIES:-6} - local i out - - for ((i=1; i<=attempts; i++)); do - if out=$(prole_render_manifest "$manifest" | kubectl apply -n "$NAMESPACE" -f - 2>&1); then - printf '%s\n' "$out" - return 0 - fi - if echo "$out" | grep -q "cnpg-webhook-service"; then - echo "CNPG webhook not ready yet (attempt $i/$attempts). Retrying..." - sleep 5 - continue - fi - echo "$out" >&2 - return 1 - done - - echo "ERROR: Failed to apply CNPG manifest after $attempts attempts." >&2 - echo "$out" >&2 - return 1 -} - -wait_for_cnpg_pods() { - local timeout=${1:-${CNPG_WAIT_TIMEOUT:-900}} - local start_time - start_time=$(date +%s) - - local target_pods="${CNPG_TARGET_PODS:-}" - if [[ -z "$target_pods" ]]; then - target_pods=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.instances}' 2>/dev/null || true) - fi - if [[ -z "$target_pods" ]]; then - target_pods=3 - fi - - echo "Waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' to be Ready..." - echo " Timeout: ${timeout}s" - local last_feedback=0 - local feedback_interval=30 # print detailed status every 30s - local _image_pull_remediated="" # set after first k3d import attempt to avoid loops - while true; do - local now elapsed - now=$(date +%s) - elapsed=$(( now - start_time )) - - local pods - pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null || true) - - if [[ -n "$pods" ]]; then - # Check for Error or CrashLoopBackOff (terminal failures) - if echo "$pods" | grep -E "Error|CrashLoopBackOff" >/dev/null; then - echo "ERROR: Some CNPG pods are in Error or CrashLoopBackOff state:" >&2 - echo "$pods" | grep -E "Error|CrashLoopBackOff" >&2 - return 1 - fi - - # Check for image pull failures — attempt remediation in k3d mode - if echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >/dev/null; then - if [[ "${PROLE_MODE:-}" == "k3d" ]]; then - if [[ -z "$_image_pull_remediated" ]]; then - _image_pull_remediated=1 - local fail_pod fail_image - fail_pod=$(echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" | awk '{print $1}' | head -1) - # Try containers then initContainers - fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \ - -o jsonpath='{.spec.containers[0].image}' 2>/dev/null || true) - if [[ -z "$fail_image" ]]; then - fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \ - -o jsonpath='{.spec.initContainers[0].image}' 2>/dev/null || true) - fi - echo "WARN: Pod '$fail_pod' cannot pull image '${fail_image:-unknown}' (ErrImagePull/ImagePullBackOff in k3d mode)." >&2 - import_dir="${DOCKER_IMPORT_DIR:-${PROLE_DATA:+${PROLE_DATA}/docker-import}}" - cluster_name="${K3D_CLUSTER_NAME:-}" - if [[ -z "$cluster_name" ]]; then - cluster_name=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | head -1) - fi - cluster_name="${cluster_name:-prole-dev-cluster}" - # Re-run full image ensure logic to push/import the failing image - echo " Attempting image remediation via _ensure_prole_db_image ..." >&2 - _ensure_prole_db_image >&2 || true - echo " Remediation complete; resuming wait ..." >&2 - fi - # Continue the wait loop — do not return 1 - else - echo "ERROR: Image pull failure in non-k3d mode — cannot auto-recover:" >&2 - echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >&2 - return 1 - fi - fi - - # Count Ready pods by name (exclude initdb) - local ready_pods - ready_pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep "True" | wc -l | xargs) - - if [[ "$ready_pods" -ge "$target_pods" ]]; then - echo "All $ready_pods/$target_pods pods are Ready." - # Verify DB connectivity if kubectl-cnpg is available - if kubectl cnpg version >/dev/null 2>&1; then - echo "Verifying database connectivity via 'kubectl cnpg psql'..." - if kubectl cnpg -n "$NAMESPACE" psql "$CNPG_CLUSTER_NAME" -- -c "SELECT 1" >/dev/null 2>&1; then - echo "Database connection verified." - return 0 - else - echo " [${elapsed}s/${timeout}s] Pods are Ready but database connection failed — waiting for database to accept connections..." - fi - else - # Fallback if no cnpg plugin: just return 0 if pods are ready - return 0 - fi - fi - - # Periodic detailed feedback - if (( now - last_feedback >= feedback_interval )); then - last_feedback=$now - echo "" - echo " [${elapsed}s/${timeout}s] Waiting — ${ready_pods:-0}/${target_pods} pods Ready" - echo " Pod status:" - echo "$pods" | sed 's/^/ /' - - # Show cluster status if available - local cluster_phase - cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true) - if [[ -n "$cluster_phase" ]]; then - echo " Cluster phase: ${cluster_phase}" - fi - - # Show recent events (last 3) - local events - events=$(kubectl -n "$NAMESPACE" get events \ - --sort-by='.lastTimestamp' \ - --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \ - -o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \ - --no-headers 2>/dev/null | tail -3 || true) - if [[ -n "$events" ]]; then - echo " Recent cluster events:" - echo "$events" | sed 's/^/ /' - fi - fi - else - # No pods yet — give feedback - if (( now - last_feedback >= feedback_interval )); then - last_feedback=$now - echo " [${elapsed}s/${timeout}s] No CNPG pods found yet for cluster '$CNPG_CLUSTER_NAME' — waiting for operator to create them..." - local cluster_phase - cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true) - if [[ -n "$cluster_phase" ]]; then - echo " Cluster phase: ${cluster_phase}" - fi - # If stuck in "Unable to create required cluster objects", show conditions and events for diagnosis - if [[ "$cluster_phase" == *"Unable to create"* ]]; then - echo " Cluster conditions:" - kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ - -o jsonpath='{range .status.conditions[*]} {.type}: {.status} — {.message}{"\n"}{end}' 2>/dev/null || true - echo " Recent events:" - kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \ - --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \ - -o custom-columns='TYPE:.type,REASON:.reason,MESSAGE:.message' \ - --no-headers 2>/dev/null | tail -5 | sed 's/^/ /' || true - fi - fi - fi - - if (( elapsed > timeout )); then - echo "" - echo "ERROR: Timed out after ${elapsed}s waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE'." >&2 - echo "Final pod status:" >&2 - kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" 2>/dev/null >&2 || true - echo "Recent events:" >&2 - kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \ - --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \ - -o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \ - --no-headers 2>/dev/null | tail -5 >&2 || true - return 1 - fi - sleep 5 - done -} - -cluster_exists() { - kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1 -} - -cluster_has_pods() { - kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null | grep -q . -} - -cluster_has_ready_pods() { - kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" \ - -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Ready")].status=="True")].metadata.name}' 2>/dev/null | grep -q . -} - -latest_backup_name() { - kubectl -n "$NAMESPACE" get backup \ - --sort-by=.metadata.creationTimestamp \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 -} - -latest_completed_backup_name() { - kubectl -n "$NAMESPACE" get backup \ - --sort-by=.metadata.creationTimestamp \ - -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null | \ - awk -F'|' '{p=tolower($2); if (p=="completed" || p=="succeeded") {name=$1}} END {print name}' -} - -wait_for_backup() { - local backup_name="$1" - local start_time now phase phase_lc - start_time=$(date +%s) - - while true; do - phase=$(kubectl -n "$NAMESPACE" get backup "$backup_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) - phase_lc=$(printf '%s' "$phase" | tr '[:upper:]' '[:lower:]') - - case "$phase_lc" in - completed|succeeded) - echo "Backup $backup_name completed." - return 0 - ;; - failed|error) - echo "Backup $backup_name failed (phase=$phase)." >&2 - return 1 - ;; - esac - - now=$(date +%s) - if (( now - start_time > BACKUP_WAIT_TIMEOUT )); then - echo "Timed out waiting for backup $backup_name." >&2 - return 1 - fi - - echo "Waiting for backup $backup_name to complete (phase=${phase:-unknown}) ..." - sleep 10 - done -} - -run_garage_backup() { - if [[ ! -x "$SCRIPT_DIR/init_cnpg_backup.sh" ]]; then - echo "WARN: init_cnpg_backup.sh not found; skipping Garage backup." >&2 - return 1 - fi - - echo "Running Garage backup via init_cnpg_backup.sh ..." - if ! "$SCRIPT_DIR/init_cnpg_backup.sh" start; then - echo "Garage backup script failed." >&2 - return 1 - fi - - sleep 2 - local backup_name - backup_name=$(latest_backup_name) - if [[ -z "$backup_name" ]]; then - echo "No backup resource detected after triggering backup." >&2 - return 1 - fi - - wait_for_backup "$backup_name" -} - -get_primary_pod() { - local primary - primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) - if [[ -z "$primary" ]]; then - primary=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - fi - printf '%s' "$primary" -} - -decode_b64() { - local data="$1" - if [[ -z "$data" ]]; then - return 1 - fi - printf '%s' "$data" | base64 -d 2>/dev/null -} - -resolve_db_credentials() { - local user_b64 pass_b64 user pass - - user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.username}' 2>/dev/null || true) - pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-user -o jsonpath='{.data.password}' 2>/dev/null || true) - user=$(decode_b64 "$user_b64" || true) - pass=$(decode_b64 "$pass_b64" || true) - - if [[ -z "$user" || -z "$pass" ]]; then - user_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.username}' 2>/dev/null || true) - pass_b64=$(kubectl -n "$NAMESPACE" get secret prole-db-superuser -o jsonpath='{.data.password}' 2>/dev/null || true) - user=$(decode_b64 "$user_b64" || true) - pass=$(decode_b64 "$pass_b64" || true) - fi - - if [[ -z "$user" || -z "$pass" ]]; then - return 1 - fi - - printf '%s\n%s' "$user" "$pass" -} - -pgdump_local() { - local pod user pass db_name dump_dir dump_file timestamp - pod=$(get_primary_pod) - if [[ -z "$pod" ]]; then - echo "ERROR: No CNPG pod available for pg_dump." >&2 - return 1 - fi - - local creds - if ! creds=$(resolve_db_credentials); then - echo "ERROR: Unable to resolve database credentials for pg_dump." >&2 - return 1 - fi - user=$(printf '%s' "$creds" | sed -n '1p') - pass=$(printf '%s' "$creds" | sed -n '2p') - - db_name=${PROLE_DB_NAME:-prole-db} - dump_dir="$BACKUP_DIR" - mkdir -p "$dump_dir" - - timestamp=$(date +%Y%m%d%H%M%S) - dump_file="$dump_dir/${CNPG_CLUSTER_NAME}-pgdump-${timestamp}.dump" - - echo "Running pg_dump against pod $pod (db=$db_name) ..." - if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- env PGPASSWORD="$pass" \ - pg_dump -U "$user" -d "$db_name" -Fc > "$dump_file"; then - echo "pg_dump saved to $dump_file" - return 0 - fi - - echo "pg_dump failed; removing partial file." >&2 - rm -f "$dump_file" - return 1 -} - -attempt_backup_if_active() { - if ! cluster_exists; then - return 0 - fi - - if ! cluster_has_pods; then - echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no pods detected; skipping backup." >&2 - return 0 - fi - - if ! cluster_has_ready_pods; then - echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no ready pods detected; skipping backup." >&2 - return 0 - fi - - echo "CNPG cluster '$CNPG_CLUSTER_NAME' detected; attempting Garage backup ..." - if run_garage_backup; then - return 0 - fi - - echo "Garage backup failed; attempting local pg_dump ..." >&2 - if pgdump_local; then - return 0 - fi - - echo "WARN: Both Garage backup and local pg_dump failed." >&2 - return 0 -} - -wait_for_pod_ready() { - local pod="$1" - if ! kubectl -n "$NAMESPACE" wait --for=condition=Ready pod "$pod" --timeout=300s >/dev/null 2>&1; then - echo "WARN: pod $pod did not become Ready within timeout." >&2 - return 1 - fi - return 0 -} - -rollout_cluster() { - ensure_tools - ensure_namespace - echo "Starting manual recreate rollout for $CNPG_CLUSTER_NAME in $NAMESPACE..." - - local has_cnpg_plugin="0" - if kubectl cnpg version >/dev/null 2>&1; then - has_cnpg_plugin="1" - kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true - fi - - local primary - primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) - if [[ -z "$primary" ]]; then - echo "ERROR: Could not identify primary instance." >&2 - return 1 - fi - echo "Primary instance: $primary" - - local instances - instances=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true) - if [[ -z "$instances" ]]; then - echo "ERROR: No pods found for cluster $CNPG_CLUSTER_NAME." >&2 - return 1 - fi - - local pod - for pod in $instances; do - if [[ "$pod" != "$primary" ]]; then - echo "Recreating non-primary pod: $pod..." - kubectl delete pod -n "$NAMESPACE" "$pod" - wait_for_pod_ready "$pod" || true - if [[ "$has_cnpg_plugin" == "1" ]]; then - kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true - fi - fi - done - - local new_primary="" - for pod in $instances; do - if [[ "$pod" != "$primary" ]]; then - new_primary="$pod" - break - fi - done - - if [[ -n "$new_primary" && "$has_cnpg_plugin" == "1" ]]; then - echo "Promoting $new_primary..." - kubectl cnpg promote "$CNPG_CLUSTER_NAME" "$new_primary" -n "$NAMESPACE" || true - echo "Waiting for $new_primary to become primary..." - for _ in {1..60}; do - local current_primary - current_primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) - if [[ "$current_primary" == "$new_primary" ]]; then - echo "$new_primary is now the primary." - break - fi - sleep 5 - done - else - if [[ -n "$new_primary" ]]; then - echo "WARN: kubectl cnpg plugin not available; skipping explicit promotion." - fi - fi - - echo "Recreating the old primary pod: $primary..." - kubectl delete pod -n "$NAMESPACE" "$primary" - wait_for_pod_ready "$primary" || true - - if [[ "$has_cnpg_plugin" == "1" ]]; then - echo "Rollout complete. Final cluster status:" - kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true - fi -} - -force_rollout() { - echo "Attempting force rollout of CNPG pods ..." - rollout_cluster -} - -reset_and_reinit() { - local backup_name tmp_manifest - backup_name=$(latest_completed_backup_name) - - echo "Resetting CNPG cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' ..." - kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found - kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true - - if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then - tmp_manifest=$(mktemp) - sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest" - echo "Re-initializing from backup $backup_name ..." - CNPG_MANIFEST_OVERRIDE="$tmp_manifest" ensure_prole_stack_resources - rm -f "$tmp_manifest" - else - if [[ -n "$backup_name" && ! -f "$RECOVERY_TEMPLATE" ]]; then - echo "WARN: Recovery template not found: $RECOVERY_TEMPLATE" >&2 - fi - if [[ -z "$backup_name" ]]; then - echo "No completed backups found; starting fresh initialization." >&2 - fi - ensure_prole_stack_resources - fi - - wait_for_cnpg_pods 300 -} - -# Resolve OpenBao URL: prefer explicit env, then in-cluster service, then localhost port-forward -bao_service_url() { - if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then - echo "$PROLE_OPENBAO_URL" - return 0 - fi - if prole_is_in_cluster; then - echo "http://$OPENBAO_NAME.${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}.svc.cluster.local:8200" - return 0 - fi - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - fi - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - fi - echo "" -} - -fetch_admin_keys_and_db_pass_from_bao_or_local() { local token url if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then token=$(cat "$OPENBAO_TOKEN_FILE") @@ -1389,19 +68,14 @@ fetch_admin_keys_and_db_pass_from_bao_or_local() { return 1 fi + ensure_local_admin_keypair if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then echo "Using local admin key pair at $SECRETS_DIR" return 0 fi - if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then - echo "Using local legacy admin key pair at $SECRETS_DIR" - # Link or copy to generic for consistent use below - cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC" - cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC" - return 0 - fi - echo "ERROR: Could not obtain admin key pair from OpenBao and no local files found." >&2 - exit 1 + + echo "ERROR: Could not obtain admin key pair from OpenBao and could not generate local keys." >&2 + return 1 } apply_cnpg_admin_secret() { diff --git a/mock_val/init_cnpg_backup.sh b/mock_val/init_cnpg_backup.sh index 858ae7d..2b901a4 100755 --- a/mock_val/init_cnpg_backup.sh +++ b/mock_val/init_cnpg_backup.sh @@ -48,6 +48,9 @@ BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600} BACKUP_STATUS_INTERVAL=${BACKUP_STATUS_INTERVAL:-10} PLUGIN_READY_TIMEOUT=${PLUGIN_READY_TIMEOUT:-180} +PLUGIN_SOCKET_DIR=${PLUGIN_SOCKET_DIR:-/plugins} +LAST_BACKUP_NAME="" + usage() { cat </dev/null || true) if printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -Fxq "$BARMAN_PLUGIN_NAME"; then deployment_rows=$(kubectl get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) @@ -257,9 +262,18 @@ wait_for_plugin_ready() { ready=${ready:-0} replicas=${replicas:-0} if (( ready >= 1 && replicas >= 1 )); then - return 0 + plugin_deploy_ready=1 + break fi done <<< "$deployment_rows" + + if (( plugin_deploy_ready == 1 )); then + pod=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + socket_path="$PLUGIN_SOCKET_DIR/$BARMAN_PLUGIN_NAME" + if [[ -n "$pod" ]] && kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- test -S "$socket_path" >/dev/null 2>&1; then + return 0 + fi + fi fi now=$(date +%s) @@ -305,6 +319,21 @@ wait_for_successful_base_backup() { local start_time now elapsed start_time=$(date +%s) while true; do + if [[ -n "$LAST_BACKUP_NAME" ]]; then + local phase err + phase=$(kubectl -n "$NAMESPACE" get backup "$LAST_BACKUP_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true) + err=$(kubectl -n "$NAMESPACE" get backup "$LAST_BACKUP_NAME" -o jsonpath='{.status.error}' 2>/dev/null || true) + case "${phase:-}" in + Completed|Succeeded|completed|succeeded) + return 0 + ;; + Failed|failed) + echo "ERROR: Backup '$LAST_BACKUP_NAME' failed: ${err:-}" >&2 + return 1 + ;; + esac + fi + if has_successful_base_backup; then return 0 fi @@ -324,6 +353,7 @@ trigger_backup() { local backup_type="${1:-full}" local backup_name backup_name="${CNPG_CLUSTER_NAME}-backup-$(date +%Y%m%d%H%M%S)" + LAST_BACKUP_NAME="$backup_name" echo "Triggering ${backup_type} backup $backup_name ..." if [[ "$backup_type" == "incremental" || "$backup_type" == "incr" ]]; then kubectl apply -n "$NAMESPACE" -f - </dev/null || true } +debug_kdc_service_target() { + local host="${KRB5_KDC:-}" + if [[ -z "$host" ]]; then + return 0 + fi + + # Only attempt Kubernetes Service diagnostics for in-cluster service FQDNs. + if [[ "$host" != *".svc.cluster.local" ]]; then + return 0 + fi + + local base svc ns + base="${host%.svc.cluster.local}" + svc="${base%%.*}" + ns="${base#*.}" + ns="${ns%%.*}" + + if [[ -z "$svc" || -z "$ns" || "$svc" == "$base" ]]; then + return 0 + fi + + echo "--- KDC service diagnostics for ${svc}.${ns} ---" >&2 + kubectl -n "$ns" get svc "$svc" -o wide 1>&2 || true + kubectl -n "$ns" get endpoints "$svc" -o yaml 1>&2 || true +} + wait_for_kdc_pod() { local kdc_ns="$1" local attempt pod @@ -242,6 +268,15 @@ run_kinit() { else echo "kinit produced no output." >&2 fi + + echo "--- Debug: kinit failure context ---" >&2 + echo "KRB5_REALM=${KRB5_REALM:-}" >&2 + echo "KRB5_KDC=${KRB5_KDC:-}" >&2 + echo "KRB5_CONFIG=${krb5_config}" >&2 + # Print the relevant realm stanza (best-effort). + kubectl -n "$kdc_ns" exec "$pod_name" -- sh -c "grep -nE '^(\[realms\]|\[libdefaults\]|\[domain_realm\]|[[:space:]]*(kdc|admin_server)[[:space:]]*=)' '$krb5_config' || true" 1>&2 || true + debug_kdc_service_target + return 1 } diff --git a/supabase/deploy.sh b/supabase/deploy.sh index 1bad01f..f1aebdd 100755 --- a/supabase/deploy.sh +++ b/supabase/deploy.sh @@ -831,36 +831,12 @@ PYFIX fi # 9. Fix realtime liveness probe: kompose emits the whole curl command as a - # single quoted string instead of a proper argv list. Wrap it in sh -c. + # single argv entry, and unquoted `Authorization: ...` fragments can be + # parsed as YAML mappings. Sanitize the probe so Kubernetes receives a + # proper list of strings. local realtime_deploy="$SUPABASE_K8S_DIR/realtime-deployment.yaml" if [[ -f "$realtime_deploy" ]]; then - python3 - "$realtime_deploy" <<'PYFIX' -import sys, re - -path = sys.argv[1] -with open(path, 'r') as f: - content = f.read() - -# Match the single-string curl command in the exec probe -m = re.search( - r"(\s+command:\n)(\s+- )'(curl .+?)'", - content, re.DOTALL -) -if m: - indent = m.group(2) - curl_cmd = m.group(3) - replacement = ( - m.group(1) - + indent + "sh\n" - + indent + "-c\n" - + indent + curl_cmd - ) - content = content[:m.start()] + replacement + content[m.end():] - -with open(path, 'w') as f: - f.write(content) -print(" [OK] realtime liveness probe -> sh -c curl ...") -PYFIX + python3 "$SCRIPT_DIR/patch_realtime_probe.py" "$realtime_deploy" fi } @@ -871,7 +847,8 @@ patch_supabase_credentials() { local jwt_secret="" # Resolve postgres password from prole-db-superuser secret - local ns="${PROLE_DB_NAMESPACE:-prole-db-a0001}" + resolve_prole_db_namespace + local ns="${PROLE_DB_NAMESPACE:-${NAMESPACE:-default}}" if kubectl get secret prole-db-superuser -n "$ns" >/dev/null 2>&1; then pg_password=$(kubectl get secret prole-db-superuser -n "$ns" \ -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) @@ -939,7 +916,8 @@ PYFIX setup_prole_db_for_supabase() { log "Ensuring Supabase roles, schemas and databases exist in prole-db..." - local ns="${PROLE_DB_NAMESPACE:-prole-db-a0001}" + resolve_prole_db_namespace + local ns="${PROLE_DB_NAMESPACE:-${NAMESPACE:-default}}" # Find the primary CNPG pod local primary @@ -1100,7 +1078,11 @@ run_helm() { kubectl cluster-info >/dev/null 2>&1 || return 1 if [[ "$FORCE" == "true" ]]; then - kubectl delete namespace supabase --ignore-not-found --wait=false >/dev/null 2>&1 || true + # Helm stores release metadata as Secrets in the namespace; it will fail if + # the namespace is stuck in Terminating. + force_reset_supabase_namespace + else + ensure_supabase_namespace fi helm_render_values @@ -1140,10 +1122,18 @@ PY } ensure_supabase_namespace() { - if ! kubectl get namespace supabase >/dev/null 2>&1; then - log "Creating 'supabase' namespace..." - kubectl create namespace supabase + if kubectl get namespace supabase >/dev/null 2>&1; then + local phase + phase=$(kubectl get namespace supabase -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "${phase:-}" == "Terminating" ]]; then + warn "Namespace 'supabase' is Terminating; resetting it" + force_reset_supabase_namespace + fi + return 0 fi + + log "Creating 'supabase' namespace..." + kubectl create namespace supabase } force_reset_supabase_namespace() { diff --git a/supabase/helm/generated/values.generated.json b/supabase/helm/generated/values.generated.json index 721c979..2c15b0a 100644 --- a/supabase/helm/generated/values.generated.json +++ b/supabase/helm/generated/values.generated.json @@ -29,13 +29,13 @@ }, "secret": { "db": { - "password": "${OPENBAO:kv/prole/prole-db001/db#password}", + "password": "fr3styl3", "database": "postgres" }, "jwt": { "secret": "4e53e1ecb98eb32ee83b890bea79e678e952c6ae237edf3f65f37bf6dd2262eb", - "anonKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiIsImlzcyI6InByb2xlLXN1cGFiYXNlIiwiaWF0IjoxNzcyNDM0NjM4LCJleHAiOjIwODc3OTQ2Mzh9.Wg05U6ioVntunXskmJs3DXMc1i9-O7UBts2cnBwPCoI", - "serviceKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaXNzIjoicHJvbGUtc3VwYWJhc2UiLCJpYXQiOjE3NzI0MzQ2MzgsImV4cCI6MjA4Nzc5NDYzOH0.xZ179fB412UFdgDxc8XUv2EQt-L1nGyhocll09u2u38" + "anonKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiIsImlzcyI6InByb2xlLXN1cGFiYXNlIiwiaWF0IjoxNzczNTM4MjkyLCJleHAiOjIwODg4OTgyOTJ9.PoUrud3YpPeuw_sR1PRgLdFatkqoi6hag8kRERh-bE0", + "serviceKey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaXNzIjoicHJvbGUtc3VwYWJhc2UiLCJpYXQiOjE3NzM1MzgyOTIsImV4cCI6MjA4ODg5ODI5Mn0.rFJn8pV-MhsG1QdqUCVFdppU69aunUh6TPjGzluh5Ic" } }, "ingress": { diff --git a/supabase/patch_realtime_probe.py b/supabase/patch_realtime_probe.py new file mode 100644 index 0000000..4bc2a96 --- /dev/null +++ b/supabase/patch_realtime_probe.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import re +import sys + + +def _quote_yaml_single(s: str) -> str: + # YAML single-quoted scalars escape a single quote by doubling it. + return "'" + s.replace("'", "''") + "'" + + +def patch_realtime_deployment_yaml(content: str) -> tuple[str, bool]: + changed = False + + # Case A: kompose sometimes emits a single-quoted curl command as a single argv entry. + # Wrap it into `sh -c ''`. + def _wrap_single_curl(m: re.Match[str]) -> str: + nonlocal changed + changed = True + cmd_hdr = m.group(1) + indent = m.group(2) + curl_cmd = m.group(3) + return ( + cmd_hdr + + indent + + "sh\n" + + indent + + "-c\n" + + indent + + _quote_yaml_single(curl_cmd) + + "\n" + ) + + content = re.sub( + r"(\s+command:\n)(\s+- )'(curl[^\n]+)'\n", + _wrap_single_curl, + content, + count=1, + ) + + # Case B: unquoted scalars containing `Authorization: ...` can be parsed as YAML mappings. + # Quote them to guarantee Kubernetes receives a string. + out_lines: list[str] = [] + for line in content.splitlines(True): + m = re.match(r"^(\s*-\s+)(curl\b.*)$", line) + if m: + prefix = m.group(1) + cmd = m.group(2).rstrip("\n") + if "Authorization:" in cmd and not cmd.lstrip().startswith(("'", '"')): + out_lines.append(prefix + _quote_yaml_single(cmd) + "\n") + changed = True + continue + out_lines.append(line) + + return "".join(out_lines), changed + + +def main() -> int: + if len(sys.argv) != 2: + print("Usage: patch_realtime_probe.py ", file=sys.stderr) + return 2 + + path = sys.argv[1] + with open(path, "r", encoding="utf-8") as f: + content = f.read() + + patched, changed = patch_realtime_deployment_yaml(content) + if changed: + with open(path, "w", encoding="utf-8") as f: + f.write(patched) + print(" [OK] realtime liveness probe YAML sanitized") + else: + print(" [OK] realtime liveness probe already sane") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/etc/test_init_cloudnative_pg_bootstrap_admin_keys.sh b/tests/etc/test_init_cloudnative_pg_bootstrap_admin_keys.sh new file mode 100644 index 0000000..0dc9837 --- /dev/null +++ b/tests/etc/test_init_cloudnative_pg_bootstrap_admin_keys.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Regression test: init_cloudnative_pg.sh should bootstrap-generate CNPG admin keys +# when OpenBao is unavailable and no local admin key files exist. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) +ETC_DIR="$PROLE_HOME/etc" +SCRIPT_UNDER_TEST="$ETC_DIR/init_cloudnative_pg.sh" + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT +export TMP_DIR + +BIN_DIR="$TMP_DIR/bin" +mkdir -p "$BIN_DIR" + +mock_tool() { + cat <"$BIN_DIR/$1" +#!/usr/bin/env bash +echo "Mocked $1 called with \$@" >>"$TMP_DIR/mock_calls.log" +exit 0 +M_EOF + chmod +x "$BIN_DIR/$1" +} + +# Custom kubectl mock: return minimal outputs so initialize() doesn't hang. +cat <<'K_EOF' >"$BIN_DIR/kubectl" +#!/usr/bin/env bash + +_log_file="${TMP_DIR}/mock_calls.log" +echo "Mocked kubectl called with $@" >>"${_log_file}" + +args="$*" + +# API server readiness probes used by wait_for_apiserver_ready() +if [[ "${args}" == *"get --raw=/readyz"* || "${args}" == *"get --raw='/readyz'"* || "${args}" == *"get --raw=\"/readyz\""* ]]; then + echo "ok" + exit 0 +fi + +if [[ "${args}" == *"version --short"* ]]; then + echo "Client Version: v0.0.0" + echo "Server Version: v0.0.0" + exit 0 +fi + +# Simulate kubectl-cnpg plugin being present, but psql connectivity not yet ready. +if [[ "${args}" == *"cnpg"*"version"* ]]; then + exit 0 +fi +if [[ "${args}" == *"cnpg"*"psql"* ]]; then + exit 1 +fi + +# Pretend required secrets exist (DB user + TLS artifacts) +if [[ "${args}" == *"get secret"*"prole-db-user"* ]]; then + exit 0 +fi +if [[ "${args}" == *"get secret"*"prole-db-tls"* || "${args}" == *"get secret"*"prole-db-ca"* ]]; then + exit 0 +fi + +# CNPG webhook wait: return an endpoint IP so wait passes quickly. +if [[ "${args}" == *"get endpoints"*"cnpg-webhook-service"* ]]; then + echo "10.42.0.10" + exit 0 +fi + +# CNPG pods listing and readiness queries +if [[ "${args}" == *"get pods"* && "${args}" == *"cnpg.io/cluster="* ]]; then + if [[ "${args}" == *"--no-headers"* ]]; then + echo "prole-db-1 1/1 Running 0 1m" + echo "prole-db-2 1/1 Running 0 1m" + echo "prole-db-3 1/1 Running 0 1m" + exit 0 + fi + + if [[ "${args}" == *"-o jsonpath="* ]]; then + printf "True\nTrue\nTrue\n" + exit 0 + fi +fi + +# Cluster instances +if [[ "${args}" == *"get cluster"*"jsonpath="*".spec.instances"* ]]; then + echo "3" + exit 0 +fi + +# For kubectl create secret --dry-run=client -o yaml, emit a minimal YAML so pipes look realistic. +if [[ "${args}" == *"create secret"*"--dry-run=client"*"-o yaml"* ]]; then + echo "apiVersion: v1" + echo "kind: Secret" + exit 0 +fi + +exit 0 +K_EOF +chmod +x "$BIN_DIR/kubectl" + +# Other tool mocks +mock_tool curl +mock_tool docker +mock_tool k3d +mock_tool skopeo +mock_tool jq + +# Mock prole.cfg: point PROLE_SERVICE at a non-writable dir so the script must fallback to HOME +mkdir -p "$TMP_DIR/conf" +mkdir -p "$TMP_DIR/service" +chmod 0500 "$TMP_DIR/service" || true + +TMP_HOME="$TMP_DIR/home" +mkdir -p "$TMP_HOME" + +cat <"$TMP_DIR/conf/prole.cfg" +[User] +NAMESPACE = test-ns +SERVICE_NAMESPACE = test-system +PROLE_HOME = $PROLE_HOME +PROLE_SERVICE = $TMP_DIR/service + +[Global] +DEPLOYMENT_MODE = k3d +KUBECONTEXT = test + +[Docker Build] +LOCAL_REGISTRY = localhost:5000 +LOCAL_REGISTRY_INTERNAL = k3d-prole-registry.localhost:5000 +C_EOF + +export PATH="$BIN_DIR:$PATH" +export HOME="$TMP_HOME" +export PROLE_HOME="$PROLE_HOME" +export PROLE_CONF="$TMP_DIR/conf" +export PROLE_PASSWD="test-password" +export PROLE_SERVICE="$TMP_DIR/service" +export CNPG_WAIT_TIMEOUT=5 + +set +e +bash "$SCRIPT_UNDER_TEST" --mode k3d initialize >"$TMP_DIR/stdout" 2>"$TMP_DIR/stderr" +RC=$? +set -e + +if [[ $RC -ne 0 ]]; then + echo "FAILURE: init_cloudnative_pg.sh initialize returned rc=$RC" + echo "--- stdout ---" + sed -n '1,200p' "$TMP_DIR/stdout" || true + echo "--- stderr ---" + sed -n '1,200p' "$TMP_DIR/stderr" || true + exit 1 +fi + +SECRETS_DIR_FALLBACK="$TMP_HOME/.prole/etc/secrets" +if [[ ! -f "$SECRETS_DIR_FALLBACK/admin.key" || ! -f "$SECRETS_DIR_FALLBACK/admin.pub" ]]; then + echo "FAILURE: expected bootstrap-generated admin.key/admin.pub in fallback secrets dir: $SECRETS_DIR_FALLBACK" + ls -la "$SECRETS_DIR_FALLBACK" 2>/dev/null || true + exit 1 +fi + +if [[ ! -f "$SECRETS_DIR_FALLBACK/admin_ed25519.key" || ! -f "$SECRETS_DIR_FALLBACK/admin_ed25519.pub" ]]; then + echo "FAILURE: expected legacy-mirrored admin_ed25519.key/.pub in fallback secrets dir: $SECRETS_DIR_FALLBACK" + ls -la "$SECRETS_DIR_FALLBACK" 2>/dev/null || true + exit 1 +fi + +echo "SUCCESS" diff --git a/tests/etc/test_init_cnpg_backup_start_waits_for_plugin_socket.sh b/tests/etc/test_init_cnpg_backup_start_waits_for_plugin_socket.sh new file mode 100644 index 0000000..166a8c8 --- /dev/null +++ b/tests/etc/test_init_cnpg_backup_start_waits_for_plugin_socket.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# Unit test for etc/init_cnpg_backup.sh (start action) +# Verifies we do not trigger the first Backup until the plugin socket exists in a CNPG pod. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME_REPO=$(cd "$SCRIPT_DIR/../.." && pwd) +ETC_DIR="$PROLE_HOME_REPO/etc" +SCRIPT_UNDER_TEST="$ETC_DIR/init_cnpg_backup.sh" + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT +export TMP_DIR + +BIN_DIR="$TMP_DIR/bin" +mkdir -p "$BIN_DIR" + +cat <<'K_EOF' >"$BIN_DIR/kubectl" +#!/usr/bin/env bash +set -euo pipefail + +_log_file="${TMP_DIR}/mock_calls.log" +echo "Mocked kubectl called with $*" >>"${_log_file}" + +args="$*" + +# API server readiness probes +if [[ "${args}" == *"get --raw=/readyz"* || "${args}" == *"get --raw='/readyz'"* || "${args}" == *"get --raw=\"/readyz\""* ]]; then + echo "ok" + exit 0 +fi +if [[ "${args}" == *"version --short"* ]]; then + echo "Client Version: v0.0.0" + echo "Server Version: v0.0.0" + exit 0 +fi + +# Namespace existence/creation +if [[ "${args}" == "get namespace test-ns" ]]; then + exit 0 +fi +if [[ "${args}" == "create namespace test-ns" ]]; then + exit 0 +fi + +# Cluster existence and plugin config presence +if [[ "${args}" == "get cluster prole-db -n test-ns"* ]]; then + exit 0 +fi +if [[ "${args}" == *"-n test-ns"*"get cluster prole-db"*"-o jsonpath="*".spec.plugins"*".name"* ]]; then + echo "barman-cloud.cloudnative-pg.io" + exit 0 +fi +if [[ "${args}" == *"-n test-ns"*"get cluster prole-db"*"-o jsonpath="*".spec.plugins"* ]]; then + echo '[{"enabled":true,"isWALArchiver":true,"name":"barman-cloud.cloudnative-pg.io","parameters":{"barmanObjectName":"prole-db-barman-objectstore"}}]' + exit 0 +fi + +# Barman CRD present +if [[ "${args}" == "get crd objectstores.barmancloud.cnpg.io"* ]]; then + exit 0 +fi + +# CNPG operator webhook endpoints ready +if [[ "${args}" == *"-n cnpg-system"*"get endpoints"*"cnpg-webhook-service"* ]]; then + echo "10.42.0.10" + exit 0 +fi + +# Barman-cloud deployment ready +if [[ "${args}" == "get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath="* ]]; then + printf 'cnpg-system\t1\t1\n' + exit 0 +fi + +# CNPG cluster pod selection for socket check +if [[ "${args}" == *"-n test-ns"*"get pods"*"-l cnpg.io/cluster=prole-db"*"-o jsonpath={.items[0].metadata.name"* ]]; then + echo "prole-db-1" + exit 0 +fi + +# Plugin socket check: fail twice, succeed on the third call +if [[ "${args}" == *"-n test-ns"*"exec prole-db-1 -c postgres -- test -S /plugins/barman-cloud.cloudnative-pg.io"* ]]; then + cfile="${TMP_DIR}/socket_checks" + c=0 + [[ -f "$cfile" ]] && c=$(cat "$cfile") + c=$((c + 1)) + echo "$c" >"$cfile" + if (( c >= 3 )); then + exit 0 + fi + exit 1 +fi + +# Garage pod selection +if [[ "${args}" == *"get pods -n test-system -l app=garage"*"-o jsonpath={.items[0].metadata.name"* ]]; then + echo "garage-0" + exit 0 +fi + +# Garage exec commands +if [[ "${args}" == *"exec -n test-system garage-0 -- /garage"*"layout show"* ]]; then + echo "Current cluster layout version: 1" + exit 0 +fi +if [[ "${args}" == *"exec -n test-system garage-0 -- /garage"*"status"* ]]; then + echo "OK" + exit 0 +fi +if [[ "${args}" == *"exec -n test-system garage-0 -- /garage"*"key info"*"--show-secret"* ]]; then + echo "Access key ID: TESTACCESSKEY" + echo "Secret access key: TESTSECRETKEY" + exit 0 +fi +if [[ "${args}" == *"exec -n test-system garage-0 -- /garage"*"bucket info"* ]]; then + exit 0 +fi +if [[ "${args}" == *"exec -n test-system garage-0 -- /garage"*"bucket allow"* ]]; then + exit 0 +fi + +# Capture applied manifests (ObjectStore, Secret, Backup) +if [[ "${args}" == apply*"-n"*"test-ns"*"-f"*"-"* ]]; then + cat >"${TMP_DIR}/applied.$(date +%s%N).yaml" + echo "applied" + exit 0 +fi + +# Simulate the triggered backup reaching Completed quickly +if [[ "${args}" == *"-n test-ns"*"get backup"*"-o jsonpath={.status.phase"* ]]; then + echo "Completed" + exit 0 +fi +if [[ "${args}" == *"-n test-ns"*"get backup"*"-o jsonpath={.status.error"* ]]; then + echo "" + exit 0 +fi +if [[ "${args}" == "get backup -n test-ns -o jsonpath="* ]]; then + # name\tphase\tmethod\tbackupType + printf 'prole-db-backup-XYZ\tCompleted\tplugin\t\n' + exit 0 +fi + +if [[ "${args}" == "get backup -n test-ns"* ]]; then + echo "prole-db-backup-XYZ Completed prole-db plugin" + exit 0 +fi + +exit 0 +K_EOF +chmod +x "$BIN_DIR/kubectl" + +export PATH="$BIN_DIR:$PATH" + +# prole_cfg.sh sources $PROLE_HOME/env.sh if present; provide a minimal env.sh that points to our temp config. +CONF_DIR="$TMP_DIR/conf" +ENV_HOME="$TMP_DIR/env-home" +mkdir -p "$CONF_DIR" "$ENV_HOME" + +cat <"$CONF_DIR/prole.cfg" +[User] +NAMESPACE = test-ns +SERVICE_NAMESPACE = test-system +PROLE_HOME = $PROLE_HOME_REPO +PROLE_SERVICE = $TMP_DIR/service + +[Global] +DEPLOYMENT_MODE = k3d +C_EOF + +cat <"$ENV_HOME/env.sh" +#!/usr/bin/env bash +export PROLE_HOME="$PROLE_HOME_REPO" +export PROLE_CONF="$CONF_DIR" +E_EOF +chmod +x "$ENV_HOME/env.sh" + +set +e +PROLE_HOME="$ENV_HOME" bash "$SCRIPT_UNDER_TEST" start >"$TMP_DIR/stdout" 2>"$TMP_DIR/stderr" +RC=$? +set -e + +if [[ $RC -ne 0 ]]; then + echo "FAILURE: init_cnpg_backup.sh start returned rc=$RC" + sed -n '1,200p' "$TMP_DIR/mock_calls.log" || true + sed -n '1,200p' "$TMP_DIR/stderr" || true + exit 1 +fi + +# Ensure socket checks happened before any Backup apply +socket_line=$(grep -n "exec prole-db-1 -c postgres -- test -S /plugins/barman-cloud.cloudnative-pg.io" "$TMP_DIR/mock_calls.log" | tail -n1 | cut -d: -f1 || true) +backup_apply_line=$(grep -n "apply -n test-ns -f -" "$TMP_DIR/mock_calls.log" | tail -n1 | cut -d: -f1 || true) + +if [[ -z "$socket_line" || -z "$backup_apply_line" || "$socket_line" -ge "$backup_apply_line" ]]; then + echo "FAILURE: expected plugin socket check before triggering backup apply" + echo "socket_line=$socket_line backup_apply_line=$backup_apply_line" + sed -n '1,220p' "$TMP_DIR/mock_calls.log" || true + exit 1 +fi + +echo "SUCCESS" diff --git a/tests/etc/test_prole_cfg_kube_context_k3d_prefix.sh b/tests/etc/test_prole_cfg_kube_context_k3d_prefix.sh new file mode 100644 index 0000000..1d97af2 --- /dev/null +++ b/tests/etc/test_prole_cfg_kube_context_k3d_prefix.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT +export TMP_DIR + +BIN_DIR="$TMP_DIR/bin" +mkdir -p "$BIN_DIR" + +cat <<'K_EOF' >"$BIN_DIR/kubectl" +#!/usr/bin/env bash +set -euo pipefail + +args="$*" + +if [[ "$args" == "config current-context"* ]]; then + echo "k3d-knoe-dev-cluster" + exit 0 +fi + +if [[ "$args" == "config get-contexts -o name"* ]]; then + echo "docker-desktop" + echo "k3d-knoe-dev-cluster" + echo "k3d-knoe-system" + exit 0 +fi + +if [[ "$args" == "config get-contexts --no-headers"* ]]; then + # Format: CURRENT NAME CLUSTER AUTHINFO NAMESPACE + echo "* k3d-knoe-dev-cluster k3d-knoe-dev-cluster user " + echo " k3d-knoe-system k3d-knoe-system user " + exit 0 +fi + +if [[ "$args" == "config use-context "* ]]; then + # shellcheck disable=SC2129 + echo "${args#config use-context }" >"${TMP_DIR}/used_context" + exit 0 +fi + +exit 0 +K_EOF +chmod +x "$BIN_DIR/kubectl" + +mkdir -p "$TMP_DIR/conf" +cat <"$TMP_DIR/conf/prole.cfg" +[Global] +DEPLOYMENT_MODE = k3d +KUBECONTEXT = knoe-system +C_EOF + +export PATH="$BIN_DIR:$PATH" +export HOME="$TMP_DIR/home" +mkdir -p "$HOME" + +export PROLE_HOME="$PROLE_HOME" +export PROLE_CONF="$TMP_DIR/conf" +export PROLE_MODE="k3d" +export KUBECONTEXT="knoe-system" + +# shellcheck disable=SC1090 +source "$PROLE_HOME/etc/prole_cfg.sh" + +prole_ensure_kube_context + +if [[ "$(cat "$TMP_DIR/used_context")" != "k3d-knoe-system" ]]; then + echo "FAILURE: expected context 'k3d-knoe-system' but got '$(cat "$TMP_DIR/used_context")'" >&2 + exit 1 +fi + +echo "SUCCESS" diff --git a/tests/etc/test_prole_cfg_kube_context_single_context_fallback.sh b/tests/etc/test_prole_cfg_kube_context_single_context_fallback.sh new file mode 100644 index 0000000..45beb20 --- /dev/null +++ b/tests/etc/test_prole_cfg_kube_context_single_context_fallback.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT +export TMP_DIR + +BIN_DIR="$TMP_DIR/bin" +mkdir -p "$BIN_DIR" + +cat <<'K_EOF' >"$BIN_DIR/kubectl" +#!/usr/bin/env bash +set -euo pipefail + +args="$*" + +if [[ "$args" == "config current-context"* ]]; then + echo "default" + exit 0 +fi + +if [[ "$args" == "config get-contexts -o name"* ]]; then + echo "default" + exit 0 +fi + +if [[ "$args" == "config get-contexts --no-headers"* ]]; then + # Format: CURRENT NAME CLUSTER AUTHINFO NAMESPACE + echo "* default default user " + exit 0 +fi + +if [[ "$args" == "config use-context "* ]]; then + ctx="${args#config use-context }" + if [[ "$ctx" == "default" ]]; then + echo "$ctx" >"${TMP_DIR}/used_context" + exit 0 + fi + exit 1 +fi + +exit 0 +K_EOF +chmod +x "$BIN_DIR/kubectl" + +export PATH="$BIN_DIR:$PATH" +export HOME="$TMP_DIR/home" +mkdir -p "$HOME" + +# Prevent prole_cfg.sh from trying to auto-discover kubeconfig during sourcing. +export KUBECONFIG="$TMP_DIR/kubeconfig" + +export PROLE_HOME="$PROLE_HOME" +export PROLE_MODE="k3s" +export KUBECONTEXT="prole-k3s" + +# shellcheck disable=SC1090 +source "$PROLE_HOME/etc/prole_cfg.sh" + +prole_ensure_kube_context + +if [[ "$(cat "$TMP_DIR/used_context")" != "default" ]]; then + echo "FAILURE: expected context 'default' but got '$(cat "$TMP_DIR/used_context")'" >&2 + exit 1 +fi + +echo "SUCCESS" diff --git a/tests/etc/test_status_k3d_common_services_namespace.sh b/tests/etc/test_status_k3d_common_services_namespace.sh new file mode 100644 index 0000000..6b91143 --- /dev/null +++ b/tests/etc/test_status_k3d_common_services_namespace.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Regression test for etc/status.sh +# - In k3d mode, it must not auto-select prole-k3s.kubeconfig via the fallback. +# - It must call status_common_services.sh using SERVICE_NAMESPACE (not DB NAMESPACE). + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME_REPO=$(cd "$SCRIPT_DIR/../.." && pwd) +ETC_DIR="$PROLE_HOME_REPO/etc" +SCRIPT_UNDER_TEST="$ETC_DIR/status.sh" + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT +export TMP_DIR +export PROLE_K3S_KUBECONFIG_PATH="$PROLE_HOME_REPO/prole-k3s.kubeconfig" + +BIN_DIR="$TMP_DIR/bin" +mkdir -p "$BIN_DIR" + +cat <<'K_EOF' >"$BIN_DIR/kubectl" +#!/usr/bin/env bash +set -euo pipefail + +_log_file="${TMP_DIR}/mock_calls.log" +echo "Mocked kubectl called with $*" >>"${_log_file}" + +args="$*" + +if [[ "$args" == "config current-context"* ]]; then + echo "k3d-knoe-system" + exit 0 +fi + +if [[ "$args" == "config view --minify"* ]]; then + echo "https://127.0.0.1:6443" + exit 0 +fi + +if [[ "$args" == "cluster-info"* ]]; then + if [[ "${KUBECONFIG:-}" == "${PROLE_K3S_KUBECONFIG_PATH}" ]]; then + echo "fallback-probe" >>"${TMP_DIR}/k3s_kubeconfig_probe" + fi + echo "Kubernetes control plane is running" + exit 0 +fi + +# Basic pods output so status.sh can iterate. +if [[ "$args" == "get pods -o wide -A"* ]]; then + echo "NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE" + echo "kube-system coredns 1/1 Running 0 1h 10.0.0.1 node" + exit 0 +fi + +# Common services checks (SERVICE_NAMESPACE should be svc-ns) +if [[ "$args" == "-n svc-ns get svc"* ]]; then + echo "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE" + echo "openbao ClusterIP 10.0.0.2 8200/TCP 1h" + echo "garage ClusterIP 10.0.0.3 3900/TCP 1h" + echo "opentofu ClusterIP 10.0.0.4 8080/TCP 1h" + exit 0 +fi +if [[ "$args" == "-n svc-ns get svc auth"* ]]; then + echo "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE" + echo "auth ClusterIP 10.0.0.5 88/TCP 1h" + exit 0 +fi +if [[ "$args" == "-n svc-ns get deploy"* || "$args" == "-n svc-ns get statefulset"* ]]; then + exit 0 +fi + +exit 0 +K_EOF +chmod +x "$BIN_DIR/kubectl" + +export PATH="$BIN_DIR:$PATH" + +# Provide a reachable prole-k3s.kubeconfig file (fallback must NOT probe it in k3d mode) +touch "$PROLE_HOME_REPO/prole-k3s.kubeconfig" + +CFG_DIR="$TMP_DIR/conf" +mkdir -p "$CFG_DIR" + +cat <"$CFG_DIR/prole.cfg" +[User] +NAMESPACE = db-ns +SERVICE_NAMESPACE = svc-ns + +[Global] +DEPLOYMENT_MODE = k3d +C_EOF + +set +e +STATUS_CHECK_TIMEOUT=2 bash "$SCRIPT_UNDER_TEST" -c "$CFG_DIR/prole.cfg" >"$TMP_DIR/stdout" 2>"$TMP_DIR/stderr" +RC=$? +set -e + +# We don't require rc=0; we only assert the correct calls were made. + +if [[ -f "$TMP_DIR/k3s_kubeconfig_probe" ]]; then + echo "FAILURE: expected status.sh not to probe prole-k3s.kubeconfig via kubectl cluster-info in k3d mode" >&2 + sed -n '1,200p' "$TMP_DIR/mock_calls.log" >&2 || true + exit 1 +fi + +if ! grep -q -- "-n svc-ns get svc opentofu garage openbao" "$TMP_DIR/mock_calls.log"; then + echo "FAILURE: expected status_common_services.sh checks to use SERVICE_NAMESPACE (svc-ns)" >&2 + sed -n '1,200p' "$TMP_DIR/mock_calls.log" >&2 || true + exit 1 +fi + +echo "SUCCESS" diff --git a/tests/etc/test_supabase_patch_realtime_probe.sh b/tests/etc/test_supabase_patch_realtime_probe.sh new file mode 100644 index 0000000..578e940 --- /dev/null +++ b/tests/etc/test_supabase_patch_realtime_probe.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +YAML_PATH="$TMP_DIR/realtime-deployment.yaml" + +cat >"$YAML_PATH" <<'YAML' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: realtime + namespace: supabase +spec: + replicas: 1 + selector: + matchLabels: + app: realtime + template: + metadata: + labels: + app: realtime + spec: + containers: + - name: realtime + image: supabase/realtime:v2.76.5 + livenessProbe: + exec: + command: + - sh + - -c + - curl -sSfL --head -o /dev/null -H "Authorization: Bearer abc:def" http://localhost:4000/health + initialDelaySeconds: 10 + periodSeconds: 30 +YAML + +python3 "$PROLE_HOME/supabase/patch_realtime_probe.py" "$YAML_PATH" >/dev/null + +# Validate that the curl command is YAML-quoted. +# (We avoid using `kubectl ... -o json` here because it may contact the cluster for discovery +# even in client dry-run mode, which makes the test flaky/offline-hostile.) + +if ! grep -Eq "^[[:space:]]*-[[:space:]]*'curl .*Authorization: Bearer" "$YAML_PATH"; then + echo "FAILURE: expected the realtime liveness curl command to be YAML single-quoted" >&2 + sed -n '1,120p' "$YAML_PATH" >&2 || true + exit 1 +fi + +if grep -Eq "^[[:space:]]*-[[:space:]]*curl .*Authorization: Bearer" "$YAML_PATH"; then + echo "FAILURE: found an unquoted realtime liveness curl command (YAML may parse it as a mapping)" >&2 + sed -n '1,120p' "$YAML_PATH" >&2 || true + exit 1 +fi + +echo "SUCCESS" diff --git a/tests/installer/test_actions_helpers.py b/tests/installer/test_actions_helpers.py index c6c13f7..829a1bb 100644 --- a/tests/installer/test_actions_helpers.py +++ b/tests/installer/test_actions_helpers.py @@ -11,8 +11,8 @@ from unittest.mock import MagicMock, patch import pytest from installer.core.actions import ( - ProleInstallerBase, - ProleSilentInstaller, + ProleInstaller, + ProleConsoleInstaller, _configure_unbuffered_io, ) @@ -22,7 +22,7 @@ from installer.core.actions import ( # --------------------------------------------------------------------------- -class _TestableInstaller(ProleInstallerBase): +class _TestableInstaller(ProleInstaller): """Minimal concrete subclass for testing base-class helpers.""" def __init__(self, inputs=None, project_root=None): @@ -268,13 +268,16 @@ spec: inst = _TestableInstaller(project_root=tmp_path) - images = inst._collect_dependent_images( - include_supabase=True, - include_kerberos_proxy=True, - ) + with mock.patch.dict( + os.environ, {"KRB5_AD_PROXY_IMAGE": "krb-proxy:test"}, clear=False + ): + images = inst._collect_dependent_images( + include_supabase=True, + include_kerberos_proxy=True, + ) assert "nginx:1.2" in images assert "supabase/postgres:15" in images - assert "ghcr.io/bsharp-tech/prole-kerberos-proxy:latest" in images + assert "krb-proxy:test" in images # --------------------------------------------------------------------------- @@ -282,8 +285,8 @@ spec: # --------------------------------------------------------------------------- -class _TestableSilentInstaller(ProleSilentInstaller): - """Testable subclass of ProleSilentInstaller.""" +class _TestableSilentInstaller(ProleConsoleInstaller): + """Testable subclass of ProleConsoleInstaller.""" def __init__(self, inputs=None, project_root=None): self._inputs_dict = inputs or {} diff --git a/tests/installer/test_argocd_canvas_only_page.py b/tests/installer/test_argocd_canvas_only_page.py new file mode 100644 index 0000000..fd17182 --- /dev/null +++ b/tests/installer/test_argocd_canvas_only_page.py @@ -0,0 +1,48 @@ +from __future__ import annotations + + +def test_show_page_argocd_config_hides_slide_area(): + """Regression: `argocd_config` is a canvas-rendered page and must not be covered by `slide_area`.""" + + from installer.ui.screens.base import ScreenBaseMixin + + class _SlideArea: + def __init__(self): + self.place_forget_called = 0 + self.place_called = 0 + + def place_forget(self): + self.place_forget_called += 1 + + def place(self, **_kwargs): + self.place_called += 1 + + def lift(self): + return None + + def configure(self, **_kwargs): + return None + + class _Dummy(ScreenBaseMixin): + def __init__(self): + self._showing_service_overlay = False + self.page_index = 0 + self.pages = [("argocd_config", None)] + self.page_frames = {} + self.canvas_renderers = {"argocd_config": lambda: None} + self.slide_area = _SlideArea() + + def _clear_canvas_page(self): + return None + + def _update_nav_highlight(self, _pid: str): + return None + + def update_footer(self): + return None + + s = _Dummy() + s.show_page("argocd_config") + + assert s.slide_area.place_forget_called == 1 + assert s.slide_area.place_called == 0 diff --git a/tests/installer/test_cluster_save_triggers_status_check.py b/tests/installer/test_cluster_save_triggers_status_check.py new file mode 100644 index 0000000..a8b8939 --- /dev/null +++ b/tests/installer/test_cluster_save_triggers_status_check.py @@ -0,0 +1,92 @@ +import sys +import tkinter as tk +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# Ensure `installer` is importable when tests are invoked directly via `pytest` +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +import installer.ui.screens.cluster as cluster_mod +from installer.ui.screens.base import ScreenBaseMixin +from installer.ui.screens.cluster import ClusterScreenMixin + + +class _DummyClusterSaveApp(ScreenBaseMixin, ClusterScreenMixin): + """Minimal surface to exercise the Cluster Environment screen Save behavior.""" + + def __init__(self, root: tk.Tk): + self.root = root + # Canvas is not required by `_validate_and_save_cluster_config`, but other mixins + # assume it exists. + self.bg_canvas = tk.Canvas(root, width=1, height=1) + self._canvas_items = [] + self._overlay_widgets = [] + + self.cluster_env = tk.StringVar(master=root, value="service") + self.selected_k3d_cluster = tk.StringVar(master=root, value="") + self.selected_kubectx = tk.StringVar(master=root, value="") + self.service_namespace = tk.StringVar(master=root, value="knoe-system") + self.prod_artifacts_path = tk.StringVar(master=root, value=str(Path.home())) + + self.k3s_server_url = tk.StringVar(master=root, value="") + self.k3s_token = tk.StringVar(master=root, value="") + + self.supabase_enabled = tk.BooleanVar(master=root, value=False) + self.gitops_enabled = tk.BooleanVar(master=root, value=False) + self.kerberos_enabled = tk.BooleanVar(master=root, value=False) + self.at_rest_encryption_enabled = tk.BooleanVar(master=root, value=False) + + # Minimal config structure required by `_validate_and_save_cluster_config`. + self.prole_cfg_data = { + "Initialize Cluster": { + "ENVIRONMENT": "service", + "K3S_SERVER_URL": "", + "K3S_TOKEN": "", + }, + "Global": { + # Keep old/new equal so we don't trigger the namespace cleanup prompt. + "SERVICE_NAMESPACE": "knoe-system", + }, + "Optional Features": {}, + "Prod Cluster (k8s)": {}, + } + + self.saved_calls = 0 + self.status_check_calls = 0 + + def _save_prole_cfg(self): + self.saved_calls += 1 + + def _verify_k3s_services(self): + self.status_check_calls += 1 + + +def test_cluster_env_save_triggers_status_check(monkeypatch): + # Some test modules replace tkinter with MagicMocks at import-time. + if isinstance(sys.modules.get("tkinter"), MagicMock): + pytest.skip("tkinter is mocked in this test run") + + # Avoid depending on a real kubeconfig file. + monkeypatch.setattr(cluster_mod, "_find_kubeconfig_file", lambda: "/tmp/kubeconfig") + # Avoid any real secret handling. + monkeypatch.setattr(cluster_mod, "_encrypt_cfg_secret", lambda s: s) + + root = tk.Tk() + root.withdraw() + try: + app = _DummyClusterSaveApp(root) + + assert app._validate_and_save_cluster_config() is True + assert app.saved_calls == 1 + + # Requirement: clicking Save should immediately run a status check. + assert app.status_check_calls == 1 + finally: + try: + root.destroy() + except Exception: + pass diff --git a/tests/installer/test_core_classes.py b/tests/installer/test_core_classes.py index a98bb52..0fc2518 100644 --- a/tests/installer/test_core_classes.py +++ b/tests/installer/test_core_classes.py @@ -12,7 +12,7 @@ from installer.state import InstallerState from installer.milestone import Milestone, ProgressCallback from installer.build import get_build_command, BuildMilestone from installer.core.controller import ProleController -from installer.core.actions import ProleSilentInstaller +from installer.core.actions import ProleConsoleInstaller # ===== InstallerState ===== @@ -271,7 +271,7 @@ class TestProleController: assert c.project_root == tmp_path assert c.verbose is False assert c.state is not None - installer = ProleSilentInstaller(c) + installer = ProleConsoleInstaller(c) defaults = installer._default_inputs() assert defaults.get("kerberos_config.user") == "administrator" assert defaults.get("kerberos_config.password") == "" @@ -326,7 +326,7 @@ class TestProleController: def test_silent_installer_runs_supabase_preload_before_deploy(self, tmp_path): c = ProleController(tmp_path) - installer = ProleSilentInstaller(c) + installer = ProleConsoleInstaller(c) with mock.patch.object(installer, "_load_inputs_from_cfg", return_value={}), \ mock.patch.object(installer, "_write_cfg"), \ @@ -349,7 +349,7 @@ class TestProleController: self, tmp_path ): c = ProleController(tmp_path) - installer = ProleSilentInstaller(c) + installer = ProleConsoleInstaller(c) openbao_ref = "${OPENBAO:kv/prole/test/db#password}" diff --git a/tests/installer/test_gitea_init_script_present.py b/tests/installer/test_gitea_init_script_present.py new file mode 100644 index 0000000..8faffbb --- /dev/null +++ b/tests/installer/test_gitea_init_script_present.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from installer.core.env import PROJECT_ROOT + + +def test_init_gitea_script_exists(): + assert (PROJECT_ROOT / "etc" / "init_gitea.sh").is_file() diff --git a/tests/installer/test_namespace_propagation.py b/tests/installer/test_namespace_propagation.py new file mode 100644 index 0000000..d2fc7c5 --- /dev/null +++ b/tests/installer/test_namespace_propagation.py @@ -0,0 +1,88 @@ +import sys +import time +import tkinter as tk +from unittest.mock import MagicMock + +import pytest + +from installer.ui.screens.cfg import ConfigMixin + + +class _DummyNamespaceApp(ConfigMixin): + """Minimal surface to exercise `ConfigMixin._propagate_namespace_change`.""" + + def __init__(self, root: tk.Tk): + self.root = root + # Keep the test fast and deterministic. + # Use a debounce long enough that the scheduled `after()` callback will not + # run between the simulated keystrokes (which call `root.update()`). + self._namespace_debounce_ms = 100 + + self.db_namespace = tk.StringVar(master=root, value="prole-db") + self.service_namespace = tk.StringVar(master=root, value="knoe-system") + + # Independent namespaces that must not be rewritten. + self.gitops_namespace = tk.StringVar(master=root, value="gitea") + self.argocd_namespace = tk.StringVar(master=root, value="argocd") + + self._last_synced_ns = self.db_namespace.get() + + # Inputs dict is used by the sweep logic; include a service namespace entry + # to ensure it is not touched. + self.inputs = { + "init_cluster.service_namespace": self.service_namespace.get(), + } + + self.save_calls = 0 + self.updated_namespace = None + + def _update_env_namespace(self, namespace: str): + self.updated_namespace = namespace + + def _save_prole_cfg(self): + self.save_calls += 1 + + +def _drain_tk_events(root: tk.Tk, timeout_s: float = 0.2) -> None: + end = time.monotonic() + timeout_s + while time.monotonic() < end: + root.update() + time.sleep(0.001) + + +def test_namespace_typing_does_not_corrupt_service_namespace(): + # Some test modules replace tkinter with MagicMocks at import-time. + if isinstance(sys.modules.get("tkinter"), MagicMock): + pytest.skip("tkinter is mocked in this test run") + + root = tk.Tk() + root.withdraw() + try: + app = _DummyNamespaceApp(root) + app.db_namespace.trace_add("write", app._propagate_namespace_change) + + # Simulate typing a new namespace in the UI entry box. + for partial in [ + "k", + "kn", + "kno", + "knoe", + "knoe-", + "knoe-d", + "knoe-db", + ]: + app.db_namespace.set(partial) + root.update() + + _drain_tk_events(root, timeout_s=0.3) + + # Regression: SERVICE_NAMESPACE must remain unchanged. + assert app.service_namespace.get() == "knoe-system" + + # Debounced propagation should result in a single save. + assert app.save_calls == 1 + finally: + try: + root.destroy() + except Exception: + pass diff --git a/tests/installer/test_push_docker_image_ca_fallback.py b/tests/installer/test_push_docker_image_ca_fallback.py new file mode 100644 index 0000000..34dff2b --- /dev/null +++ b/tests/installer/test_push_docker_image_ca_fallback.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from types import SimpleNamespace + + +def test_push_docker_image_prefers_tls_verified_skopeo_when_ca_provided( + tmp_path, monkeypatch +): + from installer.core import env + + ca = tmp_path / "registry-ca.pem" + ca.write_text("dummy-ca") + monkeypatch.setenv("PROLE_REGISTRY_CA_CERT", str(ca)) + + monkeypatch.setattr(env.shutil, "which", lambda name: "/usr/bin/skopeo" if name == "skopeo" else None) + monkeypatch.setattr(env.shutil, "copyfile", lambda _src, _dst: None) + + calls: list[list[str]] = [] + + def fake_run(cmd, capture_output=True, text=True): + calls.append(list(cmd)) + if cmd[:2] == ["docker", "push"]: + return SimpleNamespace(returncode=1, stdout="", stderr="x509: unknown authority") + + # Secure skopeo path should be used and succeed. + assert cmd[0] == "/usr/bin/skopeo" + assert "--dest-tls-verify=true" in cmd + assert "--dest-cert-dir" in cmd + assert "--dest-tls-verify=false" not in cmd + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(env.subprocess, "run", fake_run) + + ok = env._push_docker_image("myrddin.prole.org:5000/prole-db:18-118") + assert ok is True + assert len(calls) == 2 + + +def test_push_docker_image_auto_detects_repo_cert_for_myrddin(monkeypatch): + """Regression: when no env var is provided, prefer repo-shipped certs (Ansible prole_ssl role).""" + + from installer.core import env + + monkeypatch.delenv("PROLE_REGISTRY_CA_CERT", raising=False) + monkeypatch.delenv("REGISTRY_CA_CERT", raising=False) + monkeypatch.delenv("PROLE_REGISTRY_CERT_DIR", raising=False) + + monkeypatch.setattr( + env.shutil, + "which", + lambda name: "/usr/bin/skopeo" if name == "skopeo" else None, + ) + monkeypatch.setattr(env.shutil, "copyfile", lambda _src, _dst: None) + + calls: list[list[str]] = [] + + def fake_run(cmd, capture_output=True, text=True): + calls.append(list(cmd)) + if cmd[:2] == ["docker", "push"]: + return SimpleNamespace( + returncode=1, stdout="", stderr="x509: unknown authority" + ) + + assert cmd[0] == "/usr/bin/skopeo" + assert "--dest-tls-verify=true" in cmd + assert "--dest-cert-dir" in cmd + assert "--dest-tls-verify=false" not in cmd + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(env.subprocess, "run", fake_run) + + ok = env._push_docker_image("myrddin.prole.org:5000/prole-db:18-118") + assert ok is True + assert len(calls) == 2 diff --git a/tests/installer/test_registry_k3s_selection.py b/tests/installer/test_registry_k3s_selection.py new file mode 100644 index 0000000..8d27ba5 --- /dev/null +++ b/tests/installer/test_registry_k3s_selection.py @@ -0,0 +1,71 @@ +from __future__ import annotations + + +from pathlib import Path + + +import pytest + + +from installer.ui.screens.docker import DockerScreenMixin +import installer.ui.screens.docker as docker_mod + + +class _DummyDockerScreen(DockerScreenMixin): + def __init__(self, tmp_path: Path): + self._tmp_path = tmp_path + self.prole_cfg_data = { + # Global deployment mode may remain k3d, while the UI selection is Service. + "Global": {"DEPLOYMENT_MODE": "k3d"}, + "Docker Build": {}, + } + self.local_registry_url = None + self.local_registry_internal = None + + def _cluster_env_key(self, env_label=None) -> str: + return "service" + + def _deployment_mode(self) -> str: + return "k3d" + + def _k3s_registry_hostport(self) -> str: + return "myrddin.prole.org:5000" + + def _registry_namespace(self) -> str: + return "knoe-system" + + def check_docker_running(self) -> bool: + return True + + def safe_after(self, fn): + # Execute inline for tests + try: + fn() + except Exception: + pass + + def _save_prole_cfg(self): + return None + + def _registry_log_path(self) -> Path: + return self._tmp_path / "local_registry.log" + + +def test_ensure_local_registry_available_prefers_selected_env_over_global_mode(monkeypatch, tmp_path): + """Regression: Service (k3s) selection must not trigger localhost/k3d registry checks.""" + + def _unexpected_ping(*_args, **_kwargs): + raise AssertionError("_http_ping_registry should not be called for k3s branch") + + def _unexpected_run(*args, **kwargs): + raise AssertionError(f"subprocess.run should not be used for k3s branch: {args}") + + monkeypatch.setattr(docker_mod, "_http_ping_registry", _unexpected_ping) + monkeypatch.setattr(docker_mod.subprocess, "run", _unexpected_run) + + screen = _DummyDockerScreen(tmp_path) + info = screen.ensure_local_registry_available() + assert info == ( + "myrddin.prole.org:5000", + "registry.knoe-system.svc.cluster.local:5000", + ) diff --git a/tests/installer/test_registry_rescan_after_init.py b/tests/installer/test_registry_rescan_after_init.py new file mode 100644 index 0000000..2939c4b --- /dev/null +++ b/tests/installer/test_registry_rescan_after_init.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + + +def test_db_rescan_registry_status_after_init_schedules_multiple_checks(): + from installer.ui.screens.database import DatabaseScreenMixin + + class _Dummy(DatabaseScreenMixin): + def __init__(self): + self._scheduled: list[int] = [] + self._ensure_called = 0 + self._db_registry_status_label = 1 + self.bg_canvas = MagicMock() + + def safe_after(self, fn, delay: int = 0): + self._scheduled.append(delay) + if delay == 0: + return fn() + return None + + def _ensure_db_build_registry_async(self): + self._ensure_called += 1 + + d = _Dummy() + d._db_rescan_registry_status_after_init() + + # One immediate label update (delay 0), then a sequence of scheduled checks. + assert d._scheduled == [0, 0, 1000, 2000, 4000, 8000, 15000] + assert d._ensure_called == 1 diff --git a/tests/test_cluster_screen_observe_only.py b/tests/test_cluster_screen_observe_only.py new file mode 100644 index 0000000..c8646ca --- /dev/null +++ b/tests/test_cluster_screen_observe_only.py @@ -0,0 +1,141 @@ +from __future__ import annotations + + +import json + + +from installer.ui.screens.cluster import ClusterScreenMixin + + +class _Var: + def __init__(self, value: str = ""): + self._value = value + + def get(self) -> str: + return self._value + + def set(self, value: str) -> None: + self._value = value + + +class _Root: + def after(self, _delay_ms: int, fn): + fn() + + +class _Canvas: + def itemconfig(self, *_args, **_kwargs): + return None + + +class _Button: + def configure(self, **_kwargs): + return None + + +def test_cluster_status_refresh_is_observe_only(monkeypatch): + """Regression: simply navigating/refreshing the Cluster Environment screen must not mutate the cluster.""" + + class _ImmediateThread: + def __init__(self, *, target=None, daemon=None): + self._target = target + + def start(self): + # Execute synchronously so exceptions surface to the test. + if self._target: + self._target() + + monkeypatch.setattr( + "installer.ui.screens.cluster.threading.Thread", _ImmediateThread, raising=True + ) + + class Dummy(ClusterScreenMixin): + def __init__(self): + self.root = _Root() + self.bg_canvas = _Canvas() + self.k3d_status_label = 1 + self.common_services_status_label = 2 + self._init_cluster_button = _Button() + self._kong_last_repair_at = 0.0 + self._kong_repair_cooldown_s = 0.0 + self._repair_inflight = False + self._repair_last_run_at = 0.0 + self._repair_cooldown_s = 0.0 + + def _set_cluster_env_message(self, *_args, **_kwargs): + return None + + def _cluster_status_snapshot(self) -> dict: + return { + "env": "service", + "cluster_ok": True, + "cluster_msg": "ok", + "cluster_fill": "#34c759", + "common_ok": True, + "common_msg": "ok", + "common_fill": "#34c759", + } + + def _dashboard_kong_status(self, _env=None): + # Unhealthy, but observe-only behavior must not attempt repairs. + return {"ok": False, "reset_pods": ["pod-1"]} + + def _authority_context_missing(self) -> bool: + return True + + def _maybe_run_repair_pipeline(self, *_args, **_kwargs): + raise AssertionError("repair pipeline must not run during status refresh") + + Dummy().check_cluster_status_async() + + +def test_service_namespace_infers_registry_namespace(monkeypatch): + """If cluster is reachable and `registry:2` exists, infer that namespace instead of defaulting to `default`.""" + + class Dummy(ClusterScreenMixin): + def __init__(self): + self.service_namespace = _Var("") + self.cluster_env = _Var("prod") + self.prole_cfg_data = {"Global": {}} + + def _ensure_k3s_kubeconfig_merged(self): + return None + + def _deployment_mode(self) -> str: + return "k8s" + + def _kubectl_base_cmd(self, _mode=None): + return ["kubectl"] + + payload = { + "items": [ + { + "metadata": {"namespace": "knoe-system"}, + "spec": { + "template": { + "spec": {"containers": [{"image": "registry:2"}]} + } + }, + } + ] + } + + def fake_run(cmd, capture_output, text, env, timeout): + assert cmd[:3] == ["kubectl", "get", "deploy"] + + class _Res: + returncode = 0 + stdout = json.dumps(payload) + stderr = "" + + return _Res() + + monkeypatch.setattr( + "installer.ui.screens.cluster.subprocess.run", fake_run, raising=True + ) + # Avoid reliance on an actual kubeconfig in test environment. + monkeypatch.setattr( + "installer.ui.screens.cluster._find_kubeconfig_file", lambda *_a, **_k: "", raising=True + ) + + assert Dummy()._get_service_namespace() == "knoe-system" diff --git a/tests/test_init_kerberos_ad_forwarder_readiness.py b/tests/test_init_kerberos_ad_forwarder_readiness.py new file mode 100644 index 0000000..6bf6d5b --- /dev/null +++ b/tests/test_init_kerberos_ad_forwarder_readiness.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from pathlib import Path + + +def test_init_kerberos_waits_for_ad_forwarder_ready_and_endpoints(): + """Regression: Kerberos test must not run against an AD forwarder Service with no endpoints. + + Historically `init_kerberos.sh` would `kubectl apply` the AD forwarder Deployment/Service and then + proceed immediately (or ignore rollout failures), which could yield `kinit: Cannot contact any KDC`. + """ + + text = Path("etc/init_kerberos.sh").read_text(encoding="utf-8") + + assert "wait_for_ad_forwarder_ready" in text + assert "wait_for_ad_forwarder_ready \"$KRB5_AD_NAMESPACE\" \"$KRB5_AD_PROXY_NAME\" \"$KRB5_AD_SERVICE_NAME\"" in text + + # Ensure we don't silently ignore readiness. + assert "rollout status deploy/${KRB5_AD_PROXY_NAME} --timeout=120s || true" not in text diff --git a/tests/test_k3s_common_core_namespace_and_repair.py b/tests/test_k3s_common_core_namespace_and_repair.py index 904fb84..ef2bb7b 100644 --- a/tests/test_k3s_common_core_namespace_and_repair.py +++ b/tests/test_k3s_common_core_namespace_and_repair.py @@ -2,7 +2,7 @@ from __future__ import annotations from pathlib import Path -from installer.core.actions import ProleSilentInstaller +from installer.core.actions import ProleConsoleInstaller from installer.core.controller import ProleController @@ -12,7 +12,7 @@ def test_silent_installer_service_namespace_defaults_to_knoe_system_for_k3s(tmp_ monkeypatch.delenv("SERVICE_NAMESPACE", raising=False) c = ProleController(tmp_path) - installer = ProleSilentInstaller(c) + installer = ProleConsoleInstaller(c) installer.inputs["init_cluster.cluster_env"] = "service" installer.prole_cfg_data = {"Global": {}} diff --git a/tests/test_reconcile_blocked_cluster_state.py b/tests/test_reconcile_blocked_cluster_state.py index d13950a..e25e1dd 100644 --- a/tests/test_reconcile_blocked_cluster_state.py +++ b/tests/test_reconcile_blocked_cluster_state.py @@ -5,13 +5,13 @@ from types import SimpleNamespace import pytest import installer.core.actions as actions -from installer.core.actions import ProleSilentInstaller +from installer.core.actions import ProleConsoleInstaller from installer.core.controller import ProleController -def _mk_installer(tmp_path) -> ProleSilentInstaller: +def _mk_installer(tmp_path) -> ProleConsoleInstaller: c = ProleController(tmp_path) - return ProleSilentInstaller(c) + return ProleConsoleInstaller(c) def test_repair_flow_classifies_blockers_attempts_reclaim_and_avoids_blind_health_wait( diff --git a/tests/test_repair_stale_released_pv_claimref.py b/tests/test_repair_stale_released_pv_claimref.py index d3332aa..074e273 100644 --- a/tests/test_repair_stale_released_pv_claimref.py +++ b/tests/test_repair_stale_released_pv_claimref.py @@ -5,7 +5,7 @@ import os from types import SimpleNamespace import installer.core.actions as actions -from installer.core.actions import ProleSilentInstaller +from installer.core.actions import ProleConsoleInstaller from installer.core.controller import ProleController @@ -29,9 +29,9 @@ def _preserve_namespace_env(): os.environ[k] = v -def _mk_installer(tmp_path) -> ProleSilentInstaller: +def _mk_installer(tmp_path) -> ProleConsoleInstaller: c = ProleController(tmp_path) - return ProleSilentInstaller(c) + return ProleConsoleInstaller(c) def test_match_stale_released_pv_happy_path(tmp_path):