diff --git a/mock_val/init_1password.sh b/mock_val/init_1password.sh new file mode 100755 index 0000000..dfd79ab --- /dev/null +++ b/mock_val/init_1password.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# init_1password.sh +# Preflight: sign in to 1Password, create the 'knoey' vault if absent, +# and ensure the 'administrator' item (DB master password) exists. +# +# Called by install.sh before the Python installer. Skipped in --min mode. + +set -euo pipefail + +# ── helpers ──────────────────────────────────────────────────────────────── + +_info() { echo "==> [1Password] $*"; } +_warn() { echo " [WARN] $*" >&2; } +_fatal() { echo " [ERROR] $*" >&2; exit 1; } + +VAULT="knoey" +ADMIN_ITEM="administrator" + +# ── skip in --min mode ───────────────────────────────────────────────────── + +for arg in "$@"; do + if [[ "$arg" == "--min" ]]; then + _warn "Skipping 1Password preflight in --min mode." + exit 0 + fi +done + +# ── require op CLI ───────────────────────────────────────────────────────── + +if ! command -v op >/dev/null 2>&1; then + _fatal "1Password CLI (op) not found. + Install: brew install 1password-cli + Docs: https://developer.1password.com/docs/cli" +fi + +_info "op CLI found: $(op --version)" + +# ── sign in ──────────────────────────────────────────────────────────────── + +if ! op whoami >/dev/null 2>&1; then + # If running non-interactively (no TTY), skip rather than hang. + if [[ ! -t 0 ]]; then + _warn "No active 1Password session and no TTY — skipping 1Password preflight." + exit 0 + fi + _info "No active 1Password session. Signing in..." + op signin +fi + +_info "Signed in as: $(op whoami --format json 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('email','unknown'))" 2>/dev/null || op whoami)" + +# ── create knoey vault if absent ─────────────────────────────────────────── + +if op vault get "$VAULT" >/dev/null 2>&1; then + _info "Vault '$VAULT' already exists." +else + _info "Creating vault '$VAULT'..." + op vault create "$VAULT" + _info "Vault '$VAULT' created." +fi + +# ── ensure administrator item exists ─────────────────────────────────────── + +if op item get "$ADMIN_ITEM" --vault "$VAULT" >/dev/null 2>&1; then + _info "Item '$ADMIN_ITEM' already exists in vault '$VAULT'." +else + _info "Creating item '$ADMIN_ITEM' in vault '$VAULT' with a generated password..." + op item create \ + --category login \ + --title "$ADMIN_ITEM" \ + --vault "$VAULT" \ + --generate-password="32,letters,digits" + _info "Item '$ADMIN_ITEM' created." +fi + +# ── export for child processes ───────────────────────────────────────────── + +export OP_VAULT="$VAULT" +_info "OP_VAULT=$OP_VAULT" diff --git a/mock_val/init_argocd.sh b/mock_val/init_argocd.sh index 6e1e060..1c8f978 100755 --- a/mock_val/init_argocd.sh +++ b/mock_val/init_argocd.sh @@ -32,6 +32,9 @@ fi ACTION="$COMMON_CORE_ACTION" +REPO_ROOT="${KNOE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}" +GKE_MANIFEST_DIR="${GKE_MANIFEST_DIR:-$REPO_ROOT/deploy/gcp/gke}" + if [[ -d "$SCRIPT_DIR/../k8s/argocd" ]]; then ARGOCD_MANIFEST_DIR="$SCRIPT_DIR/../k8s/argocd" elif [[ -n "${KNOE_HOME:-}" && -d "$KNOE_HOME/k8s/argocd" ]]; then @@ -52,6 +55,11 @@ ARGOCD_PORT_FORWARD_LOCAL=${ARGOCD_PORT_FORWARD_LOCAL:-8081} ARGOCD_PORT_FORWARD_REMOTE=${ARGOCD_PORT_FORWARD_REMOTE:-80} ARGOCD_NODE_SELECTOR=${ARGOCD_NODE_SELECTOR:-} +# ── Google Workspace OIDC (applied when all three vars are set) ─────────────── +PLATFORM_DOMAIN="${PLATFORM_DOMAIN:-}" +FRONTDOOR_HOST="${FRONTDOOR_HOST:-}" +BOOTSTRAP_ADMIN_EMAIL="${BOOTSTRAP_ADMIN_EMAIL:-}" + ensure_tools() { command -v kubectl >/dev/null || { echo "Missing required tool: kubectl" >&2; exit 1; } } @@ -122,6 +130,24 @@ apply_argocd() { kubectl rollout status statefulset/argocd-application-controller -n "$ARGOCD_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true } +apply_argocd_oidc_config() { + local cm_file="$GKE_MANIFEST_DIR/argocd-oidc-cm.yaml" + if [[ -z "$PLATFORM_DOMAIN" || -z "$FRONTDOOR_HOST" || -z "$BOOTSTRAP_ADMIN_EMAIL" ]]; then + return 0 + fi + if [[ ! -f "$cm_file" ]]; then + echo "WARN: ArgoCD OIDC config not found at $cm_file; skipping OIDC patch." >&2 + return 0 + fi + echo "Applying ArgoCD OIDC config (issuer=https://${FRONTDOOR_HOST}/auth) ..." + PLATFORM_DOMAIN="$PLATFORM_DOMAIN" \ + FRONTDOOR_HOST="$FRONTDOOR_HOST" \ + BOOTSTRAP_ADMIN_EMAIL="$BOOTSTRAP_ADMIN_EMAIL" \ + envsubst < "$cm_file" \ + | kubectl apply --server-side --force-conflicts \ + --field-manager=knoe-installer -f - +} + rollout_restart_argocd() { kubectl -n "$ARGOCD_NAMESPACE" rollout restart deploy/argocd-server deploy/argocd-repo-server \ deploy/argocd-dex-server deploy/argocd-applicationset-controller deploy/argocd-notifications-controller \ @@ -144,6 +170,7 @@ case "$ACTION" in ensure_tools ensure_namespace apply_argocd + apply_argocd_oidc_config knoe_register_port_forward "argocd" "$ARGOCD_NAMESPACE" "svc/${ARGOCD_SERVER_SERVICE}" \ "$ARGOCD_PORT_FORWARD_LOCAL" "$ARGOCD_PORT_FORWARD_REMOTE" "0.0.0.0" "TCP" "ArgoCD" ;; @@ -152,6 +179,7 @@ case "$ACTION" in ensure_namespace apply_argocd rollout_restart_argocd + apply_argocd_oidc_config knoe_register_port_forward "argocd" "$ARGOCD_NAMESPACE" "svc/${ARGOCD_SERVER_SERVICE}" \ "$ARGOCD_PORT_FORWARD_LOCAL" "$ARGOCD_PORT_FORWARD_REMOTE" "0.0.0.0" "TCP" "ArgoCD" ;; diff --git a/mock_val/init_certmgr.sh b/mock_val/init_certmgr.sh index 7c2ff2a..f906505 100755 --- a/mock_val/init_certmgr.sh +++ b/mock_val/init_certmgr.sh @@ -18,10 +18,13 @@ _has_config=0 for _arg in "$@"; do [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1 done -if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - set -- "-c" "$SCRIPT_DIR/../conf/knoe.cfg" "$@" +if [[ $_has_config -eq 0 ]]; then + _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)" + if [[ -n "$_default_cfg" ]]; then + set -- "-c" "$_default_cfg" "$@" + fi fi -unset _has_config _arg +unset _has_config _arg _default_cfg common_core_preparse_config "$@" @@ -61,7 +64,7 @@ fi usage() { cat <&2 - exit 1 -fi - -ACTION=${1:-} -CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-knoe-db} -VERSION=${2:-latest} -OPENBAO_NAME=${OPENBAO_NAME:-openbao} -REALM=${REALM:-PROLE.ORG} -DOMAIN=${DOMAIN:-knoe.org} -CNPG_MANIFEST_OVERRIDE=${CNPG_MANIFEST_OVERRIDE:-} -KNOE_HOME=${KNOE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} -BACKUP_DIR=${BACKUP_DIR:-$KNOE_HOME/knoe/backup} -BACKUP_WAIT_TIMEOUT=${BACKUP_WAIT_TIMEOUT:-1800} -if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-900} -else - CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-300} -fi -RECOVERY_TEMPLATE="$SCRIPT_DIR/../k8s/knoe/knoe-db-recovery.yaml.tpl" -BARMAN_PLUGIN_MANIFEST_URL=${BARMAN_PLUGIN_MANIFEST_URL:-} -BARMAN_PLUGIN_FALLBACK_VERSION=${BARMAN_PLUGIN_FALLBACK_VERSION:-0.9.0} -CERT_MANAGER_MANIFEST_URL=${CERT_MANAGER_MANIFEST_URL:-} -CERT_MANAGER_FALLBACK_VERSION=${CERT_MANAGER_FALLBACK_VERSION:-1.19.3} - -# Protected storage requirements (k3s only) -PROLE_CNPG_STORAGE_CLASS=${PROLE_CNPG_STORAGE_CLASS:-synology-iscsi} -PROLE_PROTECTED_DATA_PATH=${PROLE_PROTECTED_DATA_PATH:-/synology/d001/data} -PROLE_PROTECTED_WAL_PATH=${PROLE_PROTECTED_WAL_PATH:-/synology/d001/wal} - -if [[ "$ACTION" != "deploy" && "$ACTION" != "rollout" && "$ACTION" != "force-rollout" ]]; then - if [[ -n "${2:-}" ]]; then - CNPG_CLUSTER_NAME="${2}" - fi - VERSION=${3:-latest} -fi - -# Prefer infra-managed manifests (OpenTofu/GitOps) when available so bootstrap matches Ansible. -if [[ -n "${KNOE_HOME:-}" && -d "$KNOE_HOME/deploy/opentofu/k3s/manifests/knoe" ]]; then - K8S_PROLE_DIR="$KNOE_HOME/deploy/opentofu/k3s/manifests/knoe" -elif [[ -d "$SCRIPT_DIR/../deploy/opentofu/k3s/manifests/knoe" ]]; then - K8S_PROLE_DIR="$SCRIPT_DIR/../deploy/opentofu/k3s/manifests/knoe" -elif [[ -d "$SCRIPT_DIR/../k8s/knoe" ]]; then - K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/knoe" -elif [[ -n "${KNOE_HOME:-}" && -d "$KNOE_HOME/k8s/knoe" ]]; then - K8S_PROLE_DIR="$KNOE_HOME/k8s/knoe" -else - K8S_PROLE_DIR="$SCRIPT_DIR/../k8s/knoe" -fi -CNPG_MANIFEST="$K8S_PROLE_DIR/knoe-db.yaml" -BARMAN_OBJECTSTORE_MANIFEST="$K8S_PROLE_DIR/knoe-db-barman-objectstore.yaml" - -# Secrets and token locations -# Use KNOE_SERVICE if writable locally, otherwise fallback to ~/.knoe -if [[ -n "${KNOE_SERVICE:-}" ]] && _knoe_usable_dir "${KNOE_SERVICE}/secrets" >/dev/null; then - SECRETS_DIR="$KNOE_SERVICE/secrets" -else - SECRETS_DIR="$HOME/.knoe/etc/secrets" -fi -mkdir -p "$SECRETS_DIR" 2>/dev/null || true - -# Resolving CNPG admin keys. -# We prefer names without algorithm suffixes to be more generic, matching install.py fallback strategy. -ADMIN_PRIV_ED25519="$SECRETS_DIR/admin_ed25519.key" -ADMIN_PUB_ED25519="$SECRETS_DIR/admin_ed25519.pub" -ADMIN_PRIV_GENERIC="$SECRETS_DIR/admin.key" -ADMIN_PUB_GENERIC="$SECRETS_DIR/admin.pub" -OPENBAO_TOKEN_FILE="$SECRETS_DIR/openbao-root-token" -# NAMESPACE is always derived from knoe.cfg via PROLE_NAMESPACE — never from the environment -NAMESPACE="${PROLE_NAMESPACE}" -BAO_NAMESPACE="${PROLE_NAMESPACE}" -OPENBAO_NAMESPACE="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${PROLE_NAMESPACE}}}" -BAO_PATH_PREFIX="knoe/${BAO_NAMESPACE}" -BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin" -BAO_PATH_DB="${BAO_PATH_PREFIX}/db" -BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring" - -ensure_tools() { - for t in kubectl curl openssl base64 jq; do - command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; } - done -} - -_require_tool() { - local t="$1" - command -v "$t" >/dev/null 2>&1 || { echo "ERROR: Missing required tool: $t" >&2; return 1; } -} - -_require_knoe_protected_mount() { - local path="$1" label="$2" - local findmnt_bin="${PROLE_FINDMNT_BIN:-findmnt}" - - if [[ ! -d "$path" ]]; then - echo "ERROR: Protected ${label} path missing: ${path}" >&2 - echo "Refusing CNPG deployment: Knoe protected storage must be mounted at ${PROLE_PROTECTED_DATA_PATH} and ${PROLE_PROTECTED_WAL_PATH}." >&2 - return 1 - fi - - _require_tool "$findmnt_bin" || { - echo "ERROR: '${findmnt_bin}' is required to validate protected storage mounts (k3s mode)." >&2 - return 1 - } - - local root_src path_src path_target - root_src=$($findmnt_bin -T / -n -o SOURCE 2>/dev/null || true) - path_src=$($findmnt_bin -T "$path" -n -o SOURCE 2>/dev/null || true) - path_target=$($findmnt_bin -T "$path" -n -o TARGET 2>/dev/null || true) - - if [[ -z "$root_src" || -z "$path_src" || -z "$path_target" ]]; then - echo "ERROR: Unable to determine mount backing device for '${path}' via ${findmnt_bin}." >&2 - echo "root_src='${root_src:-}' path_src='${path_src:-}' path_target='${path_target:-}'" >&2 - return 1 - fi - - if [[ "$path_target" == "/" || "$path_src" == "$root_src" ]]; then - echo "ERROR: Protected ${label} path '${path}' is backed by the root filesystem (target='${path_target}', source='${path_src}')." >&2 - echo "Refusing CNPG deployment to prevent root/SD-backed storage from being used." >&2 - return 1 - fi - return 0 -} - -ensure_knoe_protected_storage() { - if [[ "${KNOE_MODE:-}" != "k3s" ]]; then - return 0 - fi - - echo "Validating protected Knoe storage mounts (k3s mode) ..." - _require_knoe_protected_mount "$PROLE_PROTECTED_DATA_PATH" "data" || return 1 - _require_knoe_protected_mount "$PROLE_PROTECTED_WAL_PATH" "WAL" || return 1 - return 0 -} - -validate_cnpg_manifest_storage() { - local rendered_manifest="$1" - - if [[ "${KNOE_MODE:-}" != "k3s" ]]; then - return 0 - fi - - if [[ ! -f "$rendered_manifest" ]]; then - echo "ERROR: Rendered CNPG manifest not found: $rendered_manifest" >&2 - return 1 - fi - - # Guardrail: never allow default/local-path to appear in the CNPG Cluster manifest in k3s mode. - if grep -Eq '^[[:space:]]*storageClassName:[[:space:]]*local-path([[:space:]]|$)' "$rendered_manifest"; then - echo "ERROR: CNPG manifest references storageClassName=local-path (forbidden in k3s mode)." >&2 - return 1 - fi - - local in_cluster=0 block="" data_sc="" wal_sc="" data_role="" wal_role="" - while IFS= read -r line || [[ -n "$line" ]]; do - if [[ "$line" =~ ^---[[:space:]]*$ ]]; then - in_cluster=0 - block="" - continue - fi - if [[ "$line" =~ ^kind:[[:space:]]*Cluster[[:space:]]*$ ]]; then - in_cluster=1 - block="" - continue - fi - (( in_cluster )) || continue - - if [[ "$line" =~ ^[[:space:]]*storage:[[:space:]]*$ ]]; then - block="storage" - continue - fi - if [[ "$line" =~ ^[[:space:]]*walStorage:[[:space:]]*$ ]]; then - block="wal" - continue - fi - # When we leave the storage-related blocks (next top-level under spec), stop attributing matches. - if [[ "$line" =~ ^[[:space:]]{2}[a-zA-Z0-9_-]+:[[:space:]]*$ ]]; then - case "$line" in - " storage:"|" walStorage:") ;; - *) block="" ;; - esac - fi - - if [[ -n "$block" && "$line" =~ storageClassName: ]]; then - local v - v=$(printf '%s' "$line" | sed -E 's/^.*storageClassName:[[:space:]]*//; s/[[:space:]]+$//') - if [[ "$block" == "storage" && -z "$data_sc" ]]; then - data_sc="$v" - elif [[ "$block" == "wal" && -z "$wal_sc" ]]; then - wal_sc="$v" - fi - fi - if [[ -n "$block" && "$line" =~ synology\.storage/role: ]]; then - local r - r=$(printf '%s' "$line" | sed -E 's/^.*synology\.storage\/role:[[:space:]]*//; s/[[:space:]]+$//') - if [[ "$block" == "storage" && -z "$data_role" ]]; then - data_role="$r" - elif [[ "$block" == "wal" && -z "$wal_role" ]]; then - wal_role="$r" - fi - fi - done < "$rendered_manifest" - - if [[ "$data_sc" != "$PROLE_CNPG_STORAGE_CLASS" || "$wal_sc" != "$PROLE_CNPG_STORAGE_CLASS" ]]; then - echo "ERROR: CNPG Cluster manifest must set storageClassName=${PROLE_CNPG_STORAGE_CLASS} for both data and walStorage (got data='${data_sc:-}', wal='${wal_sc:-}')." >&2 - echo "Refusing deployment to prevent fallback to default storage class." >&2 - return 1 - fi - - if [[ "$data_role" != "data" || "$wal_role" != "wal" ]]; then - echo "ERROR: CNPG Cluster manifest must include explicit PV selectors for Knoe storage (synology.storage/role=data and synology.storage/role=wal)." >&2 - echo "Got selector roles: data='${data_role:-}', wal='${wal_role:-}'" >&2 - return 1 - fi - - return 0 -} - -validate_cnpg_runtime_storage() { - if [[ "${KNOE_MODE:-}" != "k3s" ]]; then - return 0 - fi - - local sc="$PROLE_CNPG_STORAGE_CLASS" - local cluster_json - cluster_json=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o json 2>/dev/null || true) - if [[ -z "$cluster_json" ]]; then - echo "ERROR: Unable to fetch CNPG Cluster '${CNPG_CLUSTER_NAME}' for runtime storage validation." >&2 - return 1 - fi - - local data_sc wal_sc data_role wal_role - data_sc=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.storageClassName // empty') - wal_sc=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.storageClassName // empty') - data_role=$(printf '%s' "$cluster_json" | jq -r '.spec.storage.pvcTemplate.selector.matchLabels["synology.storage/role"] // empty') - wal_role=$(printf '%s' "$cluster_json" | jq -r '.spec.walStorage.pvcTemplate.selector.matchLabels["synology.storage/role"] // empty') - - if [[ "$data_sc" != "$sc" || "$wal_sc" != "$sc" ]]; then - echo "ERROR: Live CNPG Cluster storageClassName must be '${sc}' for both data and walStorage (got data='${data_sc:-}', wal='${wal_sc:-}')." >&2 - return 1 - fi - if [[ "$data_role" != "data" || "$wal_role" != "wal" ]]; then - echo "ERROR: Live CNPG Cluster must select Knoe PVs via selector labels (got data role='${data_role:-}', wal role='${wal_role:-}')." >&2 - return 1 - fi - - # Wait for CNPG PVCs to exist so we can validate binding and PV paths. - local timeout=${CNPG_STORAGE_VALIDATE_TIMEOUT:-120} - local start now elapsed pvc_json pvc_count - start=$(date +%s) - while true; do - pvc_json=$(kubectl -n "$NAMESPACE" get pvc -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" -o json 2>/dev/null || true) - pvc_count=$(printf '%s' "$pvc_json" | jq -r '.items | length' 2>/dev/null || echo 0) - if [[ "$pvc_count" =~ ^[0-9]+$ ]] && (( pvc_count > 0 )); then - break - fi - now=$(date +%s) - elapsed=$(( now - start )) - if (( elapsed > timeout )); then - echo "ERROR: Timed out (${timeout}s) waiting for CNPG PVCs to appear for cluster '${CNPG_CLUSTER_NAME}'." >&2 - return 1 - fi - sleep 3 - done - - local bad_pvc - bad_pvc=$(printf '%s' "$pvc_json" | jq -r --arg sc "$sc" '.items[] | select((.spec.storageClassName // "") != $sc) | .metadata.name' | head -n 1) - if [[ -n "$bad_pvc" ]]; then - echo "ERROR: CNPG PVC '${bad_pvc}' is not using storageClassName='${sc}'. Refusing deployment." >&2 - kubectl -n "$NAMESPACE" get pvc -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" -o wide >&2 || true - return 1 - fi - - local pv - while IFS= read -r pv || [[ -n "$pv" ]]; do - [[ -n "$pv" ]] || continue - local pv_json pv_sc pv_path - pv_json=$(kubectl get pv "$pv" -o json 2>/dev/null || true) - pv_sc=$(printf '%s' "$pv_json" | jq -r '.spec.storageClassName // empty') - pv_path=$(printf '%s' "$pv_json" | jq -r '.spec.local.path // empty') - if [[ "$pv_sc" != "$sc" ]]; then - echo "ERROR: Bound PV '${pv}' has storageClassName='${pv_sc:-}' (expected '${sc}')." >&2 - return 1 - fi - if [[ -z "$pv_path" ]]; then - echo "ERROR: Bound PV '${pv}' is missing .spec.local.path; cannot verify protected storage path." >&2 - return 1 - fi - if [[ "$pv_path" == /var/lib/rancher/k3s/storage/* ]]; then - echo "ERROR: Bound PV '${pv}' points into k3s local-path storage (${pv_path}). Refusing deployment." >&2 - return 1 - fi - case "$pv_path" in - /synology/*) ;; - *) - echo "ERROR: Bound PV '${pv}' path '${pv_path}' is not under /synology/. Refusing deployment." >&2 - return 1 - ;; - esac - done < <(printf '%s' "$pvc_json" | jq -r '.items[].spec.volumeName // empty') - - return 0 -} - -ensure_local_admin_keypair() { - mkdir -p "$SECRETS_DIR" 2>/dev/null || true - - # If generic already exists, prefer it. - if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then - # Ensure legacy mirror exists for compatibility. - cp "$ADMIN_PRIV_GENERIC" "$ADMIN_PRIV_ED25519" 2>/dev/null || true - cp "$ADMIN_PUB_GENERIC" "$ADMIN_PUB_ED25519" 2>/dev/null || true - return 0 - fi - - # If legacy exists, normalize it into generic. - if [[ -f "$ADMIN_PRIV_ED25519" && -f "$ADMIN_PUB_ED25519" ]]; then - cp "$ADMIN_PRIV_ED25519" "$ADMIN_PRIV_GENERIC" 2>/dev/null || true - cp "$ADMIN_PUB_ED25519" "$ADMIN_PUB_GENERIC" 2>/dev/null || true - chmod 0600 "$ADMIN_PRIV_GENERIC" 2>/dev/null || true - return 0 - fi - - echo "Generating CNPG admin keypair in $SECRETS_DIR ..." - # Attempt ed25519, fallback to rsa if not available - if ! openssl genpkey -algorithm ED25519 -out "$ADMIN_PRIV_GENERIC" 2>/dev/null; then - echo "WARN: ED25519 not supported by openssl, falling back to RSA 4096..." >&2 - openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out "$ADMIN_PRIV_GENERIC" - fi - openssl pkey -in "$ADMIN_PRIV_GENERIC" -pubout -out "$ADMIN_PUB_GENERIC" - chmod 0600 "$ADMIN_PRIV_GENERIC" - - # Mirror to legacy filenames for compatibility with other scripts. - cp "$ADMIN_PRIV_GENERIC" "$ADMIN_PRIV_ED25519" 2>/dev/null || true - cp "$ADMIN_PUB_GENERIC" "$ADMIN_PUB_ED25519" 2>/dev/null || true - return 0 -} - -_kubectl_is_transient_error() { - local msg="${1:-}" - # Errors we frequently see on k3s under load / during restart. - # These are worth retrying rather than proceeding with partial state. - printf '%s' "$msg" | grep -Eqi \ - '(failed to download openapi|apiserver not ready|the server is currently unable to handle the request|connect: connection refused|context deadline exceeded|i/o timeout|no endpoints available for service "cnpg-webhook-service"|failed calling webhook)' -} - -wait_for_apiserver_ready() { - local timeout=${1:-120} - local start_time elapsed - start_time=$(date +%s) - - echo "Waiting for Kubernetes API server to be ready (timeout: ${timeout}s)..." - while true; do - # Prefer /readyz when available; fall back to kubectl version. - if kubectl --request-timeout=5s get --raw='/readyz' >/dev/null 2>&1; then - return 0 - fi - if kubectl --request-timeout=5s version --short >/dev/null 2>&1; then - return 0 - fi - - elapsed=$(( $(date +%s) - start_time )) - if (( elapsed > timeout )); then - echo "ERROR: Kubernetes API server not ready after ${elapsed}s." >&2 - echo "Check kubeconfig/context and k3s status, then retry." >&2 - return 1 - fi - if (( elapsed % 20 < 6 && elapsed > 5 )); then - echo " [${elapsed}s/${timeout}s] Waiting for API server..." - fi - sleep 2 - done -} - -ensure_namespace() { - if [[ -z "${PROLE_NAMESPACE:-}" ]]; then - echo "ERROR: PROLE_NAMESPACE is empty. Set NAMESPACE in conf/knoe.cfg." >&2 - exit 1 - fi - if ! kubectl get namespace "$PROLE_NAMESPACE" >/dev/null 2>&1; then - echo "Creating namespace '$PROLE_NAMESPACE' ..." - kubectl create namespace "$PROLE_NAMESPACE" >/dev/null 2>&1 || true - fi -} - -kubectl_apply_retry() { - local namespace="${1:-}" - local attempts=${2:-8} - local sleep_s=${3:-5} - local i out - - for ((i=1; i<=attempts; i++)); do - if [[ -n "$namespace" ]]; then - if out=$(kubectl apply -n "$namespace" -f - 2>&1); then - printf '%s\n' "$out" - return 0 - fi - else - if out=$(kubectl apply -f - 2>&1); then - printf '%s\n' "$out" - return 0 - fi - fi - - if _kubectl_is_transient_error "$out"; then - echo "WARN: kubectl apply failed due to transient API/webhook issue (attempt $i/$attempts); retrying..." >&2 - echo "$out" >&2 - sleep "$sleep_s" - continue - fi - - echo "$out" >&2 - return 1 - done - - echo "ERROR: kubectl apply failed after $attempts attempts." >&2 - echo "$out" >&2 - return 1 -} - -get_latest_image() { - local pg_version_file release_file pg_version release - if [[ -f "$SCRIPT_DIR/../conf/postgresql/.version" ]]; then - pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" - elif [[ -n "${KNOE_HOME:-}" && -f "$KNOE_HOME/conf/postgresql/.version" ]]; then - pg_version_file="$KNOE_HOME/conf/postgresql/.version" - else - pg_version_file="$SCRIPT_DIR/../conf/postgresql/.version" - fi - - if [[ -f "$SCRIPT_DIR/../knoe-db/.version" ]]; then - release_file="$SCRIPT_DIR/../knoe-db/.version" - elif [[ -n "${KNOE_HOME:-}" && -f "$KNOE_HOME/knoe-db/.version" ]]; then - release_file="$KNOE_HOME/knoe-db/.version" - else - release_file="$SCRIPT_DIR/../knoe-db/.version" - fi - - if [[ -f "$pg_version_file" ]]; then - pg_version=$(tr -d '[:space:]' < "$pg_version_file") - else - pg_version="17.7" - fi - - if [[ -f "$release_file" ]]; then - release=$(tr -d '[:space:]' < "$release_file") - else - release="43" - fi - - if [[ "$release" =~ ^[0-9]+$ ]]; then - release=$(printf "%03d" "$release") - fi - - echo "knoe-db:${pg_version}-${release}" -} - -resolve_cnpg_image() { - local image="$1" - if _knoe_local_registry_enabled; then - # Prefer the in-cluster/internal registry for k3d/k3s nodes pulling images. - # Using LOCAL_REGISTRY (e.g., localhost:5000) breaks inside cluster and may - # also prefer IPv6 ::1, leading to connection refused. Avoid it. - local registry="" - - _cnpg_registry_looks_like_k3d() { - local r="${1:-}" - [[ -n "$r" ]] || return 1 - case "$r" in - k3d-*|*"/k3d-"*) return 0 ;; - esac - return 1 - } - - _cnpg_registry_is_localhostish() { - local r="${1:-}" - [[ -n "$r" ]] || return 1 - case "$r" in - localhost:5000|127.0.0.1:5000|*.localhost|*.localhost:5000) return 0 ;; - esac - return 1 - } - - _cnpg_k3s_internal_registry() { - # For k3s, never use k3d registry names or localhost-ish endpoints. - local r="${LOCAL_REGISTRY_INTERNAL:-}" - if _cnpg_registry_looks_like_k3d "$r" || _cnpg_registry_is_localhostish "$r"; then - r="" - fi - if [[ -n "$r" ]]; then - printf '%s' "$r" - return 0 - fi - - local host="" - if command -v _knoe_host_from_url >/dev/null 2>&1; then - host=$(_knoe_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") - fi - if [[ -n "${host:-}" ]]; then - printf '%s' "${host}:5000" - return 0 - fi - - local ns="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}" - printf '%s' "registry.${ns}.svc.cluster.local:5000" - } - - local _mode - _mode=$(knoe_normalize_mode "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}") - case "${_mode}" in - k3s) - registry=$(_cnpg_k3s_internal_registry) - ;; - *) - if [[ -n "${LOCAL_REGISTRY_INTERNAL:-}" ]]; then - registry="${LOCAL_REGISTRY_INTERNAL}" - fi - ;; - esac - - if [[ -z "$registry" && -n "${LOCAL_REGISTRY:-}" ]]; then - # Only fall back to LOCAL_REGISTRY when INTERNAL is not available - # and it's not pointing at localhost (which is invalid for cluster pulls). - if [[ "${LOCAL_REGISTRY}" != "localhost:5000" && "${LOCAL_REGISTRY}" != "127.0.0.1:5000" ]]; then - registry="${LOCAL_REGISTRY}" - fi - fi - - if [[ -n "$registry" ]]; then - local first="${image%%/*}" - # If the image is unqualified (no registry), prefix it with the chosen registry - if [[ "$image" != */* ]]; then - image="${registry}/${image}" - # If the first path segment has no dot/colon, it's still unqualified (e.g., knoe-db:TAG) - elif [[ "$first" != *"."* && "$first" != *":"* ]]; then - image="${registry}/${image}" - fi - fi - fi - printf '%s' "$image" -} - -sync_manifest_image() { - local image="$1" - local files=() - if [[ -n "$CNPG_MANIFEST" ]]; then - files+=("$CNPG_MANIFEST") - fi - if [[ -f "$RECOVERY_TEMPLATE" ]]; then - files+=("$RECOVERY_TEMPLATE") - fi - local f tmp - for f in "${files[@]}"; do - if [[ -f "$f" ]] && grep -qE '^[[:space:]]*imageName:' "$f"; then - tmp=$(mktemp) - sed -E "s|^([[:space:]]*imageName:).*|\\1 ${image}|" "$f" > "$tmp" - mv "$tmp" "$f" - fi - done -} - -openbao_url() { - if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then - echo "$PROLE_OPENBAO_URL" - return 0 - fi - if knoe_is_in_cluster; then - echo "http://openbao.${OPENBAO_NAMESPACE}.svc.cluster.local:8200" - return 0 - fi - - # In k3s mode, scripts run outside the cluster must reach OpenBao via the k3s host - # (never via localhost or kubectl port-forward). - if [[ "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then - if command -v _knoe_host_from_url >/dev/null 2>&1; then - local host - host=$(_knoe_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") - if [[ -n "${host:-}" ]]; then - # If the LoadBalancer has no external IP, k3s exposes the service via a nodePort. - # Prefer that nodePort when available. - if command -v kubectl >/dev/null 2>&1 && command -v jq >/dev/null 2>&1; then - local ns node_port - ns="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}" - node_port=$(kubectl -n "$ns" get svc "${OPENBAO_NAME:-openbao}" -o json 2>/dev/null | jq -r '.spec.ports[] | select(.port==8200) | .nodePort // empty' | head -n 1) - if [[ -n "${node_port:-}" && "${node_port:-}" != "null" ]]; then - echo "http://${host}:${node_port}" - return 0 - fi - fi - echo "http://${host}:8200" - return 0 - fi - fi - echo "" - return 0 - fi - - if curl -sS "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - fi - echo "" - return 0 -} - -openbao_token() { - if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then - cat "$OPENBAO_TOKEN_FILE" - else - echo "${OPENBAO_ROOT_TOKEN:-}" - fi -} - -fetch_openbao_secret() { - local path="$1" - local key="$2" - local token url - token=$(openbao_token) - url=$(openbao_url) - if [[ -z "$token" || -z "$url" ]]; then - echo "" - return 0 - fi - curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo "" -} - -resolve_db_password() { - local db_pw="${DB_PASSWORD:-}" - if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${KNOE_SECRET:'* ]]; then - local fetched_db - fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password") - if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then - db_pw="$fetched_db" - fi - fi - printf '%s' "$db_pw" -} - - -ensure_cnpg_operator() { - if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then - kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true - wait_for_cnpg_webhook 180 || true - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - tune_cnpg_operator_for_k3s || true - kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true - wait_for_cnpg_webhook 300 || true - fi - return 0 - fi - - local target_version minor_version yaml_url - target_version="${CNPG_OPERATOR_VERSION:-${CNPG_VERSION:-${CNPG_OPERATOR_FALLBACK_VERSION:-1.28.1}}}" - if [[ -z "$target_version" || "$target_version" == "latest" ]]; then - target_version=$(get_latest_cnpg_version) - fi - minor_version=$(echo "$target_version" | cut -d. -f1,2) - yaml_url="https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${minor_version}/releases/cnpg-${target_version}.yaml" - - echo "Installing CloudNative-PG operator version ${target_version} ..." - kubectl apply --server-side -f "$yaml_url" - - if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then - kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=180s || true - wait_for_cnpg_webhook 180 || true - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - tune_cnpg_operator_for_k3s || true - kubectl -n cnpg-system rollout status deploy/cnpg-controller-manager --timeout=300s || true - wait_for_cnpg_webhook 300 || true - fi - fi -} - -tune_cnpg_operator_for_k3s() { - # k3s on a single node can experience short API/server or scheduling stalls under load. - # CNPG's default probes are very aggressive (timeoutSeconds=1), which can cause flapping - # readiness and webhook endpoints disappearing mid-apply. - echo "Tuning CNPG operator deployment probes/resources for k3s ..." - kubectl -n cnpg-system patch deploy cnpg-controller-manager --type merge -p ' - { - "spec": { - "template": { - "spec": { - "containers": [ - { - "name": "manager", - "resources": { - "requests": {"cpu": "250m", "memory": "512Mi"}, - "limits": {"cpu": "500m", "memory": "1Gi"} - }, - "livenessProbe": {"timeoutSeconds": 5, "failureThreshold": 6}, - "readinessProbe": {"timeoutSeconds": 5, "failureThreshold": 6}, - "startupProbe": {"timeoutSeconds": 5, "failureThreshold": 60} - } - ] - } - } - } - }' >/dev/null 2>&1 || return 1 -} - -get_latest_barman_plugin_version() { - local version tag - tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cloudnative-pg/plugin-barman-cloud/releases/latest" | jq -r '.tag_name' || echo "") - if [[ -z "$tag" || "$tag" == "null" ]]; then - echo "v${BARMAN_PLUGIN_FALLBACK_VERSION}" - return 0 - fi - echo "$tag" -} - -resolve_barman_plugin_manifest_url() { - if [[ -n "$BARMAN_PLUGIN_MANIFEST_URL" ]]; then - echo "$BARMAN_PLUGIN_MANIFEST_URL" - return 0 - fi - local tag - tag=$(get_latest_barman_plugin_version) - echo "https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/${tag}/manifest.yaml" -} - -cert_manager_ready() { - if ! kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then - return 1 - fi - if ! kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then - return 1 - fi - return 0 -} - -resolve_control_plane_selector() { - local node="" - node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - if [[ -n "$node" ]]; then - printf 'kubernetes.io/hostname=%s' "$node" - fi -} - -deployment_node_selector_value() { - local ns="$1" - local dep="$2" - local key="$3" - kubectl -n "$ns" get deployment "$dep" -o jsonpath="{.spec.template.spec.nodeSelector['${key}']}" 2>/dev/null || true -} - -ensure_deployment_node_selector() { - local ns="$1" - local dep="$2" - local key="$3" - local value="$4" - - if ! kubectl -n "$ns" get deployment "$dep" >/dev/null 2>&1; then - return 0 - fi - - local current - current=$(deployment_node_selector_value "$ns" "$dep" "$key") - if [[ "$current" == "$value" ]]; then - return 0 - fi - - kubectl -n "$ns" patch deployment "$dep" --type merge \ - -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true -} - -pin_cert_manager() { - local selector="${CERT_MANAGER_NODE_SELECTOR:-}" - if [[ -z "$selector" && "${KNOE_MODE:-}" == "k3s" ]]; then - selector="$(resolve_control_plane_selector)" - fi - if [[ -z "$selector" ]]; then - return 0 - fi - local key value - key="${selector%%=*}" - value="${selector#*=}" - if [[ -z "$key" || -z "$value" ]]; then - echo "WARN: CERT_MANAGER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 - return 0 - fi - if [[ -z "$(kubectl get nodes -l "${key}=${value}" --no-headers 2>/dev/null)" ]]; then - echo "WARN: No nodes match CERT_MANAGER_NODE_SELECTOR=${selector}. Skipping pin." >&2 - return 0 - fi - echo "Pinning cert-manager deployments to nodes with ${selector} ..." - for dep in cert-manager cert-manager-webhook cert-manager-cainjector; do - ensure_deployment_node_selector cert-manager "$dep" "$key" "$value" - done -} - -pin_barman_cloud() { - local selector="${BARMAN_NODE_SELECTOR:-}" - if [[ -z "$selector" && "${KNOE_MODE:-}" == "k3s" ]]; then - selector="$(resolve_control_plane_selector)" - fi - if [[ -z "$selector" ]]; then - return 0 - fi - local key value - key="${selector%%=*}" - value="${selector#*=}" - if [[ -z "$key" || -z "$value" ]]; then - echo "WARN: BARMAN_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 - return 0 - fi - if [[ -z "$(kubectl get nodes -l "${key}=${value}" --no-headers 2>/dev/null)" ]]; then - echo "WARN: No nodes match BARMAN_NODE_SELECTOR=${selector}. Skipping pin." >&2 - return 0 - fi - echo "Pinning barman-cloud deployment to nodes with ${selector} ..." - ensure_deployment_node_selector cnpg-system barman-cloud "$key" "$value" -} - -get_latest_cert_manager_version() { - local tag - tag=$(curl -s --connect-timeout 5 --max-time 10 "https://api.github.com/repos/cert-manager/cert-manager/releases/latest" | jq -r '.tag_name' || echo "") - if [[ -z "$tag" || "$tag" == "null" ]]; then - echo "v${CERT_MANAGER_FALLBACK_VERSION}" - return 0 - fi - echo "$tag" -} - -resolve_cert_manager_manifest_url() { - if [[ -n "$CERT_MANAGER_MANIFEST_URL" ]]; then - echo "$CERT_MANAGER_MANIFEST_URL" - return 0 - fi - local tag - tag=$(get_latest_cert_manager_version) - echo "https://github.com/cert-manager/cert-manager/releases/download/${tag}/cert-manager.yaml" -} - -ensure_cert_manager() { - if cert_manager_ready; then - pin_cert_manager - if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then - kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true - fi - return 0 - fi - - local cm_url - cm_url=$(resolve_cert_manager_manifest_url) - echo "Installing cert-manager from ${cm_url} ..." - kubectl apply -f "$cm_url" - pin_cert_manager - - if kubectl -n cert-manager get deploy cert-manager >/dev/null 2>&1; then - kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s || true - kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s || true - fi -} - -wait_for_barman_crd() { - local timeout=${1:-120} - local start_time - start_time=$(date +%s) - echo "Waiting for Barman Cloud CRD (timeout: ${timeout}s)..." - while true; do - if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then - echo "Barman Cloud CRD is available." - return 0 - fi - local elapsed=$(( $(date +%s) - start_time )) - if (( elapsed > timeout )); then - echo "Barman Cloud CRD not ready after ${elapsed}s." >&2 - return 1 - fi - if (( elapsed % 30 < 6 && elapsed > 5 )); then - echo " [${elapsed}s/${timeout}s] Still waiting for Barman Cloud CRD..." - fi - sleep 5 - done -} - -wait_for_barman_tls_secrets() { - local timeout=${1:-180} - local start_time - start_time=$(date +%s) - echo "Waiting for Barman Cloud TLS secrets (timeout: ${timeout}s)..." - while true; do - local client_crt client_key server_crt server_key - client_crt=$(kubectl -n cnpg-system get secret barman-cloud-client-tls -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true) - client_key=$(kubectl -n cnpg-system get secret barman-cloud-client-tls -o jsonpath='{.data.tls\.key}' 2>/dev/null || true) - server_crt=$(kubectl -n cnpg-system get secret barman-cloud-server-tls -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true) - server_key=$(kubectl -n cnpg-system get secret barman-cloud-server-tls -o jsonpath='{.data.tls\.key}' 2>/dev/null || true) - if [[ -n "$client_crt" && -n "$client_key" && -n "$server_crt" && -n "$server_key" ]]; then - echo "Barman Cloud TLS secrets are available." - return 0 - fi - local elapsed=$(( $(date +%s) - start_time )) - if (( elapsed > timeout )); then - echo "Barman Cloud TLS secrets not ready after ${elapsed}s." >&2 - return 1 - fi - if (( elapsed % 30 < 6 && elapsed > 5 )); then - echo " [${elapsed}s/${timeout}s] Still waiting for Barman Cloud TLS secrets..." - fi - sleep 5 - done -} - -ensure_barman_plugin() { - ensure_cert_manager - - local plugin_url - plugin_url=$(resolve_barman_plugin_manifest_url) - echo "Installing Barman Cloud plugin from ${plugin_url} ..." - local apply_out="" - if ! apply_out=$(kubectl apply -f "$plugin_url" 2>&1); then - echo "$apply_out" >&2 - if echo "$apply_out" | grep -qi "webhook.cert-manager.io"; then - echo "WARN: cert-manager webhook error detected; restarting cert-manager components and retrying..." >&2 - kubectl -n cert-manager rollout restart deploy/cert-manager deploy/cert-manager-webhook deploy/cert-manager-cainjector >/dev/null 2>&1 || true - kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s >/dev/null 2>&1 || true - kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s >/dev/null 2>&1 || true - kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s >/dev/null 2>&1 || true - kubectl apply -f "$plugin_url" || true - fi - else - printf '%s\n' "$apply_out" - fi - - if ! wait_for_barman_crd 120; then - echo "WARN: Barman Cloud ObjectStore CRD not ready after install." >&2 - fi - - if ! wait_for_barman_tls_secrets 180; then - echo "WARN: Barman Cloud TLS secrets not ready after install." >&2 - fi - - pin_barman_cloud - - if kubectl -n cnpg-system get deploy barman-cloud >/dev/null 2>&1; then - kubectl -n cnpg-system rollout status deploy/barman-cloud --timeout=180s || true - fi -} - -pin_cnpg_controller() { - local selector="${CNPG_CONTROLLER_NODE_SELECTOR:-}" - if [[ -z "$selector" && "${KNOE_MODE:-}" == "k3s" ]]; then - if [[ -n "$(kubectl get nodes -l "storage=primary" --no-headers 2>/dev/null)" ]]; then - selector="storage=primary" - else - local cp_node="" - cp_node=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - if [[ -n "$cp_node" ]]; then - selector="kubernetes.io/hostname=${cp_node}" - fi - fi - fi - if [[ -z "$selector" ]]; then - return 0 - fi - local key value - key="${selector%%=*}" - value="${selector#*=}" - if [[ -z "$key" || -z "$value" ]]; then - echo "WARN: CNPG_CONTROLLER_NODE_SELECTOR must be key=value (got '$selector'). Skipping pin." >&2 - return 0 - fi - if [[ -z "$(kubectl get nodes -l "${key}=${value}" --no-headers 2>/dev/null)" ]]; then - echo "WARN: No nodes match CNPG_CONTROLLER_NODE_SELECTOR=${selector}. Skipping pin." >&2 - return 0 - fi - echo "Pinning cnpg-controller-manager to nodes with ${selector} ..." - kubectl -n cnpg-system patch deployment cnpg-controller-manager --type merge \ - -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"${key}\":\"${value}\"}}}}}" >/dev/null 2>&1 || true -} - -wait_for_cnpg_webhook() { - local timeout=${1:-120} - local start_time - start_time=$(date +%s) - - echo "Waiting for CNPG webhook service endpoints to be ready (timeout: ${timeout}s)..." - while true; do - local endpoints - endpoints=$(kubectl -n cnpg-system get endpoints cnpg-webhook-service -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true) - if [[ -n "$endpoints" ]]; then - echo "CNPG webhook service has endpoints." - return 0 - fi - local elapsed=$(( $(date +%s) - start_time )) - if (( elapsed > timeout )); then - echo "WARN: CNPG webhook endpoints not ready after ${elapsed}s." >&2 - kubectl -n cnpg-system get pods 2>/dev/null >&2 || true - return 1 - fi - if (( elapsed % 30 < 6 && elapsed > 5 )); then - echo " [${elapsed}s/${timeout}s] Waiting for CNPG webhook endpoints..." - kubectl -n cnpg-system get pods --no-headers 2>/dev/null | sed 's/^/ /' || true - fi - sleep 5 - done -} - -apply_barman_objectstore_if_present() { - if [[ ! -f "$BARMAN_OBJECTSTORE_MANIFEST" ]]; then - return 0 - fi - if kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1; then - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - local endpoint - endpoint="http://garage.${SERVICE_NAMESPACE}.svc.cluster.local:3900" - knoe_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" \ - | sed -E "s|^[[:space:]]*endpointURL:.*| endpointURL: ${endpoint}|" \ - | kubectl_apply_retry "$NAMESPACE" - else - knoe_render_manifest "$BARMAN_OBJECTSTORE_MANIFEST" | kubectl_apply_retry "$NAMESPACE" - fi - else - echo "WARN: Barman Cloud ObjectStore CRD not found; skipping $BARMAN_OBJECTSTORE_MANIFEST." - fi -} - -detect_root_manifest_namespace() { - # Extract the root-level `metadata.namespace` for a single YAML document. - # Note: This intentionally ignores nested `metadata:` blocks (e.g., templates). - local file="$1" - awk ' - BEGIN{in_meta=0} - /^metadata:[[:space:]]*$/ {in_meta=1; next} - in_meta && /^[[:space:]]{2}namespace:[[:space:]]*/ { - sub(/^[[:space:]]{2}namespace:[[:space:]]*/, "") - gsub(/\r/, "") - print - exit - } - in_meta && /^[^[:space:]]/ {in_meta=0} - ' "$file" | head -n 1 -} - -apply_manifest_split_by_doc() { - # Apply a potentially multi-document YAML manifest, routing each document - # to either $NAMESPACE (when no namespace is specified) or to its declared - # `metadata.namespace`. - local file="$1" - local attempts=${2:-8} - - local tmpdir - tmpdir=$(mktemp -d -t knoe-manifest-docs.XXXXXX) - - awk -v dir="$tmpdir" ' - BEGIN{i=0; f=sprintf("%s/doc-%03d.yaml", dir, i)} - /^---[[:space:]]*$/ {i++; f=sprintf("%s/doc-%03d.yaml", dir, i); next} - {print > f} - ' "$file" - - local doc doc_ns apply_ns - for doc in "$tmpdir"/doc-*.yaml; do - [[ -f "$doc" ]] || continue - # Skip empty/whitespace-only docs - if ! grep -q '[^[:space:]]' "$doc"; then - continue - fi - - doc_ns=$(detect_root_manifest_namespace "$doc") - apply_ns="$NAMESPACE" - if [[ -n "$doc_ns" ]]; then - apply_ns="$doc_ns" - if ! kubectl get namespace "$apply_ns" >/dev/null 2>&1; then - echo "Creating namespace '$apply_ns' for resources in '$(basename "$file")' ..." - kubectl create namespace "$apply_ns" >/dev/null 2>&1 || true - fi - fi - - if ! kubectl_apply_retry "$apply_ns" "$attempts" <"$doc"; then - rm -rf "$tmpdir" - return 1 - fi - done - - rm -rf "$tmpdir" - return 0 -} - -apply_knoe_manifest_file() { - local file="$1" - local attempts=${KUBECTL_APPLY_RETRIES:-8} - local i output="" - - local tmp - tmp=$(mktemp -t knoe-manifest.XXXXXX) - knoe_render_manifest "$file" >"$tmp" - - for ((i=1; i<=attempts; i++)); do - if output=$(kubectl apply -n "$NAMESPACE" -f "$tmp" 2>&1); then - printf '%s\n' "$output" - rm -f "$tmp" - return 0 - fi - - if [[ "${KNOE_MODE:-}" == "k3d" && "$(basename "$file")" == "garage-statefulset.yaml" ]] \ - && echo "$output" | grep -q "updates to statefulset spec"; then - echo "WARN: Garage StatefulSet immutable in k3d; skipping apply." - rm -f "$tmp" - return 0 - fi - - # Some manifests intentionally contain resources in other namespaces - # (e.g., dashboard/supabase ingress). If we force `-n $NAMESPACE`, kubectl - # errors with a namespace mismatch. In that case, split the manifest into - # documents and apply each doc with its correct namespace. - if printf '%s' "$output" | grep -Eqi 'namespace from the provided object.*does not match the namespace'; then - echo "WARN: Namespace mismatch applying '$(basename "$file")' with -n '$NAMESPACE'; applying per-document namespaces instead." >&2 - apply_manifest_split_by_doc "$tmp" "$attempts" - local rc=$? - rm -f "$tmp" - return $rc - fi - - if _kubectl_is_transient_error "$output"; then - echo "WARN: Failed to apply '$(basename "$file")' due to transient API/webhook issue (attempt $i/$attempts); retrying..." >&2 - echo "$output" >&2 - sleep 5 - continue - fi - - echo "$output" >&2 - rm -f "$tmp" - return 1 - done - - echo "ERROR: Failed to apply '$(basename "$file")' after $attempts attempts." >&2 - echo "$output" >&2 - rm -f "$tmp" - return 1 -} - -_knoe_k3s_registry_host_for_cache() { - # Best-effort host:port for the k3s registry, used as an optional cache source in k3d mode. - # Intentionally avoids k3d registry names and localhost-ish endpoints. - local r="${PROLE_K3S_REGISTRY:-${K3S_REGISTRY_HOST:-${LOCAL_REGISTRY_INTERNAL:-${LOCAL_REGISTRY:-}}}}" - - r="${r#http://}" - r="${r#https://}" - - case "${r}" in - "") - r="myrddin.knoe.org:5000" - ;; - localhost:5000|127.0.0.1:5000|*.localhost|*.localhost:5000) - r="myrddin.knoe.org:5000" - ;; - k3d-*|*/k3d-*) - r="myrddin.knoe.org:5000" - ;; - esac - - printf '%s' "$r" -} - -_docker_build_knoe_db_image() { - # Build the knoe-db image with BuildKit and optional cache source. - # Args: [cache_ref] - local tag="$1" - local context_dir="$2" - local cache_ref="${3:-}" - - local ssh_args=() - if [[ -n "${SSH_AUTH_SOCK:-}" ]]; then - ssh_args+=(--ssh default) - fi - - if docker buildx version >/dev/null 2>&1; then - # Prefer an isolated docker-container builder to avoid host snapshot/cache corruption. - local builder_name="${PROLE_BUILDX_BUILDER:-knoe-buildkit}" - if ! docker buildx inspect "$builder_name" >/dev/null 2>&1; then - docker buildx create --name "$builder_name" --driver docker-container --use >/dev/null 2>&1 || true - else - docker buildx use "$builder_name" >/dev/null 2>&1 || true - fi - - local cache_args=() - if [[ -n "$cache_ref" ]]; then - cache_args+=(--cache-from "type=registry,ref=${cache_ref}") - fi - - if docker buildx build \ - --pull \ - --tag "$tag" \ - --load \ - "${ssh_args[@]}" \ - "${cache_args[@]}" \ - "$context_dir"; then - return 0 - fi - fi - - # Fallback to docker build (still BuildKit-enabled); cache-from here only works if the cache image is present locally. - local cache_from_args=() - if [[ -n "$cache_ref" ]]; then - docker pull "$cache_ref" >/dev/null 2>&1 || true - cache_from_args+=(--cache-from "$cache_ref") - fi - - DOCKER_BUILDKIT=1 docker build \ - --pull \ - "${ssh_args[@]}" \ - "${cache_from_args[@]}" \ - -t "$tag" \ - "$context_dir" -} - -_push_to_k3s_registry() { - local image="$1" - local push_host="${LOCAL_REGISTRY:-${LOCAL_REGISTRY_INTERNAL:-myrddin.knoe.org:5000}}" - local plain_image="${image##*/}" - - # Normalize push_host (strip scheme if provided) - push_host="${push_host#http://}" - push_host="${push_host#https://}" - - # The in-cluster registry is deployed with hostPort:5000 (binds to 0.0.0.0:5000) - # on the control-plane node. When running on that same host, localhost:5000 is - # the most reliable path — no DNS resolution, no firewall, no port-forward needed. - # Build an ordered list of candidate endpoints to try. - local -a _push_candidates=() - _push_candidates+=("$push_host") - # Add localhost:5000 as a candidate when push_host is not already localhost-ish. - case "$push_host" in - localhost:*|127.0.0.1:*|0.0.0.0:*) ;; - *) _push_candidates+=("localhost:5000") ;; - esac - - if command -v skopeo >/dev/null 2>&1; then - for _cand in "${_push_candidates[@]}"; do - # Probe the registry endpoint before attempting a (potentially slow) push. - if command -v curl >/dev/null 2>&1; then - if ! curl -k -fsS -m 2 "https://${_cand}/v2/" >/dev/null 2>&1 \ - && ! curl -fsS -m 2 "http://${_cand}/v2/" >/dev/null 2>&1; then - echo " WARN: Registry endpoint '${_cand}' is not reachable; trying next candidate ..." >&2 - continue - fi - fi - echo " Pushing to k3s registry at '${_cand}' using skopeo ..." - if skopeo copy --dest-tls-verify=false docker-daemon:"$image" docker://"${_cand}/${plain_image}"; then - echo " ✓ Image pushed to registry at '${_cand}'." - return 0 - fi - echo " WARN: skopeo push to '${_cand}' failed; trying next candidate ..." >&2 - done - fi - - # Fallback: direct node import via SSH (no port-forward needed). - if _import_image_to_k3s_nodes "$image"; then - return 0 - fi - - # Fallback to docker push if skopeo is missing or fails (might fail if daemon not configured) - for _cand in "${_push_candidates[@]}"; do - local push_ref="${_cand}/${plain_image}" - docker tag "$image" "$push_ref" 2>/dev/null || true - if docker push "$push_ref" 2>/dev/null; then - echo " ✓ Image pushed to registry at '${_cand}' via docker push." - return 0 - fi - done - - echo "ERROR: Failed to push image '$image' to k3s registry at '${push_host}'." >&2 - return 1 -} - -# Push image to local registry (via LOCAL_REGISTRY host address) with k3d import fallback. -_push_to_k3d_registry() { - local image="$1" - local cluster_name="$2" - local push_host="${LOCAL_REGISTRY:-localhost:5000}" - local plain_image="${image##*/}" # strip registry prefix, e.g. knoe-db:18-088 - - if [[ -n "$push_host" ]]; then - local push_ref="${push_host}/${plain_image}" - docker tag "$image" "$push_ref" 2>/dev/null || true - if docker push "$push_ref" 2>/dev/null; then - echo " ✓ Image pushed to registry at '${push_host}'." - return 0 - fi - echo " WARN: push to '${push_host}' failed; falling back to k3d image import ..." >&2 - fi - - if k3d image import "$image" -c "$cluster_name" 2>/dev/null; then - echo " ✓ Image '$image' imported directly into k3d cluster '$cluster_name'." - return 0 - fi - echo "ERROR: Failed to push or import image '$image'." >&2 - return 1 -} - -# Pre-flight: ensure the knoe-db image is available in the k3d cluster before -# the CNPG operator ever tries to pull it, avoiding ErrImagePull backoff loops. -# Steps: containerd cache → Docker daemon (registry tag) → Docker daemon (plain tag) -# → tar import → docker build + push/import. -_ensure_knoe_db_image() { - if [[ "${KNOE_MODE:-}" != "k3d" && "${KNOE_MODE:-}" != "k3s" ]]; then - return 0 - fi - - local image_override="${CNPG_IMAGE:-${KNOE_DB_IMAGE:-}}" - local image="" - if [[ -n "$image_override" ]]; then - image="$image_override" - else - if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then - image=$(get_latest_image) - else - image="knoe-db:$VERSION" - fi - fi - image=$(resolve_cnpg_image "$image") - local knoe_db_dir="${KNOE_HOME:-$SCRIPT_DIR/..}/knoe-db" - local plain_image="${image##*/}" # e.g. knoe-db:18-088 - local k3s_cache_ref="" - - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - echo "Pre-flight: ensuring image '$image' is available in k3s registry/import path ..." - else - # Auto-detect active k3d cluster name - local cluster_name="${K3D_CLUSTER_NAME:-}" - if [[ -z "$cluster_name" ]]; then - cluster_name=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | head -1) - fi - cluster_name="${cluster_name:-knoe-dev-cluster}" - - # In k3d mode, optionally consult the k3s registry as a cache/source of truth before building. - local k3s_host - k3s_host=$(_knoe_k3s_registry_host_for_cache) - if [[ -n "${k3s_host:-}" ]]; then - k3s_cache_ref="${k3s_host}/${plain_image}" - fi - - echo "Pre-flight: verifying image '$image' is available in k3d cluster '$cluster_name' ..." - - # Step 1: check if already present in k3d containerd with matching digest - local containerd_digest local_digest containerd_sha - containerd_digest=$(docker exec "k3d-${cluster_name}-server-0" \ - ctr images ls -q 2>/dev/null | grep -F "$image" | head -1 || true) - if [[ -n "$containerd_digest" ]]; then - local_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$image" 2>/dev/null \ - | awk -F@ '{print $2}' || true) - containerd_sha=$(docker exec "k3d-${cluster_name}-server-0" \ - ctr images ls 2>/dev/null | grep -F "$image" | awk '{print $3}' | head -1 || true) - if [[ -z "$local_digest" || "$containerd_sha" == "$local_digest" ]]; then - echo " ✓ Image '$image' already in k3d containerd (digest match); no import needed." - return 0 - fi - echo " Image '$image' in containerd but digest mismatch (local: ${local_digest:-unknown}, containerd: ${containerd_sha:-unknown}); re-importing ..." - docker exec "k3d-${cluster_name}-server-0" ctr images rm "$image" 2>/dev/null || true - fi - fi - - # Step 2a: registry-tagged image in local Docker daemon → push + import - if docker image inspect "$image" >/dev/null 2>&1; then - echo " Image '$image' found in Docker daemon; pushing to registry ..." - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - _push_to_k3s_registry "$image" - else - _push_to_k3d_registry "$image" "$cluster_name" - fi - return $? - fi - - # Step 2b: plain-tagged image in local Docker daemon → tag + push + import - if docker image inspect "$plain_image" >/dev/null 2>&1; then - echo " Plain image '$plain_image' found in Docker daemon; tagging as '$image' and pushing ..." - docker tag "$plain_image" "$image" - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - _push_to_k3s_registry "$image" - else - _push_to_k3d_registry "$image" "$cluster_name" - fi - return $? - fi - - # Step 3: look for a matching tar in the docker-import directory - local knoe_data="${PROLE_DATA:-$HOME/.knoe/data}" - local docker_import_dir="${DOCKER_IMPORT_DIR:-${knoe_data}/docker-import}" - local name_part="${plain_image%%:*}" # e.g. knoe-db - local tag_part="${plain_image##*:}" # e.g. 18-088 - local found_tar="" - if [[ -d "$docker_import_dir" ]]; then - for _t in "$docker_import_dir"/*.tar; do - [[ -f "$_t" ]] || continue - local _bn - _bn=$(basename "$_t" .tar) - if [[ "$_bn" == *"$name_part"* && "$_bn" == *"$tag_part"* ]]; then - found_tar="$_t" - break - fi - done - fi - if [[ -n "$found_tar" ]]; then - echo " Loading tar '$(basename "$found_tar")' into Docker daemon ..." - docker load -i "$found_tar" - docker tag "$plain_image" "$image" 2>/dev/null || true - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - _push_to_k3s_registry "$image" - else - _push_to_k3d_registry "$image" "$cluster_name" - fi - return $? - fi - - # Step 3b (k3d only): try to reuse a previously-built image from the k3s registry. - if [[ "${KNOE_MODE:-}" == "k3d" && -n "${k3s_cache_ref:-}" && "${k3s_cache_ref}" != "${image}" ]]; then - if docker image inspect "$k3s_cache_ref" >/dev/null 2>&1 || docker pull "$k3s_cache_ref" >/dev/null 2>&1; then - echo " ✓ Found '$plain_image' in k3s registry (${k3s_cache_ref}); re-tagging for k3d as '$image' ..." - docker tag "$k3s_cache_ref" "$image" 2>/dev/null || true - _push_to_k3d_registry "$image" "$cluster_name" - return $? - fi - echo " (cache) '$plain_image' not present in k3s registry (${k3s_cache_ref}); building locally ..." - fi - - # Step 4: image not found anywhere — build from source then push + import - if [[ ! -f "$knoe_db_dir/Dockerfile" ]]; then - echo "ERROR: Dockerfile not found in '$knoe_db_dir'; cannot build knoe-db image." >&2 - return 1 - fi - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - echo " Image '$image' not found in Docker daemon or docker-import dir." - else - echo " Image '$image' not found in k3d, Docker daemon, or docker-import dir." - fi - echo " Building knoe-db image from '$knoe_db_dir' ..." - if ! _docker_build_knoe_db_image "$plain_image" "$knoe_db_dir" "${k3s_cache_ref:-}"; then - echo "ERROR: docker build failed for image '$plain_image'." >&2 - return 1 - fi - docker tag "$plain_image" "$image" - echo " Build complete. Pushing '$image' to registry ..." - if [[ "${KNOE_MODE:-}" == "k3s" ]]; then - _push_to_k3s_registry "$image" - else - _push_to_k3d_registry "$image" "$cluster_name" - fi - return $? -} - -cleanup_unintended_cnpg_services() { - # Historical manifest bug: we used to create a CNPG-managed Service named `knoe-db-001` - # as `type: LoadBalancer`. In k3s this spawns `svclb-knoe-db-001` pods, and it can - # interfere with CNPG startup/reconciliation. Ensure it is removed if it exists. - local svc_name="knoe-db-001" - if kubectl -n "$NAMESPACE" get svc "$svc_name" >/dev/null 2>&1; then - echo "Removing unintended Service '$svc_name' from namespace '$NAMESPACE' ..." - kubectl -n "$NAMESPACE" delete svc "$svc_name" --ignore-not-found >/dev/null 2>&1 || true - fi -} - -ensure_knoe_stack_resources() { - echo "Applying CloudNative-PG cluster and related resources ..." - wait_for_apiserver_ready 180 - ensure_knoe_protected_storage - cleanup_unintended_cnpg_services || true - local image_override="${CNPG_IMAGE:-${KNOE_DB_IMAGE:-}}" - local image="" - if [[ -n "$image_override" ]]; then - image="$image_override" - else - if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then - image=$(get_latest_image) - else - image="knoe-db:$VERSION" - fi - fi - image=$(resolve_cnpg_image "$image") - sync_manifest_image "$image" - if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then - if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then - echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2 - return 1 - fi - local dir file - dir="$K8S_PROLE_DIR" - for file in "$dir"/*.yaml; do - case "$(basename "$file")" in - knoe-db.yaml|kustomization.yaml|supabase-*.yaml|knoe-db-barman-objectstore.yaml|ingress.yaml) - continue - ;; - openbao-statefulset.yaml|openbao-service.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - garage-*.yaml|grafana-*.yaml|prometheus-*.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - kong-*.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - esac - apply_knoe_manifest_file "$file" - done - # Apply ingress.yaml without -n flag so each document targets its own namespace - if [[ -f "$dir/ingress.yaml" ]]; then - # Ensure referenced namespaces exist before applying multi-namespace ingress - for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do - if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then - echo "Creating namespace '$_ing_ns' for ingress resource ..." - kubectl create namespace "$_ing_ns" 2>/dev/null || true - fi - done - knoe_render_manifest "$dir/ingress.yaml" | kubectl_apply_retry "" || true - fi - apply_barman_objectstore_if_present - apply_cnpg_cluster_manifest "$CNPG_MANIFEST_OVERRIDE" - validate_cnpg_runtime_storage - reconcile_cnpg_instances - else - local dir file - dir="$K8S_PROLE_DIR" - for file in "$dir"/*.yaml; do - case "$(basename "$file")" in - knoe-db.yaml|kustomization.yaml|supabase-*.yaml|knoe-db-barman-objectstore.yaml|ingress.yaml) - continue - ;; - openbao-statefulset.yaml|openbao-service.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - garage-*.yaml|grafana-*.yaml|prometheus-*.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - kong-*.yaml) - if [[ -n "${SERVICE_NAMESPACE:-}" && "${SERVICE_NAMESPACE}" != "${NAMESPACE}" ]]; then - continue - fi - ;; - esac - apply_knoe_manifest_file "$file" - done - # Apply ingress.yaml without -n flag so each document targets its own namespace - if [[ -f "$dir/ingress.yaml" ]]; then - # Ensure referenced namespaces exist before applying multi-namespace ingress - for _ing_ns in $(grep -E '^\s+namespace:' "$dir/ingress.yaml" | awk '{print $2}' | sort -u); do - if ! kubectl get namespace "$_ing_ns" >/dev/null 2>&1; then - echo "Creating namespace '$_ing_ns' for ingress resource ..." - kubectl create namespace "$_ing_ns" 2>/dev/null || true - fi - done - knoe_render_manifest "$dir/ingress.yaml" | kubectl_apply_retry "" || true - fi - apply_barman_objectstore_if_present - apply_cnpg_cluster_manifest "$CNPG_MANIFEST" - validate_cnpg_runtime_storage - reconcile_cnpg_instances - fi - - # Ensure knoe-index-html exists for knoe deployment readiness probe - if ! kubectl get configmap knoe-index-html -n "$NAMESPACE" >/dev/null 2>&1; then - echo "Creating knoe-index-html configmap..." - printf "

Knoe

" > /tmp/index.html - kubectl create configmap knoe-index-html --from-file=index.html=/tmp/index.html -n "$NAMESPACE" --dry-run=client -o yaml \ - | kubectl_apply_retry "$NAMESPACE" - rm /tmp/index.html - fi - - # Ensure knoe-nginx-tls exists (self-signed for dev) - if ! kubectl get secret knoe-nginx-tls -n "$NAMESPACE" >/dev/null 2>&1; then - echo "Generating self-signed knoe-nginx-tls for development..." - openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -keyout /tmp/nginx-tls.key -out /tmp/nginx-tls.crt \ - -subj "/CN=knoe.org" >/dev/null 2>&1 - kubectl create secret tls knoe-nginx-tls --key /tmp/nginx-tls.key --cert /tmp/nginx-tls.crt -n "$NAMESPACE" --dry-run=client -o yaml \ - | kubectl_apply_retry "$NAMESPACE" - rm /tmp/nginx-tls.key /tmp/nginx-tls.crt - fi -} - -_coerce_uint() { - local raw="${1:-}" default="${2:-0}" - if [[ "$raw" =~ ^[0-9]+$ ]]; then - echo "$raw" - else - echo "$default" - fi -} - -_ready_schedulable_nodes_count() { - kubectl get nodes --no-headers 2>/dev/null \ - | awk '$2 ~ /^Ready/ && $2 !~ /SchedulingDisabled/ {c++} END {print c+0}' -} - -_ready_schedulable_db_nodes_count() { - local key="${CNPG_NODE_ROLE_LABEL_KEY:-knoe.org/node-role}" - local val="${CNPG_NODE_ROLE_LABEL_VALUE:-db}" - kubectl get nodes -l "${key}=${val}" --no-headers 2>/dev/null \ - | awk '$2 ~ /^Ready/ && $2 !~ /SchedulingDisabled/ {c++} END {print c+0}' -} - -reconcile_cnpg_instances() { - # Cap instances to cluster capacity to avoid overloading a single node during degraded startup. - # Inputs are taken from the same sources Ansible uses today: - # - node labels (e.g., `knoe.org/node-role=db`) - # - optional-workloads guard (`OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES`) - - local current - current=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.instances}' 2>/dev/null || true) - [[ -z "$current" ]] && return 0 - current=$(_coerce_uint "$current" 0) - - local min_required - min_required=$(_coerce_uint "${OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES:-2}" 2) - if (( min_required < 1 )); then - min_required=1 - fi - - local ready_nodes eligible_db_nodes - ready_nodes=$(_coerce_uint "$(_ready_schedulable_nodes_count)" 0) - eligible_db_nodes=$(_coerce_uint "$(_ready_schedulable_db_nodes_count)" 0) - if (( eligible_db_nodes < 1 )); then - echo "ERROR: No Ready, schedulable CNPG DB nodes found (label ${CNPG_NODE_ROLE_LABEL_KEY:-knoe.org/node-role}=${CNPG_NODE_ROLE_LABEL_VALUE:-db})." >&2 - echo "Refusing to force CNPG pods onto an arbitrary node during bootstrap." >&2 - return 1 - fi - - local desired - desired=$(_coerce_uint "${CNPG_INSTANCES:-3}" 3) - if (( desired < 1 )); then - desired=1 - fi - if (( desired > eligible_db_nodes )); then - desired=$eligible_db_nodes - fi - - # Degraded cluster: keep instances at 1 until enough nodes are Ready+schedulable. - if (( ready_nodes < min_required )); then - desired=1 - fi - - if (( desired < 1 )); then - desired=1 - fi - - if (( current == desired )); then - return 0 - fi - - echo "Reconciling CNPG cluster '$CNPG_CLUSTER_NAME' instances=${desired} (was ${current}; ready_nodes=${ready_nodes}; eligible_db_nodes=${eligible_db_nodes}) ..." - local attempts=${KUBECTL_PATCH_RETRIES:-8} - local i out - for ((i=1; i<=attempts; i++)); do - if out=$(kubectl -n "$NAMESPACE" patch cluster "$CNPG_CLUSTER_NAME" --type merge -p "{\"spec\":{\"instances\":${desired}}}" 2>&1); then - printf '%s\n' "$out" - return 0 - fi - if _kubectl_is_transient_error "$out"; then - echo "WARN: Failed to patch CNPG instances due to transient API issue (attempt $i/$attempts); retrying..." >&2 - echo "$out" >&2 - sleep 5 - continue - fi - echo "$out" >&2 - return 1 - done - - echo "ERROR: Failed to reconcile instances=${desired} after $attempts attempts." >&2 - echo "$out" >&2 - return 1 -} - -apply_cnpg_cluster_manifest() { - local manifest="$1" - local attempts=${CNPG_APPLY_RETRIES:-6} - local i out - - for ((i=1; i<=attempts; i++)); do - local tmp - tmp=$(mktemp -t knoe-cnpg.XXXXXX) - knoe_render_manifest "$manifest" >"$tmp" - if ! validate_cnpg_manifest_storage "$tmp"; then - rm -f "$tmp" - return 1 - fi - if out=$(kubectl apply -n "$NAMESPACE" -f "$tmp" 2>&1); then - printf '%s\n' "$out" - rm -f "$tmp" - return 0 - fi - rm -f "$tmp" - - if _kubectl_is_transient_error "$out" || echo "$out" | grep -q "cnpg-webhook-service"; then - echo "CNPG apply failed due to transient API/webhook issue (attempt $i/$attempts). Retrying..." >&2 - echo "$out" >&2 - sleep 5 - continue - fi - - echo "$out" >&2 - return 1 - done - - echo "ERROR: Failed to apply CNPG manifest after $attempts attempts." >&2 - echo "$out" >&2 - return 1 -} - -wait_for_cnpg_pods() { - local timeout=${1:-${CNPG_WAIT_TIMEOUT:-900}} - local start_time - start_time=$(date +%s) - - local target_pods="${CNPG_TARGET_PODS:-}" - if [[ -z "$target_pods" ]]; then - target_pods=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.instances}' 2>/dev/null || true) - fi - if [[ -z "$target_pods" || ! "$target_pods" =~ ^[0-9]+$ ]]; then - target_pods=1 - fi - - echo "Waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' to be Ready..." - echo " Timeout: ${timeout}s" - local last_feedback=0 - local feedback_interval=30 # print detailed status every 30s - local _image_pull_remediated="" # set after first k3d import attempt to avoid loops - while true; do - local now elapsed - now=$(date +%s) - elapsed=$(( now - start_time )) - - local pods - pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null || true) - - if [[ -n "$pods" ]]; then - # Check for Error or CrashLoopBackOff (terminal failures) - if echo "$pods" | grep -E "Error|CrashLoopBackOff" >/dev/null; then - echo "ERROR: Some CNPG pods are in Error or CrashLoopBackOff state:" >&2 - echo "$pods" | grep -E "Error|CrashLoopBackOff" >&2 - return 1 - fi - - # Check for image pull failures — attempt remediation in k3d/k3s mode - if echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >/dev/null; then - if [[ "${KNOE_MODE:-}" == "k3d" || "${KNOE_MODE:-}" == "k3s" ]]; then - if [[ -z "$_image_pull_remediated" ]]; then - _image_pull_remediated=1 - local fail_pod fail_image - fail_pod=$(echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" | awk '{print $1}' | head -1) - # Try containers then initContainers - fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \ - -o jsonpath='{.spec.containers[0].image}' 2>/dev/null || true) - if [[ -z "$fail_image" ]]; then - fail_image=$(kubectl -n "$NAMESPACE" get pod "$fail_pod" \ - -o jsonpath='{.spec.initContainers[0].image}' 2>/dev/null || true) - fi - echo "WARN: Pod '$fail_pod' cannot pull image '${fail_image:-unknown}' (ErrImagePull/ImagePullBackOff)." >&2 - import_dir="${DOCKER_IMPORT_DIR:-${PROLE_DATA:+${PROLE_DATA}/docker-import}}" - # Re-run full image ensure logic to push/import the failing image - echo " Attempting image remediation via _ensure_knoe_db_image ..." >&2 - _ensure_knoe_db_image >&2 || true - echo " Remediation complete; resuming wait ..." >&2 - fi - # Continue the wait loop — do not return 1 - else - echo "ERROR: Image pull failure in non-k3d mode — cannot auto-recover:" >&2 - echo "$pods" | grep -E "ErrImagePull|ImagePullBackOff" >&2 - return 1 - fi - fi - - # Count Ready pods by name (exclude initdb) - local ready_pods - ready_pods=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep "True" | wc -l | xargs) - - if [[ "$ready_pods" -ge "$target_pods" ]]; then - echo "All $ready_pods/$target_pods pods are Ready." - # By default, pod readiness is sufficient. Connectivity checks can be enabled explicitly. - # This avoids unnecessary timeouts/resets during initial bootstrap when CNPG may still be - # initializing even though pods are already Ready. - if kubectl cnpg version >/dev/null 2>&1; then - local require_psql="${CNPG_REQUIRE_PSQL:-false}" - if [[ "$require_psql" == "1" || "$require_psql" == "true" || "$require_psql" == "True" || "$require_psql" == "yes" || "$require_psql" == "YES" ]]; then - echo "Verifying database connectivity via 'kubectl cnpg psql'..." - if kubectl cnpg -n "$NAMESPACE" psql "$CNPG_CLUSTER_NAME" -- -c "SELECT 1" >/dev/null 2>&1; then - echo "Database connection verified." - return 0 - else - echo " [${elapsed}s/${timeout}s] Pods are Ready but database connection failed — waiting for database to accept connections..." - fi - else - # Best-effort connectivity check (warn-only) - if ! kubectl cnpg -n "$NAMESPACE" psql "$CNPG_CLUSTER_NAME" -- -c "SELECT 1" >/dev/null 2>&1; then - echo "WARN: CNPG pods are Ready but 'kubectl cnpg psql' is not yet available; continuing." >&2 - fi - return 0 - fi - else - # Fallback if no cnpg plugin: just return 0 if pods are ready - return 0 - fi - fi - - # Periodic detailed feedback - if (( now - last_feedback >= feedback_interval )); then - last_feedback=$now - echo "" - echo " [${elapsed}s/${timeout}s] Waiting — ${ready_pods:-0}/${target_pods} pods Ready" - echo " Pod status:" - echo "$pods" | sed 's/^/ /' - - # Show cluster status if available - local cluster_phase - cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true) - if [[ -n "$cluster_phase" ]]; then - echo " Cluster phase: ${cluster_phase}" - fi - - # Show recent events (last 3) - local events - events=$(kubectl -n "$NAMESPACE" get events \ - --sort-by='.lastTimestamp' \ - --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \ - -o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \ - --no-headers 2>/dev/null | tail -3 || true) - if [[ -n "$events" ]]; then - echo " Recent cluster events:" - echo "$events" | sed 's/^/ /' - fi - fi - else - # No pods yet — give feedback - if (( now - last_feedback >= feedback_interval )); then - last_feedback=$now - echo " [${elapsed}s/${timeout}s] No CNPG pods found yet for cluster '$CNPG_CLUSTER_NAME' — waiting for operator to create them..." - local cluster_phase - cluster_phase=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true) - if [[ -n "$cluster_phase" ]]; then - echo " Cluster phase: ${cluster_phase}" - fi - # If stuck in "Unable to create required cluster objects", show conditions and events for diagnosis - if [[ "$cluster_phase" == *"Unable to create"* ]]; then - echo " Cluster conditions:" - kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ - -o jsonpath='{range .status.conditions[*]} {.type}: {.status} — {.message}{"\n"}{end}' 2>/dev/null || true - echo " Recent events:" - kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \ - --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \ - -o custom-columns='TYPE:.type,REASON:.reason,MESSAGE:.message' \ - --no-headers 2>/dev/null | tail -5 | sed 's/^/ /' || true - fi - fi - fi - - if (( elapsed > timeout )); then - echo "" - echo "ERROR: Timed out after ${elapsed}s waiting for $target_pods CNPG pods for cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE'." >&2 - echo "Final pod status:" >&2 - kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" 2>/dev/null >&2 || true - echo "Recent events:" >&2 - kubectl -n "$NAMESPACE" get events --sort-by='.lastTimestamp' \ - --field-selector="involvedObject.name=${CNPG_CLUSTER_NAME}" \ - -o custom-columns='TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MESSAGE:.message' \ - --no-headers 2>/dev/null | tail -5 >&2 || true - return 1 - fi - sleep 5 - done -} - -cluster_exists() { - kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1 -} - -cluster_has_pods() { - kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --no-headers 2>/dev/null | grep -q . -} - -cluster_has_ready_pods() { - kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" \ - -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Ready")].status=="True")].metadata.name}' 2>/dev/null | grep -q . -} - -latest_backup_name() { - kubectl -n "$NAMESPACE" get backup \ - --sort-by=.metadata.creationTimestamp \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | tail -n 1 -} - -latest_completed_backup_name() { - kubectl -n "$NAMESPACE" get backup \ - --sort-by=.metadata.creationTimestamp \ - -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null | \ - awk -F'|' '{p=tolower($2); if (p=="completed" || p=="succeeded") {name=$1}} END {print name}' -} - -wait_for_backup() { - local backup_name="$1" - local start_time now phase phase_lc - start_time=$(date +%s) - - while true; do - phase=$(kubectl -n "$NAMESPACE" get backup "$backup_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) - phase_lc=$(printf '%s' "$phase" | tr '[:upper:]' '[:lower:]') - - case "$phase_lc" in - completed|succeeded) - echo "Backup $backup_name completed." - return 0 - ;; - failed|error) - echo "Backup $backup_name failed (phase=$phase)." >&2 - return 1 - ;; - esac - - now=$(date +%s) - if (( now - start_time > BACKUP_WAIT_TIMEOUT )); then - echo "Timed out waiting for backup $backup_name." >&2 - return 1 - fi - - echo "Waiting for backup $backup_name to complete (phase=${phase:-unknown}) ..." - sleep 10 - done -} - -run_garage_backup() { - if [[ ! -x "$SCRIPT_DIR/init_cnpg_backup.sh" ]]; then - echo "WARN: init_cnpg_backup.sh not found; skipping Garage backup." >&2 - return 1 - fi - - echo "Running Garage backup via init_cnpg_backup.sh ..." - if ! "$SCRIPT_DIR/init_cnpg_backup.sh" start; then - echo "Garage backup script failed." >&2 - return 1 - fi - - sleep 2 - local backup_name - backup_name=$(latest_backup_name) - if [[ -z "$backup_name" ]]; then - echo "No backup resource detected after triggering backup." >&2 - return 1 - fi - - wait_for_backup "$backup_name" -} - -get_primary_pod() { - local primary - primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) - if [[ -z "$primary" ]]; then - primary=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - fi - printf '%s' "$primary" -} - -decode_b64() { - local data="$1" - if [[ -z "$data" ]]; then - return 1 - fi - printf '%s' "$data" | base64 -d 2>/dev/null -} - -resolve_db_credentials() { - local user_b64 pass_b64 user pass - - user_b64=$(kubectl -n "$NAMESPACE" get secret knoe-db-user -o jsonpath='{.data.username}' 2>/dev/null || true) - pass_b64=$(kubectl -n "$NAMESPACE" get secret knoe-db-user -o jsonpath='{.data.password}' 2>/dev/null || true) - user=$(decode_b64 "$user_b64" || true) - pass=$(decode_b64 "$pass_b64" || true) - - if [[ -z "$user" || -z "$pass" ]]; then - user_b64=$(kubectl -n "$NAMESPACE" get secret knoe-db-superuser -o jsonpath='{.data.username}' 2>/dev/null || true) - pass_b64=$(kubectl -n "$NAMESPACE" get secret knoe-db-superuser -o jsonpath='{.data.password}' 2>/dev/null || true) - user=$(decode_b64 "$user_b64" || true) - pass=$(decode_b64 "$pass_b64" || true) - fi - - if [[ -z "$user" || -z "$pass" ]]; then - return 1 - fi - - printf '%s\n%s' "$user" "$pass" -} - -pgdump_local() { - local pod user pass db_name dump_dir dump_file timestamp - pod=$(get_primary_pod) - if [[ -z "$pod" ]]; then - echo "ERROR: No CNPG pod available for pg_dump." >&2 - return 1 - fi - - local creds - if ! creds=$(resolve_db_credentials); then - echo "ERROR: Unable to resolve database credentials for pg_dump." >&2 - return 1 - fi - user=$(printf '%s' "$creds" | sed -n '1p') - pass=$(printf '%s' "$creds" | sed -n '2p') - - db_name=${KNOE_DB_NAME:-knoe-db} - dump_dir="$BACKUP_DIR" - mkdir -p "$dump_dir" - - timestamp=$(date +%Y%m%d%H%M%S) - dump_file="$dump_dir/${CNPG_CLUSTER_NAME}-pgdump-${timestamp}.dump" - - echo "Running pg_dump against pod $pod (db=$db_name) ..." - if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- env PGPASSWORD="$pass" \ - pg_dump -U "$user" -d "$db_name" -Fc > "$dump_file"; then - echo "pg_dump saved to $dump_file" - return 0 - fi - - echo "pg_dump failed; removing partial file." >&2 - rm -f "$dump_file" - return 1 -} - -attempt_backup_if_active() { - if ! cluster_exists; then - return 0 - fi - - if ! cluster_has_pods; then - echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no pods detected; skipping backup." >&2 - return 0 - fi - - if ! cluster_has_ready_pods; then - echo "CNPG cluster '$CNPG_CLUSTER_NAME' exists but no ready pods detected; skipping backup." >&2 - return 0 - fi - - echo "CNPG cluster '$CNPG_CLUSTER_NAME' detected; attempting Garage backup ..." - if run_garage_backup; then - return 0 - fi - - echo "Garage backup failed; attempting local pg_dump ..." >&2 - if pgdump_local; then - return 0 - fi - - echo "WARN: Both Garage backup and local pg_dump failed." >&2 - return 0 -} - -wait_for_pod_ready() { - local pod="$1" - if ! kubectl -n "$NAMESPACE" wait --for=condition=Ready pod "$pod" --timeout=300s >/dev/null 2>&1; then - echo "WARN: pod $pod did not become Ready within timeout." >&2 - return 1 - fi - return 0 -} - -rollout_cluster() { - ensure_tools - ensure_namespace - echo "Starting manual recreate rollout for $CNPG_CLUSTER_NAME in $NAMESPACE..." - - local has_cnpg_plugin="0" - if kubectl cnpg version >/dev/null 2>&1; then - has_cnpg_plugin="1" - kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true - fi - - local primary - primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) - if [[ -z "$primary" ]]; then - echo "ERROR: Could not identify primary instance." >&2 - return 1 - fi - echo "Primary instance: $primary" - - local instances - instances=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true) - if [[ -z "$instances" ]]; then - echo "ERROR: No pods found for cluster $CNPG_CLUSTER_NAME." >&2 - return 1 - fi - - local pod - for pod in $instances; do - if [[ "$pod" != "$primary" ]]; then - echo "Recreating non-primary pod: $pod..." - kubectl delete pod -n "$NAMESPACE" "$pod" - wait_for_pod_ready "$pod" || true - if [[ "$has_cnpg_plugin" == "1" ]]; then - kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true - fi - fi - done - - local new_primary="" - for pod in $instances; do - if [[ "$pod" != "$primary" ]]; then - new_primary="$pod" - break - fi - done - - if [[ -n "$new_primary" && "$has_cnpg_plugin" == "1" ]]; then - echo "Promoting $new_primary..." - kubectl cnpg promote "$CNPG_CLUSTER_NAME" "$new_primary" -n "$NAMESPACE" || true - echo "Waiting for $new_primary to become primary..." - for _ in {1..60}; do - local current_primary - current_primary=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true) - if [[ "$current_primary" == "$new_primary" ]]; then - echo "$new_primary is now the primary." - break - fi - sleep 5 - done - else - if [[ -n "$new_primary" ]]; then - echo "WARN: kubectl cnpg plugin not available; skipping explicit promotion." - fi - fi - - echo "Recreating the old primary pod: $primary..." - kubectl delete pod -n "$NAMESPACE" "$primary" - wait_for_pod_ready "$primary" || true - - if [[ "$has_cnpg_plugin" == "1" ]]; then - echo "Rollout complete. Final cluster status:" - kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true - fi -} - -force_rollout() { - echo "Attempting force rollout of CNPG pods ..." - rollout_cluster -} - -reset_and_reinit() { - local backup_name tmp_manifest - backup_name=$(latest_completed_backup_name) - - echo "Resetting CNPG cluster '$CNPG_CLUSTER_NAME' in namespace '$NAMESPACE' ..." - kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found - kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true - - recycle_released_knoe_iscsi_pvs || true - - if [[ -n "$backup_name" && -f "$RECOVERY_TEMPLATE" ]]; then - tmp_manifest=$(mktemp) - sed "s/{{BACKUP_NAME}}/${backup_name}/g" "$RECOVERY_TEMPLATE" > "$tmp_manifest" - echo "Re-initializing from backup $backup_name ..." - CNPG_MANIFEST_OVERRIDE="$tmp_manifest" ensure_knoe_stack_resources - rm -f "$tmp_manifest" - else - if [[ -n "$backup_name" && ! -f "$RECOVERY_TEMPLATE" ]]; then - echo "WARN: Recovery template not found: $RECOVERY_TEMPLATE" >&2 - fi - if [[ -z "$backup_name" ]]; then - echo "No completed backups found; starting fresh initialization." >&2 - fi - ensure_knoe_stack_resources - fi - - wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}" -} - -recycle_released_knoe_iscsi_pvs() { - # StorageClass `synology-iscsi` uses `Retain` PV reclaim policy. After a CNPG reset, PVs can remain - # in `Released` with a stale `claimRef`, which prevents new PVCs (same names) from binding. - # In that case, CNPG init jobs remain `Pending` with "didn't find available persistent volumes". - echo "Recycling Released synology-iscsi PVs (clearing stale claimRefs) for namespace '$NAMESPACE' ..." - local pvs - pvs=$(kubectl get pv -o json \ - | jq -r --arg ns "$NAMESPACE" '.items[] - | select(.spec.storageClassName == "synology-iscsi") - | select(.status.phase == "Released") - | select((.spec.claimRef.namespace // "") == $ns) - | .metadata.name' 2>/dev/null || true) - - if [[ -z "${pvs:-}" ]]; then - return 0 - fi - - local pv - for pv in $pvs; do - echo " - Clearing claimRef on PV: $pv" - kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true - done -} - -# Resolve OpenBao URL: prefer explicit env, then in-cluster service. -# In k3s mode, out-of-cluster runs must use the k3s host (no localhost/port-forward). -bao_service_url() { - if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then - echo "$PROLE_OPENBAO_URL" - return 0 - fi - if knoe_is_in_cluster; then - local ns="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}" - echo "http://$OPENBAO_NAME.$ns.svc.cluster.local:8200" - return 0 - fi - - if [[ "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}" == "k3s" ]]; then - if command -v _knoe_host_from_url >/dev/null 2>&1; then - local host - host=$(_knoe_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}") - if [[ -n "${host:-}" ]]; then - # If the LoadBalancer has no external IP, k3s exposes the service via a nodePort. - # Prefer that nodePort when available. - if command -v kubectl >/dev/null 2>&1 && command -v jq >/dev/null 2>&1; then - local ns node_port - ns="${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}}" - node_port=$(kubectl -n "$ns" get svc "${OPENBAO_NAME:-openbao}" -o json 2>/dev/null | jq -r '.spec.ports[] | select(.port==8200) | .nodePort // empty' | head -n 1) - if [[ -n "${node_port:-}" && "${node_port:-}" != "null" ]]; then - echo "http://${host}:${node_port}" - return 0 - fi - fi - echo "http://${host}:8200" - return 0 - fi - fi - echo "" - return 0 - fi - - if curl -sS --connect-timeout 2 "http://127.0.0.1:8200/v1/sys/health" >/dev/null 2>&1; then - echo "http://127.0.0.1:8200" - return 0 - fi - echo "" -} - -fetch_admin_keys_and_db_pass_from_bao_or_local() { - local priv_b64 pub_b64 - priv_b64=$(fetch_openbao_secret "$BAO_PATH_ADMIN" "admin_private_key_b64") - pub_b64=$(fetch_openbao_secret "$BAO_PATH_ADMIN" "admin_public_key_b64") - if [[ -n "${priv_b64:-}" && "${priv_b64:-}" != "null" && -n "${pub_b64:-}" && "${pub_b64:-}" != "null" ]]; then - echo "Attempting to read admin key pair from OpenBao kv/$BAO_PATH_ADMIN ..." - - # Prefer the generic filenames. - printf "%s" "$priv_b64" | base64 -d >"$ADMIN_PRIV_GENERIC" - printf "%s" "$pub_b64" | base64 -d >"$ADMIN_PUB_GENERIC" - chmod 0600 "$ADMIN_PRIV_GENERIC" - - # Mirror to legacy path for compatibility. - cp "$ADMIN_PRIV_GENERIC" "$ADMIN_PRIV_ED25519" 2>/dev/null || true - cp "$ADMIN_PUB_GENERIC" "$ADMIN_PUB_ED25519" 2>/dev/null || true - fi - - local db_pass db_user - db_pass=$(resolve_db_password) - db_user="${KNOE_DB_USER:-knoe}" - if [[ -n "${db_pass:-}" && "${db_pass:-}" != "null" ]]; then - echo "Ensuring database user secret 'knoe-db-user' ..." - kubectl create secret generic knoe-db-user -n "$NAMESPACE" \ - --from-literal=username="$db_user" \ - --from-literal=password="$db_pass" \ - --dry-run=client -o yaml | kubectl_apply_retry "$NAMESPACE" - - echo "Ensuring database superuser secret 'knoe-db-superuser' ..." - kubectl create secret generic knoe-db-superuser -n "$NAMESPACE" \ - --from-literal=username=postgres \ - --from-literal=password="$db_pass" \ - --dry-run=client -o yaml | kubectl_apply_retry "$NAMESPACE" - fi - - # No env fallback: if OpenBao is unreachable and secrets are missing, fail clearly - - if ! kubectl -n "$NAMESPACE" get secret knoe-db-user >/dev/null 2>&1; then - echo "ERROR: 'knoe-db-user' secret is missing in namespace '$NAMESPACE' and could not be resolved from OpenBao or local DB_PASSWORD." >&2 - return 1 - fi - - ensure_local_admin_keypair - if [[ -f "$ADMIN_PRIV_GENERIC" && -f "$ADMIN_PUB_GENERIC" ]]; then - echo "Using local admin key pair at $SECRETS_DIR" - return 0 - fi - - echo "ERROR: Could not obtain admin key pair from OpenBao and could not generate local keys." >&2 - return 1 -} - -apply_cnpg_admin_secret() { - echo "Creating/updating Secret cnpg-admin-key ..." - kubectl create secret generic cnpg-admin-key -n "$NAMESPACE" \ - --from-file=admin.key="$ADMIN_PRIV_GENERIC" \ - --from-file=admin.pub="$ADMIN_PUB_GENERIC" \ - --dry-run=client -o yaml | kubectl_apply_retry "$NAMESPACE" -} - -detect_and_reprovision_unencrypted_cluster() { - local encryption_enabled="${AT_REST_ENCRYPTION_ENABLED:-false}" - if [[ "$encryption_enabled" != "true" && "$encryption_enabled" != "True" && "$encryption_enabled" != "1" ]]; then - return 0 - fi - - if ! kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then - return 0 - fi - - local server_tls - server_tls=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.certificates.serverTLSSecret}' 2>/dev/null || echo "") - if [[ -n "$server_tls" ]]; then - echo "Existing cluster '$CNPG_CLUSTER_NAME' already has TLS configured ($server_tls); no conversion needed." - return 0 - fi - - echo "WARN: Existing cluster '$CNPG_CLUSTER_NAME' does not have TLS/at-rest-encryption configured." - echo "At-rest encryption is now enabled; the cluster must be reprovisioned." - echo "NOTE: External full backup and restore from barman backup is not yet available." - echo " Removing existing cluster to allow re-creation with encryption enabled ..." - - attempt_backup_if_active - - kubectl -n "$NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found - kubectl -n "$NAMESPACE" wait --for=delete pod -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" --timeout=180s >/dev/null 2>&1 || true - echo "Existing unencrypted cluster removed. Will be re-created with TLS and encryption." -} - -validate_ca_secret() { - local secret_name="$1" - local ns="$2" - if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then - return 1 - fi - local crt key - crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true) - key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.ca\.key}' 2>/dev/null || true) - if [[ -z "$crt" || -z "$key" ]]; then - echo "WARN: CA secret '$secret_name' exists but is missing ca.crt or ca.key data." >&2 - return 1 - fi - if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then - echo "WARN: CA secret '$secret_name' has invalid certificate data." >&2 - return 1 - fi - # CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8 - local key_header - key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1) - if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then - echo "WARN: CA secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2 - return 1 - fi - return 0 -} - -validate_tls_secret() { - local secret_name="$1" - local ns="$2" - if ! kubectl -n "$ns" get secret "$secret_name" >/dev/null 2>&1; then - return 1 - fi - local crt key - crt=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true) - key=$(kubectl -n "$ns" get secret "$secret_name" -o jsonpath='{.data.tls\.key}' 2>/dev/null || true) - if [[ -z "$crt" || -z "$key" ]]; then - echo "WARN: TLS secret '$secret_name' exists but is missing tls.crt or tls.key data." >&2 - return 1 - fi - if ! echo "$crt" | base64 -d 2>/dev/null | openssl x509 -noout 2>/dev/null; then - echo "WARN: TLS secret '$secret_name' has invalid certificate data." >&2 - return 1 - fi - # CNPG machinery expects EC PRIVATE KEY (ECDSA P-256), not RSA or PKCS#8 - local key_header - key_header=$(echo "$key" | base64 -d 2>/dev/null | head -1) - if [[ "$key_header" != *"BEGIN EC PRIVATE KEY"* ]]; then - echo "WARN: TLS secret '$secret_name' key is not EC format (found: $key_header); CNPG requires EC PRIVATE KEY." >&2 - return 1 - fi - return 0 -} - -generate_tls_if_missing() { - local ca_secret_name="${CNPG_CLUSTER_NAME}-ca" - local tls_secret_name="${CNPG_CLUSTER_NAME}-tls" - local need_ca=0 - local need_tls=0 - - if validate_ca_secret "$ca_secret_name" "$NAMESPACE"; then - echo "CA secret $ca_secret_name already exists and is valid." - else - if kubectl -n "$NAMESPACE" get secret "$ca_secret_name" >/dev/null 2>&1; then - echo "Removing invalid CA secret '$ca_secret_name' ..." - kubectl -n "$NAMESPACE" delete secret "$ca_secret_name" --ignore-not-found - fi - need_ca=1 - fi - - if validate_tls_secret "$tls_secret_name" "$NAMESPACE"; then - echo "TLS secret $tls_secret_name already exists and is valid." - else - if kubectl -n "$NAMESPACE" get secret "$tls_secret_name" >/dev/null 2>&1; then - echo "Removing invalid TLS secret '$tls_secret_name' ..." - kubectl -n "$NAMESPACE" delete secret "$tls_secret_name" --ignore-not-found - fi - need_tls=1 - fi - - if [[ "$need_ca" -eq 0 && "$need_tls" -eq 0 ]]; then - echo "TLS secrets already present and valid; skipping generation." - return 0 - fi - - local TMPD - TMPD=$(mktemp -d) - - if [[ "$need_ca" -eq 1 ]]; then - echo "Generating self-signed CA (EC P-256) for CNPG ..." - openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/ca.key" - openssl req -x509 -new -key "$TMPD/ca.key" -out "$TMPD/ca.crt" -days 3650 -subj "/CN=Knoe CNPG CA" - kubectl -n "$NAMESPACE" create secret generic "$ca_secret_name" \ - --from-file=ca.crt="$TMPD/ca.crt" \ - --from-file=ca.key="$TMPD/ca.key" \ - --dry-run=client -o yaml | kubectl_apply_retry "$NAMESPACE" - else - # Extract existing CA for signing the server certificate - kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.crt}' | base64 -d > "$TMPD/ca.crt" - kubectl -n "$NAMESPACE" get secret "$ca_secret_name" -o jsonpath='{.data.ca\.key}' | base64 -d > "$TMPD/ca.key" - fi - - if [[ "$need_tls" -eq 1 ]]; then - echo "Generating server TLS certificate for CNPG ($tls_secret_name) ..." - openssl ecparam -name prime256v1 -genkey -noout -out "$TMPD/tls.key" - openssl req -new -key "$TMPD/tls.key" -out "$TMPD/tls.csr" \ - -subj "/CN=${CNPG_CLUSTER_NAME}.${NAMESPACE}.svc" - openssl x509 -req -in "$TMPD/tls.csr" -CA "$TMPD/ca.crt" -CAkey "$TMPD/ca.key" \ - -CAcreateserial -out "$TMPD/tls.crt" -days 365 \ - -extfile <(printf "subjectAltName=DNS:%s,DNS:%s-rw,DNS:%s-rw.%s.svc,DNS:%s-r,DNS:%s-ro" \ - "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME" "$NAMESPACE" \ - "$CNPG_CLUSTER_NAME" "$CNPG_CLUSTER_NAME") - kubectl -n "$NAMESPACE" create secret tls "$tls_secret_name" \ - --cert="$TMPD/tls.crt" \ - --key="$TMPD/tls.key" \ - --dry-run=client -o yaml | kubectl_apply_retry "$NAMESPACE" - fi - - rm -rf "$TMPD" -} - -# Resolve latest CNPG version from GitHub if possible, fallback to a sensible default. -get_latest_cnpg_version() { - local version - version=$(curl -s "https://api.github.com/repos/cloudnative-pg/cloudnative-pg/releases/latest" | jq -r '.tag_name' | sed 's/^v//' || echo "") - if [[ -z "$version" || "$version" == "null" ]]; then - echo "1.27.0" - else - echo "$version" - fi -} - -initialize() { - ensure_tools - ensure_namespace - wait_for_apiserver_ready 180 - attempt_backup_if_active - ensure_cnpg_operator - pin_cnpg_controller - ensure_barman_plugin - echo "Using namespace: $NAMESPACE" - - if ! fetch_admin_keys_and_db_pass_from_bao_or_local; then - echo "ERROR: Failed to fetch/ensure database secrets and admin keys." >&2 - return 1 - fi - apply_cnpg_admin_secret - detect_and_reprovision_unencrypted_cluster - generate_tls_if_missing - - # Pre-flight: verify required secrets exist before deploying cluster - local _missing_secrets="" - if ! kubectl -n "$NAMESPACE" get secret knoe-db-user >/dev/null 2>&1; then - _missing_secrets="${_missing_secrets} knoe-db-user" - fi - if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-tls" >/dev/null 2>&1; then - _missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-tls" - fi - if ! kubectl -n "$NAMESPACE" get secret "${CNPG_CLUSTER_NAME}-ca" >/dev/null 2>&1; then - _missing_secrets="${_missing_secrets} ${CNPG_CLUSTER_NAME}-ca" - fi - if [[ -n "$_missing_secrets" ]]; then - echo "ERROR: Required secrets missing in namespace '$NAMESPACE':${_missing_secrets}" >&2 - echo "The CNPG cluster cannot start without these secrets." >&2 - return 1 - fi - echo "Pre-flight check passed: all required secrets present." - - if ! _ensure_knoe_db_image; then - echo "ERROR: Pre-flight image check failed; aborting cluster initialization." >&2 - return 1 - fi - - ensure_knoe_stack_resources - - if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then - if cluster_has_pods; then - echo "WARN: CNPG pods exist but did not become ready; attempting force rollout ..." >&2 - if force_rollout; then - if ! wait_for_cnpg_pods "${CNPG_WAIT_TIMEOUT}"; then - echo "WARN: Force rollout did not recover CNPG; attempting reset and re-init ..." >&2 - if ! reset_and_reinit; then - return 1 - fi - fi - else - echo "WARN: Force rollout failed; attempting reset and re-init ..." >&2 - if ! reset_and_reinit; then - return 1 - fi - fi - else - echo "WARN: No CNPG pods found — this is a new installation." >&2 - echo "The CNPG operator was unable to create cluster pods." >&2 - echo "Check operator logs: kubectl -n cnpg-system logs -l app.kubernetes.io/name=cloudnative-pg" >&2 - echo "Check cluster status: kubectl -n $NAMESPACE describe cluster $CNPG_CLUSTER_NAME" >&2 - return 1 - fi - fi - - echo "Ensuring CNPG continuous backups (ObjectStore + Cluster backup config + ScheduledBackup) ..." - "$SCRIPT_DIR/init_cnpg_backup.sh" start - - echo "Initialization complete for CNPG + cert artifacts." - knoe_register_port_forward "postgres" "${NAMESPACE:-default}" "svc/${CNPG_CLUSTER_NAME}-rw" "5432" "5432" "0.0.0.0" "TCP" "PostgreSQL" -} - -update_reload() { - initialize -} - -deploy_cluster() { - ensure_tools - ensure_namespace - ensure_cnpg_operator - pin_cnpg_controller - ensure_barman_plugin - - ensure_knoe_protected_storage - - local image - if [[ "$VERSION" == "latest" || -z "$VERSION" ]]; then - image=$(get_latest_image) - else - image="knoe-db:$VERSION" - fi - image=$(resolve_cnpg_image "$image") - sync_manifest_image "$image" - - generate_tls_if_missing - - echo "Deploying $image to cluster $CNPG_CLUSTER_NAME in namespace $NAMESPACE..." - _ensure_knoe_db_image || true - local manifest - manifest="$CNPG_MANIFEST" - if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then - manifest="$CNPG_MANIFEST_OVERRIDE" - fi - if [[ ! -f "$manifest" ]]; then - echo "ERROR: CNPG manifest not found at $manifest" >&2 - return 1 - fi - apply_cnpg_cluster_manifest "$manifest" - validate_cnpg_runtime_storage - reconcile_cnpg_instances - - local current_image - current_image=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.imageName}' 2>/dev/null || echo "") - if [[ -n "$current_image" && "$current_image" != "$image" ]]; then - echo "Patching cluster to use image '$image' (was '$current_image')..." - kubectl -n "$NAMESPACE" patch cluster "$CNPG_CLUSTER_NAME" --type merge -p "{\"spec\": {\"imageName\": \"$image\"}}" - if kubectl cnpg version >/dev/null 2>&1; then - kubectl cnpg restart "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" || true - fi - fi -} - -preflight_image() { - ensure_tools - ensure_namespace - _ensure_knoe_db_image -} - -case "$ACTION" in - preflight-image|image-preflight|preflight_image) - preflight_image - ;; - recreate) - ensure_tools - "$0" delete "$CNPG_CLUSTER_NAME" - # Clean up PVCs left behind by the CNPG operator (not in static manifests) - echo "Cleaning up PVCs in namespace '$NAMESPACE' ..." - kubectl -n "$NAMESPACE" delete pvc --all --ignore-not-found 2>/dev/null || true - # Wait briefly for pods to fully terminate before re-creating - echo "Waiting for pods to terminate in namespace '$NAMESPACE' ..." - _wait_term=0 - while kubectl -n "$NAMESPACE" get pods --no-headers 2>/dev/null | grep -qv '^No resources'; do - sleep 3 - _wait_term=$(( _wait_term + 3 )) - if (( _wait_term >= 60 )); then - echo "WARN: Pods still present after 60s; proceeding anyway." >&2 - break - fi - done - "$0" create "$CNPG_CLUSTER_NAME" - ;; - create) - ensure_tools - ensure_namespace - initialize - ;; - delete) - ensure_tools - echo "Deleting all resources for '$CNPG_CLUSTER_NAME' ..." - # Delete each manifest individually, mirroring the apply pattern: - # ingress.yaml contains multi-namespace resources and must be deleted without -n. - for _del_f in "$K8S_PROLE_DIR"/*.yaml; do - _del_base=$(basename "$_del_f") - case "$_del_base" in - kustomization.yaml|ingress.yaml) continue ;; - *) kubectl delete -n "$NAMESPACE" -f "$_del_f" --ignore-not-found 2>&1 \ - | grep -v "^Error from server (NotFound)" || true ;; - esac - done - if [[ -f "$K8S_PROLE_DIR/ingress.yaml" ]]; then - kubectl delete -f "$K8S_PROLE_DIR/ingress.yaml" --ignore-not-found 2>&1 \ - | grep -v "^Error from server (NotFound)" || true - fi - ;; - start) - ensure_tools - ensure_namespace - ensure_cnpg_operator - pin_cnpg_controller - ensure_barman_plugin - if [[ ! -f "$CNPG_MANIFEST" ]]; then - echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2 - exit 1 - fi - echo "Starting CloudNative-PG cluster from $CNPG_MANIFEST in namespace $NAMESPACE..." - knoe_render_manifest "$CNPG_MANIFEST" | kubectl apply -n "$NAMESPACE" -f - - reconcile_cnpg_instances - ;; - stop) - ensure_tools - if [[ ! -f "$CNPG_MANIFEST" ]]; then - echo "ERROR: CNPG manifest not found at $CNPG_MANIFEST" >&2 - exit 1 - fi - echo "Stopping CloudNative-PG cluster using $CNPG_MANIFEST ..." - kubectl delete -f "$CNPG_MANIFEST" --ignore-not-found - ;; - status) - ensure_tools - echo "--- CloudNative-PG Cluster Status ($CNPG_CLUSTER_NAME) ---" - if kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then - kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" - echo "" - echo "CNPG Plugin Status:" - if kubectl cnpg version >/dev/null 2>&1; then - kubectl cnpg status "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" - else - echo "Note: 'kubectl cnpg' plugin not found; skipping detailed status." - fi - else - echo "Cluster '$CNPG_CLUSTER_NAME' not found in namespace '$NAMESPACE'." - fi - ;; - restart) - ensure_tools - "$0" stop - "$0" start - ;; - initialize) - if ! initialize; then - exit 1 - fi - ;; - update|reload) - if ! update_reload; then - exit 1 - fi - ;; - deploy) - deploy_cluster - ;; - rollout|force-rollout) - rollout_cluster - ;; - install-barman-plugin) - ensure_tools - ensure_cnpg_operator - pin_cnpg_controller - ensure_barman_plugin - ;; - *) - echo "Usage: $0 {create|delete|recreate|start|stop|status|restart|initialize|update|reload|deploy|rollout|install-barman-plugin} [cluster] [version]" >&2 - exit 2 - ;; -esac diff --git a/mock_val/init_cnpg_backup.sh b/mock_val/init_cnpg_backup.sh index a889466..65722a8 100755 --- a/mock_val/init_cnpg_backup.sh +++ b/mock_val/init_cnpg_backup.sh @@ -24,30 +24,28 @@ fi ACTION=${1:-start} -NAMESPACE="${PROLE_NAMESPACE}" +NAMESPACE="${PROLE_NAMESPACE:-${DATABASE_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-db}}}" CNPG_CLUSTER_NAME=${CNPG_CLUSTER_NAME:-knoe-db} CNPG_OPERATOR_NAMESPACE=${CNPG_OPERATOR_NAMESPACE:-cnpg-system} GARAGE_NAME=${GARAGE_NAME:-garage} SERVICE_NAMESPACE=${SERVICE_NAMESPACE:-} -if [[ -z "${GARAGE_NAMESPACE:-}" ]]; then - if [[ -n "$SERVICE_NAMESPACE" ]]; then - GARAGE_NAMESPACE="$SERVICE_NAMESPACE" - else - GARAGE_NAMESPACE="$NAMESPACE" - fi -fi +GARAGE_NAMESPACE=${GARAGE_NAMESPACE:-} GARAGE_BACKUP_BUCKET=${GARAGE_BACKUP_BUCKET:-knoe-db-backups} GARAGE_BACKUP_KEY_NAME=${GARAGE_BACKUP_KEY_NAME:-knoe-db-backup} GARAGE_BACKUP_SECRET_NAME=${GARAGE_BACKUP_SECRET_NAME:-knoe-db-barman-s3} -GARAGE_S3_ENDPOINT=${GARAGE_S3_ENDPOINT:-http://$GARAGE_NAME.$GARAGE_NAMESPACE.svc.cluster.local:3900} +GARAGE_S3_ENDPOINT=${GARAGE_S3_ENDPOINT:-} GARAGE_S3_REGION=${GARAGE_S3_REGION:-garage} RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1} RETENTION_POLICY=${RETENTION_POLICY:-30d} +GARAGE_LAYOUT_BOOTSTRAP_ENABLED=${GARAGE_LAYOUT_BOOTSTRAP_ENABLED:-1} BARMAN_PLUGIN_NAME=${BARMAN_PLUGIN_NAME:-barman-cloud.cloudnative-pg.io} BARMAN_OBJECT_NAME=${BARMAN_OBJECT_NAME:-knoe-db-barman-objectstore} BACKUP_STATUS_TIMEOUT=${BACKUP_STATUS_TIMEOUT:-600} BACKUP_STATUS_INTERVAL=${BACKUP_STATUS_INTERVAL:-10} PLUGIN_READY_TIMEOUT=${PLUGIN_READY_TIMEOUT:-180} +OBJECTSTORE_READY_TIMEOUT=${OBJECTSTORE_READY_TIMEOUT:-180} +OBJECTSTORE_READY_INTERVAL=${OBJECTSTORE_READY_INTERVAL:-5} +KUBECTL_CONTEXT_OVERRIDE=${KUBECTL_CONTEXT_OVERRIDE:-${DB_CLUSTER_KUBECONTEXT:-${KUBECTL_CONTEXT:-${KUBECONTEXT:-}}}} SCHEDULED_BACKUP_NAME=${SCHEDULED_BACKUP_NAME:-knoe-db-scheduled-backup} SCHEDULED_BACKUP_CRON=${SCHEDULED_BACKUP_CRON:-"0 3 * * *"} @@ -76,6 +74,19 @@ ensure_tools() { done } +select_kube_context() { + local target_context="${KUBECTL_CONTEXT_OVERRIDE:-}" + if [[ -z "$target_context" ]]; then + return 0 + fi + + if kubectl config get-contexts "$target_context" >/dev/null 2>&1; then + kubectl config use-context "$target_context" >/dev/null + else + echo "WARN: Requested kubecontext '$target_context' not found; continuing with current context." >&2 + fi +} + _kubectl_is_transient_error() { local msg="${1:-}" printf '%s' "$msg" | grep -Eqi \ @@ -179,6 +190,34 @@ barman_crd_ready() { kubectl get crd objectstores.barmancloud.cnpg.io >/dev/null 2>&1 } +garage_pod_exists_in_namespace() { + local ns="${1:-}" + local pod_name + [[ -z "$ns" ]] && return 1 + pod_name=$(kubectl get pods -n "$ns" -l "app=$GARAGE_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + [[ -n "$pod_name" ]] +} + +resolve_garage_namespace_and_endpoint() { + if [[ -z "$GARAGE_NAMESPACE" ]]; then + if [[ -n "$SERVICE_NAMESPACE" ]] && garage_pod_exists_in_namespace "$SERVICE_NAMESPACE"; then + GARAGE_NAMESPACE="$SERVICE_NAMESPACE" + elif garage_pod_exists_in_namespace "knoe-system"; then + GARAGE_NAMESPACE="knoe-system" + elif garage_pod_exists_in_namespace "$NAMESPACE"; then + GARAGE_NAMESPACE="$NAMESPACE" + elif [[ -n "$SERVICE_NAMESPACE" ]]; then + GARAGE_NAMESPACE="$SERVICE_NAMESPACE" + else + GARAGE_NAMESPACE="$NAMESPACE" + fi + fi + + if [[ -z "$GARAGE_S3_ENDPOINT" ]]; then + GARAGE_S3_ENDPOINT="http://$GARAGE_NAME.$GARAGE_NAMESPACE.svc.cluster.local:3900" + fi +} + get_garage_pod() { kubectl get pods -n "$GARAGE_NAMESPACE" -l "app=$GARAGE_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true } @@ -193,6 +232,29 @@ garage_exec() { kubectl exec -n "$GARAGE_NAMESPACE" "$pod" -- /garage "$@" } +bootstrap_garage_layout_if_needed() { + if [[ "${GARAGE_LAYOUT_BOOTSTRAP_ENABLED}" != "1" ]]; then + return 0 + fi + + local init_script="$SCRIPT_DIR/init_garage_store.sh" + if [[ ! -f "$init_script" ]]; then + echo "WARN: Garage bootstrap helper not found: $init_script" >&2 + return 0 + fi + + echo "Garage layout still not applied; invoking Garage bootstrap recovery ..." + if [[ -n "${KNOE_MODE:-}" ]]; then + if ! bash "$init_script" --mode "$KNOE_MODE" start; then + echo "WARN: Garage bootstrap recovery failed (continuing wait loop)." >&2 + fi + else + if ! bash "$init_script" start; then + echo "WARN: Garage bootstrap recovery failed (continuing wait loop)." >&2 + fi + fi +} + parse_key_output() { local output="$1" local access_key secret_key @@ -207,6 +269,7 @@ parse_key_output() { ensure_garage_ready() { echo "Checking Garage readiness ..." local i status_out + local bootstrap_attempted=0 for i in {1..30}; do if status_out=$(garage_exec status 2>/dev/null); then # If layout is applied, DataAvail should eventually show something or at least the node should be healthy. @@ -219,6 +282,10 @@ ensure_garage_ready() { return 0 fi fi + if [[ "$bootstrap_attempted" -eq 0 && "$i" -ge 6 ]]; then + bootstrap_garage_layout_if_needed + bootstrap_attempted=1 + fi echo "Waiting for Garage layout to be applied... ($i/30)" sleep 5 done @@ -292,13 +359,88 @@ spec: OBJECTSTORE } +objectstore_spec_matches_expected() { + local destination endpoint retention + local access_key_secret access_key_name secret_key_secret secret_key_name + local region_secret region_name + + destination=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.destinationPath}' 2>/dev/null || true) + endpoint=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.endpointURL}' 2>/dev/null || true) + retention=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.retentionPolicy}' 2>/dev/null || true) + access_key_secret=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.s3Credentials.accessKeyId.name}' 2>/dev/null || true) + access_key_name=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.s3Credentials.accessKeyId.key}' 2>/dev/null || true) + secret_key_secret=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.s3Credentials.secretAccessKey.name}' 2>/dev/null || true) + secret_key_name=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.s3Credentials.secretAccessKey.key}' 2>/dev/null || true) + region_secret=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.s3Credentials.region.name}' 2>/dev/null || true) + region_name=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.spec.configuration.s3Credentials.region.key}' 2>/dev/null || true) + + [[ "$destination" == "s3://$GARAGE_BACKUP_BUCKET/" ]] || return 1 + [[ "$endpoint" == "$GARAGE_S3_ENDPOINT" ]] || return 1 + [[ "$retention" == "$RETENTION_POLICY" ]] || return 1 + [[ "$access_key_secret" == "$GARAGE_BACKUP_SECRET_NAME" ]] || return 1 + [[ "$access_key_name" == "ACCESS_KEY_ID" ]] || return 1 + [[ "$secret_key_secret" == "$GARAGE_BACKUP_SECRET_NAME" ]] || return 1 + [[ "$secret_key_name" == "SECRET_ACCESS_KEY" ]] || return 1 + [[ "$region_secret" == "$GARAGE_BACKUP_SECRET_NAME" ]] || return 1 + [[ "$region_name" == "REGION" ]] || return 1 + return 0 +} + +ensure_barman_object_store_config() { + local attempts=${OBJECTSTORE_RECONCILE_ATTEMPTS:-3} + local i + + for (( i=1; i<=attempts; i++ )); do + apply_barman_object_store + if objectstore_spec_matches_expected; then + return 0 + fi + echo "WARN: ObjectStore '$BARMAN_OBJECT_NAME' spec does not match expected Garage configuration (attempt ${i}/${attempts}); reapplying ..." + sleep 2 + done + + echo "ERROR: ObjectStore '$BARMAN_OBJECT_NAME' does not match expected Garage configuration after ${attempts} attempts." >&2 + kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o yaml 2>/dev/null >&2 || true + return 1 +} + ensure_barman_plugin_config() { - local plugin_names plugin_present + local plugin_names plugin_present barman_obj_name plugin_enabled plugin_wal_archiver plugin_names=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.plugins[*].name}' 2>/dev/null || true) plugin_present=$(printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -F "$BARMAN_PLUGIN_NAME" || true) - if [[ -n "$plugin_present" ]]; then - return 0 + # Plugin name present — verify it's fully active for WAL archiving and object store routing. + barman_obj_name=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ + -o jsonpath="{range .spec.plugins[?(@.name==\"$BARMAN_PLUGIN_NAME\")]}{.parameters.barmanObjectName}{end}" 2>/dev/null || true) + plugin_enabled=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ + -o jsonpath="{range .spec.plugins[?(@.name==\"$BARMAN_PLUGIN_NAME\")]}{.enabled}{end}" 2>/dev/null || true) + plugin_wal_archiver=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ + -o jsonpath="{range .spec.plugins[?(@.name==\"$BARMAN_PLUGIN_NAME\")]}{.isWALArchiver}{end}" 2>/dev/null || true) + + if [[ "$barman_obj_name" == "$BARMAN_OBJECT_NAME" && "${plugin_enabled,,}" == "true" && "${plugin_wal_archiver,,}" == "true" ]]; then + return 0 + fi + echo "Updating Barman plugin entry: ensuring enabled=true, isWALArchiver=true, and barmanObjectName=$BARMAN_OBJECT_NAME ..." + local idx + idx=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ + -o json 2>/dev/null \ + | python3 -c "import sys,json; items=json.load(sys.stdin)['spec'].get('plugins',[]); print(next((i for i,p in enumerate(items) if p.get('name')=='barman-cloud.cloudnative-pg.io'),-1))") + if [[ "$idx" -ge 0 ]]; then + kubectl patch cluster "$CNPG_CLUSTER_NAME" -n "$NAMESPACE" --type json -p "[ + { + \"op\": \"replace\", + \"path\": \"/spec/plugins/$idx\", + \"value\": { + \"enabled\": true, + \"name\": \"$BARMAN_PLUGIN_NAME\", + \"isWALArchiver\": true, + \"parameters\": {\"barmanObjectName\": \"$BARMAN_OBJECT_NAME\"} + } + } + ]" + return 0 + fi + # Fallback: append a correctly configured entry fi local plugins_json @@ -405,10 +547,11 @@ SCHEDULEDBACKUP wait_for_plugin_ready() { local start_time now plugin_names deployment_rows ns ready replicas - local plugin_deploy_ready pod socket_path + local plugin_deploy_ready plugin_ns svc_ns svc_name client_secret server_secret plugin_port start_time=$(date +%s) while true; do plugin_deploy_ready=0 + plugin_ns="" plugin_names=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.plugins[*].name}' 2>/dev/null || true) if printf '%s\n' "$plugin_names" | tr ' ' '\n' | grep -Fxq "$BARMAN_PLUGIN_NAME"; then deployment_rows=$(kubectl get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) @@ -422,22 +565,37 @@ wait_for_plugin_ready() { replicas=${replicas:-0} if (( ready >= 1 && replicas >= 1 )); then plugin_deploy_ready=1 + plugin_ns="$ns" break fi done <<< "$deployment_rows" if (( plugin_deploy_ready == 1 )); then - pod=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME,cnpg.io/instanceRole=primary" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - if [[ -z "$pod" ]]; then - pod=$(kubectl -n "$NAMESPACE" get pods -l "cnpg.io/cluster=$CNPG_CLUSTER_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + # The Barman Cloud plugin is deployed as a standalone CNPG-I plugin. + # In this model, there is no Unix socket expected inside the database pod. + # Instead, CNPG discovers the plugin via a Kubernetes Service annotated with + # cnpg.io/pluginClientSecret, cnpg.io/pluginServerSecret and cnpg.io/pluginPort. + # + # We validate that registration artifact exists and that the referenced TLS + # secrets are present. The subsequent backup operation will fail if the plugin + # is still not usable. + + svc_ns="${plugin_ns:-$CNPG_OPERATOR_NAMESPACE}" + svc_name=$(kubectl -n "$svc_ns" get svc -l "cnpg.io/pluginName=$BARMAN_PLUGIN_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$svc_name" ]]; then + # Fallback for older manifests that don't set the cnpg.io/pluginName label + svc_name=$(kubectl -n "$svc_ns" get svc barman-cloud -o jsonpath='{.metadata.name}' 2>/dev/null || true) fi - socket_path="$PLUGIN_SOCKET_DIR/$BARMAN_PLUGIN_NAME" - if [[ -n "$pod" ]]; then - if kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- test -S "$socket_path" >/dev/null 2>&1; then - return 0 - fi - if kubectl -n "$NAMESPACE" exec "$pod" -c manager -- test -S "$socket_path" >/dev/null 2>&1; then - return 0 + + if [[ -n "$svc_name" ]]; then + client_secret=$(kubectl -n "$svc_ns" get svc "$svc_name" -o jsonpath='{.metadata.annotations.cnpg\.io/pluginClientSecret}' 2>/dev/null || true) + server_secret=$(kubectl -n "$svc_ns" get svc "$svc_name" -o jsonpath='{.metadata.annotations.cnpg\.io/pluginServerSecret}' 2>/dev/null || true) + plugin_port=$(kubectl -n "$svc_ns" get svc "$svc_name" -o jsonpath='{.metadata.annotations.cnpg\.io/pluginPort}' 2>/dev/null || true) + + if [[ -n "$client_secret" && -n "$server_secret" && -n "$plugin_port" ]]; then + if kubectl -n "$svc_ns" get secret "$client_secret" >/dev/null 2>&1 && kubectl -n "$svc_ns" get secret "$server_secret" >/dev/null 2>&1; then + return 0 + fi fi fi fi @@ -454,6 +612,165 @@ wait_for_plugin_ready() { done } +wait_for_barman_plugin_infra_ready() { + # Ensure the plugin controller and its registration artifacts (Service annotations + TLS secrets) + # are present *before* patching the CNPG Cluster to depend on the plugin. + local start_time now + local deployment_rows ns ready replicas + local plugin_deploy_ready plugin_ns svc_ns svc_name client_secret server_secret plugin_port + start_time=$(date +%s) + + while true; do + plugin_deploy_ready=0 + plugin_ns="" + + deployment_rows=$(kubectl get deployment -A -l app.kubernetes.io/name=barman-cloud -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) + if [[ -z "$deployment_rows" ]]; then + deployment_rows=$(kubectl get deployment -A -o jsonpath='{range .items[?(@.metadata.name=="barman-cloud")]}{.metadata.namespace}{"\t"}{.status.readyReplicas}{"\t"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) + fi + + while IFS=$'\t' read -r ns ready replicas; do + [[ -z "$ns" ]] && continue + ready=${ready:-0} + replicas=${replicas:-0} + if (( ready >= 1 && replicas >= 1 )); then + plugin_deploy_ready=1 + plugin_ns="$ns" + break + fi + done <<< "$deployment_rows" + + if (( plugin_deploy_ready == 1 )); then + svc_ns="${plugin_ns:-$CNPG_OPERATOR_NAMESPACE}" + svc_name=$(kubectl -n "$svc_ns" get svc -l "cnpg.io/pluginName=$BARMAN_PLUGIN_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$svc_name" ]]; then + svc_name=$(kubectl -n "$svc_ns" get svc barman-cloud -o jsonpath='{.metadata.name}' 2>/dev/null || true) + fi + + if [[ -n "$svc_name" ]]; then + client_secret=$(kubectl -n "$svc_ns" get svc "$svc_name" -o jsonpath='{.metadata.annotations.cnpg\.io/pluginClientSecret}' 2>/dev/null || true) + server_secret=$(kubectl -n "$svc_ns" get svc "$svc_name" -o jsonpath='{.metadata.annotations.cnpg\.io/pluginServerSecret}' 2>/dev/null || true) + plugin_port=$(kubectl -n "$svc_ns" get svc "$svc_name" -o jsonpath='{.metadata.annotations.cnpg\.io/pluginPort}' 2>/dev/null || true) + + if [[ -n "$client_secret" && -n "$server_secret" && -n "$plugin_port" ]]; then + if kubectl -n "$svc_ns" get secret "$client_secret" >/dev/null 2>&1 && kubectl -n "$svc_ns" get secret "$server_secret" >/dev/null 2>&1; then + return 0 + fi + fi + fi + fi + + now=$(date +%s) + if (( now - start_time >= PLUGIN_READY_TIMEOUT )); then + echo "ERROR: Timed out waiting for Barman plugin infrastructure to become ready." >&2 + return 1 + fi + + echo "Waiting for Barman plugin infrastructure (deployment + service registration) ..." + sleep 5 + done +} + +continuous_archiving_condition_line() { + kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ + -o jsonpath='{range .status.conditions[?(@.type=="ContinuousArchiving")]}{.status}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' 2>/dev/null || true +} + +wait_for_continuous_archiving_ready() { + local timeout=${1:-$PLUGIN_READY_TIMEOUT} + local start_time now elapsed + local reconcile_count=0 + local reconcile_limit=${PLUGIN_BACKUP_RETRY_LIMIT:-6} + + start_time=$(date +%s) + while true; do + local ca_line ca_status ca_reason ca_message ca_status_lc ca_reason_lc ca_message_lc + ca_line=$(continuous_archiving_condition_line) + ca_status="" + ca_reason="" + ca_message="" + + if [[ -n "$ca_line" ]]; then + IFS=$'\t' read -r ca_status ca_reason ca_message <<< "$ca_line" + fi + + ca_status_lc=$(echo "${ca_status:-}" | tr '[:upper:]' '[:lower:]') + ca_reason_lc=$(echo "${ca_reason:-}" | tr '[:upper:]' '[:lower:]') + ca_message_lc=$(echo "${ca_message:-}" | tr '[:upper:]' '[:lower:]') + + if [[ "$ca_status_lc" == "true" ]]; then + return 0 + fi + + if [[ "$ca_reason_lc" == *"continuousarchivingfailing"* || "$ca_message_lc" == *"wal archive plugin is not available"* || "$ca_message_lc" == *"plugin is not available"* ]]; then + if (( reconcile_count < reconcile_limit )); then + reconcile_count=$((reconcile_count + 1)) + echo "WARN: ContinuousArchiving reports plugin unavailable. Reconciling plugin config (${reconcile_count}/${reconcile_limit}) ..." + ensure_barman_plugin_config || true + wait_for_plugin_ready || true + fi + fi + + now=$(date +%s) + elapsed=$((now - start_time)) + if (( elapsed >= timeout )); then + echo "ERROR: Timed out waiting for ContinuousArchiving to become ready (status=${ca_status:-} reason=${ca_reason:-} message=${ca_message:-})." >&2 + return 1 + fi + + sleep 5 + done +} + +wait_for_objectstore_ready() { + local timeout=${1:-$OBJECTSTORE_READY_TIMEOUT} + local start_time now elapsed + start_time=$(date +%s) + + echo "Waiting for ObjectStore '$BARMAN_OBJECT_NAME' to be Ready (timeout: ${timeout}s)..." + while true; do + local ready_cond phase + ready_cond=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" \ + -o jsonpath='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}' 2>/dev/null || true) + phase=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" \ + -o jsonpath='{.status.phase}' 2>/dev/null || true) + + # barman-cloud v0.11+ signals readiness via serverRecoveryWindow rather than + # a Ready condition; accept either form. + local recovery_window + recovery_window=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" \ + -o jsonpath='{.status.serverRecoveryWindow}' 2>/dev/null || true) + if [[ "$ready_cond" == "True" || "$ready_cond" == "true" || "$phase" == "Ready" || "$phase" == "ready" || -n "$recovery_window" ]]; then + echo "ObjectStore '$BARMAN_OBJECT_NAME' is Ready." + return 0 + fi + + now=$(date +%s) + elapsed=$((now - start_time)) + + # Some plugin/controller versions do not currently set Ready/phase status on + # ObjectStore. In that case, continue only when the ObjectStore exists, + # barman-cloud controller is available, and the ObjectStore spec still matches + # the expected Garage-backed configuration. + local object_exists barman_available generation + object_exists=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o name 2>/dev/null || true) + barman_available=$(kubectl -n "$CNPG_OPERATOR_NAMESPACE" get deploy barman-cloud -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true) + generation=$(kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o jsonpath='{.metadata.generation}' 2>/dev/null || true) + if [[ -n "$object_exists" && "$barman_available" =~ ^[1-9][0-9]*$ && "$generation" =~ ^[0-9]+$ && $elapsed -ge 30 ]] && objectstore_spec_matches_expected; then + echo "WARN: ObjectStore status fields are not populated, but barman-cloud is available and ObjectStore spec matches expected Garage configuration; continuing." + return 0 + fi + + if (( elapsed >= timeout )); then + echo "ERROR: Timed out waiting for ObjectStore '$BARMAN_OBJECT_NAME' to become Ready." >&2 + kubectl -n "$NAMESPACE" get objectstore "$BARMAN_OBJECT_NAME" -o yaml 2>/dev/null >&2 || true + return 1 + fi + + sleep "$OBJECTSTORE_READY_INTERVAL" + done +} + has_successful_base_backup() { local rows name phase method backup_type phase_lc method_lc backup_type_lc rows=$(kubectl get backup -n "$NAMESPACE" -o jsonpath="{range .items[?(@.spec.cluster.name=='$CNPG_CLUSTER_NAME')]}{.metadata.name}{\"\t\"}{.status.phase}{\"\t\"}{.spec.method}{\"\t\"}{.spec.pluginConfiguration.parameters.backupType}{\"\n\"}{end}" 2>/dev/null || true) @@ -484,6 +801,9 @@ has_successful_base_backup() { wait_for_successful_base_backup() { local start_time now elapsed + local plugin_retry_count=0 + local plugin_retry_limit=${PLUGIN_BACKUP_RETRY_LIMIT:-6} + local plugin_retry_interval=${PLUGIN_BACKUP_RETRY_INTERVAL:-20} start_time=$(date +%s) while true; do if [[ -n "$LAST_BACKUP_NAME" ]]; then @@ -495,12 +815,55 @@ wait_for_successful_base_backup() { return 0 ;; Failed|failed) + local err_lc + err_lc=$(echo "${err:-}" | tr '[:upper:]' '[:lower:]') + if [[ "$err_lc" == *"requested plugin is not available"* || "$err_lc" == *"wal archive plugin is not available"* || "$err_lc" == *"plugin is not available"* ]] && (( plugin_retry_count < plugin_retry_limit )); then + plugin_retry_count=$((plugin_retry_count + 1)) + echo "WARN: Backup '$LAST_BACKUP_NAME' failed because plugin is not yet available. Retry ${plugin_retry_count}/${plugin_retry_limit} ..." + wait_for_plugin_ready || true + local retry_sleep=$((plugin_retry_interval * plugin_retry_count)) + if (( retry_sleep > 120 )); then + retry_sleep=120 + fi + if (( retry_sleep > 0 )); then + echo "Waiting ${retry_sleep}s before retrying backup trigger ..." + sleep "$retry_sleep" + fi + LAST_BACKUP_NAME="" + trigger_backup + sleep 5 + continue + fi echo "ERROR: Backup '$LAST_BACKUP_NAME' failed: ${err:-}" >&2 return 1 ;; esac fi + local ca_line ca_status ca_reason ca_message ca_status_lc ca_reason_lc ca_message_lc + ca_line=$(continuous_archiving_condition_line) + ca_status="" + ca_reason="" + ca_message="" + if [[ -n "$ca_line" ]]; then + IFS=$'\t' read -r ca_status ca_reason ca_message <<< "$ca_line" + fi + + ca_status_lc=$(echo "${ca_status:-}" | tr '[:upper:]' '[:lower:]') + ca_reason_lc=$(echo "${ca_reason:-}" | tr '[:upper:]' '[:lower:]') + ca_message_lc=$(echo "${ca_message:-}" | tr '[:upper:]' '[:lower:]') + if [[ "$ca_status_lc" != "true" && ( "$ca_reason_lc" == *"continuousarchivingfailing"* || "$ca_message_lc" == *"wal archive plugin is not available"* || "$ca_message_lc" == *"plugin is not available"* ) ]] && (( plugin_retry_count < plugin_retry_limit )); then + plugin_retry_count=$((plugin_retry_count + 1)) + echo "WARN: ContinuousArchiving is failing due to plugin availability. Retry ${plugin_retry_count}/${plugin_retry_limit} ..." + ensure_barman_plugin_config || true + wait_for_plugin_ready || true + wait_for_continuous_archiving_ready "$PLUGIN_READY_TIMEOUT" || true + LAST_BACKUP_NAME="" + trigger_backup + sleep 5 + continue + fi + if has_successful_base_backup; then return 0 fi @@ -558,6 +921,7 @@ BACKUP status() { ensure_tools + select_kube_context echo "CNPG backup status for cluster '$CNPG_CLUSTER_NAME' (namespace: '$NAMESPACE')" local backup_spec @@ -573,7 +937,7 @@ status() { fi local ca_line - ca_line=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{range .status.conditions[?(@.type=="ContinuousArchiving")]}{.status}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' 2>/dev/null || true) + ca_line=$(continuous_archiving_condition_line) if [[ -n "$ca_line" ]]; then echo "ContinuousArchiving condition:" printf '%s' "$ca_line" | awk -F$'\t' '{print "- status=" $1 (length($2)?" reason=" $2:"") (length($3)?" message=" $3:"")}' @@ -599,14 +963,29 @@ status() { case "$ACTION" in start) ensure_tools + select_kube_context ensure_namespace wait_for_apiserver_ready 180 ensure_cluster + # Fast-path: skip re-initialization when continuous archiving is already healthy + # and at least one successful base backup exists. All the wait_for_* loops below + # are idempotent but slow — no need to re-run them on every installer pass. + _ca_status_fp=$(kubectl -n "$NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" \ + -o jsonpath='{range .status.conditions[?(@.type=="ContinuousArchiving")]}{.status}{end}' \ + 2>/dev/null || true) + if [[ "$_ca_status_fp" == "True" ]] && has_successful_base_backup; then + echo "[CNPG BACKUP] Continuous archiving healthy and base backup exists — skipping re-initialization." + exit 0 + fi + resolve_garage_namespace_and_endpoint ensure_garage_bucket_and_key - apply_barman_object_store - ensure_barman_plugin_config + wait_for_barman_plugin_infra_ready + ensure_barman_object_store_config + wait_for_objectstore_ready ensure_cluster_backup_config + ensure_barman_plugin_config wait_for_plugin_ready + wait_for_continuous_archiving_ready ensure_scheduled_backup if [[ "$RUN_FIRST_BACKUP" == "1" ]]; then wait_for_cnpg_webhook 300 @@ -623,17 +1002,23 @@ case "$ACTION" in ;; backup) ensure_tools + select_kube_context ensure_namespace wait_for_apiserver_ready 180 ensure_cluster - apply_barman_object_store - ensure_barman_plugin_config + resolve_garage_namespace_and_endpoint + wait_for_barman_plugin_infra_ready + ensure_barman_object_store_config + wait_for_objectstore_ready ensure_cluster_backup_config + ensure_barman_plugin_config wait_for_plugin_ready + wait_for_continuous_archiving_ready wait_for_cnpg_webhook 300 trigger_backup "${2:-full}" ;; status) + select_kube_context wait_for_apiserver_ready 60 || true status ;; diff --git a/mock_val/init_cnpg_gke.sh b/mock_val/init_cnpg_gke.sh new file mode 100755 index 0000000..056fd68 --- /dev/null +++ b/mock_val/init_cnpg_gke.sh @@ -0,0 +1,521 @@ +#!/usr/bin/env bash +# init_cnpg_gke.sh +# Provision CloudNativePG on GKE with GCS backup via Workload Identity. +# +# Usage: +# ./etc/init_cnpg_gke.sh [--project PROJECT_ID] [--region REGION] [--cluster CLUSTER_NAME] +# +# Env vars (override flags): +# GCP_PROJECT_ID, GCP_REGION, GKE_CLUSTER, GCS_BACKUP_BUCKET, +# CNPG_NAMESPACE, ARTIFACT_REGISTRY, DB_PASSWORD, CLOUDSDK_AUTH_ACCESS_TOKEN + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GKE_MANIFEST_DIR="$REPO_ROOT/deploy/gcp/gke" + +# ── Defaults ──────────────────────────────────────────────────────────────── +GCP_PROJECT_ID="${GCP_PROJECT_ID:-}" +GCP_REGION="${GCP_REGION:-us-central1}" +GKE_CLUSTER="${GKE_CLUSTER:-knoe-dev-0}" +GCS_BACKUP_BUCKET="${GCS_BACKUP_BUCKET:-knoe-0-backups}" +GCS_WAL_BUCKET="${GCS_WAL_BUCKET:-knoe-0-wal}" +CNPG_NAMESPACE="${CNPG_NAMESPACE:-knoe-db-0}" +CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-knoe-db}" +CNPG_BACKUP_SA="${CNPG_BACKUP_SA:-cnpg-backup}" +ARTIFACT_REGISTRY="${ARTIFACT_REGISTRY:-}" +KNOE_DB_IMAGE_TAG="${KNOE_DB_IMAGE_TAG:-}" +DB_PASSWORD="${DB_PASSWORD:-}" +CNPG_DB_OWNER="${CNPG_DB_OWNER:-knoe}" +CNPG_STORAGE_CLASS_ORDER="${CNPG_STORAGE_CLASS_ORDER:-premium-rwo,ssd,premium,standard-rwo,garage-hdd,standard,dynamic-rwo}" +CNPG_STORAGE_WAIT_TIMEOUT="${CNPG_STORAGE_WAIT_TIMEOUT:-240}" +# v1.29.0+ is required: spec.serviceAccountName lets the cluster pods run as +# cnpg-backup-sa (annotated with iam.gke.io/gcp-service-account for WI), which +# is how the GCS-backed barman ObjectStore authenticates without a static key. +# Older operators (we used 1.24.0 historically) silently drop the field. +CNPG_OPERATOR_VERSION="${CNPG_OPERATOR_VERSION:-1.29.0}" + +# ── Argument parsing ───────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --project) GCP_PROJECT_ID="$2"; shift 2 ;; + --region) GCP_REGION="$2"; shift 2 ;; + --cluster) GKE_CLUSTER="$2"; shift 2 ;; + --bucket) GCS_BACKUP_BUCKET="$2"; shift 2 ;; + --namespace) CNPG_NAMESPACE="$2"; shift 2 ;; + --registry) ARTIFACT_REGISTRY="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +[[ -z "$GCP_PROJECT_ID" ]] && { + # Try to read from gcp.cfg + GCP_CFG="$REPO_ROOT/conf/prod/gcp.cfg" + if [[ -f "$GCP_CFG" ]]; then + GCP_PROJECT_ID=$(grep '^project_id' "$GCP_CFG" | sed 's/.*=\s*"\?\([^"]*\)"\?.*/\1/' | tr -d '[:space:]') + fi +} +[[ -z "$GCP_PROJECT_ID" ]] && { echo "ERROR: GCP_PROJECT_ID not set and conf/prod/gcp.cfg not found." >&2; exit 1; } + +GCP_SA_EMAIL="${CNPG_BACKUP_SA}@${GCP_PROJECT_ID}.iam.gserviceaccount.com" + +log() { echo "[init_cnpg_gke] $*"; } + +resolve_knoe_db_image_tag() { + [[ -n "$KNOE_DB_IMAGE_TAG" ]] && return + + local mode_key="k8s" + local knoe_home + knoe_home="${KNOE_HOME:-$REPO_ROOT}" + + local pg_version_file="$knoe_home/modes/$mode_key/conf/postgresql/.version" + local release_file="$knoe_home/modes/$mode_key/knoe-db/.version" + + [[ -f "$pg_version_file" ]] || pg_version_file="$knoe_home/conf/postgresql/.version" + [[ -f "$release_file" ]] || release_file="$knoe_home/knoe-db/.version" + + local pg_version release + if [[ -f "$pg_version_file" ]]; then + pg_version="$(tr -d '[:space:]' < "$pg_version_file")" + else + pg_version="17.7" + fi + + if [[ -f "$release_file" ]]; then + release="$(tr -d '[:space:]' < "$release_file")" + else + release="43" + fi + + if [[ "$release" =~ ^[0-9]+$ ]]; then + release="$(printf '%03d' "$release")" + fi + + KNOE_DB_IMAGE_TAG="${pg_version}-${release}" +} + +storage_class_available() { + local candidate="$1" + [[ -n "$candidate" ]] || return 1 + kubectl get storageclass "$candidate" >/dev/null 2>&1 +} + +append_storage_candidate() { + local candidate="$1" + [[ -n "$candidate" ]] || return + case ",${CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED}," in + *",${candidate},"*) return ;; + esac + if [[ -n "$CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED" ]]; then + CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED+="${IFS_COMMA}${candidate}" + else + CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED="$candidate" + fi +} + +storage_class_is_compatible() { + local candidate="$1" + case "$candidate" in + *rwx*|*RWX*|gcsfuse*|parallelstore-*|enterprise-rwx|enterprise-multishare-rwx|regional-rwx|premium-rwx|standard-rwx|synology-*|*iscsi*) + return 1 + ;; + *) + return 0 + ;; + esac +} + +append_by_name_pattern() { + local pattern="$1" + local sc + while IFS= read -r sc; do + [[ -n "$sc" ]] || continue + storage_class_is_compatible "$sc" || continue + case "$sc" in + *"$pattern"*) append_storage_candidate "$sc" ;; + esac + done <<< "$AVAILABLE_STORAGE_CLASSES" +} + +resolve_storage_class_candidates() { + AVAILABLE_STORAGE_CLASSES="$(kubectl get storageclass -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)" + [[ -n "$AVAILABLE_STORAGE_CLASSES" ]] || { + echo "ERROR: unable to enumerate Kubernetes storage classes." >&2 + exit 1 + } + + CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED="" + IFS=',' read -r -a requested_classes <<< "$CNPG_STORAGE_CLASS_ORDER" + local requested + for requested in "${requested_classes[@]}"; do + requested="${requested//[[:space:]]/}" + [[ -n "$requested" ]] || continue + case "$requested" in + ssd) + append_by_name_pattern "ssd" + ;; + premium) + append_by_name_pattern "premium" + ;; + *) + if storage_class_available "$requested" && storage_class_is_compatible "$requested"; then + append_storage_candidate "$requested" + fi + ;; + esac + done + + local sc + while IFS= read -r sc; do + [[ -n "$sc" ]] || continue + storage_class_is_compatible "$sc" || continue + append_storage_candidate "$sc" + done <<< "$AVAILABLE_STORAGE_CLASSES" + + [[ -n "$CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED" ]] || { + echo "ERROR: no compatible storage classes available for CNPG provisioning." >&2 + exit 1 + } +} + +render_cnpg_manifest() { + local storage_class="$1" + local rendered_manifest="$2" + ARTIFACT_REGISTRY="${ARTIFACT_REGISTRY}" KNOE_DB_IMAGE_TAG="${KNOE_DB_IMAGE_TAG}" \ + envsubst '${ARTIFACT_REGISTRY} ${KNOE_DB_IMAGE_TAG}' < "$GKE_MANIFEST_DIR/knoe-db.yaml" > "$rendered_manifest" + sed -E "s#^([[:space:]]*storageClassName:).*#\\1 ${storage_class}#" "$rendered_manifest" > "${rendered_manifest}.tmp" + mv "${rendered_manifest}.tmp" "$rendered_manifest" +} + +cnpg_attempt_ready() { + local expected_storage_class="$1" + local pfx current_storage_class + pfx="${CNPG_CLUSTER_NAME}-" + current_storage_class="$(kubectl -n "$CNPG_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.spec.storage.pvcTemplate.storageClassName}' 2>/dev/null || true)" + [[ "$current_storage_class" == "$expected_storage_class" ]] || return 1 + + kubectl -n "$CNPG_NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.storageClassName}{"|"}{.status.phase}{"\n"}{end}' 2>/dev/null \ + | awk -F'|' -v expected="$expected_storage_class" -v prefix="$pfx" '$1 ~ ("^" prefix) && $2 == expected && $3 == "Bound" { found=1 } END { exit(found ? 0 : 1) }' +} + +cnpg_attempt_has_pvc_for_class() { + local expected_storage_class="$1" + local pfx + pfx="${CNPG_CLUSTER_NAME}-" + kubectl -n "$CNPG_NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.storageClassName}{"\n"}{end}' 2>/dev/null \ + | awk -F'|' -v expected="$expected_storage_class" -v prefix="$pfx" '$1 ~ ("^" prefix) && $2 == expected { found=1 } END { exit(found ? 0 : 1) }' +} + +cnpg_recent_events() { + kubectl -n "$CNPG_NAMESPACE" get events --sort-by=.lastTimestamp 2>/dev/null | tail -n 80 || true +} + +wait_for_storage_outcome() { + local expected_storage_class="$1" + local started_at current_ts events + started_at="$(date +%s)" + + while true; do + if cnpg_attempt_ready "$expected_storage_class"; then + return 0 + fi + + if cnpg_attempt_has_pvc_for_class "$expected_storage_class"; then + events="$(cnpg_recent_events)" + if printf '%s\n' "$events" | grep -q "SSD_TOTAL_GB"; then + return 10 + fi + if printf '%s\n' "$events" | grep -Eqi "storageclass.*not found|failed to provision volume"; then + return 11 + fi + fi + + current_ts="$(date +%s)" + if (( current_ts - started_at >= CNPG_STORAGE_WAIT_TIMEOUT )); then + return 1 + fi + sleep 10 + done +} + +reset_cnpg_cluster_attempt() { + kubectl -n "$CNPG_NAMESPACE" delete cluster "$CNPG_CLUSTER_NAME" --ignore-not-found=true --wait=true --timeout=180s >/dev/null 2>&1 || true + kubectl -n "$CNPG_NAMESPACE" delete job -l cnpg.io/cluster="$CNPG_CLUSTER_NAME" --ignore-not-found=true >/dev/null 2>&1 || true + kubectl -n "$CNPG_NAMESPACE" delete pvc -l cnpg.io/cluster="$CNPG_CLUSTER_NAME" --ignore-not-found=true >/dev/null 2>&1 || true + kubectl -n "$CNPG_NAMESPACE" delete pod -l cnpg.io/cluster="$CNPG_CLUSTER_NAME" --ignore-not-found=true >/dev/null 2>&1 || true +} + +ensure_cnpg_tls_secrets() { + local cluster_name="knoe-db" + if kubectl -n "$CNPG_NAMESPACE" get secret "${cluster_name}-ca" >/dev/null 2>&1 \ + && kubectl -n "$CNPG_NAMESPACE" get secret "${cluster_name}-tls" >/dev/null 2>&1; then + local ca_key_b64 + local ca_key_pem + ca_key_b64="$(kubectl -n "$CNPG_NAMESPACE" get secret "${cluster_name}-ca" -o jsonpath='{.data.ca\.key}' 2>/dev/null || true)" + ca_key_pem="$(printf '%s' "$ca_key_b64" | openssl base64 -d -A 2>/dev/null || true)" + if [[ -n "$ca_key_b64" && "$ca_key_pem" == *"BEGIN EC PRIVATE KEY"* ]]; then + log "CNPG TLS secrets already present in ${CNPG_NAMESPACE}." + return + fi + log "CNPG CA secret is missing/invalid for CNPG in ${CNPG_NAMESPACE}; regenerating CNPG TLS secrets ..." + fi + + log "Bootstrapping CNPG TLS secrets in ${CNPG_NAMESPACE} ..." + local tmp_dir + tmp_dir="$(mktemp -d)" + local fqdn cn + fqdn="${cluster_name}-rw.${CNPG_NAMESPACE}.svc.cluster.local" + cn="$fqdn" + if [[ ${#cn} -gt 64 ]]; then + cn="${cluster_name}-rw" + fi + + openssl ecparam -name prime256v1 -genkey -noout -out "$tmp_dir/ca.key" >/dev/null 2>&1 + openssl req -x509 -new -nodes -key "$tmp_dir/ca.key" -sha256 -days 3650 \ + -subj "/CN=${cluster_name}-ca" -out "$tmp_dir/ca.crt" >/dev/null 2>&1 + + openssl ecparam -name prime256v1 -genkey -noout -out "$tmp_dir/tls.key" >/dev/null 2>&1 + openssl req -new -key "$tmp_dir/tls.key" -subj "/CN=${cn}" -out "$tmp_dir/tls.csr" >/dev/null 2>&1 + + cat > "$tmp_dir/ext.cnf" </dev/null 2>&1 + + kubectl -n "$CNPG_NAMESPACE" create secret generic "${cluster_name}-ca" \ + --from-file=ca.crt="$tmp_dir/ca.crt" \ + --from-file=ca.key="$tmp_dir/ca.key" \ + --dry-run=client -o yaml | kubectl apply -f - + + kubectl -n "$CNPG_NAMESPACE" create secret tls "${cluster_name}-tls" \ + --cert="$tmp_dir/tls.crt" \ + --key="$tmp_dir/tls.key" \ + --dry-run=client -o yaml | kubectl apply -f - + + rm -rf "$tmp_dir" +} + +ensure_db_user_secret() { + if kubectl -n "$CNPG_NAMESPACE" get secret knoe-db-user >/dev/null 2>&1; then + log "knoe-db-user secret already present in ${CNPG_NAMESPACE}." + else + [[ -n "$DB_PASSWORD" && "$DB_PASSWORD" != \$\{* ]] || { + echo "ERROR: secret 'knoe-db-user' missing in '${CNPG_NAMESPACE}' and DB_PASSWORD is not set." >&2 + exit 1 + } + log "Creating missing knoe-db-user secret in ${CNPG_NAMESPACE} ..." + kubectl -n "$CNPG_NAMESPACE" create secret generic knoe-db-user \ + --from-literal=username="$CNPG_DB_OWNER" \ + --from-literal=password="$DB_PASSWORD" \ + --dry-run=client -o yaml | kubectl apply -f - + fi + + if kubectl -n "$CNPG_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then + return + fi + [[ -n "$DB_PASSWORD" && "$DB_PASSWORD" != \$\{* ]] || return + log "Creating missing knoe-db-superuser secret in ${CNPG_NAMESPACE} ..." + kubectl -n "$CNPG_NAMESPACE" create secret generic knoe-db-superuser \ + --from-literal=username=postgres \ + --from-literal=password="$DB_PASSWORD" \ + --dry-run=client -o yaml | kubectl apply -f - +} + +# ── Tool checks ────────────────────────────────────────────────────────────── +for tool in gcloud kubectl gsutil openssl; do + command -v "$tool" >/dev/null || { echo "ERROR: $tool not found in PATH." >&2; exit 1; } +done + +# ── 1. Acquire kubeconfig ──────────────────────────────────────────────────── +log "Fetching kubeconfig for $GKE_CLUSTER in $GCP_REGION ..." +gcloud container clusters get-credentials "$GKE_CLUSTER" \ + --project "$GCP_PROJECT_ID" \ + --region "$GCP_REGION" + +# ── 2. Create GCS buckets ──────────────────────────────────────────────────── +for bucket in "$GCS_BACKUP_BUCKET" "$GCS_WAL_BUCKET"; do + if gsutil ls -b "gs://$bucket" >/dev/null 2>&1; then + log "Bucket gs://$bucket already exists." + else + log "Creating bucket gs://$bucket ..." + gsutil mb -p "$GCP_PROJECT_ID" -l "$GCP_REGION" "gs://$bucket" + gsutil versioning set on "gs://$bucket" + gsutil lifecycle set /dev/stdin "gs://$bucket" </dev/null 2>&1; then + log "Service account $GCP_SA_EMAIL already exists." +else + log "Creating service account $GCP_SA_EMAIL ..." + gcloud iam service-accounts create "$CNPG_BACKUP_SA" \ + --display-name="CNPG GCS Backup" \ + --project="$GCP_PROJECT_ID" +fi + +# IAM propagation is eventually consistent right after service-account creation. +# Wait briefly so downstream gsutil IAM grants don't fail with transient 400. +for i in $(seq 1 12); do + if gcloud iam service-accounts describe "$GCP_SA_EMAIL" --project "$GCP_PROJECT_ID" >/dev/null 2>&1; then + break + fi + log "Waiting for service account propagation ($i/12) ..." + sleep 5 +done + +# ── 4. Grant storage access ─────────────────────────────────────────────────── +log "Granting objectAdmin on backup buckets to $GCP_SA_EMAIL ..." +for bucket in "$GCS_BACKUP_BUCKET" "$GCS_WAL_BUCKET"; do + ok=0 + for i in $(seq 1 6); do + if gsutil iam ch \ + "serviceAccount:${GCP_SA_EMAIL}:objectAdmin" \ + "gs://$bucket"; then + ok=1 + break + fi + log "WARN: IAM grant failed for gs://$bucket (attempt $i/6); retrying in 5s ..." + sleep 5 + done + [[ $ok -eq 1 ]] || { + echo "ERROR: failed to grant objectAdmin on gs://$bucket to $GCP_SA_EMAIL" >&2 + exit 1 + } +done + +# ── 5. Bind Workload Identity ───────────────────────────────────────────────── +log "Binding Workload Identity ..." +gcloud iam service-accounts add-iam-policy-binding "$GCP_SA_EMAIL" \ + --role=roles/iam.workloadIdentityUser \ + --member="serviceAccount:${GCP_PROJECT_ID}.svc.id.goog[${CNPG_NAMESPACE}/cnpg-backup-sa]" \ + --project="$GCP_PROJECT_ID" + +# ── 6. Apply namespace ──────────────────────────────────────────────────────── +log "Applying namespace $CNPG_NAMESPACE ..." +kubectl apply -f "$GKE_MANIFEST_DIR/namespace.yaml" + +# ── 7. Install CNPG operator (if not present) ───────────────────────────────── +if ! kubectl get crd clusters.postgresql.cnpg.io >/dev/null 2>&1; then + log "Installing CloudNativePG operator v${CNPG_OPERATOR_VERSION} ..." + kubectl apply --server-side -f \ + "https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-${CNPG_OPERATOR_VERSION}.yaml" + log "Waiting for CNPG operator to be ready ..." + kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=120s +else + log "CNPG operator already installed." +fi + +# ── 8. Apply ServiceAccount + annotate with WI ─────────────────────────────── +log "Applying CNPG backup ServiceAccount with Workload Identity annotation ..." +GCP_PROJECT_ID="$GCP_PROJECT_ID" envsubst '${GCP_PROJECT_ID}' < "$GKE_MANIFEST_DIR/knoe-db-backup-gcs.yaml" \ + | kubectl apply -f - + +kubectl annotate serviceaccount cnpg-backup-sa \ + -n "$CNPG_NAMESPACE" \ + "iam.gke.io/gcp-service-account=${GCP_SA_EMAIL}" \ + --overwrite + +# ── 9. Bootstrap CNPG secrets ─────────────────────────────────────────────── +ensure_cnpg_tls_secrets +ensure_db_user_secret + +# ── 10. Apply CNPG cluster ─────────────────────────────────────────────────── +IFS_COMMA="," +resolve_knoe_db_image_tag +log "Using knoe-db image tag: ${KNOE_DB_IMAGE_TAG}" +resolve_storage_class_candidates +IFS=',' read -r -a storage_classes <<< "$CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED" +selected_storage_class="" +attempt_total="${#storage_classes[@]}" +attempt_index=0 + +if kubectl -n "$CNPG_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" >/dev/null 2>&1; then + existing_phase="$(kubectl -n "$CNPG_NAMESPACE" get cluster "$CNPG_CLUSTER_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + if [[ "$existing_phase" != "Cluster in healthy state" ]]; then + log "Existing cluster phase ${existing_phase:-unknown} is not healthy; resetting stale resources before storage-class evaluation." + reset_cnpg_cluster_attempt + fi +fi + +for storage_class in "${storage_classes[@]}"; do + storage_class="${storage_class//[[:space:]]/}" + [[ -n "$storage_class" ]] || continue + attempt_index=$((attempt_index + 1)) + + log "Applying knoe-db CNPG cluster with storageClass=${storage_class} (${attempt_index}/${attempt_total}) ..." + tmp_manifest="$(mktemp)" + render_cnpg_manifest "$storage_class" "$tmp_manifest" + kubectl apply -f "$tmp_manifest" + rm -f "$tmp_manifest" + + if wait_for_storage_outcome "$storage_class"; then + selected_storage_class="$storage_class" + break + fi + + outcome_rc=$? + case "$outcome_rc" in + 10) + log "WARN: storageClass=${storage_class} failed due to SSD quota pressure; stepping down." + ;; + 11) + log "WARN: storageClass=${storage_class} failed provisioning; stepping down." + ;; + *) + log "WARN: storageClass=${storage_class} did not become ready within ${CNPG_STORAGE_WAIT_TIMEOUT}s; stepping down." + ;; + esac + + reset_cnpg_cluster_attempt +done + +[[ -n "$selected_storage_class" ]] || { + echo "ERROR: CNPG failed to provision across storage class candidates: ${CNPG_STORAGE_CLASS_CANDIDATES_RESOLVED}" >&2 + exit 1 +} + +log "CNPG provisioning selected storageClass=${selected_storage_class}." + +# ── 11. Ensure RoleBindings reference cnpg-backup-sa ───────────────────────── +# CNPG auto-creates knoe-db and knoe-db-barman-cloud RoleBindings at cluster +# creation. With spec.serviceAccountName set from the start the operator may +# use cnpg-backup-sa as the sole subject, but if it still uses the default SA +# name (cluster name) we patch both bindings to add cnpg-backup-sa — matching +# the live state after the 2026-04-29 stabilization session. +log "Ensuring CNPG-managed RoleBindings include cnpg-backup-sa ..." +for rb in "${CNPG_CLUSTER_NAME}" "${CNPG_CLUSTER_NAME}-barman-cloud"; do + for _ in 1 2 3 4 5; do + kubectl -n "${CNPG_NAMESPACE}" get rolebinding "${rb}" >/dev/null 2>&1 && break + sleep 2 + done + if kubectl -n "${CNPG_NAMESPACE}" get rolebinding "${rb}" >/dev/null 2>&1; then + if ! kubectl -n "${CNPG_NAMESPACE}" get rolebinding "${rb}" \ + -o jsonpath='{.subjects[*].name}' | grep -qw cnpg-backup-sa; then + patch_file="$(mktemp)" + cat > "${patch_file}" < -Deploys common infrastructure services (Registry, OpenTofu, Garage, OpenBao, Kong, Cert-Manager) +Deploys common infrastructure services (Registry, Garage, OpenBao, Kong, Cert-Manager; +OpenTofu on non-k8s modes) into the given Kubernetes namespace. Use -k to include the Kerberos/KDC service. EOF } @@ -83,7 +87,8 @@ if [ -z "$NS" ]; then NS="${SERVICE_NAMESPACE:-${NAMESPACE:-}}" fi if [ -z "$NS" ]; then - NS="default" + echo "ERROR: No namespace specified. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2 + exit 1 fi # Registry should live in the common-core/service namespace unless explicitly overridden. @@ -91,8 +96,8 @@ REGISTRY_NS="${REGISTRY_NAMESPACE:-${NS}}" knoe_ensure_kubeconfig >/dev/null 2>&1 || true echo "DEBUG: knoe_ensure_kubeconfig finished" >&2 -knoe_ensure_kube_context || exit 1 -echo "DEBUG: knoe_ensure_kube_context finished" >&2 +ensure_kube_context || exit 1 +echo "DEBUG: ensure_kube_context finished" >&2 # Check cluster reachability early to fail fast if [[ "$ACTION" != "status" ]]; then @@ -501,12 +506,19 @@ OPENTOFU_NAME=${OPENTOFU_NAME:-opentofu} OPENBAO_NAME=${OPENBAO_NAME:-openbao} REGISTRY_NAME=${REGISTRY_NAME:-registry} GARAGE_NAME=${GARAGE_NAME:-garage} +REDIS_NAME=${REDIS_NAME:-redis} KONG_NAME=${KONG_NAME:-knoe-svc-kong} +KONG_CONFIG_NAME=${KONG_CONFIG_NAME:-knoe-svc-kong-config} +# In k8s (GKE/prod) mode use the knoe-svc-kong naming +if [ "${KNOE_MODE:-}" = "k8s" ]; then + KONG_NAME=${KONG_NAME/knoe-svc-kong/knoe-svc-kong} + KONG_CONFIG_NAME=${KONG_CONFIG_NAME/knoe-svc-kong/knoe-svc-kong} +fi +export KONG_NAME KONG_CONFIG_NAME KNOE_MODE OPENTOFU_CONFIGMAP=${OPENTOFU_CONFIGMAP:-opentofu-nginx} OPENTOFU_SECRET=${OPENTOFU_SECRET:-opentofu-admin} GARAGE_CONFIGMAP=${GARAGE_CONFIGMAP:-garage-config} GARAGE_SECRET_NAME=${GARAGE_SECRET_NAME:-garage-secrets} -KONG_CONFIG_NAME=${KONG_CONFIG_NAME:-knoe-svc-kong-config} find_namespaces() { local kind="$1" @@ -532,17 +544,19 @@ collect_other_namespaces() { migrate_common_services() { local old_ns - for old_ns in $(collect_other_namespaces "$NS" "$OPENTOFU_NAME" deployment service); do - echo "Found OpenTofu in namespace '$old_ns'; removing before deploy to '$NS' ..." - if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then - "$SCRIPT_DIR/init_opentofu.sh" -n "$old_ns" stop || true - else - kubectl delete -n "$old_ns" deploy "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true - kubectl delete -n "$old_ns" svc "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true - fi - kubectl delete -n "$old_ns" configmap "$OPENTOFU_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true - kubectl delete -n "$old_ns" secret "$OPENTOFU_SECRET" --ignore-not-found >/dev/null 2>&1 || true - done + if [ "${KNOE_MODE:-}" != "k8s" ]; then + for old_ns in $(collect_other_namespaces "$NS" "$OPENTOFU_NAME" deployment service); do + echo "Found OpenTofu in namespace '$old_ns'; removing before deploy to '$NS' ..." + if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then + "$SCRIPT_DIR/init_opentofu.sh" -n "$old_ns" stop || true + else + kubectl delete -n "$old_ns" deploy "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true + kubectl delete -n "$old_ns" svc "$OPENTOFU_NAME" --ignore-not-found >/dev/null 2>&1 || true + fi + kubectl delete -n "$old_ns" configmap "$OPENTOFU_CONFIGMAP" --ignore-not-found >/dev/null 2>&1 || true + kubectl delete -n "$old_ns" secret "$OPENTOFU_SECRET" --ignore-not-found >/dev/null 2>&1 || true + done + fi for old_ns in $(collect_other_namespaces "$REGISTRY_NS" "$REGISTRY_NAME" deployment service); do echo "Found Registry ($REGISTRY_NAME) in namespace '$old_ns'; removing before deploy to '$REGISTRY_NS' ..." @@ -573,6 +587,16 @@ migrate_common_services() { fi done + for old_ns in $(collect_other_namespaces "$NS" "$REDIS_NAME" statefulset deployment service); do + echo "Found Redis in namespace '$old_ns'; removing before deploy to '$NS' ..." + if [ -x "$SCRIPT_DIR/init_redis.sh" ]; then + "$SCRIPT_DIR/init_redis.sh" -n "$old_ns" stop || true + else + kubectl delete -n "$old_ns" statefulset "${REDIS_NAME}-master" --ignore-not-found >/dev/null 2>&1 || true + kubectl delete -n "$old_ns" svc "${REDIS_NAME}-master" "${REDIS_NAME}-headless" --ignore-not-found >/dev/null 2>&1 || true + fi + done + # Kong API gateway (service layer) local kong_old_namespaces="" kong_old_namespaces+=$(collect_other_namespaces "$NS" "$KONG_NAME" deployment service) @@ -584,6 +608,53 @@ migrate_common_services() { kubectl delete -n "$old_ns" svc "$KONG_NAME" --ignore-not-found >/dev/null 2>&1 || true kubectl delete -n "$old_ns" configmap "$KONG_CONFIG_NAME" --ignore-not-found >/dev/null 2>&1 || true done + + # Same-namespace: remove the alternate Kong deployment name if present. + # In k8s/GKE mode KONG_NAME=knoe-svc-kong; a leftover knoe-svc-kong (or vice + # versa) in the same namespace causes duplicate pods that the deployer won't + # clean up on its own. + local kong_alt_name kong_alt_config + case "$KONG_NAME" in + *knoe-svc-kong*) kong_alt_name="${KONG_NAME//knoe-svc-kong/knoe-svc-kong}" + kong_alt_config="${KONG_CONFIG_NAME//knoe-svc-kong/knoe-svc-kong}" ;; + *knoe-svc-kong*) kong_alt_name="${KONG_NAME//knoe-svc-kong/knoe-svc-kong}" + kong_alt_config="${KONG_CONFIG_NAME//knoe-svc-kong/knoe-svc-kong}" ;; + *) kong_alt_name="" ; kong_alt_config="" ;; + esac + if [[ -n "$kong_alt_name" && "$kong_alt_name" != "$KONG_NAME" ]]; then + if kubectl -n "$NS" get deploy "$kong_alt_name" >/dev/null 2>&1; then + echo "Found stale alternate Kong deployment '$kong_alt_name' in '$NS'; removing ..." + kubectl -n "$NS" delete deploy "$kong_alt_name" --ignore-not-found >/dev/null 2>&1 || true + kubectl -n "$NS" delete svc "$kong_alt_name" --ignore-not-found >/dev/null 2>&1 || true + [[ -n "$kong_alt_config" ]] && \ + kubectl -n "$NS" delete configmap "$kong_alt_config" --ignore-not-found >/dev/null 2>&1 || true + fi + fi + + # Same-namespace: prune excess unhealthy Kong pods from a stuck rolling update. + # Happens when the old pod (e.g. OOMKilled) does not terminate cleanly before + # the new pod comes up, leaving the deployment with more pods than desired. + if kubectl -n "$NS" get deploy "$KONG_NAME" >/dev/null 2>&1; then + local _kong_desired _kong_pod_count _stale_pod + _kong_desired=$(kubectl -n "$NS" get deploy "$KONG_NAME" \ + -o jsonpath='{.spec.replicas}' 2>/dev/null) + _kong_desired="${_kong_desired:-1}" + _kong_pod_count=$(kubectl -n "$NS" get pods \ + -l "app=${KONG_NAME}" --no-headers 2>/dev/null | wc -l | tr -d '[:space:]') + if [[ "${_kong_pod_count:-0}" -gt "${_kong_desired:-1}" ]]; then + echo "Kong ($KONG_NAME) has ${_kong_pod_count} pod(s) but desired=${_kong_desired};" \ + "removing unhealthy pods ..." + while IFS= read -r _stale_pod; do + [[ -z "$_stale_pod" ]] && continue + echo " Removing unhealthy pod: $_stale_pod" + kubectl -n "$NS" delete pod "$_stale_pod" --force --grace-period=0 \ + >/dev/null 2>&1 || true + done < <( + kubectl -n "$NS" get pods -l "app=${KONG_NAME}" --no-headers 2>/dev/null \ + | awk '{split($2,r,"/"); ok=($3=="Running" && r[1]==r[2] && r[1]~/^[0-9]+$/); if(!ok) print $1}' + ) + fi + fi } echo "Deploying common services (namespace=$NS, action=$ACTION)" @@ -601,14 +672,19 @@ esac # 1. Registry – no dependencies; other services pull images from it # 2. OpenBao – secrets vault; needed by downstream services # 3. Garage – object storage -# 4. OpenTofu – IaC engine; depends on registry + secrets +# 4. OpenTofu – IaC engine; depends on registry + secrets (non-k8s) # --------------------------------------------------------------------------- -if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then - REGISTRY_NAMESPACE="$REGISTRY_NS" SERVICE_NAMESPACE="$NS" \ - "$SCRIPT_DIR/init_registry.sh" -n "$REGISTRY_NS" "$ACTION" || rc=$? +if [ "${KNOE_MODE:-}" = "k8s" ]; then + echo "[INFO] Registry update namespace=${REGISTRY_NS}" + echo "[INFO] GKE/prod mode: skipping in-cluster Docker registry (using GCP Artifact Registry)." else - echo "WARN: init_registry.sh not found; registry deploy skipped." + if [ -x "$SCRIPT_DIR/init_registry.sh" ]; then + REGISTRY_NAMESPACE="$REGISTRY_NS" SERVICE_NAMESPACE="$NS" \ + "$SCRIPT_DIR/init_registry.sh" -n "$REGISTRY_NS" "$ACTION" || rc=$? + else + echo "WARN: init_registry.sh not found; registry deploy skipped." + fi fi if [ -x "$SCRIPT_DIR/init_openbao.sh" ]; then @@ -618,6 +694,13 @@ else echo "WARN: init_openbao.sh not found; skipping OpenBao." fi +if [ -x "$SCRIPT_DIR/init_redis.sh" ]; then + REDIS_NAMESPACE="$NS" SERVICE_NAMESPACE="$NS" \ + "$SCRIPT_DIR/init_redis.sh" -n "$NS" "$ACTION" || rc=$? +else + echo "WARN: init_redis.sh not found; Redis deploy skipped." +fi + if [ -x "$SCRIPT_DIR/init_garage_store.sh" ]; then garage_action="$ACTION" case "$garage_action" in @@ -629,24 +712,32 @@ else echo "WARN: init_garage_store.sh not found; garage deploy skipped." fi -if [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then +if [ "${KNOE_MODE:-}" = "k8s" ]; then + echo "[INFO] k8s mode: skipping OpenTofu deploy." +elif [ -x "$SCRIPT_DIR/init_opentofu.sh" ]; then ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" "$SCRIPT_DIR/init_opentofu.sh" -n "$NS" "$ACTION" || rc=$? else echo "WARN: init_opentofu.sh not found; skipping OpenTofu." fi if [[ "$ENABLE_KERBEROS" == "1" ]]; then - if [ -x "$SCRIPT_DIR/init_kdc.sh" ]; then - kdc_action="$ACTION" - case "$kdc_action" in - stop) kdc_action="cleanup" ;; - status) kdc_action="status" ;; - *) kdc_action="update" ;; - esac - SERVICE_NAMESPACE="$NS" PROLE_KDC_NAMESPACE="$NS" \ - "$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$? + # KDC is now embedded in the `knoe-auth` pod (multi-container) by default. + # Only deploy a standalone KDC when explicitly requested. + if [[ "${PROLE_KDC_STANDALONE:-0}" == "1" ]]; then + if [ -x "$SCRIPT_DIR/init_kdc.sh" ]; then + kdc_action="$ACTION" + case "$kdc_action" in + stop) kdc_action="cleanup" ;; + status) kdc_action="status" ;; + *) kdc_action="update" ;; + esac + SERVICE_NAMESPACE="$NS" PROLE_KDC_NAMESPACE="$NS" \ + "$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$? + else + echo "WARN: init_kdc.sh not found; standalone KDC deploy skipped." + fi else - echo "WARN: init_kdc.sh not found; kerberos deploy skipped." + echo "[INFO] Kerberos enabled: skipping standalone KDC deploy (KDC runs as sidecar in knoe-auth)." fi fi diff --git a/mock_val/init_db_manager.sh b/mock_val/init_db_manager.sh index 702f7e1..1425976 100755 --- a/mock_val/init_db_manager.sh +++ b/mock_val/init_db_manager.sh @@ -5,7 +5,7 @@ set -euo pipefail # - Build and deploy the knoe-db-manager Node.js REST endpoint # - Provides /backup//full to trigger barman full backups into Garage # - Deploys to the knoe-db namespace; accessible via Kong endpoint /backup -# - Deploys to the current cluster (k3d or k3s) based on KNOE_MODE from conf/knoe.cfg +# - Deploys to the current cluster (k3d or k3s) based on KNOE_MODE from active config # # Usage: # ./init_db_manager.sh [--mode MODE] diff --git a/mock_val/init_forgejo.sh b/mock_val/init_forgejo.sh index d78305a..1b64202 100644 --- a/mock_val/init_forgejo.sh +++ b/mock_val/init_forgejo.sh @@ -20,7 +20,7 @@ Usage: Options: --mode Deployment mode (default: ${MODE:-k3d}) -n, --namespace Target namespace (default: forgejo) - -c, --config Path to knoe.cfg (defaults to detected) + -c, --config Path to config file (defaults to detected) --force Delete existing Forgejo resources before deploy --help Show this help @@ -49,11 +49,12 @@ while [[ $# -gt 0 ]]; do esac done -# Resolve config path and namespace defaults from knoe.cfg when present -if [[ -z "$CFG_PATH" && -n "${KNOE_CONF:-}" && -f "${KNOE_CONF}/knoe.cfg" ]]; then - CFG_PATH="${KNOE_CONF}/knoe.cfg" -elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - CFG_PATH="$SCRIPT_DIR/../conf/knoe.cfg" +# Resolve config path and namespace defaults when present +if [[ -z "$CFG_PATH" && -n "${KNOE_CONF:-}" ]]; then + CFG_PATH="$(_knoe_cfg_select_cfg_file "${KNOE_CONF}")" +fi +if [[ -z "$CFG_PATH" ]]; then + CFG_PATH="$(_knoe_cfg_select_cfg_file "$SCRIPT_DIR/../conf")" fi if [[ -z "$NAMESPACE" && -n "$CFG_PATH" ]]; then diff --git a/mock_val/init_garage_store.sh b/mock_val/init_garage_store.sh index 9b80f19..b0605a6 100755 --- a/mock_val/init_garage_store.sh +++ b/mock_val/init_garage_store.sh @@ -49,6 +49,47 @@ GARAGE_NODE_CAPACITY=${GARAGE_NODE_CAPACITY:-10GB} GARAGE_ZONE=${GARAGE_ZONE:-local} GARAGE_NAMESPACE=${GARAGE_NAMESPACE:-$RESOLVED_NAMESPACE} NAMESPACE="$GARAGE_NAMESPACE" +APP_CLUSTER_KUBECONTEXT=${APP_CLUSTER_KUBECONTEXT:-${init_cluster_app_cluster_kubecontext:-}} +DB_CLUSTER_KUBECONTEXT=${DB_CLUSTER_KUBECONTEXT:-${init_cluster_db_cluster_kubecontext:-}} +GARAGE_AUTHORITY_ROLE=${GARAGE_AUTHORITY_ROLE:-db} +GARAGE_AUTHORITY_ROLE="${GARAGE_AUTHORITY_ROLE,,}" +ACTIVE_KUBE_CONTEXT=${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}} + +garage_context_role() { + local ctx="${ACTIVE_KUBE_CONTEXT:-}" + if [[ -z "$ctx" ]]; then + ctx=$(kubectl config current-context 2>/dev/null || true) + fi + if [[ -n "$APP_CLUSTER_KUBECONTEXT" && "$ctx" == "$APP_CLUSTER_KUBECONTEXT" ]]; then + printf '%s' "app" + return 0 + fi + if [[ -n "$DB_CLUSTER_KUBECONTEXT" && "$ctx" == "$DB_CLUSTER_KUBECONTEXT" ]]; then + printf '%s' "db" + return 0 + fi + printf '%s' "unknown" +} + +garage_split_cluster_contexts_enabled() { + [[ -n "$APP_CLUSTER_KUBECONTEXT" && -n "$DB_CLUSTER_KUBECONTEXT" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]] +} + +garage_is_authoritative_context() { + local role + if ! garage_split_cluster_contexts_enabled; then + return 0 + fi + role="$(garage_context_role)" + [[ "$role" == "$GARAGE_AUTHORITY_ROLE" ]] +} + +garage_ensure_absent_non_authoritative() { + local role + role="$(garage_context_role)" + echo "[INFO] Garage authority role is '${GARAGE_AUTHORITY_ROLE}'; current role '${role}' is non-authoritative. Ensuring Garage is absent in namespace '$NAMESPACE'." + delete_manifests || true +} # Support both KNOE_HOME/k8s and sibling k8s directory if [[ -d "$SCRIPT_DIR/../k8s/knoe" ]]; then @@ -67,11 +108,22 @@ GARAGE_FILES=( "$GARAGE_MANIFEST_DIR/garage-service.yaml" ) if [[ "${KNOE_MODE:-}" == "k3d" ]]; then + # k3d: no local storage provisioner — skip synology StorageClass and static PVs. GARAGE_FILES=( "$GARAGE_MANIFEST_DIR/garage-configmap.yaml" "$GARAGE_MANIFEST_DIR/garage-statefulset.yaml" "$GARAGE_MANIFEST_DIR/garage-service.yaml" ) +elif [[ "${KNOE_MODE:-}" == "k8s" ]]; then + # GKE Autopilot: apply our custom garage-hdd StorageClass (pd-standard HDD, avoids SSD quota). + # The skip-if-exists guard in apply_manifests handles idempotent re-runs safely. + # Use GCP-specific statefulset (no synology selectors, explicit resource requests). + GARAGE_FILES=( + "$GARAGE_MANIFEST_DIR/storageclass-gcp-hdd.yaml" + "$GARAGE_MANIFEST_DIR/garage-configmap.yaml" + "$GARAGE_MANIFEST_DIR/garage-statefulset-gcp.yaml" + "$GARAGE_MANIFEST_DIR/garage-service.yaml" + ) fi GARAGE_APPLY_CHANGED=0 @@ -160,9 +212,37 @@ apply_manifests() { continue fi if [[ $diff_rc -ne 1 ]]; then + # For immutable StatefulSet VolumeClaimTemplates, kubectl diff itself returns an + # error (diff_rc != 1 but contains the immutable spec error). Handle it here. + if [[ "$(basename "$f")" == garage-statefulset*.yaml ]] \ + && echo "$diff_out" | grep -q "updates to statefulset spec"; then + echo "WARN: Garage StatefulSet VolumeClaimTemplates changed (diff-stage); deleting and recreating ..." + kubectl delete statefulset "$GARAGE_NAME" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true + local _pvc_sc_d + _pvc_sc_d=$(kubectl get pvc "data-garage-0" -n "$NAMESPACE" \ + -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + if [[ -n "$_pvc_sc_d" ]]; then + echo " Removing stale PVC 'data-garage-0' (storageClass: $_pvc_sc_d) ..." + kubectl delete pvc "data-garage-0" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true + fi + printf '%s' "$rendered" | kubectl apply --validate=false -n "$NAMESPACE" -f - + GARAGE_APPLY_CHANGED=1 + GARAGE_STATEFULSET_CHANGED=1 + continue + fi echo "$diff_out" >&2 exit 1 fi + # StorageClass resources have immutable parameters/reclaimPolicy. + # If the StorageClass already exists in the cluster, skip re-applying it. + if echo "$rendered" | grep -q "^kind: StorageClass"; then + local _sc_name + _sc_name=$(echo "$rendered" | grep "^ name:" | head -1 | awk '{print $2}') + if [[ -n "$_sc_name" ]] && kubectl get storageclass "$_sc_name" >/dev/null 2>&1; then + echo "[SKIP] StorageClass '$_sc_name' already exists; skipping apply (immutable fields)." + continue + fi + fi if output=$(printf '%s' "$rendered" | kubectl apply --validate=false -n "$NAMESPACE" -f - 2>&1); then printf '%s\n' "$output" GARAGE_APPLY_CHANGED=1 @@ -175,6 +255,23 @@ apply_manifests() { && echo "$output" | grep -q "updates to statefulset spec"; then echo "WARN: Garage StatefulSet immutable in k3d; skipping apply." continue + elif [[ "$(basename "$f")" == garage-statefulset*.yaml ]] \ + && echo "$output" | grep -q "updates to statefulset spec"; then + # VolumeClaimTemplates are immutable; delete the StatefulSet (PVCs are orphaned/preserved) + # and recreate so the new storageClass name takes effect. + echo "WARN: Garage StatefulSet VolumeClaimTemplates changed; deleting and recreating ..." + kubectl delete statefulset "$GARAGE_NAME" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true + local _pvc_sc + _pvc_sc=$(kubectl get pvc "data-garage-0" -n "$NAMESPACE" \ + -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + if [[ -n "$_pvc_sc" ]]; then + echo " Removing stale PVC 'data-garage-0' (storageClass: $_pvc_sc) ..." + kubectl delete pvc "data-garage-0" -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true + fi + printf '%s' "$rendered" | kubectl apply --validate=false -n "$NAMESPACE" -f - + GARAGE_APPLY_CHANGED=1 + GARAGE_STATEFULSET_CHANGED=1 + continue fi echo "$output" >&2 exit 1 @@ -322,7 +419,11 @@ wait_ready() { local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}" if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout="$initial_timeout"; then - echo "WARN: Garage not ready after $initial_timeout; checking for Released synology-iscsi PVs with stale claimRefs ..." >&2 + if [[ "${KNOE_MODE:-}" == "k8s" ]]; then + echo "WARN: Garage not ready after $initial_timeout; PVC provisioning may still be in progress (GKE CSI)." >&2 + else + echo "WARN: Garage not ready after $initial_timeout; checking for Released synology-iscsi PVs with stale claimRefs ..." >&2 + fi recycle_released_knoe_iscsi_pv_for_pvc "$NAMESPACE" "data-garage-0" || true fi @@ -421,6 +522,21 @@ status() { case "$ACTION" in start|initialize|update|reload|restart) ensure_tools + if [[ "$GARAGE_AUTHORITY_ROLE" != "db" && "$GARAGE_AUTHORITY_ROLE" != "app" ]]; then + echo "ERROR: GARAGE_AUTHORITY_ROLE must be 'db' or 'app' (got '$GARAGE_AUTHORITY_ROLE')." >&2 + exit 1 + fi + if garage_split_cluster_contexts_enabled; then + current_role="$(garage_context_role)" + if [[ "$current_role" == "unknown" ]]; then + echo "ERROR: Unable to resolve Garage deployment cluster role from kubecontext '${ACTIVE_KUBE_CONTEXT:-}' (APP='${APP_CLUSTER_KUBECONTEXT:-}', DB='${DB_CLUSTER_KUBECONTEXT:-}')." >&2 + exit 1 + fi + if ! garage_is_authoritative_context; then + garage_ensure_absent_non_authoritative + exit 0 + fi + fi ensure_namespace ensure_secrets k3d_cleanup_pending_pvc diff --git a/mock_val/init_gitea.sh b/mock_val/init_gitea.sh old mode 100644 new mode 100755 index 9c7759c..523999d --- a/mock_val/init_gitea.sh +++ b/mock_val/init_gitea.sh @@ -20,7 +20,7 @@ Usage: Options: --mode Deployment mode (default: ${MODE:-k3d}) -n, --namespace Target namespace (default: gitea) - -c, --config Path to knoe.cfg (defaults to detected) + -c, --config Path to config file (defaults to detected) --force Delete existing release before deploy --help Show this help @@ -49,11 +49,12 @@ while [[ $# -gt 0 ]]; do esac done -# Resolve config path and namespace defaults from knoe.cfg when present -if [[ -z "$CFG_PATH" && -n "${KNOE_CONF:-}" && -f "${KNOE_CONF}/knoe.cfg" ]]; then - CFG_PATH="${KNOE_CONF}/knoe.cfg" -elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - CFG_PATH="$SCRIPT_DIR/../conf/knoe.cfg" +# Resolve config path and namespace defaults when present +if [[ -z "$CFG_PATH" && -n "${KNOE_CONF:-}" ]]; then + CFG_PATH="$(_knoe_cfg_select_cfg_file "${KNOE_CONF}")" +fi +if [[ -z "$CFG_PATH" ]]; then + CFG_PATH="$(_knoe_cfg_select_cfg_file "$SCRIPT_DIR/../conf")" fi if [[ -z "$NAMESPACE" && -n "$CFG_PATH" ]]; then @@ -81,6 +82,124 @@ RELEASE_NAME="gitea" IMAGE_REPO_DEFAULT="gitea/gitea" IMAGE_TAG="${GITEA_IMAGE_TAG:-1.22.3}" IMAGE_REPO="$IMAGE_REPO_DEFAULT" +NODE_SELECTOR="${GITEA_NODE_SELECTOR:-${NODE_SELECTOR:-}}" +NODE_SELECTOR_KEY="${GITEA_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}" +GITEA_PV_NODE_SELECTOR_KEY="${GITEA_PV_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY}}" +GITEA_PV_NODE="${GITEA_PV_NODE:-${GITEA_NODE_SELECTOR:-}}" +GITEA_PV_BASE_DIR="${GITEA_PV_BASE_DIR:-/synology/d005}" +GITEA_STORAGE_CLASS="${GITEA_STORAGE_CLASS:-gitea-local-d005}" +GITEA_DOMAIN="${GITEA_DOMAIN:-git.prole.org}" +GITEA_SSH_DOMAIN="${GITEA_SSH_DOMAIN:-$GITEA_DOMAIN}" +KNOE_DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}" +KNOE_DB_CLUSTER="${KNOE_DB_CLUSTER:-${CLUSTER_NAME:-knoe-db}}" +KNOE_DB_SERVICE="${KNOE_DB_SERVICE:-${KNOE_DB_CLUSTER}-rw}" +KNOE_DB_PORT="${KNOE_DB_PORT:-${DB_HOST_PORT:-5432}}" +KNOE_DB_ADMIN_USER="${KNOE_DB_ADMIN_USER:-${KNOE_DB_USER:-postgres}}" +GITEA_DB_NAME="${GITEA_DB_NAME:-gitea}" +GITEA_DB_USER="${GITEA_DB_USER:-gitea}" +GITEA_DB_PASSWORD="${GITEA_DB_PASSWORD:-${DB_PASSWORD:-}}" + +if [[ "$MODE" == "k3s" && -z "$GITEA_PV_NODE" ]]; then + die "GITEA_PV_NODE (or GITEA_NODE_SELECTOR) must be set in k3s mode." +fi + +is_secret_placeholder() { + case "${1:-}" in + '${KNOE_SECRET:'*|'${OPENBAO:'*) return 0 ;; + esac + return 1 +} + +resolve_gitea_db_password() { + if [[ -n "${GITEA_DB_PASSWORD:-}" ]] && ! is_secret_placeholder "${GITEA_DB_PASSWORD}"; then + return 0 + fi + + if kubectl -n "$KNOE_DB_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then + local resolved + resolved=$(kubectl -n "$KNOE_DB_NAMESPACE" get secret knoe-db-superuser \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) + if [[ -n "$resolved" ]]; then + GITEA_DB_PASSWORD="$resolved" + fi + fi + + if [[ -z "${GITEA_DB_PASSWORD:-}" ]] || is_secret_placeholder "${GITEA_DB_PASSWORD}"; then + warn "Could not resolve a concrete Gitea DB password; database bootstrap may be skipped." + fi +} + +resolve_knoe_db_primary_pod() { + local primary + primary=$(kubectl -n "$KNOE_DB_NAMESPACE" get pods \ + -l "cnpg.io/cluster=${KNOE_DB_CLUSTER},cnpg.io/instanceRole=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$primary" ]]; then + primary=$(kubectl -n "$KNOE_DB_NAMESPACE" get pods \ + -l "cnpg.io/cluster=${KNOE_DB_CLUSTER}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + fi + printf '%s' "$primary" +} + +sql_escape_literal() { + printf '%s' "${1:-}" | sed "s/'/''/g" +} + +setup_knoe_db_for_gitea() { + local primary + primary="$(resolve_knoe_db_primary_pod)" + if [[ -z "$primary" ]]; then + warn "No knoe-db pod found in namespace '${KNOE_DB_NAMESPACE}'; skipping Gitea DB setup." + return 0 + fi + + resolve_gitea_db_password + if [[ -z "${GITEA_DB_PASSWORD:-}" ]] || is_secret_placeholder "${GITEA_DB_PASSWORD}"; then + warn "Skipping Gitea DB setup due to unresolved GITEA_DB_PASSWORD." + return 0 + fi + + local admin_user="" + local candidate + for candidate in "$KNOE_DB_ADMIN_USER" postgres root; do + [[ -z "$candidate" ]] && continue + if kubectl -n "$KNOE_DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$candidate" -d postgres -tAc "SELECT 1" >/dev/null 2>&1; then + admin_user="$candidate" + break + fi + done + + if [[ -z "$admin_user" ]]; then + warn "Unable to connect to knoe-db as admin user; skipping Gitea DB setup." + return 0 + fi + + local escaped_password + escaped_password="$(sql_escape_literal "$GITEA_DB_PASSWORD")" + + kubectl -n "$KNOE_DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$admin_user" -d postgres -c " + DO \$\$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${GITEA_DB_USER}') THEN + CREATE ROLE ${GITEA_DB_USER} LOGIN PASSWORD '${escaped_password}'; + END IF; + END \$\$; + " >/dev/null 2>&1 || warn "Could not create role '${GITEA_DB_USER}'." + + local db_exists + db_exists=$(kubectl -n "$KNOE_DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$admin_user" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='${GITEA_DB_NAME}';" 2>/dev/null || true) + + if [[ "$db_exists" != "1" ]]; then + kubectl -n "$KNOE_DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$admin_user" -d postgres -c "CREATE DATABASE ${GITEA_DB_NAME} OWNER ${GITEA_DB_USER};" \ + >/dev/null 2>&1 || warn "Could not create database '${GITEA_DB_NAME}'." + fi + + log "Gitea database '${GITEA_DB_NAME}' prepared in namespace '${KNOE_DB_NAMESPACE}'." +} # Optional reset if [[ "$FORCE" -eq 1 ]]; then @@ -119,20 +238,177 @@ deploy_with_helm() { # Minimal config: NodePort or ClusterIP + Ingress depending on environment. # Avoid heavy persistence defaults; users can override via values later. + local -a helm_args=( + --set "image.repository=$IMAGE_REPO" + --set "image.tag=$IMAGE_TAG" + --set "image.rootless=false" + --set "image.fullOverride=${IMAGE_REPO}:${IMAGE_TAG}" + --set "service.http.type=ClusterIP" + --set "service.ssh.type=ClusterIP" + --set "service.ssh.port=22" + --set "gitea.admin.username=${GITEA_ADMIN_USER:-gitea_admin}" + --set "gitea.admin.password=${GITEA_ADMIN_PASSWORD:-gitea_admin}" + --set "gitea.admin.email=${GITEA_ADMIN_EMAIL:-gitea_admin@example.local}" + --set "gitea.config.server.DOMAIN=${GITEA_DOMAIN}" + --set "gitea.config.server.SSH_DOMAIN=${GITEA_SSH_DOMAIN}" + --set "gitea.config.server.ROOT_URL=http://${GITEA_DOMAIN}/" + --set "gitea.config.server.HTTP_PORT=3000" + --set "gitea.config.server.SSH_PORT=22" + --set "gitea.config.database.DB_TYPE=postgres" + --set "gitea.config.database.HOST=${KNOE_DB_SERVICE}.${KNOE_DB_NAMESPACE}.svc.cluster.local:${KNOE_DB_PORT}" + --set "gitea.config.database.NAME=${GITEA_DB_NAME}" + --set "gitea.config.database.USER=${GITEA_DB_USER}" + --set-string "gitea.config.database.PASSWD=${GITEA_DB_PASSWORD}" + --set "gitea.config.database.SSL_MODE=disable" + --set "gitea.config.session.PROVIDER=db" + --set "postgresql.enabled=false" + --set "postgresql-ha.enabled=false" + --set "valkey-cluster.enabled=false" + --set "persistence.enabled=true" + ) + + if [[ "$MODE" == "k3s" ]]; then + helm_args+=( + --set "gitea.config.cache.ADAPTER=memory" + --set "global.storageClass=$GITEA_STORAGE_CLASS" + --set "persistence.storageClass=$GITEA_STORAGE_CLASS" + --set-json "nodeSelector={\"${GITEA_PV_NODE_SELECTOR_KEY}\":\"${GITEA_PV_NODE}\"}" + --set "deployment.strategy=Recreate" + ) + fi + helm upgrade --install "$RELEASE_NAME" "$CHART_NAME" \ -n "$NAMESPACE" \ - --set image.repository="$IMAGE_REPO" \ - --set image.tag="$IMAGE_TAG" \ - --set service.http.type=ClusterIP \ - --set service.ssh.type=ClusterIP \ - --set gitea.admin.username="${GITEA_ADMIN_USER:-gitea_admin}" \ - --set gitea.admin.password="${GITEA_ADMIN_PASSWORD:-gitea_admin}" \ - --set gitea.admin.email="${GITEA_ADMIN_EMAIL:-gitea_admin@example.local}" \ + "${helm_args[@]}" \ --wait --timeout 10m } +ensure_k3s_storage_layout() { + [[ "$MODE" == "k3s" ]] || return 0 + + log "Ensuring Gitea k3s storage on node '${GITEA_PV_NODE}' at '${GITEA_PV_BASE_DIR}' (storageClass=${GITEA_STORAGE_CLASS})" + + cat </dev/null 2>&1 || true + + cat </dev/null + kubectl -n "$NAMESPACE" delete pod "$prep_pod" --ignore-not-found >/dev/null 2>&1 || true +} + +reset_stuck_k3s_release() { + [[ "$MODE" == "k3s" ]] || return 0 + local pending + pending=$(kubectl -n "$NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.spec.storageClassName}:{.status.phase}{"\n"}{end}' 2>/dev/null | grep '^:Pending$' || true) + if [[ -z "$pending" ]]; then + return 0 + fi + + warn "Detected pending PVCs without storageClass in namespace ${NAMESPACE}; resetting stuck Gitea release" + if command -v helm >/dev/null 2>&1; then + helm uninstall "$RELEASE_NAME" -n "$NAMESPACE" >/dev/null 2>&1 || true + fi + kubectl -n "$NAMESPACE" delete pvc gitea-shared-storage data-gitea-postgresql-ha-postgresql-0 data-gitea-postgresql-ha-postgresql-1 data-gitea-postgresql-ha-postgresql-2 valkey-data-gitea-valkey-cluster-0 valkey-data-gitea-valkey-cluster-1 valkey-data-gitea-valkey-cluster-2 >/dev/null 2>&1 || true +} + +release_exists() { + command -v helm >/dev/null 2>&1 && helm status "$RELEASE_NAME" -n "$NAMESPACE" >/dev/null 2>&1 +} + +gitea_workload_exists() { + kubectl -n "$NAMESPACE" get deploy "$RELEASE_NAME" >/dev/null 2>&1 +} + apply_manifest_fallback() { - warn "Helm unavailable; applying fallback manifest" + if release_exists || gitea_workload_exists; then + warn "Helm resources detected for release '$RELEASE_NAME'; skipping fallback manifest to avoid immutable selector/port conflicts" + return 0 + fi + + warn "Applying fallback manifest" + local node_selector_block="" + if [[ -n "$NODE_SELECTOR" ]]; then + node_selector_block=$(cat </dev/null && pwd)/$(basename "$_gitlab_prescan_cfg")" + fi + if [[ -f "$_gitlab_prescan_cfg" ]]; then + export PROLE_DEPLOY_CFG="$_gitlab_prescan_cfg" + fi + fi + unset _gitlab_prescan_cfg _gitlab_prescan_tok _gitlab_prescan_want_next +fi + # shellcheck disable=SC1090 source "$SCRIPT_DIR/knoe_cfg.sh" MODE="$(knoe_normalize_mode "${KNOE_MODE:-${DEPLOYMENT_MODE:-k3d}}")" -NAMESPACE="${GITLAB_NAMESPACE:-}" +# NAMESPACE: init_gitlab.sh ALWAYS targets 'gitlab' unless explicitly overridden. +# Do NOT fall back to $NAMESPACE (may be 'knoe-db' or 'gitea' from other pipeline steps). +NAMESPACE="${GITLAB_NAMESPACE:-gitlab}" CFG_PATH="" FORCE=0 +# NODE_SELECTOR intentionally blank: only gitaly+minio are pinned (via STORAGE_NODE). +# Setting this would pin ALL global components to one node, exhausting RAM. +NODE_SELECTOR="" usage() { cat < Deployment mode (default: ${MODE:-k3d}) - -n, --namespace Target namespace (default: gitlab) - -c, --config Path to knoe.cfg (defaults to detected) - --force Uninstall existing GitLab release before deploy - --help Show this help + -n, --namespace Target namespace (default: gitlab) + -c, --config Path to config file (defaults to detected) + --node-selector Node to pin all GitLab workloads (optional) + --force Remove existing GitLab and Gitea releases before deploy + --help Show this help Behavior: - - Installs self-hosted GitLab via the official Helm chart. - - Uses CloudNativePG (CNPG) Postgres as the database backend (external DB). + - Removes any pre-existing GitLab-domain Gitea configurations (helm release, + namespace, Kong routes) when --force is specified or when gitea is detected. + - Installs the GitLab Operator via Helm into the gitlab namespace. + - Provisions a GitLab CR that uses the knoe-db CloudNativePG cluster as its + external PostgreSQL data store. + - Configures the GitLab public hostname as Kubernetes ingress pointing to GitLab. + - Legacy/local modes pin gitaly storage to GITLAB_STORAGE_NODE (required). EOF } -die() { echo "[ERROR] $*" >&2; exit 2; } -log() { echo "[INFO] $*"; } -warn() { echo "[WARN] $*" >&2; } +die() { echo "[ERROR] $*" >&2; exit 2; } +log() { echo "[INFO] $*" >&2; } +warn() { echo "[WARN] $*" >&2; } + +resolve_explicit_kube_context() { + local ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}" + if [[ -n "$ctx" ]]; then + printf '%s' "$ctx" + return 0 + fi + return 1 +} + +enforce_app_cluster_targeting() { + if [[ "$MODE" != "k8s" ]]; then + return 0 + fi + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + local target_ctx + target_ctx="$(resolve_explicit_kube_context || true)" + + log "Reconciling GitLab with APP context: ${app_ctx:-"(unset)"}" + [[ -n "$db_ctx" ]] && log "Reconciling GitLab with DB context: ${db_ctx}" + + [[ -n "$app_ctx" ]] || die "APP_CLUSTER_KUBECONTEXT is required for k8s GitLab deployment." + [[ -n "$target_ctx" ]] || die "Explicit kubectl context is required for k8s GitLab deployment." + + if [[ -n "$db_ctx" && "$target_ctx" == "$db_ctx" ]]; then + die "Refusing GitLab APP step against DB context '$target_ctx'." + fi + if [[ "$target_ctx" != "$app_ctx" ]]; then + die "GitLab APP step must target APP_CLUSTER_KUBECONTEXT='${app_ctx}' (got '${target_ctx}')." + fi + + export KUBECTL_CONTEXT="$app_ctx" + export KUBE_CONTEXT_NAME="$app_ctx" + export KUBECONTEXT="$app_ctx" +} + +gitlab_split_cluster_ownership_diagnostics() { + if [[ "$MODE" != "k8s" ]]; then + return 0 + fi + + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + local gitlab_instance="${GITLAB_RELEASE:-gitlab}" + + if [[ -z "$db_ctx" || "$db_ctx" == "$app_ctx" ]]; then + log "GitLab ownership policy: single-cluster mode (APP=${app_ctx:-unknown}); operator and app workloads are expected in this context." + return 0 + fi + + log "GitLab split-cluster ownership policy: APP context '${app_ctx}' is authoritative for GitLab operator + app workloads in namespace '${NAMESPACE}'." + log "GitLab split-cluster ownership policy: DB context '${db_ctx}' must not host GitLab app workloads (gitaly/webservice/sidekiq/kas/registry/toolbox)." + + local app_gitlab_objects db_gitlab_app_objects db_gitlab_operator_objects + app_gitlab_objects=$(kubectl -n "$NAMESPACE" get deploy,statefulset,job,cronjob \ + -l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true) + db_gitlab_app_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deploy,statefulset,job,cronjob \ + -l "app.kubernetes.io/instance=${gitlab_instance}" -o name 2>/dev/null || true) + db_gitlab_operator_objects=$(command kubectl --context "$db_ctx" -n "$NAMESPACE" get deployment -o name 2>/dev/null \ + | grep -E 'deployment.apps/(gitlab-controller-manager|gitlab-operator|.*gitlab.*controller-manager)' || true) + + if [[ -n "$app_gitlab_objects" ]]; then + log "GitLab APP-context resource snapshot (${app_ctx}):" + while IFS= read -r _obj; do + [[ -n "$_obj" ]] || continue + log " - ${_obj}" + done <<< "$app_gitlab_objects" + else + warn "GitLab APP-context snapshot has no resources with app.kubernetes.io/instance=${gitlab_instance} yet (this can be transient during first reconcile)." + fi + + if [[ -n "$db_gitlab_operator_objects" ]]; then + warn "Detected GitLab operator control-plane resources in DB context '${db_ctx}' (stale/legacy install likely):" + while IFS= read -r _obj; do + [[ -n "$_obj" ]] || continue + warn " - ${_obj}" + done <<< "$db_gitlab_operator_objects" + warn "DB-context GitLab operator resources are not authoritative for APP GitLab workloads in split-cluster mode." + fi + + if [[ -n "$db_gitlab_app_objects" ]]; then + local db_app_report="" + while IFS= read -r _obj; do + [[ -n "$_obj" ]] || continue + db_app_report+="- ${_obj}"$'\n' + done <<< "$db_gitlab_app_objects" + repair_blocked "GitLab split-cluster ownership violation: DB cluster contains GitLab app workloads" \ + "APP context: ${app_ctx}. DB context: ${db_ctx}. GitLab app resources detected in DB context:\n${db_app_report}Expected policy: GitLab app workloads run only in APP context. Remove stale DB GitLab app workloads and rerun deploy." + fi +} + +kubectl() { + local target_ctx + target_ctx="$(resolve_explicit_kube_context || true)" + if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then + die "Explicit kubectl context is required in k8s mode." + fi + + local arg has_context=0 + for arg in "$@"; do + case "$arg" in + --context|--context=*|--server|--server=*) + has_context=1 + break + ;; + esac + done + + if [[ -n "$target_ctx" && $has_context -eq 0 ]]; then + command kubectl --context "$target_ctx" "$@" + else + command kubectl "$@" + fi +} + +resolve_db_cluster_context() { + if [[ "$MODE" == "k8s" ]]; then + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + [[ -n "$db_ctx" ]] || die "DB_CLUSTER_KUBECONTEXT is required for GitLab DB setup in k8s mode." + log "Using DB cluster context: ${db_ctx}" + printf '%s' "$db_ctx" + return 0 + fi + return 1 +} + +repair_blocked() { + local reason="$1" + local remediation="${2:-}" + echo "" >&2 + echo "[REPAIR_BLOCKED] ${reason}" >&2 + if [[ -n "$remediation" ]]; then + echo "Remediation: ${remediation}" >&2 + fi + echo "" >&2 + exit 1 +} + +gitlab_selector_for_deployment() { + local deployment_name="$1" + local selector_lines selector="" + + selector_lines=$(kubectl -n "$NAMESPACE" get deployment "$deployment_name" \ + -o go-template='{{range $k, $v := .spec.selector.matchLabels}}{{printf "%s=%s\n" $k $v}}{{end}}' 2>/dev/null || true) + [[ -n "$selector_lines" ]] || return 1 + + local selector_line + while IFS= read -r selector_line; do + [[ -n "$selector_line" ]] || continue + if [[ -n "$selector" ]]; then + selector+=",${selector_line}" + else + selector="$selector_line" + fi + done <<< "$selector_lines" + + [[ -n "$selector" ]] || return 1 + printf '%s' "$selector" +} + +gitlab_non_terminal_pod_count_for_app() { + local app_name="$1" + local pod_selector="$app_name" + if [[ -z "$pod_selector" ]]; then + printf '0' + return 0 + fi + if [[ "$pod_selector" != *"="* ]]; then + pod_selector="app=${app_name}" + fi + local count + count=$(kubectl -n "$NAMESPACE" get pods -l "$pod_selector" \ + --field-selector=status.phase!=Succeeded,status.phase!=Failed \ + --no-headers 2>/dev/null | wc -l | xargs || echo "0") + if [[ -z "$count" || ! "$count" =~ ^[0-9]+$ ]]; then + count=0 + fi + printf '%s' "$count" +} + +gitlab_non_terminal_pod_names_for_app() { + local app_name="$1" + local pod_selector="$app_name" + if [[ -z "$pod_selector" ]]; then + return 0 + fi + if [[ "$pod_selector" != *"="* ]]; then + pod_selector="app=${app_name}" + fi + local names + names=$(kubectl -n "$NAMESPACE" get pods -l "$pod_selector" \ + --field-selector=status.phase!=Succeeded,status.phase!=Failed \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}' 2>/dev/null || true) + names="${names% }" + if [[ -n "$names" ]]; then + printf '%s' "$names" + fi + return 0 +} + +gitlab_old_replicaset_live_summary() { + local deployment_name="$1" + local current_revision rs_rows summary="" + + current_revision=$(kubectl -n "$NAMESPACE" get deployment "$deployment_name" \ + -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true) + [[ -n "$current_revision" ]] || return 0 + + rs_rows=$(kubectl -n "$NAMESPACE" get rs -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.ownerReferences[0].kind}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.metadata.annotations.deployment\.kubernetes\.io/revision}{"|"}{.status.replicas}{"\n"}{end}' 2>/dev/null || true) + [[ -n "$rs_rows" ]] || return 0 + + local rs_name owner_kind owner_name rs_revision rs_replicas + while IFS='|' read -r rs_name owner_kind owner_name rs_revision rs_replicas; do + [[ -n "$rs_name" ]] || continue + [[ "$owner_kind" == "Deployment" && "$owner_name" == "$deployment_name" ]] || continue + [[ "$rs_revision" != "$current_revision" ]] || continue + + if [[ -z "$rs_replicas" || ! "$rs_replicas" =~ ^[0-9]+$ ]]; then + rs_replicas=0 + fi + if (( rs_replicas > 0 )); then + summary+="${rs_name}:${rs_replicas}," + fi + done <<< "$rs_rows" + + summary="${summary%,}" + if [[ -n "$summary" ]]; then + printf '%s' "$summary" + fi + return 0 +} + +gitlab_replica_source_of_truth_report() { + local target_replicas="$1" + local report="" + local dep_suffix dep_name live_replicas + + for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2"; do + dep_name="${GITLAB_RELEASE}-${dep_suffix}" + live_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + + if [[ -z "$live_replicas" ]]; then + report+="- ${dep_name}: spec=, desired=${target_replicas}"$'\n' + continue + fi + + if [[ ! "$live_replicas" =~ ^[0-9]+$ ]]; then + report+="- ${dep_name}: spec=${live_replicas}, desired=${target_replicas}"$'\n' + continue + fi + + if [[ "$live_replicas" != "$target_replicas" ]]; then + report+="- ${dep_name}: spec=${live_replicas}, desired=${target_replicas}"$'\n' + fi + done + + printf '%s' "$report" +} + +gitlab_rendered_replica_source_fields_from_cr() { + local desired_cr="$1" + [[ -n "$desired_cr" ]] || return 0 + + echo "$desired_cr" | awk ' +BEGIN { + in_values = 0 + component = "" +} +{ + line = $0 + if (line ~ /^ values:[[:space:]]*$/) { + in_values = 1 + next + } + if (!in_values) { + next + } + + if (line ~ /^ webservice:[[:space:]]*$/) { + component = "gitlab.webservice" + next + } + if (line ~ /^ sidekiq:[[:space:]]*$/) { + component = "gitlab.sidekiq" + next + } + if (line ~ /^ gitlab-shell:[[:space:]]*$/) { + component = "gitlab.gitlab-shell" + next + } + if (line ~ /^ kas:[[:space:]]*$/) { + component = "gitlab.kas" + next + } + if (line ~ /^ registry:[[:space:]]*$/) { + component = "registry" + next + } + + if (line ~ /^ gitaly:[[:space:]]*$/ || line ~ /^ postgresql:[[:space:]]*$/ || line ~ /^ redis:[[:space:]]*$/) { + component = "" + } + if (line ~ /^ gitaly:[[:space:]]*$/ || line ~ /^ toolbox:[[:space:]]*$/) { + component = "" + } + + if (component != "" && line ~ /^[[:space:]]*(replicaCount|minReplicas|maxReplicas|hpa):[[:space:]]*/) { + sub(/^[[:space:]]+/, "", line) + print "- " component "." line + } +}' +} + +gitlab_verify_replica_source_of_truth() { + local target_replicas="$1" + local verify_timeout_s="${GITLAB_SOURCE_REPLICA_VERIFY_TIMEOUT:-180}" + local verify_poll_interval_s="${GITLAB_SOURCE_REPLICA_VERIFY_POLL_INTERVAL:-10}" + local rendered_replica_fields + + rendered_replica_fields="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")" + if [[ -z "$rendered_replica_fields" ]]; then + rendered_replica_fields="- (no replica-related fields detected in rendered CR values)" + fi + + if [[ -z "$verify_timeout_s" || ! "$verify_timeout_s" =~ ^[0-9]+$ || "$verify_timeout_s" == "0" ]]; then + verify_timeout_s=180 + fi + if [[ -z "$verify_poll_interval_s" || ! "$verify_poll_interval_s" =~ ^[0-9]+$ || "$verify_poll_interval_s" == "0" ]]; then + verify_poll_interval_s=10 + fi + + local verify_start_ts verify_now_ts mismatch_report + verify_start_ts=$(date +%s) + while true; do + mismatch_report=$(gitlab_replica_source_of_truth_report "$target_replicas") + if [[ -z "$mismatch_report" ]]; then + log "GitLab operator desired replicas are source-of-truth converged (gitlab-shell/kas/registry/sidekiq spec=${target_replicas})." + return 0 + fi + + verify_now_ts=$(date +%s) + if (( verify_now_ts - verify_start_ts >= verify_timeout_s )); then + repair_blocked "GitLab operator desired replica source-of-truth mismatch" \ + "Operator-managed Deployment specs did not converge to desired=${target_replicas} after GitLab CR apply:\n${mismatch_report}Rendered GitLab CR replica source fields:\n${rendered_replica_fields}\nThis is a source-of-truth issue (CR values still resolve to replicas>1), not a rollout lag issue." + fi + + log "Waiting for GitLab operator to apply source-of-truth replicas (desired=${target_replicas}) before settle verification..." + while IFS= read -r mismatch_line; do + [[ -n "$mismatch_line" ]] || continue + log " ${mismatch_line}" + done <<< "$mismatch_report" + sleep "$verify_poll_interval_s" + done +} + +gitlab_webservice_blocked_reasons() { + local webservice_app="$1" + local webservice_selector="$webservice_app" + if [[ -z "$webservice_selector" ]]; then + return 0 + fi + if [[ "$webservice_selector" != *"="* ]]; then + webservice_selector="app=${webservice_app}" + fi + local not_ready_block_s="${GITLAB_WEBSERVICE_NOT_READY_BLOCK_SECONDS:-180}" + if [[ -z "$not_ready_block_s" || ! "$not_ready_block_s" =~ ^[0-9]+$ ]]; then + not_ready_block_s=180 + fi + local pod_names + pod_names=$(kubectl -n "$NAMESPACE" get pods -l "$webservice_selector" \ + --field-selector=status.phase!=Succeeded,status.phase!=Failed \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true) + [[ -n "$pod_names" ]] || return 0 + + local pod_name + while IFS= read -r pod_name; do + [[ -n "$pod_name" ]] || continue + + local pod_phase pod_reason + pod_phase=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + pod_reason=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.reason}' 2>/dev/null || true) + + if [[ "$pod_reason" == "ContainerStatusUnknown" || "$pod_phase" == "Unknown" ]]; then + printf '%s\n' "pod ${pod_name} status is ${pod_reason:-$pod_phase}" + fi + + local pod_ready_status pod_ready_transition + pod_ready_status=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) + pod_ready_transition=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].lastTransitionTime}' 2>/dev/null || true) + + local container_rows + container_rows=$(kubectl -n "$NAMESPACE" get pod "$pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"|"}{.state.waiting.reason}{"|"}{.state.terminated.reason}{"\n"}{end}' 2>/dev/null || true) + [[ -n "$container_rows" ]] || continue + + local container_name waiting_reason terminated_reason + while IFS='|' read -r container_name waiting_reason terminated_reason; do + [[ -n "$container_name" ]] || continue + + case "$waiting_reason" in + CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerError|CreateContainerConfigError|RunContainerError|Error) + printf '%s\n' "pod ${pod_name}/${container_name} waiting reason=${waiting_reason}" + ;; + esac + case "$terminated_reason" in + Error|ContainerCannotRun) + printf '%s\n' "pod ${pod_name}/${container_name} terminated reason=${terminated_reason}" + ;; + esac + done <<< "$container_rows" + + if [[ "$pod_ready_status" != "True" ]]; then + local ready_false_age_s=0 + if [[ -n "$pod_ready_transition" ]]; then + ready_false_age_s=$(python3 - "$pod_ready_transition" <<'PY' +import datetime +import sys + +raw = (sys.argv[1] if len(sys.argv) > 1 else "").strip() +if not raw: + print("0") + raise SystemExit(0) + +try: + ts = datetime.datetime.fromisoformat(raw.replace("Z", "+00:00")) + now = datetime.datetime.now(datetime.timezone.utc) + age = int((now - ts).total_seconds()) + print(str(max(age, 0))) +except Exception: + print("0") +PY +) + fi + if [[ -z "$ready_false_age_s" || ! "$ready_false_age_s" =~ ^[0-9]+$ ]]; then + ready_false_age_s=0 + fi + if (( ready_false_age_s >= not_ready_block_s )); then + printf '%s\n' "pod ${pod_name} Ready=False for ${ready_false_age_s}s (threshold=${not_ready_block_s}s)" + fi + fi + done <<< "$pod_names" +} + +wait_for_gitlab_workload_convergence() { + local timeout_s="${GITLAB_WORKLOAD_CONVERGENCE_TIMEOUT:-420}" + local poll_interval_s="${GITLAB_WORKLOAD_CONVERGENCE_POLL_INTERVAL:-10}" + + if [[ -z "$timeout_s" || ! "$timeout_s" =~ ^[0-9]+$ ]]; then + timeout_s=420 + fi + if [[ -z "$poll_interval_s" || ! "$poll_interval_s" =~ ^[0-9]+$ || "$poll_interval_s" == "0" ]]; then + poll_interval_s=10 + fi + + local start_ts + start_ts=$(date +%s) + local last_report="" + + while true; do + local -a blockers=() + local dep_suffix dep_name dep_selector desired_replicas live_non_terminal old_rs_summary + local ready_replicas available_replicas + for dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do + dep_name="${GITLAB_RELEASE}-${dep_suffix}" + + desired_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + if [[ -z "$desired_replicas" ]]; then + blockers+=("${dep_name}: deployment not found") + continue + fi + if [[ ! "$desired_replicas" =~ ^[0-9]+$ ]]; then + desired_replicas=1 + fi + + dep_selector=$(gitlab_selector_for_deployment "$dep_name" 2>/dev/null || true) + if [[ -z "$dep_selector" ]]; then + blockers+=("${dep_name}: deployment selector is empty") + continue + fi + + live_non_terminal=$(gitlab_non_terminal_pod_count_for_app "$dep_selector") + if (( live_non_terminal > desired_replicas )); then + blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} > desired=${desired_replicas} (selector=${dep_selector})") + elif (( live_non_terminal < desired_replicas )); then + blockers+=("${dep_name}: non-terminal pods=${live_non_terminal} < desired=${desired_replicas} (selector=${dep_selector})") + fi + + ready_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true) + available_replicas=$(kubectl -n "$NAMESPACE" get deployment "$dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true) + if [[ -z "$ready_replicas" || ! "$ready_replicas" =~ ^[0-9]+$ ]]; then + ready_replicas=0 + fi + if [[ -z "$available_replicas" || ! "$available_replicas" =~ ^[0-9]+$ ]]; then + available_replicas=0 + fi + if (( ready_replicas < desired_replicas )); then + blockers+=("${dep_name}: readyReplicas=${ready_replicas} < desired=${desired_replicas}") + fi + if (( available_replicas < desired_replicas )); then + blockers+=("${dep_name}: availableReplicas=${available_replicas} < desired=${desired_replicas}") + fi + + old_rs_summary=$(gitlab_old_replicaset_live_summary "$dep_name") + if [[ -n "$old_rs_summary" ]]; then + blockers+=("${dep_name}: old ReplicaSet pods still running (${old_rs_summary})") + fi + + if [[ "$dep_suffix" == "webservice-default" ]]; then + local webservice_reasons + webservice_reasons=$(gitlab_webservice_blocked_reasons "$dep_selector" 2>/dev/null || true) + if [[ -n "$webservice_reasons" ]]; then + local webservice_reason + while IFS= read -r webservice_reason; do + [[ -n "$webservice_reason" ]] || continue + blockers+=("${dep_name}: ${webservice_reason}") + done <<< "$webservice_reasons" + fi + fi + done + + if (( ${#blockers[@]} == 0 )); then + log "GitLab workloads converged: pod counts, ReplicaSets, and webservice health are clean." + return 0 + fi + + local report="" + local blocker + for blocker in "${blockers[@]}"; do + report+="- ${blocker}"$'\n' + done + + local now_ts + now_ts=$(date +%s) + if (( now_ts - start_ts >= timeout_s )); then + repair_blocked "GitLab workloads are not converged after reconcile" \ + "Namespace: ${NAMESPACE}. Remaining blockers: +${report}Check: kubectl -n ${NAMESPACE} get deploy,rs,pods -o wide" + fi + + if [[ "$report" != "$last_report" ]]; then + warn "Waiting for GitLab workload convergence..." + local report_line + while IFS= read -r report_line; do + [[ -n "$report_line" ]] || continue + warn " ${report_line}" + done <<< "$report" + last_report="$report" + fi + sleep "$poll_interval_s" + done +} + +get_gitlab_migrations_diagnostics() { + local job_name="$1" + [[ -n "$job_name" ]] || return 0 + + local _pod_name=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$job_name" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + if [[ -z "$_pod_name" ]]; then + echo "No pods found for migration job $job_name." + return 0 + fi + + local _diag="" + _diag+="--- Migrations Diagnostics for Pod: $_pod_name ---"$'\n' + _diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o wide 2>/dev/null || true)"$'\n' + + _diag+=$'\n'"[Container Statuses]"$'\n' + _diag+="$(kubectl -n "$NAMESPACE" get pod "$_pod_name" -o jsonpath='{range .status.containerStatuses[*]}{.name}: state={.state.waiting.reason}{.state.terminated.reason}{.state.running.startedAt}, exitCode={.state.terminated.exitCode}, restarts={.restartCount}{"\n"}{end}' 2>/dev/null || true)"$'\n' + + _diag+=$'\n'"[Migrations Logs (tail=100)]"$'\n' + _diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --tail=100 2>/dev/null || echo "(no logs available)")"$'\n' + + _diag+=$'\n'"[Previous Migrations Logs (if any)]"$'\n' + _diag+="$(kubectl -n "$NAMESPACE" logs "$_pod_name" -c migrations --previous --tail=100 2>/dev/null || echo "(no previous logs)")"$'\n' + + _diag+=$'\n'"[Pod Events]"$'\n' + _diag+="$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$_pod_name" --sort-by='.lastTimestamp' 2>/dev/null | tail -n 10 || true)" + + echo "$_diag" +} + +check_gitlab_migrations_blocked() { + local _latest_job + _latest_job=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + [[ -n "$_latest_job" ]] || return 0 + + local _job_failed + _job_failed=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) + + # 1. Active Failure Detection + local _latest_pod + _latest_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + if [[ -n "$_latest_pod" ]]; then + local _restarts + _restarts=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0") + local _waiting_reason + _waiting_reason=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "") + local _exit_code + _exit_code=$(kubectl -n "$NAMESPACE" get pod "$_latest_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.terminated.exitCode}' 2>/dev/null || echo "") + + if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_exit_code" && "$_exit_code" != "0" ) ]]; then + log "GitLab migrations job is actively failing: ${_latest_job} (pod: ${_latest_pod}, restarts: ${_restarts}, exitCode: ${_exit_code:-unknown})" + + # Enhanced DB connectivity diagnostics for split-cluster visibility + local _db_info="DB_HOST=${DB_HOST}, DB_PORT=${DB_PORT}" + local _split_cluster="No" + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then _split_cluster="Yes (APP: ${app_ctx}, DB: ${db_ctx})"; fi + + local _diag + _diag=$(get_gitlab_migrations_diagnostics "$_latest_job") + + local _connectivity_hint="" + if echo "$_diag" | grep -qi "connecting with your hostname"; then + _connectivity_hint=" (Likely DNS/resolution failure)" + elif echo "$_diag" | grep -qiE "connection refused|timeout"; then + _connectivity_hint=" (Likely TCP connectivity/firewall failure)" + fi + + repair_blocked "GitLab migrations job is actively failing" \ + "Job: ${_latest_job}. Pod: ${_latest_pod}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_exit_code:-unknown}. Restarts: ${_restarts}. +DB Config: ${_db_info}${_connectivity_hint} +Split-cluster: ${_split_cluster} +Diagnostics: +${_diag}" + fi + fi + + # 2. Stale Failed Job Repair + local _job_creation_ts + _job_creation_ts=$(kubectl -n "$NAMESPACE" get job "$_latest_job" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true) + local _job_pods_running + _job_pods_running=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0") + local _job_pods_pending + _job_pods_pending=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_latest_job" -o jsonpath='{range .items[?(@.status.phase=="Pending")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | wc -l | xargs || echo "0") + + local _now_ts + _now_ts=$(date +%s) + local _creation_ts + _creation_ts=$(python3 -c "from datetime import datetime; print(int(datetime.strptime('$_job_creation_ts'.replace('Z', '+0000'), '%Y-%m-%dT%H:%M:%S%z').timestamp()))" 2>/dev/null || echo "0") + local _age + _age=$(( _now_ts - _creation_ts )) + + if [[ "$_job_failed" == "True" && "$_job_pods_running" == "0" && "$_job_pods_pending" == "0" && $_age -gt 600 ]]; then + log "GitLab operator is stalled on stale failed migrations job: ${_latest_job} (age: ${_age}s, no active pods)" + log "Deleting stale failed migrations job to trigger repair..." + kubectl -n "$NAMESPACE" delete job "$_latest_job" --wait=true 2>/dev/null || true + kubectl -n "$NAMESPACE" delete pods -l "job-name=$_latest_job" --force --grace-period=0 2>/dev/null || true + log "Stale migrations job deleted. Operator should recreate it shortly." + fi +} + +check_host_resolves() { + local target_host="$1" + # Strip port if present + local host_only="${target_host%:*}" + local target_ctx + target_ctx="$(resolve_explicit_kube_context || true)" + + local is_k8s_svc=0 + if [[ "$host_only" == *".svc.cluster.local" ]]; then + is_k8s_svc=1 + elif [[ "$host_only" =~ ^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+$ ]]; then + # Simple . format (exactly one dot, no numbers to avoid IPs) + is_k8s_svc=1 + fi + + if [[ $is_k8s_svc -eq 1 ]]; then + # Parse service and namespace (first and second labels) + local svc_name svc_ns + svc_name=$(echo "$host_only" | cut -d. -f1) + svc_ns=$(echo "$host_only" | cut -d. -f2) + + log "Redis validation mode: kubernetes-service (cluster: ${target_ctx:-default})" + log "Parsed REDIS_HOST ${target_host} -> service=${svc_name} namespace=${svc_ns}" + + if kubectl -n "$svc_ns" get service "$svc_name" >/dev/null 2>&1; then + log "Service ${svc_name} found in namespace ${svc_ns}" + # Preferably verify it has endpoints / ready backing pods + local endpoints_found + endpoints_found=$(kubectl -n "$svc_ns" get endpoints "$svc_name" -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true) + if [[ -n "$endpoints_found" ]]; then + log "Redis endpoints present for ${svc_name}" + else + warn "Service ${svc_name} found in namespace ${svc_ns} but has NO ready endpoints (yet)." + fi + return 0 + else + warn "Service ${svc_name} NOT found in namespace ${svc_ns} (cluster: ${target_ctx:-default})" + return 1 + fi + else + log "Redis validation mode: external-host" + if ! host "$host_only" >/dev/null 2>&1; then + # If it's an IP, host might fail. Check if it's an IP. + if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + log "Host ${host_only} is an IP address, skipping DNS check." + return 0 + fi + return 1 + fi + fi + return 0 +} + +check_db_connectivity() { + local target_host="$1" + local target_port="${2:-5432}" + local host_only="${target_host%:*}" + + log "Validating GitLab database connectivity to ${target_host}:${target_port}..." + + # DNS sanity check + if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + log "Host ${host_only} is an IP address, skipping DNS check." + else + # In k8s mode, internal DNS resolution might fail from the script runner + # but work from within the cluster. We do a basic 'host' check as a hint. + if ! host "$host_only" >/dev/null 2>&1; then + warn "DNS resolution (host) failed for database host: ${host_only}. This may be expected if only resolvable in-cluster." + else + log "DNS resolution successful for ${host_only}" + fi + fi + + # TCP check via temporary pod in the APP cluster namespace + log "Performing in-cluster TCP connectivity probe to ${host_only}:${target_port} (namespace: ${NAMESPACE})..." + if kubectl -n "$NAMESPACE" run db-probe \ + --image=alpine:latest --restart=Never --rm --attach --timeout=30s \ + --command -- sh -c "nc -zv -w 5 ${host_only} ${target_port}" >/dev/null 2>&1; then + log "TCP connectivity to ${host_only}:${target_port} SUCCESSFUL." + return 0 + else + warn "TCP connectivity probe to ${host_only}:${target_port} FAILED." + return 1 + fi +} + +check_gitlab_pre_apply_blocked() { + # --- Registry Endpoint Placeholder Check --- + if [[ "$GARAGE_S3_ENDPOINT" == *""* ]]; then + # If the configured value itself is still a placeholder, we MUST block. + # The operator hasn't provided a real value yet. + repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \ + "GARAGE_PRIVATE_S3_ENDPOINT is set to the literal placeholder. Set it to the real DB cluster Garage endpoint." + fi + + # --- Redis Host Resolution Check --- + if ! check_host_resolves "$REDIS_HOST"; then + # This is potentially repairable if we can infer a better host, + # but for now we block if it's clearly invalid. + repair_blocked "Redis host does not resolve" \ + "Resource: REDIS_HOST. Value: ${REDIS_HOST} does not resolve. Fix: Ensure Redis is deployed and REDIS_NAMESPACE is correct." + fi + + # --- Database Connectivity Check --- + if [[ "$MODE" == "k8s" ]]; then + if ! check_db_connectivity "$DB_HOST" "$DB_PORT"; then + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + repair_blocked "GitLab database host is not reachable from APP cluster" \ + "Host: ${DB_HOST}. Port: ${DB_PORT}. App Context: ${app_ctx}. DB Context: ${db_ctx}. Fix: Ensure cross-cluster networking (ILB) is functional." + fi + fi +} + +get_gitlab_blocker_context() { + local ctx="" + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then + ctx+="Split-cluster (APP: ${app_ctx}, DB: ${db_ctx}). " + fi + ctx+="DB_HOST: ${DB_HOST}. " + + local _last_mig=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + if [[ -n "$_last_mig" ]]; then + local _m_ts=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true) + local _m_failed=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) + local _m_pods=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --no-headers 2>/dev/null | wc -l | xargs || echo "0") + + local _m_pod_ctx="" + local _m_last_pod=$(kubectl -n "$NAMESPACE" get pods -l "job-name=$_last_mig" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + if [[ -n "$_m_last_pod" ]]; then + local _m_pod_status=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.phase}' 2>/dev/null || true) + local _m_restarts=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].restartCount}' 2>/dev/null || echo "0") + local _m_waiting=$(kubectl -n "$NAMESPACE" get pod "$_m_last_pod" -o jsonpath='{.status.containerStatuses[?(@.name=="migrations")].state.waiting.reason}' 2>/dev/null || echo "") + _m_pod_ctx="Pod: ${_m_last_pod} (${_m_pod_status}, restarts: ${_m_restarts}, waiting: ${_m_waiting:-None}). " + fi + ctx+="Migration Job: ${_last_mig} (Created: ${_m_ts}, Failed: ${_m_failed:-False}, Pods: ${_m_pods}). ${_m_pod_ctx}" + fi + + local _sts_name="${GITLAB_RELEASE}-gitaly" + local _sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$_sts_name" -o yaml 2>/dev/null || true) + if [[ -n "$_sts_yaml" ]]; then + local _sts_ns=$(echo "$_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true) + local _sts_sc=$(echo "$_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true) + ctx+="Gitaly STS: nodeSelector=[${_sts_ns}], storageClass=[${_sts_sc}]. " + fi + + local _pvc_name="repo-data-gitlab-gitaly-0" + local _pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$_pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ -n "$_pvc_phase" ]]; then + ctx+="Gitaly PVC: ${_pvc_phase}. " + fi + + local _op_pod=$(kubectl -n "$NAMESPACE" get pods -l "control-plane=controller-manager" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -n "$_op_pod" ]]; then + local _op_log=$(kubectl -n "$NAMESPACE" logs "$_op_pod" -c manager --tail=1 2>/dev/null || true) + ctx+="Operator Log: ${_op_log}. " + fi + echo "$ctx" +} + +gitlab_storage_class_matches_expected() { + local live_sc="$1" + local expected_sc="$2" + [[ -z "$expected_sc" ]] && return 0 + [[ "$live_sc" == "$expected_sc" ]] +} + +gitlab_rendered_storage_fields_from_cr() { + local desired_cr="$1" + local rendered_global_sc="" + local rendered_gitlab_gitaly_sc="" + local rendered_chart_gitaly_sc="" + + if [[ -n "$desired_cr" ]]; then + rendered_global_sc=$(echo "$desired_cr" | sed -n '/^ global:/,/^ postgresql:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true) + rendered_gitlab_gitaly_sc=$(echo "$desired_cr" | sed -n '/^ gitaly:/,/^ toolbox:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true) + rendered_chart_gitaly_sc=$(echo "$desired_cr" | sed -n '/^ gitaly:/,/^ registry:/p' | grep 'storageClass:' | head -n1 | cut -d: -f2 | xargs || true) + fi + + printf '%s|%s|%s\n' "$rendered_global_sc" "$rendered_gitlab_gitaly_sc" "$rendered_chart_gitaly_sc" +} + +gitlab_chart_major_version() { + local chart_version="${GITLAB_CHART_VERSION:-}" + chart_version="${chart_version#v}" + chart_version="${chart_version%%-*}" + chart_version="${chart_version%%.*}" + printf '%s' "$chart_version" +} + +gitlab_chart_is_v9_or_newer() { + local chart_major + chart_major="$(gitlab_chart_major_version)" + [[ "$chart_major" =~ ^[0-9]+$ ]] || return 1 + (( chart_major >= 9 )) +} + +gitlab_chart9_find_deprecated_top_level_replica_keys() { + local desired_cr="$1" + [[ -n "$desired_cr" ]] || return 0 + + echo "$desired_cr" | awk ' +function leading_spaces(str, i, c, n) { + n = 0 + for (i = 1; i <= length(str); i++) { + c = substr(str, i, 1) + if (c == " ") { + n++ + } else { + break + } + } + return n +} +BEGIN { + in_values = 0 + in_registry = 0 + registry_indent = -1 + in_hpa = 0 + hpa_indent = -1 +} +{ + line = $0 + if (line ~ /^ values:[[:space:]]*$/) { + in_values = 1 + next + } + if (!in_values) { + next + } + + indent = leading_spaces(line) + if (in_hpa && indent <= hpa_indent) { + in_hpa = 0 + hpa_indent = -1 + } + if (in_registry && indent <= registry_indent && line !~ /^ registry:[[:space:]]*$/) { + in_registry = 0 + registry_indent = -1 + } + + if (line ~ /^ registry:[[:space:]]*$/) { + in_registry = 1 + registry_indent = indent + next + } + if (!in_registry) { + next + } + + if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) { + in_hpa = 1 + hpa_indent = indent + next + } + + if (!in_hpa && line ~ /^[[:space:]]*(minReplicas|maxReplicas):[[:space:]]*/) { + print NR ":" line + } +}' +} + +gitlab_chart9_strip_deprecated_top_level_replica_keys() { + local desired_cr="$1" + [[ -n "$desired_cr" ]] || { + printf '%s' "$desired_cr" + return 0 + } + + echo "$desired_cr" | awk ' +function leading_spaces(str, i, c, n) { + n = 0 + for (i = 1; i <= length(str); i++) { + c = substr(str, i, 1) + if (c == " ") { + n++ + } else { + break + } + } + return n +} +BEGIN { + in_values = 0 + in_registry = 0 + registry_indent = -1 + in_hpa = 0 + hpa_indent = -1 +} +{ + line = $0 + if (line ~ /^ values:[[:space:]]*$/) { + in_values = 1 + print line + next + } + if (!in_values) { + print line + next + } + + indent = leading_spaces(line) + if (in_hpa && indent <= hpa_indent) { + in_hpa = 0 + hpa_indent = -1 + } + if (in_registry && indent <= registry_indent && line !~ /^ registry:[[:space:]]*$/) { + in_registry = 0 + registry_indent = -1 + } + + if (line ~ /^ registry:[[:space:]]*$/) { + in_registry = 1 + registry_indent = indent + print line + next + } + if (!in_registry) { + print line + next + } + + if (line ~ /^[[:space:]]*hpa:[[:space:]]*$/) { + in_hpa = 1 + hpa_indent = indent + print line + next + } + + if (!in_hpa && line ~ /^[[:space:]]*(minReplicas|maxReplicas):[[:space:]]*/) { + next + } + + print line +}' +} + +sanitize_gitlab_cr_rendered_values_for_chart_version() { + local desired_cr="$1" + [[ -n "$desired_cr" ]] || { + printf '%s' "$desired_cr" + return 0 + } + + if ! gitlab_chart_is_v9_or_newer; then + printf '%s' "$desired_cr" + return 0 + fi + + local deprecated_keys_before + deprecated_keys_before="$(gitlab_chart9_find_deprecated_top_level_replica_keys "$desired_cr")" + if [[ -n "$deprecated_keys_before" ]]; then + log "GitLab chart ${GITLAB_CHART_VERSION}: stripping deprecated top-level replica keys before CR apply:" + while IFS= read -r deprecated_key; do + [[ -n "$deprecated_key" ]] || continue + log " - ${deprecated_key}" + done <<< "$deprecated_keys_before" + desired_cr="$(gitlab_chart9_strip_deprecated_top_level_replica_keys "$desired_cr")" + fi + + printf '%s' "$desired_cr" +} + +preflight_validate_gitlab_cr_rendered_values() { + local desired_cr="$1" + [[ -n "$desired_cr" ]] || return 0 + + if ! gitlab_chart_is_v9_or_newer; then + return 0 + fi + + local deprecated_keys_remaining + deprecated_keys_remaining="$(gitlab_chart9_find_deprecated_top_level_replica_keys "$desired_cr")" + if [[ -n "$deprecated_keys_remaining" ]]; then + repair_blocked "Rendered GitLab CR values contain removed chart v9+ keys" \ + "Local preflight rejected GitLab CR apply. Deprecated top-level replica keys remain (for example registry.minReplicas/registry.maxReplicas). Remaining keys: ${deprecated_keys_remaining//$'\n'/; }." + fi +} + +gitlab_live_gitaly_repo_data_template_storage_class() { + local gitaly_sts_name="${GITLAB_RELEASE}-gitaly" + local live_template_sc + live_template_sc=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o jsonpath='{range .spec.volumeClaimTemplates[?(@.metadata.name=="repo-data")]}{.spec.storageClassName}{end}' 2>/dev/null || true) + if [[ -z "$live_template_sc" ]]; then + live_template_sc=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o jsonpath='{.spec.volumeClaimTemplates[0].spec.storageClassName}' 2>/dev/null || true) + fi + printf '%s' "$live_template_sc" +} + +cleanup_gitlab_wrong_gitaly_template_storage() { + local expected_sc="$1" + local live_template_sc="$2" + local pvc_mismatch_records="$3" + local desired_cr="$4" + local gitaly_sts_name="${GITLAB_RELEASE}-gitaly" + + export GITALY_AUTOCLEAN_PERFORMED=1 + log "AUTOCLEAN: repairing wrong-class Gitaly StatefulSet template storageClass (expected='${expected_sc}', live='${live_template_sc:-}')." + log "Rendered GitLab CR storage fields prior to repair: $(gitlab_rendered_storage_fields_from_cr "$desired_cr")" + + kubectl -n "$NAMESPACE" scale statefulset "$gitaly_sts_name" --replicas=0 --timeout=30s 2>/dev/null || true + if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then + log "Deleting StatefulSet ${gitaly_sts_name} to force operator recreation of repo-data claim template..." + kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true + fi + + if [[ -n "$pvc_mismatch_records" ]]; then + cleanup_gitlab_wrong_storage_class_records "$pvc_mismatch_records" + fi + + # Also clear retained stale GKE dynamic-provisioning leftovers for repo-data claim. + repair_stale_gke_gitaly_dynamic_storage "$expected_sc" +} + +is_gke_stale_gitaly_storage_class_for_standard_target() { + local desired_sc="$1" + local candidate_sc="$2" + [[ "$desired_sc" == "standard" ]] || return 1 + [[ -n "$candidate_sc" && "$candidate_sc" != "$desired_sc" ]] && return 0 + return 1 +} + +is_gke_blocking_disk_type_for_standard_target() { + local disk_type="$1" + case "$disk_type" in + pd-balanced|pd-ssd) + return 0 + ;; + esac + return 1 +} + +gke_fetch_disk_type_from_volume_handle() { + local volume_handle="$1" + [[ -n "$volume_handle" ]] || return 0 + command -v gcloud >/dev/null 2>&1 || return 0 + + local parsed_ref + parsed_ref=$(gke_parse_disk_ref_from_volume_handle "$volume_handle" || true) + [[ -n "$parsed_ref" ]] || return 0 + + local disk_scope disk_location disk_name + IFS='|' read -r disk_scope disk_location disk_name <<< "$parsed_ref" + [[ -n "$disk_name" ]] || return 0 + + local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}" + if [[ -z "$project_id" ]]; then + project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true) + fi + + local -a common_args=() + if [[ -n "$project_id" ]]; then + common_args+=(--project "$project_id") + fi + + local disk_type="" + if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then + disk_type=$(gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" --format='value(type.basename())' 2>/dev/null || true) + elif [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then + disk_type=$(gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" --format='value(type.basename())' 2>/dev/null || true) + else + local matched + matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](type.basename())' 2>/dev/null | head -n1 || true) + disk_type="$matched" + fi + + printf '%s' "$disk_type" +} + +collect_gitlab_storage_mismatch_records() { + local expected_sc="$1" + [[ -n "$expected_sc" ]] || return 0 + + local pvc_rows + pvc_rows=$(kubectl -n "$NAMESPACE" get pvc -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.storageClassName}{"|"}{.status.phase}{"|"}{.spec.volumeName}{"|"}{.metadata.labels.app\.kubernetes\.io/instance}{"\n"}{end}' 2>/dev/null || true) + [[ -n "$pvc_rows" ]] || return 0 + + while IFS='|' read -r pvc_name pvc_sc pvc_phase pvc_pv pvc_instance; do + [[ -n "$pvc_name" ]] || continue + + local is_gitlab_pvc=0 + if [[ "$pvc_instance" == "$GITLAB_RELEASE" || "$pvc_name" == "repo-data-${GITLAB_RELEASE}-"* || "$pvc_name" == "${GITLAB_RELEASE}-"* ]]; then + is_gitlab_pvc=1 + fi + [[ "$is_gitlab_pvc" == "1" ]] || continue + + local pv_sc="" disk_handle="" disk_type="" disk_scope="" disk_location="" disk_name="" + if [[ -n "$pvc_pv" ]]; then + local pv_details pv_disk_name + pv_details=$(kubectl get pv "$pvc_pv" -o jsonpath='{.spec.storageClassName}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"|"}{.spec.csi.volumeAttributes.type}' 2>/dev/null || true) + if [[ -n "$pv_details" ]]; then + IFS='|' read -r pv_sc disk_handle pv_disk_name disk_type <<< "$pv_details" + fi + + if [[ -z "$disk_handle" && -n "$pv_disk_name" ]]; then + disk_handle="$pv_disk_name" + fi + + if [[ -n "$disk_handle" ]]; then + local parsed_ref + parsed_ref=$(gke_parse_disk_ref_from_volume_handle "$disk_handle" || true) + if [[ -n "$parsed_ref" ]]; then + IFS='|' read -r disk_scope disk_location disk_name <<< "$parsed_ref" + fi + fi + + if [[ -z "$disk_type" && -n "$disk_handle" ]]; then + disk_type=$(gke_fetch_disk_type_from_volume_handle "$disk_handle" || true) + fi + fi + + local effective_live_sc="$pvc_sc" + if [[ -z "$effective_live_sc" && -n "$pv_sc" ]]; then + effective_live_sc="$pv_sc" + fi + + if ! gitlab_storage_class_matches_expected "$effective_live_sc" "$expected_sc"; then + printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ + "$pvc_name" "$pvc_sc" "$pvc_phase" "$pvc_pv" "$pv_sc" "$disk_handle" "$disk_scope" "$disk_location" "$disk_name" "$disk_type" + fi + done <<< "$pvc_rows" +} + +collect_gitlab_statefulset_template_mismatch_records() { + local expected_sc="$1" + [[ -n "$expected_sc" ]] || return 0 + + local sts_rows + sts_rows=$(kubectl -n "$NAMESPACE" get statefulset -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.labels.app\.kubernetes\.io/instance}{"|"}{range .spec.volumeClaimTemplates[*]}{.metadata.name}{":"}{.spec.storageClassName}{","}{end}{"\n"}{end}' 2>/dev/null || true) + [[ -n "$sts_rows" ]] || return 0 + + while IFS='|' read -r sts_name sts_instance claims; do + [[ -n "$sts_name" ]] || continue + + local is_gitlab_sts=0 + if [[ "$sts_instance" == "$GITLAB_RELEASE" || "$sts_name" == "${GITLAB_RELEASE}-"* ]]; then + is_gitlab_sts=1 + fi + [[ "$is_gitlab_sts" == "1" ]] || continue + [[ -n "$claims" ]] || continue + + IFS=',' read -ra claim_pairs <<< "$claims" + local pair + for pair in "${claim_pairs[@]}"; do + [[ -n "$pair" ]] || continue + local claim_name claim_sc + claim_name="${pair%%:*}" + claim_sc="${pair#*:}" + if ! gitlab_storage_class_matches_expected "$claim_sc" "$expected_sc"; then + printf '%s|%s|%s\n' "$sts_name" "$claim_name" "$claim_sc" + fi + done + done <<< "$sts_rows" +} + +cleanup_gitlab_wrong_storage_class_records() { + local mismatch_records="$1" + [[ -n "$mismatch_records" ]] || return 0 + + export GITALY_AUTOCLEAN_PERFORMED=1 + log "AUTOCLEAN: removing wrong-class GitLab PVC/PV artifacts to enforce storageClass='${GITALY_STORAGE_CLASS:-}'..." + + local processed_disks="|" + local _disk_key + while IFS='|' read -r pvc_name _pvc_sc _pvc_phase pvc_pv _pv_sc disk_handle disk_scope disk_location disk_name _disk_type; do + [[ -n "$pvc_name" ]] || continue + + if [[ "$pvc_name" == "repo-data-${GITLAB_RELEASE}-gitaly-0" ]]; then + kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true + fi + + if kubectl -n "$NAMESPACE" get pvc "$pvc_name" >/dev/null 2>&1; then + log "Deleting wrong-class PVC ${pvc_name}..." + kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true + fi + + if [[ -n "$pvc_pv" ]] && kubectl get pv "$pvc_pv" >/dev/null 2>&1; then + log "Deleting wrong-class PV ${pvc_pv} (PVC=${pvc_name})..." + kubectl delete pv "$pvc_pv" --wait=false 2>/dev/null || true + fi + + local disk_ref="" + if [[ -n "$disk_handle" ]]; then + disk_ref=$(gke_parse_disk_ref_from_volume_handle "$disk_handle" || true) + fi + if [[ -n "$disk_ref" ]]; then + IFS='|' read -r disk_scope disk_location disk_name <<< "$disk_ref" + fi + if [[ -z "$disk_name" && -n "$disk_handle" ]]; then + disk_scope="name" + disk_location="" + disk_name="$disk_handle" + fi + + if [[ -n "$disk_name" ]]; then + _disk_key="${disk_scope}|${disk_location}|${disk_name}" + if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then + processed_disks+="${_disk_key}|" + gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true + fi + fi + done <<< "$mismatch_records" + + sleep 2 +} + +enforce_gitlab_storage_class_target() { + local expected_sc="$1" + local desired_cr="$2" + + [[ "$MODE" == "k8s" ]] || return 0 + [[ -n "$expected_sc" ]] || return 0 + + local rendered_global_sc="" + local rendered_gitlab_gitaly_sc="" + local rendered_chart_gitaly_sc="" + IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")" + + log "GitLab storage target: configured='${expected_sc}', rendered.global.persistence.storageClass='${rendered_global_sc:-}', rendered.gitlab.gitaly.persistence.storageClass='${rendered_gitlab_gitaly_sc:-}', rendered.gitaly.persistence.storageClass='${rendered_chart_gitaly_sc:-}'" + + if [[ -n "$rendered_global_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_global_sc" "$expected_sc"; then + repair_blocked "GitLab CR global persistence storageClass mismatch" \ + "Configured target=[${expected_sc}] but rendered global.persistence.storageClass=[${rendered_global_sc}]. This indicates GitLab CR rendering is wrong." + fi + if [[ -n "$rendered_gitlab_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_gitlab_gitaly_sc" "$expected_sc"; then + repair_blocked "GitLab CR gitaly persistence storageClass mismatch" \ + "Configured target=[${expected_sc}] but rendered gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc}]. This indicates GitLab CR rendering is wrong." + fi + if [[ -n "$rendered_chart_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_chart_gitaly_sc" "$expected_sc"; then + repair_blocked "GitLab CR chart gitaly persistence storageClass mismatch" \ + "Configured target=[${expected_sc}] but rendered gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc}]. This indicates GitLab CR rendering is wrong." + fi + + local template_mismatches + template_mismatches=$(collect_gitlab_statefulset_template_mismatch_records "$expected_sc") + local pvc_mismatches + pvc_mismatches=$(collect_gitlab_storage_mismatch_records "$expected_sc") + + local gitaly_sts_name="${GITLAB_RELEASE}-gitaly" + local gitaly_sts_exists=0 + if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then + gitaly_sts_exists=1 + fi + + local live_gitaly_template_sc + live_gitaly_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class) + local live_gitaly_pvc_sc + live_gitaly_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "repo-data-${GITLAB_RELEASE}-gitaly-0" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + log "GitLab Gitaly live storage snapshot: statefulset=${gitaly_sts_name} claimTemplate.repo-data.storageClassName='${live_gitaly_template_sc:-}' pvc.repo-data-${GITLAB_RELEASE}-gitaly-0.storageClassName='${live_gitaly_pvc_sc:-}'" + if [[ "$gitaly_sts_exists" == "1" ]] && ! gitlab_storage_class_matches_expected "$live_gitaly_template_sc" "$expected_sc"; then + if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then + repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass mismatch" \ + "Configured target=[${expected_sc}] rendered.global.persistence.storageClass=[${rendered_global_sc:-}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-}] live.statefulset=${GITLAB_RELEASE}-gitaly live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_template_sc:-}]. Delete statefulset/${GITLAB_RELEASE}-gitaly, pvc/repo-data-${GITLAB_RELEASE}-gitaly-0, and any wrong-class bound/released PV+disk artifacts, or enable GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 for automatic destructive repair." + fi + + cleanup_gitlab_wrong_gitaly_template_storage "$expected_sc" "$live_gitaly_template_sc" "$pvc_mismatches" "$desired_cr" + + # Re-check after cleanup so subsequent gates reflect live post-repair state. + template_mismatches=$(collect_gitlab_statefulset_template_mismatch_records "$expected_sc") + pvc_mismatches=$(collect_gitlab_storage_mismatch_records "$expected_sc") + gitaly_sts_exists=0 + if kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" >/dev/null 2>&1; then + gitaly_sts_exists=1 + fi + live_gitaly_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class) + if [[ "$gitaly_sts_exists" == "1" ]] && ! gitlab_storage_class_matches_expected "$live_gitaly_template_sc" "$expected_sc"; then + repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass remains wrong after repair" \ + "Configured target=[${expected_sc}] rendered.global.persistence.storageClass=[${rendered_global_sc:-}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-}] live.statefulset=${GITLAB_RELEASE}-gitaly live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_template_sc:-}]." + fi + fi + + if [[ -n "$template_mismatches" ]]; then + log "GitLab StatefulSet volumeClaimTemplate storageClass diagnostics (expected='${expected_sc}')" + while IFS='|' read -r sts_name claim_name claim_sc; do + [[ -n "$sts_name" ]] || continue + log " - workload=statefulset/${sts_name} claimTemplate=${claim_name} storageClass=${claim_sc}" + done <<< "$template_mismatches" + fi + + if [[ -n "$pvc_mismatches" ]]; then + log "GitLab PVC/PV storageClass diagnostics (expected='${expected_sc}')" + while IFS='|' read -r pvc_name pvc_sc pvc_phase pvc_pv pv_sc disk_handle disk_scope disk_location disk_name disk_type; do + [[ -n "$pvc_name" ]] || continue + log " - pvc=${pvc_name} phase=${pvc_phase:-unknown} pvc.storageClass=${pvc_sc:-} pv=${pvc_pv:-} pv.storageClass=${pv_sc:-} diskHandle=${disk_handle:-} diskRef=${disk_scope:-}:${disk_location:-}/${disk_name:-} diskType=${disk_type:-}" + done <<< "$pvc_mismatches" + fi + + if [[ -z "$template_mismatches" && -z "$pvc_mismatches" ]]; then + return 0 + fi + + if [[ -n "$template_mismatches" ]]; then + repair_blocked "GitLab StatefulSet claim-template storageClass mismatch" \ + "Configured target=[${expected_sc}] but one or more live GitLab StatefulSet volumeClaimTemplates use a different storageClass. For Gitaly this is a hard mismatch and must be repaired with authoritative CR fields + destructive cleanup gate." + fi + + if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then + repair_blocked "GitLab storageClass mismatch detected" \ + "Configured target=[${expected_sc}] but one or more GitLab StatefulSet/PVC/PV resources are on a different storageClass. Enable GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to delete wrong-class GitLab PVC/PV/disk artifacts for reprovision." + fi + + cleanup_gitlab_wrong_storage_class_records "$pvc_mismatches" +} + +is_gke_cluster_detected() { + local provider_ids gke_pool_labels gke_topology_labels + provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true) + gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true) + gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true) + if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then + return 0 + fi + return 1 +} + +gke_parse_disk_ref_from_volume_handle() { + local handle="$1" + [[ -n "$handle" ]] || return 1 + + if [[ "$handle" =~ /zones/([^/]+)/disks/([^/]+)$ ]]; then + printf 'zone|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" + return 0 + fi + if [[ "$handle" =~ /regions/([^/]+)/disks/([^/]+)$ ]]; then + printf 'region|%s|%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" + return 0 + fi + if [[ "$handle" =~ ^[^/]+$ ]]; then + printf 'name||%s' "$handle" + return 0 + fi + return 1 +} + +gke_delete_disk_ref_if_present() { + local disk_scope="$1" + local disk_location="$2" + local disk_name="$3" + [[ -n "$disk_name" ]] || return 0 + + if ! command -v gcloud >/dev/null 2>&1; then + warn "gcloud not found; cannot auto-delete stale disk '${disk_name}'." + return 1 + fi + + local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}" + if [[ -z "$project_id" ]]; then + project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true) + fi + + local -a common_args=(--quiet) + if [[ -n "$project_id" ]]; then + common_args+=(--project "$project_id") + fi + + if [[ "$disk_scope" == "zone" && -n "$disk_location" ]]; then + if gcloud compute disks describe "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then + log "Deleting stale GKE disk ${disk_name} (zone=${disk_location})..." + gcloud compute disks delete "$disk_name" --zone "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${disk_location})." + else + log "Stale disk ${disk_name} already absent in zone ${disk_location}." + fi + return 0 + fi + + if [[ "$disk_scope" == "region" && -n "$disk_location" ]]; then + if gcloud compute disks describe "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1; then + log "Deleting stale GKE regional disk ${disk_name} (region=${disk_location})..." + gcloud compute disks delete "$disk_name" --region "$disk_location" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${disk_location})." + else + log "Stale regional disk ${disk_name} already absent in region ${disk_location}." + fi + return 0 + fi + + local matched + matched=$(gcloud compute disks list "${common_args[@]}" --filter="name=('${disk_name}')" --format='csv[no-heading,separator="|"](zone.basename(),region.basename())' 2>/dev/null || true) + if [[ -z "$matched" ]]; then + log "Stale disk ${disk_name} already absent." + return 0 + fi + + while IFS='|' read -r zone_name region_name; do + [[ -n "$zone_name" || -n "$region_name" ]] || continue + if [[ -n "$zone_name" ]]; then + log "Deleting stale GKE disk ${disk_name} (zone=${zone_name})..." + gcloud compute disks delete "$disk_name" --zone "$zone_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete disk ${disk_name} (zone=${zone_name})." + elif [[ -n "$region_name" ]]; then + log "Deleting stale GKE regional disk ${disk_name} (region=${region_name})..." + gcloud compute disks delete "$disk_name" --region "$region_name" "${common_args[@]}" >/dev/null 2>&1 || warn "Failed to delete regional disk ${disk_name} (region=${region_name})." + fi + done <<< "$matched" +} + +collect_stale_gke_gitaly_pv_records() { + local desired_sc="$1" + [[ "$desired_sc" == "standard" ]] || return 0 + + local pvc_name="repo-data-gitlab-gitaly-0" + local pvc_phase + pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + + local pv_rows + pv_rows=$(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"|"}{.spec.storageClassName}{"|"}{.spec.claimRef.namespace}{"|"}{.spec.claimRef.name}{"|"}{.spec.csi.volumeHandle}{"|"}{.spec.gcePersistentDisk.pdName}{"\n"}{end}' 2>/dev/null || true) + [[ -n "$pv_rows" ]] || return 0 + + while IFS='|' read -r pv_name pv_phase pv_sc pv_claim_ns pv_claim_name pv_handle pv_gce_pd; do + [[ -n "$pv_name" ]] || continue + [[ "$pv_claim_ns" == "$NAMESPACE" && "$pv_claim_name" == "$pvc_name" ]] || continue + is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$pv_sc" || continue + + local stale_reason="" + if [[ "$pv_phase" == "Released" || "$pv_phase" == "Failed" ]]; then + stale_reason="pv-phase-${pv_phase}" + elif [[ -z "$pvc_phase" ]]; then + stale_reason="missing-live-pvc" + fi + [[ -n "$stale_reason" ]] || continue + + printf '%s|%s|%s|%s|%s|%s\n' "$pv_name" "$pv_phase" "$pv_sc" "$pv_handle" "$pv_gce_pd" "$stale_reason" + done <<< "$pv_rows" +} + +collect_stale_gke_gitaly_disk_records_without_pv() { + local desired_sc="$1" + [[ "$desired_sc" == "standard" ]] || return 0 + command -v gcloud >/dev/null 2>&1 || return 0 + + local pvc_name="repo-data-gitlab-gitaly-0" + local pvc_phase + pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + [[ -z "$pvc_phase" ]] || return 0 + + local project_id="${GCP_PROJECT_ID:-${GOOGLE_CLOUD_PROJECT:-}}" + if [[ -z "$project_id" ]]; then + project_id=$(gcloud config get-value project 2>/dev/null | tr -d '[:space:]' || true) + fi + local -a common_args=(--quiet) + if [[ -n "$project_id" ]]; then + common_args+=(--project "$project_id") + fi + + local rows + rows=$(gcloud compute disks list "${common_args[@]}" \ + --filter="labels.kubernetes-io-created-for-pvc-name=${pvc_name} AND labels.kubernetes-io-created-for-pvc-namespace=${NAMESPACE}" \ + --format='csv[no-heading,separator="|"](name,zone.basename(),region.basename(),type.basename())' 2>/dev/null || true) + [[ -n "$rows" ]] || return 0 + + while IFS='|' read -r disk_name disk_zone disk_region disk_type; do + [[ -n "$disk_name" ]] || continue + is_gke_blocking_disk_type_for_standard_target "$disk_type" || continue + if [[ -n "$disk_zone" ]]; then + printf 'zone|%s|%s|%s|%s\n' "$disk_zone" "$disk_name" "$disk_type" "label-scan" + elif [[ -n "$disk_region" ]]; then + printf 'region|%s|%s|%s|%s\n' "$disk_region" "$disk_name" "$disk_type" "label-scan" + else + printf 'name||%s|%s|%s\n' "$disk_name" "$disk_type" "label-scan" + fi + done <<< "$rows" +} + +repair_stale_gke_gitaly_dynamic_storage() { + local desired_sc="$1" + [[ "$MODE" == "k8s" ]] || return 0 + [[ "$desired_sc" == "standard" ]] || return 0 + is_gke_cluster_detected || return 0 + + local pvc_name="repo-data-gitlab-gitaly-0" + local stale_pv_records + stale_pv_records=$(collect_stale_gke_gitaly_pv_records "$desired_sc") + local stale_disk_records + stale_disk_records=$(collect_stale_gke_gitaly_disk_records_without_pv "$desired_sc") + + if [[ -z "$stale_pv_records" && -z "$stale_disk_records" ]]; then + return 0 + fi + + local summary="" + if [[ -n "$stale_pv_records" ]]; then + while IFS='|' read -r pv_name pv_phase pv_sc _pv_handle _pv_gce_pd stale_reason; do + [[ -n "$pv_name" ]] || continue + summary+="PV ${pv_name} (phase=${pv_phase:-unknown}, sc=${pv_sc:-unknown}, reason=${stale_reason}). " + done <<< "$stale_pv_records" + fi + if [[ -n "$stale_disk_records" ]]; then + while IFS='|' read -r disk_scope disk_location disk_name disk_type disk_source; do + [[ -n "$disk_name" ]] || continue + summary+="Disk ${disk_name} (${disk_scope}:${disk_location:-n/a}, type=${disk_type:-unknown}, source=${disk_source}). " + done <<< "$stale_disk_records" + fi + + if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then + repair_blocked "Detected stale GKE Gitaly dynamic storage artifacts blocking class '${desired_sc}'" \ + "Artifacts are scoped to PVC ${pvc_name} only: ${summary}Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to auto-clean stale PV/PD leftovers from old standard-rwo/pd-balanced attempts." + fi + + export GITALY_AUTOCLEAN_PERFORMED=1 + log "AUTOCLEAN: repairing stale GKE Gitaly dynamic storage artifacts for PVC ${pvc_name}..." + kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true + + local live_pvc_sc live_pvc_phase + live_pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + live_pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ -n "$live_pvc_phase" ]] && is_gke_stale_gitaly_storage_class_for_standard_target "$desired_sc" "$live_pvc_sc" && [[ "$live_pvc_phase" != "Bound" ]]; then + log "Deleting stale live PVC ${pvc_name} (phase=${live_pvc_phase}, sc=${live_pvc_sc}) before retry..." + kubectl -n "$NAMESPACE" delete pvc "$pvc_name" --wait=false 2>/dev/null || true + fi + + local processed_disks="|" + local _disk_key + local _disk_ref _disk_scope _disk_location _disk_name + if [[ -n "$stale_pv_records" ]]; then + while IFS='|' read -r pv_name _pv_phase _pv_sc pv_handle pv_gce_pd _stale_reason; do + [[ -n "$pv_name" ]] || continue + log "Deleting stale Gitaly PV ${pv_name}..." + kubectl delete pv "$pv_name" --wait=false 2>/dev/null || true + + _disk_ref="" + if [[ -n "$pv_handle" ]]; then + _disk_ref=$(gke_parse_disk_ref_from_volume_handle "$pv_handle" || true) + fi + if [[ -z "$_disk_ref" && -n "$pv_gce_pd" ]]; then + _disk_ref="name||${pv_gce_pd}" + fi + if [[ -n "$_disk_ref" ]]; then + IFS='|' read -r _disk_scope _disk_location _disk_name <<< "$_disk_ref" + _disk_key="${_disk_scope}|${_disk_location}|${_disk_name}" + if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then + processed_disks+="${_disk_key}|" + gke_delete_disk_ref_if_present "$_disk_scope" "$_disk_location" "$_disk_name" || true + fi + fi + done <<< "$stale_pv_records" + fi + + if [[ -n "$stale_disk_records" ]]; then + while IFS='|' read -r disk_scope disk_location disk_name _disk_type _disk_source; do + [[ -n "$disk_name" ]] || continue + _disk_key="${disk_scope}|${disk_location}|${disk_name}" + if [[ "$processed_disks" != *"|${_disk_key}|"* ]]; then + processed_disks+="${_disk_key}|" + gke_delete_disk_ref_if_present "$disk_scope" "$disk_location" "$disk_name" || true + fi + done <<< "$stale_disk_records" + fi + + sleep 2 +} + +check_gitlab_post_apply_blocked() { + # --- Migrations Check --- + check_gitlab_migrations_blocked + + # --- Gitaly Check --- + if [[ "$MODE" == "k8s" ]]; then + local desired_cr="${GITLAB_CR_RENDERED:-}" + if [[ -n "$desired_cr" ]]; then + local rendered_global_sc="" + local rendered_gitlab_gitaly_sc="" + local rendered_chart_gitaly_sc="" + IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")" + + local desired_gitaly_node_selector + desired_gitaly_node_selector=$(echo "$desired_cr" | sed -n '/gitaly:/,/toolbox:/p' | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true) + local desired_gitaly_storage_class="$rendered_gitlab_gitaly_sc" + + log "Desired Gitaly state (from CR): nodeSelector=[${desired_gitaly_node_selector}], rendered.global.persistence.storageClass=[${rendered_global_sc:-}], rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-}], rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-}]" + log "Internal script GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]" + + if [[ "$desired_cr" == *"gandalf.prole.org"* ]]; then + repair_blocked "GitLab CR contains legacy nodeSelector" \ + "Desired GitLab CR still contains gandalf.prole.org. This is a configuration bug." + fi + if [[ "$desired_cr" == *"gitlab-gitaly-static"* ]]; then + repair_blocked "GitLab CR contains legacy storageClass" \ + "Desired GitLab CR still contains gitlab-gitaly-static. This is a configuration bug." + fi + if [[ -n "$desired_gitaly_storage_class" ]] && ! gitlab_storage_class_matches_expected "$desired_gitaly_storage_class" "${GITALY_STORAGE_CLASS:-}"; then + repair_blocked "GitLab CR storageClass mismatch" \ + "Desired GitLab CR storageClass=[${desired_gitaly_storage_class}] does not match GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]. Configuration bug." + fi + if [[ -n "$rendered_chart_gitaly_sc" ]] && ! gitlab_storage_class_matches_expected "$rendered_chart_gitaly_sc" "${GITALY_STORAGE_CLASS:-}"; then + repair_blocked "GitLab CR chart-level Gitaly storageClass mismatch" \ + "Rendered gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc}] does not match GITALY_STORAGE_CLASS=[${GITALY_STORAGE_CLASS:-}]. Configuration bug." + fi + + enforce_gitlab_storage_class_target "${GITALY_STORAGE_CLASS:-}" "$desired_cr" + fi + + local gitaly_sts_name="${GITLAB_RELEASE}-gitaly" + + # If autoclean was performed, wait for convergence before checking live state. + # This prevents false positives when the operator hasn't yet updated the stale StatefulSet. + if [[ "${GITALY_AUTOCLEAN_PERFORMED:-0}" == "1" ]]; then + log "AUTOCLEAN was performed. Waiting for Gitaly StatefulSet to converge (removing legacy storage)..." + local grace_start=$(date +%s) + local grace_timeout=60 + local converged=0 + while true; do + local live_sts_yaml + live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true) + if [[ -z "$live_sts_yaml" ]]; then + log "Gitaly StatefulSet not found (awaiting operator action)..." + elif [[ "$live_sts_yaml" != *"gandalf.prole.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then + log "Gitaly StatefulSet converged to corrected state (clean nodeSelector/storageClass)." + local _live_sc + _live_sc=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true) + if [[ -n "$_live_sc" ]] && ! gitlab_storage_class_matches_expected "$_live_sc" "${GITALY_STORAGE_CLASS:-}"; then + log "WARNING: Converged StatefulSet uses storageClass=[${_live_sc}], expected [${GITALY_STORAGE_CLASS:-}]." + fi + converged=1 + break + else + local live_gitaly_node_selector + live_gitaly_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true) + local live_gitaly_storage_class + live_gitaly_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true) + log "Still waiting for Gitaly convergence (live nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}])..." + fi + + if (( $(date +%s) - grace_start > grace_timeout )); then + log "Grace period (${grace_timeout}s) expired." + break + fi + sleep 15 + done + + if [[ "$converged" == "0" ]]; then + local live_sts_yaml + live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true) + if [[ -n "$live_sts_yaml" && ( "$live_sts_yaml" == *"gandalf.prole.org"* || "$live_sts_yaml" == *"gitlab-gitaly-static"* ) ]]; then + log "Live StatefulSet still legacy after grace period. Performing explicit replacement..." + # We already confirmed desired CR is corrected at the start of this function. + kubectl -n "$NAMESPACE" delete statefulset "$gitaly_sts_name" --wait=true 2>/dev/null || true + log "Legacy StatefulSet deleted. Waiting for operator recreation..." + + local recreate_start=$(date +%s) + local recreate_timeout=300 + while true; do + live_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true) + if [[ -n "$live_sts_yaml" ]]; then + local recreated_node_selector + recreated_node_selector=$(echo "$live_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true) + local recreated_storage_class + recreated_storage_class=$(echo "$live_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true) + log "StatefulSet recreated. nodeSelector=[${recreated_node_selector}], storageClass=[${recreated_storage_class}]" + + if [[ "$live_sts_yaml" != *"gandalf.knoe.org"* && "$live_sts_yaml" != *"gitlab-gitaly-static"* ]]; then + if [[ -n "$recreated_storage_class" ]] && ! gitlab_storage_class_matches_expected "$recreated_storage_class" "${GITALY_STORAGE_CLASS:-}"; then + repair_blocked "Recreated Gitaly StatefulSet uses wrong storageClass" \ + "Resource: statefulset/${gitaly_sts_name}. Value: storageClassName=[${recreated_storage_class}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check operator reconciliation." + fi + log "Gitaly StatefulSet recreated in corrected state." + converged=1 + break + else + log "Recreated StatefulSet STILL contains legacy fields. Waiting for operator to correct it..." + fi + fi + + if (( $(date +%s) - recreate_start > recreate_timeout )); then + repair_blocked "Gitaly failed to recreate clean StatefulSet after ${recreate_timeout}s" \ + "Resource: statefulset/${gitaly_sts_name}. Fix: Check operator logs and desired GitLab CR." + fi + sleep 15 + done + else + log "No legacy StatefulSet found after grace period (may have been deleted or converged)." + converged=1 + fi + fi + + if [[ "$converged" == "1" ]]; then + # Wait for PVC provisioning if autoclean was performed + log "Waiting for PVC repo-data-gitlab-gitaly-0 to be provisioned and Bound..." + local pvc_name="repo-data-gitlab-gitaly-0" + local pvc_start=$(date +%s) + local pvc_timeout=600 + local pvc_uid="" + while true; do + local pvc_sc=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + local pvc_phase=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + + if [[ -n "$pvc_phase" ]]; then + pvc_uid=$(kubectl -n "$NAMESPACE" get pvc "$pvc_name" -o jsonpath='{.metadata.uid}' 2>/dev/null || true) + local _sc_info="" + if [[ "$pvc_sc" == "standard-rwo" ]]; then _sc_info=" (pd-balanced / wrong-class for strict standard target)"; fi + if [[ "$pvc_sc" == "premium-rwo" ]]; then _sc_info=" (pd-ssd)"; fi + if [[ "$pvc_sc" == "standard" ]]; then _sc_info=" (pd-standard)"; fi + + log "PVC ${pvc_name}: storageClass=[${pvc_sc}${_sc_info}], phase=[${pvc_phase}]" + if [[ -n "$pvc_sc" ]] && ! gitlab_storage_class_matches_expected "$pvc_sc" "${GITALY_STORAGE_CLASS:-}"; then + repair_blocked "Gitaly PVC uses wrong storageClass" \ + "PVC: ${pvc_name}. Value: storageClass=[${pvc_sc}]. Expected: [${GITALY_STORAGE_CLASS:-}]. Fix: Check StatefulSet volumeClaimTemplates and operator reconciliation." + fi + + if [[ "$pvc_phase" == "Bound" ]]; then + if [[ "$pvc_sc" == "${GITALY_STORAGE_CLASS:-}" ]]; then + log "PVC ${pvc_name} successfully provisioned on '${pvc_sc}' storage." + break + fi + fi + + # Check provisioning failures only for the *current* PVC object. + local provisioning_fail="" + if [[ -n "$pvc_uid" ]]; then + provisioning_fail=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim -o jsonpath='{range .items[?(@.reason=="FailedBinding" || @.reason=="ProvisioningFailed")]}{.message}{"\n"}{end}' 2>/dev/null | tail -n 1 || true) + fi + if [[ -n "$provisioning_fail" ]]; then + if [[ "$provisioning_fail" == *"quota"* || "$provisioning_fail" == *"QUOTA"* ]]; then + repair_blocked "Gitaly PVC provisioning failed (Quota Exceeded)" \ + "PVC: ${pvc_name}. Error: ${provisioning_fail}. Fix: Check GKE storage quotas and ensure Gitaly uses storageClass='${GITALY_STORAGE_CLASS:-standard}'." + else + log "PVC ${pvc_name} provisioning event: ${provisioning_fail}" + fi + fi + else + # Check migrations first to fail fast + check_gitlab_migrations_blocked + log "PVC ${pvc_name} not found yet (awaiting operator/provisioner action)..." + fi + + if (( $(date +%s) - pvc_start > pvc_timeout )); then + local last_msg="" + if [[ -n "$pvc_uid" ]]; then + last_msg=$(kubectl -n "$NAMESPACE" get events --field-selector involvedObject.uid="$pvc_uid",involvedObject.kind=PersistentVolumeClaim --sort-by='.lastTimestamp' -o jsonpath='{.items[-1:].message}' 2>/dev/null || true) + else + last_msg="No live PVC UID observed yet; ignoring stale historical PVC events from previous claims." + fi + repair_blocked "Gitaly PVC failed to bind after ${pvc_timeout}s" \ + "PVC: ${pvc_name}. Status: ${pvc_phase:-NotFound}. Last event: ${last_msg}. Fix: Check storage provider and quota." + fi + sleep 15 + done + + export GITALY_AUTOCLEAN_PERFORMED=0 + fi + + # Also wait for PV deletion to settle if it exists + if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then + log "Waiting for legacy PV gitlab-gitaly-synology to be removed..." + local pv_wait_start=$(date +%s) + while kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; do + if (( $(date +%s) - pv_wait_start > 120 )); then + log "Legacy PV still exists after 120s; continuing (it may be stuck in Terminating)." + break + fi + sleep 5 + done + fi + fi + + # Final live state checks + local gitaly_sts_yaml + gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true) + if [[ -n "$gitaly_sts_yaml" ]]; then + local rendered_global_sc="" + local rendered_gitlab_gitaly_sc="" + local rendered_chart_gitaly_sc="" + IFS='|' read -r rendered_global_sc rendered_gitlab_gitaly_sc rendered_chart_gitaly_sc <<< "$(gitlab_rendered_storage_fields_from_cr "$desired_cr")" + + local live_gitaly_node_selector + live_gitaly_node_selector=$(echo "$gitaly_sts_yaml" | grep "nodeSelector:" -A 1 | tail -n 1 | xargs || true) + local live_gitaly_storage_class + live_gitaly_storage_class=$(echo "$gitaly_sts_yaml" | grep "storageClassName:" | cut -d: -f2 | xargs || true) + local live_gitaly_repo_data_template_sc + live_gitaly_repo_data_template_sc=$(gitlab_live_gitaly_repo_data_template_storage_class) + + log "Live Gitaly state: nodeSelector=[${live_gitaly_node_selector}], storageClass=[${live_gitaly_storage_class}]" + log "Live Gitaly StatefulSet repo-data claim template storageClassName=[${live_gitaly_repo_data_template_sc:-}]" + + if ! gitlab_storage_class_matches_expected "$live_gitaly_repo_data_template_sc" "${GITALY_STORAGE_CLASS:-}"; then + if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" != "1" ]]; then + repair_blocked "GitLab Gitaly StatefulSet repo-data claim-template storageClass mismatch" \ + "Configured target=[${GITALY_STORAGE_CLASS:-}] rendered.global.persistence.storageClass=[${rendered_global_sc:-}] rendered.gitlab.gitaly.persistence.storageClass=[${rendered_gitlab_gitaly_sc:-}] rendered.gitaly.persistence.storageClass=[${rendered_chart_gitaly_sc:-}] live.statefulset=${gitaly_sts_name} live.volumeClaimTemplate.repo-data.storageClassName=[${live_gitaly_repo_data_template_sc:-}]. Delete statefulset/${gitaly_sts_name}, pvc/repo-data-${GITLAB_RELEASE}-gitaly-0, and wrong-class PV+disk artifacts or set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 for automatic destructive repair." + fi + + local gitaly_pvc_mismatches + gitaly_pvc_mismatches=$(collect_gitlab_storage_mismatch_records "${GITALY_STORAGE_CLASS:-}") + cleanup_gitlab_wrong_gitaly_template_storage "${GITALY_STORAGE_CLASS:-}" "$live_gitaly_repo_data_template_sc" "$gitaly_pvc_mismatches" "$desired_cr" + fi + + if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* ]]; then + repair_blocked "Gitaly using legacy Synology storage" \ + "Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Ensure GITLAB_STORAGE_NODE is not set in k8s mode." + fi + if [[ "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then + repair_blocked "Gitaly using legacy Synology storage" \ + "Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden in k8s mode." + fi + fi + + if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then + repair_blocked "Gitaly using legacy Synology storage" \ + "Resource: pv/gitlab-gitaly-synology. Value: exists. Fix: Delete legacy PV or use GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1." + fi + + local unschedulable_gitaly + unschedulable_gitaly=$(kubectl -n "$NAMESPACE" get pods -l "app=gitaly" -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="PodScheduled")].status=="False")]}{.metadata.name}:{.status.conditions[?(@.type=="PodScheduled")].reason}{"\n"}{end}' 2>/dev/null | grep "Unschedulable" || true) + if [[ -n "$unschedulable_gitaly" ]]; then + repair_blocked "Gitaly pod(s) unschedulable" \ + "Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storageClass='${GITALY_STORAGE_CLASS:-standard}' and no hostname nodeSelector. Ensure wrong-class PV/PVC artifacts were repaired." + fi + + # --- Sidekiq Config Check --- + log "Desired Sidekiq state: concurrency=${GITLAB_SIDEKIQ_CONCURRENCY}, requests=[cpu=${GITLAB_SIDEKIQ_REQUESTS_CPU}, mem=${GITLAB_SIDEKIQ_REQUESTS_MEMORY}], limits=[cpu=${GITLAB_SIDEKIQ_LIMITS_CPU}, mem=${GITLAB_SIDEKIQ_LIMITS_MEMORY}]" + if [[ "${gitlab_apply_changed:-0}" == "1" || "${gitlab_spec_changed:-0}" == "1" ]]; then + log "GitLab CR was updated to fix configuration drift (Sidekiq or other fields)." + fi + + local sidekiq_deploy_name="${GITLAB_RELEASE}-sidekiq-all-in-1-v2" + local live_sidekiq_cpu_req + live_sidekiq_cpu_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}' 2>/dev/null || true) + local live_sidekiq_mem_req + live_sidekiq_mem_req=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.requests.memory}' 2>/dev/null || true) + local live_sidekiq_cpu_lim + live_sidekiq_cpu_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.cpu}' 2>/dev/null || true) + local live_sidekiq_mem_lim + live_sidekiq_mem_lim=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].resources.limits.memory}' 2>/dev/null || true) + local live_sidekiq_concurrency + live_sidekiq_concurrency=$(kubectl -n "$NAMESPACE" get deploy "$sidekiq_deploy_name" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="SIDEKIQ_CONCURRENCY")].value}' 2>/dev/null || true) + + if [[ -n "$live_sidekiq_cpu_req" ]]; then + log "Live Sidekiq state: concurrency=${live_sidekiq_concurrency}, requests=[cpu=${live_sidekiq_cpu_req}, mem=${live_sidekiq_mem_req}], limits=[cpu=${live_sidekiq_cpu_lim}, mem=${live_sidekiq_mem_lim}]" + fi + fi + + # --- Registry Config Check --- + local registry_secret_config + registry_secret_config=$(kubectl -n "$NAMESPACE" get secret "${GITLAB_RELEASE}-registry-storage" -o jsonpath='{.data.config}' 2>/dev/null | base64 -d 2>/dev/null || true) + if [[ "$registry_secret_config" == *""* ]]; then + if [[ "$GARAGE_S3_ENDPOINT" != *""* ]]; then + log "Registry secret still contains placeholder but config is updated. Re-applying secret..." + # Re-triggering the secret creation (this is safer than just blocking) + setup_garage_for_gitlab + else + repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder in live secret)" \ + "Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint." + fi + fi + + # --- Registry Logs Check --- + local registry_pod + registry_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=registry" -o name 2>/dev/null | head -n1 || true) + if [[ -n "$registry_pod" ]]; then + local logs + logs=$(kubectl -n "$NAMESPACE" logs "$registry_pod" --tail=100 2>&1 || true) + if [[ "$logs" == *"DNS failure"* || "$logs" == *"AccessDenied"* || "$logs" == *"No such key"* ]]; then + # If it's a "No such key" or "AccessDenied", and we have a custom endpoint, it might be the WRONG cluster Garage. + repair_blocked "Registry endpoint invalid or wrong Garage (S3 error detected)" \ + "Logs: ${logs}. Fix: Verify GARAGE_S3_ENDPOINT points to the DB cluster Garage (not the APP cluster one)." + fi + fi + + # --- KAS Logs Check --- + local kas_pod + kas_pod=$(kubectl -n "$NAMESPACE" get pods -l "app=kas" -o name 2>/dev/null | head -n1 || true) + if [[ -n "$kas_pod" ]]; then + local logs + logs=$(kubectl -n "$NAMESPACE" logs "$kas_pod" --tail=100 2>&1 || true) + if [[ "$logs" == *"no such host"* && "$logs" == *"redis"* ]]; then + repair_blocked "Redis host does not resolve (detected in KAS logs)" \ + "Logs: ${logs}. Fix: Verify REDIS_HOST (${REDIS_HOST}) and ensure Redis service is healthy." + fi + fi +} + +kubectl_db() { + local db_ctx="" + db_ctx="$(resolve_db_cluster_context || true)" + if [[ -n "$db_ctx" ]]; then + command kubectl --context "$db_ctx" "$@" + else + kubectl "$@" + fi +} + +resolve_garage_admin_context() { + if [[ "$MODE" == "k8s" ]]; then + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + local explicit_ctx="" + explicit_ctx="$(resolve_explicit_kube_context || true)" + + if [[ -n "$db_ctx" && -n "$app_ctx" && "$db_ctx" != "$app_ctx" ]]; then + printf '%s' "$db_ctx" + return 0 + fi + + if [[ -n "$db_ctx" ]]; then + printf '%s' "$db_ctx" + return 0 + fi + if [[ -n "$explicit_ctx" ]]; then + printf '%s' "$explicit_ctx" + return 0 + fi + if [[ -n "$app_ctx" ]]; then + printf '%s' "$app_ctx" + return 0 + fi + + die "Explicit Garage admin context is required in k8s mode." + fi + return 1 +} + +kubectl_garage_admin() { + local garage_ctx="" + garage_ctx="$(resolve_garage_admin_context || true)" + if [[ -n "$garage_ctx" ]]; then + command kubectl --context "$garage_ctx" "$@" + else + kubectl "$@" + fi +} + +db_kubectl() { + local db_ctx + db_ctx="$(resolve_db_cluster_context || true)" + if [[ -n "$db_ctx" ]]; then + command kubectl --context "$db_ctx" "$@" + else + kubectl "$@" + fi +} + +ensure_cross_cluster_db_host() { + if [[ "$MODE" != "k8s" ]]; then + return 0 + fi + + local app_ctx db_ctx + app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}" + if [[ -z "$app_ctx" ]]; then + app_ctx="$(kubectl config current-context 2>/dev/null || true)" + fi + db_ctx="$(resolve_db_cluster_context || true)" + + if [[ -z "$db_ctx" || -z "$app_ctx" || "$db_ctx" == "$app_ctx" ]]; then + return 0 + fi + + local db_cluster_name="${CNPG_CLUSTER_NAME:-knoe-db}" + local db_ns="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}" + local ilb_service="${GITLAB_DB_ILB_SERVICE:-knoe-db-rw-ilb}" + + log "Split-cluster mode detected (APP: ${app_ctx}, DB: ${db_ctx})" + log "Ensuring cross-cluster Postgres ILB service '${ilb_service}' in namespace '${db_ns}'..." + + db_kubectl -n "$db_ns" apply -f - </dev/null +apiVersion: v1 +kind: Service +metadata: + name: ${ilb_service} + annotations: + networking.gke.io/load-balancer-type: "Internal" +spec: + type: LoadBalancer + selector: + cnpg.io/cluster: ${db_cluster_name} + cnpg.io/instanceRole: primary + ports: + - name: postgres + port: 5432 + targetPort: 5432 + protocol: TCP +EOF + + local db_host="" + local attempts=0 + log "Waiting for cross-cluster DB host (ILB IP/hostname)..." + while (( attempts < 60 )); do + attempts=$((attempts + 1)) + db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)" + if [[ -z "$db_host" ]]; then + db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)" + fi + if [[ -n "$db_host" ]]; then + break + fi + sleep 5 + done + + if [[ -z "$db_host" ]]; then + warn "Cross-cluster Postgres ILB '${ilb_service}' has no ingress address yet after 5 minutes." + return 0 + fi + + log "Successfully resolved cross-cluster DB host: ${db_host}" + GITLAB_CROSS_CLUSTER_DB_HOST="$db_host" + export GITLAB_CROSS_CLUSTER_DB_HOST +} + +enforce_app_cluster_targeting + +is_truthy() { + case "${1:-}" in + 1|true|TRUE|True|yes|YES|on|ON|y|Y) + return 0 + ;; + *) + return 1 + ;; + esac +} + +assert_public_ingress_targeting() { + local ingress_class="${1:-}" + shift || true + local hosts=("$@") + + if [[ "$MODE" != "k8s" ]]; then + return 0 + fi + + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + local active_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}" + + if [[ -z "$app_ctx" || -z "$active_ctx" ]]; then + die "Explicit APP cluster context is required for GitLab ingress operations in k8s mode." + fi + + local host_count=0 + local h + for h in "${hosts[@]}"; do + [[ -n "${h:-}" ]] && host_count=$((host_count + 1)) + done + + if [[ "$host_count" -gt 0 && -n "$db_ctx" && "$active_ctx" == "$db_ctx" ]]; then + die "Refusing to render/apply GitLab public ingress in DB cluster context '${active_ctx}' (hosts: ${hosts[*]})." + fi + + if [[ "$host_count" -gt 0 && -n "$app_ctx" && -n "$active_ctx" && "$active_ctx" != "$app_ctx" ]]; then + die "GitLab public ingress must target APP cluster context '${app_ctx}', active context is '${active_ctx}'." + fi + + local normalized_class="${ingress_class,,}" + if [[ "$normalized_class" == traefik* ]] && ! is_truthy "${ALLOW_TRAEFIK_PUBLIC_INGRESS:-${GITLAB_ALLOW_TRAEFIK_INGRESS:-0}}"; then + die "Ingress class '${ingress_class}' is incompatible with k8s mode unless Traefik public ingress is explicitly enabled." + fi +} + +assert_unique_ingress_host_claims() { + local ingress_name="${1:-}" + local ingress_namespace="${2:-}" + local host_csv="${3:-}" + local gitlab_release="${4:-}" + local expected_backend_service="${5:-}" + [[ -n "$host_csv" ]] || return 0 + + local target_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}" + if [[ "$MODE" == "k8s" && -z "$target_ctx" ]]; then + die "Explicit kubectl context is required for ingress ownership checks in k8s mode." + fi + + INGRESS_HOST_CLAIM_DETAILS="" + + local claim_result claim_status + claim_result=$(python3 - "$host_csv" "$ingress_namespace" "$ingress_name" "$target_ctx" "$gitlab_release" "$expected_backend_service" <<'PY' +import json +import subprocess +import sys + +requested_hosts = {h.strip().lower() for h in (sys.argv[1] or "").split(",") if h.strip()} +target_ns = sys.argv[2] +target_name = sys.argv[3] +target_ctx = (sys.argv[4] or "").strip() +target_release = (sys.argv[5] or "").strip() +expected_backend_service = (sys.argv[6] or "").strip() + +cmd = ["kubectl"] +if target_ctx: + cmd.extend(["--context", target_ctx]) +cmd.extend(["get", "ingress", "-A", "-o", "json"]) + +try: + raw = subprocess.check_output(cmd, text=True) +except Exception: + print("OK|") + raise SystemExit(0) + +if not raw.strip(): + print("OK|") + raise SystemExit(0) + +try: + payload = json.loads(raw) +except Exception: + print("OK|") + raise SystemExit(0) + +same_owner_conflicts: list[str] = [] +foreign_conflicts: list[str] = [] +for item in payload.get("items", []) or []: + md = item.get("metadata", {}) or {} + ns = (md.get("namespace") or "").strip() + name = (md.get("name") or "").strip() + if ns == target_ns and name == target_name: + continue + + labels = md.get("labels", {}) or {} + spec = item.get("spec", {}) or {} + rules = spec.get("rules", []) or [] + for rule in rules: + host = (rule.get("host") or "").strip().lower() + if not host or host not in requested_hosts: + continue + http = rule.get("http", {}) or {} + paths = http.get("paths", []) or [{"path": "/"}] + for path_item in paths: + path = (path_item.get("path") or "/").strip() or "/" + if path not in {"/", ""}: + continue + + backend = path_item.get("backend", {}) or {} + backend_svc = backend.get("service", {}) or {} + backend_name = (backend_svc.get("name") or "").strip() + + same_owner = False + if ns == target_ns: + if expected_backend_service and backend_name == expected_backend_service: + same_owner = True + if target_release and backend_name.startswith(f"{target_release}-webservice"): + same_owner = True + + instance_label = (labels.get("app.kubernetes.io/instance") or "").strip() + part_of_label = (labels.get("app.kubernetes.io/part-of") or "").strip().lower() + if target_release and ( + name == f"{target_release}-webservice-default" + or name.startswith(f"{target_release}-") + or instance_label == target_release + or part_of_label == "gitlab" + ): + same_owner = True + + owner_desc = f"{host}{path} already owned by {ns}/{name}" + if backend_name: + owner_desc += f" (backend={backend_name})" + + if same_owner: + same_owner_conflicts.append(owner_desc) + else: + foreign_conflicts.append(owner_desc) + +if foreign_conflicts: + print("CONFLICT|" + "; ".join(sorted(set(foreign_conflicts)))) +elif same_owner_conflicts: + print("OWNED_BY_GITLAB|" + "; ".join(sorted(set(same_owner_conflicts)))) +else: + print("OK|") +PY +) + claim_status="${claim_result%%|*}" + INGRESS_HOST_CLAIM_DETAILS="${claim_result#*|}" + + case "$claim_status" in + OK|"") + return 0 + ;; + OWNED_BY_GITLAB) + return 10 + ;; + CONFLICT) + return 11 + ;; + *) + INGRESS_HOST_CLAIM_DETAILS="$claim_result" + return 11 + ;; + esac +} while [[ $# -gt 0 ]]; do case "$1" in - --mode) MODE="$(knoe_normalize_mode "${2:-}")"; shift 2 ;; - --mode=*) MODE="$(knoe_normalize_mode "${1#*=}")"; shift 1 ;; - -n|--namespace) NAMESPACE="${2:-}"; shift 2 ;; - --namespace=*) NAMESPACE="${1#*=}"; shift 1 ;; - -c|--config) CFG_PATH="${2:-}"; shift 2 ;; - --config=*) CFG_PATH="${1#*=}"; shift 1 ;; - --force) FORCE=1; shift 1 ;; - -h|--help) usage; exit 0 ;; + --mode) MODE="$(knoe_normalize_mode "${2:-}")"; shift 2 ;; + --mode=*) MODE="$(knoe_normalize_mode "${1#*=}")"; shift 1 ;; + -n|--namespace) NAMESPACE="${2:-}"; shift 2 ;; + --namespace=*) NAMESPACE="${1#*=}"; shift 1 ;; + -c|--config) CFG_PATH="${2:-}"; shift 2 ;; + --config=*) CFG_PATH="${1#*=}"; shift 1 ;; + --node-selector) NODE_SELECTOR="${2:-}"; shift 2 ;; + --node-selector=*) NODE_SELECTOR="${1#*=}"; shift 1 ;; + --force) FORCE=1; shift 1 ;; + -h|--help) usage; exit 0 ;; + --trace-namespace) + # Resolve namespace then exit — used by diagnostic scripts only + _TRACE_NS=1; shift 1 ;; *) break ;; esac done -# Resolve config path and namespace defaults from knoe.cfg when present +# --------------------------------------------------------------------------- +# Resolve config path +# --------------------------------------------------------------------------- if [[ -z "$CFG_PATH" && -n "${KNOE_CONF:-}" && -f "${KNOE_CONF}/knoe.cfg" ]]; then - CFG_PATH="${KNOE_CONF}/knoe.cfg" -elif [[ -z "$CFG_PATH" && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - CFG_PATH="$SCRIPT_DIR/../conf/knoe.cfg" + CFG_PATH="$(_knoe_cfg_select_cfg_file "${KNOE_CONF}")" +fi +if [[ -z "$CFG_PATH" ]]; then + CFG_PATH="$(_knoe_cfg_select_cfg_file "$SCRIPT_DIR/../conf")" fi -if [[ -z "$NAMESPACE" && -n "$CFG_PATH" ]]; then - maybe_ns="$(_knoe_cfg_extract_key "$CFG_PATH" "GITOPS_NAMESPACE")" - [[ -z "$maybe_ns" ]] && maybe_ns="$(_knoe_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")" - NAMESPACE="$maybe_ns" +# cfg-based namespace override (only GITLAB_NAMESPACE; GITOPS_NAMESPACE intentionally excluded) +if [[ -z "$NAMESPACE" || "$NAMESPACE" == "gitlab" ]] && [[ -n "$CFG_PATH" ]]; then + maybe_ns="$(_knoe_cfg_extract_key "$CFG_PATH" "GITLAB_NAMESPACE")" + [[ -n "$maybe_ns" ]] && NAMESPACE="$maybe_ns" fi +# Final safety net — always default to 'gitlab' NAMESPACE="${NAMESPACE:-gitlab}" export GITLAB_NAMESPACE="$NAMESPACE" +# Diagnostic exit — used by tmp/sim_installer_ns.sh +if [[ "${_TRACE_NS:-0}" == "1" ]]; then + echo "TRACE: NAMESPACE='$NAMESPACE' GITLAB_NAMESPACE='$GITLAB_NAMESPACE'" >&2 + echo "TRACE: env GITLAB_NAMESPACE_ENV='${GITLAB_NAMESPACE:-}' NAMESPACE_ENV_PRE='${PROLE_NAMESPACE:-}'" >&2 + exit 0 +fi + case "$MODE" in k3d|k3s|k8s|local) ;; *) die "Unsupported mode '$MODE' (use k3d, k3s, k8s, or local)" ;; @@ -70,105 +2356,2376 @@ esac export KNOE_MODE="$MODE" command -v kubectl >/dev/null || die "kubectl not found" -command -v helm >/dev/null || die "helm not found (required for GitLab install)" +command -v helm >/dev/null || die "helm not found (required for GitLab Operator install)" -DB_NAMESPACE="${PROLE_NAMESPACE:-}" -if [[ -z "$DB_NAMESPACE" && -n "$CFG_PATH" ]]; then - DB_NAMESPACE="$(_knoe_cfg_extract_key "$CFG_PATH" "NAMESPACE")" -fi -DB_NAMESPACE="${DB_NAMESPACE:-default}" +# --------------------------------------------------------------------------- +# knoe-db (CNPG) settings — reuse the same cluster gitea uses +# --------------------------------------------------------------------------- +DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}" +CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-${CLUSTER_NAME:-knoe-db}}" + +ensure_cross_cluster_db_host -CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-knoe-db}" GITLAB_DB_NAME="${GITLAB_DB_NAME:-gitlabhq_production}" GITLAB_DB_USER="${GITLAB_DB_USER:-gitlab}" GITLAB_DB_PASSWORD="${GITLAB_DB_PASSWORD:-}" -if [[ -z "$GITLAB_DB_PASSWORD" ]]; then - GITLAB_DB_PASSWORD=$(LC_ALL=C tr -dc 'A-Za-z0-9' /dev/null | head -c 32 || true) -fi - -DB_HOST="${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local" +DB_HOST="${GITLAB_DB_HOST:-${GITLAB_CROSS_CLUSTER_DB_HOST:-${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local}}" DB_PORT="${GITLAB_DB_PORT:-5432}" -kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null 2>&1 +# --------------------------------------------------------------------------- +# Redis — shared common service (deployed by init_redis.sh in knoe-system) +# --------------------------------------------------------------------------- +REDIS_NAMESPACE="${REDIS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}" +REDIS_HOST="${GITLAB_REDIS_HOST:-redis-master.${REDIS_NAMESPACE}.svc.cluster.local}" +REDIS_PORT="${GITLAB_REDIS_PORT:-6379}" -ensure_db() { - local primary_pod="" - primary_pod=$(kubectl -n "$DB_NAMESPACE" get pods \ - -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME},role=primary" \ +# GitLab public domain (configurable; default follows deployment mode) +_default_gitlab_domain="git.prole.org" +if [[ "$MODE" == "k8s" ]]; then + _default_gitlab_domain="git.knoe.dev" +fi +GITLAB_DOMAIN="${GITLAB_DOMAIN:-${GITLAB_HOSTNAME:-$_default_gitlab_domain}}" +GITLAB_PUBLIC_HOSTS_RAW="${GITLAB_PUBLIC_HOSTS:-$GITLAB_DOMAIN}" + +trim_csv_token() { + local token="$1" + token="${token#${token%%[![:space:]]*}}" + token="${token%${token##*[![:space:]]}}" + printf '%s' "$token" +} + +_gitlab_hosts_csv="" +GITLAB_PUBLIC_HOSTS=() +IFS=',' read -r -a _gitlab_host_candidates <<< "$GITLAB_PUBLIC_HOSTS_RAW" +for _gitlab_host in "${_gitlab_host_candidates[@]}"; do + _gitlab_host="$(trim_csv_token "$_gitlab_host")" + [[ -n "$_gitlab_host" ]] || continue + case ",${_gitlab_hosts_csv}," in + *,"${_gitlab_host}",*) ;; + *) + GITLAB_PUBLIC_HOSTS+=("$_gitlab_host") + _gitlab_hosts_csv="${_gitlab_hosts_csv:+${_gitlab_hosts_csv},}${_gitlab_host}" + ;; + esac +done +unset _gitlab_host_candidates _gitlab_host + +if [[ ${#GITLAB_PUBLIC_HOSTS[@]} -eq 0 ]]; then + GITLAB_PUBLIC_HOSTS=("$GITLAB_DOMAIN") +fi + +PRIMARY_GITLAB_HOST="${GITLAB_PUBLIC_HOSTS[0]}" +GITLAB_DOMAIN="$PRIMARY_GITLAB_HOST" +PRIMARY_GITLAB_DOMAIN_ROOT="${PRIMARY_GITLAB_HOST#*.}" +if [[ "$PRIMARY_GITLAB_DOMAIN_ROOT" == "$PRIMARY_GITLAB_HOST" || -z "$PRIMARY_GITLAB_DOMAIN_ROOT" ]]; then + PRIMARY_GITLAB_DOMAIN_ROOT="prole.org" +fi + +# Public ingress class (k8s/GKE defaults to gce; local clusters keep kong) +_default_gitlab_ingress_class="kong" +if [[ "$MODE" == "k8s" ]]; then + _default_gitlab_ingress_class="gce" +fi +GITLAB_INGRESS_CLASS="${GITLAB_INGRESS_CLASS:-$_default_gitlab_ingress_class}" + +# --------------------------------------------------------------------------- +# Git SSH hostname + LoadBalancer IP +# --------------------------------------------------------------------------- +# On GKE with Google-managed certs, HTTPS has to live on a GCE global L7 IP +# (ManagedCertificate CRD only binds there). Port 22 requires a regional +# Network LB which can't share an IP with a global L7. So in k8s mode we +# default to a dedicated SSH hostname (git-ssh.) bound to a +# user-reserved regional external static IP via GITLAB_SHELL_LOADBALANCER_IP. +# In k3d/k3s modes the gitlab-shell Service stays ClusterIP (typically +# port-forwarded) and global.hosts.ssh tracks the main gitlab hostname. +_default_gitlab_ssh_host="$GITLAB_DOMAIN" +if [[ "$MODE" == "k8s" ]]; then + _default_gitlab_ssh_host="git-ssh.${PRIMARY_GITLAB_DOMAIN_ROOT}" +fi +GITLAB_SSH_HOST="${GITLAB_SSH_HOST:-$_default_gitlab_ssh_host}" + +# Pre-reserved regional external static IP for the gitlab-shell LoadBalancer +# Service. If blank, no service override is emitted (ClusterIP default). +GITLAB_SHELL_LOADBALANCER_IP="${GITLAB_SHELL_LOADBALANCER_IP:-}" + +# externalTrafficPolicy preserves client source IPs in SSH auth logs — useful +# for abuse triage and rate-limiting. Requires at least one gitlab-shell pod +# per node in the LB backend; our replicaCount=1 is fine. +GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY="${GITLAB_SHELL_EXTERNAL_TRAFFIC_POLICY:-Local}" + +# Single GitLab front-door owner model. +# - operator: GitLab chart/operator-managed ingress owns ${GITLAB_PUBLIC_HOSTS} +# - fallback: custom fallback ingress owns ${GITLAB_PUBLIC_HOSTS} +# In k8s/GKE mode, default to fallback ownership so GitLab frontdoor remains +# anchored on the explicit GCE ingress path. +_default_gitlab_frontdoor_owner="fallback" +GITLAB_FRONTDOOR_OWNER="${GITLAB_FRONTDOOR_OWNER:-$_default_gitlab_frontdoor_owner}" +GITLAB_FRONTDOOR_OWNER="${GITLAB_FRONTDOOR_OWNER,,}" +GITLAB_FALLBACK_INGRESS_NAME="${GITLAB_FALLBACK_INGRESS_NAME:-gitlab-frontdoor-ingress}" +GITLAB_LEGACY_FALLBACK_INGRESS_NAME="gitlab-kong-ingress" +case "$GITLAB_FRONTDOOR_OWNER" in + operator|fallback) + ;; + *) + die "Unsupported GITLAB_FRONTDOOR_OWNER='${GITLAB_FRONTDOOR_OWNER}'. Supported values: operator, fallback." + ;; +esac + +# Trusted proxies used by GitLab Rails/Workhorse to accept forwarded host/proto +# from ingress/load-balancer hops. +_default_gitlab_trusted_proxies="127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10" +if [[ "$MODE" == "k8s" ]]; then + _default_gitlab_trusted_proxies+=" ,130.211.0.0/22,35.191.0.0/16" +fi +_default_gitlab_trusted_proxies="${_default_gitlab_trusted_proxies// /}" +GITLAB_TRUSTED_PROXIES_RAW="${GITLAB_TRUSTED_PROXIES:-${_default_gitlab_trusted_proxies}}" +_gitlab_trusted_proxies_csv="" +GITLAB_TRUSTED_PROXIES=() +IFS=',' read -r -a _gitlab_trusted_proxy_candidates <<< "$GITLAB_TRUSTED_PROXIES_RAW" +for _gitlab_trusted_proxy in "${_gitlab_trusted_proxy_candidates[@]}"; do + _gitlab_trusted_proxy="$(trim_csv_token "$_gitlab_trusted_proxy")" + [[ -n "$_gitlab_trusted_proxy" ]] || continue + case ",${_gitlab_trusted_proxies_csv}," in + *,"${_gitlab_trusted_proxy}",*) ;; + *) + GITLAB_TRUSTED_PROXIES+=("$_gitlab_trusted_proxy") + _gitlab_trusted_proxies_csv="${_gitlab_trusted_proxies_csv:+${_gitlab_trusted_proxies_csv},}${_gitlab_trusted_proxy}" + ;; + esac +done +unset _gitlab_trusted_proxy_candidates _gitlab_trusted_proxy +if [[ ${#GITLAB_TRUSTED_PROXIES[@]} -eq 0 ]]; then + GITLAB_TRUSTED_PROXIES=("127.0.0.1/32") + _gitlab_trusted_proxies_csv="127.0.0.1/32" +fi + +GITLAB_WEBSERVICE_INGRESS_ENABLED="${GITLAB_WEBSERVICE_INGRESS_ENABLED:-true}" +if [[ "$MODE" == "k8s" ]]; then + if [[ "$GITLAB_FRONTDOOR_OWNER" == "operator" ]]; then + GITLAB_WEBSERVICE_INGRESS_ENABLED=true + elif [[ "$GITLAB_FRONTDOOR_OWNER" == "fallback" ]]; then + GITLAB_WEBSERVICE_INGRESS_ENABLED=false + fi +fi + +# Google Workspace OIDC — FRONTDOOR_HOST gates OmniAuth configuration. +# On k3s: api.knoe.org is the knoe-auth SSO gateway (knoe-auth service). +# Requires k8s secret 'gitlab-google-oidc' in GITLAB_NAMESPACE with Google +# OAuth2 client credentials (client_id, client_secret, redirect_uri). +# To disable OIDC: unset FRONTDOOR_HOST before running. +_default_auth_hostname="api.prole.org" +if [[ "$MODE" == "k8s" ]]; then + _default_auth_hostname="api.knoe.dev" +fi +AUTH_HOSTNAME="${AUTH_HOSTNAME:-${FRONTDOOR_HOST:-${_default_auth_hostname}}}" +FRONTDOOR_HOST="${FRONTDOOR_HOST:-$AUTH_HOSTNAME}" +FRONTDOOR_AUTH_ENABLED="${FRONTDOOR_AUTH_ENABLED:-${AUTHORITY_ENABLED:-1}}" +AUTH_VERIFY_PATH="${AUTH_VERIFY_PATH:-/auth/verify}" +AUTH_LOGIN_PATH="${AUTH_LOGIN_PATH:-/auth/login}" +AUTH_RESPONSE_HEADERS="${AUTH_RESPONSE_HEADERS:-X-Knoe-User,X-Knoe-Email,X-Knoe-Groups}" +AUTH_VERIFY_URL="${AUTH_VERIFY_URL:-https://${AUTH_HOSTNAME}${AUTH_VERIFY_PATH}}" +AUTH_SIGNIN_URL="${AUTH_SIGNIN_URL:-https://${AUTH_HOSTNAME}${AUTH_LOGIN_PATH}?next=\$scheme://\$host\$escaped_request_uri}" +_default_gitlab_oidc_issuer="https://${AUTH_HOSTNAME}/auth" +if [[ "$MODE" == "k8s" ]]; then + _default_gitlab_oidc_issuer="https://api.knoe.dev/auth" +fi +_default_gitlab_oidc_redirect_uri="https://${GITLAB_DOMAIN}/users/auth/openid_connect/callback" +if [[ "$MODE" == "k8s" ]]; then + _default_gitlab_oidc_redirect_uri="https://git.knoe.dev/users/auth/openid_connect/callback" +fi +GITLAB_OIDC_PROVIDER_NAME="${GITLAB_OIDC_PROVIDER_NAME:-openid_connect}" +GITLAB_OIDC_ISSUER="${GITLAB_OIDC_ISSUER:-${OIDC_ISSUER:-${_default_gitlab_oidc_issuer}}}" +GITLAB_OIDC_REDIRECT_URI="${GITLAB_OIDC_REDIRECT_URI:-${_default_gitlab_oidc_redirect_uri}}" +GITLAB_OIDC_CLIENT_ID="${GITLAB_OIDC_CLIENT_ID:-${OIDC_CLIENT_ID:-${GOOGLE_OIDC_CLIENT_ID:-${GOOGLE_CLIENT_ID:-}}}}" +GITLAB_OIDC_CLIENT_SECRET="${GITLAB_OIDC_CLIENT_SECRET:-${OIDC_CLIENT_SECRET:-${GOOGLE_OIDC_CLIENT_SECRET:-${GOOGLE_CLIENT_SECRET:-}}}}" + +# Resolve `secretref://` values for standalone invocations. deploy.sh +# resolves these in Python (knoe/core/actions.py:_resolve_secretref_value) +# before exec; direct invocations need the same lookup inline. Searches env +# aliases, then KNOE_SERVICE/secrets, etc/secrets, and secrets directories +# for a file named . Returns the raw input unchanged on miss. +_init_gitlab_resolve_secretref() { + local raw="${1:-}" + local aliases="${2:-}" + case "$raw" in + secretref://*) ;; + *) printf '%s' "$raw"; return 0 ;; + esac + + local ref="${raw#secretref://}" + ref="${ref#/}"; ref="${ref%/}" + [[ -n "$ref" ]] || { printf '%s' "$raw"; return 0; } + + local normalized="${ref//-/_}" + normalized="${normalized//\//_}" + normalized="${normalized//./_}" + local upper_norm="${normalized^^}" + + local -a candidates=("$ref" "$normalized" "$upper_norm") + local alias + for alias in $aliases; do + candidates+=("$alias") + done + + local key val + for key in "${candidates[@]}"; do + [[ -z "$key" ]] && continue + [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue + val="${!key-}" + if [[ -n "$val" && "$val" != secretref://* ]]; then + printf '%s' "$val" + return 0 + fi + done + + local root="${KNOE_HOME:-${SCRIPT_DIR}/..}" + local svc="${KNOE_SERVICE:-}" + local -a paths=() + [[ -n "$svc" ]] && paths+=("$svc/secrets/$ref") + paths+=("$root/etc/secrets/$ref" "$root/secrets/$ref") + + local path + for path in "${paths[@]}"; do + if [[ -f "$path" && -r "$path" ]]; then + val="$(< "$path")" + val="${val#"${val%%[![:space:]]*}"}" + val="${val%"${val##*[![:space:]]}"}" + if [[ -n "$val" ]]; then + printf '%s' "$val" + return 0 + fi + fi + done + + printf '%s' "$raw" +} + +GITLAB_OIDC_CLIENT_ID="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_ID" \ + "GITLAB_OIDC_CLIENT_ID OIDC_CLIENT_ID GOOGLE_OIDC_CLIENT_ID GOOGLE_CLIENT_ID")" +GITLAB_OIDC_CLIENT_SECRET="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_SECRET" \ + "GITLAB_OIDC_CLIENT_SECRET OIDC_CLIENT_SECRET GOOGLE_OIDC_CLIENT_SECRET GOOGLE_CLIENT_SECRET")" + +is_unresolved_secret_ref() { + local raw="${1:-}" + case "$raw" in + ""|secretref://*|'${OPENBAO:'*|'${KNOE_SECRET:'*) + return 0 + ;; + esac + return 1 +} +GITLAB_RELEASE="gitlab" + +gitlab_split_cluster_ownership_diagnostics + +assert_public_ingress_targeting "$GITLAB_INGRESS_CLASS" "${GITLAB_PUBLIC_HOSTS[@]}" + +# --------------------------------------------------------------------------- +# Garage S3 (knoe-system) — replaces embedded minio for object storage +# --------------------------------------------------------------------------- +GARAGE_NAMESPACE="${GARAGE_NAMESPACE:-knoe-system}" +GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-}" +GARAGE_S3_ENDPOINT="${GARAGE_S3_ENDPOINT:-${GARAGE_PRIVATE_S3_ENDPOINT:-${GARAGE_PRIVATE_ENDPOINT:-}}}" +if [[ "$GARAGE_S3_ENDPOINT" == *""* ]]; then + repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \ + "Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint in your config." +fi +if [[ -z "$GARAGE_S3_ENDPOINT" ]]; then + if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]]; then + die "GARAGE_S3_ENDPOINT (or GARAGE_PRIVATE_S3_ENDPOINT) must be set to an explicit private cross-cluster endpoint when APP and DB clusters differ." + fi + GARAGE_SVC_HOST="${GARAGE_SVC_HOST:-garage.${GARAGE_NAMESPACE}.svc.cluster.local}" + GARAGE_S3_ENDPOINT="http://${GARAGE_SVC_HOST}:3900" +fi +if [[ "$MODE" == "k8s" && -n "${APP_CLUSTER_KUBECONTEXT:-}" && -n "${DB_CLUSTER_KUBECONTEXT:-}" && "$APP_CLUSTER_KUBECONTEXT" != "$DB_CLUSTER_KUBECONTEXT" ]] && [[ "$GARAGE_S3_ENDPOINT" == *".svc.cluster.local"* ]]; then + die "GARAGE_S3_ENDPOINT must use a private cross-cluster endpoint and cannot use cluster-local service DNS (${GARAGE_S3_ENDPOINT})." +fi +if [[ -z "$GARAGE_SVC_HOST" ]]; then + _garage_endpoint_hostport="${GARAGE_S3_ENDPOINT#http://}" + _garage_endpoint_hostport="${_garage_endpoint_hostport#https://}" + _garage_endpoint_hostport="${_garage_endpoint_hostport%%/*}" + GARAGE_SVC_HOST="${_garage_endpoint_hostport%%:*}" + unset _garage_endpoint_hostport +fi +GARAGE_S3_KEY_NAME="${GARAGE_S3_KEY_NAME:-gitlab-s3}" +GITLAB_OBJECT_STORAGE_REQUIRED="${GITLAB_OBJECT_STORAGE_REQUIRED:-1}" + +# --------------------------------------------------------------------------- +# Operator chart coordinates +# --------------------------------------------------------------------------- +OPERATOR_REPO_NAME="gitlab-operator" +OPERATOR_REPO_URL="https://gitlab.com/api/v4/projects/18899486/packages/helm/stable" +OPERATOR_CHART="gitlab-operator/gitlab-operator" +OPERATOR_RELEASE="gitlab-operator" +OPERATOR_NAMESPACE="${GITLAB_OPERATOR_NAMESPACE:-${NAMESPACE}}" +GITLAB_OPERATOR_CHART_VERSION="${GITLAB_OPERATOR_CHART_VERSION:-}" +# GitLab Helm chart version required in the CR; auto-detected if not set. +GITLAB_CHART_VERSION="${GITLAB_CHART_VERSION:-}" +GITLAB_CHART_REPO_NAME="gitlab" +GITLAB_CHART_REPO_URL="https://charts.gitlab.io/" +gitlab_db_secret_changed=0 +gitlab_object_storage_secret_changed=0 +gitlab_registry_storage_secret_changed=0 + +resolve_helm_release_chart_version() { + local release_namespace="$1" + local release_name="$2" + local chart_version + chart_version=$({ helm -n "$release_namespace" list -f "^${release_name}$" -o json 2>/dev/null || true; } | python3 - <<'PY' +import json +import sys + +try: + rows = json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) + +if not rows: + print("") + raise SystemExit(0) + +chart = str(rows[0].get("chart", "")) +if not chart: + print("") + raise SystemExit(0) + +if "-" in chart: + print(chart.rsplit("-", 1)[-1]) +else: + print("") +PY +) + if [[ -n "$chart_version" ]]; then + printf '%s' "$chart_version" + return 0 + fi + + { helm -n "$release_namespace" status "$release_name" -o json 2>/dev/null || true; } | python3 - <<'PY' +import json +import sys + +try: + payload = json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) + +chart = str(payload.get("chart", "")) +if not chart: + print("") + raise SystemExit(0) + +if "-" in chart: + print(chart.rsplit("-", 1)[-1]) +else: + print("") +PY +} + +resolve_operator_watch_namespace() { + local watch_namespace + watch_namespace=$({ helm -n "$OPERATOR_NAMESPACE" get values "$OPERATOR_RELEASE" --all -o json 2>/dev/null || true; } | python3 - <<'PY' +import json +import sys + +try: + values = json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) + +watch_namespace = values.get("watchNamespace", "") +if isinstance(watch_namespace, str): + print(watch_namespace.strip()) +else: + print("") +PY +) + if [[ -n "$watch_namespace" ]]; then + printf '%s' "$watch_namespace" + return 0 + fi + + kubectl -n "$OPERATOR_NAMESPACE" get deploy "$OPERATOR_RELEASE" \ + -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="WATCH_NAMESPACE")].value}' 2>/dev/null || true +} + +kubectl_apply_reports_changed() { + local apply_output="${1:-}" + if [[ -z "${apply_output//[[:space:]]/}" ]]; then + # Be conservative: unknown apply output should force full reconcile checks. + return 0 + fi + + case "$apply_output" in + *" created"*|*" configured"*|*" patched"*) return 0 ;; + esac + + case "$apply_output" in + *" unchanged"*) return 1 ;; + esac + + # Unknown token from kubectl apply output -> treat as changed. + return 0 +} + +# --------------------------------------------------------------------------- +# Helper: resolve knoe-db primary pod (same pattern as init_gitea.sh) +# --------------------------------------------------------------------------- +resolve_knoe_db_primary_pod() { + local primary + primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \ + -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME},cnpg.io/instanceRole=primary" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) - if [[ -z "$primary_pod" ]]; then - primary_pod=$(kubectl -n "$DB_NAMESPACE" get pods \ + if [[ -z "$primary" ]]; then + primary=$(kubectl_db -n "$DB_NAMESPACE" get pods \ -l "cnpg.io/cluster=${CNPG_CLUSTER_NAME}" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) fi - if [[ -z "$primary_pod" ]]; then - die "Could not find CNPG pod for cluster '${CNPG_CLUSTER_NAME}' in namespace '${DB_NAMESPACE}'" - fi - - log "Ensuring database '${GITLAB_DB_NAME}' and role '${GITLAB_DB_USER}' exist in CNPG cluster '${CNPG_CLUSTER_NAME}' (ns=${DB_NAMESPACE})" - - kubectl -n "$DB_NAMESPACE" exec "$primary_pod" -- bash -lc "psql -v ON_ERROR_STOP=1 -U postgres -d postgres" </dev/null + printf '%s' "$primary" } -ensure_db +sql_escape_literal() { + printf '%s' "${1:-}" | sed "s/'/''/g" +} -RELEASE_NAME="${GITLAB_RELEASE_NAME:-gitlab}" +# --------------------------------------------------------------------------- +# Resolve GitLab DB password (pull from knoe-db-superuser secret as fallback) +# --------------------------------------------------------------------------- +resolve_gitlab_db_password() { + if [[ -n "${GITLAB_DB_PASSWORD:-}" ]]; then + return 0 + fi + local resolved="" + if kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser >/dev/null 2>&1; then + resolved=$(kubectl_db -n "$DB_NAMESPACE" get secret knoe-db-superuser \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) + fi + if [[ -n "$resolved" ]]; then + GITLAB_DB_PASSWORD="$resolved" + else + # Generate a random password when none is available + GITLAB_DB_PASSWORD="$(LC_ALL=C tr -dc 'A-Za-z0-9' /dev/null | head -c 32 || true)" + warn "Generated random GitLab DB password; store it in GITLAB_DB_PASSWORD for future runs." + fi +} +# --------------------------------------------------------------------------- +# Provision GitLab role + database in knoe-db +# --------------------------------------------------------------------------- +setup_knoe_db_for_gitlab() { + local primary db_ctx + db_ctx="$(resolve_db_cluster_context || true)" + + primary="$(resolve_knoe_db_primary_pod)" + if [[ -z "$primary" ]]; then + if [[ "$MODE" == "k8s" ]]; then + die "No knoe-db pod found in namespace '${DB_NAMESPACE}' on DB_CLUSTER_KUBECONTEXT='${db_ctx}'." + fi + warn "No knoe-db pod found in namespace '${DB_NAMESPACE}'; skipping GitLab DB setup." + return 0 + fi + + resolve_gitlab_db_password + + local admin_user="" + local candidate + for candidate in postgres root; do + if kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$candidate" -d postgres -tAc "SELECT 1" >/dev/null 2>&1; then + admin_user="$candidate" + break + fi + done + + if [[ -z "$admin_user" ]]; then + if [[ "$MODE" == "k8s" ]]; then + die "Unable to connect to knoe-db as admin on DB_CLUSTER_KUBECONTEXT='${db_ctx}'." + fi + warn "Unable to connect to knoe-db as admin; skipping GitLab DB setup." + return 0 + fi + + local esc_pw + esc_pw="$(sql_escape_literal "$GITLAB_DB_PASSWORD")" + + if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$admin_user" -d postgres -c " + DO \$\$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${GITLAB_DB_USER}') THEN + CREATE ROLE ${GITLAB_DB_USER} LOGIN PASSWORD '${esc_pw}'; + ELSE + ALTER ROLE ${GITLAB_DB_USER} WITH PASSWORD '${esc_pw}'; + END IF; + END \$\$; + " >/dev/null 2>&1; then + if [[ "$MODE" == "k8s" ]]; then + die "Could not create/update role '${GITLAB_DB_USER}' on DB cluster context '${db_ctx}'." + fi + warn "Could not create/update role '${GITLAB_DB_USER}'." + fi + + local db_exists + db_exists=$(kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$admin_user" -d postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='${GITLAB_DB_NAME}';" 2>/dev/null || true) + + if [[ "$db_exists" != "1" ]]; then + if ! kubectl_db -n "$DB_NAMESPACE" exec "$primary" -c postgres -- \ + psql -U "$admin_user" -d postgres -c \ + "CREATE DATABASE ${GITLAB_DB_NAME} OWNER ${GITLAB_DB_USER};" \ + >/dev/null 2>&1; then + if [[ "$MODE" == "k8s" ]]; then + die "Could not create database '${GITLAB_DB_NAME}' on DB cluster context '${db_ctx}'." + fi + warn "Could not create database '${GITLAB_DB_NAME}'." + fi + fi + + if [[ -n "$db_ctx" ]]; then + log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE}, ctx=${db_ctx})." + else + log "GitLab database '${GITLAB_DB_NAME}' prepared in knoe-db (ns=${DB_NAMESPACE})." + fi +} + +# --------------------------------------------------------------------------- +# Clean up any pre-existing Gitea / git.prole.org configurations +# --------------------------------------------------------------------------- +cleanup_gitea() { + log "Checking for pre-existing Gitea deployment to remove before GitLab install..." + + local gitea_ns="${GITEA_NAMESPACE:-gitea}" + + # Remove Gitea Helm release + if helm -n "$gitea_ns" status gitea >/dev/null 2>&1; then + log "Uninstalling Gitea Helm release from namespace '$gitea_ns'..." + helm -n "$gitea_ns" uninstall gitea >/dev/null 2>&1 || true + fi + + # Remove any leftover raw Gitea resources + kubectl -n "$gitea_ns" delete deploy/gitea svc/gitea-http svc/gitea-ssh \ + >/dev/null 2>&1 || true + + # Remove Kong routes/services registered for git.prole.org (best-effort) + local kong_ns="${KONG_NAMESPACE:-${NAMESPACE:-kong}}" + for res_type in kongplugins kongingresses; do + kubectl -n "$gitea_ns" delete "$res_type" --all >/dev/null 2>&1 || true + done + + # Remove gitea namespace Ingress objects that route git.prole.org + kubectl -n "$gitea_ns" delete ingress \ + -l "app.kubernetes.io/name=gitea" >/dev/null 2>&1 || true + kubectl -n "$gitea_ns" delete ingress \ + --field-selector="metadata.name=gitea" >/dev/null 2>&1 || true + + log "Gitea cleanup complete." +} + +# --------------------------------------------------------------------------- +# Ensure gitlab namespace exists +# --------------------------------------------------------------------------- +kubectl get ns "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null + +# --------------------------------------------------------------------------- +# --force: remove existing GitLab operator and release first +# --------------------------------------------------------------------------- if [[ "$FORCE" -eq 1 ]]; then - warn "--force specified; uninstalling existing release '$RELEASE_NAME' in namespace '$NAMESPACE' (if present)" - helm -n "$NAMESPACE" uninstall "$RELEASE_NAME" >/dev/null 2>&1 || true + warn "--force: removing existing GitLab resources in namespace '$NAMESPACE'..." + if kubectl get crd gitlabs.apps.gitlab.com >/dev/null 2>&1; then + kubectl -n "$NAMESPACE" delete gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true + # Wait briefly for operator to clean up managed resources + sleep 10 + fi + helm -n "$NAMESPACE" uninstall "$GITLAB_RELEASE" >/dev/null 2>&1 || true + helm -n "$NAMESPACE" uninstall "$OPERATOR_RELEASE" >/dev/null 2>&1 || true + cleanup_gitea fi -GITLAB_DOMAIN="${GITLAB_DOMAIN:-${DOMAIN:-${PROLE_DOMAIN:-}}}" -GITLAB_DOMAIN="${GITLAB_DOMAIN:-example.local}" +# --------------------------------------------------------------------------- +# Always remove gitea if its helm release exists (non-destructive path) +# gitea and gitlab both claim git.prole.org; they cannot coexist. +# --------------------------------------------------------------------------- +if helm -n "${GITEA_NAMESPACE:-gitea}" status gitea >/dev/null 2>&1; then + warn "Gitea release detected — removing to free git.knoe.org for GitLab..." + cleanup_gitea +fi -log "Adding/updating GitLab Helm repo" -helm repo add gitlab https://charts.gitlab.io/ >/dev/null 2>&1 || true -helm repo update >/dev/null +# --------------------------------------------------------------------------- +# Prepare knoe-db +# --------------------------------------------------------------------------- +setup_knoe_db_for_gitlab -log "Installing GitLab (release=$RELEASE_NAME, ns=$NAMESPACE, domain=$GITLAB_DOMAIN)" +# Persist the DB password as a k8s Secret the operator CR can reference +resolve_gitlab_db_password +gitlab_db_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-db-password \ + --from-literal=password="$GITLAB_DB_PASSWORD" \ + --dry-run=client -o yaml | kubectl apply -f - 2>&1)" +if kubectl_apply_reports_changed "$gitlab_db_secret_apply_output"; then + gitlab_db_secret_changed=1 +fi +log "gitlab-db-password secret applied in namespace '$NAMESPACE' (${gitlab_db_secret_apply_output})." -# Notes: -# - We keep the chart install minimal to avoid assuming ingress/cert-manager setup. -# - We disable the bundled PostgreSQL and point GitLab to CNPG. -helm upgrade --install "$RELEASE_NAME" gitlab/gitlab \ - -n "$NAMESPACE" \ - --timeout 30m \ - --wait \ - --set global.hosts.domain="$GITLAB_DOMAIN" \ - --set global.hosts.https=false \ - --set certmanager.install=false \ - --set prometheus.install=false \ - --set postgresql.install=false \ - --set global.psql.host="$DB_HOST" \ - --set global.psql.port="$DB_PORT" \ - --set global.psql.username="$GITLAB_DB_USER" \ - --set global.psql.database="$GITLAB_DB_NAME" \ - --set global.psql.password.secret=gitlab-db \ - --set global.psql.password.key=password +# --------------------------------------------------------------------------- +# Install / upgrade the GitLab Operator +# --------------------------------------------------------------------------- +operator_changed=0 +operator_release_exists=0 +operator_requires_upgrade=1 +installed_operator_chart_version="" +installed_operator_watch_namespace="" +operator_watch_namespace_matches=1 -log "Done. Inspect services/pods with: kubectl -n $NAMESPACE get pods,svc" +if helm -n "$OPERATOR_NAMESPACE" status "$OPERATOR_RELEASE" >/dev/null 2>&1; then + operator_release_exists=1 + installed_operator_chart_version="$(resolve_helm_release_chart_version "$OPERATOR_NAMESPACE" "$OPERATOR_RELEASE")" + installed_operator_watch_namespace="$(resolve_operator_watch_namespace)" +fi + +if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" && -n "$installed_operator_chart_version" ]]; then + GITLAB_OPERATOR_CHART_VERSION="$installed_operator_chart_version" + log "Reusing installed GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}" +fi + +if [[ -n "$installed_operator_watch_namespace" && "$installed_operator_watch_namespace" != "$NAMESPACE" ]]; then + operator_watch_namespace_matches=0 +fi + +if [[ "$operator_release_exists" == "1" && -n "$installed_operator_chart_version" \ + && "$installed_operator_chart_version" == "$GITLAB_OPERATOR_CHART_VERSION" \ + && "$operator_watch_namespace_matches" == "1" \ + && "${GITLAB_OPERATOR_FORCE_UPGRADE:-0}" != "1" ]]; then + operator_requires_upgrade=0 +fi + +if [[ "$operator_requires_upgrade" == "1" ]]; then + if [[ "$operator_watch_namespace_matches" == "0" ]]; then + warn "Installed GitLab Operator watchNamespace (${installed_operator_watch_namespace}) differs from target (${NAMESPACE}); forcing operator upgrade." + fi + + log "Adding/updating GitLab Operator Helm repo..." + helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true + helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..." + + if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then + GITLAB_OPERATOR_CHART_VERSION=$(helm search repo "$OPERATOR_CHART" --output table 2>/dev/null \ + | awk 'NR==2{print $2}' || true) + if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then + die "Cannot detect GitLab Operator chart version. Set GITLAB_OPERATOR_CHART_VERSION explicitly and re-run." + fi + log "Auto-detected GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}" + else + log "Using GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}" + fi + + log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..." + + # The operator needs cluster-scoped RBAC; it watches all namespaces by default. + helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \ + -n "$OPERATOR_NAMESPACE" \ + --create-namespace \ + --version "$GITLAB_OPERATOR_CHART_VERSION" \ + --timeout 10m \ + --wait \ + --set watchNamespace="$NAMESPACE" + + operator_changed=1 + log "GitLab Operator ready." +else + log "GitLab Operator already matches desired chart/watchNamespace; skipping operator upgrade." +fi + +# --------------------------------------------------------------------------- +# Resolve GitLab chart version (required by the Operator CR since >= v0.28) +# --------------------------------------------------------------------------- +installed_gitlab_chart_version="" +if [[ -z "$GITLAB_CHART_VERSION" ]]; then + installed_gitlab_chart_version="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.spec.chart.version}' 2>/dev/null || true)" + if [[ -n "$installed_gitlab_chart_version" ]]; then + GITLAB_CHART_VERSION="$installed_gitlab_chart_version" + log "Reusing installed GitLab chart version from existing GitLab CR: ${GITLAB_CHART_VERSION}" + else + log "Auto-detecting latest GitLab chart version..." + helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true + helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..." + GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \ + | awk 'NR==2{print $2}' || true) + if [[ -z "$GITLAB_CHART_VERSION" ]]; then + die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run." + fi + log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}" + fi +fi +CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\"" +gitlab_chart_version_changed=0 +if [[ -z "$installed_gitlab_chart_version" || "$installed_gitlab_chart_version" != "$GITLAB_CHART_VERSION" ]]; then + gitlab_chart_version_changed=1 +fi + +# --------------------------------------------------------------------------- +# Determine node selector block for the GitLab CR +# --------------------------------------------------------------------------- +# Storage node is only used for legacy/local static local-PV mode. +if [[ "$MODE" == "k8s" ]]; then + # In k8s/GKE mode, we do NOT use static storage nodes or nodeSelectors. + STORAGE_NODE="" + if [[ "$NODE_SELECTOR" == *"gandalf.prole.org"* ]]; then + warn "Stripping legacy nodeSelector '${NODE_SELECTOR}' in k8s mode." + NODE_SELECTOR="" + fi +else + STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}" +fi +[[ "$MODE" == "k8s" || -n "$STORAGE_NODE" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set in ${MODE} mode." +# NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned +NODE_SELECTOR="${NODE_SELECTOR:-}" +NODE_SELECTOR_KEY="${GITLAB_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}" +STORAGE_NODE_SELECTOR_KEY="${GITLAB_STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY}}" +NODE_SELECTOR_YAML="" +if [[ -n "$NODE_SELECTOR" ]]; then + log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}" + NODE_SELECTOR_YAML="${NODE_SELECTOR_KEY}: ${NODE_SELECTOR}" +fi +STORAGE_NODE_SELECTOR_YAML="" +if [[ -n "$STORAGE_NODE" ]]; then + STORAGE_NODE_SELECTOR_YAML="${STORAGE_NODE_SELECTOR_KEY}: ${STORAGE_NODE}" +fi + +GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}" +if [[ "$MODE" == "k8s" ]]; then + # Strip legacy values that are meaningless in k8s mode — fall through to defaults. + if [[ "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then + warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode." + GITALY_STORAGE_CLASS="" + fi + + # Default to strict 'standard' (pd-standard / HDD) in k8s mode. + # Never auto-substitute to 'standard-rwo' for GitLab storage. + if [[ -z "$GITALY_STORAGE_CLASS" ]]; then + GITALY_STORAGE_CLASS="standard" + log "Defaulting Gitaly StorageClass to strict target 'standard' (pd-standard / HDD)." + else + log "Honoring explicit GITLAB_GITALY_STORAGE_CLASS=${GITALY_STORAGE_CLASS} (no auto-normalization)." + fi + + if [[ "$GITALY_STORAGE_CLASS" == "standard" ]] && ! kubectl get storageclass standard >/dev/null 2>&1; then + repair_blocked "Required GitLab StorageClass is missing" \ + "Configured target storageClass is 'standard' but StorageClass/standard does not exist on the cluster. Do not substitute to standard-rwo; create/restore StorageClass 'standard' or set an explicit non-standard class." + fi +else + if [[ -z "$GITALY_STORAGE_CLASS" ]]; then + GITALY_STORAGE_CLASS="gitlab-gitaly-static" + fi +fi + +# Sidekiq resources and concurrency +GITLAB_SIDEKIQ_REQUESTS_CPU="${GITLAB_SIDEKIQ_REQUESTS_CPU:-250m}" +GITLAB_SIDEKIQ_REQUESTS_MEMORY="${GITLAB_SIDEKIQ_REQUESTS_MEMORY:-1500Mi}" +GITLAB_SIDEKIQ_LIMITS_CPU="${GITLAB_SIDEKIQ_LIMITS_CPU:-1}" +GITLAB_SIDEKIQ_LIMITS_MEMORY="${GITLAB_SIDEKIQ_LIMITS_MEMORY:-3Gi}" +GITLAB_SIDEKIQ_CONCURRENCY="${GITLAB_SIDEKIQ_CONCURRENCY:-5}" + +# Webservice (Puma) resources and concurrency +# Defaults calibrated for e2-standard-2 nodes (~7.1 GB allocatable). +# - workerProcesses=1 keeps steady-state RSS under ~1.1 GB. +# - Memory *limit* must exceed the boot spike (Rails preload + worker fork), +# which peaks around 1.4–1.6 GB. 1800M gives headroom without exceeding +# the node's 1/4-node per-pod budget. +# - puma.threads.{min,max} are the correct chart paths; PUMA_THREADS_{MIN,MAX} +# env vars do NOT propagate through the chart's ERB-rendered puma config. +GITLAB_WEBSERVICE_REQUESTS_CPU="${GITLAB_WEBSERVICE_REQUESTS_CPU:-200m}" +GITLAB_WEBSERVICE_REQUESTS_MEMORY="${GITLAB_WEBSERVICE_REQUESTS_MEMORY:-900M}" +GITLAB_WEBSERVICE_LIMITS_MEMORY="${GITLAB_WEBSERVICE_LIMITS_MEMORY:-1800M}" +GITLAB_WEBSERVICE_WORKER_PROCESSES="${GITLAB_WEBSERVICE_WORKER_PROCESSES:-1}" +GITLAB_WEBSERVICE_PUMA_THREADS_MIN="${GITLAB_WEBSERVICE_PUMA_THREADS_MIN:-2}" +GITLAB_WEBSERVICE_PUMA_THREADS_MAX="${GITLAB_WEBSERVICE_PUMA_THREADS_MAX:-2}" + +JEMALLOC_HOSTPATH_DIR="/opt/gitlab-jemalloc" +JEMALLOC_HOSTPATH_LIB="${JEMALLOC_HOSTPATH_DIR}/libjemalloc.so.2" +GITLAB_JEMALLOC_MODE="${GITLAB_JEMALLOC_MODE:-auto}" +if is_truthy "${GITLAB_JEMALLOC_REQUIRED:-0}"; then + GITLAB_JEMALLOC_MODE="force" +fi + +JEMALLOC_HOSTPATH_SUPPORTED=1 +JEMALLOC_HOSTPATH_REASON="" +JEMALLOC_HOSTPATH_WANTED=0 +JEMALLOC_HOSTPATH_ACTIVE=0 + +detect_jemalloc_hostpath_support() { + local provider_ids gke_pool_labels gke_topology_labels os_images + + provider_ids=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.providerID}{"\n"}{end}' 2>/dev/null || true) + gke_pool_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.cloud\.google\.com/gke-nodepool}{"\n"}{end}' 2>/dev/null || true) + gke_topology_labels=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.topology\.gke\.io/zone}{"\n"}{end}' 2>/dev/null || true) + os_images=$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.nodeInfo.osImage}{"\n"}{end}' 2>/dev/null || true) + + if [[ "$provider_ids" == *"gce://"* || -n "${gke_pool_labels//[[:space:]]/}" || -n "${gke_topology_labels//[[:space:]]/}" ]]; then + JEMALLOC_HOSTPATH_SUPPORTED=0 + JEMALLOC_HOSTPATH_REASON="Detected GKE/GCE node metadata (providerID/labels)." + return 0 + fi + + if [[ "${os_images,,}" == *"container-optimized os"* ]]; then + JEMALLOC_HOSTPATH_SUPPORTED=0 + JEMALLOC_HOSTPATH_REASON="Detected Container-Optimized OS node image." + fi +} + +configure_jemalloc_hostpath_mode() { + local mode_normalized="${GITLAB_JEMALLOC_MODE,,}" + + detect_jemalloc_hostpath_support + + case "$mode_normalized" in + auto|"") + if (( JEMALLOC_HOSTPATH_SUPPORTED == 1 )); then + JEMALLOC_HOSTPATH_WANTED=1 + else + JEMALLOC_HOSTPATH_WANTED=0 + warn "Skipping hostPath jemalloc optimization: ${JEMALLOC_HOSTPATH_REASON}" + fi + ;; + off|false|0|disabled) + JEMALLOC_HOSTPATH_WANTED=0 + ;; + force|required|on|true|1) + JEMALLOC_HOSTPATH_WANTED=1 + if (( JEMALLOC_HOSTPATH_SUPPORTED == 0 )); then + die "HostPath jemalloc mode is forced but unsupported in this cluster: ${JEMALLOC_HOSTPATH_REASON} Disable jemalloc (GITLAB_JEMALLOC_MODE=off) or use a non-hostPath approach." + fi + ;; + *) + die "Unsupported GITLAB_JEMALLOC_MODE='${GITLAB_JEMALLOC_MODE}' (expected: auto, off, force)." + ;; + esac +} + +# --------------------------------------------------------------------------- +# Create the GitLab CR (operator reconciles this into the full deployment) +# --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# Apply gitlab-google-oidc secret (OIDC provider config for OmniAuth) +# --------------------------------------------------------------------------- +if [[ -n "$FRONTDOOR_HOST" ]]; then + oidc_secret_tmpl="${SCRIPT_DIR}/../deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml" + if is_unresolved_secret_ref "$GITLAB_OIDC_CLIENT_ID" || is_unresolved_secret_ref "$GITLAB_OIDC_CLIENT_SECRET"; then + repair_blocked "GitLab OIDC client credentials are missing or unresolved" \ + "Set non-empty OIDC client values in installer inputs/config (auth.clientId/auth.clientSecret or GITLAB_OIDC_CLIENT_ID/GITLAB_OIDC_CLIENT_SECRET) and rerun deploy." + fi + if [[ -f "$oidc_secret_tmpl" ]]; then + log "Applying gitlab-google-oidc secret (issuer=${GITLAB_OIDC_ISSUER}, redirect=${GITLAB_OIDC_REDIRECT_URI}) ..." + GITLAB_OIDC_PROVIDER_NAME="$GITLAB_OIDC_PROVIDER_NAME" \ + GITLAB_OIDC_ISSUER="$GITLAB_OIDC_ISSUER" \ + GITLAB_OIDC_CLIENT_ID="$GITLAB_OIDC_CLIENT_ID" \ + GITLAB_OIDC_CLIENT_SECRET="$GITLAB_OIDC_CLIENT_SECRET" \ + GITLAB_OIDC_REDIRECT_URI="$GITLAB_OIDC_REDIRECT_URI" \ + envsubst < "$oidc_secret_tmpl" | kubectl apply -n "$NAMESPACE" -f - >/dev/null || \ + repair_blocked "Could not apply gitlab-google-oidc secret" "GitLab OIDC provider secret apply failed for namespace ${NAMESPACE}." + else + repair_blocked "gitlab-google-oidc-secret.example.yaml not found" "Missing ${oidc_secret_tmpl}; cannot configure GitLab OIDC automatically." + fi +fi + +# --------------------------------------------------------------------------- +# ARM64 / 16KB-page jemalloc fix — build glibc jemalloc on every node once +# All RPi nodes use 16KB kernel pages; GitLab's bundled jemalloc is 4KB-only. +# We compile a compatible jemalloc-5.3.0 with --with-lg-page=14 via DaemonSet. +# --------------------------------------------------------------------------- +setup_jemalloc_on_nodes() { + local jlib="${JEMALLOC_HOSTPATH_LIB}" + local ds_name="jemalloc-builder" + + if (( JEMALLOC_HOSTPATH_WANTED == 0 )); then + log "HostPath jemalloc optimization disabled for this environment." + if kubectl -n "$NAMESPACE" get ds "$ds_name" >/dev/null 2>&1; then + log "Deleting existing ${ds_name} DaemonSet because hostPath jemalloc is disabled." + kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \ + warn "Could not delete ${ds_name} DaemonSet; continuing." + fi + JEMALLOC_HOSTPATH_ACTIVE=0 + return 0 + fi + + # Check if already present on all schedulable nodes + local nodes + nodes=$(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | tr '\n' ' ') + local all_ready=true + for node in $nodes; do + result=$(kubectl -n "$NAMESPACE" run "jcheck-${node//\./-}" \ + --image=ubuntu:22.04 --restart=Never --rm --attach --quiet \ + --overrides="{\"spec\":{\"nodeName\":\"${node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"jlib\",\"hostPath\":{\"path\":\"${JEMALLOC_HOSTPATH_DIR}\",\"type\":\"DirectoryOrCreate\"}}],\"containers\":[{\"name\":\"c\",\"image\":\"ubuntu:22.04\",\"command\":[\"bash\",\"-c\",\"test -f /jlib/libjemalloc.so.2 && echo OK || echo MISSING\"],\"volumeMounts\":[{\"name\":\"jlib\",\"mountPath\":\"/jlib\"}]}]}}" 2>/dev/null || echo "MISSING") + if [[ "$result" != *"OK"* ]]; then + all_ready=false + break + fi + done + + if $all_ready; then + log "jemalloc-16k already present on all nodes — skipping build." + JEMALLOC_HOSTPATH_ACTIVE=1 + return 0 + fi + + log "Deploying jemalloc-builder DaemonSet (compiles jemalloc-5.3.0 with --with-lg-page=14 on each node)..." + local _tmpds + _tmpds=$(mktemp /tmp/jemalloc-ds-XXXXXX.yaml) + cat > "$_tmpds" <<'JEDS' +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: jemalloc-builder + namespace: NAMESPACE_PLACEHOLDER + labels: + app: jemalloc-builder +spec: + selector: + matchLabels: + app: jemalloc-builder + template: + metadata: + labels: + app: jemalloc-builder + spec: + tolerations: + - operator: Exists + initContainers: + - name: build + image: ubuntu:22.04 + command: + - bash + - -c + - | + set -e + TARGET=/hostlib/libjemalloc.so.2 + if [ -f "$TARGET" ]; then echo "Already built"; exit 0; fi + apt-get update -qq && apt-get install -y -q gcc make wget bzip2 2>&1 + cd /tmp + wget -q -O jemalloc.tar.bz2 \ + https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 + tar xjf jemalloc.tar.bz2 && cd jemalloc-5.3.0 + ./configure --with-lg-page=14 --disable-stats --disable-prof --disable-fill 2>&1 + make -j2 lib/libjemalloc.so.2 2>&1 + cp lib/libjemalloc.so.2 "$TARGET" && chmod 755 "$TARGET" + echo "Done: $(ls -lh $TARGET)" + volumeMounts: + - name: hostlib + mountPath: /hostlib + containers: + - name: keepalive + image: ubuntu:22.04 + command: [bash, -c, "sleep infinity"] + volumes: + - name: hostlib + hostPath: + path: JEMALLOC_HOSTPATH_DIR_PLACEHOLDER + type: DirectoryOrCreate +JEDS + sed -i.bak "s/NAMESPACE_PLACEHOLDER/${NAMESPACE}/g" "$_tmpds" + sed -i.bak2 "s#JEMALLOC_HOSTPATH_DIR_PLACEHOLDER#${JEMALLOC_HOSTPATH_DIR}#g" "$_tmpds" + kubectl apply -f "$_tmpds" + rm -f "$_tmpds" "${_tmpds}.bak" "${_tmpds}.bak2" + + log "Waiting up to 20 minutes for jemalloc to build on all nodes..." + local deadline=$((SECONDS + 1200)) + while (( SECONDS < deadline )); do + ready=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \ + -o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0) + desired=$(kubectl -n "$NAMESPACE" get ds jemalloc-builder \ + -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo 1) + [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]] && break + log " jemalloc build progress: ${ready}/${desired} nodes ready..." + sleep 30 + done + + if [[ "$ready" -ge "$desired" && "$desired" -gt 0 ]]; then + JEMALLOC_HOSTPATH_ACTIVE=1 + log "jemalloc-16k ready on all nodes." + return 0 + fi + + JEMALLOC_HOSTPATH_ACTIVE=0 + if [[ "${GITLAB_JEMALLOC_MODE,,}" == "force" || "${GITLAB_JEMALLOC_MODE,,}" == "required" || "${GITLAB_JEMALLOC_MODE,,}" == "1" || "${GITLAB_JEMALLOC_MODE,,}" == "true" || "${GITLAB_JEMALLOC_MODE,,}" == "on" ]]; then + die "jemalloc-builder did not become ready (${ready}/${desired}) and jemalloc is required." + fi + kubectl -n "$NAMESPACE" delete ds "$ds_name" --ignore-not-found >/dev/null || \ + warn "Could not delete ${ds_name} DaemonSet after timeout; continuing without jemalloc." + warn "jemalloc-builder did not become ready (${ready}/${desired}); continuing without hostPath jemalloc optimization." +} + +# --------------------------------------------------------------------------- +# Garage S3 key + bucket provisioning for GitLab object storage +# Creates a dedicated "gitlab-s3" key in Garage, provisions required buckets, +# and writes the k8s secrets consumed by the GitLab CR. +# --------------------------------------------------------------------------- +setup_garage_for_gitlab() { + local garage_admin_context garage_admin_context_label + garage_admin_context="$(resolve_garage_admin_context || true)" + if [[ -n "$garage_admin_context" ]]; then + garage_admin_context_label="$garage_admin_context" + else + garage_admin_context_label="(active/default)" + fi + + log "Configuring Garage S3 (${GARAGE_NAMESPACE}) as GitLab object storage..." + log "Garage admin context: ${garage_admin_context_label}" + log "Garage namespace: ${GARAGE_NAMESPACE}" + log "GitLab object storage S3 endpoint: ${GARAGE_S3_ENDPOINT}" + + garage_setup_failure() { + local reason="$1" + if is_truthy "${GITLAB_OBJECT_STORAGE_REQUIRED:-1}"; then + die "${reason} (object storage is required for GitLab deployment)." + fi + warn "${reason} (continuing because GITLAB_OBJECT_STORAGE_REQUIRED=${GITLAB_OBJECT_STORAGE_REQUIRED})." + return 0 + } + + local garage_pod + garage_pod=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get pods -l app=garage \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$garage_pod" ]]; then + garage_setup_failure "Garage pod not found in namespace '${GARAGE_NAMESPACE}' on context '${garage_admin_context_label}'" + return 0 + fi + + local garage_admin_token_b64 garage_admin_token + garage_admin_token_b64=$(kubectl_garage_admin -n "$GARAGE_NAMESPACE" get secret garage-secrets \ + -o jsonpath='{.data.admin_token}' 2>/dev/null || true) + if [[ -z "$garage_admin_token_b64" ]]; then + garage_setup_failure "Garage admin token missing in secret '${GARAGE_NAMESPACE}/garage-secrets' on context '${garage_admin_context_label}'" + return 0 + fi + + garage_admin_token=$(printf '%s' "$garage_admin_token_b64" | base64 -d 2>/dev/null || true) + garage_admin_token="${garage_admin_token//$'\r'/}" + garage_admin_token="${garage_admin_token//$'\n'/}" + if [[ -z "$garage_admin_token" ]]; then + garage_setup_failure "Garage admin token in secret '${GARAGE_NAMESPACE}/garage-secrets' is malformed or empty on context '${garage_admin_context_label}'" + return 0 + fi + + local garage_cmd_last_reason="" + local garage_cmd_output="" + garage_exec() { + local cmd_desc="$1" + shift + + local out_file err_file stderr_text lower_stderr + out_file=$(mktemp) + err_file=$(mktemp) + + if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \ + /garage --admin-token "$garage_admin_token" "$@" >"$out_file" 2>"$err_file"; then + garage_cmd_output=$(cat "$out_file") + rm -f "$out_file" "$err_file" + garage_cmd_last_reason="" + return 0 + fi + + stderr_text=$(cat "$err_file" 2>/dev/null || true) + if [[ "$stderr_text" == *"--admin-token"* ]] && [[ "$stderr_text" == *"unknown"* || "$stderr_text" == *"unexpected"* || "$stderr_text" == *"wasn't expected"* ]]; then + if kubectl_garage_admin -n "$GARAGE_NAMESPACE" exec "$garage_pod" -- \ + /garage "$@" >"$out_file" 2>"$err_file"; then + garage_cmd_output=$(cat "$out_file") + rm -f "$out_file" "$err_file" + garage_cmd_last_reason="" + return 0 + fi + stderr_text=$(cat "$err_file" 2>/dev/null || true) + fi + + lower_stderr="${stderr_text,,}" + if [[ "$lower_stderr" == *"unauth"* || "$lower_stderr" == *"forbidden"* || "$lower_stderr" == *"invalid token"* || "$lower_stderr" == *"permission denied"* || "$lower_stderr" == *" 401"* || "$lower_stderr" == *" 403"* ]]; then + garage_cmd_last_reason="Unauthenticated admin command while running '${cmd_desc}'" + else + garage_cmd_last_reason="Garage admin command failed while running '${cmd_desc}'" + fi + if [[ -n "$stderr_text" ]]; then + garage_cmd_last_reason+=" (${stderr_text})" + fi + garage_cmd_output="" + + rm -f "$out_file" "$err_file" + return 1 + } + + # ---- Resolve or create the gitlab-s3 access key ---- + # garage v1.x outputs plaintext (not JSON) for key list / key info / key create. + # key list format: " GKxxxxxxxxx " + # key create/info format includes lines: + # "Key ID: GKxxxxxxxxx" + # "Secret key: " + local access_key secret_key existing_id key_output rotated_key_name key_list_output create_failure_reason + if ! garage_exec "garage key list" key list; then + garage_setup_failure "${garage_cmd_last_reason}" + return 0 + fi + key_list_output="$garage_cmd_output" + existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true) + + if [[ -n "$existing_id" ]]; then + log "Garage key '${GARAGE_S3_KEY_NAME}' already exists (id=${existing_id}); fetching info..." + if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then + garage_setup_failure "${garage_cmd_last_reason}" + return 0 + fi + key_output="$garage_cmd_output" + else + log "Creating Garage key '${GARAGE_S3_KEY_NAME}'..." + create_failure_reason="" + if ! garage_exec "garage key create ${GARAGE_S3_KEY_NAME}" key create "$GARAGE_S3_KEY_NAME"; then + create_failure_reason="$garage_cmd_last_reason" + fi + + if ! garage_exec "garage key list" key list; then + if [[ -n "$create_failure_reason" ]]; then + garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}" + else + garage_setup_failure "${garage_cmd_last_reason}" + fi + return 0 + fi + key_list_output="$garage_cmd_output" + existing_id=$(printf '%s\n' "$key_list_output" | awk -v name="${GARAGE_S3_KEY_NAME}" '$2 == name {print $1; exit}' || true) + + if [[ -z "$existing_id" ]]; then + if [[ -n "$create_failure_reason" ]]; then + garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}' (${create_failure_reason})" + else + garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'" + fi + return 0 + fi + + if ! garage_exec "garage key info --show-secret ${existing_id}" key info --show-secret "$existing_id"; then + if [[ -n "$create_failure_reason" ]]; then + garage_setup_failure "${garage_cmd_last_reason}; key create error: ${create_failure_reason}" + else + garage_setup_failure "${garage_cmd_last_reason}" + fi + return 0 + fi + key_output="$garage_cmd_output" + fi + + if [[ -z "$key_output" ]]; then + garage_setup_failure "Could not create/retrieve Garage key '${GARAGE_S3_KEY_NAME}'" + return 0 + fi + + # Parse plaintext output — match "Key ID: GKxxx" and "Secret key: xxx" lines. + # Also try JSON in case a future garage version changes format. + access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1) + secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1) + # JSON fallback + if [[ -z "$access_key" ]]; then + access_key=$(printf '%s' "$key_output" | python3 -c \ + "import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true) + fi + if [[ -z "$secret_key" ]]; then + secret_key=$(printf '%s' "$key_output" | python3 -c \ + "import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true) + fi + + if [[ "$secret_key" == "(redacted)" ]]; then + warn "Garage key '${GARAGE_S3_KEY_NAME}' secret is redacted; creating a deterministic fallback key for GitLab object storage." + rotated_key_name="${GARAGE_S3_KEY_NAME}-gitlab" + if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then + if ! garage_exec "garage key create ${rotated_key_name}" key create "$rotated_key_name"; then + garage_setup_failure "${garage_cmd_last_reason}" + return 0 + fi + if ! garage_exec "garage key info --show-secret ${rotated_key_name}" key info --show-secret "$rotated_key_name"; then + garage_setup_failure "${garage_cmd_last_reason}" + return 0 + fi + fi + key_output="$garage_cmd_output" + access_key=$(printf '%s' "$key_output" | sed -nE 's/^(Access key ID|Key ID):[[:space:]]+//p' | head -n1) + secret_key=$(printf '%s' "$key_output" | sed -nE 's/^(Secret access key|Secret key):[[:space:]]+//p' | head -n1) + if [[ -z "$access_key" ]]; then + access_key=$(printf '%s' "$key_output" | python3 -c \ + "import sys,json; k=json.load(sys.stdin); print(k.get('accessKeyId',''))" 2>/dev/null || true) + fi + if [[ -z "$secret_key" ]]; then + secret_key=$(printf '%s' "$key_output" | python3 -c \ + "import sys,json; k=json.load(sys.stdin); print(k.get('secretAccessKey',''))" 2>/dev/null || true) + fi + if [[ -n "$access_key" && -n "$secret_key" && "$secret_key" != "(redacted)" ]]; then + GARAGE_S3_KEY_NAME="$rotated_key_name" + log "Using rotated Garage key '${GARAGE_S3_KEY_NAME}'." + fi + fi + + if [[ -z "$access_key" || -z "$secret_key" || "$secret_key" == "(redacted)" ]]; then + garage_setup_failure "Malformed Garage key output for '${GARAGE_S3_KEY_NAME}' (missing access key or secret key)" + warn "Raw output: ${key_output}" + return 0 + fi + + # ---- Create buckets and grant permissions ---- + # --key takes the accessKeyId (not the name) in garage v1.x + # Bucket names must match GitLab Helm chart defaults because the GitLab CR's + # global.appConfig.object_store block only specifies the connection secret — + # it does NOT override per-object bucket names. Rails therefore writes to + # gitlab-uploads / gitlab-artifacts / etc. (no -storage suffix). A prior + # naming with a -storage suffix caused Excon::Error::NotFound / NoSuchBucket + # on first login when CarrierWave tried to upload the duo-bot avatar to + # /gitlab-uploads/user/avatar/8/duo-bot.png. Keep registry / terraform-state / + # ci-secure-files verbatim — those are the names both sides already agreed on. + local buckets=( + registry + gitlab-artifacts + gitlab-lfs + gitlab-uploads + gitlab-packages + gitlab-dependency-proxy + gitlab-terraform-state + gitlab-ci-secure-files + ) + for bucket in "${buckets[@]}"; do + if ! garage_exec "garage bucket create ${bucket}" bucket create "$bucket"; then + local bucket_create_failure_reason lower_bucket_create_failure_reason + bucket_create_failure_reason="${garage_cmd_last_reason}" + lower_bucket_create_failure_reason="${bucket_create_failure_reason,,}" + if [[ "$lower_bucket_create_failure_reason" == *"already exists"* ]]; then + log "Garage bucket '${bucket}' already exists; reusing existing bucket." + else + garage_setup_failure "${bucket_create_failure_reason}" + return 0 + fi + fi + # bucket name is positional; --read/--write/--owner are boolean flags + if ! garage_exec "garage bucket allow ${bucket}" bucket allow "$bucket" --key "$access_key" \ + --read --write --owner; then + garage_setup_failure "${garage_cmd_last_reason}" + return 0 + fi + done + log "Garage buckets provisioned for GitLab." + + # ---- Write k8s secrets consumed by the GitLab CR ---- + # Rails object storage connection (artifacts, LFS, uploads, packages, etc.) + local object_storage_secret_apply_output registry_storage_secret_apply_output + object_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-object-storage \ + --from-literal=connection="provider: AWS +region: garage +aws_access_key_id: ${access_key} +aws_secret_access_key: ${secret_key} +endpoint: '${GARAGE_S3_ENDPOINT}' +path_style: true" \ + --dry-run=client -o yaml | kubectl apply -f - 2>&1)" + if kubectl_apply_reports_changed "$object_storage_secret_apply_output"; then + gitlab_object_storage_secret_changed=1 + fi + log "gitlab-object-storage secret applied (${object_storage_secret_apply_output})." + + # Docker registry S3 storage driver config. + # Use s3_v2 (AWS SDK v2) — s3_v1 has signature computation issues with + # non-standard ports (Host: endpoint:3900) against Garage. SDK v2 always + # uses SigV4; v4auth is not a recognised key and must be omitted. + registry_storage_secret_apply_output="$(kubectl -n "$NAMESPACE" create secret generic gitlab-registry-storage \ + --from-literal=config="s3_v2: + accesskey: ${access_key} + secretkey: ${secret_key} + bucket: registry + regionendpoint: ${GARAGE_S3_ENDPOINT} + region: garage + pathstyle: true" \ + --dry-run=client -o yaml | kubectl apply -f - 2>&1)" + if kubectl_apply_reports_changed "$registry_storage_secret_apply_output"; then + gitlab_registry_storage_secret_changed=1 + fi + log "gitlab-registry-storage secret applied (${registry_storage_secret_apply_output})." +} + +# --------------------------------------------------------------------------- +# Legacy/local static PV + synology directory setup for gitaly +# minio PV removed — object storage is provided by knoe-system/garage +# --------------------------------------------------------------------------- +setup_gitlab_legacy_storage() { + local synology_node="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}" + local storage_selector_key="${STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}" + local base_path="${GITLAB_STORAGE_BASE:-/synology/d005/gitlab}" + local gitaly_size="${GITLAB_GITALY_PV_SIZE:-50Gi}" + + [[ -n "$synology_node" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set for legacy GitLab storage setup." + + log "Creating synology gitaly directory on ${synology_node}..." + kubectl -n "$NAMESPACE" run gitlab-dirprep \ + --image=alpine:latest --restart=Never --rm --attach \ + --overrides="{\"spec\":{\"nodeName\":\"${synology_node}\",\"tolerations\":[{\"operator\":\"Exists\"}],\"volumes\":[{\"name\":\"s\",\"hostPath\":{\"path\":\"$(dirname ${base_path})\",\"type\":\"Directory\"}}],\"containers\":[{\"name\":\"dirprep\",\"image\":\"alpine:latest\",\"command\":[\"sh\",\"-c\",\"mkdir -p ${base_path}/gitaly && chmod 777 ${base_path}/gitaly && echo done\"],\"volumeMounts\":[{\"name\":\"s\",\"mountPath\":\"$(dirname ${base_path})\"}]}]}}" \ + 2>&1 || warn "Could not create synology dirs (may already exist)" + + # A named no-provisioner StorageClass is required so the helm chart renders + # storageClassName in the PVC template (an empty string is treated as falsy + # by the chart and omitted, causing the PVC to use the cluster default). + log "Ensuring gitlab-gitaly-static StorageClass (no-provisioner)..." + kubectl apply -f - </dev/null 2>&1; then has_legacy=1; fi + + local gitaly_sts_name="${GITLAB_RELEASE}-gitaly" + local gitaly_sts_yaml + gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true) + if [[ -n "$gitaly_sts_yaml" ]]; then + if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* || "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then + has_legacy=1 + fi + fi + + # Also treat PVC as legacy only when its actual storageClassName is a + # known legacy class — not merely because the object exists. + local pvc_sc + pvc_sc=$(kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 \ + -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + if [[ "$pvc_sc" == "gitlab-gitaly-static" || "$pvc_sc" == "synology-iscsi" ]]; then + has_legacy=1 + fi + + if [[ "$has_legacy" == "1" ]]; then + if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then + export GITALY_AUTOCLEAN_PERFORMED=1 + log "AUTOCLEAN: repairing legacy Gitaly storage (scaling down and deleting legacy PV/PVC)..." + kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true + kubectl -n "$NAMESPACE" delete pvc repo-data-gitlab-gitaly-0 --wait=false 2>/dev/null || true + kubectl delete pv gitlab-gitaly-synology --wait=false 2>/dev/null || true + # Give it a moment to process deletions + sleep 2 + else + repair_blocked "Legacy Gitaly PV/PVC still bound to gitlab-gitaly-static / gandalf.prole.org" \ + "Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to automatically repair by deleting legacy PVC/PV, or run: +kubectl -n $NAMESPACE scale sts ${GITLAB_RELEASE}-gitaly --replicas=0 +kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0 +kubectl delete pv gitlab-gitaly-synology" + fi + fi + + repair_stale_gke_gitaly_dynamic_storage "${GITALY_STORAGE_CLASS:-}" + else + setup_gitlab_legacy_storage + fi + + # Object storage (registry, artifacts, LFS, uploads, etc.) is provided by + # knoe-system/garage — credentials and buckets are set up here. + setup_garage_for_gitlab +} + +configure_jemalloc_hostpath_mode +setup_jemalloc_on_nodes +check_gitlab_pre_apply_blocked +setup_gitlab_storage + +log "Pre-creating gitlab-app-nonroot ServiceAccount (required by chart v9+)..." +kubectl -n "${NAMESPACE}" apply -f - </dev/null || true)" +gitlab_exists_before=0 +if [[ -n "$gitlab_generation_before" ]]; then + gitlab_exists_before=1 +fi + +GITLAB_CR_RENDERED="$(cat </dev/null || true)" +gitlab_apply_output="$(echo "$GITLAB_CR_RENDERED" | kubectl apply -f -)" +gitlab_generation_after="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)" + +gitlab_spec_changed=1 +if [[ -n "$gitlab_generation_before" && -n "$gitlab_generation_after" && "$gitlab_generation_before" == "$gitlab_generation_after" ]]; then + gitlab_spec_changed=0 +fi +gitlab_apply_changed=0 +if kubectl_apply_reports_changed "$gitlab_apply_output"; then + if [[ "$MODE" == "k8s" && "$gitlab_spec_changed" == "0" ]]; then + # Some apply operations may report "configured" without changing generation. + # In that case, avoid triggering a full reconcile solely from apply output text. + log "GitLab CR apply reported change, but generation is unchanged. Skipping drift trigger." + else + gitlab_apply_changed=1 + fi +fi + +gitlab_config_changed=0 +if [[ "$gitlab_db_secret_changed" == "1" || "$gitlab_object_storage_secret_changed" == "1" || "$gitlab_registry_storage_secret_changed" == "1" ]]; then + gitlab_config_changed=1 +fi + +gitlab_reconcile_required=0 +# Check actual replicas to ensure they match desired count (1) - Requirement 3 +gitlab_replica_drift=0 +for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2" "webservice-default"; do + _dep_name="${GITLAB_RELEASE}-${_dep_suffix}" + _actual=$(kubectl -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1") + if [[ "$_actual" != "1" ]]; then + log "Detected replica drift for ${_dep_name}: actual=${_actual}, desired=1" + gitlab_replica_drift=1 + break + fi +done + +if [[ "$gitlab_exists_before" == "0" || "$operator_changed" == "1" || "$gitlab_chart_version_changed" == "1" || "$gitlab_spec_changed" == "1" || "$gitlab_apply_changed" == "1" || "$gitlab_config_changed" == "1" || "$gitlab_replica_drift" == "1" ]]; then + gitlab_reconcile_required=1 +fi + +check_gitlab_post_apply_blocked + +# minio is disabled (global.minio.enabled: false) — no ARM64 credential fix needed. + +# --------------------------------------------------------------------------- +# ARM64 configure check: GitLab v17+ (chart v8+) ships multi-arch CNG images +# that include ARM64 variants. The old workaround of replacing the configure +# init container with alpine:latest is no longer needed and breaks things +# because /templates/configure is baked into the gitlab-base image, not +# mounted from a ConfigMap. +# This function is kept as a no-op to avoid breaking callers; remove entirely +# once the chart version floor is confirmed stable at v9+. +# --------------------------------------------------------------------------- +fix_registry_arm64_configure() { + log "Skipping ARM64 configure patch — GitLab v18 CNG images are multi-arch natively." +} + +fix_registry_arm64_configure + +log "GitLab CR applied — operator is reconciling (this may take 10-20 minutes)." +log "Monitor progress: kubectl -n ${NAMESPACE} get gitlab ${GITLAB_RELEASE} -w" +log "Watch pods: kubectl -n ${NAMESPACE} get pods -w" + +# --------------------------------------------------------------------------- +# Optionally wait for GitLab to become available +# --------------------------------------------------------------------------- +WAIT_TIMEOUT="${GITLAB_WAIT_TIMEOUT:-1200}" # default 20 minutes +if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then + if [[ "$gitlab_reconcile_required" == "0" ]]; then + log "Fast-path: no GitLab reconcile triggers detected; skipping long wait." + gitlab_available_status="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{range .status.conditions[?(@.type=="Available")]}{.status}{end}' 2>/dev/null || true)" + if [[ "$gitlab_available_status" == "True" ]]; then + log "Short health check OK: GitLab CR condition Available=True." + check_gitlab_post_apply_blocked + else + warn "Short health check: GitLab CR Available condition is '${gitlab_available_status:-unknown}'." + kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" >/dev/null 2>&1 || true + fi + else + log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status (checking for blocked states every 30s)..." + _wait_start=$(date +%s) + while true; do + if kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \ + --for=condition=Available \ + --timeout=30s >/dev/null 2>&1; then + log "GitLab CR condition Available=True." + break + fi + + check_gitlab_post_apply_blocked + + _now=$(date +%s) + if (( _now - _wait_start >= WAIT_TIMEOUT )); then + _ctx=$(get_gitlab_blocker_context) + repair_blocked "Timed out waiting for GitLab CR condition=Available" \ + "The deployment is taking too long. Context: ${_ctx}Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}" + fi + done + fi +fi + +# --------------------------------------------------------------------------- +# Requirement: Explicitly scale down over-replicated GitLab deployments +# Some components may remain at 2 replicas even after CR is Available. +# --------------------------------------------------------------------------- +log "Verifying GitLab deployment replica counts (desired=1)..." +_gitlab_replica_target=1 + +if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then + gitlab_verify_replica_source_of_truth "$_gitlab_replica_target" +fi + +gitlab_post_remediation_success=0 +_replica_drift_found=0 +_corrected_deployments=() +_drifted_deployments=() +# Explicit list of components to verify and scale if needed (Requirement 2) +for _dep_suffix in "gitlab-shell" "kas" "registry" "sidekiq-all-in-1-v2"; do + _dep_name="${GITLAB_RELEASE}-${_dep_suffix}" + + # Read live spec.replicas (Requirement 2 & 6) + _live_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + if [[ -z "$_live_replicas" || ! "$_live_replicas" =~ ^[0-9]+$ ]]; then + _live_replicas=0 + fi + + if [[ "$_live_replicas" != "$_gitlab_replica_target" ]]; then + log "Detected replica drift for ${_dep_name}: spec=${_live_replicas}, desired=${_gitlab_replica_target}" + _replica_drift_found=1 + _drifted_deployments+=("$_dep_name") + else + log "Deployment ${_dep_name} replica count is correct: spec=${_live_replicas}, desired=${_gitlab_replica_target}" + fi +done + +if [[ "$_replica_drift_found" == "1" ]]; then + log "Replica drift detected. Re-applying GitLab CR source-of-truth (gitlab-shell/kas/registry/sidekiq desired=${_gitlab_replica_target})." + if ! echo "$GITLAB_CR_RENDERED" | kubectl --context "$KUBECTL_CONTEXT" apply -f - >/dev/null; then + repair_blocked "Failed to re-apply GitLab CR for replica drift remediation" \ + "Could not reconcile source-of-truth replica values." + fi + + gitlab_verify_replica_source_of_truth "$_gitlab_replica_target" + + _source_of_truth_mismatch_report="" + + for _dep_name in "${_drifted_deployments[@]}"; do + _post_reconcile_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + if [[ -z "$_post_reconcile_replicas" || ! "$_post_reconcile_replicas" =~ ^[0-9]+$ ]]; then + _post_reconcile_replicas=0 + fi + + _corrected_deployments+=("$_dep_name") + if [[ "$_post_reconcile_replicas" != "$_gitlab_replica_target" ]]; then + _source_of_truth_mismatch_report+="- ${_dep_name}: spec=${_post_reconcile_replicas}, desired=${_gitlab_replica_target}"$'\n' + fi + log "Deployment ${_dep_name} after CR reconcile: spec=${_post_reconcile_replicas}, desired=${_gitlab_replica_target}." + done + + if [[ -n "$_source_of_truth_mismatch_report" ]]; then + _rendered_replica_fields_report="$(gitlab_rendered_replica_source_fields_from_cr "${GITLAB_CR_RENDERED:-}")" + if [[ -z "$_rendered_replica_fields_report" ]]; then + _rendered_replica_fields_report="- (no replica-related fields detected in rendered CR values)" + fi + repair_blocked "GitLab operator desired replica source-of-truth mismatch after CR re-apply" \ + "Deployment specs still do not match desired=${_gitlab_replica_target}:\n${_source_of_truth_mismatch_report}Rendered GitLab CR replica source fields:\n${_rendered_replica_fields_report}\nThis indicates CR source-of-truth still resolves to replicas>1." + fi + + log "Waiting for rollout of reconciled deployments..." + _remediation_rollout_failed=0 + for _dep_name in "${_corrected_deployments[@]}"; do + if ! kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" rollout status deployment/"$_dep_name" --timeout=300s; then + warn "Rollout status check did not report success for ${_dep_name}; final replica state will decide pass/fail." + _remediation_rollout_failed=1 + fi + done + + # Final verification with settle window to tolerate transient old pod linger + _settle_timeout_s="${GITLAB_POST_REMEDIATION_SETTLE_TIMEOUT:-90}" + _settle_poll_interval_s="${GITLAB_POST_REMEDIATION_SETTLE_POLL_INTERVAL:-5}" + if [[ -z "$_settle_timeout_s" || ! "$_settle_timeout_s" =~ ^[0-9]+$ || "$_settle_timeout_s" == "0" ]]; then + _settle_timeout_s=90 + fi + if [[ -z "$_settle_poll_interval_s" || ! "$_settle_poll_interval_s" =~ ^[0-9]+$ || "$_settle_poll_interval_s" == "0" ]]; then + _settle_poll_interval_s=5 + fi + + log "Post-remediation settle verification for corrected deployments (timeout=${_settle_timeout_s}s, poll=${_settle_poll_interval_s}s):" + _remediation_final_failed=0 + _remediation_failure_class="" + _remediation_failure_report="" + if (( ${#_corrected_deployments[@]} > 0 )); then + _settle_start_ts=$(date +%s) + _settle_had_errexit=0 + if [[ $- == *e* ]]; then + _settle_had_errexit=1 + set +e + fi + _settle_iteration=0 + while true; do + _settle_iteration=$((_settle_iteration + 1)) + _remediation_failure_report="" + _iteration_blocked=0 + _source_of_truth_reversion=0 + for _dep_name in "${_corrected_deployments[@]}"; do + _target_desired_replicas="${_gitlab_replica_target}" + _desired_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + if [[ -z "$_desired_replicas" || ! "$_desired_replicas" =~ ^[0-9]+$ ]]; then + _desired_replicas=0 + fi + + _status_replicas=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.replicas}' 2>/dev/null || true) + _status_ready=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true) + _status_available=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true) + _status_updated=$(kubectl --context "$KUBECTL_CONTEXT" -n "$NAMESPACE" get deployment "$_dep_name" -o jsonpath='{.status.updatedReplicas}' 2>/dev/null || true) + _status_replicas=${_status_replicas:-0} + _status_ready=${_status_ready:-0} + _status_available=${_status_available:-0} + _status_updated=${_status_updated:-0} + + _dep_selector=$(gitlab_selector_for_deployment "$_dep_name" 2>/dev/null || true) + if [[ -n "$_dep_selector" ]]; then + _final_live=$(gitlab_non_terminal_pod_count_for_app "$_dep_selector" 2>/dev/null || true) + if [[ -z "$_final_live" || ! "$_final_live" =~ ^[0-9]+$ ]]; then + _final_live=0 + fi + _matching_pods=$(gitlab_non_terminal_pod_names_for_app "$_dep_selector" 2>/dev/null || true) + else + _final_live=0 + _matching_pods="" + fi + if [[ -z "$_matching_pods" ]]; then + _matching_pods="(none)" + fi + + log "Settle poll #${_settle_iteration}: deployment ${_dep_name} selector=${_dep_selector:-} spec=${_desired_replicas}, desired=${_target_desired_replicas}, live=${_final_live}, status(replicas/ready/available/updated)=${_status_replicas}/${_status_ready}/${_status_available}/${_status_updated}, pods=[${_matching_pods}]" + + if [[ -z "$_final_live" || ! "$_final_live" =~ ^[0-9]+$ || "$_desired_replicas" != "$_target_desired_replicas" || "$_final_live" != "$_target_desired_replicas" ]]; then + _iteration_blocked=1 + if [[ "$_desired_replicas" != "$_target_desired_replicas" ]]; then + _source_of_truth_reversion=1 + fi + _remediation_failure_report+="- ${_dep_name}: spec=${_desired_replicas}, final_live=${_final_live:-unknown}, desired=${_target_desired_replicas}, selector=${_dep_selector:-}, status=${_status_replicas}/${_status_ready}/${_status_available}/${_status_updated}, pods=[${_matching_pods}]"$'\n' + fi + done + + if [[ "$_source_of_truth_reversion" == "1" ]]; then + _remediation_final_failed=1 + _remediation_failure_class="source_of_truth" + break + fi + + if [[ "$_iteration_blocked" == "0" ]]; then + break + fi + + _settle_now_ts=$(date +%s) + if (( _settle_now_ts - _settle_start_ts >= _settle_timeout_s )); then + _remediation_final_failed=1 + _remediation_failure_class="convergence" + break + fi + + sleep "$_settle_poll_interval_s" + done + if [[ "$_settle_had_errexit" == "1" ]]; then + set -e + fi + fi + + if [[ "$_remediation_final_failed" == "1" ]]; then + if [[ "$_remediation_failure_class" == "source_of_truth" ]]; then + repair_blocked "GitLab operator reverted deployment spec.replicas after source-of-truth reconcile" \ + "Operator-managed desired state diverged during settle verification:\n${_remediation_failure_report}This is a source-of-truth failure (CR values path), not a rollout lag issue." + else + repair_blocked "GitLab post-remediation replica verification failed" \ + "Source-of-truth reconcile ran, but final convergence did not meet required conditions (spec.replicas==desired and live selector pod count==desired):\n${_remediation_failure_report}Check: kubectl -n ${NAMESPACE} get deploy -l app.kubernetes.io/instance=${GITLAB_RELEASE}" + fi + fi + + gitlab_replica_drift=0 + gitlab_reconcile_required=0 + gitlab_post_remediation_success=1 + if [[ "$_remediation_rollout_failed" == "1" ]]; then + log "GitLab corrective action converged by final replica state (spec and live selector counts match desired=${_gitlab_replica_target})." + else + log "GitLab corrective action succeeded: rollout completed and all spec/live selector counts match desired=${_gitlab_replica_target}." + fi +fi + +if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then + if [[ "${gitlab_post_remediation_success:-0}" == "1" ]]; then + log "GitLab corrective action success is authoritative; skipping additional post-remediation convergence gate." + else + wait_for_gitlab_workload_convergence + fi +else + warn "GITLAB_NO_WAIT=1, skipping strict GitLab workload convergence checks." +fi + +# --------------------------------------------------------------------------- +# Public ingress for configured GitLab domain -> gitlab-webservice +# The GitLab Operator creates the Ingress; this block ensures a stable public +# class/host mapping if the CR-managed ingress is absent or not usable. +# --------------------------------------------------------------------------- +WEBSERVICE_SVC="${GITLAB_RELEASE}-webservice-default" + +GITLAB_INGRESS_RULES_YAML="" +for _gitlab_host in "${GITLAB_PUBLIC_HOSTS[@]}"; do + GITLAB_INGRESS_RULES_YAML+=$'\n'" - host: ${_gitlab_host} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ${WEBSERVICE_SVC} + port: + number: 8181" +done +unset _gitlab_host + +GITLAB_AUTH_ANNOTATIONS_YAML="" +case "${FRONTDOOR_AUTH_ENABLED}" in + 1|true|TRUE|True|yes|YES|Yes|on|ON|On) + GITLAB_AUTH_ANNOTATIONS_YAML=$(cat <HTTPS redirect). We create both alongside the +# Ingress and wire them up via annotations. No-op for nginx/kong classes. +# +# We also create a BackendConfig so GCLB's health check hits +# gitlab-workhorse's `/-/readiness` endpoint instead of `/`. +# The default `/` check returns 302 (workhorse redirects to /users/sign_in), +# which GCLB interprets as an unhealthy backend — the backend flaps between +# UNHEALTHY/HEALTHY and the ingress surfaces as HTTP 502. +# The Service is annotated with cloud.google.com/backend-config below so +# GCLB picks the BackendConfig up for the workhorse port (8181). +GITLAB_GCE_TLS_ANNOTATIONS_YAML="" +GITLAB_MANAGED_CERT_NAME="gitlab-managed-cert" +GITLAB_FRONTEND_CONFIG_NAME="gitlab-frontend-config" +GITLAB_BACKEND_CONFIG_NAME="gitlab-webservice-backendconfig" +# Optional: pin the GCE L7 ingress to a pre-reserved global external static IP +# (gcloud compute addresses create --global). Prevents IP churn on ingress +# delete/recreate. Leave blank to let GCE assign ephemerally. +GITLAB_GLOBAL_STATIC_IP_NAME="${GITLAB_GLOBAL_STATIC_IP_NAME:-}" +if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then + GITLAB_GCE_TLS_ANNOTATIONS_YAML=$(cat </dev/null 2>&1; then + die "ManagedCertificate ${NAMESPACE}/${GITLAB_MANAGED_CERT_NAME} was not found after apply — cannot safely apply GCE ingress." + fi + if ! kubectl -n "$NAMESPACE" get frontendconfig "$GITLAB_FRONTEND_CONFIG_NAME" >/dev/null 2>&1; then + die "FrontendConfig ${NAMESPACE}/${GITLAB_FRONTEND_CONFIG_NAME} was not found after apply — cannot safely apply GCE ingress." + fi + log "Confirmed: ManagedCertificate/${GITLAB_MANAGED_CERT_NAME} and FrontendConfig/${GITLAB_FRONTEND_CONFIG_NAME} exist in ns=${NAMESPACE}." + + # Annotate the webservice Service so GCLB picks up the BackendConfig. + # The GitLab Operator owns this Service, so we apply the annotation + # out-of-band. It is additive (not owned by the chart) and survives + # operator reconciles. + _webservice_svc="${GITLAB_RELEASE}-webservice-default" + if kubectl -n "$NAMESPACE" get svc "$_webservice_svc" >/dev/null 2>&1; then + log "Annotating Service ${_webservice_svc} with cloud.google.com/backend-config=${GITLAB_BACKEND_CONFIG_NAME}..." + kubectl -n "$NAMESPACE" annotate svc "$_webservice_svc" \ + "cloud.google.com/backend-config={\"default\":\"${GITLAB_BACKEND_CONFIG_NAME}\"}" \ + --overwrite >/dev/null || warn "Failed to annotate ${_webservice_svc}; will retry after CR reconcile." + else + log "Service ${_webservice_svc} not present yet; BackendConfig annotation will be (re-)applied after the CR reconcile." + fi + unset _webservice_svc +fi + +# Diagnostics: show current ingress, managedcertificate, and frontendconfig state before reconcile. +if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then + log "--- GCE ingress diagnostics (pre-apply) ---" + kubectl get ingress -A --no-headers 2>/dev/null || true + log "ManagedCertificates:" + kubectl get managedcertificate -A --no-headers 2>/dev/null || true + log "FrontendConfigs:" + kubectl get frontendconfig -A --no-headers 2>/dev/null || true + # Show current annotations for each expected ingress before patch. + for _diag_ing in "${GITLAB_FALLBACK_INGRESS_NAME}" "${GITLAB_RELEASE}-webservice-default"; do + if kubectl -n "$NAMESPACE" get ingress "$_diag_ing" >/dev/null 2>&1; then + log "Current annotations for ${NAMESPACE}/${_diag_ing} (pre-apply):" + kubectl -n "$NAMESPACE" get ingress "$_diag_ing" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true + echo + fi + done + unset _diag_ing + log "---" +fi + +log "Reconciling GitLab front-door owner='${GITLAB_FRONTDOOR_OWNER}' (ingressClass=${GITLAB_INGRESS_CLASS}) for hosts=${_gitlab_hosts_csv} -> ${WEBSERVICE_SVC}:8181 ..." +log "Rendered GitLab ingress (pre-apply): ns=${NAMESPACE} owner=${GITLAB_FRONTDOOR_OWNER} class=${GITLAB_INGRESS_CLASS} managedCert=${GITLAB_MANAGED_CERT_NAME} frontendConfig=${GITLAB_FRONTEND_CONFIG_NAME} preSharedCert=- hosts=${_gitlab_hosts_csv} backend=${WEBSERVICE_SVC}:8181" + +if [[ "$GITLAB_FRONTDOOR_OWNER" == "operator" ]]; then + if assert_unique_ingress_host_claims "${GITLAB_RELEASE}-webservice-default" "$NAMESPACE" "$_gitlab_hosts_csv" "$GITLAB_RELEASE" "$WEBSERVICE_SVC"; then + : + else + _ingress_claim_rc=$? + case "$_ingress_claim_rc" in + 10) + log "GitLab operator ingress host ownership confirmed in namespace '${NAMESPACE}': ${INGRESS_HOST_CLAIM_DETAILS}" + ;; + *) + die "Duplicate ingress host/path claim detected for GitLab operator ingress '${NAMESPACE}/${GITLAB_RELEASE}-webservice-default': ${INGRESS_HOST_CLAIM_DETAILS}" + ;; + esac + fi + + if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then + _operator_ingress_name="${GITLAB_RELEASE}-webservice-default" + if kubectl -n "$NAMESPACE" get ingress "$_operator_ingress_name" >/dev/null 2>&1; then + log "Annotating ingress ${_operator_ingress_name} with ManagedCertificate=${GITLAB_MANAGED_CERT_NAME} and FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}..." + kubectl -n "$NAMESPACE" annotate ingress "$_operator_ingress_name" \ + "networking.gke.io/managed-certificates=${GITLAB_MANAGED_CERT_NAME}" \ + "networking.gke.io/v1beta1.FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}" \ + ingress.gcp.kubernetes.io/pre-shared-cert- \ + --overwrite >/dev/null || warn "Failed to annotate ${_operator_ingress_name} with GCE ingress TLS annotations." + else + warn "Operator ingress ${_operator_ingress_name} not found yet; cannot apply GCE ingress annotations in this pass." + fi + unset _operator_ingress_name + else + _operator_ingress_name="${GITLAB_RELEASE}-webservice-default" + if kubectl -n "$NAMESPACE" get ingress "$_operator_ingress_name" >/dev/null 2>&1; then + log "Removing stale GCE TLS annotations from operator ingress ${_operator_ingress_name} (class=${GITLAB_INGRESS_CLASS})..." + kubectl -n "$NAMESPACE" annotate ingress "$_operator_ingress_name" \ + networking.gke.io/managed-certificates- \ + networking.gke.io/v1beta1.FrontendConfig- \ + ingress.gcp.kubernetes.io/pre-shared-cert- \ + --overwrite >/dev/null 2>&1 || true + fi + unset _operator_ingress_name + fi + + if kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then + log "Deleting stale fallback ingress ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} to enforce single front-door ownership (${GITLAB_FRONTDOOR_OWNER})." + kubectl -n "$NAMESPACE" delete ingress "$GITLAB_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || \ + warn "Failed to delete stale fallback ingress ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME}." + fi + if [[ "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" != "$GITLAB_FALLBACK_INGRESS_NAME" ]] && kubectl -n "$NAMESPACE" get ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then + log "Deleting legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME} to avoid owner/controller ambiguity." + kubectl -n "$NAMESPACE" delete ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || \ + warn "Failed to delete legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME}." + fi +else + if assert_unique_ingress_host_claims "$GITLAB_FALLBACK_INGRESS_NAME" "$NAMESPACE" "$_gitlab_hosts_csv" "$GITLAB_RELEASE" "$WEBSERVICE_SVC"; then + : + else + _ingress_claim_rc=$? + case "$_ingress_claim_rc" in + 10) + log "GitLab-managed ingress already owns host/path in namespace '${NAMESPACE}': ${INGRESS_HOST_CLAIM_DETAILS}" + _operator_ingress_name="${GITLAB_RELEASE}-webservice-default" + if kubectl -n "$NAMESPACE" get ingress "$_operator_ingress_name" >/dev/null 2>&1; then + log "Fallback front-door owner selected; deleting stale operator ingress ${NAMESPACE}/${_operator_ingress_name}." + kubectl -n "$NAMESPACE" delete ingress "$_operator_ingress_name" --ignore-not-found >/dev/null || \ + warn "Failed to delete stale operator ingress ${NAMESPACE}/${_operator_ingress_name}." + fi + unset _operator_ingress_name + ;; + *) + die "Duplicate ingress host/path claim detected for GitLab fallback ingress '${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME}': ${INGRESS_HOST_CLAIM_DETAILS}" + ;; + esac + fi + + if [[ "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" != "$GITLAB_FALLBACK_INGRESS_NAME" ]] && kubectl -n "$NAMESPACE" get ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then + log "Deleting legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME} before applying ${GITLAB_FALLBACK_INGRESS_NAME}." + kubectl -n "$NAMESPACE" delete ingress "$GITLAB_LEGACY_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || \ + warn "Failed to delete legacy fallback ingress ${NAMESPACE}/${GITLAB_LEGACY_FALLBACK_INGRESS_NAME}." + fi + + if kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" >/dev/null 2>&1; then + _live_spec_class="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.spec.ingressClassName}' 2>/dev/null || true)" + _live_ann_class="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.kubernetes\.io/ingress\.class}' 2>/dev/null || true)" + _live_class="${_live_spec_class:-$_live_ann_class}" + _live_managed="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.networking\.gke\.io/managed-certificates}' 2>/dev/null || true)" + _live_frontend="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.networking\.gke\.io/v1beta1\.FrontendConfig}' 2>/dev/null || true)" + _live_pre_shared="$(kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations.ingress\.gcp\.kubernetes\.io/pre-shared-cert}' 2>/dev/null || true)" + _replace_reason="" + _patch_only=0 + if [[ -n "$_live_class" && "$_live_class" != "$GITLAB_INGRESS_CLASS" ]]; then + # ingressClass change requires recreation (immutable field). + _replace_reason="ingressClass drift (live=${_live_class}, desired=${GITLAB_INGRESS_CLASS})" + elif [[ "$GITLAB_INGRESS_CLASS" == "gce" ]]; then + # Requirement 2 & 3: never delete/recreate to clear stale cert annotations. + # Patch annotations in place instead. + if [[ -n "$_live_pre_shared" ]]; then + log "Patching stale pre-shared-cert annotation from ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} in place (was: ${_live_pre_shared})." + kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \ + ingress.gcp.kubernetes.io/pre-shared-cert- \ + "networking.gke.io/managed-certificates=${GITLAB_MANAGED_CERT_NAME}" \ + "networking.gke.io/v1beta1.FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}" \ + --overwrite >/dev/null 2>&1 || warn "Failed to patch pre-shared-cert annotation from ${GITLAB_FALLBACK_INGRESS_NAME}." + _patch_only=1 + elif [[ -n "$_live_managed" && "$_live_managed" != "$GITLAB_MANAGED_CERT_NAME" ]]; then + log "Patching managed certificate annotation on ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} in place (live=${_live_managed}, desired=${GITLAB_MANAGED_CERT_NAME})." + kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \ + "networking.gke.io/managed-certificates=${GITLAB_MANAGED_CERT_NAME}" \ + --overwrite >/dev/null 2>&1 || warn "Failed to patch managed-certificates annotation on ${GITLAB_FALLBACK_INGRESS_NAME}." + _patch_only=1 + elif [[ -n "$_live_frontend" && "$_live_frontend" != "$GITLAB_FRONTEND_CONFIG_NAME" ]]; then + log "Patching frontend config annotation on ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} in place (live=${_live_frontend}, desired=${GITLAB_FRONTEND_CONFIG_NAME})." + kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \ + "networking.gke.io/v1beta1.FrontendConfig=${GITLAB_FRONTEND_CONFIG_NAME}" \ + --overwrite >/dev/null 2>&1 || warn "Failed to patch FrontendConfig annotation on ${GITLAB_FALLBACK_INGRESS_NAME}." + _patch_only=1 + fi + fi + if [[ -n "$_replace_reason" && "$_patch_only" -eq 0 ]]; then + log "Ingress shape change detected for ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME}: ${_replace_reason}. Replacing ingress (host/rule shape change)." + kubectl -n "$NAMESPACE" delete ingress "$GITLAB_FALLBACK_INGRESS_NAME" --ignore-not-found >/dev/null || true + fi + unset _live_spec_class _live_ann_class _live_class _live_managed _live_frontend _live_pre_shared _replace_reason _patch_only + fi + + kubectl apply -f - </dev/null 2>&1 || true + else + kubectl -n "$NAMESPACE" annotate ingress "$GITLAB_FALLBACK_INGRESS_NAME" \ + networking.gke.io/managed-certificates- \ + networking.gke.io/v1beta1.FrontendConfig- \ + ingress.gcp.kubernetes.io/pre-shared-cert- \ + --overwrite >/dev/null 2>&1 || true + fi + + # Diagnostics: show annotations after apply. + if [[ "${GITLAB_INGRESS_CLASS}" == "gce" ]]; then + log "Annotations for ${NAMESPACE}/${GITLAB_FALLBACK_INGRESS_NAME} (post-apply):" + kubectl -n "$NAMESPACE" get ingress "$GITLAB_FALLBACK_INGRESS_NAME" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true + echo + kubectl -n "$NAMESPACE" describe ingress "$GITLAB_FALLBACK_INGRESS_NAME" 2>/dev/null || true + fi +fi + +# --------------------------------------------------------------------------- +# Registry migration: registry:2 (knoe-system) → gitlab-registry +# Runs unattended after GitLab is Ready. Delegates to init_registry.sh +# migrate which uses skopeo when available, otherwise prints commands. +# Skipped if the old registry:2 has no images or is already gone. +# Set SKIP_REGISTRY_MIGRATE=1 to suppress. +# --------------------------------------------------------------------------- +if [[ "${SKIP_REGISTRY_MIGRATE:-0}" != "1" ]]; then + _init_registry_sh="${SCRIPT_DIR}/init_registry.sh" + if [[ -x "$_init_registry_sh" ]]; then + log "--- Registry migration: registry:2 → gitlab-registry ---" + # Pass the gitlab namespace so the migrate action knows where to send images. + GITLAB_NAMESPACE="$NAMESPACE" \ + REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}" \ + "$_init_registry_sh" migrate || \ + warn "Registry migration encountered errors — check output above." + else + warn "init_registry.sh not found at ${_init_registry_sh}; skipping registry migration." + warn "Run manually: ./etc/init_registry.sh migrate" + fi + unset _init_registry_sh +fi + +log "Done." +log "GitLab will be reachable at hosts=${_gitlab_hosts_csv} once pods are Running." +log "Initial root password: kubectl -n ${NAMESPACE} get secret ${GITLAB_RELEASE}-gitlab-initial-root-password -o jsonpath='{.data.password}' | base64 -d" diff --git a/mock_val/init_grafana_oauth.sh b/mock_val/init_grafana_oauth.sh new file mode 100755 index 0000000..a7f5b2b --- /dev/null +++ b/mock_val/init_grafana_oauth.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# init_grafana_oauth.sh +# +# Bootstrap the Google-OAuth secret that kps-grafana mounts as env vars +# (GF_AUTH_GOOGLE_CLIENT_ID / GF_AUTH_GOOGLE_CLIENT_SECRET) for its native +# auth.google sign-in flow. Companion to: +# - deploy/gcp/gke/grafana-google-oidc-secret.example.yaml (envsubst template) +# - monitoring/kps-values-gke.yaml (Helm overrides for grafana subchart) +# +# Auth model: +# - Anyone in the @knoey.com Workspace can sign in (auth.google.allowed_domains). +# - chrisfu@knoey.com + ron@knoey.com get Admin (role_attribute_path JMESPath). +# - Everyone else @knoey.com gets Editor. +# +# Usage: +# ./etc/init_grafana_oauth.sh +# +# Env vars (resolved from etc/secrets/* if not set in the shell): +# GRAFANA_GOOGLE_CLIENT_ID ← from etc/secrets/grafana-google-oidc-client-id +# GRAFANA_GOOGLE_CLIENT_SECRET ← from etc/secrets/grafana-google-oidc-client-secret +# +# Optional: +# APP_CLUSTER_KUBECONTEXT (default: $KUBECONTEXT then ambient) +# NAMESPACE (default: monitoring) +# +# Pre-reqs: +# - OAuth 2.0 client created at GCP Console (see the secret template +# deploy/gcp/gke/grafana-google-oidc-secret.example.yaml for the +# exact authorized redirect URI + consent screen settings). +# - Two values saved into etc/secrets/grafana-google-oidc-client-{id,secret} +# (chmod 0600 each; etc/secrets/ is gitignored except for .keep). +# +# After this script runs and the Secret is in place, the next `helm upgrade` +# (or kubectl-apply of the chart's rendered manifest) of kps-grafana picks +# up the Secret via `envFromSecret: grafana-google-oidc`. Verify with: +# kubectl --context=$APP_CLUSTER_KUBECONTEXT -n monitoring exec -it kps-grafana-0 -c grafana -- \ +# env | grep GF_AUTH_GOOGLE_ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GKE_MANIFEST_DIR="$REPO_ROOT/deploy/gcp/gke" + +NAMESPACE="${NAMESPACE:-monitoring}" +KCTX="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}" + +if [[ -n "$KCTX" ]]; then + KCTX_FLAG=(--context="$KCTX") +else + KCTX_FLAG=() +fi + +log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +resolve_secret() { + local var="$1" file="$2" val="${!1:-}" + if [[ -z "$val" && -f "$REPO_ROOT/etc/secrets/$file" ]]; then + val="$(cat "$REPO_ROOT/etc/secrets/$file")" + fi + if [[ -z "$val" ]]; then + die "missing $var (set the env var, or save the value into etc/secrets/$file)" + fi + printf '%s' "$val" +} + +for tool in kubectl envsubst; do + command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool" +done + +GRAFANA_GOOGLE_CLIENT_ID="$(resolve_secret GRAFANA_GOOGLE_CLIENT_ID grafana-google-oidc-client-id)" +GRAFANA_GOOGLE_CLIENT_SECRET="$(resolve_secret GRAFANA_GOOGLE_CLIENT_SECRET grafana-google-oidc-client-secret)" +export GRAFANA_GOOGLE_CLIENT_ID GRAFANA_GOOGLE_CLIENT_SECRET + +SECRET_TMPL="$GKE_MANIFEST_DIR/grafana-google-oidc-secret.example.yaml" +[[ -f "$SECRET_TMPL" ]] || die "missing manifest: $SECRET_TMPL" + +log "==> grafana google-oauth bootstrap" +log " namespace : $NAMESPACE" +log " kubectx : ${KCTX:-}" + +log "Applying grafana-google-oidc Secret ..." +envsubst '${GRAFANA_GOOGLE_CLIENT_ID} ${GRAFANA_GOOGLE_CLIENT_SECRET}' \ + < "$SECRET_TMPL" \ + | kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" apply -f - + +log "==> grafana google-oauth secret applied." +echo "" +echo " Next steps:" +echo " 1. Apply the Helm values override at monitoring/kps-values-gke.yaml:" +echo " helm upgrade --reuse-values kps prometheus-community/kube-prometheus-stack \\" +echo " --namespace $NAMESPACE \\" +echo " -f $REPO_ROOT/monitoring/kps-values-gke.yaml" +echo " 2. Patch knoe-svc-kong-config to add the /grafana route (see" +echo " deploy/opentofu/k3s/manifests/knoe/kong-configmap.yaml for the canonical source)." +echo " 3. Restart Kong: kubectl rollout restart deployment/knoe-svc-kong -n knoe-system" +echo " 4. Browser-test: https://svc.knoe.dev/grafana → Google sign-in → Grafana" +echo "" diff --git a/mock_val/init_grafana_oauth_prole.sh b/mock_val/init_grafana_oauth_prole.sh new file mode 100644 index 0000000..b4f48df --- /dev/null +++ b/mock_val/init_grafana_oauth_prole.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# init_grafana_oauth_prole.sh +# +# Bootstrap the Google-OAuth secret for Grafana on the prole.org k3s homelab +# cluster. Companion to the knoe.dev version (init_grafana_oauth.sh) but uses +# prole.org GCP project credentials and targets the k3s kubecontext. +# +# Creates the grafana-google-oidc Secret in the monitoring namespace, which +# kps-grafana mounts via envFromSecret to get GF_AUTH_GOOGLE_CLIENT_ID and +# GF_AUTH_GOOGLE_CLIENT_SECRET for its auth.google sign-in flow. +# +# Auth model (monitoring/kps-values-k3s.yaml): +# - Kerberos/knoe-auth users auto-login via auth.proxy (X-WEBAUTH-USER). +# - chrisfu@prole.org (and any future @prole.org Workspace user) signs in +# with the Google button on the Grafana login page. +# - chrisfu@prole.org → Admin; all other @prole.org users → Editor. +# +# Usage: +# ./etc/init_grafana_oauth_prole.sh +# +# Env vars (resolved from etc/secrets/* if not set in the shell): +# GRAFANA_GOOGLE_CLIENT_ID ← from etc/secrets/grafana-google-oidc-client-id-prole +# GRAFANA_GOOGLE_CLIENT_SECRET ← from etc/secrets/grafana-google-oidc-client-secret-prole +# +# Optional: +# K3S_KUBECONTEXT (default: $KUBECONTEXT then ambient) +# NAMESPACE (default: monitoring) +# +# Pre-reqs: +# - OAuth 2.0 Web Application client created in the prole.org GCP project: +# Authorized JS origins: https://svc.prole.org +# Authorized redirect URI: https://svc.prole.org/grafana/login/google +# Consent screen: Internal (prole.org Workspace) +# Scopes: openid, email, profile +# See deploy/gcp/gke/grafana-google-oidc-secret-prole.example.yaml for details. +# - Client ID and secret saved (chmod 0600) to: +# etc/secrets/grafana-google-oidc-client-id-prole +# etc/secrets/grafana-google-oidc-client-secret-prole + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MANIFEST_DIR="$REPO_ROOT/deploy/gcp/gke" + +NAMESPACE="${NAMESPACE:-monitoring}" +KCTX="${K3S_KUBECONTEXT:-${KUBECONTEXT:-}}" + +if [[ -n "$KCTX" ]]; then + KCTX_FLAG=(--context="$KCTX") +else + KCTX_FLAG=() +fi + +log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +resolve_secret() { + local var="$1" file="$2" val="${!1:-}" + if [[ -z "$val" && -f "$REPO_ROOT/etc/secrets/$file" ]]; then + val="$(cat "$REPO_ROOT/etc/secrets/$file")" + fi + if [[ -z "$val" ]]; then + die "missing $var (set the env var, or save the value into etc/secrets/$file)" + fi + printf '%s' "$val" +} + +for tool in kubectl envsubst; do + command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool" +done + +GRAFANA_GOOGLE_CLIENT_ID="$(resolve_secret GRAFANA_GOOGLE_CLIENT_ID grafana-google-oidc-client-id-prole)" +GRAFANA_GOOGLE_CLIENT_SECRET="$(resolve_secret GRAFANA_GOOGLE_CLIENT_SECRET grafana-google-oidc-client-secret-prole)" +export GRAFANA_GOOGLE_CLIENT_ID GRAFANA_GOOGLE_CLIENT_SECRET + +SECRET_TMPL="$MANIFEST_DIR/grafana-google-oidc-secret-prole.example.yaml" +[[ -f "$SECRET_TMPL" ]] || die "missing manifest: $SECRET_TMPL" + +log "==> grafana google-oauth bootstrap (prole.org / k3s)" +log " namespace : $NAMESPACE" +log " kubectx : ${KCTX:-}" + +log "Applying grafana-google-oidc Secret ..." +envsubst '${GRAFANA_GOOGLE_CLIENT_ID} ${GRAFANA_GOOGLE_CLIENT_SECRET}' \ + < "$SECRET_TMPL" \ + | kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" apply -f - + +log "==> grafana google-oauth secret applied." +echo "" +echo " Next steps:" +echo " 1. Helm upgrade kube-prometheus-stack with the k3s values:" +echo " helm upgrade --reuse-values kps prometheus-community/kube-prometheus-stack \\" +echo " --namespace $NAMESPACE \\" +echo " -f $REPO_ROOT/monitoring/kps-values-k3s.yaml" +echo " 2. Restart grafana-proxy to pick up the updated configmap:" +echo " kubectl rollout restart deployment/knoe-grafana-proxy -n $NAMESPACE" +echo " 3. Browser-test: https://svc.prole.org/grafana/login → Google sign-in button" +echo " present; Kerberos users still auto-login via X-WEBAUTH-USER." +echo "" diff --git a/mock_val/init_k3s_registry.sh b/mock_val/init_k3s_registry.sh index 4c092c7..a89b2e7 100644 --- a/mock_val/init_k3s_registry.sh +++ b/mock_val/init_k3s_registry.sh @@ -22,11 +22,17 @@ registry_host_from_url() { printf '%s' "$value" } -K3S_REGISTRY_HOST=${K3S_REGISTRY_HOST:-$(registry_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")} +K3S_REGISTRY_HOST=${K3S_REGISTRY_HOST:-$(registry_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")} K3S_REGISTRY_PORT=${K3S_REGISTRY_PORT:-5000} K3S_REGISTRY_NAMESPACE=${K3S_REGISTRY_NAMESPACE:-${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}}} K3S_REGISTRY_FILE=${K3S_REGISTRY_FILE:-/etc/rancher/k3s/registries.yaml} -K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-http} +K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-} + +if [[ -z "${K3S_REGISTRY_SCHEME}" ]]; then + # Internal Knoe registry endpoints are plain HTTP by default. + # Set K3S_REGISTRY_SCHEME=https explicitly when TLS is configured. + K3S_REGISTRY_SCHEME="http" +fi ensure_root() { if [[ "$(id -u)" -ne 0 ]]; then @@ -40,6 +46,7 @@ render_registries_yaml() { local port="$2" local ns="$3" local scheme="$4" + if [[ "$scheme" == "http" ]]; then cat </dev/null 2>&1 || true -knoe_ensure_kube_context || exit 1 +ensure_kube_context || exit 1 ACTION=${1:-initialize} +resolve_kdc_mode_hint() { + local mode_hint="${KNOE_MODE:-${DEPLOYMENT_MODE:-${CLUSTER_ENV:-k3s}}}" + if command -v knoe_normalize_mode >/dev/null 2>&1; then + knoe_normalize_mode "$mode_hint" + return 0 + fi + mode_hint=$(printf '%s' "$mode_hint" | tr 'A-Z' 'a-z') + case "$mode_hint" in + prod|production) + printf 'k8s' + ;; + *) + printf '%s' "$mode_hint" + ;; + esac +} + +default_knoe_kdc_name() { + case "$(resolve_kdc_mode_hint)" in + k8s) + printf 'authority-gcp-auth' + ;; + *) + printf 'authority-knoe-auth' + ;; + esac +} + KDC_NAMESPACE=${PROLE_KDC_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}} +# Namespace where knoe-auth runs and expects the `knoe-kdc-config` ConfigMap. +# Defaults to PROLE_NAMESPACE (from knoe.cfg) when present. +PROLE_AUTH_NAMESPACE=${PROLE_AUTH_NAMESPACE:-${PROLE_NAMESPACE:-}} PROLE_KDC_ENABLED=${PROLE_KDC_ENABLED:-1} -PROLE_KDC_NAME=${PROLE_KDC_NAME:-auth} +PROLE_KDC_NAME=${PROLE_KDC_NAME:-$(default_knoe_kdc_name)} PROLE_KDC_SERVICE=${PROLE_KDC_SERVICE:-auth} PROLE_KDC_IMAGE=${PROLE_KDC_IMAGE:-} # If the user provided an explicit image, we should not require Docker unless @@ -293,7 +324,11 @@ get_secret_value() { resolve_knoe_kdc_defaults() { if [[ -z "$PROLE_KDC_REALM" ]]; then - PROLE_KDC_REALM="PROLE.LOCAL" + # Post-rebrand default — must match knoe-db/etc/init_kdc.sh. The old + # "PROLE.LOCAL" string seeded a stale KDC config on the prole k3s + # cluster that took an afternoon of cross-realm trust debugging to + # find — see ~/.claude/plans/chrisfu-myrddin-dev-prole-git-pull-*.md. + PROLE_KDC_REALM="KNOE.LOCAL" fi if [[ -z "$PROLE_KDC_DOMAIN" ]]; then PROLE_KDC_DOMAIN=$(lowercase "$PROLE_KDC_REALM") @@ -532,6 +567,13 @@ EOF host_net_block=" hostNetwork: true" dns_policy_block=" dnsPolicy: ClusterFirstWithHostNet" fi + # PVC storage class: explicit when $PROLE_KDC_STORAGE_CLASS is set; + # otherwise leave blank so the cluster's default StorageClass picks + # the binder (k3s "local-path", GKE "standard", etc.). + local storage_class_block="" + if [[ -n "${PROLE_KDC_STORAGE_CLASS:-}" ]]; then + storage_class_block=" storageClassName: ${PROLE_KDC_STORAGE_CLASS}" + fi cat < + UPN) does not match MIT's + # ( + ). RC4 derives keys from + # the password alone, so both sides converge with no salt fight. + # ------------------------------------------------------------------ + + # Outbound: KNOE.LOCAL → PROLE.ORG (issued here, decrypted by Samba) if ! kadmin.local -q "get_principal krbtgt/\${PROLE_KDC_TRUST_REALM}@\${PROLE_KDC_REALM}" >/dev/null 2>&1; then - echo "Creating trust principal krbtgt/\${PROLE_KDC_TRUST_REALM}@\${PROLE_KDC_REALM}..." - kadmin.local -q "addprinc -pw \${shared_pw} krbtgt/\${PROLE_KDC_TRUST_REALM}@\${PROLE_KDC_REALM}" + echo "Creating outbound trust principal krbtgt/\${PROLE_KDC_TRUST_REALM}@\${PROLE_KDC_REALM}..." + kadmin.local -q "addprinc -pw \${shared_pw} -e arcfour-hmac:normal krbtgt/\${PROLE_KDC_TRUST_REALM}@\${PROLE_KDC_REALM}" fi - if [[ -n "\${PROLE_KDC_TRUST_ADMIN:-}" && -n "\${PROLE_KDC_TRUST_PASSWORD:-}" ]]; then - echo "Creating reciprocal trust principal in \${PROLE_KDC_TRUST_REALM}..." - kadmin -r "\${PROLE_KDC_TRUST_REALM}" -p "\${PROLE_KDC_TRUST_ADMIN}" -w "\${PROLE_KDC_TRUST_PASSWORD}" \ - -q "addprinc -pw \${shared_pw} krbtgt/\${PROLE_KDC_REALM}@\${PROLE_KDC_TRUST_REALM}" || true - else - echo "WARN: Missing PROLE_KDC_TRUST_ADMIN/PROLE_KDC_TRUST_PASSWORD; skipping external trust principal." + # Inbound: PROLE.ORG → KNOE.LOCAL (issued by Samba, decrypted here) + if ! kadmin.local -q "get_principal krbtgt/\${PROLE_KDC_REALM}@\${PROLE_KDC_TRUST_REALM}" >/dev/null 2>&1; then + echo "Creating inbound trust principal krbtgt/\${PROLE_KDC_REALM}@\${PROLE_KDC_TRUST_REALM}..." + kadmin.local -q "addprinc -pw \${shared_pw} -e arcfour-hmac:normal krbtgt/\${PROLE_KDC_REALM}@\${PROLE_KDC_TRUST_REALM}" fi + + # NOTE: The Samba-side trust account (user "krbtgt_\${PROLE_KDC_REALM}" + # in PROLE.ORG with UPN/SPN krbtgt/\${PROLE_KDC_REALM}) is provisioned + # OUT-OF-BAND by this repo's Ansible playbook: + # infrastructure/playbooks/kerberos_trust_setup.yml + # Earlier versions of this script tried to use a remote "kadmin" + # client to write that principal into Samba, but Samba AD does not + # accept additions over MIT's kadmin protocol — it always failed + # with "Missing parameters in krb5.conf required for kadmin client". + # Run the playbook once after this KDC comes up: + # ANSIBLE_VAULT_PASSWORD_FILE=\$PWD/.vault_pass \\ + # ansible-playbook infrastructure/playbooks/kerberos_trust_setup.yml + echo "Note: Samba-side trust account is provisioned out-of-band by" + echo " infrastructure/playbooks/kerberos_trust_setup.yml" fi # Start daemons. Keep kadmind in PID 1; run krb5kdc in background and verify it binds. @@ -774,7 +840,21 @@ ${dns_policy_block} configMap: name: knoe-kdc-config - name: knoe-kdc-data - emptyDir: {} + persistentVolumeClaim: + claimName: knoe-kdc-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: knoe-kdc-data + namespace: ${KDC_NAMESPACE} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${PROLE_KDC_STORAGE_SIZE:-1Gi} +${storage_class_block} --- apiVersion: v1 kind: Service @@ -810,6 +890,112 @@ EOF apply_kdc_manifest + # knoe-auth runs in PROLE_AUTH_NAMESPACE (default: PROLE_NAMESPACE) and mounts + # ConfigMap `knoe-kdc-config` for its embedded KDC sidecar. When the KDC itself + # is deployed into a different namespace (default: SERVICE_NAMESPACE), ensure + # the configmap also exists in the knoe-auth namespace to prevent FailedMount. + if [[ -n "${PROLE_AUTH_NAMESPACE:-}" && "${PROLE_AUTH_NAMESPACE}" != "${KDC_NAMESPACE}" ]]; then + if ! kubectl get namespace "$PROLE_AUTH_NAMESPACE" >/dev/null 2>&1; then + log "Creating namespace '$PROLE_AUTH_NAMESPACE' ..." + kubectl create namespace "$PROLE_AUTH_NAMESPACE" >/dev/null 2>&1 || true + fi + log "Ensuring ConfigMap 'knoe-kdc-config' exists in namespace '${PROLE_AUTH_NAMESPACE}' for knoe-auth ..." + cat </dev/null 2>&1; then + echo "Installing Kerberos packages..." + echo "krb5-config krb5-config/default_realm string \${PROLE_KDC_REALM}" | debconf-set-selections || true + echo "krb5-config krb5-config/kerberos_servers string 127.0.0.1" | debconf-set-selections || true + echo "krb5-config krb5-config/admin_server string 127.0.0.1" | debconf-set-selections || true + apt-get update + apt-get install -y --no-install-recommends krb5-kdc krb5-admin-server krb5-user dnsutils ca-certificates + rm -rf /var/lib/apt/lists/* + fi + + mkdir -p /etc/krb5kdc /var/lib/krb5kdc + if [[ -f /opt/knoe-kdc/krb5.conf ]]; then + cp /opt/knoe-kdc/krb5.conf /etc/krb5.conf + fi + if [[ -f /opt/knoe-kdc/kdc.conf ]]; then + cp /opt/knoe-kdc/kdc.conf /etc/krb5kdc/kdc.conf + fi + if [[ -f /opt/knoe-kdc/kadm5.acl ]]; then + cp /opt/knoe-kdc/kadm5.acl /etc/krb5kdc/kadm5.acl + fi + + if [[ -z "\${PROLE_KDC_MASTER_PASSWORD:-}" ]]; then + echo "ERROR: Missing required env PROLE_KDC_MASTER_PASSWORD (secret 'knoe-kdc-secrets/master_password')." >&2 + exit 1 + fi + if [[ -z "\${PROLE_KDC_ADMIN_PASSWORD:-}" ]]; then + echo "ERROR: Missing required env PROLE_KDC_ADMIN_PASSWORD (secret 'knoe-kdc-secrets/admin_password')." >&2 + exit 1 + fi + + if [[ ! -f /var/lib/krb5kdc/principal ]]; then + echo "Initializing realm database for \${realm}..." + kdb5_util create -s -r "\${realm}" -P "\${PROLE_KDC_MASTER_PASSWORD}" + fi + + if ! kadmin.local -q "get_principal \${admin_principal}" >/dev/null 2>&1; then + echo "Creating admin principal \${admin_principal}..." + kadmin.local -q "addprinc -pw \${PROLE_KDC_ADMIN_PASSWORD} \${admin_principal}" + fi + + echo "Starting krb5kdc and kadmind ..." + krb5kdc -n & + sleep 0.5 + if ! pgrep -x krb5kdc >/dev/null 2>&1; then + echo "ERROR: krb5kdc failed to start. Check /var/log/ (syslog) for details." >&2 + exit 1 + fi + exec kadmind -nofork +EOF + fi + local rollout_timeout="$PROLE_KDC_ROLLOUT_TIMEOUT" if [[ "$deployment_present" -eq 0 ]]; then rollout_timeout="$PROLE_KDC_DEPLOY_TIMEOUT" @@ -839,6 +1025,10 @@ cleanup_knoe_kdc() { kubectl -n "$KDC_NAMESPACE" delete service "$PROLE_KDC_SERVICE" --ignore-not-found kubectl -n "$KDC_NAMESPACE" delete deployment "$PROLE_KDC_NAME" --ignore-not-found kubectl -n "$KDC_NAMESPACE" delete configmap knoe-kdc-config --ignore-not-found + + if [[ -n "${PROLE_AUTH_NAMESPACE:-}" && "${PROLE_AUTH_NAMESPACE}" != "${KDC_NAMESPACE}" ]]; then + kubectl -n "$PROLE_AUTH_NAMESPACE" delete configmap knoe-kdc-config --ignore-not-found + fi } status() { diff --git a/mock_val/init_kerberos.sh b/mock_val/init_kerberos.sh index 3d72220..3958a16 100755 --- a/mock_val/init_kerberos.sh +++ b/mock_val/init_kerberos.sh @@ -48,7 +48,7 @@ KRB5_AD_PROXY_IMAGE=${KRB5_AD_PROXY_IMAGE:-alpine/socat} KRB5_AD_PROXY_HOST_NETWORK=${KRB5_AD_PROXY_HOST_NETWORK:-1} KRB5_AD_TCP_PORTS=${KRB5_AD_TCP_PORTS:-"88 389 445 464 636"} KRB5_AD_UDP_PORTS=${KRB5_AD_UDP_PORTS:-"88 464"} -CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-300} +CNPG_WAIT_TIMEOUT=${CNPG_WAIT_TIMEOUT:-900} SERVICE_NAMESPACE=${SERVICE_NAMESPACE:-${NAMESPACE:-default}} OPENBAO_NAMESPACE=${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-default}} KRB5_AD_NAMESPACE=${KRB5_AD_NAMESPACE:-${SERVICE_NAMESPACE}} @@ -92,6 +92,12 @@ wait_for_ad_forwarder_ready() { } ensure_tools() { + if [[ "${KNOE_MODE:-}" == "min" ]]; then + for t in curl jq; do + command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; } + done + return + fi for t in kubectl curl jq; do command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; } done @@ -156,12 +162,24 @@ is_ip_address() { } detect_knoe_cfg() { - if [[ -n "${KNOE_CONF:-}" && -f "$KNOE_CONF/knoe.cfg" ]]; then - printf '%s' "$KNOE_CONF/knoe.cfg" - elif [[ -n "${KNOE_HOME:-}" && -f "$KNOE_HOME/conf/knoe.cfg" ]]; then - printf '%s' "$KNOE_HOME/conf/knoe.cfg" - elif [[ -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - printf '%s' "$SCRIPT_DIR/../conf/knoe.cfg" + local cfg="" + if [[ -n "${KNOE_CONF:-}" ]]; then + cfg="$(_knoe_cfg_select_cfg_file "$KNOE_CONF")" + if [[ -n "$cfg" ]]; then + printf '%s' "$cfg" + return 0 + fi + fi + if [[ -n "${KNOE_HOME:-}" ]]; then + cfg="$(_knoe_cfg_select_cfg_file "$KNOE_HOME/conf")" + if [[ -n "$cfg" ]]; then + printf '%s' "$cfg" + return 0 + fi + fi + cfg="$(_knoe_cfg_select_cfg_file "$SCRIPT_DIR/../conf")" + if [[ -n "$cfg" ]]; then + printf '%s' "$cfg" fi } @@ -481,6 +499,11 @@ default_port_forward_if_local() { } sync_knoe_kdc_trust() { + if [[ "${PROLE_KDC_STANDALONE:-0}" != "1" ]]; then + # Default deployment embeds the KDC as a sidecar in `knoe-auth`; do not + # attempt to manage a standalone KDC unless explicitly requested. + return 0 + fi if [[ ! -x "$SCRIPT_DIR/init_kdc.sh" ]]; then return 0 fi @@ -952,6 +975,10 @@ maybe_apply_k3d_route_fix() { } initialize() { + if [[ "${KNOE_MODE:-}" == "min" ]]; then + log "Minimal mode: skipping Kerberos Kubernetes resource initialization." + return 0 + fi ensure_tools ensure_namespace resolve_krb5_defaults @@ -1085,13 +1112,17 @@ run_test() { run_kerberos_test_loop() { local test_attempt=1 local test_max_attempts=2 + local checker_mode_args=() + if [[ -n "$mode" ]]; then + checker_mode_args=(--mode "$mode") + fi while true; do if PROLE_USE_CHILD_REALM="$test_use_child" \ KRB5_REALM="$KRB5_REALM" KRB5_KDC="$effective_kdc" KRB5_ADMIN="$KRB5_ADMIN" \ KRB5_USER="${KERBEROS_TEST_ADMIN_USER:-administrator}" KRB5_PASSWORD="$KRB5_PASSWORD" \ SAMBA_ADMIN_USER="${KERBEROS_TEST_ADMIN_USER:-administrator}" SAMBA_ADMIN_PASSWORD="$KRB5_PASSWORD" \ SAMBA_DNS_SERVER="$test_samba_dns" \ - "$kerberos_check_script" test; then + "$kerberos_check_script" "${checker_mode_args[@]}" test; then return 0 fi diff --git a/mock_val/init_knoe_auth.sh b/mock_val/init_knoe_auth.sh new file mode 100755 index 0000000..e8b430e --- /dev/null +++ b/mock_val/init_knoe_auth.sh @@ -0,0 +1,573 @@ +#!/usr/bin/env bash +# init_knoe_auth.sh +# Provision knoe-auth (Kerberos KDC + Spring Boot enrollment service). +# +# Usage: +# ./etc/init_knoe_auth.sh [--context KUBECONTEXT] [--namespace NAMESPACE] [--project PROJECT_ID] [--mode MODE] +# ./etc/init_knoe_auth.sh initialize # full setup +# ./etc/init_knoe_auth.sh schema # schema only (idempotent) +# ./etc/init_knoe_auth.sh invite EMAIL # create first admin invite +# ./etc/init_knoe_auth.sh status # check pod + principal state +# +# Modes: +# (default / gke) GKE deploy — uses 1Password for secrets, GCP Workload Identity +# k3d Local k3d dev loop — uses hardcoded dev passwords, skips GCP steps +# +# Prerequisites: +# kubectl, op (1Password CLI — GKE mode only), psql (or kubectl exec fallback) +# +# Env vars (override args): +# APP_CLUSTER_KUBECONTEXT, KNOE_NAMESPACE, GCP_PROJECT_ID, +# KNOE_DB_HOST, KNOE_DB_PORT, KNOE_DB_NAME, KNOE_DB_SUPERUSER, +# KNOE_MODE (set to 'k3d' as alternative to --mode k3d) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GKE_DIR="$REPO_ROOT/deploy/gcp/gke" + +# ── Defaults ──────────────────────────────────────────────────────────────── +MODE="${KNOE_MODE:-gke}" # gke (default) or k3d +APP_CTX="${APP_CLUSTER_KUBECONTEXT:-gke_plenary-truck-485623-p7_us-west3_knoe-dev-0}" +NAMESPACE="${KNOE_NAMESPACE:-knoe-system}" +GCP_PROJECT="${GCP_PROJECT_ID:-plenary-truck-485623-p7}" +DB_HOST="${KNOE_DB_HOST:-}" # resolved from CNPG svc if blank +DB_PORT="${KNOE_DB_PORT:-5432}" +DB_NAME="${KNOE_DB_NAME:-knoe}" +DB_SUPERUSER="${KNOE_DB_SUPERUSER:-postgres}" +REALM="KNOE.DEV" +AUTH_HOST="${KNOE_AUTH_HOST:-https://auth.knoe.dev}" + +# k3d-specific defaults (overridden when --mode k3d is active) +K3D_CLUSTER_NAME="${K3D_CLUSTER_NAME:-k3d-knoe}" +K3D_CTX="k3d-${K3D_CLUSTER_NAME}" +K3D_DB_NS="${K3D_DB_NS:-knoe-db-0}" +# Dev-only passwords — NOT used in GKE mode; safe to commit +K3D_KDC_MASTER_PASSWORD="${K3D_KDC_MASTER_PASSWORD:-knoe-local-master-dev}" +K3D_KDC_ADMIN_PASSWORD="${K3D_KDC_ADMIN_PASSWORD:-knoe-local-admin-dev}" + +log() { echo "[init_knoe_auth] $*"; } +info() { log "INFO $*"; } +warn() { log "WARN $*" >&2; } +die() { log "ERROR $*" >&2; exit 1; } + +kube() { kubectl --context="$APP_CTX" "$@"; } + +# ── Argument parsing ───────────────────────────────────────────────────────── +COMMAND="${1:-initialize}" +shift || true + +while [[ $# -gt 0 ]]; do + case "$1" in + --context) APP_CTX="$2"; shift 2 ;; + --namespace) NAMESPACE="$2"; shift 2 ;; + --project) GCP_PROJECT="$2"; shift 2 ;; + --db-host) DB_HOST="$2"; shift 2 ;; + --mode) MODE="$2"; shift 2 ;; + *) break ;; + esac +done + +# Apply k3d mode overrides after argument parsing +if [[ "$MODE" == "k3d" ]]; then + APP_CTX="${APP_CLUSTER_KUBECONTEXT:-${K3D_CTX}}" + REALM="KNOE.LOCAL" + AUTH_HOST="${KNOE_AUTH_HOST:-http://localhost:8080}" + # DB is accessed via port-forward (localhost:5432) in k3d mode + DB_HOST="${KNOE_DB_HOST:-localhost}" +fi + +# ── Helpers ────────────────────────────────────────────────────────────────── + +require_tool() { + command -v "$1" >/dev/null 2>&1 || die "Required tool not found: $1 — install it and retry." +} + +wait_for_pods() { + local label="$1" + local timeout="${2:-180}" + info "Waiting up to ${timeout}s for pods with label ${label} in ${NAMESPACE}..." + kube -n "$NAMESPACE" wait pod \ + -l "$label" \ + --for=condition=Ready \ + --timeout="${timeout}s" +} + +op_secret() { + # Retrieve a 1Password secret; fall back to prompting if op isn't authed. + local item="$1" field="${2:-password}" + if command -v op >/dev/null 2>&1; then + op item get "$item" --fields "$field" 2>/dev/null || { + warn "1Password: could not read $item/$field — prompting." + read -rsp "Enter value for $item/$field: " val; echo + printf '%s' "$val" + } + else + read -rsp "Enter value for $item/$field: " val; echo + printf '%s' "$val" + fi +} + +resolve_db_host() { + if [[ -n "$DB_HOST" ]]; then return; fi + if [[ "$MODE" == "k3d" ]]; then + # In k3d mode the engineer runs make k3d-knoe-pf first; DB is at localhost:5432 + DB_HOST="localhost" + info "k3d mode: using DB host localhost (port-forward expected on :5432)" + return + fi + # Try to resolve CNPG primary service from the DB cluster context + DB_CTX="${DB_CLUSTER_KUBECONTEXT:-gke_plenary-truck-485623-p7_us-west3_knoe-cnpg-0}" + DB_HOST=$(kubectl --context="$DB_CTX" -n knoe-db-0 \ + get svc knoe-db-rw -o jsonpath='{.spec.clusterIP}' 2>/dev/null || echo "") + [[ -z "$DB_HOST" ]] && die "Cannot resolve KNOE DB host. Set KNOE_DB_HOST or ensure knoe-db-rw svc exists." + info "Resolved DB host: $DB_HOST" +} + +psql_file() { + local file="$1" + if [[ "$MODE" == "k3d" ]]; then + # In k3d mode: exec into the CNPG primary pod directly (no port-forward needed for schema) + local pod + pod=$(kubectl --context="$APP_CTX" -n "$K3D_DB_NS" \ + get pod -l cnpg.io/cluster=knoe-db,role=primary \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + [[ -z "$pod" ]] && die "Cannot find CNPG primary pod in k3d. Is 'make k3d-knoe-up' complete?" + kubectl --context="$APP_CTX" -n "$K3D_DB_NS" cp "$file" "${pod}:/tmp/knoe_auth_schema.sql" + kubectl --context="$APP_CTX" -n "$K3D_DB_NS" exec "$pod" -- \ + psql -U "$DB_SUPERUSER" -d "$DB_NAME" -f /tmp/knoe_auth_schema.sql + return + fi + if command -v psql >/dev/null 2>&1 && [[ -n "${PGPASSWORD:-}" ]]; then + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_SUPERUSER" -d "$DB_NAME" -f "$file" + else + # Fallback: exec into CNPG primary pod + DB_CTX="${DB_CLUSTER_KUBECONTEXT:-gke_plenary-truck-485623-p7_us-west3_knoe-cnpg-0}" + local pod + pod=$(kubectl --context="$DB_CTX" -n knoe-db-0 \ + get pod -l cnpg.io/cluster=knoe-db,role=primary \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + [[ -z "$pod" ]] && die "Cannot find CNPG primary pod. Set PGPASSWORD and KNOE_DB_HOST for direct psql." + kubectl --context="$DB_CTX" -n knoe-db-0 cp "$file" "${pod}:/tmp/knoe_auth_schema.sql" + kubectl --context="$DB_CTX" -n knoe-db-0 exec "$pod" -- \ + psql -U "$DB_SUPERUSER" -d "$DB_NAME" -f /tmp/knoe_auth_schema.sql + fi +} + +# ── Schema ─────────────────────────────────────────────────────────────────── + +run_schema() { + info "Applying knoe-auth schema additions..." + resolve_db_host + + local tmpfile + tmpfile=$(mktemp /tmp/knoe_auth_schema_XXXX.sql) + + cat > "$tmpfile" <<'ENDSQL' +-- ── knoe-auth Round 1 schema additions ─────────────────────────────────────── +-- Idempotent: all CREATE TABLE ... IF NOT EXISTS + +-- Invite tokens (admin creates, single-use) +-- contact is the email/phone the invite was sent to — the OTP trust anchor. +-- knoe.dev starts with ZERO pre-knowledge of the developer's home org. +CREATE TABLE IF NOT EXISTS knoe.invitation ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + token TEXT NOT NULL UNIQUE, + contact TEXT NOT NULL, + contact_type TEXT NOT NULL DEFAULT 'email', + name_hint TEXT, + otp_hash TEXT NOT NULL, + otp_expires_at TIMESTAMPTZ NOT NULL, + otp_attempts INT NOT NULL DEFAULT 0, + otp_verified_at TIMESTAMPTZ, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL, + used_at TIMESTAMPTZ, + used_by TEXT +); +CREATE INDEX IF NOT EXISTS idx_invitation_token ON knoe.invitation(token); +CREATE INDEX IF NOT EXISTS idx_invitation_contact ON knoe.invitation(contact); + +-- External identity corroborations (Google sub → knoe user) +-- provider_hd records the developer's home domain (prole.org, gmail.com, etc.) +-- for audit purposes only — it is NOT used for access control. +CREATE TABLE IF NOT EXISTS knoe.identity ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES knoe.user(id) ON DELETE CASCADE, + provider TEXT NOT NULL, + provider_sub TEXT NOT NULL, + provider_email TEXT, + provider_hd TEXT, + verified_at TIMESTAMPTZ NOT NULL, + UNIQUE(provider, provider_sub) +); +CREATE INDEX IF NOT EXISTS idx_identity_user ON knoe.identity(user_id); + +-- TOTP 2FA credentials (encrypted secret, backup codes) +CREATE TABLE IF NOT EXISTS knoe.totp_credential ( + user_id INT PRIMARY KEY REFERENCES knoe.user(id) ON DELETE CASCADE, + secret TEXT NOT NULL, + verified_at TIMESTAMPTZ, + backup_codes TEXT[], + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Platform-managed resources (repos, db roles, policies, etc.) +CREATE TABLE IF NOT EXISTS knoe.knobject ( + id SERIAL PRIMARY KEY, + type TEXT NOT NULL, + name TEXT NOT NULL, + platform_id TEXT, + metadata JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(type, name) +); + +-- Access grants (user → knobject with role) +CREATE TABLE IF NOT EXISTS knoe.access_grant ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES knoe.user(id), + knobject_id INT NOT NULL REFERENCES knoe.knobject(id), + role TEXT NOT NULL, + granted_by TEXT NOT NULL, + granted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + revoked_at TIMESTAMPTZ, + UNIQUE(user_id, knobject_id) +); + +-- Async provisioning job queue (GitLab user, Gitea user, CNPG role, etc.) +CREATE TABLE IF NOT EXISTS knoe.provisioning_job ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES knoe.user(id), + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + payload JSONB, + result JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_provisioning_job_status + ON knoe.provisioning_job(status, created_at); + +-- Seed well-known knobjects +INSERT INTO knoe.knobject (type, name, metadata) VALUES + ('gitea_org', 'knoey.com', '{"description": "Knoey.com Gitea organisation"}'), + ('gitlab_group','knoey.com', '{"description": "Knoey.com GitLab group"}') +ON CONFLICT (type, name) DO NOTHING; + +SELECT 'knoe-auth schema v1 applied.' AS status; +ENDSQL + + psql_file "$tmpfile" + rm -f "$tmpfile" + info "Schema applied." +} + +# ── Dev-user seed (k3d only) ───────────────────────────────────────────────── +# +# Creates the `knoe_developer` group role (the production GKE deploy uses it +# too via pg_hba.conf `+knoe_developer` rules; on GKE it was hand-rolled per +# 2026-04-30 onboarding work, never baked into postInitTemplateSQL — see +# docs/db-access.md). Then creates a `chrisfu` LOGIN role with a dev +# password and grants `knoe_developer` to it, so the engineer can connect +# from the host as chrisfu@knoey.com via the port-forward. +# +# Idempotent: re-runs on every `make k3d-knoe-up` and either creates or +# updates the role's password. This makes the rebuild loop deterministic — +# after `down && up`, chrisfu's password is always `chrisfu-dev`. +seed_dev_users_k3d() { + info "Seeding dev users (knoe_developer + chrisfu) for k3d ..." + + local tmpfile + tmpfile=$(mktemp /tmp/knoe_auth_seed_XXXX.sql) + + cat > "$tmpfile" <<'ENDSQL' +-- knoe_developer group role: R/W on knoe + public, R/O on auth/storage/extensions. +-- Mirrors the GKE production layout (docs/db-access.md). NOLOGIN — group only. +DO $do$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'knoe_developer') THEN + CREATE ROLE knoe_developer NOLOGIN; + END IF; +END $do$; + +-- knoe + public — full R/W +GRANT USAGE ON SCHEMA knoe TO knoe_developer; +GRANT USAGE ON SCHEMA public TO knoe_developer; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA knoe TO knoe_developer; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO knoe_developer; +GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA knoe TO knoe_developer; +GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO knoe_developer; +ALTER DEFAULT PRIVILEGES IN SCHEMA knoe GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO knoe_developer; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO knoe_developer; +ALTER DEFAULT PRIVILEGES IN SCHEMA knoe GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO knoe_developer; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO knoe_developer; + +-- Per-engineer LOGIN role for chrisfu (local-dev only; password is fixed dev value). +-- Creates if missing, otherwise resets the password — guarantees the rebuild loop +-- always produces the same credentials. +DO $do$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'chrisfu') THEN + CREATE ROLE chrisfu LOGIN INHERIT PASSWORD 'chrisfu-dev'; + ELSE + ALTER ROLE chrisfu WITH LOGIN INHERIT PASSWORD 'chrisfu-dev'; + END IF; +END $do$; + +GRANT knoe_developer TO chrisfu; + +-- Sanity check (visible in psql_file output). +SELECT 'chrisfu' AS role, + pg_has_role('chrisfu', 'knoe_developer', 'MEMBER') AS is_developer; +ENDSQL + + psql_file "$tmpfile" + rm -f "$tmpfile" + info "Dev-user seed applied (chrisfu / chrisfu-dev, member of knoe_developer)." +} + +# ── KDC secrets ────────────────────────────────────────────────────────────── + +create_kdc_secrets() { + if kube -n "$NAMESPACE" get secret knoe-kdc-secrets >/dev/null 2>&1; then + info "knoe-kdc-secrets already exists — skipping." + return + fi + if [[ "$MODE" == "k3d" ]]; then + info "k3d mode: creating knoe-kdc-secrets with dev passwords (no 1Password)..." + kube -n "$NAMESPACE" create secret generic knoe-kdc-secrets \ + --from-literal=master_password="$K3D_KDC_MASTER_PASSWORD" \ + --from-literal=admin_password="$K3D_KDC_ADMIN_PASSWORD" + info "knoe-kdc-secrets created (dev passwords)." + return + fi + info "Creating knoe-kdc-secrets from 1Password..." + local master admin + master=$(op_secret "knoe-kdc-master" "password") + admin=$(op_secret "knoe-kdc-admin" "password") + kube -n "$NAMESPACE" create secret generic knoe-kdc-secrets \ + --from-literal=master_password="$master" \ + --from-literal=admin_password="$admin" + info "knoe-kdc-secrets created." +} + +create_google_oidc_secret() { + if kube -n "$NAMESPACE" get secret knoe-auth-google-oidc >/dev/null 2>&1; then + info "knoe-auth-google-oidc already exists — skipping." + return + fi + info "Creating knoe-auth-google-oidc secret..." + local client_id client_secret + client_id=$(op_secret "knoe-google-oidc" "client_id") + client_secret=$(op_secret "knoe-google-oidc" "client_secret") + kube -n "$NAMESPACE" create secret generic knoe-auth-google-oidc \ + --from-literal=client_id="$client_id" \ + --from-literal=client_secret="$client_secret" + info "knoe-auth-google-oidc created." +} + +create_session_secret() { + if kube -n "$NAMESPACE" get secret knoe-auth-secrets >/dev/null 2>&1; then + info "knoe-auth-secrets already exists — skipping." + return + fi + info "Creating knoe-auth-secrets (session HMAC key)..." + local session_secret + session_secret=$(op_secret "knoe-auth-session" "password") + kube -n "$NAMESPACE" create secret generic knoe-auth-secrets \ + --from-literal=sessionSecret="$session_secret" + info "knoe-auth-secrets created." +} + +create_oidc_path_b_secret() { + if kube -n "$NAMESPACE" get secret knoe-auth-oidc >/dev/null 2>&1; then + info "knoe-auth-oidc already exists — skipping." + return + fi + + info "Creating knoe-auth-oidc secret for Path B..." + local client_id client_secret signing_key + client_id=$(op_secret "knoe-auth-oidc-gitlab" "client_id") + client_secret=$(op_secret "knoe-auth-oidc-gitlab" "client_secret") + signing_key=$(op_secret "knoe-auth-oidc-signing" "private_key") + + kube -n "$NAMESPACE" create secret generic knoe-auth-oidc \ + --from-literal=client-id="$client_id" \ + --from-literal=client-secret="$client_secret" \ + --from-literal=signing-key="$signing_key" + info "knoe-auth-oidc created." +} + +# ── Manifests ──────────────────────────────────────────────────────────────── + +apply_manifests() { + if [[ "$MODE" == "k3d" ]]; then + local k3d_dir="$REPO_ROOT/k8s/knoe" + info "k3d mode: applying KDC manifests from $k3d_dir..." + kube apply -f "$k3d_dir/knoe-kdc-configmap.yaml" + kube apply -f "$k3d_dir/knoe-kdc-pvc.yaml" + kube apply -f "$k3d_dir/knoe-kdc-deployment.yaml" + kube apply -f "$k3d_dir/knoe-kdc-service.yaml" + info "k3d mode: skipping knoe-auth Deployment (runs on host via mvn spring-boot:run)." + return + fi + info "Applying KDC ConfigMap..." + kube apply -f "$GKE_DIR/knoe-kdc-configmap.yaml" + + info "Applying knoe-auth Deployment..." + kube apply -f "$GKE_DIR/knoe-auth-deployment.yaml" +} + +# ── Invite helper ───────────────────────────────────────────────────────────── + +create_first_invite() { + local contact="${1:-}" + [[ -z "$contact" ]] && { read -rp "Invite contact (email or phone): " contact; } + local name_hint="" + read -rp "Display name hint (optional, press Enter to skip): " name_hint || true + + info "Creating invite for: $contact" + local admin_token + admin_token=$(op_secret "knoe-admin-token" "credential" 2>/dev/null || \ + { read -rsp "Admin token: " t; echo; printf '%s' "$t"; }) + + local response + response=$(curl -sf -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $admin_token" \ + -d "{\"contact\":\"$contact\",\"contactType\":\"email\",\"nameHint\":\"$name_hint\"}" \ + "${AUTH_HOST}/auth/admin/invites") || { + warn "Admin API call failed. knoe-auth may not be ready yet." + warn "Retry: POST ${AUTH_HOST}/auth/admin/invites" + return 1 + } + + local invite_url + invite_url=$(printf '%s' "$response" | grep -o '"enrollUrl":"[^"]*"' | sed 's/"enrollUrl":"//;s/"//') + printf '\n\033[1;32mInvite URL:\033[0m %s\n\n' "$invite_url" + info "Send the above URL to: $contact" +} + +# ── Status ──────────────────────────────────────────────────────────────────── + +show_status() { + if [[ "$MODE" == "k3d" ]]; then + info "=== KDC pod status ===" + kube -n "$NAMESPACE" get pods -l app=knoe-kdc 2>/dev/null || true + info "=== KDC secret ===" + kube -n "$NAMESPACE" get secret knoe-kdc-secrets 2>/dev/null || true + info "=== KDC service ===" + kube -n "$NAMESPACE" get svc knoe-kdc 2>/dev/null || true + info "=== knoe-auth runs on host ===" + info " export KRB5_CONFIG=\$PWD/etc/krb5.local.conf" + info " mvn -pl authority spring-boot:run" + return + fi + info "=== Pod status ===" + kube -n "$NAMESPACE" get pods -l app=knoe-auth 2>/dev/null || true + info "=== Secrets ===" + kube -n "$NAMESPACE" get secret \ + knoe-kdc-secrets knoe-auth-google-oidc knoe-auth-secrets 2>/dev/null || true + info "=== Services ===" + kube -n "$NAMESPACE" get svc knoe-auth 2>/dev/null || true + info "=== Enrollment endpoint ===" + info "${AUTH_HOST}/auth/enroll" +} + +# ── Full initialization ─────────────────────────────────────────────────────── + +cmd_initialize() { + require_tool kubectl + + info "=== knoe-auth initialization ===" + info " Mode: $MODE" + info " Realm: $REALM" + info " Cluster: $APP_CTX" + info " NS: $NAMESPACE" + info "" + + if [[ "$MODE" == "k3d" ]]; then + cmd_initialize_k3d + return + fi + + # 1. Ensure namespace exists + kube get namespace "$NAMESPACE" >/dev/null 2>&1 || \ + kube create namespace "$NAMESPACE" + + # 2. Secrets + create_kdc_secrets + create_google_oidc_secret + create_session_secret + create_oidc_path_b_secret + + # 3. Apply ConfigMap + Deployment + apply_manifests + + # 4. Wait for pods + wait_for_pods "app=knoe-auth" 240 + + # 5. Schema + run_schema + + # 6. Done + info "" + info "=== knoe-auth is ready ===" + info "Enrollment URL: ${AUTH_HOST}/auth/enroll?token=" + info "" + info "Next: create first admin invite:" + info " $0 invite chrisfu@knoey.com" + info "" + show_status +} + +cmd_initialize_k3d() { + info "=== k3d mode: provisioning KDC + schema ===" + + # 1. Ensure namespace exists + kube get namespace "$NAMESPACE" >/dev/null 2>&1 || \ + kube create namespace "$NAMESPACE" + + # 2. KDC secret (dev passwords, no 1Password) + create_kdc_secrets + + # 3. Apply KDC manifests + apply_manifests + + # 4. Wait for KDC pod + info "Waiting for KDC deployment to be ready..." + kube -n "$NAMESPACE" rollout status deployment/knoe-kdc --timeout=120s + + # 5. Schema (via kubectl exec into CNPG primary) + run_schema + + # 6. Dev-user seed: knoe_developer group + chrisfu role (k3d-only). + seed_dev_users_k3d + + info "" + info "=== k3d knoe-auth stack is ready ===" + info "Run: make k3d-knoe-pf" + info "Then in another terminal:" + info " export KRB5_CONFIG=\$PWD/etc/krb5.local.conf" + info " export KNOE_AUTH_OIDC_SIGNING_KEY=\$(cat etc/secrets/knoe-auth-oidc-key.b64)" + info " mvn -pl authority spring-boot:run -Dspring-boot.run.profiles=k3d" + info "" + info "Dev DB access (from host, with port-forward up):" + info " PGPASSWORD=chrisfu-dev psql -h localhost -U chrisfu -d knoe-db" + info "" + show_status +} + +# ── Dispatch ────────────────────────────────────────────────────────────────── + +case "$COMMAND" in + initialize) cmd_initialize ;; + schema) run_schema ;; + invite) create_first_invite "${1:-}" ;; + status) show_status ;; + *) + echo "Usage: $0 {initialize|schema|invite EMAIL|status} [--context CTX] [--namespace NS]" >&2 + exit 1 + ;; +esac diff --git a/mock_val/init_knoe_users.sh b/mock_val/init_knoe_users.sh new file mode 100755 index 0000000..5a3a8bd --- /dev/null +++ b/mock_val/init_knoe_users.sh @@ -0,0 +1,1360 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# init_knoe_users.sh +# Purpose: +# - Provision Kerberos principals and database accounts for knoe-system users +# - Creates: admin@KNOE.LOCAL (master password), guest@KNOE.LOCAL (read-only), +# postgres service principal (keytab for GSS auth), developer group role +# - Cross-realm trust with myrddin.prole.org PROLE.ORG is referenced via +# PROLE_KDC_TRUST_REALM=PROLE.ORG in init_kdc.sh / init_knoe_auth.sh +# - Sets up service admin access: ArgoCD RBAC, Gitea, GitLab +# - Ensures Gitea SPNEGO keytab (HTTP/git.prole.org@PROLE.ORG) is provisioned +# +# Prerequisites: +# init_kdc.sh initialize (with PROLE_KDC_REALM=PROLE.LOCAL, +# PROLE_KDC_TRUST_REALM=PROLE.ORG, +# KRB5_KDC=myrddin.prole.org) +# init_cnpg_backup.sh (CNPG cluster must exist) +# init_argocd.sh (ArgoCD must be running) +# +# Usage: +# ./init_knoe_users.sh initialize # create/update all user resources +# ./init_knoe_users.sh onboard [role] # add a single new user +# ./init_knoe_users.sh status # show current state +# ./init_knoe_users.sh cleanup # remove managed secrets/patches +# +# 1Password authentication (required for recovery credential generation): +# Preferred — run from your laptop with 1Password desktop open: +# op run -- bash etc/init_knoe_users.sh --mode k3s initialize +# Headless server — store a service account token in k8s first: +# kubectl -n knoe-system create secret generic op-service-account-token \ +# --from-literal=token= +# Manual — sign in interactively, then run normally: +# op signin && bash etc/init_knoe_users.sh --mode k3s initialize +# +# Browser SPNEGO (Kerberos SSO in Chrome/Edge): +# Deploy to all managed endpoints via Ansible — NOT a per-user step: +# ansible-playbook infrastructure/playbooks/workstation_kerberos.yml +# This deploys Chrome managed policy + krb5.conf. New-hire laptops get it +# automatically as part of the standard provisioning run. + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Pre-scan argv for --mode before sourcing knoe_cfg.sh. +# knoe_cfg.sh selects the .cfg file based on KNOE_MODE at source time; +# without this, KNOE_MODE is empty and k3d.cfg wins the fallback loop even +# when --mode k3s is passed — causing k3s.cfg values (e.g. KNOE_ADMIN_PRINCIPAL) +# to never be loaded. knoe_normalize_mode is not yet available here so we +# accept k3d/k3s/k8s verbatim; the canonical normalisation runs in knoe_set_mode. +_knoe_prescan_mode() { + local _prev="" + for _arg in "$@"; do + case "$_prev" in --mode|-m) export KNOE_MODE="$_arg"; return ;; esac + case "$_arg" in --mode=*|-m=*) export KNOE_MODE="${_arg#*=}"; return ;; esac + _prev="$_arg" + done +} +_knoe_prescan_mode "$@" +unset -f _knoe_prescan_mode + +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/knoe_cfg.sh" + +if [[ "${1:-}" == "--mode" || "${1:-}" == "-m" ]]; then + knoe_set_mode "${2:-}" + shift 2 +elif [[ "${1:-}" == --mode=* || "${1:-}" == -m=* ]]; then + knoe_set_mode "${1#*=}" + shift +fi + +knoe_ensure_kubeconfig >/dev/null 2>&1 || true +ensure_kube_context || exit 1 + +# Guard: reject the 'default' context in k3d mode — it means the desired +# k3d context was not found and a placeholder was silently accepted. +# In k3s/k8s mode the k3s-generated kubeconfig legitimately uses 'default' +# as the context name (e.g. on the cluster-server node itself), so allow it. +_active_ctx=$(kubectl config current-context 2>/dev/null || true) +_active_mode="${KNOE_MODE:-}" +if [[ "${_active_ctx:-}" == "default" && "$_active_mode" != "k3s" && "$_active_mode" != "k8s" ]]; then + printf '[ERROR] Active kubectl context is '\''default'\'' — the required context was not found.\n' >&2 + printf ' In k3d mode this indicates a missed context switch.\n' >&2 + printf ' Pass --mode k3s if you intend to target a k3s cluster, e.g.:\n' >&2 + printf ' %s --mode k3s initialize\n' "$(basename "$0")" >&2 + printf ' Or add the correct kubeconfig: export KUBECONFIG=%s:~/.kube/config\n' "${KUBECONFIG:-/path/to/knoe-k3s.kubeconfig}" >&2 + exit 1 +fi +unset _active_ctx _active_mode + +ACTION="${1:-initialize}" + +resolve_knoe_mode() { + local raw_mode="${KNOE_MODE:-${DEPLOYMENT_MODE:-${CLUSTER_ENV:-k3s}}}" + if command -v knoe_normalize_mode >/dev/null 2>&1; then + knoe_normalize_mode "$raw_mode" + return 0 + fi + raw_mode=$(printf '%s' "$raw_mode" | tr 'A-Z' 'a-z') + case "$raw_mode" in + prod|production) + printf 'k8s' + ;; + *) + printf '%s' "$raw_mode" + ;; + esac +} + +default_knoe_auth_deployment() { + case "$(resolve_knoe_mode)" in + k8s) + printf 'authority-gcp-auth' + ;; + *) + printf 'authority-knoe-auth' + ;; + esac +} + +# ── Configurable variables ───────────────────────────────────────────────── +# PROLE_KDC_NAMESPACE is the canonical env set by init_kdc.sh; honour it here +# so both scripts look at the same namespace for knoe-kdc-secrets. +# Never fall through to "default" — use knoe-system as the last resort. +KNOE_USERS_NAMESPACE="${KNOE_USERS_NAMESPACE:-${SERVICE_NAMESPACE:-knoe-system}}" +KNOE_KDC_NAMESPACE="${KNOE_KDC_NAMESPACE:-${PROLE_KDC_NAMESPACE:-${KNOE_USERS_NAMESPACE}}}" +KNOE_DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}" +KNOE_DB_CLUSTER="${KNOE_DB_CLUSTER:-knoe-db}" +KNOE_DEPLOYMENT_MODE="${KNOE_DEPLOYMENT_MODE:-$(resolve_knoe_mode)}" +KNOE_AUTH_DEPLOYMENT="${KNOE_AUTH_DEPLOYMENT:-${PROLE_KDC_NAME:-$(default_knoe_auth_deployment)}}" +KNOE_ADMIN_PRINCIPAL="${KNOE_ADMIN_PRINCIPAL:-admin}" +PROLE_KDC_REALM="${PROLE_KDC_REALM:-KNOE.LOCAL}" +GITEA_HOST="${GITEA_HOST:-git.prole.org}" +GITEA_SPNEGO_HOST="${GITEA_SPNEGO_HOST:-${GITEA_HOST}}" +GITEA_KRB5_AD_REALM="${GITEA_KRB5_AD_REALM:-PROLE.ORG}" +GITEA_KRB5_AD_USER="${GITEA_KRB5_AD_USER:-gitea-http}" +GITLAB_HOST="${GITLAB_HOST:-gitlab.prole.org}" +GITEA_NAMESPACE="${GITEA_NAMESPACE:-gitea}" +GITLAB_NAMESPACE="${GITLAB_NAMESPACE:-gitlab}" +ARGOCD_NAMESPACE="${ARGOCD_NAMESPACE:-argocd}" +# Provided externally or resolved from k8s secrets / 1Password +PROLE_KDC_MASTER_PASSWORD="${PROLE_KDC_MASTER_PASSWORD:-}" +KNOE_GUEST_PASSWORD="${KNOE_GUEST_PASSWORD:-}" +# 1Password secret references (op:// URIs). Set in conf/k3s.cfg or via env. +# When set and op CLI is available, these are used as fallback if the k8s +# secret is absent or empty. +OP_KDC_MASTER_PASSWORD_REF="${OP_KDC_MASTER_PASSWORD_REF:-}" +OP_KDC_GUEST_PASSWORD_REF="${OP_KDC_GUEST_PASSWORD_REF:-}" +# UI-supplied credentials (set by knoe_users.py from the Database / Kerberos Auth screens) +PROLE_LOCAL_ADMIN_PASSWORD="${PROLE_LOCAL_ADMIN_PASSWORD:-}" +PROLE_ORG_ADMIN_PASSWORD="${PROLE_ORG_ADMIN_PASSWORD:-}" +# Optional: personal access tokens for service admin promotion +GITEA_ADMIN_TOKEN="${GITEA_ADMIN_TOKEN:-}" +GITLAB_ADMIN_TOKEN="${GITLAB_ADMIN_TOKEN:-}" +# 1Password vault for Gitea/service credentials (default: Personal) +GITEA_OP_VAULT="${GITEA_OP_VAULT:-Personal}" + +log() { printf '[INFO] %s\n' "$*"; } + +# _op_ensure_auth +# Establishes a usable 1Password session for headless/automated use. +# Priority order: +# 1. OP_SERVICE_ACCOUNT_TOKEN already in env (Teams/Business service account, +# or injected by CI/CD). +# 2. K8s Secret "op-service-account-token" in SERVICE_NAMESPACE — for servers +# that run the script directly (e.g. myrddin running as the admin user). +# Create once: +# op service-account create "prole-init" --expires-in 1y \ +# --vault "${GITEA_OP_VAULT}" +# kubectl -n knoe-system create secret generic op-service-account-token \ +# --from-literal=token= +# Requires 1Password Teams/Business. +# 3. Existing interactive session — covers admin running the script from their +# laptop with 1Password desktop app open, or after a manual "op signin". +# 4. "op run --" inheritance — when the script is invoked as: +# op run -- bash etc/init_knoe_users.sh --mode k3s initialize +# op run sets OP_SESSION_* in the environment, which the op CLI picks up +# automatically. No extra code needed; this case falls through to (3). +# +# Returns 0 when 1Password is usable, 1 otherwise (non-fatal; callers warn). +_op_ensure_auth() { + command -v op >/dev/null 2>&1 || { warn "1Password CLI (op) not installed — skipping credential steps"; return 1; } + + # Already set (service account token, CI env, or op run --) + if [[ -n "${OP_SERVICE_ACCOUNT_TOKEN:-}" ]]; then + op account list >/dev/null 2>&1 && return 0 + # Token set but op still rejects — fall through to other methods + fi + + # Try k8s-stored service account token (headless server running this script) + local _ns="${SERVICE_NAMESPACE:-knoe-system}" + local _tok + _tok=$(kubectl -n "$_ns" get secret op-service-account-token \ + -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null || true) + if [[ -n "$_tok" ]]; then + export OP_SERVICE_ACCOUNT_TOKEN="$_tok" + log "1Password: using service account token from k8s secret op-service-account-token" >&2 + return 0 + fi + + # Try existing interactive session (op signin already done, or desktop app running) + if op account list >/dev/null 2>&1; then + return 0 + fi + + warn "1Password: not authenticated. Choose one of:" + warn " A) Run from your laptop (1Password desktop app handles auth automatically):" + warn " op run -- bash etc/init_knoe_users.sh --mode k3s initialize" + warn " B) Sign in once on this host, then re-run:" + warn " op signin && bash etc/init_knoe_users.sh --mode k3s initialize" + warn " C) Create a service account (1Password Teams/Business) and store token:" + warn " op service-account create 'prole-init' --expires-in 1y --vault '${GITEA_OP_VAULT}'" + warn " kubectl -n ${_ns} create secret generic op-service-account-token --from-literal=token=" + return 1 +} +err() { printf '[ERROR] %s\n' "$*" >&2; } +warn() { printf '[WARN] %s\n' "$*" >&2; } +die() { err "$*"; exit 1; } + +ensure_tools() { + for t in kubectl curl python3; do + command -v "$t" >/dev/null || die "Missing required tool: $t" + done +} + +b64_decode() { + if base64 --decode /dev/null 2>&1; then + base64 --decode + elif base64 -d /dev/null 2>&1; then + base64 -d + else + base64 -D + fi +} + +gen_password() { + if command -v openssl >/dev/null 2>&1; then + openssl rand -base64 18 + return 0 + fi + python3 - <<'PY' +import secrets, string +alphabet = string.ascii_letters + string.digits +print(''.join(secrets.choice(alphabet) for _ in range(24))) +PY +} + +# ── Secret helpers ───────────────────────────────────────────────────────── + +# Read a value from 1Password using the op CLI. +# Returns 1 (empty output) if op is not installed or the ref is empty. +# Usage: try_op_read "op://vault/item/field" +try_op_read() { + local ref="${1:-}" + [[ -n "$ref" ]] || return 1 + command -v op >/dev/null 2>&1 || return 1 + local val + val=$(op read "$ref" 2>/dev/null) || return 1 + [[ -n "$val" ]] || return 1 + printf '%s' "$val" +} + +# Read a base64-encoded key from a k8s Secret in KNOE_KDC_NAMESPACE. +# Mirrors get_secret_value() in init_kdc.sh (which reads from KDC_NAMESPACE). +get_secret_value() { + local secret="$1" key="$2" + kubectl -n "$KNOE_KDC_NAMESPACE" get secret "$secret" \ + -o "jsonpath={.data.${key}}" 2>/dev/null | b64_decode 2>/dev/null || true +} + +# ── KDC helpers ──────────────────────────────────────────────────────────── + +get_kdc_pod() { + # The kdc container lives inside the authority deployment pod selected by mode. + # Only return Running pods — Terminating pods from a prior rollout must be excluded. + local pod + pod=$(kubectl -n "$KNOE_KDC_NAMESPACE" get pods \ + -l "app=${KNOE_AUTH_DEPLOYMENT}" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -n "$pod" ]]; then + printf '%s' "$pod" + return 0 + fi + return 0 +} + +# _resolve_kdc_registry — sets _kdc_reg_internal and _kdc_reg_host in the +# caller's scope for use as env-var prefixes on init_kdc.sh invocations. +# In k3s mode derives correct values from KNOE_IMAGE_REGISTRY / PROLE_K3S_SERVER +# so stale k3d LOCAL_REGISTRY_INTERNAL values cannot leak into subprocesses. +_resolve_kdc_registry() { + _kdc_reg_internal="" + _kdc_reg_host="" + if [[ "$(resolve_knoe_mode)" == "k3s" ]]; then + if [[ -n "${KNOE_IMAGE_REGISTRY:-}" ]]; then + _kdc_reg_internal="${KNOE_IMAGE_REGISTRY}" + else + local _rns="${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-${KNOE_KDC_NAMESPACE}}}" + _kdc_reg_internal="registry.${_rns}.svc.cluster.local:5000" + fi + local _k3s_url="${PROLE_K3S_SERVER:-${K3S_SERVER:-${K3S_SERVER_URL:-}}}" + if [[ -n "${_k3s_url:-}" ]]; then + _kdc_reg_host="${_k3s_url#https://}" + _kdc_reg_host="${_kdc_reg_host#http://}" + _kdc_reg_host="${_kdc_reg_host%%:*}" + _kdc_reg_host="${_kdc_reg_host}:5000" + fi + fi +} + +ensure_kdc_pod() { + local kdc_pod init_kdc_script + kdc_pod=$(get_kdc_pod) + if [[ -n "$kdc_pod" ]]; then + printf '%s' "$kdc_pod" + return 0 + fi + + init_kdc_script="$SCRIPT_DIR/init_kdc.sh" + [[ -f "$init_kdc_script" ]] || die "Required script not found: $init_kdc_script" + + log "No authority pod found in namespace ${KNOE_KDC_NAMESPACE}; bootstrapping ${KNOE_AUTH_DEPLOYMENT} via init_kdc.sh ..." >&2 + + _resolve_kdc_registry + LOCAL_REGISTRY_INTERNAL="${_kdc_reg_internal}" \ + PROLE_KDC_REGISTRY_INTERNAL="${_kdc_reg_internal}" \ + PROLE_KDC_REGISTRY_HOST="${_kdc_reg_host}" \ + PROLE_KDC_NAMESPACE="$KNOE_KDC_NAMESPACE" \ + PROLE_KDC_MASTER_PASSWORD="$PROLE_KDC_MASTER_PASSWORD" \ + PROLE_KDC_NAME="$KNOE_AUTH_DEPLOYMENT" \ + bash "$init_kdc_script" initialize >&2 + + kdc_pod=$(get_kdc_pod) + [[ -n "$kdc_pod" ]] || die "No KDC pod found in namespace ${KNOE_KDC_NAMESPACE} (label app=${KNOE_AUTH_DEPLOYMENT})." + printf '%s' "$kdc_pod" +} + +kdc_principal_exists() { + local pod="$1" principal="$2" + kubectl -n "$KNOE_KDC_NAMESPACE" exec "$pod" -c kdc -- \ + kadmin.local -q "get_principal ${principal}" >/dev/null 2>&1 +} + +kdc_addprinc() { + local pod="$1" principal="$2" password="$3" + kubectl -n "$KNOE_KDC_NAMESPACE" exec "$pod" -c kdc -- \ + kadmin.local -q "addprinc -pw ${password} ${principal}" +} + +kdc_addprinc_randkey() { + local pod="$1" principal="$2" + kubectl -n "$KNOE_KDC_NAMESPACE" exec "$pod" -c kdc -- \ + kadmin.local -q "addprinc -randkey ${principal}" +} + +# ── CNPG helpers ─────────────────────────────────────────────────────────── + +get_cnpg_primary() { + local pod="" + # CNPG 1.20+: instanceRole label + pod=$(kubectl -n "$KNOE_DB_NAMESPACE" get pods \ + -l "cnpg.io/cluster=${KNOE_DB_CLUSTER},cnpg.io/instanceRole=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$pod" ]]; then + # Older label set + pod=$(kubectl -n "$KNOE_DB_NAMESPACE" get pods \ + -l "cnpg.io/cluster=${KNOE_DB_CLUSTER},role=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + fi + printf '%s' "$pod" +} + +cnpg_psql() { + local pod="$1"; shift + kubectl -n "$KNOE_DB_NAMESPACE" exec "$pod" -- \ + psql -U postgres -d postgres "$@" +} + +# ── Main: initialize ─────────────────────────────────────────────────────── + +initialize() { + ensure_tools + + log "=== knoe-system user provisioning ===" + log "Namespaces: kdc/authority=${KNOE_KDC_NAMESPACE} db=${KNOE_DB_NAMESPACE}" + log "Mode: ${KNOE_DEPLOYMENT_MODE} authority deployment: ${KNOE_AUTH_DEPLOYMENT}" + + # 1. Resolve KDC master password — try multiple sources in order: + # a) env var (already set) b) knoe-kdc-secrets k8s Secret + # c) prole-kdc-secrets k8s Secret d) 1Password (OP_KDC_MASTER_PASSWORD_REF) + # e) PROLE_LOCAL_ADMIN_PASSWORD f) die + if [[ -z "$PROLE_KDC_MASTER_PASSWORD" ]]; then + PROLE_KDC_MASTER_PASSWORD=$(get_secret_value "knoe-kdc-secrets" "master_password") + fi + if [[ -z "$PROLE_KDC_MASTER_PASSWORD" ]]; then + # The live prole deployment may use a differently-named secret + PROLE_KDC_MASTER_PASSWORD=$(get_secret_value "prole-kdc-secrets" "master_password") + [[ -n "$PROLE_KDC_MASTER_PASSWORD" ]] && log "Resolved KDC master password from prole-kdc-secrets" + fi + if [[ -z "$PROLE_KDC_MASTER_PASSWORD" && -n "${OP_KDC_MASTER_PASSWORD_REF:-}" ]]; then + log "Resolving KDC master password from 1Password (${OP_KDC_MASTER_PASSWORD_REF}) ..." + PROLE_KDC_MASTER_PASSWORD=$(try_op_read "$OP_KDC_MASTER_PASSWORD_REF" || true) + [[ -n "$PROLE_KDC_MASTER_PASSWORD" ]] && log "KDC master password resolved from 1Password" + fi + # Allow the UI-supplied PROLE_LOCAL_ADMIN_PASSWORD (DB master password field) to + # override when the secret is absent or empty. + if [[ -z "$PROLE_KDC_MASTER_PASSWORD" && -n "${PROLE_LOCAL_ADMIN_PASSWORD:-}" ]]; then + PROLE_KDC_MASTER_PASSWORD="$PROLE_LOCAL_ADMIN_PASSWORD" + log "Using PROLE_LOCAL_ADMIN_PASSWORD as PROLE_KDC_MASTER_PASSWORD" + fi + if [[ -z "$PROLE_KDC_MASTER_PASSWORD" ]]; then + err "PROLE_KDC_MASTER_PASSWORD not found." + err "The KDC master password must match the running KDC database and cannot be auto-generated." + err "Resolve one of:" + err " 1. Store in k8s secret:" + err " kubectl -n ${KNOE_KDC_NAMESPACE} patch secret knoe-kdc-secrets \\" + err " --type=merge -p '{\"data\":{\"master_password\":\"BASE64PW\"}}'" + err " 2. Store in 1Password and set in conf/k3s.cfg:" + err " OP_KDC_MASTER_PASSWORD_REF = op://personal//password" + err " then: op run -- bash etc/init_knoe_users.sh --mode k3s initialize" + err " 3. Export directly: export PROLE_KDC_MASTER_PASSWORD=" + err " 4. Check existing: kubectl -n ${KNOE_KDC_NAMESPACE} get secrets" + exit 1 + fi + + if [[ -z "$KNOE_GUEST_PASSWORD" ]]; then + KNOE_GUEST_PASSWORD=$(get_secret_value "knoe-kdc-secrets" "guest_password") + fi + if [[ -z "$KNOE_GUEST_PASSWORD" ]]; then + KNOE_GUEST_PASSWORD=$(get_secret_value "prole-kdc-secrets" "guest_password") + fi + if [[ -z "$KNOE_GUEST_PASSWORD" && -n "${OP_KDC_GUEST_PASSWORD_REF:-}" ]]; then + KNOE_GUEST_PASSWORD=$(try_op_read "$OP_KDC_GUEST_PASSWORD_REF" || true) + fi + if [[ -z "$KNOE_GUEST_PASSWORD" ]]; then + KNOE_GUEST_PASSWORD=$(gen_password) + log "Auto-generated guest password — storing in knoe-kdc-secrets ..." + if kubectl -n "$KNOE_KDC_NAMESPACE" get secret knoe-kdc-secrets >/dev/null 2>&1; then + local guest_b64 + guest_b64=$(printf '%s' "$KNOE_GUEST_PASSWORD" | base64 | tr -d '\n') + kubectl -n "$KNOE_KDC_NAMESPACE" patch secret knoe-kdc-secrets \ + --type=merge \ + --patch "{\"data\":{\"guest_password\":\"${guest_b64}\"}}" >/dev/null + else + kubectl -n "$KNOE_KDC_NAMESPACE" create secret generic knoe-kdc-secrets \ + --from-literal=guest_password="$KNOE_GUEST_PASSWORD" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + fi + fi + + # 2. Locate (or bootstrap) KDC pod + local kdc_pod + kdc_pod=$(ensure_kdc_pod) + log "KDC pod: $kdc_pod" + + # 2a. Pre-flight: verify the KDC database stash file is present and kadmin.local + # can access the database. A missing stash means init_kdc.sh has not been + # run (or the pod restarted with ephemeral storage — EmptyDir volumes are + # wiped on pod restart). Auto-recover by re-initialising the database. + log "Verifying KDC database accessibility ..." + if ! kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \ + sh -c 'kadmin.local -q listprincs' >/dev/null 2>&1; then + log "KDC database not accessible — pod may have restarted with ephemeral storage." + log "Re-initialising KDC database via init_kdc.sh ..." + local init_kdc_script="$SCRIPT_DIR/init_kdc.sh" + [[ -f "$init_kdc_script" ]] || die "Required script not found: $init_kdc_script" + _resolve_kdc_registry + LOCAL_REGISTRY_INTERNAL="${_kdc_reg_internal}" \ + PROLE_KDC_REGISTRY_INTERNAL="${_kdc_reg_internal}" \ + PROLE_KDC_REGISTRY_HOST="${_kdc_reg_host}" \ + PROLE_KDC_NAMESPACE="$KNOE_KDC_NAMESPACE" \ + PROLE_KDC_MASTER_PASSWORD="$PROLE_KDC_MASTER_PASSWORD" \ + PROLE_KDC_NAME="$KNOE_AUTH_DEPLOYMENT" \ + bash "$init_kdc_script" initialize >&2 + # Poll until the new pod's entrypoint finishes kdb5_util create. + # The rollout completes before the in-container DB init finishes (no readiness probe). + local _reinit_timeout=120 _reinit_interval=5 _reinit_elapsed=0 + log "Waiting up to ${_reinit_timeout}s for KDC database to become accessible ..." + while (( _reinit_elapsed < _reinit_timeout )); do + kdc_pod=$(get_kdc_pod) + if [[ -n "$kdc_pod" ]] && kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \ + sh -c 'kadmin.local -q listprincs' >/dev/null 2>&1; then + log "KDC database re-initialised successfully (pod: $kdc_pod)" + break + fi + sleep "$_reinit_interval" + _reinit_elapsed=$(( _reinit_elapsed + _reinit_interval )) + done + if (( _reinit_elapsed >= _reinit_timeout )); then + die "KDC database still not accessible after ${_reinit_timeout}s (pod: ${kdc_pod:-}). + Check init_kdc.sh logs above for errors." + fi + else + log "KDC database accessible" + fi + + # 3. Create admin@PROLE.LOCAL (UI login with master password) + local admin_princ="${KNOE_ADMIN_PRINCIPAL}@${PROLE_KDC_REALM}" + if kdc_principal_exists "$kdc_pod" "$admin_princ"; then + log "${admin_princ} already exists" + else + log "Creating ${admin_princ} ..." + kdc_addprinc "$kdc_pod" "$admin_princ" "$PROLE_KDC_MASTER_PASSWORD" + fi + + # 4. Create guest@PROLE.LOCAL (read-only) + local guest_princ="guest@${PROLE_KDC_REALM}" + if kdc_principal_exists "$kdc_pod" "$guest_princ"; then + log "${guest_princ} already exists" + else + log "Creating ${guest_princ} ..." + kdc_addprinc "$kdc_pod" "$guest_princ" "$KNOE_GUEST_PASSWORD" + fi + + # 5. Ensure BOTH cross-realm trust principals exist in the MIT KDC + # Direction A: krbtgt/PROLE.ORG@KNOE.LOCAL — cluster users → AD services + # (created by init_kdc.sh PROLE_KDC_TRUST_REALM mechanism) + # Direction B: krbtgt/KNOE.LOCAL@PROLE.ORG — AD users → cluster services ← THIS IS WHAT WE NEED + local trust_princ_in="krbtgt/PROLE.ORG@${PROLE_KDC_REALM}" + local trust_princ_out="krbtgt/${PROLE_KDC_REALM}@PROLE.ORG" + local trust_shared_pw + trust_shared_pw=$(kubectl -n "$KNOE_KDC_NAMESPACE" get secret knoe-kdc-secrets \ + -o jsonpath='{.data.trust_shared_password}' 2>/dev/null | b64_decode || true) + if [[ -z "$trust_shared_pw" ]]; then + warn "trust_shared_password not set in knoe-kdc-secrets; cross-realm trust principals will not be created" + warn "Add trust_shared_password to knoe-kdc-secrets and re-run" + else + # Direction A (init_kdc.sh may already have created this) + if kdc_principal_exists "$kdc_pod" "$trust_princ_in"; then + log "${trust_princ_in} already exists" + else + log "Creating cross-realm principal ${trust_princ_in} ..." + kdc_addprinc "$kdc_pod" "$trust_princ_in" "$trust_shared_pw" + fi + # Direction B — the principal that allows AD users to reach cluster services + if kdc_principal_exists "$kdc_pod" "$trust_princ_out"; then + log "${trust_princ_out} already exists" + else + log "Creating cross-realm principal ${trust_princ_out} ..." + kdc_addprinc "$kdc_pod" "$trust_princ_out" "$trust_shared_pw" + fi + fi + + # 6. Create postgres service principal + export keytab. + # Always (re)create the SPN with a fresh random key — the old key is gone when + # the KDC pod restarts on ephemeral storage. kadmin.local -q exits 0 even on + # failure, so we verify success by checking the keytab file was written. + local pg_spn="postgres/knoe-db-rw.${KNOE_DB_NAMESPACE}.svc.cluster.local@${PROLE_KDC_REALM}" + log "Creating/refreshing service principal ${pg_spn} ..." + kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \ + sh -c "kadmin.local -q 'delprinc -force ${pg_spn}' 2>/dev/null; kadmin.local -q 'addprinc -randkey ${pg_spn}'" + + log "Exporting keytab for ${pg_spn} ..." + kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \ + sh -c "rm -f /tmp/pg.keytab; kadmin.local -q 'ktadd -k /tmp/pg.keytab ${pg_spn}'" + + # kadmin.local -q exits 0 even on failure — verify the file was actually written + kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \ + sh -c '[ -s /tmp/pg.keytab ]' \ + || die "Keytab export failed for ${pg_spn} — /tmp/pg.keytab is missing or empty. Check KDC logs." + + log "Storing keytab in Secret knoe-db-pg-keytab ..." + local keytab_b64 + keytab_b64=$(kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \ + sh -c 'base64 /tmp/pg.keytab | tr -d "\n"') + [[ -n "$keytab_b64" ]] || die "base64 encoding of keytab produced empty output." + # Write secret into the DB namespace where CNPG mounts it + kubectl -n "$KNOE_DB_NAMESPACE" create secret generic knoe-db-pg-keytab \ + --from-literal=pg.keytab="$(printf '%s' "$keytab_b64" | b64_decode)" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + log "knoe-db-pg-keytab stored in namespace ${KNOE_DB_NAMESPACE}" + + # 6. Patch CNPG cluster with managed.roles (idempotent merge) + log "Patching CNPG cluster ${KNOE_DB_CLUSTER} managed.roles ..." + kubectl -n "$KNOE_DB_NAMESPACE" patch cluster "$KNOE_DB_CLUSTER" \ + --type=merge --patch '{ + "spec": { + "managed": { + "roles": [ + { + "name": "admin", + "ensure": "present", + "login": true, + "superuser": true, + "comment": "Kerberos admin principal — full cluster access" + }, + { + "name": "guest", + "ensure": "present", + "login": true, + "superuser": false, + "comment": "Kerberos guest principal — read-only access to demo schema" + }, + { + "name": "developer", + "ensure": "present", + "login": false, + "superuser": false, + "comment": "Developer group role — granted to knoe-system user accounts" + } + ] + } + } + }' + + # 7. Locate CNPG primary and run SQL + local primary + primary=$(get_cnpg_primary) + if [[ -z "$primary" ]]; then + warn "CNPG primary pod not found — skipping SQL setup (run again once cluster is ready)" + else + log "CNPG primary pod: $primary" + log "Setting up demo schema and role grants ..." + cnpg_psql "$primary" <<'SQL' +-- demo schema for guest read-only access (evolves over time) +CREATE SCHEMA IF NOT EXISTS demo; + +-- Ensure roles exist (CNPG managed.roles may not have reconciled yet) +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'admin') THEN + CREATE ROLE admin LOGIN SUPERUSER; + ELSE + ALTER ROLE admin SUPERUSER LOGIN; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'guest') THEN + CREATE ROLE guest LOGIN; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'developer') THEN + CREATE ROLE developer NOLOGIN; + END IF; +END $$; + +-- guest: read-only on demo schema +GRANT USAGE ON SCHEMA demo TO guest; +GRANT SELECT ON ALL TABLES IN SCHEMA demo TO guest; +ALTER DEFAULT PRIVILEGES IN SCHEMA demo GRANT SELECT ON TABLES TO guest; + +-- developer: read/write access on public and demo schemas +GRANT USAGE ON SCHEMA public TO developer; +GRANT USAGE ON SCHEMA demo TO developer; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO developer; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA demo TO developer; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO developer; +ALTER DEFAULT PRIVILEGES IN SCHEMA demo GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO developer; +SQL + log "SQL setup complete" + fi + + # 8. Create knoe.user schema tables and bootstrap users + if [[ -n "$primary" ]]; then + create_knoe_schema "$primary" + provision_user "$primary" "chrisfu" "chrisfu@prole.org" "Chris Fu" "admin" + # ron has no prole.org Google Workspace account; identity is ron@KNOE.LOCAL (Kerberos only) + provision_user "$primary" "ron" "" "Ron" "developer" + else + warn "CNPG primary not found — skipping knoe.user schema and user provisioning" + fi + + # 9. ArgoCD RBAC patch + patch_argocd_rbac + + # 10. Gitea SPNEGO keytab (HTTP/git.prole.org@PROLE.ORG) + ensure_gitea_spnego_keytab + + # 11. Gitea admin promotion + # Set KNOE_ADMIN_EMAIL so promote_gitea_admin can update the UUID placeholder + # email that Gitea assigns on SPNEGO auto-registration. + # Must match the email passed to provision_user for KNOE_ADMIN_PRINCIPAL above. + KNOE_ADMIN_EMAIL="${KNOE_ADMIN_EMAIL:-chrisfu@prole.org}" + promote_gitea_admin + + # 12. GitLab admin promotion + promote_gitlab_admin + + log "=== Initialization complete ===" + log "" + log "Next steps:" + if [[ "$KNOE_DEPLOYMENT_MODE" != "k3d" ]]; then + log " 1. KDC cross-realm principals: ensure krbtgt/PROLE.ORG@KNOE.LOCAL and krbtgt/KNOE.LOCAL@PROLE.ORG" + log " exist in the MIT KDC (created by init_kdc.sh or manually via kadmin.local)" + log "" + log " 2. Browser Kerberos SSO — deploy to ALL managed endpoints via Ansible (one-time, not per-user):" + log " ansible-playbook infrastructure/playbooks/workstation_kerberos.yml" + log " This deploys Chrome managed policy + krb5.conf pointing to ${KRB5_KDC:-myrddin.prole.org}." + log " New-hire laptops get it as part of standard provisioning. No per-user browser configuration." + log "" + log " 3. Onboard each new engineer:" + log " op run -- bash etc/init_knoe_users.sh --mode k3s onboard " + log " Creates Kerberos principal, sets 1Password recovery credential, and sends invite." + log "" + log " 4. 1Password recovery credentials — run this script with 1Password authenticated:" + log " op run -- bash etc/init_knoe_users.sh --mode k3s initialize" + log " Or store a service account token first (see script header for setup instructions)." + else + log " 1. After admin logs in to Gitea/GitLab for the first time, re-run: $0 initialize" + fi +} + +# ── knoe.user schema + user provisioning ────────────────────────────────── + +create_knoe_schema() { + local primary="$1" + log "Creating knoe.user schema tables (idempotent)..." + cnpg_psql "$primary" <<'SQL' +CREATE TABLE IF NOT EXISTS knoe.user ( + id SERIAL PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + realm TEXT NOT NULL DEFAULT 'KNOE.LOCAL', + email TEXT, + display_name TEXT, + tenant_realm TEXT, + is_realm_admin BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); +CREATE TABLE IF NOT EXISTS knoe.user_role ( + user_id INT NOT NULL REFERENCES knoe.user(id) ON DELETE CASCADE, + role TEXT NOT NULL, + granted_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (user_id, role) +); +GRANT SELECT, INSERT, UPDATE ON knoe.user TO knoe; +GRANT SELECT, INSERT, UPDATE ON knoe.user_role TO knoe; +GRANT USAGE, SELECT ON SEQUENCE knoe.user_id_seq TO knoe; +SQL + log "knoe.user schema ready." +} + +# provision_user +# role: admin | developer | guest (default: developer) +# Idempotent — safe to re-run. +provision_user() { + local primary="$1" username="$2" email="${3:-}" display_name="${4:-$2}" role="${5:-developer}" + + # 1. Kerberos principal (randkey — user sets own password via kpasswd / kadmin) + local user_princ="${username}@${PROLE_KDC_REALM}" + if [[ -n "$email" ]]; then + log "Provisioning user: ${username} kerberos=${user_princ} email=${email} role=${role}" + else + log "Provisioning user: ${username} kerberos=${user_princ} (no contact email) role=${role}" + fi + local kdc_pod + kdc_pod=$(get_kdc_pod) + if [[ -n "$kdc_pod" ]]; then + if kdc_principal_exists "$kdc_pod" "$user_princ"; then + log " Kerberos: ${user_princ} already exists" + else + log " Kerberos: creating ${user_princ} ..." + kdc_addprinc_randkey "$kdc_pod" "$user_princ" + fi + else + warn " No KDC pod — skipping Kerberos principal for ${username}" + fi + + # 2. PostgreSQL role (GSS-API: username@PROLE.LOCAL → pg role 'username') + cnpg_psql "$primary" -v username="$username" -v role="$role" </dev/null 2>&1; then + warn "argocd-rbac-cm not found in namespace ${ARGOCD_NAMESPACE}; skipping ArgoCD RBAC patch" + return 0 + fi + + local admin_line="g, ${KNOE_ADMIN_PRINCIPAL}, role:admin" + local current_csv + current_csv=$(kubectl -n "$ARGOCD_NAMESPACE" get configmap argocd-rbac-cm \ + -o jsonpath='{.data.policy\.csv}' 2>/dev/null || true) + + if printf '%s' "$current_csv" | grep -qF "$admin_line"; then + log "ArgoCD RBAC already contains admin entry" + return 0 + fi + + local new_csv="${current_csv}"$'\n'"${admin_line}" + local escaped_csv + escaped_csv=$(python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<< "$new_csv") + + kubectl -n "$ARGOCD_NAMESPACE" patch configmap argocd-rbac-cm \ + --type=merge \ + --patch "{\"data\":{\"policy.csv\":${escaped_csv}}}" >/dev/null + log "ArgoCD RBAC: added '${admin_line}'" +} + +# ── Gitea SPNEGO keytab ───────────────────────────────────────────────────── + +# Ensures the Secret gitea-krb5-keytab exists in GITEA_NAMESPACE. +# If samba-tool is available (i.e. running on myrddin), creates the service +# account, SPN, and keytab automatically. Otherwise prints the manual steps. +ensure_gitea_spnego_keytab() { + local spnego_principal="HTTP/${GITEA_SPNEGO_HOST}" + local spn="${spnego_principal}@${GITEA_KRB5_AD_REALM}" + local keytab_tmp="/tmp/gitea-http-$$.keytab" + + log "=== Gitea SPNEGO keytab (${GITEA_NAMESPACE}/gitea-krb5-keytab) ===" + + if kubectl -n "$GITEA_NAMESPACE" get secret gitea-krb5-keytab >/dev/null 2>&1; then + log "gitea-krb5-keytab already exists in namespace ${GITEA_NAMESPACE} — OK" + return 0 + fi + + log "gitea-krb5-keytab not found — attempting to provision ..." + + if ! command -v samba-tool >/dev/null 2>&1; then + warn "samba-tool not available — run the following on myrddin.prole.org as root:" + warn " samba-tool user create ${GITEA_KRB5_AD_USER} --random-password 2>/dev/null || true" + warn " samba-tool spn add ${spnego_principal} ${GITEA_KRB5_AD_USER}" + warn " samba-tool domain exportkeytab /tmp/http.keytab --principal ${spnego_principal}" + warn " klist -k /tmp/http.keytab # verify" + warn " kubectl -n ${GITEA_NAMESPACE} create secret generic gitea-krb5-keytab \\" + warn " --from-file=http.keytab=/tmp/http.keytab" + warn " rm -f /tmp/http.keytab" + warn " kubectl -n ${GITEA_NAMESPACE} rollout restart deployment gitea-spnego-proxy" + return 0 + fi + + log "Creating AD service account '${GITEA_KRB5_AD_USER}' (idempotent) ..." + samba-tool user create "$GITEA_KRB5_AD_USER" --random-password 2>/dev/null \ + && log "Created ${GITEA_KRB5_AD_USER}" \ + || log "${GITEA_KRB5_AD_USER} already exists" + + log "Adding SPN ${spnego_principal} → ${GITEA_KRB5_AD_USER} ..." + samba-tool spn add "$spnego_principal" "$GITEA_KRB5_AD_USER" \ + || warn "SPN add returned non-zero (may already exist)" + + log "Exporting keytab for ${spn} ..." + rm -f "$keytab_tmp" + samba-tool domain exportkeytab "$keytab_tmp" --principal "$spnego_principal" + + if [[ ! -s "$keytab_tmp" ]]; then + rm -f "$keytab_tmp" + die "Keytab export failed — ${keytab_tmp} is missing or empty. Check samba-tool output." + fi + + log "Storing keytab in Secret gitea-krb5-keytab ..." + kubectl -n "$GITEA_NAMESPACE" create secret generic gitea-krb5-keytab \ + --from-file=http.keytab="$keytab_tmp" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + rm -f "$keytab_tmp" + log "gitea-krb5-keytab provisioned — ${spn}" + + log "Restarting gitea-spnego-proxy to load new keytab ..." + kubectl -n "$GITEA_NAMESPACE" rollout restart deployment gitea-spnego-proxy 2>/dev/null || true +} + +# ── Gitea admin promotion ─────────────────────────────────────────────────── + +# ── Gitea credential management (1Password-backed) ──────────────────────── +# +# Design: Gitea passwords are vault credentials — strong, random, generated +# by 1Password, never logged, never stored in k8s secrets. Primary auth is +# always Kerberos SPNEGO; the password is an emergency recovery path only. +# API tokens (automation) are stored in k8s secret and are revocable. + +# gitea_op_item_title — canonical 1Password item title +gitea_op_item_title() { + printf 'Gitea — %s@%s' "$1" "${GITEA_HOST}" +} + +# gitea_ensure_password +# Prints a strong password to stdout (never to stderr/log). +# If a 1Password item exists: retrieves it (idempotent re-runs safe). +# If not: creates a new item with a generated password. +# Returns 1 if op CLI is unavailable or unauthenticated. +gitea_ensure_password() { + local username="$1" + local title + title=$(gitea_op_item_title "$username") + + # Establish 1Password session before any op commands. + _op_ensure_auth || return 1 + + # Retrieve existing password (silent on error = item not found yet) + local pw + pw=$(op item get "$title" --vault="${GITEA_OP_VAULT}" --fields=password 2>/dev/null) || true + if [[ -n "$pw" ]]; then + log " Gitea: retrieved existing credential for '${username}' from 1Password (vault: ${GITEA_OP_VAULT})" >&2 + printf '%s' "$pw" + return 0 + fi + + # Generate and store a new credential in 1Password. + # --generate-password uses 1Password's generator — no plaintext secret + # ever appears in the process environment or shell history. + log " Gitea: generating credential for '${username}' → 1Password vault '${GITEA_OP_VAULT}'" >&2 + op item create \ + --category=Login \ + --title="$title" \ + --vault="${GITEA_OP_VAULT}" \ + --url="https://${GITEA_HOST}" \ + --generate-password='letters,digits,32' \ + "username=${username}" \ + "notesPlain=Generated by init_knoe_users.sh. Primary auth is Kerberos SPNEGO — this password is for emergency recovery only." \ + >/dev/null 2>&1 || { warn " Gitea: failed to create 1Password item '${title}' (op signed in?)"; return 1; } + + pw=$(op item get "$title" --vault="${GITEA_OP_VAULT}" --fields=password 2>/dev/null) || true + if [[ -z "$pw" ]]; then + err " Gitea: 1Password item created but password could not be retrieved" + return 1 + fi + log " Gitea: credential stored as '${title}' in vault '${GITEA_OP_VAULT}'" >&2 + printf '%s' "$pw" +} + +# gitea_set_password +# Sets the Gitea user's password via kubectl exec (no prior Gitea auth needed). +gitea_set_password() { + local username="$1" pw="$2" + local pod + pod=$(kubectl -n "$GITEA_NAMESPACE" get pods \ + -l "app.kubernetes.io/name=gitea" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + [[ -n "$pod" ]] || { err " Gitea: no running pod in namespace ${GITEA_NAMESPACE}"; return 1; } + # gitea refuses to run as root; su to the git user (uid 1000). + # Capture stderr so we can surface it on failure without ever logging the password. + local _out + if ! _out=$(kubectl -n "$GITEA_NAMESPACE" exec "$pod" -c gitea -- \ + su git -s /bin/bash -c \ + "gitea admin user change-password --username '${username}' --password '${pw}'" \ + 2>&1); then + warn " Gitea: change-password failed for '${username}': ${_out}" + return 1 + fi +} + +# gitea_api_token [token_name] +# Obtains a fresh Gitea API token via REST basic-auth. +# Deletes any existing token with the same name first (idempotency). +# Prints the token sha1 to stdout; never logs it. +gitea_api_token() { + local username="$1" pw="$2" token_name="${3:-knoe-installer}" + # Remove stale token if present (ignore errors — may not exist) + curl -s -o /dev/null \ + -u "${username}:${pw}" \ + -X DELETE \ + "https://${GITEA_HOST}/api/v1/users/${username}/tokens/${token_name}" 2>/dev/null || true + # Capture body + HTTP status separately so we can log failures without + # exposing the password. The password never appears in the log/stderr. + # Gitea ≥1.22 requires explicit scopes; request all admin + user scopes. + local resp http_code + resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \ + -u "${username}:${pw}" \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"${token_name}\",\"scopes\":[\"read:admin\",\"write:admin\",\"read:user\",\"write:user\",\"read:issue\",\"write:issue\",\"read:repository\",\"write:repository\"]}" \ + "https://${GITEA_HOST}/api/v1/users/${username}/tokens" 2>/dev/null) || true + http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true) + resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__') + if [[ "$http_code" != "201" ]]; then + warn " Gitea: token POST returned HTTP ${http_code:-???} for '${username}' (body: ${resp:0:120})" + fi + # Extract sha1 from {"id":...,"name":...,"sha1":"",...} + printf '%s' "$resp" | grep -o '"sha1":"[^"]*"' | cut -d'"' -f4 +} + +# gitea_helm_admin_token [token_name] +# Reads the Helm bootstrap admin credentials from the Gitea Deployment's +# configure-gitea init container environment variables (GITEA_ADMIN_USERNAME / +# GITEA_ADMIN_PASSWORD). The Helm chart stores these as plain values in the +# pod spec — there is no k8s Secret holding the admin password. +# +# This works even when the gitea admin CLI is broken (e.g. wrong DB hostname +# in app.ini) because the request goes through the running web server which +# has the correct GITEA__database__HOST env var. +# Prints the token sha1 to stdout; returns 1 on failure. +gitea_helm_admin_token() { + local token_name="${1:-knoe-installer-helm}" + local release="${GITEA_HELM_RELEASE:-gitea}" + + # The Helm chart renders admin credentials as plain values (not secretKeyRefs) + # in the configure-gitea init container. However, if the Deployment has been + # updated (e.g. a failed upgrade) but the new pod never became Ready, the + # RUNNING pod still carries the old init container's password. + # + # Strategy: build a list of candidate passwords by reading from: + # 1. The ReplicaSet that owns the currently RUNNING pod (highest priority — + # this is what was actually applied to the live admin account) + # 2. The current Deployment spec (fallback — used when everything is in sync) + local admin_user admin_pw="" + + admin_user=$(kubectl -n "$GITEA_NAMESPACE" get deployment "$release" \ + -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="configure-gitea")].env[?(@.name=="GITEA_ADMIN_USERNAME")].value}' \ + 2>/dev/null || true) + # Env-var override takes precedence (useful for testing / manual bootstrap) + admin_user="${GITEA_HELM_ADMIN_USER:-${admin_user}}" + + if [[ -z "$admin_user" ]]; then + warn " Gitea: could not determine Helm admin username from Deployment '${release}'" + return 1 + fi + + # Gather candidate passwords: running-pod RS first, then current deployment. + local _running_rs _rs_pw _deploy_pw _tok + _running_rs=$(kubectl -n "$GITEA_NAMESPACE" get pod \ + -l "app.kubernetes.io/name=${release}" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.ownerReferences[?(@.kind=="ReplicaSet")].name}' \ + 2>/dev/null || true) + _rs_pw="" + if [[ -n "$_running_rs" ]]; then + _rs_pw=$(kubectl -n "$GITEA_NAMESPACE" get rs "$_running_rs" \ + -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="configure-gitea")].env[?(@.name=="GITEA_ADMIN_PASSWORD")].value}' \ + 2>/dev/null || true) + fi + _deploy_pw=$(kubectl -n "$GITEA_NAMESPACE" get deployment "$release" \ + -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="configure-gitea")].env[?(@.name=="GITEA_ADMIN_PASSWORD")].value}' \ + 2>/dev/null || true) + + local tok="" + for _pw in "$_rs_pw" "$_deploy_pw"; do + [[ -z "$_pw" ]] && continue + _tok=$(gitea_api_token "$admin_user" "$_pw" "$token_name") + if [[ -n "$_tok" ]]; then + tok="$_tok" + break + fi + done + + if [[ -z "$tok" ]]; then + warn " Gitea: could not obtain API token for Helm admin '${admin_user}'" + return 1 + fi + log " Gitea: obtained bootstrap token from Helm admin '${admin_user}'" >&2 + printf '%s' "$tok" +} + +# gitea_api_set_password +# Changes a user's password via the Gitea admin REST API. +# Uses the running web server (not the CLI), so it works even when app.ini +# has a wrong database hostname (the server overrides via GITEA__ env vars). +# +# NOTE: Gitea's AdminEditUser PATCH requires 'email' alongside login_name/source_id. +# Without it the server returns 200 but silently discards the password field. +# We fetch the current email first; if that fails we fall back to a configured default. +gitea_api_set_password() { + local token="$1" username="$2" pw="$3" + + # Fetch the user's current email — required by Gitea's admin PATCH endpoint. + local user_resp user_email + user_resp=$(curl -s \ + -H "Authorization: token ${token}" \ + "https://${GITEA_HOST}/api/v1/users/${username}" 2>/dev/null) || true + user_email=$(printf '%s' "$user_resp" | grep -o '"email":"[^"]*"' | cut -d'"' -f4 || true) + + # If the stored email is a UUID placeholder (SPNEGO auto-reg), substitute a real one. + if [[ -z "$user_email" ]] || printf '%s' "$user_email" | grep -qE '^[0-9a-f-]{36}@localhost$'; then + user_email="${KNOE_ADMIN_EMAIL:-${username}@${GITEA_HOST}}" + fi + + local resp http_code + resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \ + -X PATCH \ + -H "Authorization: token ${token}" \ + -H "Content-Type: application/json" \ + -d "{\"login_name\":\"${username}\",\"source_id\":0,\"password\":\"${pw}\",\"email\":\"${user_email}\",\"must_change_password\":false}" \ + "https://${GITEA_HOST}/api/v1/admin/users/${username}" 2>/dev/null) || true + http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true) + resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__') + + if [[ "$http_code" =~ ^2 ]]; then + return 0 + else + warn " Gitea: admin API set-password returned HTTP ${http_code:-???} for '${username}': ${resp:0:200}" + return 1 + fi +} + +promote_gitea_admin() { + local token="$GITEA_ADMIN_TOKEN" + + # 1. Try GITEA_ADMIN_TOKEN from env or persisted k8s secret + if [[ -z "$token" ]]; then + token=$(kubectl -n "$GITEA_NAMESPACE" get secret gitea-admin-token \ + -o jsonpath='{.data.token}' 2>/dev/null | b64_decode || true) + fi + + # 2. Bootstrap via the Helm admin account (k8s secret → REST API token). + # This path works even when the gitea admin CLI is broken (app.ini has wrong + # DB hostname) because REST calls go through the running server which has the + # correct GITEA__database__HOST env var. + local _helm_token="" + if [[ -z "$token" ]]; then + if _helm_token=$(gitea_helm_admin_token "knoe-installer-helm"); then + token="$_helm_token" + fi + fi + + # 3. Try 1Password → set password via CLI → exchange for token. + # Kept as fallback for when the Helm admin secret is absent (e.g. non-Helm deploy). + if [[ -z "$token" ]]; then + local _pw="" + if _pw=$(gitea_ensure_password "${KNOE_ADMIN_PRINCIPAL}"); then + log "Gitea: setting password for '${KNOE_ADMIN_PRINCIPAL}' via kubectl exec ..." + if gitea_set_password "${KNOE_ADMIN_PRINCIPAL}" "$_pw"; then + token=$(gitea_api_token "${KNOE_ADMIN_PRINCIPAL}" "$_pw" "knoe-installer") + [[ -n "$token" ]] && log "Gitea: API token obtained for '${KNOE_ADMIN_PRINCIPAL}'" + else + warn "Gitea: CLI password change failed (DB host mismatch in app.ini?)" + fi + _pw="" + fi + fi + + # 4. Last resort: kubectl exec generate-access-token (CLI path, may fail if DB broken). + if [[ -z "$token" ]]; then + local _gitea_pod + _gitea_pod=$(kubectl -n "$GITEA_NAMESPACE" get pods \ + -l "app.kubernetes.io/name=gitea" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -n "$_gitea_pod" ]]; then + log "Gitea: attempting token generation via kubectl exec (last resort) ..." + local _u + for _u in "gitea_admin" "${KNOE_ADMIN_PRINCIPAL}"; do + token=$(kubectl -n "$GITEA_NAMESPACE" exec "$_gitea_pod" -c gitea -- \ + su git -s /bin/bash -c \ + "gitea admin user generate-access-token --username ${_u} --token-name knoe-installer --scopes write:admin,read:user --raw 2>/dev/null" \ + 2>/dev/null | tail -1 || true) + [[ -n "$token" ]] && { log "Gitea: token generated for user '${_u}'"; break; } + done + fi + fi + + if [[ -z "$token" ]]; then + warn "No Gitea admin token available — skipping admin promotion" + warn " Ensure 'op signin' is active and the Helm admin secret exists, then re-run." + warn " Or set GITEA_ADMIN_TOKEN and re-run." + return 0 + fi + + # Persist token in k8s secret for automation reuse (tokens are revocable) + kubectl -n "$GITEA_NAMESPACE" create secret generic gitea-admin-token \ + --from-literal=token="$token" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null 2>&1 || true + + local http_code resp + # Check whether the Gitea account for KNOE_ADMIN_PRINCIPAL exists yet. + # Gitea auto-registers users on first SPNEGO login via the reverse proxy. + resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \ + -H "Authorization: token ${token}" \ + "https://${GITEA_HOST}/api/v1/users/${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null || true) + http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true) + resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__') + + log " Gitea: user check for '${KNOE_ADMIN_PRINCIPAL}' → HTTP ${http_code:-???}" + + if [[ "$http_code" == "404" ]]; then + warn "Gitea: user '${KNOE_ADMIN_PRINCIPAL}' not found — SPNEGO auto-registration not yet triggered" + warn " Visit https://${GITEA_HOST} with a valid Kerberos ticket, then re-run: $0 initialize" + return 0 + fi + + # Fetch existing email — Gitea's edit-user API requires it to avoid resetting. + # Prefer GITEA_ADMIN_EMAIL (may be set by caller or env), then the current + # Gitea value (skip UUID placeholders from SPNEGO auto-registration), + # then fall back to the user's configured email from the users list. + local existing_email current_email + current_email=$(printf '%s' "$resp" | grep -o '"email":"[^"]*"' | cut -d'"' -f4 || true) + if [[ -n "${GITEA_ADMIN_EMAIL:-}" ]]; then + existing_email="${GITEA_ADMIN_EMAIL}" + elif [[ -n "$current_email" ]] && ! printf '%s' "$current_email" | grep -qE '^[0-9a-f-]{36}@localhost$'; then + existing_email="$current_email" + else + # Current email is a UUID placeholder; use provisioned email if available, + # otherwise fall back to a reasonable default. + existing_email="${KNOE_ADMIN_EMAIL:-${KNOE_ADMIN_PRINCIPAL}@${GITEA_HOST}}" + fi + + # Admin promotion — sets admin:true. Must happen BEFORE the recovery password + # is set, because Gitea's AdminEditUser with source_id:0 on a SPNEGO-registered + # user can reset the password field as a side effect of re-initialising the auth + # record. Setting the password last guarantees it survives the promotion PATCH. + resp=$(curl -s -w '\n__HTTP_STATUS__%{http_code}' \ + -X PATCH \ + -H "Authorization: token ${token}" \ + -H "Content-Type: application/json" \ + -d "{\"admin\":true,\"login_name\":\"${KNOE_ADMIN_PRINCIPAL}\",\"source_id\":0,\"email\":\"${existing_email}\"}" \ + "https://${GITEA_HOST}/api/v1/admin/users/${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null || true) + http_code=$(printf '%s' "$resp" | grep -o '__HTTP_STATUS__[0-9]*' | tr -d '_A-Z' || true) + resp=$(printf '%s' "$resp" | grep -v '__HTTP_STATUS__') + + if [[ "$http_code" =~ ^2 ]]; then + log "Gitea: '${KNOE_ADMIN_PRINCIPAL}' promoted to admin" + else + warn "Gitea admin promotion returned HTTP ${http_code:-???} (${resp:0:200})" + fi + + # Set recovery password AFTER admin promotion. The promotion PATCH (source_id:0) + # can reset the password as a side effect on SPNEGO-registered users; setting it + # last ensures 1Password and Gitea agree on the same value. + # Skipped gracefully if op is unavailable (automated CI, no 1Password session). + if _op_ensure_auth 2>/dev/null; then + local _pw="" + if _pw=$(gitea_ensure_password "${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null); then + log "Gitea: setting recovery password for '${KNOE_ADMIN_PRINCIPAL}' via admin REST API ..." + gitea_api_set_password "$token" "${KNOE_ADMIN_PRINCIPAL}" "$_pw" \ + && log "Gitea: recovery password set for '${KNOE_ADMIN_PRINCIPAL}'" \ + || true + _pw="" + else + warn " Gitea: 1Password unavailable (op not signed in?) — skipping recovery password set" + warn " Run 'op signin' on $(hostname) and re-run to store recovery credentials" + fi + fi +} + +# ── GitLab admin promotion ───────────────────────────────────────────────── + +promote_gitlab_admin() { + local token="$GITLAB_ADMIN_TOKEN" + if [[ -z "$token" ]]; then + warn "No GITLAB_ADMIN_TOKEN (personal access token) available — skipping GitLab admin promotion" + warn " Set GITLAB_ADMIN_TOKEN and re-run, or use:" + warn " kubectl exec -n ${GITLAB_NAMESPACE} -- gitlab-rails runner \\" + warn " \"User.find_by_username('${KNOE_ADMIN_PRINCIPAL}')&.update(admin: true)\"" + return 0 + fi + + # Look up user ID by username + local user_json user_id + user_json=$(curl -s \ + -H "PRIVATE-TOKEN: ${token}" \ + "https://${GITLAB_HOST}/api/v4/users?username=${KNOE_ADMIN_PRINCIPAL}" 2>/dev/null || true) + user_id=$(python3 -c \ + 'import sys,json; u=json.load(sys.stdin); print(u[0]["id"] if u else "")' \ + <<< "$user_json" 2>/dev/null || true) + + if [[ -z "$user_id" ]]; then + warn "GitLab: user '${KNOE_ADMIN_PRINCIPAL}' not found (must log in first)" + warn " Re-run '$0 initialize' after first login" + return 0 + fi + + local http_code + http_code=$(curl -s -o /dev/null -w '%{http_code}' \ + -X PUT \ + -H "PRIVATE-TOKEN: ${token}" \ + -H "Content-Type: application/json" \ + -d '{"admin":true}' \ + "https://${GITLAB_HOST}/api/v4/users/${user_id}" 2>/dev/null || true) + + if [[ "$http_code" =~ ^2 ]]; then + log "GitLab: '${KNOE_ADMIN_PRINCIPAL}' promoted to admin" + else + warn "GitLab admin promotion returned HTTP ${http_code}" + fi +} + +# ── Status ───────────────────────────────────────────────────────────────── + +show_status() { + log "=== KDC Principals ===" + local kdc_pod + kdc_pod=$(get_kdc_pod) + if [[ -n "$kdc_pod" ]]; then + kubectl -n "$KNOE_KDC_NAMESPACE" exec "$kdc_pod" -c kdc -- \ + kadmin.local -q "list_principals" 2>/dev/null \ + | grep -E "^(admin|guest|developer|postgres/|krbtgt/)" || true + else + warn "No KDC pod found" + fi + + log "" + log "=== CNPG Managed Roles ===" + kubectl -n "$KNOE_DB_NAMESPACE" get cluster "$KNOE_DB_CLUSTER" \ + -o jsonpath='{.spec.managed.roles}' 2>/dev/null \ + | python3 -m json.tool 2>/dev/null || warn "Could not read managed.roles" + + log "" + log "=== PostgreSQL Roles ===" + local primary + primary=$(get_cnpg_primary) + if [[ -n "$primary" ]]; then + cnpg_psql "$primary" -c "\\du admin guest developer" 2>/dev/null || true + else + warn "No CNPG primary pod found" + fi + + log "" + log "=== Keytab Secrets ===" + if kubectl -n "$KNOE_DB_NAMESPACE" get secret knoe-db-pg-keytab >/dev/null 2>&1; then + log "knoe-db-pg-keytab exists in namespace ${KNOE_DB_NAMESPACE}" + else + warn "knoe-db-pg-keytab NOT FOUND in namespace ${KNOE_DB_NAMESPACE}" + fi + if kubectl -n "$GITEA_NAMESPACE" get secret gitea-krb5-keytab >/dev/null 2>&1; then + log "gitea-krb5-keytab exists in namespace ${GITEA_NAMESPACE} (HTTP/${GITEA_SPNEGO_HOST}@${GITEA_KRB5_AD_REALM})" + else + warn "gitea-krb5-keytab NOT FOUND in namespace ${GITEA_NAMESPACE}" + fi + + log "" + log "=== ArgoCD RBAC ===" + kubectl -n "$ARGOCD_NAMESPACE" get configmap argocd-rbac-cm \ + -o jsonpath='{.data.policy\.csv}' 2>/dev/null || warn "argocd-rbac-cm not found" +} + +# ── Cleanup ──────────────────────────────────────────────────────────────── + +cleanup() { + warn "cleanup removes the pg keytab secret and clears managed.roles from CNPG." + warn "KDC principals are NOT deleted (use kadmin.local to remove them manually)." + + kubectl -n "$KNOE_DB_NAMESPACE" delete secret knoe-db-pg-keytab \ + --ignore-not-found >/dev/null && log "Deleted knoe-db-pg-keytab" + + kubectl -n "$KNOE_DB_NAMESPACE" patch cluster "$KNOE_DB_CLUSTER" \ + --type=merge --patch '{"spec":{"managed":null}}' >/dev/null 2>&1 \ + && log "Cleared managed.roles from cluster ${KNOE_DB_CLUSTER}" || true +} + +# ── Dispatch ─────────────────────────────────────────────────────────────── + +case "$ACTION" in + initialize|init) initialize ;; + status) show_status ;; + cleanup) cleanup ;; + *) + err "Unknown action: ${ACTION}" + err "Usage: $(basename "$0") [initialize|status|cleanup]" + exit 1 + ;; +esac diff --git a/mock_val/init_kong.sh b/mock_val/init_kong.sh index 3269b95..550a338 100755 --- a/mock_val/init_kong.sh +++ b/mock_val/init_kong.sh @@ -22,10 +22,13 @@ _has_config=0 for _arg in "$@"; do [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1 done -if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - set -- "-c" "$SCRIPT_DIR/../conf/knoe.cfg" "$@" +if [[ $_has_config -eq 0 ]]; then + _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)" + if [[ -n "$_default_cfg" ]]; then + set -- "-c" "$_default_cfg" "$@" + fi fi -unset _has_config _arg +unset _has_config _arg _default_cfg common_core_preparse_config "$@" @@ -39,6 +42,73 @@ if [[ -z "${KNOE_MODE:-}" ]]; then export KNOE_MODE="k3s" fi +resolve_explicit_kube_context() { + local ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}" + if [[ -n "$ctx" ]]; then + printf '%s' "$ctx" + return 0 + fi + return 1 +} + +enforce_app_cluster_targeting() { + if [[ "${KNOE_MODE:-}" != "k8s" ]]; then + return 0 + fi + local app_ctx="${APP_CLUSTER_KUBECONTEXT:-}" + local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}" + local target_ctx + target_ctx="$(resolve_explicit_kube_context || true)" + + if [[ -z "$app_ctx" ]]; then + echo "ERROR: APP_CLUSTER_KUBECONTEXT is required for k8s Kong deployment." >&2 + exit 2 + fi + if [[ -z "$target_ctx" ]]; then + echo "ERROR: explicit kubectl context is required for k8s Kong deployment." >&2 + exit 2 + fi + if [[ -n "$db_ctx" && "$target_ctx" == "$db_ctx" ]]; then + echo "ERROR: refusing Kong APP step against DB context '$target_ctx'." >&2 + exit 2 + fi + if [[ "$target_ctx" != "$app_ctx" ]]; then + echo "ERROR: Kong APP step must target APP_CLUSTER_KUBECONTEXT='$app_ctx' (got '$target_ctx')." >&2 + exit 2 + fi + + export KUBECTL_CONTEXT="$app_ctx" + export KUBE_CONTEXT_NAME="$app_ctx" + export KUBECONTEXT="$app_ctx" +} + +kubectl() { + local target_ctx + target_ctx="$(resolve_explicit_kube_context || true)" + if [[ "${KNOE_MODE:-}" == "k8s" && -z "$target_ctx" ]]; then + echo "ERROR: explicit kubectl context is required in k8s mode." >&2 + return 2 + fi + + local arg has_context=0 + for arg in "$@"; do + case "$arg" in + --context|--context=*|--server|--server=*) + has_context=1 + break + ;; + esac + done + + if [[ -n "$target_ctx" && $has_context -eq 0 ]]; then + command kubectl --context "$target_ctx" "$@" + else + command kubectl "$@" + fi +} + +enforce_app_cluster_targeting + if [[ "${COMMON_CORE_HELP:-0}" == 1 ]]; then common_core_usage "$0" exit 0 @@ -56,16 +126,43 @@ common_core_apply_namespace "$NAMESPACE" KNOE_HOME=${KNOE_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)} KONG_IMAGE="${KONG_IMAGE:-kong:3.9}" -KONG_NAME="${KONG_NAME:-knoe-svc-kong}" +# k8s/GKE prod mode uses knoe.dev domain and knoe-svc-kong; all other modes use knoe.org +if [[ "${KNOE_MODE:-}" == "k8s" ]]; then + KONG_NAME="${KONG_NAME:-knoe-svc-kong}" + KONG_CONFIG_NAME="${KONG_CONFIG_NAME:-knoe-svc-kong-config}" + SERVICE_HOSTNAME="${SERVICE_HOSTNAME:-svc.knoe.dev}" + AUTH_HOSTNAME="${AUTH_HOSTNAME:-api.knoe.dev}" + GITEA_HOSTNAME="${GITEA_HOSTNAME:-${GITEA_DOMAIN:-git.knoe.dev}}" + SERVICE_INGRESS_TLS_ENABLED="${SERVICE_INGRESS_TLS_ENABLED:-0}" +else + KONG_NAME="${KONG_NAME:-knoe-svc-kong}" + KONG_CONFIG_NAME="${KONG_CONFIG_NAME:-knoe-svc-kong-config}" + SERVICE_HOSTNAME="${SERVICE_HOSTNAME:-svc.prole.org}" + AUTH_HOSTNAME="${AUTH_HOSTNAME:-api.prole.org}" + GITEA_HOSTNAME="${GITEA_HOSTNAME:-${GITEA_DOMAIN:-git.prole.org}}" + DB_HOSTNAME="${DB_HOSTNAME:-db.prole.org}" + SERVICE_INGRESS_TLS_ENABLED="${SERVICE_INGRESS_TLS_ENABLED:-1}" +fi KONG_PROXY_PORT="${KONG_PROXY_PORT:-8000}" KONG_ADMIN_PORT="${KONG_ADMIN_PORT:-8001}" -KONG_CONFIG_NAME="${KONG_CONFIG_NAME:-knoe-svc-kong-config}" - -# Public service entrypoint (single source of truth from knoe.cfg via knoe_cfg.sh) -SERVICE_HOSTNAME="${SERVICE_HOSTNAME:-svc.knoe.org}" +KONG_GITEA_SSH_PORT="${KONG_GITEA_SSH_PORT:-3022}" SERVICE_TLS_SECRET_NAME="${SERVICE_TLS_SECRET_NAME:-${SERVICE_HOSTNAME//./-}-tls}" SERVICE_TLS_CLUSTER_ISSUER="${SERVICE_TLS_CLUSTER_ISSUER:-letsencrypt-prod}" +# Optional: pin the GCE L7 svc-knoe-ingress to a pre-reserved GLOBAL external +# static IP (gcloud compute addresses create --global). Prevents IP churn on +# ingress delete/recreate. Leave blank to let GCE assign ephemerally. +SVC_KNOE_GLOBAL_STATIC_IP_NAME="${SVC_KNOE_GLOBAL_STATIC_IP_NAME:-}" + +# BackendConfig name for the knoe-svc-kong Service. GCE's default L7 health +# check hits HTTP `/` on the backend port, which Kong responds to with 404 +# (no route) -- marking the backend UNHEALTHY and causing the LB to return +# "Server Error" instead of reaching Kong. We instead point the GCE LB at a +# TCP health check (port-level liveness) so the backend passes as long as +# Kong is accepting connections, which is sufficient for our traffic shape. +# Mirrors the pattern in etc/init_gitlab.sh (gitlab-webservice-backendconfig). +SVC_KNOE_BACKEND_CONFIG_NAME="${SVC_KNOE_BACKEND_CONFIG_NAME:-knoe-svc-kong-backendconfig}" + # Legacy: svc-check used to own svc.knoe.org. We now route the service hostname # to Grafana, so remove any leftover svc-check resources to avoid conflicts. SVC_CHECK_NAMESPACE="${SVC_CHECK_NAMESPACE:-svc-check}" @@ -76,14 +173,27 @@ KUBECTL_APPLY_RETRIES="${KUBECTL_APPLY_RETRIES:-5}" KUBECTL_APPLY_RETRY_DELAY="${KUBECTL_APPLY_RETRY_DELAY:-2}" # Upstream service defaults +# oauth2-proxy for Supabase Studio (db.prole.org) — deployed by supabase/helm/oauth2-proxy +OAUTH2_PROXY_SERVICE="${OAUTH2_PROXY_SERVICE:-oauth2-proxy}" +OAUTH2_PROXY_NAMESPACE="${OAUTH2_PROXY_NAMESPACE:-supabase}" +OAUTH2_PROXY_PORT="${OAUTH2_PROXY_PORT:-80}" DB_MANAGER_SERVICE="${DB_MANAGER_SERVICE:-knoe-db-manager}" DB_MANAGER_PORT="${DB_MANAGER_PORT:-80}" -KNOE_SERVICE_UPSTREAM_URL="${KNOE_SERVICE_UPSTREAM_URL:-http://knoe-svc.knoe-db.svc.cluster.local:8080}" +DB_MANAGER_NAMESPACE="${DB_MANAGER_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}" +KNOE_SERVICE_UPSTREAM_URL="${KNOE_SERVICE_UPSTREAM_URL:-http://knoe-svc.${NAMESPACE}.svc.cluster.local:8080}" GRAFANA_UPSTREAM_URL="${GRAFANA_UPSTREAM_URL:-http://kps-grafana.monitoring.svc.cluster.local:80}" +GITEA_HTTP_UPSTREAM_URL="${GITEA_HTTP_UPSTREAM_URL:-http://gitea-http.gitea.svc.cluster.local:3000}" +GITEA_SSH_UPSTREAM_HOST="${GITEA_SSH_UPSTREAM_HOST:-gitea-ssh.gitea.svc.cluster.local}" +GITEA_SSH_UPSTREAM_PORT="${GITEA_SSH_UPSTREAM_PORT:-22}" + +# SSO wiring knobs +PROLE_GRAFANA_SSO_ENABLED="${PROLE_GRAFANA_SSO_ENABLED:-0}" +GRAFANA_PROXY_UPSTREAM_URL="${GRAFANA_PROXY_UPSTREAM_URL:-http://knoe-grafana-proxy.${NAMESPACE}.svc.cluster.local:80}" +KNOE_AUTH_UPSTREAM_URL="${KNOE_AUTH_UPSTREAM_URL:-http://knoe-auth.${SERVICE_NAMESPACE:-${NAMESPACE}}.svc.cluster.local:8080}" usage() { cat <&2 + exit 1 + fi + + local host_count=0 + local h + for h in "${hosts[@]}"; do + [[ -n "${h:-}" ]] && host_count=$((host_count + 1)) + done + + if [[ "$host_count" -gt 0 && -n "$db_ctx" && "$active_ctx" == "$db_ctx" ]]; then + echo "ERROR: refusing to render/apply public Kong ingress in DB cluster context '${active_ctx}' (hosts: ${hosts[*]})." >&2 + exit 1 + fi + + if [[ "$host_count" -gt 0 && -n "$app_ctx" && -n "$active_ctx" && "$active_ctx" != "$app_ctx" ]]; then + echo "ERROR: public Kong ingress must target APP cluster context '${app_ctx}', active context is '${active_ctx}'." >&2 + exit 1 + fi + + if [[ -n "$ingress_class" ]]; then + local normalized_class="${ingress_class,,}" + if [[ "$normalized_class" == traefik* ]] && ! is_truthy "${ALLOW_TRAEFIK_PUBLIC_INGRESS:-${KONG_ALLOW_TRAEFIK_INGRESS:-0}}"; then + echo "ERROR: ingress class '${ingress_class}' is incompatible with k8s mode unless Traefik public ingress is explicitly enabled." >&2 + exit 1 + fi + fi +} + +_is_host_claimed_by_other_ingress() { + # Returns 0 (true) if $1 is already a host rule on any ingress other than + # $3/$2 (name/namespace). Best-effort: returns 1 on any error. + local host="${1:-}" skip_name="${2:-}" skip_ns="${3:-}" + [[ -n "$host" ]] || return 1 + local ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}" + python3 - "$host" "$skip_ns" "$skip_name" "$ctx" <<'PY' 2>/dev/null +import json, subprocess, sys +host = sys.argv[1].strip().lower() +skip_ns, skip_name = sys.argv[2].strip(), sys.argv[3].strip() +ctx = sys.argv[4].strip() +cmd = ["kubectl"] +if ctx: + cmd += ["--context", ctx] +cmd += ["get", "ingress", "-A", "-o", "json"] +try: + raw = subprocess.check_output(cmd, text=True) +except Exception: + sys.exit(1) +for item in json.loads(raw).get("items", []): + md = item.get("metadata", {}) + if (md.get("namespace") or "").strip() == skip_ns and (md.get("name") or "").strip() == skip_name: + continue + for rule in (item.get("spec", {}) or {}).get("rules", []) or []: + if (rule.get("host") or "").strip().lower() == host: + sys.exit(0) +sys.exit(1) +PY +} + +assert_unique_ingress_host_claims() { + local ingress_name="${1:-}" + local ingress_namespace="${2:-}" + local host_csv="${3:-}" + [[ -n "$host_csv" ]] || return 0 + + local target_ctx="${KUBECTL_CONTEXT:-${KUBE_CONTEXT_NAME:-${KUBECONTEXT:-}}}" + if [[ "${KNOE_MODE:-}" == "k8s" && -z "$target_ctx" ]]; then + echo "ERROR: explicit kubectl context is required for ingress ownership checks in k8s mode." >&2 + return 1 + fi + + if ! python3 - "$host_csv" "$ingress_namespace" "$ingress_name" "$target_ctx" <<'PY' +import json +import subprocess +import sys + +requested_hosts = {h.strip().lower() for h in (sys.argv[1] or "").split(",") if h.strip()} +target_ns = sys.argv[2] +target_name = sys.argv[3] +target_ctx = (sys.argv[4] or "").strip() + +cmd = ["kubectl"] +if target_ctx: + cmd.extend(["--context", target_ctx]) +cmd.extend(["get", "ingress", "-A", "-o", "json"]) + +try: + raw = subprocess.check_output(cmd, text=True) +except Exception: + raise SystemExit(0) + +if not raw.strip(): + raise SystemExit(0) + +payload = json.loads(raw) +conflicts: list[str] = [] +for item in payload.get("items", []) or []: + md = item.get("metadata", {}) or {} + ns = (md.get("namespace") or "").strip() + name = (md.get("name") or "").strip() + if ns == target_ns and name == target_name: + continue + + spec = item.get("spec", {}) or {} + rules = spec.get("rules", []) or [] + for rule in rules: + host = (rule.get("host") or "").strip().lower() + if not host or host not in requested_hosts: + continue + http = rule.get("http", {}) or {} + paths = http.get("paths", []) or [{"path": "/"}] + for path_item in paths: + path = (path_item.get("path") or "/").strip() or "/" + if path in {"/", ""}: + conflicts.append(f"{host}{path} already owned by {ns}/{name}") + +if conflicts: + raise SystemExit("; ".join(conflicts)) +PY + then + echo "ERROR: duplicate ingress host/path claim detected for Kong ingress '${ingress_namespace}/${ingress_name}'." >&2 + return 1 + fi +} + ensure_namespace() { if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then echo "Creating namespace '$NAMESPACE' ..." @@ -139,30 +399,65 @@ _KONG_CONFIG_CHANGED_FILE="" create_kong_config() { echo "Creating/updating Kong declarative config '$KONG_CONFIG_NAME' in namespace '$NAMESPACE' ..." + local grafana_url + grafana_url="$GRAFANA_UPSTREAM_URL" + case "${PROLE_GRAFANA_SSO_ENABLED:-0}" in + 1|true|TRUE|True|yes|YES|on|ON) + grafana_url="$GRAFANA_PROXY_UPSTREAM_URL" + ;; + esac + local kong_yml kong_yml=$(cat </dev/null 2>&1 || true } +cleanup_legacy_prole_kong() { + # Best-effort cleanup: prole-era installs deployed Kong as "prole-svc-kong". + # After the prole→knoe rebrand the canonical name is "knoe-svc-kong"; stale + # prole resources would conflict with the rollout-status wait in deploy(). + local found=0 + kubectl -n "$NAMESPACE" get deployment prole-svc-kong >/dev/null 2>&1 && found=1 + kubectl -n "$NAMESPACE" get svc prole-svc-kong >/dev/null 2>&1 && found=1 + kubectl -n "$NAMESPACE" get ingress svc-prole-ingress >/dev/null 2>&1 && found=1 + if [[ "$found" -eq 0 ]]; then return 0; fi + echo "Removing legacy prole-svc-kong resources from namespace '$NAMESPACE' ..." + kubectl -n "$NAMESPACE" delete pod -l app=prole-svc-kong --ignore-not-found=true >/dev/null 2>&1 || true + kubectl -n "$NAMESPACE" delete deployment prole-svc-kong --ignore-not-found=true || true + kubectl -n "$NAMESPACE" delete svc prole-svc-kong --ignore-not-found=true || true + kubectl -n "$NAMESPACE" delete configmap prole-svc-kong-config --ignore-not-found=true || true + kubectl -n "$NAMESPACE" delete ingress svc-prole-ingress --ignore-not-found=true || true + echo "Legacy prole-svc-kong cleaned up." + # Clean up any old prole-db-* ingress resources from earlier installs. + kubectl -n "$NAMESPACE" delete ingress -l knoe.dev/route=prole-db --ignore-not-found=true >/dev/null 2>&1 || true + kubectl -n "$NAMESPACE" delete ingress prole-db-ingress --ignore-not-found=true >/dev/null 2>&1 || true +} + apply_service_ingress() { local host="${SERVICE_HOSTNAME:-}" if [[ -z "$host" ]]; then @@ -216,6 +563,302 @@ apply_service_ingress() { return 0 fi + local auth_host="${AUTH_HOSTNAME:-}" + local gitea_host="${GITEA_HOSTNAME:-}" + local db_host="${DB_HOSTNAME:-}" + local include_aux_hosts=1 + local include_gitea_host=1 + # db.prole.org is only routed in k3s mode; oauth2-proxy handles auth before Supabase Studio. + # k8s: GKE has its own oauth2-proxy Deployment + Ingress in deploy/gcp/gke/. + # k3d: db access is port-forward only; no public hostname on the local cluster. + local include_db_host=0 + local _mode="${KNOE_MODE:-}" + if [[ "$_mode" == "k3s" ]]; then + include_db_host=1 + fi + if [[ "$_mode" == "k8s" || "$_mode" == "k3d" ]]; then + # k8s: GitLab/Gitea manages its own public ingress. + # k3d: git access is port-forward only; no public hostname on the local cluster. + include_gitea_host=0 + fi + # Safety net: if another ingress already owns the gitea host (e.g. the git + # component deployed before Common Services), skip rather than hard-fail. + if [[ "$include_gitea_host" -eq 1 && -n "$gitea_host" ]]; then + if _is_host_claimed_by_other_ingress "$gitea_host" "svc-knoe-ingress" "$NAMESPACE"; then + echo "NOTICE: ${gitea_host} already claimed by another ingress; skipping gitea route in svc-knoe-ingress." + include_gitea_host=0 + fi + fi + local tls_hosts_extra="" + local rules_extra="" + if [[ "$include_aux_hosts" -eq 1 && -n "$auth_host" && "$auth_host" != "$host" ]]; then + tls_hosts_extra=$'\n - '"${auth_host}" + rules_extra=$(cat </dev/null 2>&1; then + echo "ERROR: ManagedCertificate ${NAMESPACE}/${service_managed_cert_name} was not found after apply — cannot safely apply GCE ingress." >&2 + return 1 + fi + if ! kubectl -n "$NAMESPACE" get frontendconfig "$service_frontend_config_name" >/dev/null 2>&1; then + echo "ERROR: FrontendConfig ${NAMESPACE}/${service_frontend_config_name} was not found after apply — cannot safely apply GCE ingress." >&2 + return 1 + fi + echo "Confirmed: ManagedCertificate/${service_managed_cert_name} and FrontendConfig/${service_frontend_config_name} exist in ns=${NAMESPACE}." + + # BackendConfig: GCE default healthCheck is HTTP GET / on the backend port + # and Kong returns 404 on an unrouted path, so the backend never goes + # HEALTHY. We wanted TCP (Kong is alive as long as it accepts connections), + # but GCE's BackendConfig CRD rejects `type: TCP` with + # `Protocol "TCP" is not valid, must be one of [HTTP,HTTPS,HTTP2]` + # so we fall back to HTTP against the `/healthz` route we add to the + # knoe-svc-kong declarative config above (request-termination plugin + # returns 200 synchronously, no upstream dependency -- equivalent liveness + # semantics to a TCP check but over a protocol GCE accepts). + echo "Reconciling BackendConfig (${SVC_KNOE_BACKEND_CONFIG_NAME}) for ${KONG_NAME} in ns=${NAMESPACE} ..." + kubectl apply -f - <&2 + fi + unset managed_domains_yaml ingress_tls_host + fi + + if (( tls_enabled == 1 )); then + tls_annotations=$(cat </dev/null || true + echo "ManagedCertificates:" + kubectl get managedcertificate -A --no-headers 2>/dev/null || true + echo "FrontendConfigs:" + kubectl get frontendconfig -A --no-headers 2>/dev/null || true + if kubectl -n "$NAMESPACE" get ingress "$ingress_name" >/dev/null 2>&1; then + echo "Current annotations for ${NAMESPACE}/${ingress_name} (pre-apply):" + kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true + echo + fi + echo "---" + fi + + echo "Rendered svc ingress (pre-apply): ns=${NAMESPACE} ingress=${ingress_name} class=${ingress_class} managedCert=${service_managed_cert_name:--} frontendConfig=${service_frontend_config_name:--} preSharedCert=${service_pre_shared_cert:--} tlsEnabled=${tls_enabled} tlsSecret=${SERVICE_TLS_SECRET_NAME:--} hosts=${ingress_host_csv} backend=${KONG_NAME}:${KONG_PROXY_PORT}" + + if kubectl -n "$NAMESPACE" get ingress "$ingress_name" >/dev/null 2>&1; then + local live_spec_class="" + local live_ann_class="" + local live_class="" + local live_managed_cert="" + local live_frontend_config="" + local live_pre_shared_cert="" + live_spec_class="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.spec.ingressClassName}' 2>/dev/null || true)" + live_ann_class="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.kubernetes\.io/ingress\.class}' 2>/dev/null || true)" + live_class="${live_spec_class:-$live_ann_class}" + live_managed_cert="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.networking\.gke\.io/managed-certificates}' 2>/dev/null || true)" + live_frontend_config="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.networking\.gke\.io/v1beta1\.FrontendConfig}' 2>/dev/null || true)" + live_pre_shared_cert="$(kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations.ingress\.gcp\.kubernetes\.io/pre-shared-cert}' 2>/dev/null || true)" + + local replace_reason="" + local patch_only=0 + if [[ -n "$live_class" && "$live_class" != "$ingress_class" ]]; then + # ingressClass change requires recreation (immutable field). + replace_reason="ingressClass drift (live=${live_class}, desired=${ingress_class})" + elif [[ "${KNOE_MODE:-}" == "k8s" && "$ingress_class" == "gce" ]]; then + # Requirement 2 & 3: never delete/recreate to clear stale cert annotations. + # Patch annotations in place instead. + if [[ -n "$live_pre_shared_cert" ]]; then + echo "Patching stale pre-shared-cert annotation from ${NAMESPACE}/${ingress_name} in place (was: ${live_pre_shared_cert})." + kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \ + ingress.gcp.kubernetes.io/pre-shared-cert- \ + "networking.gke.io/managed-certificates=${service_managed_cert_name}" \ + "networking.gke.io/v1beta1.FrontendConfig=${service_frontend_config_name}" \ + --overwrite >/dev/null 2>&1 || echo "WARN: Failed to patch pre-shared-cert annotation from ${ingress_name}." >&2 + patch_only=1 + elif [[ -n "$live_managed_cert" && "$live_managed_cert" != "$service_managed_cert_name" ]]; then + echo "Patching managed certificate annotation on ${NAMESPACE}/${ingress_name} in place (live=${live_managed_cert}, desired=${service_managed_cert_name})." + kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \ + "networking.gke.io/managed-certificates=${service_managed_cert_name}" \ + --overwrite >/dev/null 2>&1 || echo "WARN: Failed to patch managed-certificates annotation on ${ingress_name}." >&2 + patch_only=1 + elif [[ -n "$live_frontend_config" && "$live_frontend_config" != "$service_frontend_config_name" ]]; then + echo "Patching frontend config annotation on ${NAMESPACE}/${ingress_name} in place (live=${live_frontend_config}, desired=${service_frontend_config_name})." + kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \ + "networking.gke.io/v1beta1.FrontendConfig=${service_frontend_config_name}" \ + --overwrite >/dev/null 2>&1 || echo "WARN: Failed to patch FrontendConfig annotation on ${ingress_name}." >&2 + patch_only=1 + fi + fi + + if [[ -n "$replace_reason" && "$patch_only" -eq 0 ]]; then + echo "Ingress shape change detected for ${NAMESPACE}/${ingress_name}: ${replace_reason}. Replacing ingress (host/rule shape change)." + kubectl -n "$NAMESPACE" delete ingress "$ingress_name" --ignore-not-found >/dev/null || true + fi + unset patch_only + fi + echo "Applying service Ingress for host '${host}' -> ${KONG_NAME}:${KONG_PROXY_PORT} (namespace=${NAMESPACE}) ..." ( tmp="$(mktemp)" @@ -224,17 +867,13 @@ apply_service_ingress() { apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: svc-knoe-ingress + name: ${ingress_name} namespace: ${NAMESPACE} annotations: - kubernetes.io/ingress.class: traefik - traefik.ingress.kubernetes.io/router.priority: "10" - cert-manager.io/cluster-issuer: ${SERVICE_TLS_CLUSTER_ISSUER} + kubernetes.io/ingress.class: ${ingress_class} +${extra_annotations}${tls_annotations}${gce_tls_annotations} spec: - tls: - - hosts: - - ${host} - secretName: ${SERVICE_TLS_SECRET_NAME} +${tls_block} rules: - host: ${host} http: @@ -246,15 +885,38 @@ spec: name: ${KONG_NAME} port: number: ${KONG_PROXY_PORT} +${rules_extra} EOF kubectl_apply_retry -f "$tmp" ) + + if [[ "${KNOE_MODE:-}" == "k8s" && "$ingress_class" == "gce" ]]; then + kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \ + ingress.gcp.kubernetes.io/pre-shared-cert- \ + --overwrite >/dev/null 2>&1 || true + # Diagnostics: show annotations after apply. + echo "Annotations for ${NAMESPACE}/${ingress_name} (post-apply):" + kubectl -n "$NAMESPACE" get ingress "$ingress_name" -o jsonpath='{.metadata.annotations}' 2>/dev/null || true + echo + kubectl -n "$NAMESPACE" describe ingress "$ingress_name" 2>/dev/null || true + else + kubectl -n "$NAMESPACE" annotate ingress "$ingress_name" \ + networking.gke.io/managed-certificates- \ + networking.gke.io/v1beta1.FrontendConfig- \ + ingress.gcp.kubernetes.io/pre-shared-cert- \ + --overwrite >/dev/null 2>&1 || true + fi } deploy() { echo "Deploying $KONG_NAME to namespace '$NAMESPACE' ..." - local manifests_dir="$KNOE_HOME/deploy/opentofu/k3s/manifests/knoe" + local manifests_dir + if [[ "${KNOE_MODE:-}" == "k8s" ]]; then + manifests_dir="$KNOE_HOME/deploy/opentofu/k8s/manifests/knoe" + else + manifests_dir="$KNOE_HOME/deploy/opentofu/k3s/manifests/knoe" + fi # Determine whether the Deployment already exists before applying manifests. local deployment_existed=0 @@ -267,6 +929,18 @@ deploy() { svc_out=$(kubectl_apply_retry -f "$manifests_dir/kong-service.yaml" -n "$NAMESPACE" 2>&1) echo "$svc_out" + # In k8s/GCE mode, annotate the Service so GCE LB picks up the BackendConfig + # with the TCP health check. Matches etc/init_gitlab.sh's pattern for + # gitlab-webservice-default. Additive annotation; survives manifest + # re-applies (the YAML in deploy/opentofu/ doesn't set it). + if [[ "${KNOE_MODE:-}" == "k8s" ]]; then + echo "Annotating Service ${KONG_NAME} with cloud.google.com/backend-config=${SVC_KNOE_BACKEND_CONFIG_NAME}..." + kubectl -n "$NAMESPACE" annotate svc "$KONG_NAME" \ + "cloud.google.com/backend-config={\"default\":\"${SVC_KNOE_BACKEND_CONFIG_NAME}\"}" \ + --overwrite >/dev/null || \ + echo "WARN: Failed to annotate ${KONG_NAME} with backend-config; GCE LB will fall back to default healthcheck (likely UNHEALTHY)." >&2 + fi + echo "Waiting for $KONG_NAME rollout ..." kubectl rollout status deployment/"$KONG_NAME" -n "$NAMESPACE" --timeout=120s @@ -303,6 +977,14 @@ status() { echo "" echo "=== $KONG_NAME service ===" kubectl get svc "$KONG_NAME" -n "$NAMESPACE" 2>/dev/null || echo "No service found" + echo "" + echo "=== svc-knoe-ingress hosts (includes db.prole.org in k3s mode) ===" + kubectl get ingress svc-knoe-ingress -n "$NAMESPACE" -o jsonpath='{range .spec.rules[*]}{.host}{"\n"}{end}' 2>/dev/null || echo "No ingress found" + if [[ "${KNOE_MODE:-}" == "k3s" ]]; then + echo "" + echo "=== oauth2-proxy (db.prole.org gate, ns=${OAUTH2_PROXY_NAMESPACE}) ===" + kubectl get pods -n "${OAUTH2_PROXY_NAMESPACE}" -l app.kubernetes.io/name=oauth2-proxy 2>/dev/null || echo "No oauth2-proxy pods found" + fi } restart() { @@ -316,6 +998,7 @@ action_update() { ensure_tools ensure_namespace cleanup_legacy_svc_check + cleanup_legacy_prole_kong create_kong_config apply_service_ingress deploy diff --git a/mock_val/init_min.sh b/mock_val/init_min.sh new file mode 100644 index 0000000..b951ed1 --- /dev/null +++ b/mock_val/init_min.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +# etc/init_min.sh +# Purpose: Initialize minimal containerd environment for knoe-db development + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/knoe_cfg.sh" + +log() { echo "==> $*"; } + +ACTION=${1:-initialize} + +case "$ACTION" in + initialize) + log "Initializing minimal environment..." + + # Ensure containerd is running (if on macOS via brew) + if [[ "$OSTYPE" == "darwin"* ]]; then + if ! pgrep containerd >/dev/null; then + log "Starting containerd via brew services..." + brew services start containerd + sleep 2 + fi + fi + + # Create start.sh and stop.sh aliases to knoe.sh + log "Generating start.sh and stop.sh as wrappers to knoe.sh..." + + cat > "$PROJECT_ROOT/start.sh" < "$PROJECT_ROOT/stop.sh" </dev/null | head -n1 } +monitoring_mode() { + local mode + mode="${KNOE_MODE:-${DEPLOYMENT_MODE:-}}" + mode=$(printf '%s' "$mode" | tr '[:upper:]' '[:lower:]') + case "$mode" in + gke|prod|k8s) + echo "k8s" + ;; + k3s|k3d) + echo "$mode" + ;; + *) + echo "$mode" + ;; + esac +} + choose_monitoring_storage_class() { - if [[ -n "${MONITORING_STORAGE_CLASS:-}" ]] && storage_class_exists "$MONITORING_STORAGE_CLASS"; then + local mode + mode=$(monitoring_mode) + + if [[ -n "${MONITORING_STORAGE_CLASS:-}" ]]; then echo "$MONITORING_STORAGE_CLASS" return 0 fi - if storage_class_exists "merlin-local-iscsi"; then - echo "merlin-local-iscsi" - return 0 - fi - # Backward compatibility (older runs created knoe-monitoring-) - if storage_class_exists "knoe-monitoring-d004"; then - echo "knoe-monitoring-d004" - return 0 + + if [[ "$mode" == "k3s" ]]; then + if storage_class_exists "merlin-local-iscsi"; then + echo "merlin-local-iscsi" + return 0 + fi + # Backward compatibility (older runs created knoe-monitoring-) + if storage_class_exists "knoe-monitoring-d004"; then + echo "knoe-monitoring-d004" + return 0 + fi fi + local default_sc default_sc=$(default_storage_class) if [[ -n "$default_sc" ]]; then echo "$default_sc" return 0 fi - if storage_class_exists "local-path"; then + + if [[ "$mode" == "k3s" || "$mode" == "k3d" ]] && storage_class_exists "local-path"; then echo "local-path" return 0 fi + echo "" } @@ -120,7 +146,7 @@ monitoring_storage_class_for_role() { # For the static local-PV setup on the primary k3s node, use dedicated # storageClasses per component to avoid nondeterministic PV binding. - if [[ "$base" == "merlin-local-iscsi" ]]; then + if [[ "$(monitoring_mode)" == "k3s" && "$base" == "merlin-local-iscsi" ]]; then local candidate candidate="${base}-${role}" if storage_class_exists "$candidate"; then @@ -134,8 +160,45 @@ monitoring_storage_class_for_role() { echo "$base" } +validate_monitoring_storage_classes() { + local selected="${MONITORING_STORAGE_CLASS_SELECTED:-}" + [[ -n "${selected:-}" ]] || return 0 + + local role sc + local -a missing=() + local -a seen=() + for role in prometheus alertmanager grafana; do + sc=$(monitoring_storage_class_for_role "$role") + [[ -n "${sc:-}" ]] || continue + + local already_seen=0 + local seen_sc + for seen_sc in "${seen[@]}"; do + if [[ "$seen_sc" == "$sc" ]]; then + already_seen=1 + break + fi + done + (( already_seen == 1 )) && continue + seen+=("$sc") + + if ! storage_class_exists "$sc"; then + missing+=("$sc") + fi + done + + if [[ ${#missing[@]} -gt 0 ]]; then + local mode + mode=$(monitoring_mode) + err "Monitoring storageClass validation failed for mode '${mode:-unknown}': rendered class(es) not found: ${missing[*]}. Configure MONITORING_STORAGE_CLASS in conf/gke.cfg (or the selected cluster config) to an existing StorageClass." + return 1 + fi + + return 0 +} + monitoring_nodes_available() { - kubectl get nodes -l "knoe.org/node-role=general" -o name 2>/dev/null | grep -q . + kubectl get nodes -l "prole.org/node-role=general" -o name 2>/dev/null | grep -q . } resolve_monitoring_primary_node() { @@ -144,11 +207,11 @@ resolve_monitoring_primary_node() { printf '%s' "${MONITORING_PRIMARY_NODE}" return 0 fi - if kubectl get node merlin.knoe.org >/dev/null 2>&1; then - printf '%s' "merlin.knoe.org" + if kubectl get node merlin.prole.org >/dev/null 2>&1; then + printf '%s' "merlin.prole.org" return 0 fi - node=$(kubectl get nodes -l "knoe.org/node-role=general" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + node=$(kubectl get nodes -l "prole.org/node-role=general" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "${node:-}" ]]; then printf '%s' "$node" return 0 @@ -161,7 +224,7 @@ prepare_k3s_monitoring_data_dirs() { local node="$1" # NOTE: Historically this came from knoe.cfg. For now we pin it to the # monitoring deployment default and allow overrides via env var. - local base="${PROLE_MONITORING_DATA_DIR:-/knoe/d004}" + local base="${PROLE_MONITORING_DATA_DIR:-/synology/d004}" base="${base%/}" if [[ -z "${node:-}" ]]; then err "Could not resolve a monitoring node to prepare storage on" @@ -256,7 +319,7 @@ apply_k3s_monitoring_local_pvs() { fi local base volume_id sc_base sc_prom sc_am sc_graf - base="${PROLE_MONITORING_DATA_DIR:-/knoe/d004}" + base="${PROLE_MONITORING_DATA_DIR:-/synology/d004}" base="${base%/}" volume_id=$(basename "$base") if [[ -z "${volume_id:-}" || "$volume_id" == "/" || "$volume_id" == "." ]]; then @@ -398,7 +461,7 @@ render_node_selector() { local indent="$1" cat < "$tmp" + # Pass the dashboard JSON through unchanged. The source JSON templates the + # datasource UID via the dashboard's own `DS_PROMETHEUS` variable (resolved + # at render time by Grafana, configurable per-user via the Datasource + # dropdown). The earlier sed-substitution to a fixed `prom_uid` baked + # `uid: prometheus` into every variable + panel definition, defeating that + # template — the dashboard then queried only the default Prometheus + # datasource regardless of what the user selected. With cross-cluster + # CNPG metrics on `cnpg-prometheus` (commit 09f2c1a), passing through + # unchanged is required for variable dropdowns + panels to follow the + # Datasource selector. + cp "$dashboard_path" "$tmp" + : "$prom_uid" # silence unused-var warning local cm_yaml cm_yaml=$(mktemp) @@ -613,35 +726,19 @@ apply_grafana_dashboard() { } apply_grafana_datasource() { - local ns="$1" - local prom_uid="${GRAFANA_PROMETHEUS_DATASOURCE_UID:-prometheus}" - local prom_url="${GRAFANA_PROMETHEUS_DATASOURCE_URL:-http://kps-kube-prometheus-stack-prometheus.${ns}.svc.cluster.local:9090}" - - local cm_yaml - cm_yaml=$(mktemp) - cat > "$cm_yaml" </dev/null - rm -f "$cm_yaml" + # No-op: kube-prometheus-stack already provisions the Prometheus + Alertmanager + # datasources via its own ConfigMap (`-grafana-datasource`). The + # earlier extra `knoe-grafana-datasource` ConfigMap created here was a + # duplicate of that — it registered the same `uid: prometheus`, which collided + # with the chart's datasource and caused Grafana's provisioning reload to + # error out (HTTP 500 → datasources never refreshed). + # + # Additional datasources (e.g. the cross-cluster `cnpg-prometheus` pointing + # at the DB-cluster Prometheus) are wired through the chart values at + # `monitoring/kps-values-gke.yaml` `grafana.additionalDataSources`, NOT + # through this script. Keeping this function as a no-op so call sites don't + # need to change. + return 0 } helm_release_status() { @@ -1059,6 +1156,10 @@ install_monitoring() { log "No storageClass detected; disabling persistence for Grafana and skipping Prometheus/Alertmanager storage." fi + if ! validate_monitoring_storage_classes; then + return 1 + fi + # Don’t enforce a single storageClass here: in k3s static local-PV mode we use # per-component storageClasses (e.g. merlin-local-iscsi-prometheus|grafana|alertmanager). cleanup_pending_pvcs "$monitoring_ns" "" @@ -1074,6 +1175,23 @@ install_monitoring() { ;; esac + # Derive PV names from PROLE_MONITORING_DATA_DIR (same logic as apply_monitoring_pvs) + local _mpv_base _mpv_vid + _mpv_base="${PROLE_MONITORING_DATA_DIR:-/synology/d004}" + _mpv_base="${_mpv_base%/}" + _mpv_vid=$(basename "$_mpv_base") + pv_prom="merlin-local-iscsi-${_mpv_vid}-prometheus" + pv_am="merlin-local-iscsi-${_mpv_vid}-alertmanager" + pv_graf="merlin-local-iscsi-${_mpv_vid}-grafana" + # Release any Released monitoring PVs so new PVCs can bind (idempotent) + for _mpv in "${pv_prom}" "${pv_am}" "${pv_graf}"; do + _mpv_phase=$(kubectl get pv "$_mpv" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "$_mpv_phase" == "Released" ]]; then + log "Clearing stale claimRef on Released PV '$_mpv' ..." + kubectl patch pv "$_mpv" -p '{"spec":{"claimRef":null}}' 2>/dev/null || true + fi + done + section "Installing Prometheus stack" helm repo add prometheus-community https://prometheus-community.github.io/helm-charts || true helm repo update prometheus-community || true @@ -1181,6 +1299,15 @@ prometheus-node-exporter: $(render_node_selector " ") $(render_node_affinity " ") $(render_tolerations " ") + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - pi.prole.org EOF local monitoring_helm_server_side @@ -1278,6 +1405,8 @@ fullnameOverride: kps-grafana adminPassword: "${GRAFANA_ADMIN_PASSWORD}" +$(render_grafana_external_url "") + service: port: 80 targetPort: 3000 diff --git a/mock_val/init_nginx_ingress.sh b/mock_val/init_nginx_ingress.sh index 3eac64d..b9f9392 100755 --- a/mock_val/init_nginx_ingress.sh +++ b/mock_val/init_nginx_ingress.sh @@ -61,7 +61,7 @@ if [[ -n "$MODE" && "$MODE" != "k3d" ]]; then fi knoe_ensure_kubeconfig >/dev/null 2>&1 || true -knoe_ensure_kube_context || exit 1 +ensure_kube_context || exit 1 PORT_MAPPING_FILE_PATH="${PORT_MAPPING_FILE:-}" if [[ -z "$PORT_MAPPING_FILE_PATH" ]]; then diff --git a/mock_val/init_oauth2_proxy.sh b/mock_val/init_oauth2_proxy.sh new file mode 100755 index 0000000..4d5e433 --- /dev/null +++ b/mock_val/init_oauth2_proxy.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# init_oauth2_proxy.sh +# +# Bootstrap the oauth2-proxy gate in front of Supabase Studio at +# db.0.knoe.dev. Gates access via Google Workspace OIDC (knoey.com) so any +# @knoey.com identity (chrisfu, ron) can sign in and share the Studio +# session. Outside-domain users are rejected at this layer. +# +# Usage: +# ./etc/init_oauth2_proxy.sh +# +# Env vars (resolved from etc/secrets/* if not set in the shell): +# OAUTH2_PROXY_CLIENT_ID ← from etc/secrets/oauth2-proxy-client-id +# OAUTH2_PROXY_CLIENT_SECRET ← from etc/secrets/oauth2-proxy-client-secret +# OAUTH2_PROXY_COOKIE_SECRET ← from etc/secrets/oauth2-proxy-cookie-secret +# +# Optional: +# APP_CLUSTER_KUBECONTEXT (default: $KUBECONTEXT then ambient) +# NAMESPACE (default: supabase) +# +# Pre-reqs: +# - OAuth 2.0 client created at GCP Console (see the secret template +# deploy/gcp/gke/oauth2-proxy-google-oidc-secret.example.yaml for the +# exact authorized redirect URI + consent screen settings). +# - cookie_secret generated with: openssl rand -base64 32 +# - Three values saved into etc/secrets/oauth2-proxy-{client-id,client-secret,cookie-secret} +# (chmod 0600 each; etc/secrets/ is gitignored except for .keep). +# +# After this script runs and the oauth2-proxy Deployment is Ready, two +# manual steps complete the wiring (NOT done by this script — see the plan +# in docs/plans/ for the full sequence): +# +# 1. Patch the supabase-kong Ingress to route db.0.knoe.dev through +# oauth2-proxy:80 instead of supabase-kong:8000: +# +# kubectl --context=$APP_CLUSTER_KUBECONTEXT -n supabase patch ingress \ +# supabase-kong --type=json -p '[ +# {"op": "replace", +# "path": "/spec/rules/1/http/paths/0/backend/service/name", +# "value": "oauth2-proxy"}, +# {"op": "replace", +# "path": "/spec/rules/1/http/paths/0/backend/service/port/number", +# "value": 80} +# ]' +# (verify the index by checking which rule has host=db.0.knoe.dev first; +# index may shift on future Helm reconciles) +# +# 2. Remove the basic-auth plugin from the dashboard route in the +# supabase-kong configmap (oauth2-proxy is the gate now; double-auth is +# friction). Then rollout-restart supabase-kong. +# +# When knoe-auth Round 1 ships an OIDC OP at https://api.knoe.dev/auth, change +# `--provider=google` to `--provider=oidc --oidc-issuer-url=https://api.knoe.dev/auth` +# in deploy/gcp/gke/oauth2-proxy-deployment.yaml and re-run this script. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GKE_MANIFEST_DIR="$REPO_ROOT/deploy/gcp/gke" + +NAMESPACE="${NAMESPACE:-supabase}" +KCTX="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}" + +if [[ -n "$KCTX" ]]; then + KCTX_FLAG=(--context="$KCTX") +else + KCTX_FLAG=() +fi + +log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +resolve_secret() { + # Resolve a value from env (preferred) or etc/secrets/. + local var="$1" file="$2" val="${!1:-}" + if [[ -z "$val" && -f "$REPO_ROOT/etc/secrets/$file" ]]; then + val="$(cat "$REPO_ROOT/etc/secrets/$file")" + fi + if [[ -z "$val" ]]; then + die "missing $var (set the env var, or save the value into etc/secrets/$file)" + fi + printf '%s' "$val" +} + +for tool in kubectl envsubst; do + command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool" +done + +OAUTH2_PROXY_CLIENT_ID="$(resolve_secret OAUTH2_PROXY_CLIENT_ID oauth2-proxy-client-id)" +OAUTH2_PROXY_CLIENT_SECRET="$(resolve_secret OAUTH2_PROXY_CLIENT_SECRET oauth2-proxy-client-secret)" +OAUTH2_PROXY_COOKIE_SECRET="$(resolve_secret OAUTH2_PROXY_COOKIE_SECRET oauth2-proxy-cookie-secret)" +export OAUTH2_PROXY_CLIENT_ID OAUTH2_PROXY_CLIENT_SECRET OAUTH2_PROXY_COOKIE_SECRET + +SECRET_TMPL="$GKE_MANIFEST_DIR/oauth2-proxy-google-oidc-secret.example.yaml" +DEPLOY_MANIFEST="$GKE_MANIFEST_DIR/oauth2-proxy-deployment.yaml" + +[[ -f "$SECRET_TMPL" ]] || die "missing manifest: $SECRET_TMPL" +[[ -f "$DEPLOY_MANIFEST" ]] || die "missing manifest: $DEPLOY_MANIFEST" + +log "==> oauth2-proxy bootstrap" +log " namespace : $NAMESPACE" +log " kubectx : ${KCTX:-}" + +log "Applying oauth2-proxy-google-oidc secret ..." +envsubst '${OAUTH2_PROXY_CLIENT_ID} ${OAUTH2_PROXY_CLIENT_SECRET} ${OAUTH2_PROXY_COOKIE_SECRET}' \ + < "$SECRET_TMPL" \ + | kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" apply -f - + +log "Applying oauth2-proxy ServiceAccount + BackendConfig + Service + Deployment ..." +kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" apply -f "$DEPLOY_MANIFEST" + +log "Waiting for oauth2-proxy Deployment to become Ready (timeout 180s) ..." +kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" rollout status deployment/oauth2-proxy --timeout=180s + +log "==> oauth2-proxy bootstrap complete." +echo "" +echo " Next steps (NOT performed by this script):" +echo " 1. Patch the supabase-kong Ingress so db.0.knoe.dev routes to" +echo " oauth2-proxy:80 instead of supabase-kong:8000." +echo " 2. Remove the basic-auth plugin from the dashboard route in the" +echo " supabase-kong configmap, then rollout-restart supabase-kong." +echo " 3. In a browser, sign in to https://db.0.knoe.dev/ with a" +echo " @knoey.com Google account. Try a non-knoey account too — should" +echo " receive 403 from oauth2-proxy." +echo "" +echo " See the active plan in ~/.claude/plans/ for the exact patch commands." diff --git a/mock_val/init_oauth2_proxy_prole.sh b/mock_val/init_oauth2_proxy_prole.sh new file mode 100644 index 0000000..c9736b2 --- /dev/null +++ b/mock_val/init_oauth2_proxy_prole.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# init_oauth2_proxy_prole.sh +# +# Bootstrap the oauth2-proxy gate in front of Supabase Studio at db.prole.org +# on the k3s homelab cluster. Companion to init_oauth2_proxy.sh (knoe.dev GKE) +# but uses prole.org GCP project credentials and targets the k3s kubecontext. +# +# Gates access via Google Workspace OIDC (prole.org) so @prole.org identities +# can sign in to Studio. Outside-domain users are rejected at this layer. +# +# Usage: +# ./etc/init_oauth2_proxy_prole.sh +# +# Env vars (resolved from etc/secrets/* if not set in the shell): +# OAUTH2_PROXY_CLIENT_ID ← from etc/secrets/oauth2-proxy-client-id-prole +# OAUTH2_PROXY_CLIENT_SECRET ← from etc/secrets/oauth2-proxy-client-secret-prole +# OAUTH2_PROXY_COOKIE_SECRET ← from etc/secrets/oauth2-proxy-cookie-secret-prole +# +# Optional: +# K3S_KUBECONTEXT (default: $KUBECONTEXT then ambient) +# NAMESPACE (default: supabase) +# +# Pre-reqs: +# - OAuth 2.0 Web Application client created in the prole.org GCP project: +# Authorized JS origins: https://db.prole.org +# Authorized redirect URI: https://db.prole.org/oauth2/callback +# Consent screen: Internal (prole.org Workspace) +# Scopes: openid, email, profile +# See deploy/gcp/gke/oauth2-proxy-google-oidc-secret-prole.example.yaml. +# - Cookie secret generated with: openssl rand -base64 32 +# - Three values saved (chmod 0600) to: +# etc/secrets/oauth2-proxy-client-id-prole +# etc/secrets/oauth2-proxy-client-secret-prole +# etc/secrets/oauth2-proxy-cookie-secret-prole +# +# After this script runs, patch the db.prole.org Ingress/IngressRoute to route +# through oauth2-proxy:80 instead of supabase-kong:8000 directly (see next +# steps printed at the end of this script). +# +# When knoe-auth Round 1 ships an OIDC OP at https://api.prole.org/auth, change +# --provider=google to --provider=oidc --oidc-issuer-url=https://api.prole.org/auth +# in deploy/opentofu/k3s/manifests/knoe/oauth2-proxy-deployment-prole.yaml and +# re-run this script. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MANIFEST_DIR="$REPO_ROOT/deploy/gcp/gke" +K3S_MANIFEST_DIR="$REPO_ROOT/deploy/opentofu/k3s/manifests/knoe" + +NAMESPACE="${NAMESPACE:-supabase}" +KCTX="${K3S_KUBECONTEXT:-${KUBECONTEXT:-}}" + +if [[ -n "$KCTX" ]]; then + KCTX_FLAG=(--context="$KCTX") +else + KCTX_FLAG=() +fi + +log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +resolve_secret() { + local var="$1" file="$2" val="${!1:-}" + if [[ -z "$val" && -f "$REPO_ROOT/etc/secrets/$file" ]]; then + val="$(cat "$REPO_ROOT/etc/secrets/$file")" + fi + if [[ -z "$val" ]]; then + die "missing $var (set the env var, or save the value into etc/secrets/$file)" + fi + printf '%s' "$val" +} + +for tool in kubectl envsubst; do + command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool" +done + +OAUTH2_PROXY_CLIENT_ID="$(resolve_secret OAUTH2_PROXY_CLIENT_ID oauth2-proxy-client-id-prole)" +OAUTH2_PROXY_CLIENT_SECRET="$(resolve_secret OAUTH2_PROXY_CLIENT_SECRET oauth2-proxy-client-secret-prole)" +OAUTH2_PROXY_COOKIE_SECRET="$(resolve_secret OAUTH2_PROXY_COOKIE_SECRET oauth2-proxy-cookie-secret-prole)" +export OAUTH2_PROXY_CLIENT_ID OAUTH2_PROXY_CLIENT_SECRET OAUTH2_PROXY_COOKIE_SECRET + +SECRET_TMPL="$MANIFEST_DIR/oauth2-proxy-google-oidc-secret-prole.example.yaml" +DEPLOY_MANIFEST="$K3S_MANIFEST_DIR/oauth2-proxy-deployment-prole.yaml" + +[[ -f "$SECRET_TMPL" ]] || die "missing manifest: $SECRET_TMPL" +[[ -f "$DEPLOY_MANIFEST" ]] || die "missing manifest: $DEPLOY_MANIFEST" + +log "==> oauth2-proxy bootstrap (prole.org / k3s)" +log " namespace : $NAMESPACE" +log " kubectx : ${KCTX:-}" + +log "Applying oauth2-proxy-google-oidc Secret ..." +envsubst '${OAUTH2_PROXY_CLIENT_ID} ${OAUTH2_PROXY_CLIENT_SECRET} ${OAUTH2_PROXY_COOKIE_SECRET}' \ + < "$SECRET_TMPL" \ + | kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" apply -f - + +log "Applying oauth2-proxy ServiceAccount + Service + Deployment ..." +kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" apply -f "$DEPLOY_MANIFEST" + +log "Waiting for oauth2-proxy Deployment to become Ready (timeout 180s) ..." +kubectl "${KCTX_FLAG[@]}" -n "$NAMESPACE" rollout status deployment/oauth2-proxy --timeout=180s + +log "==> oauth2-proxy bootstrap complete." +echo "" +echo " Next steps (NOT performed by this script):" +echo " 1. Patch the db.prole.org Ingress/IngressRoute so traffic routes" +echo " through oauth2-proxy:80 instead of supabase-kong:8000 directly." +echo " Check current routing:" +echo " kubectl -n supabase get ingress,ingressroute" +echo " 2. Remove or disable any basic-auth plugin on the Studio route in" +echo " the supabase-kong configmap; rollout-restart supabase-kong." +echo " 3. Browser-test: https://db.prole.org/ → Google sign-in (prole.org" +echo " account). Verify a non-prole.org account receives 403." +echo "" diff --git a/mock_val/init_openbao.sh b/mock_val/init_openbao.sh index 0705a8b..e1ece01 100755 --- a/mock_val/init_openbao.sh +++ b/mock_val/init_openbao.sh @@ -20,10 +20,13 @@ _has_config=0 for _arg in "$@"; do [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1 done -if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - set -- "-c" "$SCRIPT_DIR/../conf/knoe.cfg" "$@" +if [[ $_has_config -eq 0 ]]; then + _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)" + if [[ -n "$_default_cfg" ]]; then + set -- "-c" "$_default_cfg" "$@" + fi fi -unset _has_config _arg +unset _has_config _arg _default_cfg common_core_preparse_config "$@" @@ -100,7 +103,7 @@ root_token_file="$SECRETS_DIR/openbao-root-token" OPENBAO_NAMESPACE=${OPENBAO_NAMESPACE:-${SERVICE_NAMESPACE:-${NAMESPACE:-default}}} # OpenBao resources (StatefulSet/Service) live in `OPENBAO_NAMESPACE` (usually the service namespace), -# but secret *paths* must be keyed by the Knoe workload namespace from `conf/knoe.cfg` so that +# but secret *paths* must be keyed by the Knoe workload namespace from the active config file so that # `${OPENBAO:kv/knoe//...}` references resolve consistently across scripts. OPENBAO_PATH_NAMESPACE=${OPENBAO_PATH_NAMESPACE:-${PROLE_NAMESPACE:-${NAMESPACE:-default}}} OPENBAO_RESOURCE_NAMESPACE="$OPENBAO_NAMESPACE" @@ -416,6 +419,85 @@ should_apply_kerberos_configmap() { return 1 } +iscsi_pv_names_from_manifest() { + local manifest="$1" + [[ -f "$manifest" ]] || return 0 + # Extract PV names by scanning PersistentVolume documents. + awk ' + $1=="kind:" && $2=="PersistentVolume" {in_pv=1; next} + in_pv && $1=="name:" {print $2; in_pv=0} + ' "$manifest" | tr -d $'"\r' | grep -v '^$' || true +} + +desired_pv_path_from_manifest() { + local manifest="$1" + local pv_name="$2" + [[ -f "$manifest" ]] || return 0 + awk -v pv="$pv_name" ' + /^---/ {in_doc=0; is_pv=0; hit=0} + $1=="kind:" && $2=="PersistentVolume" {is_pv=1} + is_pv && $1=="name:" && $2==pv {hit=1} + hit && $1=="path:" {print $2; exit} + ' "$manifest" | tr -d $'"\r' | head -1 || true +} + +desired_pv_node_from_manifest() { + local manifest="$1" + local pv_name="$2" + [[ -f "$manifest" ]] || return 0 + awk -v pv="$pv_name" ' + /^---/ {is_pv=0; hit=0; in_values=0} + $1=="kind:" && $2=="PersistentVolume" {is_pv=1} + is_pv && $1=="name:" && $2==pv {hit=1} + hit && $1=="values:" {in_values=1; next} + in_values && $1=="-" {print $2; exit} + ' "$manifest" | tr -d $'"\r' | head -1 || true +} + +ensure_iscsi_pvs() { + local manifest="$1" + [[ -f "$manifest" ]] || return 0 + + local pv + for pv in $(iscsi_pv_names_from_manifest "$manifest"); do + if ! kubectl get pv "$pv" >/dev/null 2>&1; then + continue + fi + + local desired_path desired_node + desired_path=$(desired_pv_path_from_manifest "$manifest" "$pv") + desired_node=$(desired_pv_node_from_manifest "$manifest" "$pv") + + local actual_path actual_node phase + actual_path=$(kubectl get pv "$pv" -o jsonpath='{.spec.local.path}' 2>/dev/null || true) + actual_node=$(kubectl get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]}' 2>/dev/null || true) + phase=$(kubectl get pv "$pv" -o jsonpath='{.status.phase}' 2>/dev/null || true) + + local mismatch=0 + if [[ -n "$desired_path" && -n "$actual_path" && "$desired_path" != "$actual_path" ]]; then + mismatch=1 + fi + if [[ -n "$desired_node" && -n "$actual_node" && "$desired_node" != "$actual_node" ]]; then + mismatch=1 + fi + + if [[ "$mismatch" == "1" ]]; then + if [[ "$phase" == "Bound" ]]; then + echo "ERROR: PV '$pv' is Bound but differs from desired immutable fields." >&2 + echo " actual path=$actual_path node=$actual_node" >&2 + echo " desired path=$desired_path node=$desired_node" >&2 + echo "Refusing to delete a Bound PV. Resolve by draining workloads / deleting PVCs, then retry." >&2 + return 1 + fi + echo "Recreating PV '$pv' to match desired local.path/nodeAffinity (phase=${phase:-}) ..." + kubectl delete pv "$pv" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || true + fi + done + + echo "Applying iSCSI PersistentVolumes ..." + kubectl apply -f "$manifest" +} + apply_k8s() { echo "Applying OpenBao manifest to namespace '$OPENBAO_RESOURCE_NAMESPACE' ..." local use_statefulset=0 @@ -423,14 +505,17 @@ apply_k8s() { use_statefulset=1 fi if [[ "$use_statefulset" == "1" ]]; then - # Ensure storage class and PVs exist for k3s - if [[ -f "$SCRIPT_DIR/../k8s/knoe/storageclass-synology-iscsi.yaml" ]]; then - echo "Applying StorageClass 'synology-iscsi' ..." - kubectl apply -f "$SCRIPT_DIR/../k8s/knoe/storageclass-synology-iscsi.yaml" - fi - if [[ -f "$SCRIPT_DIR/../k8s/knoe/iscsi-pvs.yaml" ]]; then - echo "Applying iSCSI PersistentVolumes ..." - kubectl apply -f "$SCRIPT_DIR/../k8s/knoe/iscsi-pvs.yaml" + # Ensure storage class and PVs exist for k3s (skipped on GKE/k8s — uses CSI provisioner) + if [[ "${KNOE_MODE:-}" != "k8s" ]]; then + if [[ -f "$SCRIPT_DIR/../k8s/knoe/storageclass-synology-iscsi.yaml" ]]; then + echo "Applying StorageClass 'synology-iscsi' ..." + kubectl apply -f "$SCRIPT_DIR/../k8s/knoe/storageclass-synology-iscsi.yaml" + fi + if [[ -f "$SCRIPT_DIR/../k8s/knoe/iscsi-pvs.yaml" ]]; then + ensure_iscsi_pvs "$SCRIPT_DIR/../k8s/knoe/iscsi-pvs.yaml" + fi + else + echo "[GKE] Skipping Synology iSCSI StorageClass and PVs (not supported on GKE Autopilot; using CSI provisioner)." fi local output="" @@ -440,6 +525,20 @@ apply_k8s() { else if [[ "${KNOE_MODE:-}" == "k3d" && "$output" == *"updates to statefulset spec"* ]]; then echo "WARN: OpenBao StatefulSet immutable in k3d; skipping apply." + elif [[ "$output" == *"updates to statefulset spec"* ]]; then + # VolumeClaimTemplates are immutable; delete the StatefulSet (PVCs are orphaned/preserved) + # and recreate so the new storageClass name takes effect. + echo "WARN: OpenBao StatefulSet VolumeClaimTemplates changed; deleting and recreating ..." + kubectl delete statefulset "$OPENBAO_NAME" -n "$OPENBAO_RESOURCE_NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true + local _pvc_sc + _pvc_sc=$(kubectl get pvc "data-openbao-0" -n "$OPENBAO_RESOURCE_NAMESPACE" \ + -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + if [[ -n "$_pvc_sc" && "$_pvc_sc" != "synology-iscsi" ]]; then + echo " Removing stale PVC 'data-openbao-0' (storageClass: $_pvc_sc) ..." + kubectl delete pvc "data-openbao-0" -n "$OPENBAO_RESOURCE_NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true + fi + knoe_render_manifest "$SCRIPT_DIR/../k8s/knoe/openbao-statefulset.yaml" \ + | kubectl apply --validate=false -n "$OPENBAO_RESOURCE_NAMESPACE" -f - else echo "$output" >&2 return 1 diff --git a/mock_val/init_port_forwards.sh b/mock_val/init_port_forwards.sh index 8a15bf1..82278a3 100755 --- a/mock_val/init_port_forwards.sh +++ b/mock_val/init_port_forwards.sh @@ -32,16 +32,16 @@ Usage: $PROG [-v|--verbose] [-f|--force] [-c|--config-file=FILE] [component] Options: - -c, --config-file=FILE Path to a knoe.cfg file to source when generating port-mapping.cfg + -c, --config-file=FILE Path to a config file to source when generating port-mapping.cfg -v, --verbose Verbose output -f, --force Force: kill existing processes blocking ports Examples: - $PROG -c ./knoe.cfg start + $PROG -c ./k3d.cfg start $PROG stop openbao $PROG --verbose status -Config (knoe.cfg or port-mapping.cfg): +Config (k3d.cfg/k3s.cfg/gke.cfg or port-mapping.cfg): PORT_FORWARD_K3D_MAPPING_1 = id=dashboard;namespace=kubernetes-dashboard;target=svc/kubernetes-dashboard-kong-proxy;address=127.0.0.1;hostPort=8443;servicePort=443;protocol=TCP;description=Kubernetes Dashboard PORT_FORWARD_K3S_MAPPING_1 = id=opentofu;namespace=\${NAMESPACE};target=svc/opentofu;address=0.0.0.0;hostPort=8080;servicePort=8080;protocol=TCP;description=OpenTofu Legacy port-mapping.cfg: @@ -60,13 +60,22 @@ have() { command -v "$1" >/dev/null 2>&1; } # ---- Config helpers / derived defaults ---- cfg_file() { - if [ -n "${KNOE_CONF:-}" ] && [ -f "$KNOE_CONF/knoe.cfg" ]; then - printf '%s' "$KNOE_CONF/knoe.cfg" - return 0 - fi - if [ -n "${KNOE_HOME:-}" ] && [ -f "$KNOE_HOME/conf/knoe.cfg" ]; then - printf '%s' "$KNOE_HOME/conf/knoe.cfg" - return 0 + local cfg="" + if declare -F _knoe_cfg_select_cfg_file >/dev/null 2>&1; then + if [ -n "${KNOE_CONF:-}" ]; then + cfg="$(_knoe_cfg_select_cfg_file "$KNOE_CONF")" + if [ -n "$cfg" ]; then + printf '%s' "$cfg" + return 0 + fi + fi + if [ -n "${KNOE_HOME:-}" ]; then + cfg="$(_knoe_cfg_select_cfg_file "$KNOE_HOME/conf")" + if [ -n "$cfg" ]; then + printf '%s' "$cfg" + return 0 + fi + fi fi return 1 } diff --git a/mock_val/init_redis.sh b/mock_val/init_redis.sh new file mode 100755 index 0000000..f21e1c4 --- /dev/null +++ b/mock_val/init_redis.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# init_redis.sh +# Purpose: +# - Deploy Redis as a shared common service via Bitnami Helm chart +# - Provides a pub/sub-capable Redis instance for knoe CLI, desktop, and GitLab +# - Runs in knoe-system (or SERVICE_NAMESPACE) so all services can reach it +# - Service endpoint: redis-master..svc.cluster.local:6379 + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Shared option parsing for common core scripts +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/common_core_lib.sh" + +# Inject default config if not provided +_has_config=0 +for _arg in "$@"; do + [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1 +done +if [[ $_has_config -eq 0 ]]; then + _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)" + if [[ -n "$_default_cfg" ]]; then + set -- "-c" "$_default_cfg" "$@" + fi +fi +unset _has_config _arg _default_cfg + +common_core_preparse_config "$@" + +# shellcheck disable=SC1090 +source "$SCRIPT_DIR/knoe_cfg.sh" + +set -- "${COMMON_CORE_ARGS[@]}" +common_core_parse_args "$@" + +if [[ -z "${KNOE_MODE:-}" ]]; then + export KNOE_MODE="k3s" +fi + +if [[ "${COMMON_CORE_HELP:-0}" == 1 ]]; then + common_core_usage "$0" + exit 0 +fi + +if [[ -n "${COMMON_CORE_PARSE_ERROR:-}" ]]; then + echo "ERROR: ${COMMON_CORE_PARSE_ERROR}" >&2 + common_core_usage "$0" + exit 2 +fi + +ACTION="$COMMON_CORE_ACTION" +NAMESPACE="$(common_core_resolve_namespace "${SERVICE_NAMESPACE:-knoe-system}")" +common_core_apply_namespace "$NAMESPACE" + +REDIS_RELEASE="${REDIS_RELEASE:-redis}" +REDIS_REPO_NAME="bitnami" +REDIS_REPO_URL="https://charts.bitnami.com/bitnami" +REDIS_CHART="bitnami/redis" +REDIS_CHART_VERSION="${REDIS_CHART_VERSION:-}" +REDIS_PORT="${REDIS_PORT:-6379}" +REDIS_PVC_SIZE="${REDIS_PVC_SIZE:-1Gi}" +# No dynamic provisioner is available in the service cluster; persistence disabled by default +# (Redis is used as a pub/sub broker — durable storage is not required) +REDIS_STORAGE_CLASS="${REDIS_STORAGE_CLASS:-${STORAGE_CLASS:-}}" +REDIS_PERSISTENCE_ENABLED="${REDIS_PERSISTENCE_ENABLED:-false}" +# Image registry override — set to "registry.bitnami.com" if docker.io is unreachable +# Bitnami OCI registry works as an alternative when Docker Hub times out on constrained networks. +REDIS_IMAGE_REGISTRY="${REDIS_IMAGE_REGISTRY:-}" +# Helm timeout — increase for slow image pulls on constrained network links (e.g. Pi cluster) +REDIS_HELM_TIMEOUT="${REDIS_HELM_TIMEOUT:-8m}" + +log() { echo "[INFO] $*"; } +warn() { echo "[WARN] $*" >&2; } +die() { echo "[ERROR] $*" >&2; exit 1; } + +command -v helm >/dev/null 2>&1 || die "helm not found" +command -v kubectl >/dev/null 2>&1 || die "kubectl not found" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +redis_service_host() { + printf '%s-master.%s.svc.cluster.local' "$REDIS_RELEASE" "$NAMESPACE" +} + +redis_is_running() { + kubectl -n "$NAMESPACE" get deployment "${REDIS_RELEASE}-master" >/dev/null 2>&1 || \ + kubectl -n "$NAMESPACE" get statefulset "${REDIS_RELEASE}-master" >/dev/null 2>&1 +} + +# --------------------------------------------------------------------------- +# Deploy +# --------------------------------------------------------------------------- +deploy_redis() { + # If Redis is already running and REDIS_FORCE_UPGRADE is not set, skip the + # helm upgrade to avoid Docker Hub / OCI registry timeouts on constrained + # networks (e.g. Pi cluster where registry-1.docker.io may be unreachable). + # Set REDIS_FORCE_UPGRADE=1 to force a chart upgrade regardless. + if [[ "${REDIS_FORCE_UPGRADE:-0}" != "1" ]] && redis_is_running; then + log "Redis already running in '${NAMESPACE}'; skipping upgrade." + log "Set REDIS_FORCE_UPGRADE=1 to force a helm upgrade." + return 0 + fi + + log "Adding/updating Bitnami Helm repo..." + helm repo add "$REDIS_REPO_NAME" "$REDIS_REPO_URL" 2>&1 || true + helm repo update "$REDIS_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..." + + kubectl get namespace "$NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$NAMESPACE" >/dev/null + + local helm_args=( + upgrade --install "$REDIS_RELEASE" "$REDIS_CHART" + --namespace "$NAMESPACE" + --create-namespace + --timeout "$REDIS_HELM_TIMEOUT" + --wait + --set architecture=standalone + --set auth.enabled=false + --set master.persistence.enabled="${REDIS_PERSISTENCE_ENABLED}" + --set master.persistence.size="$REDIS_PVC_SIZE" + --set master.resources.requests.memory=128Mi + --set master.resources.requests.cpu=50m + --set master.resources.limits.memory=512Mi + --set master.resources.limits.cpu=500m + ) + + if [[ -n "$REDIS_CHART_VERSION" ]]; then + helm_args+=(--version "$REDIS_CHART_VERSION") + fi + + if [[ -n "$REDIS_STORAGE_CLASS" ]]; then + helm_args+=(--set master.persistence.storageClass="$REDIS_STORAGE_CLASS") + fi + + if [[ -n "$REDIS_IMAGE_REGISTRY" ]]; then + log "Using image registry override: ${REDIS_IMAGE_REGISTRY}" + helm_args+=(--set global.imageRegistry="$REDIS_IMAGE_REGISTRY") + fi + + log "Deploying Redis (release=${REDIS_RELEASE}, ns=${NAMESPACE})..." + helm "${helm_args[@]}" + + log "Redis ready. Endpoint: $(redis_service_host):${REDIS_PORT}" + log "Pub/sub: redis-cli -h $(redis_service_host) -p ${REDIS_PORT} subscribe " +} + +# --------------------------------------------------------------------------- +# Action dispatch +# --------------------------------------------------------------------------- +case "$ACTION" in + start|update|reload|initialize|restart) + deploy_redis + ;; + stop) + log "Removing Redis (${REDIS_RELEASE}) from namespace ${NAMESPACE}..." + helm -n "$NAMESPACE" uninstall "$REDIS_RELEASE" 2>/dev/null || warn "Redis release not found." + ;; + status) + log "Redis status in namespace ${NAMESPACE}:" + kubectl -n "$NAMESPACE" get pods -l "app.kubernetes.io/name=redis" 2>/dev/null || \ + echo " No Redis pods found." + kubectl -n "$NAMESPACE" get svc -l "app.kubernetes.io/name=redis" 2>/dev/null || true + ;; + *) + die "Unknown action: $ACTION" + ;; +esac diff --git a/mock_val/init_registry.sh b/mock_val/init_registry.sh index 5b483ec..bd18212 100755 --- a/mock_val/init_registry.sh +++ b/mock_val/init_registry.sh @@ -19,10 +19,13 @@ _has_config=0 for _arg in "$@"; do [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1 done -if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - set -- "-c" "$SCRIPT_DIR/../conf/knoe.cfg" "$@" +if [[ $_has_config -eq 0 ]]; then + _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)" + if [[ -n "$_default_cfg" ]]; then + set -- "-c" "$_default_cfg" "$@" + fi fi -unset _has_config _arg +unset _has_config _arg _default_cfg common_core_preparse_config "$@" @@ -55,13 +58,21 @@ REGISTRY_MANIFEST_FILE=${REGISTRY_MANIFEST_FILE:-"$REGISTRY_MANIFEST_DIR/deploym usage() { cat < +Usage: init_registry.sh [-n|--namespace NS] [-r|--registry-namespace NS] Deploys registry support for local clusters. - In k3s mode, deploys `registry:2` into the service namespace (default: SERVICE_NAMESPACE). - In k3d mode, manages the `k3d` registry (port 5000) and removes any in-cluster registry resources. +Actions: + start|initialize|update|reload|restart Deploy or update the registry + stop Remove registry resources + status Show registry pod/service status + migrate Copy all images from registry:2 → gitlab-registry. + Run AFTER GitLab is up, BEFORE 'stop'. + Uses skopeo if available, otherwise prints commands. + NOTE: `-r/--registry-namespace` is kept for backwards compatibility and is treated as an alias for `-n/--namespace`. EOF } @@ -125,7 +136,11 @@ elif [[ -n "${REGISTRY_NAMESPACE:-}" ]]; then REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE" else # Default to the configured service namespace (driven by knoe.cfg via knoe_cfg.sh). - REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}" + REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-}}" + if [[ -z "$REGISTRY_NAMESPACE" ]]; then + echo "ERROR: REGISTRY_NAMESPACE could not be determined. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2 + exit 1 + fi fi # In k3s mode, the in-cluster registry is a common core service and should live @@ -134,8 +149,12 @@ _mode_resolved="${KNOE_MODE:-${DEPLOYMENT_MODE:-}}" if declare -F knoe_normalize_mode >/dev/null 2>&1; then _mode_resolved="$(knoe_normalize_mode "$_mode_resolved")" fi -if [[ "$_mode_resolved" == "k3s" && ( -z "${REGISTRY_NAMESPACE:-}" || "${REGISTRY_NAMESPACE}" == "default" ) ]]; then - REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}" +if [[ "$_mode_resolved" == "k3s" && -z "${REGISTRY_NAMESPACE:-}" ]]; then + REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-}}" + if [[ -z "$REGISTRY_NAMESPACE" ]]; then + echo "ERROR: REGISTRY_NAMESPACE could not be determined for k3s mode. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2 + exit 1 + fi fi unset _mode_resolved @@ -170,7 +189,7 @@ resolve_registry_node_selector() { return 0 fi # Default: pin to the control-plane node so hostPort:5000 is reachable via - # the k3s server host (e.g. myrddin.knoe.org:5000). + # the k3s server host (e.g. myrddin.prole.org:5000). resolve_control_plane_node_selector } @@ -323,8 +342,26 @@ apply_registry() { mode=$(current_mode) if [[ "$mode" == "k3d" ]]; then if command -v k3d >/dev/null 2>&1; then + # Remove legacy prole-registry that may be squatting on port 5000. + # It may be k3d-managed (delete via k3d) or a bare Docker container (rm -f). + if k3d registry list prole-registry >/dev/null 2>&1 \ + || docker inspect k3d-prole-registry >/dev/null 2>&1; then + echo "Removing legacy k3d-prole-registry from port 5000 ..." + k3d registry delete prole-registry >/dev/null 2>&1 || true + docker stop k3d-prole-registry >/dev/null 2>&1 || true + docker rm -f k3d-prole-registry >/dev/null 2>&1 || true + fi + echo "Ensuring k3d registry 'knoe-registry' on port 5000 ..." - if ! k3d registry list knoe-registry >/dev/null 2>&1; then + # If knoe-registry was created but never started (port was taken), nuke and recreate. + if k3d registry list knoe-registry >/dev/null 2>&1; then + if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -qE '^k3d-knoe-registry$'; then + echo "k3d-knoe-registry exists but is not running; recreating ..." + k3d registry delete knoe-registry >/dev/null 2>&1 || true + docker rm -f k3d-knoe-registry >/dev/null 2>&1 || true + k3d registry create knoe-registry --port 5000 || true + fi + else k3d registry create knoe-registry --port 5000 || true fi local cluster_name @@ -377,6 +414,30 @@ apply_registry() { fi fi + # In k3s mode, if GitLab is deployed it owns port 5000 (gitlab-registry). + # Deploying registry:2 with hostPort:5000 on the same node would conflict. + # Skip the apply and warn; use init_registry.sh stop to remove the old deployment. + local _mode_for_gitlab_check + _mode_for_gitlab_check=$(current_mode) + if [[ "$_mode_for_gitlab_check" == "k3s" ]]; then + local _gitlab_ns="${GITLAB_NAMESPACE:-gitlab}" + if kubectl get namespace "$_gitlab_ns" >/dev/null 2>&1; then + echo "WARN: GitLab namespace '${_gitlab_ns}' detected in k3s mode." >&2 + echo " gitlab-registry owns port 5000 — registry:2 deployment skipped." >&2 + echo " Run 'init_registry.sh stop' to remove any existing registry:2 resources." >&2 + return 0 + fi + fi + + # Idempotency check: skip apply+wait if the registry is already available. + local _desired _available + _desired=$(kubectl get deploy/registry -n "$REGISTRY_NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + _available=$(kubectl get deploy/registry -n "$REGISTRY_NAMESPACE" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true) + if [[ -n "$_desired" && "${_available:-0}" -ge "${_desired:-1}" && "${_desired:-0}" -gt 0 ]]; then + echo "Registry already running in namespace '$REGISTRY_NAMESPACE' ($_available/$_desired replicas available); skipping apply." + return 0 + fi + echo "Applying registry manifest to namespace '$REGISTRY_NAMESPACE' ..." render_registry_manifest | kubectl apply --server-side --force-conflicts --field-manager=knoe-installer --validate=false -n "$REGISTRY_NAMESPACE" -f - apply_registry_node_selector @@ -415,6 +476,72 @@ status_registry() { kubectl -n "$REGISTRY_NAMESPACE" get pods -l "app=registry" 2>/dev/null || true } +migrate_registry_to_gitlab() { + # Enumerate images from the registry:2 instance (registry..svc.cluster.local:5000) + # and copy them to gitlab-registry using skopeo when available. + # Safe to run multiple times — skopeo copy is idempotent. + # Skipped gracefully if registry:2 has no images or is already gone. + local src_ns="${REGISTRY_NAMESPACE:-knoe-system}" + local gitlab_ns="${GITLAB_NAMESPACE:-gitlab}" + local src_registry="${REGISTRY_MIGRATE_SRC:-registry.${src_ns}.svc.cluster.local:5000}" + local dst_registry="${REGISTRY_MIGRATE_DST:-gitlab-registry.${gitlab_ns}.svc.cluster.local:5000}" + + echo "Registry migration: ${src_registry} → ${dst_registry}" + + # Verify the source registry:2 pod exists at all before trying to catalog it. + if ! kubectl -n "$src_ns" get deploy/registry >/dev/null 2>&1 && \ + ! kubectl -n "$src_ns" get pods -l app=registry --field-selector=status.phase=Running 2>/dev/null | grep -q Running; then + echo "INFO: registry:2 not found in namespace '${src_ns}' — nothing to migrate." + return 0 + fi + + # Fetch repository catalog via a temporary curl pod on the cluster. + local repos + repos=$(kubectl -n "$src_ns" run registry-catalog-migrate \ + --image=alpine/curl:latest --restart=Never --rm --attach --quiet \ + --overrides="{\"spec\":{\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"c\",\"image\":\"alpine/curl:latest\",\"command\":[\"sh\",\"-c\",\"curl -sf http://${src_registry}/v2/_catalog\"]}]}}" \ + 2>/dev/null \ + | python3 -c "import sys,json; [print(r) for r in json.load(sys.stdin).get('repositories',[])]" 2>/dev/null || true) + + if [[ -z "$repos" ]]; then + echo "INFO: No repositories found in registry:2 at ${src_registry} — nothing to migrate." + return 0 + fi + + echo "Repositories to migrate:" + printf '%s\n' "$repos" | sed 's/^/ /' + + if command -v skopeo >/dev/null 2>&1; then + echo "skopeo found — migrating images automatically..." + local _ok=0 _fail=0 + while IFS= read -r repo; do + [[ -z "$repo" ]] && continue + echo " Copying ${repo} ..." + if skopeo copy --all \ + "docker://${src_registry}/${repo}" \ + "docker://${dst_registry}/${repo}" \ + --dest-tls-verify=false --src-tls-verify=false 2>&1; then + echo " [OK] ${repo}" + (( _ok++ )) || true + else + echo " [WARN] ${repo} — copy failed, may need manual retry" + (( _fail++ )) || true + fi + done <<< "$repos" + echo "Migration complete: ${_ok} succeeded, ${_fail} failed." + if [[ $_fail -gt 0 ]]; then + echo "Failed repos can be retried with: REGISTRY_MIGRATE_SRC=${src_registry} REGISTRY_MIGRATE_DST=${dst_registry} ./etc/init_registry.sh migrate" + fi + else + echo "" + echo "skopeo not found — run these commands manually (install: sudo apt-get install skopeo):" + while IFS= read -r repo; do + [[ -z "$repo" ]] && continue + echo " skopeo copy --all docker://${src_registry}/${repo} docker://${dst_registry}/${repo} --dest-tls-verify=false --src-tls-verify=false" + done <<< "$repos" + fi +} + case "${ACTION:-}" in start|initialize|update|reload|restart) ensure_tools @@ -437,6 +564,12 @@ case "${ACTION:-}" in status_registry fi ;; + migrate) + # Migrate images from registry:2 → gitlab-registry before decommissioning registry:2. + # Run after GitLab is up and before running 'stop' on the old registry. + ensure_tools + migrate_registry_to_gitlab + ;; *) usage >&2 exit 1 diff --git a/mock_val/init_service_layer.sh b/mock_val/init_service_layer.sh index d49ca54..dbd1f79 100755 --- a/mock_val/init_service_layer.sh +++ b/mock_val/init_service_layer.sh @@ -4,7 +4,7 @@ set -euo pipefail # init_service_layer.sh # Purpose: -# - Deploy the Knoe service layer (OpenTofu, Garage, OpenBao, Kong; Kerberos optional) +# - Deploy the service layer (Garage, OpenBao, Kong; OpenTofu in non-k8s modes) # - Keep service-layer resources grouped in SERVICE_NAMESPACE # - Migrate service layer to a new namespace @@ -18,10 +18,13 @@ _has_config=0 for _arg in "$@"; do [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1 done -if [[ $_has_config -eq 0 && -f "$SCRIPT_DIR/../conf/knoe.cfg" ]]; then - set -- "-c" "$SCRIPT_DIR/../conf/knoe.cfg" "$@" +if [[ $_has_config -eq 0 ]]; then + _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)" + if [[ -n "$_default_cfg" ]]; then + set -- "-c" "$_default_cfg" "$@" + fi fi -unset _has_config _arg +unset _has_config _arg _default_cfg common_core_preparse_config "$@" @@ -29,7 +32,7 @@ common_core_preparse_config "$@" source "$SCRIPT_DIR/knoe_cfg.sh" knoe_ensure_kubeconfig >/dev/null 2>&1 || true -knoe_ensure_kube_context || exit 1 +ensure_kube_context || exit 1 ACTION="" SERVICE_NAMESPACE_OVERRIDE="" @@ -169,11 +172,16 @@ deploy_service_layer() { local action="$1" local ns="$2" local rc=0 + local manage_opentofu=1 + + if [[ "${KNOE_MODE:-}" == "k8s" ]]; then + manage_opentofu=0 + fi ensure_namespace "$ns" label_namespace "$ns" - local argocd_action opentofu_action garage_action kdc_action openbao_action kong_action registry_action + local argocd_action opentofu_action garage_action kdc_action openbao_action redis_action kong_action registry_action case "$action" in start|initialize|update|reload) argocd_action="update" ;; restart) argocd_action="restart" ;; @@ -182,13 +190,15 @@ deploy_service_layer() { *) argocd_action="update" ;; esac - case "$action" in - start|initialize|update|reload) opentofu_action="update" ;; - restart) opentofu_action="restart" ;; - stop) opentofu_action="stop" ;; - status) opentofu_action="status" ;; - *) opentofu_action="update" ;; - esac + if [[ "$manage_opentofu" == "1" ]]; then + case "$action" in + start|initialize|update|reload) opentofu_action="update" ;; + restart) opentofu_action="restart" ;; + stop) opentofu_action="stop" ;; + status) opentofu_action="status" ;; + *) opentofu_action="update" ;; + esac + fi case "$action" in start|initialize|update|reload) openbao_action="update" ;; @@ -198,6 +208,14 @@ deploy_service_layer() { *) openbao_action="update" ;; esac + case "$action" in + start|initialize|update|reload) redis_action="update" ;; + restart) redis_action="restart" ;; + stop) redis_action="stop" ;; + status) redis_action="status" ;; + *) redis_action="update" ;; + esac + case "$action" in start|initialize|update|reload) garage_action="start" ;; restart) garage_action="restart" ;; @@ -239,7 +257,7 @@ deploy_service_layer() { # 2. OpenBao – secrets vault; needed by downstream services # 3. Garage – object storage # 4. Kong – API gateway - # 5. OpenTofu – IaC engine; depends on registry + secrets (last) + # 5. OpenTofu – IaC engine; depends on registry + secrets (non-k8s, last) # ------------------------------------------------------------------------- if [[ -x "$SCRIPT_DIR/init_registry.sh" ]]; then @@ -250,6 +268,14 @@ deploy_service_layer() { OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_openbao.sh" -n "$ns" "$openbao_action" || rc=$? + # Redis — shared pub/sub broker; deploy before Kong so GitLab and knoe services can reach it + if [[ -x "$SCRIPT_DIR/init_redis.sh" ]]; then + REDIS_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_redis.sh" -n "$ns" "$redis_action" || rc=$? + else + log "[WARN] init_redis.sh not found; Redis deploy skipped." + fi + # cert-manager is cluster-scoped and managed independently via init_certmgr.sh # in its own dedicated 'cert-manager' namespace; it is not part of the service layer. @@ -259,12 +285,21 @@ deploy_service_layer() { KONG_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_kong.sh" -n "$ns" "$kong_action" || rc=$? - OPENTOFU_NAMESPACE="$ns" OPENTOFU_SECRET_NAMESPACE="${NAMESPACE:-$ns}" OPENTOFU_OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" "$opentofu_action" || rc=$? + if [[ "$manage_opentofu" == "1" ]]; then + OPENTOFU_NAMESPACE="$ns" OPENTOFU_SECRET_NAMESPACE="${NAMESPACE:-$ns}" OPENTOFU_OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" "$opentofu_action" || rc=$? + else + log "[INFO] k8s mode: skipping OpenTofu deploy." + fi if [[ "$ENABLE_KERBEROS" == "1" ]]; then - SERVICE_NAMESPACE="$ns" PROLE_KDC_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$? + # KDC is embedded in `knoe-auth` by default. Only deploy standalone KDC when requested. + if [[ "${PROLE_KDC_STANDALONE:-0}" == "1" ]]; then + SERVICE_NAMESPACE="$ns" PROLE_KDC_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$? + else + log "[INFO] Kerberos enabled: skipping standalone KDC deploy (KDC runs as sidecar in 'knoe-auth')." + fi fi return "$rc" @@ -286,13 +321,17 @@ cleanup_old_namespace() { "$SCRIPT_DIR/init_garage_store.sh" stop || true OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_openbao.sh" -n "$ns" stop || true + if [[ -x "$SCRIPT_DIR/init_redis.sh" ]]; then + REDIS_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_redis.sh" -n "$ns" stop || true + fi CERTMGR_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_certmgr.sh" -n "$ns" stop || true if [[ -x "$SCRIPT_DIR/init_registry.sh" ]]; then REGISTRY_NAMESPACE="$REGISTRY_NS" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_registry.sh" -n "$REGISTRY_NS" stop || true fi - if [[ "$ENABLE_KERBEROS" == "1" ]]; then + if [[ "$ENABLE_KERBEROS" == "1" && "${PROLE_KDC_STANDALONE:-0}" == "1" ]]; then SERVICE_NAMESPACE="$ns" PROLE_KDC_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_kdc.sh" cleanup || true fi