From 492caabf60932d85c855421e14dfb1502690da89 Mon Sep 17 00:00:00 2001 From: chrisfu Date: Tue, 14 Apr 2026 06:01:50 -0700 Subject: [PATCH] chore: make node selector keys configurable and improve scheduling logic - Refactored `init_gitlab.sh` and `init_gitea.sh` to use configurable node selector keys, removing hardcoded defaults like `gandalf.prole.org`. - Enhanced scheduling logic to validate required fields and prevent stale node constraints during reruns. - Added fast-path guards in GitLab init to skip redundant operations when no changes occur in operator or CR specifications. - Updated Supabase deployment to clear stale topology spread constraints alongside node selectors and affinity. - Added tests for configurable node selector keys, fast-path guards, and stale constraint cleanups. --- etc/init_gitea.sh | 16 +- etc/init_gitlab.sh | 178 ++++++++++++++---- supabase/deploy.sh | 15 +- .../test_repair_update_and_supabase_flags.py | 45 +++++ 4 files changed, 216 insertions(+), 38 deletions(-) diff --git a/etc/init_gitea.sh b/etc/init_gitea.sh index 292f93d..a0fc9e9 100755 --- a/etc/init_gitea.sh +++ b/etc/init_gitea.sh @@ -83,7 +83,9 @@ IMAGE_REPO_DEFAULT="gitea/gitea" IMAGE_TAG="${GITEA_IMAGE_TAG:-1.22.3}" IMAGE_REPO="$IMAGE_REPO_DEFAULT" NODE_SELECTOR="${GITEA_NODE_SELECTOR:-${NODE_SELECTOR:-}}" -GITEA_PV_NODE="${GITEA_PV_NODE:-${GITEA_NODE_SELECTOR:-gandalf.prole.org}}" +NODE_SELECTOR_KEY="${GITEA_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}" +GITEA_PV_NODE_SELECTOR_KEY="${GITEA_PV_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY}}" +GITEA_PV_NODE="${GITEA_PV_NODE:-${GITEA_NODE_SELECTOR:-}}" GITEA_PV_BASE_DIR="${GITEA_PV_BASE_DIR:-/synology/d005}" GITEA_STORAGE_CLASS="${GITEA_STORAGE_CLASS:-gitea-local-d005}" GITEA_DOMAIN="${GITEA_DOMAIN:-git.prole.org}" @@ -97,6 +99,10 @@ GITEA_DB_NAME="${GITEA_DB_NAME:-gitea}" GITEA_DB_USER="${GITEA_DB_USER:-gitea}" GITEA_DB_PASSWORD="${GITEA_DB_PASSWORD:-${DB_PASSWORD:-}}" +if [[ "$MODE" == "k3s" && -z "$GITEA_PV_NODE" ]]; then + die "GITEA_PV_NODE (or GITEA_NODE_SELECTOR) must be set in k3s mode." +fi + is_secret_placeholder() { case "${1:-}" in '${PROLE_SECRET:'*|'${OPENBAO:'*) return 0 ;; @@ -266,7 +272,7 @@ deploy_with_helm() { --set "gitea.config.cache.ADAPTER=memory" --set "global.storageClass=$GITEA_STORAGE_CLASS" --set "persistence.storageClass=$GITEA_STORAGE_CLASS" - --set "nodeSelector.kubernetes\\.io/hostname=$GITEA_PV_NODE" + --set-json "nodeSelector={\"${GITEA_PV_NODE_SELECTOR_KEY}\":\"${GITEA_PV_NODE}\"}" --set "deployment.strategy=Recreate" ) fi @@ -314,7 +320,7 @@ spec: required: nodeSelectorTerms: - matchExpressions: - - key: kubernetes.io/hostname + - key: ${GITEA_PV_NODE_SELECTOR_KEY} operator: In values: - ${GITEA_PV_NODE} @@ -335,7 +341,7 @@ metadata: spec: restartPolicy: Never nodeSelector: - kubernetes.io/hostname: ${GITEA_PV_NODE} + ${GITEA_PV_NODE_SELECTOR_KEY}: ${GITEA_PV_NODE} containers: - name: perms image: busybox:1.36 @@ -398,7 +404,7 @@ apply_manifest_fallback() { if [[ -n "$NODE_SELECTOR" ]]; then node_selector_block=$(cat </dev/null || true; } | python3 - <<'PY' +import json +import sys + +try: + rows = json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) + +if not rows: + print("") + raise SystemExit(0) + +chart = str(rows[0].get("chart", "")) +if not chart: + print("") + raise SystemExit(0) + +if "-" in chart: + print(chart.rsplit("-", 1)[-1]) +else: + print("") +PY +} + +resolve_operator_watch_namespace() { + { helm -n "$OPERATOR_NAMESPACE" get values "$OPERATOR_RELEASE" -o json 2>/dev/null || true; } | python3 - <<'PY' +import json +import sys + +try: + values = json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) + +watch_namespace = values.get("watchNamespace", "") +if isinstance(watch_namespace, str): + print(watch_namespace.strip()) +else: + print("") +PY +} + # --------------------------------------------------------------------------- # Helper: resolve knoe-db primary pod (same pattern as init_gitea.sh) # --------------------------------------------------------------------------- @@ -720,35 +769,82 @@ log "gitlab-db-password secret applied in namespace '$NAMESPACE'." # --------------------------------------------------------------------------- # Install / upgrade the GitLab Operator # --------------------------------------------------------------------------- -log "Adding/updating GitLab Operator Helm repo..." -helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true -helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..." +operator_changed=0 +operator_release_exists=0 +operator_requires_upgrade=1 +installed_operator_chart_version="" +installed_operator_watch_namespace="" -log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..." +if helm -n "$OPERATOR_NAMESPACE" status "$OPERATOR_RELEASE" >/dev/null 2>&1; then + operator_release_exists=1 + installed_operator_chart_version="$(resolve_helm_release_chart_version "$OPERATOR_NAMESPACE" "$OPERATOR_RELEASE")" + installed_operator_watch_namespace="$(resolve_operator_watch_namespace)" +fi -# The operator needs cluster-scoped RBAC; it watches all namespaces by default. -helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \ - -n "$OPERATOR_NAMESPACE" \ - --create-namespace \ - --timeout 10m \ - --wait \ - --set watchNamespace="$NAMESPACE" +if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" && -n "$installed_operator_chart_version" ]]; then + GITLAB_OPERATOR_CHART_VERSION="$installed_operator_chart_version" + log "Reusing installed GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}" +fi -log "GitLab Operator ready." +if [[ "$operator_release_exists" == "1" && -n "$installed_operator_chart_version" && -n "$installed_operator_watch_namespace" \ + && "$installed_operator_chart_version" == "$GITLAB_OPERATOR_CHART_VERSION" \ + && "$installed_operator_watch_namespace" == "$NAMESPACE" \ + && "${GITLAB_OPERATOR_FORCE_UPGRADE:-0}" != "1" ]]; then + operator_requires_upgrade=0 +fi + +if [[ "$operator_requires_upgrade" == "1" ]]; then + log "Adding/updating GitLab Operator Helm repo..." + helm repo add "$OPERATOR_REPO_NAME" "$OPERATOR_REPO_URL" 2>&1 || true + helm repo update "$OPERATOR_REPO_NAME" 2>&1 || warn "helm repo update returned non-zero; continuing..." + + if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then + GITLAB_OPERATOR_CHART_VERSION=$(helm search repo "$OPERATOR_CHART" --output table 2>/dev/null \ + | awk 'NR==2{print $2}' || true) + if [[ -z "$GITLAB_OPERATOR_CHART_VERSION" ]]; then + die "Cannot detect GitLab Operator chart version. Set GITLAB_OPERATOR_CHART_VERSION explicitly and re-run." + fi + log "Auto-detected GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}" + else + log "Using GitLab Operator chart version: ${GITLAB_OPERATOR_CHART_VERSION}" + fi + + log "Installing GitLab Operator (release=${OPERATOR_RELEASE}, ns=${OPERATOR_NAMESPACE})..." + + # The operator needs cluster-scoped RBAC; it watches all namespaces by default. + helm upgrade --install "$OPERATOR_RELEASE" "$OPERATOR_CHART" \ + -n "$OPERATOR_NAMESPACE" \ + --create-namespace \ + --version "$GITLAB_OPERATOR_CHART_VERSION" \ + --timeout 10m \ + --wait \ + --set watchNamespace="$NAMESPACE" + + operator_changed=1 + log "GitLab Operator ready." +else + log "GitLab Operator already matches desired chart/watchNamespace; skipping operator upgrade." +fi # --------------------------------------------------------------------------- # Resolve GitLab chart version (required by the Operator CR since >= v0.28) # --------------------------------------------------------------------------- if [[ -z "$GITLAB_CHART_VERSION" ]]; then - log "Auto-detecting latest GitLab chart version..." - helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true - helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..." - GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \ - | awk 'NR==2{print $2}' || true) - if [[ -z "$GITLAB_CHART_VERSION" ]]; then - die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run." + installed_gitlab_chart_version="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.spec.chart.version}' 2>/dev/null || true)" + if [[ -n "$installed_gitlab_chart_version" ]]; then + GITLAB_CHART_VERSION="$installed_gitlab_chart_version" + log "Reusing installed GitLab chart version from existing GitLab CR: ${GITLAB_CHART_VERSION}" + else + log "Auto-detecting latest GitLab chart version..." + helm repo add "$GITLAB_CHART_REPO_NAME" "$GITLAB_CHART_REPO_URL" 2>&1 || true + helm repo update "$GITLAB_CHART_REPO_NAME" 2>&1 || warn "gitlab chart repo update warning; continuing..." + GITLAB_CHART_VERSION=$(helm search repo gitlab/gitlab --output table 2>/dev/null \ + | awk 'NR==2{print $2}' || true) + if [[ -z "$GITLAB_CHART_VERSION" ]]; then + die "Cannot detect GitLab chart version. Set GITLAB_CHART_VERSION explicitly (e.g. export GITLAB_CHART_VERSION=8.9.1) and re-run." + fi + log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}" fi - log "Auto-detected GitLab chart version: ${GITLAB_CHART_VERSION}" fi CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\"" @@ -759,18 +855,21 @@ CHART_VERSION_YAML="version: \"${GITLAB_CHART_VERSION}\"" if [[ "$MODE" == "k8s" ]]; then STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}" else - STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-gandalf.prole.org}}" + STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}" fi +[[ "$MODE" == "k8s" || -n "$STORAGE_NODE" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set in ${MODE} mode." # NODE_SELECTOR intentionally NOT defaulted — only storage components get pinned NODE_SELECTOR="${NODE_SELECTOR:-}" +NODE_SELECTOR_KEY="${GITLAB_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}" +STORAGE_NODE_SELECTOR_KEY="${GITLAB_STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY}}" NODE_SELECTOR_YAML="" if [[ -n "$NODE_SELECTOR" ]]; then log "Pinning all GitLab workloads to node: ${NODE_SELECTOR}" - NODE_SELECTOR_YAML="kubernetes.io/hostname: ${NODE_SELECTOR}" + NODE_SELECTOR_YAML="${NODE_SELECTOR_KEY}: ${NODE_SELECTOR}" fi STORAGE_NODE_SELECTOR_YAML="" if [[ -n "$STORAGE_NODE" ]]; then - STORAGE_NODE_SELECTOR_YAML="kubernetes.io/hostname: ${STORAGE_NODE}" + STORAGE_NODE_SELECTOR_YAML="${STORAGE_NODE_SELECTOR_KEY}: ${STORAGE_NODE}" fi GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}" @@ -1265,10 +1364,13 @@ path_style: true" \ # minio PV removed — object storage is provided by knoe-system/garage # --------------------------------------------------------------------------- setup_gitlab_legacy_storage() { - local synology_node="${GITLAB_STORAGE_NODE:-gandalf.prole.org}" + local synology_node="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}" + local storage_selector_key="${STORAGE_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}" local base_path="${GITLAB_STORAGE_BASE:-/synology/d005/gitlab}" local gitaly_size="${GITLAB_GITALY_PV_SIZE:-50Gi}" + [[ -n "$synology_node" ]] || die "GITLAB_STORAGE_NODE (or STORAGE_NODE) must be set for legacy GitLab storage setup." + log "Creating synology gitaly directory on ${synology_node}..." kubectl -n "$NAMESPACE" run gitlab-dirprep \ --image=alpine:latest --restart=Never --rm --attach \ @@ -1307,7 +1409,7 @@ spec: required: nodeSelectorTerms: - matchExpressions: - - key: kubernetes.io/hostname + - key: ${storage_selector_key} operator: In values: [${synology_node}] PVYAML @@ -1399,6 +1501,8 @@ fi log "Applying GitLab CR (hosts=${_gitlab_hosts_csv}, db=${GITLAB_DB_NAME}@${DB_HOST})..." +gitlab_generation_before="$(kubectl -n "$NAMESPACE" get gitlab "$GITLAB_RELEASE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true)" + kubectl apply -f - </dev/null || true)" +gitlab_spec_changed=1 +if [[ -n "$gitlab_generation_before" && -n "$gitlab_generation_after" && "$gitlab_generation_before" == "$gitlab_generation_after" ]]; then + gitlab_spec_changed=0 +fi + # minio is disabled (global.minio.enabled: false) — no ARM64 credential fix needed. # --------------------------------------------------------------------------- @@ -1612,12 +1722,16 @@ log "Watch pods: kubectl -n ${NAMESPACE} get pods -w" # --------------------------------------------------------------------------- WAIT_TIMEOUT="${GITLAB_WAIT_TIMEOUT:-1200}" # default 20 minutes if [[ "${GITLAB_NO_WAIT:-0}" != "1" ]]; then - log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status..." - if ! kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \ - --for=condition=Available \ - --timeout="${WAIT_TIMEOUT}s" 2>/dev/null; then - warn "Timed out waiting for GitLab CR condition=Available." - warn "The deployment may still be in progress. Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}" + if [[ "$operator_changed" == "0" && "$gitlab_spec_changed" == "0" ]]; then + log "Skipping GitLab reconcile wait: operator and GitLab CR spec unchanged." + else + log "Waiting up to ${WAIT_TIMEOUT}s for GitLab CR to reach Ready status..." + if ! kubectl -n "$NAMESPACE" wait gitlab/"$GITLAB_RELEASE" \ + --for=condition=Available \ + --timeout="${WAIT_TIMEOUT}s" 2>/dev/null; then + warn "Timed out waiting for GitLab CR condition=Available." + warn "The deployment may still be in progress. Check: kubectl -n ${NAMESPACE} describe gitlab ${GITLAB_RELEASE}" + fi fi fi diff --git a/supabase/deploy.sh b/supabase/deploy.sh index 3d4aac3..501f330 100755 --- a/supabase/deploy.sh +++ b/supabase/deploy.sh @@ -85,12 +85,25 @@ def ensure_enabled(data, component, enabled): cfg = deployment.setdefault(component, {}) cfg["enabled"] = bool(enabled) +def clear_stale_app_scheduling(data): + deployment = data.get("deployment", {}) + if not isinstance(deployment, dict): + return + + for cfg in deployment.values(): + if not isinstance(cfg, dict): + continue + cfg["nodeSelector"] = {} + cfg["affinity"] = {} + cfg["topologySpreadConstraints"] = [] + app_values = deepcopy(base) ensure_enabled(app_values, "studio", False) ensure_enabled(app_values, "kong", False) app_values.setdefault("studioIngress", {})["enabled"] = False app_values.setdefault("ingress", {})["enabled"] = False app_values.setdefault("scheduling", {})["enforceGeneralNodeRole"] = False +clear_stale_app_scheduling(app_values) db_values = deepcopy(base) for component in ( @@ -1944,7 +1957,7 @@ enforce_supabase_workload_node() { for clear_kind in deployment statefulset; do while IFS= read -r clear_resource; do [[ -n "$clear_resource" ]] || continue - kubectl -n "$clear_ns" patch "$clear_resource" --type merge -p '{"spec":{"template":{"spec":{"nodeSelector":{"kubernetes.io/hostname":null,"knoe.dev/node-role":null,"prole.org/node-role":null},"affinity":{"nodeAffinity":null}}}}}' >/dev/null 2>&1 || true + kubectl -n "$clear_ns" patch "$clear_resource" --type merge -p '{"spec":{"template":{"spec":{"nodeSelector":{"kubernetes.io/hostname":null,"knoe.dev/node-role":null,"prole.org/node-role":null},"affinity":{"nodeAffinity":null},"topologySpreadConstraints":null}}}}' >/dev/null 2>&1 || true done < <(kubectl -n "$clear_ns" get "$clear_kind" -o name 2>/dev/null || true) done } diff --git a/tests/test_repair_update_and_supabase_flags.py b/tests/test_repair_update_and_supabase_flags.py index 745f4ce..480a239 100644 --- a/tests/test_repair_update_and_supabase_flags.py +++ b/tests/test_repair_update_and_supabase_flags.py @@ -583,6 +583,51 @@ def test_supabase_deploy_split_app_and_db_values_disable_general_node_role_enfor assert 'db_values.setdefault("scheduling", {})["enforceGeneralNodeRole"] = False' in script +def test_supabase_deploy_split_app_values_clear_stale_scheduling_constraints(): + """APP split values must scrub stale node/affinity/topology placement constraints for GKE scheduling.""" + script = (REPO_ROOT / "supabase" / "deploy.sh").read_text(encoding="utf-8") + + assert "def clear_stale_app_scheduling(data):" in script + assert 'cfg["nodeSelector"] = {}' in script + assert 'cfg["affinity"] = {}' in script + assert 'cfg["topologySpreadConstraints"] = []' in script + assert "clear_stale_app_scheduling(app_values)" in script + + +def test_supabase_deploy_runtime_constraint_cleanup_clears_topology_spread_constraints(): + """Runtime stale-constraint cleanup must clear topology spread constraints in addition to node selectors/affinity.""" + script = (REPO_ROOT / "supabase" / "deploy.sh").read_text(encoding="utf-8") + + assert '"topologySpreadConstraints":null' in script + + +def test_init_gitlab_adds_noop_rerun_fast_path_guards(): + """GitLab init must include fast-path guards to skip heavy rerun work when operator/CR are unchanged.""" + script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8") + + assert 'GITLAB_OPERATOR_CHART_VERSION="${GITLAB_OPERATOR_CHART_VERSION:-}"' in script + assert "resolve_helm_release_chart_version()" in script + assert "resolve_operator_watch_namespace()" in script + assert "operator_requires_upgrade=1" in script + assert "Reusing installed GitLab chart version from existing GitLab CR" in script + assert "Skipping GitLab reconcile wait: operator and GitLab CR spec unchanged." in script + assert "gitlab_generation_before=" in script + assert "gitlab_generation_after=" in script + + +def test_gitlab_and_gitea_init_storage_node_defaults_are_config_driven(): + """GitLab/Gitea init scripts must not hardcode physical host defaults for storage node pinning.""" + gitlab_script = (REPO_ROOT / "etc" / "init_gitlab.sh").read_text(encoding="utf-8") + gitea_script = (REPO_ROOT / "etc" / "init_gitea.sh").read_text(encoding="utf-8") + + assert "GITLAB_STORAGE_NODE:-gandalf.prole.org" not in gitlab_script + assert "GITEA_NODE_SELECTOR:-gandalf.prole.org" not in gitea_script + assert "gandalf.prole.org" not in gitlab_script + assert "gandalf.prole.org" not in gitea_script + assert 'NODE_SELECTOR_KEY="${GITLAB_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"' in gitlab_script + assert 'NODE_SELECTOR_KEY="${GITEA_NODE_SELECTOR_KEY:-${NODE_SELECTOR_KEY:-kubernetes.io/hostname}}"' in gitea_script + + def test_init_kong_allows_disabling_svc_ingress_tls_for_k8s_path(): """etc/init_kong.sh must support non-TLS svc ingress rendering for k8s/GKE.""" script = (REPO_ROOT / "etc" / "init_kong.sh").read_text(encoding="utf-8")