From 1b08f725381b666a859739ee87cdfacb2c9e8c40 Mon Sep 17 00:00:00 2001 From: chrisfu Date: Mon, 20 Apr 2026 22:51:41 -0700 Subject: [PATCH] fix(gitlab): stop AUTOCLEAN from wiping gitaly PVC on every run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovered today after the SSH-ingress reconcile: init_gitlab.sh had a destructive false-positive in its "legacy storage detection" at the top of setup_gitlab_storage(). The bare check if kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then has_legacy=1; fi flagged has_legacy=1 whenever the PVC object simply existed — which is always true after the first successful deploy. Combined with GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1, every init_gitlab.sh invocation triggered the "repairing legacy Gitaly storage" branch, which scales the Gitaly StatefulSet to 0 and deletes repo-data-gitlab-gitaly-0 immediately, wiping all git-repo data. We caught it when the knoe-db repo (pushed via HTTPS earlier in the day) disappeared after an unrelated init_gitlab.sh run for SSH ingress wiring. The PVC age on inspection was ~26 min — newer than the push. git ls-remote against the project now returns a garbled gzip body, consistent with an empty gitaly backing an existing DB project. Fix: - etc/init_gitlab.sh:setup_gitlab_storage() * Remove the bare `pvc exists` legacy check. * Keep the real legacy markers: synology PV, gandalf.prole.org nodeSelector, gitlab-gitaly-static storageClass on the STS. * Add a new check that flags has_legacy=1 only when the PVC's actual storageClassName is a known-legacy value (gitlab-gitaly-static or synology-iscsi), never on mere existence. - conf/gke.cfg * GITLAB_REPAIR_BLOCKED_AUTOCLEAN: 1 -> 0. The script fix above makes re-enabling safe, but the conservative default is to let the user see repair_blocked and consent rather than silently wipe data. Co-Authored-By: Claude Opus 4.7 (1M context) --- conf/gke.cfg | 38 ++++++++++++++------------------------ etc/init_gitlab.sh | 20 ++++++++++++++++++-- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/conf/gke.cfg b/conf/gke.cfg index 22ac47a..9caa8fd 100644 --- a/conf/gke.cfg +++ b/conf/gke.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-20 14:42:17 +; Generated by install.py on 2026-04-20 22:44:30 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -79,8 +79,8 @@ init_password.cluster_name = ${CLUSTER_NAME} init_password.db_cluster_name = knoe-dev-cnpg-0 init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password = ${PROLE_SECRET:v1:t152p6lozmV1olHq:cJCCJAqk8iusivgp9akabPgYgi5RZYqVI9grBY4pRzaWcuQviqizZg==} -init_password.db_password_confirm = ${PROLE_SECRET:v1:t152p6lozmV1olHq:cJCCJAqk8iusivgp9akabPgYgi5RZYqVI9grBY4pRzaWcuQviqizZg==} +init_password.db_password = ${PROLE_SECRET:v1:rNg0cdK0dMk1kxsb:0Eu8yWnNpdwTDLnAaUItgMV6VjMZPDTQRMrU09mKQzozWE05U-7LLQ==} +init_password.db_password_confirm = ${PROLE_SECRET:v1:rNg0cdK0dMk1kxsb:0Eu8yWnNpdwTDLnAaUItgMV6VjMZPDTQRMrU09mKQzozWE05U-7LLQ==} init_password.db_username = chrisfu init_password.generate_ssh_key = true init_scripts.run_scripts = true @@ -122,33 +122,20 @@ DOCKER_PRELOAD = false GARAGE_PRIVATE_S3_ENDPOINT = http://10.180.15.239:3900 GITLAB_DOMAIN = git.knoe.dev GITLAB_GITALY_STORAGE_CLASS = standard +GITLAB_GLOBAL_STATIC_IP_NAME = GITLAB_INGRESS_CLASS = gce GITLAB_OIDC_CLIENT_ID = secretref://google-oidc-client-id GITLAB_OIDC_CLIENT_SECRET = secretref://google-oidc-client-secret GITLAB_OIDC_ISSUER = https://accounts.google.com GITLAB_PUBLIC_HOSTS = git.knoe.dev -; Pin the GCE L7 HTTPS ingresses to reserved GLOBAL static IPs (gcloud compute -; addresses create --global). Prevents IP churn on ingress delete/recreate. -; git.knoe.dev is intentionally LEFT UNPINNED — the ingress was already bound -; to 34.107.159.254 (working, cert Active, DNS pointed at it) when we added -; the pin surface. Swapping it would force a ~5-15 min cert re-Provision for -; no benefit. Defer the pin to the next time the ingress recreates for -; another reason; reserve the IP under `git-knoe-dev` in gcloud first, then -; set the key below. -GITLAB_GLOBAL_STATIC_IP_NAME = -; Kong (knoe-system/svc-knoe-ingress, serves svc.knoe.dev + api.knoe.dev) -SVC_KNOE_GLOBAL_STATIC_IP_NAME = svc-knoe -; Supabase Kong API ingress (supabase/supabase-kong, serves api.0.knoe.dev) -SUPABASE_API_GLOBAL_STATIC_IP_NAME = supabase-api -; Supabase Studio ingress (supabase/supabase-studio, serves db.0.knoe.dev) -SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME = supabase-studio -; SSH ingress — dedicated REGIONAL external static IP (us-west3) bound to -; gitlab-shell Service of type LoadBalancer. Must be created via -; gcloud compute addresses create git-knoe-ssh --region=us-west3 -; then fill in the .address value below. -GITLAB_SSH_HOST = git-ssh.knoe.dev +; AUTOCLEAN was turned OFF on 2026-04-20 after it destructively deleted the +; gitaly PVC on every init_gitlab.sh run (has_legacy false-positive via +; bare PVC-exists check — fixed in setup_gitlab_storage). With the fix in +; place the flag is safe to re-enable, but 0 is the conservative default: +; a repair_blocked message is preferable to a silent data wipe. +GITLAB_REPAIR_BLOCKED_AUTOCLEAN = 0 GITLAB_SHELL_LOADBALANCER_IP = 34.106.243.154 -GITLAB_REPAIR_BLOCKED_AUTOCLEAN = 1 +GITLAB_SSH_HOST = git-ssh.knoe.dev GITLAB_WEBSERVICE_LIMITS_MEMORY = 3Gi GITLAB_WEBSERVICE_PUMA_THREADS_MAX = 2 GITLAB_WEBSERVICE_PUMA_THREADS_MIN = 2 @@ -173,12 +160,15 @@ PROTECTED_GIT_HOSTS = git.knoe.dev REDIS_HOST = redis-master.knoe-system.svc.cluster.local REGISTRY_NAMESPACE = knoe-system SERVICE_NAMESPACE = knoe-system +SUPABASE_API_GLOBAL_STATIC_IP_NAME = supabase-api SUPABASE_API_HOSTNAME = api.0.knoe.dev SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS = true SUPABASE_AUTO_CLEAN_RETAINED_PVS = true SUPABASE_HOSTNAME = db.0.knoe.dev SUPABASE_INGRESS_CLASS = gce +SUPABASE_STUDIO_GLOBAL_STATIC_IP_NAME = supabase-studio SUPABASE_STUDIO_HOSTNAME = db.0.knoe.dev +SVC_KNOE_GLOBAL_STATIC_IP_NAME = svc-knoe [Welcome] ; No configuration values captured yet for this section. diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh index c1d0a4a..f9da55c 100755 --- a/etc/init_gitlab.sh +++ b/etc/init_gitlab.sh @@ -3703,10 +3703,17 @@ setup_gitlab_storage() { if [[ "$MODE" == "k8s" ]]; then log "Skipping legacy Synology/local-PV gitaly storage prep in k8s mode; using dynamic StorageClass '${GITALY_STORAGE_CLASS}'." - # Check for legacy resources on GKE/k8s + # Check for ACTUAL legacy resources on GKE/k8s. + # + # Historical bug: the bare check `kubectl get pvc repo-data-gitlab-gitaly-0` + # flagged has_legacy=1 on every run once the PVC existed — which is always + # true after the first deploy. Combined with GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 + # that triggered a destructive PVC delete on every init_gitlab.sh run, + # silently wiping the gitaly git-repo PV. The chart's normal PVC is not + # legacy; the real legacy markers are synology PV, gandalf.prole.org + # nodeSelector, or gitlab-gitaly-static storageClass. local has_legacy=0 if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then has_legacy=1; fi - if kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then has_legacy=1; fi local gitaly_sts_name="${GITLAB_RELEASE}-gitaly" local gitaly_sts_yaml @@ -3717,6 +3724,15 @@ setup_gitlab_storage() { fi fi + # Also treat PVC as legacy only when its actual storageClassName is a + # known legacy class — not merely because the object exists. + local pvc_sc + pvc_sc=$(kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 \ + -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + if [[ "$pvc_sc" == "gitlab-gitaly-static" || "$pvc_sc" == "synology-iscsi" ]]; then + has_legacy=1 + fi + if [[ "$has_legacy" == "1" ]]; then if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then export GITALY_AUTOCLEAN_PERFORMED=1