From e44ff8e4765f8cde1499b9e6592a1d9cadb3b39b Mon Sep 17 00:00:00 2001 From: chrisfu Date: Mon, 20 Apr 2026 18:51:41 -0700 Subject: [PATCH] fix(gitlab): unblock standalone reconcile + OIDC + object storage Enables iterating on GitLab reconciles without a full deploy.sh run and unblocks the end-to-end login flow: - etc/init_gitlab.sh * Pre-scan $@ for --config before sourcing prole_cfg.sh so PROLE_DEPLOY_CFG is set for standalone invocations; previously prole_cfg.sh auto-picked conf/k3d.cfg and leaked KUBECONTEXT=dev, failing with `error: context "dev" does not exist`. * _init_gitlab_resolve_secretref: bash mirror of knoe/core/actions.py:_resolve_secretref_value so secretref://google-oidc-client-* resolves from etc/secrets/ files when run outside deploy.sh Python env-injection. * Garage bucket array: drop -storage suffix from uploads/artifacts/ lfs/packages/dependency-proxy. Chart defaults for these have no suffix, and the object_store block in the CR does not override per-object bucket names; the prior mismatch caused first-login 500s (NoSuchBucket on avatar PUT). - deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml * Add discovery: true so omniauth-openid_connect fetches Google .well-known/openid-configuration; fixes "Could not authenticate from OpenIDConnect: No host info" on the callback. - conf/gke.cfg * GITLAB_WEBSERVICE_LIMITS_MEMORY 1800M -> 3Gi, REQUESTS_MEMORY 900M -> 2Gi, REQUESTS_CPU 200m -> 500m. Live pod was sitting at 1706Mi/1800M (99%) in OOMKilled loop. - conf/port-mapping.cfg: add supabase + gitea forwards, fix postgres namespace knoe-db -> knoe-db-0 for split-cluster CNPG layout. Adds read-only diagnostics used to chase the above: - etc/diag_gitlab_boot.sh, diag_gitlab_webservice_oom.sh, diag_gke_storage.sh - etc/ensure_default_storage_class.sh + k8s/prole/storageclass-gcp-standard-hdd.yaml (preflight + HDD-default SC manifest for SSD-quota-constrained GKE projects). Co-Authored-By: Claude Opus 4.7 (1M context) --- conf/gke.cfg | 28 +- conf/port-mapping.cfg | 9 +- .../gitlab-google-oidc-secret.example.yaml | 6 + etc/diag_gitlab_boot.sh | 134 +++++++++ etc/diag_gitlab_webservice_oom.sh | 132 +++++++++ etc/diag_gke_storage.sh | 179 ++++++++++++ etc/ensure_default_storage_class.sh | 255 ++++++++++++++++++ etc/init_gitlab.sh | 115 +++++++- k8s/prole/storageclass-gcp-standard-hdd.yaml | 30 +++ 9 files changed, 864 insertions(+), 24 deletions(-) create mode 100755 etc/diag_gitlab_boot.sh create mode 100755 etc/diag_gitlab_webservice_oom.sh create mode 100755 etc/diag_gke_storage.sh create mode 100755 etc/ensure_default_storage_class.sh create mode 100644 k8s/prole/storageclass-gcp-standard-hdd.yaml diff --git a/conf/gke.cfg b/conf/gke.cfg index 913bff8..34a7e90 100644 --- a/conf/gke.cfg +++ b/conf/gke.cfg @@ -1,5 +1,5 @@ ; Prole Master Configuration File -; Generated by install.py on 2026-04-18 20:45:32 +; Generated by install.py on 2026-04-20 14:42:17 ; This file is used as input for Ansible deployment and k8s cluster creation. [User] @@ -8,8 +8,8 @@ [Inputs] ; Screen-scoped inputs used for unattended replays (-S) -auth.clientId = secretref://google-oidc-client-id -auth.clientSecret = secretref://google-oidc-client-secret +auth.clientId = 200699021716-pbaekj5q8i3fplhof0u7gccrvkvvacug.apps.googleusercontent.com +auth.clientSecret = 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 build.deploy_env = Prod build.run_build = false dependencies.ansible.install = true @@ -79,8 +79,8 @@ init_password.cluster_name = ${CLUSTER_NAME} init_password.db_cluster_name = knoe-dev-cnpg-0 init_password.db_host_port = 5432 init_password.db_namespace = ${DATABASE_NAMESPACE} -init_password.db_password = ${PROLE_SECRET:v1:5RlyU1vwz3CDoq6u:UfdWQw93uyPDIY_TWxRzdIbyZQCGv4sBpjNFmQMDRdmrypdVxcwz0w==} -init_password.db_password_confirm = ${PROLE_SECRET:v1:5RlyU1vwz3CDoq6u:UfdWQw93uyPDIY_TWxRzdIbyZQCGv4sBpjNFmQMDRdmrypdVxcwz0w==} +init_password.db_password = ${PROLE_SECRET:v1:t152p6lozmV1olHq:cJCCJAqk8iusivgp9akabPgYgi5RZYqVI9grBY4pRzaWcuQviqizZg==} +init_password.db_password_confirm = ${PROLE_SECRET:v1:t152p6lozmV1olHq:cJCCJAqk8iusivgp9akabPgYgi5RZYqVI9grBY4pRzaWcuQviqizZg==} init_password.db_username = chrisfu init_password.generate_ssh_key = true init_scripts.run_scripts = true @@ -114,7 +114,7 @@ CNPG_SIZE_PROFILE = small DATABASE_NAMESPACE = knoe-db-0 DB_CLUSTER_KUBECONTEXT = gke_plenary-truck-485623-p7_us-west3_knoe-dev-cnpg-0 DB_HOST_PORT = 5432 -DB_PASSWORD = ${PROLE_SECRET:v1:v42vq25mZFVFIwuZ:YK1Bz8WwwhkEQk712JAa2aCWqokdAyVG_Z1bP_Rj5RxVCoU6_lDYYA==} +DB_PASSWORD = ${PROLE_SECRET:v1:AX1bdMcT41sPTZbS:wvvf8AVN-hvKOcJWg1EIBtly78nrTL7rBrjfLXboqqMUD_qw-lSoSQ==} DEPLOYMENT_MODE = k8s DEPLOYMENT_TARGET = prole-prod-cluster DOCKER_IMPORT_DIR = @@ -123,22 +123,16 @@ GARAGE_PRIVATE_S3_ENDPOINT = http://10.180.15.239:3900 GITLAB_DOMAIN = git.knoe.dev GITLAB_GITALY_STORAGE_CLASS = standard GITLAB_INGRESS_CLASS = gce -; OIDC client credentials are indirected through secretref://; the loader -; (knoe/core/actions.py:_resolve_secretref_value) reads them from env or from -; etc/secrets/ (the etc/secrets/ directory is gitignored). This keeps -; plaintext client_secret out of the repo so partial public forks stay clean. GITLAB_OIDC_CLIENT_ID = secretref://google-oidc-client-id GITLAB_OIDC_CLIENT_SECRET = secretref://google-oidc-client-secret -; Path A: Google Workspace as OIDC OP directly (bypassing knoe-auth, which -; doesn't yet implement an OIDC provider surface). See docs/oidc-setup.md. GITLAB_OIDC_ISSUER = https://accounts.google.com GITLAB_PUBLIC_HOSTS = git.knoe.dev GITLAB_REPAIR_BLOCKED_AUTOCLEAN = 1 -GITLAB_WEBSERVICE_LIMITS_MEMORY = 1800M +GITLAB_WEBSERVICE_LIMITS_MEMORY = 3Gi GITLAB_WEBSERVICE_PUMA_THREADS_MAX = 2 GITLAB_WEBSERVICE_PUMA_THREADS_MIN = 2 -GITLAB_WEBSERVICE_REQUESTS_CPU = 200m -GITLAB_WEBSERVICE_REQUESTS_MEMORY = 900M +GITLAB_WEBSERVICE_REQUESTS_CPU = 500m +GITLAB_WEBSERVICE_REQUESTS_MEMORY = 2Gi GITLAB_WEBSERVICE_WORKER_PROCESSES = 1 GRAFANA_HOSTNAME = svc.knoe.dev K3S_SERVER = @@ -159,11 +153,11 @@ REDIS_HOST = redis-master.knoe-system.svc.cluster.local REGISTRY_NAMESPACE = knoe-system SERVICE_NAMESPACE = knoe-system SUPABASE_API_HOSTNAME = api.0.knoe.dev +SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS = true +SUPABASE_AUTO_CLEAN_RETAINED_PVS = true SUPABASE_HOSTNAME = db.0.knoe.dev SUPABASE_INGRESS_CLASS = gce SUPABASE_STUDIO_HOSTNAME = db.0.knoe.dev -SUPABASE_AUTO_CLEAN_RETAINED_PVS = true -SUPABASE_AUTO_CLEAN_RETAINED_GCE_DISKS = true [Welcome] ; No configuration values captured yet for this section. diff --git a/conf/port-mapping.cfg b/conf/port-mapping.cfg index 8ad477c..1486b6b 100644 --- a/conf/port-mapping.cfg +++ b/conf/port-mapping.cfg @@ -2,12 +2,17 @@ # Format: key: local=... remote=... ns=... svc=... address=... argocd: local=8081 remote=80 ns=argocd svc=argocd-server address=0.0.0.0 +supabase-studio: local=18080 remote=3000 ns=supabase svc=studio address=0.0.0.0 +supabase-auth: local=9999 remote=9999 ns=supabase svc=auth address=127.0.0.1 +supabase-rest: local=3001 remote=3000 ns=supabase svc=rest address=0.0.0.0 +supabase-realtime: local=4000 remote=4000 ns=supabase svc=realtime address=0.0.0.0 +gitea-http: local=13000 remote=3000 ns=gitea svc=gitea-http address=0.0.0.0 gitea-ssh: local=22 remote=22 ns=gitea svc=gitea-ssh address=0.0.0.0 garage: local=3900 remote=3900 ns=knoe-system svc=garage address=0.0.0.0 openbao: local=8200 remote=8200 ns=knoe-system svc=openbao address=0.0.0.0 opentofu: local=8080 remote=8080 ns=knoe-system svc=opentofu address=0.0.0.0 dashboard: local=8443 remote=443 ns=kubernetes-dashboard svc=kubernetes-dashboard-kong-proxy address=127.0.0.1 -postgres: local=5432 remote=5432 ns=knoe-db svc=knoe-db-rw address=0.0.0.0 +postgres: local=5432 remote=5432 ns=knoe-db-0 svc=knoe-db-rw address=0.0.0.0 prometheus: local=9090 remote=9090 ns=monitoring svc=kps-kube-prometheus-stack-prometheus address=127.0.0.1 grafana: local=3000 remote=80 ns=monitoring svc=kps-grafana address=0.0.0.0 -gitea-http: local=13000 remote=3000 ns=gitea svc=gitea-http address=0.0.0.0 +supabase-kong: local=8000 remote=8000 ns=supabase svc=kong address=0.0.0.0 diff --git a/deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml b/deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml index 1324a58..ec2f573 100644 --- a/deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml +++ b/deploy/gcp/gke/gitlab-google-oidc-secret.example.yaml @@ -20,6 +20,12 @@ stringData: - profile - email response_type: code + # discovery: true triggers a one-time fetch of + # ${issuer}/.well-known/openid-configuration at webservice boot, which + # populates the OpenIDConnect client's host/port/scheme/endpoints. + # Without it omniauth-openid_connect fails every callback with + # "Could not authenticate you from OpenIDConnect because 'No host info'". + discovery: true issuer: "${GITLAB_OIDC_ISSUER}" client_auth_method: query uid_field: email diff --git a/etc/diag_gitlab_boot.sh b/etc/diag_gitlab_boot.sh new file mode 100755 index 0000000..4862523 --- /dev/null +++ b/etc/diag_gitlab_boot.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# diag_gitlab_boot.sh — read-only snapshot of GitLab backend health. +# +# Investigates why the frontdoor returns "Waiting for GitLab to boot" even +# after DNS + ingress are green. Focuses on: +# - webservice pod states + last-terminated reasons (OOM, probe failures) +# - webservice + workhorse log tail on the likely-serving pod +# - Gitaly readiness +# - recent Warning events +# +# Usage: ./etc/diag_gitlab_boot.sh [--namespace NS] [--kube-context CTX] +# --namespace defaults to 'gitlab' + +set -euo pipefail + +NS="gitlab" +KUBE_CONTEXT="" +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) NS="$2"; shift 2 ;; + --kube-context) KUBE_CONTEXT="$2"; shift 2 ;; + -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +kctl() { + if [[ -n "$KUBE_CONTEXT" ]]; then + kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@" + else + kubectl -n "$NS" "$@" + fi +} + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +WEB_JSON="$TMPDIR/web.json" +kctl get pods -l app=webservice -o json > "$WEB_JSON" + +echo "=== Namespace: $NS ===" +echo +echo "=== Webservice pods (restarts + last termination) ===" + +PY_SCRIPT="$TMPDIR/analyze.py" +cat > "$PY_SCRIPT" <<'PY' +import json, sys + +with open(sys.argv[1]) as f: + data = json.load(f) + +pods = data.get("items") or [] +if not pods: + print(" (no webservice pods found)") + +for pod in pods: + name = (pod.get("metadata") or {}).get("name", "") + phase = (pod.get("status") or {}).get("phase", "") + print(f"\n-- {name} phase={phase} --") + for cs in (pod.get("status") or {}).get("containerStatuses") or []: + cname = cs.get("name", "") + ready = cs.get("ready") + restart = cs.get("restartCount", 0) + state = cs.get("state") or {} + last = cs.get("lastState") or {} + state_key = next(iter(state.keys()), "unknown") + state_detail = state.get(state_key) or {} + last_term = (last.get("terminated") or {}) if last else {} + print(f" container={cname:<22} ready={ready} restarts={restart} state={state_key}") + if state_detail.get("reason"): + print(f" state.reason={state_detail.get('reason')} msg={(state_detail.get('message') or '')[:180]}") + if last_term: + print(f" last terminated: exit={last_term.get('exitCode')} reason={last_term.get('reason')} " + f"signal={last_term.get('signal')}") + print(f" last start={last_term.get('startedAt')} finish={last_term.get('finishedAt')}") + if last_term.get("message"): + print(f" last msg: {(last_term.get('message') or '')[:240]}") + +# Emit the best-candidate pod name to a sidecar file so the shell can pick it up +def score(p): + cs = (p.get("status") or {}).get("containerStatuses") or [] + ready = all(c.get("ready") for c in cs) if cs else False + restarts = max((c.get("restartCount", 0) for c in cs), default=0) + return (1 if ready else 0, -restarts) + +pods.sort(key=score, reverse=True) +target = pods[0]["metadata"]["name"] if pods else "" +with open(sys.argv[2], "w") as f: + f.write(target) +PY + +TARGET_FILE="$TMPDIR/target.txt" +python3 "$PY_SCRIPT" "$WEB_JSON" "$TARGET_FILE" +POD="$(cat "$TARGET_FILE")" + +echo +echo "=== Most-likely serving webservice pod: tail 60 lines (webservice container) ===" +if [[ -n "$POD" ]]; then + echo "Pod: $POD" + kctl logs "$POD" -c webservice --tail=60 2>&1 | sed 's/^/ /' || true + echo + echo "=== ... gitlab-workhorse container (tail 30) ===" + kctl logs "$POD" -c gitlab-workhorse --tail=30 2>&1 | sed 's/^/ /' || true +else + echo " (no webservice pod found)" +fi + +echo +echo "=== Gitaly pods ===" +kctl get pods -l app=gitaly -o wide || true + +echo +echo "=== Recent Warning events (last 25) ===" +kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 25 || true + +echo +echo "=== GitLab CR status (if present) ===" +GL_JSON="$TMPDIR/gl.json" +if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then + python3 - "$GL_JSON" <<'PY' +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +for it in data.get("items") or []: + m = it.get("metadata") or {} + s = it.get("status") or {} + print(f" {m.get('name')}: phase={s.get('phase','?')}") + for c in (s.get("conditions") or [])[:10]: + print(f" {c.get('type')}={c.get('status')} reason={c.get('reason')} " + f"msg={(c.get('message') or '')[:140]}") +PY +else + echo " (no GitLab CR or CRD not installed)" +fi diff --git a/etc/diag_gitlab_webservice_oom.sh b/etc/diag_gitlab_webservice_oom.sh new file mode 100755 index 0000000..169765c --- /dev/null +++ b/etc/diag_gitlab_webservice_oom.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# diag_gitlab_webservice_oom.sh — memory ceilings + Puma tuning for webservice. +# +# Read-only. Prints: +# 1. webservice container resources (requests/limits) +# 2. Puma worker/thread env vars from pod spec +# 3. node allocatable memory so we know upper bound +# 4. prometheus-style /metrics snapshot from the running pod if available +# +# Usage: ./etc/diag_gitlab_webservice_oom.sh [--namespace NS] [--kube-context CTX] + +set -euo pipefail + +NS="gitlab" +KUBE_CONTEXT="" +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) NS="$2"; shift 2 ;; + --kube-context) KUBE_CONTEXT="$2"; shift 2 ;; + -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +kctl() { + if [[ -n "$KUBE_CONTEXT" ]]; then + kubectl --context "$KUBE_CONTEXT" -n "$NS" "$@" + else + kubectl -n "$NS" "$@" + fi +} + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +DEPLOY_JSON="$TMPDIR/deploy.json" +POD_JSON="$TMPDIR/pod.json" +NODE_JSON="$TMPDIR/node.json" + +kctl get deployment -l app=webservice -o json > "$DEPLOY_JSON" +kctl get pods -l app=webservice -o json > "$POD_JSON" + +echo "=== webservice container resources (from deployment) ===" +python3 - "$DEPLOY_JSON" <<'PY' +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +for d in data.get("items") or []: + name = (d.get("metadata") or {}).get("name", "") + print(f"\n-- deployment/{name} --") + spec = ((d.get("spec") or {}).get("template") or {}).get("spec") or {} + for c in spec.get("containers") or []: + cname = c.get("name", "") + res = c.get("resources") or {} + req = res.get("requests") or {} + lim = res.get("limits") or {} + print(f" container={cname}") + print(f" requests: cpu={req.get('cpu','?')} mem={req.get('memory','?')}") + print(f" limits: cpu={lim.get('cpu','?')} mem={lim.get('memory','?')}") + # Pick out Puma-relevant env vars + env = c.get("env") or [] + interesting = { + "PUMA_WORKERS", "PUMA_THREADS_MIN", "PUMA_THREADS_MAX", + "WORKER_PROCESSES", "SIDEKIQ_CONCURRENCY", + "GITLAB_MEMORY_WATCHDOG_ENABLED", "GITLAB_MEMORY_WATCHDOG_PUMA_ENABLED", + "GITLAB_MEMORY_WATCHDOG_MAX_HEAP_FRAG_THRESHOLD_MB", + "GITLAB_MEMORY_WATCHDOG_MAX_STRIKES", + "MALLOC_ARENA_MAX", + } + seen = [e for e in env if e.get("name") in interesting] + if seen: + print(" puma/watchdog env:") + for e in seen: + val = e.get("value", "") if "value" in e else "" + print(f" {e.get('name')}={val}") +PY + +echo +echo "=== Node memory allocatable ===" +kctl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory,MEM_CAPACITY:.status.capacity.memory 2>/dev/null \ + || kubectl get nodes -o custom-columns=NAME:.metadata.name,MEM_ALLOCATABLE:.status.allocatable.memory + +echo +echo "=== Running webservice pod memory usage (kubectl top if metrics-server present) ===" +kctl top pod -l app=webservice --containers 2>&1 || echo " (metrics-server not installed or not returning data)" + +echo +echo "=== Pod QoS class + restart history ===" +python3 - "$POD_JSON" <<'PY' +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +for p in data.get("items") or []: + m = p.get("metadata") or {} + s = p.get("status") or {} + cs = s.get("containerStatuses") or [] + ws = next((c for c in cs if c.get("name") == "webservice"), None) + print(f" {m.get('name'):<48} qos={s.get('qosClass','?'):<10} " + f"phase={s.get('phase','?'):<10} restarts={ws.get('restartCount', '?') if ws else '?'}") +PY + +echo +echo "=== Last 40 Warning events (whole namespace, for noise diagnosis) ===" +kctl get events --field-selector type=Warning --sort-by=.lastTimestamp | tail -n 40 || true + +echo +echo "=== All PVCs in namespace (what the operator might be trying to reconcile) ===" +kctl get pvc -o wide || true + +echo +echo "=== GitLab operator CR spec.gitaly (replicas + storage) ===" +GL_JSON="$TMPDIR/gl.json" +if kctl get gitlab -o json > "$GL_JSON" 2>/dev/null; then + python3 - "$GL_JSON" <<'PY' +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +for g in data.get("items") or []: + m = g.get("metadata") or {} + name = m.get("name", "") + chart = ((g.get("spec") or {}).get("chart") or {}).get("values") or {} + gitaly = ((chart.get("global") or {}).get("gitaly") or {}) + wsvc = (chart.get("gitlab") or {}).get("webservice") or {} + print(f" gitlab/{name}") + print(f" global.gitaly: {json.dumps(gitaly, indent=6)[:800]}") + print(f" gitlab.webservice.replicas: {wsvc.get('replicaCount', wsvc.get('minReplicas','?'))}") + print(f" gitlab.webservice.resources: {json.dumps(wsvc.get('resources', {}), indent=6)}") + print(f" gitlab.webservice.workerProcesses: {wsvc.get('workerProcesses','?')}") +PY +else + echo " (no GitLab CR or not readable)" +fi diff --git a/etc/diag_gke_storage.sh b/etc/diag_gke_storage.sh new file mode 100755 index 0000000..236a168 --- /dev/null +++ b/etc/diag_gke_storage.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# diag_gke_storage.sh — inspect StorageClass + SSD quota state on a GKE cluster. +# +# Safe, read-only. Prints: +# 1. Every StorageClass, whether it's default, and its underlying pd type +# 2. All PVs grouped by storage class, with capacity + bound claim +# 3. Totals: how many GiB are pinned to each disk type (pd-ssd / pd-balanced +# count against SSD_TOTAL_GB; pd-standard is the separate HDD quota) +# 4. Any PVC stuck in Pending with a QUOTA_EXCEEDED / ProvisioningFailed event +# +# Usage: ./etc/diag_gke_storage.sh [--kube-context CTX] + +set -euo pipefail + +KUBE_CONTEXT="" +while [[ $# -gt 0 ]]; do + case "$1" in + --kube-context) KUBE_CONTEXT="$2"; shift 2 ;; + -h|--help) sed -n '1,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +kctl() { + if [[ -n "$KUBE_CONTEXT" ]]; then + kubectl --context "$KUBE_CONTEXT" "$@" + else + kubectl "$@" + fi +} + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +SC_JSON="$TMPDIR/sc.json" +PV_JSON="$TMPDIR/pv.json" +PVC_JSON="$TMPDIR/pvc.json" +PY_SCRIPT="$TMPDIR/diag.py" + +kctl get storageclass -o json > "$SC_JSON" +kctl get pv -o json > "$PV_JSON" +kctl get pvc -A -o json > "$PVC_JSON" + +# Single python script does all four sections — reads the three JSON files by path. +cat > "$PY_SCRIPT" <<'PY' +import json, re, sys +from collections import defaultdict + +sc_path, pv_path, pvc_path = sys.argv[1], sys.argv[2], sys.argv[3] + +with open(sc_path) as f: + scs_raw = json.load(f).get("items") or [] +with open(pv_path) as f: + pvs = json.load(f).get("items") or [] +with open(pvc_path) as f: + pvcs = json.load(f).get("items") or [] + +# --- Section 1: StorageClasses ------------------------------------------------ +print("=== StorageClasses ===") +print(f"{'NAME':<24} {'PROVISIONER':<30} {'DISK_TYPE':<14} {'DEFAULT':<7} {'RECLAIM'}") +sc_by_name = {} +for sc in scs_raw: + m = sc.get("metadata") or {} + name = m.get("name", "") + ann = m.get("annotations") or {} + is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "")).lower() == "true" + prov = sc.get("provisioner", "") or "" + disk = ((sc.get("parameters") or {}).get("type") or "") + reclaim = sc.get("reclaimPolicy", "") or "" + sc_by_name[name] = {"disk": disk, "prov": prov, "default": is_default} + print(f"{name:<24} {prov:<30} {disk:<14} {'YES' if is_default else '-':<7} {reclaim}") + +# --- Section 2: PVs grouped by SC -------------------------------------------- +print() +print("=== PVs by StorageClass ===") +groups = defaultdict(list) +for pv in pvs: + spec = pv.get("spec", {}) or {} + sc = spec.get("storageClassName", "") or "" + cap = (spec.get("capacity") or {}).get("storage", "") + phase = (pv.get("status") or {}).get("phase", "") + cr = spec.get("claimRef") or {} + claim = f"{cr.get('namespace','')}/{cr.get('name','')}" if cr else "" + groups[sc].append((pv.get("metadata", {}).get("name", ""), cap, phase, claim)) + +if not groups: + print(" (no PVs)") +else: + for sc, rows in sorted(groups.items()): + print(f"-- {sc} --") + for name, cap, phase, claim in rows: + print(f" {name:<50} {cap:<8} {phase:<10} {claim}") + +# --- Section 3: GiB per disk type -------------------------------------------- +def to_gib(cap): + if not cap: + return 0.0 + m = re.match(r"^(\d+(?:\.\d+)?)\s*([KMGTP]i?)?$", cap) + if not m: + return 0.0 + n = float(m.group(1)) + u = m.group(2) or "" + unit = { + "": 1/1024**3, "Ki": 1/1024**2, "Mi": 1/1024, "Gi": 1, "Ti": 1024, "Pi": 1024**2, + "K": 1e3/1024**3, "M": 1e6/1024**3, "G": 1e9/1024**3, "T": 1e12/1024**3, + }.get(u, 0) + return n * unit + +print() +print("=== GiB per disk type (what counts against which quota) ===") +totals = defaultdict(float) +for pv in pvs: + spec = pv.get("spec", {}) or {} + sc = spec.get("storageClassName", "") or "" + disk = sc_by_name.get(sc, {}).get("disk", "") + cap = (spec.get("capacity") or {}).get("storage", "") + totals[disk] += to_gib(cap) + +ssd_quota = 0.0 +hdd_quota = 0.0 +if not totals: + print(" (no PVs)") +else: + for disk, gib in sorted(totals.items()): + if disk in ("pd-balanced", "pd-ssd", "pd-extreme"): + bucket = "SSD_TOTAL_GB" + ssd_quota += gib + elif disk == "pd-standard": + bucket = "HDD (pd-standard)" + hdd_quota += gib + else: + bucket = "other" + print(f" {disk:<16} {gib:>8.1f} GiB ({bucket})") + print() + print(f" TOTAL counted against SSD_TOTAL_GB: {ssd_quota:.1f} GiB") + print(f" TOTAL on HDD (pd-standard): {hdd_quota:.1f} GiB") + +# --- Section 4: Pending PVCs -------------------------------------------------- +print() +print("=== Pending PVCs ===") +pending = [] +for pvc in pvcs: + phase = (pvc.get("status") or {}).get("phase", "") + if phase != "Pending": + continue + m = pvc.get("metadata") or {} + sc = (pvc.get("spec") or {}).get("storageClassName") or "" + pending.append((m.get("namespace", ""), m.get("name", ""), sc)) + +if not pending: + print(" (none)") +else: + for ns, name, sc in pending: + print(f" {ns}/{name} (storageClass={sc})") + +# Print the namespace/name list on stderr so the shell can loop over it. +import sys as _sys +for ns, name, _sc in pending: + print(f"{ns}/{name}", file=_sys.stderr) +PY + +# Run the python analysis; stderr carries the pending-PVC list for the shell +# loop below. +PENDING_LIST="$(python3 "$PY_SCRIPT" "$SC_JSON" "$PV_JSON" "$PVC_JSON" 2>"$TMPDIR/pending.txt")" +printf '%s\n' "$PENDING_LIST" + +echo +echo "=== Provisioning events for any Pending PVCs (tail) ===" +if [[ ! -s "$TMPDIR/pending.txt" ]]; then + echo " (no Pending PVCs)" +else + while IFS='/' read -r ns pvc; do + [[ -n "$ns" && -n "$pvc" ]] || continue + echo "-- ${ns}/${pvc} --" + kctl -n "$ns" describe pvc "$pvc" \ + | awk '/^Events:/{flag=1} flag' \ + | head -n 20 + done < "$TMPDIR/pending.txt" +fi diff --git a/etc/ensure_default_storage_class.sh b/etc/ensure_default_storage_class.sh new file mode 100755 index 0000000..2bb01d7 --- /dev/null +++ b/etc/ensure_default_storage_class.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# ensure_default_storage_class.sh +# +# Preflight / remediator that asserts the cluster-wide default StorageClass is +# pd-standard (HDD), not pd-balanced / pd-ssd. GKE ships `standard-rwo` as the +# default, which provisions `pd-balanced` under the hood and draws from the +# SSD_TOTAL_GB quota. That has repeatedly wedged provisioning on low-SSD-quota +# projects (see init_cnpg_gke.sh step-down logic, which compensates on the +# consumer side). +# +# This script fixes the root cause cluster-side: it un-defaults any built-in +# SSD-backed class and promotes `standard-hdd` (pd-standard) as the default. +# Workloads that actually need SSD latency (CNPG) opt in explicitly via +# `storageClassName: premium-rwo` in their PVC template — they are unaffected +# by the default change. +# +# Usage +# ----- +# ./etc/ensure_default_storage_class.sh [--check | --apply] [--kube-context CTX] +# +# --check (default) Report current state; exit 0 if compliant, 1 if drift +# detected. No cluster mutations. +# --apply Remediate: create standard-hdd if absent, clear the +# default annotation from any SSD-backed class, set standard-hdd +# as the default. Idempotent. +# --kube-context CTX Use the named kube-context (default: current). +# +# Exit codes +# ---------- +# 0 Compliant (or successfully remediated) +# 1 Drift detected in --check mode +# 2 Remediation failed (in --apply mode) +# 3 Cluster not a GKE cluster (no pd.csi.storage.gke.io provisioner seen) +# 4 Invalid arguments / missing tooling + +set -euo pipefail + +SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")" +DEFAULT_SC_NAME="${PROLE_DEFAULT_SC_NAME:-standard-hdd}" +DEFAULT_SC_DISK_TYPE="pd-standard" +GKE_CSI_PROVISIONER="pd.csi.storage.gke.io" +MODE="check" +KUBE_CONTEXT="" + +log() { printf '[%s] %s\n' "$SCRIPT_NAME" "$*"; } +warn() { printf '[%s] WARN: %s\n' "$SCRIPT_NAME" "$*" >&2; } +die() { printf '[%s] ERROR: %s\n' "$SCRIPT_NAME" "$*" >&2; exit "${2:-4}"; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) MODE="check"; shift ;; + --apply) MODE="apply"; shift ;; + --kube-context) + [[ -n "${2:-}" ]] || die "--kube-context requires an argument" 4 + KUBE_CONTEXT="$2"; shift 2 ;; + -h|--help) + sed -n '1,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit 0 ;; + *) die "unknown argument: $1" 4 ;; + esac +done + +command -v kubectl >/dev/null || die "kubectl not found in PATH" 4 +command -v python3 >/dev/null || die "python3 not found in PATH" 4 + +kctl() { + if [[ -n "$KUBE_CONTEXT" ]]; then + kubectl --context "$KUBE_CONTEXT" "$@" + else + kubectl "$@" + fi +} + +# ── 1. Confirm we're talking to a GKE cluster ──────────────────────────────── +sc_json="$(kctl get storageclass -o json 2>/dev/null || true)" +if [[ -z "$sc_json" ]] || ! printf '%s' "$sc_json" | python3 -c ' +import json, sys +data = json.load(sys.stdin) +items = data.get("items") or [] +sys.exit(0 if any(it.get("provisioner") == "pd.csi.storage.gke.io" for it in items) else 1) +'; then + die "no ${GKE_CSI_PROVISIONER} StorageClasses found; this does not look like a GKE cluster" 3 +fi + +# ── 2. Inspect current state ───────────────────────────────────────────────── +# Emits: ||| for every StorageClass. +sc_summary="$(printf '%s' "$sc_json" | python3 - <<'PY' +import json, sys +data = json.load(sys.stdin) +for it in data.get("items") or []: + meta = it.get("metadata") or {} + name = meta.get("name", "") + ann = (meta.get("annotations") or {}) + is_default = str(ann.get("storageclass.kubernetes.io/is-default-class", "") or "").lower() == "true" + prov = it.get("provisioner", "") or "" + disk_type = ((it.get("parameters") or {}).get("type", "")) or "" + print(f"{name}|{prov}|{disk_type}|{'true' if is_default else 'false'}") +PY +)" + +log "Current StorageClasses:" +while IFS='|' read -r name prov disk_type is_default; do + [[ -n "$name" ]] || continue + if [[ "$is_default" == "true" ]]; then + log " * ${name} prov=${prov} type=${disk_type} (default)" + else + log " ${name} prov=${prov} type=${disk_type}" + fi +done <<< "$sc_summary" + +# Identify the current default + its disk type. +current_default="" +current_default_type="" +ssd_backed_defaults=() +while IFS='|' read -r name prov disk_type is_default; do + [[ -n "$name" ]] || continue + if [[ "$is_default" == "true" ]]; then + current_default="$name" + current_default_type="$disk_type" + if [[ "$disk_type" == "pd-balanced" || "$disk_type" == "pd-ssd" ]]; then + ssd_backed_defaults+=("$name") + fi + fi +done <<< "$sc_summary" + +# Does standard-hdd (or PROLE_DEFAULT_SC_NAME override) exist with the right shape? +target_exists="false" +target_disk_type="" +target_is_default="false" +while IFS='|' read -r name prov disk_type is_default; do + if [[ "$name" == "$DEFAULT_SC_NAME" ]]; then + target_exists="true" + target_disk_type="$disk_type" + target_is_default="$is_default" + fi +done <<< "$sc_summary" + +compliant="true" +issues=() + +if [[ "$target_exists" != "true" ]]; then + compliant="false" + issues+=("target StorageClass '${DEFAULT_SC_NAME}' does not exist") +elif [[ "$target_disk_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then + compliant="false" + issues+=("'${DEFAULT_SC_NAME}' has type='${target_disk_type}', want '${DEFAULT_SC_DISK_TYPE}'") +fi + +if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then + compliant="false" + issues+=("SSD-backed class(es) still marked default: ${ssd_backed_defaults[*]}") +fi + +if [[ "$target_exists" == "true" && "$target_is_default" != "true" ]]; then + compliant="false" + issues+=("'${DEFAULT_SC_NAME}' is not annotated is-default-class=true") +fi + +if [[ "$compliant" == "true" ]]; then + log "OK — default StorageClass is '${current_default}' (type=${current_default_type}). No action needed." + exit 0 +fi + +log "Drift detected:" +for issue in "${issues[@]}"; do + log " - ${issue}" +done + +if [[ "$MODE" == "check" ]]; then + log "Run with --apply to remediate." + exit 1 +fi + +# ── 3. Remediate ───────────────────────────────────────────────────────────── +log "Applying remediation ..." + +# 3a. Create/update standard-hdd — but NOT yet marked default so we never have +# a window with two defaults racing. +cat </dev/null || true)" +if [[ "$live_type" != "$DEFAULT_SC_DISK_TYPE" ]]; then + warn "live '${DEFAULT_SC_NAME}' has type='${live_type}' (want '${DEFAULT_SC_DISK_TYPE}'). Recreating ..." + kctl delete storageclass "$DEFAULT_SC_NAME" --wait=true >/dev/null 2>&1 || die "failed to delete '${DEFAULT_SC_NAME}' for recreation" 2 + cat </dev/null || true)" + [[ "$live_type" == "$DEFAULT_SC_DISK_TYPE" ]] || die "after recreate, '${DEFAULT_SC_NAME}' still has type='${live_type}'" 2 +fi + +# 3b. Clear the default annotation from any SSD-backed class that currently +# holds it. Patch to "false" (not remove) so we leave a clear audit trail. +if [[ "${#ssd_backed_defaults[@]}" -gt 0 ]]; then + for sc in "${ssd_backed_defaults[@]}"; do + log "Un-defaulting SSD-backed class: ${sc}" + kctl annotate storageclass "$sc" \ + "storageclass.kubernetes.io/is-default-class=false" --overwrite >/dev/null \ + || die "failed to un-default '${sc}'" 2 + done +fi + +# 3c. Promote standard-hdd to default — atomic with respect to other defaults +# because we cleared them in 3b first. +kctl annotate storageclass "$DEFAULT_SC_NAME" \ + "storageclass.kubernetes.io/is-default-class=true" --overwrite >/dev/null \ + || die "failed to mark '${DEFAULT_SC_NAME}' as default" 2 + +# ── 4. Re-verify ───────────────────────────────────────────────────────────── +final_default="$(kctl get storageclass \ + -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | head -n1)" + +if [[ "$final_default" != "$DEFAULT_SC_NAME" ]]; then + die "post-remediation: default is '${final_default:-}', expected '${DEFAULT_SC_NAME}'" 2 +fi + +# Count defaults — having two is a worse state than having zero. +default_count="$(kctl get storageclass \ + -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | grep -cv '^$' || true)" +if [[ "$default_count" != "1" ]]; then + die "post-remediation: ${default_count} default StorageClasses exist (want exactly 1)" 2 +fi + +log "Remediation OK. Default StorageClass is now '${DEFAULT_SC_NAME}' (${DEFAULT_SC_DISK_TYPE})." +log "Re-run 'kubectl get storageclass' to confirm." +exit 0 diff --git a/etc/init_gitlab.sh b/etc/init_gitlab.sh index c493dc4..2bbc687 100755 --- a/etc/init_gitlab.sh +++ b/etc/init_gitlab.sh @@ -4,6 +4,37 @@ set -euo pipefail PROG="init_gitlab" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Standalone-invocation fix: pre-scan $@ for --config and export PROLE_DEPLOY_CFG +# BEFORE sourcing prole_cfg.sh. deploy.sh injects PROLE_DEPLOY_CFG in its env, +# but direct invocations (./etc/init_gitlab.sh --config conf/gke.cfg deploy) do +# not — prole_cfg.sh's auto-selection then picks conf/k3d.cfg first, which +# exports KUBECONTEXT=dev and downstream kubectl calls fail with +# `error: context "dev" does not exist`. +if [[ -z "${PROLE_DEPLOY_CFG:-}" ]]; then + _gitlab_prescan_cfg="" + _gitlab_prescan_want_next=0 + for _gitlab_prescan_tok in "$@"; do + if [[ "$_gitlab_prescan_want_next" == "1" ]]; then + _gitlab_prescan_cfg="$_gitlab_prescan_tok" + _gitlab_prescan_want_next=0 + break + fi + case "$_gitlab_prescan_tok" in + --config=*) _gitlab_prescan_cfg="${_gitlab_prescan_tok#*=}"; break ;; + -c|--config) _gitlab_prescan_want_next=1 ;; + esac + done + if [[ -n "$_gitlab_prescan_cfg" ]]; then + if [[ "$_gitlab_prescan_cfg" != /* ]]; then + _gitlab_prescan_cfg="$(cd "$(dirname "$_gitlab_prescan_cfg")" 2>/dev/null && pwd)/$(basename "$_gitlab_prescan_cfg")" + fi + if [[ -f "$_gitlab_prescan_cfg" ]]; then + export PROLE_DEPLOY_CFG="$_gitlab_prescan_cfg" + fi + fi + unset _gitlab_prescan_cfg _gitlab_prescan_tok _gitlab_prescan_want_next +fi + # shellcheck disable=SC1090 source "$SCRIPT_DIR/prole_cfg.sh" @@ -2483,6 +2514,72 @@ GITLAB_OIDC_REDIRECT_URI="${GITLAB_OIDC_REDIRECT_URI:-${_default_gitlab_oidc_red GITLAB_OIDC_CLIENT_ID="${GITLAB_OIDC_CLIENT_ID:-${OIDC_CLIENT_ID:-${GOOGLE_OIDC_CLIENT_ID:-${GOOGLE_CLIENT_ID:-}}}}" GITLAB_OIDC_CLIENT_SECRET="${GITLAB_OIDC_CLIENT_SECRET:-${OIDC_CLIENT_SECRET:-${GOOGLE_OIDC_CLIENT_SECRET:-${GOOGLE_CLIENT_SECRET:-}}}}" +# Resolve `secretref://` values for standalone invocations. deploy.sh +# resolves these in Python (knoe/core/actions.py:_resolve_secretref_value) +# before exec; direct invocations need the same lookup inline. Searches env +# aliases, then PROLE_SERVICE/secrets, etc/secrets, and secrets directories +# for a file named . Returns the raw input unchanged on miss. +_init_gitlab_resolve_secretref() { + local raw="${1:-}" + local aliases="${2:-}" + case "$raw" in + secretref://*) ;; + *) printf '%s' "$raw"; return 0 ;; + esac + + local ref="${raw#secretref://}" + ref="${ref#/}"; ref="${ref%/}" + [[ -n "$ref" ]] || { printf '%s' "$raw"; return 0; } + + local normalized="${ref//-/_}" + normalized="${normalized//\//_}" + normalized="${normalized//./_}" + local upper_norm="${normalized^^}" + + local -a candidates=("$ref" "$normalized" "$upper_norm") + local alias + for alias in $aliases; do + candidates+=("$alias") + done + + local key val + for key in "${candidates[@]}"; do + [[ -z "$key" ]] && continue + [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue + val="${!key-}" + if [[ -n "$val" && "$val" != secretref://* ]]; then + printf '%s' "$val" + return 0 + fi + done + + local root="${PROLE_HOME:-${SCRIPT_DIR}/..}" + local svc="${PROLE_SERVICE:-}" + local -a paths=() + [[ -n "$svc" ]] && paths+=("$svc/secrets/$ref") + paths+=("$root/etc/secrets/$ref" "$root/secrets/$ref") + + local path + for path in "${paths[@]}"; do + if [[ -f "$path" && -r "$path" ]]; then + val="$(< "$path")" + val="${val#"${val%%[![:space:]]*}"}" + val="${val%"${val##*[![:space:]]}"}" + if [[ -n "$val" ]]; then + printf '%s' "$val" + return 0 + fi + fi + done + + printf '%s' "$raw" +} + +GITLAB_OIDC_CLIENT_ID="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_ID" \ + "GITLAB_OIDC_CLIENT_ID OIDC_CLIENT_ID GOOGLE_OIDC_CLIENT_ID GOOGLE_CLIENT_ID")" +GITLAB_OIDC_CLIENT_SECRET="$(_init_gitlab_resolve_secretref "$GITLAB_OIDC_CLIENT_SECRET" \ + "GITLAB_OIDC_CLIENT_SECRET OIDC_CLIENT_SECRET GOOGLE_OIDC_CLIENT_SECRET GOOGLE_CLIENT_SECRET")" + is_unresolved_secret_ref() { local raw="${1:-}" case "$raw" in @@ -3445,13 +3542,21 @@ setup_garage_for_gitlab() { # ---- Create buckets and grant permissions ---- # --key takes the accessKeyId (not the name) in garage v1.x + # Bucket names must match GitLab Helm chart defaults because the GitLab CR's + # global.appConfig.object_store block only specifies the connection secret — + # it does NOT override per-object bucket names. Rails therefore writes to + # gitlab-uploads / gitlab-artifacts / etc. (no -storage suffix). A prior + # naming with a -storage suffix caused Excon::Error::NotFound / NoSuchBucket + # on first login when CarrierWave tried to upload the duo-bot avatar to + # /gitlab-uploads/user/avatar/8/duo-bot.png. Keep registry / terraform-state / + # ci-secure-files verbatim — those are the names both sides already agreed on. local buckets=( registry - gitlab-artifacts-storage - gitlab-lfs-storage - gitlab-uploads-storage - gitlab-packages-storage - gitlab-dependency-proxy-storage + gitlab-artifacts + gitlab-lfs + gitlab-uploads + gitlab-packages + gitlab-dependency-proxy gitlab-terraform-state gitlab-ci-secure-files ) diff --git a/k8s/prole/storageclass-gcp-standard-hdd.yaml b/k8s/prole/storageclass-gcp-standard-hdd.yaml new file mode 100644 index 0000000..470ed77 --- /dev/null +++ b/k8s/prole/storageclass-gcp-standard-hdd.yaml @@ -0,0 +1,30 @@ +# Generic HDD (pd-standard) StorageClass for GKE — the cluster-wide default. +# +# Why this exists +# --------------- +# GKE ships with `standard-rwo` as the default StorageClass, which under the +# hood provisions `pd-balanced` (SSD-backed hybrid). That draws from the +# SSD_TOTAL_GB quota and has repeatedly caused provisioning to wedge on small +# projects. For workloads that don't need SSD latency (OpenBao, registry +# storage, generic scratch PVCs, Garage) we want `pd-standard` (HDD) — which +# uses the separate HDD quota and is cheaper per GiB. +# +# This class is the canonical cluster-wide default. Workloads that DO need +# SSD latency (CNPG) opt in explicitly via storageClassName=premium-rwo in +# their PVC template. +# +# To make this the default, apply this manifest and then run +# etc/ensure_default_storage_class.sh --apply (which clears the default +# annotation off standard-rwo and sets it here). +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: standard-hdd + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-standard +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true