mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
chore: enhance GitLab init to block misconfigurations and repair legacy setups
- Improved `check_gitlab_pre_apply_blocked` to better detect invalid placeholders, legacy storage configurations, and DNS issues for Redis and Garage endpoints. - Added auto-repair for Garage endpoint secrets and storage class mismatches in k8s mode. - Introduced warnings and blocking logic for unsupported configurations like `gitlab-gitaly-static`. - Refined storage class handling to dynamically align with `standard-rwo` in k8s/GKE.
This commit is contained in:
chrisfu
parent
cf293aa751
commit
8c4a965ded
@ -145,12 +145,16 @@ check_host_resolves() {
|
||||
check_gitlab_pre_apply_blocked() {
|
||||
# --- Registry Endpoint Placeholder Check ---
|
||||
if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
|
||||
repair_blocked "Registry endpoint invalid or wrong Garage" \
|
||||
"Resource: GARAGE_S3_ENDPOINT. Value: contains placeholder. Fix: Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint."
|
||||
# If the configured value itself is still a placeholder, we MUST block.
|
||||
# The operator hasn't provided a real value yet.
|
||||
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder)" \
|
||||
"GARAGE_PRIVATE_S3_ENDPOINT is set to the literal placeholder. Set it to the real DB cluster Garage endpoint."
|
||||
fi
|
||||
|
||||
# --- Redis Host Resolution Check ---
|
||||
if ! check_host_resolves "$REDIS_HOST"; then
|
||||
# This is potentially repairable if we can infer a better host,
|
||||
# but for now we block if it's clearly invalid.
|
||||
repair_blocked "Redis host does not resolve" \
|
||||
"Resource: REDIS_HOST. Value: ${REDIS_HOST} does not resolve. Fix: Ensure Redis is deployed and REDIS_NAMESPACE is correct."
|
||||
fi
|
||||
@ -165,11 +169,11 @@ check_gitlab_post_apply_blocked() {
|
||||
if [[ -n "$gitaly_sts_yaml" ]]; then
|
||||
if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* ]]; then
|
||||
repair_blocked "Gitaly using legacy Synology storage" \
|
||||
"Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Unset GITLAB_STORAGE_NODE in config."
|
||||
"Resource: statefulset/${gitaly_sts_name}. Value: nodeSelector contains gandalf.prole.org. Fix: Ensure GITLAB_STORAGE_NODE is not set in k8s mode."
|
||||
fi
|
||||
if [[ "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
|
||||
repair_blocked "Gitaly using legacy Synology storage" \
|
||||
"Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden."
|
||||
"Resource: statefulset/${gitaly_sts_name}. Value: storageClassName is gitlab-gitaly-static. Fix: Ensure GITLAB_GITALY_STORAGE_CLASS is not overridden in k8s mode."
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -182,7 +186,7 @@ check_gitlab_post_apply_blocked() {
|
||||
unschedulable_gitaly=$(kubectl -n "$NAMESPACE" get pods -l "app=gitaly" -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="PodScheduled")].status=="False")]}{.metadata.name}:{.status.conditions[?(@.type=="PodScheduled")].reason}{"\n"}{end}' 2>/dev/null | grep "Unschedulable" || true)
|
||||
if [[ -n "$unschedulable_gitaly" ]]; then
|
||||
repair_blocked "Gitaly pod(s) unschedulable" \
|
||||
"Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storage (standard-rwo) and no hostname nodeSelector."
|
||||
"Resource: pod -l app=gitaly. Value: Unschedulable. Fix: GKE requires dynamic storage (standard-rwo) and no hostname nodeSelector. Ensure PV/PVC were repaired."
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -190,8 +194,14 @@ check_gitlab_post_apply_blocked() {
|
||||
local registry_secret_config
|
||||
registry_secret_config=$(kubectl -n "$NAMESPACE" get secret "${GITLAB_RELEASE}-registry-storage" -o jsonpath='{.data.config}' 2>/dev/null | base64 -d 2>/dev/null || true)
|
||||
if [[ "$registry_secret_config" == *"<private-db-cluster-garage-endpoint>"* ]]; then
|
||||
repair_blocked "Registry endpoint invalid or wrong Garage" \
|
||||
"Resource: secret/${GITLAB_RELEASE}-registry-storage. Value: contains placeholder. Fix: Set GARAGE_PRIVATE_S3_ENDPOINT correctly."
|
||||
if [[ "$GARAGE_S3_ENDPOINT" != *"<private-db-cluster-garage-endpoint>"* ]]; then
|
||||
log "Registry secret still contains placeholder but config is updated. Re-applying secret..."
|
||||
# Re-triggering the secret creation (this is safer than just blocking)
|
||||
setup_garage_for_gitlab
|
||||
else
|
||||
repair_blocked "Registry endpoint invalid or wrong Garage (contains placeholder in live secret)" \
|
||||
"Set GARAGE_PRIVATE_S3_ENDPOINT to the real DB cluster Garage endpoint."
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Registry Logs Check ---
|
||||
@ -201,8 +211,9 @@ check_gitlab_post_apply_blocked() {
|
||||
local logs
|
||||
logs=$(kubectl -n "$NAMESPACE" logs "$registry_pod" --tail=100 2>&1 || true)
|
||||
if [[ "$logs" == *"DNS failure"* || "$logs" == *"AccessDenied"* || "$logs" == *"No such key"* ]]; then
|
||||
repair_blocked "Registry endpoint invalid or wrong Garage" \
|
||||
"Resource: registry pod logs. Value: S3 error detected. Fix: Verify Garage endpoint and bucket permissions."
|
||||
# If it's a "No such key" or "AccessDenied", and we have a custom endpoint, it might be the WRONG cluster Garage.
|
||||
repair_blocked "Registry endpoint invalid or wrong Garage (S3 error detected)" \
|
||||
"Logs: ${logs}. Fix: Verify GARAGE_S3_ENDPOINT points to the DB cluster Garage (not the APP cluster one)."
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -213,8 +224,8 @@ check_gitlab_post_apply_blocked() {
|
||||
local logs
|
||||
logs=$(kubectl -n "$NAMESPACE" logs "$kas_pod" --tail=100 2>&1 || true)
|
||||
if [[ "$logs" == *"no such host"* && "$logs" == *"redis"* ]]; then
|
||||
repair_blocked "Redis host does not resolve" \
|
||||
"Resource: kas pod logs. Value: DNS failure for redis. Fix: Verify REDIS_HOST and REDIS_NAMESPACE."
|
||||
repair_blocked "Redis host does not resolve (detected in KAS logs)" \
|
||||
"Logs: ${logs}. Fix: Verify REDIS_HOST (${REDIS_HOST}) and ensure Redis service is healthy."
|
||||
fi
|
||||
fi
|
||||
}
|
||||
@ -1039,7 +1050,8 @@ fi
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage node is only used for legacy/local static local-PV mode.
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
|
||||
# In k8s/GKE mode, we do NOT use static storage nodes or nodeSelectors.
|
||||
STORAGE_NODE=""
|
||||
else
|
||||
STORAGE_NODE="${STORAGE_NODE:-${GITLAB_STORAGE_NODE:-}}"
|
||||
fi
|
||||
@ -1059,10 +1071,15 @@ if [[ -n "$STORAGE_NODE" ]]; then
|
||||
fi
|
||||
|
||||
GITALY_STORAGE_CLASS="${GITLAB_GITALY_STORAGE_CLASS:-}"
|
||||
if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
if [[ -z "$GITALY_STORAGE_CLASS" || "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
|
||||
if [[ "$GITALY_STORAGE_CLASS" == "gitlab-gitaly-static" ]]; then
|
||||
warn "Stripping legacy storageClass 'gitlab-gitaly-static' in k8s mode; using 'standard-rwo'."
|
||||
fi
|
||||
GITALY_STORAGE_CLASS="standard-rwo"
|
||||
else
|
||||
fi
|
||||
else
|
||||
if [[ -z "$GITALY_STORAGE_CLASS" ]]; then
|
||||
GITALY_STORAGE_CLASS="gitlab-gitaly-static"
|
||||
fi
|
||||
fi
|
||||
@ -1615,14 +1632,33 @@ setup_gitlab_storage() {
|
||||
log "Skipping legacy Synology/local-PV gitaly storage prep in k8s mode; using dynamic StorageClass '${GITALY_STORAGE_CLASS}'."
|
||||
|
||||
# Check for legacy resources on GKE/k8s
|
||||
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1 || \
|
||||
kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then
|
||||
local has_legacy=0
|
||||
if kubectl get pv gitlab-gitaly-synology >/dev/null 2>&1; then has_legacy=1; fi
|
||||
if kubectl -n "$NAMESPACE" get pvc repo-data-gitlab-gitaly-0 >/dev/null 2>&1; then has_legacy=1; fi
|
||||
|
||||
local gitaly_sts_name="${GITLAB_RELEASE}-gitaly"
|
||||
local gitaly_sts_yaml
|
||||
gitaly_sts_yaml=$(kubectl -n "$NAMESPACE" get statefulset "$gitaly_sts_name" -o yaml 2>/dev/null || true)
|
||||
if [[ -n "$gitaly_sts_yaml" ]]; then
|
||||
if [[ "$gitaly_sts_yaml" == *"gandalf.prole.org"* || "$gitaly_sts_yaml" == *"gitlab-gitaly-static"* ]]; then
|
||||
has_legacy=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$has_legacy" == "1" ]]; then
|
||||
if [[ "${GITLAB_REPAIR_BLOCKED_AUTOCLEAN:-0}" == "1" ]]; then
|
||||
log "AUTOCLEAN: repairing legacy Gitaly storage (scaling down and deleting legacy PV/PVC)..."
|
||||
kubectl -n "$NAMESPACE" scale statefulset "${GITLAB_RELEASE}-gitaly" --replicas=0 --timeout=30s 2>/dev/null || true
|
||||
kubectl -n "$NAMESPACE" delete pvc repo-data-gitlab-gitaly-0 --wait=false 2>/dev/null || true
|
||||
kubectl delete pv gitlab-gitaly-synology --wait=false 2>/dev/null || true
|
||||
# Give it a moment to process deletions
|
||||
sleep 2
|
||||
else
|
||||
repair_blocked "Legacy Gitaly PV/PVC still bound to gitlab-gitaly-static / gandalf.prole.org" \
|
||||
"Set GITLAB_REPAIR_BLOCKED_AUTOCLEAN=1 to automatically repair by deleting legacy PVC/PV, or run:
|
||||
kubectl -n $NAMESPACE scale sts ${GITLAB_RELEASE}-gitaly --replicas=0
|
||||
kubectl -n $NAMESPACE delete pvc repo-data-gitlab-gitaly-0
|
||||
kubectl delete pv gitlab-gitaly-synology"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
|
||||
Loading…
Reference in New Issue
Block a user