mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
chore: enhance DB connectivity diagnostics and cross-cluster support
- Added split-cluster detection with detailed app and DB context logging. - Introduced robust DB connectivity checks for DNS resolution and TCP probes. - Enabled automated cross-cluster DB ILB service creation and validation. - Improved logging for DB migration failures with enhanced diagnostics and connectivity hints.
This commit is contained in:
chrisfu
parent
ff5b73a435
commit
e6374e3e46
@ -174,10 +174,28 @@ check_gitlab_migrations_blocked() {
|
||||
|
||||
if [[ "$_waiting_reason" == "CrashLoopBackOff" || ( -n "$_exit_code" && "$_exit_code" != "0" ) ]]; then
|
||||
log "GitLab migrations job is actively failing: ${_latest_job} (pod: ${_latest_pod}, restarts: ${_restarts}, exitCode: ${_exit_code:-unknown})"
|
||||
|
||||
# Enhanced DB connectivity diagnostics for split-cluster visibility
|
||||
local _db_info="DB_HOST=${DB_HOST}, DB_PORT=${DB_PORT}"
|
||||
local _split_cluster="No"
|
||||
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
|
||||
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
|
||||
if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then _split_cluster="Yes (APP: ${app_ctx}, DB: ${db_ctx})"; fi
|
||||
|
||||
local _diag
|
||||
_diag=$(get_gitlab_migrations_diagnostics "$_latest_job")
|
||||
|
||||
local _connectivity_hint=""
|
||||
if echo "$_diag" | grep -qi "connecting with your hostname"; then
|
||||
_connectivity_hint=" (Likely DNS/resolution failure)"
|
||||
elif echo "$_diag" | grep -qiE "connection refused|timeout"; then
|
||||
_connectivity_hint=" (Likely TCP connectivity/firewall failure)"
|
||||
fi
|
||||
|
||||
repair_blocked "GitLab migrations job is actively failing" \
|
||||
"Job: ${_latest_job}. Pod: ${_latest_pod}. Status: ${_waiting_reason:-Terminated}. ExitCode: ${_exit_code:-unknown}. Restarts: ${_restarts}.
|
||||
DB Config: ${_db_info}${_connectivity_hint}
|
||||
Split-cluster: ${_split_cluster}
|
||||
Diagnostics:
|
||||
${_diag}"
|
||||
fi
|
||||
@ -260,6 +278,39 @@ check_host_resolves() {
|
||||
return 0
|
||||
}
|
||||
|
||||
check_db_connectivity() {
|
||||
local target_host="$1"
|
||||
local target_port="${2:-5432}"
|
||||
local host_only="${target_host%:*}"
|
||||
|
||||
log "Validating GitLab database connectivity to ${target_host}:${target_port}..."
|
||||
|
||||
# DNS sanity check
|
||||
if [[ "$host_only" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
log "Host ${host_only} is an IP address, skipping DNS check."
|
||||
else
|
||||
# In k8s mode, internal DNS resolution might fail from the script runner
|
||||
# but work from within the cluster. We do a basic 'host' check as a hint.
|
||||
if ! host "$host_only" >/dev/null 2>&1; then
|
||||
warn "DNS resolution (host) failed for database host: ${host_only}. This may be expected if only resolvable in-cluster."
|
||||
else
|
||||
log "DNS resolution successful for ${host_only}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# TCP check via temporary pod in the APP cluster namespace
|
||||
log "Performing in-cluster TCP connectivity probe to ${host_only}:${target_port} (namespace: ${NAMESPACE})..."
|
||||
if kubectl -n "$NAMESPACE" run db-probe \
|
||||
--image=alpine:latest --restart=Never --rm --attach --timeout=30s \
|
||||
--command -- sh -c "nc -zv -w 5 ${host_only} ${target_port}" >/dev/null 2>&1; then
|
||||
log "TCP connectivity to ${host_only}:${target_port} SUCCESSFUL."
|
||||
return 0
|
||||
else
|
||||
warn "TCP connectivity probe to ${host_only}:${target_port} FAILED."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_gitlab_pre_apply_blocked() {
|
||||
# --- Registry Endpoint Placeholder Check ---
|
||||
if [[ "$GARAGE_S3_ENDPOINT" == *"<private-db-cluster-garage-endpoint>"* ]]; then
|
||||
@ -276,10 +327,27 @@ check_gitlab_pre_apply_blocked() {
|
||||
repair_blocked "Redis host does not resolve" \
|
||||
"Resource: REDIS_HOST. Value: ${REDIS_HOST} does not resolve. Fix: Ensure Redis is deployed and REDIS_NAMESPACE is correct."
|
||||
fi
|
||||
|
||||
# --- Database Connectivity Check ---
|
||||
if [[ "$MODE" == "k8s" ]]; then
|
||||
if ! check_db_connectivity "$DB_HOST" "$DB_PORT"; then
|
||||
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
|
||||
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
|
||||
repair_blocked "GitLab database host is not reachable from APP cluster" \
|
||||
"Host: ${DB_HOST}. Port: ${DB_PORT}. App Context: ${app_ctx}. DB Context: ${db_ctx}. Fix: Ensure cross-cluster networking (ILB) is functional."
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
get_gitlab_blocker_context() {
|
||||
local ctx=""
|
||||
local app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
|
||||
local db_ctx="${DB_CLUSTER_KUBECONTEXT:-}"
|
||||
if [[ -n "$db_ctx" && "$db_ctx" != "$app_ctx" ]]; then
|
||||
ctx+="Split-cluster (APP: ${app_ctx}, DB: ${db_ctx}). "
|
||||
fi
|
||||
ctx+="DB_HOST: ${DB_HOST}. "
|
||||
|
||||
local _last_mig=$(kubectl -n "$NAMESPACE" get jobs -l "app=migrations" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
|
||||
if [[ -n "$_last_mig" ]]; then
|
||||
local _m_ts=$(kubectl -n "$NAMESPACE" get job "$_last_mig" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || true)
|
||||
@ -626,6 +694,83 @@ kubectl_garage_admin() {
|
||||
fi
|
||||
}
|
||||
|
||||
db_kubectl() {
|
||||
local db_ctx
|
||||
db_ctx="$(resolve_db_cluster_context || true)"
|
||||
if [[ -n "$db_ctx" ]]; then
|
||||
command kubectl --context "$db_ctx" "$@"
|
||||
else
|
||||
kubectl "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_cross_cluster_db_host() {
|
||||
if [[ "$MODE" != "k8s" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local app_ctx db_ctx
|
||||
app_ctx="${APP_CLUSTER_KUBECONTEXT:-${KUBECONTEXT:-}}"
|
||||
if [[ -z "$app_ctx" ]]; then
|
||||
app_ctx="$(kubectl config current-context 2>/dev/null || true)"
|
||||
fi
|
||||
db_ctx="$(resolve_db_cluster_context || true)"
|
||||
|
||||
if [[ -z "$db_ctx" || -z "$app_ctx" || "$db_ctx" == "$app_ctx" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local db_cluster_name="${CNPG_CLUSTER_NAME:-knoe-db}"
|
||||
local db_ns="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
|
||||
local ilb_service="${GITLAB_DB_ILB_SERVICE:-knoe-db-rw-ilb}"
|
||||
|
||||
log "Split-cluster mode detected (APP: ${app_ctx}, DB: ${db_ctx})"
|
||||
log "Ensuring cross-cluster Postgres ILB service '${ilb_service}' in namespace '${db_ns}'..."
|
||||
|
||||
db_kubectl -n "$db_ns" apply -f - <<EOF >/dev/null
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ${ilb_service}
|
||||
annotations:
|
||||
networking.gke.io/load-balancer-type: "Internal"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
selector:
|
||||
cnpg.io/cluster: ${db_cluster_name}
|
||||
cnpg.io/instanceRole: primary
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
protocol: TCP
|
||||
EOF
|
||||
|
||||
local db_host=""
|
||||
local attempts=0
|
||||
log "Waiting for cross-cluster DB host (ILB IP/hostname)..."
|
||||
while (( attempts < 60 )); do
|
||||
attempts=$((attempts + 1))
|
||||
db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)"
|
||||
if [[ -z "$db_host" ]]; then
|
||||
db_host="$(db_kubectl -n "$db_ns" get svc "$ilb_service" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -n "$db_host" ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if [[ -z "$db_host" ]]; then
|
||||
warn "Cross-cluster Postgres ILB '${ilb_service}' has no ingress address yet after 5 minutes."
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Successfully resolved cross-cluster DB host: ${db_host}"
|
||||
GITLAB_CROSS_CLUSTER_DB_HOST="$db_host"
|
||||
export GITLAB_CROSS_CLUSTER_DB_HOST
|
||||
}
|
||||
|
||||
enforce_app_cluster_targeting
|
||||
|
||||
is_truthy() {
|
||||
@ -865,10 +1010,13 @@ command -v helm >/dev/null || die "helm not found (required for GitLab Operat
|
||||
# ---------------------------------------------------------------------------
|
||||
DB_NAMESPACE="${KNOE_DB_NAMESPACE:-${DATABASE_NAMESPACE:-knoe-db}}"
|
||||
CNPG_CLUSTER_NAME="${CNPG_CLUSTER_NAME:-${CLUSTER_NAME:-knoe-db}}"
|
||||
|
||||
ensure_cross_cluster_db_host
|
||||
|
||||
GITLAB_DB_NAME="${GITLAB_DB_NAME:-gitlabhq_production}"
|
||||
GITLAB_DB_USER="${GITLAB_DB_USER:-gitlab}"
|
||||
GITLAB_DB_PASSWORD="${GITLAB_DB_PASSWORD:-}"
|
||||
DB_HOST="${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local"
|
||||
DB_HOST="${GITLAB_DB_HOST:-${GITLAB_CROSS_CLUSTER_DB_HOST:-${CNPG_CLUSTER_NAME}-rw.${DB_NAMESPACE}.svc.cluster.local}}"
|
||||
DB_PORT="${GITLAB_DB_PORT:-5432}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Loading…
Reference in New Issue
Block a user