mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
- add idempotent CNPG operator reconciliation to prevent duplicate active controller ReplicaSets - restore and validate CNPG 3-instance convergence with PV claimRef recovery and storage path readiness - wire canonical launcher aliases/reset behavior and improve namespace cleanup semantics - harden backup/objectstore readiness handling and retry behavior - enforce service namespace usage for common services and remove default-namespace drift - enable Kerberos milestone auto-activation when realm/kdc are configured and apply CNPG GSS pg_hba rules - keep final runtime healthy: knoe-db 3/3, operator stable, perfsnap captured Co-authored-by: Junie <junie@jetbrains.com>
253 lines
8.3 KiB
Bash
Executable File
253 lines
8.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
reset-ns.sh -n <namespace> [options]
|
|
|
|
Required:
|
|
-n <namespace> Namespace to reset (REQUIRED; no implicit default)
|
|
|
|
Options:
|
|
--delete-pvcs Delete all PVCs in the namespace (DANGEROUS: wipes state)
|
|
--delete-config Delete ConfigMaps and Secrets (VERY DANGEROUS)
|
|
--keep-registry Preserve registry workloads/services in the target namespace
|
|
--clear-node-reservations
|
|
Best-effort clear node reservations by uncordoning nodes
|
|
--no-clear-node-reservations
|
|
Disable node reservation cleanup (default)
|
|
-h, --help Show this help
|
|
|
|
Notes:
|
|
• This script NEVER deletes the namespace itself — only resources inside it.
|
|
• It will NOT delete the CNPG operator (cluster-wide), but if CNPG CRDs exist,
|
|
it WILL delete CNPG Cluster resources IN THE TARGET NAMESPACE so pods stop respawning.
|
|
• It will NEVER delete Service "kubernetes" in the default namespace.
|
|
|
|
Examples:
|
|
./reset-ns.sh -n default
|
|
./reset-ns.sh -n default --delete-pvcs
|
|
EOF
|
|
}
|
|
|
|
NS=""
|
|
DELETE_PVCS=false
|
|
DELETE_CONFIG=false
|
|
KEEP_REGISTRY=false
|
|
CLEAR_NODE_RESERVATIONS=false
|
|
|
|
[[ $# -eq 0 ]] && usage && exit 1
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-n) NS="${2:-}"; shift 2 ;;
|
|
--delete-pvcs) DELETE_PVCS=true; shift ;;
|
|
--delete-config) DELETE_CONFIG=true; shift ;;
|
|
--keep-registry) KEEP_REGISTRY=true; shift ;;
|
|
--clear-node-reservations) CLEAR_NODE_RESERVATIONS=true; shift ;;
|
|
--no-clear-node-reservations) CLEAR_NODE_RESERVATIONS=false; shift ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "Unknown argument: $1"; echo; usage; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "${NS}" ]]; then
|
|
echo "ERROR: -n <namespace> is required"
|
|
echo
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
# Refuse truly critical system namespaces; allow "default" if explicitly passed.
|
|
case "$NS" in
|
|
kube-system|kube-public|kube-node-lease)
|
|
echo "REFUSING to operate on protected namespace: $NS"
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
echo "Resetting namespace: $NS"
|
|
echo " DELETE_PVCS=$DELETE_PVCS"
|
|
echo " DELETE_CONFIG=$DELETE_CONFIG"
|
|
echo " KEEP_REGISTRY=$KEEP_REGISTRY"
|
|
echo " CLEAR_NODE_RESERVATIONS=$CLEAR_NODE_RESERVATIONS"
|
|
echo
|
|
|
|
if ! kubectl get namespace "$NS" >/dev/null 2>&1; then
|
|
echo "Namespace '$NS' does not exist; skipping."
|
|
exit 0
|
|
fi
|
|
|
|
is_registry_resource() {
|
|
local kind="$1"
|
|
local name="$2"
|
|
|
|
[[ "$KEEP_REGISTRY" == "true" ]] || return 1
|
|
|
|
if [[ "$name" == "registry" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local app_label=""
|
|
app_label=$(kubectl -n "$NS" get "$kind" "$name" -o jsonpath='{.metadata.labels.app}' 2>/dev/null || true)
|
|
[[ "$app_label" == "registry" ]]
|
|
}
|
|
|
|
scale_down_workloads() {
|
|
local kind="$1"
|
|
local item
|
|
while IFS= read -r item; do
|
|
[[ -z "$item" ]] && continue
|
|
if is_registry_resource "$kind" "$item"; then
|
|
echo " preserving ${kind}/${item}"
|
|
continue
|
|
fi
|
|
kubectl -n "$NS" scale "$kind" "$item" --replicas=0 2>/dev/null || true
|
|
done < <(kubectl -n "$NS" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
|
}
|
|
|
|
delete_kind_resources() {
|
|
local kind="$1"
|
|
local item
|
|
while IFS= read -r item; do
|
|
[[ -z "$item" ]] && continue
|
|
if is_registry_resource "$kind" "$item"; then
|
|
echo " preserving ${kind}/${item}"
|
|
continue
|
|
fi
|
|
kubectl -n "$NS" delete "$kind" "$item" --ignore-not-found --wait=false 2>/dev/null || true
|
|
done < <(kubectl -n "$NS" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
|
}
|
|
|
|
# -------------------------
|
|
# CNPG: stop respawning in THIS namespace by deleting CNPG Cluster CRs
|
|
# Detect via CRD existence (more reliable than kubectl api-resources on some setups)
|
|
# -------------------------
|
|
if kubectl get crd clusters.postgresql.cnpg.io >/dev/null 2>&1; then
|
|
echo "CNPG detected (CRD clusters.postgresql.cnpg.io exists)."
|
|
echo "Deleting CNPG resources in namespace '$NS' (operator NOT touched)..."
|
|
|
|
# List first (helpful visibility; won't fail the script)
|
|
kubectl -n "$NS" get clusters.postgresql.cnpg.io 2>/dev/null || true
|
|
|
|
# Delete Cluster CRs (this is what causes StatefulSets/Pods to be recreated)
|
|
kubectl -n "$NS" delete clusters.postgresql.cnpg.io --all --ignore-not-found --wait=false
|
|
|
|
# Related CNPG CRs (safe best-effort)
|
|
if kubectl get crd poolers.postgresql.cnpg.io >/dev/null 2>&1; then
|
|
kubectl -n "$NS" delete poolers.postgresql.cnpg.io --all --ignore-not-found --wait=false || true
|
|
fi
|
|
if kubectl get crd backups.postgresql.cnpg.io >/dev/null 2>&1; then
|
|
kubectl -n "$NS" delete backups.postgresql.cnpg.io --all --ignore-not-found --wait=false || true
|
|
fi
|
|
if kubectl get crd scheduledbackups.postgresql.cnpg.io >/dev/null 2>&1; then
|
|
kubectl -n "$NS" delete scheduledbackups.postgresql.cnpg.io --all --ignore-not-found --wait=false || true
|
|
fi
|
|
|
|
# Small pause so the operator observes deletion and stops reconciling
|
|
sleep 2
|
|
else
|
|
echo "CNPG not detected via CRD check; skipping CNPG cleanup."
|
|
fi
|
|
|
|
# -------------------------
|
|
# Scale down controllers to reduce churn
|
|
# -------------------------
|
|
echo "Scaling down Deployments/StatefulSets..."
|
|
scale_down_workloads deployment
|
|
scale_down_workloads statefulset
|
|
|
|
# -------------------------
|
|
# Delete common workload controllers
|
|
# -------------------------
|
|
echo "Deleting workload controllers..."
|
|
delete_kind_resources deployment
|
|
delete_kind_resources statefulset
|
|
delete_kind_resources daemonset
|
|
delete_kind_resources replicaset
|
|
delete_kind_resources job
|
|
delete_kind_resources cronjob
|
|
|
|
# -------------------------
|
|
# Delete services safely (NEVER delete "kubernetes" service in default)
|
|
# -------------------------
|
|
echo "Deleting services (excluding service/kubernetes)..."
|
|
for s in $(kubectl -n "$NS" get svc -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true); do
|
|
if [[ "$NS" == "default" && "$s" == "kubernetes" ]]; then
|
|
echo " skipping protected service: default/kubernetes"
|
|
continue
|
|
fi
|
|
if is_registry_resource service "$s"; then
|
|
echo " preserving service/$s"
|
|
continue
|
|
fi
|
|
kubectl -n "$NS" delete svc "$s" --ignore-not-found --wait=false
|
|
done
|
|
|
|
echo "Deleting ingress & networkpolicy..."
|
|
kubectl -n "$NS" delete ingress,networkpolicy --all --ignore-not-found --wait=false
|
|
|
|
# -------------------------
|
|
# Delete HPA / PDB
|
|
# -------------------------
|
|
echo "Deleting HPA / PDB (if any)..."
|
|
kubectl -n "$NS" delete hpa,pdb --all --ignore-not-found --wait=false 2>/dev/null || true
|
|
|
|
# -------------------------
|
|
# Optional config wipe
|
|
# -------------------------
|
|
if [[ "$DELETE_CONFIG" == "true" ]]; then
|
|
echo "Deleting ConfigMaps..."
|
|
kubectl -n "$NS" delete configmap --all --ignore-not-found --wait=false
|
|
|
|
echo "Deleting Secrets (excluding service-account tokens)..."
|
|
for s in $(kubectl -n "$NS" get secret -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true); do
|
|
type=$(kubectl -n "$NS" get secret "$s" -o jsonpath='{.type}' 2>/dev/null || true)
|
|
[[ "$type" == "kubernetes.io/service-account-token" ]] && continue
|
|
kubectl -n "$NS" delete secret "$s" --ignore-not-found --wait=false
|
|
done
|
|
fi
|
|
|
|
# -------------------------
|
|
# Optional PVC wipe
|
|
# -------------------------
|
|
if [[ "$DELETE_PVCS" == "true" ]]; then
|
|
echo "Deleting PVCs..."
|
|
kubectl -n "$NS" delete pvc --all --ignore-not-found --wait=false
|
|
fi
|
|
|
|
# -------------------------
|
|
# Final pod cleanup
|
|
# -------------------------
|
|
echo "Deleting remaining pods..."
|
|
deleted_pods=()
|
|
for p in $(kubectl -n "$NS" get pod -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true); do
|
|
if is_registry_resource pod "$p"; then
|
|
echo " preserving pod/$p"
|
|
continue
|
|
fi
|
|
kubectl -n "$NS" delete pod "$p" --ignore-not-found --wait=false 2>/dev/null || true
|
|
deleted_pods+=("$p")
|
|
done
|
|
|
|
echo "Waiting for pods to terminate..."
|
|
if [[ ${#deleted_pods[@]} -gt 0 ]]; then
|
|
wait_args=()
|
|
for p in "${deleted_pods[@]}"; do
|
|
wait_args+=("pod/$p")
|
|
done
|
|
kubectl -n "$NS" wait --for=delete "${wait_args[@]}" --timeout=180s 2>/dev/null || true
|
|
fi
|
|
|
|
if [[ "$CLEAR_NODE_RESERVATIONS" == "true" ]]; then
|
|
echo "Clearing node reservations (uncordon all nodes)..."
|
|
while IFS= read -r node; do
|
|
[[ -z "$node" ]] && continue
|
|
kubectl uncordon "$node" >/dev/null 2>&1 || true
|
|
done < <(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
|
fi
|
|
|
|
echo
|
|
echo "Namespace '$NS' reset complete (namespace preserved)."
|