diff --git a/etc/init_garage_store.sh b/etc/init_garage_store.sh index ef71ed9..a605a59 100755 --- a/etc/init_garage_store.sh +++ b/etc/init_garage_store.sh @@ -282,8 +282,50 @@ dump_debug() { echo "---- Garage debug end ----" } +recycle_released_prole_iscsi_pv_for_pvc() { + # StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain + # in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding. + local ns="$1" + local pvc="$2" + + if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then + return 0 + fi + + local pvc_phase pvc_uid + pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "$pvc_phase" != "Pending" ]]; then + return 0 + fi + pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true) + + local pv sc rp phase claim_ns claim_name claim_uid + while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do + [[ "$sc" == "prole-iscsi" ]] || continue + [[ "$rp" == "Retain" ]] || continue + [[ "$phase" == "Released" ]] || continue + [[ "$claim_ns" == "$ns" ]] || continue + [[ "$claim_name" == "$pvc" ]] || continue + + # If the PV claimRef UID matches the current PVC UID, don't touch it. + if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then + continue + fi + + echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..." + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true + done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true) +} + wait_ready() { echo "Waiting for Garage StatefulSet to become ready ..." + + local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}" + if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout="$initial_timeout"; then + echo "WARN: Garage not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2 + recycle_released_prole_iscsi_pv_for_pvc "$NAMESPACE" "data-garage-0" || true + fi + if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s}; then echo "ERROR: Garage StatefulSet did not become ready in time." >&2 dump_debug diff --git a/etc/init_openbao.sh b/etc/init_openbao.sh index 3ea1d73..631644e 100755 --- a/etc/init_openbao.sh +++ b/etc/init_openbao.sh @@ -483,6 +483,41 @@ k3d_cleanup_pending_openbao_pvc() { fi } +recycle_released_prole_iscsi_pv_for_pvc() { + # StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain + # in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding. + local ns="$1" + local pvc="$2" + + if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then + return 0 + fi + + local pvc_phase pvc_uid + pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "$pvc_phase" != "Pending" ]]; then + return 0 + fi + pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true) + + local pv sc rp phase claim_ns claim_name claim_uid + while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do + [[ "$sc" == "prole-iscsi" ]] || continue + [[ "$rp" == "Retain" ]] || continue + [[ "$phase" == "Released" ]] || continue + [[ "$claim_ns" == "$ns" ]] || continue + [[ "$claim_name" == "$pvc" ]] || continue + + # If the PV claimRef UID matches the current PVC UID, don't touch it. + if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then + continue + fi + + echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..." + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true + done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true) +} + apply_openbao_host_network_patch() { if [[ "${OPENBAO_HOST_NETWORK:-0}" != "1" ]]; then return @@ -503,10 +538,20 @@ wait_for_openbao() { wait_ns="$PRIMARY_SERVICE_NAMESPACE" fi + local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}" + echo "Waiting for OpenBao to become ready in namespace '$wait_ns' ..." if kubectl get statefulset/$OPENBAO_NAME -n "$wait_ns" >/dev/null 2>&1; then + if ! kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then + echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2 + recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true + fi kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s} else + if ! kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then + echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2 + recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true + fi kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s} fi } diff --git a/mock_val/init_garage_store.sh b/mock_val/init_garage_store.sh index ef71ed9..a605a59 100755 --- a/mock_val/init_garage_store.sh +++ b/mock_val/init_garage_store.sh @@ -282,8 +282,50 @@ dump_debug() { echo "---- Garage debug end ----" } +recycle_released_prole_iscsi_pv_for_pvc() { + # StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain + # in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding. + local ns="$1" + local pvc="$2" + + if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then + return 0 + fi + + local pvc_phase pvc_uid + pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "$pvc_phase" != "Pending" ]]; then + return 0 + fi + pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true) + + local pv sc rp phase claim_ns claim_name claim_uid + while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do + [[ "$sc" == "prole-iscsi" ]] || continue + [[ "$rp" == "Retain" ]] || continue + [[ "$phase" == "Released" ]] || continue + [[ "$claim_ns" == "$ns" ]] || continue + [[ "$claim_name" == "$pvc" ]] || continue + + # If the PV claimRef UID matches the current PVC UID, don't touch it. + if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then + continue + fi + + echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..." + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true + done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true) +} + wait_ready() { echo "Waiting for Garage StatefulSet to become ready ..." + + local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}" + if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout="$initial_timeout"; then + echo "WARN: Garage not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2 + recycle_released_prole_iscsi_pv_for_pvc "$NAMESPACE" "data-garage-0" || true + fi + if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s}; then echo "ERROR: Garage StatefulSet did not become ready in time." >&2 dump_debug diff --git a/mock_val/init_openbao.sh b/mock_val/init_openbao.sh index 5359f75..ae561c0 100755 --- a/mock_val/init_openbao.sh +++ b/mock_val/init_openbao.sh @@ -467,6 +467,41 @@ k3d_cleanup_pending_openbao_pvc() { fi } +recycle_released_prole_iscsi_pv_for_pvc() { + # StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain + # in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding. + local ns="$1" + local pvc="$2" + + if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then + return 0 + fi + + local pvc_phase pvc_uid + pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "$pvc_phase" != "Pending" ]]; then + return 0 + fi + pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true) + + local pv sc rp phase claim_ns claim_name claim_uid + while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do + [[ "$sc" == "prole-iscsi" ]] || continue + [[ "$rp" == "Retain" ]] || continue + [[ "$phase" == "Released" ]] || continue + [[ "$claim_ns" == "$ns" ]] || continue + [[ "$claim_name" == "$pvc" ]] || continue + + # If the PV claimRef UID matches the current PVC UID, don't touch it. + if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then + continue + fi + + echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..." + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true + done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true) +} + apply_openbao_host_network_patch() { if [[ "${OPENBAO_HOST_NETWORK:-0}" != "1" ]]; then return @@ -487,10 +522,20 @@ wait_for_openbao() { wait_ns="$PRIMARY_SERVICE_NAMESPACE" fi + local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}" + echo "Waiting for OpenBao to become ready in namespace '$wait_ns' ..." if kubectl get statefulset/$OPENBAO_NAME -n "$wait_ns" >/dev/null 2>&1; then + if ! kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then + echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2 + recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true + fi kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s} else + if ! kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then + echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2 + recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true + fi kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s} fi } diff --git a/tests/etc/test_init_garage_recycles_released_pv_claimref.sh b/tests/etc/test_init_garage_recycles_released_pv_claimref.sh new file mode 100644 index 0000000..c1a99f2 --- /dev/null +++ b/tests/etc/test_init_garage_recycles_released_pv_claimref.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Regression test: Garage startup should auto-recycle a Released prole-iscsi PV with a stale +# claimRef when the PVC is Pending and rollout is stuck beyond the initial timeout. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) +SOURCE_ETC_DIR="$PROLE_HOME/etc" + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +WORK_DIR="$TMP_DIR/work" +BIN_DIR="$TMP_DIR/bin" +mkdir -p "$WORK_DIR/etc" "$WORK_DIR/conf" "$WORK_DIR/k8s/prole" "$BIN_DIR" + +cp "$SOURCE_ETC_DIR/init_garage_store.sh" "$WORK_DIR/etc/init_garage_store.sh" +cp "$SOURCE_ETC_DIR/common_core_lib.sh" "$WORK_DIR/etc/common_core_lib.sh" +cp "$SOURCE_ETC_DIR/prole_cfg.sh" "$WORK_DIR/etc/prole_cfg.sh" +chmod +x "$WORK_DIR/etc/init_garage_store.sh" + +# Dummy manifests required by init_garage_store.sh +for f in storageclass-prole-iscsi.yaml iscsi-pvs.yaml garage-configmap.yaml garage-statefulset.yaml garage-service.yaml; do + printf '%s\n' "apiVersion: v1" > "$WORK_DIR/k8s/prole/$f" +done + +cat < "$WORK_DIR/conf/prole.cfg" +[globals] +prole.home = $WORK_DIR +prole.mode = k3s +NAMESPACE = test-ns +SERVICE_NAMESPACE = knoe-system +EOF + +export KUBECTL_LOG="$TMP_DIR/kubectl.log" + +cat <<'EOF' > "$BIN_DIR/kubectl" +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >> "${KUBECTL_LOG}" + +# Skip global options like --request-timeout=... +while [[ "${1:-}" == --* ]]; do + shift +done + +sub="${1:-}" +case "$sub" in + config) + case "${2:-}" in + current-context) + printf '%s' "default" + exit 0 + ;; + use-context|get-contexts) + exit 0 + ;; + esac + exit 0 + ;; + get) + # apiserver readiness + if [[ "$*" == "get nodes"* ]]; then + exit 0 + fi + # namespace/secret checks + if [[ "$*" == *"get namespace"* || "$*" == *"get secret"* ]]; then + exit 1 + fi + # PVC existence + phase/uid + if [[ "$*" == *"get pvc data-garage-0"*"-o jsonpath={.status.phase}"* ]]; then + printf '%s' "Pending" + exit 0 + fi + if [[ "$*" == *"get pvc data-garage-0"*"-o jsonpath={.metadata.uid}"* ]]; then + printf '%s' "newuid" + exit 0 + fi + if [[ "$*" == *"get pvc data-garage-0"* ]]; then + exit 0 + fi + # PV list for recycle helper + if [[ "$*" == *"get pv"*"-o jsonpath="* ]]; then + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "prole-iscsi-d001-garage" "prole-iscsi" "Retain" "Released" "test-ns" "data-garage-0" "olduid" + exit 0 + fi + # Pod lookup for init_layout + if [[ "$*" == *"get pods"*"-l app=garage"*"-o jsonpath={.items[0].metadata.name}"* ]]; then + printf '%s' "garage-0" + exit 0 + fi + exit 0 + ;; + create) + # namespace + secret creation + exit 0 + ;; + diff) + exit 1 + ;; + apply) + exit 0 + ;; + rollout) + if [[ "$*" == *"rollout status"*"--timeout=0s"* ]]; then + exit 1 + fi + exit 0 + ;; + patch) + exit 0 + ;; + exec) + if [[ "$*" == *"/garage node id"* ]]; then + printf '%s\n' "deadbeefdeadbeef@node" + exit 0 + fi + if [[ "$*" == *"/garage layout show"* ]]; then + printf '%s\n' "deadbeefdeadbeef" + exit 0 + fi + exit 0 + ;; + *) + exit 0 + ;; +esac +EOF +chmod +x "$BIN_DIR/kubectl" + +export PATH="$BIN_DIR:$PATH" + +export PROLE_HOME="$WORK_DIR" +export PROLE_CONF="$WORK_DIR/conf" +export PROLE_MODE="k3s" +export PVC_REPAIR_WAIT_TIMEOUT="0s" + +if "$WORK_DIR/etc/init_garage_store.sh" -n "test-ns" restart >"$TMP_DIR/stdout" 2>"$TMP_DIR/stderr"; then + : +else + echo "FAILURE: init_garage_store.sh restart exited non-zero" >&2 + cat "$TMP_DIR/stderr" >&2 + exit 1 +fi + +if ! grep -q "patch pv prole-iscsi-d001-garage" "$KUBECTL_LOG"; then + echo "FAILURE: expected PV claimRef recycle (kubectl patch pv) was not invoked" >&2 + echo "--- kubectl log ---" >&2 + sed -n '1,200p' "$KUBECTL_LOG" >&2 || true + exit 1 +fi + +echo "SUCCESS" diff --git a/tests/etc/test_init_openbao_recycles_released_pv_claimref.sh b/tests/etc/test_init_openbao_recycles_released_pv_claimref.sh new file mode 100644 index 0000000..bed0c57 --- /dev/null +++ b/tests/etc/test_init_openbao_recycles_released_pv_claimref.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Regression test: OpenBao restart should auto-recycle a Released prole-iscsi PV with a stale +# claimRef when the PVC is Pending and rollout is stuck beyond the initial timeout. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd) +SOURCE_ETC_DIR="$PROLE_HOME/etc" + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +WORK_DIR="$TMP_DIR/work" +BIN_DIR="$TMP_DIR/bin" +mkdir -p "$WORK_DIR/etc" "$WORK_DIR/conf" "$BIN_DIR" + +cp "$SOURCE_ETC_DIR/init_openbao.sh" "$WORK_DIR/etc/init_openbao.sh" +cp "$SOURCE_ETC_DIR/common_core_lib.sh" "$WORK_DIR/etc/common_core_lib.sh" +cp "$SOURCE_ETC_DIR/prole_cfg.sh" "$WORK_DIR/etc/prole_cfg.sh" +chmod +x "$WORK_DIR/etc/init_openbao.sh" + +cat < "$WORK_DIR/conf/prole.cfg" +[globals] +prole.home = $WORK_DIR +prole.mode = k3s +NAMESPACE = test-ns +SERVICE_NAMESPACE = knoe-system +EOF + +export KUBECTL_LOG="$TMP_DIR/kubectl.log" + +cat <<'EOF' > "$BIN_DIR/kubectl" +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >> "${KUBECTL_LOG}" + +# Skip global options like --request-timeout=... +while [[ "${1:-}" == --* ]]; do + shift +done + +sub="${1:-}" +case "$sub" in + config) + case "${2:-}" in + current-context) + printf '%s' "default" + exit 0 + ;; + use-context|get-contexts) + exit 0 + ;; + esac + exit 0 + ;; + get) + # statefulset existence check + if [[ "$*" == *"get statefulset/openbao"* ]]; then + exit 0 + fi + # PVC existence + phase/uid + if [[ "$*" == *"get pvc data-openbao-0"*"-o jsonpath={.status.phase}"* ]]; then + printf '%s' "Pending" + exit 0 + fi + if [[ "$*" == *"get pvc data-openbao-0"*"-o jsonpath={.metadata.uid}"* ]]; then + printf '%s' "newuid" + exit 0 + fi + if [[ "$*" == *"get pvc data-openbao-0"* ]]; then + exit 0 + fi + # PV list for recycle helper + if [[ "$*" == *"get pv"*"-o jsonpath="* ]]; then + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "prole-iscsi-d001-openbao" "prole-iscsi" "Retain" "Released" "knoe-system" "data-openbao-0" "olduid" + exit 0 + fi + exit 0 + ;; + rollout) + if [[ "$*" == *"rollout status"*"--timeout=0s"* ]]; then + exit 1 + fi + exit 0 + ;; + scale) + exit 0 + ;; + patch) + exit 0 + ;; + *) + exit 0 + ;; +esac +EOF +chmod +x "$BIN_DIR/kubectl" + +export PATH="$BIN_DIR:$PATH" + +export PROLE_HOME="$WORK_DIR" +export PROLE_CONF="$WORK_DIR/conf" +export PROLE_MODE="k3s" +export PVC_REPAIR_WAIT_TIMEOUT="0s" + +if "$WORK_DIR/etc/init_openbao.sh" -n "knoe-system" restart >"$TMP_DIR/stdout" 2>"$TMP_DIR/stderr"; then + : +else + echo "FAILURE: init_openbao.sh restart exited non-zero" >&2 + cat "$TMP_DIR/stderr" >&2 + exit 1 +fi + +if ! grep -q "patch pv prole-iscsi-d001-openbao" "$KUBECTL_LOG"; then + echo "FAILURE: expected PV claimRef recycle (kubectl patch pv) was not invoked" >&2 + echo "--- kubectl log ---" >&2 + sed -n '1,200p' "$KUBECTL_LOG" >&2 || true + exit 1 +fi + +echo "SUCCESS"