mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
Add regression tests and logic to handle recycling Released prole-iscsi PVs with stale claimRefs
- Introduced `recycle_released_prole_iscsi_pv_for_pvc` to resolve stale claimRef issues during restarts. - Updated `init_garage_store.sh` and `init_openbao.sh` to invoke the recycling logic when rollouts are stuck. - Added regression tests to validate PV recycling for Garage and OpenBao services.
This commit is contained in:
chrisfu
parent
6852f036cb
commit
52f65b9929
@ -282,8 +282,50 @@ dump_debug() {
|
||||
echo "---- Garage debug end ----"
|
||||
}
|
||||
|
||||
recycle_released_prole_iscsi_pv_for_pvc() {
|
||||
# StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain
|
||||
# in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding.
|
||||
local ns="$1"
|
||||
local pvc="$2"
|
||||
|
||||
if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local pvc_phase pvc_uid
|
||||
pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
if [[ "$pvc_phase" != "Pending" ]]; then
|
||||
return 0
|
||||
fi
|
||||
pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
|
||||
|
||||
local pv sc rp phase claim_ns claim_name claim_uid
|
||||
while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do
|
||||
[[ "$sc" == "prole-iscsi" ]] || continue
|
||||
[[ "$rp" == "Retain" ]] || continue
|
||||
[[ "$phase" == "Released" ]] || continue
|
||||
[[ "$claim_ns" == "$ns" ]] || continue
|
||||
[[ "$claim_name" == "$pvc" ]] || continue
|
||||
|
||||
# If the PV claimRef UID matches the current PVC UID, don't touch it.
|
||||
if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..."
|
||||
kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
|
||||
done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true)
|
||||
}
|
||||
|
||||
wait_ready() {
|
||||
echo "Waiting for Garage StatefulSet to become ready ..."
|
||||
|
||||
local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}"
|
||||
if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout="$initial_timeout"; then
|
||||
echo "WARN: Garage not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2
|
||||
recycle_released_prole_iscsi_pv_for_pvc "$NAMESPACE" "data-garage-0" || true
|
||||
fi
|
||||
|
||||
if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s}; then
|
||||
echo "ERROR: Garage StatefulSet did not become ready in time." >&2
|
||||
dump_debug
|
||||
|
||||
@ -483,6 +483,41 @@ k3d_cleanup_pending_openbao_pvc() {
|
||||
fi
|
||||
}
|
||||
|
||||
recycle_released_prole_iscsi_pv_for_pvc() {
|
||||
# StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain
|
||||
# in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding.
|
||||
local ns="$1"
|
||||
local pvc="$2"
|
||||
|
||||
if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local pvc_phase pvc_uid
|
||||
pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
if [[ "$pvc_phase" != "Pending" ]]; then
|
||||
return 0
|
||||
fi
|
||||
pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
|
||||
|
||||
local pv sc rp phase claim_ns claim_name claim_uid
|
||||
while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do
|
||||
[[ "$sc" == "prole-iscsi" ]] || continue
|
||||
[[ "$rp" == "Retain" ]] || continue
|
||||
[[ "$phase" == "Released" ]] || continue
|
||||
[[ "$claim_ns" == "$ns" ]] || continue
|
||||
[[ "$claim_name" == "$pvc" ]] || continue
|
||||
|
||||
# If the PV claimRef UID matches the current PVC UID, don't touch it.
|
||||
if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..."
|
||||
kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
|
||||
done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true)
|
||||
}
|
||||
|
||||
apply_openbao_host_network_patch() {
|
||||
if [[ "${OPENBAO_HOST_NETWORK:-0}" != "1" ]]; then
|
||||
return
|
||||
@ -503,10 +538,20 @@ wait_for_openbao() {
|
||||
wait_ns="$PRIMARY_SERVICE_NAMESPACE"
|
||||
fi
|
||||
|
||||
local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}"
|
||||
|
||||
echo "Waiting for OpenBao to become ready in namespace '$wait_ns' ..."
|
||||
if kubectl get statefulset/$OPENBAO_NAME -n "$wait_ns" >/dev/null 2>&1; then
|
||||
if ! kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then
|
||||
echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2
|
||||
recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true
|
||||
fi
|
||||
kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s}
|
||||
else
|
||||
if ! kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then
|
||||
echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2
|
||||
recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true
|
||||
fi
|
||||
kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s}
|
||||
fi
|
||||
}
|
||||
|
||||
@ -282,8 +282,50 @@ dump_debug() {
|
||||
echo "---- Garage debug end ----"
|
||||
}
|
||||
|
||||
recycle_released_prole_iscsi_pv_for_pvc() {
|
||||
# StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain
|
||||
# in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding.
|
||||
local ns="$1"
|
||||
local pvc="$2"
|
||||
|
||||
if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local pvc_phase pvc_uid
|
||||
pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
if [[ "$pvc_phase" != "Pending" ]]; then
|
||||
return 0
|
||||
fi
|
||||
pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
|
||||
|
||||
local pv sc rp phase claim_ns claim_name claim_uid
|
||||
while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do
|
||||
[[ "$sc" == "prole-iscsi" ]] || continue
|
||||
[[ "$rp" == "Retain" ]] || continue
|
||||
[[ "$phase" == "Released" ]] || continue
|
||||
[[ "$claim_ns" == "$ns" ]] || continue
|
||||
[[ "$claim_name" == "$pvc" ]] || continue
|
||||
|
||||
# If the PV claimRef UID matches the current PVC UID, don't touch it.
|
||||
if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..."
|
||||
kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
|
||||
done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true)
|
||||
}
|
||||
|
||||
wait_ready() {
|
||||
echo "Waiting for Garage StatefulSet to become ready ..."
|
||||
|
||||
local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}"
|
||||
if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout="$initial_timeout"; then
|
||||
echo "WARN: Garage not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2
|
||||
recycle_released_prole_iscsi_pv_for_pvc "$NAMESPACE" "data-garage-0" || true
|
||||
fi
|
||||
|
||||
if ! kubectl rollout status statefulset/$GARAGE_NAME -n "$NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s}; then
|
||||
echo "ERROR: Garage StatefulSet did not become ready in time." >&2
|
||||
dump_debug
|
||||
|
||||
@ -467,6 +467,41 @@ k3d_cleanup_pending_openbao_pvc() {
|
||||
fi
|
||||
}
|
||||
|
||||
recycle_released_prole_iscsi_pv_for_pvc() {
|
||||
# StorageClass `prole-iscsi` uses `Retain` PV reclaim policy. On restart, PVs can remain
|
||||
# in `Released` with a stale `claimRef`, which prevents a same-named PVC from binding.
|
||||
local ns="$1"
|
||||
local pvc="$2"
|
||||
|
||||
if ! kubectl get pvc "$pvc" -n "$ns" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local pvc_phase pvc_uid
|
||||
pvc_phase=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
if [[ "$pvc_phase" != "Pending" ]]; then
|
||||
return 0
|
||||
fi
|
||||
pvc_uid=$(kubectl get pvc "$pvc" -n "$ns" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)
|
||||
|
||||
local pv sc rp phase claim_ns claim_name claim_uid
|
||||
while IFS=$'\t' read -r pv sc rp phase claim_ns claim_name claim_uid; do
|
||||
[[ "$sc" == "prole-iscsi" ]] || continue
|
||||
[[ "$rp" == "Retain" ]] || continue
|
||||
[[ "$phase" == "Released" ]] || continue
|
||||
[[ "$claim_ns" == "$ns" ]] || continue
|
||||
[[ "$claim_name" == "$pvc" ]] || continue
|
||||
|
||||
# If the PV claimRef UID matches the current PVC UID, don't touch it.
|
||||
if [[ -n "${claim_uid:-}" && -n "${pvc_uid:-}" && "${claim_uid:-}" == "${pvc_uid:-}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Recycling Released PV '$pv' for PVC '$ns/$pvc' (clearing stale claimRef) ..."
|
||||
kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' >/dev/null 2>&1 || true
|
||||
done < <(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.storageClassName}{"\t"}{.spec.persistentVolumeReclaimPolicy}{"\t"}{.status.phase}{"\t"}{.spec.claimRef.namespace}{"\t"}{.spec.claimRef.name}{"\t"}{.spec.claimRef.uid}{"\n"}{end}' 2>/dev/null || true)
|
||||
}
|
||||
|
||||
apply_openbao_host_network_patch() {
|
||||
if [[ "${OPENBAO_HOST_NETWORK:-0}" != "1" ]]; then
|
||||
return
|
||||
@ -487,10 +522,20 @@ wait_for_openbao() {
|
||||
wait_ns="$PRIMARY_SERVICE_NAMESPACE"
|
||||
fi
|
||||
|
||||
local initial_timeout="${PVC_REPAIR_WAIT_TIMEOUT:-30s}"
|
||||
|
||||
echo "Waiting for OpenBao to become ready in namespace '$wait_ns' ..."
|
||||
if kubectl get statefulset/$OPENBAO_NAME -n "$wait_ns" >/dev/null 2>&1; then
|
||||
if ! kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then
|
||||
echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2
|
||||
recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true
|
||||
fi
|
||||
kubectl rollout status statefulset/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s}
|
||||
else
|
||||
if ! kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout="$initial_timeout"; then
|
||||
echo "WARN: OpenBao not ready after $initial_timeout; checking for Released prole-iscsi PVs with stale claimRefs ..." >&2
|
||||
recycle_released_prole_iscsi_pv_for_pvc "$wait_ns" "data-openbao-0" || true
|
||||
fi
|
||||
kubectl rollout status deploy/$OPENBAO_NAME -n "$wait_ns" --timeout=${ROLLOUT_TIMEOUT:-300s}
|
||||
fi
|
||||
}
|
||||
|
||||
156
tests/etc/test_init_garage_recycles_released_pv_claimref.sh
Normal file
156
tests/etc/test_init_garage_recycles_released_pv_claimref.sh
Normal file
@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regression test: Garage startup should auto-recycle a Released prole-iscsi PV with a stale
|
||||
# claimRef when the PVC is Pending and rollout is stuck beyond the initial timeout.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd)
|
||||
SOURCE_ETC_DIR="$PROLE_HOME/etc"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
WORK_DIR="$TMP_DIR/work"
|
||||
BIN_DIR="$TMP_DIR/bin"
|
||||
mkdir -p "$WORK_DIR/etc" "$WORK_DIR/conf" "$WORK_DIR/k8s/prole" "$BIN_DIR"
|
||||
|
||||
cp "$SOURCE_ETC_DIR/init_garage_store.sh" "$WORK_DIR/etc/init_garage_store.sh"
|
||||
cp "$SOURCE_ETC_DIR/common_core_lib.sh" "$WORK_DIR/etc/common_core_lib.sh"
|
||||
cp "$SOURCE_ETC_DIR/prole_cfg.sh" "$WORK_DIR/etc/prole_cfg.sh"
|
||||
chmod +x "$WORK_DIR/etc/init_garage_store.sh"
|
||||
|
||||
# Dummy manifests required by init_garage_store.sh
|
||||
for f in storageclass-prole-iscsi.yaml iscsi-pvs.yaml garage-configmap.yaml garage-statefulset.yaml garage-service.yaml; do
|
||||
printf '%s\n' "apiVersion: v1" > "$WORK_DIR/k8s/prole/$f"
|
||||
done
|
||||
|
||||
cat <<EOF > "$WORK_DIR/conf/prole.cfg"
|
||||
[globals]
|
||||
prole.home = $WORK_DIR
|
||||
prole.mode = k3s
|
||||
NAMESPACE = test-ns
|
||||
SERVICE_NAMESPACE = knoe-system
|
||||
EOF
|
||||
|
||||
export KUBECTL_LOG="$TMP_DIR/kubectl.log"
|
||||
|
||||
cat <<'EOF' > "$BIN_DIR/kubectl"
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
printf '%s\n' "$*" >> "${KUBECTL_LOG}"
|
||||
|
||||
# Skip global options like --request-timeout=...
|
||||
while [[ "${1:-}" == --* ]]; do
|
||||
shift
|
||||
done
|
||||
|
||||
sub="${1:-}"
|
||||
case "$sub" in
|
||||
config)
|
||||
case "${2:-}" in
|
||||
current-context)
|
||||
printf '%s' "default"
|
||||
exit 0
|
||||
;;
|
||||
use-context|get-contexts)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
;;
|
||||
get)
|
||||
# apiserver readiness
|
||||
if [[ "$*" == "get nodes"* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
# namespace/secret checks
|
||||
if [[ "$*" == *"get namespace"* || "$*" == *"get secret"* ]]; then
|
||||
exit 1
|
||||
fi
|
||||
# PVC existence + phase/uid
|
||||
if [[ "$*" == *"get pvc data-garage-0"*"-o jsonpath={.status.phase}"* ]]; then
|
||||
printf '%s' "Pending"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$*" == *"get pvc data-garage-0"*"-o jsonpath={.metadata.uid}"* ]]; then
|
||||
printf '%s' "newuid"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$*" == *"get pvc data-garage-0"* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
# PV list for recycle helper
|
||||
if [[ "$*" == *"get pv"*"-o jsonpath="* ]]; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"prole-iscsi-d001-garage" "prole-iscsi" "Retain" "Released" "test-ns" "data-garage-0" "olduid"
|
||||
exit 0
|
||||
fi
|
||||
# Pod lookup for init_layout
|
||||
if [[ "$*" == *"get pods"*"-l app=garage"*"-o jsonpath={.items[0].metadata.name}"* ]]; then
|
||||
printf '%s' "garage-0"
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
create)
|
||||
# namespace + secret creation
|
||||
exit 0
|
||||
;;
|
||||
diff)
|
||||
exit 1
|
||||
;;
|
||||
apply)
|
||||
exit 0
|
||||
;;
|
||||
rollout)
|
||||
if [[ "$*" == *"rollout status"*"--timeout=0s"* ]]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
patch)
|
||||
exit 0
|
||||
;;
|
||||
exec)
|
||||
if [[ "$*" == *"/garage node id"* ]]; then
|
||||
printf '%s\n' "deadbeefdeadbeef@node"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$*" == *"/garage layout show"* ]]; then
|
||||
printf '%s\n' "deadbeefdeadbeef"
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$BIN_DIR/kubectl"
|
||||
|
||||
export PATH="$BIN_DIR:$PATH"
|
||||
|
||||
export PROLE_HOME="$WORK_DIR"
|
||||
export PROLE_CONF="$WORK_DIR/conf"
|
||||
export PROLE_MODE="k3s"
|
||||
export PVC_REPAIR_WAIT_TIMEOUT="0s"
|
||||
|
||||
if "$WORK_DIR/etc/init_garage_store.sh" -n "test-ns" restart >"$TMP_DIR/stdout" 2>"$TMP_DIR/stderr"; then
|
||||
:
|
||||
else
|
||||
echo "FAILURE: init_garage_store.sh restart exited non-zero" >&2
|
||||
cat "$TMP_DIR/stderr" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q "patch pv prole-iscsi-d001-garage" "$KUBECTL_LOG"; then
|
||||
echo "FAILURE: expected PV claimRef recycle (kubectl patch pv) was not invoked" >&2
|
||||
echo "--- kubectl log ---" >&2
|
||||
sed -n '1,200p' "$KUBECTL_LOG" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "SUCCESS"
|
||||
124
tests/etc/test_init_openbao_recycles_released_pv_claimref.sh
Normal file
124
tests/etc/test_init_openbao_recycles_released_pv_claimref.sh
Normal file
@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regression test: OpenBao restart should auto-recycle a Released prole-iscsi PV with a stale
|
||||
# claimRef when the PVC is Pending and rollout is stuck beyond the initial timeout.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
PROLE_HOME=$(cd "$SCRIPT_DIR/../.." && pwd)
|
||||
SOURCE_ETC_DIR="$PROLE_HOME/etc"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
WORK_DIR="$TMP_DIR/work"
|
||||
BIN_DIR="$TMP_DIR/bin"
|
||||
mkdir -p "$WORK_DIR/etc" "$WORK_DIR/conf" "$BIN_DIR"
|
||||
|
||||
cp "$SOURCE_ETC_DIR/init_openbao.sh" "$WORK_DIR/etc/init_openbao.sh"
|
||||
cp "$SOURCE_ETC_DIR/common_core_lib.sh" "$WORK_DIR/etc/common_core_lib.sh"
|
||||
cp "$SOURCE_ETC_DIR/prole_cfg.sh" "$WORK_DIR/etc/prole_cfg.sh"
|
||||
chmod +x "$WORK_DIR/etc/init_openbao.sh"
|
||||
|
||||
cat <<EOF > "$WORK_DIR/conf/prole.cfg"
|
||||
[globals]
|
||||
prole.home = $WORK_DIR
|
||||
prole.mode = k3s
|
||||
NAMESPACE = test-ns
|
||||
SERVICE_NAMESPACE = knoe-system
|
||||
EOF
|
||||
|
||||
export KUBECTL_LOG="$TMP_DIR/kubectl.log"
|
||||
|
||||
cat <<'EOF' > "$BIN_DIR/kubectl"
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
printf '%s\n' "$*" >> "${KUBECTL_LOG}"
|
||||
|
||||
# Skip global options like --request-timeout=...
|
||||
while [[ "${1:-}" == --* ]]; do
|
||||
shift
|
||||
done
|
||||
|
||||
sub="${1:-}"
|
||||
case "$sub" in
|
||||
config)
|
||||
case "${2:-}" in
|
||||
current-context)
|
||||
printf '%s' "default"
|
||||
exit 0
|
||||
;;
|
||||
use-context|get-contexts)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
;;
|
||||
get)
|
||||
# statefulset existence check
|
||||
if [[ "$*" == *"get statefulset/openbao"* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
# PVC existence + phase/uid
|
||||
if [[ "$*" == *"get pvc data-openbao-0"*"-o jsonpath={.status.phase}"* ]]; then
|
||||
printf '%s' "Pending"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$*" == *"get pvc data-openbao-0"*"-o jsonpath={.metadata.uid}"* ]]; then
|
||||
printf '%s' "newuid"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$*" == *"get pvc data-openbao-0"* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
# PV list for recycle helper
|
||||
if [[ "$*" == *"get pv"*"-o jsonpath="* ]]; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"prole-iscsi-d001-openbao" "prole-iscsi" "Retain" "Released" "knoe-system" "data-openbao-0" "olduid"
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
rollout)
|
||||
if [[ "$*" == *"rollout status"*"--timeout=0s"* ]]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
scale)
|
||||
exit 0
|
||||
;;
|
||||
patch)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$BIN_DIR/kubectl"
|
||||
|
||||
export PATH="$BIN_DIR:$PATH"
|
||||
|
||||
export PROLE_HOME="$WORK_DIR"
|
||||
export PROLE_CONF="$WORK_DIR/conf"
|
||||
export PROLE_MODE="k3s"
|
||||
export PVC_REPAIR_WAIT_TIMEOUT="0s"
|
||||
|
||||
if "$WORK_DIR/etc/init_openbao.sh" -n "knoe-system" restart >"$TMP_DIR/stdout" 2>"$TMP_DIR/stderr"; then
|
||||
:
|
||||
else
|
||||
echo "FAILURE: init_openbao.sh restart exited non-zero" >&2
|
||||
cat "$TMP_DIR/stderr" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q "patch pv prole-iscsi-d001-openbao" "$KUBECTL_LOG"; then
|
||||
echo "FAILURE: expected PV claimRef recycle (kubectl patch pv) was not invoked" >&2
|
||||
echo "--- kubectl log ---" >&2
|
||||
sed -n '1,200p' "$KUBECTL_LOG" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "SUCCESS"
|
||||
Loading…
Reference in New Issue
Block a user