From ad812cc504da750326732a7b1cb27d9721a0ca6a Mon Sep 17 00:00:00 2001 From: chrisfu Date: Sun, 22 Mar 2026 23:26:02 -0700 Subject: [PATCH] Reorder service initialization and cleanup flows to enforce dependency sequencing, ensuring OpenTofu deploys after OpenBao, Garage, and Kong. --- etc/init_service_layer.sh | 21 +++++++++++++++------ etc/status_common_services.sh | 2 +- mock_val/init_service_layer.sh | 21 +++++++++++++++------ mock_val/status_common_services.sh | 2 +- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/etc/init_service_layer.sh b/etc/init_service_layer.sh index 04edba4..2d99331 100755 --- a/etc/init_service_layer.sh +++ b/etc/init_service_layer.sh @@ -225,8 +225,13 @@ deploy_service_layer() { "$SCRIPT_DIR/init_argocd.sh" -n "$ARGOCD_NS" "$argocd_action" || rc=$? fi - OPENTOFU_NAMESPACE="$ns" OPENTOFU_SECRET_NAMESPACE="${NAMESPACE:-$ns}" OPENTOFU_OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" "$opentofu_action" || rc=$? + # ------------------------------------------------------------------------- + # Deploy services in dependency order: + # 1. OpenBao – secrets vault; needed by downstream services + # 2. Garage – object storage + # 3. Kong – API gateway + # 4. OpenTofu – IaC engine; depends on registry + secrets (last) + # ------------------------------------------------------------------------- OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_openbao.sh" -n "$ns" "$openbao_action" || rc=$? @@ -240,6 +245,9 @@ deploy_service_layer() { KONG_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_kong.sh" -n "$ns" "$kong_action" || rc=$? + OPENTOFU_NAMESPACE="$ns" OPENTOFU_SECRET_NAMESPACE="${NAMESPACE:-$ns}" OPENTOFU_OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" "$opentofu_action" || rc=$? + if [[ "$ENABLE_KERBEROS" == "1" ]]; then # KDC is embedded in `knoe-auth` by default. Only deploy standalone KDC when requested. if [[ "${PROLE_KDC_STANDALONE:-0}" == "1" ]]; then @@ -260,16 +268,17 @@ cleanup_old_namespace() { ARGOCD_NAMESPACE="$ARGOCD_NS" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_argocd.sh" -n "$ARGOCD_NS" stop || true fi + # Cleanup in reverse dependency order (OpenTofu first, then downstream deps) OPENTOFU_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" stop || true + KONG_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_kong.sh" -n "$ns" stop || true + NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" GARAGE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_garage_store.sh" stop || true OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_openbao.sh" -n "$ns" stop || true CERTMGR_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_certmgr.sh" -n "$ns" stop || true - NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" GARAGE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_garage_store.sh" stop || true - KONG_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_kong.sh" -n "$ns" stop || true if [[ "$ENABLE_KERBEROS" == "1" && "${PROLE_KDC_STANDALONE:-0}" == "1" ]]; then SERVICE_NAMESPACE="$ns" PROLE_KDC_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_kdc.sh" cleanup || true diff --git a/etc/status_common_services.sh b/etc/status_common_services.sh index 0e7e136..8be40a3 100755 --- a/etc/status_common_services.sh +++ b/etc/status_common_services.sh @@ -420,7 +420,7 @@ _recheck_pods() { # Print the name of each component that currently has blocked pods. _blocked_comps() { - for _bc in registry openbao opentofu garage auth kong certmgr; do + for _bc in registry openbao garage opentofu auth kong certmgr; do [[ "$_bc" == "auth" && "$ENABLE_KERBEROS" != "1" ]] && continue [[ "$_bc" == "registry" && ( "$REGISTRY_CHECK" -ne 1 || "$MODE" == "k3d" ) ]] && continue [[ ${BLOCKED_COUNT["$_bc"]:-0} -gt 0 ]] && echo "$_bc" diff --git a/mock_val/init_service_layer.sh b/mock_val/init_service_layer.sh index 7438c77..6f52364 100755 --- a/mock_val/init_service_layer.sh +++ b/mock_val/init_service_layer.sh @@ -225,8 +225,13 @@ deploy_service_layer() { "$SCRIPT_DIR/init_argocd.sh" -n "$ARGOCD_NS" "$argocd_action" || rc=$? fi - OPENTOFU_NAMESPACE="$ns" OPENTOFU_SECRET_NAMESPACE="${NAMESPACE:-$ns}" OPENTOFU_OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" "$opentofu_action" || rc=$? + # ------------------------------------------------------------------------- + # Deploy services in dependency order: + # 1. OpenBao – secrets vault; needed by downstream services + # 2. Garage – object storage + # 3. Kong – API gateway + # 4. OpenTofu – IaC engine; depends on registry + secrets (last) + # ------------------------------------------------------------------------- OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_openbao.sh" -n "$ns" "$openbao_action" || rc=$? @@ -240,6 +245,9 @@ deploy_service_layer() { KONG_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_kong.sh" -n "$ns" "$kong_action" || rc=$? + OPENTOFU_NAMESPACE="$ns" OPENTOFU_SECRET_NAMESPACE="${NAMESPACE:-$ns}" OPENTOFU_OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" "$opentofu_action" || rc=$? + if [[ "$ENABLE_KERBEROS" == "1" ]]; then SERVICE_NAMESPACE="$ns" PROLE_KDC_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_kdc.sh" "$kdc_action" || rc=$? @@ -255,16 +263,17 @@ cleanup_old_namespace() { ARGOCD_NAMESPACE="$ARGOCD_NS" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_argocd.sh" -n "$ARGOCD_NS" stop || true fi + # Cleanup in reverse dependency order (OpenTofu first, then downstream deps) OPENTOFU_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_opentofu.sh" -n "$ns" stop || true + KONG_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_kong.sh" -n "$ns" stop || true + NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" GARAGE_NAMESPACE="$ns" \ + "$SCRIPT_DIR/init_garage_store.sh" stop || true OPENBAO_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_openbao.sh" -n "$ns" stop || true CERTMGR_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_certmgr.sh" -n "$ns" stop || true - NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" GARAGE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_garage_store.sh" stop || true - KONG_NAMESPACE="$ns" SERVICE_NAMESPACE="$ns" \ - "$SCRIPT_DIR/init_kong.sh" -n "$ns" stop || true if [[ "$ENABLE_KERBEROS" == "1" ]]; then SERVICE_NAMESPACE="$ns" PROLE_KDC_NAMESPACE="$ns" \ "$SCRIPT_DIR/init_kdc.sh" cleanup || true diff --git a/mock_val/status_common_services.sh b/mock_val/status_common_services.sh index 0e7e136..8be40a3 100755 --- a/mock_val/status_common_services.sh +++ b/mock_val/status_common_services.sh @@ -420,7 +420,7 @@ _recheck_pods() { # Print the name of each component that currently has blocked pods. _blocked_comps() { - for _bc in registry openbao opentofu garage auth kong certmgr; do + for _bc in registry openbao garage opentofu auth kong certmgr; do [[ "$_bc" == "auth" && "$ENABLE_KERBEROS" != "1" ]] && continue [[ "$_bc" == "registry" && ( "$REGISTRY_CHECK" -ne 1 || "$MODE" == "k3d" ) ]] && continue [[ ${BLOCKED_COUNT["$_bc"]:-0} -gt 0 ]] && echo "$_bc"