mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
chore: extend PVC diagnostics and enhance StorageClass reconciliation
- Improved PVC error diagnostics in `deploy.sh` with additional context. - Added robust StorageClass reconciliation logic, including dynamic creation and parameter validation. - Updated Helm templates to support advanced StorageClass attributes like `reclaimPolicy` and `volumeBindingMode`.
This commit is contained in:
chrisfu
parent
5ebe4d6c06
commit
d14605ea23
@ -103,26 +103,44 @@ check_supabase_pvc_quota_blocked() {
|
||||
error_info=$(kubectl "${ctx_args[@]}" get events -n "$ns" -o json 2>/dev/null | python3 - <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
def kubectl_json(*args):
|
||||
try:
|
||||
out = subprocess.check_output(["kubectl"] + list(args), text=True)
|
||||
return json.loads(out)
|
||||
except:
|
||||
return {}
|
||||
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
except:
|
||||
sys.exit(0)
|
||||
|
||||
for event in data.get("items", []):
|
||||
obj = event.get("involvedObject", {})
|
||||
if obj.get("kind") == "PersistentVolumeClaim":
|
||||
msg = event.get("message", "")
|
||||
reason = event.get("reason", "")
|
||||
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason:
|
||||
if "exceeded" in msg.lower() or "quota" in msg.lower():
|
||||
print(f"{obj.get('name')}|{msg}")
|
||||
sys.exit(0)
|
||||
if "QUOTA_EXCEEDED" in msg or "ProvisioningFailed" in reason or "exceeded" in msg.lower():
|
||||
pvc_name = obj.get("name")
|
||||
pvc_ns = obj.get("namespace")
|
||||
|
||||
# Fetch extra context
|
||||
pvc_spec = kubectl_json("get", "pvc", pvc_name, "-n", pvc_ns, "-o", "json").get("spec", {})
|
||||
sc_name = pvc_spec.get("storageClassName", "<unset>")
|
||||
sc_type = "unknown"
|
||||
if sc_name != "<unset>":
|
||||
sc_type = kubectl_json("get", "storageclass", sc_name, "-o", "json").get("parameters", {}).get("type", "unknown")
|
||||
|
||||
print(f"{pvc_name}|{sc_name}|{sc_type}|{msg}")
|
||||
sys.exit(0)
|
||||
PY
|
||||
)
|
||||
if [[ -n "$error_info" ]]; then
|
||||
local pvc_name="${error_info%%|*}"
|
||||
local gke_msg="${error_info#*|}"
|
||||
repair_blocked "Supabase PVC provisioning failed (Storage Quota Exceeded)" \
|
||||
"PVC '${pvc_name}' is blocked. GKE Error: ${gke_msg}"
|
||||
IFS='|' read -r pvc_name sc_name sc_type gke_msg <<< "$error_info"
|
||||
repair_blocked "Supabase PVC provisioning failed (Quota/Storage Error)" \
|
||||
"PVC '${pvc_name}' is blocked. storageClassName: ${sc_name}, parameters.type: ${sc_type}. GKE Error: ${gke_msg}"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -384,12 +402,12 @@ ns = sys.argv[1]
|
||||
kube_context = sys.argv[2].strip()
|
||||
selector = sys.argv[3].strip()
|
||||
|
||||
def kubectl_json(*args):
|
||||
def kubectl_json(*args, use_selector=True):
|
||||
cmd = ["kubectl"]
|
||||
if kube_context:
|
||||
cmd.extend(["--context", kube_context])
|
||||
cmd.extend(args)
|
||||
if selector:
|
||||
if use_selector and selector:
|
||||
cmd.extend(["-l", selector])
|
||||
out = subprocess.check_output(cmd, text=True)
|
||||
return json.loads(out)
|
||||
@ -404,6 +422,8 @@ def first_non_empty(*values):
|
||||
|
||||
pvc = kubectl_json("get", "pvc", "-n", ns, "-o", "json")
|
||||
pods = kubectl_json("get", "pods", "-n", ns, "-o", "json")
|
||||
sc = kubectl_json("get", "storageclass", "-o", "json", use_selector=False)
|
||||
sc_type_map = {item["metadata"]["name"]: item.get("parameters", {}).get("type", "unknown") for item in sc.get("items", [])}
|
||||
|
||||
pvc_pending = []
|
||||
for item in pvc.get("items", []) or []:
|
||||
@ -421,10 +441,14 @@ for item in pvc.get("items", []) or []:
|
||||
break
|
||||
if not reason:
|
||||
reason = phase or "Pending"
|
||||
|
||||
sc_name = first_non_empty(spec.get("storageClassName"), "<unset>")
|
||||
sc_type = sc_type_map.get(sc_name, "unknown")
|
||||
|
||||
pvc_pending.append({
|
||||
"name": name,
|
||||
"phase": phase or "Unknown",
|
||||
"storageClass": first_non_empty(spec.get("storageClassName"), "<unset>"),
|
||||
"storageClass": f"{sc_name} (type: {sc_type})",
|
||||
"reason": reason,
|
||||
})
|
||||
|
||||
@ -2178,21 +2202,48 @@ ensure_gke_supabase_storage_class() {
|
||||
local sc_json
|
||||
sc_json=$(kubectl get storageclass "$sc_name" -o json 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$sc_json" ]]; then
|
||||
repair_blocked "storage-class configuration error: Required StorageClass '${sc_name}' does not exist on this cluster." \
|
||||
"Create the HDD-backed StorageClass (provisioner: pd.csi.storage.gke.io, type: pd-standard) before deploying Supabase."
|
||||
local provisioner=""
|
||||
local disk_type=""
|
||||
local reclaim_policy=""
|
||||
local binding_mode=""
|
||||
|
||||
if [[ -n "$sc_json" ]]; then
|
||||
provisioner=$(echo "$sc_json" | python3 -c "import sys, json; print(json.load(sys.stdin).get('provisioner', ''))")
|
||||
disk_type=$(echo "$sc_json" | python3 -c "import sys, json; print(json.load(sys.stdin).get('parameters', {}).get('type', ''))")
|
||||
reclaim_policy=$(echo "$sc_json" | python3 -c "import sys, json; print(json.load(sys.stdin).get('reclaimPolicy', 'Retain'))")
|
||||
binding_mode=$(echo "$sc_json" | python3 -c "import sys, json; print(json.load(sys.stdin).get('volumeBindingMode', 'WaitForFirstConsumer'))")
|
||||
fi
|
||||
|
||||
local provisioner disk_type
|
||||
provisioner=$(echo "$sc_json" | python3 -c "import sys, json; print(json.load(sys.stdin).get('provisioner', ''))")
|
||||
disk_type=$(echo "$sc_json" | python3 -c "import sys, json; print(json.load(sys.stdin).get('parameters', {}).get('type', ''))")
|
||||
|
||||
if [[ "$provisioner" != "pd.csi.storage.gke.io" ]]; then
|
||||
repair_blocked "storage-class configuration error: StorageClass '${sc_name}' must use provisioner 'pd.csi.storage.gke.io', but uses '${provisioner}'."
|
||||
fi
|
||||
|
||||
if [[ "$disk_type" != "pd-standard" ]]; then
|
||||
repair_blocked "storage-class configuration error: StorageClass '${sc_name}' must use parameters.type 'pd-standard', but uses '${disk_type}'."
|
||||
if [[ -z "$sc_json" ]] || [[ "$provisioner" != "pd.csi.storage.gke.io" ]] || [[ "$disk_type" != "pd-standard" ]] || [[ "$binding_mode" != "WaitForFirstConsumer" ]]; then
|
||||
if [[ -n "$sc_json" ]]; then
|
||||
warn "StorageClass '${sc_name}' exists but has incorrect parameters (type=${disk_type}, provisioner=${provisioner}, mode=${binding_mode}). Reconciling..."
|
||||
kubectl delete storageclass "$sc_name" --wait=true >/dev/null 2>&1 || true
|
||||
else
|
||||
log "StorageClass '${sc_name}' does not exist. Creating HDD-backed StorageClass..."
|
||||
fi
|
||||
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ${sc_name}
|
||||
provisioner: pd.csi.storage.gke.io
|
||||
parameters:
|
||||
type: pd-standard
|
||||
reclaimPolicy: Retain
|
||||
volumeBindingMode: WaitForFirstConsumer
|
||||
allowVolumeExpansion: true
|
||||
EOF
|
||||
|
||||
# Re-verify after creation
|
||||
sc_json=$(kubectl get storageclass "$sc_name" -o json 2>/dev/null || true)
|
||||
if [[ -z "$sc_json" ]]; then
|
||||
die "Failed to create StorageClass '${sc_name}'."
|
||||
fi
|
||||
disk_type=$(echo "$sc_json" | python3 -c "import sys, json; print(json.load(sys.stdin).get('parameters', {}).get('type', ''))")
|
||||
if [[ "$disk_type" != "pd-standard" ]]; then
|
||||
die "Created StorageClass '${sc_name}' but it still reports type '${disk_type}'. This is unexpected."
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Verified live Supabase StorageClass '${sc_name}' (type: ${disk_type})."
|
||||
@ -2297,9 +2348,13 @@ run_helm() {
|
||||
local is_gke_cluster="false"
|
||||
local force_gke_functions_single_replica="false"
|
||||
|
||||
# Robust GKE detection: check for gke nodes OR the gke-cluster-id label on nodes
|
||||
first_node_name=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||||
is_gke_cluster="false"
|
||||
if [[ "$first_node_name" == gke-* ]]; then
|
||||
is_gke_cluster="true"
|
||||
elif kubectl get nodes -o jsonpath='{.items[0].metadata.labels}' 2>/dev/null | grep -q "cloud.google.com/gke-cluster-id"; then
|
||||
is_gke_cluster="true"
|
||||
fi
|
||||
|
||||
# Resolve storage class name strictly in k8s mode
|
||||
|
||||
@ -831,9 +831,11 @@ def _build_overlay(cfg: configparser.ConfigParser, args: argparse.Namespace) ->
|
||||
"fullnameOverride": "supabase",
|
||||
"storageClass": {
|
||||
"enabled": mode == "k8s",
|
||||
"name": "supabase-standard",
|
||||
"name": supabase_storage_class or "supabase-standard",
|
||||
"provisioner": "pd.csi.storage.gke.io",
|
||||
"type": "pd-standard",
|
||||
"reclaimPolicy": "Retain",
|
||||
"volumeBindingMode": "WaitForFirstConsumer",
|
||||
},
|
||||
"publicIngress": {
|
||||
"enabled": False,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user