checkpoint: update infrastructure scripts, manifests, and installer. etc/init_*.sh: added namespace support and OpenBao monitoring configuration; k8s/prole: added grafana-pvc, updated prole-db with S3 region support; install.py: implemented config secret encryption and improved ARM64 platform detection; .gitignore: expanded ignore patterns; prole-net: updated prole-agent binary

This commit is contained in:
chrisfu 2026-02-04 01:37:46 -08:00
parent bcc8f23a0d
commit 620ce63190
12 changed files with 836 additions and 155 deletions

31
.gitignore vendored
View File

@ -1,6 +1,32 @@
/.DS_Store
**/.DS_Store
/.gitignore
/__pycache__/
# Python artifacts
**/__pycache__/
*.py[cod]
*$py.class
*.pyc
# Build and distribution
/build/
/dist/
/installer/build/
/installer/dist/
/prole-app/dist/
/prole-app/dist-*/
/prole-tools-app/dist/
/prole-tools-app/.build/
/prole-tools-app/.build-cli/
# Binaries and generated tools
/net/prole-agent
/net/prole-scan
/bin/prole-agent
/bin/prole-scan
# MagicMock artifacts (likely accidental)
/MagicMock/
/bin/*
!/bin/prole-env.sh
!/bin/prole-kpf.sh
@ -10,12 +36,13 @@
/.vagrant/
/.vscode/
/.venv/
/prole-tools-app/dist/
/prole-tools-app/.build/
/prole-tools-app/.build-cli/
/prole/backup/
# Secrets and local config
etc/secrets/
prole/conf/
*-password.txt
*secret.yaml
.vault_pass

View File

@ -60,6 +60,7 @@ BAO_NAMESPACE="${NAMESPACE:-default}"
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
BAO_PATH_DB="${BAO_PATH_PREFIX}/db"
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"
ensure_tools() {
for t in kubectl curl openssl base64 jq; do
@ -78,6 +79,71 @@ ensure_namespace() {
fi
}
openbao_url() {
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
echo "$PROLE_OPENBAO_URL"
elif curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:18200"
else
echo "http://openbao.${NAMESPACE}.svc.cluster.local:8200"
fi
}
openbao_token() {
if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
cat "$OPENBAO_TOKEN_FILE"
else
echo "${OPENBAO_ROOT_TOKEN:-}"
fi
}
fetch_openbao_secret() {
local path="$1"
local key="$2"
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -z "$token" ]]; then
echo ""
return 0
fi
curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
}
resolve_db_password() {
local db_pw="${DB_PASSWORD:-}"
if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
local fetched_db
fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password")
if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
db_pw="$fetched_db"
fi
fi
printf '%s' "$db_pw"
}
ensure_grafana_admin_secret() {
local db_pw
db_pw=$(resolve_db_password)
if [[ -z "$db_pw" ]]; then
echo "ERROR: DB_PASSWORD unavailable; cannot create Grafana admin secret." >&2
return 1
fi
kubectl create secret generic grafana-admin \
-n "$NAMESPACE" \
--from-literal=admin-password="$db_pw" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -n "$token" ]]; then
curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
-X POST "$url/v1/kv/data/$BAO_PATH_MONITORING" \
-d "{\"data\":{\"grafana_admin_password\":\"$db_pw\"}}" >/dev/null || true
fi
}
ensure_cnpg_operator() {
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
return 0
@ -98,6 +164,7 @@ ensure_cnpg_operator() {
ensure_prole_stack_resources() {
echo "Applying CloudNative-PG cluster and related resources ..."
ensure_grafana_admin_secret
if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then
if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then
echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2

View File

@ -27,7 +27,7 @@ err() {
}
ensure_tools() {
for t in helm kubectl; do
for t in helm kubectl curl jq; do
command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; }
done
}
@ -40,6 +40,73 @@ ensure_namespace() {
fi
}
openbao_url() {
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
echo "$PROLE_OPENBAO_URL"
elif curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
echo "http://127.0.0.1:18200"
else
echo "http://openbao.${NAMESPACE}.svc.cluster.local:8200"
fi
}
openbao_token() {
if [[ -f "$PROLE_SERVICE/secrets/openbao-root-token" ]]; then
cat "$PROLE_SERVICE/secrets/openbao-root-token"
else
echo "${OPENBAO_ROOT_TOKEN:-}"
fi
}
fetch_openbao_secret() {
local path="$1"
local key="$2"
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -z "$token" ]]; then
echo ""
return 0
fi
curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
}
resolve_grafana_password() {
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then
local fetched
fetched=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/monitoring" "grafana_admin_password")
if [[ -n "$fetched" && "$fetched" != "null" ]]; then
GRAFANA_ADMIN_PASSWORD="$fetched"
fi
fi
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
local db_pw="${DB_PASSWORD:-}"
if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
local fetched_db
fetched_db=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/db" "password")
if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
db_pw="$fetched_db"
fi
fi
if [[ -n "$db_pw" ]]; then
GRAFANA_ADMIN_PASSWORD="$db_pw"
fi
fi
}
write_grafana_password_to_openbao() {
local token url
token=$(openbao_token)
url=$(openbao_url)
if [[ -z "$token" || -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
return 0
fi
curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
-X POST "$url/v1/kv/data/prole/${NAMESPACE:-default}/monitoring" \
-d "{\"data\":{\"grafana_admin_password\":\"$GRAFANA_ADMIN_PASSWORD\"}}" >/dev/null || true
}
cleanup_grafana_rbac_conflicts() {
local release="grafana-prole"
local ns="$NAMESPACE"
@ -112,6 +179,13 @@ install_monitoring() {
local prometheus_svc="http://prometheus-community-kube-prometheus.$monitoring_ns.svc.cluster.local:9090"
cleanup_grafana_rbac_conflicts
resolve_grafana_password
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
err "GRAFANA_ADMIN_PASSWORD is empty. Set it or ensure DB_PASSWORD is available."
exit 1
fi
write_grafana_password_to_openbao
# Note: kube-prometheus-stack may have already been installed with Grafana enabled in 'default'.
# If we want to move Grafana to $NAMESPACE, we install it there.
@ -122,13 +196,15 @@ install_monitoring() {
--force-conflicts \
--server-side=true \
--set "rbac.namespaced=true" \
--set "persistence.enabled=true" \
--set "persistence.size=5Gi" \
--set "datasources.datasources\.yaml.apiVersion=1" \
--set "datasources.datasources\.yaml.datasources[0].name=Prometheus" \
--set "datasources.datasources\.yaml.datasources[0].type=prometheus" \
--set "datasources.datasources\.yaml.datasources[0].url=$prometheus_svc" \
--set "datasources.datasources\.yaml.datasources[0].access=proxy" \
--set "datasources.datasources\.yaml.datasources[0].isDefault=true" \
--set "adminPassword=admin" \
--set "adminPassword=$GRAFANA_ADMIN_PASSWORD" \
grafana-prole grafana/grafana
log "Applying CNPG prometheus rules in namespace '$NAMESPACE'..."

View File

@ -20,7 +20,8 @@ if [[ -z "${PROLE_SERVICE:-}" ]]; then
exit 1
fi
ACTION=${1:-}
ACTION=""
NAMESPACE_OVERRIDE=""
# Defaults
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
@ -47,6 +48,49 @@ mkdir -p "$SECRETS_DIR"
admin_key_priv="$SECRETS_DIR/admin_ed25519.key"
admin_key_pub="$SECRETS_DIR/admin_ed25519.pub"
root_token_file="$SECRETS_DIR/openbao-root-token"
while [[ $# -gt 0 ]]; do
case "$1" in
-n|--namespace)
shift
if [[ -z "${1:-}" ]]; then
echo "ERROR: -n/--namespace requires a value" >&2
exit 2
fi
NAMESPACE_OVERRIDE="$1"
;;
-n=*|--namespace=*)
NAMESPACE_OVERRIDE="${1#*=}"
;;
start|stop|status|restart|initialize|update|reload)
ACTION="$1"
;;
--)
shift
break
;;
*)
if [[ -z "$ACTION" && "$1" != -* ]]; then
ACTION="$1"
else
echo "Unknown argument: $1" >&2
exit 2
fi
;;
esac
shift
done
if [[ -n "$NAMESPACE_OVERRIDE" ]]; then
NAMESPACE="$NAMESPACE_OVERRIDE"
export NAMESPACE
fi
if [[ -z "${NAMESPACE:-}" ]]; then
NAMESPACE="default"
export NAMESPACE
fi
BAO_NAMESPACE="${NAMESPACE:-default}"
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
@ -56,6 +100,9 @@ BAO_PATH_TDE="${BAO_PATH_PREFIX}/tde"
BAO_PATH_KERBEROS="${BAO_PATH_PREFIX}/kerberos"
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"
# Default Grafana password to the database root password when not explicitly set.
GRAFANA_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-${DB_PASSWORD:-}}
ensure_tools() {
for t in curl openssl base64 jq; do
command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
@ -651,7 +698,7 @@ case "$ACTION" in
echo "Re-applied manifests."
;;
*)
echo "Usage: $0 {start|stop|status|restart|initialize|update|reload}" >&2
echo "Usage: $0 [-n|--namespace NAME] {start|stop|status|restart|initialize|update|reload}" >&2
exit 2
;;
esac

View File

@ -23,6 +23,7 @@ GARAGE_BACKUP_BUCKET=${GARAGE_BACKUP_BUCKET:-prole-db-backups}
GARAGE_BACKUP_KEY_NAME=${GARAGE_BACKUP_KEY_NAME:-prole-db-backup}
GARAGE_BACKUP_SECRET_NAME=${GARAGE_BACKUP_SECRET_NAME:-prole-db-barman-s3}
GARAGE_S3_ENDPOINT=${GARAGE_S3_ENDPOINT:-http://$GARAGE_NAME.$NAMESPACE.svc.cluster.local:3900}
GARAGE_S3_REGION=${GARAGE_S3_REGION:-garage}
RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1}
usage() {
@ -132,6 +133,7 @@ ensure_garage_bucket_and_key() {
kubectl create secret generic "$GARAGE_BACKUP_SECRET_NAME" -n "$NAMESPACE" \
--from-literal=ACCESS_KEY_ID="$access_key" \
--from-literal=SECRET_ACCESS_KEY="$secret_key" \
--from-literal=REGION="$GARAGE_S3_REGION" \
--dry-run=client -o yaml | kubectl apply -f -
}
@ -145,7 +147,8 @@ configure_cnpg_backup() {
\"endpointURL\": \"$GARAGE_S3_ENDPOINT\",
\"s3Credentials\": {
\"accessKeyId\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"ACCESS_KEY_ID\"},
\"secretAccessKey\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"SECRET_ACCESS_KEY\"}
\"secretAccessKey\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"SECRET_ACCESS_KEY\"},
\"region\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"REGION\"}
},
\"wal\": {\"compression\": \"gzip\"},
\"data\": {\"compression\": \"gzip\"}

File diff suppressed because it is too large Load Diff

View File

@ -110,7 +110,13 @@ def is_apple_silicon() -> bool:
return platform.machine() == "arm64" and platform.system() == "Darwin"
def get_docker_build_platform_args() -> list[str]:
def get_docker_build_platform_args(target_env: Optional[str] = None) -> list[str]:
override = os.environ.get("PROLE_DOCKER_PLATFORM", "").strip()
if override:
return ["--platform", override]
env_key = (target_env or "").strip().lower()
if env_key in ("service", "k3s", "prole-service-cluster"):
return ["--platform", "linux/arm64"]
if is_apple_silicon():
return ["--platform", "linux/amd64"]
return []

View File

@ -24,10 +24,15 @@ spec:
- name: GF_SECURITY_ADMIN_USER
value: admin
- name: GF_SECURITY_ADMIN_PASSWORD
value: admin
valueFrom:
secretKeyRef:
name: grafana-admin
key: admin-password
volumeMounts:
- name: provisioning
mountPath: /etc/grafana/provisioning/datasources
- name: grafana-data
mountPath: /var/lib/grafana
volumes:
- name: provisioning
configMap:
@ -35,3 +40,6 @@ spec:
items:
- key: datasource.yml
path: datasource.yml
- name: grafana-data
persistentVolumeClaim:
claimName: grafana-data

View File

@ -0,0 +1,10 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: grafana-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi

View File

@ -18,4 +18,5 @@ resources:
- prometheus-service.yaml
- grafana-configmap.yaml
- grafana-deployment.yaml
- grafana-pvc.yaml
- grafana-service.yaml

View File

@ -48,6 +48,11 @@ spec:
- GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA topology TO prole;
enableSuperuserAccess: true
env:
- name: AWS_REGION
value: garage
- name: AWS_DEFAULT_REGION
value: garage
storage:
size: 1Gi
@ -65,7 +70,7 @@ spec:
backup:
barmanObjectStore:
destinationPath: s3://prole-db-backups/
endpointURL: http://garage.default.svc.cluster.local:3900
endpointURL: http://garage:3900
s3Credentials:
accessKeyId:
name: prole-db-barman-s3
@ -73,6 +78,9 @@ spec:
secretAccessKey:
name: prole-db-barman-s3
key: SECRET_ACCESS_KEY
region:
name: prole-db-barman-s3
key: REGION
wal:
compression: gzip
data:

Binary file not shown.