mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 12:03:59 +00:00
checkpoint: update infrastructure scripts, manifests, and installer. etc/init_*.sh: added namespace support and OpenBao monitoring configuration; k8s/prole: added grafana-pvc, updated prole-db with S3 region support; install.py: implemented config secret encryption and improved ARM64 platform detection; .gitignore: expanded ignore patterns; prole-net: updated prole-agent binary
This commit is contained in:
parent
bcc8f23a0d
commit
620ce63190
31
.gitignore
vendored
31
.gitignore
vendored
@ -1,6 +1,32 @@
|
||||
/.DS_Store
|
||||
**/.DS_Store
|
||||
/.gitignore
|
||||
/__pycache__/
|
||||
# Python artifacts
|
||||
**/__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.pyc
|
||||
|
||||
# Build and distribution
|
||||
/build/
|
||||
/dist/
|
||||
/installer/build/
|
||||
/installer/dist/
|
||||
/prole-app/dist/
|
||||
/prole-app/dist-*/
|
||||
/prole-tools-app/dist/
|
||||
/prole-tools-app/.build/
|
||||
/prole-tools-app/.build-cli/
|
||||
|
||||
# Binaries and generated tools
|
||||
/net/prole-agent
|
||||
/net/prole-scan
|
||||
/bin/prole-agent
|
||||
/bin/prole-scan
|
||||
|
||||
# MagicMock artifacts (likely accidental)
|
||||
/MagicMock/
|
||||
|
||||
/bin/*
|
||||
!/bin/prole-env.sh
|
||||
!/bin/prole-kpf.sh
|
||||
@ -10,12 +36,13 @@
|
||||
/.vagrant/
|
||||
/.vscode/
|
||||
/.venv/
|
||||
/prole-tools-app/dist/
|
||||
/prole-tools-app/.build/
|
||||
/prole-tools-app/.build-cli/
|
||||
/prole/backup/
|
||||
|
||||
# Secrets and local config
|
||||
etc/secrets/
|
||||
prole/conf/
|
||||
*-password.txt
|
||||
*secret.yaml
|
||||
.vault_pass
|
||||
|
||||
@ -60,6 +60,7 @@ BAO_NAMESPACE="${NAMESPACE:-default}"
|
||||
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
|
||||
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
|
||||
BAO_PATH_DB="${BAO_PATH_PREFIX}/db"
|
||||
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"
|
||||
|
||||
ensure_tools() {
|
||||
for t in kubectl curl openssl base64 jq; do
|
||||
@ -78,6 +79,71 @@ ensure_namespace() {
|
||||
fi
|
||||
}
|
||||
|
||||
openbao_url() {
|
||||
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
|
||||
echo "$PROLE_OPENBAO_URL"
|
||||
elif curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
|
||||
echo "http://127.0.0.1:18200"
|
||||
else
|
||||
echo "http://openbao.${NAMESPACE}.svc.cluster.local:8200"
|
||||
fi
|
||||
}
|
||||
|
||||
openbao_token() {
|
||||
if [[ -f "$OPENBAO_TOKEN_FILE" ]]; then
|
||||
cat "$OPENBAO_TOKEN_FILE"
|
||||
else
|
||||
echo "${OPENBAO_ROOT_TOKEN:-}"
|
||||
fi
|
||||
}
|
||||
|
||||
fetch_openbao_secret() {
|
||||
local path="$1"
|
||||
local key="$2"
|
||||
local token url
|
||||
token=$(openbao_token)
|
||||
url=$(openbao_url)
|
||||
if [[ -z "$token" ]]; then
|
||||
echo ""
|
||||
return 0
|
||||
fi
|
||||
curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
|
||||
}
|
||||
|
||||
resolve_db_password() {
|
||||
local db_pw="${DB_PASSWORD:-}"
|
||||
if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
|
||||
local fetched_db
|
||||
fetched_db=$(fetch_openbao_secret "$BAO_PATH_DB" "password")
|
||||
if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
|
||||
db_pw="$fetched_db"
|
||||
fi
|
||||
fi
|
||||
printf '%s' "$db_pw"
|
||||
}
|
||||
|
||||
ensure_grafana_admin_secret() {
|
||||
local db_pw
|
||||
db_pw=$(resolve_db_password)
|
||||
if [[ -z "$db_pw" ]]; then
|
||||
echo "ERROR: DB_PASSWORD unavailable; cannot create Grafana admin secret." >&2
|
||||
return 1
|
||||
fi
|
||||
kubectl create secret generic grafana-admin \
|
||||
-n "$NAMESPACE" \
|
||||
--from-literal=admin-password="$db_pw" \
|
||||
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
|
||||
local token url
|
||||
token=$(openbao_token)
|
||||
url=$(openbao_url)
|
||||
if [[ -n "$token" ]]; then
|
||||
curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
|
||||
-X POST "$url/v1/kv/data/$BAO_PATH_MONITORING" \
|
||||
-d "{\"data\":{\"grafana_admin_password\":\"$db_pw\"}}" >/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_cnpg_operator() {
|
||||
if kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
|
||||
return 0
|
||||
@ -98,6 +164,7 @@ ensure_cnpg_operator() {
|
||||
|
||||
ensure_prole_stack_resources() {
|
||||
echo "Applying CloudNative-PG cluster and related resources ..."
|
||||
ensure_grafana_admin_secret
|
||||
if [[ -n "$CNPG_MANIFEST_OVERRIDE" ]]; then
|
||||
if [[ ! -f "$CNPG_MANIFEST_OVERRIDE" ]]; then
|
||||
echo "ERROR: CNPG_MANIFEST_OVERRIDE not found: $CNPG_MANIFEST_OVERRIDE" >&2
|
||||
|
||||
@ -27,7 +27,7 @@ err() {
|
||||
}
|
||||
|
||||
ensure_tools() {
|
||||
for t in helm kubectl; do
|
||||
for t in helm kubectl curl jq; do
|
||||
command -v "$t" >/dev/null || { err "Missing required tool: $t"; exit 1; }
|
||||
done
|
||||
}
|
||||
@ -40,6 +40,73 @@ ensure_namespace() {
|
||||
fi
|
||||
}
|
||||
|
||||
openbao_url() {
|
||||
if [[ -n "${PROLE_OPENBAO_URL:-}" ]]; then
|
||||
echo "$PROLE_OPENBAO_URL"
|
||||
elif curl -sS "http://127.0.0.1:18200/v1/sys/health" >/dev/null 2>&1; then
|
||||
echo "http://127.0.0.1:18200"
|
||||
else
|
||||
echo "http://openbao.${NAMESPACE}.svc.cluster.local:8200"
|
||||
fi
|
||||
}
|
||||
|
||||
openbao_token() {
|
||||
if [[ -f "$PROLE_SERVICE/secrets/openbao-root-token" ]]; then
|
||||
cat "$PROLE_SERVICE/secrets/openbao-root-token"
|
||||
else
|
||||
echo "${OPENBAO_ROOT_TOKEN:-}"
|
||||
fi
|
||||
}
|
||||
|
||||
fetch_openbao_secret() {
|
||||
local path="$1"
|
||||
local key="$2"
|
||||
local token url
|
||||
token=$(openbao_token)
|
||||
url=$(openbao_url)
|
||||
if [[ -z "$token" ]]; then
|
||||
echo ""
|
||||
return 0
|
||||
fi
|
||||
curl -sS -H "X-Vault-Token: $token" "$url/v1/kv/data/$path" | jq -r ".data.data.\"$key\"" || echo ""
|
||||
}
|
||||
|
||||
resolve_grafana_password() {
|
||||
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" || "${GRAFANA_ADMIN_PASSWORD}" == '${OPENBAO:'* || "${GRAFANA_ADMIN_PASSWORD}" == '${PROLE_SECRET:'* ]]; then
|
||||
local fetched
|
||||
fetched=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/monitoring" "grafana_admin_password")
|
||||
if [[ -n "$fetched" && "$fetched" != "null" ]]; then
|
||||
GRAFANA_ADMIN_PASSWORD="$fetched"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
|
||||
local db_pw="${DB_PASSWORD:-}"
|
||||
if [[ -z "$db_pw" || "$db_pw" == '${OPENBAO:'* || "$db_pw" == '${PROLE_SECRET:'* ]]; then
|
||||
local fetched_db
|
||||
fetched_db=$(fetch_openbao_secret "prole/${NAMESPACE:-default}/db" "password")
|
||||
if [[ -n "$fetched_db" && "$fetched_db" != "null" ]]; then
|
||||
db_pw="$fetched_db"
|
||||
fi
|
||||
fi
|
||||
if [[ -n "$db_pw" ]]; then
|
||||
GRAFANA_ADMIN_PASSWORD="$db_pw"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
write_grafana_password_to_openbao() {
|
||||
local token url
|
||||
token=$(openbao_token)
|
||||
url=$(openbao_url)
|
||||
if [[ -z "$token" || -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
curl -sS -H "X-Vault-Token: $token" -H 'Content-Type: application/json' \
|
||||
-X POST "$url/v1/kv/data/prole/${NAMESPACE:-default}/monitoring" \
|
||||
-d "{\"data\":{\"grafana_admin_password\":\"$GRAFANA_ADMIN_PASSWORD\"}}" >/dev/null || true
|
||||
}
|
||||
|
||||
cleanup_grafana_rbac_conflicts() {
|
||||
local release="grafana-prole"
|
||||
local ns="$NAMESPACE"
|
||||
@ -112,6 +179,13 @@ install_monitoring() {
|
||||
local prometheus_svc="http://prometheus-community-kube-prometheus.$monitoring_ns.svc.cluster.local:9090"
|
||||
|
||||
cleanup_grafana_rbac_conflicts
|
||||
|
||||
resolve_grafana_password
|
||||
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
|
||||
err "GRAFANA_ADMIN_PASSWORD is empty. Set it or ensure DB_PASSWORD is available."
|
||||
exit 1
|
||||
fi
|
||||
write_grafana_password_to_openbao
|
||||
|
||||
# Note: kube-prometheus-stack may have already been installed with Grafana enabled in 'default'.
|
||||
# If we want to move Grafana to $NAMESPACE, we install it there.
|
||||
@ -122,13 +196,15 @@ install_monitoring() {
|
||||
--force-conflicts \
|
||||
--server-side=true \
|
||||
--set "rbac.namespaced=true" \
|
||||
--set "persistence.enabled=true" \
|
||||
--set "persistence.size=5Gi" \
|
||||
--set "datasources.datasources\.yaml.apiVersion=1" \
|
||||
--set "datasources.datasources\.yaml.datasources[0].name=Prometheus" \
|
||||
--set "datasources.datasources\.yaml.datasources[0].type=prometheus" \
|
||||
--set "datasources.datasources\.yaml.datasources[0].url=$prometheus_svc" \
|
||||
--set "datasources.datasources\.yaml.datasources[0].access=proxy" \
|
||||
--set "datasources.datasources\.yaml.datasources[0].isDefault=true" \
|
||||
--set "adminPassword=admin" \
|
||||
--set "adminPassword=$GRAFANA_ADMIN_PASSWORD" \
|
||||
grafana-prole grafana/grafana
|
||||
|
||||
log "Applying CNPG prometheus rules in namespace '$NAMESPACE'..."
|
||||
|
||||
@ -20,7 +20,8 @@ if [[ -z "${PROLE_SERVICE:-}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ACTION=${1:-}
|
||||
ACTION=""
|
||||
NAMESPACE_OVERRIDE=""
|
||||
|
||||
# Defaults
|
||||
OPENBAO_NAME=${OPENBAO_NAME:-openbao}
|
||||
@ -47,6 +48,49 @@ mkdir -p "$SECRETS_DIR"
|
||||
admin_key_priv="$SECRETS_DIR/admin_ed25519.key"
|
||||
admin_key_pub="$SECRETS_DIR/admin_ed25519.pub"
|
||||
root_token_file="$SECRETS_DIR/openbao-root-token"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-n|--namespace)
|
||||
shift
|
||||
if [[ -z "${1:-}" ]]; then
|
||||
echo "ERROR: -n/--namespace requires a value" >&2
|
||||
exit 2
|
||||
fi
|
||||
NAMESPACE_OVERRIDE="$1"
|
||||
;;
|
||||
-n=*|--namespace=*)
|
||||
NAMESPACE_OVERRIDE="${1#*=}"
|
||||
;;
|
||||
start|stop|status|restart|initialize|update|reload)
|
||||
ACTION="$1"
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$ACTION" && "$1" != -* ]]; then
|
||||
ACTION="$1"
|
||||
else
|
||||
echo "Unknown argument: $1" >&2
|
||||
exit 2
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [[ -n "$NAMESPACE_OVERRIDE" ]]; then
|
||||
NAMESPACE="$NAMESPACE_OVERRIDE"
|
||||
export NAMESPACE
|
||||
fi
|
||||
|
||||
if [[ -z "${NAMESPACE:-}" ]]; then
|
||||
NAMESPACE="default"
|
||||
export NAMESPACE
|
||||
fi
|
||||
|
||||
BAO_NAMESPACE="${NAMESPACE:-default}"
|
||||
BAO_PATH_PREFIX="prole/${BAO_NAMESPACE}"
|
||||
BAO_PATH_ADMIN="${BAO_PATH_PREFIX}/admin"
|
||||
@ -56,6 +100,9 @@ BAO_PATH_TDE="${BAO_PATH_PREFIX}/tde"
|
||||
BAO_PATH_KERBEROS="${BAO_PATH_PREFIX}/kerberos"
|
||||
BAO_PATH_MONITORING="${BAO_PATH_PREFIX}/monitoring"
|
||||
|
||||
# Default Grafana password to the database root password when not explicitly set.
|
||||
GRAFANA_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-${DB_PASSWORD:-}}
|
||||
|
||||
ensure_tools() {
|
||||
for t in curl openssl base64 jq; do
|
||||
command -v "$t" >/dev/null || { echo "Missing required tool: $t" >&2; exit 1; }
|
||||
@ -651,7 +698,7 @@ case "$ACTION" in
|
||||
echo "Re-applied manifests."
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {start|stop|status|restart|initialize|update|reload}" >&2
|
||||
echo "Usage: $0 [-n|--namespace NAME] {start|stop|status|restart|initialize|update|reload}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
@ -23,6 +23,7 @@ GARAGE_BACKUP_BUCKET=${GARAGE_BACKUP_BUCKET:-prole-db-backups}
|
||||
GARAGE_BACKUP_KEY_NAME=${GARAGE_BACKUP_KEY_NAME:-prole-db-backup}
|
||||
GARAGE_BACKUP_SECRET_NAME=${GARAGE_BACKUP_SECRET_NAME:-prole-db-barman-s3}
|
||||
GARAGE_S3_ENDPOINT=${GARAGE_S3_ENDPOINT:-http://$GARAGE_NAME.$NAMESPACE.svc.cluster.local:3900}
|
||||
GARAGE_S3_REGION=${GARAGE_S3_REGION:-garage}
|
||||
RUN_FIRST_BACKUP=${RUN_FIRST_BACKUP:-1}
|
||||
|
||||
usage() {
|
||||
@ -132,6 +133,7 @@ ensure_garage_bucket_and_key() {
|
||||
kubectl create secret generic "$GARAGE_BACKUP_SECRET_NAME" -n "$NAMESPACE" \
|
||||
--from-literal=ACCESS_KEY_ID="$access_key" \
|
||||
--from-literal=SECRET_ACCESS_KEY="$secret_key" \
|
||||
--from-literal=REGION="$GARAGE_S3_REGION" \
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
}
|
||||
|
||||
@ -145,7 +147,8 @@ configure_cnpg_backup() {
|
||||
\"endpointURL\": \"$GARAGE_S3_ENDPOINT\",
|
||||
\"s3Credentials\": {
|
||||
\"accessKeyId\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"ACCESS_KEY_ID\"},
|
||||
\"secretAccessKey\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"SECRET_ACCESS_KEY\"}
|
||||
\"secretAccessKey\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"SECRET_ACCESS_KEY\"},
|
||||
\"region\": {\"name\": \"$GARAGE_BACKUP_SECRET_NAME\", \"key\": \"REGION\"}
|
||||
},
|
||||
\"wal\": {\"compression\": \"gzip\"},
|
||||
\"data\": {\"compression\": \"gzip\"}
|
||||
|
||||
718
install.py
718
install.py
File diff suppressed because it is too large
Load Diff
@ -110,7 +110,13 @@ def is_apple_silicon() -> bool:
|
||||
return platform.machine() == "arm64" and platform.system() == "Darwin"
|
||||
|
||||
|
||||
def get_docker_build_platform_args() -> list[str]:
|
||||
def get_docker_build_platform_args(target_env: Optional[str] = None) -> list[str]:
|
||||
override = os.environ.get("PROLE_DOCKER_PLATFORM", "").strip()
|
||||
if override:
|
||||
return ["--platform", override]
|
||||
env_key = (target_env or "").strip().lower()
|
||||
if env_key in ("service", "k3s", "prole-service-cluster"):
|
||||
return ["--platform", "linux/arm64"]
|
||||
if is_apple_silicon():
|
||||
return ["--platform", "linux/amd64"]
|
||||
return []
|
||||
|
||||
@ -24,10 +24,15 @@ spec:
|
||||
- name: GF_SECURITY_ADMIN_USER
|
||||
value: admin
|
||||
- name: GF_SECURITY_ADMIN_PASSWORD
|
||||
value: admin
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: grafana-admin
|
||||
key: admin-password
|
||||
volumeMounts:
|
||||
- name: provisioning
|
||||
mountPath: /etc/grafana/provisioning/datasources
|
||||
- name: grafana-data
|
||||
mountPath: /var/lib/grafana
|
||||
volumes:
|
||||
- name: provisioning
|
||||
configMap:
|
||||
@ -35,3 +40,6 @@ spec:
|
||||
items:
|
||||
- key: datasource.yml
|
||||
path: datasource.yml
|
||||
- name: grafana-data
|
||||
persistentVolumeClaim:
|
||||
claimName: grafana-data
|
||||
|
||||
10
k8s/prole/grafana-pvc.yaml
Normal file
10
k8s/prole/grafana-pvc.yaml
Normal file
@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: grafana-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
@ -18,4 +18,5 @@ resources:
|
||||
- prometheus-service.yaml
|
||||
- grafana-configmap.yaml
|
||||
- grafana-deployment.yaml
|
||||
- grafana-pvc.yaml
|
||||
- grafana-service.yaml
|
||||
|
||||
@ -48,6 +48,11 @@ spec:
|
||||
- GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA topology TO prole;
|
||||
|
||||
enableSuperuserAccess: true
|
||||
env:
|
||||
- name: AWS_REGION
|
||||
value: garage
|
||||
- name: AWS_DEFAULT_REGION
|
||||
value: garage
|
||||
|
||||
storage:
|
||||
size: 1Gi
|
||||
@ -65,7 +70,7 @@ spec:
|
||||
backup:
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://prole-db-backups/
|
||||
endpointURL: http://garage.default.svc.cluster.local:3900
|
||||
endpointURL: http://garage:3900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: prole-db-barman-s3
|
||||
@ -73,6 +78,9 @@ spec:
|
||||
secretAccessKey:
|
||||
name: prole-db-barman-s3
|
||||
key: SECRET_ACCESS_KEY
|
||||
region:
|
||||
name: prole-db-barman-s3
|
||||
key: REGION
|
||||
wal:
|
||||
compression: gzip
|
||||
data:
|
||||
|
||||
Binary file not shown.
Loading…
Reference in New Issue
Block a user