prole/update.sh
chrisfu e3c2e625f1 refactor(config): separate k3d k3s and gke config entrypoints
Rename env config files from conf/*/prole.cfg to conf/k3d.cfg, conf/k3s.cfg, and conf/gke.cfg. Update shell/Python loaders and etc/deploy scripts to resolve named configs cleanly while keeping legacy fallback behavior. Align k3s Ansible tasks, docs, and regression coverage with the new configuration layout.

Co-authored-by: Junie <junie@jetbrains.com>
2026-04-11 22:20:45 -07:00

584 lines
21 KiB
Bash
Executable File

#!/usr/bin/env bash
# update.sh — Apply the vault master password to all prole resources.
#
# Reads vault_prole_db_master_password from:
# infrastructure/inventory/group_vars/all/vault_db_master.yml
#
# Applies the password to:
# - k8s secrets: knoe-db-user, knoe-db-superuser (knoe-cnpg-0, knoe-db-0)
# - PostgreSQL users: prole, postgres (via CNPG primary pod)
# - k8s secret: prometheus-grafana (knoe-dev-0, monitoring)
# - Grafana CLI reset (via grafana pod exec)
# - config password fields (conf/gke.cfg, conf/service/prod.cfg)
# - OpenBao kv secrets (if reachable)
#
# Usage:
# ./update.sh [--dry-run] [--prompt] [--vault-pass-file <path>]
# [--skip-db] [--skip-grafana] [--skip-cfg]
#
# --prompt Read password interactively (masked, confirmed twice) and
# re-create vault_db_master.yml with ansible-vault encrypt (whole-file).
# Use this when the vault file is missing or corrupt.
#
# The script is idempotent — safe to run on every deploy or rotation.
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# ---------------------------------------------------------------------------
# Auto-activate project virtualenv (provides ansible-vault, python3 packages)
# ---------------------------------------------------------------------------
if [[ -z "${VIRTUAL_ENV:-}" && -f "${ROOT_DIR}/bin/activate" ]]; then
# shellcheck disable=SC1091
source "${ROOT_DIR}/bin/activate" 2>/dev/null || true
fi
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
DRY_RUN=false
PROMPT_MODE=false
SKIP_DB=false
SKIP_GRAFANA=false
SKIP_CFG=false
VAULT_PASS_FILE="${ROOT_DIR}/.vault_pass"
VAULT_FILE="${ROOT_DIR}/infrastructure/inventory/group_vars/all/vault_db_master.yml"
VAULT_KEY="vault_prole_db_master_password"
# Cluster contexts — override via env or auto-detect from config files
APP_CTX="${APP_CLUSTER_KUBECONTEXT:-}"
DB_CTX="${DB_CLUSTER_KUBECONTEXT:-}"
DB_NS="${DATABASE_NAMESPACE:-knoe-db-0}"
MON_NS="${MONITORING_NAMESPACE:-monitoring}"
SVC_NS="${SERVICE_NAMESPACE:-knoe-system}"
# ---------------------------------------------------------------------------
# Args
# ---------------------------------------------------------------------------
while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run) DRY_RUN=true; shift ;;
--prompt) PROMPT_MODE=true; shift ;;
--skip-db) SKIP_DB=true; shift ;;
--skip-grafana) SKIP_GRAFANA=true; shift ;;
--skip-cfg) SKIP_CFG=true; shift ;;
--vault-pass-file) VAULT_PASS_FILE="${2:?}"; shift 2 ;;
-h|--help)
grep '^#' "$0" | head -25 | sed 's/^# \?//'
exit 0 ;;
*) echo "Unknown argument: $1" >&2; exit 2 ;;
esac
done
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
log() { echo "[update.sh] $*" >&2; }
warn() { echo "[update.sh] WARN: $*" >&2; }
fail() { echo "[update.sh] ERROR: $*" >&2; exit 1; }
run() {
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY-RUN] $*"
else
"$@"
fi
}
b64enc() { printf '%s' "$1" | base64 | tr -d '\n'; }
# b64dec: decodes base64 — accepts argument OR piped stdin
b64dec() {
if [[ $# -gt 0 ]]; then
printf '%s' "$1" | base64 -d
else
base64 -d
fi
}
# Read a key from any section of an INI-style config
cfg_get() {
local file="$1" key="$2"
python3 - "$file" "$key" <<'PY'
import configparser, sys
cfg = configparser.ConfigParser(interpolation=None)
cfg.optionxform = str
cfg.read(sys.argv[1])
for s in cfg.sections():
if cfg.has_option(s, sys.argv[2]):
v = cfg.get(s, sys.argv[2], fallback="").strip()
if v: print(v); break
PY
}
# ---------------------------------------------------------------------------
# Step 0: Resolve cluster contexts from config if not set in env
# ---------------------------------------------------------------------------
_cfg_prod="${ROOT_DIR}/conf/gke.cfg"
_cfg_svc="${ROOT_DIR}/conf/service/prod.cfg"
resolve_contexts() {
if [[ -z "$APP_CTX" ]]; then
APP_CTX=$(cfg_get "$_cfg_svc" "APP_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$APP_CTX" && -f "$_cfg_prod" ]]; then
APP_CTX=$(cfg_get "$_cfg_prod" "APP_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$DB_CTX" ]]; then
DB_CTX=$(cfg_get "$_cfg_svc" "DB_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$DB_CTX" && -f "$_cfg_prod" ]]; then
DB_CTX=$(cfg_get "$_cfg_prod" "DB_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$APP_CTX" ]]; then
warn "APP_CLUSTER_KUBECONTEXT not set — will use ambient kubectl context for app cluster ops."
fi
if [[ -z "$DB_CTX" ]]; then
warn "DB_CLUSTER_KUBECONTEXT not set — will use ambient kubectl context for DB cluster ops."
fi
log "App cluster context : ${APP_CTX:-<ambient>}"
log "DB cluster context : ${DB_CTX:-<ambient>}"
}
kapp() { kubectl ${APP_CTX:+--context="$APP_CTX"} "$@"; }
kdb() { kubectl ${DB_CTX:+--context="$DB_CTX"} "$@"; }
# ---------------------------------------------------------------------------
# Step 1a: Decrypt master password from Ansible vault
# ---------------------------------------------------------------------------
decrypt_vault_password() {
log "Decrypting master password from vault..."
if [[ ! -f "$VAULT_FILE" ]]; then
fail "Vault file not found: $VAULT_FILE (run with --prompt to create it)"
fi
if [[ ! -f "$VAULT_PASS_FILE" ]]; then
fail "Vault password file not found: $VAULT_PASS_FILE (set --vault-pass-file)"
fi
if ! command -v ansible-vault >/dev/null 2>&1; then
fail "ansible-vault not found — install ansible or activate the project virtualenv."
fi
local decrypted
decrypted=$(ansible-vault view \
--vault-password-file "$VAULT_PASS_FILE" \
"$VAULT_FILE" 2>&1) || fail "ansible-vault view failed — vault file may be corrupt or unencrypted. Run: ./update.sh --prompt"
# Extract the value from YAML
local pw
pw=$(python3 - "$VAULT_KEY" <<PY
import sys, yaml
try:
data = yaml.safe_load("""${decrypted}""")
except Exception as e:
raise SystemExit(f"YAML parse error: {e}")
key = sys.argv[1]
if not isinstance(data, dict) or key not in data:
raise SystemExit(f"Key '{key}' not found in vault file. Available keys: {list(data.keys()) if isinstance(data, dict) else 'none'}")
print(data[key])
PY
)
if [[ -z "$pw" ]]; then
fail "Decrypted password is empty — check vault file key: $VAULT_KEY"
fi
printf '%s' "$pw"
}
# ---------------------------------------------------------------------------
# Step 1b: Prompt for new password + recreate vault file (--prompt mode)
# ---------------------------------------------------------------------------
prompt_and_recreate_vault() {
log "Creating new master password and recreating vault file..."
if [[ ! -f "$VAULT_PASS_FILE" ]]; then
fail "Vault password file not found: $VAULT_PASS_FILE (set --vault-pass-file)"
fi
if ! command -v ansible-vault >/dev/null 2>&1; then
fail "ansible-vault not found — activate the project virtualenv first."
fi
# Read new password twice with masked echo
local pw1 pw2
# Use /dev/tty so prompt works even when stdout is redirected
if [[ -t 0 ]]; then
read -r -s -p "New master password: " pw1 </dev/tty; echo >/dev/tty
read -r -s -p "Confirm master password: " pw2 </dev/tty; echo >/dev/tty
else
# Non-interactive fallback (e.g. piped input)
read -r pw1
pw2="$pw1"
fi
if [[ -z "$pw1" ]]; then
fail "Password cannot be empty."
fi
if [[ "$pw1" != "$pw2" ]]; then
fail "Passwords do not match — please try again."
fi
# Write a plain YAML file then encrypt the whole file.
# We use whole-file encryption (ansible-vault encrypt) NOT encrypt_string,
# so that ansible-vault view can read it back correctly.
log "Encrypting password with ansible-vault (whole-file)..."
mkdir -p "$(dirname "$VAULT_FILE")"
local plain_tmp="${VAULT_FILE}.plain.tmp"
printf -- '---\n%s: %s\n' "$VAULT_KEY" "$pw1" > "$plain_tmp"
ansible-vault encrypt \
--vault-password-file "$VAULT_PASS_FILE" \
--output "$VAULT_FILE" \
"$plain_tmp" 2>/dev/null \
|| { rm -f "$plain_tmp"; fail "ansible-vault encrypt failed — check vault password file."; }
rm -f "$plain_tmp"
log "Vault file recreated: $VAULT_FILE"
# Return the plaintext password for immediate use
printf '%s' "$pw1"
}
# ---------------------------------------------------------------------------
# Step 2: Update k8s DB secrets (knoe-cnpg-0, knoe-db-0)
# ---------------------------------------------------------------------------
update_db_secrets() {
local pw="$1"
log "Updating DB k8s secrets in ${DB_NS} on DB cluster..."
if ! kdb get namespace "$DB_NS" >/dev/null 2>&1; then
warn "Namespace ${DB_NS} not found on DB cluster — skipping DB secrets."
return
fi
run kdb create secret generic knoe-db-user \
--namespace="$DB_NS" \
--from-literal=username=prole \
--from-literal=password="$pw" \
--dry-run=client -o yaml | run kdb apply -f - >/dev/null
run kdb create secret generic knoe-db-superuser \
--namespace="$DB_NS" \
--from-literal=username=postgres \
--from-literal=password="$pw" \
--dry-run=client -o yaml | run kdb apply -f - >/dev/null
# knoe schema-owner credentials (PostGIS / extensions catalog)
run kdb create secret generic knoe-db-knoe \
--namespace="$DB_NS" \
--from-literal=username=knoe \
--from-literal=password="$pw" \
--dry-run=client -o yaml | run kdb apply -f - >/dev/null
[[ "$DRY_RUN" != "true" ]] && log " [DONE] DB k8s secrets updated (knoe-db-user, knoe-db-superuser, knoe-db-knoe)."
}
# ---------------------------------------------------------------------------
# Step 3: ALTER USER in PostgreSQL via CNPG primary pod
# ---------------------------------------------------------------------------
update_postgres_password() {
local pw="$1"
log "Rotating PostgreSQL user passwords via CNPG primary pod..."
if ! kdb get namespace "$DB_NS" >/dev/null 2>&1; then
warn "Namespace ${DB_NS} not found — skipping PostgreSQL ALTER USER."
return
fi
# Find primary pod (CNPG labels it with role=primary)
local primary_pod
primary_pod=$(kdb get pod -n "$DB_NS" \
-l "cnpg.io/cluster=knoe-db,cnpg.io/instanceRole=primary" \
--field-selector=status.phase=Running \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -z "$primary_pod" ]]; then
# Fallback: any running pod with cluster label
primary_pod=$(kdb get pod -n "$DB_NS" \
-l "cnpg.io/cluster=knoe-db" \
--field-selector=status.phase=Running \
--sort-by=.metadata.creationTimestamp \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
fi
if [[ -z "$primary_pod" ]]; then
warn "No running CNPG pod found in ${DB_NS} — skipping PostgreSQL ALTER USER."
return
fi
log " Primary pod: ${primary_pod}"
# Escape single quotes in password for SQL
local pw_sql="${pw//\'/\'\'}"
# Rotate all LOGIN roles that use the master password:
# prole — application user (k8s secret: knoe-db-user)
# postgres — superuser (k8s secret: knoe-db-superuser)
# knoe — schema owner of PostGIS/extensions (created in postInitSQL,
# password initially copied from prole but NOT auto-synced)
# authenticator — Supabase JWT auth role (LOGIN, no independent secret yet)
#
# Roles that are NOLOGIN (no rotation needed):
# knoe_catalog_executor, anon, guest (NOLOGIN), developer (NOLOGIN)
#
# Managed roles (admin, guest LOGIN) have passwords controlled by CNPG secrets —
# rotate separately if/when those secrets are created.
local sql="
ALTER USER prole WITH PASSWORD '${pw_sql}';
ALTER USER postgres WITH PASSWORD '${pw_sql}';
DO \$\$ BEGIN
IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'knoe') THEN
EXECUTE format('ALTER USER knoe WITH PASSWORD %L', '${pw_sql}');
END IF;
END \$\$;
DO \$\$ BEGIN
IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'authenticator') THEN
EXECUTE format('ALTER USER authenticator WITH PASSWORD %L', '${pw_sql}');
END IF;
END \$\$;
"
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY-RUN] Would exec psql ALTER USER prole/postgres/knoe/authenticator in ${primary_pod}"
else
echo "$sql" | kdb exec -i "$primary_pod" -n "$DB_NS" -c postgres -- \
env PGPASSWORD="$pw" psql -U postgres -d postgres \
|| warn "ALTER USER via pod exec failed — DB may need a pod restart to pick up new password."
log " [DONE] PostgreSQL users rotated (prole, postgres, knoe, authenticator)."
fi
}
# ---------------------------------------------------------------------------
# Step 4: Update Grafana k8s secret + CLI reset (knoe-dev-0, monitoring)
# ---------------------------------------------------------------------------
update_grafana() {
local pw="$1"
log "Updating Grafana on app cluster (namespace: ${MON_NS})..."
if ! kapp get namespace "$MON_NS" >/dev/null 2>&1; then
warn "Namespace ${MON_NS} not found on app cluster — skipping Grafana update."
return
fi
local pw_b64
pw_b64=$(b64enc "$pw")
# Update k8s secret — try both common secret names
local secret_name=""
for s in prometheus-grafana kps-grafana grafana; do
if kapp get secret "$s" -n "$MON_NS" >/dev/null 2>&1; then
secret_name="$s"
break
fi
done
if [[ -n "$secret_name" ]]; then
run kapp patch secret "$secret_name" -n "$MON_NS" --type merge \
-p "{\"data\":{\"admin-password\":\"${pw_b64}\",\"admin-user\":\"$(b64enc admin)\"}}" >/dev/null
[[ "$DRY_RUN" != "true" ]] && log " [DONE] Grafana k8s secret '${secret_name}' updated."
else
warn "No Grafana k8s secret found in ${MON_NS} — skipping secret patch."
fi
# CLI reset inside the running pod (most reliable — bypasses Grafana's secret cache)
local grafana_pod
grafana_pod=$(kapp get pod -n "$MON_NS" \
-l "app.kubernetes.io/name=grafana" \
--field-selector=status.phase=Running \
--sort-by=.metadata.creationTimestamp \
-o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
if [[ -z "$grafana_pod" ]]; then
warn "No running Grafana pod found — secret updated but CLI reset skipped. Grafana needs restart."
return
fi
log " Grafana pod: ${grafana_pod}"
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY-RUN] Would exec: grafana cli admin reset-admin-password in ${grafana_pod}"
echo " [DRY-RUN] Would rollout restart statefulset/prometheus-grafana in ${MON_NS}"
else
local attempts=5 i cli_ok=false
for ((i=1; i<=attempts; i++)); do
if kapp exec "$grafana_pod" -n "$MON_NS" -c grafana -- \
grafana cli admin reset-admin-password "$pw" 2>/dev/null; then
log " [DONE] Grafana CLI reset successful."
cli_ok=true
break
fi
# Try alternate binary name
if kapp exec "$grafana_pod" -n "$MON_NS" -c grafana -- \
grafana-cli admin reset-admin-password "$pw" 2>/dev/null; then
log " [DONE] Grafana CLI reset successful (grafana-cli)."
cli_ok=true
break
fi
log " Attempt ${i}/${attempts} failed — waiting 5s..."
sleep 5
done
[[ "$cli_ok" == "false" ]] && warn "Grafana CLI reset failed after ${attempts} attempts."
# Restart Grafana so the pod picks up GF_SECURITY_ADMIN_PASSWORD from the
# updated k8s secret. The CLI reset updates SQLite but the env var loaded at
# pod startup still holds the old value — a restart is required for consistency.
log " Restarting Grafana StatefulSet to apply new secret..."
if kapp rollout restart statefulset/prometheus-grafana -n "$MON_NS" 2>/dev/null; then
log " Waiting for Grafana rollout..."
kapp rollout status statefulset/prometheus-grafana -n "$MON_NS" --timeout=120s 2>/dev/null \
&& log " [DONE] Grafana restarted and ready." \
|| warn "Grafana rollout status check timed out — pod may still be starting."
else
# Fallback: delete the pod (works for both Deployment and StatefulSet)
kapp delete pod "$grafana_pod" -n "$MON_NS" --grace-period=10 2>/dev/null \
&& log " [DONE] Grafana pod deleted — will be recreated with new secret." \
|| warn "Could not restart Grafana — port-forward into the new pod once it's Running."
fi
fi
}
# ---------------------------------------------------------------------------
# Step 5: Update prole.cfg password fields
# ---------------------------------------------------------------------------
update_cfg_files() {
local pw="$1"
log "Updating password fields in prole.cfg files..."
# Keys to update with the plaintext password (pre-1Password era)
local password_keys=(
"init_password.db_password"
"init_password.db_password_confirm"
"GRAFANA_ADMIN_PASSWORD"
"DB_PASSWORD"
)
update_cfg() {
local cfg_file="$1"
[[ -f "$cfg_file" ]] || return 0
python3 - "$cfg_file" "$pw" "${password_keys[@]}" <<'PY'
import sys, re
cfg_path = sys.argv[1]
new_pw = sys.argv[2]
keys = set(sys.argv[3:])
pattern = re.compile(r'^(?P<lead>\s*)(?P<key>[^=\s][^=]*)(?P<mid>\s*=\s*)(?P<val>.*)$')
out = []
with open(cfg_path) as f:
for line in f:
line = line.rstrip('\n')
stripped = line.strip()
if not stripped or stripped.startswith(('#', ';')) or stripped.startswith('['):
out.append(line)
continue
m = pattern.match(line)
if m and m.group('key').strip() in keys:
out.append(f"{m.group('lead')}{m.group('key').strip()}{m.group('mid')}{new_pw}")
else:
out.append(line)
with open(cfg_path, 'w') as f:
f.write('\n'.join(out) + '\n')
PY
log " Updated: $cfg_file"
}
if [[ "$DRY_RUN" == "true" ]]; then
log " [DRY-RUN] Would update password fields in conf/gke.cfg and conf/service/prod.cfg"
else
update_cfg "${ROOT_DIR}/conf/gke.cfg"
update_cfg "${ROOT_DIR}/conf/service/prod.cfg"
fi
}
# ---------------------------------------------------------------------------
# Step 6: Verify
# ---------------------------------------------------------------------------
verify() {
local pw="$1"
if [[ "$DRY_RUN" == "true" ]]; then
log "--- Verification skipped (dry-run — no changes were applied) ---"
return 0
fi
log "--- Verification ---"
local ok=true
# DB secrets
if [[ "$SKIP_DB" != "true" ]]; then
for secret in knoe-db-user knoe-db-superuser knoe-db-knoe; do
if kdb get secret "$secret" -n "$DB_NS" >/dev/null 2>&1; then
local stored
stored=$(kdb get secret "$secret" -n "$DB_NS" \
-o jsonpath='{.data.password}' 2>/dev/null | b64dec 2>/dev/null || true)
if [[ "$stored" == "$pw" ]]; then
log " [PASS] DB secret ${DB_NS}/${secret}"
else
warn "[FAIL] DB secret ${DB_NS}/${secret} — value mismatch"
ok=false
fi
fi
done
fi
# Grafana secret
if [[ "$SKIP_GRAFANA" != "true" ]]; then
for s in prometheus-grafana kps-grafana grafana; do
if kapp get secret "$s" -n "$MON_NS" >/dev/null 2>&1; then
local stored
stored=$(kapp get secret "$s" -n "$MON_NS" \
-o jsonpath='{.data.admin-password}' 2>/dev/null | b64dec 2>/dev/null || true)
if [[ "$stored" == "$pw" ]]; then
log " [PASS] Grafana secret ${MON_NS}/${s}"
else
warn "[FAIL] Grafana secret ${MON_NS}/${s} — value mismatch"
ok=false
fi
break
fi
done
fi
if [[ "$ok" == "true" ]]; then
log "All secret verifications passed."
else
warn "Some verifications failed — review warnings above."
fi
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
log "=== prole update.sh — master password rotation ==="
[[ "$DRY_RUN" == "true" ]] && log "(DRY-RUN mode — no changes will be made)"
[[ "$PROMPT_MODE" == "true" ]] && log "(PROMPT mode — will recreate vault file)"
resolve_contexts
local MASTER_PW
if [[ "$PROMPT_MODE" == "true" ]]; then
MASTER_PW=$(prompt_and_recreate_vault)
log "Vault file recreated and master password ready."
else
MASTER_PW=$(decrypt_vault_password)
log "Vault master password decrypted successfully."
fi
if [[ "$SKIP_DB" != "true" ]]; then
update_db_secrets "$MASTER_PW"
update_postgres_password "$MASTER_PW"
fi
if [[ "$SKIP_GRAFANA" != "true" ]]; then
update_grafana "$MASTER_PW"
fi
if [[ "$SKIP_CFG" != "true" ]]; then
update_cfg_files "$MASTER_PW"
fi
verify "$MASTER_PW"
log "=== update.sh complete ==="
}
main "$@"