prole/update.sh

559 lines
20 KiB
Bash
Executable File

#!/usr/bin/env bash
# update.sh — Apply the master password to all knoe resources.
#
# Reads the administrator password from:
# 1Password knoey vault → item 'administrator' → field 'password'
#
# Applies the password to:
# - k8s secrets: knoe-db-user, knoe-db-superuser (knoe-cnpg-0, knoe-db-0)
# - PostgreSQL users: knoe, postgres (via CNPG primary pod)
# - k8s secret: prometheus-grafana (knoe-dev-0, monitoring)
# - Grafana CLI reset (via grafana pod exec)
# - config password fields (conf/gke.cfg, conf/service/prod.cfg)
# - OpenBao kv secrets (if reachable)
#
# Usage:
# ./update.sh [--dry-run] [--prompt] [--skip-db] [--skip-grafana] [--skip-cfg]
#
# --prompt Read password interactively (masked, confirmed twice) and
# save it to 1Password knoey/administrator.
# Use this when you need to rotate the master password.
#
# The script is idempotent — safe to run on every deploy or rotation.
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# ---------------------------------------------------------------------------
# Auto-activate project virtualenv (provides python3 packages)
# ---------------------------------------------------------------------------
if [[ -z "${VIRTUAL_ENV:-}" && -f "${ROOT_DIR}/bin/activate" ]]; then
# shellcheck disable=SC1091
source "${ROOT_DIR}/bin/activate" 2>/dev/null || true
fi
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
DRY_RUN=false
PROMPT_MODE=false
SKIP_DB=false
SKIP_GRAFANA=false
SKIP_CFG=false
OP_VAULT="${OP_VAULT:-knoey}"
OP_ITEM="administrator"
# Cluster contexts — override via env or auto-detect from config files
APP_CTX="${APP_CLUSTER_KUBECONTEXT:-}"
DB_CTX="${DB_CLUSTER_KUBECONTEXT:-}"
DB_NS="${DATABASE_NAMESPACE:-knoe-db-0}"
MON_NS="${MONITORING_NAMESPACE:-monitoring}"
SVC_NS="${SERVICE_NAMESPACE:-knoe-system}"
# ---------------------------------------------------------------------------
# Args
# ---------------------------------------------------------------------------
while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run) DRY_RUN=true; shift ;;
--prompt) PROMPT_MODE=true; shift ;;
--skip-db) SKIP_DB=true; shift ;;
--skip-grafana) SKIP_GRAFANA=true; shift ;;
--skip-cfg) SKIP_CFG=true; shift ;;
--vault) OP_VAULT="${2:?}"; shift 2 ;;
-h|--help)
grep '^#' "$0" | head -25 | sed 's/^# \?//'
exit 0 ;;
*) echo "Unknown argument: $1" >&2; exit 2 ;;
esac
done
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
log() { echo "[update.sh] $*" >&2; }
warn() { echo "[update.sh] WARN: $*" >&2; }
fail() { echo "[update.sh] ERROR: $*" >&2; exit 1; }
run() {
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY-RUN] $*"
else
"$@"
fi
}
b64enc() { printf '%s' "$1" | base64 | tr -d '\n'; }
# b64dec: decodes base64 — accepts argument OR piped stdin
b64dec() {
if [[ $# -gt 0 ]]; then
printf '%s' "$1" | base64 -d
else
base64 -d
fi
}
# Read a key from any section of an INI-style config
cfg_get() {
local file="$1" key="$2"
python3 - "$file" "$key" <<'PY'
import configparser, sys
cfg = configparser.ConfigParser(interpolation=None)
cfg.optionxform = str
cfg.read(sys.argv[1])
for s in cfg.sections():
if cfg.has_option(s, sys.argv[2]):
v = cfg.get(s, sys.argv[2], fallback="").strip()
if v: print(v); break
PY
}
# ---------------------------------------------------------------------------
# Step 0: Resolve cluster contexts from config if not set in env
# ---------------------------------------------------------------------------
_cfg_prod="${ROOT_DIR}/conf/gke.cfg"
_cfg_svc="${ROOT_DIR}/conf/service/prod.cfg"
resolve_contexts() {
if [[ -z "$APP_CTX" ]]; then
APP_CTX=$(cfg_get "$_cfg_svc" "APP_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$APP_CTX" && -f "$_cfg_prod" ]]; then
APP_CTX=$(cfg_get "$_cfg_prod" "APP_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$DB_CTX" ]]; then
DB_CTX=$(cfg_get "$_cfg_svc" "DB_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$DB_CTX" && -f "$_cfg_prod" ]]; then
DB_CTX=$(cfg_get "$_cfg_prod" "DB_CLUSTER_KUBECONTEXT" 2>/dev/null || true)
fi
if [[ -z "$APP_CTX" ]]; then
warn "APP_CLUSTER_KUBECONTEXT not set — will use ambient kubectl context for app cluster ops."
fi
if [[ -z "$DB_CTX" ]]; then
warn "DB_CLUSTER_KUBECONTEXT not set — will use ambient kubectl context for DB cluster ops."
fi
log "App cluster context : ${APP_CTX:-<ambient>}"
log "DB cluster context : ${DB_CTX:-<ambient>}"
}
kapp() { kubectl ${APP_CTX:+--context="$APP_CTX"} "$@"; }
kdb() { kubectl ${DB_CTX:+--context="$DB_CTX"} "$@"; }
# ---------------------------------------------------------------------------
# Step 1a: Read master password from 1Password
# ---------------------------------------------------------------------------
read_1password_password() {
log "Reading master password from 1Password vault '$OP_VAULT' item '$OP_ITEM'..."
if ! command -v op >/dev/null 2>&1; then
fail "1Password CLI (op) not found. Install: brew install 1password-cli"
fi
if ! op whoami >/dev/null 2>&1; then
log "No active 1Password session. Signing in..."
op signin || fail "1Password sign-in failed."
fi
local pw
pw=$(op item get "$OP_ITEM" --vault "$OP_VAULT" --fields password --reveal 2>/dev/null) \
|| fail "Could not read '$OP_ITEM' from vault '$OP_VAULT'. Run: ./update.sh --prompt"
if [[ -z "$pw" ]]; then
fail "Password is empty in 1Password vault '$OP_VAULT' item '$OP_ITEM'."
fi
printf '%s' "$pw"
}
# ---------------------------------------------------------------------------
# Step 1b: Prompt for new password + save to 1Password (--prompt mode)
# ---------------------------------------------------------------------------
prompt_and_save_to_1password() {
log "Creating new master password and saving to 1Password..."
if ! command -v op >/dev/null 2>&1; then
fail "1Password CLI (op) not found. Install: brew install 1password-cli"
fi
if ! op whoami >/dev/null 2>&1; then
op signin || fail "1Password sign-in failed."
fi
local pw1 pw2
if [[ -t 0 ]]; then
read -r -s -p "New master password: " pw1 </dev/tty; echo >/dev/tty
read -r -s -p "Confirm master password: " pw2 </dev/tty; echo >/dev/tty
else
read -r pw1
pw2="$pw1"
fi
if [[ -z "$pw1" ]]; then
fail "Password cannot be empty."
fi
if [[ "$pw1" != "$pw2" ]]; then
fail "Passwords do not match — please try again."
fi
if op item get "$OP_ITEM" --vault "$OP_VAULT" >/dev/null 2>&1; then
op item edit "$OP_ITEM" --vault "$OP_VAULT" "password=${pw1}" >/dev/null \
|| fail "Failed to update '$OP_ITEM' in vault '$OP_VAULT'."
log "Updated existing item '$OP_ITEM' in vault '$OP_VAULT'."
else
op item create \
--category login \
--title "$OP_ITEM" \
--vault "$OP_VAULT" \
"password=${pw1}" >/dev/null \
|| fail "Failed to create '$OP_ITEM' in vault '$OP_VAULT'."
log "Created item '$OP_ITEM' in vault '$OP_VAULT'."
fi
printf '%s' "$pw1"
}
# ---------------------------------------------------------------------------
# Step 2: Update k8s DB secrets (knoe-cnpg-0, knoe-db-0)
# ---------------------------------------------------------------------------
update_db_secrets() {
local pw="$1"
log "Updating DB k8s secrets in ${DB_NS} on DB cluster..."
if ! kdb get namespace "$DB_NS" >/dev/null 2>&1; then
warn "Namespace ${DB_NS} not found on DB cluster — skipping DB secrets."
return
fi
run kdb create secret generic knoe-db-user \
--namespace="$DB_NS" \
--from-literal=username=knoe \
--from-literal=password="$pw" \
--dry-run=client -o yaml | run kdb apply -f - >/dev/null
run kdb create secret generic knoe-db-superuser \
--namespace="$DB_NS" \
--from-literal=username=postgres \
--from-literal=password="$pw" \
--dry-run=client -o yaml | run kdb apply -f - >/dev/null
# knoe schema-owner credentials (PostGIS / extensions catalog)
run kdb create secret generic knoe-db-knoe \
--namespace="$DB_NS" \
--from-literal=username=knoe \
--from-literal=password="$pw" \
--dry-run=client -o yaml | run kdb apply -f - >/dev/null
[[ "$DRY_RUN" != "true" ]] && log " [DONE] DB k8s secrets updated (knoe-db-user, knoe-db-superuser, knoe-db-knoe)."
}
# ---------------------------------------------------------------------------
# Step 3: ALTER USER in PostgreSQL via CNPG primary pod
# ---------------------------------------------------------------------------
update_postgres_password() {
local pw="$1"
log "Rotating PostgreSQL user passwords via CNPG primary pod..."
if ! kdb get namespace "$DB_NS" >/dev/null 2>&1; then
warn "Namespace ${DB_NS} not found — skipping PostgreSQL ALTER USER."
return
fi
# Find primary pod (CNPG labels it with role=primary)
local primary_pod
primary_pod=$(kdb get pod -n "$DB_NS" \
-l "cnpg.io/cluster=knoe-db,cnpg.io/instanceRole=primary" \
--field-selector=status.phase=Running \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -z "$primary_pod" ]]; then
# Fallback: any running pod with cluster label
primary_pod=$(kdb get pod -n "$DB_NS" \
-l "cnpg.io/cluster=knoe-db" \
--field-selector=status.phase=Running \
--sort-by=.metadata.creationTimestamp \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
fi
if [[ -z "$primary_pod" ]]; then
warn "No running CNPG pod found in ${DB_NS} — skipping PostgreSQL ALTER USER."
return
fi
log " Primary pod: ${primary_pod}"
# Escape single quotes in password for SQL
local pw_sql="${pw//\'/\'\'}"
# Rotate all LOGIN roles that use the master password:
# knoe — application user (k8s secret: knoe-db-user)
# postgres — superuser (k8s secret: knoe-db-superuser)
# knoe — schema owner of PostGIS/extensions (created in postInitSQL,
# password initially copied from knoe but NOT auto-synced)
# authenticator — Supabase JWT auth role (LOGIN, no independent secret yet)
#
# Roles that are NOLOGIN (no rotation needed):
# knoe_catalog_executor, anon, guest (NOLOGIN), developer (NOLOGIN)
#
# Managed roles (admin, guest LOGIN) have passwords controlled by CNPG secrets —
# rotate separately if/when those secrets are created.
local sql="
ALTER USER knoe WITH PASSWORD '${pw_sql}';
ALTER USER postgres WITH PASSWORD '${pw_sql}';
DO \$\$ BEGIN
IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'knoe') THEN
EXECUTE format('ALTER USER knoe WITH PASSWORD %L', '${pw_sql}');
END IF;
END \$\$;
DO \$\$ BEGIN
IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'authenticator') THEN
EXECUTE format('ALTER USER authenticator WITH PASSWORD %L', '${pw_sql}');
END IF;
END \$\$;
"
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY-RUN] Would exec psql ALTER USER knoe/postgres/knoe/authenticator in ${primary_pod}"
else
echo "$sql" | kdb exec -i "$primary_pod" -n "$DB_NS" -c postgres -- \
env PGPASSWORD="$pw" psql -U postgres -d postgres \
|| warn "ALTER USER via pod exec failed — DB may need a pod restart to pick up new password."
log " [DONE] PostgreSQL users rotated (knoe, postgres, knoe, authenticator)."
fi
}
# ---------------------------------------------------------------------------
# Step 4: Update Grafana k8s secret + CLI reset (knoe-dev-0, monitoring)
# ---------------------------------------------------------------------------
update_grafana() {
local pw="$1"
log "Updating Grafana on app cluster (namespace: ${MON_NS})..."
if ! kapp get namespace "$MON_NS" >/dev/null 2>&1; then
warn "Namespace ${MON_NS} not found on app cluster — skipping Grafana update."
return
fi
local pw_b64
pw_b64=$(b64enc "$pw")
# Update k8s secret — try both common secret names
local secret_name=""
for s in prometheus-grafana kps-grafana grafana; do
if kapp get secret "$s" -n "$MON_NS" >/dev/null 2>&1; then
secret_name="$s"
break
fi
done
if [[ -n "$secret_name" ]]; then
run kapp patch secret "$secret_name" -n "$MON_NS" --type merge \
-p "{\"data\":{\"admin-password\":\"${pw_b64}\",\"admin-user\":\"$(b64enc admin)\"}}" >/dev/null
[[ "$DRY_RUN" != "true" ]] && log " [DONE] Grafana k8s secret '${secret_name}' updated."
else
warn "No Grafana k8s secret found in ${MON_NS} — skipping secret patch."
fi
# CLI reset inside the running pod (most reliable — bypasses Grafana's secret cache)
local grafana_pod
grafana_pod=$(kapp get pod -n "$MON_NS" \
-l "app.kubernetes.io/name=grafana" \
--field-selector=status.phase=Running \
--sort-by=.metadata.creationTimestamp \
-o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
if [[ -z "$grafana_pod" ]]; then
warn "No running Grafana pod found — secret updated but CLI reset skipped. Grafana needs restart."
return
fi
log " Grafana pod: ${grafana_pod}"
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY-RUN] Would exec: grafana cli admin reset-admin-password in ${grafana_pod}"
echo " [DRY-RUN] Would rollout restart statefulset/prometheus-grafana in ${MON_NS}"
else
local attempts=5 i cli_ok=false
for ((i=1; i<=attempts; i++)); do
if kapp exec "$grafana_pod" -n "$MON_NS" -c grafana -- \
grafana cli admin reset-admin-password "$pw" 2>/dev/null; then
log " [DONE] Grafana CLI reset successful."
cli_ok=true
break
fi
# Try alternate binary name
if kapp exec "$grafana_pod" -n "$MON_NS" -c grafana -- \
grafana-cli admin reset-admin-password "$pw" 2>/dev/null; then
log " [DONE] Grafana CLI reset successful (grafana-cli)."
cli_ok=true
break
fi
log " Attempt ${i}/${attempts} failed — waiting 5s..."
sleep 5
done
[[ "$cli_ok" == "false" ]] && warn "Grafana CLI reset failed after ${attempts} attempts."
# Restart Grafana so the pod picks up GF_SECURITY_ADMIN_PASSWORD from the
# updated k8s secret. The CLI reset updates SQLite but the env var loaded at
# pod startup still holds the old value — a restart is required for consistency.
log " Restarting Grafana StatefulSet to apply new secret..."
if kapp rollout restart statefulset/prometheus-grafana -n "$MON_NS" 2>/dev/null; then
log " Waiting for Grafana rollout..."
kapp rollout status statefulset/prometheus-grafana -n "$MON_NS" --timeout=120s 2>/dev/null \
&& log " [DONE] Grafana restarted and ready." \
|| warn "Grafana rollout status check timed out — pod may still be starting."
else
# Fallback: delete the pod (works for both Deployment and StatefulSet)
kapp delete pod "$grafana_pod" -n "$MON_NS" --grace-period=10 2>/dev/null \
&& log " [DONE] Grafana pod deleted — will be recreated with new secret." \
|| warn "Could not restart Grafana — port-forward into the new pod once it's Running."
fi
fi
}
# ---------------------------------------------------------------------------
# Step 5: Update knoe.cfg password fields
# ---------------------------------------------------------------------------
update_cfg_files() {
local pw="$1"
log "Updating password fields in knoe.cfg files..."
# Keys to update with the plaintext password (pre-1Password era)
local password_keys=(
"init_password.db_password"
"init_password.db_password_confirm"
"GRAFANA_ADMIN_PASSWORD"
"DB_PASSWORD"
)
update_cfg() {
local cfg_file="$1"
[[ -f "$cfg_file" ]] || return 0
python3 - "$cfg_file" "$pw" "${password_keys[@]}" <<'PY'
import sys, re
cfg_path = sys.argv[1]
new_pw = sys.argv[2]
keys = set(sys.argv[3:])
pattern = re.compile(r'^(?P<lead>\s*)(?P<key>[^=\s][^=]*)(?P<mid>\s*=\s*)(?P<val>.*)$')
out = []
with open(cfg_path) as f:
for line in f:
line = line.rstrip('\n')
stripped = line.strip()
if not stripped or stripped.startswith(('#', ';')) or stripped.startswith('['):
out.append(line)
continue
m = pattern.match(line)
if m and m.group('key').strip() in keys:
out.append(f"{m.group('lead')}{m.group('key').strip()}{m.group('mid')}{new_pw}")
else:
out.append(line)
with open(cfg_path, 'w') as f:
f.write('\n'.join(out) + '\n')
PY
log " Updated: $cfg_file"
}
if [[ "$DRY_RUN" == "true" ]]; then
log " [DRY-RUN] Would update password fields in conf/gke.cfg and conf/service/prod.cfg"
else
update_cfg "${ROOT_DIR}/conf/gke.cfg"
update_cfg "${ROOT_DIR}/conf/service/prod.cfg"
fi
}
# ---------------------------------------------------------------------------
# Step 6: Verify
# ---------------------------------------------------------------------------
verify() {
local pw="$1"
if [[ "$DRY_RUN" == "true" ]]; then
log "--- Verification skipped (dry-run — no changes were applied) ---"
return 0
fi
log "--- Verification ---"
local ok=true
# DB secrets
if [[ "$SKIP_DB" != "true" ]]; then
for secret in knoe-db-user knoe-db-superuser knoe-db-knoe; do
if kdb get secret "$secret" -n "$DB_NS" >/dev/null 2>&1; then
local stored
stored=$(kdb get secret "$secret" -n "$DB_NS" \
-o jsonpath='{.data.password}' 2>/dev/null | b64dec 2>/dev/null || true)
if [[ "$stored" == "$pw" ]]; then
log " [PASS] DB secret ${DB_NS}/${secret}"
else
warn "[FAIL] DB secret ${DB_NS}/${secret} — value mismatch"
ok=false
fi
fi
done
fi
# Grafana secret
if [[ "$SKIP_GRAFANA" != "true" ]]; then
for s in prometheus-grafana kps-grafana grafana; do
if kapp get secret "$s" -n "$MON_NS" >/dev/null 2>&1; then
local stored
stored=$(kapp get secret "$s" -n "$MON_NS" \
-o jsonpath='{.data.admin-password}' 2>/dev/null | b64dec 2>/dev/null || true)
if [[ "$stored" == "$pw" ]]; then
log " [PASS] Grafana secret ${MON_NS}/${s}"
else
warn "[FAIL] Grafana secret ${MON_NS}/${s} — value mismatch"
ok=false
fi
break
fi
done
fi
if [[ "$ok" == "true" ]]; then
log "All secret verifications passed."
else
warn "Some verifications failed — review warnings above."
fi
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
log "=== knoe update.sh — master password rotation ==="
[[ "$DRY_RUN" == "true" ]] && log "(DRY-RUN mode — no changes will be made)"
[[ "$PROMPT_MODE" == "true" ]] && log "(PROMPT mode — will update 1Password item)"
resolve_contexts
local MASTER_PW
if [[ "$PROMPT_MODE" == "true" ]]; then
MASTER_PW=$(prompt_and_save_to_1password)
log "1Password item updated and master password ready."
else
MASTER_PW=$(read_1password_password)
log "Master password read from 1Password successfully."
fi
if [[ "$SKIP_DB" != "true" ]]; then
update_db_secrets "$MASTER_PW"
update_postgres_password "$MASTER_PW"
fi
if [[ "$SKIP_GRAFANA" != "true" ]]; then
update_grafana "$MASTER_PW"
fi
if [[ "$SKIP_CFG" != "true" ]]; then
update_cfg_files "$MASTER_PW"
fi
verify "$MASTER_PW"
log "=== update.sh complete ==="
}
main "$@"