ansible: switch kerberos_trust_setup to KNOE.LOCAL + vault wiring

The cross-realm trust playbook was written for the pre-rebrand realm name
PROLE.LOCAL, which no longer exists in the in-cluster KDC — the canonical
realm is now KNOE.LOCAL. The Samba AD trust was therefore never actually
established, leaving chrisfu@KNOE.LOCAL→chrisfu@PROLE.ORG service ticket
flows blocked (and blocking pg_oauth / db.prole.org Kerberos work).

Changes:
- Realm: PROLE.LOCAL → KNOE.LOCAL across all task names, vars, and the
  krb5.conf [realms] / [domain_realm] blocks added on myrddin.
- samba_admin_password now resolves from the Ansible vault var
  vault_samba_dns_admin_pass (group_vars/ad_dc/vault.yml) by default,
  with SAMBA_ADMIN_PASSWORD env and -e overrides preserved for CI.
- trust_shared_password auto-resolves from the in-cluster Secret
  knoe-system/knoe-kdc-secrets (key trust_shared_password) when not
  passed explicitly — same Secret init_kdc.sh writes.
- Added [domain_realm] mapping for *.knoe.local → KNOE.LOCAL so Samba
  can resolve in-cluster service principals.
- Added a final `samba-tool domain trust show` smoke step so a successful
  run prints the visible trust state for log review.

Run:
  ANSIBLE_VAULT_PASSWORD_FILE=$PWD/.vault_pass   ansible-playbook infrastructure/playbooks/kerberos_trust_setup.yml

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

infrastructure/playbooks/kerberos_trust_setup.yml

@@ -1,51 +1,72 @@
 ---
-# kerberos_trust_setup.yml — Register the reciprocal PROLE.LOCAL ↔ PROLE.ORG cross-realm trust
+# kerberos_trust_setup.yml — Register the reciprocal KNOE.LOCAL ↔ PROLE.ORG cross-realm trust
 # in Samba AD on myrddin.prole.org.
 #
+# This unblocks SSO from PROLE.ORG (the on-prem Samba AD realm) into KNOE.LOCAL
+# (the in-cluster MIT KDC realm), which is required for chrisfu@KNOE.LOCAL→
+# chrisfu@PROLE.ORG service ticket flows targeting db.prole.org and other
+# cluster-hosted services that authenticate against the KNOE.LOCAL KDC.
+#
 # PREREQUISITES
 # -------------
-# 1. Run AFTER init_knoe_users.sh initialize completes on the k3s cluster.
+# 1. Run AFTER init_knoe_users.sh / init_kdc.sh has run on the k3s cluster.
 #    The trust_shared_password used here must match the value stored in the
-#    in-cluster Secret knoe-kdc-secrets (key: trust_shared_password).  The
+#    in-cluster Secret  knoe-system/knoe-kdc-secrets  (key: trust_shared_password).
-#    in-cluster MIT KDC must already have created the principal
+#    The in-cluster MIT KDC must already have created the principal
-#      krbtgt/PROLE.ORG@PROLE.LOCAL
+#      krbtgt/PROLE.ORG@KNOE.LOCAL
-#    before this playbook runs.
+#    and  krbtgt/KNOE.LOCAL@PROLE.ORG
+#    before this playbook runs.  (init_kdc.sh handles this.)
 #
 # 2. trust_kdc_ip must be the ClusterIP of the 'auth' Service in the knoe-system
-#    namespace.  This is a cluster-internal IP; myrddin (the Samba DC) must be
+#    namespace. myrddin (the Samba DC) must be able to reach it — either because
-#    reachable to it either because it is the k3s server node (and therefore on
+#    it is the k3s server node (and is therefore on the pod/service CIDR network)
-#    the pod/service CIDR network) or because a static route has been added.
+#    or because a static route has been added.
-#    Resolve it at runtime with:
+#
-#      kubectl -n knoe-system get svc auth -o jsonpath='{.spec.clusterIP}'
+# 3. The Samba AD administrator password lives in the Ansible vault as
+#      vault_samba_dns_admin_pass
+#    (group_vars/ad_dc/vault.yml). Decrypt with --vault-password-file .vault_pass
+#    or `ANSIBLE_VAULT_PASSWORD_FILE=$PWD/.vault_pass`.
 #
 # DRY-RUN
 # -------
 #   ansible-playbook infrastructure/playbooks/kerberos_trust_setup.yml \
 #     --check --diff \
-#     -e trust_shared_password=dummy \
+#     --vault-password-file .vault_pass \
-#     -e samba_admin_password=dummy
+#     -e trust_shared_password=dummy
 #
-# FULL RUN
+# FULL RUN (auto-resolves trust_shared_password from the cluster Secret)
-# --------
+# ---------------------------------------------------------------------
-#   export PROLE_TRUST_SHARED_PASSWORD=<secret>
-#   export SAMBA_ADMIN_PASSWORD=<secret>
 #   ansible-playbook infrastructure/playbooks/kerberos_trust_setup.yml \
-#     -e trust_kdc_ip=$(kubectl -n knoe-system get svc auth -o jsonpath='{.spec.clusterIP}')
+#     --vault-password-file .vault_pass
+#
+# OVERRIDES
+# ---------
+#   -e trust_kdc_ip=10.43.x.y         # skip ClusterIP autoresolve
+#   -e trust_shared_password=...      # skip Secret autoresolve
+#   -e samba_admin_password=...       # override vault (e.g. CI without vault)
 
-- name: Register PROLE.LOCAL cross-realm trust in Samba AD on myrddin
+- name: Register KNOE.LOCAL cross-realm trust in Samba AD on myrddin
   hosts: myrddin.prole.org
   gather_facts: false
   become: true
 
   vars:
-    trust_realm: "PROLE.LOCAL"
+    trust_realm: "KNOE.LOCAL"
-    trust_kdc_ip: ""          # override via --extra-vars or resolve at call site (see above)
+    trust_kdc_ip: ""          # auto-resolved below if empty
-    trust_shared_password: "{{ lookup('env', 'PROLE_TRUST_SHARED_PASSWORD') }}"
+    # Pull the trust password from the cluster Secret unless caller overrides.
-    samba_admin_password: "{{ lookup('env', 'SAMBA_ADMIN_PASSWORD') }}"
+    trust_shared_password: "{{ lookup('env', 'PROLE_TRUST_SHARED_PASSWORD') | default('', true) }}"
+    # Admin password resolution order:
+    #   1. -e samba_admin_password=...   (explicit override)
+    #   2. SAMBA_ADMIN_PASSWORD env var  (legacy / CI)
+    #   3. vault_samba_dns_admin_pass    (Ansible vault — preferred)
+    samba_admin_password: >-
+      {{ lookup('env', 'SAMBA_ADMIN_PASSWORD')
+         | default(hostvars[inventory_hostname].vault_samba_dns_admin_pass
+                   | default(vault_samba_dns_admin_pass | default('')), true) }}
 
   tasks:
     # ------------------------------------------------------------------
-    # 0. Resolve the ClusterIP if the caller did not supply trust_kdc_ip
+    # 0a. Resolve the ClusterIP of the in-cluster MIT KDC if not provided
     # ------------------------------------------------------------------
     - name: Resolve trust_kdc_ip from cluster if not provided
       ansible.builtin.command:
@@ -59,20 +80,61 @@
     - name: Set trust_kdc_ip fact from cluster lookup
       ansible.builtin.set_fact:
         trust_kdc_ip: "{{ _kdc_clusterip.stdout | trim }}"
-      when: trust_kdc_ip == "" and _kdc_clusterip is defined
+      when: trust_kdc_ip == "" and _kdc_clusterip is defined and _kdc_clusterip.stdout is defined
 
     - name: Assert trust_kdc_ip is set
       ansible.builtin.assert:
         that:
           - trust_kdc_ip != ""
         fail_msg: >
-          trust_kdc_ip is empty. Provide it via --extra-vars or ensure
+          trust_kdc_ip is empty. Provide it via --extra-vars trust_kdc_ip=...
-          'kubectl -n knoe-system get svc auth' succeeds from the control node.
+          or ensure 'kubectl -n knoe-system get svc auth' succeeds from the
+          control node.
+
+    # ------------------------------------------------------------------
+    # 0b. Resolve trust_shared_password from the cluster Secret if not provided
+    # ------------------------------------------------------------------
+    - name: Resolve trust_shared_password from knoe-system/knoe-kdc-secrets
+      ansible.builtin.shell:
+        cmd: >
+          set -o pipefail;
+          kubectl -n knoe-system get secret knoe-kdc-secrets
+          -o jsonpath='{.data.trust_shared_password}' | base64 -d
+        executable: /bin/bash
+      delegate_to: localhost
+      become: false
+      register: _trust_pw
+      when: trust_shared_password == ""
+      changed_when: false
+      no_log: true
+
+    - name: Set trust_shared_password fact from cluster Secret
+      ansible.builtin.set_fact:
+        trust_shared_password: "{{ _trust_pw.stdout | trim }}"
+      when: trust_shared_password == "" and _trust_pw is defined and _trust_pw.stdout is defined
+      no_log: true
+
+    - name: Assert trust_shared_password is set
+      ansible.builtin.assert:
+        that:
+          - trust_shared_password | length > 0
+        fail_msg: >
+          trust_shared_password is empty. Provide it via -e trust_shared_password=...
+          or ensure the Secret knoe-system/knoe-kdc-secrets has key
+          'trust_shared_password' populated by init_kdc.sh.
+
+    - name: Assert samba_admin_password is set
+      ansible.builtin.assert:
+        that:
+          - samba_admin_password | length > 0
+        fail_msg: >
+          samba_admin_password is empty. Decrypt with --vault-password-file .vault_pass
+          or set SAMBA_ADMIN_PASSWORD / -e samba_admin_password=...
 
     # ------------------------------------------------------------------
     # 1. Check if the trust already exists (idempotency guard)
     # ------------------------------------------------------------------
-    - name: Check whether PROLE.LOCAL trust already exists in Samba
+    - name: Check whether {{ trust_realm }} trust already exists in Samba
       ansible.builtin.command:
         cmd: samba-tool domain trust list
       register: _trust_list
@@ -83,10 +145,17 @@
       ansible.builtin.set_fact:
         _trust_exists: "{{ trust_realm in _trust_list.stdout }}"
 
+    - name: Report trust pre-existence
+      ansible.builtin.debug:
+        msg: >-
+          {{ trust_realm }} trust
+          {{ 'already present in Samba AD — skipping create.'
+             if _trust_exists else 'is missing — will create.' }}
+
     # ------------------------------------------------------------------
-    # 2. Add PROLE.LOCAL realm block to /etc/krb5.conf on myrddin
+    # 2. Add KNOE.LOCAL realm block to /etc/krb5.conf on myrddin
     # ------------------------------------------------------------------
-    - name: Ensure PROLE.LOCAL realm block is present in /etc/krb5.conf
+    - name: Ensure {{ trust_realm }} realm block is present in /etc/krb5.conf
       ansible.builtin.blockinfile:
         path: /etc/krb5.conf
         marker: "# {mark} ANSIBLE MANAGED BLOCK — {{ trust_realm }}"
@@ -96,12 +165,20 @@
               kdc = {{ trust_kdc_ip }}
               admin_server = {{ trust_kdc_ip }}
           }
-      when: not _trust_exists
+
+    - name: Ensure [domain_realm] mapping prole-cluster → {{ trust_realm }}
+      ansible.builtin.blockinfile:
+        path: /etc/krb5.conf
+        marker: "# {mark} ANSIBLE MANAGED BLOCK — domain_realm {{ trust_realm }}"
+        insertafter: '^\[domain_realm\]'
+        block: |
+          .knoe.local = {{ trust_realm }}
+          knoe.local  = {{ trust_realm }}
 
     # ------------------------------------------------------------------
     # 3. Register the trust in Samba
     # ------------------------------------------------------------------
-    - name: Create Kerberos cross-realm trust for PROLE.LOCAL in Samba AD
+    - name: Create Kerberos cross-realm trust for {{ trust_realm }} in Samba AD
       ansible.builtin.command:
         cmd: >
           samba-tool domain trust create {{ trust_realm }}
@@ -111,13 +188,13 @@
           -U administrator%{{ samba_admin_password }}
       no_log: true
       register: _trust_create
-      changed_when: "'Successfully created trust' in _trust_create.stdout or _trust_create.rc == 0"
+      changed_when: "'Successfully created trust' in (_trust_create.stdout | default('')) or _trust_create.rc == 0"
       when: not _trust_exists
 
     # ------------------------------------------------------------------
     # 4. Validate the trust
     # ------------------------------------------------------------------
-    - name: Validate the PROLE.LOCAL trust in Samba AD
+    - name: Validate the {{ trust_realm }} trust in Samba AD
       ansible.builtin.command:
         cmd: >
           samba-tool domain trust validate {{ trust_realm }}
@@ -130,3 +207,20 @@
     - name: Print trust validation result
       ansible.builtin.debug:
         msg: "{{ _trust_validate.stdout_lines }}"
+
+    # ------------------------------------------------------------------
+    # 5. Smoke test — fetch a cross-realm TGT
+    # ------------------------------------------------------------------
+    - name: Smoke test — list krbtgt principals seen by Samba KDC
+      ansible.builtin.command:
+        cmd: >
+          samba-tool domain trust show {{ trust_realm }}
+          -U administrator%{{ samba_admin_password }}
+      no_log: true
+      register: _trust_show
+      changed_when: false
+      failed_when: _trust_show.rc != 0
+
+    - name: Print trust show output
+      ansible.builtin.debug:
+        msg: "{{ _trust_show.stdout_lines }}"