feat(mariadb): add temporary external storage support for Merlin

- Enable mounting borrowed USB disk (exFAT) persistently at `/external`.
- Migrate MariaDB datadir to `/external/mariadb` with bind mount at `/srv/mariadb/mariadb`.
- Add migration marker and rollback support to prevent accidental remigration.
- Include Ansible tasks for external storage setup and migration validation.

docs/MERLIN-MARIADB-EXTERNAL-TEMP.md

@@ -0,0 +1,82 @@
+# Merlin: temporary MariaDB storage on borrowed USB (`/external`)
+
+This is a temporary, host-specific setup for `merlin.prole.org` to move MariaDB storage off the current path and onto a borrowed USB disk.
+
+## What changed
+
+- Ansible now mounts the borrowed USB disk partition (`/dev/sda1`, `exFAT`) persistently at `/external` **by filesystem UUID**.
+- MariaDB data is migrated from `/srv/mariadb/mariadb/` to `/external/mariadb/`.
+- After a successful copy, the original datadir is moved aside to `/srv/mariadb/mariadb.pre-external` (kept for rollback).
+- `/srv/mariadb/mariadb` is replaced with a **bind mount** of `/external/mariadb` (so MariaDB can keep using the same `datadir` path).
+- A marker file is written to avoid repeating the migration destructively: `/external/mariadb/.prole-mariadb-external-migrated`.
+
+## Why this is temporary
+
+- The disk is borrowed and formatted as `exFAT`.
+- Running MariaDB on `exFAT` is not ideal (permissions/ownership/ACLs/xattrs are limited), but this is an accepted short-term tradeoff until a better USB device is installed.
+
+## Verify the mount
+
+On `merlin`:
+
+```bash
+lsblk -f
+findmnt /external
+mountpoint -q /external && echo "/external is mounted"
+```
+
+You should see `/external` mounted from `UUID=...` with `FSTYPE=exfat`.
+
+## Verify MariaDB is using `/external/mariadb`
+
+1) Confirm MariaDB’s configured `datadir` path:
+
+```bash
+sudo mysql --protocol=socket --user=root \
+  --execute="SHOW VARIABLES LIKE 'datadir';"
+```
+
+Expected: `datadir` is `/srv/mariadb/mariadb/` (the traditional path).
+
+2) Confirm that path is backed by the external disk (bind mount):
+
+```bash
+findmnt -T /srv/mariadb/mariadb
+mountpoint -q /srv/mariadb/mariadb && echo "/srv/mariadb/mariadb is a mountpoint"
+```
+
+Expected: `/srv/mariadb/mariadb` is a bind mount whose source is `/external/mariadb`.
+
+## Rollback (move back to `/srv/mariadb/mariadb`)
+
+This rollback keeps the external disk untouched and restores the original datadir directory.
+
+1) Stop MariaDB:
+
+```bash
+sudo systemctl stop mariadb
+```
+
+2) Unmount the bind mount:
+
+```bash
+sudo umount /srv/mariadb/mariadb
+```
+
+3) Restore the original datadir directory:
+
+```bash
+sudo rm -rf /srv/mariadb/mariadb
+sudo mv /srv/mariadb/mariadb.pre-external /srv/mariadb/mariadb
+```
+
+4) Start MariaDB:
+
+```bash
+sudo systemctl start mariadb
+```
+
+5) Remove/disable the Ansible external-storage configuration:
+
+- In `infrastructure/inventory/host_vars/merlin.prole.org.yml`, set `mariadb_external_enabled: false` (or remove the `mariadb_external_*` vars).
+- Re-run the MariaDB provisioning playbook so it removes the persistent mount entries it previously created.

infrastructure/inventory/host_vars/merlin.prole.org.yml

@@ -10,6 +10,21 @@ netplan_static_nameservers:
 
 # MariaDB primary (USB-backed)
 
+# Temporary borrowed USB disk for MariaDB (merlin only)
+# - Disk: /dev/sda1 (exFAT)
+# - Mount: /external (by UUID, discovered at runtime)
+# - MariaDB datadir is expected at /srv/mariadb/mariadb, but is bind-mounted to /external/mariadb
+mariadb_external_enabled: true
+mariadb_external_device: /dev/sda1
+mariadb_external_mountpoint: /external
+mariadb_external_fstype: exfat
+mariadb_external_mount_opts: "defaults,nofail,x-systemd.device-timeout=10,uid=mysql,gid=mysql,umask=0077"
+
+mariadb_external_src_datadir: /srv/mariadb/mariadb
+mariadb_external_dst_datadir: /external/mariadb
+mariadb_external_backup_datadir: /srv/mariadb/mariadb.pre-external
+mariadb_external_migration_marker: /external/mariadb/.prole-mariadb-external-migrated
+
 # k3s worker (stateless workloads)
 k3s_enabled: true
 

infrastructure/roles/mariadb_primary/tasks/main.yml

@@ -8,6 +8,62 @@
       - rsync
     state: present
 
+- name: Temporary external storage for MariaDB (merlin)
+  block:
+    - name: Ensure exFAT support packages are installed (Debian/Ubuntu)
+      block:
+        - name: Install exfatprogs
+          ansible.builtin.package:
+            name: exfatprogs
+            state: present
+      rescue:
+        - name: Install legacy exFAT packages
+          ansible.builtin.package:
+            name:
+              - exfat-fuse
+              - exfat-utils
+            state: present
+      when: ansible_facts['os_family'] | default('') == 'Debian'
+
+    - name: Ensure external mountpoint directory exists
+      ansible.builtin.file:
+        path: "{{ mariadb_external_mountpoint | default('/external') }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Ensure external partition device exists
+      ansible.builtin.stat:
+        path: "{{ mariadb_external_device | default('/dev/sda1') }}"
+      register: _mariadb_external_device_stat
+
+    - name: Fail if external partition device is missing
+      ansible.builtin.assert:
+        that:
+          - _mariadb_external_device_stat.stat.exists
+        fail_msg: "External device {{ mariadb_external_device | default('/dev/sda1') }} does not exist on {{ inventory_hostname }}."
+
+    - name: Discover filesystem UUID for external partition
+      ansible.builtin.command: >-
+        blkid -s UUID -o value {{ mariadb_external_device | default('/dev/sda1') }}
+      register: _mariadb_external_uuid
+      changed_when: false
+      failed_when: _mariadb_external_uuid.rc != 0 or (_mariadb_external_uuid.stdout | trim | length) == 0
+
+    - name: Mount external filesystem at /external (persistent)
+      ansible.posix.mount:
+        src: "UUID={{ _mariadb_external_uuid.stdout | trim }}"
+        path: "{{ mariadb_external_mountpoint | default('/external') }}"
+        fstype: "{{ mariadb_external_fstype | default('exfat') }}"
+        opts: "{{ mariadb_external_mount_opts | default('defaults,nofail') }}"
+        state: mounted
+
+    - name: Fail if /external is not mounted
+      ansible.builtin.command: "mountpoint -q {{ mariadb_external_mountpoint | default('/external') }}"
+      changed_when: false
+  when: mariadb_external_enabled | default(false) | bool
+
 - name: Ensure USB mountpoint directory exists
   ansible.builtin.file:
     path: "{{ mariadb_usb_mountpoint | default('/srv/mariadb') }}"
@@ -15,6 +71,7 @@
     owner: root
     group: root
     mode: "0755"
+  when: not (mariadb_external_enabled | default(false) | bool)
 
 - name: Mount MariaDB USB filesystem
   ansible.builtin.mount:
@@ -23,6 +80,7 @@
     fstype: "{{ mariadb_usb_fstype | default('ext4') }}"
     opts: "{{ mariadb_usb_mount_opts | default('noatime,nofail') }}"
     state: mounted
+  when: not (mariadb_external_enabled | default(false) | bool)
 
 - name: Stop MariaDB before datadir changes
   ansible.builtin.service:
@@ -30,6 +88,102 @@
     state: stopped
   failed_when: false
 
+- name: Migrate MariaDB datadir to /external (merlin temporary setup)
+  vars:
+    mariadb_external_src: "{{ mariadb_external_src_datadir | default('/srv/mariadb/mariadb') }}"
+    mariadb_external_dst: "{{ mariadb_external_dst_datadir | default('/external/mariadb') }}"
+    mariadb_external_backup: "{{ mariadb_external_backup_datadir | default('/srv/mariadb/mariadb.pre-external') }}"
+    mariadb_external_marker: "{{ mariadb_external_migration_marker | default('/external/mariadb/.prole-mariadb-external-migrated') }}"
+  block:
+    - name: Fail if external mount is not present
+      ansible.builtin.command: "mountpoint -q {{ mariadb_external_mountpoint | default('/external') }}"
+      changed_when: false
+
+    - name: Ensure destination directory exists on external disk
+      ansible.builtin.file:
+        path: "{{ mariadb_external_dst }}"
+        state: directory
+
+    - name: Check migration marker
+      ansible.builtin.stat:
+        path: "{{ mariadb_external_marker }}"
+      register: _mariadb_external_marker_stat
+
+    - name: Fail if source datadir does not exist
+      ansible.builtin.stat:
+        path: "{{ mariadb_external_src }}"
+      register: _mariadb_external_src_stat
+
+    - name: Assert source datadir exists
+      ansible.builtin.assert:
+        that:
+          - _mariadb_external_src_stat.stat.exists
+          - _mariadb_external_src_stat.stat.isdir
+        fail_msg: "Source MariaDB datadir {{ mariadb_external_src }} is missing or not a directory on {{ inventory_hostname }}."
+
+    - name: Fail clearly if rsync is not available
+      ansible.builtin.command: rsync --version
+      register: _mariadb_external_rsync_version
+      changed_when: false
+
+    - name: Check whether source datadir is already a mountpoint (bind-mounted)
+      ansible.builtin.command: "mountpoint -q {{ mariadb_external_src }}"
+      register: _mariadb_external_src_is_mountpoint
+      changed_when: false
+      failed_when: false
+
+    - name: Rsync MariaDB data to external disk (copy; non-destructive)
+      ansible.builtin.command: >-
+        rsync -aH --numeric-ids {{ mariadb_external_src }}/ {{ mariadb_external_dst }}/
+      register: _mariadb_external_rsync
+      changed_when: true
+      when: not _mariadb_external_marker_stat.stat.exists
+
+    - name: Write migration marker
+      ansible.builtin.copy:
+        dest: "{{ mariadb_external_marker }}"
+        owner: root
+        group: root
+        mode: "0644"
+        content: |
+          migrated_from={{ mariadb_external_src }}
+          migrated_to={{ mariadb_external_dst }}
+          host={{ inventory_hostname }}
+      when: not _mariadb_external_marker_stat.stat.exists
+
+    - name: Check whether backup datadir already exists
+      ansible.builtin.stat:
+        path: "{{ mariadb_external_backup }}"
+      register: _mariadb_external_backup_stat
+
+    - name: Move original datadir aside (kept for rollback)
+      ansible.builtin.command: >-
+        mv {{ mariadb_external_src }} {{ mariadb_external_backup }}
+      when:
+        - _mariadb_external_marker_stat.stat.exists or (not _mariadb_external_marker_stat.stat.exists and (_mariadb_external_rsync is defined))
+        - _mariadb_external_src_is_mountpoint.rc != 0
+        - not _mariadb_external_backup_stat.stat.exists
+
+    - name: Re-create source datadir path as a mountpoint directory
+      ansible.builtin.file:
+        path: "{{ mariadb_external_src }}"
+        state: directory
+      when: _mariadb_external_src_is_mountpoint.rc != 0
+
+    - name: Bind-mount external MariaDB directory to the expected datadir path
+      ansible.posix.mount:
+        src: "{{ mariadb_external_dst }}"
+        path: "{{ mariadb_external_src }}"
+        fstype: none
+        opts: bind
+        state: mounted
+      when: _mariadb_external_src_is_mountpoint.rc != 0
+
+    - name: Fail if bind mount is not active
+      ansible.builtin.command: "mountpoint -q {{ mariadb_external_src }}"
+      changed_when: false
+  when: mariadb_external_enabled | default(false) | bool
+
 - name: Ensure MariaDB datadir exists on USB
   ansible.builtin.file:
     path: "{{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}"
@@ -37,6 +191,7 @@
     owner: mysql
     group: mysql
     mode: "0700"
+  when: not (mariadb_external_enabled | default(false) | bool)
 
 - name: Allow MariaDB datadir in AppArmor (Ubuntu/Debian)
   ansible.builtin.copy:
@@ -91,6 +246,7 @@
   when:
     - _mariadb_src_tables.stat.exists
     - not _mariadb_dst_tables.stat.exists
+    - not (mariadb_external_enabled | default(false) | bool)
   register: _mariadb_rsync
   changed_when: true
 
@@ -100,6 +256,7 @@
   when:
     - _mariadb_rsync is defined
     - _mariadb_rsync is changed
+    - not (mariadb_external_enabled | default(false) | bool)
   changed_when: true
 
 - name: Ensure MariaDB service is started and enabled