prole/infrastructure/roles/k3s/tasks/main.yml

---

- name: Skip k3s when state is absent
  ansible.builtin.meta: end_host
  when: k3s_state == "absent"

- name: Set k3s service name
  ansible.builtin.set_fact:
    k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}"

- name: Check if k3s service is installed
  ansible.builtin.stat:
    path: "/etc/systemd/system/{{ k3s_service_name }}.service"
  register: k3s_service

- name: Check if k3s binary exists
  ansible.builtin.stat:
    path: /usr/local/bin/k3s
  register: k3s_bin

- name: Determine if k3s needs installation
  ansible.builtin.set_fact:
    k3s_needs_install: "{{ not k3s_bin.stat.exists or not k3s_service.stat.exists }}"

- name: Gather service facts for health check
  ansible.builtin.service_facts:
  when: not k3s_needs_install

- name: Check k3s API responsiveness
  ansible.builtin.command: k3s kubectl get nodes
  register: k3s_api_check
  changed_when: false
  failed_when: false
  when:
    - not k3s_needs_install
    - (k3s_service_name + ".service") in ansible_facts.services
    - ansible_facts.services[k3s_service_name + ".service"].state == 'running'

- name: Force k3s repair if not responsive
  ansible.builtin.set_fact:
    k3s_needs_install: true
  when:
    - not k3s_needs_install
    - (k3s_service_name + ".service") in ansible_facts.services
    - ansible_facts.services[k3s_service_name + ".service"].state == 'running'
    - k3s_api_check.rc | default(0) != 0

- name: Install k3s dependencies
  ansible.builtin.package:
    name:
      - curl
      - iptables
    state: present

- name: Address Retropie networking (wifi power save)
  ansible.builtin.shell: |
    if [ -d /sys/class/net/wlan0 ]; then
       /sbin/iwconfig wlan0 power off || /usr/sbin/iw dev wlan0 set power_save off || true
    fi    
  when: inventory_hostname in ['retropie.prole.org', 'pi.prole.org']
  changed_when: false
  failed_when: false

- name: Configure connman to ignore k8s/cni interfaces
  ansible.builtin.shell: |
    if [ -f /etc/connman/main.conf ]; then
      if ! grep -q "^NetworkInterfaceBlacklist=" /etc/connman/main.conf; then
        echo "NetworkInterfaceBlacklist=veth,cni,flannel,docker,virbr" >> /etc/connman/main.conf
        echo "RESTART_CONNMAN"
      elif ! grep -q "cni" /etc/connman/main.conf; then
        # Ensure we have our required prefixes in the blacklist
        sed -i 's/^NetworkInterfaceBlacklist=/NetworkInterfaceBlacklist=veth,cni,flannel,docker,virbr,/' /etc/connman/main.conf
        # Clean up any potential double commas or wildcards we might have added before
        sed -i 's/\*//g' /etc/connman/main.conf
        sed -i 's/lo,//g' /etc/connman/main.conf
        sed -i 's/,,/,/g' /etc/connman/main.conf
        echo "RESTART_CONNMAN"
      fi
    fi    
  register: _connman_config
  when: inventory_hostname in ['retropie.prole.org', 'pi.prole.org']
  changed_when: "'RESTART_CONNMAN' in _connman_config.stdout"

- name: Restart connman to apply blacklist changes
  ansible.builtin.service:
    name: connman
    state: restarted
  async: 1
  poll: 0
  when:
    - inventory_hostname in ['retropie.prole.org', 'pi.prole.org']
    - _connman_config is changed
  ignore_errors: true

- name: Wait for connection to return after network restart
  ansible.builtin.wait_for_connection:
    delay: 5
    timeout: 300
  when:
    - inventory_hostname in ['retropie.prole.org', 'pi.prole.org']
    - _connman_config is changed

- name: Address Retropie routing (wlan0 priority)
  ansible.builtin.shell: |
    # If eth0 has a default route but we are using wlan0, it might block internet
    if [ "{{ k3s_flannel_iface | default('') }}" = "wlan0" ] || [ -d /sys/class/net/wlan0 ]; then
      if ip route show default dev eth0 | grep -q default; then
        if ip route show default dev wlan0 | grep -q default; then
           ip route del default dev eth0 || true
        fi
      fi
    fi    
  when: inventory_hostname in ['retropie.prole.org', 'pi.prole.org']
  changed_when: false
  failed_when: false

- name: Preflight - ensure rancher mountpoint is mounted
  ansible.builtin.command: "findmnt -n {{ k3s_rancher_mountpoint | default('/var/lib/rancher') }}"
  register: rancher_mounted
  changed_when: false
  failed_when: rancher_mounted.rc != 0

- name: Preflight - ensure cgroup kernel params are present
  ansible.builtin.command: "cat /proc/cmdline"
  register: k3s_proc_cmdline
  changed_when: false

- name: Fail if cgroup params are missing (reboot required)
  ansible.builtin.fail:
    msg: "Missing cgroup params in /proc/cmdline: {{ ['cgroup_memory=1', 'cgroup_enable=memory'] | reject('in', k3s_proc_cmdline.stdout) | list }}. Reboot required before k3s install."
  when: (['cgroup_memory=1', 'cgroup_enable=memory'] | reject('in', k3s_proc_cmdline.stdout) | list) | length > 0

- name: Guardrail - ensure rancher mountpoint is not on SD/rootfs
  ansible.builtin.command: "findmnt -n -o SOURCE {{ k3s_rancher_mountpoint | default('/var/lib/rancher') }}"
  register: rancher_source
  changed_when: false

- name: Fail if rancher mountpoint is on SD
  ansible.builtin.fail:
    msg: "{{ k3s_rancher_mountpoint | default('/var/lib/rancher') }} is on SD/rootfs ({{ rancher_source.stdout }}). Refusing to proceed."
  when: rancher_source.stdout is search("mmcblk0") or rancher_source.stdout is search("/dev/mmc")

- name: Preflight - ensure required mountpoints are mounted
  ansible.builtin.command: "findmnt -n {{ item }}"
  register: k3s_required_mounts_present
  changed_when: false
  failed_when: k3s_required_mounts_present.rc != 0
  loop: "{{ k3s_required_mounts | default([]) }}"
  when: (k3s_required_mounts | default([])) | length > 0

- name: Guardrail - ensure required mountpoints are not on SD
  ansible.builtin.command: "findmnt -n -o SOURCE {{ item }}"
  register: k3s_required_mounts_sources
  changed_when: false
  loop: "{{ k3s_required_mounts | default([]) }}"
  when: (k3s_required_mounts | default([])) | length > 0

- name: Fail if required mountpoint is on SD
  ansible.builtin.fail:
    msg: "{{ item.item }} is on SD/rootfs ({{ item.stdout }}). Refusing to proceed."
  loop: "{{ k3s_required_mounts_sources.results }}"
  when: item.stdout is search("mmcblk0") or item.stdout is search("/dev/mmc")

- name: Read k3s node token for sharing
  ansible.builtin.slurp:
    src: /var/lib/rancher/k3s/server/node-token
  register: k3s_node_token_slurp
  when: 
    - k3s_role == "server"
    - k3s_cluster_init | bool or k3s_server_url | default('') | length == 0
  ignore_errors: true

- name: Set k3s_token_discovered fact
  ansible.builtin.set_fact:
    k3s_token_discovered: "{{ k3s_node_token_slurp.content | b64decode | trim }}"
  when:
    - k3s_node_token_slurp is defined
    - k3s_node_token_slurp.content is defined

- name: Wait for k3s token from server
  ansible.builtin.command: /usr/bin/true
  until: hostvars[item].k3s_token_discovered | default('') | length > 0
  retries: 60
  delay: 10
  loop: "{{ groups['k3s_hosts'] }}"
  when:
    - k3s_role == 'agent'
    - hostvars[item].k3s_role | default('agent') == 'server'
    - (k3s_token | default('') | length == 0) or (item in ansible_play_hosts)
  changed_when: false

- name: Discover k3s token from other hosts
  ansible.builtin.set_fact:
    k3s_token: "{{ hostvars[item].k3s_token_discovered }}"
  loop: "{{ groups['k3s_hosts'] }}"
  when:
    - hostvars[item].k3s_token_discovered | default('') | length > 0
    - hostvars[item].k3s_role | default('agent') == 'server'
    - (k3s_token | default('') | length == 0) or (k3s_role == 'agent' and k3s_token != hostvars[item].k3s_token_discovered)

- name: Detect k3s CA hash mismatch on agents (logs)
  ansible.builtin.shell: |
    journalctl -u {{ k3s_service_name }} --since "1 hour ago" | grep "token CA hash does not match"    
  register: k3s_ca_mismatch_logs
  changed_when: false
  failed_when: false
  when:
    - k3s_role == 'agent'
    - k3s_service.stat.exists

- name: Proactively check for CA hash mismatch (files)
  block:
    - name: Get local agent CA cert hash (DER)
      ansible.builtin.shell: |
        if [ -f /var/lib/rancher/k3s/agent/server-ca.crt ]; then
          openssl x509 -in /var/lib/rancher/k3s/agent/server-ca.crt -outform DER | openssl dgst -sha256 | awk '{print $NF}'
        else
          echo "MISSING"
        fi        
      register: k3s_agent_ca_hash_cmd
      changed_when: false

    - name: Extract expected CA hash from token
      ansible.builtin.set_fact:
        k3s_expected_ca_hash: "{{ (k3s_token | regex_replace('^K10([a-f0-9]+)::.*$', '\\1')) if (k3s_token is search('^K10[a-f0-9]+::')) else '' }}"

    - name: Detect mismatch from files
      ansible.builtin.set_fact:
        k3s_ca_mismatch_files: "{{ k3s_agent_ca_hash_cmd.stdout | trim != k3s_expected_ca_hash }}"
      when:
        - k3s_agent_ca_hash_cmd.stdout | trim != "MISSING"
        - k3s_expected_ca_hash | length > 0
  when:
    - k3s_role == 'agent'
    - k3s_token | default('') | length > 0

- name: Handle k3s CA hash mismatch
  block:
    - name: Stop k3s-agent for CA repair
      ansible.builtin.shell: "systemctl stop {{ k3s_service_name }} || true"
      changed_when: false
      failed_when: false

    - name: Kill any remaining k3s processes
      ansible.builtin.shell: "killall -9 k3s || true"
      changed_when: false

    - name: Wipe stale agent state
      ansible.builtin.file:
        path: /var/lib/rancher/k3s/agent
        state: absent

    - name: Force k3s re-installation after CA wipe
      ansible.builtin.set_fact:
        k3s_needs_install: true
  when:
    - k3s_role == 'agent'
    - (k3s_ca_mismatch_logs.rc | default(1) == 0) or (k3s_ca_mismatch_files | default(false) | bool)

- name: Validate join configuration
  ansible.builtin.assert:
    that:
      - k3s_server_url is defined
      - k3s_server_url | length > 0
      - k3s_token is defined
      - k3s_token | length > 0
    fail_msg: "k3s_server_url and k3s_token are required for non-init join nodes."
  when: 
    - not k3s_cluster_init | bool
    - k3s_role == 'agent' or k3s_server_url | default('') | length > 0

- name: Check for existing k3s server node token
  ansible.builtin.stat:
    path: /var/lib/rancher/k3s/server/node-token
  register: k3s_server_node_token
  when:
    - k3s_guard_token_drift | bool
    - k3s_role == "server"
    - k3s_cluster_init | bool

- name: Read existing k3s server node token
  ansible.builtin.slurp:
    src: /var/lib/rancher/k3s/server/node-token
  register: k3s_server_node_token_raw
  when:
    - k3s_guard_token_drift | bool
    - k3s_role == "server"
    - k3s_cluster_init | bool
    - k3s_server_node_token.stat.exists

- name: Fail when vault k3s token is missing for existing cluster
  ansible.builtin.fail:
    msg: >-
      vault_k3s_token is empty but /var/lib/rancher/k3s/server/node-token exists.
      Refusing to overwrite k3s config. Update vault_k3s.yml or run the k3s sync playbook.      
  when:
    - k3s_guard_token_drift | bool
    - k3s_role == "server"
    - k3s_cluster_init | bool
    - k3s_server_node_token.stat.exists
    - (k3s_token | default('') | trim) | length == 0

- name: Fail on k3s token drift from live cluster
  ansible.builtin.fail:
    msg: >-
      vault_k3s_token does not match the live k3s server node-token.
      Refusing to overwrite k3s config. Run infrastructure/playbooks/k3s_sync.yml
      or update vault_k3s.yml to match the live token.      
  when:
    - k3s_guard_token_drift | bool
    - k3s_role == "server"
    - k3s_cluster_init | bool
    - k3s_server_node_token.stat.exists
    - (k3s_token | default('') | trim) | length > 0
    - (k3s_server_node_token_raw.content | b64decode | trim) != (k3s_token | default('') | trim)

- name: Ensure k3s config directory exists
  ansible.builtin.file:
    path: /etc/rancher/k3s
    state: directory
    mode: "0755"

- name: Load registry defaults from prole.cfg
  ansible.builtin.set_fact:
    k3s_prole_k3s_server_cfg: "{{ lookup('ansible.builtin.ini', 'PROLE_K3S_SERVER section=Global file=' + playbook_dir + '/../../conf/prole.cfg', default='') }}"
    k3s_prole_service_namespace_cfg: "{{ lookup('ansible.builtin.ini', 'SERVICE_NAMESPACE section=Global file=' + playbook_dir + '/../../conf/prole.cfg', default='') }}"
  when: k3s_state == "present" and k3s_registry_config_enabled | bool

- name: Resolve k3s registry host
  ansible.builtin.set_fact:
    k3s_registry_host_resolved: >-
      {{ (k3s_registry_host | default('')) if (k3s_registry_host | default('') | length > 0)
         else (k3s_prole_k3s_server_cfg | default('')) if (k3s_prole_k3s_server_cfg | default('') | length > 0)
         else (k3s_server_url | default('')) if (k3s_server_url | default('') | length > 0)
         else inventory_hostname }}      
  when: k3s_state == "present" and k3s_registry_config_enabled | bool

- name: Normalize k3s registry host
  ansible.builtin.set_fact:
    k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}"
  when: k3s_state == "present" and k3s_registry_config_enabled | bool

- name: Resolve k3s registry namespace
  ansible.builtin.set_fact:
    k3s_registry_namespace_resolved: >-
      {{ (k3s_registry_namespace | default('')) if (k3s_registry_namespace | default('') | length > 0 and k3s_registry_namespace != '${SERVICE_NAMESPACE}')
         else (k3s_prole_service_namespace_cfg | default('')) if (k3s_prole_service_namespace_cfg | default('') | length > 0 and k3s_prole_service_namespace_cfg != '${SERVICE_NAMESPACE}')
         else 'default' }}      
  when: k3s_state == "present" and k3s_registry_config_enabled | bool

- name: Validate k3s registry host
  ansible.builtin.assert:
    that:
      - k3s_registry_host_resolved | length > 0
    fail_msg: "k3s registry host could not be resolved (set k3s_registry_host or PROLE_K3S_SERVER)."
  when: k3s_state == "present" and k3s_registry_config_enabled | bool

- name: Render k3s registries config
  ansible.builtin.template:
    src: registries.yaml.j2
    dest: "{{ k3s_registry_config_path }}"
    mode: "0644"
  notify: Restart k3s
  when: k3s_state == "present" and k3s_registry_config_enabled | bool

- name: Ensure kubeconfig group exists
  ansible.builtin.group:
    name: "{{ k3s_kubeconfig_group }}"
    state: present
    local: "{{ (ansible_facts['os_family'] | default('')) in ['Darwin', 'FreeBSD', 'OpenBSD', 'NetBSD'] }}"

- name: Ensure kubeconfig users are in group (server)
  ansible.builtin.user:
    name: "{{ item }}"
    groups: "{{ k3s_kubeconfig_group }}"
    append: true
  loop: "{{ k3s_kubeconfig_users }}"
  when: k3s_role == "server" and (k3s_kubeconfig_users | length) > 0

- name: Ensure rancher directories are group-accessible
  ansible.builtin.file:
    path: "{{ item }}"
    state: directory
    owner: root
    group: "{{ k3s_kubeconfig_group }}"
    mode: "0750"
  loop:
    - /etc/rancher
    - /etc/rancher/k3s
    - /etc/rancher/k3s/config.yaml.d
    - "{{ k3s_rancher_mountpoint | default('/var/lib/rancher') }}"
  when: k3s_state != "absent"

- name: Detect active interface for k3s_node_ip (Retropie/Pi only)
  ansible.builtin.shell: |
    # Find interface that has the configured k3s_node_ip
    ip -4 addr show | grep -B2 "{{ k3s_node_ip }}" | grep -oE "^[0-9]+: [^:]+" | head -n 1 | awk '{print $2}'    
  register: _detected_k3s_iface
  when:
    - inventory_hostname in ['retropie.prole.org', 'pi.prole.org']
    - k3s_node_ip is defined and k3s_node_ip | length > 0
  changed_when: false

- name: Update k3s_flannel_iface based on detection (Retropie/Pi only)
  ansible.builtin.set_fact:
    k3s_flannel_iface: "{{ _detected_k3s_iface.stdout | trim }}"
  when:
    - inventory_hostname in ['retropie.prole.org', 'pi.prole.org']
    - _detected_k3s_iface.stdout | default('') | length > 0
    - _detected_k3s_iface.stdout | trim != k3s_flannel_iface

- name: Render k3s config
  ansible.builtin.template:
    src: config.yaml.j2
    dest: /etc/rancher/k3s/config.yaml
    owner: root
    group: "{{ k3s_kubeconfig_group }}"
    mode: "0640"
  notify: Restart k3s

- name: Build k3s install environment
  ansible.builtin.set_fact:
    k3s_install_env: >-
      {{ {'INSTALL_K3S_EXEC': k3s_role}
         | combine((k3s_version | length > 0) | ternary({'INSTALL_K3S_VERSION': k3s_version}, {})) }}      

- name: Ensure k3s service is stopped before install to prevent hangs
  ansible.builtin.service:
    name: "{{ k3s_service_name }}"
    state: stopped
  when:
    - k3s_needs_install
    - k3s_service.stat.exists
  failed_when: false

- name: Install k3s
  ansible.builtin.shell: |
    curl -sfL https://get.k3s.io | sh -    
  environment:
    "{{ k3s_install_env }}"
  when: k3s_needs_install

- name: Ensure k3s service state
  ansible.builtin.service:
    name: "{{ k3s_service_name }}"
    state: "{{ 'started' if k3s_state == 'present' else 'stopped' }}"
    enabled: "{{ k3s_state == 'present' }}"
  when: k3s_service.stat.exists or k3s_bin.stat.exists

- name: Apply node taints
  ansible.builtin.command: "k3s kubectl taint node {{ inventory_hostname }} {{ item }} --overwrite"
  delegate_to: "{{ (groups['k3s_hosts'] | map('extract', hostvars) | selectattr('k3s_role', 'defined') | selectattr('k3s_role', 'equalto', 'server') | map(attribute='inventory_hostname') | first) | default(groups['k3s_hosts'] | first) }}"
  loop: "{{ k3s_node_taints }}"
  when:
    - k3s_state == "present"
    - k3s_node_taints | length > 0
  register: _taint_result
  changed_when: "'modified' in _taint_result.stdout or 'tainted' in _taint_result.stdout"
  failed_when: false # Avoid failing if node is not yet ready

- name: Wait for k3s to be ready
  ansible.builtin.shell: k3s kubectl get nodes | grep -w "Ready"
  register: k3s_ready
  until: k3s_ready.rc == 0
  retries: 20
  delay: 5
  changed_when: false
  when: k3s_state == "present" and k3s_role == "server"

- name: Pause to allow k3s API to stabilize
  ansible.builtin.pause:
    seconds: 10
  when: k3s_state == "present" and k3s_role == "server" and k3s_ready.changed | default(true)

- name: Read k3s node token after install for sharing
  ansible.builtin.slurp:
    src: /var/lib/rancher/k3s/server/node-token
  register: k3s_node_token_slurp_post
  when: 
    - k3s_role == "server"
    - k3s_cluster_init | bool or k3s_server_url | default('') | length == 0
    - k3s_token_discovered is not defined
  ignore_errors: true

- name: Set k3s_token_discovered fact (post-install)
  ansible.builtin.set_fact:
    k3s_token_discovered: "{{ k3s_node_token_slurp_post.content | b64decode | trim }}"
  when:
    - k3s_node_token_slurp_post is defined
    - k3s_node_token_slurp_post.content is defined

- name: Sync k3s token to local vault
  ansible.builtin.include_tasks: sync_vault.yml
  when:
    - k3s_role == "server"
    - k3s_cluster_init | bool or k3s_server_url | default('') | length == 0
    - k3s_token_discovered is defined
    - k3s_token_discovered | length > 0

- name: Pre-stage k3s images (deliberate arch)
  ansible.builtin.shell: "k3s ctr -n k8s.io images pull --platform linux/arm64 {{ item }}"
  loop:
    - docker.io/library/registry:2
    - docker.io/prom/prometheus:v2.55.1
    - docker.io/prom/alertmanager:v0.27.0
    - quay.io/prometheus/node-exporter:v1.8.2
    - quay.io/prometheus-operator/prometheus-operator:v0.71.2
    - ghcr.io/openbao/openbao:2.0.0
    - ghcr.io/opentofu/opentofu:1.8.2
    - docker.io/library/kong:3.8
  register: _image_pull
  until: _image_pull.rc == 0
  retries: 10
  delay: 20
  when: k3s_state == "present"
  ignore_errors: true
  tags: [images]

- name: Create monitoring directory on pi.prole.org
  ansible.builtin.file:
    path: /var/lib/rancher/monitoring
    state: directory
    mode: "0777"
  when:
    - k3s_state == "present"
    - inventory_hostname == 'pi.prole.org'

- name: Apply pi-local-iscsi StorageClass
  ansible.builtin.shell: |
    k3s kubectl apply -f - <<'EOF'
    apiVersion: storage.k8s.io/v1
    kind: StorageClass
    metadata:
      name: pi-local-iscsi
    provisioner: rancher.io/local-path
    volumeBindingMode: WaitForFirstConsumer
    reclaimPolicy: Retain
    parameters:
      nodePathMap: |
        [
          {
            "node": "pi.prole.org",
            "paths": ["/var/lib/rancher/monitoring"]
          }
        ]
    EOF    
  when:
    - k3s_state == "present"
    - k3s_role == 'server'
  run_once: true

- name: Ensure CloudNative-PG operator and kubectl plugin
  ansible.builtin.include_tasks: cnpg.yml
  when: k3s_state == "present"