prole/infrastructure/playbooks/k3s_diagnose_repair.yml
chrisfu 5dcd3b9581 ansible: treat /var/lib/rancher as local K3s state
Update inventory/roles to keep K3s state on local storage and prevent iSCSI from managing /var/lib/rancher.

Also add a single-host k3s install playbook, a systemd override template, and docs describing the installer ↔ Ansible boundary and slow-storage knobs.

Co-authored-by: Junie <junie@jetbrains.com>
2026-03-15 23:20:55 -07:00

212 lines
7.8 KiB
YAML

---
- name: Prepare k3s repair metadata
hosts: k3s_hosts
gather_facts: false
run_once: true
tasks:
- name: Initialize k3s repair source host
ansible.builtin.set_fact:
k3s_repair_source_host: ""
delegate_to: localhost
delegate_facts: true
- name: Select k3s init server as repair source
ansible.builtin.set_fact:
k3s_repair_source_host: "{{ item }}"
loop: "{{ groups['k3s_hosts'] }}"
when: hostvars[item].k3s_cluster_init | default(false) | bool
delegate_to: localhost
delegate_facts: true
- name: Require a k3s init server as repair source
ansible.builtin.assert:
that:
- hostvars['localhost'].k3s_repair_source_host | length > 0
fail_msg: "No k3s init server found. Ensure a host has k3s_cluster_init: true."
- name: Ensure k3s node token exists on source
ansible.builtin.stat:
path: "{{ k3s_data_dir }}/server/node-token"
register: k3s_node_token
delegate_to: "{{ hostvars['localhost'].k3s_repair_source_host }}"
- name: Fail when k3s node token is missing
ansible.builtin.fail:
msg: "k3s node token not found at {{ k3s_data_dir }}/server/node-token"
when: not k3s_node_token.stat.exists
- name: Read k3s node token from source
ansible.builtin.slurp:
src: "{{ k3s_data_dir }}/server/node-token"
register: k3s_node_token_raw
delegate_to: "{{ hostvars['localhost'].k3s_repair_source_host }}"
- name: Read k3s server CA from source
ansible.builtin.slurp:
src: "{{ k3s_data_dir }}/server/tls/server-ca.crt"
register: k3s_server_ca_raw
delegate_to: "{{ hostvars['localhost'].k3s_repair_source_host }}"
- name: Store k3s repair metadata on controller
ansible.builtin.set_fact:
k3s_repair_token: "{{ k3s_node_token_raw.content | b64decode | trim }}"
k3s_repair_server_ca: "{{ k3s_server_ca_raw.content | b64decode }}"
k3s_repair_server_ca_sha: "{{ (k3s_server_ca_raw.content | b64decode) | hash('sha256') }}"
delegate_to: localhost
delegate_facts: true
- name: Update inventory vault token to match source
hosts: localhost
gather_facts: false
vars:
k3s_repair_token: "{{ hostvars['localhost'].k3s_repair_token | default('') }}"
vault_k3s_path: "{{ playbook_dir }}/../inventory/group_vars/all/vault_k3s.yml"
tasks:
- name: Require k3s repair token
ansible.builtin.assert:
that:
- k3s_repair_token | length > 0
fail_msg: "k3s repair token is empty. Cannot update vault_k3s.yml."
- name: Update vault k3s token
ansible.builtin.lineinfile:
path: "{{ vault_k3s_path }}"
regexp: '^vault_k3s_token:'
line: "vault_k3s_token: \"{{ k3s_repair_token }}\""
- name: Diagnose and repair k3s configuration
hosts: k3s_hosts
become: true
gather_facts: false
vars:
k3s_repair_source_host: "{{ hostvars['localhost'].k3s_repair_source_host }}"
k3s_repair_token: "{{ hostvars['localhost'].k3s_repair_token | default('') }}"
k3s_repair_server_ca: "{{ hostvars['localhost'].k3s_repair_server_ca | default('') }}"
k3s_repair_server_ca_sha: "{{ hostvars['localhost'].k3s_repair_server_ca_sha | default('') }}"
k3s_config_path: /etc/rancher/k3s/config.yaml
k3s_agent_ca_path: "{{ k3s_data_dir }}/agent/server-ca.crt"
k3s_server_ca_path: "{{ k3s_data_dir }}/server/tls/server-ca.crt"
pre_tasks:
- name: Require k3s repair metadata
ansible.builtin.assert:
that:
- k3s_repair_token | length > 0
- k3s_repair_server_ca | length > 0
fail_msg: "Missing k3s repair metadata from controller."
- name: Set k3s service name
ansible.builtin.set_fact:
k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}"
tasks:
- name: Ensure k3s config exists
ansible.builtin.stat:
path: "{{ k3s_config_path }}"
register: k3s_config
- name: Fail when k3s config is missing
ansible.builtin.fail:
msg: "k3s config is missing at {{ k3s_config_path }}. Run the k3s role first."
when: not k3s_config.stat.exists
- name: Read k3s config
ansible.builtin.slurp:
src: "{{ k3s_config_path }}"
register: k3s_config_raw
when: k3s_config.stat.exists
- name: Read k3s token from config
ansible.builtin.shell: |
awk -F': ' '/^token:/ {gsub(/"/, "", $2); print $2}' "{{ k3s_config_path }}"
register: k3s_config_token_cmd
changed_when: false
- name: Read k3s server URL from config
ansible.builtin.shell: |
awk -F': ' '/^server:/ {gsub(/"/, "", $2); print $2}' "{{ k3s_config_path }}"
register: k3s_config_server_cmd
changed_when: false
- name: Read k3s data-dir from config
ansible.builtin.shell: |
awk -F': ' '/^data-dir:/ {gsub(/"/, "", $2); print $2}' "{{ k3s_config_path }}"
register: k3s_config_data_dir_cmd
changed_when: false
- name: Store parsed k3s config values
ansible.builtin.set_fact:
k3s_config_token: "{{ k3s_config_token_cmd.stdout | trim }}"
k3s_config_server: "{{ k3s_config_server_cmd.stdout | trim }}"
k3s_config_data_dir: "{{ k3s_config_data_dir_cmd.stdout | trim }}"
- name: Read agent server CA when present
ansible.builtin.slurp:
src: "{{ k3s_agent_ca_path }}"
register: k3s_agent_ca_raw
when: k3s_role != 'server'
failed_when: false
- name: Compute agent CA checksum
ansible.builtin.set_fact:
k3s_agent_ca_sha: >-
{{ (k3s_agent_ca_raw.content | default('') | b64decode | hash('sha256')) if (k3s_agent_ca_raw.content is defined) else '' }}
when: k3s_role != 'server'
- name: Report k3s config state
ansible.builtin.debug:
msg:
host: "{{ inventory_hostname }}"
role: "{{ k3s_role }}"
service: "{{ k3s_service_name }}"
config_token: "{{ k3s_config_token | default('') }}"
token_matches_source: "{{ (k3s_config_token | default('')) == k3s_repair_token }}"
config_server: "{{ k3s_config_server | default('') }}"
server_matches_inventory: "{{ (k3s_role == 'server') or ((k3s_config_server | default('')) == (k3s_server_url | default(''))) }}"
config_data_dir: "{{ k3s_config_data_dir | default('') }}"
data_dir_matches_inventory: "{{ (k3s_config_data_dir | default('')) == (k3s_data_dir | default('')) }}"
agent_ca_matches_source: "{{ (k3s_role == 'server') or ((k3s_agent_ca_sha | default('')) == k3s_repair_server_ca_sha) }}"
- name: Sync k3s token into config
ansible.builtin.lineinfile:
path: "{{ k3s_config_path }}"
regexp: '^token:'
line: "token: \"{{ k3s_repair_token }}\""
mode: "0640"
notify: Restart k3s service
- name: Sync k3s server URL into config for non-init nodes
ansible.builtin.lineinfile:
path: "{{ k3s_config_path }}"
regexp: '^server:'
line: "server: \"{{ k3s_server_url }}\""
mode: "0640"
when:
- not (k3s_cluster_init | default(false) | bool)
- k3s_server_url | default('') | length > 0
notify: Restart k3s service
- name: Ensure agent cert directory exists
ansible.builtin.file:
path: "{{ k3s_data_dir }}/agent"
state: directory
mode: "0755"
when: k3s_role != 'server'
- name: Sync server CA to agent
ansible.builtin.copy:
content: "{{ k3s_repair_server_ca }}"
dest: "{{ k3s_agent_ca_path }}"
owner: root
group: root
mode: "0644"
when: k3s_role != 'server'
notify: Restart k3s service
handlers:
- name: Restart k3s service
ansible.builtin.service:
name: "{{ k3s_service_name }}"
state: restarted
- import_playbook: k3s_fetch_kubeconfig.yml