chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 12:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
e44ff8e476
prole/infrastructure/roles/k3s/tasks/cleanup.yml
chrisfu 133b719479 Align CNPG bootstrap placement with Ansible policy 
Prefer infra-managed manifests for bootstrap; reconcile CNPG instances based on Ready+schedulable labeled db nodes; update manifests to use node-role affinity + anti-affinity; add policy/tests and config touch-ups (incl. prole.cfg).
2026-03-18 22:06:41 -07:00

315 lines
14 KiB
YAML
Raw Blame History

---
- name: Set effective k3s data-dir
ansible.builtin.set_fact:
k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}"
# k3s reset: cluster-side node removal (best-effort)
#
# Why this exists:
# - Prevents stale `NotReady` node objects after a node reset.
# - Prevents DaemonSet pods stuck `Terminating` (the scheduler still thinks the node exists).
# - Prevents lingering `kube-node-lease` entries which can cause subsequent install/join hangs.
#
# Important safety rules:
# - We run these `kubectl` operations on a healthy control-plane host via `delegate_to`.
# - We skip if there is no reachable control-plane host or if its kubeconfig is missing.
# - We do not attempt this when resetting the *last* control-plane node (there is no other
# healthy server to delegate to, and we don't want the reset to break the cluster).
- name: Build node-name candidates for cluster-side cleanup (prefer inventory_hostname)
ansible.builtin.set_fact:
k3s_reset_node_name_candidates: >-
{{ [inventory_hostname,
ansible_fqdn | default(''),
ansible_hostname | default('')]
| reject('equalto', '')
| unique
| list }}
k3s_reset_node_name: >-
{{ ([inventory_hostname,
ansible_fqdn | default(''),
ansible_hostname | default('')]
| reject('equalto', '')
| unique
| list)[0] }}
when: k3s_reset_force_cluster_node_delete | bool
- name: Pick control-plane delegate host for cluster-side cleanup (first available k3s server)
ansible.builtin.set_fact:
k3s_reset_control_plane_hosts: "{{ groups['k3s_servers'] | default([]) }}"
k3s_reset_is_last_control_plane: >-
{{ (inventory_hostname in (groups['k3s_servers'] | default([])))
and ((groups['k3s_servers'] | default([])) | length == 1) }}
k3s_reset_delegate_host: >-
{{ ((groups['k3s_servers'] | default([]))
| difference([inventory_hostname])
| first) | default('') }}
when: k3s_reset_force_cluster_node_delete | bool
- name: Log why cluster-side node removal is skipped (last control-plane)
ansible.builtin.debug:
msg: >-
Skipping cluster-side node deletion for {{ inventory_hostname }} because it is the last
control-plane host in inventory (no other server to delegate kubectl to).
when:
- k3s_reset_force_cluster_node_delete | bool
- k3s_reset_is_last_control_plane | bool
- name: Log why cluster-side node removal is skipped (no control-plane delegate host)
ansible.builtin.debug:
msg: >-
Skipping cluster-side node deletion for {{ inventory_hostname }} because no control-plane
host is available to delegate to (inventory group `k3s_servers` is empty, or only contains
this host).
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (k3s_reset_delegate_host | default('') | length) == 0
- name: Check whether delegated control-plane host is reachable
ansible.builtin.ping:
delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}"
register: _k3s_reset_delegate_ping
failed_when: false
changed_when: false
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (k3s_reset_delegate_host | default('') | length) > 0
- name: Log why cluster-side node removal is skipped (control-plane host unreachable)
ansible.builtin.debug:
msg: >-
Skipping cluster-side node deletion for {{ inventory_hostname }} because delegated
control-plane host '{{ k3s_reset_delegate_host }}' is unreachable.
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (k3s_reset_delegate_host | default('') | length) > 0
- _k3s_reset_delegate_ping is not defined or (_k3s_reset_delegate_ping.ping | default('')) != 'pong'
- name: Check for kubeconfig on delegated control-plane host
ansible.builtin.stat:
path: "{{ k3s_reset_kubeconfig_path }}"
delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}"
register: _k3s_reset_kubeconfig_stat
failed_when: false
changed_when: false
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- name: Log why cluster-side node removal is skipped (kubeconfig missing on delegate)
ansible.builtin.debug:
msg: >-
Skipping cluster-side node deletion for {{ inventory_hostname }} because kubeconfig
'{{ k3s_reset_kubeconfig_path }}' is missing on delegated control-plane host
'{{ k3s_reset_delegate_host }}'.
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- _k3s_reset_kubeconfig_stat is defined
- not (_k3s_reset_kubeconfig_stat.stat.exists | default(false) | bool)
- name: Fetch cluster node list (best-effort) to resolve the exact Kubernetes node name
ansible.builtin.command: >-
timeout 15s kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'
--request-timeout=5s
delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}"
environment:
KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}"
register: _k3s_reset_nodes_list
failed_when: false
changed_when: false
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool
- name: Resolve Kubernetes node name from cluster (fallback to inventory_hostname)
ansible.builtin.set_fact:
k3s_reset_node_name: >-
{{ (k3s_reset_node_name_candidates
| select('in', _k3s_reset_nodes_list.stdout_lines | default([]))
| list
| first)
| default(k3s_reset_node_name) }}
when:
- k3s_reset_force_cluster_node_delete | bool
- _k3s_reset_nodes_list is defined
- name: Log planned cluster-side cleanup target
ansible.builtin.debug:
msg: >-
Cluster-side cleanup enabled. Will attempt to delete node/lease '{{ k3s_reset_node_name }}'
via delegated control-plane host '{{ k3s_reset_delegate_host }}' (kubeconfig: {{ k3s_reset_kubeconfig_path }}).
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool
- name: Cluster-side cleanup - delete node object (best-effort)
ansible.builtin.command: >-
timeout 15s kubectl delete node {{ k3s_reset_node_name }} --ignore-not-found=true
--wait=false --request-timeout=5s
delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}"
environment:
KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}"
register: _k3s_reset_delete_node_1
failed_when: false
changed_when: "'deleted' in (_k3s_reset_delete_node_1.stdout | default('') | lower)"
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool
- name: Cluster-side cleanup - patch away node finalizers if deletion is stuck (best-effort)
ansible.builtin.command: >-
timeout 15s kubectl patch node {{ k3s_reset_node_name }} --type=merge
-p '{"metadata":{"finalizers":[]}}' --request-timeout=5s
delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}"
environment:
KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}"
register: _k3s_reset_patch_node_finalizers
failed_when: false
changed_when: "'patched' in (_k3s_reset_patch_node_finalizers.stdout | default('') | lower)"
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool
- name: Cluster-side cleanup - delete node object again after patch (best-effort)
ansible.builtin.command: >-
timeout 15s kubectl delete node {{ k3s_reset_node_name }} --ignore-not-found=true
--wait=false --request-timeout=5s
delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}"
environment:
KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}"
register: _k3s_reset_delete_node_2
failed_when: false
changed_when: "'deleted' in (_k3s_reset_delete_node_2.stdout | default('') | lower)"
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool
- name: Cluster-side cleanup - delete kube-node-lease for the node (best-effort)
ansible.builtin.command: >-
timeout 15s kubectl delete lease {{ k3s_reset_node_name }} -n kube-node-lease --ignore-not-found=true
--wait=false --request-timeout=5s
delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}"
environment:
KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}"
register: _k3s_reset_delete_lease
failed_when: false
changed_when: "'deleted' in (_k3s_reset_delete_lease.stdout | default('') | lower)"
when:
- k3s_reset_force_cluster_node_delete | bool
- not (k3s_reset_is_last_control_plane | default(false) | bool)
- (_k3s_reset_delegate_ping.ping | default('')) == 'pong'
- (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool
- name: Log cluster-side cleanup results (best-effort)
ansible.builtin.debug:
msg:
- "kubectl delete node (1): rc={{ _k3s_reset_delete_node_1.rc | default('n/a') }} stdout={{ _k3s_reset_delete_node_1.stdout | default('') | trim }} stderr={{ _k3s_reset_delete_node_1.stderr | default('') | trim }}"
- "kubectl patch node finalizers: rc={{ _k3s_reset_patch_node_finalizers.rc | default('n/a') }} stdout={{ _k3s_reset_patch_node_finalizers.stdout | default('') | trim }} stderr={{ _k3s_reset_patch_node_finalizers.stderr | default('') | trim }}"
- "kubectl delete node (2): rc={{ _k3s_reset_delete_node_2.rc | default('n/a') }} stdout={{ _k3s_reset_delete_node_2.stdout | default('') | trim }} stderr={{ _k3s_reset_delete_node_2.stderr | default('') | trim }}"
- "kubectl delete lease: rc={{ _k3s_reset_delete_lease.rc | default('n/a') }} stdout={{ _k3s_reset_delete_lease.stdout | default('') | trim }} stderr={{ _k3s_reset_delete_lease.stderr | default('') | trim }}"
when:
- k3s_reset_force_cluster_node_delete | bool
- _k3s_reset_delete_node_1 is defined or _k3s_reset_delete_lease is defined
- name: Best-effort purge kube-system workloads before uninstall
ansible.builtin.command: >-
timeout 15s kubectl -n kube-system delete all --all --ignore-not-found
--wait=false --request-timeout=5s
failed_when: false
changed_when: false
- name: Best-effort purge kube-system namespaces
ansible.builtin.command: >-
timeout 15s kubectl delete namespace kube-system --ignore-not-found
--wait=false --request-timeout=5s
failed_when: false
changed_when: false
- name: Force-remove kube-system finalizers if namespace is stuck
ansible.builtin.command: >-
timeout 15s kubectl patch namespace kube-system
-p '{"spec":{"finalizers":[]}}' --type=merge --request-timeout=5s
failed_when: false
changed_when: false
- name: Stop k3s services if present
ansible.builtin.service:
name: "{{ item }}"
state: stopped
enabled: false
loop:
- k3s
- k3s-agent
failed_when: false
- name: Check for k3s uninstall scripts
ansible.builtin.stat:
path: "{{ item }}"
loop:
- /usr/local/bin/k3s-uninstall.sh
- /usr/local/bin/k3s-agent-uninstall.sh
register: k3s_uninstall_scripts
- name: Run k3s uninstall scripts if present
ansible.builtin.command: "{{ item.stat.path }}"
loop: "{{ k3s_uninstall_scripts.results }}"
when: item.stat.exists
- name: Remove k3s systemd units and directories
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- /etc/rancher
- /etc/systemd/system/k3s.service
- /etc/systemd/system/k3s-agent.service
- /etc/systemd/system/k3s.service.d
- /etc/systemd/system/k3s-agent.service.d
- /etc/rancher/node
- /etc/rancher/k3s
- /etc/cni
- /opt/cni
- "{{ k3s_effective_data_dir }}"
- "{{ k3s_effective_data_dir }}/server/tls"
- "{{ k3s_effective_data_dir }}/server/cred"
- "{{ k3s_effective_data_dir }}/server/token"
- "{{ k3s_effective_data_dir }}/server/node-token"
- "{{ k3s_effective_data_dir }}/server/agent-token"
- "{{ k3s_effective_data_dir }}/server/db"
- "{{ k3s_effective_data_dir }}/agent"
- "{{ k3s_effective_data_dir }}/data"
- /var/lib/kubelet
- /var/lib/etcd
- /var/lib/cni
- /var/lib/containerd
- /run/k3s
- /run/flannel
- /var/log/k3s
- /usr/local/bin/k3s
- /usr/local/bin/k3s-killall.sh
- /usr/local/bin/k3s-uninstall.sh
- /usr/local/bin/k3s-agent-uninstall.sh
- /usr/local/bin/kubectl
- /usr/local/bin/crictl
- /usr/local/bin/ctr
- name: Reload systemd after k3s cleanup
ansible.builtin.systemd:
daemon_reload: true
Reference in New Issue View Git Blame Copy Permalink