diff --git a/Executing b/Executing new file mode 100644 index 0000000..e69de29 diff --git a/ansible_recovery.log b/ansible_recovery.log new file mode 100644 index 0000000..e071695 --- /dev/null +++ b/ansible_recovery.log @@ -0,0 +1,48 @@ +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/Users/chrisfu/dev/prole/.ansible/logs/ansible/ansible-20260216-053756.log +CMD: ansible-playbook infrastructure/playbooks/site.yml --limit k3s_hosts --vault-password-file .vault_pass + +PLAY [iSCSI hosts and filesystem mounts] *************************************** + +TASK [Gathering Facts] ********************************************************* +ok: [retropie.prole.org] +ok: [myrddin.prole.org] +ok: [pi.prole.org] + +TASK [iscsi : Install iSCSI dependencies] ************************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [iscsi : Ensure iscsid enabled] ******************************************* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [iscsi : Configure and login iSCSI targets] ******************************* +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for pi.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-14.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/lib/rancher', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=466eefa2-9a7c-4bac-a136-d1d3ec35dae1'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for pi.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-19.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/log/pihole', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=56b21ec3-2826-4171-b909-a6715223f9a4'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for retropie.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-16.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/lib/rancher', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=73a886bb-3f01-4e8a-9c6d-ab431c671824'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-11.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/d001', 'fstype': 'xfs', 'opts': '_netdev,noatime', 'src': 'UUID=c07dc0f2-cf60-4da5-b68e-ea45e86d473e'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-12.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/d002', 'fstype': 'xfs', 'opts': '_netdev,noatime', 'src': 'UUID=fe087fb0-a321-4767-b838-b4385e81693e'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-13.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/d003', 'fstype': 'xfs', 'opts': '_netdev,noatime', 'src': 'UUID=757f1ee4-dc23-414b-b595-e3058c0744f0'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-17.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/home', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=607718c8-467a-474f-8e36-80ceafe1beb4'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-1.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/logs', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=5335affd-b1ab-4134-a1c0-be88ab965309'}]}) +included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-15.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/lib/rancher', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=c8320979-d3eb-4b40-93e0-ee452a4b9780'}]}) + +TASK [iscsi : Ensure node record exists for target] **************************** +ok: [pi.prole.org] + +TASK [iscsi : Create node record if missing] *********************************** +skipping: [pi.prole.org] + +TASK [iscsi : Configure CHAP authmethod] *************************************** +changed: [pi.prole.org] + +TASK [iscsi : Configure CHAP username] ***************************************** +changed: [pi.prole.org] + +TASK [iscsi : Configure CHAP password] ***************************************** +changed: [pi.prole.org] + +TASK [iscsi : Check if target session is already logged in] ******************** diff --git a/deploy/opentofu/k3s/argocd/application-openbao.yaml b/deploy/opentofu/k3s/argocd/application-openbao.yaml new file mode 100644 index 0000000..f754c31 --- /dev/null +++ b/deploy/opentofu/k3s/argocd/application-openbao.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: prole-openbao + namespace: argocd +spec: + project: default + source: + repoURL: git@github.com:dredx/prole.git + targetRevision: main + path: deploy/opentofu/k3s/manifests/openbao + destination: + server: https://myrddin.prole.org:6443 + namespace: prole-db-a0001 + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/deploy/opentofu/k3s/argocd/application-opentofu.yaml b/deploy/opentofu/k3s/argocd/application-opentofu.yaml new file mode 100644 index 0000000..add7a28 --- /dev/null +++ b/deploy/opentofu/k3s/argocd/application-opentofu.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: prole-opentofu + namespace: argocd +spec: + project: default + source: + repoURL: git@github.com:dredx/prole.git + targetRevision: main + path: deploy/opentofu/k3s/manifests/opentofu + destination: + server: https://myrddin.prole.org:6443 + namespace: prole-db-a0001 + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/deploy/opentofu/k3s/argocd/application-prole.yaml b/deploy/opentofu/k3s/argocd/application-prole.yaml new file mode 100644 index 0000000..bc288b2 --- /dev/null +++ b/deploy/opentofu/k3s/argocd/application-prole.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: prole-prole + namespace: argocd +spec: + project: default + source: + repoURL: git@github.com:dredx/prole.git + targetRevision: main + path: deploy/opentofu/k3s/manifests/prole + destination: + server: https://myrddin.prole.org:6443 + namespace: prole-db-a0001 + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml b/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml index fb94494..1caad77 100644 --- a/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml +++ b/deploy/opentofu/k3s/manifests/prole/kong-deployment.yaml @@ -31,10 +31,10 @@ spec: - name: KONG_ADMIN_LISTEN value: "0.0.0.0:8001" - name: KONG_DECLARATIVE_CONFIG - value: "/etc/kong/kong.yml" + value: "/etc/kong/declarative/kong.yml" volumeMounts: - name: kong-config - mountPath: /etc/kong + mountPath: /etc/kong/declarative readinessProbe: httpGet: path: /status @@ -50,10 +50,10 @@ spec: resources: requests: cpu: 100m - memory: 128Mi + memory: 256Mi limits: cpu: 500m - memory: 512Mi + memory: 1Gi volumes: - name: kong-config configMap: diff --git a/deploy/opentofu/prod/README.md b/deploy/opentofu/prod/README.md new file mode 100644 index 0000000..8690283 --- /dev/null +++ b/deploy/opentofu/prod/README.md @@ -0,0 +1,13 @@ +# OpenTofu Production Pipeline + +This pipeline deploys Prole into a production cluster (AWS EKS / GCloud GKE). + +## Usage +1. Copy manifests from the k3s pipeline into `manifests/`. +2. Set `cluster_endpoint`, `cluster_token`, and optionally `cluster_ca_cert`. +3. Run: +```bash +tofu init +tofu plan +tofu apply +``` diff --git a/deploy/opentofu/prod/main.tf b/deploy/opentofu/prod/main.tf new file mode 100644 index 0000000..f96d848 --- /dev/null +++ b/deploy/opentofu/prod/main.tf @@ -0,0 +1,57 @@ +terraform { + required_version = ">= 1.6.0" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.30" + } + } +} + +# Configure for AWS EKS or GCloud GKE via variables +provider "kubernetes" { + host = var.cluster_endpoint + token = var.cluster_token + cluster_ca_certificate = var.cluster_ca_cert != "" ? base64decode(var.cluster_ca_cert) : null + insecure = var.cluster_ca_cert == "" +} + +locals { + manifest_dir = "${path.module}/manifests" + manifest_files = fileset(local.manifest_dir, "**/*.yaml") + raw_documents = flatten([ + for f in local.manifest_files : [ + for doc in split("\n---", trimspace(file("${local.manifest_dir}/${f}"))) : + trimspace(doc) + if trimspace(doc) != "" + ] + ]) + decoded_documents = [ + for doc in local.raw_documents : yamldecode(doc) + if try(yamldecode(doc).kind, "") != "" + ] + cluster_scoped_kinds = toset([ + "Namespace", "CustomResourceDefinition", "ClusterRole", + "ClusterRoleBinding", "PersistentVolume", "StorageClass", + ]) + namespaced_documents = [ + for m in local.decoded_documents : + contains(local.cluster_scoped_kinds, m.kind) ? m : merge( + m, { metadata = merge(lookup(m, "metadata", {}), { namespace = var.namespace }) } + ) + ] +} + +resource "kubernetes_manifest" "namespace" { + manifest = { + apiVersion = "v1" + kind = "Namespace" + metadata = { name = var.namespace } + } +} + +resource "kubernetes_manifest" "resources" { + for_each = { for idx, m in local.namespaced_documents : tostring(idx) => m } + manifest = each.value + depends_on = [kubernetes_manifest.namespace] +} diff --git a/deploy/opentofu/prod/manifests/.gitkeep b/deploy/opentofu/prod/manifests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deploy/opentofu/prod/variables.tf b/deploy/opentofu/prod/variables.tf new file mode 100644 index 0000000..da33a90 --- /dev/null +++ b/deploy/opentofu/prod/variables.tf @@ -0,0 +1,22 @@ +variable "cluster_endpoint" { + type = string + description = "Production cluster API endpoint (EKS/GKE)" +} + +variable "cluster_token" { + type = string + description = "Production cluster auth token" + sensitive = true +} + +variable "cluster_ca_cert" { + type = string + description = "Base64-encoded CA certificate (leave empty for insecure)" + default = "" +} + +variable "namespace" { + type = string + description = "Target namespace for Prole resources" + default = "default" +} diff --git a/end_time.txt b/end_time.txt new file mode 100644 index 0000000..d9994a0 --- /dev/null +++ b/end_time.txt @@ -0,0 +1 @@ +Mon Feb 16 04:29:00 PST 2026 diff --git a/install.log b/install.log new file mode 100644 index 0000000..089cdaf --- /dev/null +++ b/install.log @@ -0,0 +1,777 @@ +[INFO] Using Ansible Vault password file: /Users/chrisfu/dev/prole/.vault_pass +[CONFIG] Using conf/prole.cfg +==> Dependencies +[OK] Homebrew Homebrew 5.0.14 +[OK] python Python 3.14.2 +[OK] ansible ansible [core 2.20.2] +[OK] kubectl Client Version: v1.35.0 +[OK] kubectl-cnpg Build: {Version:1.28.0 Commit:a9696201f760013182c6cdba7c4ed3c236a6423b Date:2025-12-09T17:09:28Z} +[OK] Docker Docker version 29.2.0, build 0b9d1985db +[OK] k3d k3d version v5.8.3 +[OK] Network scan already performed. KDC detected: 10.0.0.3 +==> Environment setup +[OK] /Users/chrisfu/.prole/env.sh is up to date. +==> Database creation +[RESET] OpenBao anchor detected without token; generated new DB password for initialization. +[OK] /Users/chrisfu/.prole/env.sh is up to date. +[SKIP] Local registry pre-pull only supported for k3d. +==> Environment Preparation (Secure Access) +[SKIP] Secure access keys already exist at /Users/chrisfu/.ssh/id_prole_ed25519 +[INFO] Deferring OpenBao initialization until cluster is ready. +==> Cluster setup +[RESET] Resetting k3s cluster (mode k3s)... +Running: ./ansible.sh -p infrastructure/playbooks/k3s_delete.yml -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-022843.log +CMD: ansible-playbook infrastructure/playbooks/k3s_delete.yml --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Delete k3s nodes and remove cluster token from vault] ******************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +changed: [myrddin.prole.org] => (item=k3s) +ok: [retropie.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) +ok: [pi.prole.org] => (item=k3s) +changed: [retropie.prole.org] => (item=k3s-agent) +changed: [pi.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2011, 'inode': 305262, 'dev': 45826, 'nlink': 1, 'atime': 1771236200.9584086, 'mtime': 1771236200.9584086, 'ctime': 1771236200.9584086, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'c9554235f592bf87e98ffcd75871b6d749ea2729', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '860247187', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 4103, 'dev': 45826, 'nlink': 1, 'atime': 1771236988.7820203, 'mtime': 1771236988.7820203, 'ctime': 1771236988.7920203, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '2537109730', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 16764, 'dev': 45826, 'nlink': 1, 'atime': 1771236404.6359234, 'mtime': 1771236404.6519232, 'ctime': 1771236404.659923, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '1795183155', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) + +TASK [k3s : Remove k3s systemd units and directories] ************************** +changed: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +changed: [retropie.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +changed: [pi.prole.org] => (item=/etc/rancher) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [retropie.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [retropie.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [retropie.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) +ok: [pi.prole.org] => (item=/etc/rancher/node) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [pi.prole.org] => (item=/etc/cni) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [pi.prole.org] => (item=/opt/cni) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [retropie.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/etcd) +ok: [retropie.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [retropie.prole.org] => (item=/var/lib/containerd) +ok: [retropie.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [retropie.prole.org] => (item=/run/flannel) +ok: [retropie.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [retropie.prole.org] => (item=/usr/local/bin/kubectl) +ok: [retropie.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/usr/local/bin/ctr) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/etcd) +ok: [pi.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +PLAY [Remove k3s token from Ansible vault] ************************************* + +TASK [Require vault password file for token deletion] ************************** +ok: [localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure vault file exists] ************************************************ +ok: [localhost] + +TASK [Fail when vault file is missing] ***************************************** +skipping: [localhost] + +TASK [Read vault_k3s.yml header] *********************************************** +ok: [localhost] + +TASK [Mark vault_k3s.yml encryption state] ************************************* +ok: [localhost] + +TASK [Decrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +TASK [Remove vault_k3s_token entry] ******************************************** +changed: [localhost] + +TASK [Encrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +PLAY RECAP ********************************************************************* +localhost : ok=5 changed=1 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0 +myrddin.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +pi.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +retropie.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-023020.log +CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Reset k3s nodes and reinstall with vault token] ************************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Find kernel cmdline file] ************************************** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +ok: [myrddin.prole.org] => (item=/boot/cmdline.txt) + +TASK [cgroups : Select kernel cmdline path] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select kernel cmdline update targets] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when kernel cmdline file is missing] ********************** +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] **** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************ +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read kernel cmdline files] ************************************* +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read current /proc/cmdline] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Check current cgroup params in /proc/cmdline] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Normalize kernel cmdline content] ****************************** +ok: [myrddin.prole.org] => (item={'content': 'cmVib290PXcgY29oZXJlbnRfcG9vbD0xTSA4MjUwLm5yX3VhcnRzPTEgcGNpPXBjaWVfYnVzX3NhZmUgbnVtYV9wb2xpY3k9aW50ZXJsZWF2ZSBudm1lLm1heF9ob3N0X21lbV9zaXplX21iPTAgbnVtYT1mYWtlPTggc3lzdGVtX2hlYXAubWF4X29yZGVyPTAgaW9tbXVfZG1hX251bWFfcG9saWN5PWludGVybGVhdmUgc21zYzk1eHgubWFjYWRkcj0yQzpDRjo2Nzo4QzoyQjo0NyB2Y19tZW0ubWVtX2Jhc2U9MHgzZmMwMDAwMCB2Y19tZW0ubWVtX3NpemU9MHg0MDAwMDAwMCBjb25zb2xlPXR0eUFNQTEwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD1hMzhiMTY1OS0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IA==', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'}) + +TASK [cgroups : Select kernel cmdline line] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select cmdline source] ***************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Tokenize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Remove conflicting cgroup params] ****************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Calculate missing cgroup params] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline tokens] *************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline line] ***************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize updated kernel cmdline line] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline needs update] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Update kernel cmdline when params are missing] ***************** +skipping: [myrddin.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory', 'sanitized': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory'}) +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from updated cmdline files] *** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Normalize updated cmdline whitespace] ************************** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline was updated] *********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if reboot is required] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Reboot to apply cgroup settings] ******************************* +skipping: [myrddin.prole.org] + +TASK [cgroups : Verify cgroup kernel parameters after reboot] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Read cgroup v2 controllers] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Record cgroup memory controller presence] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when cgroup params are missing (reboot required)] ********* +skipping: [myrddin.prole.org] + +TASK [cgroups : Fail when conflicting cgroup params are present] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +ok: [myrddin.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Remove k3s systemd units and directories] ************************** +ok: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] + +TASK [k3s : Skip k3s when state is absent] ************************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s service is installed] ********************************* +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s binary exists] **************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Determine if k3s needs installation] ******************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s dependencies] ****************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ****************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure cgroup kernel params are present] *************** +ok: [myrddin.prole.org] + +TASK [k3s : Fail if cgroup params are missing (reboot required)] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Fail if rancher mountpoint is on SD] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure required mountpoints are mounted] *************** +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Guardrail - ensure required mountpoints are not on SD] ************* +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Fail if required mountpoint is on SD] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdc1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d001'], 'start': '2026-02-16 02:30:47.792562', 'end': '2026-02-16 02:30:47.796164', 'delta': '0:00:00.003602', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d001', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdc1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d001', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdd1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d002'], 'start': '2026-02-16 02:30:48.135485', 'end': '2026-02-16 02:30:48.139254', 'delta': '0:00:00.003769', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d002', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdd1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d002', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sde1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d003'], 'start': '2026-02-16 02:30:48.441553', 'end': '2026-02-16 02:30:48.445293', 'delta': '0:00:00.003740', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d003', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sde1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d003', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Validate join configuration] *************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Check for existing k3s server node token] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Read existing k3s server node token] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail when vault k3s token is missing for existing cluster] ********* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail on k3s token drift from live cluster] ************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config directory exists] ******************************** +changed: [myrddin.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Render k3s registries config] ************************************** +changed: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig group exists] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig users are in group (server)] ********************* +ok: [myrddin.prole.org] => (item=pi) + +TASK [k3s : Ensure rancher directories are group-accessible] ******************* +changed: [myrddin.prole.org] => (item=/etc/rancher) +changed: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher) + +TASK [k3s : Render k3s config] ************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Build k3s install environment] ************************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s] ******************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure k3s service state] ****************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure prole config directory exists] ****************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole.cfg to k3s hosts] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole home directory exists] ******************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole service directory exists] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync init_port_forwards.sh to k3s hosts] *************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole_cfg.sh to k3s hosts] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Select k3s init server for port-forward token] ********************* +skipping: [myrddin.prole.org] => (item=pi.prole.org) +ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org) +skipping: [myrddin.prole.org] => (item=retropie.prole.org) + +TASK [k3s : Require k3s init server for port-forward token] ******************** +ok: [myrddin.prole.org -> localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Ensure prole-installer service account exists] ********************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure prole-installer token secret exists] ************************ +changed: [myrddin.prole.org] + +TASK [k3s : Read prole-installer token for port forwards] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Store prole-installer token on controller] ************************* +ok: [myrddin.prole.org -> localhost] + +TASK [k3s : Set prole port-forward token for host] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Render kubeconfig for port forwards] ******************************* +changed: [myrddin.prole.org] + +TASK [k3s : Stop and disable prole port-forwards service] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Remove prole port-forwards systemd unit] *************************** +ok: [myrddin.prole.org] + +RUNNING HANDLER [k3s : Restart k3s] ******************************************** +changed: [myrddin.prole.org] + +PLAY RECAP ********************************************************************* +myrddin.prole.org : ok=73 changed=9 unreachable=0 failed=0 skipped=16 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_sync.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-023148.log +CMD: ansible-playbook infrastructure/playbooks/k3s_sync.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Prepare k3s synchronization metadata] ************************************ + +TASK [Initialize k3s sync source host] ***************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Select k3s init server as sync source] *********************************** +skipping: [myrddin.prole.org] => (item=pi.prole.org) +ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org) +skipping: [myrddin.prole.org] => (item=retropie.prole.org) + +TASK [Require a k3s init server as sync source] ******************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure sync source is included in this run] ****************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Create local temp directory for k3s sync] ******************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Ensure k3s sync temp directory is writable] ****************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Store k3s sync metadata on controller] *********************************** +ok: [myrddin.prole.org -> localhost] + +PLAY [Collect k3s token and certs from init server] **************************** + +TASK [Require a k3s sync source] *********************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure k3s node token exists] ******************************************** +ok: [myrddin.prole.org] + +TASK [Fail when k3s node token is missing] ************************************* +skipping: [myrddin.prole.org] + +TASK [Read k3s node token] ***************************************************** +ok: [myrddin.prole.org] + +TASK [Store k3s sync token on controller] ************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Check for k3s tls directory] ********************************************* +ok: [myrddin.prole.org] + +TASK [Create k3s tls bundle] *************************************************** +changed: [myrddin.prole.org] + +TASK [Fetch k3s tls bundle] **************************************************** +changed: [myrddin.prole.org] + +TASK [Mark tls bundle presence on controller] ********************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Remove temporary tls bundle from source] ********************************* +changed: [myrddin.prole.org] + +PLAY [Synchronize k3s token and certs to servers] ****************************** + +TASK [Require k3s sync token] ************************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Gather service facts] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +skipping: [myrddin.prole.org] => (item=k3s) +skipping: [myrddin.prole.org] => (item=k3s-agent) +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config exists] ****************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Fail when k3s config is missing] *********************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config directory exists when missing] ******************* +skipping: [myrddin.prole.org] + +TASK [k3s : Create minimal k3s config when missing] **************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Sync k3s token into config] **************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +skipping: [myrddin.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Render k3s registries config] ************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s server directory exists] ******************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Remove existing k3s tls directory before sync] ********************* +skipping: [myrddin.prole.org] + +TASK [k3s : Restore k3s tls bundle] ******************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Gather service facts] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Build k3s install environment when service is missing] ************* +skipping: [myrddin.prole.org] + +TASK [k3s : Install k3s when service is missing] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Refresh service facts after install] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s service is installed] *********************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Start k3s service] ************************************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Wait for k3s to become active] ************************************* +skipping: [myrddin.prole.org] + +PLAY [Update k3s vault token on controller] ************************************ + +TASK [Skip vault update when disabled] ***************************************** +skipping: [myrddin.prole.org] + +TASK [Require k3s sync token for vault update] ********************************* +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Check for default vault password file] *********************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Determine vault password file] ******************************************* +ok: [myrddin.prole.org -> localhost] + +TASK [Fallback to default vault password file] ********************************* +skipping: [myrddin.prole.org] + +TASK [Require vault password file] ********************************************* +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure vault file exists] ************************************************ +ok: [myrddin.prole.org -> localhost] + +TASK [Fail when vault file is missing] ***************************************** +skipping: [myrddin.prole.org] + +TASK [Check if vault file is encrypted] **************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Mark vault encryption state] ********************************************* +ok: [myrddin.prole.org -> localhost] + +TASK [Decrypt vault_k3s.yml] *************************************************** +skipping: [myrddin.prole.org] + +TASK [Update vault k3s token] ************************************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Encrypt vault_k3s.yml] *************************************************** +skipping: [myrddin.prole.org] + +PLAY [Update local kubeconfig on controller] *********************************** + +TASK [Fetch k3s.yaml from init server] ***************************************** +ok: [myrddin.prole.org] + +TASK [Write k3s kubeconfig to local project] *********************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Debug kubeconfig write] ************************************************** +ok: [myrddin.prole.org] => { + "kubeconfig_write_debug.stdout_lines": [ + "Writing to /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig", + "-rw-r--r--@ 1 chrisfu staff 2946 Feb 16 02:31 /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig" + ] +} + +PLAY RECAP ********************************************************************* +myrddin.prole.org : ok=28 changed=7 unreachable=0 failed=0 skipped=29 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l pi.prole.org,retropie.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-023155.log +CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit pi.prole.org,retropie.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass +Fatal Python error: init_sys_streams: can't initialize sys standard streams +Python runtime state: core initialized +OSError: [Errno 9] Bad file descriptor + +Current thread 0x00000002060bb080 (most recent call first): + +[FATAL] k3s reset (agents) failed with code 1 diff --git a/install.pid b/install.pid new file mode 100644 index 0000000..c3e7fee --- /dev/null +++ b/install.pid @@ -0,0 +1 @@ +81081 diff --git a/install_final.log b/install_final.log new file mode 100644 index 0000000..6eeae60 --- /dev/null +++ b/install_final.log @@ -0,0 +1,1037 @@ +[INFO] Using Ansible Vault password file: /Users/chrisfu/dev/prole/.vault_pass +[CONFIG] Using conf/prole.cfg +==> Dependencies +[OK] Homebrew Homebrew 5.0.14 +[OK] python Python 3.14.2 +[OK] ansible ansible [core 2.20.2] +[OK] kubectl Client Version: v1.35.0 +[OK] kubectl-cnpg Build: {Version:1.28.0 Commit:a9696201f760013182c6cdba7c4ed3c236a6423b Date:2025-12-09T17:09:28Z} +[OK] Docker Docker version 29.2.0, build 0b9d1985db +[OK] k3d k3d version v5.8.3 +[OK] Network scan already performed. KDC detected: 10.0.0.3 +==> Environment setup +[OK] /Users/chrisfu/.prole/env.sh is up to date. +==> Database creation +[RESET] OpenBao anchor detected without token; generated new DB password for initialization. +[OK] /Users/chrisfu/.prole/env.sh is up to date. +[SKIP] Local registry pre-pull only supported for k3d. +==> Environment Preparation (Secure Access) +[SKIP] Secure access keys already exist at /Users/chrisfu/.ssh/id_prole_ed25519 +[INFO] Deferring OpenBao initialization until cluster is ready. +==> Cluster setup +Using local kubeconfig: /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig +[RESET] Resetting k3s cluster (mode k3s)... +[RESET] Removed local kubeconfig /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig +Running: ./ansible.sh -p infrastructure/playbooks/k3s_delete.yml -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-050334.log +CMD: ansible-playbook infrastructure/playbooks/k3s_delete.yml --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Delete k3s nodes and remove cluster token from vault] ******************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [retropie.prole.org] +ok: [myrddin.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +changed: [myrddin.prole.org] => (item=k3s) +ok: [retropie.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) +ok: [pi.prole.org] => (item=k3s) +changed: [retropie.prole.org] => (item=k3s-agent) +changed: [pi.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2011, 'inode': 305262, 'dev': 45826, 'nlink': 1, 'atime': 1771245076.6917439, 'mtime': 1771245076.6917439, 'ctime': 1771245076.6917439, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'c9554235f592bf87e98ffcd75871b6d749ea2729', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '40454479', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 4103, 'dev': 45826, 'nlink': 1, 'atime': 1771246024.4932477, 'mtime': 1771246024.4932477, 'ctime': 1771246024.4932477, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '601476158', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 17206, 'dev': 45826, 'nlink': 1, 'atime': 1771245276.1998293, 'mtime': 1771245276.1998293, 'ctime': 1771245276.2078295, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '3443958598', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) + +TASK [k3s : Remove k3s systemd units and directories] ************************** +changed: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +changed: [retropie.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +changed: [pi.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [retropie.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [retropie.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [retropie.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [retropie.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [pi.prole.org] => (item=/etc/rancher/node) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [pi.prole.org] => (item=/etc/cni) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/opt/cni) +ok: [retropie.prole.org] => (item=/var/lib/kubelet) +ok: [retropie.prole.org] => (item=/var/lib/etcd) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/cni) +ok: [retropie.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [retropie.prole.org] => (item=/run/k3s) +ok: [retropie.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [retropie.prole.org] => (item=/var/log/k3s) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [retropie.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/usr/local/bin/ctr) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/etcd) +ok: [pi.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +PLAY [Remove k3s token from Ansible vault] ************************************* + +TASK [Require vault password file for token deletion] ************************** +ok: [localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure vault file exists] ************************************************ +ok: [localhost] + +TASK [Fail when vault file is missing] ***************************************** +skipping: [localhost] + +TASK [Read vault_k3s.yml header] *********************************************** +ok: [localhost] + +TASK [Mark vault_k3s.yml encryption state] ************************************* +ok: [localhost] + +TASK [Decrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +TASK [Remove vault_k3s_token entry] ******************************************** +changed: [localhost] + +TASK [Encrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +PLAY RECAP ********************************************************************* +localhost : ok=5 changed=1 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0 +myrddin.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +pi.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +retropie.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-050507.log +CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Reset k3s nodes and reinstall with vault token] ************************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Find kernel cmdline file] ************************************** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +ok: [myrddin.prole.org] => (item=/boot/cmdline.txt) + +TASK [cgroups : Select kernel cmdline path] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select kernel cmdline update targets] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when kernel cmdline file is missing] ********************** +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] **** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************ +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read kernel cmdline files] ************************************* +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read current /proc/cmdline] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Check current cgroup params in /proc/cmdline] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Normalize kernel cmdline content] ****************************** +ok: [myrddin.prole.org] => (item={'content': 'cmVib290PXcgY29oZXJlbnRfcG9vbD0xTSA4MjUwLm5yX3VhcnRzPTEgcGNpPXBjaWVfYnVzX3NhZmUgbnVtYV9wb2xpY3k9aW50ZXJsZWF2ZSBudm1lLm1heF9ob3N0X21lbV9zaXplX21iPTAgbnVtYT1mYWtlPTggc3lzdGVtX2hlYXAubWF4X29yZGVyPTAgaW9tbXVfZG1hX251bWFfcG9saWN5PWludGVybGVhdmUgc21zYzk1eHgubWFjYWRkcj0yQzpDRjo2Nzo4QzoyQjo0NyB2Y19tZW0ubWVtX2Jhc2U9MHgzZmMwMDAwMCB2Y19tZW0ubWVtX3NpemU9MHg0MDAwMDAwMCBjb25zb2xlPXR0eUFNQTEwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD1hMzhiMTY1OS0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IA==', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'}) + +TASK [cgroups : Select kernel cmdline line] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select cmdline source] ***************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Tokenize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Remove conflicting cgroup params] ****************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Calculate missing cgroup params] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline tokens] *************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline line] ***************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize updated kernel cmdline line] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline needs update] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Update kernel cmdline when params are missing] ***************** +skipping: [myrddin.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory', 'sanitized': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory'}) +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from updated cmdline files] *** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Normalize updated cmdline whitespace] ************************** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline was updated] *********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if reboot is required] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Reboot to apply cgroup settings] ******************************* +skipping: [myrddin.prole.org] + +TASK [cgroups : Verify cgroup kernel parameters after reboot] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Read cgroup v2 controllers] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Record cgroup memory controller presence] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when cgroup params are missing (reboot required)] ********* +skipping: [myrddin.prole.org] + +TASK [cgroups : Fail when conflicting cgroup params are present] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +ok: [myrddin.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Remove k3s systemd units and directories] ************************** +ok: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] + +TASK [k3s : Skip k3s when state is absent] ************************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s service is installed] ********************************* +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s binary exists] **************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Determine if k3s needs installation] ******************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s dependencies] ****************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ****************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure cgroup kernel params are present] *************** +ok: [myrddin.prole.org] + +TASK [k3s : Fail if cgroup params are missing (reboot required)] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Fail if rancher mountpoint is on SD] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure required mountpoints are mounted] *************** +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Guardrail - ensure required mountpoints are not on SD] ************* +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Fail if required mountpoint is on SD] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdc1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d001'], 'start': '2026-02-16 05:05:34.800232', 'end': '2026-02-16 05:05:34.804029', 'delta': '0:00:00.003797', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d001', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdc1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d001', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdd1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d002'], 'start': '2026-02-16 05:05:35.112555', 'end': '2026-02-16 05:05:35.116284', 'delta': '0:00:00.003729', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d002', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdd1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d002', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sde1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d003'], 'start': '2026-02-16 05:05:35.416937', 'end': '2026-02-16 05:05:35.420618', 'delta': '0:00:00.003681', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d003', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sde1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d003', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Validate join configuration] *************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Check for existing k3s server node token] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Read existing k3s server node token] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail when vault k3s token is missing for existing cluster] ********* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail on k3s token drift from live cluster] ************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config directory exists] ******************************** +changed: [myrddin.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Render k3s registries config] ************************************** +changed: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig group exists] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig users are in group (server)] ********************* +ok: [myrddin.prole.org] => (item=pi) + +TASK [k3s : Ensure rancher directories are group-accessible] ******************* +changed: [myrddin.prole.org] => (item=/etc/rancher) +changed: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher) + +TASK [k3s : Render k3s config] ************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Build k3s install environment] ************************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s] ******************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure k3s service state] ****************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure prole config directory exists] ****************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole.cfg to k3s hosts] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole home directory exists] ******************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole service directory exists] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync init_port_forwards.sh to k3s hosts] *************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole_cfg.sh to k3s hosts] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Select k3s init server for port-forward token] ********************* +skipping: [myrddin.prole.org] => (item=pi.prole.org) +ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org) +skipping: [myrddin.prole.org] => (item=retropie.prole.org) + +TASK [k3s : Require k3s init server for port-forward token] ******************** +ok: [myrddin.prole.org -> localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Ensure prole-installer service account exists] ********************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure prole-installer token secret exists] ************************ +changed: [myrddin.prole.org] + +TASK [k3s : Read prole-installer token for port forwards] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Store prole-installer token on controller] ************************* +ok: [myrddin.prole.org -> localhost] + +TASK [k3s : Set prole port-forward token for host] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Render kubeconfig for port forwards] ******************************* +changed: [myrddin.prole.org] + +TASK [k3s : Stop and disable prole port-forwards service] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Remove prole port-forwards systemd unit] *************************** +ok: [myrddin.prole.org] + +RUNNING HANDLER [k3s : Restart k3s] ******************************************** +changed: [myrddin.prole.org] + +PLAY RECAP ********************************************************************* +myrddin.prole.org : ok=73 changed=9 unreachable=0 failed=0 skipped=16 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_sync.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-050640.log +CMD: ansible-playbook infrastructure/playbooks/k3s_sync.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Prepare k3s synchronization metadata] ************************************ + +TASK [Initialize k3s sync source host] ***************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Select k3s init server as sync source] *********************************** +skipping: [myrddin.prole.org] => (item=pi.prole.org) +ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org) +skipping: [myrddin.prole.org] => (item=retropie.prole.org) + +TASK [Require a k3s init server as sync source] ******************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure sync source is included in this run] ****************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Create local temp directory for k3s sync] ******************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Ensure k3s sync temp directory is writable] ****************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Store k3s sync metadata on controller] *********************************** +ok: [myrddin.prole.org -> localhost] + +PLAY [Collect k3s token and certs from init server] **************************** + +TASK [Require a k3s sync source] *********************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure k3s node token exists] ******************************************** +ok: [myrddin.prole.org] + +TASK [Fail when k3s node token is missing] ************************************* +skipping: [myrddin.prole.org] + +TASK [Read k3s node token] ***************************************************** +ok: [myrddin.prole.org] + +TASK [Store k3s sync token on controller] ************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Check for k3s tls directory] ********************************************* +ok: [myrddin.prole.org] + +TASK [Create k3s tls bundle] *************************************************** +changed: [myrddin.prole.org] + +TASK [Fetch k3s tls bundle] **************************************************** +changed: [myrddin.prole.org] + +TASK [Mark tls bundle presence on controller] ********************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Remove temporary tls bundle from source] ********************************* +changed: [myrddin.prole.org] + +PLAY [Synchronize k3s token and certs to servers] ****************************** + +TASK [Require k3s sync token] ************************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Gather service facts] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +skipping: [myrddin.prole.org] => (item=k3s) +skipping: [myrddin.prole.org] => (item=k3s-agent) +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config exists] ****************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Fail when k3s config is missing] *********************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config directory exists when missing] ******************* +skipping: [myrddin.prole.org] + +TASK [k3s : Create minimal k3s config when missing] **************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Sync k3s token into config] **************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +skipping: [myrddin.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Render k3s registries config] ************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s server directory exists] ******************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Remove existing k3s tls directory before sync] ********************* +skipping: [myrddin.prole.org] + +TASK [k3s : Restore k3s tls bundle] ******************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Gather service facts] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Build k3s install environment when service is missing] ************* +skipping: [myrddin.prole.org] + +TASK [k3s : Install k3s when service is missing] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Refresh service facts after install] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s service is installed] *********************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Start k3s service] ************************************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Wait for k3s to become active] ************************************* +skipping: [myrddin.prole.org] + +PLAY [Update k3s vault token on controller] ************************************ + +TASK [Skip vault update when disabled] ***************************************** +skipping: [myrddin.prole.org] + +TASK [Require k3s sync token for vault update] ********************************* +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Check for default vault password file] *********************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Determine vault password file] ******************************************* +ok: [myrddin.prole.org -> localhost] + +TASK [Fallback to default vault password file] ********************************* +skipping: [myrddin.prole.org] + +TASK [Require vault password file] ********************************************* +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure vault file exists] ************************************************ +ok: [myrddin.prole.org -> localhost] + +TASK [Fail when vault file is missing] ***************************************** +skipping: [myrddin.prole.org] + +TASK [Check if vault file is encrypted] **************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Mark vault encryption state] ********************************************* +ok: [myrddin.prole.org -> localhost] + +TASK [Decrypt vault_k3s.yml] *************************************************** +skipping: [myrddin.prole.org] + +TASK [Update vault k3s token] ************************************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Encrypt vault_k3s.yml] *************************************************** +skipping: [myrddin.prole.org] + +PLAY [Update local kubeconfig on controller] *********************************** + +TASK [Fetch k3s.yaml from init server] ***************************************** +ok: [myrddin.prole.org] + +TASK [Write k3s kubeconfig to local project] *********************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Debug kubeconfig write] ************************************************** +ok: [myrddin.prole.org] => { + "kubeconfig_write_debug.stdout_lines": [ + "Writing to /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig", + "-rw-r--r--@ 1 chrisfu staff 2954 Feb 16 05:06 /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig" + ] +} + +PLAY RECAP ********************************************************************* +myrddin.prole.org : ok=28 changed=7 unreachable=0 failed=0 skipped=29 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l pi.prole.org,retropie.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-050648.log +CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit pi.prole.org,retropie.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Reset k3s nodes and reinstall with vault token] ************************** + +TASK [Gathering Facts] ********************************************************* +ok: [pi.prole.org] + +TASK [cgroups : Find kernel cmdline file] ************************************** +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) +ok: [pi.prole.org] => (item=/boot/cmdline.txt) + +TASK [cgroups : Select kernel cmdline path] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Select kernel cmdline update targets] ************************** +ok: [pi.prole.org] + +TASK [cgroups : Fail when kernel cmdline file is missing] ********************** +skipping: [pi.prole.org] + +TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] **** +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************ +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read kernel cmdline files] ************************************* +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read current /proc/cmdline] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Check current cgroup params in /proc/cmdline] ****************** +ok: [pi.prole.org] + +TASK [cgroups : Normalize kernel cmdline content] ****************************** +ok: [pi.prole.org] => (item={'content': 'Y29uc29sZT1zZXJpYWwwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD0zOWRjYjQwNi0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX2VuYWJsZT1jcHVzZXQgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IHNlbGludXg9MCBlbmZvcmNpbmc9MCBzeXN0ZW1kLnVuaXQ9bXVsdGktdXNlci50YXJnZXQg', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'}) + +TASK [cgroups : Select kernel cmdline line] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Select cmdline source] ***************************************** +ok: [pi.prole.org] + +TASK [cgroups : Sanitize kernel cmdline] *************************************** +ok: [pi.prole.org] + +TASK [cgroups : Tokenize kernel cmdline] *************************************** +ok: [pi.prole.org] + +TASK [cgroups : Remove conflicting cgroup params] ****************************** +ok: [pi.prole.org] + +TASK [cgroups : Calculate missing cgroup params] ******************************* +ok: [pi.prole.org] + +TASK [cgroups : Build updated kernel cmdline tokens] *************************** +ok: [pi.prole.org] + +TASK [cgroups : Build updated kernel cmdline line] ***************************** +ok: [pi.prole.org] + +TASK [cgroups : Sanitize updated kernel cmdline line] ************************** +ok: [pi.prole.org] + +TASK [cgroups : Determine if kernel cmdline needs update] ********************** +ok: [pi.prole.org] + +TASK [cgroups : Update kernel cmdline when params are missing] ***************** +skipping: [pi.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'console=serial0,115200 console=tty1 root=PARTUUID=39dcb406-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory selinux=0 enforcing=0 systemd.unit=multi-user.target', 'sanitized': 'console=serial0,115200 console=tty1 root=PARTUUID=39dcb406-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory selinux=0 enforcing=0 systemd.unit=multi-user.target'}) +skipping: [pi.prole.org] + +TASK [cgroups : Remove literal backslash escapes from updated cmdline files] *** +skipping: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [pi.prole.org] + +TASK [cgroups : Normalize updated cmdline whitespace] ************************** +skipping: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [pi.prole.org] + +TASK [cgroups : Determine if kernel cmdline was updated] *********************** +ok: [pi.prole.org] + +TASK [cgroups : Determine if reboot is required] ******************************* +ok: [pi.prole.org] + +TASK [cgroups : Reboot to apply cgroup settings] ******************************* +skipping: [pi.prole.org] + +TASK [cgroups : Verify cgroup kernel parameters after reboot] ****************** +ok: [pi.prole.org] + +TASK [cgroups : Read cgroup v2 controllers] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Record cgroup memory controller presence] ********************** +ok: [pi.prole.org] + +TASK [cgroups : Fail when cgroup params are missing (reboot required)] ********* +skipping: [pi.prole.org] + +TASK [cgroups : Fail when conflicting cgroup params are present] *************** +skipping: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [pi.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [pi.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +ok: [pi.prole.org] => (item=k3s) +ok: [pi.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [pi.prole.org] + +TASK [k3s : Remove k3s systemd units and directories] ************************** +ok: [pi.prole.org] => (item=/etc/rancher) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [pi.prole.org] => (item=/etc/rancher/node) +ok: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [pi.prole.org] => (item=/etc/cni) +ok: [pi.prole.org] => (item=/opt/cni) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/etcd) +ok: [pi.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [pi.prole.org] + +TASK [k3s : Skip k3s when state is absent] ************************************* +skipping: [pi.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +ok: [pi.prole.org] + +TASK [k3s : Check if k3s service is installed] ********************************* +ok: [pi.prole.org] + +TASK [k3s : Check if k3s binary exists] **************************************** +ok: [pi.prole.org] + +TASK [k3s : Determine if k3s needs installation] ******************************* +ok: [pi.prole.org] + +TASK [k3s : Install k3s dependencies] ****************************************** +ok: [pi.prole.org] + +TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ****************** +ok: [pi.prole.org] + +TASK [k3s : Preflight - ensure cgroup kernel params are present] *************** +ok: [pi.prole.org] + +TASK [k3s : Fail if cgroup params are missing (reboot required)] *************** +skipping: [pi.prole.org] + +TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] ********* +ok: [pi.prole.org] + +TASK [k3s : Fail if rancher mountpoint is on SD] ******************************* +skipping: [pi.prole.org] + +TASK [k3s : Preflight - ensure required mountpoints are mounted] *************** +skipping: [pi.prole.org] + +TASK [k3s : Guardrail - ensure required mountpoints are not on SD] ************* +skipping: [pi.prole.org] + +TASK [k3s : Fail if required mountpoint is on SD] ****************************** +skipping: [pi.prole.org] + +TASK [k3s : Validate join configuration] *************************************** +ok: [pi.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Check for existing k3s server node token] ************************** +skipping: [pi.prole.org] + +TASK [k3s : Read existing k3s server node token] ******************************* +skipping: [pi.prole.org] + +TASK [k3s : Fail when vault k3s token is missing for existing cluster] ********* +skipping: [pi.prole.org] + +TASK [k3s : Fail on k3s token drift from live cluster] ************************* +skipping: [pi.prole.org] + +TASK [k3s : Ensure k3s config directory exists] ******************************** +changed: [pi.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +ok: [pi.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +ok: [pi.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +ok: [pi.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +ok: [pi.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +ok: [pi.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Render k3s registries config] ************************************** +changed: [pi.prole.org] + +TASK [k3s : Ensure kubeconfig group exists] ************************************ +ok: [pi.prole.org] + +TASK [k3s : Ensure kubeconfig users are in group (server)] ********************* +skipping: [pi.prole.org] => (item=pi) +skipping: [pi.prole.org] + +TASK [k3s : Ensure rancher directories are group-accessible] ******************* +changed: [pi.prole.org] => (item=/etc/rancher) +changed: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher) + +TASK [k3s : Render k3s config] ************************************************* +changed: [pi.prole.org] + +TASK [k3s : Build k3s install environment] ************************************* +ok: [pi.prole.org] + +TASK [k3s : Install k3s] ******************************************************* diff --git a/install_reset.log b/install_reset.log new file mode 100644 index 0000000..c568ab0 --- /dev/null +++ b/install_reset.log @@ -0,0 +1,561 @@ +[INFO] Using Ansible Vault password file: /Users/chrisfu/dev/prole/.vault_pass +[CONFIG] Using conf/prole.cfg +==> Dependencies +[OK] Homebrew Homebrew 5.0.14 +[OK] python Python 3.14.2 +[OK] ansible ansible [core 2.20.2] +[OK] kubectl Client Version: v1.35.0 +[OK] kubectl-cnpg Build: {Version:1.28.0 Commit:a9696201f760013182c6cdba7c4ed3c236a6423b Date:2025-12-09T17:09:28Z} +[OK] Docker Docker version 29.2.0, build 0b9d1985db +[OK] k3d k3d version v5.8.3 +[OK] Network scan already performed. KDC detected: 10.0.0.3 +==> Environment setup +[OK] /Users/chrisfu/.prole/env.sh is up to date. +==> Database creation +[RESET] OpenBao anchor detected without token; generated new DB password for initialization. +[OK] /Users/chrisfu/.prole/env.sh is up to date. +[SKIP] Local registry pre-pull only supported for k3d. +==> Environment Preparation (Secure Access) +[SKIP] Secure access keys already exist at /Users/chrisfu/.ssh/id_prole_ed25519 +[INFO] Deferring OpenBao initialization until cluster is ready. +==> Cluster setup +Using local kubeconfig: /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig +[RESET] Resetting k3s cluster (mode k3s)... +[RESET] Removed local kubeconfig /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig +Running: ./ansible.sh -p infrastructure/playbooks/k3s_delete.yml -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-055810.log +CMD: ansible-playbook infrastructure/playbooks/k3s_delete.yml --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Delete k3s nodes and remove cluster token from vault] ******************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [retropie.prole.org] +ok: [pi.prole.org] +ok: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +changed: [myrddin.prole.org] => (item=k3s) +ok: [retropie.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) +ok: [pi.prole.org] => (item=k3s) +changed: [retropie.prole.org] => (item=k3s-agent) +changed: [pi.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2011, 'inode': 305262, 'dev': 45826, 'nlink': 1, 'atime': 1771248194.1921544, 'mtime': 1771248194.1921544, 'ctime': 1771248194.1961544, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'c9554235f592bf87e98ffcd75871b6d749ea2729', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '3113320857', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 4118, 'dev': 45826, 'nlink': 1, 'atime': 1771248199.8591769, 'mtime': 1771248199.8591769, 'ctime': 1771248199.8691769, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '2098092759', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +changed: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 17206, 'dev': 45826, 'nlink': 1, 'atime': 1771248200.1626952, 'mtime': 1771248200.1626952, 'ctime': 1771248200.170695, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '181296699', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) + +TASK [k3s : Remove k3s systemd units and directories] ************************** +changed: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +changed: [retropie.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +changed: [pi.prole.org] => (item=/etc/rancher) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [retropie.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [retropie.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [retropie.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [retropie.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [pi.prole.org] => (item=/etc/cni) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [pi.prole.org] => (item=/opt/cni) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [retropie.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [retropie.prole.org] => (item=/var/lib/etcd) +ok: [retropie.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [retropie.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/run/k3s) +ok: [retropie.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [retropie.prole.org] => (item=/var/log/k3s) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [retropie.prole.org] => (item=/usr/local/bin/crictl) +ok: [retropie.prole.org] => (item=/usr/local/bin/ctr) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/etcd) +ok: [pi.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +PLAY [Remove k3s token from Ansible vault] ************************************* + +TASK [Require vault password file for token deletion] ************************** +ok: [localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure vault file exists] ************************************************ +ok: [localhost] + +TASK [Fail when vault file is missing] ***************************************** +skipping: [localhost] + +TASK [Read vault_k3s.yml header] *********************************************** +ok: [localhost] + +TASK [Mark vault_k3s.yml encryption state] ************************************* +ok: [localhost] + +TASK [Decrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +TASK [Remove vault_k3s_token entry] ******************************************** +changed: [localhost] + +TASK [Encrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +PLAY RECAP ********************************************************************* +localhost : ok=5 changed=1 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0 +myrddin.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +pi.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +retropie.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-060531.log +CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Reset k3s nodes and reinstall with vault token] ************************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Find kernel cmdline file] ************************************** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +ok: [myrddin.prole.org] => (item=/boot/cmdline.txt) + +TASK [cgroups : Select kernel cmdline path] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select kernel cmdline update targets] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when kernel cmdline file is missing] ********************** +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] **** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************ +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read kernel cmdline files] ************************************* +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read current /proc/cmdline] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Check current cgroup params in /proc/cmdline] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Normalize kernel cmdline content] ****************************** +ok: [myrddin.prole.org] => (item={'content': 'cmVib290PXcgY29oZXJlbnRfcG9vbD0xTSA4MjUwLm5yX3VhcnRzPTEgcGNpPXBjaWVfYnVzX3NhZmUgbnVtYV9wb2xpY3k9aW50ZXJsZWF2ZSBudm1lLm1heF9ob3N0X21lbV9zaXplX21iPTAgbnVtYT1mYWtlPTggc3lzdGVtX2hlYXAubWF4X29yZGVyPTAgaW9tbXVfZG1hX251bWFfcG9saWN5PWludGVybGVhdmUgc21zYzk1eHgubWFjYWRkcj0yQzpDRjo2Nzo4QzoyQjo0NyB2Y19tZW0ubWVtX2Jhc2U9MHgzZmMwMDAwMCB2Y19tZW0ubWVtX3NpemU9MHg0MDAwMDAwMCBjb25zb2xlPXR0eUFNQTEwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD1hMzhiMTY1OS0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IA==', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'}) + +TASK [cgroups : Select kernel cmdline line] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select cmdline source] ***************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Tokenize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Remove conflicting cgroup params] ****************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Calculate missing cgroup params] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline tokens] *************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline line] ***************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize updated kernel cmdline line] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline needs update] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Update kernel cmdline when params are missing] ***************** +skipping: [myrddin.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory', 'sanitized': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory'}) +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from updated cmdline files] *** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Normalize updated cmdline whitespace] ************************** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline was updated] *********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if reboot is required] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Reboot to apply cgroup settings] ******************************* +skipping: [myrddin.prole.org] + +TASK [cgroups : Verify cgroup kernel parameters after reboot] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Read cgroup v2 controllers] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Record cgroup memory controller presence] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when cgroup params are missing (reboot required)] ********* +skipping: [myrddin.prole.org] + +TASK [cgroups : Fail when conflicting cgroup params are present] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +ok: [myrddin.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Remove k3s systemd units and directories] ************************** +ok: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] + +TASK [k3s : Skip k3s when state is absent] ************************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s service is installed] ********************************* +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s binary exists] **************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Determine if k3s needs installation] ******************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s dependencies] ****************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ****************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure cgroup kernel params are present] *************** +ok: [myrddin.prole.org] + +TASK [k3s : Fail if cgroup params are missing (reboot required)] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Fail if rancher mountpoint is on SD] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure required mountpoints are mounted] *************** +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Guardrail - ensure required mountpoints are not on SD] ************* +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Fail if required mountpoint is on SD] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdc1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d001'], 'start': '2026-02-16 06:05:58.067260', 'end': '2026-02-16 06:05:59.072024', 'delta': '0:00:01.004764', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d001', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdc1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d001', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdd1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d002'], 'start': '2026-02-16 06:05:59.414480', 'end': '2026-02-16 06:05:59.418044', 'delta': '0:00:00.003564', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d002', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdd1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d002', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sde1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d003'], 'start': '2026-02-16 06:05:59.730321', 'end': '2026-02-16 06:05:59.734126', 'delta': '0:00:00.003805', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d003', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sde1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d003', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Validate join configuration] *************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Check for existing k3s server node token] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Read existing k3s server node token] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail when vault k3s token is missing for existing cluster] ********* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail on k3s token drift from live cluster] ************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config directory exists] ******************************** +changed: [myrddin.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Render k3s registries config] ************************************** +changed: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig group exists] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig users are in group (server)] ********************* +ok: [myrddin.prole.org] => (item=pi) + +TASK [k3s : Ensure rancher directories are group-accessible] ******************* +changed: [myrddin.prole.org] => (item=/etc/rancher) +changed: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher) + +TASK [k3s : Render k3s config] ************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Build k3s install environment] ************************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s] ******************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure k3s service state] ****************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Pre-stage k3s images] ********************************************** +changed: [myrddin.prole.org] => (item=docker.io/library/registry:2) +changed: [myrddin.prole.org] => (item=docker.io/prom/prometheus:latest) +changed: [myrddin.prole.org] => (item=docker.io/prom/alertmanager:latest) +changed: [myrddin.prole.org] => (item=quay.io/prometheus/node-exporter:latest) +[ERROR]: Task failed: Module failed: non-zero return code +Origin: /Users/chrisfu/dev/prole/infrastructure/roles/k3s/tasks/main.yml:238:3 + +236 when: k3s_service.stat.exists or k3s_bin.stat.exists +237 +238 - name: Pre-stage k3s images + ^ column 3 + +failed: [myrddin.prole.org] (item=quay.io/prometheus-operator/prometheus-operator:latest) => {"ansible_loop_var": "item", "changed": true, "cmd": "crictl pull quay.io/prometheus-operator/prometheus-operator:latest", "delta": "0:00:00.465932", "end": "2026-02-16 06:08:50.890945", "item": "quay.io/prometheus-operator/prometheus-operator:latest", "msg": "non-zero return code", "rc": 1, "start": "2026-02-16 06:08:50.425013", "stderr": "E0216 06:08:50.887164 38586 log.go:32] \"PullImage from image service failed\" err=\"rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\" image=\"quay.io/prometheus-operator/prometheus-operator:latest\"\ntime=\"2026-02-16T06:08:50-08:00\" level=fatal msg=\"pulling image: rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\"", "stderr_lines": ["E0216 06:08:50.887164 38586 log.go:32] \"PullImage from image service failed\" err=\"rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\" image=\"quay.io/prometheus-operator/prometheus-operator:latest\"", "time=\"2026-02-16T06:08:50-08:00\" level=fatal msg=\"pulling image: rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\""], "stdout": "", "stdout_lines": []} +changed: [myrddin.prole.org] => (item=ghcr.io/openbao/openbao:latest) +...ignoring + +TASK [k3s : Ensure prole config directory exists] ****************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole.cfg to k3s hosts] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole home directory exists] ******************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole service directory exists] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync init_port_forwards.sh to k3s hosts] *************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole_cfg.sh to k3s hosts] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Select k3s init server for port-forward token] ********************* +skipping: [myrddin.prole.org] => (item=pi.prole.org) +ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org) +skipping: [myrddin.prole.org] => (item=retropie.prole.org) + +TASK [k3s : Require k3s init server for port-forward token] ******************** +ok: [myrddin.prole.org -> localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Ensure prole-installer service account exists] ********************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure prole-installer token secret exists] ************************ +changed: [myrddin.prole.org] + +TASK [k3s : Read prole-installer token for port forwards] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Store prole-installer token on controller] ************************* +ok: [myrddin.prole.org -> localhost] + +TASK [k3s : Set prole port-forward token for host] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Render kubeconfig for port forwards] ******************************* +changed: [myrddin.prole.org] + +TASK [k3s : Stop and disable prole port-forwards service] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Remove prole port-forwards systemd unit] *************************** +ok: [myrddin.prole.org] + +RUNNING HANDLER [k3s : Restart k3s] ******************************************** diff --git a/install_retry.log b/install_retry.log new file mode 100644 index 0000000..5bdbe34 --- /dev/null +++ b/install_retry.log @@ -0,0 +1,1039 @@ +[INFO] Using Ansible Vault password file: /Users/chrisfu/dev/prole/.vault_pass +[CONFIG] Using conf/prole.cfg +==> Dependencies +[OK] Homebrew Homebrew 5.0.14 +[OK] python Python 3.14.2 +[OK] ansible ansible [core 2.20.2] +[OK] kubectl Client Version: v1.35.0 +[OK] kubectl-cnpg Build: {Version:1.28.0 Commit:a9696201f760013182c6cdba7c4ed3c236a6423b Date:2025-12-09T17:09:28Z} +[OK] Docker Docker version 29.2.0, build 0b9d1985db +[OK] k3d k3d version v5.8.3 +[OK] Network scan already performed. KDC detected: 10.0.0.3 +==> Environment setup +[OK] /Users/chrisfu/.prole/env.sh is up to date. +==> Database creation +[RESET] OpenBao anchor detected without token; generated new DB password for initialization. +[OK] /Users/chrisfu/.prole/env.sh is up to date. +[SKIP] Local registry pre-pull only supported for k3d. +==> Environment Preparation (Secure Access) +[SKIP] Secure access keys already exist at /Users/chrisfu/.ssh/id_prole_ed25519 +[INFO] Deferring OpenBao initialization until cluster is ready. +==> Cluster setup +Using local kubeconfig: /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig +[RESET] Resetting k3s cluster (mode k3s)... +[RESET] Removed local kubeconfig /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig +Running: ./ansible.sh -p infrastructure/playbooks/k3s_delete.yml -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-042917.log +CMD: ansible-playbook infrastructure/playbooks/k3s_delete.yml --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Delete k3s nodes and remove cluster token from vault] ******************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [retropie.prole.org] +ok: [pi.prole.org] +ok: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [retropie.prole.org] +ok: [myrddin.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +ok: [retropie.prole.org] => (item=k3s) +changed: [myrddin.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) +ok: [pi.prole.org] => (item=k3s) +ok: [retropie.prole.org] => (item=k3s-agent) +ok: [pi.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [pi.prole.org] +skipping: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [retropie.prole.org] +changed: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2011, 'inode': 305262, 'dev': 45826, 'nlink': 1, 'atime': 1771237860.9947455, 'mtime': 1771237860.9947455, 'ctime': 1771237860.9947455, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'c9554235f592bf87e98ffcd75871b6d749ea2729', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '306154775', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) + +TASK [k3s : Remove k3s systemd units and directories] ************************** +changed: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [retropie.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [pi.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [retropie.prole.org] => (item=/etc/rancher/k3s) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [retropie.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [retropie.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [pi.prole.org] => (item=/etc/cni) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [pi.prole.org] => (item=/opt/cni) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s) +ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [retropie.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [retropie.prole.org] => (item=/var/lib/etcd) +ok: [retropie.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [retropie.prole.org] => (item=/var/lib/containerd) +ok: [retropie.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [retropie.prole.org] => (item=/run/flannel) +ok: [retropie.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [retropie.prole.org] => (item=/usr/local/bin/kubectl) +ok: [retropie.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [retropie.prole.org] => (item=/usr/local/bin/ctr) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/etcd) +ok: [pi.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] +ok: [retropie.prole.org] +ok: [pi.prole.org] + +PLAY [Remove k3s token from Ansible vault] ************************************* + +TASK [Require vault password file for token deletion] ************************** +ok: [localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure vault file exists] ************************************************ +ok: [localhost] + +TASK [Fail when vault file is missing] ***************************************** +skipping: [localhost] + +TASK [Read vault_k3s.yml header] *********************************************** +ok: [localhost] + +TASK [Mark vault_k3s.yml encryption state] ************************************* +ok: [localhost] + +TASK [Decrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +TASK [Remove vault_k3s_token entry] ******************************************** +changed: [localhost] + +TASK [Encrypt vault_k3s.yml] *************************************************** +skipping: [localhost] + +PLAY RECAP ********************************************************************* +localhost : ok=5 changed=1 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0 +myrddin.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +pi.prole.org : ok=8 changed=0 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 +retropie.prole.org : ok=8 changed=0 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-043037.log +CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Reset k3s nodes and reinstall with vault token] ************************** + +TASK [Gathering Facts] ********************************************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Find kernel cmdline file] ************************************** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +ok: [myrddin.prole.org] => (item=/boot/cmdline.txt) + +TASK [cgroups : Select kernel cmdline path] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select kernel cmdline update targets] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when kernel cmdline file is missing] ********************** +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] **** +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************ +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read kernel cmdline files] ************************************* +ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read current /proc/cmdline] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Check current cgroup params in /proc/cmdline] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Normalize kernel cmdline content] ****************************** +ok: [myrddin.prole.org] => (item={'content': 'cmVib290PXcgY29oZXJlbnRfcG9vbD0xTSA4MjUwLm5yX3VhcnRzPTEgcGNpPXBjaWVfYnVzX3NhZmUgbnVtYV9wb2xpY3k9aW50ZXJsZWF2ZSBudm1lLm1heF9ob3N0X21lbV9zaXplX21iPTAgbnVtYT1mYWtlPTggc3lzdGVtX2hlYXAubWF4X29yZGVyPTAgaW9tbXVfZG1hX251bWFfcG9saWN5PWludGVybGVhdmUgc21zYzk1eHgubWFjYWRkcj0yQzpDRjo2Nzo4QzoyQjo0NyB2Y19tZW0ubWVtX2Jhc2U9MHgzZmMwMDAwMCB2Y19tZW0ubWVtX3NpemU9MHg0MDAwMDAwMCBjb25zb2xlPXR0eUFNQTEwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD1hMzhiMTY1OS0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IA==', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'}) + +TASK [cgroups : Select kernel cmdline line] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Select cmdline source] ***************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Tokenize kernel cmdline] *************************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Remove conflicting cgroup params] ****************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Calculate missing cgroup params] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline tokens] *************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Build updated kernel cmdline line] ***************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Sanitize updated kernel cmdline line] ************************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline needs update] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Update kernel cmdline when params are missing] ***************** +skipping: [myrddin.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory', 'sanitized': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory'}) +skipping: [myrddin.prole.org] + +TASK [cgroups : Remove literal backslash escapes from updated cmdline files] *** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Normalize updated cmdline whitespace] ************************** +skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [myrddin.prole.org] + +TASK [cgroups : Determine if kernel cmdline was updated] *********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Determine if reboot is required] ******************************* +ok: [myrddin.prole.org] + +TASK [cgroups : Reboot to apply cgroup settings] ******************************* +skipping: [myrddin.prole.org] + +TASK [cgroups : Verify cgroup kernel parameters after reboot] ****************** +ok: [myrddin.prole.org] + +TASK [cgroups : Read cgroup v2 controllers] ************************************ +ok: [myrddin.prole.org] + +TASK [cgroups : Record cgroup memory controller presence] ********************** +ok: [myrddin.prole.org] + +TASK [cgroups : Fail when cgroup params are missing (reboot required)] ********* +skipping: [myrddin.prole.org] + +TASK [cgroups : Fail when conflicting cgroup params are present] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [myrddin.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +ok: [myrddin.prole.org] => (item=k3s) +ok: [myrddin.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Remove k3s systemd units and directories] ************************** +ok: [myrddin.prole.org] => (item=/etc/rancher) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [myrddin.prole.org] => (item=/etc/rancher/node) +ok: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/etc/cni) +ok: [myrddin.prole.org] => (item=/opt/cni) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [myrddin.prole.org] => (item=/var/lib/kubelet) +ok: [myrddin.prole.org] => (item=/var/lib/etcd) +ok: [myrddin.prole.org] => (item=/var/lib/cni) +ok: [myrddin.prole.org] => (item=/var/lib/containerd) +ok: [myrddin.prole.org] => (item=/run/k3s) +ok: [myrddin.prole.org] => (item=/run/flannel) +ok: [myrddin.prole.org] => (item=/var/log/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl) +ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [myrddin.prole.org] + +TASK [k3s : Skip k3s when state is absent] ************************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s service is installed] ********************************* +ok: [myrddin.prole.org] + +TASK [k3s : Check if k3s binary exists] **************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Determine if k3s needs installation] ******************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s dependencies] ****************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ****************** +ok: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure cgroup kernel params are present] *************** +ok: [myrddin.prole.org] + +TASK [k3s : Fail if cgroup params are missing (reboot required)] *************** +skipping: [myrddin.prole.org] + +TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] ********* +ok: [myrddin.prole.org] + +TASK [k3s : Fail if rancher mountpoint is on SD] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Preflight - ensure required mountpoints are mounted] *************** +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Guardrail - ensure required mountpoints are not on SD] ************* +ok: [myrddin.prole.org] => (item=/prole/d001) +ok: [myrddin.prole.org] => (item=/prole/d002) +ok: [myrddin.prole.org] => (item=/prole/d003) + +TASK [k3s : Fail if required mountpoint is on SD] ****************************** +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdc1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d001'], 'start': '2026-02-16 04:31:02.981699', 'end': '2026-02-16 04:31:02.985283', 'delta': '0:00:00.003584', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d001', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdc1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d001', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdd1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d002'], 'start': '2026-02-16 04:31:03.302218', 'end': '2026-02-16 04:31:03.305986', 'delta': '0:00:00.003768', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d002', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdd1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d002', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sde1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d003'], 'start': '2026-02-16 04:31:03.627476', 'end': '2026-02-16 04:31:03.631273', 'delta': '0:00:00.003797', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d003', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sde1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d003', 'ansible_loop_var': 'item'}) +skipping: [myrddin.prole.org] + +TASK [k3s : Validate join configuration] *************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Check for existing k3s server node token] ************************** +ok: [myrddin.prole.org] + +TASK [k3s : Read existing k3s server node token] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail when vault k3s token is missing for existing cluster] ********* +skipping: [myrddin.prole.org] + +TASK [k3s : Fail on k3s token drift from live cluster] ************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config directory exists] ******************************** +changed: [myrddin.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Render k3s registries config] ************************************** +changed: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig group exists] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Ensure kubeconfig users are in group (server)] ********************* +ok: [myrddin.prole.org] => (item=pi) + +TASK [k3s : Ensure rancher directories are group-accessible] ******************* +changed: [myrddin.prole.org] => (item=/etc/rancher) +changed: [myrddin.prole.org] => (item=/etc/rancher/k3s) +ok: [myrddin.prole.org] => (item=/var/lib/rancher) + +TASK [k3s : Render k3s config] ************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Build k3s install environment] ************************************* +ok: [myrddin.prole.org] + +TASK [k3s : Install k3s] ******************************************************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure k3s service state] ****************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure prole config directory exists] ****************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole.cfg to k3s hosts] *************************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole home directory exists] ******************************** +ok: [myrddin.prole.org] + +TASK [k3s : Ensure prole service directory exists] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync init_port_forwards.sh to k3s hosts] *************************** +ok: [myrddin.prole.org] + +TASK [k3s : Sync prole_cfg.sh to k3s hosts] ************************************ +ok: [myrddin.prole.org] + +TASK [k3s : Select k3s init server for port-forward token] ********************* +skipping: [myrddin.prole.org] => (item=pi.prole.org) +ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org) +skipping: [myrddin.prole.org] => (item=retropie.prole.org) + +TASK [k3s : Require k3s init server for port-forward token] ******************** +ok: [myrddin.prole.org -> localhost] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Ensure prole-installer service account exists] ********************* +changed: [myrddin.prole.org] + +TASK [k3s : Ensure prole-installer token secret exists] ************************ +changed: [myrddin.prole.org] + +TASK [k3s : Read prole-installer token for port forwards] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Store prole-installer token on controller] ************************* +ok: [myrddin.prole.org -> localhost] + +TASK [k3s : Set prole port-forward token for host] ***************************** +ok: [myrddin.prole.org] + +TASK [k3s : Render kubeconfig for port forwards] ******************************* +changed: [myrddin.prole.org] + +TASK [k3s : Stop and disable prole port-forwards service] ********************** +ok: [myrddin.prole.org] + +TASK [k3s : Remove prole port-forwards systemd unit] *************************** +ok: [myrddin.prole.org] + +RUNNING HANDLER [k3s : Restart k3s] ******************************************** +changed: [myrddin.prole.org] + +PLAY RECAP ********************************************************************* +myrddin.prole.org : ok=73 changed=9 unreachable=0 failed=0 skipped=16 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_sync.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-043209.log +CMD: ansible-playbook infrastructure/playbooks/k3s_sync.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Prepare k3s synchronization metadata] ************************************ + +TASK [Initialize k3s sync source host] ***************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Select k3s init server as sync source] *********************************** +skipping: [myrddin.prole.org] => (item=pi.prole.org) +ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org) +skipping: [myrddin.prole.org] => (item=retropie.prole.org) + +TASK [Require a k3s init server as sync source] ******************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure sync source is included in this run] ****************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Create local temp directory for k3s sync] ******************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Ensure k3s sync temp directory is writable] ****************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Store k3s sync metadata on controller] *********************************** +ok: [myrddin.prole.org -> localhost] + +PLAY [Collect k3s token and certs from init server] **************************** + +TASK [Require a k3s sync source] *********************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure k3s node token exists] ******************************************** +ok: [myrddin.prole.org] + +TASK [Fail when k3s node token is missing] ************************************* +skipping: [myrddin.prole.org] + +TASK [Read k3s node token] ***************************************************** +ok: [myrddin.prole.org] + +TASK [Store k3s sync token on controller] ************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Check for k3s tls directory] ********************************************* +ok: [myrddin.prole.org] + +TASK [Create k3s tls bundle] *************************************************** +changed: [myrddin.prole.org] + +TASK [Fetch k3s tls bundle] **************************************************** +changed: [myrddin.prole.org] + +TASK [Mark tls bundle presence on controller] ********************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Remove temporary tls bundle from source] ********************************* +changed: [myrddin.prole.org] + +PLAY [Synchronize k3s token and certs to servers] ****************************** + +TASK [Require k3s sync token] ************************************************** +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Gather service facts] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +skipping: [myrddin.prole.org] => (item=k3s) +skipping: [myrddin.prole.org] => (item=k3s-agent) +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config exists] ****************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Fail when k3s config is missing] *********************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s config directory exists when missing] ******************* +skipping: [myrddin.prole.org] + +TASK [k3s : Create minimal k3s config when missing] **************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Sync k3s token into config] **************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +skipping: [myrddin.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Render k3s registries config] ************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s server directory exists] ******************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Remove existing k3s tls directory before sync] ********************* +skipping: [myrddin.prole.org] + +TASK [k3s : Restore k3s tls bundle] ******************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Gather service facts] ********************************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Build k3s install environment when service is missing] ************* +skipping: [myrddin.prole.org] + +TASK [k3s : Install k3s when service is missing] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Refresh service facts after install] ******************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Ensure k3s service is installed] *********************************** +skipping: [myrddin.prole.org] + +TASK [k3s : Start k3s service] ************************************************* +skipping: [myrddin.prole.org] + +TASK [k3s : Wait for k3s to become active] ************************************* +skipping: [myrddin.prole.org] + +PLAY [Update k3s vault token on controller] ************************************ + +TASK [Skip vault update when disabled] ***************************************** +skipping: [myrddin.prole.org] + +TASK [Require k3s sync token for vault update] ********************************* +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Check for default vault password file] *********************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Determine vault password file] ******************************************* +ok: [myrddin.prole.org -> localhost] + +TASK [Fallback to default vault password file] ********************************* +skipping: [myrddin.prole.org] + +TASK [Require vault password file] ********************************************* +ok: [myrddin.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [Ensure vault file exists] ************************************************ +ok: [myrddin.prole.org -> localhost] + +TASK [Fail when vault file is missing] ***************************************** +skipping: [myrddin.prole.org] + +TASK [Check if vault file is encrypted] **************************************** +ok: [myrddin.prole.org -> localhost] + +TASK [Mark vault encryption state] ********************************************* +ok: [myrddin.prole.org -> localhost] + +TASK [Decrypt vault_k3s.yml] *************************************************** +skipping: [myrddin.prole.org] + +TASK [Update vault k3s token] ************************************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Encrypt vault_k3s.yml] *************************************************** +skipping: [myrddin.prole.org] + +PLAY [Update local kubeconfig on controller] *********************************** + +TASK [Fetch k3s.yaml from init server] ***************************************** +ok: [myrddin.prole.org] + +TASK [Write k3s kubeconfig to local project] *********************************** +changed: [myrddin.prole.org -> localhost] + +TASK [Debug kubeconfig write] ************************************************** +ok: [myrddin.prole.org] => { + "kubeconfig_write_debug.stdout_lines": [ + "Writing to /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig", + "-rw-r--r--@ 1 chrisfu staff 2950 Feb 16 04:32 /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig" + ] +} + +PLAY RECAP ********************************************************************* +myrddin.prole.org : ok=28 changed=7 unreachable=0 failed=0 skipped=29 rescued=0 ignored=0 + +Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l pi.prole.org,retropie.prole.org -v /Users/chrisfu/dev/prole/.vault_pass +ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg +LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-043217.log +CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit pi.prole.org,retropie.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass + +PLAY [Reset k3s nodes and reinstall with vault token] ************************** + +TASK [Gathering Facts] ********************************************************* +ok: [pi.prole.org] + +TASK [cgroups : Find kernel cmdline file] ************************************** +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) +ok: [pi.prole.org] => (item=/boot/cmdline.txt) + +TASK [cgroups : Select kernel cmdline path] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Select kernel cmdline update targets] ************************** +ok: [pi.prole.org] + +TASK [cgroups : Fail when kernel cmdline file is missing] ********************** +skipping: [pi.prole.org] + +TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] **** +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************ +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read kernel cmdline files] ************************************* +ok: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) + +TASK [cgroups : Read current /proc/cmdline] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Check current cgroup params in /proc/cmdline] ****************** +ok: [pi.prole.org] + +TASK [cgroups : Normalize kernel cmdline content] ****************************** +ok: [pi.prole.org] => (item={'content': 'Y29uc29sZT1zZXJpYWwwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD0zOWRjYjQwNi0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX2VuYWJsZT1jcHVzZXQgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IHNlbGludXg9MCBlbmZvcmNpbmc9MCBzeXN0ZW1kLnVuaXQ9bXVsdGktdXNlci50YXJnZXQg', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'}) + +TASK [cgroups : Select kernel cmdline line] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Select cmdline source] ***************************************** +ok: [pi.prole.org] + +TASK [cgroups : Sanitize kernel cmdline] *************************************** +ok: [pi.prole.org] + +TASK [cgroups : Tokenize kernel cmdline] *************************************** +ok: [pi.prole.org] + +TASK [cgroups : Remove conflicting cgroup params] ****************************** +ok: [pi.prole.org] + +TASK [cgroups : Calculate missing cgroup params] ******************************* +ok: [pi.prole.org] + +TASK [cgroups : Build updated kernel cmdline tokens] *************************** +ok: [pi.prole.org] + +TASK [cgroups : Build updated kernel cmdline line] ***************************** +ok: [pi.prole.org] + +TASK [cgroups : Sanitize updated kernel cmdline line] ************************** +ok: [pi.prole.org] + +TASK [cgroups : Determine if kernel cmdline needs update] ********************** +ok: [pi.prole.org] + +TASK [cgroups : Update kernel cmdline when params are missing] ***************** +skipping: [pi.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'console=serial0,115200 console=tty1 root=PARTUUID=39dcb406-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory selinux=0 enforcing=0 systemd.unit=multi-user.target', 'sanitized': 'console=serial0,115200 console=tty1 root=PARTUUID=39dcb406-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory selinux=0 enforcing=0 systemd.unit=multi-user.target'}) +skipping: [pi.prole.org] + +TASK [cgroups : Remove literal backslash escapes from updated cmdline files] *** +skipping: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [pi.prole.org] + +TASK [cgroups : Normalize updated cmdline whitespace] ************************** +skipping: [pi.prole.org] => (item=/boot/firmware/cmdline.txt) +skipping: [pi.prole.org] + +TASK [cgroups : Determine if kernel cmdline was updated] *********************** +ok: [pi.prole.org] + +TASK [cgroups : Determine if reboot is required] ******************************* +ok: [pi.prole.org] + +TASK [cgroups : Reboot to apply cgroup settings] ******************************* +skipping: [pi.prole.org] + +TASK [cgroups : Verify cgroup kernel parameters after reboot] ****************** +ok: [pi.prole.org] + +TASK [cgroups : Read cgroup v2 controllers] ************************************ +ok: [pi.prole.org] + +TASK [cgroups : Record cgroup memory controller presence] ********************** +ok: [pi.prole.org] + +TASK [cgroups : Fail when cgroup params are missing (reboot required)] ********* +skipping: [pi.prole.org] + +TASK [cgroups : Fail when conflicting cgroup params are present] *************** +skipping: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system workloads before uninstall] ********** +ok: [pi.prole.org] + +TASK [k3s : Best-effort purge kube-system namespaces] ************************** +ok: [pi.prole.org] + +TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] ********* +ok: [pi.prole.org] + +TASK [k3s : Stop k3s services if present] ************************************** +ok: [pi.prole.org] => (item=k3s) +ok: [pi.prole.org] => (item=k3s-agent) + +TASK [k3s : Check for k3s uninstall scripts] *********************************** +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) + +TASK [k3s : Run k3s uninstall scripts if present] ****************************** +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'}) +skipping: [pi.prole.org] + +TASK [k3s : Remove k3s systemd units and directories] ************************** +ok: [pi.prole.org] => (item=/etc/rancher) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d) +ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d) +ok: [pi.prole.org] => (item=/etc/rancher/node) +ok: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [pi.prole.org] => (item=/etc/cni) +ok: [pi.prole.org] => (item=/opt/cni) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent) +ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data) +ok: [pi.prole.org] => (item=/var/lib/kubelet) +ok: [pi.prole.org] => (item=/var/lib/etcd) +ok: [pi.prole.org] => (item=/var/lib/cni) +ok: [pi.prole.org] => (item=/var/lib/containerd) +ok: [pi.prole.org] => (item=/run/k3s) +ok: [pi.prole.org] => (item=/run/flannel) +ok: [pi.prole.org] => (item=/var/log/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh) +ok: [pi.prole.org] => (item=/usr/local/bin/kubectl) +ok: [pi.prole.org] => (item=/usr/local/bin/crictl) +ok: [pi.prole.org] => (item=/usr/local/bin/ctr) + +TASK [k3s : Reload systemd after k3s cleanup] ********************************** +ok: [pi.prole.org] + +TASK [k3s : Skip k3s when state is absent] ************************************* +skipping: [pi.prole.org] + +TASK [k3s : Set k3s service name] ********************************************** +ok: [pi.prole.org] + +TASK [k3s : Check if k3s service is installed] ********************************* +ok: [pi.prole.org] + +TASK [k3s : Check if k3s binary exists] **************************************** +ok: [pi.prole.org] + +TASK [k3s : Determine if k3s needs installation] ******************************* +ok: [pi.prole.org] + +TASK [k3s : Install k3s dependencies] ****************************************** +ok: [pi.prole.org] + +TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ****************** +ok: [pi.prole.org] + +TASK [k3s : Preflight - ensure cgroup kernel params are present] *************** +ok: [pi.prole.org] + +TASK [k3s : Fail if cgroup params are missing (reboot required)] *************** +skipping: [pi.prole.org] + +TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] ********* +ok: [pi.prole.org] + +TASK [k3s : Fail if rancher mountpoint is on SD] ******************************* +skipping: [pi.prole.org] + +TASK [k3s : Preflight - ensure required mountpoints are mounted] *************** +skipping: [pi.prole.org] + +TASK [k3s : Guardrail - ensure required mountpoints are not on SD] ************* +skipping: [pi.prole.org] + +TASK [k3s : Fail if required mountpoint is on SD] ****************************** +skipping: [pi.prole.org] + +TASK [k3s : Validate join configuration] *************************************** +ok: [pi.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Check for existing k3s server node token] ************************** +skipping: [pi.prole.org] + +TASK [k3s : Read existing k3s server node token] ******************************* +skipping: [pi.prole.org] + +TASK [k3s : Fail when vault k3s token is missing for existing cluster] ********* +skipping: [pi.prole.org] + +TASK [k3s : Fail on k3s token drift from live cluster] ************************* +skipping: [pi.prole.org] + +TASK [k3s : Ensure k3s config directory exists] ******************************** +changed: [pi.prole.org] + +TASK [k3s : Load registry defaults from prole.cfg] ***************************** +ok: [pi.prole.org] + +TASK [k3s : Resolve k3s registry host] ***************************************** +ok: [pi.prole.org] + +TASK [k3s : Normalize k3s registry host] *************************************** +ok: [pi.prole.org] + +TASK [k3s : Resolve k3s registry namespace] ************************************ +ok: [pi.prole.org] + +TASK [k3s : Validate k3s registry host] **************************************** +ok: [pi.prole.org] => { + "changed": false, + "msg": "All assertions passed" +} + +TASK [k3s : Render k3s registries config] ************************************** +changed: [pi.prole.org] + +TASK [k3s : Ensure kubeconfig group exists] ************************************ +ok: [pi.prole.org] + +TASK [k3s : Ensure kubeconfig users are in group (server)] ********************* +skipping: [pi.prole.org] => (item=pi) +skipping: [pi.prole.org] + +TASK [k3s : Ensure rancher directories are group-accessible] ******************* +changed: [pi.prole.org] => (item=/etc/rancher) +changed: [pi.prole.org] => (item=/etc/rancher/k3s) +ok: [pi.prole.org] => (item=/var/lib/rancher) + +TASK [k3s : Render k3s config] ************************************************* +changed: [pi.prole.org] + +TASK [k3s : Build k3s install environment] ************************************* +ok: [pi.prole.org] + +TASK [k3s : Install k3s] ******************************************************* diff --git a/installer/scripts/install_prole_opentofu.sh b/installer/scripts/install_prole_opentofu.sh new file mode 100755 index 0000000..54a380d --- /dev/null +++ b/installer/scripts/install_prole_opentofu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Prole Installer: OpenTofu +# Installs OpenTofu via Homebrew if missing + +set -euo pipefail + +echo "[prole] Checking OpenTofu..." +if ! command -v tofu >/dev/null 2>&1; then + echo "[prole] Installing OpenTofu via brew..." + brew install opentofu +else + echo "[prole] OpenTofu already installed" +fi + +echo "[prole] OpenTofu OK" diff --git a/start_time.txt b/start_time.txt new file mode 100644 index 0000000..7b63f67 --- /dev/null +++ b/start_time.txt @@ -0,0 +1 @@ +Mon Feb 16 04:29:12 PST 2026 diff --git a/start_time_final.txt b/start_time_final.txt new file mode 100644 index 0000000..939c64a --- /dev/null +++ b/start_time_final.txt @@ -0,0 +1 @@ +Mon Feb 16 05:03:29 PST 2026