mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 10:13:58 +00:00
Fix Kong OOM crash: increase memory limit to 1Gi, mount ConfigMap to subdirectory
This commit is contained in:
chrisfu
parent
b03efa8f69
commit
a98c6d9d1c
48
ansible_recovery.log
Normal file
48
ansible_recovery.log
Normal file
@ -0,0 +1,48 @@
|
||||
ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg
|
||||
LOGFILE=/Users/chrisfu/dev/prole/.ansible/logs/ansible/ansible-20260216-053756.log
|
||||
CMD: ansible-playbook infrastructure/playbooks/site.yml --limit k3s_hosts --vault-password-file .vault_pass
|
||||
|
||||
PLAY [iSCSI hosts and filesystem mounts] ***************************************
|
||||
|
||||
TASK [Gathering Facts] *********************************************************
|
||||
ok: [retropie.prole.org]
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Install iSCSI dependencies] **************************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Ensure iscsid enabled] *******************************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Configure and login iSCSI targets] *******************************
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for pi.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-14.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/lib/rancher', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=466eefa2-9a7c-4bac-a136-d1d3ec35dae1'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for pi.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-19.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/log/pihole', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=56b21ec3-2826-4171-b909-a6715223f9a4'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for retropie.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-16.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/lib/rancher', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=73a886bb-3f01-4e8a-9c6d-ab431c671824'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-11.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/d001', 'fstype': 'xfs', 'opts': '_netdev,noatime', 'src': 'UUID=c07dc0f2-cf60-4da5-b68e-ea45e86d473e'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-12.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/d002', 'fstype': 'xfs', 'opts': '_netdev,noatime', 'src': 'UUID=fe087fb0-a321-4767-b838-b4385e81693e'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-13.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/d003', 'fstype': 'xfs', 'opts': '_netdev,noatime', 'src': 'UUID=757f1ee4-dc23-414b-b595-e3058c0744f0'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-17.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/home', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=607718c8-467a-474f-8e36-80ceafe1beb4'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-1.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/prole/logs', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=5335affd-b1ab-4134-a1c0-be88ab965309'}]})
|
||||
included: /Users/chrisfu/dev/prole/infrastructure/roles/iscsi/tasks/iscsi_target.yml for myrddin.prole.org => (item={'iqn': 'iqn.2000-01.com.synology:synology.Target-15.292d45194a1', 'chap_user': 'prole', 'chap_password': 'zVAx33rQK6RkU6Bv', 'mounts': [{'path': '/var/lib/rancher', 'fstype': 'ext4', 'opts': '_netdev,noatime', 'src': 'UUID=c8320979-d3eb-4b40-93e0-ee452a4b9780'}]})
|
||||
|
||||
TASK [iscsi : Ensure node record exists for target] ****************************
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Create node record if missing] ***********************************
|
||||
skipping: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Configure CHAP authmethod] ***************************************
|
||||
changed: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Configure CHAP username] *****************************************
|
||||
changed: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Configure CHAP password] *****************************************
|
||||
changed: [pi.prole.org]
|
||||
|
||||
TASK [iscsi : Check if target session is already logged in] ********************
|
||||
20
deploy/opentofu/k3s/argocd/application-openbao.yaml
Normal file
20
deploy/opentofu/k3s/argocd/application-openbao.yaml
Normal file
@ -0,0 +1,20 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: prole-openbao
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@github.com:dredx/prole.git
|
||||
targetRevision: main
|
||||
path: deploy/opentofu/k3s/manifests/openbao
|
||||
destination:
|
||||
server: https://myrddin.prole.org:6443
|
||||
namespace: prole-db-a0001
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
20
deploy/opentofu/k3s/argocd/application-opentofu.yaml
Normal file
20
deploy/opentofu/k3s/argocd/application-opentofu.yaml
Normal file
@ -0,0 +1,20 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: prole-opentofu
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@github.com:dredx/prole.git
|
||||
targetRevision: main
|
||||
path: deploy/opentofu/k3s/manifests/opentofu
|
||||
destination:
|
||||
server: https://myrddin.prole.org:6443
|
||||
namespace: prole-db-a0001
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
20
deploy/opentofu/k3s/argocd/application-prole.yaml
Normal file
20
deploy/opentofu/k3s/argocd/application-prole.yaml
Normal file
@ -0,0 +1,20 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: prole-prole
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@github.com:dredx/prole.git
|
||||
targetRevision: main
|
||||
path: deploy/opentofu/k3s/manifests/prole
|
||||
destination:
|
||||
server: https://myrddin.prole.org:6443
|
||||
namespace: prole-db-a0001
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
@ -31,10 +31,10 @@ spec:
|
||||
- name: KONG_ADMIN_LISTEN
|
||||
value: "0.0.0.0:8001"
|
||||
- name: KONG_DECLARATIVE_CONFIG
|
||||
value: "/etc/kong/kong.yml"
|
||||
value: "/etc/kong/declarative/kong.yml"
|
||||
volumeMounts:
|
||||
- name: kong-config
|
||||
mountPath: /etc/kong
|
||||
mountPath: /etc/kong/declarative
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
@ -50,10 +50,10 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: kong-config
|
||||
configMap:
|
||||
|
||||
13
deploy/opentofu/prod/README.md
Normal file
13
deploy/opentofu/prod/README.md
Normal file
@ -0,0 +1,13 @@
|
||||
# OpenTofu Production Pipeline
|
||||
|
||||
This pipeline deploys Prole into a production cluster (AWS EKS / GCloud GKE).
|
||||
|
||||
## Usage
|
||||
1. Copy manifests from the k3s pipeline into `manifests/`.
|
||||
2. Set `cluster_endpoint`, `cluster_token`, and optionally `cluster_ca_cert`.
|
||||
3. Run:
|
||||
```bash
|
||||
tofu init
|
||||
tofu plan
|
||||
tofu apply
|
||||
```
|
||||
57
deploy/opentofu/prod/main.tf
Normal file
57
deploy/opentofu/prod/main.tf
Normal file
@ -0,0 +1,57 @@
|
||||
terraform {
|
||||
required_version = ">= 1.6.0"
|
||||
required_providers {
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = "~> 2.30"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Configure for AWS EKS or GCloud GKE via variables
|
||||
provider "kubernetes" {
|
||||
host = var.cluster_endpoint
|
||||
token = var.cluster_token
|
||||
cluster_ca_certificate = var.cluster_ca_cert != "" ? base64decode(var.cluster_ca_cert) : null
|
||||
insecure = var.cluster_ca_cert == ""
|
||||
}
|
||||
|
||||
locals {
|
||||
manifest_dir = "${path.module}/manifests"
|
||||
manifest_files = fileset(local.manifest_dir, "**/*.yaml")
|
||||
raw_documents = flatten([
|
||||
for f in local.manifest_files : [
|
||||
for doc in split("\n---", trimspace(file("${local.manifest_dir}/${f}"))) :
|
||||
trimspace(doc)
|
||||
if trimspace(doc) != ""
|
||||
]
|
||||
])
|
||||
decoded_documents = [
|
||||
for doc in local.raw_documents : yamldecode(doc)
|
||||
if try(yamldecode(doc).kind, "") != ""
|
||||
]
|
||||
cluster_scoped_kinds = toset([
|
||||
"Namespace", "CustomResourceDefinition", "ClusterRole",
|
||||
"ClusterRoleBinding", "PersistentVolume", "StorageClass",
|
||||
])
|
||||
namespaced_documents = [
|
||||
for m in local.decoded_documents :
|
||||
contains(local.cluster_scoped_kinds, m.kind) ? m : merge(
|
||||
m, { metadata = merge(lookup(m, "metadata", {}), { namespace = var.namespace }) }
|
||||
)
|
||||
]
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "namespace" {
|
||||
manifest = {
|
||||
apiVersion = "v1"
|
||||
kind = "Namespace"
|
||||
metadata = { name = var.namespace }
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "resources" {
|
||||
for_each = { for idx, m in local.namespaced_documents : tostring(idx) => m }
|
||||
manifest = each.value
|
||||
depends_on = [kubernetes_manifest.namespace]
|
||||
}
|
||||
0
deploy/opentofu/prod/manifests/.gitkeep
Normal file
0
deploy/opentofu/prod/manifests/.gitkeep
Normal file
22
deploy/opentofu/prod/variables.tf
Normal file
22
deploy/opentofu/prod/variables.tf
Normal file
@ -0,0 +1,22 @@
|
||||
variable "cluster_endpoint" {
|
||||
type = string
|
||||
description = "Production cluster API endpoint (EKS/GKE)"
|
||||
}
|
||||
|
||||
variable "cluster_token" {
|
||||
type = string
|
||||
description = "Production cluster auth token"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "cluster_ca_cert" {
|
||||
type = string
|
||||
description = "Base64-encoded CA certificate (leave empty for insecure)"
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "namespace" {
|
||||
type = string
|
||||
description = "Target namespace for Prole resources"
|
||||
default = "default"
|
||||
}
|
||||
1
end_time.txt
Normal file
1
end_time.txt
Normal file
@ -0,0 +1 @@
|
||||
Mon Feb 16 04:29:00 PST 2026
|
||||
777
install.log
Normal file
777
install.log
Normal file
@ -0,0 +1,777 @@
|
||||
[INFO] Using Ansible Vault password file: /Users/chrisfu/dev/prole/.vault_pass
|
||||
[CONFIG] Using conf/prole.cfg
|
||||
==> Dependencies
|
||||
[OK] Homebrew Homebrew 5.0.14
|
||||
[OK] python Python 3.14.2
|
||||
[OK] ansible ansible [core 2.20.2]
|
||||
[OK] kubectl Client Version: v1.35.0
|
||||
[OK] kubectl-cnpg Build: {Version:1.28.0 Commit:a9696201f760013182c6cdba7c4ed3c236a6423b Date:2025-12-09T17:09:28Z}
|
||||
[OK] Docker Docker version 29.2.0, build 0b9d1985db
|
||||
[OK] k3d k3d version v5.8.3
|
||||
[OK] Network scan already performed. KDC detected: 10.0.0.3
|
||||
==> Environment setup
|
||||
[OK] /Users/chrisfu/.prole/env.sh is up to date.
|
||||
==> Database creation
|
||||
[RESET] OpenBao anchor detected without token; generated new DB password for initialization.
|
||||
[OK] /Users/chrisfu/.prole/env.sh is up to date.
|
||||
[SKIP] Local registry pre-pull only supported for k3d.
|
||||
==> Environment Preparation (Secure Access)
|
||||
[SKIP] Secure access keys already exist at /Users/chrisfu/.ssh/id_prole_ed25519
|
||||
[INFO] Deferring OpenBao initialization until cluster is ready.
|
||||
==> Cluster setup
|
||||
[RESET] Resetting k3s cluster (mode k3s)...
|
||||
Running: ./ansible.sh -p infrastructure/playbooks/k3s_delete.yml -v /Users/chrisfu/dev/prole/.vault_pass
|
||||
ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg
|
||||
LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-022843.log
|
||||
CMD: ansible-playbook infrastructure/playbooks/k3s_delete.yml --vault-password-file /Users/chrisfu/dev/prole/.vault_pass
|
||||
|
||||
PLAY [Delete k3s nodes and remove cluster token from vault] ********************
|
||||
|
||||
TASK [Gathering Facts] *********************************************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system workloads before uninstall] **********
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system namespaces] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] *********
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [k3s : Stop k3s services if present] **************************************
|
||||
changed: [myrddin.prole.org] => (item=k3s)
|
||||
ok: [retropie.prole.org] => (item=k3s)
|
||||
ok: [myrddin.prole.org] => (item=k3s-agent)
|
||||
ok: [pi.prole.org] => (item=k3s)
|
||||
changed: [retropie.prole.org] => (item=k3s-agent)
|
||||
changed: [pi.prole.org] => (item=k3s-agent)
|
||||
|
||||
TASK [k3s : Check for k3s uninstall scripts] ***********************************
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
|
||||
TASK [k3s : Run k3s uninstall scripts if present] ******************************
|
||||
skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
changed: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2011, 'inode': 305262, 'dev': 45826, 'nlink': 1, 'atime': 1771236200.9584086, 'mtime': 1771236200.9584086, 'ctime': 1771236200.9584086, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'c9554235f592bf87e98ffcd75871b6d749ea2729', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '860247187', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
changed: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 4103, 'dev': 45826, 'nlink': 1, 'atime': 1771236988.7820203, 'mtime': 1771236988.7820203, 'ctime': 1771236988.7920203, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '2537109730', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
changed: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 16764, 'dev': 45826, 'nlink': 1, 'atime': 1771236404.6359234, 'mtime': 1771236404.6519232, 'ctime': 1771236404.659923, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '1795183155', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
|
||||
TASK [k3s : Remove k3s systemd units and directories] **************************
|
||||
changed: [myrddin.prole.org] => (item=/etc/rancher)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
changed: [retropie.prole.org] => (item=/etc/rancher)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
changed: [pi.prole.org] => (item=/etc/rancher)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/etc/cni)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/opt/cni)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [retropie.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [retropie.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/cni)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [retropie.prole.org] => (item=/etc/cni)
|
||||
ok: [myrddin.prole.org] => (item=/run/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/run/flannel)
|
||||
ok: [myrddin.prole.org] => (item=/var/log/k3s)
|
||||
ok: [retropie.prole.org] => (item=/opt/cni)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr)
|
||||
ok: [pi.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [pi.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [pi.prole.org] => (item=/etc/cni)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [pi.prole.org] => (item=/opt/cni)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/cni)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [retropie.prole.org] => (item=/run/k3s)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [retropie.prole.org] => (item=/run/flannel)
|
||||
ok: [retropie.prole.org] => (item=/var/log/k3s)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/ctr)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [pi.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [pi.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [pi.prole.org] => (item=/var/lib/cni)
|
||||
ok: [pi.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [pi.prole.org] => (item=/run/k3s)
|
||||
ok: [pi.prole.org] => (item=/run/flannel)
|
||||
ok: [pi.prole.org] => (item=/var/log/k3s)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/ctr)
|
||||
|
||||
TASK [k3s : Reload systemd after k3s cleanup] **********************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
PLAY [Remove k3s token from Ansible vault] *************************************
|
||||
|
||||
TASK [Require vault password file for token deletion] **************************
|
||||
ok: [localhost] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [Ensure vault file exists] ************************************************
|
||||
ok: [localhost]
|
||||
|
||||
TASK [Fail when vault file is missing] *****************************************
|
||||
skipping: [localhost]
|
||||
|
||||
TASK [Read vault_k3s.yml header] ***********************************************
|
||||
ok: [localhost]
|
||||
|
||||
TASK [Mark vault_k3s.yml encryption state] *************************************
|
||||
ok: [localhost]
|
||||
|
||||
TASK [Decrypt vault_k3s.yml] ***************************************************
|
||||
skipping: [localhost]
|
||||
|
||||
TASK [Remove vault_k3s_token entry] ********************************************
|
||||
changed: [localhost]
|
||||
|
||||
TASK [Encrypt vault_k3s.yml] ***************************************************
|
||||
skipping: [localhost]
|
||||
|
||||
PLAY RECAP *********************************************************************
|
||||
localhost : ok=5 changed=1 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0
|
||||
myrddin.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
||||
pi.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
||||
retropie.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
||||
|
||||
Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass
|
||||
ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg
|
||||
LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-023020.log
|
||||
CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass
|
||||
|
||||
PLAY [Reset k3s nodes and reinstall with vault token] **************************
|
||||
|
||||
TASK [Gathering Facts] *********************************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Find kernel cmdline file] **************************************
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
ok: [myrddin.prole.org] => (item=/boot/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Select kernel cmdline path] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Select kernel cmdline update targets] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Fail when kernel cmdline file is missing] **********************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] ****
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Read kernel cmdline files] *************************************
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Read current /proc/cmdline] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Check current cgroup params in /proc/cmdline] ******************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Normalize kernel cmdline content] ******************************
|
||||
ok: [myrddin.prole.org] => (item={'content': 'cmVib290PXcgY29oZXJlbnRfcG9vbD0xTSA4MjUwLm5yX3VhcnRzPTEgcGNpPXBjaWVfYnVzX3NhZmUgbnVtYV9wb2xpY3k9aW50ZXJsZWF2ZSBudm1lLm1heF9ob3N0X21lbV9zaXplX21iPTAgbnVtYT1mYWtlPTggc3lzdGVtX2hlYXAubWF4X29yZGVyPTAgaW9tbXVfZG1hX251bWFfcG9saWN5PWludGVybGVhdmUgc21zYzk1eHgubWFjYWRkcj0yQzpDRjo2Nzo4QzoyQjo0NyB2Y19tZW0ubWVtX2Jhc2U9MHgzZmMwMDAwMCB2Y19tZW0ubWVtX3NpemU9MHg0MDAwMDAwMCBjb25zb2xlPXR0eUFNQTEwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD1hMzhiMTY1OS0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IA==', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'})
|
||||
|
||||
TASK [cgroups : Select kernel cmdline line] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Select cmdline source] *****************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Sanitize kernel cmdline] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Tokenize kernel cmdline] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Remove conflicting cgroup params] ******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Calculate missing cgroup params] *******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Build updated kernel cmdline tokens] ***************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Build updated kernel cmdline line] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Sanitize updated kernel cmdline line] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Determine if kernel cmdline needs update] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Update kernel cmdline when params are missing] *****************
|
||||
skipping: [myrddin.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory', 'sanitized': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory'})
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Remove literal backslash escapes from updated cmdline files] ***
|
||||
skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Normalize updated cmdline whitespace] **************************
|
||||
skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Determine if kernel cmdline was updated] ***********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Determine if reboot is required] *******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Reboot to apply cgroup settings] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Verify cgroup kernel parameters after reboot] ******************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Read cgroup v2 controllers] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Record cgroup memory controller presence] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Fail when cgroup params are missing (reboot required)] *********
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Fail when conflicting cgroup params are present] ***************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system workloads before uninstall] **********
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system namespaces] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] *********
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Stop k3s services if present] **************************************
|
||||
ok: [myrddin.prole.org] => (item=k3s)
|
||||
ok: [myrddin.prole.org] => (item=k3s-agent)
|
||||
|
||||
TASK [k3s : Check for k3s uninstall scripts] ***********************************
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
|
||||
TASK [k3s : Run k3s uninstall scripts if present] ******************************
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Remove k3s systemd units and directories] **************************
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/etc/cni)
|
||||
ok: [myrddin.prole.org] => (item=/opt/cni)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/cni)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [myrddin.prole.org] => (item=/run/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/run/flannel)
|
||||
ok: [myrddin.prole.org] => (item=/var/log/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr)
|
||||
|
||||
TASK [k3s : Reload systemd after k3s cleanup] **********************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Skip k3s when state is absent] *************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Set k3s service name] **********************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Check if k3s service is installed] *********************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Check if k3s binary exists] ****************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Determine if k3s needs installation] *******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Install k3s dependencies] ******************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ******************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Preflight - ensure cgroup kernel params are present] ***************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail if cgroup params are missing (reboot required)] ***************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] *********
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail if rancher mountpoint is on SD] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Preflight - ensure required mountpoints are mounted] ***************
|
||||
ok: [myrddin.prole.org] => (item=/prole/d001)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d002)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d003)
|
||||
|
||||
TASK [k3s : Guardrail - ensure required mountpoints are not on SD] *************
|
||||
ok: [myrddin.prole.org] => (item=/prole/d001)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d002)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d003)
|
||||
|
||||
TASK [k3s : Fail if required mountpoint is on SD] ******************************
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdc1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d001'], 'start': '2026-02-16 02:30:47.792562', 'end': '2026-02-16 02:30:47.796164', 'delta': '0:00:00.003602', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d001', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdc1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d001', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdd1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d002'], 'start': '2026-02-16 02:30:48.135485', 'end': '2026-02-16 02:30:48.139254', 'delta': '0:00:00.003769', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d002', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdd1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d002', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sde1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d003'], 'start': '2026-02-16 02:30:48.441553', 'end': '2026-02-16 02:30:48.445293', 'delta': '0:00:00.003740', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d003', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sde1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d003', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Validate join configuration] ***************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Check for existing k3s server node token] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Read existing k3s server node token] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail when vault k3s token is missing for existing cluster] *********
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail on k3s token drift from live cluster] *************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s config directory exists] ********************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Load registry defaults from prole.cfg] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Resolve k3s registry host] *****************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Normalize k3s registry host] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Resolve k3s registry namespace] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Validate k3s registry host] ****************************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [k3s : Render k3s registries config] **************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure kubeconfig group exists] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure kubeconfig users are in group (server)] *********************
|
||||
ok: [myrddin.prole.org] => (item=pi)
|
||||
|
||||
TASK [k3s : Ensure rancher directories are group-accessible] *******************
|
||||
changed: [myrddin.prole.org] => (item=/etc/rancher)
|
||||
changed: [myrddin.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher)
|
||||
|
||||
TASK [k3s : Render k3s config] *************************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Build k3s install environment] *************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Install k3s] *******************************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s service state] ******************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure prole config directory exists] ******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Sync prole.cfg to k3s hosts] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure prole home directory exists] ********************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure prole service directory exists] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Sync init_port_forwards.sh to k3s hosts] ***************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Sync prole_cfg.sh to k3s hosts] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Select k3s init server for port-forward token] *********************
|
||||
skipping: [myrddin.prole.org] => (item=pi.prole.org)
|
||||
ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org)
|
||||
skipping: [myrddin.prole.org] => (item=retropie.prole.org)
|
||||
|
||||
TASK [k3s : Require k3s init server for port-forward token] ********************
|
||||
ok: [myrddin.prole.org -> localhost] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [k3s : Ensure prole-installer service account exists] *********************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure prole-installer token secret exists] ************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Read prole-installer token for port forwards] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Store prole-installer token on controller] *************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [k3s : Set prole port-forward token for host] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Render kubeconfig for port forwards] *******************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Stop and disable prole port-forwards service] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Remove prole port-forwards systemd unit] ***************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
RUNNING HANDLER [k3s : Restart k3s] ********************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
PLAY RECAP *********************************************************************
|
||||
myrddin.prole.org : ok=73 changed=9 unreachable=0 failed=0 skipped=16 rescued=0 ignored=0
|
||||
|
||||
Running: ./ansible.sh -p infrastructure/playbooks/k3s_sync.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass
|
||||
ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg
|
||||
LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-023148.log
|
||||
CMD: ansible-playbook infrastructure/playbooks/k3s_sync.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass
|
||||
|
||||
PLAY [Prepare k3s synchronization metadata] ************************************
|
||||
|
||||
TASK [Initialize k3s sync source host] *****************************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Select k3s init server as sync source] ***********************************
|
||||
skipping: [myrddin.prole.org] => (item=pi.prole.org)
|
||||
ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org)
|
||||
skipping: [myrddin.prole.org] => (item=retropie.prole.org)
|
||||
|
||||
TASK [Require a k3s init server as sync source] ********************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [Ensure sync source is included in this run] ******************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [Create local temp directory for k3s sync] ********************************
|
||||
changed: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Ensure k3s sync temp directory is writable] ******************************
|
||||
changed: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Store k3s sync metadata on controller] ***********************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
PLAY [Collect k3s token and certs from init server] ****************************
|
||||
|
||||
TASK [Require a k3s sync source] ***********************************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [Ensure k3s node token exists] ********************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [Fail when k3s node token is missing] *************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [Read k3s node token] *****************************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [Store k3s sync token on controller] **************************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Check for k3s tls directory] *********************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [Create k3s tls bundle] ***************************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [Fetch k3s tls bundle] ****************************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [Mark tls bundle presence on controller] **********************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Remove temporary tls bundle from source] *********************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
PLAY [Synchronize k3s token and certs to servers] ******************************
|
||||
|
||||
TASK [Require k3s sync token] **************************************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [k3s : Gather service facts] **********************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Stop k3s services if present] **************************************
|
||||
skipping: [myrddin.prole.org] => (item=k3s)
|
||||
skipping: [myrddin.prole.org] => (item=k3s-agent)
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s config exists] ******************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail when k3s config is missing] ***********************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s config directory exists when missing] *******************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Create minimal k3s config when missing] ****************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Sync k3s token into config] ****************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Load registry defaults from prole.cfg] *****************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Resolve k3s registry host] *****************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Normalize k3s registry host] ***************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Resolve k3s registry namespace] ************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Validate k3s registry host] ****************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Render k3s registries config] **************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s server directory exists] ********************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Remove existing k3s tls directory before sync] *********************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Restore k3s tls bundle] ********************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Set k3s service name] **********************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Gather service facts] **********************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Build k3s install environment when service is missing] *************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Install k3s when service is missing] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Refresh service facts after install] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s service is installed] ***********************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Start k3s service] *************************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Wait for k3s to become active] *************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
PLAY [Update k3s vault token on controller] ************************************
|
||||
|
||||
TASK [Skip vault update when disabled] *****************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [Require k3s sync token for vault update] *********************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [Check for default vault password file] ***********************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Determine vault password file] *******************************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Fallback to default vault password file] *********************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [Require vault password file] *********************************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [Ensure vault file exists] ************************************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Fail when vault file is missing] *****************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [Check if vault file is encrypted] ****************************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Mark vault encryption state] *********************************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Decrypt vault_k3s.yml] ***************************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [Update vault k3s token] **************************************************
|
||||
changed: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Encrypt vault_k3s.yml] ***************************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
PLAY [Update local kubeconfig on controller] ***********************************
|
||||
|
||||
TASK [Fetch k3s.yaml from init server] *****************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [Write k3s kubeconfig to local project] ***********************************
|
||||
changed: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [Debug kubeconfig write] **************************************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"kubeconfig_write_debug.stdout_lines": [
|
||||
"Writing to /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig",
|
||||
"-rw-r--r--@ 1 chrisfu staff 2946 Feb 16 02:31 /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig"
|
||||
]
|
||||
}
|
||||
|
||||
PLAY RECAP *********************************************************************
|
||||
myrddin.prole.org : ok=28 changed=7 unreachable=0 failed=0 skipped=29 rescued=0 ignored=0
|
||||
|
||||
Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l pi.prole.org,retropie.prole.org -v /Users/chrisfu/dev/prole/.vault_pass
|
||||
ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg
|
||||
LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-023155.log
|
||||
CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit pi.prole.org,retropie.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass
|
||||
Fatal Python error: init_sys_streams: can't initialize sys standard streams
|
||||
Python runtime state: core initialized
|
||||
OSError: [Errno 9] Bad file descriptor
|
||||
|
||||
Current thread 0x00000002060bb080 (most recent call first):
|
||||
<no Python frame>
|
||||
[FATAL] k3s reset (agents) failed with code 1
|
||||
1
install.pid
Normal file
1
install.pid
Normal file
@ -0,0 +1 @@
|
||||
81081
|
||||
1037
install_final.log
Normal file
1037
install_final.log
Normal file
File diff suppressed because it is too large
Load Diff
561
install_reset.log
Normal file
561
install_reset.log
Normal file
@ -0,0 +1,561 @@
|
||||
[INFO] Using Ansible Vault password file: /Users/chrisfu/dev/prole/.vault_pass
|
||||
[CONFIG] Using conf/prole.cfg
|
||||
==> Dependencies
|
||||
[OK] Homebrew Homebrew 5.0.14
|
||||
[OK] python Python 3.14.2
|
||||
[OK] ansible ansible [core 2.20.2]
|
||||
[OK] kubectl Client Version: v1.35.0
|
||||
[OK] kubectl-cnpg Build: {Version:1.28.0 Commit:a9696201f760013182c6cdba7c4ed3c236a6423b Date:2025-12-09T17:09:28Z}
|
||||
[OK] Docker Docker version 29.2.0, build 0b9d1985db
|
||||
[OK] k3d k3d version v5.8.3
|
||||
[OK] Network scan already performed. KDC detected: 10.0.0.3
|
||||
==> Environment setup
|
||||
[OK] /Users/chrisfu/.prole/env.sh is up to date.
|
||||
==> Database creation
|
||||
[RESET] OpenBao anchor detected without token; generated new DB password for initialization.
|
||||
[OK] /Users/chrisfu/.prole/env.sh is up to date.
|
||||
[SKIP] Local registry pre-pull only supported for k3d.
|
||||
==> Environment Preparation (Secure Access)
|
||||
[SKIP] Secure access keys already exist at /Users/chrisfu/.ssh/id_prole_ed25519
|
||||
[INFO] Deferring OpenBao initialization until cluster is ready.
|
||||
==> Cluster setup
|
||||
Using local kubeconfig: /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig
|
||||
[RESET] Resetting k3s cluster (mode k3s)...
|
||||
[RESET] Removed local kubeconfig /Users/chrisfu/dev/prole/etc/secrets/k3s.kubeconfig
|
||||
Running: ./ansible.sh -p infrastructure/playbooks/k3s_delete.yml -v /Users/chrisfu/dev/prole/.vault_pass
|
||||
ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg
|
||||
LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-055810.log
|
||||
CMD: ansible-playbook infrastructure/playbooks/k3s_delete.yml --vault-password-file /Users/chrisfu/dev/prole/.vault_pass
|
||||
|
||||
PLAY [Delete k3s nodes and remove cluster token from vault] ********************
|
||||
|
||||
TASK [Gathering Facts] *********************************************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system workloads before uninstall] **********
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system namespaces] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] *********
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
TASK [k3s : Stop k3s services if present] **************************************
|
||||
changed: [myrddin.prole.org] => (item=k3s)
|
||||
ok: [retropie.prole.org] => (item=k3s)
|
||||
ok: [myrddin.prole.org] => (item=k3s-agent)
|
||||
ok: [pi.prole.org] => (item=k3s)
|
||||
changed: [retropie.prole.org] => (item=k3s-agent)
|
||||
changed: [pi.prole.org] => (item=k3s-agent)
|
||||
|
||||
TASK [k3s : Check for k3s uninstall scripts] ***********************************
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
|
||||
TASK [k3s : Run k3s uninstall scripts if present] ******************************
|
||||
skipping: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
changed: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2011, 'inode': 305262, 'dev': 45826, 'nlink': 1, 'atime': 1771248194.1921544, 'mtime': 1771248194.1921544, 'ctime': 1771248194.1961544, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'c9554235f592bf87e98ffcd75871b6d749ea2729', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '3113320857', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
changed: [retropie.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 4118, 'dev': 45826, 'nlink': 1, 'atime': 1771248199.8591769, 'mtime': 1771248199.8591769, 'ctime': 1771248199.8691769, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '2098092759', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
changed: [pi.prole.org] => (item={'changed': False, 'stat': {'exists': True, 'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'mode': '0755', 'isdir': False, 'ischr': False, 'isblk': False, 'isreg': True, 'isfifo': False, 'islnk': False, 'issock': False, 'uid': 0, 'gid': 0, 'size': 2047, 'inode': 17206, 'dev': 45826, 'nlink': 1, 'atime': 1771248200.1626952, 'mtime': 1771248200.1626952, 'ctime': 1771248200.170695, 'wusr': True, 'rusr': True, 'xusr': True, 'wgrp': False, 'rgrp': True, 'xgrp': True, 'woth': False, 'roth': True, 'xoth': True, 'isuid': False, 'isgid': False, 'blocks': 8, 'block_size': 4096, 'device_type': 0, 'readable': True, 'writeable': True, 'executable': True, 'pw_name': 'root', 'gr_name': 'root', 'checksum': 'f3e6ce8bc198982270efc0a73bd0464113fafab7', 'mimetype': 'text/x-shellscript', 'charset': 'us-ascii', 'version': '181296699', 'attributes': ['extents'], 'attr_flags': 'e'}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
|
||||
TASK [k3s : Remove k3s systemd units and directories] **************************
|
||||
changed: [myrddin.prole.org] => (item=/etc/rancher)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
changed: [retropie.prole.org] => (item=/etc/rancher)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
changed: [pi.prole.org] => (item=/etc/rancher)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/cni)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
ok: [myrddin.prole.org] => (item=/opt/cni)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [retropie.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [retropie.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [retropie.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/cni)
|
||||
ok: [retropie.prole.org] => (item=/etc/cni)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [myrddin.prole.org] => (item=/run/k3s)
|
||||
ok: [pi.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
ok: [retropie.prole.org] => (item=/opt/cni)
|
||||
ok: [myrddin.prole.org] => (item=/run/flannel)
|
||||
ok: [myrddin.prole.org] => (item=/var/log/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [pi.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [pi.prole.org] => (item=/etc/cni)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [pi.prole.org] => (item=/opt/cni)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/cni)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [retropie.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [retropie.prole.org] => (item=/run/k3s)
|
||||
ok: [retropie.prole.org] => (item=/run/flannel)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [retropie.prole.org] => (item=/var/log/k3s)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [retropie.prole.org] => (item=/usr/local/bin/ctr)
|
||||
ok: [pi.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [pi.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [pi.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [pi.prole.org] => (item=/var/lib/cni)
|
||||
ok: [pi.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [pi.prole.org] => (item=/run/k3s)
|
||||
ok: [pi.prole.org] => (item=/run/flannel)
|
||||
ok: [pi.prole.org] => (item=/var/log/k3s)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [pi.prole.org] => (item=/usr/local/bin/ctr)
|
||||
|
||||
TASK [k3s : Reload systemd after k3s cleanup] **********************************
|
||||
ok: [myrddin.prole.org]
|
||||
ok: [retropie.prole.org]
|
||||
ok: [pi.prole.org]
|
||||
|
||||
PLAY [Remove k3s token from Ansible vault] *************************************
|
||||
|
||||
TASK [Require vault password file for token deletion] **************************
|
||||
ok: [localhost] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [Ensure vault file exists] ************************************************
|
||||
ok: [localhost]
|
||||
|
||||
TASK [Fail when vault file is missing] *****************************************
|
||||
skipping: [localhost]
|
||||
|
||||
TASK [Read vault_k3s.yml header] ***********************************************
|
||||
ok: [localhost]
|
||||
|
||||
TASK [Mark vault_k3s.yml encryption state] *************************************
|
||||
ok: [localhost]
|
||||
|
||||
TASK [Decrypt vault_k3s.yml] ***************************************************
|
||||
skipping: [localhost]
|
||||
|
||||
TASK [Remove vault_k3s_token entry] ********************************************
|
||||
changed: [localhost]
|
||||
|
||||
TASK [Encrypt vault_k3s.yml] ***************************************************
|
||||
skipping: [localhost]
|
||||
|
||||
PLAY RECAP *********************************************************************
|
||||
localhost : ok=5 changed=1 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0
|
||||
myrddin.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
||||
pi.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
||||
retropie.prole.org : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
||||
|
||||
Running: ./ansible.sh -p infrastructure/playbooks/k3s_reset.yml -l myrddin.prole.org -v /Users/chrisfu/dev/prole/.vault_pass
|
||||
ANSIBLE_CONFIG=/Users/chrisfu/dev/prole/ansible.cfg
|
||||
LOGFILE=/opt/prole/logs/chrisfu/ansible/ansible-20260216-060531.log
|
||||
CMD: ansible-playbook infrastructure/playbooks/k3s_reset.yml --limit myrddin.prole.org --vault-password-file /Users/chrisfu/dev/prole/.vault_pass
|
||||
|
||||
PLAY [Reset k3s nodes and reinstall with vault token] **************************
|
||||
|
||||
TASK [Gathering Facts] *********************************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Find kernel cmdline file] **************************************
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
ok: [myrddin.prole.org] => (item=/boot/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Select kernel cmdline path] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Select kernel cmdline update targets] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Fail when kernel cmdline file is missing] **********************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Remove literal backslash escapes from kernel cmdline files] ****
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Remove stray backslashes from kernel cmdline files] ************
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Read kernel cmdline files] *************************************
|
||||
ok: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
|
||||
TASK [cgroups : Read current /proc/cmdline] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Check current cgroup params in /proc/cmdline] ******************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Normalize kernel cmdline content] ******************************
|
||||
ok: [myrddin.prole.org] => (item={'content': 'cmVib290PXcgY29oZXJlbnRfcG9vbD0xTSA4MjUwLm5yX3VhcnRzPTEgcGNpPXBjaWVfYnVzX3NhZmUgbnVtYV9wb2xpY3k9aW50ZXJsZWF2ZSBudm1lLm1heF9ob3N0X21lbV9zaXplX21iPTAgbnVtYT1mYWtlPTggc3lzdGVtX2hlYXAubWF4X29yZGVyPTAgaW9tbXVfZG1hX251bWFfcG9saWN5PWludGVybGVhdmUgc21zYzk1eHgubWFjYWRkcj0yQzpDRjo2Nzo4QzoyQjo0NyB2Y19tZW0ubWVtX2Jhc2U9MHgzZmMwMDAwMCB2Y19tZW0ubWVtX3NpemU9MHg0MDAwMDAwMCBjb25zb2xlPXR0eUFNQTEwLDExNTIwMCBjb25zb2xlPXR0eTEgcm9vdD1QQVJUVVVJRD1hMzhiMTY1OS0wMiByb290ZnN0eXBlPWV4dDQgZnNjay5yZXBhaXI9eWVzIHJvb3R3YWl0IGNmZzgwMjExLmllZWU4MDIxMV9yZWdkb209VVMgY2dyb3VwX21lbW9yeT0xIGNncm91cF9lbmFibGU9bWVtb3J5IA==', 'source': '/boot/firmware/cmdline.txt', 'encoding': 'base64', 'invocation': {'module_args': {'path': '/boot/firmware/cmdline.txt', 'src': '/boot/firmware/cmdline.txt'}}, 'failed': False, 'changed': False, 'item': '/boot/firmware/cmdline.txt', 'ansible_loop_var': 'item'})
|
||||
|
||||
TASK [cgroups : Select kernel cmdline line] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Select cmdline source] *****************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Sanitize kernel cmdline] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Tokenize kernel cmdline] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Remove conflicting cgroup params] ******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Calculate missing cgroup params] *******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Build updated kernel cmdline tokens] ***************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Build updated kernel cmdline line] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Sanitize updated kernel cmdline line] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Determine if kernel cmdline needs update] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Update kernel cmdline when params are missing] *****************
|
||||
skipping: [myrddin.prole.org] => (item={'path': '/boot/firmware/cmdline.txt', 'raw_line': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory', 'sanitized': 'reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave nvme.max_host_mem_size_mb=0 numa=fake=8 system_heap.max_order=0 iommu_dma_numa_policy=interleave smsc95xx.macaddr=2C:CF:67:8C:2B:47 vc_mem.mem_base=0x3fc00000 vc_mem.mem_size=0x40000000 console=ttyAMA10,115200 console=tty1 root=PARTUUID=a38b1659-02 rootfstype=ext4 fsck.repair=yes rootwait cfg80211.ieee80211_regdom=US cgroup_memory=1 cgroup_enable=memory'})
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Remove literal backslash escapes from updated cmdline files] ***
|
||||
skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Normalize updated cmdline whitespace] **************************
|
||||
skipping: [myrddin.prole.org] => (item=/boot/firmware/cmdline.txt)
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Determine if kernel cmdline was updated] ***********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Determine if reboot is required] *******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Reboot to apply cgroup settings] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Verify cgroup kernel parameters after reboot] ******************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Read cgroup v2 controllers] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Record cgroup memory controller presence] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Fail when cgroup params are missing (reboot required)] *********
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [cgroups : Fail when conflicting cgroup params are present] ***************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system workloads before uninstall] **********
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Best-effort purge kube-system namespaces] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Force-remove kube-system finalizers if namespace is stuck] *********
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Stop k3s services if present] **************************************
|
||||
ok: [myrddin.prole.org] => (item=k3s)
|
||||
ok: [myrddin.prole.org] => (item=k3s-agent)
|
||||
|
||||
TASK [k3s : Check for k3s uninstall scripts] ***********************************
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
|
||||
TASK [k3s : Run k3s uninstall scripts if present] ******************************
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stat': {'exists': False}, 'invocation': {'module_args': {'path': '/usr/local/bin/k3s-agent-uninstall.sh', 'follow': False, 'get_checksum': True, 'get_mime': True, 'get_attributes': True, 'get_selinux_context': False, 'checksum_algorithm': 'sha1'}}, 'failed': False, 'item': '/usr/local/bin/k3s-agent-uninstall.sh', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Remove k3s systemd units and directories] **************************
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/etc/systemd/system/k3s-agent.service.d)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/node)
|
||||
ok: [myrddin.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/etc/cni)
|
||||
ok: [myrddin.prole.org] => (item=/opt/cni)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/tls)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/cred)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/node-token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/agent-token)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/server/db)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/agent)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher/k3s/data)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/kubelet)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/etcd)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/cni)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/containerd)
|
||||
ok: [myrddin.prole.org] => (item=/run/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/run/flannel)
|
||||
ok: [myrddin.prole.org] => (item=/var/log/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-killall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/k3s-agent-uninstall.sh)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/kubectl)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/crictl)
|
||||
ok: [myrddin.prole.org] => (item=/usr/local/bin/ctr)
|
||||
|
||||
TASK [k3s : Reload systemd after k3s cleanup] **********************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Skip k3s when state is absent] *************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Set k3s service name] **********************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Check if k3s service is installed] *********************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Check if k3s binary exists] ****************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Determine if k3s needs installation] *******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Install k3s dependencies] ******************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Preflight - ensure rancher mountpoint is mounted] ******************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Preflight - ensure cgroup kernel params are present] ***************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail if cgroup params are missing (reboot required)] ***************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Guardrail - ensure rancher mountpoint is not on SD/rootfs] *********
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail if rancher mountpoint is on SD] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Preflight - ensure required mountpoints are mounted] ***************
|
||||
ok: [myrddin.prole.org] => (item=/prole/d001)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d002)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d003)
|
||||
|
||||
TASK [k3s : Guardrail - ensure required mountpoints are not on SD] *************
|
||||
ok: [myrddin.prole.org] => (item=/prole/d001)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d002)
|
||||
ok: [myrddin.prole.org] => (item=/prole/d003)
|
||||
|
||||
TASK [k3s : Fail if required mountpoint is on SD] ******************************
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdc1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d001'], 'start': '2026-02-16 06:05:58.067260', 'end': '2026-02-16 06:05:59.072024', 'delta': '0:00:01.004764', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d001', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdc1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d001', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sdd1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d002'], 'start': '2026-02-16 06:05:59.414480', 'end': '2026-02-16 06:05:59.418044', 'delta': '0:00:00.003564', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d002', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sdd1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d002', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org] => (item={'changed': False, 'stdout': '/dev/sde1', 'stderr': '', 'rc': 0, 'cmd': ['findmnt', '-n', '-o', 'SOURCE', '/prole/d003'], 'start': '2026-02-16 06:05:59.730321', 'end': '2026-02-16 06:05:59.734126', 'delta': '0:00:00.003805', 'msg': '', 'invocation': {'module_args': {'_raw_params': 'findmnt -n -o SOURCE /prole/d003', '_uses_shell': False, 'expand_argument_vars': True, 'stdin_add_newline': True, 'strip_empty_ends': True, 'cmd': None, 'argv': None, 'chdir': None, 'executable': None, 'creates': None, 'removes': None, 'stdin': None}}, 'stdout_lines': ['/dev/sde1'], 'stderr_lines': [], 'failed': False, 'item': '/prole/d003', 'ansible_loop_var': 'item'})
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Validate join configuration] ***************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Check for existing k3s server node token] **************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Read existing k3s server node token] *******************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail when vault k3s token is missing for existing cluster] *********
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Fail on k3s token drift from live cluster] *************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s config directory exists] ********************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Load registry defaults from prole.cfg] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Resolve k3s registry host] *****************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Normalize k3s registry host] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Resolve k3s registry namespace] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Validate k3s registry host] ****************************************
|
||||
ok: [myrddin.prole.org] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [k3s : Render k3s registries config] **************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure kubeconfig group exists] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure kubeconfig users are in group (server)] *********************
|
||||
ok: [myrddin.prole.org] => (item=pi)
|
||||
|
||||
TASK [k3s : Ensure rancher directories are group-accessible] *******************
|
||||
changed: [myrddin.prole.org] => (item=/etc/rancher)
|
||||
changed: [myrddin.prole.org] => (item=/etc/rancher/k3s)
|
||||
ok: [myrddin.prole.org] => (item=/var/lib/rancher)
|
||||
|
||||
TASK [k3s : Render k3s config] *************************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Build k3s install environment] *************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Install k3s] *******************************************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure k3s service state] ******************************************
|
||||
skipping: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Pre-stage k3s images] **********************************************
|
||||
changed: [myrddin.prole.org] => (item=docker.io/library/registry:2)
|
||||
changed: [myrddin.prole.org] => (item=docker.io/prom/prometheus:latest)
|
||||
changed: [myrddin.prole.org] => (item=docker.io/prom/alertmanager:latest)
|
||||
changed: [myrddin.prole.org] => (item=quay.io/prometheus/node-exporter:latest)
|
||||
[ERROR]: Task failed: Module failed: non-zero return code
|
||||
Origin: /Users/chrisfu/dev/prole/infrastructure/roles/k3s/tasks/main.yml:238:3
|
||||
|
||||
236 when: k3s_service.stat.exists or k3s_bin.stat.exists
|
||||
237
|
||||
238 - name: Pre-stage k3s images
|
||||
^ column 3
|
||||
|
||||
failed: [myrddin.prole.org] (item=quay.io/prometheus-operator/prometheus-operator:latest) => {"ansible_loop_var": "item", "changed": true, "cmd": "crictl pull quay.io/prometheus-operator/prometheus-operator:latest", "delta": "0:00:00.465932", "end": "2026-02-16 06:08:50.890945", "item": "quay.io/prometheus-operator/prometheus-operator:latest", "msg": "non-zero return code", "rc": 1, "start": "2026-02-16 06:08:50.425013", "stderr": "E0216 06:08:50.887164 38586 log.go:32] \"PullImage from image service failed\" err=\"rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\" image=\"quay.io/prometheus-operator/prometheus-operator:latest\"\ntime=\"2026-02-16T06:08:50-08:00\" level=fatal msg=\"pulling image: rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\"", "stderr_lines": ["E0216 06:08:50.887164 38586 log.go:32] \"PullImage from image service failed\" err=\"rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\" image=\"quay.io/prometheus-operator/prometheus-operator:latest\"", "time=\"2026-02-16T06:08:50-08:00\" level=fatal msg=\"pulling image: rpc error: code = NotFound desc = failed to pull and unpack image \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": failed to resolve reference \\\"quay.io/prometheus-operator/prometheus-operator:latest\\\": quay.io/prometheus-operator/prometheus-operator:latest: not found\""], "stdout": "", "stdout_lines": []}
|
||||
changed: [myrddin.prole.org] => (item=ghcr.io/openbao/openbao:latest)
|
||||
...ignoring
|
||||
|
||||
TASK [k3s : Ensure prole config directory exists] ******************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Sync prole.cfg to k3s hosts] ***************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure prole home directory exists] ********************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure prole service directory exists] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Sync init_port_forwards.sh to k3s hosts] ***************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Sync prole_cfg.sh to k3s hosts] ************************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Select k3s init server for port-forward token] *********************
|
||||
skipping: [myrddin.prole.org] => (item=pi.prole.org)
|
||||
ok: [myrddin.prole.org -> localhost] => (item=myrddin.prole.org)
|
||||
skipping: [myrddin.prole.org] => (item=retropie.prole.org)
|
||||
|
||||
TASK [k3s : Require k3s init server for port-forward token] ********************
|
||||
ok: [myrddin.prole.org -> localhost] => {
|
||||
"changed": false,
|
||||
"msg": "All assertions passed"
|
||||
}
|
||||
|
||||
TASK [k3s : Ensure prole-installer service account exists] *********************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Ensure prole-installer token secret exists] ************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Read prole-installer token for port forwards] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Store prole-installer token on controller] *************************
|
||||
ok: [myrddin.prole.org -> localhost]
|
||||
|
||||
TASK [k3s : Set prole port-forward token for host] *****************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Render kubeconfig for port forwards] *******************************
|
||||
changed: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Stop and disable prole port-forwards service] **********************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
TASK [k3s : Remove prole port-forwards systemd unit] ***************************
|
||||
ok: [myrddin.prole.org]
|
||||
|
||||
RUNNING HANDLER [k3s : Restart k3s] ********************************************
|
||||
1039
install_retry.log
Normal file
1039
install_retry.log
Normal file
File diff suppressed because it is too large
Load Diff
15
installer/scripts/install_prole_opentofu.sh
Executable file
15
installer/scripts/install_prole_opentofu.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
# Prole Installer: OpenTofu
|
||||
# Installs OpenTofu via Homebrew if missing
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "[prole] Checking OpenTofu..."
|
||||
if ! command -v tofu >/dev/null 2>&1; then
|
||||
echo "[prole] Installing OpenTofu via brew..."
|
||||
brew install opentofu
|
||||
else
|
||||
echo "[prole] OpenTofu already installed"
|
||||
fi
|
||||
|
||||
echo "[prole] OpenTofu OK"
|
||||
1
start_time.txt
Normal file
1
start_time.txt
Normal file
@ -0,0 +1 @@
|
||||
Mon Feb 16 04:29:12 PST 2026
|
||||
1
start_time_final.txt
Normal file
1
start_time_final.txt
Normal file
@ -0,0 +1 @@
|
||||
Mon Feb 16 05:03:29 PST 2026
|
||||
Loading…
Reference in New Issue
Block a user