diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..2f60899 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,26 @@ +[defaults] +inventory = infrastructure/inventory/hosts.ini +roles_path = infrastructure/roles +collections_paths = infrastructure/collections +interpreter_python = auto_silent +deprecation_warnings = False +callback_plugins = lib/ansible/plugins/callback +callbacks_enabled = run_logger +stdout_callback = default +result_format = yaml +forks = 20 +timeout = 30 +host_key_checking = True + +[privilege_escalation] +become = True +become_method = sudo +become_ask_pass = False + +[ssh_connection] +pipelining = True + +# Use a repo-local known_hosts file and accept new keys non-interactively. +# This prevents unattended runs from failing when inventory uses IPs via `ansible_host` +# and the key is not yet present in the user's `~/.ssh/known_hosts`. +ssh_common_args = -o UserKnownHostsFile=.ansible/known_hosts -o StrictHostKeyChecking=accept-new diff --git a/ansible.sh b/ansible.sh new file mode 100755 index 0000000..eccb08a --- /dev/null +++ b/ansible.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export ANSIBLE_CONFIG="${ROOT_DIR}/ansible.cfg" + +# Defaults +MODE="fg" # fg|bg +PLAYBOOK="infrastructure/playbooks/site.yml" +VAULT_PASS_FILE="" +if [[ -f "${ROOT_DIR}/.vault_pass" ]]; then + VAULT_PASS_FILE="${ROOT_DIR}/.vault_pass" +fi +LIMIT="" # optional +TAGS="" # optional +EXTRA_ARGS=() # passthrough +# Use local logs if PROLE_LOGS is not writable or looks like a remote path +LOG_BASE="${ROOT_DIR}/.ansible/logs" +if [[ -n "${PROLE_LOGS:-}" ]]; then + # If it's a relative path, or it exists and is writable, or its parent is writable + if [[ ! "${PROLE_LOGS}" =~ ^/ ]] || [[ -w "${PROLE_LOGS}" ]] || [[ -w "$(dirname "${PROLE_LOGS}" 2>/dev/null)" ]]; then + LOG_BASE="${PROLE_LOGS}" + fi +fi +LOG_BASE="${LOG_BASE%/}" +LOG_DIR="${LOG_BASE}/ansible" +SYSLOG_HOST="" # e.g. loghost.prole.org +SYSLOG_PORT="514" +SYSLOG_TAG="ansible" + +usage() { + cat <] + +Options: + -p, --playbook PATH Playbook path (default: ${PLAYBOOK}) + -l, --limit HOSTS Limit hosts + -t, --tags TAGS Tags + -v, --vault-pass-file FILE Vault password file + -m, --mode fg|bg Run in foreground or background (default: fg) + --syslog-host HOST Send start/end markers via UDP syslog to HOST + --syslog-port PORT Syslog UDP port (default: 514) + --syslog-tag TAG Syslog tag (default: ansible) + -h, --help Show help + +Examples: + ./ansible.sh -l pi.prole.org -t iscsi -v .vault_pass + ./ansible.sh -m bg -p infrastructure/playbooks/site.yml -v .vault_pass + ./ansible.sh --syslog-host loghost.prole.org -m bg -v .vault_pass -- -vv +EOF +} + +send_syslog() { + local msg="$1" + if [[ -n "${SYSLOG_HOST}" ]]; then + # -d = UDP, -n host, -P port + logger -d -n "${SYSLOG_HOST}" -P "${SYSLOG_PORT}" -t "${SYSLOG_TAG}" -- "${msg}" || true + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -p|--playbook) PLAYBOOK="$2"; shift 2 ;; + -l|--limit) LIMIT="$2"; shift 2 ;; + -t|--tags) TAGS="$2"; shift 2 ;; + -v|--vault-pass-file) VAULT_PASS_FILE="$2"; shift 2 ;; + -m|--mode) MODE="$2"; shift 2 ;; + --syslog-host) SYSLOG_HOST="$2"; shift 2 ;; + --syslog-port) SYSLOG_PORT="$2"; shift 2 ;; + --syslog-tag) SYSLOG_TAG="$2"; shift 2 ;; + --) shift; EXTRA_ARGS+=("$@"); break ;; + -h|--help) usage; exit 0 ;; + *) EXTRA_ARGS+=("$1"); shift ;; + esac +done + +# Default to k3s hosts for the main site run (avoid touching non-k3s Linux/Pi hosts unless explicitly requested) +if [[ -z "${LIMIT}" ]]; then + if [[ "${PLAYBOOK}" =~ (^|/)infrastructure/playbooks/site\.yml$ ]]; then + LIMIT="k3s_hosts" + fi +fi + +mkdir -p "${LOG_DIR}" + +ts="$(date +%Y%m%d-%H%M%S)" +logfile="${LOG_DIR}/ansible-${ts}.log" + +cmd=(ansible-playbook "${PLAYBOOK}") +[[ -n "${LIMIT}" ]] && cmd+=("--limit" "${LIMIT}") +[[ -n "${TAGS}" ]] && cmd+=("--tags" "${TAGS}") +[[ -n "${VAULT_PASS_FILE}" ]] && cmd+=("--vault-password-file" "${VAULT_PASS_FILE}") +cmd+=("${EXTRA_ARGS[@]}") + +send_syslog "START playbook=${PLAYBOOK} limit=${LIMIT:-} tags=${TAGS:-} log=${logfile}" + +echo "ANSIBLE_CONFIG=${ANSIBLE_CONFIG}" +echo "LOGFILE=${logfile}" +echo "CMD: ${cmd[*]}" + +if [[ "${MODE}" == "fg" ]]; then + # Stream to terminal and file + color_env=() + use_pty=false + if [[ -z "${NO_COLOR:-}" ]] && [[ -z "${ANSIBLE_NOCOLOR:-}" ]]; then + [[ -z "${ANSIBLE_FORCE_COLOR:-}" ]] && color_env+=("ANSIBLE_FORCE_COLOR=true") + [[ -z "${PY_COLORS:-}" ]] && color_env+=("PY_COLORS=1") + + # Ensure a useful terminal type for ANSI colors when invoked from wrappers. + if [[ -z "${TERM:-}" ]] || [[ "${TERM}" == "dumb" ]]; then + color_env+=("TERM=xterm-256color") + fi + + if command -v script >/dev/null 2>&1; then + use_pty=true + fi + fi + + run_cmd=("${cmd[@]}") + if [[ "${use_pty}" == "true" ]]; then + # `tee` breaks TTY detection; wrap in a pseudo-tty so Ansible keeps colors. + run_cmd=(script -qF /dev/null "${cmd[@]}") + fi + + set +e + env "${color_env[@]}" "${run_cmd[@]}" 2>&1 | tee "${logfile}" + rc=${PIPESTATUS[0]} + set -e +else + # Background: nohup to logfile + nohup "${cmd[@]}" >"${logfile}" 2>&1 & + rc=0 + echo "Started in background (pid $!)" +fi + +if [[ "${MODE}" == "fg" ]]; then + if [[ $rc -eq 0 ]]; then + send_syslog "END OK playbook=${PLAYBOOK} limit=${LIMIT:-} tags=${TAGS:-} log=${logfile}" + else + send_syslog "END FAIL rc=${rc} playbook=${PLAYBOOK} limit=${LIMIT:-} tags=${TAGS:-} log=${logfile}" + fi + exit $rc +fi diff --git a/ansible_min.cfg b/ansible_min.cfg new file mode 100644 index 0000000..88a7a55 --- /dev/null +++ b/ansible_min.cfg @@ -0,0 +1,5 @@ +[defaults] +stdout_callback = default +interpreter_python = auto_silent +host_key_checking = False +forks = 1 diff --git a/infrastructure/ansible.cfg b/infrastructure/ansible.cfg new file mode 100644 index 0000000..41fef95 --- /dev/null +++ b/infrastructure/ansible.cfg @@ -0,0 +1,25 @@ +[defaults] +# Note: When running from the project root, the root ansible.cfg is used. +# This file is for running ansible-playbook from within the infrastructure/ directory. +inventory = inventory/hosts.ini +roles_path = roles +collections_paths = collections +interpreter_python = auto_silent +callback_plugins = ../lib/ansible/plugins/callback +callbacks_enabled = run_logger +# For Ansible < 2.13, we use community.general.yaml +# For Ansible >= 2.13, we should use result_format=yaml in ansible.builtin.default +# To maintain compatibility, we use the default callback and set result_format +stdout_callback = default +result_format = yaml +forks = 20 +timeout = 30 +host_key_checking = True + +[privilege_escalation] +become = True +become_method = sudo +become_ask_pass = False + +[ssh_connection] +pipelining = True diff --git a/infrastructure/deployments/svc-check-helm/Chart.yaml b/infrastructure/deployments/svc-check-helm/Chart.yaml new file mode 100644 index 0000000..fed01a2 --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: svc-check +description: Static svc.prole.org check site served via Kong +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/infrastructure/deployments/svc-check-helm/templates/_helpers.tpl b/infrastructure/deployments/svc-check-helm/templates/_helpers.tpl new file mode 100644 index 0000000..12741a9 --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "svc-check.name" -}} +svc-check +{{- end -}} + +{{- define "svc-check.fullname" -}} +{{- printf "%s" (include "svc-check.name" .) -}} +{{- end -}} diff --git a/infrastructure/deployments/svc-check-helm/templates/content-configmap.yaml b/infrastructure/deployments/svc-check-helm/templates/content-configmap.yaml new file mode 100644 index 0000000..568e9f5 --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/templates/content-configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-check-content + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + app: svc-check-web +data: + index.html: |- +{{ .Values.svcCheck.html | nindent 4 }} +binaryData: + prole-type.gif: {{ .Values.svcCheck.gifB64 | default "" | quote }} diff --git a/infrastructure/deployments/svc-check-helm/templates/deployment.yaml b/infrastructure/deployments/svc-check-helm/templates/deployment.yaml new file mode 100644 index 0000000..48f271e --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/templates/deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: svc-check-web + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + app: svc-check-web +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: svc-check-web + template: + metadata: + labels: + app: svc-check-web + spec: + containers: + - name: nginx + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + volumeMounts: + - name: content + mountPath: /usr/share/nginx/html + volumes: + - name: content + configMap: + name: svc-check-content + items: + - key: index.html + path: index.html + - key: prole-type.gif + path: prole-type.gif diff --git a/infrastructure/deployments/svc-check-helm/templates/ingress.yaml b/infrastructure/deployments/svc-check-helm/templates/ingress.yaml new file mode 100644 index 0000000..467168a --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/templates/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: svc-prole-ingress + namespace: {{ .Values.kong.namespace }} + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.className }} + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} + traefik.ingress.kubernetes.io/router.priority: "10" +spec: + ingressClassName: {{ .Values.ingress.className }} + tls: + - hosts: + - {{ .Values.svcCheck.domain }} + secretName: {{ .Values.ingress.tlsSecretName }} + rules: + - host: {{ .Values.svcCheck.domain }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Values.kong.proxyServiceName }} + port: + number: 8000 diff --git a/infrastructure/deployments/svc-check-helm/templates/kong-configmap.yaml b/infrastructure/deployments/svc-check-helm/templates/kong-configmap.yaml new file mode 100644 index 0000000..c698255 --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/templates/kong-configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.kong.configMapName }} + namespace: {{ .Values.kong.namespace }} +data: + kong.yml: | + _format_version: "3.0" + _transform: true + services: + - name: kubeconfig + url: http://prole-svc.knoe-db.svc.cluster.local:8080 + routes: + - name: kubeconfig + paths: + - /k3s/kube_config.sh + - name: svc-check + url: http://svc-check-web.{{ .Values.namespace | default .Release.Namespace }}.svc.cluster.local:80 + routes: + - name: svc-check-root + hosts: + - {{ .Values.svcCheck.domain }} + paths: + - / + strip_path: false diff --git a/infrastructure/deployments/svc-check-helm/templates/namespace.yaml b/infrastructure/deployments/svc-check-helm/templates/namespace.yaml new file mode 100644 index 0000000..d6c4473 --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/templates/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.namespace | default .Release.Namespace }} diff --git a/infrastructure/deployments/svc-check-helm/templates/service.yaml b/infrastructure/deployments/svc-check-helm/templates/service.yaml new file mode 100644 index 0000000..d1bf007 --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: svc-check-web + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + app: svc-check-web +spec: + selector: + app: svc-check-web + ports: + - name: http + port: 80 + targetPort: http diff --git a/infrastructure/deployments/svc-check-helm/values.yaml b/infrastructure/deployments/svc-check-helm/values.yaml new file mode 100644 index 0000000..35b3bed --- /dev/null +++ b/infrastructure/deployments/svc-check-helm/values.yaml @@ -0,0 +1,47 @@ +namespace: svc-check + +replicaCount: 1 + +image: + repository: nginx + tag: "1.27-alpine" + pullPolicy: IfNotPresent + +svcCheck: + domain: svc.prole.org + title: svc.prole.org + subtitle: served by k3s -> Kong on myrddin.prole.org + gifB64: "" + html: |- + + + + + svc.prole.org + + + +
+ Prole type +

svc.prole.org

+

served by k3s -> Kong on myrddin.prole.org

+
+ + + +kong: + namespace: default + configMapName: prole-svc-kong-config + proxyServiceName: prole-svc-kong + +ingress: + className: traefik + clusterIssuer: letsencrypt-prod + tlsSecretName: svc-prole-org-tls diff --git a/infrastructure/inventory/group_vars/ad_dc/vars.yml b/infrastructure/inventory/group_vars/ad_dc/vars.yml new file mode 100644 index 0000000..4906c65 --- /dev/null +++ b/infrastructure/inventory/group_vars/ad_dc/vars.yml @@ -0,0 +1,8 @@ +--- +samba_dns_server: "127.0.0.1" +samba_dns_admin_user: "administrator@PROLE.ORG" + +# We'll wire the password with vault next +samba_dns_admin_pass: "{{ vault_samba_dns_admin_pass }}" + +rsyslog_role: receiver diff --git a/infrastructure/inventory/group_vars/ad_dc/vault.yml b/infrastructure/inventory/group_vars/ad_dc/vault.yml new file mode 100644 index 0000000..2357dc5 --- /dev/null +++ b/infrastructure/inventory/group_vars/ad_dc/vault.yml @@ -0,0 +1,8 @@ +$ANSIBLE_VAULT;1.1;AES256 +39636637353364306639373863656430333863373635663936373464303431643761393034373061 +3365323233353233393437333535636136386162353231320a303431336432316464346538303366 +34323662643737623963313431613930646131653130663762633130626162386435656239386435 +3566323235666531330a306366616438653230383438626630626236363839376138376336626634 +34646532363036303761383939303838373136663562386661366363316666623837616661303831 +65363062313533383936323864316336633130313237366661663133663234643462356136656432 +326565363832386462393864393434656337 diff --git a/infrastructure/inventory/group_vars/all/dns.yml b/infrastructure/inventory/group_vars/all/dns.yml new file mode 100644 index 0000000..f025201 --- /dev/null +++ b/infrastructure/inventory/group_vars/all/dns.yml @@ -0,0 +1,202 @@ +--- +prole_domain: prole.org + +# Source of truth imported from name.com export (A/CNAME/MX/TXT) +# Notes: +# - "name: '@'" means the zone apex (prole.org) +# - no IPv6 records included (per request) +prole_dns_records: + - name: aventage + type: A + value: 10.0.0.206 + ttl: 300 + + - name: book.svc + type: A + value: 185.158.133.1 + ttl: 300 + + - name: fairyland + type: A + value: 10.0.0.208 + ttl: 300 + + - name: k8s + type: CNAME + value: zinfandel.prole.org + ttl: 300 + + - name: loghost + type: A + value: 10.0.0.3 + ttl: 300 + + - name: mc + type: A + value: 73.15.20.166 + ttl: 300 + + - name: merlin + type: A + value: 10.0.0.6 + ttl: 300 + + - name: morana + type: A + value: 10.0.0.66 + ttl: 300 + + - name: morgoth + type: A + value: 10.0.0.204 + ttl: 300 + + - name: myrddin + type: A + value: 10.0.0.3 + ttl: 300 + + - name: ollama + type: A + value: 73.15.20.166 + ttl: 300 + + - name: pi + type: A + value: 10.0.0.5 + ttl: 300 + + - name: "@" + type: MX + value: alt1.aspmx.l.google.com + ttl: 3600 + priority: 5 + + - name: "@" + type: MX + value: alt2.aspmx.l.google.com + ttl: 3600 + priority: 5 + + - name: "@" + type: MX + value: alt3.aspmx.l.google.com + ttl: 3600 + priority: 10 + + - name: "@" + type: MX + value: alt4.aspmx.l.google.com + ttl: 3600 + priority: 10 + + - name: "@" + type: MX + value: aspmx.l.google.com + ttl: 3600 + priority: 1 + + - name: raspberry + type: A + value: 10.0.0.4 + ttl: 300 + + - name: retropie + type: A + value: 10.0.0.207 + ttl: 300 + + - name: svc + type: A + value: 73.15.20.166 + ttl: 300 + + - name: synology + type: A + value: 10.0.0.203 + ttl: 300 + + - name: www + type: CNAME + value: ghs.googlehosted.com + ttl: 300 + + - name: zinfandel + type: A + value: 10.0.0.205 + ttl: 300 + + - name: _lovable.book.svc + type: TXT + value: lovable_verify=a6369596ca0ae0b00fde155591c566c05a9844302a70727ba69a74b56c58927f + ttl: 300 + +# Traefik Load Balancer addresses for k3s front-door +prole_traefik_lb_ips: + - 10.0.0.3 + - 10.0.0.6 + +# k3s Front Door names (normalized to prole.org) +prole_k3s_front_door_names: + - git + - svc + - api + - db + - supabase + - registry + +# Convenience list: internal RFC1918 A records only (useful for Samba AD DNS population) +prole_internal_a_records: + - fqdn: aventage.prole.org + ipv4s: [10.0.0.206] + - fqdn: fairyland.prole.org + ipv4s: [10.0.0.208] + - fqdn: loghost.prole.org + ipv4s: [10.0.0.3] + - fqdn: merlin.prole.org + ipv4s: [10.0.0.6] + - fqdn: morana.prole.org + ipv4s: [10.0.0.66] + - fqdn: morgoth.prole.org + ipv4s: [10.0.0.204] + - fqdn: myrddin.prole.org + ipv4s: [10.0.0.3] + - fqdn: pi.prole.org + ipv4s: [10.0.0.5] + - fqdn: raspberry.prole.org + ipv4s: [10.0.0.4] + - fqdn: retropie.prole.org + ipv4s: [10.0.0.207] + - fqdn: synology.prole.org + ipv4s: [10.0.0.203] + - fqdn: zinfandel.prole.org + ipv4s: [10.0.0.205] + +# k3s front-door records (public and internal.prole.org) +prole_k3s_dns_records: + - fqdn: git.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: git.internal.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: svc.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: svc.internal.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: api.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: api.internal.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: db.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: db.internal.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: supabase.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + - fqdn: supabase.internal.prole.org + ipv4s: [10.0.0.3, 10.0.0.6] + +prole_k3s_cname_records: + - fqdn: registry.prole.org + target: svc.prole.org + - fqdn: registry.internal.prole.org + target: svc.internal.prole.org diff --git a/infrastructure/inventory/group_vars/all/k3s.yml b/infrastructure/inventory/group_vars/all/k3s.yml new file mode 100644 index 0000000..d5407c1 --- /dev/null +++ b/infrastructure/inventory/group_vars/all/k3s.yml @@ -0,0 +1,27 @@ +k3s_state: present # present|stopped|absent +k3s_version: "" # e.g. "v1.29.7+k3s1" or empty for latest +k3s_disable: + - local-storage + +k3s_write_kubeconfig_mode: "0640" +k3s_kubeconfig_group: kubeadm +k3s_kubeconfig_users: + - pi + +# K3s state storage +# `/var/lib/rancher` is treated as local host storage for all k3s nodes. +# (On `myrddin.prole.org` this is backed by a locally attached USB3 SSD; shared/iSCSI +# storage is reserved for other data paths such as `/prole/*`.) +k3s_rancher_mountpoint: /var/lib/rancher +k3s_rancher_storage_class: local +k3s_data_dir: "{{ k3s_rancher_mountpoint }}/k3s" + +# Post-provisioning, persisted labels applied via kubectl (delegated to control-plane) +k3s_node_labels: + myrddin.prole.org: + prole.org/storage: fast + prole.org/ingress: primary + prole.org/stateful: "true" + merlin.prole.org: + prole.org/role: stateless + prole.org/storage: slow diff --git a/infrastructure/inventory/group_vars/all/production.yml b/infrastructure/inventory/group_vars/all/production.yml new file mode 100644 index 0000000..9ab9b48 --- /dev/null +++ b/infrastructure/inventory/group_vars/all/production.yml @@ -0,0 +1,10 @@ +--- +# Production service inventory (pre-orchestration reference list). +# Keep this list minimal and explicit; production currently only includes Pi-hole. +production_services: + - name: pihole + hosts: + - pi.prole.org + - raspberry.prole.org + dependencies: + - iscsi diff --git a/infrastructure/inventory/group_vars/all/prole_vault.yml b/infrastructure/inventory/group_vars/all/prole_vault.yml new file mode 100644 index 0000000..f596c3a --- /dev/null +++ b/infrastructure/inventory/group_vars/all/prole_vault.yml @@ -0,0 +1,7 @@ +$ANSIBLE_VAULT;1.1;AES256 +64373935366135353230383931313131666463323262663363623934346165653933323636386635 +6437306634623062323464646431323333306464316634620a306466353266663262396435613537 +37383737653930633932336633666430653462343634363838383439666163326638616264323562 +6138393633396631650a643864373833656232393164616138386638373932393363363731383033 +65366163323932633033376338646231656264363431636334656363336462616665346634613966 +3637646539373535353234653461613036353435626561396466 diff --git a/infrastructure/inventory/group_vars/all/vars.yml b/infrastructure/inventory/group_vars/all/vars.yml new file mode 100644 index 0000000..21c624e --- /dev/null +++ b/infrastructure/inventory/group_vars/all/vars.yml @@ -0,0 +1,35 @@ +--- +ansible_user: ansible +ansible_ssh_private_key_file: "~/.ssh/id_ed25519_ansible" + +ansible_become: true +ansible_become_method: sudo + +# DNS topology vars shared across roles +prole_domain: "prole.org" +ad_dc_ip: "10.0.0.3" +lan_reverse_zone: "0.0.10.in-addr.arpa" + +# k3s default Pod/Service CIDR reverse zones +k3s_reverse_zones: + - "42.10.in-addr.arpa" + - "43.10.in-addr.arpa" + +# Seed PTRs (adjust octets to match your actual IPs) +ptr_records: + - last_octet: "3" + fqdn: "myrddin.prole.org" + - last_octet: "4" + fqdn: "raspberry.prole.org" + - last_octet: "5" + fqdn: "pi.prole.org" + - last_octet: "207" + fqdn: "retropie.prole.org" + +rsyslog_server: myrddin.prole.org + +# Prole management +prole_home: "/opt/prole" +prole_repo_url: "https://github.com/prole-org/prole.git" +prole_version: "main" +prole_logs_dir: "/opt/prole/logs/chrisfu" diff --git a/infrastructure/inventory/group_vars/all/vault_db_master.yml b/infrastructure/inventory/group_vars/all/vault_db_master.yml new file mode 100644 index 0000000..9f94788 --- /dev/null +++ b/infrastructure/inventory/group_vars/all/vault_db_master.yml @@ -0,0 +1,8 @@ +$ANSIBLE_VAULT;1.1;AES256 +33623434353765353030333339626631656163343239353230643430356466306461663835383566 +3564333737623034343837616632386462306462366538390a623538373034383839376261653734 +36653238393437656332373965663866653730343864333063366462303661356366323262363839 +3961633266353131310a313930346164393031623037393339356539616639343536353163343036 +61353535633234363535633437356162626234356139323531643534613961633166393135356562 +30306634646130353665656262393132656632373634353765316630643665356331363435366165 +356261383336383736336336356564386337 diff --git a/infrastructure/inventory/group_vars/all/vault_k3s.yml b/infrastructure/inventory/group_vars/all/vault_k3s.yml new file mode 100644 index 0000000..42a1bd3 --- /dev/null +++ b/infrastructure/inventory/group_vars/all/vault_k3s.yml @@ -0,0 +1 @@ +vault_k3s_token: "K107c8c6000488eca4a067d8a73119bbae2f07b4ea1bac7d8d3dc9c500cbb8acb18::server:04572345810eae2f9619a6ed4239702b" diff --git a/infrastructure/inventory/group_vars/iscsi/vars.yml b/infrastructure/inventory/group_vars/iscsi/vars.yml new file mode 100644 index 0000000..4c54153 --- /dev/null +++ b/infrastructure/inventory/group_vars/iscsi/vars.yml @@ -0,0 +1,7 @@ +--- +# iSCSI (override per-host) +iscsi_portal: 10.0.0.203 +iscsi_chap_user: "prole" +iscsi_chap_password: "" +iscsi_target_iqn: "" +iscsi_device_hint: "" # optional: /dev/disk/by-path/... or leave empty diff --git a/infrastructure/inventory/group_vars/iscsi/vault.yml b/infrastructure/inventory/group_vars/iscsi/vault.yml new file mode 100644 index 0000000..06004df --- /dev/null +++ b/infrastructure/inventory/group_vars/iscsi/vault.yml @@ -0,0 +1,7 @@ +$ANSIBLE_VAULT;1.1;AES256 +38323535303264613662616137643166323864306366663631346332383333313638303863383363 +6536636562663531313431376664313562323966383964330a636361666235623337346636343936 +35313563393764623162386666333133653536643230313431633766346436343430353364333537 +3033396638346431610a356533346663336332326633396134656466396331336566663165633439 +38326531313532383361613535396666333165313536633035323934353666656332356530363430 +6237616366346364626537326132613666613232373864643230 diff --git a/infrastructure/inventory/group_vars/mariadb/vars.yml b/infrastructure/inventory/group_vars/mariadb/vars.yml new file mode 100644 index 0000000..73ae5dd --- /dev/null +++ b/infrastructure/inventory/group_vars/mariadb/vars.yml @@ -0,0 +1,10 @@ +--- +# MariaDB provisioning (non-secret defaults). + +# Remote TCP admin user created by `mariadb_primary` for management. +k3s_mariadb_admin_user: prole_admin + +# Keep the k3s datastore password vaulted. +# For consistency with existing k3s server configuration, reuse the AD DC's vaulted Samba DNS admin password. +# (This ensures k3s can authenticate to the datastore after migration.) +k3s_datastore_mariadb_password: "{{ hostvars[groups['ad_dc'][0]].vault_samba_dns_admin_pass }}" \ No newline at end of file diff --git a/infrastructure/inventory/group_vars/mariadb/vault.yml b/infrastructure/inventory/group_vars/mariadb/vault.yml new file mode 100644 index 0000000..ad55dc2 --- /dev/null +++ b/infrastructure/inventory/group_vars/mariadb/vault.yml @@ -0,0 +1,9 @@ +$ANSIBLE_VAULT;1.1;AES256 +62353131343066373436383738303262393762653331356334376236343636643531643666373965 +3832393033653232313165636239326163336138643134310a343639346433353933363836616561 +37303838303230663632313430613231323162376436333766316530323436666161343935363537 +3638653238316261650a383931353462356636366663316532636230656639363165376136326138 +38616265346438333465306539643433313464633633363965303366326362323632643032633361 +62386261333634626131333832383339636537663261393662613861663032336330383737626161 +62313533396231623561653133303832663765646635386230343662336231353432326437363132 +33363361396433623533 diff --git a/infrastructure/inventory/group_vars/pihole.yml b/infrastructure/inventory/group_vars/pihole.yml new file mode 100644 index 0000000..77acafa --- /dev/null +++ b/infrastructure/inventory/group_vars/pihole.yml @@ -0,0 +1,13 @@ +--- +# Your internal domain used by Samba AD (split-horizon is fine) +prole_domain: "prole.org" + +# Samba AD DC IP +ad_dc_ip: "10.0.0.3" + +# Reverse zone for 10.0.0.0/24 +lan_reverse_zone: "0.0.10.in-addr.arpa" + +# Performance/resilience knobs +pihole_dns_forward_max: 300 +pihole_sqlite_busy_timeout: 5000 diff --git a/infrastructure/inventory/host_vars/gandalf.prole.org.yml b/infrastructure/inventory/host_vars/gandalf.prole.org.yml new file mode 100644 index 0000000..c04a24a --- /dev/null +++ b/infrastructure/inventory/host_vars/gandalf.prole.org.yml @@ -0,0 +1,38 @@ +--- +hostname: gandalf +fqdn: gandalf.prole.org +ansible_host: 10.0.0.7 +ansible_user: ansible + +k3s_enabled: true +k3s_role: agent +k3s_cluster_init: false +k3s_server_url: "https://myrddin.prole.org:6443" + +k3s_service_node_labels: + - "prole.org/node-role=general" +k3s_node_taints: [] + +k3s_rancher_mount_required: true +k3s_rancher_mount_src: /external/rancher +k3s_rancher_mount_fstype: none +k3s_rancher_mount_opts: bind +k3s_rancher_mount_passno: 0 + +k3s_required_mounts: + - /var/lib/rancher + - /synology/d005 + +iscsi_portal: 10.0.0.203:3260 + +iscsi_targets: + # PROLE-DATA-5 + - iqn: "iqn.2000-01.com.synology:synology.Target-14.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - name: d005 + path: /synology/d005 + device: /dev/disk/by-path/ip-10.0.0.203:3260-iscsi-iqn.2000-01.com.synology:synology.Target-14.292d45194a1-lun-1-part1 + fstype: ext4 + opts: "_netdev,noatime,nofail" \ No newline at end of file diff --git a/infrastructure/inventory/host_vars/localhost.yml b/infrastructure/inventory/host_vars/localhost.yml new file mode 100644 index 0000000..c48a430 --- /dev/null +++ b/infrastructure/inventory/host_vars/localhost.yml @@ -0,0 +1,3 @@ +--- +ansible_become: false +ansible_connection: local diff --git a/infrastructure/inventory/host_vars/merlin.prole.org.yml b/infrastructure/inventory/host_vars/merlin.prole.org.yml new file mode 100644 index 0000000..d8c0549 --- /dev/null +++ b/infrastructure/inventory/host_vars/merlin.prole.org.yml @@ -0,0 +1,85 @@ +--- +ansible_host: 10.0.0.6 + +netplan_static_enabled: true +netplan_static_iface: eth0 +netplan_static_address: 10.0.0.6/24 +netplan_static_gateway4: 10.0.0.1 +netplan_static_nameservers: + - 10.0.0.1 + +# MariaDB primary (USB-backed) + +# Permanent portable USB disk for MariaDB (merlin) +# - Disk: UUID=3669ddf4-5884-43ae-84b2-150ec117a2b1 (ext4, ~1.82T) +# - Mount: /external (by UUID; device name varies after USB re-enumeration) +# - MariaDB datadir is expected at /srv/mariadb/mariadb, but is bind-mounted to /external/mariadb +# - Replaced borrowed WD My Book (exFAT) after disk-full kine deadlock incident (2026-04-03) +mariadb_external_enabled: true +mariadb_external_device: /dev/disk/by-uuid/3669ddf4-5884-43ae-84b2-150ec117a2b1 +mariadb_external_mountpoint: /external +mariadb_external_fstype: ext4 +# ext4 supports POSIX ownership; chown -R mysql:mysql was applied during migration from sda1. +mariadb_external_mount_opts: "defaults,nofail,x-systemd.device-timeout=10" + +mariadb_external_src_datadir: /srv/mariadb/mariadb +mariadb_external_dst_datadir: /external/mariadb +mariadb_external_backup_datadir: /srv/mariadb/mariadb.pre-external +mariadb_external_migration_marker: /external/mariadb/.prole-mariadb-external-migrated + +# k3s worker (stateless workloads) +k3s_enabled: true + +k3s_role: agent +k3s_cluster_init: false +k3s_server_url: "https://myrddin.prole.org:6443" + +# k3s datastore now lives on merlin. +# NOTE: Use IP to avoid DNS drift during migration/cutover. +k3s_datastore_mariadb_host: 10.0.0.6 +k3s_datastore_mariadb_port: 3306 +k3s_datastore_mariadb_db: k3s +k3s_datastore_mariadb_user: prole_k3s +k3s_datastore_endpoint: "mysql://{{ k3s_datastore_mariadb_user }}:{{ k3s_datastore_mariadb_password | urlencode }}@tcp({{ k3s_datastore_mariadb_host }}:{{ k3s_datastore_mariadb_port }})/{{ k3s_datastore_mariadb_db }}" + +k3s_tls_sans: + - merlin.prole.org + +k3s_service_node_labels: + - "prole.org/node-role=general" +k3s_node_taints: [] + +k3s_required_mounts: + - /synology/d002 + - /synology/d004 + +# iSCSI +# `/var/lib/rancher` is local host storage (do not manage it via iSCSI). +iscsi_portal: 10.0.0.203:3260 + +iscsi_targets: + # PROLE-DATA-2 + - iqn: "iqn.2000-01.com.synology:synology.Target-12.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - name: d002 + path: /synology/d002 + fstype: xfs + opts: "_netdev,noatime" + src: "UUID=fe087fb0-a321-4767-b838-b4385e81693e" + + # PROLE-DATA-4 + # Repurposed from myrddin's former `/var/lib/rancher` Synology LUN. + # Repartitioned 2026-04-03: raw XFS on /dev/sdb → GPT + /dev/sdb1 (XFS). + - iqn: "iqn.2000-01.com.synology:synology.Target-15.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - name: d004 + path: /synology/d004 + fstype: xfs + opts: "_netdev,noatime" + src: "UUID=45143352-142d-4e47-8508-9eb84c4c1f29" + +iscsi_absent_mounts: [] diff --git a/infrastructure/inventory/host_vars/myrddin.prole.org.yml b/infrastructure/inventory/host_vars/myrddin.prole.org.yml new file mode 100644 index 0000000..c82a22b --- /dev/null +++ b/infrastructure/inventory/host_vars/myrddin.prole.org.yml @@ -0,0 +1,117 @@ +iscsi_portal: 10.0.0.203 + +iscsi_targets: + # PROLE-DATA-1 + - iqn: "iqn.2000-01.com.synology:synology.Target-11.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - path: /synology/d001 + fstype: xfs + opts: "_netdev,noatime" + src: "UUID=c07dc0f2-cf60-4da5-b68e-ea45e86d473e" + + # PROLE-HOME + - iqn: "iqn.2000-01.com.synology:synology.Target-17.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - path: /prole/home + fstype: ext4 + opts: "_netdev,noatime" + src: "UUID=607718c8-467a-474f-8e36-80ceafe1beb4" + + # PROLE-LOGS + - iqn: "iqn.2000-01.com.synology:synology.Target-1.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - path: /prole/logs + fstype: ext4 + opts: "_netdev,noatime" + src: "UUID=5335affd-b1ab-4134-a1c0-be88ab965309" + +iscsi_absent_mounts: + - /opt/prole/logs/chrisfu + # Migration cleanup: PROLE-DATA-2/3 are moving off myrddin + - /prole/d002 + - /prole/d003 + - /synology/d002 + - /synology/d003 + +# Migration cleanup: ensure myrddin fully logs out of LUNs that are moving to other hosts +iscsi_absent_targets: + # PROLE-DATA-2 + - "iqn.2000-01.com.synology:synology.Target-12.292d45194a1" + # PROLE-DATA-3 + - "iqn.2000-01.com.synology:synology.Target-13.292d45194a1" + +ad_dc_enabled: true + +k3s_enabled: true +k3s_cluster_init: false +k3s_role: server + +# K3s state storage (local) +# `/var/lib/rancher` is now backed by a locally attached USB3 SSD on myrddin. +# Shared/iSCSI storage remains reserved for `/prole/*` data paths. +k3s_rancher_mountpoint: /var/lib/rancher +k3s_rancher_storage_class: local_usb3_ssd + +# Guardrail: `/var/lib/rancher` must be a dedicated mount (external SSD) and k3s +# must never start if it resolves to the same backing device as `/`. +k3s_rancher_mount_required: true +k3s_rancher_mount_src: "UUID=2e457f8f-fc85-48e7-a7fc-29aad0059268" +k3s_rancher_mount_fstype: ext4 +k3s_rancher_mount_opts: "noatime" +k3s_rancher_mount_passno: 2 +k3s_token: "{{ vault_k3s_token | default('') }}" # store this in vault +k3s_datastore_mariadb_host: 10.0.0.6 +k3s_datastore_mariadb_port: 3306 +k3s_datastore_mariadb_db: k3s +k3s_datastore_mariadb_user: prole_k3s +# Password is the same as the Samba administrator password and must remain in Ansible Vault. +k3s_datastore_mariadb_password: "{{ vault_samba_dns_admin_pass }}" +k3s_datastore_endpoint: "mysql://{{ k3s_datastore_mariadb_user }}:{{ k3s_datastore_mariadb_password | urlencode }}@tcp({{ k3s_datastore_mariadb_host }}:{{ k3s_datastore_mariadb_port }})/{{ k3s_datastore_mariadb_db }}" +k3s_tls_sans: + - myrddin.prole.org +k3s_service_node_labels: + - "prole.org/node-role=db" + - "prole.org/role=control" + +# Keep general workloads off the control-plane node; allow only explicitly-tolerating pods. +k3s_node_taints: + - "node-role.kubernetes.io/control-plane:NoSchedule" + +# More forgiving IO/etcd timeouts to reduce error-correction churn on slow storage +k3s_kube_apiserver_args: + - "request-timeout=60s" + - "min-request-timeout=60" + +k3s_kube_controller_manager_args: + - "leader-elect-lease-duration=600s" + - "leader-elect-renew-deadline=420s" + - "leader-elect-retry-period=60s" + - "node-monitor-grace-period=10m" + - "node-startup-grace-period=10m" + +k3s_kube_scheduler_args: + - "leader-elect-lease-duration=600s" + - "leader-elect-renew-deadline=420s" + - "leader-elect-retry-period=60s" + +k3s_cloud_controller_manager_args: + - "leader-elect-lease-duration=900s" + - "leader-elect-renew-deadline=600s" + - "leader-elect-retry-period=90s" + +k3s_kubelet_args: + - "housekeeping-interval=30s" + +k3s_write_kubeconfig_mode: "0640" +k3s_kubeconfig_group: kubeadm +k3s_kubeconfig_users: + - pi + +k3s_required_mounts: + - /synology/d001 diff --git a/infrastructure/inventory/host_vars/pi.prole.org.yml b/infrastructure/inventory/host_vars/pi.prole.org.yml new file mode 100644 index 0000000..ac75b4e --- /dev/null +++ b/infrastructure/inventory/host_vars/pi.prole.org.yml @@ -0,0 +1,51 @@ +iscsi_portal: 10.0.0.203 + +k3s_required_mounts: + - /synology/d003 + +iscsi_targets: + # `/var/lib/rancher` is local host storage (do not manage it via iSCSI). + + # PROLE-DATA-3 + - iqn: "iqn.2000-01.com.synology:synology.Target-13.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - path: /synology/d003 + fstype: xfs + opts: "_netdev,noatime" + src: "UUID=757f1ee4-dc23-414b-b595-e3058c0744f0" + + # PROLE-PI-2 + - iqn: "iqn.2000-01.com.synology:synology.Target-19.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - path: /var/log/pihole + fstype: ext4 + opts: "_netdev,noatime" + src: "UUID=56b21ec3-2826-4171-b909-a6715223f9a4" + +# k3s intentionally disabled — pi.prole.org is a dedicated pihole node; +# insufficient RAM for k3s workloads; agent manually stopped 2026-04-01. +k3s_enabled: false +k3s_state: absent + +k3s_rancher_mount_required: true +k3s_rancher_mount_src: /synology/d003/rancher +k3s_rancher_mount_fstype: none +k3s_rancher_mount_opts: bind +k3s_rancher_mount_passno: 0 + +k3s_role: agent +k3s_cluster_init: false +k3s_server_url: "https://myrddin.prole.org:6443" +k3s_service_node_labels: + - "prole.org/node-role=general" + - "prole.org/role=observer" +k3s_node_taints: [] + +k3s_write_kubeconfig_mode: "0640" +k3s_kubeconfig_group: kubeadm +k3s_kubeconfig_users: + - pi diff --git a/infrastructure/inventory/host_vars/raspberry.prole.org.yml b/infrastructure/inventory/host_vars/raspberry.prole.org.yml new file mode 100644 index 0000000..8eb3be0 --- /dev/null +++ b/infrastructure/inventory/host_vars/raspberry.prole.org.yml @@ -0,0 +1,12 @@ +iscsi_portal: 10.0.0.203 + +iscsi_targets: + # PROLE-PI-1 + - iqn: "iqn.2000-01.com.synology:synology.Target-18.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - path: /var/log/pihole + fstype: ext4 + opts: "_netdev,noatime" + src: "UUID=17b375e9-c615-48bb-a3cd-b60672d67996" diff --git a/infrastructure/inventory/host_vars/retropie.prole.org.yml b/infrastructure/inventory/host_vars/retropie.prole.org.yml new file mode 100644 index 0000000..44ccb46 --- /dev/null +++ b/infrastructure/inventory/host_vars/retropie.prole.org.yml @@ -0,0 +1,8 @@ +iscsi_targets: [] +iscsi_absent_mounts: [] + +k3s_enabled: false + +cgroups_cmdline_candidates: + - /boot/cmdline.txt + - /boot/firmware/cmdline.txt diff --git a/infrastructure/inventory/hosts.ini b/infrastructure/inventory/hosts.ini new file mode 100644 index 0000000..15f25c4 --- /dev/null +++ b/infrastructure/inventory/hosts.ini @@ -0,0 +1,50 @@ +[iscsi] +pi.prole.org +raspberry.prole.org +myrddin.prole.org +retropie.prole.org +merlin.prole.org +gandalf.prole.org + +[pihole] +pi.prole.org +raspberry.prole.org + +[ad_dc] +myrddin.prole.org + +[k3s_servers] +myrddin.prole.org + +[k3s_agents] +merlin.prole.org +gandalf.prole.org + +[k3s_hosts:children] +k3s_servers +k3s_agents + +[linux_hosts] +pi.prole.org +raspberry.prole.org +myrddin.prole.org +retropie.prole.org +merlin.prole.org +gandalf.prole.org + +[ssl_hosts] +myrddin.prole.org + +[mariadb_primary] +merlin.prole.org + +[mariadb_replica] +raspberry.prole.org + +[mariadb:children] +mariadb_primary +mariadb_replica + +[merlin_bootstrap] +merlin ansible_host=10.0.0.36 ansible_user=ansible + diff --git a/infrastructure/playbooks/audit.yml b/infrastructure/playbooks/audit.yml new file mode 100644 index 0000000..dc1e348 --- /dev/null +++ b/infrastructure/playbooks/audit.yml @@ -0,0 +1,9 @@ +--- +- name: Debian audit mode + hosts: linux_hosts + become: true + gather_facts: true + check_mode: true + diff: true + roles: + - audit diff --git a/infrastructure/playbooks/bootstrap_local_ansible_user.yml b/infrastructure/playbooks/bootstrap_local_ansible_user.yml new file mode 100644 index 0000000..476f62b --- /dev/null +++ b/infrastructure/playbooks/bootstrap_local_ansible_user.yml @@ -0,0 +1,24 @@ +--- +- name: Bootstrap local ansible user + key + sudo + hosts: all + become: true + tasks: + - name: Ensure local ansible user exists + ansible.builtin.user: + name: ansible + shell: /bin/bash + create_home: true + + - name: Install authorized key for local ansible user + ansible.posix.authorized_key: + user: ansible + state: present + key: "{{ lookup('file', lookup('env','HOME') + '/.ssh/id_ed25519_ansible.pub') }}" + + - name: Allow passwordless sudo for local ansible + ansible.builtin.copy: + dest: /etc/sudoers.d/90-ansible + content: "ansible ALL=(ALL) NOPASSWD:ALL\n" + owner: root + group: root + mode: "0440" diff --git a/infrastructure/playbooks/cgroups.yml b/infrastructure/playbooks/cgroups.yml new file mode 100644 index 0000000..7fe713a --- /dev/null +++ b/infrastructure/playbooks/cgroups.yml @@ -0,0 +1,7 @@ +--- +- name: Ensure cgroup kernel parameters + hosts: k3s_hosts + become: true + serial: 1 + roles: + - cgroups diff --git a/infrastructure/playbooks/check_k3s_endpoint.yml b/infrastructure/playbooks/check_k3s_endpoint.yml new file mode 100644 index 0000000..dcf1f7c --- /dev/null +++ b/infrastructure/playbooks/check_k3s_endpoint.yml @@ -0,0 +1,59 @@ +--- +- name: Validate k3s API endpoint reachability + hosts: k3s_hosts + gather_facts: false + become: false + vars: + k3s_default_port: 6443 + tasks: + - name: Select k3s init host + ansible.builtin.set_fact: + k3s_init_host: "{{ item }}" + when: hostvars[item].k3s_cluster_init | default(false) | bool + loop: "{{ groups['k3s_hosts'] }}" + run_once: true + + - name: Select k3s server URL (if provided) + ansible.builtin.set_fact: + k3s_api_url: "{{ hostvars[item].k3s_server_url }}" + when: + - hostvars[item].k3s_server_url is defined + - hostvars[item].k3s_server_url | length > 0 + loop: "{{ groups['k3s_hosts'] }}" + run_once: true + + - name: Fallback to init host URL + ansible.builtin.set_fact: + k3s_api_url: "https://{{ k3s_init_host | default(groups['k3s_hosts'][0]) }}:{{ k3s_default_port }}" + when: k3s_api_url is not defined or k3s_api_url | length == 0 + run_once: true + + - name: Extract k3s API host + ansible.builtin.set_fact: + k3s_api_host: "{{ k3s_api_url | regex_replace('^https?://', '') | regex_replace(':.*$', '') }}" + run_once: true + + - name: Check TCP port 6443 + ansible.builtin.wait_for: + host: "{{ k3s_api_host }}" + port: "{{ k3s_default_port }}" + timeout: 10 + delegate_to: localhost + become: false + run_once: true + + - name: Check /readyz endpoint + ansible.builtin.uri: + url: "{{ k3s_api_url }}/readyz" + method: GET + status_code: [200, 401, 403] + validate_certs: false + timeout: 5 + delegate_to: localhost + become: false + run_once: true + + - name: Report k3s API endpoint + ansible.builtin.debug: + msg: "k3s API reachable at {{ k3s_api_url }}" + run_once: true diff --git a/infrastructure/playbooks/disable_pi_k3s.yml b/infrastructure/playbooks/disable_pi_k3s.yml new file mode 100644 index 0000000..ccc37eb --- /dev/null +++ b/infrastructure/playbooks/disable_pi_k3s.yml @@ -0,0 +1,86 @@ +--- +# Disable k3s-agent on pi.prole.org and remove it from the cluster. +# Safe to run against a live cluster — only touches pi.prole.org. +# Storage (iSCSI) and pihole configuration are intentionally preserved. +# +# Usage: +# ansible-playbook -i inventory/hosts.ini playbooks/disable_pi_k3s.yml +# ansible-playbook -i inventory/hosts.ini playbooks/disable_pi_k3s.yml --check + +- name: Drain and delete pi node from k3s control-plane + hosts: k3s_servers + gather_facts: false + become: true + tasks: + - name: Drain pi.prole.org (evict pods, ignore DaemonSets) + ansible.builtin.command: + cmd: kubectl drain pi.prole.org + --ignore-daemonsets + --delete-emptydir-data + --force + --timeout=120s + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: _drain + changed_when: _drain.rc == 0 + failed_when: false + + - name: Delete pi node from cluster + ansible.builtin.command: + cmd: kubectl delete node pi.prole.org --ignore-not-found + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: _delete_node + changed_when: "'deleted' in _delete_node.stdout" + failed_when: false + +- name: Stop and disable k3s-agent on pi.prole.org + hosts: pi.prole.org + gather_facts: false + become: true + tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + + - name: Check for k3s-agent systemd unit + ansible.builtin.stat: + path: /etc/systemd/system/k3s-agent.service + register: _agent_unit + + - name: Stop k3s-agent service + ansible.builtin.systemd: + name: k3s-agent + state: stopped + enabled: false + when: + - _agent_unit.stat.exists | default(false) + - not ansible_check_mode + + - name: Check for generic k3s systemd unit (agent variant) + ansible.builtin.stat: + path: /etc/systemd/system/k3s.service + register: _k3s_unit + + - name: Stop k3s service if present (some installs use k3s not k3s-agent) + ansible.builtin.systemd: + name: k3s + state: stopped + enabled: false + when: + - _k3s_unit.stat.exists | default(false) + - not ansible_check_mode + failed_when: false + + - name: Confirm k3s processes are not running + ansible.builtin.command: pgrep -c k3s + register: _k3s_procs + changed_when: false + failed_when: false + + - name: Report k3s process status + ansible.builtin.debug: + msg: >- + k3s processes on pi.prole.org: + {{ 'NONE (clean)' if _k3s_procs.rc != 0 else _k3s_procs.stdout + ' process(es) still running' }} diff --git a/infrastructure/playbooks/iscsi_cleanup.yml b/infrastructure/playbooks/iscsi_cleanup.yml new file mode 100644 index 0000000..b8befb7 --- /dev/null +++ b/infrastructure/playbooks/iscsi_cleanup.yml @@ -0,0 +1,58 @@ +--- +- name: Cleanup stale iSCSI nodes and restart open-iscsi + hosts: myrddin.prole.org + become: true + tasks: + - name: List configured iSCSI nodes + ansible.builtin.command: iscsiadm -m node + register: iscsiadm_nodes + changed_when: false + failed_when: false + + - name: Build desired iSCSI target list + ansible.builtin.set_fact: + desired_iqns: "{{ iscsi_targets | default([]) | map(attribute='iqn') | list }}" + + - name: Build current iSCSI node list + ansible.builtin.set_fact: + iscsi_node_iqns: "{{ iscsiadm_nodes.stdout_lines | default([]) | map('split') | map('first') | list | unique }}" + + - name: Determine stale iSCSI nodes + ansible.builtin.set_fact: + iscsi_stale_iqns: "{{ iscsi_node_iqns | difference(desired_iqns | default([])) }}" + + - name: Logout stale iSCSI nodes + ansible.builtin.command: > + iscsiadm -m node -T {{ item }} -p {{ iscsi_portal }} --logout + loop: "{{ iscsi_stale_iqns | default([]) }}" + loop_control: + label: "{{ item }}" + failed_when: false + + - name: Delete stale iSCSI node records + ansible.builtin.command: > + iscsiadm -m node -o delete -T {{ item }} -p {{ iscsi_portal }} + loop: "{{ iscsi_stale_iqns | default([]) }}" + loop_control: + label: "{{ item }}" + failed_when: false + + - name: Reset open-iscsi failed state + ansible.builtin.command: systemctl reset-failed open-iscsi + changed_when: false + failed_when: false + + - name: Restart open-iscsi + ansible.builtin.service: + name: open-iscsi + state: restarted + + - name: Check open-iscsi status + ansible.builtin.command: systemctl is-failed open-iscsi + register: open_iscsi_state + changed_when: false + failed_when: false + + - name: Report open-iscsi status + ansible.builtin.debug: + msg: "open-iscsi state: {{ open_iscsi_state.stdout | default('unknown') }}" diff --git a/infrastructure/playbooks/iscsi_login.yml b/infrastructure/playbooks/iscsi_login.yml new file mode 100644 index 0000000..4415e4d --- /dev/null +++ b/infrastructure/playbooks/iscsi_login.yml @@ -0,0 +1,11 @@ +--- +- name: iSCSI auth/login + hosts: iscsi + gather_facts: false + become: true + + tasks: + - name: iSCSI auth/login + ansible.builtin.import_role: + name: iscsi + tasks_from: login diff --git a/infrastructure/playbooks/iscsi_mount.yml b/infrastructure/playbooks/iscsi_mount.yml new file mode 100644 index 0000000..266c26f --- /dev/null +++ b/infrastructure/playbooks/iscsi_mount.yml @@ -0,0 +1,11 @@ +--- +- name: iSCSI mounts + hosts: iscsi + gather_facts: false + become: true + + tasks: + - name: iSCSI mounts + ansible.builtin.import_role: + name: iscsi + tasks_from: mount diff --git a/infrastructure/playbooks/k3s_cleanup.yml b/infrastructure/playbooks/k3s_cleanup.yml new file mode 100644 index 0000000..7e179a8 --- /dev/null +++ b/infrastructure/playbooks/k3s_cleanup.yml @@ -0,0 +1,13 @@ +--- +- name: Cleanup / uninstall k3s (operator action) + # Intentionally broad targeting; use --limit to select the host(s) to clean. + hosts: all + gather_facts: false + become: true + serial: 1 + + tasks: + - name: Uninstall k3s and remove local state + ansible.builtin.import_role: + name: k3s + tasks_from: cleanup diff --git a/infrastructure/playbooks/k3s_delete.yml b/infrastructure/playbooks/k3s_delete.yml new file mode 100644 index 0000000..dbd3c60 --- /dev/null +++ b/infrastructure/playbooks/k3s_delete.yml @@ -0,0 +1,63 @@ +--- +- name: Delete k3s nodes and remove cluster token from vault + hosts: k3s_hosts + become: true + tasks: + - name: Remove existing k3s installation and data + ansible.builtin.import_role: + name: k3s + tasks_from: cleanup + +- name: Remove k3s token from Ansible vault + hosts: localhost + connection: local + gather_facts: false + vars: + vault_k3s_path: "{{ playbook_dir }}/../inventory/group_vars/all/vault_k3s.yml" + k3s_vault_password_file: "{{ lookup('env', 'ANSIBLE_VAULT_PASSWORD_FILE') | default('', true) }}" + tasks: + - name: Require vault password file for token deletion + ansible.builtin.assert: + that: + - k3s_vault_password_file is defined + - k3s_vault_password_file | length > 0 + fail_msg: "Set k3s_vault_password_file to a vault password file path." + + - name: Ensure vault file exists + ansible.builtin.stat: + path: "{{ vault_k3s_path }}" + register: vault_k3s_file + + - name: Fail when vault file is missing + ansible.builtin.fail: + msg: "Vault file not found at {{ vault_k3s_path }}" + when: not vault_k3s_file.stat.exists + + - name: Read vault_k3s.yml header + ansible.builtin.command: "head -n 1 {{ vault_k3s_path }}" + register: vault_k3s_header + changed_when: false + + - name: Mark vault_k3s.yml encryption state + ansible.builtin.set_fact: + vault_k3s_encrypted: "{{ vault_k3s_header.stdout is search('^\\$ANSIBLE_VAULT') }}" + + - name: Decrypt vault_k3s.yml + ansible.builtin.command: >- + ansible-vault decrypt {{ vault_k3s_path }} + --vault-password-file {{ k3s_vault_password_file }} + changed_when: true + when: vault_k3s_encrypted | bool + + - name: Remove vault_k3s_token entry + ansible.builtin.lineinfile: + path: "{{ vault_k3s_path }}" + regexp: '^vault_k3s_token:' + state: absent + + - name: Encrypt vault_k3s.yml + ansible.builtin.command: >- + ansible-vault encrypt {{ vault_k3s_path }} + --vault-password-file {{ k3s_vault_password_file }} + changed_when: true + when: vault_k3s_encrypted | bool diff --git a/infrastructure/playbooks/k3s_diagnose_repair.yml b/infrastructure/playbooks/k3s_diagnose_repair.yml new file mode 100644 index 0000000..03d2925 --- /dev/null +++ b/infrastructure/playbooks/k3s_diagnose_repair.yml @@ -0,0 +1,211 @@ +--- +- name: Prepare k3s repair metadata + hosts: k3s_hosts + gather_facts: false + run_once: true + tasks: + - name: Initialize k3s repair source host + ansible.builtin.set_fact: + k3s_repair_source_host: "" + delegate_to: localhost + delegate_facts: true + + - name: Select k3s init server as repair source + ansible.builtin.set_fact: + k3s_repair_source_host: "{{ item }}" + loop: "{{ groups['k3s_hosts'] }}" + when: hostvars[item].k3s_cluster_init | default(false) | bool + delegate_to: localhost + delegate_facts: true + + - name: Require a k3s init server as repair source + ansible.builtin.assert: + that: + - hostvars['localhost'].k3s_repair_source_host | length > 0 + fail_msg: "No k3s init server found. Ensure a host has k3s_cluster_init: true." + + - name: Ensure k3s node token exists on source + ansible.builtin.stat: + path: "{{ k3s_data_dir }}/server/node-token" + register: k3s_node_token + delegate_to: "{{ hostvars['localhost'].k3s_repair_source_host }}" + + - name: Fail when k3s node token is missing + ansible.builtin.fail: + msg: "k3s node token not found at {{ k3s_data_dir }}/server/node-token" + when: not k3s_node_token.stat.exists + + - name: Read k3s node token from source + ansible.builtin.slurp: + src: "{{ k3s_data_dir }}/server/node-token" + register: k3s_node_token_raw + delegate_to: "{{ hostvars['localhost'].k3s_repair_source_host }}" + + - name: Read k3s server CA from source + ansible.builtin.slurp: + src: "{{ k3s_data_dir }}/server/tls/server-ca.crt" + register: k3s_server_ca_raw + delegate_to: "{{ hostvars['localhost'].k3s_repair_source_host }}" + + - name: Store k3s repair metadata on controller + ansible.builtin.set_fact: + k3s_repair_token: "{{ k3s_node_token_raw.content | b64decode | trim }}" + k3s_repair_server_ca: "{{ k3s_server_ca_raw.content | b64decode }}" + k3s_repair_server_ca_sha: "{{ (k3s_server_ca_raw.content | b64decode) | hash('sha256') }}" + delegate_to: localhost + delegate_facts: true + +- name: Update inventory vault token to match source + hosts: localhost + gather_facts: false + vars: + k3s_repair_token: "{{ hostvars['localhost'].k3s_repair_token | default('') }}" + vault_k3s_path: "{{ playbook_dir }}/../inventory/group_vars/all/vault_k3s.yml" + tasks: + - name: Require k3s repair token + ansible.builtin.assert: + that: + - k3s_repair_token | length > 0 + fail_msg: "k3s repair token is empty. Cannot update vault_k3s.yml." + + - name: Update vault k3s token + ansible.builtin.lineinfile: + path: "{{ vault_k3s_path }}" + regexp: '^vault_k3s_token:' + line: "vault_k3s_token: \"{{ k3s_repair_token }}\"" + +- name: Diagnose and repair k3s configuration + hosts: k3s_hosts + become: true + gather_facts: false + vars: + k3s_repair_source_host: "{{ hostvars['localhost'].k3s_repair_source_host }}" + k3s_repair_token: "{{ hostvars['localhost'].k3s_repair_token | default('') }}" + k3s_repair_server_ca: "{{ hostvars['localhost'].k3s_repair_server_ca | default('') }}" + k3s_repair_server_ca_sha: "{{ hostvars['localhost'].k3s_repair_server_ca_sha | default('') }}" + k3s_config_path: /etc/rancher/k3s/config.yaml + k3s_agent_ca_path: "{{ k3s_data_dir }}/agent/server-ca.crt" + k3s_server_ca_path: "{{ k3s_data_dir }}/server/tls/server-ca.crt" + pre_tasks: + - name: Require k3s repair metadata + ansible.builtin.assert: + that: + - k3s_repair_token | length > 0 + - k3s_repair_server_ca | length > 0 + fail_msg: "Missing k3s repair metadata from controller." + + - name: Set k3s service name + ansible.builtin.set_fact: + k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}" + + tasks: + - name: Ensure k3s config exists + ansible.builtin.stat: + path: "{{ k3s_config_path }}" + register: k3s_config + + - name: Fail when k3s config is missing + ansible.builtin.fail: + msg: "k3s config is missing at {{ k3s_config_path }}. Run the k3s role first." + when: not k3s_config.stat.exists + + - name: Read k3s config + ansible.builtin.slurp: + src: "{{ k3s_config_path }}" + register: k3s_config_raw + when: k3s_config.stat.exists + + - name: Read k3s token from config + ansible.builtin.shell: | + awk -F': ' '/^token:/ {gsub(/"/, "", $2); print $2}' "{{ k3s_config_path }}" + register: k3s_config_token_cmd + changed_when: false + + - name: Read k3s server URL from config + ansible.builtin.shell: | + awk -F': ' '/^server:/ {gsub(/"/, "", $2); print $2}' "{{ k3s_config_path }}" + register: k3s_config_server_cmd + changed_when: false + + - name: Read k3s data-dir from config + ansible.builtin.shell: | + awk -F': ' '/^data-dir:/ {gsub(/"/, "", $2); print $2}' "{{ k3s_config_path }}" + register: k3s_config_data_dir_cmd + changed_when: false + + - name: Store parsed k3s config values + ansible.builtin.set_fact: + k3s_config_token: "{{ k3s_config_token_cmd.stdout | trim }}" + k3s_config_server: "{{ k3s_config_server_cmd.stdout | trim }}" + k3s_config_data_dir: "{{ k3s_config_data_dir_cmd.stdout | trim }}" + + - name: Read agent server CA when present + ansible.builtin.slurp: + src: "{{ k3s_agent_ca_path }}" + register: k3s_agent_ca_raw + when: k3s_role != 'server' + failed_when: false + + - name: Compute agent CA checksum + ansible.builtin.set_fact: + k3s_agent_ca_sha: >- + {{ (k3s_agent_ca_raw.content | default('') | b64decode | hash('sha256')) if (k3s_agent_ca_raw.content is defined) else '' }} + when: k3s_role != 'server' + + - name: Report k3s config state + ansible.builtin.debug: + msg: + host: "{{ inventory_hostname }}" + role: "{{ k3s_role }}" + service: "{{ k3s_service_name }}" + config_token: "{{ k3s_config_token | default('') }}" + token_matches_source: "{{ (k3s_config_token | default('')) == k3s_repair_token }}" + config_server: "{{ k3s_config_server | default('') }}" + server_matches_inventory: "{{ (k3s_role == 'server') or ((k3s_config_server | default('')) == (k3s_server_url | default(''))) }}" + config_data_dir: "{{ k3s_config_data_dir | default('') }}" + data_dir_matches_inventory: "{{ (k3s_config_data_dir | default('')) == (k3s_data_dir | default('')) }}" + agent_ca_matches_source: "{{ (k3s_role == 'server') or ((k3s_agent_ca_sha | default('')) == k3s_repair_server_ca_sha) }}" + + - name: Sync k3s token into config + ansible.builtin.lineinfile: + path: "{{ k3s_config_path }}" + regexp: '^token:' + line: "token: \"{{ k3s_repair_token }}\"" + mode: "0640" + notify: Restart k3s service + + - name: Sync k3s server URL into config for non-init nodes + ansible.builtin.lineinfile: + path: "{{ k3s_config_path }}" + regexp: '^server:' + line: "server: \"{{ k3s_server_url }}\"" + mode: "0640" + when: + - not (k3s_cluster_init | default(false) | bool) + - k3s_server_url | default('') | length > 0 + notify: Restart k3s service + + - name: Ensure agent cert directory exists + ansible.builtin.file: + path: "{{ k3s_data_dir }}/agent" + state: directory + mode: "0755" + when: k3s_role != 'server' + + - name: Sync server CA to agent + ansible.builtin.copy: + content: "{{ k3s_repair_server_ca }}" + dest: "{{ k3s_agent_ca_path }}" + owner: root + group: root + mode: "0644" + when: k3s_role != 'server' + notify: Restart k3s service + + handlers: + - name: Restart k3s service + ansible.builtin.service: + name: "{{ k3s_service_name }}" + state: restarted + +- import_playbook: k3s_fetch_kubeconfig.yml diff --git a/infrastructure/playbooks/k3s_fetch_kubeconfig.yml b/infrastructure/playbooks/k3s_fetch_kubeconfig.yml new file mode 100644 index 0000000..647da6f --- /dev/null +++ b/infrastructure/playbooks/k3s_fetch_kubeconfig.yml @@ -0,0 +1,26 @@ +--- +- name: Fetch k3s kubeconfig from primary node + hosts: myrddin.prole.org + gather_facts: false + vars: + k3s_kubeconfig_src: /etc/rancher/k3s/k3s.yaml + k3s_api_server: "https://{{ inventory_hostname }}:6443" + local_kubeconfig_path: "{{ playbook_dir }}/../../prole-k3s.kubeconfig" + tasks: + - name: Read k3s kubeconfig from primary node + ansible.builtin.slurp: + src: "{{ k3s_kubeconfig_src }}" + register: k3s_kubeconfig_raw + become: true + + - name: Rewrite kubeconfig server to primary endpoint + ansible.builtin.set_fact: + k3s_kubeconfig_rendered: "{{ k3s_kubeconfig_raw.content | b64decode | regex_replace('server: https://[^\\s]+', 'server: ' ~ k3s_api_server) }}" + + - name: Save kubeconfig locally for etc/init scripts + ansible.builtin.copy: + content: "{{ k3s_kubeconfig_rendered }}" + dest: "{{ local_kubeconfig_path }}" + mode: "0600" + delegate_to: localhost + run_once: true diff --git a/infrastructure/playbooks/k3s_import_images.yml b/infrastructure/playbooks/k3s_import_images.yml new file mode 100644 index 0000000..9af8fc5 --- /dev/null +++ b/infrastructure/playbooks/k3s_import_images.yml @@ -0,0 +1,30 @@ +--- +# Import local Docker image tarballs directly into k3s containerd on all nodes. +# +# Usage: +# ansible-playbook k3s_import_images.yml \ +# -e '{"k3s_import_image_tars": ["/path/to/image1.tar", "/path/to/image2.tar"]}' +# +# This avoids the slow docker-push-to-registry flow by copying tarballs +# directly to each k3s node and importing via `k3s ctr images import`. + +- name: Import container images into k3s nodes + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + + - name: Skip import on hosts where k3s is disabled + ansible.builtin.meta: end_host + when: not (k3s_enabled | default(true) | bool) + tasks: + - name: Import image tarballs + ansible.builtin.import_role: + name: k3s + tasks_from: import_images + when: (k3s_import_images | default([])) | length > 0 diff --git a/infrastructure/playbooks/k3s_install_single.yml b/infrastructure/playbooks/k3s_install_single.yml new file mode 100644 index 0000000..dbce6ab --- /dev/null +++ b/infrastructure/playbooks/k3s_install_single.yml @@ -0,0 +1,24 @@ +--- +- name: Install and start k3s on a single host + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + + pre_tasks: + - name: Require a single host target + ansible.builtin.assert: + that: + - ansible_play_hosts_all | length == 1 + fail_msg: "This playbook targets one host. Use --limit ." + + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + + tasks: + - name: Install and start k3s + ansible.builtin.import_role: + name: k3s + tasks_from: install diff --git a/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml b/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml new file mode 100644 index 0000000..af0460e --- /dev/null +++ b/infrastructure/playbooks/k3s_mariadb_datastore_prepare.yml @@ -0,0 +1,102 @@ +--- +- name: Prepare MariaDB datastore for k3s (create DB/user/grants) + hosts: k3s_hosts + gather_facts: false + become: false + serial: 1 + vars: + k3s_mariadb_host: "{{ k3s_datastore_mariadb_host | default('synology.prole.org') }}" + k3s_mariadb_port: "{{ k3s_datastore_mariadb_port | default(3306) }}" + k3s_mariadb_db: "{{ k3s_datastore_mariadb_db | default('k3s') }}" + k3s_mariadb_user: "{{ k3s_datastore_mariadb_user | default('prole_k3s') }}" + k3s_mariadb_user_password: "{{ k3s_datastore_mariadb_password | default('') }}" + + # MariaDB admin credentials (must be stored in Ansible Vault by the operator) + k3s_mariadb_admin_user: "{{ k3s_mariadb_admin_user | default('root') }}" + k3s_mariadb_admin_password: "{{ k3s_mariadb_admin_password | default('') }}" + + # Where to allow the k3s user to connect from (tighten this if possible) + k3s_mariadb_user_host: "{{ k3s_mariadb_user_host | default('%') }}" + + pre_tasks: + - name: Only run on k3s server nodes + ansible.builtin.meta: end_host + when: k3s_role | default('') != 'server' + + - name: Require k3s MariaDB user password (vaulted) + ansible.builtin.assert: + that: + - k3s_mariadb_user_password | length > 0 + fail_msg: >- + k3s_mariadb_user_password is empty. Ensure k3s_datastore_mariadb_password is set + (and sourced from Ansible Vault, e.g. vault_samba_dns_admin_pass). + + - name: Require MariaDB admin password for non-check runs + ansible.builtin.assert: + that: + - k3s_mariadb_admin_password | length > 0 + fail_msg: >- + k3s_mariadb_admin_password is empty. + Provide a vaulted MariaDB admin password (e.g. in an encrypted group_vars/*/vault.yml). + when: not ansible_check_mode + + - name: Verify mysql client is available on this host + ansible.builtin.command: mysql --version + changed_when: false + + tasks: + - name: Create k3s database (if missing) + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="CREATE DATABASE IF NOT EXISTS `{{ k3s_mariadb_db }}`" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true + + - name: Create/ensure k3s MariaDB user exists + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="CREATE USER IF NOT EXISTS '{{ k3s_mariadb_user }}'@'{{ k3s_mariadb_user_host }}' IDENTIFIED BY '{{ k3s_mariadb_user_password }}'" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true + + - name: Grant admin privileges on k3s database to k3s MariaDB user + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="GRANT ALL PRIVILEGES ON `{{ k3s_mariadb_db }}`.* TO '{{ k3s_mariadb_user }}'@'{{ k3s_mariadb_user_host }}'" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true + + - name: Flush privileges + ansible.builtin.command: >- + mysql + --protocol=tcp + --host={{ k3s_mariadb_host }} + --port={{ k3s_mariadb_port }} + --user={{ k3s_mariadb_admin_user }} + --execute="FLUSH PRIVILEGES" + environment: + MYSQL_PWD: "{{ k3s_mariadb_admin_password }}" + changed_when: false + when: not ansible_check_mode + no_log: true diff --git a/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml b/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml new file mode 100644 index 0000000..0e0a1c9 --- /dev/null +++ b/infrastructure/playbooks/k3s_mariadb_datastore_rollout.yml @@ -0,0 +1,54 @@ +--- +- name: Roll out k3s config for external MariaDB datastore (servers) + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Require explicit acknowledgement (this changes the k3s datastore) + ansible.builtin.assert: + that: + - k3s_datastore_migration_ack | default(false) | bool + fail_msg: >- + Refusing to proceed without k3s_datastore_migration_ack=true. + Switching k3s datastore backends may require a rebuild/migration plan. + + - name: Only run on k3s server nodes + ansible.builtin.meta: end_host + when: k3s_role | default('') != 'server' + + - name: Require k3s_datastore_endpoint for MariaDB datastore + ansible.builtin.assert: + that: + - (k3s_datastore_endpoint | default('')) | length > 0 + fail_msg: "k3s_datastore_endpoint is empty on {{ inventory_hostname }}" + + tasks: + - name: Apply k3s install/config changes (server) + ansible.builtin.import_role: + name: k3s + tasks_from: install + +- name: Roll out k3s config for external MariaDB datastore (agents) + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Require explicit acknowledgement (this changes the k3s datastore) + ansible.builtin.assert: + that: + - k3s_datastore_migration_ack | default(false) | bool + fail_msg: >- + Refusing to proceed without k3s_datastore_migration_ack=true. + Switching k3s datastore backends may require a rebuild/migration plan. + + - name: Only run on k3s agent nodes + ansible.builtin.meta: end_host + when: k3s_role | default('') != 'agent' + + tasks: + - name: Apply k3s install/config changes (agent) + ansible.builtin.import_role: + name: k3s + tasks_from: install diff --git a/infrastructure/playbooks/k3s_reset.yml b/infrastructure/playbooks/k3s_reset.yml new file mode 100644 index 0000000..fb81dc8 --- /dev/null +++ b/infrastructure/playbooks/k3s_reset.yml @@ -0,0 +1,34 @@ +--- +- name: Reset k3s nodes and reinstall with vault token + hosts: k3s_hosts + become: true + serial: 1 + vars: + # During a reset we want to proactively remove the node object from the cluster + # (via a healthy control-plane host) before shutting down local k3s services. + # This prevents stale `NotReady` nodes, DaemonSet pods stuck `Terminating`, and + # `kube-node-lease` entries that can cause future joins/reinstalls to hang. + k3s_reset_force_cluster_node_delete: true + tasks: + - name: Ensure cgroup kernel params are set + ansible.builtin.import_role: + name: cgroups + + - name: Remove existing k3s installation and data + ansible.builtin.import_role: + name: k3s + tasks_from: cleanup + + - name: Reinstall and configure k3s + ansible.builtin.import_role: + name: k3s + +- name: Refresh kubeconfig on controller after reset + hosts: k3s_hosts + gather_facts: false + run_once: true + tasks: + - name: Fetch and save kubeconfig on controller + ansible.builtin.import_role: + name: k3s + tasks_from: fetch_kubeconfig diff --git a/infrastructure/playbooks/k3s_server_refresh.yml b/infrastructure/playbooks/k3s_server_refresh.yml new file mode 100644 index 0000000..3fe6a79 --- /dev/null +++ b/infrastructure/playbooks/k3s_server_refresh.yml @@ -0,0 +1,19 @@ +--- +- name: Refresh k3s server (blank slate except token/CA) + hosts: k3s_hosts + become: true + serial: 1 + vars: + # Safety guard: you must also pass `-e k3s_refresh_confirm=YES`. + # Example: + # ./ansible.sh -v .vault_pass -p infrastructure/playbooks/k3s_server_refresh.yml -l myrddin.prole.org -- -e k3s_refresh_confirm=YES + k3s_refresh_run_configure: true + tasks: + - name: Ensure cgroup kernel params are set + ansible.builtin.import_role: + name: cgroups + + - name: Refresh k3s server (preserve token/CA) + ansible.builtin.import_role: + name: k3s + tasks_from: refresh_server diff --git a/infrastructure/playbooks/k3s_start_single.yml b/infrastructure/playbooks/k3s_start_single.yml new file mode 100644 index 0000000..3c8431d --- /dev/null +++ b/infrastructure/playbooks/k3s_start_single.yml @@ -0,0 +1,16 @@ +--- +- name: Start k3s on a single host + hosts: k3s_hosts + become: true + serial: 1 + pre_tasks: + - name: Require a single host target + ansible.builtin.assert: + that: + - ansible_play_hosts_all | length == 1 + fail_msg: "This playbook targets one host. Use --limit ." + tasks: + - name: Start k3s service + ansible.builtin.import_role: + name: k3s + tasks_from: start diff --git a/infrastructure/playbooks/k3s_stop.yml b/infrastructure/playbooks/k3s_stop.yml new file mode 100644 index 0000000..c041b66 --- /dev/null +++ b/infrastructure/playbooks/k3s_stop.yml @@ -0,0 +1,33 @@ +--- +- name: Stop k3s agents first + hosts: k3s_hosts + become: true + serial: 1 + tasks: + - name: Stop k3s services on agent nodes + ansible.builtin.import_role: + name: k3s + tasks_from: stop + when: k3s_role != "server" + +- name: Stop non-init k3s servers + hosts: k3s_hosts + become: true + serial: 1 + tasks: + - name: Stop k3s services on non-init servers + ansible.builtin.import_role: + name: k3s + tasks_from: stop + when: k3s_role == "server" and not (k3s_cluster_init | bool) + +- name: Stop init k3s servers last + hosts: k3s_hosts + become: true + serial: 1 + tasks: + - name: Stop k3s services on init servers + ansible.builtin.import_role: + name: k3s + tasks_from: stop + when: k3s_role == "server" and (k3s_cluster_init | bool) diff --git a/infrastructure/playbooks/k3s_sync.yml b/infrastructure/playbooks/k3s_sync.yml new file mode 100644 index 0000000..5b7c5e6 --- /dev/null +++ b/infrastructure/playbooks/k3s_sync.yml @@ -0,0 +1,349 @@ +--- +- name: Prepare k3s synchronization metadata + hosts: k3s_hosts + gather_facts: false + run_once: true + tasks: + - name: Initialize k3s sync source host + ansible.builtin.set_fact: + k3s_sync_source_host: "" + delegate_to: localhost + delegate_facts: true + + - name: Select k3s init server as sync source + ansible.builtin.set_fact: + k3s_sync_source_host: "{{ item }}" + loop: "{{ groups['k3s_hosts'] }}" + when: + - hostvars[item].k3s_role | default('agent') == 'server' + - hostvars[item].k3s_cluster_init | default(false) | bool + delegate_to: localhost + delegate_facts: true + + - name: Fallback to first k3s server as sync source + ansible.builtin.set_fact: + k3s_sync_source_host: "{{ item }}" + loop: "{{ groups['k3s_hosts'] }}" + when: + - (hostvars['localhost'].k3s_sync_source_host | length) == 0 + - hostvars[item].k3s_role | default('agent') == 'server' + delegate_to: localhost + delegate_facts: true + + - name: Require a k3s server as sync source + ansible.builtin.assert: + that: + - hostvars['localhost'].k3s_sync_source_host | length > 0 + fail_msg: >- + No k3s sync source found. Ensure at least one host has `k3s_role: server`. + (If you have multiple servers and want to prefer the bootstrap server, + set `k3s_cluster_init: true` on exactly one server.) + + - name: Ensure sync source is included in this run + ansible.builtin.assert: + that: + - hostvars['localhost'].k3s_sync_source_host in ansible_play_hosts_all + fail_msg: "Sync source {{ hostvars['localhost'].k3s_sync_source_host }} is not in this run. Include it in --limit." + + - name: Create local temp directory for k3s sync + ansible.builtin.command: mktemp -d -p /tmp k3s-sync-XXXXXX + register: k3s_sync_tmpdir + changed_when: true + delegate_to: localhost + become: false + + - name: Ensure k3s sync temp directory is writable + ansible.builtin.file: + path: "{{ k3s_sync_tmpdir.stdout }}" + state: directory + mode: "1777" + delegate_to: localhost + + - name: Store k3s sync metadata on controller + ansible.builtin.set_fact: + k3s_sync_source_host: "{{ hostvars['localhost'].k3s_sync_source_host }}" + k3s_sync_tmpdir_path: "{{ k3s_sync_tmpdir.stdout }}" + k3s_sync_tls_bundle: "{{ k3s_sync_tmpdir.stdout }}/k3s-tls.tgz" + delegate_to: localhost + delegate_facts: true + +- name: Collect k3s token and certs from init server + hosts: k3s_hosts + become: true + gather_facts: false + run_once: true + vars: + k3s_sync_source_host: "{{ hostvars['localhost'].k3s_sync_source_host | default('') }}" + k3s_sync_tls_bundle: "{{ hostvars['localhost'].k3s_sync_tls_bundle | default('') }}" + tasks: + - name: Require a k3s sync source + ansible.builtin.assert: + that: + - k3s_sync_source_host | length > 0 + fail_msg: "k3s sync source is empty. Ensure the init server is reachable." + + - name: Ensure k3s node token exists + ansible.builtin.stat: + path: "{{ k3s_data_dir }}/server/node-token" + register: k3s_node_token + delegate_to: "{{ k3s_sync_source_host }}" + + - name: Fail when k3s node token is missing + ansible.builtin.fail: + msg: "k3s node token not found at {{ k3s_data_dir }}/server/node-token" + when: not k3s_node_token.stat.exists + + - name: Read k3s node token + ansible.builtin.slurp: + src: "{{ k3s_data_dir }}/server/node-token" + register: k3s_node_token_raw + delegate_to: "{{ k3s_sync_source_host }}" + + - name: Store k3s sync token on controller + ansible.builtin.set_fact: + k3s_sync_token: "{{ k3s_node_token_raw.content | b64decode | trim }}" + delegate_to: localhost + delegate_facts: true + + - name: Check for k3s tls directory + ansible.builtin.stat: + path: "{{ k3s_data_dir }}/server/tls" + register: k3s_tls_dir + delegate_to: "{{ k3s_sync_source_host }}" + + - name: Create k3s tls bundle + ansible.builtin.archive: + path: "{{ k3s_data_dir }}/server/tls" + dest: /tmp/k3s-tls.tgz + format: gz + when: k3s_tls_dir.stat.exists + delegate_to: "{{ k3s_sync_source_host }}" + + - name: Fetch k3s tls bundle + ansible.builtin.fetch: + src: /tmp/k3s-tls.tgz + dest: "{{ k3s_sync_tls_bundle }}" + flat: true + when: k3s_tls_dir.stat.exists + delegate_to: "{{ k3s_sync_source_host }}" + + - name: Mark tls bundle presence on controller + ansible.builtin.set_fact: + k3s_sync_tls_bundle_present: "{{ k3s_tls_dir.stat.exists }}" + delegate_to: localhost + delegate_facts: true + + - name: Remove temporary tls bundle from source + ansible.builtin.file: + path: /tmp/k3s-tls.tgz + state: absent + when: k3s_tls_dir.stat.exists + delegate_to: "{{ k3s_sync_source_host }}" + +- name: Synchronize k3s token and certs to servers + hosts: k3s_hosts + become: true + serial: 1 + gather_facts: false + vars: + k3s_sync_source_host: "{{ hostvars['localhost'].k3s_sync_source_host }}" + k3s_sync_token: "{{ hostvars['localhost'].k3s_sync_token | default('') }}" + k3s_sync_tls_bundle: "{{ hostvars['localhost'].k3s_sync_tls_bundle | default('') }}" + k3s_sync_tls_bundle_present: "{{ hostvars['localhost'].k3s_sync_tls_bundle_present | default(false) }}" + pre_tasks: + - name: Require k3s sync token + ansible.builtin.assert: + that: + - k3s_sync_token | length > 0 + fail_msg: "k3s sync token is empty. Check the init server token." + tasks: + - name: Stop k3s before syncing + ansible.builtin.import_role: + name: k3s + tasks_from: stop + when: inventory_hostname != k3s_sync_source_host + + - name: Sync k3s token and certs + ansible.builtin.import_role: + name: k3s + tasks_from: sync + when: inventory_hostname != k3s_sync_source_host + + - name: Start k3s after syncing + ansible.builtin.import_role: + name: k3s + tasks_from: start + when: inventory_hostname != k3s_sync_source_host + +- name: Update k3s vault token on controller + hosts: k3s_hosts + gather_facts: false + run_once: true + vars: + k3s_sync_token: "{{ hostvars['localhost'].k3s_sync_token | default('') }}" + vault_k3s_path: "{{ playbook_dir }}/../inventory/group_vars/all/vault_k3s.yml" + vault_pass_default: "{{ playbook_dir }}/../../.vault_pass" + tasks: + - name: Skip vault update when disabled + ansible.builtin.meta: end_play + when: not (k3s_sync_update_vault | default(true) | bool) + + - name: Require k3s sync token for vault update + ansible.builtin.assert: + that: + - k3s_sync_token | length > 0 + fail_msg: "k3s sync token is empty. Unable to update vault." + + - name: Check for default vault password file + ansible.builtin.stat: + path: "{{ vault_pass_default }}" + register: vault_pass_default_stat + delegate_to: localhost + + - name: Determine vault password file + ansible.builtin.set_fact: + k3s_vault_password_file: >- + {{ k3s_vault_password_file + | default(lookup('env', 'ANSIBLE_VAULT_PASSWORD_FILE') | default('', true), true) }} + delegate_to: localhost + delegate_facts: true + + - name: Fallback to default vault password file + ansible.builtin.set_fact: + k3s_vault_password_file: "{{ vault_pass_default }}" + when: + - (hostvars['localhost'].k3s_vault_password_file | default('')) | length == 0 + - vault_pass_default_stat.stat.exists + delegate_to: localhost + delegate_facts: true + + - name: Require vault password file + ansible.builtin.assert: + that: + - (hostvars['localhost'].k3s_vault_password_file | default('')) | length > 0 + fail_msg: "Set k3s_vault_password_file or ANSIBLE_VAULT_PASSWORD_FILE to update vault." + + - name: Ensure vault file exists + ansible.builtin.stat: + path: "{{ vault_k3s_path }}" + register: vault_k3s_file + delegate_to: localhost + + - name: Fail when vault file is missing + ansible.builtin.fail: + msg: "Vault file not found at {{ vault_k3s_path }}" + when: not vault_k3s_file.stat.exists + + - name: Check if vault file is encrypted + ansible.builtin.command: "head -n 1 {{ vault_k3s_path }}" + register: vault_k3s_head + changed_when: false + delegate_to: localhost + + - name: Mark vault encryption state + ansible.builtin.set_fact: + vault_k3s_encrypted: "{{ (vault_k3s_head.stdout | default('')) is search('^\\$ANSIBLE_VAULT') }}" + delegate_to: localhost + delegate_facts: true + + - name: Decrypt vault_k3s.yml + ansible.builtin.command: >- + ansible-vault decrypt {{ vault_k3s_path }} + --vault-password-file {{ hostvars['localhost'].k3s_vault_password_file }} + changed_when: true + delegate_to: localhost + when: hostvars['localhost'].vault_k3s_encrypted | default(false) + + - name: Update vault k3s token + ansible.builtin.lineinfile: + path: "{{ vault_k3s_path }}" + regexp: '^vault_k3s_token:' + line: "vault_k3s_token: \"{{ k3s_sync_token }}\"" + delegate_to: localhost + + - name: Encrypt vault_k3s.yml + ansible.builtin.command: >- + ansible-vault encrypt {{ vault_k3s_path }} + --vault-password-file {{ hostvars['localhost'].k3s_vault_password_file }} + changed_when: true + delegate_to: localhost + when: hostvars['localhost'].vault_k3s_encrypted | default(false) + +- name: Refresh kubeconfig on controller + hosts: k3s_hosts + gather_facts: false + run_once: true + tasks: + - name: Fetch and save kubeconfig on controller + ansible.builtin.import_role: + name: k3s + tasks_from: fetch_kubeconfig + +- name: Validate k3s nodes from controller + hosts: k3s_hosts + gather_facts: false + run_once: true + vars: + k3s_kubeconfig_service_path: "{{ playbook_dir }}/../../etc/secrets/k3s.kubeconfig" + k3s_kubeconfig_project_path: "{{ playbook_dir }}/../../prole-k3s.kubeconfig" + k3s_expected_node_count: "{{ groups['k3s_hosts'] | length }}" + tasks: + - name: Check for service kubeconfig + ansible.builtin.stat: + path: "{{ k3s_kubeconfig_service_path }}" + register: _k3s_kubeconfig_service_stat + delegate_to: localhost + become: false + + - name: Check for project kubeconfig + ansible.builtin.stat: + path: "{{ k3s_kubeconfig_project_path }}" + register: _k3s_kubeconfig_project_stat + delegate_to: localhost + become: false + + - name: Select kubeconfig path for validation + ansible.builtin.set_fact: + k3s_kubeconfig_validation_path: >- + {{ k3s_kubeconfig_service_path + if _k3s_kubeconfig_service_stat.stat.exists + else k3s_kubeconfig_project_path }} + changed_when: false + + - name: Require a kubeconfig for validation + ansible.builtin.assert: + that: + - _k3s_kubeconfig_service_stat.stat.exists or _k3s_kubeconfig_project_stat.stat.exists + fail_msg: >- + No kubeconfig found at {{ k3s_kubeconfig_service_path }} or {{ k3s_kubeconfig_project_path }}. + Run `k3s_reset` or `k3s_sync` again and ensure the init server is reachable. + + - name: Show nodes (kubectl get nodes -o wide) + ansible.builtin.command: "kubectl --kubeconfig {{ k3s_kubeconfig_validation_path }} get nodes -o wide" + register: _k3s_nodes_wide + changed_when: false + delegate_to: localhost + become: false + + - name: Debug nodes output + ansible.builtin.debug: + var: _k3s_nodes_wide.stdout_lines + + - name: Count nodes + ansible.builtin.shell: | + set -euo pipefail + kubectl --kubeconfig "{{ k3s_kubeconfig_validation_path }}" get nodes --no-headers | wc -l | tr -d ' ' + args: + executable: /bin/bash + register: _k3s_node_count + changed_when: false + delegate_to: localhost + become: false + + - name: Require expected node count + ansible.builtin.assert: + that: + - (_k3s_node_count.stdout | int) >= (k3s_expected_node_count | int) + fail_msg: >- + Expected at least {{ k3s_expected_node_count }} nodes, but kubectl reported {{ _k3s_node_count.stdout | default('0') }}. + Ensure all `k3s_hosts` are reachable and have rejoined, then re-run `k3s_sync`. diff --git a/infrastructure/playbooks/merlin_mariadb_full_export.yml b/infrastructure/playbooks/merlin_mariadb_full_export.yml new file mode 100644 index 0000000..4581e48 --- /dev/null +++ b/infrastructure/playbooks/merlin_mariadb_full_export.yml @@ -0,0 +1,12 @@ +--- +- name: Export k3s datastore database (mysqldump) + hosts: mariadb_primary + gather_facts: false + become: true + serial: 1 + + tasks: + - name: Export k3s datastore database + ansible.builtin.import_role: + name: mariadb_tools + tasks_from: k3s_datastore_export diff --git a/infrastructure/playbooks/merlin_mariadb_full_import.yml b/infrastructure/playbooks/merlin_mariadb_full_import.yml new file mode 100644 index 0000000..fa36afb --- /dev/null +++ b/infrastructure/playbooks/merlin_mariadb_full_import.yml @@ -0,0 +1,12 @@ +--- +- name: Import k3s datastore database dump into Merlin MariaDB + hosts: mariadb_primary + gather_facts: false + become: true + serial: 1 + + tasks: + - name: Import k3s datastore database + ansible.builtin.import_role: + name: mariadb_tools + tasks_from: k3s_datastore_import diff --git a/infrastructure/playbooks/merlin_mariadb_provision.yml b/infrastructure/playbooks/merlin_mariadb_provision.yml new file mode 100644 index 0000000..3d4b470 --- /dev/null +++ b/infrastructure/playbooks/merlin_mariadb_provision.yml @@ -0,0 +1,15 @@ +--- +- name: Provision Merlin MariaDB primary (users/grants/datadir) + hosts: mariadb_primary + gather_facts: false + become: true + serial: 1 + + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + + roles: + - mariadb_primary diff --git a/infrastructure/playbooks/prole_logs_migrate.yml b/infrastructure/playbooks/prole_logs_migrate.yml new file mode 100644 index 0000000..d8b7bf0 --- /dev/null +++ b/infrastructure/playbooks/prole_logs_migrate.yml @@ -0,0 +1,79 @@ +--- +- name: Migrate /prole/logs to iSCSI + hosts: myrddin.prole.org + become: true + vars: + prole_logs_iscsi_target: + iqn: "iqn.2000-01.com.synology:synology.Target-1.292d45194a1" + chap_user: "prole" + chap_password: "{{ vault_iscsi_prole_password }}" + mounts: + - path: /prole/logs/synology + fstype: ext4 + opts: "_netdev,noatime" + src: "UUID=5335affd-b1ab-4134-a1c0-be88ab965309" + tasks: + - name: Ensure rsync installed + ansible.builtin.package: + name: rsync + state: present + + - name: Ensure staging mountpoint exists + ansible.builtin.file: + path: /prole/logs/synology + state: directory + mode: "0755" + + - name: Login and mount logs volume at staging path + ansible.builtin.include_role: + name: iscsi + vars: + iscsi_targets: "{{ [prole_logs_iscsi_target] }}" + iscsi_absent_mounts: [] + + - name: Initial rsync from /prole/logs to staging + ansible.builtin.command: > + rsync -aHAX --info=stats2 /prole/logs/ /prole/logs/synology/ + register: rsync_initial + failed_when: rsync_initial.rc not in [0, 24] + + - name: Stop rsyslog + ansible.builtin.service: + name: rsyslog + state: stopped + + - name: Final rsync after rsyslog stop + ansible.builtin.command: > + rsync -aHAX --delete --info=stats2 /prole/logs/ /prole/logs/synology/ + register: rsync_final + failed_when: rsync_final.rc not in [0, 24] + + - name: Unmount staging logs volume + ansible.builtin.mount: + path: /prole/logs/synology + state: unmounted + + - name: Remove staging fstab entry + ansible.builtin.lineinfile: + path: /etc/fstab + state: absent + regexp: '^\\S+\\s+/prole/logs/synology\\s+' + + - name: Ensure /prole/logs exists + ansible.builtin.file: + path: /prole/logs + state: directory + mode: "0755" + + - name: Mount logs volume at /prole/logs + ansible.builtin.mount: + path: /prole/logs + src: "UUID=5335affd-b1ab-4134-a1c0-be88ab965309" + fstype: ext4 + opts: "_netdev,noatime" + state: mounted + + - name: Start rsyslog + ansible.builtin.service: + name: rsyslog + state: started diff --git a/infrastructure/playbooks/rotate_pihole_db.yml b/infrastructure/playbooks/rotate_pihole_db.yml new file mode 100644 index 0000000..37f9838 --- /dev/null +++ b/infrastructure/playbooks/rotate_pihole_db.yml @@ -0,0 +1,7 @@ +--- +- name: Rotate Pi-hole FTL database safely + hosts: pihole + become: true + serial: 1 + roles: + - pihole_db_rotate diff --git a/infrastructure/playbooks/site.yml b/infrastructure/playbooks/site.yml new file mode 100644 index 0000000..4fd778a --- /dev/null +++ b/infrastructure/playbooks/site.yml @@ -0,0 +1,232 @@ +--- +# Runbook tiers (systemd-like startup ordering): +# 1) boot / kernel +# 2) storage: iscsi auth, iscsi login +# 3) filesystem mount +# 4) OS configuration (e.g., samba) +# 5) OS package installation (k3s) +# 6) OS package configuration (helm/kubectl apply) +# +# This playbook is intentionally ordered so each tier fully completes before +# moving on to the next tier. + +- name: Tier 1 - Boot / kernel (cgroups) + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - cgroups + +- name: Tier 1.5 - Quiesce k3s and detach migrated iSCSI LUNs (myrddin) + hosts: myrddin.prole.org + gather_facts: false + become: true + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + tasks: + - name: Check if k3s systemd unit exists + ansible.builtin.stat: + path: /etc/systemd/system/k3s.service + register: _k3s_unit + + - name: Stop k3s (quiesce before iSCSI detach) + ansible.builtin.systemd: + name: k3s + state: stopped + when: + - not ansible_check_mode + - _k3s_unit.stat.exists | default(false) + - (iscsi_absent_mounts | default([]) | length) > 0 or (iscsi_absent_targets | default([]) | length) > 0 + + - name: Detach migrated iSCSI targets/mounts (best-effort cleanup) + ansible.builtin.import_role: + name: iscsi + tasks_from: detach + when: + - (iscsi_absent_mounts | default([]) | length) > 0 or (iscsi_absent_targets | default([]) | length) > 0 + +- name: Tier 2 - Storage (iSCSI auth/login) + hosts: iscsi + gather_facts: false + become: true + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + tasks: + - name: iSCSI auth/login + ansible.builtin.import_role: + name: iscsi + tasks_from: login + +- name: Tier 3 - Filesystem mounts (iSCSI) + hosts: iscsi + gather_facts: false + become: true + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + tasks: + - name: iSCSI mounts + ansible.builtin.import_role: + name: iscsi + tasks_from: mount + +- name: Tier 4 - OS configuration (Pi-hole DNS and resilience) + hosts: pihole + gather_facts: false + become: true + tags: + - pihole + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - pihole_dns + +- name: Tier 4 - OS configuration (Samba AD DNS reverse zones and PTRs) + hosts: ad_dc + gather_facts: false + become: true + tags: + - samba + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - samba_ad_dc + - samba_dns + - samba_reverse_dns + +- name: Tier 4 - OS configuration (base linux) + hosts: linux_hosts + gather_facts: false + become: true + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - role: netplan_static + when: netplan_static_enabled | default(false) | bool + - local_user + - rsyslog + - prole + +- name: Tier 4.1 - OS configuration (Prole SSL certs) + hosts: ssl_hosts + gather_facts: false + become: true + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - prole_ssl + +- name: Tier 4 - OS configuration (console dashboard) + hosts: k3s_hosts + gather_facts: false + become: true + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - dashboard + +- name: Tier 4.5 - Database (MariaDB primary) + hosts: mariadb_primary + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - mariadb_primary + +- name: Tier 4.6 - Database (MariaDB replica) + hosts: mariadb_replica + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + roles: + - mariadb_replica + +- name: Tier 5 - OS package installation (k3s) + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + tags: [k3s] + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + tasks: + - name: Install and start k3s + ansible.builtin.import_role: + name: k3s + tasks_from: install + +- name: Tier 5.5 - Kubernetes node operations (labels) + hosts: k3s_servers + gather_facts: false + become: true + serial: 1 + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + tasks: + - name: Persist Kubernetes node labels (post-provisioning) + ansible.builtin.import_role: + name: k3s + tasks_from: label_nodes + when: (k3s_node_labels | default({}) | length) > 0 + +- name: Tier 6 - OS package configuration (helm/kubectl apply) + hosts: k3s_hosts + gather_facts: false + become: true + serial: 1 + tags: [k3s] + pre_tasks: + - name: Gather minimal facts + ansible.builtin.setup: + gather_subset: + - min + tasks: + - name: Configure k3s add-ons + ansible.builtin.import_role: + name: k3s + tasks_from: configure diff --git a/infrastructure/playbooks/smoke_ping.yml b/infrastructure/playbooks/smoke_ping.yml new file mode 100644 index 0000000..b3d63c1 --- /dev/null +++ b/infrastructure/playbooks/smoke_ping.yml @@ -0,0 +1,7 @@ +--- +- name: Smoke test connectivity (no roles) + hosts: all + gather_facts: false + tasks: + - name: Ping + ansible.builtin.ping: diff --git a/infrastructure/playbooks/start_k3s_agents.yml b/infrastructure/playbooks/start_k3s_agents.yml new file mode 100644 index 0000000..0fefa84 --- /dev/null +++ b/infrastructure/playbooks/start_k3s_agents.yml @@ -0,0 +1,12 @@ +--- +- name: Start k3s agents + hosts: k3s_agents + gather_facts: false + become: true + serial: 1 + + tasks: + - name: Start k3s-agent service + ansible.builtin.import_role: + name: k3s + tasks_from: start_k3s_agents diff --git a/infrastructure/playbooks/start_k3s_servers.yml b/infrastructure/playbooks/start_k3s_servers.yml new file mode 100644 index 0000000..42e8a6b --- /dev/null +++ b/infrastructure/playbooks/start_k3s_servers.yml @@ -0,0 +1,12 @@ +--- +- name: Start k3s servers + hosts: k3s_servers + gather_facts: false + become: true + serial: 1 + + tasks: + - name: Start k3s service + ansible.builtin.import_role: + name: k3s + tasks_from: start_k3s_servers diff --git a/infrastructure/playbooks/stop_k3s_agents.yml b/infrastructure/playbooks/stop_k3s_agents.yml new file mode 100644 index 0000000..5efca04 --- /dev/null +++ b/infrastructure/playbooks/stop_k3s_agents.yml @@ -0,0 +1,12 @@ +--- +- name: Stop k3s agents + hosts: k3s_agents + gather_facts: false + become: true + serial: 1 + + tasks: + - name: Stop k3s-agent service + ansible.builtin.import_role: + name: k3s + tasks_from: stop_k3s_agents diff --git a/infrastructure/playbooks/stop_k3s_servers.yml b/infrastructure/playbooks/stop_k3s_servers.yml new file mode 100644 index 0000000..dfec2f2 --- /dev/null +++ b/infrastructure/playbooks/stop_k3s_servers.yml @@ -0,0 +1,12 @@ +--- +- name: Stop k3s servers + hosts: k3s_servers + gather_facts: false + become: true + serial: 1 + + tasks: + - name: Stop k3s service + ansible.builtin.import_role: + name: k3s + tasks_from: stop_k3s_servers diff --git a/infrastructure/playbooks/swap.yml b/infrastructure/playbooks/swap.yml new file mode 100644 index 0000000..74aff16 --- /dev/null +++ b/infrastructure/playbooks/swap.yml @@ -0,0 +1,6 @@ +--- +- name: Configure swap on Raspberry Pi hosts + hosts: raspberry.prole.org + become: true + roles: + - swap diff --git a/infrastructure/playbooks/sync_pihole_toml.yml b/infrastructure/playbooks/sync_pihole_toml.yml new file mode 100644 index 0000000..739822a --- /dev/null +++ b/infrastructure/playbooks/sync_pihole_toml.yml @@ -0,0 +1,37 @@ +--- +- name: Fetch gold Pi-hole config from pi.prole.org + hosts: pi.prole.org + become: true + gather_facts: false + tasks: + - name: Fetch /etc/pihole/pihole.toml to control node + ansible.builtin.fetch: + src: /etc/pihole/pihole.toml + dest: /tmp/pihole.toml.gold + flat: true + +# - name: Show diff between current and gold +# ansible.builtin.command: diff -u /etc/pihole/pihole.toml /tmp/pihole.toml.gold +# register: toml_diff +# failed_when: false +# changed_when: false + +- name: Apply gold Pi-hole config to raspberry.prole.org + hosts: raspberry.prole.org + become: true + gather_facts: false + tasks: + - name: Copy gold pihole.toml to raspberry + ansible.builtin.copy: + src: /tmp/pihole.toml.gold + dest: /etc/pihole/pihole.toml + owner: root + group: root + mode: "0644" + register: toml_copy + + - name: Restart Pi-hole FTL if config changed + ansible.builtin.systemd: + name: pihole-FTL + state: restarted + when: toml_copy.changed diff --git a/infrastructure/playbooks/test_k3s_kubeconfig_rewrite.yml b/infrastructure/playbooks/test_k3s_kubeconfig_rewrite.yml new file mode 100644 index 0000000..59795bb --- /dev/null +++ b/infrastructure/playbooks/test_k3s_kubeconfig_rewrite.yml @@ -0,0 +1,77 @@ +--- +- name: Test - k3s kubeconfig server rewrite (local) + hosts: localhost + connection: local + gather_facts: false + + vars: + _sample_kubeconfig: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: ZmFrZQ== + server: https://127.0.0.1:6443 + name: default + contexts: + - context: + cluster: default + user: default + name: default + current-context: default + kind: Config + users: + - name: default + user: + client-certificate-data: ZmFrZQ== + client-key-data: ZmFrZQ== + + _desired_server: https://myrddin.prole.org:6443 + + tasks: + - name: Old kubeconfig rewrite regex should break YAML (expected) + block: + - name: Apply old buggy regex + ansible.builtin.set_fact: + _kubeconfig_bad: >- + {{ _sample_kubeconfig + | regex_replace('server: https://[^\\s]+', 'server: ' ~ _desired_server) }} + + - name: Attempt YAML parse (expected to fail) + ansible.builtin.set_fact: + _kubeconfig_bad_parsed: "{{ _kubeconfig_bad | from_yaml }}" + + - name: Fail if YAML parse unexpectedly succeeded + ansible.builtin.fail: + msg: "Old regex unexpectedly produced valid YAML; test sample may need adjustment." + + rescue: + - name: Mark expected failure as observed + ansible.builtin.set_fact: + _old_regex_failed_as_expected: true + + always: + - name: Assert old regex failure was observed + ansible.builtin.assert: + that: + - _old_regex_failed_as_expected | default(false) | bool + + - name: New kubeconfig rewrite regex should preserve YAML structure + ansible.builtin.set_fact: + _kubeconfig_good: >- + {{ _sample_kubeconfig + | regex_replace('server:\s*https?://\S+', 'server: ' ~ _desired_server) }} + + - name: Parse rewritten kubeconfig + ansible.builtin.set_fact: + _kubeconfig_good_parsed: "{{ _kubeconfig_good | from_yaml }}" + changed_when: false + + - name: Assert kubeconfig keys and server were rewritten + ansible.builtin.assert: + that: + - _kubeconfig_good_parsed is mapping + - (_kubeconfig_good_parsed.contexts | default([])) | length == 1 + - (_kubeconfig_good_parsed.clusters | default([])) | length == 1 + - (_kubeconfig_good_parsed.users | default([])) | length == 1 + - _kubeconfig_good_parsed['current-context'] == 'default' + - _kubeconfig_good_parsed.clusters[0].cluster.server == _desired_server diff --git a/infrastructure/playbooks/test_k3s_validate_args.yml b/infrastructure/playbooks/test_k3s_validate_args.yml new file mode 100644 index 0000000..ea16892 --- /dev/null +++ b/infrastructure/playbooks/test_k3s_validate_args.yml @@ -0,0 +1,35 @@ +--- +- name: Test - k3s argument validation (local) + hosts: localhost + connection: local + gather_facts: false + + tasks: + - name: Validation should fail when a known-bad arg is present (etcd-timeout) + block: + - name: Run validation with a bad arg (expected to fail) + ansible.builtin.include_role: + name: k3s + tasks_from: validate_args + vars: + k3s_kube_apiserver_args: + - "etcd-timeout=50s" + rescue: + - name: Mark expected failure as observed + ansible.builtin.set_fact: + _bad_arg_validation_failed: true + + always: + - name: Assert the bad-arg validation failed as expected + ansible.builtin.assert: + that: + - _bad_arg_validation_failed | default(false) | bool + + - name: Validation should pass with supported args + ansible.builtin.include_role: + name: k3s + tasks_from: validate_args + vars: + k3s_kube_apiserver_args: + - "request-timeout=10s" + - "min-request-timeout=10" diff --git a/infrastructure/playbooks/tmp_bao_dir.yml b/infrastructure/playbooks/tmp_bao_dir.yml new file mode 100644 index 0000000..9418d46 --- /dev/null +++ b/infrastructure/playbooks/tmp_bao_dir.yml @@ -0,0 +1,9 @@ +--- +- hosts: myrddin.prole.org + become: true + tasks: + - name: Ensure OpenBao storage directory exists + ansible.builtin.file: + path: /synology/d001/openbao + state: directory + mode: "0777" # OpenBao pod runs as non-root usually, but k3s local-path often needs this or specific UID diff --git a/infrastructure/playbooks/tmp_mount.yml b/infrastructure/playbooks/tmp_mount.yml new file mode 100644 index 0000000..113f9b4 --- /dev/null +++ b/infrastructure/playbooks/tmp_mount.yml @@ -0,0 +1,12 @@ +--- +- hosts: myrddin.prole.org + become: true + tasks: + - name: Try to mount all filesystems + ansible.builtin.command: mount -a + - name: Check if required mount is present + ansible.builtin.command: findmnt -n /opt/prole/logs/chrisfu + register: mount_check + failed_when: false + - debug: + var: mount_check.stdout diff --git a/infrastructure/roles/audit/defaults/main.yml b/infrastructure/roles/audit/defaults/main.yml new file mode 100644 index 0000000..cfbeb84 --- /dev/null +++ b/infrastructure/roles/audit/defaults/main.yml @@ -0,0 +1,46 @@ +--- +prole_audit_scan_paths: + - /etc + - /usr/local + - /opt + - /srv + +prole_audit_scan_paths_extra: [] + +prole_audit_allow_prefixes: + - "{{ prole_home | default('/opt/prole') }}" + - "{{ prole_conf | default('/opt/prole/conf') }}" + - /etc/rancher/k3s + - /var/lib/rancher + - /usr/local/bin/k3s + - /usr/local/bin/kubectl + - /usr/local/bin/crictl + - /usr/local/bin/ctr + - /usr/local/bin/k3s-killall.sh + - /usr/local/bin/k3s-uninstall.sh + - /etc/systemd/system/k3s.service + - /etc/systemd/system/k3s.service.env + - /etc/systemd/system/k3s-agent.service + - /etc/systemd/system/k3s-agent.service.env + - /usr/local/bin/helm + - /etc/rsyslog.d/10-server.conf + - /etc/rsyslog.d/90-forward-all.conf + - /etc/dnsmasq.d/05-samba-ad.conf + - /etc/dnsmasq.d/99-dns-forward-max.conf + - /usr/local/sbin/rotate-pihole-db + - /var/lib/pihole/db-archive + +prole_audit_allow_prefixes_extra: [] + +prole_audit_dpkg_verify_packages: [] + +prole_audit_tracked_packages: + - curl + - iptables + - rsyslog + - samba + - krb5-user + - open-iscsi + - util-linux + - dphys-swapfile + - zram-tools diff --git a/infrastructure/roles/audit/tasks/main.yml b/infrastructure/roles/audit/tasks/main.yml new file mode 100644 index 0000000..bbaa00b --- /dev/null +++ b/infrastructure/roles/audit/tasks/main.yml @@ -0,0 +1,204 @@ +--- +- name: Build audit allowlists + ansible.builtin.set_fact: + prole_audit_scan_paths_combined: >- + {{ (prole_audit_scan_paths | default([])) + + (prole_audit_scan_paths_extra | default([])) + | unique }} + prole_audit_allow_prefixes_combined: >- + {{ (prole_audit_allow_prefixes | default([])) + + (prole_audit_allow_prefixes_extra | default([])) + | unique }} + changed_when: false + +- name: Verify dpkg database (Debian only) + ansible.builtin.command: >- + dpkg -V {{ prole_audit_dpkg_verify_packages | default([]) | join(' ') }} + register: prole_audit_dpkg_verify + changed_when: false + failed_when: false + check_mode: false + when: ansible_facts.os_family == "Debian" + +- name: List files not owned by dpkg and not managed by prole + ansible.builtin.shell: + cmd: | + set -euo pipefail + paths=({{ prole_audit_scan_paths_combined | map('quote') | join(' ') }}) + ALLOW_PREFIXES="$(printf '%s\n' {{ prole_audit_allow_prefixes_combined | map('quote') | join(' ') }})" + export ALLOW_PREFIXES + + valid_paths=() + for p in "${paths[@]}"; do + if [ -e "$p" ]; then + valid_paths+=("$p") + fi + done + + if [ "${#valid_paths[@]}" -eq 0 ]; then + exit 0 + fi + + { + find "${valid_paths[@]}" -xdev \( -type f -o -type l \) -print0 | + awk -v RS='\0' ' + BEGIN { n = split(ENVIRON["ALLOW_PREFIXES"], a, "\n") } + { + allowed = 0 + for (i = 1; i <= n; i++) { + p = a[i] + if (p != "" && index($0, p) == 1) { + if (length($0) == length(p) || substr($0, length(p) + 1, 1) == "/") { + allowed = 1 + break + } + } + } + if (!allowed) { printf "%s\0", $0 } + }' | + xargs -0 -r -n 200 dpkg -S 2>&1 >/dev/null || true + } | sed -n 's/^dpkg-query: no path found matching pattern //p' | sort -u + args: + executable: /bin/bash + register: prole_audit_unowned + changed_when: false + check_mode: false + when: + - ansible_facts.os_family == "Debian" + - prole_audit_scan_paths_combined | length > 0 + +- name: Check k3s binary + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: prole_audit_k3s_stat + changed_when: false + check_mode: false + +- name: Read k3s version + ansible.builtin.command: /usr/local/bin/k3s --version + register: prole_audit_k3s_version + changed_when: false + failed_when: false + check_mode: false + when: prole_audit_k3s_stat.stat.exists + +- name: Hash k3s binary + ansible.builtin.command: sha256sum /usr/local/bin/k3s + register: prole_audit_k3s_sha + changed_when: false + check_mode: false + when: prole_audit_k3s_stat.stat.exists + +- name: Check k3s dpkg owner + ansible.builtin.command: dpkg -S /usr/local/bin/k3s + register: prole_audit_k3s_owner + changed_when: false + failed_when: false + check_mode: false + when: prole_audit_k3s_stat.stat.exists and ansible_facts.os_family == "Debian" + +- name: Build k3s provenance + ansible.builtin.set_fact: + prole_audit_k3s_provenance: + path: /usr/local/bin/k3s + exists: "{{ prole_audit_k3s_stat.stat.exists | default(false) }}" + sha256: "{{ (prole_audit_k3s_sha.stdout | default('')) | regex_replace('\\s+.*$', '') }}" + version: "{{ prole_audit_k3s_version.stdout | default('') }}" + dpkg_owner: >- + {{ (prole_audit_k3s_owner.stdout | default('')) if (prole_audit_k3s_owner is defined and prole_audit_k3s_owner.rc == 0) + else 'unowned' }} + changed_when: false + +- name: Locate helm binary + ansible.builtin.shell: command -v helm + register: prole_audit_helm_path + changed_when: false + failed_when: false + check_mode: false + +- name: Read helm version + ansible.builtin.command: + argv: + - "{{ prole_audit_helm_path.stdout | trim }}" + - version + - --short + register: prole_audit_helm_version + changed_when: false + failed_when: false + check_mode: false + when: prole_audit_helm_path.rc == 0 + +- name: Hash helm binary + ansible.builtin.command: sha256sum "{{ prole_audit_helm_path.stdout | trim }}" + register: prole_audit_helm_sha + changed_when: false + check_mode: false + when: prole_audit_helm_path.rc == 0 + +- name: Check helm dpkg owner + ansible.builtin.command: dpkg -S "{{ prole_audit_helm_path.stdout | trim }}" + register: prole_audit_helm_owner + changed_when: false + failed_when: false + check_mode: false + when: prole_audit_helm_path.rc == 0 and ansible_facts.os_family == "Debian" + +- name: Build helm provenance + ansible.builtin.set_fact: + prole_audit_helm_provenance: + path: "{{ prole_audit_helm_path.stdout | default('') | trim }}" + exists: "{{ prole_audit_helm_path.rc == 0 }}" + sha256: "{{ (prole_audit_helm_sha.stdout | default('')) | regex_replace('\\s+.*$', '') }}" + version: "{{ prole_audit_helm_version.stdout | default('') }}" + dpkg_owner: >- + {{ (prole_audit_helm_owner.stdout | default('')) if (prole_audit_helm_owner is defined and prole_audit_helm_owner.rc == 0) + else 'unowned' }} + changed_when: false + +- name: Query tracked package versions + ansible.builtin.shell: >- + dpkg-query -W -f='${Package} ${Version} ${Architecture}\n' + {{ prole_audit_tracked_packages | default([]) | join(' ') }} + register: prole_audit_tracked_pkgs + changed_when: false + failed_when: false + check_mode: false + when: + - ansible_facts.os_family == "Debian" + - prole_audit_tracked_packages | default([]) | length > 0 + +- name: Report dpkg verification changes + ansible.builtin.debug: + msg: >- + {{ ((prole_audit_dpkg_verify.stdout_lines | default([])) + + (prole_audit_dpkg_verify.stderr_lines | default([]))) + if ((prole_audit_dpkg_verify.stdout_lines | default([]) | length) > 0 + or (prole_audit_dpkg_verify.stderr_lines | default([]) | length) > 0) + else ['dpkg -V returned no differences'] }} + when: prole_audit_dpkg_verify is defined + +- name: Report files not owned by dpkg and not managed by prole + ansible.builtin.debug: + msg: >- + {{ (prole_audit_unowned.stdout_lines | default([])) + if (prole_audit_unowned.stdout_lines | default([]) | length) > 0 + else ['no unowned files found in scan paths'] }} + when: prole_audit_unowned is defined + +- name: Report tracked package versions + ansible.builtin.debug: + msg: >- + {{ ((prole_audit_tracked_pkgs.stdout_lines | default([])) + + (prole_audit_tracked_pkgs.stderr_lines | default([]))) + if ((prole_audit_tracked_pkgs.stdout_lines | default([]) | length) > 0 + or (prole_audit_tracked_pkgs.stderr_lines | default([]) | length) > 0) + else ['no tracked packages reported'] }} + when: prole_audit_tracked_pkgs is defined + +- name: Report k3s provenance + ansible.builtin.debug: + var: prole_audit_k3s_provenance + +- name: Report helm provenance + ansible.builtin.debug: + var: prole_audit_helm_provenance diff --git a/infrastructure/roles/cgroups/defaults/main.yml b/infrastructure/roles/cgroups/defaults/main.yml new file mode 100644 index 0000000..83d59ec --- /dev/null +++ b/infrastructure/roles/cgroups/defaults/main.yml @@ -0,0 +1,13 @@ +--- +cgroups_cmdline_candidates: + - /boot/firmware/cmdline.txt + - /boot/cmdline.txt + +cgroups_required_params: + - cgroup_memory=1 + - cgroup_enable=memory + +cgroups_remove_params: + - cgroup_disable=memory + +cgroups_reboot: true diff --git a/infrastructure/roles/cgroups/tasks/main.yml b/infrastructure/roles/cgroups/tasks/main.yml new file mode 100644 index 0000000..74095c4 --- /dev/null +++ b/infrastructure/roles/cgroups/tasks/main.yml @@ -0,0 +1,196 @@ +--- +- name: Find kernel cmdline file + ansible.builtin.stat: + path: "{{ item }}" + loop: "{{ cgroups_cmdline_candidates }}" + register: cmdline_stats + +- name: Select kernel cmdline path + ansible.builtin.set_fact: + cgroups_cmdline_all_paths: "{{ cmdline_stats.results | selectattr('stat.exists') | map(attribute='stat.path') | list }}" + +- name: Select kernel cmdline update targets + ansible.builtin.set_fact: + cgroups_cmdline_paths: "{{ cgroups_cmdline_all_paths }}" + +- name: Fail when kernel cmdline file is missing + ansible.builtin.fail: + msg: "No kernel cmdline file found. Checked: {{ cgroups_cmdline_candidates | join(', ') }}" + when: cgroups_cmdline_paths | length == 0 + +- name: Remove literal backslash escapes from kernel cmdline files + ansible.builtin.replace: + path: "{{ item }}" + regexp: "\\\\[nrt]" + replace: " " + loop: "{{ cgroups_cmdline_paths }}" + register: cmdline_escape_cleanup + +- name: Remove stray backslashes from kernel cmdline files + ansible.builtin.replace: + path: "{{ item }}" + regexp: "\\\\" + replace: " " + loop: "{{ cgroups_cmdline_paths }}" + register: cmdline_backslash_cleanup + +- name: Read kernel cmdline files + ansible.builtin.slurp: + path: "{{ item }}" + loop: "{{ cgroups_cmdline_paths }}" + register: cmdline_files + +- name: Read current /proc/cmdline + ansible.builtin.command: "cat /proc/cmdline" + register: proc_cmdline_current + changed_when: false + check_mode: no + +- name: Check current cgroup params in /proc/cmdline + ansible.builtin.set_fact: + cgroups_missing_in_proc: "{{ cgroups_required_params | reject('in', proc_cmdline_current.stdout | default('')) | list }}" + cgroups_conflicting_in_proc: "{{ cgroups_remove_params | select('in', proc_cmdline_current.stdout | default('')) | list }}" + +- name: Normalize kernel cmdline content + ansible.builtin.set_fact: + cgroups_cmdline_files: >- + {{ (cgroups_cmdline_files | default([])) + [{ + 'path': item.item, + 'raw_line': ((item.content | b64decode).splitlines() | join(' ') | trim), + 'sanitized': (((item.content | b64decode).splitlines() | join(' ') | trim) + | regex_replace('\\\\[nrt]', ' ') + | regex_replace('\\\\', ' ') + | regex_replace('\\s+', ' ') + | trim) + }] }} + loop: "{{ cmdline_files.results }}" + +- name: Select kernel cmdline line + ansible.builtin.set_fact: + cgroups_cmdline_current: "{{ (cgroups_cmdline_files | map(attribute='sanitized') | list | first) | default('') }}" + +- name: Select cmdline source + ansible.builtin.set_fact: + cgroups_cmdline_source: "{{ (cgroups_cmdline_current | length > 0) | ternary(cgroups_cmdline_current, proc_cmdline_current.stdout | default('')) }}" + +- name: Sanitize kernel cmdline + ansible.builtin.set_fact: + cgroups_cmdline_sanitized: >- + {{ cgroups_cmdline_source + | regex_replace('\\\\[nrt]', ' ') + | regex_replace('\\\\', ' ') + | regex_replace('\\s+', ' ') + | trim }} + +- name: Tokenize kernel cmdline + ansible.builtin.set_fact: + cgroups_cmdline_tokens: "{{ cgroups_cmdline_sanitized.split() }}" + +- name: Remove conflicting cgroup params + ansible.builtin.set_fact: + cgroups_cmdline_filtered: "{{ cgroups_cmdline_tokens | reject('in', cgroups_remove_params) | list }}" + +- name: Calculate missing cgroup params + ansible.builtin.set_fact: + cgroups_missing_params: "{{ cgroups_required_params | reject('in', cgroups_cmdline_filtered) | list }}" + +- name: Build updated kernel cmdline tokens + ansible.builtin.set_fact: + cgroups_cmdline_new_tokens: "{{ (cgroups_cmdline_filtered + cgroups_missing_params) | unique | list }}" + +- name: Build updated kernel cmdline line + ansible.builtin.set_fact: + cgroups_cmdline_new: "{{ cgroups_cmdline_new_tokens | join(' ') }}" + +- name: Sanitize updated kernel cmdline line + ansible.builtin.set_fact: + cgroups_cmdline_new: >- + {{ cgroups_cmdline_new + | regex_replace('\\\\[nrt]', ' ') + | regex_replace('\\\\', ' ') + | regex_replace('\\s+', ' ') + | trim }} + +- name: Determine if kernel cmdline needs update + ansible.builtin.set_fact: + cgroups_cmdline_needs_update: >- + {{ cgroups_cmdline_new != cgroups_cmdline_sanitized or + (cgroups_cmdline_paths | length) > 1 }} + +- name: Update kernel cmdline when params are missing + ansible.builtin.copy: + dest: "{{ item.path }}" + content: >- + {{ cgroups_cmdline_new ~ '\n' }} + mode: "0644" + register: cmdline_update + when: item.raw_line != cgroups_cmdline_new + loop: "{{ cgroups_cmdline_files }}" + +- name: Remove literal backslash escapes from updated cmdline files + ansible.builtin.replace: + path: "{{ item }}" + regexp: "\\\\[nrt]" + replace: " " + loop: "{{ cgroups_cmdline_paths }}" + when: (cmdline_update.results | default([]) | selectattr('changed') | list | length) > 0 + +- name: Normalize updated cmdline whitespace + ansible.builtin.replace: + path: "{{ item }}" + regexp: "\\s+" + replace: " " + loop: "{{ cgroups_cmdline_paths }}" + when: (cmdline_update.results | default([]) | selectattr('changed') | list | length) > 0 + +- name: Determine if kernel cmdline was updated + ansible.builtin.set_fact: + cgroups_cmdline_changed: >- + {{ (cmdline_escape_cleanup.results | default([]) | selectattr('changed') | list | length) > 0 or + (cmdline_backslash_cleanup.results | default([]) | selectattr('changed') | list | length) > 0 or + (cmdline_update.results | default([]) | selectattr('changed') | list | length) > 0 }} + +- name: Determine if reboot is required + ansible.builtin.set_fact: + cgroups_reboot_needed: >- + {{ cgroups_cmdline_changed | bool or + (cgroups_missing_in_proc | default([]) | length) > 0 }} + +- name: Reboot to apply cgroup settings + ansible.builtin.reboot: + msg: "Rebooting to apply cgroup kernel parameters" + pre_reboot_delay: 3 + reboot_timeout: 600 + when: cgroups_reboot_needed | bool and cgroups_reboot | bool + +- name: Verify cgroup kernel parameters after reboot + ansible.builtin.command: "cat /proc/cmdline" + register: proc_cmdline + changed_when: false + check_mode: no + +- name: Read cgroup v2 controllers + ansible.builtin.command: "cat /sys/fs/cgroup/cgroup.controllers" + register: cgroup_controllers + changed_when: false + check_mode: no + +- name: Record cgroup memory controller presence + ansible.builtin.set_fact: + cgroups_memory_controller_present: "{{ 'memory' in (cgroup_controllers.stdout | default('')) }}" + +- name: Fail when cgroup params are missing (reboot required) + ansible.builtin.fail: + msg: "Missing cgroup params in /proc/cmdline: {{ cgroups_required_params | reject('in', proc_cmdline.stdout | default('')) | list }}. Reboot required." + when: + - not ansible_check_mode + - (cgroups_required_params | reject('in', proc_cmdline.stdout | default('')) | list) | length > 0 + - not cgroups_memory_controller_present + +- name: Fail when conflicting cgroup params are present + ansible.builtin.fail: + msg: "Conflicting cgroup params in /proc/cmdline: {{ cgroups_remove_params | select('in', proc_cmdline.stdout | default('')) | list }}. Reboot required." + when: + - not ansible_check_mode + - (cgroups_remove_params | select('in', proc_cmdline.stdout | default('')) | list) | length > 0 + - not cgroups_memory_controller_present diff --git a/infrastructure/roles/dashboard/defaults/main.yml b/infrastructure/roles/dashboard/defaults/main.yml new file mode 100644 index 0000000..13c47a5 --- /dev/null +++ b/infrastructure/roles/dashboard/defaults/main.yml @@ -0,0 +1,23 @@ +--- + +# Local console dashboard (X + conky) +dashboard_user: pi +dashboard_script_dest: /usr/local/bin/dashboard.sh + +# Primary display (Linux VT) to use for the dashboard session. +dashboard_vt: 1 + +# Whether to enable the service at boot. (Starting it manually is enough.) +dashboard_enable: false + +# Start the service as part of the playbook run. +dashboard_start: false + +# Debian-family package list (kept explicit for reproducibility). +dashboard_packages_debian: + # `conky` is a virtual package on some Debian releases; install a concrete provider. + - conky-all + - fonts-dejavu-core + - x11-xserver-utils + - xinit + - xserver-xorg diff --git a/infrastructure/roles/dashboard/handlers/main.yml b/infrastructure/roles/dashboard/handlers/main.yml new file mode 100644 index 0000000..9314521 --- /dev/null +++ b/infrastructure/roles/dashboard/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: reload systemd + ansible.builtin.command: systemctl daemon-reload + changed_when: false diff --git a/infrastructure/roles/dashboard/tasks/main.yml b/infrastructure/roles/dashboard/tasks/main.yml new file mode 100644 index 0000000..47ddf3f --- /dev/null +++ b/infrastructure/roles/dashboard/tasks/main.yml @@ -0,0 +1,41 @@ +--- + +- name: Install dashboard dependencies (Debian) + ansible.builtin.apt: + name: "{{ dashboard_packages | default(dashboard_packages_debian) }}" + state: present + update_cache: true + when: ansible_facts.os_family == 'Debian' + +- name: Fail on unsupported OS family + ansible.builtin.fail: + msg: "dashboard role currently supports Debian-family hosts only (got os_family={{ ansible_facts.os_family }})." + when: ansible_facts.os_family != 'Debian' + +- name: Deploy dashboard script + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../tools/dashboard.sh" + dest: "{{ dashboard_script_dest }}" + owner: root + group: root + mode: '0755' + +- name: Deploy systemd unit + ansible.builtin.template: + src: dashboard.service.j2 + dest: /etc/systemd/system/dashboard.service + owner: root + group: root + mode: '0644' + notify: reload systemd + +- name: Enable dashboard service + ansible.builtin.systemd: + name: dashboard + enabled: "{{ dashboard_enable | bool }}" + +- name: Start dashboard service (optional) + ansible.builtin.systemd: + name: dashboard + state: started + when: dashboard_start | bool diff --git a/infrastructure/roles/dashboard/templates/dashboard.service.j2 b/infrastructure/roles/dashboard/templates/dashboard.service.j2 new file mode 100644 index 0000000..4a60c87 --- /dev/null +++ b/infrastructure/roles/dashboard/templates/dashboard.service.j2 @@ -0,0 +1,40 @@ +[Unit] +Description=Prole console dashboard (X + conky) +After=systemd-user-sessions.service +Wants=systemd-user-sessions.service + +# If a graphical desktop is running, starting this service will stop it. +Conflicts=graphical.target +Before=graphical.target + +# If a getty is running on the dashboard VT, stop it so Xorg can take over. +Conflicts=getty@tty{{ dashboard_vt }}.service + +[Service] +Type=simple + +# Attach the service to the dashboard VT so Xorg can take over the display. +TTYPath=/dev/tty{{ dashboard_vt }} +StandardInput=tty +StandardOutput=journal+console +StandardError=journal+console +TTYReset=yes +TTYVHangup=yes +TTYVTDisallocate=yes + +Environment=DASHBOARD_VT={{ dashboard_vt }} + +ExecStart={{ dashboard_script_dest }} + +# When the dashboard stops, try to restore the login prompt on that VT. +ExecStopPost=-/usr/bin/systemctl --no-block start getty@tty{{ dashboard_vt }}.service + +Restart=always +RestartSec=2 + +# Ensure X/conky are terminated when the service stops. +KillMode=control-group +TimeoutStopSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/infrastructure/roles/dashboard/tests/defaults.yml b/infrastructure/roles/dashboard/tests/defaults.yml new file mode 100644 index 0000000..c025a1d --- /dev/null +++ b/infrastructure/roles/dashboard/tests/defaults.yml @@ -0,0 +1,19 @@ +--- +- name: Verify dashboard role defaults are installable on Debian (no virtual packages) + hosts: localhost + connection: local + gather_facts: false + + tasks: + - name: Load role defaults + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../defaults/main.yml" + + - name: Ensure conky package is a concrete provider (not the virtual package) + ansible.builtin.assert: + that: + - "'conky' not in dashboard_packages_debian" + - "'conky-all' in dashboard_packages_debian" + fail_msg: >- + dashboard_packages_debian must not include the virtual 'conky' package. + Use a concrete provider such as 'conky-all', 'conky-std', or 'conky-cli'. diff --git a/infrastructure/roles/iscsi/handlers/main.yml b/infrastructure/roles/iscsi/handlers/main.yml new file mode 100644 index 0000000..db9173d --- /dev/null +++ b/infrastructure/roles/iscsi/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: restart open-iscsi + ansible.builtin.systemd: + name: open-iscsi + state: restarted + enabled: true diff --git a/infrastructure/roles/iscsi/tasks/detach.yml b/infrastructure/roles/iscsi/tasks/detach.yml new file mode 100644 index 0000000..3c106a8 --- /dev/null +++ b/infrastructure/roles/iscsi/tasks/detach.yml @@ -0,0 +1,69 @@ +--- +- name: Require iscsi_portal when logging out absent targets + ansible.builtin.assert: + that: + - iscsi_portal is defined + - (iscsi_portal | string | length) > 0 + fail_msg: "iscsi_absent_targets is set but iscsi_portal is missing/empty." + when: (iscsi_absent_targets | default([]) | length) > 0 + +- name: Unmount absent iSCSI mountpoints + ansible.builtin.mount: + path: "{{ item.path | default(item) }}" + state: unmounted + loop: "{{ iscsi_absent_mounts | default([]) }}" + loop_control: + label: "{{ item.path | default(item) }}" + tags: + - iscsi_cleanup + +- name: Remove absent iSCSI fstab entries + ansible.builtin.lineinfile: + path: /etc/fstab + state: absent + regexp: "^\\s*\\S+\\s+{{ (item.path | default(item)) | regex_escape }}\\s+" + loop: "{{ iscsi_absent_mounts | default([]) }}" + loop_control: + label: "{{ item.path | default(item) }}" + tags: + - iscsi_cleanup + +- name: Check for active sessions for absent targets + ansible.builtin.shell: >- + iscsiadm -m session 2>/dev/null | grep -Fq -- {{ item | quote }} + register: _iscsi_absent_target_session_checks + changed_when: false + failed_when: false + loop: "{{ iscsi_absent_targets | default([]) }}" + loop_control: + label: "{{ item }}" + when: (iscsi_absent_targets | default([]) | length) > 0 + tags: + - iscsi_cleanup + +- name: Logout absent iSCSI targets (active sessions only) + ansible.builtin.command: >- + iscsiadm -m node -T {{ item.item }} -p {{ iscsi_portal }} --logout + register: _iscsi_absent_target_logout + changed_when: false + failed_when: false + loop: "{{ _iscsi_absent_target_session_checks.results | default([]) }}" + loop_control: + label: "{{ item.item }}" + when: + - (iscsi_absent_targets | default([]) | length) > 0 + - item.rc == 0 + tags: + - iscsi_cleanup + +- name: Delete node records for absent iSCSI targets (best-effort) + ansible.builtin.command: >- + iscsiadm -m node -o delete -T {{ item }} -p {{ iscsi_portal }} + changed_when: false + failed_when: false + loop: "{{ iscsi_absent_targets | default([]) }}" + loop_control: + label: "{{ item }}" + when: (iscsi_absent_targets | default([]) | length) > 0 + tags: + - iscsi_cleanup \ No newline at end of file diff --git a/infrastructure/roles/iscsi/tasks/iscsi_mount.yml b/infrastructure/roles/iscsi/tasks/iscsi_mount.yml new file mode 100644 index 0000000..bb9f82a --- /dev/null +++ b/infrastructure/roles/iscsi/tasks/iscsi_mount.yml @@ -0,0 +1,338 @@ +--- +- name: Ensure mountpoint exists + ansible.builtin.file: + path: "{{ m.path }}" + state: directory + mode: "0755" + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Reset computed mount source + ansible.builtin.set_fact: + _iscsi_mount_src: "" + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Decide whether a block device is required for this mount + ansible.builtin.set_fact: + _iscsi_need_blockdev: >- + {{ (m.mkfs_once | default(false) | bool) + or (m.mkfs_if_missing | default(false) | bool) + or (m.device is defined and (m.device | string | length) > 0) + or (m.src is not defined) or ((m.src | default('') | string | length) == 0) }} + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Parse iSCSI portal (host/port) + ansible.builtin.set_fact: + _iscsi_portal_host: "{{ (iscsi_portal | string).split(':')[0] }}" + _iscsi_portal_port: "{{ ((iscsi_portal | string).split(':') | length > 1) | ternary((iscsi_portal | string).split(':')[1], '3260') }}" + when: + - _iscsi_need_blockdev | default(false) | bool + - iscsi_portal is defined + - (iscsi_portal | string | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Use explicitly provided block device (if any) + ansible.builtin.set_fact: + _iscsi_blockdev: "{{ m.device }}" + when: + - _iscsi_need_blockdev | default(false) | bool + - m.device is defined + - (m.device | string | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Compute iSCSI by-path pattern + ansible.builtin.set_fact: + _iscsi_by_path_pattern: >- + ip-{{ _iscsi_portal_host }}:{{ _iscsi_portal_port }}-iscsi-{{ iscsi_target.iqn }}-lun-{{ (m.lun is defined) | ternary((m.lun | string), '*') }} + when: + - _iscsi_need_blockdev | default(false) | bool + - (_iscsi_blockdev is not defined) or ((_iscsi_blockdev | string | length) == 0) + - iscsi_target is defined + - iscsi_target.iqn is defined + - _iscsi_portal_host is defined + - (_iscsi_portal_host | string | length) > 0 + - _iscsi_portal_port is defined + - (_iscsi_portal_port | string | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Wait for iSCSI by-path device to appear + ansible.builtin.find: + paths: + - /dev/disk/by-path + patterns: + - "{{ _iscsi_by_path_pattern }}" + file_type: any + register: _iscsi_by_path_find + until: (_iscsi_by_path_find.matched | default(0) | int) > 0 + retries: "{{ ((m.device_timeout | default(60)) | int // 5) + 1 }}" + delay: 5 + when: + - _iscsi_need_blockdev | default(false) | bool + - (_iscsi_blockdev is not defined) or ((_iscsi_blockdev | string | length) == 0) + - _iscsi_by_path_pattern is defined + - (_iscsi_by_path_pattern | string | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Select iSCSI block device from by-path matches + ansible.builtin.set_fact: + _iscsi_blockdev: "{{ (_iscsi_by_path_find.files | map(attribute='path') | list | sort | first) }}" + when: + - _iscsi_need_blockdev | default(false) | bool + - (_iscsi_blockdev is not defined) or ((_iscsi_blockdev | string | length) == 0) + - _iscsi_by_path_find is defined + - (_iscsi_by_path_find.matched | default(0) | int) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Read existing filesystem type (if any) + ansible.builtin.command: "blkid -o value -s TYPE {{ _iscsi_blockdev }}" + register: _iscsi_blkid_type + changed_when: false + failed_when: false + check_mode: no + when: + - _iscsi_need_blockdev | default(false) | bool + - _iscsi_blockdev is defined + - (_iscsi_blockdev | string | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Ensure marker directory exists (mkfs_once) + ansible.builtin.file: + path: /var/lib/prole/iscsi + state: directory + owner: root + group: root + mode: "0755" + when: m.mkfs_once | default(false) | bool + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Compute mkfs marker path (mkfs_once) + ansible.builtin.set_fact: + _iscsi_mkfs_marker: >- + /var/lib/prole/iscsi/mkfs- + {{ (((iscsi_target.iqn | default('unknown-iqn')) ~ '-' ~ (m.path | default('unknown-path'))) + | regex_replace('[^A-Za-z0-9_.-]', '_')) }} + .done + when: m.mkfs_once | default(false) | bool + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Check mkfs marker presence (mkfs_once) + ansible.builtin.stat: + path: "{{ _iscsi_mkfs_marker }}" + register: _iscsi_mkfs_marker_stat + when: m.mkfs_once | default(false) | bool + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Fail if mountpoint is already mounted but mkfs_once requested + ansible.builtin.command: "findmnt -n {{ m.path }}" + register: _iscsi_findmnt_before_mkfs + changed_when: false + failed_when: false + when: + - m.mkfs_once | default(false) | bool + - _iscsi_mkfs_marker_stat.stat.exists is not defined or not _iscsi_mkfs_marker_stat.stat.exists + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Refuse to mkfs when mountpoint is mounted + ansible.builtin.fail: + msg: "Refusing to mkfs for {{ m.path }} because it is already mounted ({{ _iscsi_findmnt_before_mkfs.stdout | default('') }})." + when: + - m.mkfs_once | default(false) | bool + - _iscsi_mkfs_marker_stat.stat.exists is not defined or not _iscsi_mkfs_marker_stat.stat.exists + - _iscsi_findmnt_before_mkfs.rc == 0 + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Require iSCSI block device path for mkfs_once + ansible.builtin.assert: + that: + - _iscsi_blockdev is defined + - (_iscsi_blockdev | string | length) > 0 + fail_msg: "mkfs_once requested for {{ m.path }} but no block device was computed (set m.device or ensure iscsi_target+iscsi_portal are available)." + when: + - m.mkfs_once | default(false) | bool + - _iscsi_mkfs_marker_stat.stat.exists is not defined or not _iscsi_mkfs_marker_stat.stat.exists + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Re-initialize filesystem (mkfs_once) + ansible.builtin.command: >- + {{ ((m.fstype | default('ext4')) == 'xfs') + | ternary( + 'mkfs.xfs -f ' ~ _iscsi_blockdev, + (((m.fstype | default('ext4')) == 'ext4') + | ternary( + 'mkfs.ext4 -F ' ~ _iscsi_blockdev, + 'mkfs -t ' ~ (m.fstype | default('ext4')) ~ ' ' ~ _iscsi_blockdev + ) + ) + ) }} + changed_when: true + when: + - m.mkfs_once | default(false) | bool + - _iscsi_mkfs_marker_stat.stat.exists is not defined or not _iscsi_mkfs_marker_stat.stat.exists + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Create filesystem when missing (mkfs_if_missing) + ansible.builtin.command: >- + {{ ((m.fstype | default('ext4')) == 'xfs') + | ternary( + 'mkfs.xfs -f ' ~ _iscsi_blockdev, + (((m.fstype | default('ext4')) == 'ext4') + | ternary( + 'mkfs.ext4 -F ' ~ _iscsi_blockdev, + 'mkfs -t ' ~ (m.fstype | default('ext4')) ~ ' ' ~ _iscsi_blockdev + ) + ) + ) }} + changed_when: true + when: + - m.mkfs_if_missing | default(false) | bool + - _iscsi_blockdev is defined + - (_iscsi_blockdev | string | length) > 0 + - _iscsi_blkid_type is not defined or (_iscsi_blkid_type.stdout | default('') | trim) == '' + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Refuse to mount when existing filesystem type does not match + ansible.builtin.fail: + msg: >- + Existing filesystem type on {{ _iscsi_blockdev }} is + '{{ _iscsi_blkid_type.stdout | default('') | trim }}' but mount {{ m.path }} + requests fstype='{{ m.fstype | default('ext4') }}'. + Wipe the device signature (e.g. wipefs) or set mkfs_once=true if you intend + to reformat this LUN. + when: + - _iscsi_blkid_type is defined + - (_iscsi_blkid_type.stdout | default('') | trim) != '' + - (_iscsi_blkid_type.stdout | trim) != (m.fstype | default('ext4')) + - not (m.mkfs_once | default(false) | bool) + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Write mkfs marker (mkfs_once) + ansible.builtin.file: + path: "{{ _iscsi_mkfs_marker }}" + state: touch + owner: root + group: root + mode: "0644" + when: + - m.mkfs_once | default(false) | bool + - _iscsi_mkfs_marker_stat.stat.exists is not defined or not _iscsi_mkfs_marker_stat.stat.exists + tags: + - iscsi + - iscsi_mkfs + - iscsi_storage + +- name: Resolve mount source + ansible.builtin.set_fact: + _iscsi_mount_src: "{{ m.src }}" + when: + - m.src is defined + - (m.src | string | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Resolve mount source UUID from block device + ansible.builtin.command: "blkid -o value -s UUID {{ _iscsi_blockdev }}" + register: _iscsi_blkid_uuid + changed_when: false + check_mode: no + when: + - (_iscsi_mount_src | default('') | string | length) == 0 + - _iscsi_blockdev is defined + - (_iscsi_blockdev | string | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Require resolved UUID for mounting + ansible.builtin.assert: + that: + - _iscsi_blkid_uuid is defined + - (_iscsi_blkid_uuid.stdout | default('') | trim) != '' + fail_msg: "Unable to resolve a filesystem UUID for {{ m.path }} from {{ _iscsi_blockdev }}." + when: + - (_iscsi_mount_src | default('') | string | length) == 0 + - not (ansible_check_mode and ((m.mkfs_once | default(false) | bool) or (m.mkfs_if_missing | default(false) | bool))) + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Set mount source to UUID + ansible.builtin.set_fact: + _iscsi_mount_src: "UUID={{ _iscsi_blkid_uuid.stdout | trim }}" + when: + - (_iscsi_mount_src | default('') | string | length) == 0 + - _iscsi_blkid_uuid is defined + - (_iscsi_blkid_uuid.stdout | default('') | trim) != '' + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Mount {{ m.path }} + ansible.builtin.mount: + path: "{{ m.path }}" + src: "{{ _iscsi_mount_src }}" + fstype: "{{ m.fstype | default('ext4') }}" + opts: "{{ m.opts | default('_netdev,noatime') }}" + state: mounted + when: (_iscsi_mount_src | default('') | string | trim | length) > 0 + tags: + - iscsi + - iscsi_mount + - iscsi_storage diff --git a/infrastructure/roles/iscsi/tasks/iscsi_target.yml b/infrastructure/roles/iscsi/tasks/iscsi_target.yml new file mode 100644 index 0000000..8829812 --- /dev/null +++ b/infrastructure/roles/iscsi/tasks/iscsi_target.yml @@ -0,0 +1,74 @@ +--- +- name: Ensure node record exists for target + ansible.builtin.command: > + iscsiadm -m node -T {{ t.iqn }} -p {{ iscsi_portal }} + register: node_check + changed_when: false + failed_when: false + +- name: Create node record if missing + ansible.builtin.command: > + iscsiadm -m node -o new -T {{ t.iqn }} -p {{ iscsi_portal }} + when: node_check.rc != 0 + +- name: Configure CHAP authmethod + ansible.builtin.command: > + iscsiadm -m node -T {{ t.iqn }} -p {{ iscsi_portal }} + --op update -n node.session.auth.authmethod -v CHAP + when: (t.chap_user | default('')) | length > 0 + +- name: Configure CHAP username + ansible.builtin.command: > + iscsiadm -m node -T {{ t.iqn }} -p {{ iscsi_portal }} + --op update -n node.session.auth.username -v {{ t.chap_user }} + when: (t.chap_user | default('')) | length > 0 + +- name: Configure CHAP password + ansible.builtin.command: > + iscsiadm -m node -T {{ t.iqn }} -p {{ iscsi_portal }} + --op update -n node.session.auth.password -v {{ t.chap_password }} + when: (t.chap_user | default('')) | length > 0 + no_log: true + +- name: Check if target session is already logged in + ansible.builtin.shell: >- + iscsiadm -m session 2>/dev/null | grep -Fq -- {{ t.iqn | quote }} + register: session_check + changed_when: false + failed_when: false + +- name: Login to target + ansible.builtin.command: > + iscsiadm -m node -T {{ t.iqn }} -p {{ iscsi_portal }} --login + register: login + changed_when: false + failed_when: false + when: session_check.rc != 0 + +- name: Re-check session after login attempt + ansible.builtin.shell: >- + iscsiadm -m session 2>/dev/null | grep -Fq -- {{ t.iqn | quote }} + register: session_after_login + changed_when: false + failed_when: false + when: session_check.rc != 0 + +- name: Fail if login did not establish a session + ansible.builtin.fail: + msg: |- + iSCSI login failed for {{ t.iqn }} at {{ iscsi_portal }}. + + login.rc={{ login.rc | default('') }} + login.stdout={{ (login.stdout | default('')) | trim }} + login.stderr={{ (login.stderr | default('')) | trim }} + when: + - session_check.rc != 0 + - session_after_login is defined + - session_after_login.rc != 0 + - login is defined + - login.rc not in [0, 15] + +- name: Ensure iSCSI autologin + ansible.builtin.command: > + iscsiadm -m node -T {{ t.iqn }} -p {{ iscsi_portal }} + --op update -n node.startup -v automatic diff --git a/infrastructure/roles/iscsi/tasks/iscsi_target_mounts.yml b/infrastructure/roles/iscsi/tasks/iscsi_target_mounts.yml new file mode 100644 index 0000000..1efb017 --- /dev/null +++ b/infrastructure/roles/iscsi/tasks/iscsi_target_mounts.yml @@ -0,0 +1,9 @@ +--- +- name: Mount target filesystems + ansible.builtin.include_tasks: iscsi_mount.yml + loop: "{{ t.mounts | default([]) }}" + loop_control: + loop_var: m + label: "{{ m.path | default('') }}" + vars: + iscsi_target: "{{ t }}" diff --git a/infrastructure/roles/iscsi/tasks/login.yml b/infrastructure/roles/iscsi/tasks/login.yml new file mode 100644 index 0000000..158eb0f --- /dev/null +++ b/infrastructure/roles/iscsi/tasks/login.yml @@ -0,0 +1,160 @@ +--- +- name: Install iSCSI dependencies + ansible.builtin.package: + name: + - open-iscsi + - util-linux + - xfsprogs + state: present + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Ensure iscsid enabled + ansible.builtin.service: + name: iscsid + state: started + enabled: true + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Check if k3s systemd unit exists + ansible.builtin.stat: + path: /etc/systemd/system/k3s.service + register: _k3s_unit + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Check if k3s-agent systemd unit exists + ansible.builtin.stat: + path: /etc/systemd/system/k3s-agent.service + register: _k3s_agent_unit + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Ensure k3s stops/restarts before open-iscsi (systemd drop-in) + block: + - name: Ensure k3s.service.d exists + ansible.builtin.file: + path: /etc/systemd/system/k3s.service.d + state: directory + mode: "0755" + + - name: Install k3s open-iscsi ordering drop-in + ansible.builtin.copy: + dest: /etc/systemd/system/k3s.service.d/open-iscsi.conf + mode: "0644" + content: | + [Unit] + Wants=open-iscsi.service + After=open-iscsi.service + PartOf=open-iscsi.service + notify: + - reload systemd + when: _k3s_unit.stat.exists | default(false) + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Ensure k3s-agent stops/restarts before open-iscsi (systemd drop-in) + block: + - name: Ensure k3s-agent.service.d exists + ansible.builtin.file: + path: /etc/systemd/system/k3s-agent.service.d + state: directory + mode: "0755" + + - name: Install k3s-agent open-iscsi ordering drop-in + ansible.builtin.copy: + dest: /etc/systemd/system/k3s-agent.service.d/open-iscsi.conf + mode: "0644" + content: | + [Unit] + Wants=open-iscsi.service + After=open-iscsi.service + PartOf=open-iscsi.service + notify: + - reload systemd + when: _k3s_agent_unit.stat.exists | default(false) + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Ensure open-iscsi systemd override directory exists + ansible.builtin.file: + path: /etc/systemd/system/open-iscsi.service.d + state: directory + mode: "0755" + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Install open-iscsi retry-on-boot override + ansible.builtin.copy: + dest: /etc/systemd/system/open-iscsi.service.d/retry-on-boot.conf + mode: "0644" + content: | + [Unit] + Wants=network-online.target iscsid.service + After=network-online.target iscsid.service + + [Service] + Restart=on-failure + RestartSec=10s + StartLimitIntervalSec=120 + StartLimitBurst=10 + notify: + - reload systemd + - restart open-iscsi + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Apply open-iscsi override immediately (if changed) + ansible.builtin.meta: flush_handlers + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Ensure open-iscsi enabled + ansible.builtin.systemd: + name: open-iscsi + enabled: true + tags: + - iscsi + - iscsi_login + - iscsi_storage + +- name: Discover iSCSI targets + ansible.builtin.command: >- + iscsiadm -m discovery -t st -p {{ iscsi_portal }} + register: _iscsi_discovery + changed_when: false + tags: + - iscsi + - iscsi_discovery + - iscsi_login + - iscsi_storage + +- name: Configure and login iSCSI targets + ansible.builtin.include_tasks: iscsi_target.yml + loop: "{{ iscsi_targets | default([]) }}" + loop_control: + loop_var: t + label: "{{ t.iqn | default('') }}" + tags: + - iscsi + - iscsi_login + - iscsi_storage diff --git a/infrastructure/roles/iscsi/tasks/main.yml b/infrastructure/roles/iscsi/tasks/main.yml new file mode 100644 index 0000000..70f55c7 --- /dev/null +++ b/infrastructure/roles/iscsi/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: iSCSI auth/login (storage) + ansible.builtin.include_tasks: login.yml + +- name: iSCSI mounts (filesystem) + ansible.builtin.include_tasks: mount.yml diff --git a/infrastructure/roles/iscsi/tasks/mount.yml b/infrastructure/roles/iscsi/tasks/mount.yml new file mode 100644 index 0000000..a1f1bd7 --- /dev/null +++ b/infrastructure/roles/iscsi/tasks/mount.yml @@ -0,0 +1,97 @@ +--- +- name: Compute legacy mountpoints to clean up (/prole/d00x -> /synology/d00x) + ansible.builtin.set_fact: + _iscsi_legacy_absent_mounts: >- + {{ + (iscsi_targets | default([]) + | map(attribute='mounts') | list | flatten + | selectattr('path', 'defined') + | map(attribute='path') | list + | select('match', '^/synology/d[0-9]{3}$') + | map('regex_replace', '^/synology/', '/prole/') + | list) + }} + changed_when: false + tags: + - iscsi_cleanup + +- name: Unmount legacy iSCSI mounts (old /prole/d00x layout) + ansible.builtin.mount: + path: "{{ item }}" + state: unmounted + loop: "{{ _iscsi_legacy_absent_mounts | default([]) }}" + loop_control: + label: "{{ item }}" + tags: + - iscsi_cleanup + +- name: Remove legacy iSCSI fstab entries (old /prole/d00x layout) + ansible.builtin.lineinfile: + path: /etc/fstab + state: absent + regexp: "^\\s*\\S+\\s+{{ item | regex_escape }}\\s+" + loop: "{{ _iscsi_legacy_absent_mounts | default([]) }}" + loop_control: + label: "{{ item }}" + tags: + - iscsi_cleanup + +- name: Unmount stale iSCSI mounts + ansible.builtin.mount: + path: "{{ item.path | default(item) }}" + state: unmounted + loop: "{{ iscsi_absent_mounts | default([]) }}" + loop_control: + label: "{{ item.path | default(item) }}" + tags: + - iscsi_cleanup + +- name: Guardrail - iSCSI must not manage /var/lib/rancher + ansible.builtin.assert: + that: + - >- + (iscsi_targets | default([]) + | map(attribute='mounts') | list | flatten + | selectattr('path', 'equalto', '/var/lib/rancher') + | list | length) == 0 + fail_msg: >- + Inventory attempts to mount `/var/lib/rancher` via iSCSI. K3s state storage is + intended to be local host storage (e.g. USB3 SSD on myrddin). Remove + `/var/lib/rancher` from `iscsi_targets[*].mounts` (or set + `iscsi_allow_rancher_mount: true` if you are intentionally overriding this). + when: not (iscsi_allow_rancher_mount | default(false) | bool) + +- name: Remove stale iSCSI fstab entries + ansible.builtin.lineinfile: + path: /etc/fstab + state: absent + regexp: "^\\s*\\S+\\s+{{ (item.path | default(item)) | regex_escape }}\\s+" + loop: "{{ iscsi_absent_mounts | default([]) }}" + loop_control: + label: "{{ item.path | default(item) }}" + tags: + - iscsi_cleanup + +- name: Mount filesystems for iSCSI targets + ansible.builtin.include_tasks: iscsi_target_mounts.yml + loop: "{{ iscsi_targets | default([]) }}" + loop_control: + loop_var: t + label: "{{ t.iqn | default('') }}" + tags: + - iscsi + - iscsi_mount + - iscsi_storage + +- name: Guardrail - ensure required mountpoints are mounted + ansible.builtin.command: "findmnt -n {{ item.path }}" + register: _findmnt + changed_when: false + failed_when: _findmnt.rc != 0 + loop: "{{ iscsi_targets | default([]) | map(attribute='mounts') | list | flatten }}" + loop_control: + label: "{{ item.path }}" + tags: + - iscsi + - iscsi_mount + - iscsi_storage diff --git a/infrastructure/roles/k3s/defaults/main.yml b/infrastructure/roles/k3s/defaults/main.yml new file mode 100644 index 0000000..6637a9e --- /dev/null +++ b/infrastructure/roles/k3s/defaults/main.yml @@ -0,0 +1,113 @@ +--- +k3s_enabled: true +k3s_state: present +k3s_version: "" +k3s_guard_token_drift: true + +# k3s reset workflow +# By default, cleanup/uninstall is local-only. The `k3s_reset` playbook enables +# cluster-side node removal explicitly. +k3s_reset_force_cluster_node_delete: false +k3s_reset_kubeconfig_path: /etc/rancher/k3s/k3s.yaml + +# k3s service startup can be slow on first boot (image pulls, containerd unpacking, etc.). +# The service task is best-effort and then Ansible polls `systemctl is-active` using these knobs. +k3s_service_start_retries: 60 +k3s_service_start_delay: 5 + +# Destructive repair/refresh workflow (server only) +# A refresh wipes the k3s installation/state to a blank slate but preserves the +# join token and CA material so existing agents can rejoin without changing the token. +k3s_refresh_backup_root: /root/k3s-refresh-backups +k3s_refresh_confirm: "" +k3s_refresh_run_configure: true + +# CloudNative-PG +# Pin operator/plugin version by default to avoid unanticipated upgrades and load spikes. +# Upgrades must be explicitly enabled via `k3s_cnpg_upgrade: true`. +k3s_cnpg_version_default: "1.28.1" +k3s_cnpg_upgrade: false + +k3s_role: agent +k3s_cluster_init: false +k3s_server_url: "" +k3s_token: "{{ vault_k3s_token | default('') }}" +# Optional: if `k3s_token` is a full node-token (`K10...::server:...`), set this +# to the raw secret (`...` after `::server:`) for the server config. +k3s_server_token: "" +k3s_disable: [] +k3s_tls_sans: [] +k3s_service_node_labels: [] +k3s_node_taints: [] +k3s_disable_agent: false + +# K3s state storage +# `/var/lib/rancher` is treated as local host storage (not iSCSI/shared). +k3s_rancher_mountpoint: /var/lib/rancher +k3s_rancher_storage_class: local +k3s_data_dir: "{{ k3s_rancher_mountpoint }}/k3s" + +# Guardrail: when enabled, k3s must not run unless `k3s_rancher_mountpoint` is a +# dedicated mount (i.e. not backed by the same device as `/`). +# +# This is intentionally opt-in because some nodes may legitimately run k3s with +# `/var/lib/rancher` on the root filesystem. +k3s_rancher_mount_required: false + +# If `k3s_rancher_mount_required` is true, these must be set to manage the mount +# via Ansible (/etc/fstab + mount). +k3s_rancher_mount_src: "" +k3s_rancher_mount_fstype: "" +k3s_rancher_mount_opts: "noatime" +k3s_rancher_mount_passno: 2 + +k3s_write_kubeconfig_mode: "0640" +k3s_kubeconfig_group: kubeadm +k3s_kubeconfig_users: [] + +k3s_registry_config_enabled: true +k3s_registry_config_path: /etc/rancher/k3s/registries.yaml +k3s_registry_host: "" +k3s_registry_namespace: "" +k3s_registry_port: 5000 +# Internal Prole registry is HTTP by default; set to "https" when TLS is configured. +k3s_registry_endpoint_scheme: "http" + +# Pulling large images on every run can cause major load spikes; keep this opt-in. +k3s_prestage_images: false +k3s_prestage_images_platform: "linux/arm64" +k3s_prestage_images_list: + - docker.io/library/registry:2 + - docker.io/prom/prometheus:v2.55.1 + - docker.io/prom/alertmanager:v0.27.0 + - quay.io/prometheus/node-exporter:v1.8.2 + - quay.io/prometheus-operator/prometheus-operator:v0.71.2 + - ghcr.io/openbao/openbao:2.0.0 + - ghcr.io/opentofu/opentofu:1.8.2 + - docker.io/library/kong:3.8 + +k3s_cert_manager_version: "v1.19.4" +k3s_service_hostname: "{{ SERVICE_HOSTNAME | default('svc.prole.org') }}" +k3s_service_cert_name: "{{ k3s_service_hostname | replace('.', '-') }}" +k3s_service_tls_secret_name: "{{ k3s_service_cert_name }}-tls" +k3s_service_cluster_issuer: "letsencrypt-prod" + +# Host firewall: keep svc.prole.org reachable even when Tailscale inserts +# restrictive filter rules (e.g., `ts-input`/REJECT) near the top of `INPUT`. +# This role installs a small systemd oneshot that re-applies the rules at boot. +k3s_firewall_svc_allow_enabled: true +k3s_firewall_svc_allow_chain: "PROLE-SVC-ALLOW" +k3s_firewall_svc_allow_tcp_ports: + - 80 + - 443 + # Traefik NodePorts observed on k3s; keep configurable. + - 31936 + - 32370 + +# Legacy alias (svc-check used to own svc.prole.org). Keep for backward compatibility. +k3s_svc_check_domain: "{{ k3s_service_hostname }}" +k3s_svc_check_namespace: "svc-check" +k3s_svc_check_kong_namespace: "${SERVICE_NAMESPACE}" +k3s_svc_check_kong_configmap_name: "prole-svc-kong-config" +k3s_kong_namespace: "${SERVICE_NAMESPACE}" +k3s_kong_deployment: "prole-svc-kong" diff --git a/infrastructure/roles/k3s/handlers/main.yml b/infrastructure/roles/k3s/handlers/main.yml new file mode 100644 index 0000000..fac99f7 --- /dev/null +++ b/infrastructure/roles/k3s/handlers/main.yml @@ -0,0 +1,40 @@ +--- +- name: Restart k3s + ansible.builtin.service: + name: "{{ k3s_service_name | default('k3s') }}" + state: restarted + +- name: Upgrade CNPG operator + listen: Upgrade CNPG operator + ansible.builtin.command: kubectl apply --server-side -f {{ cnpg_manifest_url }} + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: cnpg_upgrade_apply + changed_when: "'created' in cnpg_upgrade_apply.stdout or 'configured' in cnpg_upgrade_apply.stdout" + +- name: Verify CNPG controller rollout (upgrade) + listen: Upgrade CNPG operator + ansible.builtin.command: kubectl rollout status deployment -n cnpg-system cnpg-controller-manager --timeout=300s + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: cnpg_upgrade_rollout_check + changed_when: false + failed_when: false + +- name: Restart CNPG controller if rollout failed (upgrade) + listen: Upgrade CNPG operator + ansible.builtin.command: kubectl rollout restart deployment -n cnpg-system cnpg-controller-manager + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + changed_when: true + when: + - cnpg_upgrade_rollout_check is defined + - cnpg_upgrade_rollout_check.rc != 0 + +- name: Final CNPG controller rollout check (upgrade) + listen: Upgrade CNPG operator + ansible.builtin.command: kubectl rollout status deployment -n cnpg-system cnpg-controller-manager --timeout=300s + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + changed_when: false + failed_when: false diff --git a/infrastructure/roles/k3s/tasks/acme_cert.yml b/infrastructure/roles/k3s/tasks/acme_cert.yml new file mode 100644 index 0000000..ab593b6 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/acme_cert.yml @@ -0,0 +1,119 @@ +--- +- name: Ensure cert-manager CRDs exist for ACME + ansible.builtin.command: k3s kubectl get crd certificates.cert-manager.io + register: k3s_acme_crd + changed_when: false + failed_when: k3s_acme_crd.rc != 0 + run_once: true + +- name: Check for flannel DaemonSet (k3s default CNI) + ansible.builtin.command: k3s kubectl -n kube-system get daemonset kube-flannel-ds + register: _k3s_flannel_ds + changed_when: false + failed_when: false + run_once: true + +- name: Wait for flannel DaemonSet rollout (ensures pod network is routable) + ansible.builtin.command: k3s kubectl -n kube-system rollout status daemonset/kube-flannel-ds --timeout=300s + register: _k3s_flannel_rollout + changed_when: false + retries: 6 + delay: 10 + until: _k3s_flannel_rollout.rc == 0 + when: _k3s_flannel_ds.rc == 0 + run_once: true + +- name: Check for kube-proxy DaemonSet + ansible.builtin.command: k3s kubectl -n kube-system get daemonset kube-proxy + register: _k3s_kube_proxy_ds + changed_when: false + failed_when: false + run_once: true + +- name: Wait for kube-proxy DaemonSet rollout + ansible.builtin.command: k3s kubectl -n kube-system rollout status daemonset/kube-proxy --timeout=300s + register: _k3s_kube_proxy_rollout + changed_when: false + retries: 6 + delay: 10 + until: _k3s_kube_proxy_rollout.rc == 0 + when: _k3s_kube_proxy_ds.rc == 0 + run_once: true + +- name: Wait for cert-manager-webhook deployment (required for validating ClusterIssuer/Certificate) + ansible.builtin.command: k3s kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=900s + register: k3s_certmgr_webhook_rollout + changed_when: false + retries: 3 + delay: 20 + until: k3s_certmgr_webhook_rollout.rc == 0 + run_once: true + +- name: Wait for cert-manager-webhook service endpoints + ansible.builtin.command: >- + k3s kubectl -n cert-manager get endpoints cert-manager-webhook + -o jsonpath='{.subsets[*].addresses[*].ip}' + register: k3s_certmgr_webhook_endpoints + changed_when: false + retries: 90 + delay: 10 + until: + - k3s_certmgr_webhook_endpoints.rc == 0 + - (k3s_certmgr_webhook_endpoints.stdout | trim) | length > 0 + run_once: true + +- name: Set ACME contact email (defaults to admin@prole.org) + ansible.builtin.set_fact: + svc_check_acme_email: "{{ letsencrypt_email | default('admin@prole.org') }}" + run_once: true + +- name: Ensure namespace exists for ACME certificate secret + ansible.builtin.shell: | + k3s kubectl apply -f - <<'EOF' + apiVersion: v1 + kind: Namespace + metadata: + name: {{ k3s_svc_check_kong_namespace }} + EOF + args: + executable: /bin/bash + run_once: true + +- name: Configure ACME ClusterIssuer and Certificate for service hostname + ansible.builtin.shell: | + k3s kubectl apply -f - <<'EOF' + apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: {{ k3s_service_cluster_issuer }} + spec: + acme: + email: {{ svc_check_acme_email }} + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: {{ k3s_service_cluster_issuer }} + solvers: + - http01: + ingress: + class: traefik + --- + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: {{ k3s_service_cert_name }} + namespace: {{ k3s_svc_check_kong_namespace }} + spec: + secretName: {{ k3s_service_tls_secret_name }} + issuerRef: + name: {{ k3s_service_cluster_issuer }} + kind: ClusterIssuer + dnsNames: + - {{ k3s_service_hostname }} + EOF + args: + executable: /bin/bash + register: k3s_acme_apply + retries: 90 + delay: 10 + until: k3s_acme_apply.rc == 0 + run_once: true diff --git a/infrastructure/roles/k3s/tasks/certmgr.yml b/infrastructure/roles/k3s/tasks/certmgr.yml new file mode 100644 index 0000000..0f37766 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/certmgr.yml @@ -0,0 +1,53 @@ +--- +- name: Check for cert-manager CRD + ansible.builtin.command: k3s kubectl get crd certificates.cert-manager.io + register: k3s_certmgr_crd + changed_when: false + failed_when: false + run_once: true + +- name: Apply cert-manager (CRDs + controller) + ansible.builtin.command: >- + k3s kubectl apply --validate=false -f + https://github.com/cert-manager/cert-manager/releases/download/{{ k3s_cert_manager_version }}/cert-manager.yaml + register: _k3s_certmgr_apply + changed_when: >- + (_k3s_certmgr_apply.stdout | default('') | lower) is search('configured') + or (_k3s_certmgr_apply.stdout | default('') | lower) is search('created') + retries: 30 + delay: 10 + until: _k3s_certmgr_apply.rc == 0 + run_once: true + +- name: Wait for cert-manager deployment to have a ready replica + ansible.builtin.command: >- + k3s kubectl -n cert-manager get deployment cert-manager + -o jsonpath='{.status.readyReplicas}' + register: _k3s_certmgr_ready + changed_when: false + retries: 90 + delay: 10 + until: (_k3s_certmgr_ready.stdout | default('0') | int) >= 1 + run_once: true + +- name: Wait for cert-manager-cainjector deployment to have a ready replica + ansible.builtin.command: >- + k3s kubectl -n cert-manager get deployment cert-manager-cainjector + -o jsonpath='{.status.readyReplicas}' + register: _k3s_certmgr_cainjector_ready + changed_when: false + retries: 90 + delay: 10 + until: (_k3s_certmgr_cainjector_ready.stdout | default('0') | int) >= 1 + run_once: true + +- name: Wait for cert-manager-webhook deployment to have a ready replica + ansible.builtin.command: >- + k3s kubectl -n cert-manager get deployment cert-manager-webhook + -o jsonpath='{.status.readyReplicas}' + register: _k3s_certmgr_webhook_ready + changed_when: false + retries: 90 + delay: 10 + until: (_k3s_certmgr_webhook_ready.stdout | default('0') | int) >= 1 + run_once: true diff --git a/infrastructure/roles/k3s/tasks/cleanup.yml b/infrastructure/roles/k3s/tasks/cleanup.yml new file mode 100644 index 0000000..037b21a --- /dev/null +++ b/infrastructure/roles/k3s/tasks/cleanup.yml @@ -0,0 +1,314 @@ +--- + +- name: Set effective k3s data-dir + ansible.builtin.set_fact: + k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}" + +# k3s reset: cluster-side node removal (best-effort) +# +# Why this exists: +# - Prevents stale `NotReady` node objects after a node reset. +# - Prevents DaemonSet pods stuck `Terminating` (the scheduler still thinks the node exists). +# - Prevents lingering `kube-node-lease` entries which can cause subsequent install/join hangs. +# +# Important safety rules: +# - We run these `kubectl` operations on a healthy control-plane host via `delegate_to`. +# - We skip if there is no reachable control-plane host or if its kubeconfig is missing. +# - We do not attempt this when resetting the *last* control-plane node (there is no other +# healthy server to delegate to, and we don't want the reset to break the cluster). +- name: Build node-name candidates for cluster-side cleanup (prefer inventory_hostname) + ansible.builtin.set_fact: + k3s_reset_node_name_candidates: >- + {{ [inventory_hostname, + ansible_fqdn | default(''), + ansible_hostname | default('')] + | reject('equalto', '') + | unique + | list }} + k3s_reset_node_name: >- + {{ ([inventory_hostname, + ansible_fqdn | default(''), + ansible_hostname | default('')] + | reject('equalto', '') + | unique + | list)[0] }} + when: k3s_reset_force_cluster_node_delete | bool + +- name: Pick control-plane delegate host for cluster-side cleanup (first available k3s server) + ansible.builtin.set_fact: + k3s_reset_control_plane_hosts: "{{ groups['k3s_servers'] | default([]) }}" + k3s_reset_is_last_control_plane: >- + {{ (inventory_hostname in (groups['k3s_servers'] | default([]))) + and ((groups['k3s_servers'] | default([])) | length == 1) }} + k3s_reset_delegate_host: >- + {{ ((groups['k3s_servers'] | default([])) + | difference([inventory_hostname]) + | first) | default('') }} + when: k3s_reset_force_cluster_node_delete | bool + +- name: Log why cluster-side node removal is skipped (last control-plane) + ansible.builtin.debug: + msg: >- + Skipping cluster-side node deletion for {{ inventory_hostname }} because it is the last + control-plane host in inventory (no other server to delegate kubectl to). + when: + - k3s_reset_force_cluster_node_delete | bool + - k3s_reset_is_last_control_plane | bool + +- name: Log why cluster-side node removal is skipped (no control-plane delegate host) + ansible.builtin.debug: + msg: >- + Skipping cluster-side node deletion for {{ inventory_hostname }} because no control-plane + host is available to delegate to (inventory group `k3s_servers` is empty, or only contains + this host). + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (k3s_reset_delegate_host | default('') | length) == 0 + +- name: Check whether delegated control-plane host is reachable + ansible.builtin.ping: + delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}" + register: _k3s_reset_delegate_ping + failed_when: false + changed_when: false + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (k3s_reset_delegate_host | default('') | length) > 0 + +- name: Log why cluster-side node removal is skipped (control-plane host unreachable) + ansible.builtin.debug: + msg: >- + Skipping cluster-side node deletion for {{ inventory_hostname }} because delegated + control-plane host '{{ k3s_reset_delegate_host }}' is unreachable. + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (k3s_reset_delegate_host | default('') | length) > 0 + - _k3s_reset_delegate_ping is not defined or (_k3s_reset_delegate_ping.ping | default('')) != 'pong' + +- name: Check for kubeconfig on delegated control-plane host + ansible.builtin.stat: + path: "{{ k3s_reset_kubeconfig_path }}" + delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}" + register: _k3s_reset_kubeconfig_stat + failed_when: false + changed_when: false + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + +- name: Log why cluster-side node removal is skipped (kubeconfig missing on delegate) + ansible.builtin.debug: + msg: >- + Skipping cluster-side node deletion for {{ inventory_hostname }} because kubeconfig + '{{ k3s_reset_kubeconfig_path }}' is missing on delegated control-plane host + '{{ k3s_reset_delegate_host }}'. + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + - _k3s_reset_kubeconfig_stat is defined + - not (_k3s_reset_kubeconfig_stat.stat.exists | default(false) | bool) + +- name: Fetch cluster node list (best-effort) to resolve the exact Kubernetes node name + ansible.builtin.command: >- + timeout 15s kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' + --request-timeout=5s + delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}" + environment: + KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}" + register: _k3s_reset_nodes_list + failed_when: false + changed_when: false + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + - (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool + +- name: Resolve Kubernetes node name from cluster (fallback to inventory_hostname) + ansible.builtin.set_fact: + k3s_reset_node_name: >- + {{ (k3s_reset_node_name_candidates + | select('in', _k3s_reset_nodes_list.stdout_lines | default([])) + | list + | first) + | default(k3s_reset_node_name) }} + when: + - k3s_reset_force_cluster_node_delete | bool + - _k3s_reset_nodes_list is defined + +- name: Log planned cluster-side cleanup target + ansible.builtin.debug: + msg: >- + Cluster-side cleanup enabled. Will attempt to delete node/lease '{{ k3s_reset_node_name }}' + via delegated control-plane host '{{ k3s_reset_delegate_host }}' (kubeconfig: {{ k3s_reset_kubeconfig_path }}). + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + - (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool + +- name: Cluster-side cleanup - delete node object (best-effort) + ansible.builtin.command: >- + timeout 15s kubectl delete node {{ k3s_reset_node_name }} --ignore-not-found=true + --wait=false --request-timeout=5s + delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}" + environment: + KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}" + register: _k3s_reset_delete_node_1 + failed_when: false + changed_when: "'deleted' in (_k3s_reset_delete_node_1.stdout | default('') | lower)" + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + - (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool + +- name: Cluster-side cleanup - patch away node finalizers if deletion is stuck (best-effort) + ansible.builtin.command: >- + timeout 15s kubectl patch node {{ k3s_reset_node_name }} --type=merge + -p '{"metadata":{"finalizers":[]}}' --request-timeout=5s + delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}" + environment: + KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}" + register: _k3s_reset_patch_node_finalizers + failed_when: false + changed_when: "'patched' in (_k3s_reset_patch_node_finalizers.stdout | default('') | lower)" + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + - (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool + +- name: Cluster-side cleanup - delete node object again after patch (best-effort) + ansible.builtin.command: >- + timeout 15s kubectl delete node {{ k3s_reset_node_name }} --ignore-not-found=true + --wait=false --request-timeout=5s + delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}" + environment: + KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}" + register: _k3s_reset_delete_node_2 + failed_when: false + changed_when: "'deleted' in (_k3s_reset_delete_node_2.stdout | default('') | lower)" + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + - (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool + +- name: Cluster-side cleanup - delete kube-node-lease for the node (best-effort) + ansible.builtin.command: >- + timeout 15s kubectl delete lease {{ k3s_reset_node_name }} -n kube-node-lease --ignore-not-found=true + --wait=false --request-timeout=5s + delegate_to: "{{ (k3s_reset_delegate_host | default('') | trim | length > 0) | ternary(k3s_reset_delegate_host, omit) }}" + environment: + KUBECONFIG: "{{ k3s_reset_kubeconfig_path }}" + register: _k3s_reset_delete_lease + failed_when: false + changed_when: "'deleted' in (_k3s_reset_delete_lease.stdout | default('') | lower)" + when: + - k3s_reset_force_cluster_node_delete | bool + - not (k3s_reset_is_last_control_plane | default(false) | bool) + - (_k3s_reset_delegate_ping.ping | default('')) == 'pong' + - (_k3s_reset_kubeconfig_stat.stat.exists | default(false)) | bool + +- name: Log cluster-side cleanup results (best-effort) + ansible.builtin.debug: + msg: + - "kubectl delete node (1): rc={{ _k3s_reset_delete_node_1.rc | default('n/a') }} stdout={{ _k3s_reset_delete_node_1.stdout | default('') | trim }} stderr={{ _k3s_reset_delete_node_1.stderr | default('') | trim }}" + - "kubectl patch node finalizers: rc={{ _k3s_reset_patch_node_finalizers.rc | default('n/a') }} stdout={{ _k3s_reset_patch_node_finalizers.stdout | default('') | trim }} stderr={{ _k3s_reset_patch_node_finalizers.stderr | default('') | trim }}" + - "kubectl delete node (2): rc={{ _k3s_reset_delete_node_2.rc | default('n/a') }} stdout={{ _k3s_reset_delete_node_2.stdout | default('') | trim }} stderr={{ _k3s_reset_delete_node_2.stderr | default('') | trim }}" + - "kubectl delete lease: rc={{ _k3s_reset_delete_lease.rc | default('n/a') }} stdout={{ _k3s_reset_delete_lease.stdout | default('') | trim }} stderr={{ _k3s_reset_delete_lease.stderr | default('') | trim }}" + when: + - k3s_reset_force_cluster_node_delete | bool + - _k3s_reset_delete_node_1 is defined or _k3s_reset_delete_lease is defined + +- name: Best-effort purge kube-system workloads before uninstall + ansible.builtin.command: >- + timeout 15s kubectl -n kube-system delete all --all --ignore-not-found + --wait=false --request-timeout=5s + failed_when: false + changed_when: false + +- name: Best-effort purge kube-system namespaces + ansible.builtin.command: >- + timeout 15s kubectl delete namespace kube-system --ignore-not-found + --wait=false --request-timeout=5s + failed_when: false + changed_when: false + +- name: Force-remove kube-system finalizers if namespace is stuck + ansible.builtin.command: >- + timeout 15s kubectl patch namespace kube-system + -p '{"spec":{"finalizers":[]}}' --type=merge --request-timeout=5s + failed_when: false + changed_when: false + +- name: Stop k3s services if present + ansible.builtin.service: + name: "{{ item }}" + state: stopped + enabled: false + loop: + - k3s + - k3s-agent + failed_when: false + +- name: Check for k3s uninstall scripts + ansible.builtin.stat: + path: "{{ item }}" + loop: + - /usr/local/bin/k3s-uninstall.sh + - /usr/local/bin/k3s-agent-uninstall.sh + register: k3s_uninstall_scripts + +- name: Run k3s uninstall scripts if present + ansible.builtin.command: "{{ item.stat.path }}" + loop: "{{ k3s_uninstall_scripts.results }}" + when: item.stat.exists + +- name: Remove k3s systemd units and directories + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/rancher + - /etc/systemd/system/k3s.service + - /etc/systemd/system/k3s-agent.service + - /etc/systemd/system/k3s.service.d + - /etc/systemd/system/k3s-agent.service.d + - /etc/rancher/node + - /etc/rancher/k3s + - /etc/cni + - /opt/cni + - "{{ k3s_effective_data_dir }}" + - "{{ k3s_effective_data_dir }}/server/tls" + - "{{ k3s_effective_data_dir }}/server/cred" + - "{{ k3s_effective_data_dir }}/server/token" + - "{{ k3s_effective_data_dir }}/server/node-token" + - "{{ k3s_effective_data_dir }}/server/agent-token" + - "{{ k3s_effective_data_dir }}/server/db" + - "{{ k3s_effective_data_dir }}/agent" + - "{{ k3s_effective_data_dir }}/data" + - /var/lib/kubelet + - /var/lib/etcd + - /var/lib/cni + - /var/lib/containerd + - /run/k3s + - /run/flannel + - /var/log/k3s + - /usr/local/bin/k3s + - /usr/local/bin/k3s-killall.sh + - /usr/local/bin/k3s-uninstall.sh + - /usr/local/bin/k3s-agent-uninstall.sh + - /usr/local/bin/kubectl + - /usr/local/bin/crictl + - /usr/local/bin/ctr + +- name: Reload systemd after k3s cleanup + ansible.builtin.systemd: + daemon_reload: true diff --git a/infrastructure/roles/k3s/tasks/cnpg.yml b/infrastructure/roles/k3s/tasks/cnpg.yml new file mode 100644 index 0000000..f272e8e --- /dev/null +++ b/infrastructure/roles/k3s/tasks/cnpg.yml @@ -0,0 +1,213 @@ +--- + +- name: Check if controller has conf/k3s.cfg (optional) + ansible.builtin.stat: + path: "{{ role_path }}/../../../conf/k3s.cfg" + register: _k3s_prole_cfg_stat + delegate_to: localhost + +- name: Load CNPG version from prole.cfg (pinned) + ansible.builtin.set_fact: + cnpg_version_raw: >- + {{ lookup( + 'ansible.builtin.ini', + 'CNPG_VERSION section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default=(k3s_cnpg_version_default | default('1.28.1')), + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else (k3s_cnpg_version_default | default('1.28.1')) }} + +- name: Normalize CNPG version + ansible.builtin.set_fact: + cnpg_version: "{{ cnpg_version_raw | trim | regex_replace('^v', '') }}" + +- name: Derive CNPG release series from version + ansible.builtin.set_fact: + cnpg_release_series: "{{ (cnpg_version.split('.')[:2]) | join('.') }}" + +- name: Build CNPG manifest URL + ansible.builtin.set_fact: + cnpg_manifest_url: "https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-{{ cnpg_release_series }}/releases/cnpg-{{ cnpg_version }}.yaml" + +- name: Load k3s server URL from prole.cfg + ansible.builtin.set_fact: + cnpg_prole_k3s_server_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'PROLE_K3S_SERVER section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + +- name: Resolve k3s API URL for kubectl + ansible.builtin.set_fact: + cnpg_k3s_server_url_raw: >- + {{ (k3s_server_url | default('')) if (k3s_server_url | default('') | length > 0) + else (cnpg_prole_k3s_server_cfg | default('')) if (cnpg_prole_k3s_server_cfg | default('') | length > 0) + else 'https://' + inventory_hostname + ':6443' }} + +- name: Normalize k3s API URL for kubectl + ansible.builtin.set_fact: + cnpg_k3s_server_url: "{{ cnpg_k3s_server_url_raw if cnpg_k3s_server_url_raw.startswith('http') else 'https://' + cnpg_k3s_server_url_raw }}" + +- name: Check for local k3s kubeconfig + ansible.builtin.stat: + path: /etc/rancher/k3s/k3s.yaml + register: cnpg_kubeconfig_stat + +- name: Ensure k3s kubeconfig for kubectl on hosts + ansible.builtin.copy: + dest: /etc/rancher/k3s/k3s.yaml + owner: root + group: "{{ k3s_kubeconfig_group | default('root') }}" + mode: "{{ k3s_write_kubeconfig_mode | default('0640') }}" + content: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + server: {{ cnpg_k3s_server_url }} + insecure-skip-tls-verify: true + name: prole-k3s + contexts: + - context: + cluster: prole-k3s + user: prole-k3s + name: prole-k3s + current-context: prole-k3s + users: + - name: prole-k3s + user: + token: {{ k3s_token }} + when: + - not cnpg_kubeconfig_stat.stat.exists + - cnpg_k3s_server_url | length > 0 + - k3s_token | default('') | length > 0 + +- name: Check k3s API availability for CNPG + ansible.builtin.command: kubectl get nodes + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: cnpg_k3s_api_check + changed_when: false + failed_when: false + +- name: Map kubectl-cnpg architecture + ansible.builtin.set_fact: + cnpg_arch_map: + x86_64: x86_64 + amd64: x86_64 + aarch64: arm64 + arm64: arm64 + ppc64le: ppc64le + s390x: s390x + +- name: Set cnpg architecture and supported arches + ansible.builtin.set_fact: + cnpg_arch: "{{ cnpg_arch_map.get(ansible_architecture, ansible_architecture) }}" + cnpg_supported_arches: + - x86_64 + - arm64 + - ppc64le + - s390x + +- name: Check kubectl-cnpg version + ansible.builtin.command: kubectl cnpg version + register: cnpg_plugin_version_check + changed_when: false + failed_when: false + +- name: Parse installed kubectl-cnpg version + ansible.builtin.set_fact: + cnpg_plugin_installed_version: "{{ (cnpg_plugin_version_check.stdout ~ ' ' ~ cnpg_plugin_version_check.stderr) | regex_search('Version:([0-9]+\\.[0-9]+\\.[0-9]+)', '\\1') | default('') }}" + +- name: Build kubectl-cnpg download URL + ansible.builtin.set_fact: + cnpg_plugin_asset: "kubectl-cnpg_{{ cnpg_version }}_linux_{{ cnpg_arch }}.tar.gz" + cnpg_plugin_url: "https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v{{ cnpg_version }}/kubectl-cnpg_{{ cnpg_version }}_linux_{{ cnpg_arch }}.tar.gz" + when: cnpg_arch in cnpg_supported_arches + +- name: Download kubectl-cnpg + ansible.builtin.get_url: + url: "{{ cnpg_plugin_url }}" + dest: "/tmp/{{ cnpg_plugin_asset }}" + mode: "0644" + when: + - cnpg_arch in cnpg_supported_arches + - cnpg_plugin_installed_version == '' or ((k3s_cnpg_upgrade | default(false)) | bool and cnpg_plugin_installed_version != cnpg_version) + +- name: Install kubectl-cnpg + ansible.builtin.unarchive: + src: "/tmp/{{ cnpg_plugin_asset }}" + dest: /usr/local/bin + remote_src: true + when: + - cnpg_arch in cnpg_supported_arches + - cnpg_plugin_installed_version == '' or ((k3s_cnpg_upgrade | default(false)) | bool and cnpg_plugin_installed_version != cnpg_version) + +- name: Ensure kubectl-cnpg is executable + ansible.builtin.file: + path: /usr/local/bin/kubectl-cnpg + mode: "0755" + when: + - cnpg_arch in cnpg_supported_arches + - cnpg_plugin_installed_version == '' or ((k3s_cnpg_upgrade | default(false)) | bool and cnpg_plugin_installed_version != cnpg_version) + +- name: Check for CNPG controller deployment + ansible.builtin.command: kubectl get deployment -n cnpg-system cnpg-controller-manager + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: cnpg_controller_deploy + changed_when: false + failed_when: false + when: cnpg_k3s_api_check.rc == 0 + +- name: Apply CNPG operator manifest (install only; pinned) + ansible.builtin.command: kubectl apply --server-side -f {{ cnpg_manifest_url }} + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: cnpg_apply + changed_when: "'created' in cnpg_apply.stdout or 'configured' in cnpg_apply.stdout" + when: + - cnpg_k3s_api_check.rc == 0 + - cnpg_controller_deploy.rc != 0 + +- name: Notify explicit CNPG operator upgrade handler (opt-in) + ansible.builtin.debug: + msg: "CNPG operator upgrade requested (k3s_cnpg_upgrade=true). Upgrading to pinned CNPG_VERSION={{ cnpg_version }} via handler." + changed_when: true + notify: + - Upgrade CNPG operator + when: + - cnpg_k3s_api_check.rc == 0 + - cnpg_controller_deploy.rc == 0 + - (k3s_cnpg_upgrade | default(false)) | bool + +- name: Verify CNPG controller rollout (new install) + ansible.builtin.command: kubectl rollout status deployment -n cnpg-system cnpg-controller-manager --timeout=60s + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + register: cnpg_rollout_check + changed_when: false + failed_when: false + when: + - cnpg_k3s_api_check.rc == 0 + - cnpg_controller_deploy.rc != 0 + +- name: Restart CNPG controller if rollout failed (new install; possibly stuck on tainted node) + ansible.builtin.command: kubectl rollout restart deployment -n cnpg-system cnpg-controller-manager + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + when: + - cnpg_k3s_api_check.rc == 0 + - cnpg_controller_deploy.rc != 0 + - cnpg_rollout_check.rc != 0 + changed_when: true + +- name: Final CNPG controller rollout check (new install) + ansible.builtin.command: kubectl rollout status deployment -n cnpg-system cnpg-controller-manager --timeout=300s + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + changed_when: false + when: + - cnpg_k3s_api_check.rc == 0 + - cnpg_controller_deploy.rc != 0 diff --git a/infrastructure/roles/k3s/tasks/configure.yml b/infrastructure/roles/k3s/tasks/configure.yml new file mode 100644 index 0000000..35333d7 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/configure.yml @@ -0,0 +1,302 @@ +--- + +- name: Skip k3s configuration when disabled + ansible.builtin.meta: end_host + when: not (k3s_enabled | default(true) | bool) + +- name: Skip k3s configuration when state is absent + ansible.builtin.meta: end_host + when: k3s_state == "absent" + +- name: Set k3s service name + ansible.builtin.set_fact: + k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}" + +- name: Check if controller has conf/k3s.cfg (optional) + ansible.builtin.stat: + path: "{{ role_path }}/../../../conf/k3s.cfg" + register: _k3s_prole_cfg_stat + delegate_to: localhost + +- name: Load service namespace from k3s.cfg (controller) + ansible.builtin.set_fact: + k3s_prole_service_namespace_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'SERVICE_NAMESPACE section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + changed_when: false + +- name: Load service hostname from k3s.cfg (controller) + ansible.builtin.set_fact: + k3s_prole_service_hostname_user_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'SERVICE_HOSTNAME section=User file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + k3s_prole_service_hostname_global_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'SERVICE_HOSTNAME section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + changed_when: false + +- name: Normalize service hostname variable + ansible.builtin.set_fact: + k3s_service_hostname: >- + {{ (k3s_prole_service_hostname_user_cfg | default('') | trim) + if ((k3s_prole_service_hostname_user_cfg | default('') | trim) | length > 0 + and (k3s_prole_service_hostname_user_cfg | trim) != '${SERVICE_HOSTNAME}') + else (k3s_prole_service_hostname_global_cfg | default('') | trim) + if ((k3s_prole_service_hostname_global_cfg | default('') | trim) | length > 0 + and (k3s_prole_service_hostname_global_cfg | trim) != '${SERVICE_HOSTNAME}') + else (k3s_service_hostname | default('svc.prole.org')) }} + changed_when: false + +- name: Resolve Kong namespaces from k3s.cfg (avoid implicit default namespace) + ansible.builtin.set_fact: + _k3s_service_namespace_resolved: >- + {{ (k3s_prole_service_namespace_cfg | default('')) + if (k3s_prole_service_namespace_cfg | default('') | length > 0 and k3s_prole_service_namespace_cfg != '${SERVICE_NAMESPACE}') + else 'default' }} + changed_when: false + +- name: Normalize Kong namespace variables + ansible.builtin.set_fact: + k3s_kong_namespace: >- + {{ (k3s_kong_namespace | default('')) + if (k3s_kong_namespace | default('') | length > 0 and k3s_kong_namespace != '${SERVICE_NAMESPACE}') + else _k3s_service_namespace_resolved }} + changed_when: false + +- name: Normalize svc-check Kong namespace variable + ansible.builtin.set_fact: + k3s_svc_check_kong_namespace: >- + {{ (k3s_svc_check_kong_namespace | default('')) + if (k3s_svc_check_kong_namespace | default('') | length > 0 and k3s_svc_check_kong_namespace != '${SERVICE_NAMESPACE}') + else k3s_kong_namespace }} + changed_when: false + +- name: Validate resolved Kong namespace variables + ansible.builtin.assert: + that: + - k3s_kong_namespace | length > 0 + - k3s_svc_check_kong_namespace | length > 0 + fail_msg: >- + Resolved Kong namespace variables are empty. Ensure `SERVICE_NAMESPACE` is set in `conf/k3s.cfg` + or explicitly set `k3s_kong_namespace` / `k3s_svc_check_kong_namespace` in Ansible inventory. + +- name: Pre-stage critical images (registry:2, kong, etc.) + ansible.builtin.include_tasks: prestage_images.yml + when: + - k3s_state == "present" + - k3s_prestage_images | bool + tags: [images] + +- name: Create ArgoCD hostPath directories on myrddin.prole.org (/synology/d001) + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0777" + loop: + - /synology/d001/argocd + - /synology/d001/argocd/home + - /synology/d001/argocd/tmp + when: + - k3s_state == "present" + - inventory_hostname == 'myrddin.prole.org' + +- name: Create ArgoCD hostPath directories on merlin.prole.org (/synology/d002) + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0777" + loop: + - /synology/d002/argocd + - /synology/d002/argocd/data + - /synology/d002/argocd/tmp + when: + - k3s_state == "present" + - inventory_hostname == 'merlin.prole.org' + +- name: Create ArgoCD hostPath directories on pi.prole.org (/synology/d003) + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0777" + loop: + - /synology/d003/argocd + - /synology/d003/argocd/gpg-keyring + - /synology/d003/argocd/tmp + - /synology/d003/argocd/helm-working-dir + - /synology/d003/argocd/var-files + - /synology/d003/argocd/plugins + when: + - k3s_state == "present" + - inventory_hostname == 'pi.prole.org' + +- name: Wait for k3s Kubernetes API to become ready + ansible.builtin.command: k3s kubectl get --raw='/readyz' + register: _k3s_readyz + changed_when: false + retries: 60 + delay: 5 + until: _k3s_readyz.rc == 0 + when: + - k3s_state == "present" + - k3s_role == 'server' + run_once: true + +- name: Load optional workloads threshold from prole.cfg (controller) + ansible.builtin.set_fact: + k3s_optional_workloads_min_nodes_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + changed_when: false + when: + - k3s_state == "present" + - k3s_role == 'server' + run_once: true + +- name: Validate optional-workloads policy config is present + ansible.builtin.assert: + that: + - _k3s_prole_cfg_stat.stat.exists + - (k3s_optional_workloads_min_nodes_cfg | string | trim | length) > 0 + - (k3s_optional_workloads_min_nodes_cfg | int) > 0 + fail_msg: >- + Missing or invalid optional-workloads policy configuration. + Expected [Global] OPTIONAL_WORKLOADS_MIN_READY_SCHEDULABLE_NODES in conf/k3s.cfg + (got '{{ k3s_optional_workloads_min_nodes_cfg | default('') }}'). + changed_when: false + when: + - k3s_state == "present" + - k3s_role == 'server' + run_once: true + +- name: Query cluster nodes for optional-workloads policy + ansible.builtin.command: k3s kubectl get nodes --no-headers + register: _k3s_nodes_list + changed_when: false + failed_when: false + when: + - k3s_state == "present" + - k3s_role == 'server' + run_once: true + +- name: Log optional-workloads policy node query failure (treat as insufficient capacity) + ansible.builtin.debug: + msg: >- + Could not query Kubernetes nodes for optional-workloads policy. + Treating as ready_schedulable_nodes=0 so optional workloads will be disabled for this run. + rc={{ _k3s_nodes_list.rc | default('n/a') }} stderr={{ _k3s_nodes_list.stderr | default('') | trim }} + when: + - k3s_state == "present" + - k3s_role == 'server' + - _k3s_nodes_list is defined + - (_k3s_nodes_list.rc | default(1) | int) != 0 + run_once: true + +- name: Evaluate optional-workloads policy (ready + schedulable) + ansible.builtin.include_tasks: optional_workloads_policy_eval.yml + when: + - k3s_state == "present" + - k3s_role == 'server' + run_once: true + +- name: Log optional-workloads policy decision + ansible.builtin.debug: + msg: >- + optional_workloads_allowed={{ optional_workloads_allowed }} + (ready_schedulable_nodes={{ ready_schedulable_nodes }} min_required={{ optional_workloads_min_ready_schedulable_nodes }}) + when: + - k3s_state == "present" + - k3s_role == 'server' + run_once: true + +- name: Apply merlin-local-iscsi StorageClass (monitoring) + ansible.builtin.shell: | + k3s kubectl apply --validate=false -f - <<'EOF' + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: merlin-local-iscsi + provisioner: kubernetes.io/no-provisioner + volumeBindingMode: WaitForFirstConsumer + reclaimPolicy: Retain + EOF + register: _merlin_local_iscsi_apply + changed_when: >- + (_merlin_local_iscsi_apply.stdout | default('') | lower) is search('configured') + or (_merlin_local_iscsi_apply.stdout | default('') | lower) is search('created') + failed_when: false + when: + - k3s_state == "present" + - k3s_role == 'server' + - optional_workloads_allowed | default(false) | bool + run_once: true + +- name: Log monitoring StorageClass apply failure (non-fatal) + ansible.builtin.debug: + msg: >- + Failed to apply monitoring StorageClass (non-fatal): rc={{ _merlin_local_iscsi_apply.rc | default('n/a') }} + stderr={{ _merlin_local_iscsi_apply.stderr | default('') | trim }} + when: + - k3s_state == "present" + - k3s_role == 'server' + - optional_workloads_allowed | default(false) | bool + - _merlin_local_iscsi_apply is defined + - (_merlin_local_iscsi_apply.rc | default(0) | int) != 0 + run_once: true + +- name: Skip monitoring StorageClass when optional workloads are disallowed + ansible.builtin.debug: + msg: >- + Skipping monitoring StorageClass because optional workloads are disallowed: + optional_workloads_allowed={{ optional_workloads_allowed | default(false) }} + (ready_schedulable_nodes={{ ready_schedulable_nodes | default('unknown') }} min_required={{ optional_workloads_min_ready_schedulable_nodes | default('unknown') }}) + when: + - k3s_state == "present" + - k3s_role == 'server' + - not (optional_workloads_allowed | default(false) | bool) + run_once: true + +- name: Fetch kubeconfig to controller for controller-side installs + ansible.builtin.include_tasks: fetch_kubeconfig.yml + when: + - k3s_state == "present" + - k3s_role == 'server' + run_once: true + +- name: Ensure CloudNative-PG operator and kubectl plugin + ansible.builtin.include_tasks: cnpg.yml + when: k3s_state == "present" + +- name: Ensure cert-manager is installed for ACME + ansible.builtin.include_tasks: certmgr.yml + when: + - k3s_state == "present" + - k3s_role == 'server' + +- name: Configure ACME issuer and certificate for service hostname + ansible.builtin.include_tasks: acme_cert.yml + when: + - k3s_state == "present" + - k3s_role == 'server' + +- name: Ensure Kong is present and refreshed + ansible.builtin.include_tasks: kong.yml + when: + - k3s_state == "present" + - k3s_role == 'server' + diff --git a/infrastructure/roles/k3s/tasks/fetch_kubeconfig.yml b/infrastructure/roles/k3s/tasks/fetch_kubeconfig.yml new file mode 100644 index 0000000..b481503 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/fetch_kubeconfig.yml @@ -0,0 +1,109 @@ +--- + +- name: Build kubeconfig fetch candidates + ansible.builtin.set_fact: + k3s_kubeconfig_source_candidates: "{{ groups['k3s_hosts'] | default([inventory_hostname]) }}" + +- name: Initialize kubeconfig source host + ansible.builtin.set_fact: + k3s_kubeconfig_source_host: "" + +- name: Select k3s init server as kubeconfig source (preferred) + ansible.builtin.set_fact: + k3s_kubeconfig_source_host: "{{ item }}" + loop: "{{ k3s_kubeconfig_source_candidates }}" + when: + - (k3s_kubeconfig_source_host | length) == 0 + - hostvars[item].k3s_role | default('agent') == 'server' + - hostvars[item].k3s_cluster_init | default(false) | bool + +- name: Fallback to first k3s server as kubeconfig source + ansible.builtin.set_fact: + k3s_kubeconfig_source_host: "{{ item }}" + loop: "{{ k3s_kubeconfig_source_candidates }}" + when: + - (k3s_kubeconfig_source_host | length) == 0 + - hostvars[item].k3s_role | default('agent') == 'server' + +- name: Final fallback to current host as kubeconfig source + ansible.builtin.set_fact: + k3s_kubeconfig_source_host: "{{ inventory_hostname }}" + when: (k3s_kubeconfig_source_host | length) == 0 + +- name: Require kubeconfig source host + ansible.builtin.assert: + that: + - (k3s_kubeconfig_source_host | length) > 0 + fail_msg: "Unable to select a k3s kubeconfig source host." + +- name: Determine kubeconfig API server URL + ansible.builtin.set_fact: + k3s_kubeconfig_api_server: >- + {{ (k3s_server_url | trim) if (k3s_server_url | default('') | trim | length) > 0 + else 'https://' ~ k3s_kubeconfig_source_host ~ ':6443' }} + +- name: Read k3s kubeconfig from source node + ansible.builtin.slurp: + src: /etc/rancher/k3s/k3s.yaml + register: _k3s_kubeconfig_raw + delegate_to: "{{ k3s_kubeconfig_source_host }}" + become: true + +- name: Rewrite kubeconfig server and context to primary endpoint + ansible.builtin.set_fact: + k3s_kubeconfig_rendered: >- + {{ _k3s_kubeconfig_raw.content + | b64decode + | regex_replace('server:\s*https?://\S+', 'server: ' ~ k3s_kubeconfig_api_server) + | regex_replace('name:\s*default', 'name: ' ~ (k3s_kubeconfig_context | default('prole-service-cluster'))) + | regex_replace('context:\s*default', 'context: ' ~ (k3s_kubeconfig_context | default('prole-service-cluster'))) + | regex_replace('user:\s*default', 'user: ' ~ (k3s_kubeconfig_context | default('prole-service-cluster'))) + | regex_replace('cluster:\s*default', 'cluster: ' ~ (k3s_kubeconfig_context | default('prole-service-cluster'))) + | regex_replace('current-context:\s*default', 'current-context: ' ~ (k3s_kubeconfig_context | default('prole-service-cluster'))) }} + +- name: Validate rendered kubeconfig YAML (controller-side scripts depend on this) + ansible.builtin.set_fact: + _k3s_kubeconfig_parsed: "{{ k3s_kubeconfig_rendered | from_yaml }}" + changed_when: false + +- name: Require kubeconfig to contain required keys + ansible.builtin.assert: + that: + - _k3s_kubeconfig_parsed is mapping + - (_k3s_kubeconfig_parsed.clusters | default([])) | length > 0 + - (_k3s_kubeconfig_parsed.contexts | default([])) | length > 0 + - (_k3s_kubeconfig_parsed.users | default([])) | length > 0 + - (_k3s_kubeconfig_parsed['current-context'] | default('') | length) > 0 + fail_msg: >- + Rendered kubeconfig is invalid or missing required keys. Refusing to save + kubeconfig to the controller. + +- name: Define controller kubeconfig destinations + ansible.builtin.set_fact: + k3s_kubeconfig_controller_project_path: "{{ playbook_dir }}/../../prole-k3s.kubeconfig" + k3s_kubeconfig_controller_service_path: "{{ playbook_dir }}/../../etc/secrets/k3s.kubeconfig" + changed_when: false + +- name: Ensure controller kubeconfig directory exists (service path) + ansible.builtin.file: + path: "{{ k3s_kubeconfig_controller_service_path | dirname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + +- name: Save kubeconfig locally for controller-side init scripts (project root) + ansible.builtin.copy: + content: "{{ k3s_kubeconfig_rendered }}" + dest: "{{ k3s_kubeconfig_controller_project_path }}" + mode: "0600" + delegate_to: localhost + become: false + +- name: Save kubeconfig locally for controller-side init scripts (etc/secrets) + ansible.builtin.copy: + content: "{{ k3s_kubeconfig_rendered }}" + dest: "{{ k3s_kubeconfig_controller_service_path }}" + mode: "0600" + delegate_to: localhost + become: false diff --git a/infrastructure/roles/k3s/tasks/firewall_svc_allow.yml b/infrastructure/roles/k3s/tasks/firewall_svc_allow.yml new file mode 100644 index 0000000..09184b1 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/firewall_svc_allow.yml @@ -0,0 +1,42 @@ +--- + +- name: Install prole svc firewall enforcement script + ansible.builtin.template: + src: prole-svc-iptables-allow.sh.j2 + dest: /usr/local/sbin/prole-svc-iptables-allow + owner: root + group: root + mode: "0755" + when: + - k3s_firewall_svc_allow_enabled | default(true) | bool + - ansible_service_mgr | default('') == 'systemd' + +- name: Install prole svc firewall systemd unit + ansible.builtin.template: + src: prole-svc-iptables-allow.service.j2 + dest: /etc/systemd/system/prole-svc-iptables-allow.service + owner: root + group: root + mode: "0644" + register: _prole_svc_fw_unit + when: + - k3s_firewall_svc_allow_enabled | default(true) | bool + - ansible_service_mgr | default('') == 'systemd' + +- name: Reload systemd daemon (svc firewall unit) + ansible.builtin.systemd: + daemon_reload: true + when: + - k3s_firewall_svc_allow_enabled | default(true) | bool + - ansible_service_mgr | default('') == 'systemd' + - _prole_svc_fw_unit is defined + - _prole_svc_fw_unit.changed + +- name: Enable and run prole svc firewall unit + ansible.builtin.systemd: + name: prole-svc-iptables-allow.service + enabled: true + state: restarted + when: + - k3s_firewall_svc_allow_enabled | default(true) | bool + - ansible_service_mgr | default('') == 'systemd' diff --git a/infrastructure/roles/k3s/tasks/import_image.yml b/infrastructure/roles/k3s/tasks/import_image.yml new file mode 100644 index 0000000..dc411db --- /dev/null +++ b/infrastructure/roles/k3s/tasks/import_image.yml @@ -0,0 +1,71 @@ +--- + +- name: Determine node platform (best-effort) + ansible.builtin.set_fact: + _k3s_node_platform: >- + linux/{{ + { + 'x86_64': 'amd64', + 'amd64': 'amd64', + 'aarch64': 'arm64', + 'arm64': 'arm64', + }.get((ansible_architecture | default('x86_64')), (ansible_architecture | default('x86_64'))) + }} + changed_when: false + +- name: Infer tarball platform from filename (best-effort) + ansible.builtin.set_fact: + _k3s_tar_platform_hint: >- + {{ + 'linux/amd64' if (k3s_import_image.path | basename) is search('linux-amd64') else ( + 'linux/arm64' if (k3s_import_image.path | basename) is search('linux-arm64') else '' + ) + }} + changed_when: false + +- name: Fail fast on obvious platform mismatch to avoid heavy import I/O + ansible.builtin.fail: + msg: >- + Refusing to import {{ k3s_import_image.path | basename }} on {{ inventory_hostname }}: + tarball appears to target {{ _k3s_tar_platform_hint }}, but node platform is {{ _k3s_node_platform }}. + Use a matching tarball (or build/export a multi-arch tar) for this node. + when: + - _k3s_tar_platform_hint | default('') != '' + - _k3s_node_platform | default('') != '' + - _k3s_tar_platform_hint != _k3s_node_platform + +- name: "Check if image {{ k3s_import_image.image }} already exists" + ansible.builtin.shell: "k3s ctr -n k8s.io images list -q | grep -qx -e '{{ k3s_import_image.image }}' -e 'docker.io/{{ k3s_import_image.image }}'" + register: _k3s_image_check + changed_when: false + failed_when: false + when: k3s_import_image.image | default('') != '' + +- name: "Copy image tarball {{ k3s_import_image.path | basename }} to k3s node" + block: + - name: "Copy image tarball {{ k3s_import_image.path | basename }} to k3s node" + ansible.builtin.copy: + src: "{{ k3s_import_image.path }}" + dest: "/tmp/{{ k3s_import_image.path | basename }}" + mode: "0644" + + - name: "Import image {{ k3s_import_image.path | basename }} into k3s containerd" + ansible.builtin.command: k3s ctr -n k8s.io images import --no-unpack "/tmp/{{ k3s_import_image.path | basename }}" + register: _k3s_ctr_import + changed_when: _k3s_ctr_import.rc == 0 + failed_when: false + + - name: Fail with actionable diagnostics when import failed + ansible.builtin.fail: + msg: >- + Failed to import {{ k3s_import_image.path | basename }} into k3s containerd on {{ inventory_hostname }} (rc={{ _k3s_ctr_import.rc }}). + stderr={{ _k3s_ctr_import.stderr | default('') | trim }} + stdout={{ _k3s_ctr_import.stdout | default('') | trim }} + Hint: if you see "image might be filtered out", the tarball usually does not contain a manifest matching this node platform ({{ _k3s_node_platform }}). + when: _k3s_ctr_import.rc != 0 + always: + - name: "Remove temporary tarball for {{ k3s_import_image.path | basename }}" + ansible.builtin.file: + path: "/tmp/{{ k3s_import_image.path | basename }}" + state: absent + when: k3s_import_image.image | default('') == '' or _k3s_image_check.rc != 0 diff --git a/infrastructure/roles/k3s/tasks/import_images.yml b/infrastructure/roles/k3s/tasks/import_images.yml new file mode 100644 index 0000000..321769e --- /dev/null +++ b/infrastructure/roles/k3s/tasks/import_images.yml @@ -0,0 +1,34 @@ +--- + +- name: Wait for k3s containerd socket to be present + ansible.builtin.wait_for: + path: /run/k3s/containerd/containerd.sock + state: present + timeout: "{{ k3s_containerd_socket_wait_timeout | default(120) }}" + changed_when: false + +- name: Verify k3s container runtime is available + ansible.builtin.command: k3s ctr -n k8s.io version + register: _k3s_ctr_version + changed_when: false + failed_when: false + retries: 12 + delay: 5 + until: _k3s_ctr_version.rc == 0 + +- name: Fail if k3s container runtime is not available + ansible.builtin.fail: + msg: >- + k3s/containerd is not ready on {{ inventory_hostname }}. + Expected /run/k3s/containerd/containerd.sock and a working `k3s ctr`. + Check the k3s service status and node networking, then retry the import. + when: _k3s_ctr_version.rc != 0 + +- name: Import image tarballs into k3s containerd + ansible.builtin.include_tasks: import_image.yml + loop: "{{ k3s_import_images }}" + loop_control: + loop_var: k3s_import_image + label: "{{ k3s_import_image.path | basename }}" + when: + - (k3s_import_images | length) > 0 diff --git a/infrastructure/roles/k3s/tasks/install.yml b/infrastructure/roles/k3s/tasks/install.yml new file mode 100644 index 0000000..5441367 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/install.yml @@ -0,0 +1,688 @@ +--- + +- name: Skip k3s when disabled + ansible.builtin.meta: end_host + when: not (k3s_enabled | default(true) | bool) + +- name: Skip k3s when state is absent + ansible.builtin.meta: end_host + when: k3s_state == "absent" + +- name: Set k3s service name + ansible.builtin.set_fact: + k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}" + +- name: Derive server token secret from node-token (server only) + ansible.builtin.set_fact: + k3s_server_token: "{{ k3s_token | regex_replace('^K10[0-9a-f]+::server:', '') }}" + when: + - k3s_role == 'server' + - (k3s_token | default('')) is match('^K10[0-9a-f]+::server:.+') + no_log: true + +- name: Set effective k3s data-dir + ansible.builtin.set_fact: + k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}" + +- name: Guardrail - enforce dedicated rancher mount (opt-in) + block: + - name: Assert rancher mount configuration is present + ansible.builtin.assert: + that: + - (k3s_rancher_mountpoint | default('') | length) > 0 + - (k3s_rancher_mount_src | default('') | length) > 0 + - (k3s_rancher_mount_fstype | default('') | length) > 0 + - (k3s_rancher_mount_opts | default('') | length) > 0 + fail_msg: >- + `k3s_rancher_mount_required: true` requires `k3s_rancher_mount_src`, + `k3s_rancher_mount_fstype`, and `k3s_rancher_mount_opts` to be set. + + - name: Ensure rancher mountpoint exists + ansible.builtin.file: + path: "{{ k3s_rancher_mountpoint }}" + state: directory + owner: root + group: root + mode: "0755" + + - name: Ensure rancher mount is present in fstab and mounted (no nofail) + ansible.builtin.mount: + path: "{{ k3s_rancher_mountpoint }}" + src: "{{ k3s_rancher_mount_src }}" + fstype: "{{ k3s_rancher_mount_fstype }}" + opts: "{{ k3s_rancher_mount_opts }}" + dump: 0 + passno: "{{ k3s_rancher_mount_passno | default(2) }}" + state: mounted + + - name: Ensure k3s systemd drop-in directory exists (rancher mount guard) + ansible.builtin.file: + path: "/etc/systemd/system/{{ k3s_service_name }}.service.d" + state: directory + mode: "0755" + when: ansible_service_mgr | default('') == 'systemd' + + - name: Install k3s systemd drop-in (require rancher mount + fail if on rootfs) + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ k3s_service_name }}.service.d/rancher-mount-guard.conf" + mode: "0644" + content: | + [Unit] + RequiresMountsFor={{ k3s_rancher_mountpoint }} + + [Service] + ExecStartPre=/bin/sh -ec 'rancher_src="$(findmnt -n -o SOURCE -T {{ k3s_rancher_mountpoint }})"; root_src="$(findmnt -n -o SOURCE -T /)"; if [ "$${rancher_src}" = "$${root_src}" ]; then echo "Refusing to start k3s: {{ k3s_rancher_mountpoint }} resolves to rootfs ($${root_src}). Expected a dedicated mount." >&2; exit 1; fi' + register: _k3s_rancher_mount_guard_dropin + when: ansible_service_mgr | default('') == 'systemd' + + - name: Reload systemd daemon (k3s rancher mount guard) + ansible.builtin.systemd: + daemon_reload: true + when: + - ansible_service_mgr | default('') == 'systemd' + - _k3s_rancher_mount_guard_dropin is defined + - _k3s_rancher_mount_guard_dropin.changed + + - name: Preflight - fail if rancher mount is backed by the same device as / + ansible.builtin.shell: | + rancher_src="$(findmnt -n -o SOURCE -T {{ k3s_rancher_mountpoint }})" + root_src="$(findmnt -n -o SOURCE -T /)" + if [ "${rancher_src}" = "${root_src}" ]; then + echo "{{ k3s_rancher_mountpoint }} is on rootfs (${root_src}); refusing to proceed." >&2 + exit 1 + fi + changed_when: false + check_mode: no + when: k3s_rancher_mount_required | default(false) | bool + +- name: Check if k3s service is installed + ansible.builtin.stat: + path: "/etc/systemd/system/{{ k3s_service_name }}.service" + register: k3s_service + +- name: Check if k3s binary exists + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: k3s_bin + +- name: Determine if k3s needs installation + ansible.builtin.set_fact: + k3s_needs_install: "{{ not k3s_bin.stat.exists or not k3s_service.stat.exists }}" + +- name: Gather service facts for health check + ansible.builtin.service_facts: + when: not k3s_needs_install + +- name: Check k3s API responsiveness + ansible.builtin.command: k3s kubectl get nodes + register: k3s_api_check + changed_when: false + failed_when: false + when: + - not k3s_needs_install + - (k3s_service_name + ".service") in ansible_facts.services + - ansible_facts.services[k3s_service_name + ".service"].state == 'running' + +- name: Restart k3s if API is not responsive (try recovery before reinstall) + ansible.builtin.service: + name: "{{ k3s_service_name }}" + state: restarted + register: k3s_api_restart + failed_when: false + when: + - not k3s_needs_install + - (k3s_service_name + ".service") in ansible_facts.services + - ansible_facts.services[k3s_service_name + ".service"].state == 'running' + - k3s_api_check.rc | default(0) != 0 + +- name: Re-check k3s API after restart + ansible.builtin.command: k3s kubectl get nodes + register: k3s_api_check_post_restart + changed_when: false + failed_when: false + until: k3s_api_check_post_restart.rc == 0 + retries: 6 + delay: 10 + when: + - not k3s_needs_install + - (k3s_service_name + ".service") in ansible_facts.services + - ansible_facts.services[k3s_service_name + ".service"].state == 'running' + - k3s_api_check.rc | default(0) != 0 + +- name: Force k3s install if still not responsive after restart + ansible.builtin.set_fact: + k3s_needs_install: true + when: + - not k3s_needs_install + - (k3s_service_name + ".service") in ansible_facts.services + - ansible_facts.services[k3s_service_name + ".service"].state == 'running' + - k3s_api_check.rc | default(0) != 0 + - k3s_api_check_post_restart.rc | default(0) != 0 + +- name: Install k3s dependencies + ansible.builtin.package: + name: + - curl + - iptables + state: present + +- name: Ensure svc reachability firewall rules are persistent + ansible.builtin.import_tasks: firewall_svc_allow.yml + when: + - k3s_state != "absent" + +- name: Address Retropie networking (wifi power save) + ansible.builtin.shell: | + if [ -d /sys/class/net/wlan0 ]; then + /sbin/iwconfig wlan0 power off || /usr/sbin/iw dev wlan0 set power_save off || true + fi + when: inventory_hostname in ['retropie.prole.org', 'pi.prole.org'] + changed_when: false + failed_when: false + +- name: Configure connman to ignore k8s/cni interfaces + ansible.builtin.shell: | + if [ -f /etc/connman/main.conf ]; then + if ! grep -q "^NetworkInterfaceBlacklist=" /etc/connman/main.conf; then + echo "NetworkInterfaceBlacklist=veth,cni,flannel,docker,virbr" >> /etc/connman/main.conf + echo "RESTART_CONNMAN" + elif ! grep -q "cni" /etc/connman/main.conf; then + # Ensure we have our required prefixes in the blacklist + sed -i 's/^NetworkInterfaceBlacklist=/NetworkInterfaceBlacklist=veth,cni,flannel,docker,virbr,/' /etc/connman/main.conf + # Clean up any potential double commas or wildcards we might have added before + sed -i 's/\*//g' /etc/connman/main.conf + sed -i 's/lo,//g' /etc/connman/main.conf + sed -i 's/,,/,/g' /etc/connman/main.conf + echo "RESTART_CONNMAN" + fi + fi + register: _connman_config + when: inventory_hostname in ['retropie.prole.org', 'pi.prole.org'] + changed_when: "'RESTART_CONNMAN' in _connman_config.stdout" + +- name: Restart connman to apply blacklist changes + ansible.builtin.service: + name: connman + state: restarted + async: 1 + poll: 0 + when: + - inventory_hostname in ['retropie.prole.org', 'pi.prole.org'] + - _connman_config is changed + ignore_errors: true + +- name: Wait for connection to return after network restart + ansible.builtin.wait_for_connection: + delay: 5 + timeout: 300 + when: + - inventory_hostname in ['retropie.prole.org', 'pi.prole.org'] + - _connman_config is changed + +- name: Address Retropie routing (wlan0 priority) + ansible.builtin.shell: | + # If eth0 has a default route but we are using wlan0, it might block internet + if [ "{{ k3s_flannel_iface | default('') }}" = "wlan0" ] || [ -d /sys/class/net/wlan0 ]; then + if ip route show default dev eth0 | grep -q default; then + if ip route show default dev wlan0 | grep -q default; then + ip route del default dev eth0 || true + fi + fi + fi + when: inventory_hostname in ['retropie.prole.org', 'pi.prole.org'] + changed_when: false + failed_when: false + +- name: Preflight - ensure cgroup kernel params are present + ansible.builtin.command: "cat /proc/cmdline" + register: k3s_proc_cmdline + changed_when: false + check_mode: no + +- name: Fail if cgroup params are missing (reboot required) + ansible.builtin.fail: + msg: "Missing cgroup params in /proc/cmdline: {{ ['cgroup_memory=1', 'cgroup_enable=memory'] | reject('in', k3s_proc_cmdline.stdout | default('')) | list }}. Reboot required before k3s install." + when: + - not ansible_check_mode + - (['cgroup_memory=1', 'cgroup_enable=memory'] | reject('in', k3s_proc_cmdline.stdout | default('')) | list) | length > 0 + +- name: Preflight - ensure required mountpoints are mounted + ansible.builtin.shell: | + awk -v mp="{{ item }}" '$5==mp {found=1} END {exit(found?0:1)}' /proc/self/mountinfo + register: k3s_required_mounts_present + changed_when: false + failed_when: k3s_required_mounts_present.rc != 0 + check_mode: no + loop: "{{ k3s_required_mounts | default([]) }}" + when: (k3s_required_mounts | default([])) | length > 0 + +- name: Guardrail - ensure required mountpoints are not on SD + ansible.builtin.shell: | + awk -v mp="{{ item }}" '$5==mp {for(i=1;i<=NF;i++) if($i=="-") {print $(i+2); exit}}' /proc/self/mountinfo + register: k3s_required_mounts_sources + changed_when: false + check_mode: no + loop: "{{ k3s_required_mounts | default([]) }}" + when: (k3s_required_mounts | default([])) | length > 0 + +- name: Fail if required mountpoint is on SD + ansible.builtin.fail: + msg: "{{ item.item }} is on SD/rootfs ({{ item.stdout | default('') }}). Refusing to proceed." + loop: "{{ k3s_required_mounts_sources.results | default([]) }}" + when: + - not ansible_check_mode + - (item.stdout | default('')) is search("mmcblk0") or (item.stdout | default('')) is search("/dev/mmc") + +- name: Read k3s node token for sharing + ansible.builtin.slurp: + src: "{{ k3s_effective_data_dir }}/server/node-token" + register: k3s_node_token_slurp + when: + - k3s_role == "server" + - k3s_cluster_init | bool or k3s_server_url | default('') | length == 0 + ignore_errors: true + +- name: Set k3s_token_discovered fact + ansible.builtin.set_fact: + k3s_token_discovered: "{{ k3s_node_token_slurp.content | b64decode | trim }}" + when: + - k3s_node_token_slurp is defined + - k3s_node_token_slurp.content is defined + +- name: Wait for k3s token from server + ansible.builtin.command: /usr/bin/true + until: hostvars[item].k3s_token_discovered | default('') | length > 0 + retries: 60 + delay: 10 + loop: "{{ groups['k3s_hosts'] }}" + when: + - k3s_role == 'agent' + - hostvars[item].k3s_role | default('agent') == 'server' + - (k3s_token | default('') | length == 0) or (item in ansible_play_hosts) + changed_when: false + +- name: Discover k3s token from other hosts + ansible.builtin.set_fact: + k3s_token: "{{ hostvars[item].k3s_token_discovered }}" + loop: "{{ groups['k3s_hosts'] }}" + when: + - hostvars[item].k3s_token_discovered | default('') | length > 0 + - hostvars[item].k3s_role | default('agent') == 'server' + - (k3s_token | default('') | length == 0) or (k3s_role == 'agent' and k3s_token != hostvars[item].k3s_token_discovered) + +- name: Detect k3s CA hash mismatch on agents (logs) + ansible.builtin.shell: | + journalctl -u {{ k3s_service_name }} --since "1 hour ago" | grep "token CA hash does not match" + register: k3s_ca_mismatch_logs + changed_when: false + failed_when: false + when: + - k3s_role == 'agent' + - k3s_service.stat.exists + +- name: Proactively check for CA hash mismatch (files) + block: + - name: Get local agent CA cert hash (DER) + ansible.builtin.shell: | + if [ -f {{ k3s_effective_data_dir }}/agent/server-ca.crt ]; then + openssl x509 -in {{ k3s_effective_data_dir }}/agent/server-ca.crt -outform DER | openssl dgst -sha256 | awk '{print $NF}' + else + echo "MISSING" + fi + register: k3s_agent_ca_hash_cmd + changed_when: false + + - name: Extract expected CA hash from token + ansible.builtin.set_fact: + k3s_expected_ca_hash: "{{ (k3s_token | regex_replace('^K10([a-f0-9]+)::.*$', '\\1')) if (k3s_token is search('^K10[a-f0-9]+::')) else '' }}" + + - name: Detect mismatch from files + ansible.builtin.set_fact: + k3s_ca_mismatch_files: "{{ k3s_agent_ca_hash_cmd.stdout | trim != k3s_expected_ca_hash }}" + when: + - k3s_agent_ca_hash_cmd.stdout | trim != "MISSING" + - k3s_expected_ca_hash | length > 0 + when: + - k3s_role == 'agent' + - k3s_token | default('') | length > 0 + +- name: Handle k3s CA hash mismatch + block: + - name: Stop k3s-agent for CA repair + ansible.builtin.shell: "systemctl stop {{ k3s_service_name }} || true" + changed_when: false + failed_when: false + + - name: Kill any remaining k3s processes + ansible.builtin.shell: "killall -9 k3s || true" + changed_when: false + + - name: Wipe stale agent state + ansible.builtin.file: + path: "{{ k3s_effective_data_dir }}/agent" + state: absent + + - name: Force k3s re-installation after CA wipe + ansible.builtin.set_fact: + k3s_needs_install: true + when: + - k3s_role == 'agent' + - (k3s_ca_mismatch_logs.rc | default(1) == 0) or (k3s_ca_mismatch_files | default(false) | bool) + +- name: Validate join configuration + ansible.builtin.assert: + that: + - k3s_server_url is defined + - k3s_server_url | length > 0 + - k3s_token is defined + - k3s_token | length > 0 + fail_msg: "k3s_server_url and k3s_token are required for non-init join nodes." + when: + - not k3s_cluster_init | bool + - k3s_role == 'agent' or k3s_server_url | default('') | length > 0 + +- name: Check for existing k3s server node token + ansible.builtin.stat: + path: "{{ k3s_effective_data_dir }}/server/node-token" + register: k3s_server_node_token + when: + - k3s_guard_token_drift | bool + - k3s_role == "server" + - k3s_cluster_init | bool + +- name: Read existing k3s server node token + ansible.builtin.slurp: + src: "{{ k3s_effective_data_dir }}/server/node-token" + register: k3s_server_node_token_raw + when: + - k3s_guard_token_drift | bool + - k3s_role == "server" + - k3s_cluster_init | bool + - k3s_server_node_token.stat.exists + +- name: Fail when vault k3s token is missing for existing cluster + ansible.builtin.fail: + msg: >- + vault_k3s_token is empty but {{ k3s_effective_data_dir }}/server/node-token exists. + Refusing to overwrite k3s config. Update vault_k3s.yml or run the k3s sync playbook. + when: + - k3s_guard_token_drift | bool + - k3s_role == "server" + - k3s_cluster_init | bool + - k3s_server_node_token.stat.exists + - (k3s_token | default('') | trim) | length == 0 + +- name: Fail on k3s token drift from live cluster + ansible.builtin.fail: + msg: >- + vault_k3s_token does not match the live k3s server node-token. + Refusing to overwrite k3s config. Run infrastructure/playbooks/k3s_sync.yml + or update vault_k3s.yml to match the live token. + when: + - k3s_guard_token_drift | bool + - k3s_role == "server" + - k3s_cluster_init | bool + - k3s_server_node_token.stat.exists + - (k3s_token | default('') | trim) | length > 0 + - (k3s_server_node_token_raw.content | b64decode | trim) != (k3s_token | default('') | trim) + +- name: Ensure k3s config directory exists + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + mode: "0755" + +- name: Check if controller has conf/k3s.cfg (optional) + ansible.builtin.stat: + path: "{{ role_path }}/../../../conf/k3s.cfg" + register: _k3s_prole_cfg_stat + delegate_to: localhost + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Load registry defaults from prole.cfg + ansible.builtin.set_fact: + k3s_prole_k3s_server_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'PROLE_K3S_SERVER section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + k3s_prole_service_namespace_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'SERVICE_NAMESPACE section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Resolve k3s registry host + ansible.builtin.set_fact: + k3s_registry_host_resolved: >- + {{ (k3s_registry_host | default('')) if (k3s_registry_host | default('') | length > 0) + else (k3s_prole_k3s_server_cfg | default('')) if (k3s_prole_k3s_server_cfg | default('') | length > 0) + else (k3s_server_url | default('')) if (k3s_server_url | default('') | length > 0) + else inventory_hostname }} + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Resolve k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: >- + {{ 'https' if (k3s_registry_host | default('') | regex_search('^https://')) + else 'http' if (k3s_registry_host | default('') | regex_search('^http://')) + else (k3s_registry_endpoint_scheme | default('http')) }} + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Normalize k3s registry host + ansible.builtin.set_fact: + k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}" + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Normalize k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}" + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Resolve k3s registry namespace + ansible.builtin.set_fact: + k3s_registry_namespace_resolved: >- + {{ (k3s_registry_namespace | default('')) if (k3s_registry_namespace | default('') | length > 0 and k3s_registry_namespace != '${SERVICE_NAMESPACE}') + else (k3s_prole_service_namespace_cfg | default('')) if (k3s_prole_service_namespace_cfg | default('') | length > 0 and k3s_prole_service_namespace_cfg != '${SERVICE_NAMESPACE}') + else 'knoe-system' }} + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Validate k3s registry host + ansible.builtin.assert: + that: + - k3s_registry_host_resolved | length > 0 + fail_msg: "k3s registry host could not be resolved (set k3s_registry_host or PROLE_K3S_SERVER)." + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Render k3s registries config + ansible.builtin.template: + src: registries.yaml.j2 + dest: "{{ k3s_registry_config_path }}" + mode: "0644" + notify: Restart k3s + when: k3s_state == "present" and k3s_registry_config_enabled | bool + +- name: Ensure kubeconfig group exists + ansible.builtin.group: + name: "{{ k3s_kubeconfig_group }}" + state: present + local: "{{ (ansible_facts['os_family'] | default('')) in ['Darwin', 'FreeBSD', 'OpenBSD', 'NetBSD'] }}" + +- name: Ensure kubeconfig users are in group (server) + ansible.builtin.user: + name: "{{ item }}" + groups: "{{ k3s_kubeconfig_group }}" + append: true + loop: "{{ k3s_kubeconfig_users }}" + when: k3s_role == "server" and (k3s_kubeconfig_users | length) > 0 + +- name: Ensure rancher directories are group-accessible + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: "{{ k3s_kubeconfig_group }}" + mode: "0750" + loop: + - /etc/rancher + - /etc/rancher/k3s + - /etc/rancher/k3s/config.yaml.d + - "{{ k3s_effective_data_dir | dirname }}" + when: k3s_state != "absent" + +- name: Detect active interface for k3s_node_ip (Retropie/Pi only) + ansible.builtin.shell: | + # Find interface that has the configured k3s_node_ip + ip -4 addr show | grep -B2 "{{ k3s_node_ip }}" | grep -oE "^[0-9]+: [^:]+" | head -n 1 | awk '{print $2}' + register: _detected_k3s_iface + when: + - inventory_hostname in ['retropie.prole.org', 'pi.prole.org'] + - k3s_node_ip is defined and k3s_node_ip | length > 0 + changed_when: false + +- name: Update k3s_flannel_iface based on detection (Retropie/Pi only) + ansible.builtin.set_fact: + k3s_flannel_iface: "{{ _detected_k3s_iface.stdout | trim }}" + when: + - inventory_hostname in ['retropie.prole.org', 'pi.prole.org'] + - _detected_k3s_iface.stdout | default('') | length > 0 + - _detected_k3s_iface.stdout | trim != k3s_flannel_iface + +- name: Validate k3s config args + ansible.builtin.import_tasks: validate_args.yml + when: k3s_state != "absent" + +- name: Render k3s config + ansible.builtin.template: + src: config.yaml.j2 + dest: /etc/rancher/k3s/config.yaml + owner: root + group: "{{ k3s_kubeconfig_group }}" + mode: "0640" + notify: Restart k3s + +- name: Build k3s install environment + ansible.builtin.set_fact: + k3s_install_env: >- + {{ {'INSTALL_K3S_EXEC': k3s_role} + | combine((k3s_version | length > 0) | ternary({'INSTALL_K3S_VERSION': k3s_version}, {})) }} + +- name: Ensure k3s service is stopped before install to prevent hangs + ansible.builtin.service: + name: "{{ k3s_service_name }}" + state: stopped + when: + - k3s_needs_install + - k3s_service.stat.exists + failed_when: false + +- name: Install k3s + ansible.builtin.shell: | + curl -sfL https://get.k3s.io | sh - + environment: + "{{ k3s_install_env }}" + when: k3s_needs_install + +- name: Re-check if k3s service is installed (post-install) + ansible.builtin.stat: + path: "/etc/systemd/system/{{ k3s_service_name }}.service" + register: k3s_service + +- name: Re-check if k3s binary exists (post-install) + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: k3s_bin + +- name: Ensure k3s service state + block: + - name: Start/stop k3s service (best-effort) + ansible.builtin.service: + name: "{{ k3s_service_name }}" + state: "{{ 'started' if k3s_state == 'present' else 'stopped' }}" + enabled: "{{ k3s_state == 'present' }}" + register: _k3s_service_manage + failed_when: false + + - name: Wait for k3s service to become active + ansible.builtin.command: "systemctl is-active --quiet {{ k3s_service_name }}" + register: _k3s_service_active + until: _k3s_service_active.rc == 0 + retries: "{{ k3s_service_start_retries | default(60) }}" + delay: "{{ k3s_service_start_delay | default(5) }}" + changed_when: false + when: k3s_state == 'present' + when: k3s_service.stat.exists or k3s_bin.stat.exists + +- name: Flush k3s restart handler before Kubernetes operations + ansible.builtin.meta: flush_handlers + when: k3s_state == "present" + +- name: Apply node taints + ansible.builtin.command: "k3s kubectl taint node {{ inventory_hostname }} {{ item }} --overwrite" + delegate_to: "{{ (groups['k3s_hosts'] | map('extract', hostvars) | selectattr('k3s_role', 'defined') | selectattr('k3s_role', 'equalto', 'server') | map(attribute='inventory_hostname') | first) | default(groups['k3s_hosts'] | first) }}" + loop: "{{ k3s_node_taints }}" + when: + - k3s_state == "present" + - k3s_node_taints | length > 0 + register: _taint_result + changed_when: "'modified' in _taint_result.stdout or 'tainted' in _taint_result.stdout" + failed_when: false # Avoid failing if node is not yet ready + +- name: Wait for k3s API readiness + ansible.builtin.command: k3s kubectl get --raw=/readyz + register: k3s_readyz + until: + - k3s_readyz.rc == 0 + - (k3s_readyz.stdout | default('') | trim) in ['ok', 'OK'] + retries: 60 + delay: 5 + changed_when: false + when: k3s_state == "present" and k3s_role == "server" + +- name: Wait for local node to report Ready + ansible.builtin.shell: | + k3s kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true + register: k3s_node_ready + until: (k3s_node_ready.stdout | default('') | trim) == 'True' + retries: 60 + delay: 5 + changed_when: false + failed_when: false + when: k3s_state == "present" and k3s_role == "server" + +- name: Read k3s node token after install for sharing + ansible.builtin.slurp: + src: "{{ k3s_effective_data_dir }}/server/node-token" + register: k3s_node_token_slurp_post + when: + - k3s_role == "server" + - k3s_cluster_init | bool or k3s_server_url | default('') | length == 0 + - k3s_token_discovered is not defined + ignore_errors: true + +- name: Set k3s_token_discovered fact (post-install) + ansible.builtin.set_fact: + k3s_token_discovered: "{{ k3s_node_token_slurp_post.content | b64decode | trim }}" + when: + - k3s_node_token_slurp_post is defined + - k3s_node_token_slurp_post.content is defined + +- name: Sync k3s token to local vault + ansible.builtin.include_tasks: sync_vault.yml + when: + - k3s_role == "server" + - k3s_cluster_init | bool or k3s_server_url | default('') | length == 0 + - k3s_token_discovered is defined + - k3s_token_discovered | length > 0 + +- name: Pre-stage k3s images (deliberate arch) + ansible.builtin.include_tasks: prestage_images.yml + when: + - k3s_state == "present" + - k3s_prestage_images | bool + tags: [images] diff --git a/infrastructure/roles/k3s/tasks/kong.yml b/infrastructure/roles/k3s/tasks/kong.yml new file mode 100644 index 0000000..515e42d --- /dev/null +++ b/infrastructure/roles/k3s/tasks/kong.yml @@ -0,0 +1,12 @@ +--- +- name: Ensure Kong is present and refreshed (common core) + ansible.builtin.shell: | + kubectl config use-context prole-k3s >/dev/null 2>&1 || true + bash {{ playbook_dir }}/../../etc/init_kong.sh -n {{ k3s_kong_namespace }} update + environment: + PROLE_MODE: k3s + KUBECONFIG: "{{ playbook_dir }}/../../prole-k3s.kubeconfig" + delegate_to: localhost + become: false + register: k3s_kong_install + run_once: true diff --git a/infrastructure/roles/k3s/tasks/kong_restart.yml b/infrastructure/roles/k3s/tasks/kong_restart.yml new file mode 100644 index 0000000..241e03e --- /dev/null +++ b/infrastructure/roles/k3s/tasks/kong_restart.yml @@ -0,0 +1,18 @@ +--- + +- name: Wait for Kong deployment to exist (for declarative config reload) + ansible.builtin.command: k3s kubectl get deployment {{ k3s_kong_deployment }} -n {{ k3s_kong_namespace }} + register: _kong_deploy_check + changed_when: false + until: _kong_deploy_check.rc == 0 + retries: 30 + delay: 5 + when: svc_check_kong_config_changed | default(false) + run_once: true + +- name: Restart Kong to reload declarative config + ansible.builtin.command: k3s kubectl rollout restart deployment/{{ k3s_kong_deployment }} -n {{ k3s_kong_namespace }} + changed_when: true + failed_when: false + when: svc_check_kong_config_changed | default(false) + run_once: true \ No newline at end of file diff --git a/infrastructure/roles/k3s/tasks/label_node.yml b/infrastructure/roles/k3s/tasks/label_node.yml new file mode 100644 index 0000000..160a39b --- /dev/null +++ b/infrastructure/roles/k3s/tasks/label_node.yml @@ -0,0 +1,20 @@ +--- + +- name: Wait for node to register in Kubernetes + ansible.builtin.command: "k3s kubectl get node {{ k3s_label_node.key }}" + register: _k3s_get_node + until: _k3s_get_node.rc == 0 + retries: 60 + delay: 5 + changed_when: false + delegate_to: "{{ k3s_kubectl_delegate_host }}" + +- name: Apply persisted labels to node + ansible.builtin.command: >- + k3s kubectl label node {{ k3s_label_node.key }} {{ item.key }}={{ item.value }} --overwrite + register: _k3s_label_result + changed_when: "'not labeled' not in (_k3s_label_result.stdout | default(''))" + loop: "{{ k3s_label_node.value | dict2items }}" + loop_control: + label: "{{ item.key }}={{ item.value }}" + delegate_to: "{{ k3s_kubectl_delegate_host }}" diff --git a/infrastructure/roles/k3s/tasks/label_nodes.yml b/infrastructure/roles/k3s/tasks/label_nodes.yml new file mode 100644 index 0000000..58ae9fa --- /dev/null +++ b/infrastructure/roles/k3s/tasks/label_nodes.yml @@ -0,0 +1,23 @@ +--- + +- name: Persist Kubernetes node labels + ansible.builtin.include_tasks: label_node.yml + loop: "{{ k3s_node_labels | dict2items }}" + loop_control: + loop_var: k3s_label_node + label: "{{ k3s_label_node.key }}" + when: + - hostvars[k3s_label_node.key] is defined + - hostvars[k3s_label_node.key].k3s_enabled | default(true) | bool + vars: + k3s_kubectl_delegate_host: >- + {{ + (groups['k3s_hosts'] + | map('extract', hostvars) + | selectattr('k3s_role', 'defined') + | selectattr('k3s_role', 'equalto', 'server') + | map(attribute='inventory_hostname') + | first) + | default(groups['k3s_hosts'] | first) + }} + run_once: true diff --git a/infrastructure/roles/k3s/tasks/main.yml b/infrastructure/roles/k3s/tasks/main.yml new file mode 100644 index 0000000..fab9056 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/main.yml @@ -0,0 +1,7 @@ +--- + +- name: Tier 5 - k3s installation and service readiness + ansible.builtin.import_tasks: install.yml + +- name: Tier 6 - k3s cluster add-ons (kubectl/helm apply) + ansible.builtin.import_tasks: configure.yml diff --git a/infrastructure/roles/k3s/tasks/optional_workloads_policy_eval.yml b/infrastructure/roles/k3s/tasks/optional_workloads_policy_eval.yml new file mode 100644 index 0000000..312920b --- /dev/null +++ b/infrastructure/roles/k3s/tasks/optional_workloads_policy_eval.yml @@ -0,0 +1,29 @@ +--- + +- name: Evaluate optional-workloads policy (ready + schedulable) - build candidates + ansible.builtin.set_fact: + _k3s_ready_node_candidates: >- + {{ (_k3s_nodes_list.stdout_lines | default([]) + | select('search', ' Ready') + | list) }} + _k3s_scheduling_disabled_node_candidates: >- + {{ (_k3s_nodes_list.stdout_lines | default([]) + | select('search', 'SchedulingDisabled') + | list) }} + optional_workloads_min_ready_schedulable_nodes: >- + {{ (k3s_optional_workloads_min_nodes_cfg | int) }} + changed_when: false + +- name: Evaluate optional-workloads policy (ready + schedulable) - count ready schedulable nodes + ansible.builtin.set_fact: + ready_schedulable_nodes: >- + {{ ((_k3s_ready_node_candidates | default([])) + | difference(_k3s_scheduling_disabled_node_candidates | default([])) + | length) | int }} + changed_when: false + +- name: Evaluate optional-workloads policy (ready + schedulable) - compute allow/deny + ansible.builtin.set_fact: + optional_workloads_allowed: >- + {{ (ready_schedulable_nodes | int) >= (optional_workloads_min_ready_schedulable_nodes | int) }} + changed_when: false diff --git a/infrastructure/roles/k3s/tasks/prestage_image.yml b/infrastructure/roles/k3s/tasks/prestage_image.yml new file mode 100644 index 0000000..ef40732 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/prestage_image.yml @@ -0,0 +1,12 @@ +--- + +- name: Pull k3s image {{ k3s_prestage_image }} + ansible.builtin.shell: >- + k3s ctr -n k8s.io images pull + --platform {{ k3s_prestage_images_platform }} + {{ k3s_prestage_image }} + register: _image_pull + until: _image_pull.rc == 0 + retries: 10 + delay: 20 + ignore_errors: true diff --git a/infrastructure/roles/k3s/tasks/prestage_images.yml b/infrastructure/roles/k3s/tasks/prestage_images.yml new file mode 100644 index 0000000..ab3389d --- /dev/null +++ b/infrastructure/roles/k3s/tasks/prestage_images.yml @@ -0,0 +1,33 @@ +--- + +- name: Verify k3s container runtime is available + ansible.builtin.command: k3s ctr -n k8s.io version + register: _k3s_ctr_version + changed_when: false + failed_when: false + +- name: List local k3s images + ansible.builtin.command: k3s ctr -n k8s.io images ls -q + register: _local_images + changed_when: false + failed_when: false + +- name: Compute missing k3s images + ansible.builtin.set_fact: + k3s_prestage_missing_images: "{{ k3s_prestage_images_list | difference(_local_images.stdout_lines | default([])) }}" + +- name: Report k3s image pre-stage plan + ansible.builtin.debug: + msg: + - "k3s_prestage_images_platform={{ k3s_prestage_images_platform }}" + - "missing_images={{ k3s_prestage_missing_images | length }}" + changed_when: false + +- name: Pull missing k3s images (one image per step) + ansible.builtin.include_tasks: prestage_image.yml + loop: "{{ k3s_prestage_missing_images }}" + loop_control: + label: "{{ item }}" + vars: + k3s_prestage_image: "{{ item }}" + when: (k3s_prestage_missing_images | length) > 0 diff --git a/infrastructure/roles/k3s/tasks/refresh_server.yml b/infrastructure/roles/k3s/tasks/refresh_server.yml new file mode 100644 index 0000000..9034ccd --- /dev/null +++ b/infrastructure/roles/k3s/tasks/refresh_server.yml @@ -0,0 +1,195 @@ +--- + +- name: Guard - require explicit confirmation for k3s server refresh + ansible.builtin.assert: + that: + - k3s_role == 'server' + - k3s_state == 'present' + - (k3s_refresh_confirm | default('')) == 'YES' + fail_msg: >- + Refusing to refresh k3s server without explicit confirmation. + Set k3s_refresh_confirm=YES and re-run (this is destructive: it wipes k3s state). + +- name: Read current k3s config (to detect current data-dir) + ansible.builtin.stat: + path: /etc/rancher/k3s/config.yaml + register: _k3s_refresh_config_stat + +- name: Slurp current k3s config + ansible.builtin.slurp: + src: /etc/rancher/k3s/config.yaml + register: _k3s_refresh_config_slurp + when: _k3s_refresh_config_stat.stat.exists + +- name: Set refresh source/target data-dir + ansible.builtin.set_fact: + _k3s_refresh_cfg: >- + {{ + (_k3s_refresh_config_slurp.content | b64decode | from_yaml) + if (_k3s_refresh_config_stat.stat.exists | bool) + else {} + }} + k3s_refresh_source_data_dir: >- + {{ + _k3s_refresh_cfg.get('data-dir') + | default((k3s_data_dir | default('/var/lib/rancher/k3s')), true) + }} + k3s_refresh_target_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}" + +- name: Preflight - check external datastore TCP reachability (best-effort) + ansible.builtin.wait_for: + host: "{{ k3s_datastore_mariadb_host }}" + port: "{{ k3s_datastore_mariadb_port | default(3306) }}" + connect_timeout: 2 + timeout: 5 + state: started + when: + - (k3s_datastore_endpoint | default('') | length) > 0 + - k3s_datastore_mariadb_host is defined + changed_when: false + failed_when: false + +- name: Create refresh backup directory + ansible.builtin.file: + path: "{{ k3s_refresh_backup_root }}/{{ inventory_hostname }}/{{ ansible_date_time.iso8601_basic_short }}" + state: directory + mode: "0700" + register: _k3s_refresh_backup_dir + +- name: Stat k3s token/tls artifacts (source data-dir) + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ k3s_refresh_source_data_dir }}/server/token" + - "{{ k3s_refresh_source_data_dir }}/server/node-token" + # Minimal TLS set to preserve agent trust (CA hash) while allowing k3s to + # regenerate component creds and `server/cred/*.kubeconfig`. + - "{{ k3s_refresh_source_data_dir }}/server/tls/server-ca.crt" + - "{{ k3s_refresh_source_data_dir }}/server/tls/server-ca.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/client-ca.crt" + - "{{ k3s_refresh_source_data_dir }}/server/tls/client-ca.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/request-header-ca.crt" + - "{{ k3s_refresh_source_data_dir }}/server/tls/request-header-ca.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/service.key" + - "{{ k3s_refresh_source_data_dir }}/server/tls/service.current.key" + register: _k3s_refresh_artifacts + +- name: Back up k3s token/tls artifacts (remote->remote) + ansible.builtin.copy: + remote_src: true + src: "{{ item.item }}" + dest: "{{ _k3s_refresh_backup_dir.path }}/" + mode: preserve + loop: "{{ _k3s_refresh_artifacts.results }}" + when: item.stat.exists + +- name: Read preserved server token (source data-dir) + ansible.builtin.slurp: + src: "{{ k3s_refresh_source_data_dir }}/server/token" + register: _k3s_refresh_token_slurp + when: >- + {{ + ( + _k3s_refresh_artifacts.results + | selectattr('item', 'equalto', (k3s_refresh_source_data_dir ~ '/server/token')) + | map(attribute='stat.exists') + | first + ) + | default(false) + }} + no_log: true + +- name: Ensure we have a token to preserve + ansible.builtin.assert: + that: + - (k3s_token | default('') | length) > 0 or (_k3s_refresh_token_slurp is defined) + fail_msg: >- + No existing token file was found under {{ k3s_refresh_source_data_dir }}/server/token, + and k3s_token is empty. Refusing to proceed because this would change the join token. + +- name: Force k3s_token to preserved token (so rendered config matches what we restore) + ansible.builtin.set_fact: + k3s_token: "{{ (_k3s_refresh_token_slurp.content | b64decode | trim) if (_k3s_refresh_token_slurp is defined) else k3s_token }}" + no_log: true + +- name: Stop k3s services before refresh + ansible.builtin.service: + name: "{{ item }}" + state: stopped + enabled: false + loop: + - k3s + - k3s-agent + failed_when: false + +- name: Uninstall and wipe k3s state (blank slate) + ansible.builtin.import_tasks: cleanup.yml + +- name: Wipe target k3s data-dir (blank slate except restored token/CA) + ansible.builtin.file: + path: "{{ k3s_refresh_target_data_dir }}" + state: absent + +- name: Recreate target server directory + ansible.builtin.file: + path: "{{ k3s_refresh_target_data_dir }}/server" + state: directory + mode: "0700" + +- name: Recreate target tls directory + ansible.builtin.file: + path: "{{ k3s_refresh_target_data_dir }}/server/tls" + state: directory + mode: "0700" + +- name: Restore preserved minimal tls artifacts into target data-dir + ansible.builtin.copy: + remote_src: true + src: "{{ item }}" + dest: "{{ k3s_refresh_target_data_dir }}/server/tls/" + mode: preserve + loop: + - "{{ _k3s_refresh_backup_dir.path }}/server-ca.crt" + - "{{ _k3s_refresh_backup_dir.path }}/server-ca.key" + - "{{ _k3s_refresh_backup_dir.path }}/client-ca.crt" + - "{{ _k3s_refresh_backup_dir.path }}/client-ca.key" + - "{{ _k3s_refresh_backup_dir.path }}/request-header-ca.crt" + - "{{ _k3s_refresh_backup_dir.path }}/request-header-ca.key" + - "{{ _k3s_refresh_backup_dir.path }}/service.key" + - "{{ _k3s_refresh_backup_dir.path }}/service.current.key" + failed_when: false + +- name: Install and start k3s + ansible.builtin.import_tasks: install.yml + +- name: Configure k3s add-ons + ansible.builtin.import_tasks: configure.yml + when: k3s_refresh_run_configure | bool + +- name: Verify k3s API is responsive after refresh + ansible.builtin.command: k3s kubectl get nodes + register: _k3s_refresh_nodes + changed_when: false + until: _k3s_refresh_nodes.rc == 0 + retries: 18 + delay: 10 + +- name: Read k3s node-token after refresh + ansible.builtin.slurp: + src: "{{ k3s_refresh_target_data_dir }}/server/node-token" + register: _k3s_refresh_node_token + changed_when: false + until: _k3s_refresh_node_token is succeeded + retries: 18 + delay: 10 + no_log: true + +- name: Assert node-token is preserved (when token is in node-token format) + ansible.builtin.assert: + that: + - (k3s_token | default('')) == (_k3s_refresh_node_token.content | b64decode | trim) + fail_msg: >- + Refusing to continue: expected regenerated node-token to match preserved k3s_token, + but it differs. This indicates the join token changed. + when: + - (k3s_token | default('')) is match('^K10[0-9a-f]+::server:.+') diff --git a/infrastructure/roles/k3s/tasks/start.yml b/infrastructure/roles/k3s/tasks/start.yml new file mode 100644 index 0000000..70fff9b --- /dev/null +++ b/infrastructure/roles/k3s/tasks/start.yml @@ -0,0 +1,51 @@ +--- +- name: Set k3s service name + ansible.builtin.set_fact: + k3s_service_name: "{{ 'k3s' if k3s_role == 'server' else 'k3s-agent' }}" + +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Build k3s install environment when service is missing + ansible.builtin.set_fact: + k3s_install_env: >- + {{ {'INSTALL_K3S_EXEC': k3s_role} + | combine((k3s_version | default('') | length > 0) | ternary({'INSTALL_K3S_VERSION': k3s_version}, {})) }} + when: (k3s_service_name + ".service") not in ansible_facts.services + +- name: Install k3s when service is missing + ansible.builtin.shell: | + set -euo pipefail + timeout "{{ k3s_install_timeout | default(600) }}" sh -c \ + 'curl -sfL --connect-timeout 20 --max-time 300 https://get.k3s.io | sh -' + environment: + "{{ k3s_install_env }}" + args: + creates: /usr/local/bin/k3s + executable: /bin/bash + when: (k3s_service_name + ".service") not in ansible_facts.services + +- name: Refresh service facts after install + ansible.builtin.service_facts: + +- name: Ensure k3s service is installed + ansible.builtin.assert: + that: + - (k3s_service_name + ".service") in ansible_facts.services + fail_msg: "k3s service {{ k3s_service_name }} is not installed. Run the k3s role or reset playbook first." + +- name: Start k3s service + ansible.builtin.systemd: + name: "{{ k3s_service_name }}" + state: started + enabled: true + no_block: true + +- name: Wait for k3s to become active + ansible.builtin.command: "systemctl is-active {{ k3s_service_name }}" + register: k3s_service_state + changed_when: false + retries: "{{ [1, (k3s_start_timeout_seconds | default(120) | int) // 10] | max }}" + delay: 10 + until: k3s_service_state.stdout in ["active", "activating"] + failed_when: k3s_service_state.stdout not in ["active", "activating"] diff --git a/infrastructure/roles/k3s/tasks/start_k3s_agents.yml b/infrastructure/roles/k3s/tasks/start_k3s_agents.yml new file mode 100644 index 0000000..0cf01e8 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/start_k3s_agents.yml @@ -0,0 +1,21 @@ +--- +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Require k3s-agent service to exist + ansible.builtin.assert: + that: + - ('k3s-agent' + '.service') in ansible_facts.services + - ansible_facts.services['k3s-agent' + '.service'].status != 'not-found' + fail_msg: "k3s-agent service is not installed on {{ inventory_hostname }}." + when: not ansible_check_mode + +- name: Start k3s-agent + ansible.builtin.systemd: + name: k3s-agent + state: started + enabled: true + no_block: true + when: + - ('k3s-agent' + '.service') in ansible_facts.services + - ansible_facts.services['k3s-agent' + '.service'].status != 'not-found' diff --git a/infrastructure/roles/k3s/tasks/start_k3s_servers.yml b/infrastructure/roles/k3s/tasks/start_k3s_servers.yml new file mode 100644 index 0000000..5c0fef3 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/start_k3s_servers.yml @@ -0,0 +1,21 @@ +--- +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Require k3s service to exist + ansible.builtin.assert: + that: + - ('k3s' + '.service') in ansible_facts.services + - ansible_facts.services['k3s' + '.service'].status != 'not-found' + fail_msg: "k3s service is not installed on {{ inventory_hostname }}." + when: not ansible_check_mode + +- name: Start k3s + ansible.builtin.systemd: + name: k3s + state: started + enabled: true + no_block: true + when: + - ('k3s' + '.service') in ansible_facts.services + - ansible_facts.services['k3s' + '.service'].status != 'not-found' diff --git a/infrastructure/roles/k3s/tasks/stop.yml b/infrastructure/roles/k3s/tasks/stop.yml new file mode 100644 index 0000000..9ebd163 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/stop.yml @@ -0,0 +1,15 @@ +--- +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Stop k3s services if present + ansible.builtin.service: + name: "{{ item }}" + state: stopped + enabled: false + loop: + - k3s + - k3s-agent + when: + - (item + ".service") in ansible_facts.services + - ansible_facts.services[item + ".service"].status != 'not-found' diff --git a/infrastructure/roles/k3s/tasks/stop_k3s_agents.yml b/infrastructure/roles/k3s/tasks/stop_k3s_agents.yml new file mode 100644 index 0000000..979d8c5 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/stop_k3s_agents.yml @@ -0,0 +1,21 @@ +--- +- name: Gather service facts + ansible.builtin.service_facts: + +# Agents can sometimes be in a Restart=always loop; disabling stops the loop. +- name: Stop and disable k3s-agent service when present + ansible.builtin.systemd: + name: k3s-agent + state: stopped + enabled: false + when: + - ('k3s-agent' + '.service') in ansible_facts.services + - ansible_facts.services['k3s-agent' + '.service'].status != 'not-found' + +- name: Clear failed state for k3s-agent (if any) + ansible.builtin.command: systemctl reset-failed k3s-agent + changed_when: false + failed_when: false + when: + - ('k3s-agent' + '.service') in ansible_facts.services + - ansible_facts.services['k3s-agent' + '.service'].status != 'not-found' diff --git a/infrastructure/roles/k3s/tasks/stop_k3s_servers.yml b/infrastructure/roles/k3s/tasks/stop_k3s_servers.yml new file mode 100644 index 0000000..11477ab --- /dev/null +++ b/infrastructure/roles/k3s/tasks/stop_k3s_servers.yml @@ -0,0 +1,19 @@ +--- +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Stop k3s service when present + ansible.builtin.systemd: + name: k3s + state: stopped + when: + - ('k3s' + '.service') in ansible_facts.services + - ansible_facts.services['k3s' + '.service'].status != 'not-found' + +- name: Clear failed state for k3s (if any) + ansible.builtin.command: systemctl reset-failed k3s + changed_when: false + failed_when: false + when: + - ('k3s' + '.service') in ansible_facts.services + - ansible_facts.services['k3s' + '.service'].status != 'not-found' diff --git a/infrastructure/roles/k3s/tasks/svc_check.yml b/infrastructure/roles/k3s/tasks/svc_check.yml new file mode 100644 index 0000000..6c13e92 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/svc_check.yml @@ -0,0 +1,54 @@ +--- +# Deploy a tiny static check site and route it through Kong as svc.prole.org + +- name: Read prole type gif (controller) + ansible.builtin.slurp: + src: "{{ playbook_dir }}/../../img/prole-type.gif" + register: svc_check_gif + delegate_to: localhost + become: false + run_once: true + +- name: Render svc-check Helm chart + ansible.builtin.command: + argv: + - helm + - template + - svc-check + - "{{ playbook_dir }}/../../infrastructure/deployments/svc-check-helm" + - --namespace + - "{{ k3s_svc_check_namespace }}" + - --set-string + - "svcCheck.domain={{ k3s_svc_check_domain }}" + - --set-string + - "svcCheck.gifB64={{ svc_check_gif.content | trim }}" + - --set-string + - "kong.namespace={{ k3s_svc_check_kong_namespace }}" + - --set-string + - "kong.configMapName={{ k3s_svc_check_kong_configmap_name }}" + - --set-string + - "ingress.clusterIssuer=letsencrypt-prod" + register: svc_check_manifest + delegate_to: localhost + become: false + changed_when: false + run_once: true + +- name: Apply svc-check manifest + ansible.builtin.command: k3s kubectl apply -f - + args: + stdin: "{{ svc_check_manifest.stdout }}" + register: svc_check_apply + changed_when: "'configured' in svc_check_apply.stdout or 'created' in svc_check_apply.stdout" + run_once: true + +- name: Detect whether Kong declarative config changed + ansible.builtin.set_fact: + svc_check_kong_config_changed: >- + {{ (svc_check_apply.stdout_lines + | select('search', '^configmap/' ~ k3s_svc_check_kong_configmap_name ~ ' ') + | select('search', '(configured|created)$') + | list + | length) > 0 }} + when: svc_check_apply is defined + run_once: true diff --git a/infrastructure/roles/k3s/tasks/sync.yml b/infrastructure/roles/k3s/tasks/sync.yml new file mode 100644 index 0000000..40f7720 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/sync.yml @@ -0,0 +1,161 @@ +--- + +- name: Set effective k3s data-dir + ansible.builtin.set_fact: + k3s_effective_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s', true) }}" + +- name: Ensure k3s config exists + ansible.builtin.stat: + path: /etc/rancher/k3s/config.yaml + register: k3s_config + +- name: Fail when k3s config is missing + ansible.builtin.debug: + msg: "k3s config missing at /etc/rancher/k3s/config.yaml; creating minimal config for sync." + when: not k3s_config.stat.exists + +- name: Ensure k3s config directory exists when missing + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + owner: root + group: "{{ k3s_kubeconfig_group | default('kubeadm') }}" + mode: "0750" + when: not k3s_config.stat.exists + +- name: Create minimal k3s config when missing + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml + owner: root + group: "{{ k3s_kubeconfig_group | default('kubeadm') }}" + mode: "0640" + content: | + {% if k3s_cluster_init | default(false) | bool %} + cluster-init: true + {% elif k3s_server_url | default('') %} + server: "{{ k3s_server_url }}" + {% endif %} + {% if k3s_sync_token | default('') %} + token: "{{ k3s_sync_token }}" + {% endif %} + when: not k3s_config.stat.exists + +- name: Sync k3s token into config + ansible.builtin.lineinfile: + path: /etc/rancher/k3s/config.yaml + regexp: "^token:" + line: "token: \"{{ k3s_sync_token }}\"" + owner: root + group: "{{ k3s_kubeconfig_group | default('kubeadm') }}" + mode: "0640" + +- name: Sync k3s server URL into config (join nodes) + ansible.builtin.lineinfile: + path: /etc/rancher/k3s/config.yaml + regexp: "^server:" + line: "server: \"{{ k3s_server_url }}\"" + owner: root + group: "{{ k3s_kubeconfig_group | default('kubeadm') }}" + mode: "0640" + when: + - k3s_server_url | default('') | length > 0 + +- name: Check if controller has conf/k3s.cfg (optional) + ansible.builtin.stat: + path: "{{ role_path }}/../../../conf/k3s.cfg" + register: _k3s_prole_cfg_stat + delegate_to: localhost + when: k3s_registry_config_enabled | bool + +- name: Load registry defaults from prole.cfg + ansible.builtin.set_fact: + k3s_prole_k3s_server_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'PROLE_K3S_SERVER section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + k3s_prole_service_namespace_cfg: >- + {{ lookup( + 'ansible.builtin.ini', + 'SERVICE_NAMESPACE section=Global file=' ~ (role_path ~ '/../../../conf/k3s.cfg'), + default='', + errors='ignore' + ) if _k3s_prole_cfg_stat.stat.exists else '' }} + when: k3s_registry_config_enabled | bool + +- name: Resolve k3s registry host + ansible.builtin.set_fact: + k3s_registry_host_resolved: >- + {{ (k3s_registry_host | default('')) if (k3s_registry_host | default('') | length > 0) + else (k3s_prole_k3s_server_cfg | default('')) if (k3s_prole_k3s_server_cfg | default('') | length > 0) + else inventory_hostname }} + when: k3s_registry_config_enabled | bool + +- name: Resolve k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: >- + {{ 'https' if (k3s_registry_host | default('') | regex_search('^https://')) + else 'http' if (k3s_registry_host | default('') | regex_search('^http://')) + else (k3s_registry_endpoint_scheme | default('http')) }} + when: k3s_registry_config_enabled | bool + +- name: Normalize k3s registry host + ansible.builtin.set_fact: + k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}" + when: k3s_registry_config_enabled | bool + +- name: Normalize k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}" + when: k3s_registry_config_enabled | bool + +- name: Resolve k3s registry namespace + ansible.builtin.set_fact: + k3s_registry_namespace_resolved: >- + {{ (k3s_registry_namespace | default('')) if (k3s_registry_namespace | default('') | length > 0 and k3s_registry_namespace != '${SERVICE_NAMESPACE}') + else (k3s_prole_service_namespace_cfg | default('')) if (k3s_prole_service_namespace_cfg | default('') | length > 0 and k3s_prole_service_namespace_cfg != '${SERVICE_NAMESPACE}') + else 'knoe-system' }} + when: k3s_registry_config_enabled | bool + +- name: Validate k3s registry host + ansible.builtin.assert: + that: + - k3s_registry_host_resolved | length > 0 + fail_msg: "k3s registry host could not be resolved (set k3s_registry_host or PROLE_K3S_SERVER)." + when: k3s_registry_config_enabled | bool + +- name: Render k3s registries config + ansible.builtin.template: + src: registries.yaml.j2 + dest: "{{ k3s_registry_config_path }}" + mode: "0644" + when: k3s_registry_config_enabled | bool + +- name: Ensure k3s server directory exists + ansible.builtin.file: + path: "{{ k3s_effective_data_dir }}/server" + state: directory + mode: "0755" + when: + - k3s_role == 'server' + - k3s_sync_tls_bundle_present | default(false) + +- name: Remove existing k3s tls directory before sync + ansible.builtin.file: + path: "{{ k3s_effective_data_dir }}/server/tls" + state: absent + when: + - k3s_role == 'server' + - k3s_sync_tls_bundle_present | default(false) + +- name: Restore k3s tls bundle + ansible.builtin.unarchive: + src: "{{ k3s_sync_tls_bundle }}" + dest: "{{ k3s_effective_data_dir }}/server" + owner: root + group: root + when: + - k3s_role == 'server' + - k3s_sync_tls_bundle_present | default(false) diff --git a/infrastructure/roles/k3s/tasks/sync_vault.yml b/infrastructure/roles/k3s/tasks/sync_vault.yml new file mode 100644 index 0000000..a3fff9a --- /dev/null +++ b/infrastructure/roles/k3s/tasks/sync_vault.yml @@ -0,0 +1,73 @@ +--- +- name: Update k3s vault token on controller + block: + - name: Set vault k3s path + ansible.builtin.set_fact: + vault_k3s_path: "{{ role_path }}/../../inventory/group_vars/all/vault_k3s.yml" + vault_pass_default: "{{ role_path }}/../../../.vault_pass" + + - name: Determine vault password file + ansible.builtin.set_fact: + k3s_vault_password_file: >- + {{ k3s_vault_password_file + | default(lookup('env', 'ANSIBLE_VAULT_PASSWORD_FILE') | default('', true), true) }} + delegate_to: localhost + delegate_facts: true + + - name: Check for default vault password file + ansible.builtin.stat: + path: "{{ vault_pass_default }}" + register: vault_pass_default_stat + delegate_to: localhost + + - name: Fallback to default vault password file + ansible.builtin.set_fact: + k3s_vault_password_file: "{{ vault_pass_default }}" + when: + - (hostvars['localhost'].k3s_vault_password_file | default('')) | length == 0 + - vault_pass_default_stat.stat.exists + delegate_to: localhost + delegate_facts: true + + - name: Check if vault file is encrypted + ansible.builtin.command: "head -n 1 {{ vault_k3s_path }}" + register: vault_k3s_head + changed_when: false + delegate_to: localhost + failed_when: false + + - name: Mark vault encryption state + ansible.builtin.set_fact: + vault_k3s_encrypted: "{{ (vault_k3s_head.stdout | default('')) is search('^\\$ANSIBLE_VAULT') }}" + delegate_to: localhost + delegate_facts: true + + - name: Decrypt vault_k3s.yml + ansible.builtin.command: >- + ansible-vault decrypt {{ vault_k3s_path }} + --vault-password-file {{ hostvars['localhost'].k3s_vault_password_file }} + changed_when: true + delegate_to: localhost + when: + - hostvars['localhost'].vault_k3s_encrypted | default(false) + - (hostvars['localhost'].k3s_vault_password_file | default('')) | length > 0 + + - name: Update vault k3s token + ansible.builtin.lineinfile: + path: "{{ vault_k3s_path }}" + regexp: '^vault_k3s_token:' + line: "vault_k3s_token: \"{{ k3s_token_discovered }}\"" + create: true + mode: "0644" + delegate_to: localhost + + - name: Encrypt vault_k3s.yml + ansible.builtin.command: >- + ansible-vault encrypt {{ vault_k3s_path }} + --vault-password-file {{ hostvars['localhost'].k3s_vault_password_file }} + changed_when: true + delegate_to: localhost + when: + - hostvars['localhost'].vault_k3s_encrypted | default(false) + - (hostvars['localhost'].k3s_vault_password_file | default('')) | length > 0 + run_once: true diff --git a/infrastructure/roles/k3s/tasks/validate_args.yml b/infrastructure/roles/k3s/tasks/validate_args.yml new file mode 100644 index 0000000..dac8bc4 --- /dev/null +++ b/infrastructure/roles/k3s/tasks/validate_args.yml @@ -0,0 +1,26 @@ +--- +# Fail fast on known-bad flags that prevent k3s from starting. + +- name: Validate kube-apiserver args (known unsupported flags) + ansible.builtin.assert: + that: + - _bad_kube_apiserver_args | length == 0 + fail_msg: >- + k3s_kube_apiserver_args contains unsupported flag(s): {{ _bad_kube_apiserver_args | join(', ') }}. + This can prevent k3s from starting (e.g. kube-apiserver exits with "unknown flag"). + vars: + _bad_kube_apiserver_args: >- + {{ + (k3s_kube_apiserver_args | default([])) + | select('search', '^etcd-timeout=') + | list + }} + +- name: Validate datastore configuration (embedded etcd vs external datastore) + ansible.builtin.assert: + that: + - not ((k3s_cluster_init | default(false) | bool) and ((k3s_datastore_endpoint | default('')) | length > 0)) + fail_msg: >- + Invalid k3s datastore config: k3s_cluster_init=true cannot be used with an external datastore. + Set k3s_cluster_init: false (or unset k3s_datastore_endpoint). + when: k3s_role == 'server' diff --git a/infrastructure/roles/k3s/templates/config.yaml.j2 b/infrastructure/roles/k3s/templates/config.yaml.j2 new file mode 100644 index 0000000..08d450c --- /dev/null +++ b/infrastructure/roles/k3s/templates/config.yaml.j2 @@ -0,0 +1,115 @@ +{% set _k3s_server_line_needed = (k3s_role != 'server') or ((k3s_role == 'server') and (not (k3s_cluster_init | bool)) and (k3s_server_url is defined) and k3s_server_url) %} +{% if k3s_role == 'server' and k3s_cluster_init | bool %} +cluster-init: true +{% endif %} +{% if _k3s_server_line_needed %} +server: "{{ k3s_server_url }}" +{% endif %} +{% set _k3s_effective_token = k3s_token %} +{% if k3s_role == 'server' and (k3s_server_token is defined) and k3s_server_token %} +{% set _k3s_effective_token = k3s_server_token %} +{% endif %} +{% if _k3s_effective_token %} +token: "{{ _k3s_effective_token }}" +{% endif %} +{% if k3s_role == 'server' and k3s_datastore_endpoint is defined and k3s_datastore_endpoint | length %} +datastore-endpoint: "{{ k3s_datastore_endpoint }}" +{% if k3s_datastore_cafile is defined and k3s_datastore_cafile | length %} +datastore-cafile: "{{ k3s_datastore_cafile }}" +{% endif %} +{% if k3s_datastore_certfile is defined and k3s_datastore_certfile | length %} +datastore-certfile: "{{ k3s_datastore_certfile }}" +{% endif %} +{% if k3s_datastore_keyfile is defined and k3s_datastore_keyfile | length %} +datastore-keyfile: "{{ k3s_datastore_keyfile }}" +{% endif %} +{% endif %} +{% if k3s_role == 'server' and k3s_write_kubeconfig_mode %} +write-kubeconfig-mode: "{{ k3s_write_kubeconfig_mode }}" +{% endif %} +{% if k3s_role == 'server' and k3s_kubeconfig_group %} +write-kubeconfig-group: "{{ k3s_kubeconfig_group }}" +{% endif %} +{% if k3s_role == 'server' and k3s_tls_sans | length %} +tls-san: +{% for item in k3s_tls_sans %} + - "{{ item }}" +{% endfor %} +{% endif %} +{% if k3s_role == 'server' and k3s_disable | length %} +disable: +{% for item in k3s_disable %} + - "{{ item }}" +{% endfor %} +{% endif %} +{% if k3s_role == 'server' and k3s_disable_agent | bool %} +disable-agent: true +{% endif %} +{% if k3s_data_dir is defined and k3s_data_dir %} +data-dir: "{{ k3s_data_dir }}" +{% endif %} +{% if k3s_service_node_labels | length %} +node-label: +{% for item in k3s_service_node_labels %} + - "{{ item }}" +{% endfor %} +{% endif %} +{% if k3s_node_taints | length %} +node-taint: +{% for item in k3s_node_taints %} + - "{{ item }}" +{% endfor %} +{% endif %} + +{# --- kube-apiserver tuning (request/etcd timeouts) --- #} +{% if k3s_role == 'server' and k3s_kube_apiserver_args is defined and k3s_kube_apiserver_args | length %} +kube-apiserver-arg: +{% for item in k3s_kube_apiserver_args %} + - "{{ item }}" +{% endfor %} +{% endif %} + +{# --- Control-plane leader election tuning (forgiving leases) --- #} +{% if k3s_role == 'server' and k3s_kube_controller_manager_args is defined and k3s_kube_controller_manager_args | length %} +kube-controller-manager-arg: +{% for item in k3s_kube_controller_manager_args %} + - "{{ item }}" +{% endfor %} +{% endif %} + +{% if k3s_role == 'server' and k3s_kube_scheduler_args is defined and k3s_kube_scheduler_args | length %} +kube-scheduler-arg: +{% for item in k3s_kube_scheduler_args %} + - "{{ item }}" +{% endfor %} +{% endif %} + +{% if k3s_role == 'server' and k3s_cloud_controller_manager_args is defined and k3s_cloud_controller_manager_args | length %} +kube-cloud-controller-manager-arg: +{% for item in k3s_cloud_controller_manager_args %} + - "{{ item }}" +{% endfor %} +{% endif %} + +{# --- Embedded etcd tuning (optional) --- #} +{% if k3s_role == 'server' and k3s_etcd_args is defined and k3s_etcd_args | length %} +etcd-arg: +{% for item in k3s_etcd_args %} + - "{{ item }}" +{% endfor %} +{% endif %} + +{# --- kubelet tuning (node housekeeping/heartbeats) --- #} +{% if k3s_kubelet_args is defined and k3s_kubelet_args | length %} +kubelet-arg: +{% for item in k3s_kubelet_args %} + - "{{ item }}" +{% endfor %} +{% endif %} + +{% if k3s_node_ip is defined and k3s_node_ip %} +node-ip: "{{ k3s_node_ip }}" +{% endif %} +{% if k3s_flannel_iface is defined and k3s_flannel_iface %} +flannel-iface: "{{ k3s_flannel_iface }}" +{% endif %} diff --git a/infrastructure/roles/k3s/templates/prole-svc-iptables-allow.service.j2 b/infrastructure/roles/k3s/templates/prole-svc-iptables-allow.service.j2 new file mode 100644 index 0000000..e91eb63 --- /dev/null +++ b/infrastructure/roles/k3s/templates/prole-svc-iptables-allow.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Prole: ensure svc.prole.org ports are ACCEPTed before Tailscale filter rules +Wants=network-online.target +After=network-online.target +After=tailscaled.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/prole-svc-iptables-allow +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/infrastructure/roles/k3s/templates/prole-svc-iptables-allow.sh.j2 b/infrastructure/roles/k3s/templates/prole-svc-iptables-allow.sh.j2 new file mode 100644 index 0000000..b177566 --- /dev/null +++ b/infrastructure/roles/k3s/templates/prole-svc-iptables-allow.sh.j2 @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CHAIN="{{ k3s_firewall_svc_allow_chain | default('PROLE-SVC-ALLOW') }}" +PORTS="{{ (k3s_firewall_svc_allow_tcp_ports | default([80, 443])) | join(' ') }}" + +log() { + echo "[prole-svc-iptables-allow] $*" >&2 +} + +apply_rules() { + local ipt="$1" + + command -v "$ipt" >/dev/null 2>&1 || return 0 + + # Ensure chain exists. + "$ipt" -t filter -N "$CHAIN" 2>/dev/null || true + "$ipt" -t filter -F "$CHAIN" || true + + # Populate chain with port accepts. + for p in $PORTS; do + if [[ ! "$p" =~ ^[0-9]+$ ]]; then + log "Skipping non-numeric port: $p" + continue + fi + "$ipt" -t filter -A "$CHAIN" -p tcp --dport "$p" -j ACCEPT + done + + # Ensure we jump to our chain at the very top of INPUT. + # Delete any existing jumps (wherever they are), then insert at position 1. + while "$ipt" -t filter -D INPUT -j "$CHAIN" 2>/dev/null; do + : + done + "$ipt" -t filter -I INPUT 1 -j "$CHAIN" +} + +apply_rules iptables +apply_rules ip6tables + +log "Applied ACCEPT rules for TCP ports: $PORTS" diff --git a/infrastructure/roles/k3s/templates/registries.yaml.j2 b/infrastructure/roles/k3s/templates/registries.yaml.j2 new file mode 100644 index 0000000..1e2a7df --- /dev/null +++ b/infrastructure/roles/k3s/templates/registries.yaml.j2 @@ -0,0 +1,29 @@ +{% set _scheme = (k3s_registry_endpoint_scheme_resolved | default(k3s_registry_endpoint_scheme | default('http')) | lower) %} +mirrors: + "{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}": + endpoint: + - "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}" + "registry.prole.org": + endpoint: + - "http://myrddin.prole.org:5000" + "registry.prole.org:80": + endpoint: + - "http://myrddin.prole.org:5000" + "registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}": + endpoint: + - "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}" +configs: + "registry.prole.org": + tls: + insecure_skip_verify: true + "registry.prole.org:80": + tls: + insecure_skip_verify: true +{% if _scheme != 'http' %} + "{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}": + tls: + insecure_skip_verify: true + "registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}": + tls: + insecure_skip_verify: true +{% endif %} diff --git a/infrastructure/roles/k3s/templates/systemd-override.conf.j2 b/infrastructure/roles/k3s/templates/systemd-override.conf.j2 new file mode 100644 index 0000000..d244ec1 --- /dev/null +++ b/infrastructure/roles/k3s/templates/systemd-override.conf.j2 @@ -0,0 +1,13 @@ +[Service] +{% if k3s_systemd_timeout_start_sec is defined and (k3s_systemd_timeout_start_sec | string | length) > 0 %} +TimeoutStartSec={{ k3s_systemd_timeout_start_sec }} +{% endif %} +{% if k3s_systemd_timeout_stop_sec is defined and (k3s_systemd_timeout_stop_sec | string | length) > 0 %} +TimeoutStopSec={{ k3s_systemd_timeout_stop_sec }} +{% endif %} +{% if k3s_systemd_restart is defined and (k3s_systemd_restart | string | length) > 0 %} +Restart={{ k3s_systemd_restart }} +{% endif %} +{% if k3s_systemd_restart_sec is defined and (k3s_systemd_restart_sec | string | length) > 0 %} +RestartSec={{ k3s_systemd_restart_sec }} +{% endif %} \ No newline at end of file diff --git a/infrastructure/roles/k3s/tests/no_rancher_sd_guardrail.yml b/infrastructure/roles/k3s/tests/no_rancher_sd_guardrail.yml new file mode 100644 index 0000000..f244577 --- /dev/null +++ b/infrastructure/roles/k3s/tests/no_rancher_sd_guardrail.yml @@ -0,0 +1,26 @@ +--- +- name: Ensure k3s rancher mount guardrail is opt-in and implemented via systemd drop-in + hosts: localhost + connection: local + gather_facts: false + become: false + + tasks: + - name: Read k3s install tasks + ansible.builtin.set_fact: + _k3s_install_tasks: "{{ lookup('ansible.builtin.file', playbook_dir ~ '/../tasks/install.yml') }}" + + - name: Assert rancher mount guardrail exists and is opt-in + ansible.builtin.assert: + that: + - (_k3s_install_tasks is search('Guardrail - enforce dedicated rancher mount')) == true + - (_k3s_install_tasks is search('RequiresMountsFor=')) == true + - (_k3s_install_tasks is search('k3s_rancher_mountpoint')) == true + - (_k3s_install_tasks is search('findmnt -n -o SOURCE -T')) == true + - (_k3s_install_tasks is search('findmnt -n -o SOURCE -T /')) == true + fail_msg: >- + k3s role should implement an opt-in guardrail that forces k3s to require + a dedicated `/var/lib/rancher` mount (via systemd drop-in + preflight), + without impacting nodes that intentionally keep rancher storage on `/`. + +# end \ No newline at end of file diff --git a/infrastructure/roles/k3s/tests/optional_workloads_policy_eval.yml b/infrastructure/roles/k3s/tests/optional_workloads_policy_eval.yml new file mode 100644 index 0000000..8641462 --- /dev/null +++ b/infrastructure/roles/k3s/tests/optional_workloads_policy_eval.yml @@ -0,0 +1,37 @@ +--- + +- name: Unit test - optional workloads policy evaluation + hosts: localhost + gather_facts: false + + vars: + _k3s_nodes_list: + stdout_lines: + - "node-a Ready" + - "node-b Ready,SchedulingDisabled" + - "node-c NotReady" + + tasks: + - name: Evaluate with min_required=2 (should be false) + ansible.builtin.include_tasks: "{{ playbook_dir }}/../tasks/optional_workloads_policy_eval.yml" + vars: + k3s_optional_workloads_min_nodes_cfg: "2" + + - name: Assert optional_workloads_allowed is false when only one schedulable node is Ready + ansible.builtin.assert: + that: + - ready_schedulable_nodes == 1 + - optional_workloads_min_ready_schedulable_nodes == 2 + - not (optional_workloads_allowed | bool) + + - name: Evaluate with min_required=1 (should be true) + ansible.builtin.include_tasks: "{{ playbook_dir }}/../tasks/optional_workloads_policy_eval.yml" + vars: + k3s_optional_workloads_min_nodes_cfg: "1" + + - name: Assert optional_workloads_allowed is true when min_required is satisfied + ansible.builtin.assert: + that: + - ready_schedulable_nodes == 1 + - optional_workloads_min_ready_schedulable_nodes == 1 + - optional_workloads_allowed | bool diff --git a/infrastructure/roles/k3s/tests/optional_workloads_policy_query_nonfatal.yml b/infrastructure/roles/k3s/tests/optional_workloads_policy_query_nonfatal.yml new file mode 100644 index 0000000..c671590 --- /dev/null +++ b/infrastructure/roles/k3s/tests/optional_workloads_policy_query_nonfatal.yml @@ -0,0 +1,21 @@ +--- +- name: Ensure optional-workloads node query does not hard-fail k3s configure + hosts: localhost + connection: local + gather_facts: false + become: false + + tasks: + - name: Read k3s configure tasks + ansible.builtin.set_fact: + _k3s_configure_tasks: "{{ lookup('ansible.builtin.file', playbook_dir ~ '/../tasks/configure.yml') }}" + + - name: Assert optional-workloads node query is non-fatal and logs clearly + ansible.builtin.assert: + that: + - "_k3s_configure_tasks is search('Query cluster nodes for optional-workloads policy')" + - "_k3s_configure_tasks is search('failed_when\\s*:\\s*false')" + - "_k3s_configure_tasks is search('Log optional-workloads policy node query failure')" + fail_msg: >- + k3s configure should not abort when the Kubernetes API is temporarily unavailable; + the optional-workloads node query must be non-fatal and log the reason. diff --git a/infrastructure/roles/k3s/tests/render_config_agent_server_only_keys.yml b/infrastructure/roles/k3s/tests/render_config_agent_server_only_keys.yml new file mode 100644 index 0000000..7d309eb --- /dev/null +++ b/infrastructure/roles/k3s/tests/render_config_agent_server_only_keys.yml @@ -0,0 +1,70 @@ +--- +- name: Ensure server-only config keys are not rendered for k3s agents + hosts: localhost + connection: local + gather_facts: false + become: false + + vars: + k3s_role: agent + k3s_cluster_init: false + k3s_server_url: "https://myrddin.prole.org:6443" + k3s_token: "K10deadbeef::server:dummy" + + # Provide defaults normally coming from the role so this standalone test can render the template. + k3s_disable: [] + k3s_service_node_labels: [] + k3s_node_taints: [] + k3s_data_dir: /var/lib/rancher/k3s + + # Intentionally set server-only knobs; the template must not render them for agents. + k3s_tls_sans: + - merlin.prole.org + k3s_disable_agent: true + k3s_kube_apiserver_args: + - "request-timeout=10s" + k3s_kube_controller_manager_args: + - "leader-elect-lease-duration=150s" + k3s_kube_scheduler_args: + - "leader-elect-lease-duration=150s" + k3s_cloud_controller_manager_args: + - "leader-elect-lease-duration=300s" + k3s_etcd_args: + - "experimental-initial-corrupt-check=true" + + tasks: + - name: Create temp dir + ansible.builtin.tempfile: + state: directory + prefix: k3s-config-test- + register: _tmp + + - name: Render config.yaml + ansible.builtin.template: + src: "{{ playbook_dir }}/../templates/config.yaml.j2" + dest: "{{ _tmp.path }}/config.yaml" + + - name: Read rendered config + ansible.builtin.set_fact: + _rendered: "{{ lookup('ansible.builtin.file', _tmp.path ~ '/config.yaml') }}" + + - name: Assert agent config contains join keys + ansible.builtin.assert: + that: + - _rendered is search('(?m)^server:') + - _rendered is search('(?m)^token:') + fail_msg: >- + Expected agent config to include join parameters (server/token). Rendered:\n{{ _rendered }} + + - name: Assert server-only keys are absent for agents + ansible.builtin.assert: + that: + - (_rendered is search('(?m)^tls-san:')) == false + - (_rendered is search('(?m)^disable-agent:')) == false + - (_rendered is search('(?m)^kube-apiserver-arg:')) == false + - (_rendered is search('(?m)^kube-controller-manager-arg:')) == false + - (_rendered is search('(?m)^kube-scheduler-arg:')) == false + - (_rendered is search('(?m)^kube-cloud-controller-manager-arg:')) == false + - (_rendered is search('(?m)^etcd-arg:')) == false + fail_msg: >- + Server-only keys were rendered for an agent. Rendered:\n{{ _rendered }} diff --git a/infrastructure/roles/k3s/tests/render_config_server_myrddin_timing_args.yml b/infrastructure/roles/k3s/tests/render_config_server_myrddin_timing_args.yml new file mode 100644 index 0000000..ea12a0d --- /dev/null +++ b/infrastructure/roles/k3s/tests/render_config_server_myrddin_timing_args.yml @@ -0,0 +1,63 @@ +--- +- name: Ensure myrddin k3s server timing args are rendered into config.yaml + hosts: localhost + connection: local + gather_facts: false + become: false + + vars: + # Provide defaults normally coming from the role so this standalone test can render the template. + k3s_disable: [] + k3s_disable_agent: false + k3s_service_node_labels: [] + k3s_node_taints: [] + k3s_data_dir: /var/lib/rancher/k3s + + tasks: + - name: Load common k3s group vars + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../../../inventory/group_vars/all/k3s.yml" + + - name: Load myrddin host vars + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../../../inventory/host_vars/myrddin.prole.org.yml" + + - name: Create temp dir + ansible.builtin.tempfile: + state: directory + prefix: k3s-config-test- + register: _tmp + + - name: Render config.yaml + ansible.builtin.template: + src: "{{ playbook_dir }}/../templates/config.yaml.j2" + dest: "{{ _tmp.path }}/config.yaml" + + - name: Read rendered config + ansible.builtin.set_fact: + _rendered: "{{ lookup('ansible.builtin.file', _tmp.path ~ '/config.yaml') }}" + + - name: Assert k3s server timing args are present + ansible.builtin.assert: + that: + - _rendered is search('(?m)^kube-apiserver-arg:$') + - _rendered is search('(?m)^ - "request-timeout=60s"$') + - _rendered is search('(?m)^ - "min-request-timeout=60"$') + - _rendered is search('(?m)^kube-controller-manager-arg:$') + - _rendered is search('(?m)^ - "leader-elect-lease-duration=600s"$') + - _rendered is search('(?m)^ - "leader-elect-renew-deadline=420s"$') + - _rendered is search('(?m)^ - "leader-elect-retry-period=60s"$') + - _rendered is search('(?m)^ - "node-monitor-grace-period=10m"$') + - _rendered is search('(?m)^ - "node-startup-grace-period=10m"$') + - _rendered is search('(?m)^kube-scheduler-arg:$') + - _rendered is search('(?m)^ - "leader-elect-lease-duration=600s"$') + - _rendered is search('(?m)^ - "leader-elect-renew-deadline=420s"$') + - _rendered is search('(?m)^ - "leader-elect-retry-period=60s"$') + - _rendered is search('(?m)^kube-cloud-controller-manager-arg:$') + - _rendered is search('(?m)^ - "leader-elect-lease-duration=900s"$') + - _rendered is search('(?m)^ - "leader-elect-renew-deadline=600s"$') + - _rendered is search('(?m)^ - "leader-elect-retry-period=90s"$') + - _rendered is search('(?m)^kubelet-arg:$') + - _rendered is search('(?m)^ - "housekeeping-interval=30s"$') + fail_msg: >- + Expected myrddin timing args to be present in rendered config. Rendered:\n{{ _rendered }} diff --git a/infrastructure/roles/local_user/defaults/main.yml b/infrastructure/roles/local_user/defaults/main.yml new file mode 100644 index 0000000..425da2e --- /dev/null +++ b/infrastructure/roles/local_user/defaults/main.yml @@ -0,0 +1,14 @@ +--- +local_user_required_packages: + - vim + - screen + - wget + +local_user_k3s_packages: + - kubecolor + +local_user_screenrc_targets: + - path: /home/pi/.screenrc + owner: pi + group: pi + mode: '0644' \ No newline at end of file diff --git a/infrastructure/roles/local_user/files/screenrc b/infrastructure/roles/local_user/files/screenrc new file mode 100644 index 0000000..11d98f3 --- /dev/null +++ b/infrastructure/roles/local_user/files/screenrc @@ -0,0 +1,47 @@ +defflow auto +defscrollback 5000 +altscreen on +autodetach on +msgwait 2 # 1 second messages +#change the hardstatus settings to +#give an window list at the bottom of the +#screen, with the time and date and with the current window highlighted +# term screen-256color + +# Unbreak scrollback in xterm-like things +termcapinfo xterm*|rxvt*|kterm*|Eterm* ti@:te@ + +defutf8 on +hardstatus alwayslastline +hardstatus string '%{gk}[ %{G}%H %{g}][%= %{wk}%?%-Lw%?%{=b kR}(%{W}%n*%f %t%?(%u)%?%{=b kR})%{= kw}%?%+Lw%?%?%= %{g}] %{=b C}[ %m/%d %c ]%{W}' + +vbell_msg "[[[ ding ]]]" +vbell off +startup_message off + +# remove some stupid / dangerous key bindings +bind k +bind W +bind ^k +bind . +bind ^\ +bind \\ +bind ^h +bind h +#make them better +bind 'K' kill +bind 'W' windowlist +#f1 and f2, forward and back +bindkey -k k1 prev +bindkey -k k2 next + +screen 0 +screen 1 +screen 2 +screen 3 +screen 4 +screen 5 +screen 6 +screen 7 +screen 8 +screen 9 \ No newline at end of file diff --git a/infrastructure/roles/local_user/tasks/main.yml b/infrastructure/roles/local_user/tasks/main.yml new file mode 100644 index 0000000..6ad11d9 --- /dev/null +++ b/infrastructure/roles/local_user/tasks/main.yml @@ -0,0 +1,45 @@ +--- +- name: Install baseline Linux packages + ansible.builtin.package: + name: "{{ local_user_required_packages }}" + state: present + +- name: Install k3s-specific packages + ansible.builtin.package: + name: "{{ local_user_k3s_packages }}" + state: present + when: + - local_user_k3s_packages | length > 0 + - (k3s_host | default(false) | bool) or ('k3s_hosts' in group_names) + +- name: Ensure kubeadm group exists + ansible.builtin.group: + name: "{{ k3s_kubeconfig_group | default('kubeadm') }}" + state: present + +- name: Ensure local user pi exists + ansible.builtin.user: + name: pi + state: present + shell: /bin/bash + home: /home/pi + create_home: yes + groups: "{{ k3s_kubeconfig_group | default('kubeadm') }}" + append: true + +- name: Ensure /home/pi directory exists and has correct ownership + ansible.builtin.file: + path: /home/pi + state: directory + owner: pi + group: pi + mode: '0755' + +- name: Install default screen configuration + ansible.builtin.copy: + src: screenrc + dest: "{{ item.path }}" + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default(item.owner | default('root')) }}" + mode: "{{ item.mode | default('0644') }}" + loop: "{{ local_user_screenrc_targets }}" diff --git a/infrastructure/roles/mariadb_primary/tasks/main.yml b/infrastructure/roles/mariadb_primary/tasks/main.yml new file mode 100644 index 0000000..47846d8 --- /dev/null +++ b/infrastructure/roles/mariadb_primary/tasks/main.yml @@ -0,0 +1,298 @@ +--- + +- name: Install MariaDB and tooling + ansible.builtin.package: + name: + - mariadb-server + - mariadb-client + - rsync + state: present + +- name: Temporary external storage for MariaDB (merlin) + block: + - name: Ensure exFAT support packages are installed (Debian/Ubuntu) + block: + - name: Install exfatprogs + ansible.builtin.package: + name: exfatprogs + state: present + rescue: + - name: Install legacy exFAT packages + ansible.builtin.package: + name: + - exfat-fuse + - exfat-utils + state: present + when: ansible_facts['os_family'] | default('') == 'Debian' + + - name: Ensure external mountpoint directory exists + ansible.builtin.file: + path: "{{ mariadb_external_mountpoint | default('/external') }}" + state: directory + + - name: Ensure external partition device exists + ansible.builtin.stat: + path: "{{ mariadb_external_device | default('/dev/sda1') }}" + register: _mariadb_external_device_stat + + - name: Fail if external partition device is missing + ansible.builtin.assert: + that: + - _mariadb_external_device_stat.stat.exists + fail_msg: "External device {{ mariadb_external_device | default('/dev/sda1') }} does not exist on {{ inventory_hostname }}." + + - name: Discover filesystem UUID for external partition + ansible.builtin.command: >- + blkid -s UUID -o value {{ mariadb_external_device | default('/dev/sda1') }} + register: _mariadb_external_uuid + changed_when: false + failed_when: _mariadb_external_uuid.rc != 0 or (_mariadb_external_uuid.stdout | trim | length) == 0 + + - name: Mount external filesystem at /external (persistent) + ansible.posix.mount: + src: "UUID={{ _mariadb_external_uuid.stdout | trim }}" + path: "{{ mariadb_external_mountpoint | default('/external') }}" + fstype: "{{ mariadb_external_fstype | default('exfat') }}" + opts: "{{ mariadb_external_mount_opts | default('defaults,nofail') }}" + state: mounted + + - name: Fail if /external is not mounted + ansible.builtin.command: "mountpoint -q {{ mariadb_external_mountpoint | default('/external') }}" + changed_when: false + when: mariadb_external_enabled | default(false) | bool + +- name: Ensure USB mountpoint directory exists + ansible.builtin.file: + path: "{{ mariadb_usb_mountpoint | default('/srv/mariadb') }}" + state: directory + owner: root + group: root + mode: "0755" + when: not (mariadb_external_enabled | default(false) | bool) + +- name: Mount MariaDB USB filesystem + ansible.builtin.mount: + src: "LABEL={{ mariadb_usb_label | default('MARIADB') }}" + path: "{{ mariadb_usb_mountpoint | default('/srv/mariadb') }}" + fstype: "{{ mariadb_usb_fstype | default('ext4') }}" + opts: "{{ mariadb_usb_mount_opts | default('noatime,nofail') }}" + state: mounted + when: not (mariadb_external_enabled | default(false) | bool) + +- name: Stop MariaDB before datadir changes + ansible.builtin.service: + name: mariadb + state: stopped + failed_when: false + +- name: Migrate MariaDB datadir to /external (merlin temporary setup) + vars: + mariadb_external_src: "{{ mariadb_external_src_datadir | default('/srv/mariadb/mariadb') }}" + mariadb_external_dst: "{{ mariadb_external_dst_datadir | default('/external/mariadb') }}" + mariadb_external_backup: "{{ mariadb_external_backup_datadir | default('/srv/mariadb/mariadb.pre-external') }}" + mariadb_external_marker: "{{ mariadb_external_migration_marker | default('/external/mariadb/.prole-mariadb-external-migrated') }}" + block: + - name: Fail if external mount is not present + ansible.builtin.command: "mountpoint -q {{ mariadb_external_mountpoint | default('/external') }}" + changed_when: false + + - name: Ensure destination directory exists on external disk + ansible.builtin.file: + path: "{{ mariadb_external_dst }}" + state: directory + + - name: Check migration marker + ansible.builtin.stat: + path: "{{ mariadb_external_marker }}" + register: _mariadb_external_marker_stat + + - name: Fail if source datadir does not exist + ansible.builtin.stat: + path: "{{ mariadb_external_src }}" + register: _mariadb_external_src_stat + + - name: Assert source datadir exists + ansible.builtin.assert: + that: + - _mariadb_external_src_stat.stat.exists + - _mariadb_external_src_stat.stat.isdir + fail_msg: "Source MariaDB datadir {{ mariadb_external_src }} is missing or not a directory on {{ inventory_hostname }}." + + - name: Fail clearly if rsync is not available + ansible.builtin.command: rsync --version + register: _mariadb_external_rsync_version + changed_when: false + + - name: Check whether source datadir is already a mountpoint (bind-mounted) + ansible.builtin.command: "mountpoint -q {{ mariadb_external_src }}" + register: _mariadb_external_src_is_mountpoint + changed_when: false + failed_when: false + + - name: Rsync MariaDB data to external disk (copy; non-destructive) + ansible.builtin.command: >- + rsync -aH --numeric-ids {{ mariadb_external_src }}/ {{ mariadb_external_dst }}/ + register: _mariadb_external_rsync + changed_when: true + when: not _mariadb_external_marker_stat.stat.exists + + - name: Write migration marker + ansible.builtin.copy: + dest: "{{ mariadb_external_marker }}" + content: | + migrated_from={{ mariadb_external_src }} + migrated_to={{ mariadb_external_dst }} + host={{ inventory_hostname }} + when: not _mariadb_external_marker_stat.stat.exists + + - name: Check whether backup datadir already exists + ansible.builtin.stat: + path: "{{ mariadb_external_backup }}" + register: _mariadb_external_backup_stat + + - name: Move original datadir aside (kept for rollback) + ansible.builtin.command: >- + mv {{ mariadb_external_src }} {{ mariadb_external_backup }} + when: + - _mariadb_external_marker_stat.stat.exists or (not _mariadb_external_marker_stat.stat.exists and (_mariadb_external_rsync is defined)) + - _mariadb_external_src_is_mountpoint.rc != 0 + - not _mariadb_external_backup_stat.stat.exists + + - name: Re-create source datadir path as a mountpoint directory + ansible.builtin.file: + path: "{{ mariadb_external_src }}" + state: directory + when: _mariadb_external_src_is_mountpoint.rc != 0 + + - name: Bind-mount external MariaDB directory to the expected datadir path + ansible.posix.mount: + src: "{{ mariadb_external_dst }}" + path: "{{ mariadb_external_src }}" + fstype: none + opts: bind + state: mounted + when: _mariadb_external_src_is_mountpoint.rc != 0 + + - name: Fail if bind mount is not active + ansible.builtin.command: "mountpoint -q {{ mariadb_external_src }}" + changed_when: false + when: mariadb_external_enabled | default(false) | bool + +- name: Ensure MariaDB datadir exists on USB + ansible.builtin.file: + path: "{{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}" + state: directory + owner: mysql + group: mysql + mode: "0700" + when: not (mariadb_external_enabled | default(false) | bool) + +- name: Allow MariaDB datadir in AppArmor (Ubuntu/Debian) + ansible.builtin.copy: + dest: /etc/apparmor.d/local/usr.sbin.mysqld + owner: root + group: root + mode: "0644" + content: | + # Managed by Ansible (mariadb_primary) + {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}/ r, + {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}/** rwk, + register: _mariadb_apparmor_local + when: ansible_facts['os_family'] | default('') == 'Debian' + +- name: Reload AppArmor profile for mysqld + ansible.builtin.command: apparmor_parser -r /etc/apparmor.d/usr.sbin.mysqld + changed_when: false + when: + - ansible_facts['os_family'] | default('') == 'Debian' + - _mariadb_apparmor_local is changed + failed_when: false + +- name: Configure MariaDB bind-address + ansible.builtin.lineinfile: + path: /etc/mysql/mariadb.conf.d/50-server.cnf + regexp: '^bind-address\s*=' + line: "bind-address = {{ mariadb_bind_address | default('0.0.0.0') }}" + insertafter: '^\[mysqld\]' + register: _mariadb_cfg_bind + +- name: Configure MariaDB datadir + ansible.builtin.lineinfile: + path: /etc/mysql/mariadb.conf.d/50-server.cnf + regexp: '^datadir\s*=' + line: "datadir = {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}" + insertafter: '^\[mysqld\]' + register: _mariadb_cfg_datadir + +- name: Check if source MariaDB system tables exist + ansible.builtin.stat: + path: /var/lib/mysql/mysql + register: _mariadb_src_tables + +- name: Check if destination MariaDB system tables exist + ansible.builtin.stat: + path: "{{ (mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb')) ~ '/mysql' }}" + register: _mariadb_dst_tables + +- name: Copy initial MariaDB data to USB datadir (first migration only) + ansible.builtin.command: >- + rsync -aHAX --numeric-ids /var/lib/mysql/ {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}/ + when: + - _mariadb_src_tables.stat.exists + - not _mariadb_dst_tables.stat.exists + - not (mariadb_external_enabled | default(false) | bool) + register: _mariadb_rsync + changed_when: true + +- name: Ensure MariaDB datadir ownership after migration + ansible.builtin.command: >- + chown -R mysql:mysql {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }} + when: + - _mariadb_rsync is defined + - _mariadb_rsync is changed + - not (mariadb_external_enabled | default(false) | bool) + changed_when: true + +- name: Ensure MariaDB service is started and enabled + ansible.builtin.service: + name: mariadb + state: started + enabled: true + +- name: Require MariaDB admin credentials + ansible.builtin.assert: + that: + - k3s_mariadb_admin_user | default('') | length > 0 + - k3s_mariadb_admin_password | default('') | length > 0 + fail_msg: "Set k3s_mariadb_admin_user and vaulted k3s_mariadb_admin_password for MariaDB provisioning." + when: not ansible_check_mode + +- name: Create/ensure MariaDB admin user for remote TCP management + ansible.builtin.command: >- + mysql --protocol=socket --user=root + --execute=" + CREATE USER IF NOT EXISTS '{{ k3s_mariadb_admin_user }}'@'%' IDENTIFIED BY '{{ k3s_mariadb_admin_password }}'; + ALTER USER '{{ k3s_mariadb_admin_user }}'@'%' IDENTIFIED BY '{{ k3s_mariadb_admin_password }}'; + GRANT ALL PRIVILEGES ON *.* TO '{{ k3s_mariadb_admin_user }}'@'%' WITH GRANT OPTION; + FLUSH PRIVILEGES;" + when: not ansible_check_mode + no_log: true + +- name: Require k3s MariaDB user password (vaulted) + ansible.builtin.assert: + that: + - k3s_datastore_mariadb_password | default('') | length > 0 + fail_msg: "k3s_datastore_mariadb_password is empty; set it from Ansible Vault." + when: not ansible_check_mode + +- name: Create k3s database/user/grants (local socket) + ansible.builtin.command: >- + mysql --protocol=socket --user=root + --execute=" + CREATE DATABASE IF NOT EXISTS `{{ k3s_datastore_mariadb_db | default('k3s') }}`; + CREATE USER IF NOT EXISTS '{{ k3s_datastore_mariadb_user | default('prole_k3s') }}'@'%' IDENTIFIED BY '{{ k3s_datastore_mariadb_password }}'; + ALTER USER '{{ k3s_datastore_mariadb_user | default('prole_k3s') }}'@'%' IDENTIFIED BY '{{ k3s_datastore_mariadb_password }}'; + GRANT ALL PRIVILEGES ON `{{ k3s_datastore_mariadb_db | default('k3s') }}`.* TO '{{ k3s_datastore_mariadb_user | default('prole_k3s') }}'@'%'; + FLUSH PRIVILEGES;" + when: not ansible_check_mode + no_log: true diff --git a/infrastructure/roles/mariadb_primary/tests/vars.yml b/infrastructure/roles/mariadb_primary/tests/vars.yml new file mode 100644 index 0000000..7ef2940 --- /dev/null +++ b/infrastructure/roles/mariadb_primary/tests/vars.yml @@ -0,0 +1,20 @@ +--- +- name: Verify mariadb_primary inventory variables resolve (including vault) + hosts: mariadb_primary + connection: local + gather_facts: false + become: false + + tasks: + - name: Require MariaDB admin credentials + ansible.builtin.assert: + that: + - k3s_mariadb_admin_user | default('') | length > 0 + - k3s_mariadb_admin_password | default('') | length > 0 + fail_msg: "Expected k3s_mariadb_admin_user and vaulted k3s_mariadb_admin_password to be defined." + + - name: Require k3s datastore MariaDB password + ansible.builtin.assert: + that: + - k3s_datastore_mariadb_password | default('') | length > 0 + fail_msg: "Expected vaulted k3s_datastore_mariadb_password to be defined (directly or derived from a vaulted var)." \ No newline at end of file diff --git a/infrastructure/roles/mariadb_replica/tasks/main.yml b/infrastructure/roles/mariadb_replica/tasks/main.yml new file mode 100644 index 0000000..9c123bd --- /dev/null +++ b/infrastructure/roles/mariadb_replica/tasks/main.yml @@ -0,0 +1,84 @@ +--- + +- name: Skip MariaDB replica provisioning when disabled + ansible.builtin.meta: end_host + when: not (mariadb_replica_enabled | default(false) | bool) + +- name: Install MariaDB and tooling + ansible.builtin.package: + name: + - mariadb-server + - mariadb-client + - rsync + state: present + +- name: Ensure USB mountpoint directory exists + ansible.builtin.file: + path: "{{ mariadb_usb_mountpoint | default('/srv/mariadb') }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Mount MariaDB USB filesystem + ansible.builtin.mount: + src: "LABEL={{ mariadb_usb_label | default('MARIADB') }}" + path: "{{ mariadb_usb_mountpoint | default('/srv/mariadb') }}" + fstype: "{{ mariadb_usb_fstype | default('ext4') }}" + opts: "{{ mariadb_usb_mount_opts | default('noatime,nofail') }}" + state: mounted + +- name: Stop MariaDB before datadir changes + ansible.builtin.service: + name: mariadb + state: stopped + failed_when: false + +- name: Ensure MariaDB datadir exists on USB + ansible.builtin.file: + path: "{{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}" + state: directory + owner: mysql + group: mysql + mode: "0700" + +- name: Allow MariaDB datadir in AppArmor (Ubuntu/Debian) + ansible.builtin.copy: + dest: /etc/apparmor.d/local/usr.sbin.mysqld + owner: root + group: root + mode: "0644" + content: | + # Managed by Ansible (mariadb_replica) + {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}/ r, + {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}/** rwk, + register: _mariadb_apparmor_local + when: ansible_facts['os_family'] | default('') == 'Debian' + +- name: Reload AppArmor profile for mysqld + ansible.builtin.command: apparmor_parser -r /etc/apparmor.d/usr.sbin.mysqld + changed_when: false + when: + - ansible_facts['os_family'] | default('') == 'Debian' + - _mariadb_apparmor_local is changed + failed_when: false + +- name: Configure MariaDB bind-address + ansible.builtin.lineinfile: + path: /etc/mysql/mariadb.conf.d/50-server.cnf + regexp: '^bind-address\s*=' + line: "bind-address = {{ mariadb_bind_address | default('0.0.0.0') }}" + insertafter: '^\[mysqld\]' + +- name: Configure MariaDB datadir + ansible.builtin.lineinfile: + path: /etc/mysql/mariadb.conf.d/50-server.cnf + regexp: '^datadir\s*=' + line: "datadir = {{ mariadb_datadir | default((mariadb_usb_mountpoint | default('/srv/mariadb')) ~ '/mariadb') }}" + insertafter: '^\[mysqld\]' + +- name: Ensure MariaDB service is started and enabled + ansible.builtin.service: + name: mariadb + state: started + enabled: true diff --git a/infrastructure/roles/mariadb_tools/defaults/main.yml b/infrastructure/roles/mariadb_tools/defaults/main.yml new file mode 100644 index 0000000..e274ae2 --- /dev/null +++ b/infrastructure/roles/mariadb_tools/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# Predictable dump directory for MariaDB exports created by this role. +mariadb_dump_dir: /var/backups/mariadb/k3s-datastore diff --git a/infrastructure/roles/mariadb_tools/tasks/k3s_datastore_export.yml b/infrastructure/roles/mariadb_tools/tasks/k3s_datastore_export.yml new file mode 100644 index 0000000..327c60d --- /dev/null +++ b/infrastructure/roles/mariadb_tools/tasks/k3s_datastore_export.yml @@ -0,0 +1,72 @@ +--- +- name: Require k3s datastore MariaDB connection variables + ansible.builtin.assert: + that: + - (k3s_datastore_mariadb_host | default('')) | length > 0 + - (k3s_datastore_mariadb_port | default('') | string) | length > 0 + - (k3s_datastore_mariadb_db | default('')) | length > 0 + - (k3s_datastore_mariadb_user | default('')) | length > 0 + - (k3s_datastore_mariadb_password | default('')) | length > 0 + fail_msg: >- + Missing one or more required vars: k3s_datastore_mariadb_host/port/db/user/password. + Ensure the k3s datastore credentials are sourced from Ansible Vault. + when: not ansible_check_mode + +- name: Ensure dump directory exists + ansible.builtin.file: + path: "{{ mariadb_dump_dir }}" + state: directory + owner: root + group: root + mode: "0700" + +- name: Verify mysqldump is available + ansible.builtin.command: mysqldump --version + changed_when: false + +- name: Get timestamp for dump filename + ansible.builtin.command: date +%Y%m%d-%H%M%S + register: _mariadb_dump_ts + changed_when: false + +- name: Set dumpfile path + ansible.builtin.set_fact: + mariadb_dumpfile: >- + {{ mariadb_dump_dir }}/k3s-datastore-{{ k3s_datastore_mariadb_db }}-{{ _mariadb_dump_ts.stdout }}.sql + +- name: Export k3s datastore database with mysqldump + ansible.builtin.command: >- + mysqldump + --protocol=tcp + --host={{ k3s_datastore_mariadb_host }} + --port={{ k3s_datastore_mariadb_port }} + --user={{ k3s_datastore_mariadb_user }} + --default-character-set=utf8mb4 + --single-transaction + --quick + --routines + --events + --triggers + --skip-lock-tables + --set-gtid-purged=OFF + --result-file={{ mariadb_dumpfile }} + {{ k3s_datastore_mariadb_db }} + environment: + MYSQL_PWD: "{{ k3s_datastore_mariadb_password }}" + changed_when: true + no_log: true + when: not ansible_check_mode + +- name: Tighten dumpfile permissions + ansible.builtin.file: + path: "{{ mariadb_dumpfile }}" + owner: root + group: root + mode: "0600" + when: + - mariadb_dumpfile is defined + - not ansible_check_mode + +- name: Print resulting dumpfile path + ansible.builtin.debug: + msg: "Dump created: {{ mariadb_dumpfile }}" diff --git a/infrastructure/roles/mariadb_tools/tasks/k3s_datastore_import.yml b/infrastructure/roles/mariadb_tools/tasks/k3s_datastore_import.yml new file mode 100644 index 0000000..436d957 --- /dev/null +++ b/infrastructure/roles/mariadb_tools/tasks/k3s_datastore_import.yml @@ -0,0 +1,72 @@ +--- +- name: Require mariadb_import_dumpfile + ansible.builtin.assert: + that: + - mariadb_import_dumpfile is defined + - (mariadb_import_dumpfile | string) | length > 0 + fail_msg: "Set -e mariadb_import_dumpfile=/path/to/dump.sql (path must exist on the target host)." + +- name: Require k3s datastore database name + ansible.builtin.assert: + that: + - (k3s_datastore_mariadb_db | default('')) | length > 0 + fail_msg: "k3s_datastore_mariadb_db is empty." + +- name: Check dumpfile exists on target host + ansible.builtin.stat: + path: "{{ mariadb_import_dumpfile }}" + register: _mariadb_import_dump_stat + +- name: Fail when dumpfile is missing + ansible.builtin.fail: + msg: "Dumpfile does not exist on {{ inventory_hostname }}: {{ mariadb_import_dumpfile }}" + when: not _mariadb_import_dump_stat.stat.exists + +- name: Ensure MariaDB service is running + ansible.builtin.service: + name: mariadb + state: started + +- name: Ensure target k3s datastore database exists + ansible.builtin.command: >- + mysql --protocol=socket --user=root + --execute="CREATE DATABASE IF NOT EXISTS `{{ k3s_datastore_mariadb_db }}`" + changed_when: false + +- name: Refuse dumpfiles that appear to reference other databases + ansible.builtin.shell: | + set -euo pipefail + python3 - "{{ mariadb_import_dumpfile }}" "{{ k3s_datastore_mariadb_db }}" <<'PY' + import re + import sys + + dump = sys.argv[1] + target_db = sys.argv[2] + bad = set() + + with open(dump, 'r', encoding='utf-8', errors='ignore') as f: + for ln in f: + if ln.startswith('CREATE DATABASE'): + bad.add('CREATE DATABASE') + if ln.startswith('USE '): + m = re.match(r'^USE\s+`?([^` ;]+)`?;?', ln.strip()) + if m and m.group(1) != target_db: + bad.add(m.group(1)) + + if bad: + print('Dumpfile references non-target DB(s) or contains CREATE DATABASE:', ','.join(sorted(bad))) + sys.exit(2) + + print('Dumpfile scope OK') + PY + args: + executable: /bin/bash + changed_when: false + +- name: Import dumpfile into k3s datastore database (deliberate operator action) + ansible.builtin.shell: >- + set -euo pipefail; + mysql --protocol=socket --user=root "{{ k3s_datastore_mariadb_db }}" < "{{ mariadb_import_dumpfile }}" + args: + executable: /bin/bash + changed_when: true diff --git a/infrastructure/roles/mariadb_tools/tests/sanity.yml b/infrastructure/roles/mariadb_tools/tests/sanity.yml new file mode 100644 index 0000000..8d75412 --- /dev/null +++ b/infrastructure/roles/mariadb_tools/tests/sanity.yml @@ -0,0 +1,30 @@ +--- +- name: Sanity check mariadb_tools task files + hosts: localhost + connection: local + gather_facts: false + become: false + + tasks: + - name: Read export task file + ansible.builtin.set_fact: + _export_tasks: "{{ lookup('ansible.builtin.file', playbook_dir ~ '/../tasks/k3s_datastore_export.yml') }}" + + - name: Assert export task file contains expected primitives + ansible.builtin.assert: + that: + - _export_tasks is search('mysqldump') + - _export_tasks is search('--single-transaction') + - _export_tasks is search('mariadb_dumpfile') + fail_msg: "Export task file missing expected mysqldump/flags/output handling." + + - name: Read import task file + ansible.builtin.set_fact: + _import_tasks: "{{ lookup('ansible.builtin.file', playbook_dir ~ '/../tasks/k3s_datastore_import.yml') }}" + + - name: Assert import task requires mariadb_import_dumpfile + ansible.builtin.assert: + that: + - _import_tasks is search('mariadb_import_dumpfile') + - _import_tasks is search('mysql --protocol=socket') + fail_msg: "Import task file missing required var checks and/or socket import path." diff --git a/infrastructure/roles/netplan_static/tasks/main.yml b/infrastructure/roles/netplan_static/tasks/main.yml new file mode 100644 index 0000000..01243af --- /dev/null +++ b/infrastructure/roles/netplan_static/tasks/main.yml @@ -0,0 +1,86 @@ +--- + +- name: Skip static IP configuration when disabled + ansible.builtin.meta: end_host + when: not (netplan_static_enabled | default(false) | bool) + +- name: Require netplan_static_address + ansible.builtin.assert: + that: + - netplan_static_address is defined + - netplan_static_address | length > 0 + fail_msg: "netplan_static_address is required when netplan_static_enabled=true (example: 10.0.0.6/24)." + +- name: Detect default interface + ansible.builtin.shell: "ip route show default | awk '/^default/ {print $5; exit}'" + register: _netplan_default_iface + changed_when: false + when: (netplan_static_iface | default('') | length) == 0 + +- name: Set effective interface + ansible.builtin.set_fact: + netplan_static_iface_effective: >- + {{ (netplan_static_iface | default('')) + if (netplan_static_iface | default('') | length) > 0 + else (_netplan_default_iface.stdout | default('') | trim) }} + +- name: Require detected interface + ansible.builtin.assert: + that: + - netplan_static_iface_effective | length > 0 + fail_msg: "Unable to detect default interface for static IP. Set netplan_static_iface explicitly (e.g., eth0)." + +- name: Detect default gateway + ansible.builtin.shell: "ip route show default | awk '/^default/ {print $3; exit}'" + register: _netplan_default_gw + changed_when: false + when: (netplan_static_gateway4 | default('') | length) == 0 + +- name: Set effective gateway + ansible.builtin.set_fact: + netplan_static_gateway4_effective: >- + {{ (netplan_static_gateway4 | default('')) + if (netplan_static_gateway4 | default('') | length) > 0 + else (_netplan_default_gw.stdout | default('') | trim) }} + +- name: Detect current DNS servers + ansible.builtin.shell: "awk '/^nameserver[[:space:]]+/ {print $2}' /etc/resolv.conf | head -n 5" + register: _netplan_dns_detect + changed_when: false + when: (netplan_static_nameservers | default([])) | length == 0 + +- name: Set effective DNS servers + ansible.builtin.set_fact: + netplan_static_nameservers_effective: >- + {{ (netplan_static_nameservers | default([])) + if ((netplan_static_nameservers | default([])) | length) > 0 + else (_netplan_dns_detect.stdout_lines | default([])) }} + +- name: Write netplan static config (NetworkManager) + ansible.builtin.template: + src: 99-ansible-static.yaml.j2 + dest: /etc/netplan/99-ansible-static.yaml + owner: root + group: root + # Netplan warns (and some versions error) if configuration is readable by non-root. + mode: "0600" + register: _netplan_cfg + +- name: Validate netplan config + ansible.builtin.command: netplan generate + changed_when: false + when: _netplan_cfg is changed + +- name: Apply netplan config (async; may disrupt SSH) + block: + - name: Apply netplan + ansible.builtin.command: netplan apply + async: 60 + poll: 0 + changed_when: false + + - name: Wait for host to come back after netplan apply + ansible.builtin.wait_for_connection: + delay: 3 + timeout: 180 + when: _netplan_cfg is changed diff --git a/infrastructure/roles/netplan_static/templates/99-ansible-static.yaml.j2 b/infrastructure/roles/netplan_static/templates/99-ansible-static.yaml.j2 new file mode 100644 index 0000000..2d4f7e2 --- /dev/null +++ b/infrastructure/roles/netplan_static/templates/99-ansible-static.yaml.j2 @@ -0,0 +1,19 @@ +network: + version: 2 + renderer: NetworkManager + ethernets: + {{ netplan_static_iface_effective }}: + dhcp4: false + dhcp6: false + addresses: + - {{ netplan_static_address }} +{% if (netplan_static_gateway4_effective | default('') | length) > 0 %} + gateway4: {{ netplan_static_gateway4_effective }} +{% endif %} +{% if (netplan_static_nameservers_effective | default([])) | length > 0 %} + nameservers: + addresses: +{% for ns in (netplan_static_nameservers_effective | map('trim') | reject('equalto','') | unique | sort) %} + - {{ ns }} +{% endfor %} +{% endif %} diff --git a/infrastructure/roles/netplan_static/tests/render_template.yml b/infrastructure/roles/netplan_static/tests/render_template.yml new file mode 100644 index 0000000..92ee7d5 --- /dev/null +++ b/infrastructure/roles/netplan_static/tests/render_template.yml @@ -0,0 +1,29 @@ +--- +- name: Render netplan_static template and verify YAML is valid + hosts: localhost + connection: local + gather_facts: false + become: false + + vars: + # Representative values (mirrors `inventory/host_vars/merlin.prole.org.yml`) + netplan_static_iface_effective: eth0 + netplan_static_address: 10.0.0.6/24 + netplan_static_gateway4_effective: 10.0.0.1 + netplan_static_nameservers_effective: + - 10.0.0.1 + + _rendered_path: /tmp/netplan_static_rendered.yaml + + tasks: + - name: Render template + ansible.builtin.template: + src: "{{ playbook_dir }}/../templates/99-ansible-static.yaml.j2" + dest: "{{ _rendered_path }}" + mode: "0600" + + - name: Parse rendered YAML with the same Python used by ansible-playbook + ansible.builtin.command: >- + {{ ansible_playbook_python }} -c + "import yaml; yaml.safe_load(open('{{ _rendered_path }}', 'r', encoding='utf-8').read()); print('YAML ok')" + changed_when: false diff --git a/infrastructure/roles/noswap/defaults/main.yml b/infrastructure/roles/noswap/defaults/main.yml new file mode 100644 index 0000000..c1c84bf --- /dev/null +++ b/infrastructure/roles/noswap/defaults/main.yml @@ -0,0 +1,2 @@ +--- +swap_enabled: false diff --git a/infrastructure/roles/noswap/handlers/main.yml b/infrastructure/roles/noswap/handlers/main.yml new file mode 100644 index 0000000..9314521 --- /dev/null +++ b/infrastructure/roles/noswap/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: reload systemd + ansible.builtin.command: systemctl daemon-reload + changed_when: false diff --git a/infrastructure/roles/noswap/tasks/main.yml b/infrastructure/roles/noswap/tasks/main.yml new file mode 100644 index 0000000..8299c67 --- /dev/null +++ b/infrastructure/roles/noswap/tasks/main.yml @@ -0,0 +1,108 @@ +--- +- name: Normalize swap_enabled fact + ansible.builtin.set_fact: + swap_enabled: "{{ swap_enabled | default(false) | bool }}" + +# --- DISABLING SWAP --- + +- name: Turn off all swap immediately + ansible.builtin.command: swapoff -a + changed_when: false + failed_when: false + when: not swap_enabled + +- name: Remove swap entries from /etc/fstab + ansible.builtin.lineinfile: + path: /etc/fstab + state: absent + regexp: '^\s*[^#].*\s+swap\s+' + notify: reload systemd + when: not swap_enabled + +- name: Check if dphys-swapfile is installed + ansible.builtin.command: dpkg -s dphys-swapfile + register: dphys_pkg + changed_when: false + failed_when: false + when: not swap_enabled + +- name: Stop and disable dphys-swapfile service if present + ansible.builtin.systemd: + name: dphys-swapfile + state: stopped + enabled: false + when: not swap_enabled and dphys_pkg.rc == 0 + failed_when: false + +- name: Uninstall dphys-swapfile if present + ansible.builtin.apt: + name: dphys-swapfile + state: absent + purge: true + update_cache: false + when: not swap_enabled and dphys_pkg.rc == 0 + +- name: Check if zram-tools is installed + ansible.builtin.command: dpkg -s zram-tools + register: zram_pkg + changed_when: false + failed_when: false + when: not swap_enabled + +- name: Stop and disable zramswap service if present + ansible.builtin.systemd: + name: zramswap + state: stopped + enabled: false + when: not swap_enabled and zram_pkg.rc == 0 + failed_when: false + +- name: Set vm.swappiness to 0 + ansible.builtin.sysctl: + name: vm.swappiness + value: "0" + state: present + reload: true + when: not swap_enabled + +# --- ENABLING SWAP --- + +- name: Install dphys-swapfile for Raspberry Pi OS + ansible.builtin.apt: + name: dphys-swapfile + state: present + update_cache: true + when: swap_enabled + +- name: Ensure dphys-swapfile is enabled and started + ansible.builtin.systemd: + name: dphys-swapfile + state: started + enabled: true + when: swap_enabled + +- name: Set vm.swappiness to 10 (default) + ansible.builtin.sysctl: + name: vm.swappiness + value: "10" + state: present + reload: true + when: swap_enabled + +- name: Turn on all swap + ansible.builtin.command: swapon -a + changed_when: false + failed_when: false + when: swap_enabled + +# --- VERIFICATION --- + +- name: Show active swap devices + ansible.builtin.command: swapon --show + register: swapon_show + changed_when: false + failed_when: false + +- name: Print swapon --show + ansible.builtin.debug: + var: swapon_show.stdout_lines diff --git a/infrastructure/roles/pihole_db_rotate/files/rotate-pihole-db.sh b/infrastructure/roles/pihole_db_rotate/files/rotate-pihole-db.sh new file mode 100755 index 0000000..25a8b38 --- /dev/null +++ b/infrastructure/roles/pihole_db_rotate/files/rotate-pihole-db.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +DB="/etc/pihole/pihole-FTL.db" +ARCHIVE_DIR="/var/lib/pihole/db-archive" +TS="$(date +%F_%H%M%S)" + +mkdir -p "$ARCHIVE_DIR" + +if [[ -f "$DB" ]]; then + gzip -c "$DB" > "$ARCHIVE_DIR/pihole-FTL.db.$TS.gz" + rm -f "$DB" +fi diff --git a/infrastructure/roles/pihole_db_rotate/tasks/main.yml b/infrastructure/roles/pihole_db_rotate/tasks/main.yml new file mode 100644 index 0000000..ad8359e --- /dev/null +++ b/infrastructure/roles/pihole_db_rotate/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Ensure Pi-hole DB archive directory exists + ansible.builtin.file: + path: /var/lib/pihole/db-archive + state: directory + owner: root + group: root + mode: "0755" + +- name: Install Pi-hole DB rotation script + ansible.builtin.copy: + src: rotate-pihole-db.sh + dest: /usr/local/sbin/rotate-pihole-db + owner: root + group: root + mode: "0755" + +- name: Stop Pi-hole FTL + ansible.builtin.systemd: + name: pihole-FTL + state: stopped + +- name: Rotate Pi-hole FTL database + ansible.builtin.command: + cmd: /usr/local/sbin/rotate-pihole-db + changed_when: true + +- name: Start Pi-hole FTL + ansible.builtin.systemd: + name: pihole-FTL + state: started diff --git a/infrastructure/roles/pihole_dns/handlers/main.yml b/infrastructure/roles/pihole_dns/handlers/main.yml new file mode 100644 index 0000000..c48bce4 --- /dev/null +++ b/infrastructure/roles/pihole_dns/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart pihole-FTL + ansible.builtin.systemd: + name: pihole-FTL + state: restarted diff --git a/infrastructure/roles/pihole_dns/tasks/main.yml b/infrastructure/roles/pihole_dns/tasks/main.yml new file mode 100644 index 0000000..4fb0283 --- /dev/null +++ b/infrastructure/roles/pihole_dns/tasks/main.yml @@ -0,0 +1,55 @@ +--- +- name: Ensure dnsmasq.d exists + ansible.builtin.file: + path: /etc/dnsmasq.d + state: directory + mode: "0755" + +- name: Configure dns-forward-max + ansible.builtin.template: + src: 99-dns-forward-max.conf.j2 + dest: /etc/dnsmasq.d/99-dns-forward-max.conf + owner: root + group: root + mode: "0644" + notify: restart pihole-FTL + +- name: Configure Pi-hole forwarding for Samba AD + reverse zone + ansible.builtin.template: + src: 05-samba-ad.conf.j2 + dest: /etc/dnsmasq.d/05-samba-ad.conf + owner: root + group: root + mode: "0644" + notify: restart pihole-FTL + +- name: Read current Pi-hole FTL DB journal mode (WAL?) + ansible.builtin.command: sqlite3 /etc/pihole/pihole-FTL.db "PRAGMA journal_mode;" + register: pihole_journal + changed_when: false + failed_when: false + +- name: Parse current journal mode (safe default) + ansible.builtin.set_fact: + pihole_journal_mode: "{{ (pihole_journal.stdout | default('') | trim | lower) }}" + +- name: Determine whether WAL update is needed + ansible.builtin.set_fact: + pihole_need_wal: "{{ (pihole_journal.rc | default(1) == 0) and (pihole_journal_mode != 'wal') }}" + +# Only stop/start FTL if we are actually changing the journal mode. +- name: Enable WAL mode on Pi-hole FTL DB (one-time) + block: + - name: Stop pihole-FTL before switching SQLite journal mode + ansible.builtin.service: + name: pihole-FTL + state: stopped + + - name: Set SQLite journal_mode=WAL + ansible.builtin.command: sqlite3 /etc/pihole/pihole-FTL.db "PRAGMA journal_mode=WAL;" + + - name: Start pihole-FTL after switching SQLite journal mode + ansible.builtin.service: + name: pihole-FTL + state: started + when: pihole_need_wal diff --git a/infrastructure/roles/pihole_dns/templates/05-samba-ad.conf.j2 b/infrastructure/roles/pihole_dns/templates/05-samba-ad.conf.j2 new file mode 100644 index 0000000..9544be3 --- /dev/null +++ b/infrastructure/roles/pihole_dns/templates/05-samba-ad.conf.j2 @@ -0,0 +1,6 @@ +# Forward AD zones to Samba DC +server=/{{ prole_domain }}/{{ ad_dc_ip }} +server=/_msdcs.{{ prole_domain }}/{{ ad_dc_ip }} + +# Forward reverse lookups for LAN to Samba DC +server=/{{ lan_reverse_zone }}/{{ ad_dc_ip }} diff --git a/infrastructure/roles/pihole_dns/templates/99-dns-forward-max.conf.j2 b/infrastructure/roles/pihole_dns/templates/99-dns-forward-max.conf.j2 new file mode 100644 index 0000000..fd7dbec --- /dev/null +++ b/infrastructure/roles/pihole_dns/templates/99-dns-forward-max.conf.j2 @@ -0,0 +1 @@ +dns-forward-max={{ pihole_dns_forward_max | default(300) }} diff --git a/infrastructure/roles/prole/tasks/main.yml b/infrastructure/roles/prole/tasks/main.yml new file mode 100644 index 0000000..695b7e2 --- /dev/null +++ b/infrastructure/roles/prole/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Ensure prole_home directory exists + ansible.builtin.file: + path: "{{ prole_home }}" + state: directory + mode: '0755' diff --git a/infrastructure/roles/prole_ssl/defaults/main.yml b/infrastructure/roles/prole_ssl/defaults/main.yml new file mode 100644 index 0000000..51f0390 --- /dev/null +++ b/infrastructure/roles/prole_ssl/defaults/main.yml @@ -0,0 +1,10 @@ +--- +prole_ssl_dest_dir: /etc/ssl/certs/prole + +# Source directory on the Ansible controller (this repo) +prole_ssl_src_dir: "{{ playbook_dir }}/../../ssl/prole" + +# Files to deploy (relative to prole_ssl_src_dir) +prole_ssl_files: + - myrddin-registry.crt + - myrddin-registry.key diff --git a/infrastructure/roles/prole_ssl/tasks/main.yml b/infrastructure/roles/prole_ssl/tasks/main.yml new file mode 100644 index 0000000..c6b1396 --- /dev/null +++ b/infrastructure/roles/prole_ssl/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Ensure Prole SSL directory exists + ansible.builtin.file: + path: "{{ prole_ssl_dest_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Deploy Prole SSL files + ansible.builtin.copy: + src: "{{ prole_ssl_src_dir }}/{{ item }}" + dest: "{{ prole_ssl_dest_dir }}/{{ item }}" + owner: root + group: root + mode: "{{ '0600' if (item | regex_search('\\.key$')) else '0644' }}" + loop: "{{ prole_ssl_files }}" diff --git a/infrastructure/roles/rsyslog/handlers/main.yml b/infrastructure/roles/rsyslog/handlers/main.yml new file mode 100644 index 0000000..6500301 --- /dev/null +++ b/infrastructure/roles/rsyslog/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart rsyslog + service: + name: rsyslog + state: restarted diff --git a/infrastructure/roles/rsyslog/tasks/client.yml b/infrastructure/roles/rsyslog/tasks/client.yml new file mode 100644 index 0000000..d898336 --- /dev/null +++ b/infrastructure/roles/rsyslog/tasks/client.yml @@ -0,0 +1,13 @@ +- name: Install rsyslog + apt: + name: rsyslog + state: present + +- name: Configure forwarding to central syslog + template: + src: 90-forward-all.conf.j2 + dest: /etc/rsyslog.d/90-forward-all.conf + owner: root + group: root + mode: '0644' + notify: restart rsyslog diff --git a/infrastructure/roles/rsyslog/tasks/main.yml b/infrastructure/roles/rsyslog/tasks/main.yml new file mode 100644 index 0000000..b162ac7 --- /dev/null +++ b/infrastructure/roles/rsyslog/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- include_tasks: server.yml + when: inventory_hostname in groups['ad_dc'] + +- include_tasks: client.yml diff --git a/infrastructure/roles/rsyslog/tasks/server.yml b/infrastructure/roles/rsyslog/tasks/server.yml new file mode 100644 index 0000000..9849a5e --- /dev/null +++ b/infrastructure/roles/rsyslog/tasks/server.yml @@ -0,0 +1,35 @@ +- name: Ensure syslog system user exists (Debian sometimes doesn't create it in minimal images) + user: + name: syslog + system: true + shell: /usr/sbin/nologin + create_home: false + ignore_errors: false + +- name: Ensure syslog is in prole group (so it can write to {{ prole_logs_dir }} on NFS) + user: + name: syslog + groups: prole + append: true + +- name: Install rsyslog + apt: + name: rsyslog + state: present + +- name: Enable rsyslog server config + template: + src: 10-server.conf.j2 + dest: /etc/rsyslog.d/10-server.conf + owner: root + group: root + mode: '0644' + notify: restart rsyslog + +- name: Ensure log root exists and is group-writable via prole (setgid) + file: + path: "{{ prole_logs_dir }}" + state: directory + owner: root + group: prole + mode: '2775' diff --git a/infrastructure/roles/rsyslog/templates/10-server.conf.j2 b/infrastructure/roles/rsyslog/templates/10-server.conf.j2 new file mode 100644 index 0000000..5266f2b --- /dev/null +++ b/infrastructure/roles/rsyslog/templates/10-server.conf.j2 @@ -0,0 +1,27 @@ +# Accept remote syslog (UDP/TCP 514) +module(load="imudp") +input(type="imudp" port="514") + +module(load="imtcp") +input(type="imtcp" port="514") + +# Prefer FQDN derived from sender IP (reverse DNS), not whatever the sender claims +global( + preserveFQDN="on" + net.enableDNS="on" +) + +# Write to {{ prole_logs_dir }}/{fqdn}/{program}.log +template(name="PerHostLogs" type="string" + string="{{ prole_logs_dir }}/%HOSTNAME%/%programname%.log"); + +*.* action( + type="omfile" + dynaFile="PerHostLogs" + createDirs="on" + dirCreateMode="02775" + fileCreateMode="0640" + dirGroup="prole" + fileGroup="prole" +) +& stop diff --git a/infrastructure/roles/rsyslog/templates/90-forward-all.conf.j2 b/infrastructure/roles/rsyslog/templates/90-forward-all.conf.j2 new file mode 100644 index 0000000..c7e084e --- /dev/null +++ b/infrastructure/roles/rsyslog/templates/90-forward-all.conf.j2 @@ -0,0 +1,2 @@ +# Forward everything to myrddin over TCP +*.* @@10.0.0.3:514 diff --git a/infrastructure/roles/samba_ad_dc/defaults/main.yml b/infrastructure/roles/samba_ad_dc/defaults/main.yml new file mode 100644 index 0000000..7adf06f --- /dev/null +++ b/infrastructure/roles/samba_ad_dc/defaults/main.yml @@ -0,0 +1,42 @@ +--- +# roles/samba_ad_dc_defaults/defaults/main.yml + +# DNS +samba_ad_dc_dns_forwarders: + - 10.0.0.5 + - 10.0.0.4 + +# Identity +# When samba_ad_dc_child_id is set (e.g., A000001), the realm/workgroup/netbios +# will be derived automatically as a child of samba_ad_dc_parent_realm. +samba_ad_dc_parent_realm: "PROLE.ORG" +samba_ad_dc_parent_netbios: "{{ samba_ad_dc_parent_realm.split('.')[0] | upper }}" +samba_ad_dc_child_id: "" +samba_ad_dc_realm: >- + {{ (samba_ad_dc_child_id | length > 0) + | ternary((samba_ad_dc_child_id | upper) ~ '.' ~ samba_ad_dc_parent_realm, samba_ad_dc_parent_realm) }} +samba_ad_dc_workgroup: >- + {{ (samba_ad_dc_child_id | length > 0) + | ternary((samba_ad_dc_child_id | upper), "PROLE") }} +samba_ad_dc_netbios_name: "{{ inventory_hostname_short | upper }}" +samba_ad_dc_server_string: "{{ samba_ad_dc_realm }} AD DC" + +# Role/services +samba_ad_dc_server_role: "active directory domain controller" +samba_ad_dc_server_services: "-smb -winbind" + +# RFC2307 +samba_ad_dc_rfc2307: true + +# Homes +samba_ad_dc_template_homedir: "/prole/home/%U" +samba_ad_dc_template_shell: "/bin/bash" + +# Shares +samba_ad_dc_sysvol_path: "/var/lib/samba/sysvol" +samba_ad_dc_netlogon_path: "/var/lib/samba/sysvol/{{ samba_ad_dc_realm | lower }}/scripts" + +# K3s/Kubeadm access group +samba_kubeadm_group: "kubeadm" +samba_kubeadm_members: + - chrisfu diff --git a/infrastructure/roles/samba_ad_dc/handlers/main.yml b/infrastructure/roles/samba_ad_dc/handlers/main.yml new file mode 100644 index 0000000..87135d8 --- /dev/null +++ b/infrastructure/roles/samba_ad_dc/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart samba-ad-dc + ansible.builtin.systemd: + name: samba-ad-dc + state: restarted diff --git a/infrastructure/roles/samba_ad_dc/tasks/main.yml b/infrastructure/roles/samba_ad_dc/tasks/main.yml new file mode 100644 index 0000000..a4068c2 --- /dev/null +++ b/infrastructure/roles/samba_ad_dc/tasks/main.yml @@ -0,0 +1,97 @@ +--- +- name: Install Samba AD DC packages + ansible.builtin.apt: + name: + - samba + - krb5-user + state: present + update_cache: true + +- name: Derive child realm from namespace when unset + ansible.builtin.set_fact: + samba_ad_dc_child_id: "{{ derived_id | upper }}" + vars: + prole_ns: "{{ lookup('env', 'PROLE_NAMESPACE') | default(lookup('env', 'NAMESPACE'), true) | default('', true) }}" + derived_id: >- + {{ (prole_ns | regex_findall('^knoe-db-([A-Za-z0-9]+)$') | first) + | default('', true) }} + when: + - samba_ad_dc_child_id is not defined or samba_ad_dc_child_id | length == 0 + - derived_id | length > 0 + +- name: Check if Samba AD is provisioned + ansible.builtin.stat: + path: /var/lib/samba/private/sam.ldb + register: samba_ad_provisioned + +- name: Fail if not provisioned (this role does not provision) + ansible.builtin.fail: + msg: |- + Samba AD is not provisioned on this host (missing /var/lib/samba/private/sam.ldb). + Provision the child realm first, for example: + samba-tool domain provision \ + --use-rfc2307 \ + --realm={{ samba_ad_dc_realm }} \ + --domain={{ samba_ad_dc_workgroup }} \ + --server-role=dc \ + --dns-backend=SAMBA_INTERNAL \ + --adminpass='' \ + --parent-realm={{ samba_ad_dc_parent_realm }} \ + --username='{{ samba_ad_dc_parent_netbios }}\\Administrator' + when: not samba_ad_provisioned.stat.exists + +- name: Deploy smb.conf + ansible.builtin.template: + src: smb.conf.j2 + dest: /etc/samba/smb.conf + owner: root + group: root + mode: "0644" + notify: Restart samba-ad-dc + +- name: Ensure samba-ad-dc enabled and running + ansible.builtin.systemd: + name: samba-ad-dc + enabled: true + state: started + +- name: Check for kubeadm Samba group + ansible.builtin.command: >- + samba-tool group show {{ samba_kubeadm_group }} + -U {{ samba_dns_admin_user }}%{{ samba_dns_admin_pass }} + register: samba_kubeadm_group_show + changed_when: false + failed_when: false + when: + - samba_dns_admin_pass is defined + - samba_dns_admin_pass | length > 0 + +- name: Create kubeadm Samba group when missing + ansible.builtin.command: >- + samba-tool group add {{ samba_kubeadm_group }} + -U {{ samba_dns_admin_user }}%{{ samba_dns_admin_pass }} + when: + - samba_dns_admin_pass is defined + - samba_dns_admin_pass | length > 0 + - samba_kubeadm_group_show.rc != 0 + +- name: List kubeadm Samba group members + ansible.builtin.command: >- + samba-tool group listmembers {{ samba_kubeadm_group }} + -U {{ samba_dns_admin_user }}%{{ samba_dns_admin_pass }} + register: samba_kubeadm_group_members + changed_when: false + when: + - samba_dns_admin_pass is defined + - samba_dns_admin_pass | length > 0 + +- name: Ensure kubeadm Samba group members + ansible.builtin.command: >- + samba-tool group addmembers {{ samba_kubeadm_group }} {{ item }} + -U {{ samba_dns_admin_user }}%{{ samba_dns_admin_pass }} + loop: "{{ samba_kubeadm_members }}" + when: + - samba_dns_admin_pass is defined + - samba_dns_admin_pass | length > 0 + - samba_kubeadm_members | length > 0 + - item not in (samba_kubeadm_group_members.stdout_lines | default([])) diff --git a/infrastructure/roles/samba_ad_dc/templates/smb.conf.j2 b/infrastructure/roles/samba_ad_dc/templates/smb.conf.j2 new file mode 100644 index 0000000..81c0ea9 --- /dev/null +++ b/infrastructure/roles/samba_ad_dc/templates/smb.conf.j2 @@ -0,0 +1,21 @@ +# Global parameters +[global] + dns forwarder = {{ samba_ad_dc_dns_forwarders | join(' ') }} + + netbios name = {{ (samba_ad_dc_netbios_name | default(inventory_hostname_short)) | upper }} + realm = {{ samba_ad_dc_realm }} + server string = {{ samba_ad_dc_server_string }} + server role = {{ samba_ad_dc_server_role }} + server services = {{ samba_ad_dc_server_services }} + workgroup = {{ samba_ad_dc_workgroup }} + idmap_ldb:use rfc2307 = {{ "yes" if samba_ad_dc_rfc2307 else "no" }} + template homedir = {{ samba_ad_dc_template_homedir }} + template shell = {{ samba_ad_dc_template_shell }} + +[sysvol] + path = {{ samba_ad_dc_sysvol_path }} + read only = No + +[netlogon] + path = {{ samba_ad_dc_netlogon_path }} + read only = No diff --git a/infrastructure/roles/samba_ad_dc/tests/netbios_name.yml b/infrastructure/roles/samba_ad_dc/tests/netbios_name.yml new file mode 100644 index 0000000..77544e5 --- /dev/null +++ b/infrastructure/roles/samba_ad_dc/tests/netbios_name.yml @@ -0,0 +1,45 @@ +--- +- name: Add a representative AD DC host for template rendering + hosts: localhost + connection: local + gather_facts: false + become: false + + tasks: + - name: Add myrddin.prole.org as a local-connection host + ansible.builtin.add_host: + name: myrddin.prole.org + groups: samba_ad_dc_test_hosts + ansible_connection: local + +- name: Render smb.conf and assert netbios name derives from short hostname + hosts: samba_ad_dc_test_hosts + connection: local + gather_facts: false + become: false + + vars: + _rendered_path: /tmp/samba_ad_dc_smb.conf + + tasks: + - name: Load samba_ad_dc role defaults + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../defaults/main.yml" + + - name: Render smb.conf template + ansible.builtin.template: + src: "{{ playbook_dir }}/../templates/smb.conf.j2" + dest: "{{ _rendered_path }}" + mode: "0600" + + - name: Read rendered smb.conf + ansible.builtin.slurp: + src: "{{ _rendered_path }}" + register: _smb_conf + + - name: Assert netbios name is the uppercase short hostname + ansible.builtin.assert: + that: + - (_smb_conf.content | b64decode) is search('(?m)^\\s*netbios name\\s*=\\s*MYRDDIN\\s*$') + fail_msg: >- + Expected smb.conf to contain: netbios name = MYRDDIN \ No newline at end of file diff --git a/infrastructure/roles/samba_dns/tasks/ensure_a.yml b/infrastructure/roles/samba_dns/tasks/ensure_a.yml new file mode 100644 index 0000000..63e5b68 --- /dev/null +++ b/infrastructure/roles/samba_dns/tasks/ensure_a.yml @@ -0,0 +1,49 @@ +--- +- name: Determine zone and record name + ansible.builtin.set_fact: + samba_target_zone: "{{ 'internal.' ~ prole_domain if item.fqdn.endswith('.internal.' ~ prole_domain) or item.fqdn == 'internal.' ~ prole_domain else prole_domain }}" + samba_dns_record_name: >- + {% set fqdn_clean = item.fqdn | regex_replace('\\.?$', '') %} + {% if fqdn_clean == ('internal.' ~ prole_domain if item.fqdn.endswith('.internal.' ~ prole_domain) or item.fqdn == 'internal.' ~ prole_domain else prole_domain) %} + @ + {% else %} + {{ fqdn_clean | regex_replace('\\.' ~ (('internal.' ~ prole_domain if item.fqdn.endswith('.internal.' ~ prole_domain) or item.fqdn == 'internal.' ~ prole_domain else prole_domain) | regex_escape) ~ '$', '') }} + {% endif %} + +- name: Query existing A records + ansible.builtin.command: + cmd: samba-tool dns query {{ samba_dns_server }} {{ samba_target_zone }} {{ samba_dns_record_name }} A -P + register: a_query + changed_when: false + failed_when: false + +- name: Parse existing A record values + ansible.builtin.set_fact: + a_existing_values: >- + {{ + (a_query.stdout | default('') | + regex_findall('\\bA\\s+([0-9]{1,3}(?:\\.[0-9]{1,3}){3})\\b') | + list) + }} + +- name: Remove stale A records + ansible.builtin.command: + cmd: samba-tool dns delete {{ samba_dns_server }} {{ samba_target_zone }} {{ samba_dns_record_name }} A {{ a_value }} -P + loop: "{{ a_existing_values | difference(item.ipv4s) }}" + loop_control: + loop_var: a_value + when: (a_existing_values | difference(item.ipv4s) | length) > 0 + +- name: Add missing A records + ansible.builtin.command: + cmd: samba-tool dns add {{ samba_dns_server }} {{ samba_target_zone }} {{ samba_dns_record_name }} A {{ a_target_ip }} -P + register: a_add + changed_when: a_add.rc == 0 + failed_when: > + a_add.rc != 0 and + ('WERR_DNS_ERROR_RECORD_ALREADY_EXISTS' not in (a_add.stderr | default(''))) and + ('Record already exists' not in (a_add.stderr | default(''))) + loop: "{{ item.ipv4s | difference(a_existing_values) }}" + loop_control: + loop_var: a_target_ip + when: (item.ipv4s | difference(a_existing_values) | length) > 0 diff --git a/infrastructure/roles/samba_dns/tasks/ensure_cname.yml b/infrastructure/roles/samba_dns/tasks/ensure_cname.yml new file mode 100644 index 0000000..37492b4 --- /dev/null +++ b/infrastructure/roles/samba_dns/tasks/ensure_cname.yml @@ -0,0 +1,45 @@ +--- +- name: Determine zone and record name + ansible.builtin.set_fact: + samba_target_zone: "{{ 'internal.' ~ prole_domain if item.fqdn.endswith('.internal.' ~ prole_domain) or item.fqdn == 'internal.' ~ prole_domain else prole_domain }}" + samba_dns_record_name: >- + {% set fqdn_clean = item.fqdn | regex_replace('\\.?$', '') %} + {% if fqdn_clean == ('internal.' ~ prole_domain if item.fqdn.endswith('.internal.' ~ prole_domain) or item.fqdn == 'internal.' ~ prole_domain else prole_domain) %} + @ + {% else %} + {{ fqdn_clean | regex_replace('\\.' ~ (('internal.' ~ prole_domain if item.fqdn.endswith('.internal.' ~ prole_domain) or item.fqdn == 'internal.' ~ prole_domain else prole_domain) | regex_escape) ~ '$', '') }} + {% endif %} + +- name: Query existing CNAME records + ansible.builtin.command: + cmd: samba-tool dns query {{ samba_dns_server }} {{ samba_target_zone }} {{ samba_dns_record_name }} CNAME -P + register: cname_query + changed_when: false + failed_when: false + +- name: Parse existing CNAME record value + ansible.builtin.set_fact: + cname_existing_value: >- + {{ + (cname_query.stdout | default('') | + regex_findall('\\bCNAME\\s+([a-zA-Z0-9.-]+)\\b') | + first | default('')) + }} + +- name: Remove stale CNAME record + ansible.builtin.command: + cmd: samba-tool dns delete {{ samba_dns_server }} {{ samba_target_zone }} {{ samba_dns_record_name }} CNAME {{ cname_existing_value }} -P + when: + - cname_existing_value | length > 0 + - cname_existing_value != item.target + +- name: Add missing CNAME record + ansible.builtin.command: + cmd: samba-tool dns add {{ samba_dns_server }} {{ samba_target_zone }} {{ samba_dns_record_name }} CNAME {{ item.target }} -P + register: cname_add + changed_when: cname_add.rc == 0 + failed_when: > + cname_add.rc != 0 and + ('WERR_DNS_ERROR_RECORD_ALREADY_EXISTS' not in (cname_add.stderr | default(''))) and + ('Record already exists' not in (cname_add.stderr | default(''))) + when: cname_existing_value != item.target diff --git a/infrastructure/roles/samba_dns/tasks/main.yml b/infrastructure/roles/samba_dns/tasks/main.yml new file mode 100644 index 0000000..99d24d2 --- /dev/null +++ b/infrastructure/roles/samba_dns/tasks/main.yml @@ -0,0 +1,34 @@ +--- +- name: Assert Samba DNS admin password is set (vault loaded) + ansible.builtin.assert: + that: + - samba_dns_admin_pass is defined + - samba_dns_admin_pass | length > 0 + fail_msg: "Missing samba_dns_admin_pass. Create inventory/group_vars/ad_dc.vault.yml with vault_samba_dns_admin_pass." + tags: [samba, samba_dns] + +- name: List Samba DNS zones + ansible.builtin.command: + cmd: samba-tool dns zonelist {{ samba_dns_server }} -P + register: samba_zones + changed_when: false + tags: [samba, samba_dns] + +- name: Create DNS zones if missing + ansible.builtin.command: + cmd: samba-tool dns zonecreate {{ samba_dns_server }} {{ item }} -P + loop: + - "{{ prole_domain }}" + - "internal.{{ prole_domain }}" + when: item not in samba_zones.stdout + tags: [samba, samba_dns] + +- name: Ensure forward A records (internal RFC1918 hosts and k3s front-door) + ansible.builtin.include_tasks: ensure_a.yml + loop: "{{ (prole_internal_a_records | default([])) + (prole_k3s_dns_records | default([])) }}" + tags: [samba, samba_dns] + +- name: Ensure forward CNAME records + ansible.builtin.include_tasks: ensure_cname.yml + loop: "{{ prole_k3s_cname_records | default([]) }}" + tags: [samba, samba_dns] diff --git a/infrastructure/roles/samba_reverse_dns/tasks/ensure_ptr.yml b/infrastructure/roles/samba_reverse_dns/tasks/ensure_ptr.yml new file mode 100644 index 0000000..0a06968 --- /dev/null +++ b/infrastructure/roles/samba_reverse_dns/tasks/ensure_ptr.yml @@ -0,0 +1,36 @@ +--- +- name: Query existing PTR records + ansible.builtin.command: + cmd: samba-tool dns query {{ samba_dns_server }} {{ lan_reverse_zone }} {{ item.last_octet }} PTR -P + register: ptr_query + changed_when: false + failed_when: false + +- name: Parse existing PTR records + ansible.builtin.set_fact: + ptr_existing_values: >- + {{ + (ptr_query.stdout | default('') | + regex_findall('PTR\\s+([A-Za-z0-9.-]+)\\.?') | + map('regex_replace', '\\.$', '') | + list) + }} + +- name: Remove stale PTR records + ansible.builtin.command: + cmd: samba-tool dns delete {{ samba_dns_server }} {{ lan_reverse_zone }} {{ item.last_octet }} PTR {{ ptr_value }} -P + loop: "{{ ptr_existing_values | reject('equalto', item.fqdn) | list }}" + loop_control: + loop_var: ptr_value + when: (ptr_existing_values | reject('equalto', item.fqdn) | list | length) > 0 + +- name: Add expected PTR record + ansible.builtin.command: + cmd: samba-tool dns add {{ samba_dns_server }} {{ lan_reverse_zone }} {{ item.last_octet }} PTR {{ item.fqdn }} -P + register: ptr_add + changed_when: ptr_add.rc == 0 + failed_when: > + ptr_add.rc != 0 and + ('WERR_DNS_ERROR_RECORD_ALREADY_EXISTS' not in (ptr_add.stderr | default(''))) and + ('Record already exists' not in (ptr_add.stderr | default(''))) + when: item.fqdn not in ptr_existing_values diff --git a/infrastructure/roles/samba_reverse_dns/tasks/main.yml b/infrastructure/roles/samba_reverse_dns/tasks/main.yml new file mode 100644 index 0000000..7369bc8 --- /dev/null +++ b/infrastructure/roles/samba_reverse_dns/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: Assert Samba DNS admin password is set (vault loaded) + ansible.builtin.assert: + that: + - samba_dns_admin_pass is defined + - samba_dns_admin_pass | length > 0 + fail_msg: "Missing samba_dns_admin_pass. Create inventory/group_vars/ad_dc.vault.yml with vault_samba_dns_admin_pass." + +- name: List Samba DNS zones + ansible.builtin.command: + cmd: samba-tool dns zonelist {{ samba_dns_server }} -P + register: samba_zones + changed_when: false + tags: [samba, samba_reverse_dns] + +- name: Build reverse DNS zone list + ansible.builtin.set_fact: + samba_reverse_zones: >- + {{ ([lan_reverse_zone] + (k3s_reverse_zones | default([]))) | unique }} + +- name: Create reverse DNS zones if missing + ansible.builtin.command: + cmd: samba-tool dns zonecreate {{ samba_dns_server }} {{ reverse_zone }} -P + loop: "{{ samba_reverse_zones }}" + loop_control: + loop_var: reverse_zone + when: reverse_zone not in samba_zones.stdout + tags: [samba, samba_reverse_dns] + +- name: Ensure minimum PTR records (DC + Pi-holes) + ansible.builtin.include_tasks: ensure_ptr.yml + loop: "{{ ptr_records | default([]) }}" + tags: [samba, samba_reverse_dns] diff --git a/infrastructure/setup/cr_ansible_user.sh b/infrastructure/setup/cr_ansible_user.sh new file mode 100644 index 0000000..f69e018 --- /dev/null +++ b/infrastructure/setup/cr_ansible_user.sh @@ -0,0 +1,15 @@ +set -x +HOST=$1 +PUBKEY="$(cat ~/.ssh/id_ed25519_ansible.pub)" + +ssh $HOST "sudo bash -s" </dev/null 2>&1 || useradd -m -s /bin/bash ansible +install -d -m 700 -o ansible -g ansible /home/ansible/.ssh +touch /home/ansible/.ssh/authorized_keys +chmod 600 /home/ansible/.ssh/authorized_keys +chown ansible:ansible /home/ansible/.ssh/authorized_keys +grep -qxF '$PUBKEY' /home/ansible/.ssh/authorized_keys || echo '$PUBKEY' >> /home/ansible/.ssh/authorized_keys +echo 'ansible ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/90-ansible +chmod 440 /etc/sudoers.d/90-ansible +EOF diff --git a/infrastructure/setup/cr_samba_family_users.sh b/infrastructure/setup/cr_samba_family_users.sh new file mode 100755 index 0000000..dfdcdc3 --- /dev/null +++ b/infrastructure/setup/cr_samba_family_users.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +GROUP="family" +PSO="family-relaxed" +PRECEDENCE="200" + +[[ $# -eq 1 ]] || { echo "Usage: $0 user1,user2,user3"; exit 1; } + +SUDO=(sudo) +[[ "${EUID}" -eq 0 ]] && SUDO=() + +IFS=',' read -ra USERS <<< "$1" + +# Return 0 if the given option string appears in pso create --help +pso_supports() { + local opt="$1" + "${SUDO[@]}" samba-tool domain passwordsettings pso create --help 2>&1 | grep -q -- "${opt}" +} + +echo "Ensuring group exists: ${GROUP}" +if ! "${SUDO[@]}" samba-tool group show "${GROUP}" >/dev/null 2>&1; then + "${SUDO[@]}" samba-tool group add "${GROUP}" +else + echo "Group ${GROUP} already exists" +fi + +echo "Ensuring relaxed password policy exists: ${PSO}" +if ! "${SUDO[@]}" samba-tool domain passwordsettings pso show "${PSO}" >/dev/null 2>&1; then + # Build args using only options supported by this samba-tool build. + ARGS=( domain passwordsettings pso create "${PSO}" "${PRECEDENCE}" ) + + # Min length + if pso_supports --min-pwd-length; then + ARGS+=( --min-pwd-length=4 ) + fi + + # Complexity off + if pso_supports --complexity; then + ARGS+=( --complexity=off ) + fi + + # Never expire + if pso_supports --max-pwd-age; then + ARGS+=( --max-pwd-age=0 ) + fi + if pso_supports --min-pwd-age; then + ARGS+=( --min-pwd-age=0 ) + fi + + # Password history option name varies across builds + if pso_supports --pwd-history-length; then + ARGS+=( --pwd-history-length=0 ) + elif pso_supports --password-history; then + ARGS+=( --password-history=0 ) + fi + + # Lockout option names vary too; only add if supported + if pso_supports --lockout-threshold; then + ARGS+=( --lockout-threshold=0 ) + elif pso_supports --lockout-limit; then + ARGS+=( --lockout-limit=0 ) + fi + + echo "Creating PSO ${PSO} with supported options..." + "${SUDO[@]}" samba-tool "${ARGS[@]}" +else + echo "PSO ${PSO} already exists" +fi + +echo "Applying PSO ${PSO} to group ${GROUP}" +out="$("${SUDO[@]}" samba-tool domain passwordsettings pso apply "${PSO}" "${GROUP}" 2>&1)" || true +if echo "$out" | grep -q "already applies"; then + echo "PSO ${PSO} already applies to ${GROUP}" +elif echo "$out" | grep -qi "^ERROR:"; then + echo "$out" >&2 + exit 1 +else + echo "$out" +fi + +echo "Creating/ensuring users and adding to ${GROUP}" +for user in "${USERS[@]}"; do + user="$(echo "$user" | xargs)" + [[ -n "$user" ]] || continue + + echo " - ${user}" + if ! "${SUDO[@]}" samba-tool user show "${user}" >/dev/null 2>&1; then + TMPPW="$(openssl rand -base64 36 | tr -d '\n' | tr '/+' 'Aa' | cut -c1-20)1aA!" + "${SUDO[@]}" samba-tool user create "${user}" "${TMPPW}" + echo " created with temporary strong password" + else + echo " user exists" + fi + + # membership first + "${SUDO[@]}" samba-tool group addmembers "${GROUP}" "${user}" >/dev/null 2>&1 || true + + # ensure password never expires + "${SUDO[@]}" samba-tool user setexpiry "${user}" --noexpiry >/dev/null + "${SUDO[@]}" samba-tool user setpassword "${user}" --must-change-at-next-login=no + + # now set the final (possibly weak) password under the PSO + echo " set final password for ${user}" + "${SUDO[@]}" samba-tool user setpassword "${user}" +done + +echo "Done." +echo "Verify:" +echo " sudo samba-tool group listmembers ${GROUP}" +echo " sudo samba-tool domain passwordsettings pso show-user anna" diff --git a/infrastructure/setup/filesystem.txt b/infrastructure/setup/filesystem.txt new file mode 100644 index 0000000..5337882 --- /dev/null +++ b/infrastructure/setup/filesystem.txt @@ -0,0 +1,12 @@ +Prole Storage + +# NFS export +/prole +├── home ← exported (users, autohome) +└── logs ← exported (central logging) + +# Database Storage +/knoe-db +├── d001 ← local-only XFS (k3s/db) +├── d002 ← local-only XFS +└── d003 ← local-only XFS diff --git a/knoe-db.iml b/knoe-db.iml index ad9fd88..d701879 100644 --- a/knoe-db.iml +++ b/knoe-db.iml @@ -3,6 +3,8 @@ + + \ No newline at end of file