chrisfu/prole
1
0
Fork 0
mirror of https://github.com/dredx/prole.git synced 2026-09-23 11:03:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

nodeSelector and storage updates across k8s manifests

Browse Source 
- Added `nodeSelector` for multiple Kubernetes resources to ensure scheduling on `myrddin.prole.org`.
- Modified `storage` requests and set `storageClassName` in `garage-statefulset.yaml`.
- Enhanced `install.py` for dynamic environment configuration and kubeconfig handling.
- Improved cgroup management tasks in Ansible with conflict resolution and parameter updates.
- Simplified vault token update process in playbooks and updated encryption checks.
This commit is contained in:
chrisfu 2026-02-06 01:17:54 -08:00
parent 1e87f4684e
commit 6b6e5e2aec
13 changed files with 122 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/init_garage_store.sh
View File

@ -44,6 +44,8 @@ else
fi
GARAGE_FILES=(
"$GARAGE_MANIFEST_DIR/storageclass-prole-iscsi.yaml"
"$GARAGE_MANIFEST_DIR/iscsi-pvs.yaml"
"$GARAGE_MANIFEST_DIR/garage-configmap.yaml"
"$GARAGE_MANIFEST_DIR/garage-statefulset.yaml"
"$GARAGE_MANIFEST_DIR/garage-service.yaml"

12
infrastructure/inventory/group_vars/all/vault_k3s.yml
View File

@ -1,11 +1 @@
vault_k3s_token: !vault |
$ANSIBLE_VAULT;1.1;AES256
63653139653862326430333034343737616234386639663461643535653362343639666530323039
3337303264656331623038623866396366316335666437310a373862626164636463613166326264
34396631366361343365623761343361383734346664366664356262373764313961313338336137
3832373063633439350a326466303830626130333461646136623334323738616339383637373935
37313266346437303461653937363035643737373632613332363339386164356464613663616435
66353533666434323466383034326531653337323932303636353436623566316537326561366539
62386437313037386334323363343232643032663064383130656465633630306535623162353563
38306432396432346535326638613132353163363734633861666662666239646432333061626431
32653766616131336635636231646263323736323335386364356232643339333136
vault_k3s_token: "K1018eb84d6d66202ed1980bb428a68c4ed122c23a11c7a025d91441094108ba145::server:7ad0aa18511842387814d4fb4bf6461f"

14
infrastructure/playbooks/k3s_sync.yml
View File

@ -213,12 +213,25 @@
msg: "Vault file not found at {{ vault_k3s_path }}"
when: not vault_k3s_file.stat.exists
- name: Check if vault file is encrypted
ansible.builtin.command: "head -n 1 {{ vault_k3s_path }}"
register: vault_k3s_head
changed_when: false
delegate_to: localhost
- name: Mark vault encryption state
ansible.builtin.set_fact:
vault_k3s_encrypted: "{{ (vault_k3s_head.stdout | default('')) is search('^\\$ANSIBLE_VAULT') }}"
delegate_to: localhost
delegate_facts: true
- name: Decrypt vault_k3s.yml
ansible.builtin.command: >-
ansible-vault decrypt {{ vault_k3s_path }}
--vault-password-file {{ hostvars['localhost'].k3s_vault_password_file }}
changed_when: true
delegate_to: localhost
when: hostvars['localhost'].vault_k3s_encrypted | default(false)
- name: Update vault k3s token
ansible.builtin.lineinfile:
@ -233,3 +246,4 @@
--vault-password-file {{ hostvars['localhost'].k3s_vault_password_file }}
changed_when: true
delegate_to: localhost
when: hostvars['localhost'].vault_k3s_encrypted | default(false)

3
infrastructure/roles/cgroups/defaults/main.yml
View File

@ -7,4 +7,7 @@ cgroups_required_params:
- cgroup_memory=1
- cgroup_enable=memory
cgroups_remove_params:
- cgroup_disable=memory
cgroups_reboot: true

62
infrastructure/roles/cgroups/tasks/main.yml
View File

@ -7,6 +7,7 @@
- name: Select kernel cmdline path
ansible.builtin.set_fact:
cgroups_cmdline_paths: "{{ cmdline_stats.results | selectattr('stat.exists') | map(attribute='stat.path') | list }}"
cgroups_cmdline_path: "{{ (cmdline_stats.results | selectattr('stat.exists') | map(attribute='stat.path') | list | first) | default('') }}"
- name: Fail when kernel cmdline file is missing
@ -19,24 +20,66 @@
path: "{{ cgroups_cmdline_path }}"
register: cmdline_raw
- name: Read current /proc/cmdline
ansible.builtin.command: "cat /proc/cmdline"
register: proc_cmdline_current
changed_when: false
- name: Parse kernel cmdline line
ansible.builtin.set_fact:
cgroups_cmdline_current: "{{ (cmdline_raw.content | b64decode).splitlines()[0] | default('') }}"
cgroups_cmdline_current: "{{ (cmdline_raw.content | b64decode).splitlines() | join(' ') | default('') }}"
- name: Select cmdline source
ansible.builtin.set_fact:
cgroups_cmdline_source: "{{ (proc_cmdline_current.stdout | default('') | length > 0) | ternary(proc_cmdline_current.stdout, cgroups_cmdline_current) }}"
- name: Sanitize kernel cmdline
ansible.builtin.set_fact:
cgroups_cmdline_sanitized: >-
{{ cgroups_cmdline_source
| regex_replace('\\\\n', ' ')
| regex_replace('\\\\r', ' ')
| regex_replace('\\\\[a-zA-Z]', ' ')
| regex_replace('\\\\', ' ')
| regex_replace('\\s+', ' ')
| trim }}
- name: Tokenize kernel cmdline
ansible.builtin.set_fact:
cgroups_cmdline_tokens: "{{ cgroups_cmdline_sanitized.split() }}"
- name: Remove conflicting cgroup params
ansible.builtin.set_fact:
cgroups_cmdline_filtered: "{{ cgroups_cmdline_tokens | reject('in', cgroups_remove_params) | list }}"
- name: Calculate missing cgroup params
ansible.builtin.set_fact:
cgroups_missing_params: "{{ cgroups_required_params | reject('in', cgroups_cmdline_current) | list }}"
cgroups_missing_params: "{{ cgroups_required_params | reject('in', cgroups_cmdline_filtered) | list }}"
- name: Build updated kernel cmdline tokens
ansible.builtin.set_fact:
cgroups_cmdline_new_tokens: "{{ (cgroups_cmdline_filtered + cgroups_missing_params) | unique | list }}"
- name: Build updated kernel cmdline line
ansible.builtin.set_fact:
cgroups_cmdline_new: "{{ cgroups_cmdline_new_tokens | join(' ') }}"
- name: Determine if kernel cmdline needs update
ansible.builtin.set_fact:
cgroups_cmdline_needs_update: >-
{{ cgroups_cmdline_new != cgroups_cmdline_sanitized or
cgroups_cmdline_sanitized != cgroups_cmdline_source or
(cgroups_cmdline_paths | length) > 1 }}
- name: Update kernel cmdline when params are missing
ansible.builtin.copy:
dest: "{{ cgroups_cmdline_path }}"
dest: "{{ item }}"
content: >-
{{ cgroups_cmdline_current ~
(' ' ~ (cgroups_missing_params | join(' ')) if (cgroups_missing_params | length) > 0 else '') ~
'\n' }}
{{ cgroups_cmdline_new ~ '\n' }}
mode: "0644"
register: cmdline_update
when: cgroups_missing_params | length > 0
when: cgroups_cmdline_needs_update
loop: "{{ cgroups_cmdline_paths }}"
- name: Reboot to apply cgroup settings
ansible.builtin.reboot:
@ -54,3 +97,8 @@
ansible.builtin.fail:
msg: "Missing cgroup params in /proc/cmdline: {{ cgroups_required_params | reject('in', proc_cmdline.stdout) | list }}. Reboot required."
when: (cgroups_required_params | reject('in', proc_cmdline.stdout) | list) | length > 0
- name: Fail when conflicting cgroup params are present
ansible.builtin.fail:
msg: "Conflicting cgroup params in /proc/cmdline: {{ cgroups_remove_params | select('in', proc_cmdline.stdout) | list }}. Reboot required."
when: (cgroups_remove_params | select('in', proc_cmdline.stdout) | list) | length > 0

31
install.py
View File

@ -10735,6 +10735,9 @@ class ProleSilentInstaller:
env["PROLE_DB_USER"] = self._get_input('init_password.db_username', '')
env["DB_PASSWORD"] = password
env["NAMESPACE"] = (self._get_input('init_password.db_namespace', '') or '').strip()
env_key = _normalize_cluster_env(self._get_input('init_cluster.cluster_env', 'dev'))
if env_key:
env["PROLE_MODE"] = _deployment_mode_from_env(env_key) or env_key
realm = self._get_input('kerberos_config.realm', '').strip()
kdc = self._get_input('kerberos_config.kdc', '').strip()
user = self._get_input('kerberos_config.user', '').strip()
@ -10752,6 +10755,14 @@ class ProleSilentInstaller:
env["KRB5_PASSWORD"] = krb_pw
env.setdefault("KRB5_AD_PORT_FORWARD", "1")
kubeconfig_path = None
if env_key == 'service':
server = (self._get_input('init_cluster.k3s_server_url', '') or '').strip()
token = (self._get_input('init_cluster.k3s_token', '') or '').strip()
if server and token:
kubeconfig_path = _write_k3s_kubeconfig(server, token)
env["KUBECONFIG"] = str(kubeconfig_path)
self.log("Stopping existing port-forwards to avoid conflicts...")
self._run_script("init_port_forwards.sh", args=["stop"], env=env)
@ -10788,6 +10799,11 @@ class ProleSilentInstaller:
if rc != 0:
self.err(f"[ERROR] {script} failed (code {rc})")
overall_success = False
if kubeconfig_path:
try:
os.unlink(kubeconfig_path)
except Exception:
pass
self._scripts_success = overall_success
self.prole_cfg_data['Initialization Scripts']['STATUS'] = 'Completed' if overall_success else 'Attempted'
@ -10802,6 +10818,16 @@ class ProleSilentInstaller:
env["PROLE_HOME"] = str(self.project_root)
env["PROLE_SERVICE"] = str(self.project_root)
env["NAMESPACE"] = (self._get_input('init_password.db_namespace', '') or '').strip()
env_key = _normalize_cluster_env(self._get_input('init_cluster.cluster_env', 'dev'))
if env_key:
env["PROLE_MODE"] = _deployment_mode_from_env(env_key) or env_key
kubeconfig_path = None
if env_key == 'service':
server = (self._get_input('init_cluster.k3s_server_url', '') or '').strip()
token = (self._get_input('init_cluster.k3s_token', '') or '').strip()
if server and token:
kubeconfig_path = _write_k3s_kubeconfig(server, token)
env["KUBECONFIG"] = str(kubeconfig_path)
rc = self._run_cmd(['bash', str(etc_dir / 'init_prole-db.sh'), 'deploy', 'latest'], env=env)
if rc == 0:
self._cnpg_success = True
@ -10816,6 +10842,11 @@ class ProleSilentInstaller:
rc2 = self._run_cmd(['bash', str(etc_dir / 'init_prole-db.sh'), 'rollout'], env=env)
if rc2 != 0:
self.err(f"[ERROR] Rollout failed (code {rc2})")
if kubeconfig_path:
try:
os.unlink(kubeconfig_path)
except Exception:
pass
def _step_supabase(self) -> None:
if not self._get_input_bool('supabase_config.run_deploy', False):

2
k8s/opentofu/deployment.yaml
View File

@ -55,6 +55,8 @@ spec:
labels:
app: opentofu
spec:
nodeSelector:
kubernetes.io/hostname: myrddin.prole.org
containers:
- name: opentofu-ui
image: nginx:1.27-alpine

5
k8s/prole/garage-statefulset.yaml
View File

@ -15,6 +15,8 @@ spec:
labels:
app: garage
spec:
nodeSelector:
kubernetes.io/hostname: myrddin.prole.org
containers:
- name: garage
image: dxflrs/garage:v1.3.1
@ -68,4 +70,5 @@ spec:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storage: 29Gi
storageClassName: prole-iscsi

2
k8s/prole/grafana-deployment.yaml
View File

@ -14,6 +14,8 @@ spec:
labels:
app: grafana
spec:
nodeSelector:
kubernetes.io/hostname: myrddin.prole.org
containers:
- name: grafana
image: grafana/grafana-oss:11.2.0

2
k8s/prole/openbao-statefulset.yaml
View File

@ -15,6 +15,8 @@ spec:
labels:
app: openbao
spec:
nodeSelector:
kubernetes.io/hostname: myrddin.prole.org
containers:
- name: openbao
image: ghcr.io/openbao/openbao:latest

2
k8s/prole/prole-db.yaml
View File

@ -8,6 +8,8 @@ spec:
postgresUID: 100
postgresGID: 101
maxSyncReplicas: 1
nodeSelector:
kubernetes.io/hostname: myrddin.prole.org
postgresql:
parameters:

2
k8s/prole/prole-deployment.yaml
View File

@ -14,6 +14,8 @@ spec:
labels:
app: prole
spec:
nodeSelector:
kubernetes.io/hostname: myrddin.prole.org
containers:
- name: nginx
image: nginx:1.27-alpine

2
k8s/prole/prometheus-deployment.yaml
View File

@ -14,6 +14,8 @@ spec:
labels:
app: prometheus
spec:
nodeSelector:
kubernetes.io/hostname: myrddin.prole.org
containers:
- name: prometheus
image: prom/prometheus:v2.55.1