diff --git a/etc/init_k3s_registry.sh b/etc/init_k3s_registry.sh index f4c5088..c8da75b 100644 --- a/etc/init_k3s_registry.sh +++ b/etc/init_k3s_registry.sh @@ -4,7 +4,7 @@ set -euo pipefail # init_k3s_registry.sh # Purpose: -# - Configure k3s/containerd to allow HTTPS access to the Prole registry +# - Configure k3s/containerd to allow access to the Prole registry # - Writes /etc/rancher/k3s/registries.yaml on the k3s node SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -29,12 +29,9 @@ K3S_REGISTRY_FILE=${K3S_REGISTRY_FILE:-/etc/rancher/k3s/registries.yaml} K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-} if [[ -z "${K3S_REGISTRY_SCHEME}" ]]; then - # k3d's local registry is plain HTTP by default. - if [[ "${K3S_REGISTRY_HOST}" == k3d-* ]]; then - K3S_REGISTRY_SCHEME="http" - else - K3S_REGISTRY_SCHEME="https" - fi + # Internal Prole registry endpoints are plain HTTP by default. + # Set K3S_REGISTRY_SCHEME=https explicitly when TLS is configured. + K3S_REGISTRY_SCHEME="http" fi ensure_root() { diff --git a/infrastructure/roles/k3s/defaults/main.yml b/infrastructure/roles/k3s/defaults/main.yml index 8c8c4c6..6637a9e 100644 --- a/infrastructure/roles/k3s/defaults/main.yml +++ b/infrastructure/roles/k3s/defaults/main.yml @@ -70,6 +70,8 @@ k3s_registry_config_path: /etc/rancher/k3s/registries.yaml k3s_registry_host: "" k3s_registry_namespace: "" k3s_registry_port: 5000 +# Internal Prole registry is HTTP by default; set to "https" when TLS is configured. +k3s_registry_endpoint_scheme: "http" # Pulling large images on every run can cause major load spikes; keep this opt-in. k3s_prestage_images: false diff --git a/infrastructure/roles/k3s/tasks/install.yml b/infrastructure/roles/k3s/tasks/install.yml index 90a0d2e..74d2484 100644 --- a/infrastructure/roles/k3s/tasks/install.yml +++ b/infrastructure/roles/k3s/tasks/install.yml @@ -466,11 +466,24 @@ else inventory_hostname }} when: k3s_state == "present" and k3s_registry_config_enabled | bool +- name: Resolve k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: >- + {{ 'https' if (k3s_registry_host | default('') | regex_search('^https://')) + else 'http' if (k3s_registry_host | default('') | regex_search('^http://')) + else (k3s_registry_endpoint_scheme | default('http')) }} + when: k3s_state == "present" and k3s_registry_config_enabled | bool + - name: Normalize k3s registry host ansible.builtin.set_fact: k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}" when: k3s_state == "present" and k3s_registry_config_enabled | bool +- name: Normalize k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}" + when: k3s_state == "present" and k3s_registry_config_enabled | bool + - name: Resolve k3s registry namespace ansible.builtin.set_fact: k3s_registry_namespace_resolved: >- diff --git a/infrastructure/roles/k3s/tasks/sync.yml b/infrastructure/roles/k3s/tasks/sync.yml index dc88413..880687c 100644 --- a/infrastructure/roles/k3s/tasks/sync.yml +++ b/infrastructure/roles/k3s/tasks/sync.yml @@ -93,11 +93,24 @@ else inventory_hostname }} when: k3s_registry_config_enabled | bool +- name: Resolve k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: >- + {{ 'https' if (k3s_registry_host | default('') | regex_search('^https://')) + else 'http' if (k3s_registry_host | default('') | regex_search('^http://')) + else (k3s_registry_endpoint_scheme | default('http')) }} + when: k3s_registry_config_enabled | bool + - name: Normalize k3s registry host ansible.builtin.set_fact: k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}" when: k3s_registry_config_enabled | bool +- name: Normalize k3s registry endpoint scheme + ansible.builtin.set_fact: + k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}" + when: k3s_registry_config_enabled | bool + - name: Resolve k3s registry namespace ansible.builtin.set_fact: k3s_registry_namespace_resolved: >- diff --git a/infrastructure/roles/k3s/templates/registries.yaml.j2 b/infrastructure/roles/k3s/templates/registries.yaml.j2 index 422dda5..17c73dd 100644 --- a/infrastructure/roles/k3s/templates/registries.yaml.j2 +++ b/infrastructure/roles/k3s/templates/registries.yaml.j2 @@ -1,10 +1,12 @@ +{% set _scheme = (k3s_registry_endpoint_scheme_resolved | default(k3s_registry_endpoint_scheme | default('http')) | lower) %} mirrors: '{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}': endpoint: - - "https://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}" + - "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}" 'registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}': endpoint: - - "https://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}" + - "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}" +{% if _scheme != 'http' %} configs: '{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}': tls: @@ -12,3 +14,4 @@ configs: 'registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}': tls: insecure_skip_verify: true +{% endif %} diff --git a/knoe-db/Dockerfile b/knoe-db/Dockerfile index eada580..4aacc05 100644 --- a/knoe-db/Dockerfile +++ b/knoe-db/Dockerfile @@ -12,7 +12,7 @@ RUN set -eux; \ ca-certificates curl wget gnupg2 lsb-release \ locales \ gosu \ - build-essential git openssh-client \ + build-essential git openssh-client pkg-config \ ; \ rm -rf /var/lib/apt/lists/*; \ locale-gen en_US.UTF-8 @@ -119,9 +119,17 @@ ENV PATH=/root/.cargo/bin:$PATH RUN set -eux; \ cd /tmp; \ git clone https://github.com/tembo-io/pgmq.git; \ - cd pgmq/pgmq-extension; \ - cargo pgrx init --pg18=$(which pg_config); \ - cargo pgrx install --release; \ + if [ -f pgmq/pgmq-extension/Cargo.toml ]; then \ + cd pgmq/pgmq-extension; \ + cargo pgrx init --pg18=$(which pg_config); \ + cargo pgrx install --release; \ + elif [ -f pgmq/pgmq-extension/Makefile ]; then \ + make -C pgmq/pgmq-extension USE_PGXS=1; \ + make -C pgmq/pgmq-extension USE_PGXS=1 install; \ + else \ + echo "Unable to locate pgmq extension build files"; \ + exit 1; \ + fi; \ cd /; \ rm -rf /tmp/pgmq @@ -135,7 +143,9 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/*; \ cd /tmp; \ git clone https://github.com/supabase/wrappers.git; \ - cd wrappers; \ + cd wrappers/wrappers; \ + /root/.cargo/bin/cargo install --locked cargo-pgrx --version 0.16.1; \ + cargo pgrx init --pg18=$(which pg_config); \ cargo pgrx install --release; \ cd /; \ rm -rf /tmp/wrappers diff --git a/knoe-db/Dockerfile.percona.template b/knoe-db/Dockerfile.percona.template index 8de75be..861ae55 100644 --- a/knoe-db/Dockerfile.percona.template +++ b/knoe-db/Dockerfile.percona.template @@ -12,7 +12,7 @@ RUN set -eux; \ ca-certificates curl wget gnupg2 lsb-release \ locales \ gosu \ - build-essential git openssh-client \ + build-essential git openssh-client pkg-config \ ; \ rm -rf /var/lib/apt/lists/*; \ locale-gen en_US.UTF-8 @@ -114,9 +114,17 @@ ENV PATH=/root/.cargo/bin:$PATH RUN set -eux; \ cd /tmp; \ git clone https://github.com/tembo-io/pgmq.git; \ - cd pgmq/pgmq-extension; \ - cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \ - cargo pgrx install --release; \ + if [ -f pgmq/pgmq-extension/Cargo.toml ]; then \ + cd pgmq/pgmq-extension; \ + cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \ + cargo pgrx install --release; \ + elif [ -f pgmq/pgmq-extension/Makefile ]; then \ + make -C pgmq/pgmq-extension USE_PGXS=1; \ + make -C pgmq/pgmq-extension USE_PGXS=1 install; \ + else \ + echo "Unable to locate pgmq extension build files"; \ + exit 1; \ + fi; \ cd /; \ rm -rf /tmp/pgmq @@ -130,7 +138,9 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/*; \ cd /tmp; \ git clone https://github.com/supabase/wrappers.git; \ - cd wrappers; \ + cd wrappers/wrappers; \ + /root/.cargo/bin/cargo install --locked cargo-pgrx --version 0.16.1; \ + cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \ cargo pgrx install --release; \ cd /; \ rm -rf /tmp/wrappers diff --git a/knoe/core/ops/cloudnative_pg.py b/knoe/core/ops/cloudnative_pg.py index a7fd623..9786595 100644 --- a/knoe/core/ops/cloudnative_pg.py +++ b/knoe/core/ops/cloudnative_pg.py @@ -950,7 +950,20 @@ def bootstrap_cnpg_tls_secrets( f"{cluster_name}-r", f"{cluster_name}-ro", ] - srv_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, f"{cluster_name}.{namespace}.svc")]) + primary_srv_cn = f"{cluster_name}.{namespace}.svc" + srv_cn = primary_srv_cn + if len(srv_cn) > 64: + for candidate in (f"{cluster_name}-rw", cluster_name, "cnpg-server"): + if len(candidate) <= 64: + srv_cn = candidate + break + _log( + log, + "CNPG TLS subject CN exceeds X.509 limit; " + f"using fallback CN '{srv_cn}' instead of '{primary_srv_cn}'.", + ) + + srv_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, srv_cn)]) srv_cert = ( x509.CertificateBuilder() .subject_name(srv_name) diff --git a/mock_val/init_k3s_registry.sh b/mock_val/init_k3s_registry.sh index df64827..3101f7e 100644 --- a/mock_val/init_k3s_registry.sh +++ b/mock_val/init_k3s_registry.sh @@ -4,7 +4,7 @@ set -euo pipefail # init_k3s_registry.sh # Purpose: -# - Configure k3s/containerd to allow HTTPS access to the Prole registry +# - Configure k3s/containerd to allow access to the Prole registry # - Writes /etc/rancher/k3s/registries.yaml on the k3s node SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -22,10 +22,11 @@ registry_host_from_url() { printf '%s' "$value" } -K3S_REGISTRY_HOST=${K3S_REGISTRY_HOST:-$(registry_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")} +K3S_REGISTRY_HOST=${K3S_REGISTRY_HOST:-$(registry_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")} K3S_REGISTRY_PORT=${K3S_REGISTRY_PORT:-5000} K3S_REGISTRY_NAMESPACE=${K3S_REGISTRY_NAMESPACE:-${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}}} K3S_REGISTRY_FILE=${K3S_REGISTRY_FILE:-/etc/rancher/k3s/registries.yaml} +K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-http} ensure_root() { if [[ "$(id -u)" -ne 0 ]]; then @@ -38,14 +39,27 @@ render_registries_yaml() { local host="$1" local port="$2" local ns="$3" + local scheme="$4" + if [[ "$scheme" == "http" ]]; then + cat <"$K3S_REGISTRY_FILE" - echo "Wrote $K3S_REGISTRY_FILE for ${K3S_REGISTRY_HOST}:${K3S_REGISTRY_PORT}" + render_registries_yaml "$K3S_REGISTRY_HOST" "$K3S_REGISTRY_PORT" "$K3S_REGISTRY_NAMESPACE" "$K3S_REGISTRY_SCHEME" >"$K3S_REGISTRY_FILE" + echo "Wrote $K3S_REGISTRY_FILE for ${K3S_REGISTRY_SCHEME}://${K3S_REGISTRY_HOST}:${K3S_REGISTRY_PORT}" echo "Restart k3s to apply: sudo systemctl restart k3s" ;; status) diff --git a/modes/k3s/knoe-db/.version b/modes/k3s/knoe-db/.version index aa59885..aca544d 100644 --- a/modes/k3s/knoe-db/.version +++ b/modes/k3s/knoe-db/.version @@ -1 +1 @@ -143 \ No newline at end of file +145 \ No newline at end of file diff --git a/scripts/reset-ns.sh b/scripts/reset-ns.sh index 156c290..07ac59f 100755 --- a/scripts/reset-ns.sh +++ b/scripts/reset-ns.sh @@ -233,11 +233,19 @@ done echo "Waiting for pods to terminate..." if [[ ${#deleted_pods[@]} -gt 0 ]]; then - wait_args=() + deadline=$((SECONDS + 180)) for p in "${deleted_pods[@]}"; do - wait_args+=("pod/$p") + while kubectl -n "$NS" get pod "$p" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo " timeout waiting for pod/$p to terminate; continuing" + break + fi + sleep 2 + done + if ! kubectl -n "$NS" get pod "$p" >/dev/null 2>&1; then + echo " pod/$p terminated" + fi done - kubectl -n "$NS" wait --for=delete "${wait_args[@]}" --timeout=180s 2>/dev/null || true fi if [[ "$CLEAR_NODE_RESERVATIONS" == "true" ]]; then diff --git a/tests/installer/test_cnpg_tls_bootstrap.py b/tests/installer/test_cnpg_tls_bootstrap.py new file mode 100644 index 0000000..dd785b4 --- /dev/null +++ b/tests/installer/test_cnpg_tls_bootstrap.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import base64 +from types import SimpleNamespace + +from cryptography import x509 +from cryptography.x509.oid import NameOID + +from knoe.core.ops import cloudnative_pg + + +def _decode_tls_cert_pem(manifest: str) -> bytes: + for line in manifest.splitlines(): + key, _, value = line.partition(":") + if key.strip() == "tls.crt": + return base64.b64decode(value.strip()) + raise AssertionError("tls.crt not found in manifest") + + +def test_bootstrap_cnpg_tls_secrets_falls_back_to_short_cn_for_long_namespace(monkeypatch): + namespace = "knoe-db-17-7-043-18-140-17-7-043-18-140-17-7-043-18-140" + cluster_name = "knoe-db" + applied_manifests: list[str] = [] + + monkeypatch.setattr(cloudnative_pg, "_ensure_namespace", lambda *_args, **_kwargs: None) + monkeypatch.setattr(cloudnative_pg, "_kubectl_ok", lambda *_args, **_kwargs: False) + + def _fake_run(args, input=None, **_kwargs): + assert args == ["kubectl", "apply", "-n", namespace, "-f", "-"] + assert input is not None + applied_manifests.append(input) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(cloudnative_pg.subprocess, "run", _fake_run) + + cloudnative_pg.bootstrap_cnpg_tls_secrets(namespace=namespace, cluster_name=cluster_name) + + tls_manifest = next(m for m in applied_manifests if "kubernetes.io/tls" in m) + cert = x509.load_pem_x509_certificate(_decode_tls_cert_pem(tls_manifest)) + + cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + assert cn == "knoe-db-rw" + assert len(cn) <= 64 + + san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + san_dns = san.get_values_for_type(x509.DNSName) + assert f"{cluster_name}-rw.{namespace}.svc" in san_dns + + +def test_bootstrap_cnpg_tls_secrets_keeps_fqdn_cn_when_within_limit(monkeypatch): + namespace = "knoe-db" + cluster_name = "knoe-db" + applied_manifests: list[str] = [] + + monkeypatch.setattr(cloudnative_pg, "_ensure_namespace", lambda *_args, **_kwargs: None) + monkeypatch.setattr(cloudnative_pg, "_kubectl_ok", lambda *_args, **_kwargs: False) + + def _fake_run(args, input=None, **_kwargs): + assert args == ["kubectl", "apply", "-n", namespace, "-f", "-"] + assert input is not None + applied_manifests.append(input) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(cloudnative_pg.subprocess, "run", _fake_run) + + cloudnative_pg.bootstrap_cnpg_tls_secrets(namespace=namespace, cluster_name=cluster_name) + + tls_manifest = next(m for m in applied_manifests if "kubernetes.io/tls" in m) + cert = x509.load_pem_x509_certificate(_decode_tls_cert_pem(tls_manifest)) + + cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + assert cn == f"{cluster_name}.{namespace}.svc" + assert len(cn) <= 64