Fix DB image build and k3s registry/CNPG robustness

- update Percona Dockerfiles for compatible extension/tooling install flow\n- add HTTP/HTTPS-aware k3s registry configuration path across scripts/Ansible\n- harden CNPG TLS bootstrap CN handling for long namespaces and add regression test\n- improve namespace reset pod-deletion wait behavior

Co-authored-by: Junie <junie@jetbrains.com>

etc/init_k3s_registry.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 # init_k3s_registry.sh
 # Purpose:
-# - Configure k3s/containerd to allow HTTPS access to the Prole registry
+# - Configure k3s/containerd to allow access to the Prole registry
 # - Writes /etc/rancher/k3s/registries.yaml on the k3s node
 
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -29,12 +29,9 @@ K3S_REGISTRY_FILE=${K3S_REGISTRY_FILE:-/etc/rancher/k3s/registries.yaml}
 K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-}
 
 if [[ -z "${K3S_REGISTRY_SCHEME}" ]]; then
-  # k3d's local registry is plain HTTP by default.
+  # Internal Prole registry endpoints are plain HTTP by default.
-  if [[ "${K3S_REGISTRY_HOST}" == k3d-* ]]; then
+  # Set K3S_REGISTRY_SCHEME=https explicitly when TLS is configured.
   K3S_REGISTRY_SCHEME="http"
-  else
-    K3S_REGISTRY_SCHEME="https"
-  fi
 fi
 
 ensure_root() {

infrastructure/roles/k3s/defaults/main.yml

@@ -70,6 +70,8 @@ k3s_registry_config_path: /etc/rancher/k3s/registries.yaml
 k3s_registry_host: ""
 k3s_registry_namespace: ""
 k3s_registry_port: 5000
+# Internal Prole registry is HTTP by default; set to "https" when TLS is configured.
+k3s_registry_endpoint_scheme: "http"
 
 # Pulling large images on every run can cause major load spikes; keep this opt-in.
 k3s_prestage_images: false

infrastructure/roles/k3s/tasks/install.yml

@@ -466,11 +466,24 @@
          else inventory_hostname }}
   when: k3s_state == "present" and k3s_registry_config_enabled | bool
 
+- name: Resolve k3s registry endpoint scheme
+  ansible.builtin.set_fact:
+    k3s_registry_endpoint_scheme_resolved: >-
+      {{ 'https' if (k3s_registry_host | default('') | regex_search('^https://'))
+         else 'http' if (k3s_registry_host | default('') | regex_search('^http://'))
+         else (k3s_registry_endpoint_scheme | default('http')) }}
+  when: k3s_state == "present" and k3s_registry_config_enabled | bool
+
 - name: Normalize k3s registry host
   ansible.builtin.set_fact:
     k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}"
   when: k3s_state == "present" and k3s_registry_config_enabled | bool
 
+- name: Normalize k3s registry endpoint scheme
+  ansible.builtin.set_fact:
+    k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}"
+  when: k3s_state == "present" and k3s_registry_config_enabled | bool
+
 - name: Resolve k3s registry namespace
   ansible.builtin.set_fact:
     k3s_registry_namespace_resolved: >-

infrastructure/roles/k3s/tasks/sync.yml

@@ -93,11 +93,24 @@
          else inventory_hostname }}
   when: k3s_registry_config_enabled | bool
 
+- name: Resolve k3s registry endpoint scheme
+  ansible.builtin.set_fact:
+    k3s_registry_endpoint_scheme_resolved: >-
+      {{ 'https' if (k3s_registry_host | default('') | regex_search('^https://'))
+         else 'http' if (k3s_registry_host | default('') | regex_search('^http://'))
+         else (k3s_registry_endpoint_scheme | default('http')) }}
+  when: k3s_registry_config_enabled | bool
+
 - name: Normalize k3s registry host
   ansible.builtin.set_fact:
     k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}"
   when: k3s_registry_config_enabled | bool
 
+- name: Normalize k3s registry endpoint scheme
+  ansible.builtin.set_fact:
+    k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}"
+  when: k3s_registry_config_enabled | bool
+
 - name: Resolve k3s registry namespace
   ansible.builtin.set_fact:
     k3s_registry_namespace_resolved: >-

infrastructure/roles/k3s/templates/registries.yaml.j2

@@ -1,10 +1,12 @@
+{% set _scheme = (k3s_registry_endpoint_scheme_resolved | default(k3s_registry_endpoint_scheme | default('http')) | lower) %}
 mirrors:
   '{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}':
     endpoint:
-      - "https://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
+      - "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
   'registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}':
     endpoint:
-      - "https://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
+      - "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
+{% if _scheme != 'http' %}
 configs:
   '{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}':
     tls:
@@ -12,3 +14,4 @@ configs:
   'registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}':
     tls:
       insecure_skip_verify: true
+{% endif %}

knoe-db/Dockerfile

@@ -12,7 +12,7 @@ RUN set -eux; \
     ca-certificates curl wget gnupg2 lsb-release \
     locales \
     gosu \
-    build-essential git openssh-client \
+    build-essential git openssh-client pkg-config \
   ; \
   rm -rf /var/lib/apt/lists/*; \
   locale-gen en_US.UTF-8
@@ -119,9 +119,17 @@ ENV PATH=/root/.cargo/bin:$PATH
 RUN set -eux; \
   cd /tmp; \
   git clone https://github.com/tembo-io/pgmq.git; \
+  if [ -f pgmq/pgmq-extension/Cargo.toml ]; then \
     cd pgmq/pgmq-extension; \
     cargo pgrx init --pg18=$(which pg_config); \
     cargo pgrx install --release; \
+  elif [ -f pgmq/pgmq-extension/Makefile ]; then \
+    make -C pgmq/pgmq-extension USE_PGXS=1; \
+    make -C pgmq/pgmq-extension USE_PGXS=1 install; \
+  else \
+    echo "Unable to locate pgmq extension build files"; \
+    exit 1; \
+  fi; \
   cd /; \
   rm -rf /tmp/pgmq
 
@@ -135,7 +143,9 @@ RUN set -eux; \
   rm -rf /var/lib/apt/lists/*; \
   cd /tmp; \
   git clone https://github.com/supabase/wrappers.git; \
-  cd wrappers; \
+  cd wrappers/wrappers; \
+  /root/.cargo/bin/cargo install --locked cargo-pgrx --version 0.16.1; \
+  cargo pgrx init --pg18=$(which pg_config); \
   cargo pgrx install --release; \
   cd /; \
   rm -rf /tmp/wrappers

knoe-db/Dockerfile.percona.template

@@ -12,7 +12,7 @@ RUN set -eux; \
     ca-certificates curl wget gnupg2 lsb-release \
     locales \
     gosu \
-    build-essential git openssh-client \
+    build-essential git openssh-client pkg-config \
   ; \
   rm -rf /var/lib/apt/lists/*; \
   locale-gen en_US.UTF-8
@@ -114,9 +114,17 @@ ENV PATH=/root/.cargo/bin:$PATH
 RUN set -eux; \
   cd /tmp; \
   git clone https://github.com/tembo-io/pgmq.git; \
+  if [ -f pgmq/pgmq-extension/Cargo.toml ]; then \
     cd pgmq/pgmq-extension; \
     cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \
     cargo pgrx install --release; \
+  elif [ -f pgmq/pgmq-extension/Makefile ]; then \
+    make -C pgmq/pgmq-extension USE_PGXS=1; \
+    make -C pgmq/pgmq-extension USE_PGXS=1 install; \
+  else \
+    echo "Unable to locate pgmq extension build files"; \
+    exit 1; \
+  fi; \
   cd /; \
   rm -rf /tmp/pgmq
 
@@ -130,7 +138,9 @@ RUN set -eux; \
   rm -rf /var/lib/apt/lists/*; \
   cd /tmp; \
   git clone https://github.com/supabase/wrappers.git; \
-  cd wrappers; \
+  cd wrappers/wrappers; \
+  /root/.cargo/bin/cargo install --locked cargo-pgrx --version 0.16.1; \
+  cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \
   cargo pgrx install --release; \
   cd /; \
   rm -rf /tmp/wrappers

knoe/core/ops/cloudnative_pg.py

@@ -950,7 +950,20 @@ def bootstrap_cnpg_tls_secrets(
         f"{cluster_name}-r",
         f"{cluster_name}-ro",
     ]
-    srv_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, f"{cluster_name}.{namespace}.svc")])
+    primary_srv_cn = f"{cluster_name}.{namespace}.svc"
+    srv_cn = primary_srv_cn
+    if len(srv_cn) > 64:
+        for candidate in (f"{cluster_name}-rw", cluster_name, "cnpg-server"):
+            if len(candidate) <= 64:
+                srv_cn = candidate
+                break
+        _log(
+            log,
+            "CNPG TLS subject CN exceeds X.509 limit; "
+            f"using fallback CN '{srv_cn}' instead of '{primary_srv_cn}'.",
+        )
+
+    srv_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, srv_cn)])
     srv_cert = (
         x509.CertificateBuilder()
         .subject_name(srv_name)

mock_val/init_k3s_registry.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 # init_k3s_registry.sh
 # Purpose:
-# - Configure k3s/containerd to allow HTTPS access to the Prole registry
+# - Configure k3s/containerd to allow access to the Prole registry
 # - Writes /etc/rancher/k3s/registries.yaml on the k3s node
 
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -26,6 +26,7 @@ K3S_REGISTRY_HOST=${K3S_REGISTRY_HOST:-$(registry_host_from_url "${PROLE_K3S_SER
 K3S_REGISTRY_PORT=${K3S_REGISTRY_PORT:-5000}
 K3S_REGISTRY_NAMESPACE=${K3S_REGISTRY_NAMESPACE:-${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}}}
 K3S_REGISTRY_FILE=${K3S_REGISTRY_FILE:-/etc/rancher/k3s/registries.yaml}
+K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-http}
 
 ensure_root() {
   if [[ "$(id -u)" -ne 0 ]]; then
@@ -38,14 +39,27 @@ render_registries_yaml() {
   local host="$1"
   local port="$2"
   local ns="$3"
+  local scheme="$4"
+  if [[ "$scheme" == "http" ]]; then
     cat <<EOF
 mirrors:
   "${host}:${port}":
     endpoint:
-      - "https://${host}:${port}"
+      - "http://${host}:${port}"
   "registry.${ns}.svc.cluster.local:${port}":
     endpoint:
-      - "https://${host}:${port}"
+      - "http://${host}:${port}"
+EOF
+    return 0
+  fi
+  cat <<EOF
+mirrors:
+  "${host}:${port}":
+    endpoint:
+      - "${scheme}://${host}:${port}"
+  "registry.${ns}.svc.cluster.local:${port}":
+    endpoint:
+      - "${scheme}://${host}:${port}"
 configs:
   "${host}:${port}":
     tls:
@@ -64,8 +78,8 @@ case "$ACTION" in
     fi
     ensure_root
     mkdir -p "$(dirname "$K3S_REGISTRY_FILE")"
-    render_registries_yaml "$K3S_REGISTRY_HOST" "$K3S_REGISTRY_PORT" "$K3S_REGISTRY_NAMESPACE" >"$K3S_REGISTRY_FILE"
+    render_registries_yaml "$K3S_REGISTRY_HOST" "$K3S_REGISTRY_PORT" "$K3S_REGISTRY_NAMESPACE" "$K3S_REGISTRY_SCHEME" >"$K3S_REGISTRY_FILE"
-    echo "Wrote $K3S_REGISTRY_FILE for ${K3S_REGISTRY_HOST}:${K3S_REGISTRY_PORT}"
+    echo "Wrote $K3S_REGISTRY_FILE for ${K3S_REGISTRY_SCHEME}://${K3S_REGISTRY_HOST}:${K3S_REGISTRY_PORT}"
     echo "Restart k3s to apply: sudo systemctl restart k3s"
     ;;
   status)

modes/k3s/knoe-db/.version

@@ -1 +1 @@
-143
+145

scripts/reset-ns.sh

@@ -233,11 +233,19 @@ done
 
 echo "Waiting for pods to terminate..."
 if [[ ${#deleted_pods[@]} -gt 0 ]]; then
-  wait_args=()
+  deadline=$((SECONDS + 180))
   for p in "${deleted_pods[@]}"; do
-    wait_args+=("pod/$p")
+    while kubectl -n "$NS" get pod "$p" >/dev/null 2>&1; do
+      if (( SECONDS >= deadline )); then
+        echo "  timeout waiting for pod/$p to terminate; continuing"
+        break
+      fi
+      sleep 2
+    done
+    if ! kubectl -n "$NS" get pod "$p" >/dev/null 2>&1; then
+      echo "  pod/$p terminated"
+    fi
   done
-  kubectl -n "$NS" wait --for=delete "${wait_args[@]}" --timeout=180s 2>/dev/null || true
 fi
 
 if [[ "$CLEAR_NODE_RESERVATIONS" == "true" ]]; then

tests/installer/test_cnpg_tls_bootstrap.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import base64
+from types import SimpleNamespace
+
+from cryptography import x509
+from cryptography.x509.oid import NameOID
+
+from knoe.core.ops import cloudnative_pg
+
+
+def _decode_tls_cert_pem(manifest: str) -> bytes:
+    for line in manifest.splitlines():
+        key, _, value = line.partition(":")
+        if key.strip() == "tls.crt":
+            return base64.b64decode(value.strip())
+    raise AssertionError("tls.crt not found in manifest")
+
+
+def test_bootstrap_cnpg_tls_secrets_falls_back_to_short_cn_for_long_namespace(monkeypatch):
+    namespace = "knoe-db-17-7-043-18-140-17-7-043-18-140-17-7-043-18-140"
+    cluster_name = "knoe-db"
+    applied_manifests: list[str] = []
+
+    monkeypatch.setattr(cloudnative_pg, "_ensure_namespace", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(cloudnative_pg, "_kubectl_ok", lambda *_args, **_kwargs: False)
+
+    def _fake_run(args, input=None, **_kwargs):
+        assert args == ["kubectl", "apply", "-n", namespace, "-f", "-"]
+        assert input is not None
+        applied_manifests.append(input)
+        return SimpleNamespace(returncode=0, stdout="", stderr="")
+
+    monkeypatch.setattr(cloudnative_pg.subprocess, "run", _fake_run)
+
+    cloudnative_pg.bootstrap_cnpg_tls_secrets(namespace=namespace, cluster_name=cluster_name)
+
+    tls_manifest = next(m for m in applied_manifests if "kubernetes.io/tls" in m)
+    cert = x509.load_pem_x509_certificate(_decode_tls_cert_pem(tls_manifest))
+
+    cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
+    assert cn == "knoe-db-rw"
+    assert len(cn) <= 64
+
+    san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
+    san_dns = san.get_values_for_type(x509.DNSName)
+    assert f"{cluster_name}-rw.{namespace}.svc" in san_dns
+
+
+def test_bootstrap_cnpg_tls_secrets_keeps_fqdn_cn_when_within_limit(monkeypatch):
+    namespace = "knoe-db"
+    cluster_name = "knoe-db"
+    applied_manifests: list[str] = []
+
+    monkeypatch.setattr(cloudnative_pg, "_ensure_namespace", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(cloudnative_pg, "_kubectl_ok", lambda *_args, **_kwargs: False)
+
+    def _fake_run(args, input=None, **_kwargs):
+        assert args == ["kubectl", "apply", "-n", namespace, "-f", "-"]
+        assert input is not None
+        applied_manifests.append(input)
+        return SimpleNamespace(returncode=0, stdout="", stderr="")
+
+    monkeypatch.setattr(cloudnative_pg.subprocess, "run", _fake_run)
+
+    cloudnative_pg.bootstrap_cnpg_tls_secrets(namespace=namespace, cluster_name=cluster_name)
+
+    tls_manifest = next(m for m in applied_manifests if "kubernetes.io/tls" in m)
+    cert = x509.load_pem_x509_certificate(_decode_tls_cert_pem(tls_manifest))
+
+    cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
+    assert cn == f"{cluster_name}.{namespace}.svc"
+    assert len(cn) <= 64