mirror of
https://github.com/dredx/prole.git
synced 2026-09-23 11:03:59 +00:00
Fix DB image build and k3s registry/CNPG robustness
- update Percona Dockerfiles for compatible extension/tooling install flow\n- add HTTP/HTTPS-aware k3s registry configuration path across scripts/Ansible\n- harden CNPG TLS bootstrap CN handling for long namespaces and add regression test\n- improve namespace reset pod-deletion wait behavior Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
chrisfu
parent
d02aadae89
commit
fdc1582bd0
@ -4,7 +4,7 @@ set -euo pipefail
|
||||
|
||||
# init_k3s_registry.sh
|
||||
# Purpose:
|
||||
# - Configure k3s/containerd to allow HTTPS access to the Prole registry
|
||||
# - Configure k3s/containerd to allow access to the Prole registry
|
||||
# - Writes /etc/rancher/k3s/registries.yaml on the k3s node
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
@ -29,12 +29,9 @@ K3S_REGISTRY_FILE=${K3S_REGISTRY_FILE:-/etc/rancher/k3s/registries.yaml}
|
||||
K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-}
|
||||
|
||||
if [[ -z "${K3S_REGISTRY_SCHEME}" ]]; then
|
||||
# k3d's local registry is plain HTTP by default.
|
||||
if [[ "${K3S_REGISTRY_HOST}" == k3d-* ]]; then
|
||||
K3S_REGISTRY_SCHEME="http"
|
||||
else
|
||||
K3S_REGISTRY_SCHEME="https"
|
||||
fi
|
||||
# Internal Prole registry endpoints are plain HTTP by default.
|
||||
# Set K3S_REGISTRY_SCHEME=https explicitly when TLS is configured.
|
||||
K3S_REGISTRY_SCHEME="http"
|
||||
fi
|
||||
|
||||
ensure_root() {
|
||||
|
||||
@ -70,6 +70,8 @@ k3s_registry_config_path: /etc/rancher/k3s/registries.yaml
|
||||
k3s_registry_host: ""
|
||||
k3s_registry_namespace: ""
|
||||
k3s_registry_port: 5000
|
||||
# Internal Prole registry is HTTP by default; set to "https" when TLS is configured.
|
||||
k3s_registry_endpoint_scheme: "http"
|
||||
|
||||
# Pulling large images on every run can cause major load spikes; keep this opt-in.
|
||||
k3s_prestage_images: false
|
||||
|
||||
@ -466,11 +466,24 @@
|
||||
else inventory_hostname }}
|
||||
when: k3s_state == "present" and k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Resolve k3s registry endpoint scheme
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_endpoint_scheme_resolved: >-
|
||||
{{ 'https' if (k3s_registry_host | default('') | regex_search('^https://'))
|
||||
else 'http' if (k3s_registry_host | default('') | regex_search('^http://'))
|
||||
else (k3s_registry_endpoint_scheme | default('http')) }}
|
||||
when: k3s_state == "present" and k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Normalize k3s registry host
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}"
|
||||
when: k3s_state == "present" and k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Normalize k3s registry endpoint scheme
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}"
|
||||
when: k3s_state == "present" and k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Resolve k3s registry namespace
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_namespace_resolved: >-
|
||||
|
||||
@ -93,11 +93,24 @@
|
||||
else inventory_hostname }}
|
||||
when: k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Resolve k3s registry endpoint scheme
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_endpoint_scheme_resolved: >-
|
||||
{{ 'https' if (k3s_registry_host | default('') | regex_search('^https://'))
|
||||
else 'http' if (k3s_registry_host | default('') | regex_search('^http://'))
|
||||
else (k3s_registry_endpoint_scheme | default('http')) }}
|
||||
when: k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Normalize k3s registry host
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_host_resolved: "{{ k3s_registry_host_resolved | regex_replace('^https?://', '') | regex_replace('/.*$', '') | regex_replace(':.*$', '') }}"
|
||||
when: k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Normalize k3s registry endpoint scheme
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_endpoint_scheme_resolved: "{{ (k3s_registry_endpoint_scheme_resolved | default('http') | lower) if (k3s_registry_endpoint_scheme_resolved | default('http') | lower) in ['http', 'https'] else 'http' }}"
|
||||
when: k3s_registry_config_enabled | bool
|
||||
|
||||
- name: Resolve k3s registry namespace
|
||||
ansible.builtin.set_fact:
|
||||
k3s_registry_namespace_resolved: >-
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
{% set _scheme = (k3s_registry_endpoint_scheme_resolved | default(k3s_registry_endpoint_scheme | default('http')) | lower) %}
|
||||
mirrors:
|
||||
'{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}':
|
||||
endpoint:
|
||||
- "https://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
|
||||
- "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
|
||||
'registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}':
|
||||
endpoint:
|
||||
- "https://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
|
||||
- "{{ _scheme }}://{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}"
|
||||
{% if _scheme != 'http' %}
|
||||
configs:
|
||||
'{{ k3s_registry_host_resolved }}:{{ k3s_registry_port }}':
|
||||
tls:
|
||||
@ -12,3 +14,4 @@ configs:
|
||||
'registry.{{ k3s_registry_namespace_resolved }}.svc.cluster.local:{{ k3s_registry_port }}':
|
||||
tls:
|
||||
insecure_skip_verify: true
|
||||
{% endif %}
|
||||
|
||||
@ -12,7 +12,7 @@ RUN set -eux; \
|
||||
ca-certificates curl wget gnupg2 lsb-release \
|
||||
locales \
|
||||
gosu \
|
||||
build-essential git openssh-client \
|
||||
build-essential git openssh-client pkg-config \
|
||||
; \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
locale-gen en_US.UTF-8
|
||||
@ -119,9 +119,17 @@ ENV PATH=/root/.cargo/bin:$PATH
|
||||
RUN set -eux; \
|
||||
cd /tmp; \
|
||||
git clone https://github.com/tembo-io/pgmq.git; \
|
||||
cd pgmq/pgmq-extension; \
|
||||
cargo pgrx init --pg18=$(which pg_config); \
|
||||
cargo pgrx install --release; \
|
||||
if [ -f pgmq/pgmq-extension/Cargo.toml ]; then \
|
||||
cd pgmq/pgmq-extension; \
|
||||
cargo pgrx init --pg18=$(which pg_config); \
|
||||
cargo pgrx install --release; \
|
||||
elif [ -f pgmq/pgmq-extension/Makefile ]; then \
|
||||
make -C pgmq/pgmq-extension USE_PGXS=1; \
|
||||
make -C pgmq/pgmq-extension USE_PGXS=1 install; \
|
||||
else \
|
||||
echo "Unable to locate pgmq extension build files"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
cd /; \
|
||||
rm -rf /tmp/pgmq
|
||||
|
||||
@ -135,7 +143,9 @@ RUN set -eux; \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
cd /tmp; \
|
||||
git clone https://github.com/supabase/wrappers.git; \
|
||||
cd wrappers; \
|
||||
cd wrappers/wrappers; \
|
||||
/root/.cargo/bin/cargo install --locked cargo-pgrx --version 0.16.1; \
|
||||
cargo pgrx init --pg18=$(which pg_config); \
|
||||
cargo pgrx install --release; \
|
||||
cd /; \
|
||||
rm -rf /tmp/wrappers
|
||||
|
||||
@ -12,7 +12,7 @@ RUN set -eux; \
|
||||
ca-certificates curl wget gnupg2 lsb-release \
|
||||
locales \
|
||||
gosu \
|
||||
build-essential git openssh-client \
|
||||
build-essential git openssh-client pkg-config \
|
||||
; \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
locale-gen en_US.UTF-8
|
||||
@ -114,9 +114,17 @@ ENV PATH=/root/.cargo/bin:$PATH
|
||||
RUN set -eux; \
|
||||
cd /tmp; \
|
||||
git clone https://github.com/tembo-io/pgmq.git; \
|
||||
cd pgmq/pgmq-extension; \
|
||||
cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \
|
||||
cargo pgrx install --release; \
|
||||
if [ -f pgmq/pgmq-extension/Cargo.toml ]; then \
|
||||
cd pgmq/pgmq-extension; \
|
||||
cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \
|
||||
cargo pgrx install --release; \
|
||||
elif [ -f pgmq/pgmq-extension/Makefile ]; then \
|
||||
make -C pgmq/pgmq-extension USE_PGXS=1; \
|
||||
make -C pgmq/pgmq-extension USE_PGXS=1 install; \
|
||||
else \
|
||||
echo "Unable to locate pgmq extension build files"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
cd /; \
|
||||
rm -rf /tmp/pgmq
|
||||
|
||||
@ -130,7 +138,9 @@ RUN set -eux; \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
cd /tmp; \
|
||||
git clone https://github.com/supabase/wrappers.git; \
|
||||
cd wrappers; \
|
||||
cd wrappers/wrappers; \
|
||||
/root/.cargo/bin/cargo install --locked cargo-pgrx --version 0.16.1; \
|
||||
cargo pgrx init --pg{{MAJOR_VERSION}}=$(which pg_config); \
|
||||
cargo pgrx install --release; \
|
||||
cd /; \
|
||||
rm -rf /tmp/wrappers
|
||||
|
||||
@ -950,7 +950,20 @@ def bootstrap_cnpg_tls_secrets(
|
||||
f"{cluster_name}-r",
|
||||
f"{cluster_name}-ro",
|
||||
]
|
||||
srv_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, f"{cluster_name}.{namespace}.svc")])
|
||||
primary_srv_cn = f"{cluster_name}.{namespace}.svc"
|
||||
srv_cn = primary_srv_cn
|
||||
if len(srv_cn) > 64:
|
||||
for candidate in (f"{cluster_name}-rw", cluster_name, "cnpg-server"):
|
||||
if len(candidate) <= 64:
|
||||
srv_cn = candidate
|
||||
break
|
||||
_log(
|
||||
log,
|
||||
"CNPG TLS subject CN exceeds X.509 limit; "
|
||||
f"using fallback CN '{srv_cn}' instead of '{primary_srv_cn}'.",
|
||||
)
|
||||
|
||||
srv_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, srv_cn)])
|
||||
srv_cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(srv_name)
|
||||
|
||||
@ -4,7 +4,7 @@ set -euo pipefail
|
||||
|
||||
# init_k3s_registry.sh
|
||||
# Purpose:
|
||||
# - Configure k3s/containerd to allow HTTPS access to the Prole registry
|
||||
# - Configure k3s/containerd to allow access to the Prole registry
|
||||
# - Writes /etc/rancher/k3s/registries.yaml on the k3s node
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
@ -22,10 +22,11 @@ registry_host_from_url() {
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
K3S_REGISTRY_HOST=${K3S_REGISTRY_HOST:-$(registry_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")}
|
||||
K3S_REGISTRY_HOST=${K3S_REGISTRY_HOST:-$(registry_host_from_url "${PROLE_K3S_SERVER:-${K3S_SERVER_URL:-}}")}
|
||||
K3S_REGISTRY_PORT=${K3S_REGISTRY_PORT:-5000}
|
||||
K3S_REGISTRY_NAMESPACE=${K3S_REGISTRY_NAMESPACE:-${REGISTRY_NAMESPACE:-${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-default}}}}
|
||||
K3S_REGISTRY_FILE=${K3S_REGISTRY_FILE:-/etc/rancher/k3s/registries.yaml}
|
||||
K3S_REGISTRY_SCHEME=${K3S_REGISTRY_SCHEME:-http}
|
||||
|
||||
ensure_root() {
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
@ -38,14 +39,27 @@ render_registries_yaml() {
|
||||
local host="$1"
|
||||
local port="$2"
|
||||
local ns="$3"
|
||||
local scheme="$4"
|
||||
if [[ "$scheme" == "http" ]]; then
|
||||
cat <<EOF
|
||||
mirrors:
|
||||
"${host}:${port}":
|
||||
endpoint:
|
||||
- "http://${host}:${port}"
|
||||
"registry.${ns}.svc.cluster.local:${port}":
|
||||
endpoint:
|
||||
- "http://${host}:${port}"
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
cat <<EOF
|
||||
mirrors:
|
||||
"${host}:${port}":
|
||||
endpoint:
|
||||
- "https://${host}:${port}"
|
||||
- "${scheme}://${host}:${port}"
|
||||
"registry.${ns}.svc.cluster.local:${port}":
|
||||
endpoint:
|
||||
- "https://${host}:${port}"
|
||||
- "${scheme}://${host}:${port}"
|
||||
configs:
|
||||
"${host}:${port}":
|
||||
tls:
|
||||
@ -64,8 +78,8 @@ case "$ACTION" in
|
||||
fi
|
||||
ensure_root
|
||||
mkdir -p "$(dirname "$K3S_REGISTRY_FILE")"
|
||||
render_registries_yaml "$K3S_REGISTRY_HOST" "$K3S_REGISTRY_PORT" "$K3S_REGISTRY_NAMESPACE" >"$K3S_REGISTRY_FILE"
|
||||
echo "Wrote $K3S_REGISTRY_FILE for ${K3S_REGISTRY_HOST}:${K3S_REGISTRY_PORT}"
|
||||
render_registries_yaml "$K3S_REGISTRY_HOST" "$K3S_REGISTRY_PORT" "$K3S_REGISTRY_NAMESPACE" "$K3S_REGISTRY_SCHEME" >"$K3S_REGISTRY_FILE"
|
||||
echo "Wrote $K3S_REGISTRY_FILE for ${K3S_REGISTRY_SCHEME}://${K3S_REGISTRY_HOST}:${K3S_REGISTRY_PORT}"
|
||||
echo "Restart k3s to apply: sudo systemctl restart k3s"
|
||||
;;
|
||||
status)
|
||||
|
||||
@ -1 +1 @@
|
||||
143
|
||||
145
|
||||
@ -233,11 +233,19 @@ done
|
||||
|
||||
echo "Waiting for pods to terminate..."
|
||||
if [[ ${#deleted_pods[@]} -gt 0 ]]; then
|
||||
wait_args=()
|
||||
deadline=$((SECONDS + 180))
|
||||
for p in "${deleted_pods[@]}"; do
|
||||
wait_args+=("pod/$p")
|
||||
while kubectl -n "$NS" get pod "$p" >/dev/null 2>&1; do
|
||||
if (( SECONDS >= deadline )); then
|
||||
echo " timeout waiting for pod/$p to terminate; continuing"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if ! kubectl -n "$NS" get pod "$p" >/dev/null 2>&1; then
|
||||
echo " pod/$p terminated"
|
||||
fi
|
||||
done
|
||||
kubectl -n "$NS" wait --for=delete "${wait_args[@]}" --timeout=180s 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [[ "$CLEAR_NODE_RESERVATIONS" == "true" ]]; then
|
||||
|
||||
73
tests/installer/test_cnpg_tls_bootstrap.py
Normal file
73
tests/installer/test_cnpg_tls_bootstrap.py
Normal file
@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.x509.oid import NameOID
|
||||
|
||||
from knoe.core.ops import cloudnative_pg
|
||||
|
||||
|
||||
def _decode_tls_cert_pem(manifest: str) -> bytes:
|
||||
for line in manifest.splitlines():
|
||||
key, _, value = line.partition(":")
|
||||
if key.strip() == "tls.crt":
|
||||
return base64.b64decode(value.strip())
|
||||
raise AssertionError("tls.crt not found in manifest")
|
||||
|
||||
|
||||
def test_bootstrap_cnpg_tls_secrets_falls_back_to_short_cn_for_long_namespace(monkeypatch):
|
||||
namespace = "knoe-db-17-7-043-18-140-17-7-043-18-140-17-7-043-18-140"
|
||||
cluster_name = "knoe-db"
|
||||
applied_manifests: list[str] = []
|
||||
|
||||
monkeypatch.setattr(cloudnative_pg, "_ensure_namespace", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(cloudnative_pg, "_kubectl_ok", lambda *_args, **_kwargs: False)
|
||||
|
||||
def _fake_run(args, input=None, **_kwargs):
|
||||
assert args == ["kubectl", "apply", "-n", namespace, "-f", "-"]
|
||||
assert input is not None
|
||||
applied_manifests.append(input)
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(cloudnative_pg.subprocess, "run", _fake_run)
|
||||
|
||||
cloudnative_pg.bootstrap_cnpg_tls_secrets(namespace=namespace, cluster_name=cluster_name)
|
||||
|
||||
tls_manifest = next(m for m in applied_manifests if "kubernetes.io/tls" in m)
|
||||
cert = x509.load_pem_x509_certificate(_decode_tls_cert_pem(tls_manifest))
|
||||
|
||||
cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
|
||||
assert cn == "knoe-db-rw"
|
||||
assert len(cn) <= 64
|
||||
|
||||
san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
|
||||
san_dns = san.get_values_for_type(x509.DNSName)
|
||||
assert f"{cluster_name}-rw.{namespace}.svc" in san_dns
|
||||
|
||||
|
||||
def test_bootstrap_cnpg_tls_secrets_keeps_fqdn_cn_when_within_limit(monkeypatch):
|
||||
namespace = "knoe-db"
|
||||
cluster_name = "knoe-db"
|
||||
applied_manifests: list[str] = []
|
||||
|
||||
monkeypatch.setattr(cloudnative_pg, "_ensure_namespace", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(cloudnative_pg, "_kubectl_ok", lambda *_args, **_kwargs: False)
|
||||
|
||||
def _fake_run(args, input=None, **_kwargs):
|
||||
assert args == ["kubectl", "apply", "-n", namespace, "-f", "-"]
|
||||
assert input is not None
|
||||
applied_manifests.append(input)
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(cloudnative_pg.subprocess, "run", _fake_run)
|
||||
|
||||
cloudnative_pg.bootstrap_cnpg_tls_secrets(namespace=namespace, cluster_name=cluster_name)
|
||||
|
||||
tls_manifest = next(m for m in applied_manifests if "kubernetes.io/tls" in m)
|
||||
cert = x509.load_pem_x509_certificate(_decode_tls_cert_pem(tls_manifest))
|
||||
|
||||
cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
|
||||
assert cn == f"{cluster_name}.{namespace}.svc"
|
||||
assert len(cn) <= 64
|
||||
Loading…
Reference in New Issue
Block a user