prole/etc/init_registry.sh
chrisfu e10d4f3782 fix(k3d): fix prole-registry squatting port 5000 in init_registry.sh
The shell path (init_registry.sh apply_registry) is the real k3d registry
entry point. It checked 'k3d registry list knoe-registry' which returned 0
because k3d-knoe-registry existed in 'created' state, so the create was
skipped and k3d-prole-registry (the port-5000 squatter) was never touched.

- init_registry.sh: before the knoe-registry ensure, detect k3d-prole-registry
  via both k3d and docker inspect, then stop/rm the Docker container regardless
  of whether k3d manages it; also detect knoe-registry in non-running state and
  nuke+recreate it (port was unavailable on the previous attempt)
- k3d_registry.py: same cleanup logic for the Python fallback path

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-03 00:43:54 -07:00

578 lines
20 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
# init_registry.sh
# Purpose:
# - Deploy registry support for local clusters
# - k3s: deploy in-cluster `registry:2` into the service namespace
# - k3d: create/connect a `k3d` registry on port 5000
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Shared option parsing for common core scripts
# shellcheck disable=SC1090
source "$SCRIPT_DIR/common_core_lib.sh"
# Inject default config if not provided
_has_config=0
for _arg in "$@"; do
[[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1
done
if [[ $_has_config -eq 0 ]]; then
_default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)"
if [[ -n "$_default_cfg" ]]; then
set -- "-c" "$_default_cfg" "$@"
fi
fi
unset _has_config _arg _default_cfg
common_core_preparse_config "$@"
# Remove -c/--config from the remaining args now that KNOE_CONF is set.
# `common_core_preparse_config` exposes the stripped args as `COMMON_CORE_ARGS`.
set -- "${COMMON_CORE_ARGS[@]}"
# Load environment and config via knoe_cfg.sh
# shellcheck disable=SC1090
source "$SCRIPT_DIR/knoe_cfg.sh"
if [[ -z "${KNOE_SERVICE:-}" ]]; then
echo "ERROR: KNOE_SERVICE is not defined. Provide KNOE_HOME/env.sh or ~/.knoe/env.sh" >&2
exit 1
fi
ACTION=""
NAMESPACE_OVERRIDE=""
REGISTRY_NAMESPACE_OVERRIDE=""
if [[ -d "$SCRIPT_DIR/../k8s/registry" ]]; then
REGISTRY_MANIFEST_DIR="$SCRIPT_DIR/../k8s/registry"
elif [[ -n "${KNOE_HOME:-}" && -d "$KNOE_HOME/k8s/registry" ]]; then
REGISTRY_MANIFEST_DIR="$KNOE_HOME/k8s/registry"
else
REGISTRY_MANIFEST_DIR="$SCRIPT_DIR/../k8s/registry"
fi
REGISTRY_MANIFEST_FILE=${REGISTRY_MANIFEST_FILE:-"$REGISTRY_MANIFEST_DIR/deployment.yaml"}
usage() {
cat <<EOF
Usage: init_registry.sh [-n|--namespace NS] [-r|--registry-namespace NS] <start|stop|status|restart|initialize|update|reload|migrate>
Deploys registry support for local clusters.
- In k3s mode, deploys `registry:2` into the service namespace (default: SERVICE_NAMESPACE).
- In k3d mode, manages the `k3d` registry (port 5000) and removes any in-cluster registry resources.
Actions:
start|initialize|update|reload|restart Deploy or update the registry
stop Remove registry resources
status Show registry pod/service status
migrate Copy all images from registry:2 → gitlab-registry.
Run AFTER GitLab is up, BEFORE 'stop'.
Uses skopeo if available, otherwise prints commands.
NOTE: `-r/--registry-namespace` is kept for backwards compatibility and is treated as an alias for `-n/--namespace`.
EOF
}
while [[ $# -gt 0 ]]; do
case "$1" in
-m|--mode)
shift
knoe_set_mode "${1:-}"
;;
-m=*|--mode=*)
knoe_set_mode "${1#*=}"
;;
-n|--namespace)
shift
if [[ -z "${1:-}" ]]; then
echo "ERROR: -n/--namespace requires a value" >&2
exit 2
fi
NAMESPACE_OVERRIDE="$1"
;;
-n=*|--namespace=*)
NAMESPACE_OVERRIDE="${1#*=}"
;;
-r|--registry-namespace)
shift
if [[ -z "${1:-}" ]]; then
echo "ERROR: -r/--registry-namespace requires a value" >&2
exit 2
fi
REGISTRY_NAMESPACE_OVERRIDE="$1"
;;
-r=*|--registry-namespace=*)
REGISTRY_NAMESPACE_OVERRIDE="${1#*=}"
;;
start|stop|status|restart|initialize|update|reload)
ACTION="$1"
;;
-h|--help)
usage
exit 0
;;
*)
if [[ -z "$ACTION" && "$1" != -* ]]; then
ACTION="$1"
else
echo "Unknown argument: $1" >&2
usage
exit 2
fi
;;
esac
shift
done
if [[ -n "$REGISTRY_NAMESPACE_OVERRIDE" ]]; then
REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE_OVERRIDE"
elif [[ -n "$NAMESPACE_OVERRIDE" ]]; then
REGISTRY_NAMESPACE="$NAMESPACE_OVERRIDE"
elif [[ -n "${REGISTRY_NAMESPACE:-}" ]]; then
REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE"
else
# Default to the configured service namespace (driven by knoe.cfg via knoe_cfg.sh).
REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-}}"
if [[ -z "$REGISTRY_NAMESPACE" ]]; then
echo "ERROR: REGISTRY_NAMESPACE could not be determined. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2
exit 1
fi
fi
# In k3s mode, the in-cluster registry is a common core service and should live
# in the common services namespace.
_mode_resolved="${KNOE_MODE:-${DEPLOYMENT_MODE:-}}"
if declare -F knoe_normalize_mode >/dev/null 2>&1; then
_mode_resolved="$(knoe_normalize_mode "$_mode_resolved")"
fi
if [[ "$_mode_resolved" == "k3s" && -z "${REGISTRY_NAMESPACE:-}" ]]; then
REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-}}"
if [[ -z "$REGISTRY_NAMESPACE" ]]; then
echo "ERROR: REGISTRY_NAMESPACE could not be determined for k3s mode. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2
exit 1
fi
fi
unset _mode_resolved
export REGISTRY_NAMESPACE
REGISTRY_NODE_SELECTOR=${REGISTRY_NODE_SELECTOR:-}
current_mode() {
if command -v knoe_normalize_mode >/dev/null 2>&1; then
knoe_normalize_mode "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}"
return 0
fi
printf '%s' "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}"
}
registry_enabled() {
local mode
mode=$(current_mode)
[[ "$mode" == "k3s" || "$mode" == "k3d" ]]
}
resolve_control_plane_node_selector() {
local node=""
node=$(kubectl get nodes -l 'node-role.kubernetes.io/control-plane' -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -n "$node" ]]; then
printf 'kubernetes.io/hostname=%s' "$node"
fi
}
resolve_registry_node_selector() {
if [[ -n "$REGISTRY_NODE_SELECTOR" ]]; then
printf '%s' "$REGISTRY_NODE_SELECTOR"
return 0
fi
# Default: pin to the control-plane node so hostPort:5000 is reachable via
# the k3s server host (e.g. myrddin.prole.org:5000).
resolve_control_plane_node_selector
}
apply_registry_node_selector() {
local selector
selector="$(resolve_registry_node_selector)"
if [[ -z "$selector" ]]; then
return 0
fi
local key="${selector%%=*}"
local val="${selector#*=}"
if [[ -z "$key" || -z "$val" ]]; then
echo "WARN: invalid REGISTRY_NODE_SELECTOR '$selector' (expected key=value); skipping." >&2
return 0
fi
kubectl -n "$REGISTRY_NAMESPACE" patch deployment/registry \
--type merge \
-p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"$key\":\"$val\"}}}}}" >/dev/null 2>&1 || true
}
ensure_tools() {
command -v kubectl >/dev/null || { echo "Missing required tool: kubectl" >&2; exit 1; }
}
ensure_namespace() {
local ns="$1"
if [[ -z "$ns" ]]; then
return 0
fi
if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
echo "Creating namespace '$ns' ..."
kubectl create namespace "$ns" >/dev/null 2>&1 || true
fi
}
render_registry_manifest() {
if [[ ! -f "$REGISTRY_MANIFEST_FILE" ]]; then
echo "ERROR: Registry manifest not found: $REGISTRY_MANIFEST_FILE" >&2
exit 1
fi
knoe_render_manifest "$REGISTRY_MANIFEST_FILE"
}
cleanup_registry_k8s() {
render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found >/dev/null 2>&1 || true
}
configure_k3d_insecure_registry() {
local cluster_name="${1:-knoe-dev-cluster}"
local reg_host="k3d-knoe-registry"
local reg_port="5000"
local registries_yaml
registries_yaml=$(cat <<YAML
mirrors:
"${reg_host}:${reg_port}":
endpoint:
- "http://${reg_host}:${reg_port}"
YAML
)
echo "Configuring k3d nodes to use ${reg_host}:${reg_port} over HTTP ..."
local node
for node in $(docker ps --filter "label=k3d.cluster=${cluster_name}" --format '{{.Names}}'); do
docker exec "$node" sh -c "mkdir -p /etc/rancher/k3s && cat > /etc/rancher/k3s/registries.yaml <<'EOF'
${registries_yaml}
EOF" || { echo "WARN: failed to write registries.yaml to $node" >&2; continue; }
# Restart k3s inside the node so it picks up the new mirror config.
docker restart "$node" >/dev/null 2>&1 || true
echo " configured $node"
done
# Wait briefly for nodes to come back up
sleep 5
echo "Waiting for k3d nodes to become Ready ..."
kubectl wait --for=condition=Ready nodes --all --timeout=${ROLLOUT_TIMEOUT:-300s} 2>/dev/null || true
}
resolve_k3d_cluster_name() {
# Prefer an explicitly provided cluster name.
if [[ -n "${K3D_CLUSTER_NAME:-}" ]]; then
printf '%s' "$K3D_CLUSTER_NAME"
return 0
fi
# If kubectl is already pointed at a k3d context, derive the name from it.
local ctx
ctx=$(kubectl config current-context 2>/dev/null || true)
if [[ "${ctx:-}" == k3d-* ]]; then
printf '%s' "${ctx#k3d-}"
return 0
fi
# If there is exactly one k3d cluster, pick it.
if command -v k3d >/dev/null 2>&1; then
local clusters count
clusters=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | grep -v '^$' || true)
count=$(printf '%s\n' "$clusters" | grep -c . 2>/dev/null || true)
if [[ "${count:-0}" -eq 1 ]]; then
printf '%s' "$clusters"
return 0
fi
fi
# Stable fallback.
printf '%s' "knoe-dev-cluster"
}
resolve_k3d_network_name() {
local cluster_name="$1"
local net_name="k3d-${cluster_name}"
if command -v docker >/dev/null 2>&1 && docker network inspect "$net_name" >/dev/null 2>&1; then
printf '%s' "$net_name"
return 0
fi
# Fallback: infer from the first server node if available.
local node_name="k3d-${cluster_name}-server-0"
if command -v docker >/dev/null 2>&1 && docker inspect "$node_name" >/dev/null 2>&1; then
docker inspect "$node_name" -f '{{range $k, $v := .NetworkSettings.Networks}}{{println $k}}{{end}}' 2>/dev/null | head -1 || true
return 0
fi
printf '%s' ""
}
ensure_k3d_registry_dns() {
local cluster_name="$1"
local reg_host="${2:-k3d-knoe-registry}"
command -v docker >/dev/null 2>&1 || return 0
local node_name="k3d-${cluster_name}-server-0"
if ! docker inspect "$node_name" >/dev/null 2>&1; then
return 0
fi
# Validate that the node can resolve the registry hostname via Docker DNS.
if docker exec "$node_name" sh -c "command -v getent >/dev/null 2>&1 && getent hosts '$reg_host' >/dev/null 2>&1"; then
return 0
fi
if docker exec "$node_name" sh -c "ping -c 1 -W 1 '$reg_host' >/dev/null 2>&1"; then
return 0
fi
echo "WARN: k3d node '${node_name}' cannot resolve registry host '${reg_host}'." >&2
echo " This will cause image pulls like '${reg_host}:5000/<image>' to fail." >&2
return 1
}
apply_registry() {
local mode
mode=$(current_mode)
if [[ "$mode" == "k3d" ]]; then
if command -v k3d >/dev/null 2>&1; then
# Remove legacy prole-registry that may be squatting on port 5000.
# It may be k3d-managed (delete via k3d) or a bare Docker container (rm -f).
if k3d registry list prole-registry >/dev/null 2>&1 \
|| docker inspect k3d-prole-registry >/dev/null 2>&1; then
echo "Removing legacy k3d-prole-registry from port 5000 ..."
k3d registry delete prole-registry >/dev/null 2>&1 || true
docker stop k3d-prole-registry >/dev/null 2>&1 || true
docker rm -f k3d-prole-registry >/dev/null 2>&1 || true
fi
echo "Ensuring k3d registry 'knoe-registry' on port 5000 ..."
# If knoe-registry was created but never started (port was taken), nuke and recreate.
if k3d registry list knoe-registry >/dev/null 2>&1; then
if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -qE '^k3d-knoe-registry$'; then
echo "k3d-knoe-registry exists but is not running; recreating ..."
k3d registry delete knoe-registry >/dev/null 2>&1 || true
docker rm -f k3d-knoe-registry >/dev/null 2>&1 || true
k3d registry create knoe-registry --port 5000 || true
fi
else
k3d registry create knoe-registry --port 5000 || true
fi
local cluster_name
cluster_name=$(resolve_k3d_cluster_name)
if k3d cluster list "$cluster_name" >/dev/null 2>&1; then
echo "Connecting knoe-registry to $cluster_name ..."
# k3d v5+ uses 'k3d-NAME' as container name for registries
local reg_container="k3d-knoe-registry"
local net_name
net_name=$(resolve_k3d_network_name "$cluster_name")
if command -v docker >/dev/null 2>&1; then
if [[ -n "${net_name:-}" ]] && docker inspect "$reg_container" >/dev/null 2>&1 && docker network inspect "$net_name" >/dev/null 2>&1; then
if ! docker inspect "$reg_container" -f '{{json .NetworkSettings.Networks}}' | grep -q "$net_name"; then
# Ensure the registry is reachable at the stable hostname used in manifests.
docker network connect --alias k3d-knoe-registry "$net_name" "$reg_container" || true
fi
fi
fi
fi
# Configure k3d nodes to use the registry over plain HTTP (insecure).
configure_k3d_insecure_registry "$cluster_name"
# Validate registry DNS from a node so image pulls won't fail later.
ensure_k3d_registry_dns "$cluster_name" "k3d-knoe-registry" || true
# Ensure no k8s registry resources remain in k3d clusters.
echo "Removing k8s registry resources (k3d uses k3d-knoe-registry only) ..."
cleanup_registry_k8s
return 0
else
echo "k3d not found; falling back to Docker for registry ..."
fi
fi
# Fallback or k3s mode: use Docker container if k8s deploy fails or as alternative
if [[ "$mode" == "k3d" ]] || [[ "${PROLE_REGISTRY_STANDALONE:-}" == "1" ]]; then
if command -v docker >/dev/null 2>&1; then
if ! docker ps --format '{{.Names}}' | grep -q "^knoe-registry$"; then
if docker ps -a --format '{{.Names}}' | grep -q "^knoe-registry$"; then
docker start knoe-registry || true
else
docker run -d -p 5000:5000 --restart always --name knoe-registry registry:2 || true
fi
fi
if [[ "$mode" == "k3d" ]]; then
echo "Removing k8s registry resources (k3d uses k3d-knoe-registry only) ..."
cleanup_registry_k8s
fi
return 0
fi
fi
# In k3s mode, if GitLab is deployed it owns port 5000 (gitlab-registry).
# Deploying registry:2 with hostPort:5000 on the same node would conflict.
# Skip the apply and warn; use init_registry.sh stop to remove the old deployment.
local _mode_for_gitlab_check
_mode_for_gitlab_check=$(current_mode)
if [[ "$_mode_for_gitlab_check" == "k3s" ]]; then
local _gitlab_ns="${GITLAB_NAMESPACE:-gitlab}"
if kubectl get namespace "$_gitlab_ns" >/dev/null 2>&1; then
echo "WARN: GitLab namespace '${_gitlab_ns}' detected in k3s mode." >&2
echo " gitlab-registry owns port 5000 — registry:2 deployment skipped." >&2
echo " Run 'init_registry.sh stop' to remove any existing registry:2 resources." >&2
return 0
fi
fi
# Idempotency check: skip apply+wait if the registry is already available.
local _desired _available
_desired=$(kubectl get deploy/registry -n "$REGISTRY_NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
_available=$(kubectl get deploy/registry -n "$REGISTRY_NAMESPACE" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)
if [[ -n "$_desired" && "${_available:-0}" -ge "${_desired:-1}" && "${_desired:-0}" -gt 0 ]]; then
echo "Registry already running in namespace '$REGISTRY_NAMESPACE' ($_available/$_desired replicas available); skipping apply."
return 0
fi
echo "Applying registry manifest to namespace '$REGISTRY_NAMESPACE' ..."
render_registry_manifest | kubectl apply --server-side --force-conflicts --field-manager=knoe-installer --validate=false -n "$REGISTRY_NAMESPACE" -f -
apply_registry_node_selector
kubectl rollout status deploy/registry -n "$REGISTRY_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true
}
delete_registry() {
local mode
mode=$(current_mode)
if [[ "$mode" == "k3d" ]]; then
if command -v k3d >/dev/null 2>&1; then
echo "Removing k3d registry 'knoe-registry' ..."
k3d registry delete knoe-registry || true
fi
echo "Removing registry resources from namespace '$REGISTRY_NAMESPACE' ..."
render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found >/dev/null 2>&1 || true
return 0
fi
echo "Removing registry resources from namespace '$REGISTRY_NAMESPACE' ..."
render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found
}
status_registry() {
local mode
mode=$(current_mode)
if [[ "$mode" == "k3d" ]]; then
if command -v k3d >/dev/null 2>&1; then
k3d registry list 2>/dev/null || true
fi
if command -v docker >/dev/null 2>&1; then
docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}' 2>/dev/null | grep -E 'k3d-knoe-registry|knoe-registry' || true
fi
return 0
fi
kubectl -n "$REGISTRY_NAMESPACE" get deploy,svc -l "app=registry" 2>/dev/null || true
kubectl -n "$REGISTRY_NAMESPACE" get pods -l "app=registry" 2>/dev/null || true
}
migrate_registry_to_gitlab() {
# Enumerate images from the registry:2 instance (registry.<src_ns>.svc.cluster.local:5000)
# and copy them to gitlab-registry using skopeo when available.
# Safe to run multiple times — skopeo copy is idempotent.
# Skipped gracefully if registry:2 has no images or is already gone.
local src_ns="${REGISTRY_NAMESPACE:-knoe-system}"
local gitlab_ns="${GITLAB_NAMESPACE:-gitlab}"
local src_registry="${REGISTRY_MIGRATE_SRC:-registry.${src_ns}.svc.cluster.local:5000}"
local dst_registry="${REGISTRY_MIGRATE_DST:-gitlab-registry.${gitlab_ns}.svc.cluster.local:5000}"
echo "Registry migration: ${src_registry}${dst_registry}"
# Verify the source registry:2 pod exists at all before trying to catalog it.
if ! kubectl -n "$src_ns" get deploy/registry >/dev/null 2>&1 && \
! kubectl -n "$src_ns" get pods -l app=registry --field-selector=status.phase=Running 2>/dev/null | grep -q Running; then
echo "INFO: registry:2 not found in namespace '${src_ns}' — nothing to migrate."
return 0
fi
# Fetch repository catalog via a temporary curl pod on the cluster.
local repos
repos=$(kubectl -n "$src_ns" run registry-catalog-migrate \
--image=alpine/curl:latest --restart=Never --rm --attach --quiet \
--overrides="{\"spec\":{\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"c\",\"image\":\"alpine/curl:latest\",\"command\":[\"sh\",\"-c\",\"curl -sf http://${src_registry}/v2/_catalog\"]}]}}" \
2>/dev/null \
| python3 -c "import sys,json; [print(r) for r in json.load(sys.stdin).get('repositories',[])]" 2>/dev/null || true)
if [[ -z "$repos" ]]; then
echo "INFO: No repositories found in registry:2 at ${src_registry} — nothing to migrate."
return 0
fi
echo "Repositories to migrate:"
printf '%s\n' "$repos" | sed 's/^/ /'
if command -v skopeo >/dev/null 2>&1; then
echo "skopeo found — migrating images automatically..."
local _ok=0 _fail=0
while IFS= read -r repo; do
[[ -z "$repo" ]] && continue
echo " Copying ${repo} ..."
if skopeo copy --all \
"docker://${src_registry}/${repo}" \
"docker://${dst_registry}/${repo}" \
--dest-tls-verify=false --src-tls-verify=false 2>&1; then
echo " [OK] ${repo}"
(( _ok++ )) || true
else
echo " [WARN] ${repo} — copy failed, may need manual retry"
(( _fail++ )) || true
fi
done <<< "$repos"
echo "Migration complete: ${_ok} succeeded, ${_fail} failed."
if [[ $_fail -gt 0 ]]; then
echo "Failed repos can be retried with: REGISTRY_MIGRATE_SRC=${src_registry} REGISTRY_MIGRATE_DST=${dst_registry} ./etc/init_registry.sh migrate"
fi
else
echo ""
echo "skopeo not found — run these commands manually (install: sudo apt-get install skopeo):"
while IFS= read -r repo; do
[[ -z "$repo" ]] && continue
echo " skopeo copy --all docker://${src_registry}/${repo} docker://${dst_registry}/${repo} --dest-tls-verify=false --src-tls-verify=false"
done <<< "$repos"
fi
}
case "${ACTION:-}" in
start|initialize|update|reload|restart)
ensure_tools
if registry_enabled; then
ensure_namespace "$REGISTRY_NAMESPACE"
apply_registry
else
echo "Skipping registry deploy (mode=$(current_mode))"
fi
;;
stop)
ensure_tools
if registry_enabled; then
delete_registry
fi
;;
status)
ensure_tools
if registry_enabled; then
status_registry
fi
;;
migrate)
# Migrate images from registry:2 → gitlab-registry before decommissioning registry:2.
# Run after GitLab is up and before running 'stop' on the old registry.
ensure_tools
migrate_registry_to_gitlab
;;
*)
usage >&2
exit 1
;;
esac