#!/usr/bin/env bash set -euo pipefail # init_registry.sh # Purpose: # - Deploy registry support for local clusters # - k3s: deploy in-cluster `registry:2` into the service namespace # - k3d: create/connect a `k3d` registry on port 5000 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Shared option parsing for common core scripts # shellcheck disable=SC1090 source "$SCRIPT_DIR/common_core_lib.sh" # Inject default config if not provided _has_config=0 for _arg in "$@"; do [[ "$_arg" == "-c" || "$_arg" == "--config" || "$_arg" == -c=* || "$_arg" == --config=* ]] && _has_config=1 done if [[ $_has_config -eq 0 ]]; then _default_cfg="$(common_core_default_config_path "$SCRIPT_DIR" || true)" if [[ -n "$_default_cfg" ]]; then set -- "-c" "$_default_cfg" "$@" fi fi unset _has_config _arg _default_cfg common_core_preparse_config "$@" # Remove -c/--config from the remaining args now that KNOE_CONF is set. # `common_core_preparse_config` exposes the stripped args as `COMMON_CORE_ARGS`. set -- "${COMMON_CORE_ARGS[@]}" # Load environment and config via knoe_cfg.sh # shellcheck disable=SC1090 source "$SCRIPT_DIR/knoe_cfg.sh" if [[ -z "${KNOE_SERVICE:-}" ]]; then echo "ERROR: KNOE_SERVICE is not defined. Provide KNOE_HOME/env.sh or ~/.knoe/env.sh" >&2 exit 1 fi ACTION="" NAMESPACE_OVERRIDE="" REGISTRY_NAMESPACE_OVERRIDE="" if [[ -d "$SCRIPT_DIR/../k8s/registry" ]]; then REGISTRY_MANIFEST_DIR="$SCRIPT_DIR/../k8s/registry" elif [[ -n "${KNOE_HOME:-}" && -d "$KNOE_HOME/k8s/registry" ]]; then REGISTRY_MANIFEST_DIR="$KNOE_HOME/k8s/registry" else REGISTRY_MANIFEST_DIR="$SCRIPT_DIR/../k8s/registry" fi REGISTRY_MANIFEST_FILE=${REGISTRY_MANIFEST_FILE:-"$REGISTRY_MANIFEST_DIR/deployment.yaml"} usage() { cat < Deploys registry support for local clusters. - In k3s mode, deploys `registry:2` into the service namespace (default: SERVICE_NAMESPACE). - In k3d mode, manages the `k3d` registry (port 5000) and removes any in-cluster registry resources. Actions: start|initialize|update|reload|restart Deploy or update the registry stop Remove registry resources status Show registry pod/service status migrate Copy all images from registry:2 → gitlab-registry. Run AFTER GitLab is up, BEFORE 'stop'. Uses skopeo if available, otherwise prints commands. NOTE: `-r/--registry-namespace` is kept for backwards compatibility and is treated as an alias for `-n/--namespace`. EOF } while [[ $# -gt 0 ]]; do case "$1" in -m|--mode) shift knoe_set_mode "${1:-}" ;; -m=*|--mode=*) knoe_set_mode "${1#*=}" ;; -n|--namespace) shift if [[ -z "${1:-}" ]]; then echo "ERROR: -n/--namespace requires a value" >&2 exit 2 fi NAMESPACE_OVERRIDE="$1" ;; -n=*|--namespace=*) NAMESPACE_OVERRIDE="${1#*=}" ;; -r|--registry-namespace) shift if [[ -z "${1:-}" ]]; then echo "ERROR: -r/--registry-namespace requires a value" >&2 exit 2 fi REGISTRY_NAMESPACE_OVERRIDE="$1" ;; -r=*|--registry-namespace=*) REGISTRY_NAMESPACE_OVERRIDE="${1#*=}" ;; start|stop|status|restart|initialize|update|reload) ACTION="$1" ;; -h|--help) usage exit 0 ;; *) if [[ -z "$ACTION" && "$1" != -* ]]; then ACTION="$1" else echo "Unknown argument: $1" >&2 usage exit 2 fi ;; esac shift done if [[ -n "$REGISTRY_NAMESPACE_OVERRIDE" ]]; then REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE_OVERRIDE" elif [[ -n "$NAMESPACE_OVERRIDE" ]]; then REGISTRY_NAMESPACE="$NAMESPACE_OVERRIDE" elif [[ -n "${REGISTRY_NAMESPACE:-}" ]]; then REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE" else # Default to the configured service namespace (driven by knoe.cfg via knoe_cfg.sh). REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-}}" if [[ -z "$REGISTRY_NAMESPACE" ]]; then echo "ERROR: REGISTRY_NAMESPACE could not be determined. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2 exit 1 fi fi # In k3s mode, the in-cluster registry is a common core service and should live # in the common services namespace. _mode_resolved="${KNOE_MODE:-${DEPLOYMENT_MODE:-}}" if declare -F knoe_normalize_mode >/dev/null 2>&1; then _mode_resolved="$(knoe_normalize_mode "$_mode_resolved")" fi if [[ "$_mode_resolved" == "k3s" && -z "${REGISTRY_NAMESPACE:-}" ]]; then REGISTRY_NAMESPACE="${SERVICE_NAMESPACE:-${PROLE_NAMESPACE:-}}" if [[ -z "$REGISTRY_NAMESPACE" ]]; then echo "ERROR: REGISTRY_NAMESPACE could not be determined for k3s mode. Set SERVICE_NAMESPACE in knoe.cfg or pass -n/--namespace." >&2 exit 1 fi fi unset _mode_resolved export REGISTRY_NAMESPACE REGISTRY_NODE_SELECTOR=${REGISTRY_NODE_SELECTOR:-} current_mode() { if command -v knoe_normalize_mode >/dev/null 2>&1; then knoe_normalize_mode "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}" return 0 fi printf '%s' "${KNOE_MODE:-${DEPLOYMENT_MODE:-}}" } registry_enabled() { local mode mode=$(current_mode) [[ "$mode" == "k3s" || "$mode" == "k3d" ]] } resolve_control_plane_node_selector() { local node="" node=$(kubectl get nodes -l 'node-role.kubernetes.io/control-plane' -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "$node" ]]; then printf 'kubernetes.io/hostname=%s' "$node" fi } resolve_registry_node_selector() { if [[ -n "$REGISTRY_NODE_SELECTOR" ]]; then printf '%s' "$REGISTRY_NODE_SELECTOR" return 0 fi # Default: pin to the control-plane node so hostPort:5000 is reachable via # the k3s server host (e.g. myrddin.prole.org:5000). resolve_control_plane_node_selector } apply_registry_node_selector() { local selector selector="$(resolve_registry_node_selector)" if [[ -z "$selector" ]]; then return 0 fi local key="${selector%%=*}" local val="${selector#*=}" if [[ -z "$key" || -z "$val" ]]; then echo "WARN: invalid REGISTRY_NODE_SELECTOR '$selector' (expected key=value); skipping." >&2 return 0 fi kubectl -n "$REGISTRY_NAMESPACE" patch deployment/registry \ --type merge \ -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"$key\":\"$val\"}}}}}" >/dev/null 2>&1 || true } ensure_tools() { command -v kubectl >/dev/null || { echo "Missing required tool: kubectl" >&2; exit 1; } } ensure_namespace() { local ns="$1" if [[ -z "$ns" ]]; then return 0 fi if ! kubectl get namespace "$ns" >/dev/null 2>&1; then echo "Creating namespace '$ns' ..." kubectl create namespace "$ns" >/dev/null 2>&1 || true fi } render_registry_manifest() { if [[ ! -f "$REGISTRY_MANIFEST_FILE" ]]; then echo "ERROR: Registry manifest not found: $REGISTRY_MANIFEST_FILE" >&2 exit 1 fi knoe_render_manifest "$REGISTRY_MANIFEST_FILE" } cleanup_registry_k8s() { render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found >/dev/null 2>&1 || true } configure_k3d_insecure_registry() { local cluster_name="${1:-knoe-dev-cluster}" local reg_host="k3d-knoe-registry" local reg_port="5000" local registries_yaml registries_yaml=$(cat < /etc/rancher/k3s/registries.yaml <<'EOF' ${registries_yaml} EOF" || { echo "WARN: failed to write registries.yaml to $node" >&2; continue; } # Restart k3s inside the node so it picks up the new mirror config. docker restart "$node" >/dev/null 2>&1 || true echo " configured $node" done # Wait briefly for nodes to come back up sleep 5 echo "Waiting for k3d nodes to become Ready ..." kubectl wait --for=condition=Ready nodes --all --timeout=${ROLLOUT_TIMEOUT:-300s} 2>/dev/null || true } resolve_k3d_cluster_name() { # Prefer an explicitly provided cluster name. if [[ -n "${K3D_CLUSTER_NAME:-}" ]]; then printf '%s' "$K3D_CLUSTER_NAME" return 0 fi # If kubectl is already pointed at a k3d context, derive the name from it. local ctx ctx=$(kubectl config current-context 2>/dev/null || true) if [[ "${ctx:-}" == k3d-* ]]; then printf '%s' "${ctx#k3d-}" return 0 fi # If there is exactly one k3d cluster, pick it. if command -v k3d >/dev/null 2>&1; then local clusters count clusters=$(k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | grep -v '^$' || true) count=$(printf '%s\n' "$clusters" | grep -c . 2>/dev/null || true) if [[ "${count:-0}" -eq 1 ]]; then printf '%s' "$clusters" return 0 fi fi # Stable fallback. printf '%s' "knoe-dev-cluster" } resolve_k3d_network_name() { local cluster_name="$1" local net_name="k3d-${cluster_name}" if command -v docker >/dev/null 2>&1 && docker network inspect "$net_name" >/dev/null 2>&1; then printf '%s' "$net_name" return 0 fi # Fallback: infer from the first server node if available. local node_name="k3d-${cluster_name}-server-0" if command -v docker >/dev/null 2>&1 && docker inspect "$node_name" >/dev/null 2>&1; then docker inspect "$node_name" -f '{{range $k, $v := .NetworkSettings.Networks}}{{println $k}}{{end}}' 2>/dev/null | head -1 || true return 0 fi printf '%s' "" } ensure_k3d_registry_dns() { local cluster_name="$1" local reg_host="${2:-k3d-knoe-registry}" command -v docker >/dev/null 2>&1 || return 0 local node_name="k3d-${cluster_name}-server-0" if ! docker inspect "$node_name" >/dev/null 2>&1; then return 0 fi # Validate that the node can resolve the registry hostname via Docker DNS. if docker exec "$node_name" sh -c "command -v getent >/dev/null 2>&1 && getent hosts '$reg_host' >/dev/null 2>&1"; then return 0 fi if docker exec "$node_name" sh -c "ping -c 1 -W 1 '$reg_host' >/dev/null 2>&1"; then return 0 fi echo "WARN: k3d node '${node_name}' cannot resolve registry host '${reg_host}'." >&2 echo " This will cause image pulls like '${reg_host}:5000/' to fail." >&2 return 1 } apply_registry() { local mode mode=$(current_mode) if [[ "$mode" == "k3d" ]]; then if command -v k3d >/dev/null 2>&1; then echo "Ensuring k3d registry 'knoe-registry' on port 5000 ..." if ! k3d registry list knoe-registry >/dev/null 2>&1; then k3d registry create knoe-registry --port 5000 || true fi local cluster_name cluster_name=$(resolve_k3d_cluster_name) if k3d cluster list "$cluster_name" >/dev/null 2>&1; then echo "Connecting knoe-registry to $cluster_name ..." # k3d v5+ uses 'k3d-NAME' as container name for registries local reg_container="k3d-knoe-registry" local net_name net_name=$(resolve_k3d_network_name "$cluster_name") if command -v docker >/dev/null 2>&1; then if [[ -n "${net_name:-}" ]] && docker inspect "$reg_container" >/dev/null 2>&1 && docker network inspect "$net_name" >/dev/null 2>&1; then if ! docker inspect "$reg_container" -f '{{json .NetworkSettings.Networks}}' | grep -q "$net_name"; then # Ensure the registry is reachable at the stable hostname used in manifests. docker network connect --alias k3d-knoe-registry "$net_name" "$reg_container" || true fi fi fi fi # Configure k3d nodes to use the registry over plain HTTP (insecure). configure_k3d_insecure_registry "$cluster_name" # Validate registry DNS from a node so image pulls won't fail later. ensure_k3d_registry_dns "$cluster_name" "k3d-knoe-registry" || true # Ensure no k8s registry resources remain in k3d clusters. echo "Removing k8s registry resources (k3d uses k3d-knoe-registry only) ..." cleanup_registry_k8s return 0 else echo "k3d not found; falling back to Docker for registry ..." fi fi # Fallback or k3s mode: use Docker container if k8s deploy fails or as alternative if [[ "$mode" == "k3d" ]] || [[ "${PROLE_REGISTRY_STANDALONE:-}" == "1" ]]; then if command -v docker >/dev/null 2>&1; then if ! docker ps --format '{{.Names}}' | grep -q "^knoe-registry$"; then if docker ps -a --format '{{.Names}}' | grep -q "^knoe-registry$"; then docker start knoe-registry || true else docker run -d -p 5000:5000 --restart always --name knoe-registry registry:2 || true fi fi if [[ "$mode" == "k3d" ]]; then echo "Removing k8s registry resources (k3d uses k3d-knoe-registry only) ..." cleanup_registry_k8s fi return 0 fi fi # In k3s mode, if GitLab is deployed it owns port 5000 (gitlab-registry). # Deploying registry:2 with hostPort:5000 on the same node would conflict. # Skip the apply and warn; use init_registry.sh stop to remove the old deployment. local _mode_for_gitlab_check _mode_for_gitlab_check=$(current_mode) if [[ "$_mode_for_gitlab_check" == "k3s" ]]; then local _gitlab_ns="${GITLAB_NAMESPACE:-gitlab}" if kubectl get namespace "$_gitlab_ns" >/dev/null 2>&1; then echo "WARN: GitLab namespace '${_gitlab_ns}' detected in k3s mode." >&2 echo " gitlab-registry owns port 5000 — registry:2 deployment skipped." >&2 echo " Run 'init_registry.sh stop' to remove any existing registry:2 resources." >&2 return 0 fi fi # Idempotency check: skip apply+wait if the registry is already available. local _desired _available _desired=$(kubectl get deploy/registry -n "$REGISTRY_NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || true) _available=$(kubectl get deploy/registry -n "$REGISTRY_NAMESPACE" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true) if [[ -n "$_desired" && "${_available:-0}" -ge "${_desired:-1}" && "${_desired:-0}" -gt 0 ]]; then echo "Registry already running in namespace '$REGISTRY_NAMESPACE' ($_available/$_desired replicas available); skipping apply." return 0 fi echo "Applying registry manifest to namespace '$REGISTRY_NAMESPACE' ..." render_registry_manifest | kubectl apply --server-side --force-conflicts --field-manager=knoe-installer --validate=false -n "$REGISTRY_NAMESPACE" -f - apply_registry_node_selector kubectl rollout status deploy/registry -n "$REGISTRY_NAMESPACE" --timeout=${ROLLOUT_TIMEOUT:-300s} || true } delete_registry() { local mode mode=$(current_mode) if [[ "$mode" == "k3d" ]]; then if command -v k3d >/dev/null 2>&1; then echo "Removing k3d registry 'knoe-registry' ..." k3d registry delete knoe-registry || true fi echo "Removing registry resources from namespace '$REGISTRY_NAMESPACE' ..." render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found >/dev/null 2>&1 || true return 0 fi echo "Removing registry resources from namespace '$REGISTRY_NAMESPACE' ..." render_registry_manifest | kubectl delete -n "$REGISTRY_NAMESPACE" -f - --ignore-not-found } status_registry() { local mode mode=$(current_mode) if [[ "$mode" == "k3d" ]]; then if command -v k3d >/dev/null 2>&1; then k3d registry list 2>/dev/null || true fi if command -v docker >/dev/null 2>&1; then docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}' 2>/dev/null | grep -E 'k3d-knoe-registry|knoe-registry' || true fi return 0 fi kubectl -n "$REGISTRY_NAMESPACE" get deploy,svc -l "app=registry" 2>/dev/null || true kubectl -n "$REGISTRY_NAMESPACE" get pods -l "app=registry" 2>/dev/null || true } migrate_registry_to_gitlab() { # Enumerate images from the registry:2 instance (registry..svc.cluster.local:5000) # and copy them to gitlab-registry using skopeo when available. # Safe to run multiple times — skopeo copy is idempotent. # Skipped gracefully if registry:2 has no images or is already gone. local src_ns="${REGISTRY_NAMESPACE:-knoe-system}" local gitlab_ns="${GITLAB_NAMESPACE:-gitlab}" local src_registry="${REGISTRY_MIGRATE_SRC:-registry.${src_ns}.svc.cluster.local:5000}" local dst_registry="${REGISTRY_MIGRATE_DST:-gitlab-registry.${gitlab_ns}.svc.cluster.local:5000}" echo "Registry migration: ${src_registry} → ${dst_registry}" # Verify the source registry:2 pod exists at all before trying to catalog it. if ! kubectl -n "$src_ns" get deploy/registry >/dev/null 2>&1 && \ ! kubectl -n "$src_ns" get pods -l app=registry --field-selector=status.phase=Running 2>/dev/null | grep -q Running; then echo "INFO: registry:2 not found in namespace '${src_ns}' — nothing to migrate." return 0 fi # Fetch repository catalog via a temporary curl pod on the cluster. local repos repos=$(kubectl -n "$src_ns" run registry-catalog-migrate \ --image=alpine/curl:latest --restart=Never --rm --attach --quiet \ --overrides="{\"spec\":{\"tolerations\":[{\"operator\":\"Exists\"}],\"containers\":[{\"name\":\"c\",\"image\":\"alpine/curl:latest\",\"command\":[\"sh\",\"-c\",\"curl -sf http://${src_registry}/v2/_catalog\"]}]}}" \ 2>/dev/null \ | python3 -c "import sys,json; [print(r) for r in json.load(sys.stdin).get('repositories',[])]" 2>/dev/null || true) if [[ -z "$repos" ]]; then echo "INFO: No repositories found in registry:2 at ${src_registry} — nothing to migrate." return 0 fi echo "Repositories to migrate:" printf '%s\n' "$repos" | sed 's/^/ /' if command -v skopeo >/dev/null 2>&1; then echo "skopeo found — migrating images automatically..." local _ok=0 _fail=0 while IFS= read -r repo; do [[ -z "$repo" ]] && continue echo " Copying ${repo} ..." if skopeo copy --all \ "docker://${src_registry}/${repo}" \ "docker://${dst_registry}/${repo}" \ --dest-tls-verify=false --src-tls-verify=false 2>&1; then echo " [OK] ${repo}" (( _ok++ )) || true else echo " [WARN] ${repo} — copy failed, may need manual retry" (( _fail++ )) || true fi done <<< "$repos" echo "Migration complete: ${_ok} succeeded, ${_fail} failed." if [[ $_fail -gt 0 ]]; then echo "Failed repos can be retried with: REGISTRY_MIGRATE_SRC=${src_registry} REGISTRY_MIGRATE_DST=${dst_registry} ./etc/init_registry.sh migrate" fi else echo "" echo "skopeo not found — run these commands manually (install: sudo apt-get install skopeo):" while IFS= read -r repo; do [[ -z "$repo" ]] && continue echo " skopeo copy --all docker://${src_registry}/${repo} docker://${dst_registry}/${repo} --dest-tls-verify=false --src-tls-verify=false" done <<< "$repos" fi } case "${ACTION:-}" in start|initialize|update|reload|restart) ensure_tools if registry_enabled; then ensure_namespace "$REGISTRY_NAMESPACE" apply_registry else echo "Skipping registry deploy (mode=$(current_mode))" fi ;; stop) ensure_tools if registry_enabled; then delete_registry fi ;; status) ensure_tools if registry_enabled; then status_registry fi ;; migrate) # Migrate images from registry:2 → gitlab-registry before decommissioning registry:2. # Run after GitLab is up and before running 'stop' on the old registry. ensure_tools migrate_registry_to_gitlab ;; *) usage >&2 exit 1 ;; esac