crictl can import images but slow

2026-03-24 20:52:17 +08:00
parent ac05d3e5dc
commit 58da0aada6
15 changed files with 474 additions and 19 deletions
--- a/alpine/build-rootfs.sh
+++ b/alpine/build-rootfs.sh
@@ -1,8 +1,19 @@
 #!/bin/bash

+/preload-k8s-images.sh || exit 1
+
+REGISTRY_DATA_DIR="${KUBE_IMG_CACHE}/registry-data/${ARCH}/${KUBE_VERSION}"
+
+mkdir -p "${ROOTFS}/var/lib/registry"
+
+echo "Copying registry data into rootfs..."
+cp -a "${REGISTRY_DATA_DIR}/." "${ROOTFS}/var/lib/registry/"
+
 mkdir -p "$ROOTFS/var/cache/apk"
+mkdir -p "$ROOTFS/var/cache/k8s-images"
 mkdir -p "$ROOTFS/build"
 mount --bind /var/cache/apk "$ROOTFS/var/cache/apk"
+mount --bind /var/cache/k8s-images "$ROOTFS/var/cache/k8s-images"
 mount --bind /dev  "$ROOTFS/dev"
 mount --bind /proc "$ROOTFS/proc"
 mount --bind /sys  "$ROOTFS/sys"
@@ -11,6 +22,7 @@ mount --bind /run  "$ROOTFS/run"
 cp /usr/bin/qemu-aarch64-static "$ROOTFS/usr/bin/"
 cp /etc/resolv.conf "$ROOTFS/etc/resolv.conf"
 cp /build/crio.tar.gz "$ROOTFS/build/"
+cp /build/registry.tar.gz "$ROOTFS/build/"
 cp -r /build/rootfs/* "$ROOTFS/"

 chroot "$ROOTFS" /bin/sh -c "ln -s /var/cache/apk /etc/apk/cache"
@@ -21,6 +33,7 @@ chroot "$ROOTFS" /bin/bash /install-packages.sh || exit 1
 rm "$ROOTFS/install-packages.sh"

 umount "$ROOTFS/var/cache/apk"
+umount "$ROOTFS/var/cache/k8s-images"
 umount "$ROOTFS/dev"
 umount "$ROOTFS/proc"
 umount "$ROOTFS/sys"
@@ -28,11 +41,13 @@ umount "$ROOTFS/run"

 rm -r "$ROOTFS/build"

+/merge-rootfs.sh "/build/rootfs-extra" "$ROOTFS"
+
 ### Begin making full disk image for the device
 echo "=========================== RootFS "$( du -sh "$ROOTFS/" )

 IMG=output.img
-SIZE=1024MB
+SIZE=1536MB

 dd if=/dev/zero of="$IMG" bs=1 count=0 seek=$SIZE

--- a/alpine/install-packages.sh
+++ b/alpine/install-packages.sh
@@ -10,11 +10,20 @@ rc-update add devfs sysinit
 rc-update add procfs sysinit
 rc-update add sysfs sysinit
 rc-update add loopback boot
+rc-update add hostname boot
+rc-update add localmount boot
 rc-update add fancontrol default
-echo "ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100 -n -l /bin/sh" >> "/etc/inittab"
 echo '[ -x /bin/bash ] && exec /bin/bash -l' >> "/root/.profile"
-echo "export PATH=\"/usr/local/bin:$PATH\"" >> "/etc/profile.d/settings.sh"

+# We need this to ship k8s components. (coredns, kube-apiserver, etc)
+echo "##################################################### Install Local Registry"
+mkdir -p /usr/local/bin
+
+apk add skopeo
+
+tar zxf registry.tar.gz
+mv registry /usr/local/bin/registry
+/usr/local/bin/registry --version

 echo "##################################################### Installing CRI-O"

@@ -30,6 +39,9 @@ if [ $? -ne 0 ]; then
 	exit $?
 fi

+mv /etc/cni/net.d/10-crio-bridge.conflist.disabled \
+   /etc/cni/net.d/10-crio-bridge.conflist
+
 echo "--------------"
 sed -i "s/default_runtime = \"crun\"/\0\ncgroup_manager = \"cgroupfs\"/g" /etc/crio/crio.conf.d/10-crio.conf
 grep cgroup_manager /etc/crio/crio.conf.d/10-crio.conf || exit 1
--- a/alpine/merge-rootfs.sh
+++ b/alpine/merge-rootfs.sh
@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage:
+#   export HOSTNAME=mybox
+#   ./merge-rootfs.sh rootfs-extra /out/rootfs
+#
+# Naming rules:
+#   foo                 -> normal file
+#   foo.tmpl            -> render with envsubst, then normal handling
+#   foo.override        -> replace target directly
+#   foo.tmpl.override   -> render with envsubst, then replace target directly
+#
+# Default handling:
+#   etc/*               -> merge missing lines
+#   opt/scripts/*       -> replace
+#   everything else     -> copy only if missing
+
+SRC_ROOT="${1:?source rootfs path required}"
+DST_ROOT="${2:?target rootfs path required}"
+
+if [[ ! -d "$SRC_ROOT" ]]; then
+    echo "Source rootfs does not exist or is not a directory: $SRC_ROOT" >&2
+    exit 1
+fi
+
+if [[ ! -d "$DST_ROOT" ]]; then
+    echo "Target rootfs does not exist or is not a directory: $DST_ROOT" >&2
+    exit 1
+fi
+
+if ! command -v envsubst >/dev/null 2>&1; then
+    echo "envsubst not found. Install gettext or gettext-envsubst." >&2
+    exit 1
+fi
+
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+append_missing_lines() {
+    local src="$1"
+    local dst="$2"
+    local changed=1
+    local line
+
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        if ! grep -Fqx -- "$line" "$dst"; then
+            printf '%s\n' "$line" >> "$dst"
+            changed=0
+        fi
+    done < "$src"
+
+    return "$changed"
+}
+
+check_template_vars() {
+    local src="$1"
+    local missing=0
+    local vars var
+
+    vars="$(
+        grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}|\$[A-Za-z_][A-Za-z0-9_]*' "$src" \
+        | sed -E 's/^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/\1/' \
+        | sort -u || true
+    )"
+
+    while IFS= read -r var; do
+        [[ -z "$var" ]] && continue
+        if [[ -z "${!var+x}" ]]; then
+            echo "Missing required env var: $var (used in $src)" >&2
+            missing=1
+        fi
+    done <<< "$vars"
+
+    [[ "$missing" -eq 0 ]]
+}
+
+render_template() {
+    local src="$1"
+    local out="$2"
+
+    mkdir -p "$(dirname "$out")"
+    check_template_vars "$src"
+    envsubst < "$src" > "$out"
+}
+
+replace_file() {
+    local src="$1"
+    local dst="$2"
+
+    mkdir -p "$(dirname "$dst")"
+    cp -a "$src" "$dst"
+    echo "Replaced file: $dst"
+}
+
+merge_or_copy_file() {
+    local src="$1"
+    local dst="$2"
+    local rel="$3"
+
+    mkdir -p "$(dirname "$dst")"
+
+    # scripts: replace
+    if [[ "$rel" == opt/scripts/* ]]; then
+        cp -a "$src" "$dst"
+        echo "Replaced script: $dst"
+        return
+    fi
+
+    # /etc: merge missing lines
+    if [[ "$rel" == etc/* ]]; then
+        if [[ ! -e "$dst" ]]; then
+            cp -a "$src" "$dst"
+            echo "Copied new config: $dst"
+            return
+        fi
+
+        if [[ ! -f "$dst" ]]; then
+            echo "Skipping existing non-regular path: $dst" >&2
+            return
+        fi
+
+        if append_missing_lines "$src" "$dst"; then
+            echo "Appended missing lines: $dst"
+        else
+            echo "No changes needed: $dst"
+        fi
+        return
+    fi
+
+    # default: copy only if missing
+    if [[ ! -e "$dst" ]]; then
+        cp -a "$src" "$dst"
+        echo "Copied new file: $dst"
+    else
+        echo "Skipped existing file: $dst"
+    fi
+}
+
+find "$SRC_ROOT" -mindepth 1 | while IFS= read -r src_path; do
+    rel_path="${src_path#"$SRC_ROOT"/}"
+
+    if [[ -d "$src_path" ]]; then
+        mkdir -p "$DST_ROOT/$rel_path"
+        continue
+    fi
+
+    if [[ -L "$src_path" ]]; then
+        dst_path="$DST_ROOT/$rel_path"
+        if [[ -e "$dst_path" || -L "$dst_path" ]]; then
+            echo "Symlink exists, skipping: $dst_path"
+        else
+            mkdir -p "$(dirname "$dst_path")"
+            ln -s "$(readlink "$src_path")" "$dst_path"
+            echo "Created symlink: $dst_path"
+        fi
+        continue
+    fi
+
+    if [[ ! -f "$src_path" ]]; then
+        echo "Skipping unsupported file type: $src_path" >&2
+        continue
+    fi
+
+    src_for_merge="$src_path"
+    rel_for_target="$rel_path"
+    mode="default"
+
+    if [[ "$rel_for_target" == *.tmpl.override ]]; then
+        mode="override"
+        rel_for_target="${rel_for_target%.tmpl.override}"
+        rendered="$TMP_DIR/$rel_for_target"
+        render_template "$src_path" "$rendered"
+        src_for_merge="$rendered"
+
+    elif [[ "$rel_for_target" == *.override ]]; then
+        mode="override"
+        rel_for_target="${rel_for_target%.override}"
+
+    elif [[ "$rel_for_target" == *.tmpl ]]; then
+        rel_for_target="${rel_for_target%.tmpl}"
+        rendered="$TMP_DIR/$rel_for_target"
+        render_template "$src_path" "$rendered"
+        src_for_merge="$rendered"
+    fi
+
+    dst_path="$DST_ROOT/$rel_for_target"
+
+    if [[ "$mode" == "override" ]]; then
+        replace_file "$src_for_merge" "$dst_path"
+    else
+        merge_or_copy_file "$src_for_merge" "$dst_path" "$rel_for_target"
+    fi
+done
--- a/alpine/preload-k8s-images.sh
+++ b/alpine/preload-k8s-images.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+set -euo pipefail
+
+: "${KUBE_IMG_CACHE:?KUBE_IMG_CACHE is required}"
+: "${KUBE_VERSION:?KUBE_VERSION is required}"
+: "${ARCH:?ARCH is required}"
+
+OS=linux
+REGISTRY_PORT=5000
+
+# Keep everything version/arch scoped so caches do not get mixed.
+ARCHIVE_DIR="${KUBE_IMG_CACHE}/archives/${ARCH}/${KUBE_VERSION}"
+REGISTRY_DATA_DIR="${KUBE_IMG_CACHE}/registry-data/${ARCH}/${KUBE_VERSION}"
+REGISTRY_TAR="${KUBE_IMG_CACHE}/registry.tar"
+REGISTRY_IMAGE="docker.io/library/registry:2"
+
+if podman image exists "${REGISTRY_IMAGE}"; then
+    echo "Registry image already present: ${REGISTRY_IMAGE}"
+
+elif [ -f "${REGISTRY_TAR}" ]; then
+    echo "Loading registry image from cache: ${REGISTRY_TAR}"
+    podman load -i "${REGISTRY_TAR}"
+
+else
+    echo "Cache miss → pulling and saving registry image"
+
+    podman pull "${REGISTRY_IMAGE}"
+    podman save -o "${REGISTRY_TAR}" "${REGISTRY_IMAGE}"
+fi
+
+mkdir -p "${ARCHIVE_DIR}"
+rm -rf "${REGISTRY_DATA_DIR}"
+mkdir -p "${REGISTRY_DATA_DIR}"
+
+echo "============================================================"
+echo "Preparing Kubernetes image cache"
+echo "  KUBE_VERSION      = ${KUBE_VERSION}"
+echo "  ARCH              = ${ARCH}"
+echo "  ARCHIVE_DIR       = ${ARCHIVE_DIR}"
+echo "  REGISTRY_DATA_DIR = ${REGISTRY_DATA_DIR}"
+echo "============================================================"
+
+# Start a temporary local registry backed by REGISTRY_DATA_DIR.
+# We use host network here to avoid messing with nested networking.
+podman rm -f temp-registry >/dev/null 2>&1 || true
+
+podman run -d \
+    --name temp-registry \
+    --network host \
+    --cgroups=disabled \
+    -v "${REGISTRY_DATA_DIR}:/var/lib/registry" \
+    docker.io/library/registry:2
+
+cleanup() {
+    podman rm -f temp-registry >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+# Wait for registry to answer.
+for _ in $(seq 1 30); do
+    if curl -fsS "http://127.0.0.1:${REGISTRY_PORT}/v2/" >/dev/null; then
+        break
+    fi
+    sleep 1
+done
+
+if ! curl -fsS "http://127.0.0.1:${REGISTRY_PORT}/v2/" >/dev/null; then
+    echo "Temporary registry did not start"
+    exit 1
+fi
+
+# Cache and seed all kubeadm-required images.
+while read -r img; do
+    [ -n "$img" ] || continue
+
+    safe_name=$(printf '%s' "$img" | sed 's#/#_#g; s#:#__#g')
+    archive="${ARCHIVE_DIR}/${safe_name}.tar"
+    ref="${img#registry.k8s.io/}"
+
+    if [ ! -f "$archive" ]; then
+        echo "Caching: $img"
+        skopeo copy \
+            --override-os "${OS}" \
+            --override-arch "${ARCH}" \
+            "docker://$img" \
+            "oci-archive:$archive"
+    else
+        echo "Cache hit: $img"
+    fi
+
+    if skopeo inspect --tls-verify=false "docker://127.0.0.1:${REGISTRY_PORT}/${ref}" >/dev/null 2>&1; then
+        echo "Registry hit: 127.0.0.1:${REGISTRY_PORT}/${ref}"
+        continue
+    fi
+
+    echo "Seeding registry: $ref"
+    skopeo copy \
+        --dest-tls-verify=false \
+        "oci-archive:$archive" \
+        "docker://127.0.0.1:${REGISTRY_PORT}/${ref}"
+done < <(qemu-aarch64-static "$ROOTFS/usr/local/bin/kubeadm" config images list --kubernetes-version "${KUBE_VERSION}")
+
+echo
+echo "Done."
+echo "Archives:      ${ARCHIVE_DIR}"
+echo "Registry data: ${REGISTRY_DATA_DIR}"
--- a/alpine/rootfs-extra/etc/fancontrol
+++ b/alpine/rootfs-extra/etc/fancontrol
--- a/alpine/rootfs-extra/etc/fstab
+++ b/alpine/rootfs-extra/etc/fstab
@@ -0,0 +1 @@
+none /sys/fs/cgroup cgroup2 defaults 0 0
--- a/alpine/rootfs-extra/etc/hostname.tmpl.override
+++ b/alpine/rootfs-extra/etc/hostname.tmpl.override
@@ -0,0 +1 @@
+${ALPINE_HOSTNAME}
--- a/alpine/rootfs-extra/etc/inittab
+++ b/alpine/rootfs-extra/etc/inittab
@@ -0,0 +1,2 @@
+# Dev only
+ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100 -n -l /bin/bash
--- a/alpine/rootfs-extra/opt/scripts/load-images.sh
+++ b/alpine/rootfs-extra/opt/scripts/load-images.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+set -eu
+
+KUBE_VERSION="${KUBE_VERSION:-v1.35.3}"
+REGISTRY_ADDR="${REGISTRY_ADDR:-127.0.0.1:5000}"
+
+mkdir -p /opt/registry
+cat >/opt/registry/config.yml <<EOF
+version: 0.1
+log:
+  level: info
+storage:
+  filesystem:
+    rootdirectory: /var/lib/registry
+http:
+  addr: ${REGISTRY_ADDR}
+EOF
+
+cat >/opt/crictl.yaml <<'EOF'
+runtime-endpoint: unix:///var/run/crio/crio.sock
+image-endpoint: unix:///var/run/crio/crio.sock
+timeout: 10
+debug: false
+EOF
+
+registry serve /opt/registry/config.yml >/var/log/registry.log 2>&1 &
+REG_PID=$!
+
+cleanup() {
+    kill "$REG_PID" 2>/dev/null || true
+}
+trap cleanup EXIT INT TERM
+
+for _ in $(seq 1 30); do
+    if curl -fsS "http://${REGISTRY_ADDR}/v2/_catalog" >/dev/null; then
+        break
+    fi
+    sleep 1
+done
+
+curl -fsS "http://${REGISTRY_ADDR}/v2/_catalog" >/dev/null
+
+for img in $(kubeadm config images list --kubernetes-version "${KUBE_VERSION}"); do
+    name="${img#registry.k8s.io/}"
+    echo "Importing ${img} from ${REGISTRY_ADDR}/${name}"
+    skopeo copy --src-tls-verify=false \
+      "docker://${REGISTRY_ADDR}/${name}" \
+      "containers-storage:${img}"
+done
+
+echo "Imported images now visible to CRI-O:"
+crictl images
+
+kill "$REG_PID" 2>/dev/null || true
+wait "$REG_PID" 2>/dev/null || true
+trap - EXIT INT TERM