From 185a6948aaa12bef7ec63475be0d1079b44d284ea1c79f122a02d21aa372fb48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=9F=E9=85=8C=20=E9=B5=AC=E5=85=84?= Date: Sat, 2 May 2026 01:49:26 +0800 Subject: [PATCH] Successfully "make userspace" --- build.env | 2 + docker/ask.Dockerfile | 68 +++++++++++++++++++++-------- docker/build-base.Dockerfile | 34 ++++++++------- docker/download-packages.Dockerfile | 29 ++++++++++-- makefile | 10 +++++ patches/mono-ask.mk | 20 ++++++--- 6 files changed, 122 insertions(+), 41 deletions(-) diff --git a/build.env b/build.env index d982585..ff44e15 100644 --- a/build.env +++ b/build.env @@ -15,11 +15,13 @@ VPP_UPSTREAM_VERSION=23.10 # ASK's deps MONO_ASK_VERSION=mt-$(LINUX_FACTORY) LIBNFNETLINK_VERSION=1.0.2 +LIBMNL_VERSION=1.0.5 LIBNFCT_VERSION=1.1.0 LIBCLI_VERSION=1.10.7 # Check the package version for Debian trixies (what ASK uses) LIBXML2_VERSION=2.11.7 TCLAP_VERSION=1.2.5 +LIBPCAP_VERSION=1.10.4 CRIO_VERSION=cri-o.arm64.v1.35.2 KUBE_VERSION=v1.35.3 diff --git a/docker/ask.Dockerfile b/docker/ask.Dockerfile index 6c4d207..f4c73e7 100644 --- a/docker/ask.Dockerfile +++ b/docker/ask.Dockerfile @@ -4,48 +4,53 @@ ARG DOCKER_IMAGE_ROOT=monok8s FROM --platform=$BUILDPLATFORM ${DOCKER_IMAGE_ROOT}/build-base:${BUILD_BASE_TAG} AS build # Install glibc cross-compiler for kernel and standard build dependencies -RUN apt-get update && apt-get install -y \ - crossbuild-essential-arm64 \ - patch bzip2 pkg-config \ - libxml2-dev libtclap-dev - -# Provision the musl cross-compiler from musl.cc -RUN curl -O https://musl.cc/aarch64-linux-musl-cross.tgz && \ - tar xzf aarch64-linux-musl-cross.tgz -C /opt && \ - rm aarch64-linux-musl-cross.tgz - -# Expose the musl compiler to the PATH -ENV PATH="/opt/aarch64-linux-musl-cross/bin:${PATH}" +RUN apt-get update && apt-get install -y pkg-config WORKDIR /src +ARG AARCH64_MUSL_CC_TAR ARG NXP_TAR ARG MONO_ASK_TAR ARG LIBNFNETLINK_TAR +ARG LIBMNL_TAR ARG LIBNFCT_TAR ARG FMLIB_TAR ARG FMC_TAR ARG LIBXML2_TAR +ARG LIBPCAP_TAR +ARG LIBCLI_TAR ARG TCLAP_TAR # ASK's version pins (hardcoded wget) ARG LIBNFNETLINK_VERSION ARG LIBNFCT_VERSION +# MUSL Cross Compiler +COPY "${AARCH64_MUSL_CC_TAR}" ./aarch64_musl_cc.tar.gz + # Linux kernel COPY "${NXP_TAR}" ./kernel.tar.gz -# Copy the ASK and Netfilter tarballs +# Copy the ASK deps COPY "${MONO_ASK_TAR}" ./mono-ask.tar.gz COPY "${FMC_TAR}" ./fmc.tar.gz COPY "${FMLIB_TAR}" ./fmlib.tar.gz COPY "${LIBXML2_TAR}" ./libxml2.tar.xz +COPY "${LIBPCAP_TAR}" ./libpcap.tar.xz COPY "${TCLAP_TAR}" ./tclap.tar.gz +COPY "${LIBMNL_TAR}" ./libmnl.tar.bz2 +COPY "${LIBCLI_TAR}" ./libcli.tar.gz # Pinned version should keep version names COPY "${LIBNFNETLINK_TAR}" ./libnfnetlink-${LIBNFNETLINK_VERSION}.tar.bz2 -COPY "${LIBNFCT_TAR}" ./libnetfilter_conntrack-${LIBNFCT_VERSION}.tar.xz +COPY "${LIBNFCT_TAR}" ./libnetfilter_conntrack-${LIBNFCT_VERSION}.tar.xz -# Extract ASK and place Netfilter dependencies where the Makefile expects them +# Provision the musl cross-compiler from musl.cc +RUN tar zxf "aarch64_musl_cc.tar.gz" -C /opt + +# Expose the musl compiler to the PATH +ENV PATH="/opt/aarch64-linux-musl-cross/bin:${PATH}" + +# Extract and build the dependency libraries RUN mkdir -p ASK/sources/tarballs && \ tar zxf "mono-ask.tar.gz" -C "ASK" --strip-components=1 && \ mv libnfnetlink-${LIBNFNETLINK_VERSION}.tar.bz2 ASK/sources/tarballs/ && \ @@ -53,16 +58,18 @@ RUN mkdir -p ASK/sources/tarballs && \ RUN mkdir linux && tar zxf "kernel.tar.gz" -C "linux" --strip-components=1 -# Extract fmc & fmlib to prevent git fetching +# fmc & fmlib RUN mkdir -p ASK/sources/fmc && \ mkdir -p ASK/sources/fmlib && \ tar zxf "fmc.tar.gz" -C "ASK/sources/fmc" --strip-components=1 && \ tar zxf "fmlib.tar.gz" -C "ASK/sources/fmlib" --strip-components=1 +# tclap RUN mkdir -p tclap && tar zxf "tclap.tar.gz" -C "tclap" --strip-components=1 && \ cp -r tclap/include/tclap /opt/aarch64-linux-musl-cross/aarch64-linux-musl/include/ && \ rm -rf tclap +# libxml2 RUN mkdir -p libxml2 && tar xf "libxml2.tar.xz" -C "libxml2" --strip-components=1 && \ cd libxml2 && \ CC=aarch64-linux-musl-gcc ./configure --host=aarch64-linux-musl \ @@ -71,13 +78,40 @@ RUN mkdir -p libxml2 && tar xf "libxml2.tar.xz" -C "libxml2" --strip-components= make -j$(nproc) && make install && \ cd .. && rm -rf libxml2 +# libmnl +RUN mkdir -p libmnl && tar xjf "libmnl.tar.bz2" -C "libmnl" --strip-components=1 && \ + cd libmnl && \ + CC=aarch64-linux-musl-gcc ./configure --host=aarch64-linux-musl \ + --prefix=/opt/aarch64-linux-musl-cross/aarch64-linux-musl \ + --enable-static --disable-shared && \ + make -j$(nproc) && make install && \ + cd .. && rm -rf libmnl + +# libcli +RUN mkdir -p libcli && tar zxf "libcli.tar.gz" -C "libcli" --strip-components=1 && \ + cd libcli && \ + make CC=aarch64-linux-musl-gcc AR=aarch64-linux-musl-ar libcli.a && \ + cp libcli.h /opt/aarch64-linux-musl-cross/aarch64-linux-musl/include/ && \ + cp libcli.a /opt/aarch64-linux-musl-cross/aarch64-linux-musl/lib/ && \ + cd .. && rm -rf libcli + +# libpcap +RUN mkdir -p libpcap && tar xf "libpcap.tar.xz" -C "libpcap" --strip-components=1 && \ + cd libpcap && \ + CC=aarch64-linux-musl-gcc ./configure --host=aarch64-linux-musl \ + --prefix=/opt/aarch64-linux-musl-cross/aarch64-linux-musl \ + --with-pcap=linux --enable-static --disable-shared \ + --disable-usb --disable-netmap --disable-bluetooth --disable-dbus && \ + make -j$(nproc) && make install && \ + cd .. && rm -rf libpcap + WORKDIR /src/ASK COPY patches/mono-ask.mk . COPY kernel-extra.config /src/kernel-extra.config COPY kernel-build/ensure-kconfig.sh /src/ensure-kconfig.sh -# 1. This step patches the kernel, merges your k8s config, and builds the modules +# 1. This step patches the kernel, merges kernel-extra.config, and builds the modules RUN make -f mono-ask.mk modules # 2. This step cross-compiles fmlib, fmc, cmm, and dpa_app for Alpine diff --git a/docker/build-base.Dockerfile b/docker/build-base.Dockerfile index df21616..872a5ed 100644 --- a/docker/build-base.Dockerfile +++ b/docker/build-base.Dockerfile @@ -13,44 +13,46 @@ RUN if [ -n "${APT_PROXY}" ]; then \ RUN apt-get update && apt-get install -y --no-install-recommends \ bash \ bc \ + binutils-aarch64-linux-gnu \ bison \ build-essential \ - cpio \ + bzip2 \ ca-certificates \ + cpio \ curl \ + device-tree-compiler \ dosfstools \ - file \ - fdisk \ - fuse-overlayfs \ - gdisk \ + dwarves \ e2fsprogs \ + fdisk \ + file \ flex \ - git \ + fuse-overlayfs \ + gcc-aarch64-linux-gnu \ + gdisk \ gettext-base \ + git \ jq \ kmod \ + libc6-dev-arm64-cross \ + libelf-dev \ libelf-dev \ libssl-dev \ + linux-libc-dev-arm64-cross \ make \ pahole \ parted \ + patch \ perl \ + podman \ pv \ python3 \ qemu-user-static \ - podman \ - skopeo \ rsync \ + skopeo \ tar \ + u-boot-tools \ udev \ xz-utils \ zstd \ - dwarves \ - gcc-aarch64-linux-gnu \ - binutils-aarch64-linux-gnu \ - libc6-dev-arm64-cross \ - libelf-dev \ - linux-libc-dev-arm64-cross \ - u-boot-tools \ - device-tree-compiler \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/download-packages.Dockerfile b/docker/download-packages.Dockerfile index a258773..94919b1 100644 --- a/docker/download-packages.Dockerfile +++ b/docker/download-packages.Dockerfile @@ -70,6 +70,12 @@ WORKDIR /out/nxp/vpp RUN curl -fL --retry 3 -o "${VPP_VERSION}.tar.gz" \ "https://github.com/nxp-qoriq/vpp/archive/refs/tags/${VPP_VERSION}.tar.gz" +# ---- MUSL CC ---- +FROM base AS aarch64_musl_cc +WORKDIR /out +RUN curl -fL --retry 3 -o "aarch64-linux-musl-cross.tgz" \ + "https://musl.cc/aarch64-linux-musl-cross.tgz" + # ---- ASK ---- FROM base AS mono_ask ARG MONO_ASK_VERSION @@ -91,6 +97,13 @@ WORKDIR /out/ask/libnfct RUN curl -fL --retry 3 -o "${LIBNFCT_VERSION}.tar.xz" \ "https://www.netfilter.org/projects/libnetfilter_conntrack/files/libnetfilter_conntrack-${LIBNFCT_VERSION}.tar.xz" +# ---- libmnl ---- +FROM base AS libmnl +ARG LIBMNL_VERSION +WORKDIR /out/ask/libmnl +RUN curl -fL --retry 3 -o "${LIBMNL_VERSION}.tar.bz2" \ + "https://www.netfilter.org/projects/libmnl/files/libmnl-${LIBMNL_VERSION}.tar.bz2" + # ---- tclap ---- FROM base AS tclap ARG TCLAP_VERSION @@ -103,7 +116,7 @@ FROM base AS libxml2 ARG LIBXML2_VERSION WORKDIR /out/ask/libxml2 RUN curl -fL --retry 3 -o "${LIBXML2_VERSION}.tar.xz" \ - "https://download.gnome.org/sources/libxml2/2.12/libxml2-${LIBXML2_VERSION}.tar.xz" + "https://download.gnome.org/sources/libxml2/2.11/libxml2-${LIBXML2_VERSION}.tar.xz" # ---- libcli ---- FROM base AS libcli @@ -112,6 +125,13 @@ WORKDIR /out/ask/libcli RUN curl -fL --retry 3 -o "${LIBCLI_VERSION}.tar.gz" \ "https://github.com/dparrish/libcli/archive/refs/tags/V${LIBCLI_VERSION}.tar.gz" +# ---- libpcap ---- +FROM base AS libpcap +ARG LIBPCAP_VERSION +WORKDIR /out/ask/libpcap +RUN curl -fL --retry 3 -o "${LIBPCAP_VERSION}.tar.xz" \ + "https://www.tcpdump.org/release/libpcap-${LIBPCAP_VERSION}.tar.xz" + # ---- alpine rootfs ---- FROM base AS alpine_rootfs ARG ALPINE_SERIES @@ -143,13 +163,16 @@ COPY --from=kubectl /out/ / COPY --from=busybox /out/ / COPY --from=e2fsprogs /out/ / COPY --from=dpdk /out/ / -COPY --from=fmlib /out/ / -COPY --from=fmc /out/ / +COPY --from=aarch64_musl_cc /out/ / COPY --from=mono_ask /out/ / COPY --from=vpp /out/ / +COPY --from=fmlib /out/ / +COPY --from=fmc /out/ / COPY --from=libnfnetlink /out/ / COPY --from=libnfct /out/ / +COPY --from=libmnl /out/ / COPY --from=libcli /out/ / +COPY --from=libpcap /out/ / COPY --from=libxml2 /out/ / COPY --from=tclap /out/ / COPY --from=alpine_rootfs /out/ / diff --git a/makefile b/makefile index 4b85ba8..0e5b4b3 100644 --- a/makefile +++ b/makefile @@ -15,15 +15,19 @@ VPP_TAR := $(PACKAGES_DIR)/nxp/vpp/$(VPP_VERSION).tar.gz DPDK_TAR := $(PACKAGES_DIR)/nxp/dpdk/$(DPDK_VERSION).tar.gz CRIO_TAR := $(PACKAGES_DIR)/$(CRIO_VERSION).tar.gz +AARCH64_MUSL_CC_TAR := $(PACKAGES_DIR)/aarch64-linux-musl-cross.tgz + # ASK-specific MONO_ASK_TAR := $(PACKAGES_DIR)/ask/$(MONO_ASK_VERSION).tar.gz FMLIB_TAR := $(PACKAGES_DIR)/nxp/fmlib/$(FMLIB_VERSION).tar.gz FMC_TAR := $(PACKAGES_DIR)/nxp/fmc/$(FMC_VERSION).tar.gz LIBNFNETLINK_TAR := $(PACKAGES_DIR)/ask/libnfnetlink/$(LIBNFNETLINK_VERSION).tar.bz2 LIBNFCT_TAR := $(PACKAGES_DIR)/ask/libnfct/$(LIBNFCT_VERSION).tar.xz +LIBMNL_TAR := $(PACKAGES_DIR)/ask/libmnl/$(LIBMNL_VERSION).tar.bz2 LIBCLI_TAR := $(PACKAGES_DIR)/ask/libcli/$(LIBCLI_VERSION).tar.gz TCLAP_TAR := $(PACKAGES_DIR)/ask/tclap/$(TCLAP_VERSION).tar.gz LIBXML2_TAR := $(PACKAGES_DIR)/ask/libxml2/$(LIBXML2_VERSION).tar.xz +LIBPCAP_TAR := $(PACKAGES_DIR)/ask/libpcap/$(LIBPCAP_VERSION).tar.xz # Kubernetes components KUBELET_BIN := $(PACKAGES_DIR)/kubernetes/kubelet-$(KUBE_VERSION) @@ -133,9 +137,11 @@ $(DOWNLOAD_PACKAGES_STAMP): docker/download-packages.Dockerfile build.env makefi --build-arg VPP_VERSION=$(VPP_VERSION) \ --build-arg MONO_ASK_VERSION=$(MONO_ASK_VERSION) \ --build-arg LIBNFNETLINK_VERSION=$(LIBNFNETLINK_VERSION) \ + --build-arg LIBMNL_VERSION=$(LIBMNL_VERSION) \ --build-arg LIBNFCT_VERSION=$(LIBNFCT_VERSION) \ --build-arg LIBCLI_VERSION=$(LIBCLI_VERSION) \ --build-arg LIBXML2_VERSION=$(LIBXML2_VERSION) \ + --build-arg LIBPCAP_VERSION=$(LIBPCAP_VERSION) \ --build-arg TCLAP_VERSION=$(TCLAP_VERSION) \ --build-arg ALPINE_SERIES=$(ALPINE_SERIES) \ --build-arg ALPINE_ARCH=$(ALPINE_ARCH) \ @@ -212,13 +218,17 @@ ASK: $(ASK_TAR) $(LIBNFCT_TAR) $(LIBNFNETLINK_TAR) $(TCLAP_TAR) $(LIBXML2_TAR) | --build-arg DOCKER_IMAGE_ROOT=$(DOCKER_IMAGE_ROOT) \ --build-arg BUILD_BASE_TAG=$$build_base_tag \ --build-arg MONO_ASK_TAR=$(MONO_ASK_TAR) \ + --build-arg AARCH64_MUSL_CC_TAR=$(AARCH64_MUSL_CC_TAR) \ --build-arg NXP_TAR=$(NXP_TAR) \ --build-arg FMLIB_TAR=$(FMLIB_TAR) \ --build-arg FMC_TAR=$(FMC_TAR) \ --build-arg LIBNFNETLINK_TAR=$(LIBNFNETLINK_TAR) \ + --build-arg LIBMNL_TAR=$(LIBMNL_TAR) \ --build-arg LIBNFCT_TAR=$(LIBNFCT_TAR) \ --build-arg LIBXML2_TAR=$(LIBXML2_TAR) \ + --build-arg LIBPCAP_TAR=$(LIBPCAP_TAR) \ --build-arg TCLAP_TAR=$(TCLAP_TAR) \ + --build-arg LIBCLI_TAR=$(LIBCLI_TAR) \ --build-arg LIBNFNETLINK_VERSION=$(LIBNFNETLINK_VERSION) \ --build-arg LIBNFCT_VERSION=$(LIBNFCT_VERSION) \ --output type=local,dest=./$(OUT_DIR)/ASK . diff --git a/patches/mono-ask.mk b/patches/mono-ask.mk index 050a567..9b226e8 100644 --- a/patches/mono-ask.mk +++ b/patches/mono-ask.mk @@ -20,21 +20,26 @@ SYSROOT_PATH := /opt/aarch64-linux-musl-cross/aarch64-linux-musl prepare-preloaded-sources: @echo "--> Initializing dummy git repos and building preloaded fmlib/fmc..." mkdir -p $(STAMPS_DIR) - + git config --global user.email "monok8s@localhost" git config --global user.name "monok8s authors" - + git config --global --add safe.directory '*' + + # Inject libmnl.pc into the vendor's isolated sysroot to satisfy pkg-config + mkdir -p $(ASK_DIR)/sources/sysroot/lib/pkgconfig + cp -a $(SYSROOT_PATH)/lib/pkgconfig/libmnl.pc $(ASK_DIR)/sources/sysroot/lib/pkgconfig/ + # Handle fmlib: Initialize dummy repo, patch, and build cd $(FMLIB_DIR) && git init -q && git add -A && git commit -q -m "base" cd $(FMLIB_DIR) && git apply $(ASK_DIR)/patches/fmlib/01-mono-ask-extensions.patch $(MAKE) -C $(FMLIB_DIR) CROSS_COMPILE=$(MUSL_HOST)- KERNEL_SRC=$(KDIR) libfm-arm.a ln -sf libfm-arm.a $(FMLIB_DIR)/libfm.a touch $(STAMPS_DIR)/fmlib - + # Handle fmc: Initialize dummy repo, patch, and build cd $(FMC_BASE) && git init -q && git add -A && git commit -q -m "base" cd $(FMC_BASE) && git apply $(ASK_DIR)/patches/fmc/01-mono-ask-extensions.patch - $(MAKE) -C $(FMC_DIR) CC=$(MUSL_HOST)-gcc CXX=$(MUSL_HOST)-g++ AR=$(MUSL_HOST)-ar \ + $(MAKE) -C $(FMC_DIR) CC="$(MUSL_HOST)-gcc -static" CXX="$(MUSL_HOST)-g++ -static" AR=$(MUSL_HOST)-ar \ MACHINE=ls1046 \ FMD_USPACE_HEADER_PATH=$(FMLIB_DIR)/include/fmd \ FMD_USPACE_LIB_PATH=$(FMLIB_DIR) \ @@ -66,8 +71,13 @@ modules: build-kernel userspace: prepare-preloaded-sources @echo "--> Building ASK sources and libraries..." $(MAKE) -f Makefile sources KDIR=$(KDIR) CROSS_COMPILE=$(MUSL_HOST)- ARCH=$(ARCH) HOST=$(MUSL_HOST) + + @echo "--> Stripping -Werror from vendor Makefiles for modern GCC compatibility..." + sed -i 's/-Werror//g' $(ASK_DIR)/cmm/Makefile + sed -i 's/-Werror//g' $(ASK_DIR)/dpa_app/Makefile + @echo "--> Building ASK userspace executables statically..." - $(MAKE) -f Makefile userspace KDIR=$(KDIR) CROSS_COMPILE=$(MUSL_HOST)- ARCH=$(ARCH) HOST=$(MUSL_HOST) CC="$(MUSL_HOST)-gcc -static" CXX="$(MUSL_HOST)-g++ -static" + $(MAKE) -f Makefile userspace KDIR=$(KDIR) CROSS_COMPILE=$(MUSL_HOST)- ARCH=$(ARCH) HOST=$(MUSL_HOST) CC='"$(MUSL_HOST)-gcc -static"' CXX='"$(MUSL_HOST)-g++ -static"' # 5. Extract artifacts dist: