VPP = Very Painful Process

2026-04-09 21:30:12 +08:00
parent 4eae2621c9
commit b8bc6a13cf
8 changed files with 392 additions and 74 deletions
--- a/docker/build-base.Dockerfile
+++ b/docker/build-base.Dockerfile
@@ -49,6 +49,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
    gcc-aarch64-linux-gnu \
    binutils-aarch64-linux-gnu \
    libc6-dev-arm64-cross \
+    libelf-dev \
    linux-libc-dev-arm64-cross \
    u-boot-tools \
    device-tree-compiler \
--- a/docker/kernel-build.Dockerfile
+++ b/docker/kernel-build.Dockerfile
@@ -23,7 +23,6 @@ WORKDIR /build/nxplinux

 COPY kernel-extra.config /tmp/kernel-extra.config
 COPY kernel-build/dts/*.dts ./arch/arm64/boot/dts/freescale/
-COPY kernel-build/scripts/patch_usdpaa_dts.py /usr/local/bin/patch_usdpaa_dts.py

 RUN grep -q "^dtb-\\\$(CONFIG_ARCH_LAYERSCAPE) += ${DEVICE_TREE_TARGET}.dtb$" \
      arch/arm64/boot/dts/freescale/Makefile \
--- a/docker/vpp-container.Dockerfile
+++ b/docker/vpp-container.Dockerfile
@@ -0,0 +1,40 @@
+ARG BUILD_BASE_TAG=dev
+ARG DOCKER_IMAGE_ROOT=monok8s
+
+FROM debian:trixie
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG APT_PROXY
+RUN if [ -n "${APT_PROXY}" ]; then \
+      echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy; \
+    fi
+
+RUN apt-get update && apt-get install -y \
+    libnuma1 \
+    libnl-3-200 \
+    libnl-route-3-200 \
+    libxml2 \
+    # libtclap is usually header-only, but fmc might link against it
+    # libstdc++ is essential for the fmc binary
+    libstdc++6 \
+    iproute2 \
+    ca-certificates \
+    # Useful for debugging portals later
+    pciutils \
+    kmod \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG VPP_VERSION
+COPY /out/vpp_${VPP_VERSION#lf-}-1_arm64.deb /tmp/vpp.deb
+
+RUN apt-get update && \
+    apt-get install -y /tmp/vpp.deb && \
+    rm /tmp/vpp.deb && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN fmc -v || echo "FMC installed"
+
+COPY ./vpp/start-vpp.sh /usr/bin/start-vpp.sh
+RUN chmod +x /usr/bin/start-vpp.sh
+
+ENTRYPOINT ["/usr/bin/start-vpp.sh"]
--- a/docker/vpp.Dockerfile
+++ b/docker/vpp.Dockerfile
@@ -1,73 +1,311 @@
-FROM ubuntu:22.04
+ARG BUILD_BASE_TAG=dev
+ARG DOCKER_IMAGE_ROOT=monok8s

-ENV DEBIAN_FRONTEND=noninteractive
+FROM --platform=$BUILDPLATFORM ${DOCKER_IMAGE_ROOT}/build-base:${BUILD_BASE_TAG} AS build
+
+RUN mkdir /src
 WORKDIR /src

 RUN apt-get update && apt-get install -y --no-install-recommends \
-    ca-certificates \
-    curl \
-    gnupg \
-    lsb-release \
+    devscripts debootstrap qemu-user-static qemu-utils parted kpartx build-essential make cmake ncurses-dev flex bison meson bc vim curl git pkg-config rsync nfs-kernel-server debootstrap debhelper dh-python python3-pyelftools python3-ply chrpath tftpd-hpa htop \
 && rm -rf /var/lib/apt/lists/*

-# Add FD.io release repo (Jammy / arm64 packages exist for v26.02)
+RUN debootstrap \
+  --arch=arm64 \
+  --foreign \
+  trixie \
+  /mnt/rootfs \
+  http://deb.debian.org/debian
+
+ARG APT_PROXY
+
+RUN if [ -n "${APT_PROXY}" ]; then \
+      echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /mnt/rootfs/etc/apt/apt.conf.d/01proxy; \
+    fi
+
+RUN chroot /mnt/rootfs apt-get update && \
+    chroot /mnt/rootfs apt-get install -y --no-install-recommends \
+    libelf-dev \
+    libssl-dev \
+    pkg-config \
+    zlib1g-dev \
+    libpcap-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+
+RUN tee -a /usr/bin/aarch64-linux-gnu-pkg-config << 'EOF'
+#!/bin/sh -e
+SYSROOT=/mnt/rootfs
+export PKG_CONFIG_PATH=${SYSROOT}/usr/lib/pkgconfig:${SYSROOT}/usr/local/lib/pkgconfig:${SYSROOT}/usr/lib/aarch64-linux-gnu/pkgconfig
+export PKG_CONFIG_SYSROOT_DIR=${SYSROOT}
+exec pkg-config "$@"
+EOF
+RUN chmod +x /usr/bin/aarch64-linux-gnu-pkg-config
+
+ARG DPDK_TAR
+ARG DPDK_VERSION
+
+COPY ${DPDK_TAR} /tmp/
+
 RUN set -eux; \
-    mkdir -p /etc/apt/keyrings; \
-    curl -fsSL https://packagecloud.io/fdio/release/gpgkey \
-      | gpg --dearmor -o /etc/apt/keyrings/fdio-release.gpg; \
-    echo "deb [signed-by=/etc/apt/keyrings/fdio-release.gpg] https://packagecloud.io/fdio/release/ubuntu/ jammy main" \
-      > /etc/apt/sources.list.d/fdio-release.list; \
-    apt-get update; \
-    apt-get install -y --no-install-recommends \
-      vpp=26.02-release \
-      vpp-plugin-core=26.02-release \
-      vpp-plugin-dpdk=26.02-release; \
-    rm -rf /var/lib/apt/lists/*
+    mkdir -p /src/dpdk; \
+    tar -xf "/tmp/$(basename "${DPDK_TAR}")" -C /src/dpdk --strip-components=1

-RUN mkdir -p /etc/vpp /run/vpp
+WORKDIR /src/dpdk

-RUN cat > /etc/vpp/startup.conf <<'EOF'
-unix {
-  nodaemon
-  cli-listen /run/vpp/cli.sock
-  log /dev/stderr
-  full-coredump
-}

-api-trace {
-  on
-}
+RUN tee -a dpdk.config << 'END'
+[binaries]
+c = ['aarch64-linux-gnu-gcc', '--sysroot=/mnt/rootfs']
+cpp = ['aarch64-linux-gnu-g++', '--sysroot=/mnt/rootfs']
+ar = 'aarch64-linux-gnu-ar'
+as = 'aarch64-linux-gnu-as'
+ld = ['aarch64-linux-gnu-ld', '--sysroot=/mnt/rootfs']
+strip = 'aarch64-linux-gnu-strip'
+pkgconfig = '/usr/bin/aarch64-linux-gnu-pkg-config'
+pcap-config = ''

-cpu {
-  main-core 0
-}
+[host_machine]
+system = 'linux'
+cpu_family = 'aarch64'
+cpu = 'armv8-a'
+endian = 'little'

-plugins {
-  plugin default { disable }
-  plugin dpdk_plugin.so { enable }
-}
+[properties]
+platform = 'dpaa'
+END

-dpdk {
-  no-pci
-  dev dpaa_bus:fm1-mac9
-  dev dpaa_bus:fm1-mac10
-}
-EOF
+RUN rm -rf build-aarch64 && \
+  meson setup build-aarch64 \
+  --prefix=/usr \
+  --buildtype=release \
+  --default-library=shared \
+  --cross-file=dpdk.config \
+  -Dkernel_dir=/usr/src/linux \
+  -Dexamples=l2fwd,l3fwd,helloworld \
+  -Dmax_lcores=4 \
+  -Dc_args="-g -Ofast -fPIC -ftls-model=local-dynamic -Wno-error=implicit-function-declaration -Wno-error=maybe-uninitialized \
+    -I/mnt/rootfs/usr/include \
+    -I/mnt/rootfs/usr/include/aarch64-linux-gnu" \
+  -Dc_link_args="-Wl,-rpath-link=/mnt/rootfs/usr/lib/aarch64-linux-gnu \
+    -L/mnt/rootfs/usr/lib/aarch64-linux-gnu \
+    -L/mnt/rootfs/usr/lib \
+    -L/mnt/rootfs/usr/local/lib \
+    -L/src/dpdk/build-aarch64/drivers \
+    -Wl,--no-as-needed"

-RUN cat > /usr/local/bin/docker-entrypoint.sh <<'EOF'
-#!/bin/sh
-set -eu
+RUN DESTDIR=/mnt/rootfs ninja -C build-aarch64 install
+RUN install -m 644 ./drivers/bus/pci/bus_pci_driver.h /mnt/rootfs/usr/include
+RUN find ./build-aarch64/examples/ -type f -name "dpdk-*" -type f -exec sh -c "install -m 755 {} /mnt/rootfs/usr/local/bin/" \;

-mkdir -p /run/vpp
+# Copy the internal bus/driver headers that NXP's DPAA/VPP plugin needs
+RUN install -m 644 /src/dpdk/lib/eal/include/dev_driver.h /mnt/rootfs/usr/include/
+RUN install -m 644 /src/dpdk/lib/eal/arm/include/rte_cpuflags.h /mnt/rootfs/usr/include/ || true
+RUN install -m 644 /src/dpdk/drivers/bus/pci/bus_pci_driver.h /mnt/rootfs/usr/include/

-echo "=== device check ==="
-ls -l /dev/fsl-usdpaa /dev/fsl-usdpaa-irq 2>/dev/null || true
-echo "=== sysfs check ==="
-ls -l /sys/class/net 2>/dev/null || true
+# Copy the internal bus headers required by NXP VPP DPDK plugin
+RUN install -m 644 /src/dpdk/drivers/bus/pci/bus_pci_driver.h /mnt/rootfs/usr/include/
+RUN install -m 644 /src/dpdk/drivers/bus/vmbus/bus_vmbus_driver.h /mnt/rootfs/usr/include/
+RUN install -m 644 /src/dpdk/lib/eal/include/dev_driver.h /mnt/rootfs/usr/include/

-exec /usr/bin/vpp -c /etc/vpp/startup.conf "$@"
-EOF
+# Sometimes it also looks for the vdev bus header
+RUN install -m 644 /src/dpdk/drivers/bus/vdev/bus_vdev_driver.h /mnt/rootfs/usr/include/ || true

-RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+# Cryptodev internal headers (needed for VPP IPsec plugin)
+RUN install -m 644 /src/dpdk/lib/cryptodev/cryptodev_pmd.h /mnt/rootfs/usr/include/

-ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+# Eventdev internal headers (often needed for NXP DPAA2 scheduler)
+RUN install -m 644 /src/dpdk/lib/eventdev/eventdev_pmd.h /mnt/rootfs/usr/include/ || true
+RUN install -m 644 /src/dpdk/lib/eventdev/event_timer_adapter_pmd.h /mnt/rootfs/usr/include/ || true
+
+RUN ln -s /mnt/rootfs/usr/include /mnt/rootfs/usr/include/dpdk || true
+
+ARG VPP_TAR
+ARG VPP_VERSION
+ARG VPP_UPSTREAM_VERSION
+
+COPY ${VPP_TAR} /tmp/
+
+RUN set -eux; \
+    mkdir -p /src/vpp; \
+    tar -xf "/tmp/$(basename "${VPP_TAR}")" -C /src/vpp --strip-components=1; \
+    cd /src/vpp; \
+    # Strings of surreal hacks from Gemini that make vpp builds
+    git init && git add . && git config user.email "you@example.com" && \
+    git config user.name "Your Name" && git commit -m "init" && \
+    git tag -a v${VPP_UPSTREAM_VERSION} -m "v${VPP_UPSTREAM_VERSION}"
+
+WORKDIR /src/vpp
+
+# Sync libs for the linker
+RUN ln -s /mnt/rootfs/usr/lib/aarch64-linux-gnu /mnt/rootfs/usr/lib/lib64 || true
+
+RUN cd build-root && \
+    export DEB_BUILD_OPTIONS="nocheck nodoc noautodbgsym" && \
+    make V=1 \
+    PLATFORM=dpaa \
+    TAG=dpaa \
+    CROSS_SYSROOT=/mnt/rootfs \
+    CROSS_PREFIX=aarch64-linux-gnu \
+    DPDK_PATH=/mnt/rootfs/usr \
+    VPP_VERSION=${VPP_VERSION} \
+    vpp-install
+
+ARG FMLIB_TAR
+ARG FMC_TAR
+ARG FMLIB_VERSION
+ARG FMC_VERSION
+ARG NXP_VERSION
+
+COPY packages/${NXP_VERSION}.tar.gz /tmp/
+RUN set -eux; \
+    mkdir -p /src/nxplinux; \
+    tar -xf "/tmp/${NXP_VERSION}.tar.gz" -C /src/nxplinux --strip-components=1 \
+    && rm -f /tmp/${NXP_VERSION}.tar.gz
+
+COPY ${FMLIB_TAR} /tmp/
+RUN set -eux; \
+    mkdir -p /src/fmlib; \
+    tar -xf "/tmp/$(basename "${FMLIB_TAR}")" -C /src/fmlib --strip-components=1;
+
+COPY ${FMC_TAR} /tmp/
+RUN set -eux; \
+    mkdir -p /src/fmc; \
+    tar -xf "/tmp/$(basename "${FMC_TAR}")" -C /src/fmc --strip-components=1;
+
+WORKDIR /src/fmlib
+
+RUN set -eux; KERNEL_SRC="/src/nxplinux" \
+    # We add CFLAGS to point to the specific UAPI headers for DPAA/FMan
+    EXTRA_CFLAGS="-I${KERNEL_SRC}/include/uapi -I${KERNEL_SRC}/drivers/net/ethernet/freescale/fman" \
+    make all -j8 && \
+    make install-libfm-arm DESTDIR=/mnt/rootfs PREFIX=/usr
+
+WORKDIR /src/fmc
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    g++-aarch64-linux-gnu \
+    gcc-aarch64-linux-gnu \
+    binutils-aarch64-linux-gnu
+
+RUN chroot /mnt/rootfs apt-get install -y --no-install-recommends \
+    libxml2-dev libtclap-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN set -eux; make -C source clean && \
+  make -C source -j8 \
+    # Force the use of the ARM64 compiler triple if on x86 host,
+    # or ensure we pass the right machine flags
+    CC="aarch64-linux-gnu-gcc --sysroot=/mnt/rootfs" \
+    CXX="aarch64-linux-gnu-g++ --sysroot=/mnt/rootfs" \
+    # We add -v to see exactly where the linker is looking if it fails
+    # and we point to the fmlib source directory so it finds -lfmc and -lfm
+    LDFLAGS="-L/mnt/rootfs/usr/lib/aarch64-linux-gnu \
+             -L/mnt/rootfs/lib/aarch64-linux-gnu \
+             -L/src/fmlib/lib \
+             -L/src/fmc/source" \
+    CFLAGS="-Wno-write-strings -fpermissive -DLS1046 -DNCSW_LINUX \
+      -I/mnt/rootfs/usr/include/aarch64-linux-gnu \
+      -I/mnt/rootfs/usr/include/libxml2 \
+      -I/src/fmlib/include \
+      -I/src/fmlib/include/fmd \
+      -I/src/fmlib/include/fmd/Peripherals \
+      -I/src/fmlib/include/fmd/integrations" \
+    FMD_USPACE_HEADER_PATH=/src/nxplinux/include/uapi/linux/fmd \
+    FMD_USPACE_LIB_PATH=/mnt/rootfs/usr/lib \
+    LIBXML2_HEADER_PATH=/mnt/rootfs/usr/include/libxml2 \
+    TCLAP_HEADER_PATH=/mnt/rootfs/usr/include
+
+# Locate and Fix the XML + Schemas + Sub-configs
+RUN set -eux; \
+    XML_SRC=$(find /src -name "usdpaa_config_ls1046.xml" | head -n 1); \
+    SCHEMA_DIR=$(dirname $(find /src -name "cfgdata.xsd" | head -n 1)); \
+    HXS_FILE=$(find /src -name "hxs_pdl_v3.xml" | head -n 1); \
+    \
+    # Create a structured staging area
+    mkdir -p /tmp/fmc_staging/config; \
+    \
+    # Copy main files to the root of staging
+    cp "$XML_SRC" /tmp/fmc_staging/config.xml; \
+    # Copy the HXS and schemas into the 'config' subfolder where fmc expects them
+    cp "$HXS_FILE" /tmp/fmc_staging/config/; \
+    cp "$SCHEMA_DIR"/*.xsd /tmp/fmc_staging/config/; \
+    \
+    # Production Cleanup (Remove MAC 3 from the main config)
+    sed -i '/<macport name="fm0-gb3"/{N;N;N;N;d;}' /tmp/fmc_staging/config.xml; \
+    cp /tmp/fmc_staging/config.xml /tmp/fmc_staging/policy.xml
+
+RUN set -eux; \
+    cd /src/vpp; \
+    mkdir -p debian; \
+    echo "10" > debian/compat; \
+    \
+    # 1. Start the Rules file
+    echo '#!/usr/bin/make -f' > debian/rules; \
+    echo '%:' >> debian/rules; \
+    printf '\tdh $@\n' >> debian/rules; \
+    \
+    echo 'override_dh_auto_configure:' >> debian/rules; \
+    printf '\t@echo "Skipping configure"\n' >> debian/rules; \
+    \
+    echo 'override_dh_auto_clean:' >> debian/rules; \
+    printf '\t@echo "Skipping clean"\n' >> debian/rules; \
+    \
+    echo 'override_dh_auto_build:' >> debian/rules; \
+    printf '\t@echo "Skipping build"\n' >> debian/rules; \
+    \
+    # 2. THE INSTALL STEP (Crucial: everything here must have a Tab)
+    echo 'override_dh_auto_install:' >> debian/rules; \
+    printf '\tmkdir -p debian/vpp/usr/bin\n' >> debian/rules; \
+    printf '\tmkdir -p debian/vpp/usr/lib\n' >> debian/rules; \
+    printf '\tmkdir -p debian/vpp/etc/fmc/config\n' >> debian/rules; \
+    \
+    # Copy VPP binaries from build-root
+    printf '\tcp -a build-root/install-dpaa-aarch64/vpp/. debian/vpp/usr/\n' >> debian/rules; \
+    \
+    # Copy FMC binary
+    printf '\tcp /src/fmc/source/fmc debian/vpp/usr/bin/\n' >> debian/rules; \
+    \
+    # PULL FROM THE STAGING AREA
+    printf '\tcp /tmp/fmc_staging/config.xml debian/vpp/etc/fmc/\n' >> debian/rules; \
+    printf '\tcp /tmp/fmc_staging/policy.xml debian/vpp/etc/fmc/\n' >> debian/rules; \
+    printf '\tcp /tmp/fmc_staging/config/* debian/vpp/etc/fmc/config/\n' >> debian/rules; \
+    \
+    # Copy system libs
+    printf '\tcp /mnt/rootfs/usr/lib/libfm-arm.so* debian/vpp/usr/lib/ 2>/dev/null || true\n' >> debian/rules; \
+    printf '\tcp /mnt/rootfs/usr/lib/libusdpaa.so* debian/vpp/usr/lib/ 2>/dev/null || true\n' >> debian/rules; \
+    \
+    echo 'override_dh_usrlocal:' >> debian/rules; \
+    printf '\t@echo "Skipping usrlocal fixup"\n' >> debian/rules; \
+    \
+    echo 'override_dh_shlibdeps:' >> debian/rules; \
+    printf '\t@echo "Skipping shlibdeps (cross-compile)"\n' >> debian/rules; \
+    \
+    chmod +x debian/rules; \
+    \
+    # 3. Control and Changelog (echo is fine here because these aren't Makefiles)
+    echo "Source: vpp" > debian/control; \
+    echo "Maintainer: NXP Builder <builder@nxp.com>" >> debian/control; \
+    echo "Section: net" >> debian/control; \
+    echo "Priority: optional" >> debian/control; \
+    echo "Build-Depends: debhelper (>= 10)" >> debian/control; \
+    echo "" >> debian/control; \
+    echo "Package: vpp" >> debian/control; \
+    echo "Architecture: arm64" >> debian/control; \
+    echo "Depends: \${shlibs:Depends}, \${misc:Depends}, libxml2, libtclap-dev" >> debian/control; \
+    echo "Description: VPP for NXP DPAA" >> debian/control; \
+    \
+    echo "vpp (${VPP_VERSION#lf-}-1) stable; urgency=low" > debian/changelog; \
+    echo "  * Custom NXP Build" >> debian/changelog; \
+    echo " -- NXP Builder <builder@nxp.com>  $(date -R)" >> debian/changelog
+
+# 5. Final Packaging
+RUN cd /src/vpp && \
+    export DEB_BUILD_OPTIONS="nocheck nodoc noautodbgsym" && \
+    export CC=aarch64-linux-gnu-gcc && \
+    export CXX=aarch64-linux-gnu-g++ && \
+    dpkg-buildpackage -us -uc -b -aarm64 -d
+
+FROM scratch
+ARG VPP_VERSION
+COPY --from=build /src/vpp_${VPP_VERSION#lf-}-1_arm64.deb /