From 4eae2621c9925f47fb656fd3e9e86c8df88cc0aed3ccd458e85b419b271d67b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=9F=E9=85=8C=20=E9=B5=AC=E5=85=84?= Date: Wed, 8 Apr 2026 21:13:21 +0800 Subject: [PATCH] Trying to build vpp --- build.env | 5 +- clitools/docker/dpdk.Dockerfile | 2 +- clitools/makefile | 2 +- .../pkg/apis/monok8s/v1alpha1/osupgrade.go | 29 +++++- docker/alpine.Dockerfile | 6 +- docker/build-base.Dockerfile | 29 +++--- docker/kernel-build.Dockerfile | 1 + docker/vpp.Dockerfile | 73 +++++++++++++++ docs/dpdk.md | 12 --- docs/dpdk_vpp.md | 92 +++++++++++++++++++ kernel-build/dts/mono-gateway-dk-sdk.dts | 66 ++++++++----- makefile | 21 ++++- 12 files changed, 284 insertions(+), 54 deletions(-) create mode 100644 docker/vpp.Dockerfile delete mode 100644 docs/dpdk.md create mode 100644 docs/dpdk_vpp.md diff --git a/build.env b/build.env index 13dea85..71afd4c 100644 --- a/build.env +++ b/build.env @@ -6,6 +6,7 @@ TAG=dev # The Linux kernel, from NXP NXP_VERSION=lf-6.18.2-1.0.0 DPDK_VERSION=lf-6.18.2-1.0.0 +VPP_VERSION=lf-6.18.2-1.0.0 CRIO_VERSION=cri-o.arm64.v1.33.3 KUBE_VERSION=v1.33.3 @@ -14,7 +15,6 @@ KUBE_VERSION=v1.33.3 # We need fsl-ls1046a-rdb-sdk.dtb here DEVICE_TREE_TARGET=mono-gateway-dk-sdk - # Arch, should always be arm64 for our board. This is here in case branching off to other devices ARCH=arm64 @@ -34,3 +34,6 @@ ALPINE_HOSTNAME=monok8s-hostname # Upper case [A-Z]_ only, used for naming env vars BUILD_TAG=MONOK8S + +# Optional apt cache +APT_PROXY=apt-cacher-ng.eco-system.svc.cluster.local:3142 diff --git a/clitools/docker/dpdk.Dockerfile b/clitools/docker/dpdk.Dockerfile index 4452ac3..48ce0fd 100644 --- a/clitools/docker/dpdk.Dockerfile +++ b/clitools/docker/dpdk.Dockerfile @@ -44,7 +44,7 @@ RUN set -eux; \ RUN set -eux; \ mkdir -p /artifact/bin; \ test -x /src/dpdk/build/app/dpdk-testpmd; \ - cp /src/dpdk/build/app/dpdk-testpmd /artifact/bin/ + cp -r /src/dpdk/build/app/dpdk-testpmd /artifact/bin/ FROM scratch AS export COPY --from=build /out/ / diff --git a/clitools/makefile b/clitools/makefile index b8529e2..fd37282 100644 --- a/clitools/makefile +++ b/clitools/makefile @@ -53,7 +53,7 @@ uboot-tools: $(UBOOT_TAR) --build-arg UBOOT_TAR=$(UBOOT_TAR) \ --output type=local,dest=./$(OUT_DIR) . -dpdk: $(UBOOT_TAR) +dpdk: $(DPDK_TAR) @mkdir -p $(OUT_DIR)/dpdk docker buildx build --platform linux/arm64 \ -f docker/dpdk.Dockerfile \ diff --git a/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go b/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go index 9c5a248..fee1d86 100644 --- a/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go +++ b/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go @@ -52,12 +52,39 @@ type OSUpgradeList struct { Items []OSUpgrade `json:"items" yaml:"items"` } +// OSUpgradeSpec defines the desired state of an OSUpgrade. type OSUpgradeSpec struct { - // User request, can be "stable" or an explicit version like "v1.35.3". + // DesiredVersion is the requested target version. + // + // It may be either: + // - "stable" to use the catalog's current stable version + // - an explicit version such as "v1.35.3" + // + // The resolved target version is reported in status. + // +kubebuilder:validation:MinLength=1 DesiredVersion string `json:"desiredVersion,omitempty" yaml:"desiredVersion,omitempty"` + // FlashProfile controls how aggressively the image is written to disk. + // + // Supported values are: + // - "fast": prioritize speed + // - "balanced": default tradeoff between speed and system impact + // - "safe": minimize I/O pressure on the node + // + // If unset, the controller should use a default profile. + // +kubebuilder:validation:Enum=fast;balanced;safe + // +kubebuilder:default=balanced + FlashProfile string `json:"flashProfile,omitempty"` + + // Catalog specifies how available versions and images are resolved. + // + // This may point to a remote catalog, inline catalog data, or another source, + // depending on the VersionCatalogSource definition. Catalog *VersionCatalogSource `json:"catalog,omitempty"` + // NodeSelector limits the upgrade to nodes whose labels match this selector. + // + // If unset, the upgrade applies to all eligible nodes. NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty" yaml:"nodeSelector,omitempty"` } diff --git a/docker/alpine.Dockerfile b/docker/alpine.Dockerfile index 21ef2c9..fbee291 100644 --- a/docker/alpine.Dockerfile +++ b/docker/alpine.Dockerfile @@ -27,9 +27,9 @@ RUN mkdir -p /out/rootfs/usr/local/bin/ COPY packages/kubernetes/kubelet-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubelet COPY packages/kubernetes/kubeadm-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubeadm COPY packages/kubernetes/kubectl-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubectl -COPY clitools/out/dpdk/bin/dpdk-testpmd /out/rootfs/usr/local/bin/dpdk-testpmd -COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/ -COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/ +# COPY clitools/out/dpdk/bin/dpdk-testpmd /out/rootfs/usr/local/bin/dpdk-testpmd +# COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/ +# COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/ COPY alpine/rootfs-extra ./rootfs-extra COPY out/build-info ./rootfs-extra/etc/profile.d/build-info.sh COPY alpine/*.sh / diff --git a/docker/build-base.Dockerfile b/docker/build-base.Dockerfile index 656e670..6e6f605 100644 --- a/docker/build-base.Dockerfile +++ b/docker/build-base.Dockerfile @@ -4,41 +4,47 @@ ENV DEBIAN_FRONTEND=noninteractive WORKDIR /build +ARG APT_PROXY + +RUN if [ -n "${APT_PROXY}" ]; then \ + echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy; \ + fi + RUN apt-get update && apt-get install -y --no-install-recommends \ bash \ bc \ bison \ build-essential \ cpio \ - ca-certificates \ + ca-certificates \ curl \ dosfstools \ file \ - fdisk \ + fdisk \ fuse-overlayfs \ - gdisk \ + gdisk \ e2fsprogs \ flex \ git \ gettext-base \ - jq \ + jq \ kmod \ libelf-dev \ libssl-dev \ make \ pahole \ - parted \ + parted \ perl \ - pv \ + pv \ python3 \ - qemu-user-static \ - podman \ - skopeo \ + qemu-user-static \ + podman \ + skopeo \ rsync \ tar \ - udev \ + udev \ xz-utils \ - zstd \ + zstd \ dwarves \ gcc-aarch64-linux-gnu \ binutils-aarch64-linux-gnu \ @@ -47,4 +53,3 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ u-boot-tools \ device-tree-compiler \ && rm -rf /var/lib/apt/lists/* - diff --git a/docker/kernel-build.Dockerfile b/docker/kernel-build.Dockerfile index 9673b6f..5ffae1a 100644 --- a/docker/kernel-build.Dockerfile +++ b/docker/kernel-build.Dockerfile @@ -23,6 +23,7 @@ WORKDIR /build/nxplinux COPY kernel-extra.config /tmp/kernel-extra.config COPY kernel-build/dts/*.dts ./arch/arm64/boot/dts/freescale/ +COPY kernel-build/scripts/patch_usdpaa_dts.py /usr/local/bin/patch_usdpaa_dts.py RUN grep -q "^dtb-\\\$(CONFIG_ARCH_LAYERSCAPE) += ${DEVICE_TREE_TARGET}.dtb$" \ arch/arm64/boot/dts/freescale/Makefile \ diff --git a/docker/vpp.Dockerfile b/docker/vpp.Dockerfile new file mode 100644 index 0000000..2272bc5 --- /dev/null +++ b/docker/vpp.Dockerfile @@ -0,0 +1,73 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive +WORKDIR /src + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + && rm -rf /var/lib/apt/lists/* + +# Add FD.io release repo (Jammy / arm64 packages exist for v26.02) +RUN set -eux; \ + mkdir -p /etc/apt/keyrings; \ + curl -fsSL https://packagecloud.io/fdio/release/gpgkey \ + | gpg --dearmor -o /etc/apt/keyrings/fdio-release.gpg; \ + echo "deb [signed-by=/etc/apt/keyrings/fdio-release.gpg] https://packagecloud.io/fdio/release/ubuntu/ jammy main" \ + > /etc/apt/sources.list.d/fdio-release.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + vpp=26.02-release \ + vpp-plugin-core=26.02-release \ + vpp-plugin-dpdk=26.02-release; \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /etc/vpp /run/vpp + +RUN cat > /etc/vpp/startup.conf <<'EOF' +unix { + nodaemon + cli-listen /run/vpp/cli.sock + log /dev/stderr + full-coredump +} + +api-trace { + on +} + +cpu { + main-core 0 +} + +plugins { + plugin default { disable } + plugin dpdk_plugin.so { enable } +} + +dpdk { + no-pci + dev dpaa_bus:fm1-mac9 + dev dpaa_bus:fm1-mac10 +} +EOF + +RUN cat > /usr/local/bin/docker-entrypoint.sh <<'EOF' +#!/bin/sh +set -eu + +mkdir -p /run/vpp + +echo "=== device check ===" +ls -l /dev/fsl-usdpaa /dev/fsl-usdpaa-irq 2>/dev/null || true +echo "=== sysfs check ===" +ls -l /sys/class/net 2>/dev/null || true + +exec /usr/bin/vpp -c /etc/vpp/startup.conf "$@" +EOF + +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] diff --git a/docs/dpdk.md b/docs/dpdk.md deleted file mode 100644 index 2f4206f..0000000 --- a/docs/dpdk.md +++ /dev/null @@ -1,12 +0,0 @@ -Testing - -Run -``` -ls /sys/bus/platform/devices | grep -i dpaa -ls /sys/devices/platform | grep -i dpaa -``` - -You should see entries for: - * fman - * dpaa ethernet - * qman/bman portals diff --git a/docs/dpdk_vpp.md b/docs/dpdk_vpp.md new file mode 100644 index 0000000..6167c1d --- /dev/null +++ b/docs/dpdk_vpp.md @@ -0,0 +1,92 @@ +# DPDK and VPP + +This is pain. + +I'm not able to build dpdk / vpp for alpine aarch64. + +## DPDK +Testing with dpdk-testpmd. Does not seem to like the NIC layout we have here. + +We get this error complaining when some NICs controlled by linux +``` +# dpdk-testpmd -a dpaa_bus:fm1-mac9 -a dpaa_bus:fm1-mac10 -- -i +EAL: Detected CPU lcores: 4estpmd -a dpaa_bus:fm1-mac9 -a dpaa_bus:fm1-mac10 -- -i +EAL: Detected NUMA nodes: 1 +EAL: Detected static linkage of DPDK +EAL: Multi-process socket /var/run/dpdk/rte/mp_socket +EAL: Selected IOVA mode 'PA' +EAL: Error parsing /proc/mounts +EAL: Error parsing /proc/mounts +EAL: Error parsing /proc/mounts +EAL: VFIO support initialized +dpaa: This is LS1046A family SoC. +dpaa: FMan version is 0x06 +dpaa: /soc/fsl,dpaa/ethernet@1: no fsl,qman-frame-queues-rx(2) +dpaa: if_init(/soc/fsl,dpaa/ethernet@1)(2) +dpaa: FMAN driver init failed (2) +dpaa: netcfg failed: /dev/fsl_usdpaa device not available +dpaa: Check if you are using USDPAA based device tree +TELEMETRY: No legacy callbacks, legacy socket not created +testpmd: No probed ethernet devices +Interactive-mode selected +testpmd: create a new mbuf pool : n=171456, size=2176, socket=0 +testpmd: preferred mempool ops selected: ring_mp_mc +Done +testpmd> quit +``` + +Works only when all interfaces are handed over +``` +# dpdk-testpmd -a dpaa_bus:fm1-mac9 -a dpaa_bus:fm1-mac10 -- -i +EAL: Detected CPU lcores: 4 +EAL: Detected NUMA nodes: 1 +EAL: Detected static linkage of DPDK +EAL: Multi-process socket /var/run/dpdk/rte/mp_socket +EAL: Selected IOVA mode 'PA' +EAL: VFIO support initialized +dpaa: This is LS1046A family SoC. +dpaa: FMan version is 0x06 +dpaa: DPAA Bus Detected +TELEMETRY: No legacy callbacks, legacy socket not created +Interactive-mode selected +testpmd: create a new mbuf pool : n=171456, size=2176, socket=0 +testpmd: preferred mempool ops selected: dpaa +Configuring Port 0 (socket 0) + +Port 0: link state change event +Port 0: E8:F6:D7:00:1E:95 +Configuring Port 1 (socket 0) + +Port 1: link state change event +Port 1: E8:F6:D7:00:1E:96 +Checking link statuses... +Done +testpmd> quit + +Stopping port 0... +Stopping ports... +Done + +Stopping port 1... +Stopping ports... +Done + +Shutting down port 0... +Closing ports... + +Port 0: link state change event +Port 0 is closed +Done + +Shutting down port 1... +Closing ports... + +Port 1: link state change event +Port 1 is closed +Done + +Bye... +``` + +## VPP +Can't even build. The toolchain is messy. So many errors to plow through. I gave up. diff --git a/kernel-build/dts/mono-gateway-dk-sdk.dts b/kernel-build/dts/mono-gateway-dk-sdk.dts index a0320cd..80fea88 100644 --- a/kernel-build/dts/mono-gateway-dk-sdk.dts +++ b/kernel-build/dts/mono-gateway-dk-sdk.dts @@ -212,6 +212,29 @@ * SDK driver expectations (0-5 instead of mainline 2-7). * Port@82000 with cell-index=0 becomes the HC port for PCD. */ + bp7: buffer-pool@7 { + compatible = "fsl,ls1046a-bpool", "fsl,bpool"; + fsl,bpid = <7>; + fsl,bpool-ethernet-cfg = <0 0 0 192 0 0xdeadbeef>; + fsl,bpool-thresholds = <0x400 0xc00 0x0 0x0>; + dma-coherent; + }; + + bp8: buffer-pool@8 { + compatible = "fsl,ls1046a-bpool", "fsl,bpool"; + fsl,bpid = <8>; + fsl,bpool-ethernet-cfg = <0 0 0 576 0 0xabbaf00d>; + fsl,bpool-thresholds = <0x100 0x300 0x0 0x0>; + dma-coherent; + }; + + bp9: buffer-pool@9 { + compatible = "fsl,ls1046a-bpool", "fsl,bpool"; + fsl,bpid = <9>; + fsl,bpool-ethernet-cfg = <0 0 0 2048 0 0xfeedabba>; + fsl,bpool-thresholds = <0x100 0x300 0x0 0x0>; + dma-coherent; + }; pcie@3400000 { /delete-property/ iommu-map; @@ -229,33 +252,34 @@ }; &fsldpaa { - ethernet@0 { - status = "disabled"; /* MAC1 - not on Mono Gateway DK */ - }; + compatible = "fsl,ls1046a", "fsl,dpaa", "simple-bus"; + dma-coherent; - ethernet@2 { - status = "disabled"; /* MAC3 - not on Mono Gateway DK */ - }; - ethernet@3 { - status = "disabled"; /* MAC4 - not on Mono Gateway DK */ - }; + ethernet@0 { status = "disabled"; }; /* MAC1 - not on Mono Gateway DK */ + ethernet@2 { status = "disabled"; }; /* MAC3 - not on Mono Gateway DK */ + ethernet@3 { status = "disabled"; }; /* MAC4 - not on Mono Gateway DK */ - /* - * Keep Linux management/data-plane-visible kernel interfaces: - * ethernet@1 = MAC2 - * ethernet@4 = MAC5 - * ethernet@5 = MAC6 - * - * Remove the two 10G SFP+ ports from Linux DPAA netdev ownership - * so they can be owned by DPDK/VPP instead. + /* Enabled from qoriq-dpaa-eth.dtsi: + * ethernet@1 = enet1 = MAC2 + * ethernet@4 = enet4 = MAC5 + * ethernet@5 = enet5 = MAC6 + * ethernet@8 = enet6 = MAC9 (10G) */ - ethernet@8 { - status = "disabled"; /* MAC9 / 10G / SFP+ */ - }; + ethernet@8 { + compatible = "fsl,dpa-ethernet-init"; + fsl,bman-buffer-pools = <&bp7 &bp8 &bp9>; + fsl,qman-frame-queues-rx = <0x5c 1 0x5d 1>; + fsl,qman-frame-queues-tx = <0x7c 1 0x7d 1>; + }; + + /* Add MAC10 - not in qoriq-dpaa-eth.dtsi */ ethernet@9 { - status = "disabled"; /* MAC10 / 10G / SFP+ */ + compatible = "fsl,dpa-ethernet-init"; + fsl,bman-buffer-pools = <&bp7 &bp8 &bp9>; + fsl,qman-frame-queues-rx = <0x5c 1 0x5d 1>; + fsl,qman-frame-queues-tx = <0x7c 1 0x7d 1>; }; }; diff --git a/makefile b/makefile index 8a94499..b130b01 100644 --- a/makefile +++ b/makefile @@ -10,6 +10,7 @@ E2FSPROGS_TAR := $(PACKAGES_DIR)/e2fsprogs-$(E2FSPROGS_VERSION).tar.gz BUSYBOX_TAR := $(PACKAGES_DIR)/busybox-$(BUSYBOX_VERSION).tar.gz ALPINE_TAR := $(PACKAGES_DIR)/alpine-minirootfs-$(ALPINE_VER)-$(ALPINE_ARCH).tar.gz NXP_TAR := $(PACKAGES_DIR)/$(NXP_VERSION).tar.gz +VPP_TAR := $(PACKAGES_DIR)/vpp/v26.02.tar.gz CRIO_TAR := $(PACKAGES_DIR)/$(CRIO_VERSION).tar.gz # Kubernetes components @@ -105,7 +106,8 @@ RELEASE_DEPS := \ $(PACKAGES_DIR): mkdir -p $@ \ - $@/kubernetes + $@/kubernetes \ + $@/vpp $(OUT_DIR): mkdir -p $@ @@ -126,6 +128,9 @@ $(BUSYBOX_TAR): | $(PACKAGES_DIR) $(E2FSPROGS_TAR): | $(PACKAGES_DIR) curl -L -o $@ "https://github.com/tytso/e2fsprogs/archive/refs/tags/v$(E2FSPROGS_VERSION).tar.gz" +$(VPP_TAR): | $(PACKAGES_DIR) + curl -L -o $@ "https://github.com/nxp-qoriq/vpp/archive/refs/tags/$(VPP_VERSION).tar.gz" + $(ALPINE_TAR): | $(PACKAGES_DIR) curl -L -o $@ "https://dl-cdn.alpinelinux.org/alpine/v$(ALPINE_SERIES)/releases/$(ALPINE_ARCH)/alpine-minirootfs-$(ALPINE_VER)-$(ALPINE_ARCH).tar.gz" @@ -149,6 +154,7 @@ $(BUILD_INFO_FILE): $(BUILD_BASE_STAMP): $(BUILD_BASE_DEPS) | $(OUT_DIR) docker build \ -f docker/build-base.Dockerfile \ + --build-arg APT_PROXY=$(APT_PROXY) \ --build-arg TAG=$(TAG) \ -t $(DOCKER_IMAGE_ROOT)/build-base:$(TAG) . @iid=$$(docker image inspect monok8s/build-base:$(TAG) | jq -r '.[].Id' | cut -d':' -f2 | cut -c -8); \ @@ -181,7 +187,17 @@ $(INITRAMFS): $(INITRAMFS_DEPS) | $(OUT_DIR) test -f $@ $(CLITOOLS_BIN): $(CLITOOLS_SRCS) - $(MAKE) -C clitools build-agent dpdk + $(MAKE) -C clitools build-agent + +vpp: $(VPP_TAR) + @mkdir -p $(OUT_DIR)/vpp + docker buildx build --platform linux/arm64 \ + -f docker/vpp.Dockerfile \ + --build-arg APT_PROXY=$(APT_PROXY) \ + --build-arg DOCKER_IMAGE_ROOT=$(DOCKER_IMAGE_ROOT) \ + --build-arg TAG=$(TAG) \ + --build-arg VPP_TAR=$(VPP_TAR) \ + -t $(DOCKER_IMAGE_ROOT)/vpp:$(TAG) . $(BOARD_ITB): $(ITB_DEPS) | $(OUT_DIR) docker build \ @@ -273,4 +289,5 @@ pkgclean: rm -rf $(PACKAGES_DIR) .PHONY: release kernel initramfs itb build-base clitools clean distclean pkgclean \ + vpp \ cluster-config cluster-defconfig cluster-print