diff --git a/alpine/build-rootfs.sh b/alpine/build-rootfs.sh index 8779a65..2978a99 100755 --- a/alpine/build-rootfs.sh +++ b/alpine/build-rootfs.sh @@ -64,7 +64,7 @@ DISK_SIZE="${DISK_SIZE:-8G}" ROOTFS_IMG="${ROOTFS_IMG:-rootfs.ext4}" ROOTFS_IMG_ZST="${ROOTFS_IMG_ZST:-rootfs.ext4.zst}" -ROOTFS_PART_SIZE_MIB="${ROOTFS_PART_SIZE_MIB:-2048}" +ROOTFS_PART_SIZE_MIB="${ROOTFS_PART_SIZE_MIB:-2560}" FAKE_DEV="/tmp/dev" MNT_ROOTFS_IMG="/mnt/rootfs-img" @@ -183,8 +183,8 @@ truncate -s "$DISK_SIZE" "$IMG" sgdisk -o "$IMG" \ -n 1:2048:+64M -t 1:0700 -c 1:config \ - -n 2:0:+2G -t 2:8300 -c 2:rootfsA \ - -n 3:0:+2G -t 3:8300 -c 3:rootfsB \ + -n 2:0:+2560M -t 2:8300 -c 2:rootfsA \ + -n 3:0:+2560M -t 3:8300 -c 3:rootfsB \ -n 4:0:0 -t 4:8300 -c 4:data losetup -D diff --git a/alpine/rootfs-extra/etc/local.d/monok8s.start b/alpine/rootfs-extra/etc/local.d/monok8s.start index 4f44f2b..f1044c1 100755 --- a/alpine/rootfs-extra/etc/local.d/monok8s.start +++ b/alpine/rootfs-extra/etc/local.d/monok8s.start @@ -1,4 +1,9 @@ #!/bin/sh export PATH="/usr/local/bin:/usr/local/sbin:$PATH" + +mkdir -p /dev/hugepages +mountpoint -q /dev/hugepages || mount -t hugetlbfs none /dev/hugepages +echo 256 > /proc/sys/vm/nr_hugepages + /usr/local/bin/ctl init --env-file /opt/monok8s/config/cluster.env >>/var/log/monok8s/bootstrap.log 2>&1 & diff --git a/build.env b/build.env index ecffdcb..13dea85 100644 --- a/build.env +++ b/build.env @@ -5,6 +5,8 @@ TAG=dev # The Linux kernel, from NXP NXP_VERSION=lf-6.18.2-1.0.0 +DPDK_VERSION=lf-6.18.2-1.0.0 + CRIO_VERSION=cri-o.arm64.v1.33.3 KUBE_VERSION=v1.33.3 diff --git a/clitools/docker/dpdk.Dockerfile b/clitools/docker/dpdk.Dockerfile new file mode 100644 index 0000000..4452ac3 --- /dev/null +++ b/clitools/docker/dpdk.Dockerfile @@ -0,0 +1,51 @@ +ARG TAG=dev +ARG DOCKER_IMAGE_ROOT=monok8s + +FROM alpine:3.22 AS build + +RUN apk add --no-cache \ + bash \ + build-base \ + linux-headers \ + meson \ + ninja \ + pkgconf \ + python3 \ + py3-elftools \ + coreutils \ + file \ + git \ + bsd-compat-headers + +RUN mkdir /src + +WORKDIR /src + +ARG DPDK_TAR +ARG DPDK_VERSION + +COPY ${DPDK_TAR} /tmp/ + +RUN set -eux; \ + mkdir -p /src/dpdk; \ + tar -xf "/tmp/$(basename "${DPDK_TAR}")" -C /src/dpdk --strip-components=1 + +RUN set -eux; \ + meson setup /src/dpdk/build /src/dpdk \ + --buildtype=release \ + -Dplatform=dpaa \ + -Dtests=false \ + -Ddisable_drivers=crypto/*,compress/*,baseband/*,dma/*,event/*,regex/*,ml/*,gpu/*,raw/*,net/pcap,net/tap,net/vhost,net/virtio,net/ixgbe,net/i40e,net/txgbe,net/ring,net/af_packet; \ + meson configure /src/dpdk/build | tee /tmp/meson-config.txt; \ + grep -Ei 'dpaa|platform|disable_drivers' /tmp/meson-config.txt || true; \ + ninja -C /src/dpdk/build; \ + DESTDIR=/out ninja -C /src/dpdk/build install + +RUN set -eux; \ + mkdir -p /artifact/bin; \ + test -x /src/dpdk/build/app/dpdk-testpmd; \ + cp /src/dpdk/build/app/dpdk-testpmd /artifact/bin/ + +FROM scratch AS export +COPY --from=build /out/ / +COPY --from=build /artifact/ / diff --git a/clitools/makefile b/clitools/makefile index 4d3fdbb..b8529e2 100644 --- a/clitools/makefile +++ b/clitools/makefile @@ -1,3 +1,5 @@ +include ../build.env + # Should be the same as upstream version in production VERSION ?= dev UBOOT_VERSION=v2026.01 @@ -12,12 +14,14 @@ BIN_DIR := bin OUT_DIR := out UBOOT_TAR := $(PACKAGES_DIR)/uboot-$(UBOOT_VERSION).tar.gz +DPDK_TAR := $(PACKAGES_DIR)/dpdk/$(DPDK_VERSION).tar.gz BUILDINFO_FILE := pkg/buildinfo/buildinfo_gen.go CRD_PATHS := ./pkg/apis/... $(PACKAGES_DIR): - mkdir -p $@ + mkdir -p $@ \ + $@/dpdk # Never cache this .buildinfo: @@ -33,6 +37,9 @@ $(PACKAGES_DIR): ')' \ > $(BUILDINFO_FILE) +$(DPDK_TAR): | $(PACKAGES_DIR) + curl -L -o $@ "https://github.com/nxp-qoriq/dpdk/archive/refs/tags/$(DPDK_VERSION).tar.gz" + $(UBOOT_TAR): | $(PACKAGES_DIR) git clone --depth 1 --branch v2026.01 --filter=blob:none https://github.com/u-boot/u-boot.git $(OUT_DIR)/u-boot-$(UBOOT_VERSION) tar -C "$(OUT_DIR)/u-boot-$(UBOOT_VERSION)" -zcf "$@" . @@ -46,6 +53,15 @@ uboot-tools: $(UBOOT_TAR) --build-arg UBOOT_TAR=$(UBOOT_TAR) \ --output type=local,dest=./$(OUT_DIR) . +dpdk: $(UBOOT_TAR) + @mkdir -p $(OUT_DIR)/dpdk + docker buildx build --platform linux/arm64 \ + -f docker/dpdk.Dockerfile \ + --build-arg DOCKER_IMAGE_ROOT=$(DOCKER_IMAGE_ROOT) \ + --build-arg TAG=$(TAG) \ + --build-arg DPDK_TAR=$(DPDK_TAR) \ + --output type=local,dest=./$(OUT_DIR)/dpdk . + build: .buildinfo mkdir -p $(BIN_DIR) $(OUT_DIR)/crds controller-gen crd paths=$(CRD_PATHS) output:crd:dir=$(OUT_DIR)/crds @@ -73,4 +89,4 @@ clean: all: build build-agent build-local -.PHONY: clean all run .buildinfo build build-local build-agent uboot-tools +.PHONY: clean all run .buildinfo build build-local build-agent uboot-tools dpdk diff --git a/docker/alpine.Dockerfile b/docker/alpine.Dockerfile index 68c4d41..21ef2c9 100644 --- a/docker/alpine.Dockerfile +++ b/docker/alpine.Dockerfile @@ -27,8 +27,10 @@ RUN mkdir -p /out/rootfs/usr/local/bin/ COPY packages/kubernetes/kubelet-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubelet COPY packages/kubernetes/kubeadm-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubeadm COPY packages/kubernetes/kubectl-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubectl -RUN chmod +x /out/rootfs/usr/local/bin/* - +COPY clitools/out/dpdk/bin/dpdk-testpmd /out/rootfs/usr/local/bin/dpdk-testpmd +COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/ +COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/ COPY alpine/rootfs-extra ./rootfs-extra COPY out/build-info ./rootfs-extra/etc/profile.d/build-info.sh COPY alpine/*.sh / +RUN chmod +x /out/rootfs/usr/local/bin/* diff --git a/docs/dpdk.md b/docs/dpdk.md new file mode 100644 index 0000000..2f4206f --- /dev/null +++ b/docs/dpdk.md @@ -0,0 +1,12 @@ +Testing + +Run +``` +ls /sys/bus/platform/devices | grep -i dpaa +ls /sys/devices/platform | grep -i dpaa +``` + +You should see entries for: + * fman + * dpaa ethernet + * qman/bman portals diff --git a/kernel-build/dts/mono-gateway-dk-sdk.dts b/kernel-build/dts/mono-gateway-dk-sdk.dts index e74bfc5..a0320cd 100644 --- a/kernel-build/dts/mono-gateway-dk-sdk.dts +++ b/kernel-build/dts/mono-gateway-dk-sdk.dts @@ -21,6 +21,19 @@ reg = <0x00 0xaa000000 0x00 0x4000000 0x00 0xae000000 0x00 0x6000000>; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + usdpaa_mem: usdpaa_mem { + compatible = "fsl,usdpaa-mem"; + alloc-ranges = <0 0 0x10000 0>; + size = <0 0x10000000>; // 256MB + alignment = <0 0x10000000>; + }; + }; + cpus { fman0-extended-args { cell-index = <0x00>; @@ -219,25 +232,30 @@ ethernet@0 { status = "disabled"; /* MAC1 - not on Mono Gateway DK */ }; + ethernet@2 { status = "disabled"; /* MAC3 - not on Mono Gateway DK */ }; + ethernet@3 { status = "disabled"; /* MAC4 - not on Mono Gateway DK */ }; - /* Enabled from qoriq-dpaa-eth.dtsi: - * ethernet@1 = enet1 = MAC2 - * ethernet@4 = enet4 = MAC5 - * ethernet@5 = enet5 = MAC6 - * ethernet@8 = enet6 = MAC9 (10G) + /* + * Keep Linux management/data-plane-visible kernel interfaces: + * ethernet@1 = MAC2 + * ethernet@4 = MAC5 + * ethernet@5 = MAC6 + * + * Remove the two 10G SFP+ ports from Linux DPAA netdev ownership + * so they can be owned by DPDK/VPP instead. */ + ethernet@8 { + status = "disabled"; /* MAC9 / 10G / SFP+ */ + }; - /* Add MAC10 - not in qoriq-dpaa-eth.dtsi */ ethernet@9 { - compatible = "fsl,dpa-ethernet"; - fsl,fman-mac = <&enet7>; - dma-coherent; + status = "disabled"; /* MAC10 / 10G / SFP+ */ }; }; diff --git a/makefile b/makefile index 66d00e1..8a94499 100644 --- a/makefile +++ b/makefile @@ -104,8 +104,8 @@ RELEASE_DEPS := \ # ---- Directory creation ------------------------------------------------------ $(PACKAGES_DIR): - mkdir -p $@ - mkdir -p $@/kubernetes + mkdir -p $@ \ + $@/kubernetes $(OUT_DIR): mkdir -p $@ @@ -181,7 +181,7 @@ $(INITRAMFS): $(INITRAMFS_DEPS) | $(OUT_DIR) test -f $@ $(CLITOOLS_BIN): $(CLITOOLS_SRCS) - $(MAKE) -C clitools build-agent + $(MAKE) -C clitools build-agent dpdk $(BOARD_ITB): $(ITB_DEPS) | $(OUT_DIR) docker build \