Trying to build vpp

build.env

@@ -6,6 +6,7 @@ TAG=dev
 # The Linux kernel, from NXP
 NXP_VERSION=lf-6.18.2-1.0.0
 DPDK_VERSION=lf-6.18.2-1.0.0
+VPP_VERSION=lf-6.18.2-1.0.0
 
 CRIO_VERSION=cri-o.arm64.v1.33.3
 KUBE_VERSION=v1.33.3
@@ -14,7 +15,6 @@ KUBE_VERSION=v1.33.3
 # We need fsl-ls1046a-rdb-sdk.dtb here
 DEVICE_TREE_TARGET=mono-gateway-dk-sdk
 
-
 # Arch, should always be arm64 for our board. This is here in case branching off to other devices
 ARCH=arm64
 
@@ -34,3 +34,6 @@ ALPINE_HOSTNAME=monok8s-hostname
 
 # Upper case [A-Z]_ only, used for naming env vars
 BUILD_TAG=MONOK8S
+
+# Optional apt cache
+APT_PROXY=apt-cacher-ng.eco-system.svc.cluster.local:3142

clitools/docker/dpdk.Dockerfile

@@ -44,7 +44,7 @@ RUN set -eux; \
 RUN set -eux; \
     mkdir -p /artifact/bin; \
     test -x /src/dpdk/build/app/dpdk-testpmd; \
-    cp /src/dpdk/build/app/dpdk-testpmd /artifact/bin/
+    cp -r /src/dpdk/build/app/dpdk-testpmd /artifact/bin/
 
 FROM scratch AS export
 COPY --from=build /out/ /

clitools/makefile

@@ -53,7 +53,7 @@ uboot-tools: $(UBOOT_TAR)
 		--build-arg UBOOT_TAR=$(UBOOT_TAR) \
 		--output type=local,dest=./$(OUT_DIR) .
 
-dpdk: $(UBOOT_TAR)
+dpdk: $(DPDK_TAR)
 	@mkdir -p $(OUT_DIR)/dpdk
 	docker buildx build --platform linux/arm64 \
 		-f docker/dpdk.Dockerfile \

clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go

@@ -52,12 +52,39 @@ type OSUpgradeList struct {
 	Items           []OSUpgrade `json:"items" yaml:"items"`
 }
 
+// OSUpgradeSpec defines the desired state of an OSUpgrade.
 type OSUpgradeSpec struct {
-	// User request, can be "stable" or an explicit version like "v1.35.3".
+	// DesiredVersion is the requested target version.
+	//
+	// It may be either:
+	// - "stable" to use the catalog's current stable version
+	// - an explicit version such as "v1.35.3"
+	//
+	// The resolved target version is reported in status.
+	// +kubebuilder:validation:MinLength=1
 	DesiredVersion string `json:"desiredVersion,omitempty" yaml:"desiredVersion,omitempty"`
 
+	// FlashProfile controls how aggressively the image is written to disk.
+	//
+	// Supported values are:
+	// - "fast": prioritize speed
+	// - "balanced": default tradeoff between speed and system impact
+	// - "safe": minimize I/O pressure on the node
+	//
+	// If unset, the controller should use a default profile.
+	// +kubebuilder:validation:Enum=fast;balanced;safe
+	// +kubebuilder:default=balanced
+	FlashProfile string `json:"flashProfile,omitempty"`
+
+	// Catalog specifies how available versions and images are resolved.
+	//
+	// This may point to a remote catalog, inline catalog data, or another source,
+	// depending on the VersionCatalogSource definition.
 	Catalog *VersionCatalogSource `json:"catalog,omitempty"`
 
+	// NodeSelector limits the upgrade to nodes whose labels match this selector.
+	//
+	// If unset, the upgrade applies to all eligible nodes.
 	NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty" yaml:"nodeSelector,omitempty"`
 }
 

docker/alpine.Dockerfile

@@ -27,9 +27,9 @@ RUN mkdir -p /out/rootfs/usr/local/bin/
 COPY packages/kubernetes/kubelet-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubelet
 COPY packages/kubernetes/kubeadm-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubeadm
 COPY packages/kubernetes/kubectl-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubectl
-COPY clitools/out/dpdk/bin/dpdk-testpmd /out/rootfs/usr/local/bin/dpdk-testpmd
+# COPY clitools/out/dpdk/bin/dpdk-testpmd /out/rootfs/usr/local/bin/dpdk-testpmd
-COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/
+# COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/
-COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/
+# COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/
 COPY alpine/rootfs-extra ./rootfs-extra
 COPY out/build-info ./rootfs-extra/etc/profile.d/build-info.sh
 COPY alpine/*.sh /

docker/build-base.Dockerfile

@@ -4,6 +4,12 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 WORKDIR /build
 
+ARG APT_PROXY
+
+RUN if [ -n "${APT_PROXY}" ]; then \
+      echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy; \
+    fi
+
 RUN apt-get update && apt-get install -y --no-install-recommends \
     bash \
     bc \
@@ -47,4 +53,3 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     u-boot-tools \
     device-tree-compiler \
  && rm -rf /var/lib/apt/lists/*
-

docker/kernel-build.Dockerfile

@@ -23,6 +23,7 @@ WORKDIR /build/nxplinux
 
 COPY kernel-extra.config /tmp/kernel-extra.config
 COPY kernel-build/dts/*.dts ./arch/arm64/boot/dts/freescale/
+COPY kernel-build/scripts/patch_usdpaa_dts.py /usr/local/bin/patch_usdpaa_dts.py
 
 RUN grep -q "^dtb-\\\$(CONFIG_ARCH_LAYERSCAPE) += ${DEVICE_TREE_TARGET}.dtb$" \
       arch/arm64/boot/dts/freescale/Makefile \

docker/vpp.Dockerfile

@@ -0,0 +1,73 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /src
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release \
+ && rm -rf /var/lib/apt/lists/*
+
+# Add FD.io release repo (Jammy / arm64 packages exist for v26.02)
+RUN set -eux; \
+    mkdir -p /etc/apt/keyrings; \
+    curl -fsSL https://packagecloud.io/fdio/release/gpgkey \
+      | gpg --dearmor -o /etc/apt/keyrings/fdio-release.gpg; \
+    echo "deb [signed-by=/etc/apt/keyrings/fdio-release.gpg] https://packagecloud.io/fdio/release/ubuntu/ jammy main" \
+      > /etc/apt/sources.list.d/fdio-release.list; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+      vpp=26.02-release \
+      vpp-plugin-core=26.02-release \
+      vpp-plugin-dpdk=26.02-release; \
+    rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /etc/vpp /run/vpp
+
+RUN cat > /etc/vpp/startup.conf <<'EOF'
+unix {
+  nodaemon
+  cli-listen /run/vpp/cli.sock
+  log /dev/stderr
+  full-coredump
+}
+
+api-trace {
+  on
+}
+
+cpu {
+  main-core 0
+}
+
+plugins {
+  plugin default { disable }
+  plugin dpdk_plugin.so { enable }
+}
+
+dpdk {
+  no-pci
+  dev dpaa_bus:fm1-mac9
+  dev dpaa_bus:fm1-mac10
+}
+EOF
+
+RUN cat > /usr/local/bin/docker-entrypoint.sh <<'EOF'
+#!/bin/sh
+set -eu
+
+mkdir -p /run/vpp
+
+echo "=== device check ==="
+ls -l /dev/fsl-usdpaa /dev/fsl-usdpaa-irq 2>/dev/null || true
+echo "=== sysfs check ==="
+ls -l /sys/class/net 2>/dev/null || true
+
+exec /usr/bin/vpp -c /etc/vpp/startup.conf "$@"
+EOF
+
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

docs/dpdk.md

@@ -1,12 +0,0 @@
-Testing
-
-Run
-```
-ls /sys/bus/platform/devices | grep -i dpaa
-ls /sys/devices/platform | grep -i dpaa
-```
-
-You should see entries for:
-    * fman
-    * dpaa ethernet
-    * qman/bman portals

docs/dpdk_vpp.md

@@ -0,0 +1,92 @@
+# DPDK and VPP
+
+This is pain.
+
+I'm not able to build dpdk / vpp for alpine aarch64.
+
+## DPDK
+Testing with dpdk-testpmd. Does not seem to like the NIC layout we have here.
+
+We get this error complaining when some NICs controlled by linux
+```
+# dpdk-testpmd -a dpaa_bus:fm1-mac9 -a dpaa_bus:fm1-mac10 -- -i
+EAL: Detected CPU lcores: 4estpmd -a dpaa_bus:fm1-mac9 -a dpaa_bus:fm1-mac10 -- -i
+EAL: Detected NUMA nodes: 1
+EAL: Detected static linkage of DPDK
+EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
+EAL: Selected IOVA mode 'PA'
+EAL: Error parsing /proc/mounts
+EAL: Error parsing /proc/mounts
+EAL: Error parsing /proc/mounts
+EAL: VFIO support initialized
+dpaa: This is LS1046A family SoC.
+dpaa: FMan version is 0x06
+dpaa: /soc/fsl,dpaa/ethernet@1: no fsl,qman-frame-queues-rx(2)
+dpaa: if_init(/soc/fsl,dpaa/ethernet@1)(2)
+dpaa: FMAN driver init failed (2)
+dpaa: netcfg failed: /dev/fsl_usdpaa device not available
+dpaa: Check if you are using USDPAA based device tree
+TELEMETRY: No legacy callbacks, legacy socket not created
+testpmd: No probed ethernet devices
+Interactive-mode selected
+testpmd: create a new mbuf pool <mb_pool_0>: n=171456, size=2176, socket=0
+testpmd: preferred mempool ops selected: ring_mp_mc
+Done
+testpmd> quit
+```
+
+Works only when all interfaces are handed over
+```
+# dpdk-testpmd -a dpaa_bus:fm1-mac9 -a dpaa_bus:fm1-mac10 -- -i
+EAL: Detected CPU lcores: 4
+EAL: Detected NUMA nodes: 1
+EAL: Detected static linkage of DPDK
+EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
+EAL: Selected IOVA mode 'PA'
+EAL: VFIO support initialized
+dpaa: This is LS1046A family SoC.
+dpaa: FMan version is 0x06
+dpaa: DPAA Bus Detected
+TELEMETRY: No legacy callbacks, legacy socket not created
+Interactive-mode selected
+testpmd: create a new mbuf pool <mb_pool_0>: n=171456, size=2176, socket=0
+testpmd: preferred mempool ops selected: dpaa
+Configuring Port 0 (socket 0)
+
+Port 0: link state change event
+Port 0: E8:F6:D7:00:1E:95
+Configuring Port 1 (socket 0)
+
+Port 1: link state change event
+Port 1: E8:F6:D7:00:1E:96
+Checking link statuses...
+Done
+testpmd> quit
+
+Stopping port 0...
+Stopping ports...
+Done
+
+Stopping port 1...
+Stopping ports...
+Done
+
+Shutting down port 0...
+Closing ports...
+
+Port 0: link state change event
+Port 0 is closed
+Done
+
+Shutting down port 1...
+Closing ports...
+
+Port 1: link state change event
+Port 1 is closed
+Done
+
+Bye...
+```
+
+## VPP
+Can't even build. The toolchain is messy. So many errors to plow through. I gave up.

kernel-build/dts/mono-gateway-dk-sdk.dts

@@ -212,6 +212,29 @@
 	 * SDK driver expectations (0-5 instead of mainline 2-7).
 	 * Port@82000 with cell-index=0 becomes the HC port for PCD.
 	 */
+	bp7: buffer-pool@7 {
+		compatible = "fsl,ls1046a-bpool", "fsl,bpool";
+		fsl,bpid = <7>;
+		fsl,bpool-ethernet-cfg = <0 0 0 192 0 0xdeadbeef>;
+		fsl,bpool-thresholds = <0x400 0xc00 0x0 0x0>;
+		dma-coherent;
+	};
+
+	bp8: buffer-pool@8 {
+		compatible = "fsl,ls1046a-bpool", "fsl,bpool";
+		fsl,bpid = <8>;
+		fsl,bpool-ethernet-cfg = <0 0 0 576 0 0xabbaf00d>;
+		fsl,bpool-thresholds = <0x100 0x300 0x0 0x0>;
+		dma-coherent;
+	};
+
+	bp9: buffer-pool@9 {
+		compatible = "fsl,ls1046a-bpool", "fsl,bpool";
+		fsl,bpid = <9>;
+		fsl,bpool-ethernet-cfg = <0 0 0 2048 0 0xfeedabba>;
+		fsl,bpool-thresholds = <0x100 0x300 0x0 0x0>;
+		dma-coherent;
+	};
 
 	pcie@3400000 {
 		/delete-property/ iommu-map;
@@ -229,33 +252,34 @@
 };
 
 &fsldpaa {
-	ethernet@0 {
+    compatible = "fsl,ls1046a", "fsl,dpaa", "simple-bus";
-		status = "disabled";  /* MAC1 - not on Mono Gateway DK */
+	dma-coherent;
-	};
 
-	ethernet@2 {
-		status = "disabled";  /* MAC3 - not on Mono Gateway DK */
-	};
 
-	ethernet@3 {
+	ethernet@0 { status = "disabled"; }; /* MAC1 - not on Mono Gateway DK */
-		status = "disabled";  /* MAC4 - not on Mono Gateway DK */
+	ethernet@2 { status = "disabled"; }; /* MAC3 - not on Mono Gateway DK */
-	};
+	ethernet@3 { status = "disabled"; }; /* MAC4 - not on Mono Gateway DK */
 
-	/*
+	/* Enabled from qoriq-dpaa-eth.dtsi:
-	 * Keep Linux management/data-plane-visible kernel interfaces:
+	 * ethernet@1 = enet1 = MAC2
-	 *   ethernet@1 = MAC2
+	 * ethernet@4 = enet4 = MAC5
-	 *   ethernet@4 = MAC5
+	 * ethernet@5 = enet5 = MAC6
-	 *   ethernet@5 = MAC6
+	 * ethernet@8 = enet6 = MAC9 (10G)
-	 *
-	 * Remove the two 10G SFP+ ports from Linux DPAA netdev ownership
-	 * so they can be owned by DPDK/VPP instead.
 	 */
+
     ethernet@8 {
-		status = "disabled";  /* MAC9 / 10G / SFP+ */
+        compatible = "fsl,dpa-ethernet-init";
+        fsl,bman-buffer-pools = <&bp7 &bp8 &bp9>;
+        fsl,qman-frame-queues-rx = <0x5c 1 0x5d 1>;
+        fsl,qman-frame-queues-tx = <0x7c 1 0x7d 1>;
     };
 
+	/* Add MAC10 - not in qoriq-dpaa-eth.dtsi */
 	ethernet@9 {
-		status = "disabled";  /* MAC10 / 10G / SFP+ */
+        compatible = "fsl,dpa-ethernet-init";
+        fsl,bman-buffer-pools = <&bp7 &bp8 &bp9>;
+        fsl,qman-frame-queues-rx = <0x5c 1 0x5d 1>;
+        fsl,qman-frame-queues-tx = <0x7c 1 0x7d 1>;
 	};
 };
 

makefile

@@ -10,6 +10,7 @@ E2FSPROGS_TAR := $(PACKAGES_DIR)/e2fsprogs-$(E2FSPROGS_VERSION).tar.gz
 BUSYBOX_TAR   := $(PACKAGES_DIR)/busybox-$(BUSYBOX_VERSION).tar.gz
 ALPINE_TAR    := $(PACKAGES_DIR)/alpine-minirootfs-$(ALPINE_VER)-$(ALPINE_ARCH).tar.gz
 NXP_TAR       := $(PACKAGES_DIR)/$(NXP_VERSION).tar.gz
+VPP_TAR       := $(PACKAGES_DIR)/vpp/v26.02.tar.gz
 CRIO_TAR      := $(PACKAGES_DIR)/$(CRIO_VERSION).tar.gz
 
 # Kubernetes components
@@ -105,7 +106,8 @@ RELEASE_DEPS := \
 
 $(PACKAGES_DIR):
 	mkdir -p $@ \
-		$@/kubernetes
+		$@/kubernetes \
+		$@/vpp
 
 $(OUT_DIR):
 	mkdir -p $@
@@ -126,6 +128,9 @@ $(BUSYBOX_TAR): | $(PACKAGES_DIR)
 $(E2FSPROGS_TAR): | $(PACKAGES_DIR)
 	curl -L -o $@ "https://github.com/tytso/e2fsprogs/archive/refs/tags/v$(E2FSPROGS_VERSION).tar.gz"
 
+$(VPP_TAR): | $(PACKAGES_DIR)
+	curl -L -o $@ "https://github.com/nxp-qoriq/vpp/archive/refs/tags/$(VPP_VERSION).tar.gz"
+
 $(ALPINE_TAR): | $(PACKAGES_DIR)
 	curl -L -o $@ "https://dl-cdn.alpinelinux.org/alpine/v$(ALPINE_SERIES)/releases/$(ALPINE_ARCH)/alpine-minirootfs-$(ALPINE_VER)-$(ALPINE_ARCH).tar.gz"
 
@@ -149,6 +154,7 @@ $(BUILD_INFO_FILE):
 $(BUILD_BASE_STAMP): $(BUILD_BASE_DEPS) | $(OUT_DIR)
 	docker build \
 		-f docker/build-base.Dockerfile \
+		--build-arg APT_PROXY=$(APT_PROXY) \
 		--build-arg TAG=$(TAG) \
 		-t $(DOCKER_IMAGE_ROOT)/build-base:$(TAG) .
 	@iid=$$(docker image inspect monok8s/build-base:$(TAG) | jq -r '.[].Id' | cut -d':' -f2 | cut -c -8); \
@@ -181,7 +187,17 @@ $(INITRAMFS): $(INITRAMFS_DEPS) | $(OUT_DIR)
 	test -f $@
 
 $(CLITOOLS_BIN): $(CLITOOLS_SRCS)
-	$(MAKE) -C clitools build-agent dpdk
+	$(MAKE) -C clitools build-agent
+
+vpp: $(VPP_TAR)
+	@mkdir -p $(OUT_DIR)/vpp
+	docker buildx build --platform linux/arm64 \
+		-f docker/vpp.Dockerfile \
+		--build-arg APT_PROXY=$(APT_PROXY) \
+		--build-arg DOCKER_IMAGE_ROOT=$(DOCKER_IMAGE_ROOT) \
+		--build-arg TAG=$(TAG) \
+		--build-arg VPP_TAR=$(VPP_TAR) \
+		-t $(DOCKER_IMAGE_ROOT)/vpp:$(TAG) .
 
 $(BOARD_ITB): $(ITB_DEPS) | $(OUT_DIR)
 	docker build \
@@ -273,4 +289,5 @@ pkgclean:
 	rm -rf $(PACKAGES_DIR)
 
 .PHONY: release kernel initramfs itb build-base clitools clean distclean pkgclean \
+	vpp \
 	cluster-config cluster-defconfig cluster-print