ctl create cmm

alpine/preload-k8s-images.sh

@@ -27,6 +27,7 @@ FUSE_OVERLAYFS="${FUSE_OVERLAYFS:-/usr/bin/fuse-overlayfs}"
 EXTRA_IMAGES=(
 	"${EXTRA_IMAGES[@]:-}"
 	"docker-daemon:localhost/monok8s/node-control:$TAG"
+	"docker-daemon:localhost/monok8s/cmm:$TAG"
 )
 
 # Keep archive cache version/arch scoped so downloads do not get mixed.

alpine/rootfs-extra/etc/local.d/monok8s.start

@@ -11,6 +11,19 @@ BOOT_STATE=/run/monok8s/boot-state.env
 BOOTPART_FILE="$CONFIG_DIR/.bootpart"
 MIGRATION_STATE_DIR="$CONFIG_DIR/migration-state"
 
+load_module_optional() {
+	module="$1"
+
+	if ! modprobe "$module"; then
+		echo "WARNING: failed to load optional module: $module"
+	fi
+}
+
+echo "Loading optional hardware offload modules..."
+load_module_optional cdx
+load_module_optional fci
+load_module_optional auto_bridge
+
 mkdir -p /dev/hugepages
 mountpoint -q /dev/hugepages || mount -t hugetlbfs none /dev/hugepages
 echo 256 > /proc/sys/vm/nr_hugepages

ask/cmm/entrypoint.sh

@@ -3,36 +3,9 @@ set -eu
 
 CMM_CONFIG="${CMM_CONFIG:-/etc/cmm/cmm.conf}"
 
-# Vendor default from cmm.service: (131072 = 128 * 1024) max active conntrack/offload entries
+# Vendor default from cmm.service:
+# 131072 = 128 * 1024 max active conntrack/offload entries.
 CMM_MAX_CONNECTIONS="${CMM_MAX_CONNECTIONS:-131072}"
 
-mkdir -p /run/ask /var/log
-
-echo "[ask] loading auto_bridge"
-modprobe auto_bridge || true
-
-echo "[ask] loading cdx"
-modprobe cdx
-
-echo "[ask] waiting for /dev/cdx_ctrl"
-for i in $(seq 1 40); do
-	if [ -e /dev/cdx_ctrl ]; then
-		break
-	fi
-	sleep 0.25
-done
-test -e /dev/cdx_ctrl
-
-if [ ! -e /run/ask/dpa_app.loaded ]; then
-	echo "[ask] running dpa_app"
-	/bin/dpa_app
-	touch /run/ask/dpa_app.loaded
-else
-	echo "[ask] dpa_app already loaded; skipping"
-fi
-
-echo "[ask] loading fci"
-modprobe fci
-
 echo "[ask] starting cmm"
 exec /bin/cmm -D -f "$CMM_CONFIG" -n "$CMM_MAX_CONNECTIONS"

ask/cmm/init_dpa.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -eu
+
+STATE_DIR="${CMM_STATE_DIR:-/host/run/monok8s/cmm}"
+DONE_FILE="$STATE_DIR/dpa_app.loaded"
+LOCK_DIR="$STATE_DIR/dpa_app.lock"
+
+mkdir -p "$STATE_DIR"
+
+if [ -e "$DONE_FILE" ]; then
+	echo "[ask] dpa_app already loaded for this boot; skipping"
+	exit 0
+fi
+
+if ! mkdir "$LOCK_DIR" 2>/dev/null; then
+	echo "[ask] another dpa_app initialization is running; waiting"
+
+	while [ ! -e "$DONE_FILE" ]; do
+		sleep 1
+	done
+
+	echo "[ask] dpa_app was loaded by another process; skipping"
+	exit 0
+fi
+
+trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT
+
+if [ -e "$DONE_FILE" ]; then
+	echo "[ask] dpa_app already loaded for this boot; skipping"
+	exit 0
+fi
+
+echo "[ask] running dpa_app"
+/bin/dpa_app
+
+date -u +"%Y-%m-%dT%H:%M:%SZ" > "$DONE_FILE"
+echo "[ask] dpa_app loaded"

clitools/pkg/cmd/create/create.go

@@ -165,6 +165,47 @@ func NewCmdCreate(flags *genericclioptions.ConfigFlags) *cobra.Command {
 
 	cmd.AddCommand(&agentcmd)
 
+	cmmconf := render.CMMConf{}
+	cmmcmd := cobra.Command{
+		Use:   "cmm",
+		Short: "Print CMM daemonsets template",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			if len(cmmconf.ImagePullSecrets) > 0 && strings.TrimSpace(cmmconf.Image) == "" {
+				return fmt.Errorf("--image-pull-secret requires --image")
+			}
+
+			ns, _, err := flags.ToRawKubeConfigLoader().Namespace()
+			if err != nil {
+				return err
+			}
+
+			cmmconf.Namespace = ns
+
+			out, err := render.RenderCMMDaemonSets(cmmconf)
+			if err != nil {
+				return err
+			}
+
+			_, err = fmt.Fprint(cmd.OutOrStdout(), out)
+			return err
+		},
+	}
+
+	cmmcmd.Flags().StringVar(
+		&cmmconf.Image,
+		"image",
+		"",
+		"CMM image, including optional registry and tag",
+	)
+	cmmcmd.Flags().StringSliceVar(
+		&cmmconf.ImagePullSecrets,
+		"image-pull-secret",
+		nil,
+		"Image pull secret name for the CMM image; may be specified multiple times or as a comma-separated list",
+	)
+
+	cmd.AddCommand(&cmmcmd)
+
 	return cmd
 }
 

clitools/pkg/render/cmm.go

@@ -0,0 +1,219 @@
+package render
+
+import (
+	"fmt"
+	"strings"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	buildinfo "example.com/monok8s/pkg/buildinfo"
+)
+
+const cmmName = "cmm"
+
+type CMMConf struct {
+	Namespace        string
+	Image            string
+	ImagePullSecrets []string
+	Labels           map[string]string
+}
+
+func RenderCMMDaemonSets(conf CMMConf) (string, error) {
+	objs, err := buildCMMDaemonSetObjects(conf)
+	if err != nil {
+		return "", err
+	}
+
+	return renderObjects(objs)
+}
+
+func buildCMMDaemonSetObjects(conf CMMConf) ([]runtime.Object, error) {
+	if strings.TrimSpace(conf.Namespace) == "" {
+		return nil, fmt.Errorf("namespace is required")
+	}
+
+	conf.Labels = map[string]string{
+		"app.kubernetes.io/name":       cmmName,
+		"app.kubernetes.io/component":  "hardware-offload",
+		"app.kubernetes.io/part-of":    "monok8s",
+		"app.kubernetes.io/managed-by": "monok8s",
+	}
+
+	return []runtime.Object{
+		buildCMMServiceAccount(conf),
+		buildCMMDaemonSet(conf),
+	}, nil
+}
+
+func buildCMMServiceAccount(conf CMMConf) *corev1.ServiceAccount {
+	return &corev1.ServiceAccount{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ServiceAccount",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cmmName,
+			Namespace: conf.Namespace,
+			Labels:    copyStringMap(conf.Labels),
+		},
+	}
+}
+
+func buildCMMDaemonSet(conf CMMConf) *appsv1.DaemonSet {
+	privileged := true
+	dsLabels := map[string]string{
+		"app.kubernetes.io/name":       cmmName,
+		"app.kubernetes.io/component":  "hardware-offload",
+		"app.kubernetes.io/part-of":    "monok8s",
+		"app.kubernetes.io/managed-by": "monok8s",
+	}
+
+	image, pullPolicy := cmmImage(conf)
+
+	return &appsv1.DaemonSet{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "apps/v1",
+			Kind:       "DaemonSet",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cmmName,
+			Namespace: conf.Namespace,
+			Labels:    copyStringMap(conf.Labels),
+		},
+		Spec: appsv1.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app.kubernetes.io/name": cmmName,
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: dsLabels,
+				},
+				Spec: corev1.PodSpec{
+					ServiceAccountName: cmmName,
+					HostNetwork:        true,
+					DNSPolicy:          corev1.DNSClusterFirstWithHostNet,
+					ImagePullSecrets:   imagePullSecrets(conf.ImagePullSecrets),
+					Tolerations: []corev1.Toleration{
+						{Operator: corev1.TolerationOpExists},
+					},
+					InitContainers: []corev1.Container{
+						{
+							Name:            "dpa-app",
+							Image:           image,
+							ImagePullPolicy: pullPolicy,
+							Command:         []string{"/init_dpa.sh"},
+							Env:             cdxEnv(),
+							SecurityContext: &corev1.SecurityContext{
+								Privileged: &privileged,
+							},
+							VolumeMounts: append(
+								[]corev1.VolumeMount{
+									{
+										Name:      "host-run-cmm",
+										MountPath: "/host/run/monok8s/cmm",
+									},
+								},
+								cdxVolumeMounts()...,
+							),
+						},
+					},
+					Containers: []corev1.Container{
+						{
+							Name:            cmmName,
+							Image:           image,
+							ImagePullPolicy: pullPolicy,
+							Env:             cmmEnv(),
+							SecurityContext: &corev1.SecurityContext{
+								Privileged: &privileged,
+							},
+							VolumeMounts: cdxVolumeMounts(),
+						},
+					},
+					Volumes: cmmVolumes(),
+					NodeSelector: map[string]string{
+						"node.kubernetes.io/instance-type": "mono-gateway",
+					},
+				},
+			},
+		},
+	}
+}
+
+func cdxEnv() []corev1.EnvVar {
+	return []corev1.EnvVar{
+		{
+			Name:  "CDX_CFG_FILE",
+			Value: "/etc/dpa/cdx_cfg.xml",
+		},
+		{
+			Name:  "CDX_PCD_FILE",
+			Value: "/etc/dpa/cdx_pcd.xml",
+		},
+		{
+			Name:  "CDX_PDL_FILE",
+			Value: "/etc/fmc/config/hxs_pdl_v3.xml",
+		},
+		{
+			Name:  "CDX_SP_FILE",
+			Value: "/etc/dpa/cdx_sp.xml",
+		},
+	}
+}
+
+func cmmEnv() []corev1.EnvVar {
+	return []corev1.EnvVar{
+		{
+			Name:  "CMM_CONFIG",
+			Value: "/etc/cmm/cmm.conf",
+		},
+		{
+			Name:  "CMM_MAX_CONNECTIONS",
+			Value: "131072",
+		},
+	}
+}
+
+func cdxVolumeMounts() []corev1.VolumeMount {
+	return []corev1.VolumeMount{
+		{
+			Name:      "cdx-ctrl",
+			MountPath: "/dev/cdx_ctrl",
+		},
+	}
+}
+
+func cmmVolumes() []corev1.Volume {
+	return []corev1.Volume{
+		{
+			Name: "cdx-ctrl",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: "/dev/cdx_ctrl",
+					Type: hostPathType(corev1.HostPathCharDev),
+				},
+			},
+		},
+		{
+			Name: "host-run-cmm",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: "/run/monok8s/cmm",
+					Type: hostPathType(corev1.HostPathDirectoryOrCreate),
+				},
+			},
+		},
+	}
+}
+
+func cmmImage(conf CMMConf) (string, corev1.PullPolicy) {
+	if conf.Image != "" {
+		return conf.Image, corev1.PullIfNotPresent
+	}
+
+	return fmt.Sprintf("localhost/monok8s/cmm:%s", buildinfo.Version), corev1.PullNever
+}

docker/cmm.Dockerfile

@@ -1,15 +1,13 @@
 ARG ALPINE_SERIES=3.23
 FROM alpine:${ALPINE_SERIES}
 
-RUN apk add --no-cache kmod busybox-extras
-
 COPY ./out/ASK/rootfs-cfg/etc /etc
 COPY ./out/ASK/bin/cmm /bin/cmm
 COPY ./out/ASK/bin/dpa_app /bin/dpa_app
 COPY ./ask/cmm/cmm.conf /etc/cmm/cmm.conf
 COPY ./ask/cmm/entrypoint.sh /entrypoint.sh
+COPY ./ask/cmm/init_dpa.sh /init_dpa.sh
 
-RUN chmod +x /bin/cmm /bin/dpa_app /entrypoint.sh \
- && mkdir -p /run/ask /var/log
+RUN chmod +x /bin/cmm /bin/dpa_app /init_dpa.sh /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]

makefile

@@ -105,6 +105,7 @@ RELEASE_DEPS := \
 	$(BOARD_ITB) \
 	$(CLITOOLS_BIN) \
 	docker/alpine.Dockerfile \
+	cmm-image \
 	$(ALPINE_SRCS) \
 	build.env \
 	makefile
@@ -253,9 +254,10 @@ cmm-image: ASK
 		-f docker/cmm.Dockerfile \
 		--build-arg ALPINE_SERIES=$(ALPINE_SERIES) \
 		--load \
-		-t $(IMAGE_REPOSITORY)/cmm:$(KUBE_VERSION)-$(TAG) .
+		-t localhost/monok8s/cmm:$(TAG) .
 
 push-cmm-image: cmm-image
+	docker tag -t localhost/monok8s/cmm:$(TAG) $(IMAGE_REPOSITORY)/cmm:$(KUBE_VERSION)-$(TAG) .
 	docker push $(IMAGE_REPOSITORY)/cmm:$(KUBE_VERSION)-$(TAG)
 
 vpp: $(BUILD_BASE_STAMP) $(VPP_TAR) $(DPDK_TAR) $(FMLIB_TAR) $(FMC_TAR) $(NXP_TAR)