diff --git a/alpine/preload-k8s-images.sh b/alpine/preload-k8s-images.sh index d963384..b449780 100755 --- a/alpine/preload-k8s-images.sh +++ b/alpine/preload-k8s-images.sh @@ -27,6 +27,7 @@ FUSE_OVERLAYFS="${FUSE_OVERLAYFS:-/usr/bin/fuse-overlayfs}" EXTRA_IMAGES=( "${EXTRA_IMAGES[@]:-}" "docker-daemon:localhost/monok8s/node-control:$TAG" + "docker-daemon:localhost/monok8s/cmm:$TAG" ) # Keep archive cache version/arch scoped so downloads do not get mixed. diff --git a/alpine/rootfs-extra/etc/local.d/monok8s.start b/alpine/rootfs-extra/etc/local.d/monok8s.start index 30428c9..8d350fe 100755 --- a/alpine/rootfs-extra/etc/local.d/monok8s.start +++ b/alpine/rootfs-extra/etc/local.d/monok8s.start @@ -11,6 +11,19 @@ BOOT_STATE=/run/monok8s/boot-state.env BOOTPART_FILE="$CONFIG_DIR/.bootpart" MIGRATION_STATE_DIR="$CONFIG_DIR/migration-state" +load_module_optional() { + module="$1" + + if ! modprobe "$module"; then + echo "WARNING: failed to load optional module: $module" + fi +} + +echo "Loading optional hardware offload modules..." +load_module_optional cdx +load_module_optional fci +load_module_optional auto_bridge + mkdir -p /dev/hugepages mountpoint -q /dev/hugepages || mount -t hugetlbfs none /dev/hugepages echo 256 > /proc/sys/vm/nr_hugepages diff --git a/ask/cmm/entrypoint.sh b/ask/cmm/entrypoint.sh index c8c91ec..a9dac4e 100644 --- a/ask/cmm/entrypoint.sh +++ b/ask/cmm/entrypoint.sh @@ -3,36 +3,9 @@ set -eu CMM_CONFIG="${CMM_CONFIG:-/etc/cmm/cmm.conf}" -# Vendor default from cmm.service: (131072 = 128 * 1024) max active conntrack/offload entries +# Vendor default from cmm.service: +# 131072 = 128 * 1024 max active conntrack/offload entries. CMM_MAX_CONNECTIONS="${CMM_MAX_CONNECTIONS:-131072}" -mkdir -p /run/ask /var/log - -echo "[ask] loading auto_bridge" -modprobe auto_bridge || true - -echo "[ask] loading cdx" -modprobe cdx - -echo "[ask] waiting for /dev/cdx_ctrl" -for i in $(seq 1 40); do - if [ -e /dev/cdx_ctrl ]; then - break - fi - sleep 0.25 -done -test -e /dev/cdx_ctrl - -if [ ! -e /run/ask/dpa_app.loaded ]; then - echo "[ask] running dpa_app" - /bin/dpa_app - touch /run/ask/dpa_app.loaded -else - echo "[ask] dpa_app already loaded; skipping" -fi - -echo "[ask] loading fci" -modprobe fci - echo "[ask] starting cmm" exec /bin/cmm -D -f "$CMM_CONFIG" -n "$CMM_MAX_CONNECTIONS" diff --git a/ask/cmm/init_dpa.sh b/ask/cmm/init_dpa.sh new file mode 100755 index 0000000..e11b624 --- /dev/null +++ b/ask/cmm/init_dpa.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +STATE_DIR="${CMM_STATE_DIR:-/host/run/monok8s/cmm}" +DONE_FILE="$STATE_DIR/dpa_app.loaded" +LOCK_DIR="$STATE_DIR/dpa_app.lock" + +mkdir -p "$STATE_DIR" + +if [ -e "$DONE_FILE" ]; then + echo "[ask] dpa_app already loaded for this boot; skipping" + exit 0 +fi + +if ! mkdir "$LOCK_DIR" 2>/dev/null; then + echo "[ask] another dpa_app initialization is running; waiting" + + while [ ! -e "$DONE_FILE" ]; do + sleep 1 + done + + echo "[ask] dpa_app was loaded by another process; skipping" + exit 0 +fi + +trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT + +if [ -e "$DONE_FILE" ]; then + echo "[ask] dpa_app already loaded for this boot; skipping" + exit 0 +fi + +echo "[ask] running dpa_app" +/bin/dpa_app + +date -u +"%Y-%m-%dT%H:%M:%SZ" > "$DONE_FILE" +echo "[ask] dpa_app loaded" diff --git a/clitools/pkg/cmd/create/create.go b/clitools/pkg/cmd/create/create.go index e3b5fd8..9ec1a23 100644 --- a/clitools/pkg/cmd/create/create.go +++ b/clitools/pkg/cmd/create/create.go @@ -165,6 +165,47 @@ func NewCmdCreate(flags *genericclioptions.ConfigFlags) *cobra.Command { cmd.AddCommand(&agentcmd) + cmmconf := render.CMMConf{} + cmmcmd := cobra.Command{ + Use: "cmm", + Short: "Print CMM daemonsets template", + RunE: func(cmd *cobra.Command, _ []string) error { + if len(cmmconf.ImagePullSecrets) > 0 && strings.TrimSpace(cmmconf.Image) == "" { + return fmt.Errorf("--image-pull-secret requires --image") + } + + ns, _, err := flags.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + cmmconf.Namespace = ns + + out, err := render.RenderCMMDaemonSets(cmmconf) + if err != nil { + return err + } + + _, err = fmt.Fprint(cmd.OutOrStdout(), out) + return err + }, + } + + cmmcmd.Flags().StringVar( + &cmmconf.Image, + "image", + "", + "CMM image, including optional registry and tag", + ) + cmmcmd.Flags().StringSliceVar( + &cmmconf.ImagePullSecrets, + "image-pull-secret", + nil, + "Image pull secret name for the CMM image; may be specified multiple times or as a comma-separated list", + ) + + cmd.AddCommand(&cmmcmd) + return cmd } diff --git a/clitools/pkg/render/cmm.go b/clitools/pkg/render/cmm.go new file mode 100644 index 0000000..4b85275 --- /dev/null +++ b/clitools/pkg/render/cmm.go @@ -0,0 +1,219 @@ +package render + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + buildinfo "example.com/monok8s/pkg/buildinfo" +) + +const cmmName = "cmm" + +type CMMConf struct { + Namespace string + Image string + ImagePullSecrets []string + Labels map[string]string +} + +func RenderCMMDaemonSets(conf CMMConf) (string, error) { + objs, err := buildCMMDaemonSetObjects(conf) + if err != nil { + return "", err + } + + return renderObjects(objs) +} + +func buildCMMDaemonSetObjects(conf CMMConf) ([]runtime.Object, error) { + if strings.TrimSpace(conf.Namespace) == "" { + return nil, fmt.Errorf("namespace is required") + } + + conf.Labels = map[string]string{ + "app.kubernetes.io/name": cmmName, + "app.kubernetes.io/component": "hardware-offload", + "app.kubernetes.io/part-of": "monok8s", + "app.kubernetes.io/managed-by": "monok8s", + } + + return []runtime.Object{ + buildCMMServiceAccount(conf), + buildCMMDaemonSet(conf), + }, nil +} + +func buildCMMServiceAccount(conf CMMConf) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmmName, + Namespace: conf.Namespace, + Labels: copyStringMap(conf.Labels), + }, + } +} + +func buildCMMDaemonSet(conf CMMConf) *appsv1.DaemonSet { + privileged := true + dsLabels := map[string]string{ + "app.kubernetes.io/name": cmmName, + "app.kubernetes.io/component": "hardware-offload", + "app.kubernetes.io/part-of": "monok8s", + "app.kubernetes.io/managed-by": "monok8s", + } + + image, pullPolicy := cmmImage(conf) + + return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmmName, + Namespace: conf.Namespace, + Labels: copyStringMap(conf.Labels), + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": cmmName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: dsLabels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: cmmName, + HostNetwork: true, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + ImagePullSecrets: imagePullSecrets(conf.ImagePullSecrets), + Tolerations: []corev1.Toleration{ + {Operator: corev1.TolerationOpExists}, + }, + InitContainers: []corev1.Container{ + { + Name: "dpa-app", + Image: image, + ImagePullPolicy: pullPolicy, + Command: []string{"/init_dpa.sh"}, + Env: cdxEnv(), + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + VolumeMounts: append( + []corev1.VolumeMount{ + { + Name: "host-run-cmm", + MountPath: "/host/run/monok8s/cmm", + }, + }, + cdxVolumeMounts()..., + ), + }, + }, + Containers: []corev1.Container{ + { + Name: cmmName, + Image: image, + ImagePullPolicy: pullPolicy, + Env: cmmEnv(), + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + VolumeMounts: cdxVolumeMounts(), + }, + }, + Volumes: cmmVolumes(), + NodeSelector: map[string]string{ + "node.kubernetes.io/instance-type": "mono-gateway", + }, + }, + }, + }, + } +} + +func cdxEnv() []corev1.EnvVar { + return []corev1.EnvVar{ + { + Name: "CDX_CFG_FILE", + Value: "/etc/dpa/cdx_cfg.xml", + }, + { + Name: "CDX_PCD_FILE", + Value: "/etc/dpa/cdx_pcd.xml", + }, + { + Name: "CDX_PDL_FILE", + Value: "/etc/fmc/config/hxs_pdl_v3.xml", + }, + { + Name: "CDX_SP_FILE", + Value: "/etc/dpa/cdx_sp.xml", + }, + } +} + +func cmmEnv() []corev1.EnvVar { + return []corev1.EnvVar{ + { + Name: "CMM_CONFIG", + Value: "/etc/cmm/cmm.conf", + }, + { + Name: "CMM_MAX_CONNECTIONS", + Value: "131072", + }, + } +} + +func cdxVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "cdx-ctrl", + MountPath: "/dev/cdx_ctrl", + }, + } +} + +func cmmVolumes() []corev1.Volume { + return []corev1.Volume{ + { + Name: "cdx-ctrl", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/cdx_ctrl", + Type: hostPathType(corev1.HostPathCharDev), + }, + }, + }, + { + Name: "host-run-cmm", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/monok8s/cmm", + Type: hostPathType(corev1.HostPathDirectoryOrCreate), + }, + }, + }, + } +} + +func cmmImage(conf CMMConf) (string, corev1.PullPolicy) { + if conf.Image != "" { + return conf.Image, corev1.PullIfNotPresent + } + + return fmt.Sprintf("localhost/monok8s/cmm:%s", buildinfo.Version), corev1.PullNever +} diff --git a/docker/cmm.Dockerfile b/docker/cmm.Dockerfile index a1f877f..498c094 100644 --- a/docker/cmm.Dockerfile +++ b/docker/cmm.Dockerfile @@ -1,15 +1,13 @@ ARG ALPINE_SERIES=3.23 FROM alpine:${ALPINE_SERIES} -RUN apk add --no-cache kmod busybox-extras - COPY ./out/ASK/rootfs-cfg/etc /etc COPY ./out/ASK/bin/cmm /bin/cmm COPY ./out/ASK/bin/dpa_app /bin/dpa_app COPY ./ask/cmm/cmm.conf /etc/cmm/cmm.conf COPY ./ask/cmm/entrypoint.sh /entrypoint.sh +COPY ./ask/cmm/init_dpa.sh /init_dpa.sh -RUN chmod +x /bin/cmm /bin/dpa_app /entrypoint.sh \ - && mkdir -p /run/ask /var/log +RUN chmod +x /bin/cmm /bin/dpa_app /init_dpa.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/makefile b/makefile index f5197a5..32500a6 100644 --- a/makefile +++ b/makefile @@ -105,6 +105,7 @@ RELEASE_DEPS := \ $(BOARD_ITB) \ $(CLITOOLS_BIN) \ docker/alpine.Dockerfile \ + cmm-image \ $(ALPINE_SRCS) \ build.env \ makefile @@ -253,9 +254,10 @@ cmm-image: ASK -f docker/cmm.Dockerfile \ --build-arg ALPINE_SERIES=$(ALPINE_SERIES) \ --load \ - -t $(IMAGE_REPOSITORY)/cmm:$(KUBE_VERSION)-$(TAG) . + -t localhost/monok8s/cmm:$(TAG) . push-cmm-image: cmm-image + docker tag -t localhost/monok8s/cmm:$(TAG) $(IMAGE_REPOSITORY)/cmm:$(KUBE_VERSION)-$(TAG) . docker push $(IMAGE_REPOSITORY)/cmm:$(KUBE_VERSION)-$(TAG) vpp: $(BUILD_BASE_STAMP) $(VPP_TAR) $(DPDK_TAR) $(FMLIB_TAR) $(FMC_TAR) $(NXP_TAR)