control agent can now uboot commands

alpine/configure-system.sh

@@ -7,6 +7,7 @@ rc-update add devfs sysinit
 rc-update add procfs sysinit
 rc-update add sysfs sysinit
 rc-update add cgroups sysinit
+rc-update add hwclock boot
 rc-update add fancontrol boot
 rc-update add loopback boot
 rc-update add hostname boot

alpine/install-packages.sh

@@ -31,7 +31,8 @@ if [ $? -ne 0 ]; then
 	exit $?
 fi
 
-sed -i "s/default_runtime = \"crun\"/\0\ncgroup_manager = \"cgroupfs\"/g" /etc/crio/crio.conf.d/10-crio.conf
+sed -i -e "s/default_runtime = \"crun\"/\0\ncgroup_manager = \"cgroupfs\"/g" \
+	/etc/crio/crio.conf.d/10-crio.conf
 grep cgroup_manager /etc/crio/crio.conf.d/10-crio.conf || exit 1
 
 echo "##################################################### Installing Control Agent"

alpine/rootfs-extra/etc/crio/crio.conf.d/11-coredns.conf

@@ -0,0 +1,4 @@
+[crio.runtime]
+default_sysctls = [
+    "net.ipv4.ip_unprivileged_port_start=20"
+]

build.env

@@ -5,8 +5,8 @@ TAG=dev
 
 # The Linux kernel, from NXP
 NXP_VERSION=lf-6.18.2-1.0.0
-CRIO_VERSION=cri-o.arm64.v1.35.1
+CRIO_VERSION=cri-o.arm64.v1.35.2
-KUBE_VERSION=v1.35.1
+KUBE_VERSION=v1.35.3
 
 # Mono's tutorial said fsl-ls1046a-rdb.dtb but our shipped board is not that one
 # We need fsl-ls1046a-rdb-sdk.dtb here
@@ -21,6 +21,7 @@ CROSS_COMPILE=aarch64-linux-gnu-
 # Tools for initramfs
 BUSYBOX_VERSION=1_36_1
 E2FSPROGS_VERSION=1.47.4
+UBOOT_VERSION=v2026.01
 
 ## Alpine Linux
 ALPINE_VER=3.23.3

clitools/docker/ctl-agent.Dockerfile

@@ -7,6 +7,8 @@ ENV VERSION=${VERSION}
 WORKDIR /
 
 COPY bin/ctl-linux-aarch64-${VERSION} ./ctl
+COPY out/fw_printenv ./
+COPY out/fw_setenv ./
 COPY --from=cacerts /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
 ENTRYPOINT ["/ctl"]

clitools/docker/uboot-tools.Dockerfile

@@ -0,0 +1,41 @@
+FROM alpine:3.22 AS build
+
+RUN apk add --no-cache \
+    build-base \
+    bison \
+    flex \
+    linux-headers \
+    file \
+    binutils \
+    tar
+
+WORKDIR /src
+
+ARG UBOOT_TAR
+ARG UBOOT_VERSION
+
+COPY ${UBOOT_TAR} /tmp/
+RUN tar zxf "/tmp/$(basename "${UBOOT_TAR}")"
+
+RUN make tools-only_defconfig
+
+# Build the env tools using the supported target.
+RUN make -j"$(nproc)" \
+    HOSTCC=gcc \
+    HOSTLD=gcc \
+    HOSTCFLAGS='-O2' \
+    HOSTLDFLAGS='-static' \
+    envtools
+
+# fw_setenv is the same program; create the link ourselves.
+RUN ln -sf fw_printenv tools/env/fw_setenv
+
+RUN file tools/env/fw_printenv tools/env/fw_setenv
+
+RUN readelf -d tools/env/fw_printenv || true
+
+RUN ! readelf -d tools/env/fw_printenv | grep -q '(NEEDED)'
+
+FROM scratch AS export
+COPY --from=build /src/tools/env/fw_printenv /fw_printenv
+COPY --from=build /src/tools/env/fw_setenv /fw_setenv

clitools/makefile

@@ -1,17 +1,24 @@
 # Should be the same as upstream version in production
 VERSION ?= dev
+UBOOT_VERSION=v2026.01
 
 # Target kube version
 KUBE_VERSION ?= v1.35.0
 
 GIT_REV := $(shell git rev-parse HEAD)
 
-BIN_DIR := bin
+PACKAGES_DIR := packages
-OUT_DIR := out
+BIN_DIR      := bin
+OUT_DIR      := out
+
+UBOOT_TAR     := $(PACKAGES_DIR)/uboot-$(UBOOT_VERSION).tar.gz
 
 BUILDINFO_FILE := pkg/buildinfo/buildinfo_gen.go
 CRD_PATHS := ./pkg/apis/...
 
+$(PACKAGES_DIR):
+	mkdir -p $@
+
 # Never cache this
 .buildinfo:
 	@mkdir -p $(dir $(BUILDINFO_FILE))
@@ -26,13 +33,27 @@ CRD_PATHS := ./pkg/apis/...
 		')' \
 	> $(BUILDINFO_FILE)
 
+$(UBOOT_TAR): | $(PACKAGES_DIR)
+	git clone --depth 1 --branch v2026.01 --filter=blob:none https://github.com/u-boot/u-boot.git $(OUT_DIR)/u-boot-$(UBOOT_VERSION)
+	tar -C "$(OUT_DIR)/u-boot-$(UBOOT_VERSION)" -zcf "$@" .
+	rm -rf $(OUT_DIR)/u-boot-$(UBOOT_VERSION)
+	test -f $@
+
+uboot-tools: $(UBOOT_TAR)
+	docker buildx build --platform linux/arm64 \
+		-f docker/uboot-tools.Dockerfile \
+		--build-arg UBOOT_VERSION=$(UBOOT_VERSION) \
+		--build-arg UBOOT_TAR=$(UBOOT_TAR) \
+		--output type=local,dest=./$(OUT_DIR) .
+
 build: .buildinfo
 	mkdir -p $(BIN_DIR) $(OUT_DIR)/crds
 	controller-gen crd paths=$(CRD_PATHS) output:crd:dir=$(OUT_DIR)/crds
 	GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/ctl-linux-aarch64-$(VERSION) ./cmd/ctl
 
-build-agent: build
+build-agent: build uboot-tools
 	docker build \
+		-f docker/ctl-agent.Dockerfile \
 		--platform=linux/arm64 \
 		--build-arg VERSION=$(VERSION) \
 		-t localhost/monok8s/control-agent:$(VERSION) .
@@ -52,4 +73,4 @@ clean:
 
 all: build build-agent build-local
 
-.PHONY: clean all run .buildinfo build build-local build-agent
+.PHONY: clean all run .buildinfo build build-local build-agent uboot-tools

clitools/pkg/bootstrap/runner.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 
-	"k8s.io/klog/v2"
 	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
 	"example.com/monok8s/pkg/node"
 	"example.com/monok8s/pkg/system"
+	"k8s.io/klog/v2"
 )
 
 type Runner struct {
@@ -132,6 +132,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
 				Name:   "Apply node metadata",
 				Desc:   "Apply labels/annotations to the local node if API server is reachable",
 			},
+			{
+				RegKey: "ConfigureUBootCommands",
+				Name:   "Ensure fw_env config and u-boot-tools availablilty",
+				Desc:   "Install or generate /etc/fw_env.config for U-Boot environment access",
+			},
 			{
 				RegKey: "ApplyControlAgentDaemonSetResources",
 				Name:   "Apply daemonset for control agent",

clitools/pkg/cmd/internal/fwprintenv.go

@@ -0,0 +1,89 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/klog/v2"
+
+	"example.com/monok8s/pkg/system"
+)
+
+func newInternalFWPrintEnvCmd() *cobra.Command {
+	var key string
+	var configPath string
+
+	cmd := &cobra.Command{
+		Use:   "fw-printenv",
+		Short: "Run fw_printenv",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+
+			key = strings.TrimSpace(key)
+			configPath = strings.TrimSpace(configPath)
+
+			if configPath == "" {
+				configPath = defaultFWEnvConfigPath
+			}
+
+			if _, err := os.Stat(configPath); err != nil {
+				return fmt.Errorf("stat fw env config %q: %w", configPath, err)
+			}
+
+			runner := system.NewRunner(system.RunnerConfig{
+				DefaultTimeout: 15 * time.Second,
+				StreamOutput:   false,
+				Logger:         &system.StdLogger{},
+			})
+
+			runArgs := []string{"-c", configPath}
+			if key != "" {
+				runArgs = append(runArgs, key)
+			}
+
+			res, err := runner.RunWithOptions(
+				ctx,
+				"/fw_printenv",
+				runArgs,
+				system.RunOptions{
+					Quiet: true,
+				},
+			)
+			if err != nil {
+				if res != nil {
+					klog.ErrorS(err, "fw_printenv failed",
+						"key", key,
+						"stdout", strings.TrimSpace(res.Stdout),
+						"stderr", strings.TrimSpace(res.Stderr),
+					)
+				}
+				return err
+			}
+
+			stdout := strings.TrimSpace(res.Stdout)
+			stderr := strings.TrimSpace(res.Stderr)
+
+			if stdout != "" {
+				fmt.Println(stdout)
+			}
+			if stderr != "" {
+				klog.InfoS("fw_printenv stderr", "output", stderr)
+			}
+
+			klog.InfoS("fw_printenv succeeded",
+				"key", key,
+				"configPath", configPath,
+			)
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&key, "key", "", "U-Boot environment variable name to print")
+	cmd.Flags().StringVar(&configPath, "config", defaultFWEnvConfigPath, "Path to fw_env.config")
+
+	return cmd
+}

clitools/pkg/cmd/internal/fwsetenv.go

@@ -0,0 +1,115 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/klog/v2"
+
+	"example.com/monok8s/pkg/system"
+)
+
+const defaultFWEnvConfigPath = "/host/etc/fw_env.config"
+
+func newInternalFWSetEnvCmd() *cobra.Command {
+	var key string
+	var value string
+	var configPath string
+
+	cmd := &cobra.Command{
+		Use:   "fw-setenv",
+		Short: "Run fw_setenv",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+
+			key = strings.TrimSpace(key)
+			value = strings.TrimSpace(value)
+			configPath = strings.TrimSpace(configPath)
+
+			if key == "" {
+				return fmt.Errorf("--key is required")
+			}
+			if value == "" {
+				return fmt.Errorf("--value is required")
+			}
+			if configPath == "" {
+				configPath = defaultFWEnvConfigPath
+			}
+
+			if _, err := os.Stat(configPath); err != nil {
+				return fmt.Errorf("stat fw env config %q: %w", configPath, err)
+			}
+
+			runner := system.NewRunner(system.RunnerConfig{
+				DefaultTimeout: 15 * time.Second,
+				StreamOutput:   false,
+				Logger:         &system.StdLogger{},
+			})
+
+			// Preflight first so failure is clearer than blindly writing.
+			preflightRes, err := runner.RunWithOptions(
+				ctx,
+				"/fw_printenv",
+				[]string{"-c", configPath},
+				system.RunOptions{
+					Quiet: true,
+				},
+			)
+			if err != nil {
+				if preflightRes != nil {
+					klog.ErrorS(err, "fw_printenv preflight failed",
+						"stdout", strings.TrimSpace(preflightRes.Stdout),
+						"stderr", strings.TrimSpace(preflightRes.Stderr),
+					)
+				}
+				return fmt.Errorf("fw_printenv preflight: %w", err)
+			}
+
+			res, err := runner.RunWithOptions(
+				ctx,
+				"/fw_setenv",
+				[]string{
+					"-c", configPath,
+					key, value,
+				},
+				system.RunOptions{
+					Quiet: true,
+				},
+			)
+			if err != nil {
+				if res != nil {
+					klog.ErrorS(err, "fw_setenv failed",
+						"key", key,
+						"value", value,
+						"stdout", strings.TrimSpace(res.Stdout),
+						"stderr", strings.TrimSpace(res.Stderr),
+					)
+				}
+				return err
+			}
+
+			if strings.TrimSpace(res.Stdout) != "" {
+				klog.InfoS("fw_setenv stdout", "output", strings.TrimSpace(res.Stdout))
+			}
+			if strings.TrimSpace(res.Stderr) != "" {
+				klog.InfoS("fw_setenv stderr", "output", strings.TrimSpace(res.Stderr))
+			}
+
+			klog.InfoS("fw_setenv succeeded",
+				"key", key,
+				"value", value,
+				"configPath", configPath,
+			)
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&key, "key", "", "U-Boot environment variable name")
+	cmd.Flags().StringVar(&value, "value", "", "U-Boot environment variable value")
+	cmd.Flags().StringVar(&configPath, "config", defaultFWEnvConfigPath, "Path to fw_env.config")
+
+	return cmd
+}

clitools/pkg/cmd/internal/internal.go

@@ -1,9 +1,9 @@
 package internal
 
 import (
-	"github.com/spf13/cobra"
 	"example.com/monok8s/pkg/bootstrap"
 	"example.com/monok8s/pkg/config"
+	"github.com/spf13/cobra"
 )
 
 func NewCmdInternal() *cobra.Command {
@@ -33,6 +33,9 @@ func NewCmdInternal() *cobra.Command {
 			return nil
 		},
 	})
+	cmd.AddCommand(newInternalFWSetEnvCmd())
+	cmd.AddCommand(newInternalFWPrintEnvCmd())
+
 	cmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "path to MonoKSConfig yaml")
 	return cmd
 }

clitools/pkg/controller/osupgrade/bootenv.go

@@ -0,0 +1,37 @@
+package osupgrade
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+)
+
+type NextBootConfig struct {
+	Key   string
+	Value string
+}
+
+func SetNextBootEnv(ctx context.Context, cfg NextBootConfig) error {
+	if cfg.Key == "" {
+		return fmt.Errorf("boot env key is empty")
+	}
+	if cfg.Value == "" {
+		return fmt.Errorf("boot env value is empty")
+	}
+
+	cmd := exec.CommandContext(
+		ctx,
+		"/proc/self/exe",
+		"internal",
+		"fw-setenv",
+		"--key", cfg.Key,
+		"--value", cfg.Value,
+	)
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("invoke internal fw-setenv: %w: %s", err, string(out))
+	}
+
+	return nil
+}

clitools/pkg/controller/osupgrade/handler.go

@@ -128,7 +128,22 @@ func HandleOSUpgrade(ctx context.Context, clients *kube.Clients,
 	}
 
 	klog.Info(result)
-	// TODO: fw_setenv
+	if err := SetNextBootEnv(ctx, NextBootConfig{
+		Key:   "boot_part",
+		Value: "B",
+	}); err != nil {
+		return failProgress(ctx, clients, osup, "set boot env", err)
+	}
+
+	now = metav1.Now()
+	osup.Status.LastUpdatedAt = &now
+	osup.Status.Message = "image applied, verified, and next boot environment updated"
+	osup.Status.Phase = monov1alpha1.OSUpgradeProgressPhaseRebooting
+
+	osup, err = updateProgressStatus(ctx, clients, osup_gvr, osup)
+	if err != nil {
+		return fmt.Errorf("update progress status: %w", err)
+	}
 
 	return nil
 }

clitools/pkg/node/agent.go

@@ -273,20 +273,8 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter
 									Value: "/host/opt/monok8s/config/cluster.env",
 								},
 								{
-									Name:  "HOST_MOUNT_ROOT",
+									Name:  "FW_ENV_CONFIG_FILE",
-									Value: "/host/mnt/control-agent",
+									Value: "/host/etc/fw_env.config",
-								},
-								{
-									Name:  "HOST_DEV_DIR",
-									Value: "/host/dev",
-								},
-								{
-									Name:  "HOST_PROC_DIR",
-									Value: "/host/proc",
-								},
-								{
-									Name:  "HOST_RUN_DIR",
-									Value: "/host/run",
 								},
 							},
 							SecurityContext: &corev1.SecurityContext{
@@ -295,22 +283,18 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter
 							VolumeMounts: []corev1.VolumeMount{
 								{
 									Name:      "host-dev",
-									MountPath: "/host/dev",
+									MountPath: "/dev",
+								},
+								{
+									Name:      "host-etc",
+									MountPath: "/host/etc",
+									ReadOnly:  true,
 								},
 								{
 									Name:      "host-config",
 									MountPath: "/host/opt/monok8s/config",
 									ReadOnly:  true,
 								},
-								{
-									Name:      "host-run",
-									MountPath: "/host/run",
-								},
-								{
-									Name:      "host-proc",
-									MountPath: "/host/proc",
-									ReadOnly:  true,
-								},
 							},
 						},
 					},
@@ -324,6 +308,15 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter
 								},
 							},
 						},
+						{
+							Name: "host-etc",
+							VolumeSource: corev1.VolumeSource{
+								HostPath: &corev1.HostPathVolumeSource{
+									Path: "/etc",
+									Type: hostPathType(corev1.HostPathDirectory),
+								},
+							},
+						},
 						{
 							Name: "host-config",
 							VolumeSource: corev1.VolumeSource{
@@ -333,24 +326,6 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter
 								},
 							},
 						},
-						{
-							Name: "host-run",
-							VolumeSource: corev1.VolumeSource{
-								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/run",
-									Type: hostPathType(corev1.HostPathDirectory),
-								},
-							},
-						},
-						{
-							Name: "host-proc",
-							VolumeSource: corev1.VolumeSource{
-								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/proc",
-									Type: hostPathType(corev1.HostPathDirectory),
-								},
-							},
-						},
 					},
 				},
 			},

clitools/pkg/node/network.go

@@ -7,8 +7,8 @@ import (
 	"os"
 	"strings"
 
-	"k8s.io/klog/v2"
 	system "example.com/monok8s/pkg/system"
+	"k8s.io/klog/v2"
 )
 
 type NetworkConfig struct {

docs/coredns.md

@@ -0,0 +1,198 @@
+## About /etc/crio/crio.conf.d/11-coredns.conf
+
+CoreDNS fails with permission denied. Probably because of readonly RootFS.
+The current solution is to just allow port 20 up to bind without root
+
+well this is just brute force allowing any ports 20 < to be yeah go ahead
+```
+[crio.runtime]
+default_sysctls = [
+    "net.ipv4.ip_unprivileged_port_start=20"
+]
+```
+
+CoreDNS wants 53. And users may want to have ssh servers
+
+So just allow all the ports!
+
+
+## Diagnostics
+
+I'll just put some of my findings in here.
+
+Catching effective cap being 0x0000 (this is the issue?)
+```
+kubectl rollout restart deployment -n kube-system coredns
+while true; do
+  PID=$(pgrep coredns)
+  if [ ! -z "$PID" ]; then
+    echo "Found CoreDNS PID: $PID"
+    grep Cap /proc/$PID/status
+    break
+  fi
+done
+
+CapInh:	0000000000000000
+CapPrm:	0000000000000000
+CapEff:	0000000000000000
+CapBnd:	0000000000000400
+CapAmb:	0000000000000000
+```
+
+# find /var/lib/containers/storage/overlay-containers -name config.json | xargs grep "noNewPrivileges"
+
+Current kernel config
+```
+gunzip -c /proc/config.gz | grep -e SECURITY -e LSM -e SECCOMP
+CONFIG_HAVE_ARCH_SECCOMP=y
+CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
+CONFIG_SECCOMP=y
+CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
+CONFIG_IIO_ST_LSM6DSX=m
+CONFIG_IIO_ST_LSM6DSX_I2C=m
+CONFIG_IIO_ST_LSM6DSX_SPI=m
+CONFIG_IIO_ST_LSM6DSX_I3C=m
+# CONFIG_IIO_ST_LSM9DS0 is not set
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_UBIFS_FS_SECURITY=y
+CONFIG_NFS_V4_SECURITY_LABEL=y
+# CONFIG_9P_FS_SECURITY is not set
+# CONFIG_SECURITY_DMESG_RESTRICT is not set
+CONFIG_SECURITY=y
+CONFIG_HAS_SECURITY_AUDIT=y
+CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+CONFIG_SECURITY_NETWORK_XFRM=y
+CONFIG_SECURITY_PATH=y
+# CONFIG_SECURITY_SELINUX is not set
+# CONFIG_SECURITY_SMACK is not set
+# CONFIG_SECURITY_TOMOYO is not set
+# CONFIG_SECURITY_APPARMOR is not set
+# CONFIG_SECURITY_LOADPIN is not set
+# CONFIG_SECURITY_YAMA is not set
+# CONFIG_SECURITY_SAFESETID is not set
+# CONFIG_SECURITY_LOCKDOWN_LSM is not set
+# CONFIG_SECURITY_LANDLOCK is not set
+# CONFIG_SECURITY_IPE is not set
+CONFIG_DEFAULT_SECURITY_DAC=y
+CONFIG_LSM="landlock,lockdown,yama,loadpin,safesetid,ipe,bpf"
+```
+
+`CONFIG_LSM` does not need `capability`. It works when testing CAP_NET_BIND manually.
+```
+# capsh --keep=1 --user=nobody --inh=cap_net_bind_service --addamb=cap_net_bind_service -- -c "grep Cap /proc/self/status"
+CapInh:	0000000000000400
+CapPrm:	0000000000000400
+CapEff:	0000000000000400
+CapBnd:	000001ffffffffff
+CapAmb:	0000000000000400
+```
+
+And this also aligns with debian's `CONFIG_LSM`. They don't have `capability` hardcoded in.
+
+Debian's working kernel config
+```
+grep -e SECURITY -e LSM -e SECCOMP /boot/config-$(uname -r)
+CONFIG_BPF_LSM=y
+# CONFIG_NFIT_SECURITY_DEBUG is not set
+CONFIG_HAVE_ARCH_SECCOMP=y
+CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
+CONFIG_SECCOMP=y
+CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
+CONFIG_IP_NF_SECURITY=m
+CONFIG_IP6_NF_SECURITY=m
+CONFIG_IIO_ST_LSM6DSX=m
+CONFIG_IIO_ST_LSM6DSX_I2C=m
+CONFIG_IIO_ST_LSM6DSX_SPI=m
+# CONFIG_IIO_ST_LSM9DS0 is not set
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_REISERFS_FS_SECURITY=y
+CONFIG_JFS_SECURITY=y
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_JFFS2_FS_SECURITY=y
+CONFIG_UBIFS_FS_SECURITY=y
+CONFIG_EROFS_FS_SECURITY=y
+CONFIG_NFS_V4_SECURITY_LABEL=y
+CONFIG_NFSD_V4_SECURITY_LABEL=y
+# CONFIG_CEPH_FS_SECURITY_LABEL is not set
+CONFIG_9P_FS_SECURITY=y
+CONFIG_SECURITY_DMESG_RESTRICT=y
+CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y
+CONFIG_SECURITY=y
+CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+# CONFIG_SECURITY_INFINIBAND is not set
+CONFIG_SECURITY_NETWORK_XFRM=y
+CONFIG_SECURITY_PATH=y
+CONFIG_LSM_MMAP_MIN_ADDR=65536
+CONFIG_SECURITY_SELINUX=y
+# CONFIG_SECURITY_SELINUX_BOOTPARAM is not set
+# CONFIG_SECURITY_SELINUX_DISABLE is not set
+CONFIG_SECURITY_SELINUX_DEVELOP=y
+CONFIG_SECURITY_SELINUX_AVC_STATS=y
+CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=0
+CONFIG_SECURITY_SELINUX_SIDTAB_HASH_BITS=9
+CONFIG_SECURITY_SELINUX_SID2STR_CACHE_SIZE=256
+# CONFIG_SECURITY_SMACK is not set
+CONFIG_SECURITY_TOMOYO=y
+CONFIG_SECURITY_TOMOYO_MAX_ACCEPT_ENTRY=2048
+CONFIG_SECURITY_TOMOYO_MAX_AUDIT_LOG=1024
+# CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER is not set
+CONFIG_SECURITY_TOMOYO_POLICY_LOADER="/sbin/tomoyo-init"
+CONFIG_SECURITY_TOMOYO_ACTIVATION_TRIGGER="/sbin/init"
+# CONFIG_SECURITY_TOMOYO_INSECURE_BUILTIN_SETTING is not set
+CONFIG_SECURITY_APPARMOR=y
+# CONFIG_SECURITY_APPARMOR_DEBUG is not set
+CONFIG_SECURITY_APPARMOR_INTROSPECT_POLICY=y
+CONFIG_SECURITY_APPARMOR_HASH=y
+CONFIG_SECURITY_APPARMOR_HASH_DEFAULT=y
+CONFIG_SECURITY_APPARMOR_EXPORT_BINARY=y
+CONFIG_SECURITY_APPARMOR_PARANOID_LOAD=y
+# CONFIG_SECURITY_LOADPIN is not set
+CONFIG_SECURITY_YAMA=y
+# CONFIG_SECURITY_SAFESETID is not set
+CONFIG_SECURITY_LOCKDOWN_LSM=y
+CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y
+CONFIG_SECURITY_LANDLOCK=y
+CONFIG_IMA_LSM_RULES=y
+# CONFIG_DEFAULT_SECURITY_SELINUX is not set
+# CONFIG_DEFAULT_SECURITY_TOMOYO is not set
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+# CONFIG_DEFAULT_SECURITY_DAC is not set
+CONFIG_LSM="landlock,lockdown,yama,loadpin,safesetid,integrity,apparmor,selinux,smack,tomoyo,bpf"
+```
+
+Debian's CoreDNS cap
+```
+# grep Cap /proc/1911/status
+CapInh:	0000000000000000
+CapPrm:	0000000000000400
+CapEff:	0000000000000400
+CapBnd:	0000000000000400
+CapAmb:	0000000000000000
+```
+
+Debian's CRIO config
+```
+# cat /etc/crio/crio.conf.d/10-crio.conf
+[crio.image]
+signature_policy = "/etc/crio/policy.json"
+
+[crio.runtime]
+default_runtime = "crun"
+
+[crio.runtime.runtimes.crun]
+runtime_path = "/usr/libexec/crio/crun"
+runtime_root = "/run/crun"
+monitor_path = "/usr/libexec/crio/conmon"
+allowed_annotations = [
+    "io.containers.trace-syscall",
+]
+
+[crio.runtime.runtimes.runc]
+runtime_path = "/usr/libexec/crio/runc"
+runtime_root = "/run/runc"
+monitor_path = "/usr/libexec/crio/conmon"
+```

docs/uboot.md

@@ -1,4 +1,4 @@
-Booting release image with bootusb
+## Booting release image with bootusb
  - Use this if you are flashing image from macos/flashusb.sh
 ```
 setenv bootusb '
@@ -18,8 +18,16 @@ setenv bootusb 'usb start; ext4load usb 0:2 ${mks_loadaddr} /boot/kernel.itb; bo
 run bootusb
 ```
 
+## Run fw_printenv and fw_setenv from kubectl
+```
+# Avoid using daemonset/control-agent if you have multiple nodes
+kubectl exec -n kube-system control-agent-abcdef1 -- /ctl internal fw-setenv --key foo --value bar
 
-Original uboot env from mono
+# fw_printenv
+kubectl exec -n kube-system ds/control-agent -- /ctl internal fw-printenv --key foo
+```
+
+## Original uboot env from mono
 ```
 arch=arm
 baudrate=115200

kernel-extra.config

@@ -293,6 +293,14 @@ CONFIG_SECCOMP_FILTER=y
 # CONFIG_SECURITY=y
 # CONFIG_SECURITYFS=y
 
+CONFIG_SECURITY_NETWORK=y
+# enables Security Module (LSM) hooks for network operations. CoreDNS needs this
+
+CONFIG_SECURITY_PATH=y
+# Recommended for container isolation
+
+CONFIG_SECURITY_NETWORK_XFRM=y
+
 
 ###############################################################################
 # Misc userspace/container compatibility

makefile

@@ -231,20 +231,6 @@ $(RELEASE_IMAGE): $(RELEASE_DEPS) | $(OUT_DIR)
 
 	test -f $@
 
-check-functions:
-	@echo "Checking function calls vs definitions..."
-	@awk '/^main\(\)/ { in_main=1; next } \
-	      in_main && /^\}/ { in_main=0 } \
-	      in_main && /^[[:space:]]*[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]*$$/ { \
-	          gsub(/^[ \t]+/, "", $$0); \
-	          print $$0 \
-	      }' ./alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh \
-	      | sort -u > /tmp/called.txt
-	@grep -E '^[a-zA-Z_][a-zA-Z0-9_]*\(\)' ./alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh \
-	      | sed 's/().*//' \
-	      | sort -u > /tmp/defined.txt
-	@echo "Missing functions:"
-	@comm -23 /tmp/called.txt /tmp/defined.txt || true
 
 # ---- config targets ------------------------------------------------------------